python3 -u test.py RUN ./ol new -p=test-cluster Init OL dir at /root/open-lambda/test-cluster Create lambda base at /root/open-lambda/test-cluster/lambda (may take several minutes) Working Directory: /root/open-lambda/test-cluster Worker Defaults: { "registry": "/root/open-lambda/test-cluster/registry", "registry_cache_ms": 5000, "sandbox": "sock", "cluster_name": "/root/open-lambda/test-cluster", "pip_mirror": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "max_runners": 0, "handler_cache_mb": 239, "import_cache_mb": 239, "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-cluster/worker", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "worker_port": "5000", "Skip_pull_existing": false, "sandbox_config": { "processes": 10 }, "benchmark_log": "", "timing": false, "startup_pkgs": null, "docker_runtime": "", "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 } } You may modify the defaults here: /root/open-lambda/test-cluster/config.json You may now start a server using the "ol worker" command PUSH conf: {'server_mode': 'sock'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7307, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7307 { "test": "sock_churn", "params": { "fork": true, "procs": 1, "seconds": 15, "baseline": 0 }, "pass": true, "seconds": 16.9004123210907, "total_seconds": 17.172160387039185, "stats": { "sandboxes_per_sec": 6.533333333333333 }, "conf": { "startup_pkgs": null, "import_cache_mb": 239, "registry_cache_ms": 5000, "worker_dir": "/root/open-lambda/test-cluster/worker", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "benchmark_log": "", "pip_mirror": "", "sandbox_config": { "processes": 10 }, "cluster_name": "/root/open-lambda/test-cluster", "handler_cache_mb": 239, "Skip_pull_existing": false, "server_mode": "sock", "max_runners": 0, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sandbox": "sock", "registry": "/root/open-lambda/test-cluster/registry", "docker_runtime": "", "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "timing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:12:46 remove /sys/fs/cgroup/devices/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:12:46 remove /sys/fs/cgroup/freezer/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:12:46 remove /sys/fs/cgroup/hugetlb/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:12:46 remove /sys/fs/cgroup/memory/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:12:46 remove /sys/fs/cgroup/perf_event/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:12:46 remove /sys/fs/cgroup/systemd/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:12:46 remove /sys/fs/cgroup/pids/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:12:46 remove worker.pid", "2019/06/21 20:12:46 exiting", "" ] } POP conf: {'server_mode': 'sock'} PUSH conf: {'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson'], 'registry': '/root/open-lambda/test-registry'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7729, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7729 { "test": "ping_test", "params": {}, "pass": true, "seconds": 2.128896951675415, "total_seconds": 17.614634037017822, "stats": { "pings_per_sec": 469.73097811418364 }, "conf": { "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "import_cache_mb": 239, "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "benchmark_log": "", "pip_mirror": "", "sandbox_config": { "processes": 10 }, "cluster_name": "/root/open-lambda/test-cluster", "handler_cache_mb": 239, "Skip_pull_existing": false, "server_mode": "lambda", "max_runners": 0, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "sandbox": "sock", "registry": "/root/open-lambda/test-registry", "registry_cache_ms": 5000, "docker_runtime": "" }, "exception": null, "worker_tail": [ "2019/06/21 20:13:04 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:13:04 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:13:04 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:13:04 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:13:04 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:13:04 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:13:04 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:13:04 remove worker.pid", "2019/06/21 20:13:04 exiting", "" ] } PUSH conf: {'handler_cache_mb': 0, 'import_cache_mb': 0} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7803, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7803 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 1.935175895690918, "total_seconds": 10.528533935546875, "stats": null, "conf": { "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "import_cache_mb": 0, "registry": "/root/open-lambda/test-registry", "worker_dir": "/root/open-lambda/test-cluster/worker", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "benchmark_log": "", "pip_mirror": "", "sandbox_config": { "processes": 10 }, "cluster_name": "/root/open-lambda/test-cluster", "handler_cache_mb": 0, "Skip_pull_existing": false, "server_mode": "lambda", "max_runners": 0, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "sandbox": "sock", "docker_runtime": "", "registry_cache_ms": 5000, "timing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:13:14 remove /sys/fs/cgroup/devices/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:13:14 remove /sys/fs/cgroup/freezer/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:13:14 remove /sys/fs/cgroup/hugetlb/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:13:14 remove /sys/fs/cgroup/memory/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:13:14 remove /sys/fs/cgroup/perf_event/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:13:14 remove /sys/fs/cgroup/systemd/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:13:14 remove /sys/fs/cgroup/pids/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:13:14 remove worker.pid", "2019/06/21 20:13:14 exiting", "" ] } POP conf: {'handler_cache_mb': 0, 'import_cache_mb': 0} PUSH conf: {'handler_cache_mb': 256, 'import_cache_mb': 0} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7878, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7878 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 1.5504720211029053, "total_seconds": 9.623545408248901, "stats": null, "conf": { "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "import_cache_mb": 0, "registry_cache_ms": 5000, "worker_dir": "/root/open-lambda/test-cluster/worker", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "benchmark_log": "", "pip_mirror": "", "sandbox_config": { "processes": 10 }, "cluster_name": "/root/open-lambda/test-cluster", "handler_cache_mb": 256, "Skip_pull_existing": false, "registry": "/root/open-lambda/test-registry", "max_runners": 0, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sandbox": "sock", "server_mode": "lambda", "docker_runtime": "", "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "timing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:13:24 remove /sys/fs/cgroup/devices/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:13:24 remove /sys/fs/cgroup/freezer/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:13:24 remove /sys/fs/cgroup/hugetlb/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:13:24 remove /sys/fs/cgroup/memory/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:13:24 remove /sys/fs/cgroup/perf_event/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:13:24 remove /sys/fs/cgroup/systemd/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:13:24 remove /sys/fs/cgroup/pids/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:13:24 remove worker.pid", "2019/06/21 20:13:24 exiting", "" ] } POP conf: {'handler_cache_mb': 256, 'import_cache_mb': 0} PUSH conf: {'handler_cache_mb': 0, 'import_cache_mb': 256} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7956, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7956 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 1.4812328815460205, "total_seconds": 9.556445837020874, "stats": null, "conf": { "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "import_cache_mb": 256, "worker_dir": "/root/open-lambda/test-cluster/worker", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "benchmark_log": "", "pip_mirror": "", "sandbox_config": { "processes": 10 }, "cluster_name": "/root/open-lambda/test-cluster", "handler_cache_mb": 0, "Skip_pull_existing": false, "registry": "/root/open-lambda/test-registry", "max_runners": 0, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "sandbox": "sock", "docker_runtime": "", "registry_cache_ms": 5000, "server_mode": "lambda", "timing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:13:34 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:13:34 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:13:34 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:13:34 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:13:34 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:13:34 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:13:34 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:13:34 remove worker.pid", "2019/06/21 20:13:34 exiting", "" ] } POP conf: {'handler_cache_mb': 0, 'import_cache_mb': 256} PUSH conf: {'handler_cache_mb': 256, 'import_cache_mb': 256} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=8039, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 8039 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 1.7655677795410156, "total_seconds": 10.02767276763916, "stats": null, "conf": { "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "import_cache_mb": 256, "registry_cache_ms": 5000, "worker_dir": "/root/open-lambda/test-cluster/worker", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "benchmark_log": "", "pip_mirror": "", "sandbox_config": { "processes": 10 }, "cluster_name": "/root/open-lambda/test-cluster", "handler_cache_mb": 256, "Skip_pull_existing": false, "registry": "/root/open-lambda/test-registry", "max_runners": 0, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sandbox": "sock", "docker_runtime": "", "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "server_mode": "lambda", "timing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:13:44 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:13:44 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:13:44 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:13:44 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:13:44 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:13:44 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:13:44 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:13:44 remove worker.pid", "2019/06/21 20:13:44 exiting", "" ] } POP conf: {'handler_cache_mb': 256, 'import_cache_mb': 256} PUSH conf: {'sandbox': 'docker', 'import_cache_mb': 0, 'handler_cache_mb': 0} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=8124, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 8124 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 6.828116178512573, "total_seconds": 15.083780527114868, "stats": null, "conf": { "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "import_cache_mb": 0, "worker_dir": "/root/open-lambda/test-cluster/worker", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "benchmark_log": "", "pip_mirror": "", "sandbox_config": { "processes": 10 }, "cluster_name": "/root/open-lambda/test-cluster", "handler_cache_mb": 0, "Skip_pull_existing": false, "registry": "/root/open-lambda/test-registry", "max_runners": 0, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "sandbox": "docker", "docker_runtime": "", "registry_cache_ms": 5000, "server_mode": "lambda", "timing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:13:58 EVICTING INSTANCE: 23175168 used / 0 limit", "2019/06/21 20:13:58 Destroy() [SB 3]", "2019/06/21 20:13:58 received kill signal, cleaning up", "2019/06/21 20:13:58 Destroy() [SB 0]", "2019/06/21 20:13:58 Destroy() [SB 1]", "2019/06/21 20:13:58 Destroy() [SB 2]", "2019/06/21 20:13:58 Destroy() [SB 3]", "2019/06/21 20:13:59 remove worker.pid", "2019/06/21 20:13:59 exiting", "" ] } POP conf: {'sandbox': 'docker', 'import_cache_mb': 0, 'handler_cache_mb': 0} PUSH conf: {'sandbox': 'docker', 'import_cache_mb': 0, 'handler_cache_mb': 256} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=9155, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 9155 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 3.364075183868408, "total_seconds": 12.223693370819092, "stats": null, "conf": { "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "import_cache_mb": 0, "registry_cache_ms": 5000, "worker_dir": "/root/open-lambda/test-cluster/worker", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "benchmark_log": "", "pip_mirror": "", "sandbox_config": { "processes": 10 }, "cluster_name": "/root/open-lambda/test-cluster", "handler_cache_mb": 256, "Skip_pull_existing": false, "registry": "/root/open-lambda/test-registry", "max_runners": 0, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sandbox": "docker", "docker_runtime": "", "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "server_mode": "lambda", "timing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:14:10 Pause() [SB 3]", "2019/06/21 20:14:10 MemUsageKB() [SB 3]", "2019/06/21 20:14:10 received kill signal, cleaning up", "2019/06/21 20:14:10 Destroy() [SB 0]", "2019/06/21 20:14:10 Destroy() [SB 1]", "2019/06/21 20:14:10 Destroy() [SB 2]", "2019/06/21 20:14:11 Destroy() [SB 3]", "2019/06/21 20:14:11 remove worker.pid", "2019/06/21 20:14:11 exiting", "" ] } POP conf: {'sandbox': 'docker', 'import_cache_mb': 0, 'handler_cache_mb': 256} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=10202, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 10202 { "test": "fork_bomb", "params": {}, "pass": true, "seconds": 0.16570234298706055, "total_seconds": 8.529446363449097, "stats": null, "conf": { "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "import_cache_mb": 239, "worker_dir": "/root/open-lambda/test-cluster/worker", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "benchmark_log": "", "pip_mirror": "", "sandbox_config": { "processes": 10 }, "cluster_name": "/root/open-lambda/test-cluster", "handler_cache_mb": 239, "Skip_pull_existing": false, "registry": "/root/open-lambda/test-registry", "max_runners": 0, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "sandbox": "sock", "docker_runtime": "", "registry_cache_ms": 5000, "server_mode": "lambda", "timing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:14:19 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:19 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:19 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:19 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:19 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:19 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:19 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:19 remove worker.pid", "2019/06/21 20:14:19 exiting", "" ] } RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=10279, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 10279 { "test": "max_mem_alloc", "params": {}, "pass": true, "seconds": 0.9946351051330566, "total_seconds": 9.156257629394531, "stats": null, "conf": { "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "import_cache_mb": 239, "worker_dir": "/root/open-lambda/test-cluster/worker", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "benchmark_log": "", "pip_mirror": "", "sandbox_config": { "processes": 10 }, "cluster_name": "/root/open-lambda/test-cluster", "handler_cache_mb": 239, "Skip_pull_existing": false, "registry": "/root/open-lambda/test-registry", "max_runners": 0, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "sandbox": "sock", "docker_runtime": "", "registry_cache_ms": 5000, "server_mode": "lambda", "timing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:14:29 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:29 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:29 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:29 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:29 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:29 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:29 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:29 remove worker.pid", "2019/06/21 20:14:29 exiting", "" ] } POP conf: {'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson'], 'registry': '/root/open-lambda/test-registry'} PUSH conf: {'sandbox': 'sock', 'import_cache_mb': 256, 'registry': '/root/open-lambda/test-registry', 'handler_cache_mb': 256} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=10396, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 10396 { "test": "stress_one_lambda", "params": { "seconds": 15, "procs": 1 }, "pass": true, "seconds": 15.04509711265564, "total_seconds": 15.380200624465942, "stats": { "reqs_per_sec": 275.53333333333336 }, "conf": { "startup_pkgs": null, "import_cache_mb": 256, "registry_cache_ms": 5000, "worker_dir": "/root/open-lambda/test-cluster/worker", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "benchmark_log": "", "pip_mirror": "", "docker_runtime": "", "sandbox_config": { "processes": 10 }, "cluster_name": "/root/open-lambda/test-cluster", "handler_cache_mb": 256, "Skip_pull_existing": false, "server_mode": "lambda", "max_runners": 0, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sandbox": "sock", "registry": "/root/open-lambda/test-registry", "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "timing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:14:44 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:44 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:44 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:44 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:44 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:44 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:44 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:44 remove worker.pid", "2019/06/21 20:14:44 exiting", "" ] } RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=10437, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 10437 { "test": "stress_one_lambda", "params": { "seconds": 15, "procs": 2 }, "pass": true, "seconds": 15.047284126281738, "total_seconds": 15.383710384368896, "stats": { "reqs_per_sec": 270.3333333333333 }, "conf": { "startup_pkgs": null, "import_cache_mb": 256, "registry_cache_ms": 5000, "worker_dir": "/root/open-lambda/test-cluster/worker", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "benchmark_log": "", "pip_mirror": "", "docker_runtime": "", "sandbox_config": { "processes": 10 }, "cluster_name": "/root/open-lambda/test-cluster", "handler_cache_mb": 256, "Skip_pull_existing": false, "server_mode": "lambda", "max_runners": 0, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sandbox": "sock", "registry": "/root/open-lambda/test-registry", "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "timing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:14:59 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:59 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:59 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:59 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:59 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:59 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:59 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:14:59 remove worker.pid", "2019/06/21 20:14:59 exiting", "" ] } RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=10483, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 10483 { "test": "stress_one_lambda", "params": { "seconds": 15, "procs": 8 }, "pass": true, "seconds": 15.07036566734314, "total_seconds": 15.40476393699646, "stats": { "reqs_per_sec": 229.2 }, "conf": { "startup_pkgs": null, "import_cache_mb": 256, "registry_cache_ms": 5000, "worker_dir": "/root/open-lambda/test-cluster/worker", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "benchmark_log": "", "pip_mirror": "", "docker_runtime": "", "sandbox_config": { "processes": 10 }, "cluster_name": "/root/open-lambda/test-cluster", "handler_cache_mb": 256, "Skip_pull_existing": false, "server_mode": "lambda", "max_runners": 0, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sandbox": "sock", "registry": "/root/open-lambda/test-registry", "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "timing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:15:15 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:15:15 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:15:15 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:15:15 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:15:15 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:15:15 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:15:15 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:15:15 remove worker.pid", "2019/06/21 20:15:15 exiting", "" ] } POP conf: {'sandbox': 'sock', 'import_cache_mb': 256, 'registry': '/root/open-lambda/test-registry', 'handler_cache_mb': 256} PUSH conf: {'sandbox': 'sock', 'import_cache_mb': 256, 'handler_cache_mb': 256} PUSH conf: {'registry': '/tmp/tmp60hv8tqm'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=10567, port=5000, log=/root/open-lambda/test-cluster/worker.out ready out of memory, trying to kill OL { "test": "call_each_once_exec", "params": { "lambda_count": 100, "alloc_mb": 1 }, "pass": false, "seconds": null, "total_seconds": 5.9607298374176025, "stats": null, "conf": { "startup_pkgs": null, "import_cache_mb": 256, "worker_dir": "/root/open-lambda/test-cluster/worker", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "benchmark_log": "", "pip_mirror": "", "sandbox_config": { "processes": 10 }, "cluster_name": "/root/open-lambda/test-cluster", "handler_cache_mb": 256, "Skip_pull_existing": false, "registry": "/tmp/tmp60hv8tqm", "max_runners": 0, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "sandbox": "sock", "docker_runtime": "", "registry_cache_ms": 5000, "server_mode": "lambda", "timing": false }, "exception": [ "Traceback (most recent call last):", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py\", line 603, in urlopen", " chunked=chunked)", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py\", line 387, in _make_request", " six.raise_from(e, None)", " File \"\", line 2, in raise_from", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py\", line 383, in _make_request", " httplib_response = conn.getresponse()", " File \"/usr/lib/python3.5/http/client.py\", line 1197, in getresponse", " response.begin()", " File \"/usr/lib/python3.5/http/client.py\", line 297, in begin", " version, status, reason = self._read_status()", " File \"/usr/lib/python3.5/http/client.py\", line 266, in _read_status", " raise RemoteDisconnected(\"Remote end closed connection without\"", "http.client.RemoteDisconnected: Remote end closed connection without response", "", "During handling of the above exception, another exception occurred:", "", "Traceback (most recent call last):", " File \"/usr/lib/python3/dist-packages/requests/adapters.py\", line 376, in send", " timeout=timeout", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py\", line 641, in urlopen", " _stacktrace=sys.exc_info()[2])", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/util/retry.py\", line 368, in increment", " raise six.reraise(type(error), error, _stacktrace)", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/packages/six.py\", line 685, in reraise", " raise value.with_traceback(tb)", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py\", line 603, in urlopen", " chunked=chunked)", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py\", line 387, in _make_request", " six.raise_from(e, None)", " File \"\", line 2, in raise_from", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py\", line 383, in _make_request", " httplib_response = conn.getresponse()", " File \"/usr/lib/python3.5/http/client.py\", line 1197, in getresponse", " response.begin()", " File \"/usr/lib/python3.5/http/client.py\", line 297, in begin", " version, status, reason = self._read_status()", " File \"/usr/lib/python3.5/http/client.py\", line 266, in _read_status", " raise RemoteDisconnected(\"Remote end closed connection without\"", "requests.packages.urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))", "", "During handling of the above exception, another exception occurred:", "", "Traceback (most recent call last):", " File \"test.py\", line 55, in wrapper", " rv = fn(**kwargs)", " File \"test.py\", line 175, in call_each_once_exec", " r = requests.post(\"http://localhost:5000/run/L%d\"%i, data=json.dumps({\"alloc_mb\": alloc_mb}))", " File \"/usr/lib/python3/dist-packages/requests/api.py\", line 107, in post", " return request('post', url, data=data, json=json, **kwargs)", " File \"/usr/lib/python3/dist-packages/requests/api.py\", line 53, in request", " return session.request(method=method, url=url, **kwargs)", " File \"/usr/lib/python3/dist-packages/requests/sessions.py\", line 480, in request", " resp = self.send(prep, **send_kwargs)", " File \"/usr/lib/python3/dist-packages/requests/sessions.py\", line 588, in send", " r = adapter.send(request, **kwargs)", " File \"/usr/lib/python3/dist-packages/requests/adapters.py\", line 426, in send", " raise ConnectionError(err, request=request)", "requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))", "" ], "worker_tail": [ "using existing OL directory at /root/open-lambda/test-cluster", "2019/06/21 20:15:15 save PID 10567 to file /root/open-lambda/test-cluster/worker/worker.pid", "2019/06/21 20:15:15 Start Lambda Server", "2019/06/21 20:15:15 Create CodePuller", "2019/06/21 20:15:15 Initialized CodePuller (took 21.693\u00b5s)", "2019/06/21 20:15:15 Create InstallManager", "2019/06/21 20:15:15 Create InstallManager (took 2.206\u00b5s)", "2019/06/21 20:15:15 Create ContainerFactory", "2019/06/21 20:15:15 .Create(, /root/open-lambda/test-cluster/worker/import-cache, [], )", "2019/06/21 20:15:15 create /sys/fs/cgroup/blkio/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:15:15 create /sys/fs/cgroup/cpu/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:15:15 create /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:15:15 create /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:15:15 create /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:15:15 create /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:15:15 create /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:15:15 create /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:15:15 create /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:15:15 start creating/serving CGs [CGROUP POOL sock-cache]", "2019/06/21 20:15:15 create /sys/fs/cgroup/blkio/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:15:15 create /sys/fs/cgroup/cpu/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:15:15 create /sys/fs/cgroup/devices/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:15:15 create /sys/fs/cgroup/freezer/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:15:15 create /sys/fs/cgroup/hugetlb/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:15:15 create /sys/fs/cgroup/memory/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:15:15 create /sys/fs/cgroup/perf_event/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:15:15 create /sys/fs/cgroup/systemd/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:15:15 create /sys/fs/cgroup/pids/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:15:15 start creating/serving CGs [CGROUP POOL sock-handlers]", "2019/06/21 20:15:15 Initialized handler container factory (took 150.239869ms)", "2019/06/21 20:15:15 Create Benchmark", "2019/06/21 20:15:15 Setups Handlers", "2019/06/21 20:15:15 Execute handler by POSTing to localhost:5000/run/", "2019/06/21 20:15:15 Get status by sending request to localhost:5000/status", "2019/06/21 20:15:15 Receive request to /pid", "2019/06/21 20:15:15 Receive request to /run/L0", "2019/06/21 20:15:15 .Create(/root/open-lambda/test-cluster/worker/lambda_code/1-L0, /root/open-lambda/test-cluster/worker/handlers/L0, [], )", "2019/06/21 20:15:15 Channel() [SB 2]", "2019/06/21 20:15:15 Pause() [SB 2]", "2019/06/21 20:15:15 MemUsageKB() [SB 2]", "2019/06/21 20:15:15 Receive request to /run/L1", "2019/06/21 20:15:15 .Create(/root/open-lambda/test-cluster/worker/lambda_code/2-L1, /root/open-lambda/test-cluster/worker/handlers/L1, [], )", "2019/06/21 20:15:15 Channel() [SB 3]", "2019/06/21 20:15:15 Pause() [SB 3]", "2019/06/21 20:15:15 MemUsageKB() [SB 3]", "2019/06/21 20:15:15 Receive request to /run/L2", "2019/06/21 20:15:15 .Create(/root/open-lambda/test-cluster/worker/lambda_code/3-L2, /root/open-lambda/test-cluster/worker/handlers/L2, [], )", "2019/06/21 20:15:16 Channel() [SB 4]", "2019/06/21 20:15:16 Pause() [SB 4]", "2019/06/21 20:15:16 MemUsageKB() [SB 4]", "2019/06/21 20:15:16 Receive request to /run/L3", "2019/06/21 20:15:16 .Create(/root/open-lambda/test-cluster/worker/lambda_code/4-L3, /root/open-lambda/test-cluster/worker/handlers/L3, [], )", "2019/06/21 20:15:16 Channel() [SB 5]", "2019/06/21 20:15:16 Pause() [SB 5]", "2019/06/21 20:15:16 MemUsageKB() [SB 5]", "2019/06/21 20:15:16 Receive request to /run/L4", "2019/06/21 20:15:16 .Create(/root/open-lambda/test-cluster/worker/lambda_code/5-L4, /root/open-lambda/test-cluster/worker/handlers/L4, [], )", "2019/06/21 20:15:16 Channel() [SB 6]", "2019/06/21 20:15:16 Pause() [SB 6]", "2019/06/21 20:15:16 MemUsageKB() [SB 6]", "2019/06/21 20:15:16 Receive request to /run/L5", "2019/06/21 20:15:16 .Create(/root/open-lambda/test-cluster/worker/lambda_code/6-L5, /root/open-lambda/test-cluster/worker/handlers/L5, [], )", "2019/06/21 20:15:16 Channel() [SB 7]", "2019/06/21 20:15:16 Pause() [SB 7]", "2019/06/21 20:15:16 MemUsageKB() [SB 7]", "2019/06/21 20:15:16 Receive request to /run/L6", "2019/06/21 20:15:16 .Create(/root/open-lambda/test-cluster/worker/lambda_code/7-L6, /root/open-lambda/test-cluster/worker/handlers/L6, [], )", "2019/06/21 20:15:16 Channel() [SB 8]", "2019/06/21 20:15:16 Pause() [SB 8]", "2019/06/21 20:15:16 MemUsageKB() [SB 8]", "2019/06/21 20:15:16 Receive request to /run/L7", "2019/06/21 20:15:16 .Create(/root/open-lambda/test-cluster/worker/lambda_code/8-L7, /root/open-lambda/test-cluster/worker/handlers/L7, [], )", "2019/06/21 20:15:16 Channel() [SB 9]", "2019/06/21 20:15:16 Pause() [SB 9]", "2019/06/21 20:15:16 MemUsageKB() [SB 9]", "2019/06/21 20:15:16 Receive request to /run/L8", "2019/06/21 20:15:16 .Create(/root/open-lambda/test-cluster/worker/lambda_code/9-L8, /root/open-lambda/test-cluster/worker/handlers/L8, [], )", "2019/06/21 20:15:16 Channel() [SB 10]", "2019/06/21 20:15:16 Pause() [SB 10]", "2019/06/21 20:15:16 MemUsageKB() [SB 10]", "2019/06/21 20:15:16 Receive request to /run/L9", "2019/06/21 20:15:16 .Create(/root/open-lambda/test-cluster/worker/lambda_code/10-L9, /root/open-lambda/test-cluster/worker/handlers/L9, [], )", "2019/06/21 20:15:17 Channel() [SB 11]", "2019/06/21 20:15:17 Pause() [SB 11]", "2019/06/21 20:15:17 MemUsageKB() [SB 11]", "2019/06/21 20:15:17 Receive request to /run/L10", "2019/06/21 20:15:17 .Create(/root/open-lambda/test-cluster/worker/lambda_code/11-L10, /root/open-lambda/test-cluster/worker/handlers/L10, [], )", "2019/06/21 20:15:17 Channel() [SB 12]", "2019/06/21 20:15:17 Pause() [SB 12]", "2019/06/21 20:15:17 MemUsageKB() [SB 12]", "2019/06/21 20:15:17 Receive request to /run/L11", "2019/06/21 20:15:17 .Create(/root/open-lambda/test-cluster/worker/lambda_code/12-L11, /root/open-lambda/test-cluster/worker/handlers/L11, [], )", "2019/06/21 20:15:17 Channel() [SB 13]", "2019/06/21 20:15:17 Pause() [SB 13]", "2019/06/21 20:15:17 MemUsageKB() [SB 13]", "2019/06/21 20:15:17 Receive request to /run/L12", "2019/06/21 20:15:17 .Create(/root/open-lambda/test-cluster/worker/lambda_code/13-L12, /root/open-lambda/test-cluster/worker/handlers/L12, [], )", "2019/06/21 20:15:17 Channel() [SB 14]", "2019/06/21 20:15:17 Pause() [SB 14]", "2019/06/21 20:15:17 MemUsageKB() [SB 14]", "2019/06/21 20:15:17 Receive request to /run/L13", "2019/06/21 20:15:17 .Create(/root/open-lambda/test-cluster/worker/lambda_code/14-L13, /root/open-lambda/test-cluster/worker/handlers/L13, [], )", "2019/06/21 20:15:17 Channel() [SB 15]", "2019/06/21 20:15:17 Pause() [SB 15]", "2019/06/21 20:15:17 MemUsageKB() [SB 15]", "2019/06/21 20:15:17 Receive request to /run/L14", "2019/06/21 20:15:17 .Create(/root/open-lambda/test-cluster/worker/lambda_code/15-L14, /root/open-lambda/test-cluster/worker/handlers/L14, [], )", "2019/06/21 20:15:17 Channel() [SB 16]", "2019/06/21 20:15:17 Pause() [SB 16]", "2019/06/21 20:15:17 MemUsageKB() [SB 16]", "2019/06/21 20:15:17 Receive request to /run/L15", "2019/06/21 20:15:17 .Create(/root/open-lambda/test-cluster/worker/lambda_code/16-L15, /root/open-lambda/test-cluster/worker/handlers/L15, [], )", "2019/06/21 20:15:18 Channel() [SB 17]", "2019/06/21 20:15:18 Pause() [SB 17]", "2019/06/21 20:15:18 MemUsageKB() [SB 17]", "2019/06/21 20:15:18 Receive request to /run/L16", "2019/06/21 20:15:18 .Create(/root/open-lambda/test-cluster/worker/lambda_code/17-L16, /root/open-lambda/test-cluster/worker/handlers/L16, [], )", "2019/06/21 20:15:18 Channel() [SB 18]", "2019/06/21 20:15:18 Pause() [SB 18]", "2019/06/21 20:15:18 MemUsageKB() [SB 18]", "2019/06/21 20:15:18 Receive request to /run/L17", "2019/06/21 20:15:18 .Create(/root/open-lambda/test-cluster/worker/lambda_code/18-L17, /root/open-lambda/test-cluster/worker/handlers/L17, [], )", "2019/06/21 20:15:18 Channel() [SB 19]", "2019/06/21 20:15:18 Pause() [SB 19]", "2019/06/21 20:15:18 MemUsageKB() [SB 19]", "2019/06/21 20:15:18 Receive request to /run/L18", "2019/06/21 20:15:18 .Create(/root/open-lambda/test-cluster/worker/lambda_code/19-L18, /root/open-lambda/test-cluster/worker/handlers/L18, [], )", "2019/06/21 20:15:18 Channel() [SB 20]", "2019/06/21 20:15:18 Pause() [SB 20]", "2019/06/21 20:15:18 MemUsageKB() [SB 20]", "2019/06/21 20:15:18 Receive request to /run/L19", "2019/06/21 20:15:18 .Create(/root/open-lambda/test-cluster/worker/lambda_code/20-L19, /root/open-lambda/test-cluster/worker/handlers/L19, [], )", "2019/06/21 20:15:18 Channel() [SB 21]", "2019/06/21 20:15:18 Pause() [SB 21]", "2019/06/21 20:15:18 MemUsageKB() [SB 21]", "2019/06/21 20:15:18 Receive request to /run/L20", "2019/06/21 20:15:18 .Create(/root/open-lambda/test-cluster/worker/lambda_code/21-L20, /root/open-lambda/test-cluster/worker/handlers/L20, [], )", "2019/06/21 20:15:18 Channel() [SB 22]", "2019/06/21 20:15:18 Pause() [SB 22]", "2019/06/21 20:15:18 MemUsageKB() [SB 22]", "2019/06/21 20:15:18 Receive request to /run/L21", "2019/06/21 20:15:18 .Create(/root/open-lambda/test-cluster/worker/lambda_code/22-L21, /root/open-lambda/test-cluster/worker/handlers/L21, [], )", "2019/06/21 20:15:19 Channel() [SB 23]", "2019/06/21 20:15:19 Pause() [SB 23]", "2019/06/21 20:15:19 MemUsageKB() [SB 23]", "2019/06/21 20:15:19 Receive request to /run/L22", "2019/06/21 20:15:19 .Create(/root/open-lambda/test-cluster/worker/lambda_code/23-L22, /root/open-lambda/test-cluster/worker/handlers/L22, [], )", "2019/06/21 20:15:19 Channel() [SB 24]", "2019/06/21 20:15:19 Pause() [SB 24]", "2019/06/21 20:15:19 MemUsageKB() [SB 24]", "2019/06/21 20:15:19 Receive request to /run/L23", "2019/06/21 20:15:19 .Create(/root/open-lambda/test-cluster/worker/lambda_code/24-L23, /root/open-lambda/test-cluster/worker/handlers/L23, [], )", "2019/06/21 20:15:19 Channel() [SB 25]", "2019/06/21 20:15:19 Pause() [SB 25]", "2019/06/21 20:15:19 MemUsageKB() [SB 25]", "2019/06/21 20:15:19 Receive request to /run/L24", "2019/06/21 20:15:19 .Create(/root/open-lambda/test-cluster/worker/lambda_code/25-L24, /root/open-lambda/test-cluster/worker/handlers/L24, [], )", "2019/06/21 20:15:19 Channel() [SB 26]", "2019/06/21 20:15:19 Pause() [SB 26]", "2019/06/21 20:15:19 MemUsageKB() [SB 26]", "2019/06/21 20:15:19 Receive request to /run/L25", "2019/06/21 20:15:19 .Create(/root/open-lambda/test-cluster/worker/lambda_code/26-L25, /root/open-lambda/test-cluster/worker/handlers/L25, [], )", "2019/06/21 20:15:19 Channel() [SB 27]", "2019/06/21 20:15:19 Pause() [SB 27]", "2019/06/21 20:15:19 MemUsageKB() [SB 27]", "2019/06/21 20:15:19 Receive request to /run/L26", "2019/06/21 20:15:19 .Create(/root/open-lambda/test-cluster/worker/lambda_code/27-L26, /root/open-lambda/test-cluster/worker/handlers/L26, [], )", "2019/06/21 20:15:19 Channel() [SB 28]", "2019/06/21 20:15:19 Pause() [SB 28]", "2019/06/21 20:15:19 MemUsageKB() [SB 28]", "2019/06/21 20:15:19 Receive request to /run/L27", "2019/06/21 20:15:19 .Create(/root/open-lambda/test-cluster/worker/lambda_code/28-L27, /root/open-lambda/test-cluster/worker/handlers/L27, [], )", "2019/06/21 20:15:20 Channel() [SB 29]", "2019/06/21 20:15:20 Pause() [SB 29]", "2019/06/21 20:15:20 MemUsageKB() [SB 29]", "2019/06/21 20:15:20 Receive request to /run/L28", "2019/06/21 20:15:20 .Create(/root/open-lambda/test-cluster/worker/lambda_code/29-L28, /root/open-lambda/test-cluster/worker/handlers/L28, [], )", "2019/06/21 20:15:20 Channel() [SB 30]", "2019/06/21 20:15:20 Pause() [SB 30]", "2019/06/21 20:15:20 MemUsageKB() [SB 30]", "2019/06/21 20:15:20 Receive request to /run/L29", "2019/06/21 20:15:20 .Create(/root/open-lambda/test-cluster/worker/lambda_code/30-L29, /root/open-lambda/test-cluster/worker/handlers/L29, [], )", "2019/06/21 20:15:20 Channel() [SB 31]", "2019/06/21 20:15:20 Pause() [SB 31]", "2019/06/21 20:15:20 MemUsageKB() [SB 31]", "2019/06/21 20:15:20 Receive request to /run/L30", "2019/06/21 20:15:20 .Create(/root/open-lambda/test-cluster/worker/lambda_code/31-L30, /root/open-lambda/test-cluster/worker/handlers/L30, [], )", "2019/06/21 20:15:21 received kill signal, cleaning up", "2019/06/21 20:15:21 Destroy() [SB 2]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 2]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 2]", "2019/06/21 20:15:21 Destroy() [SB 3]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 3]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 3]", "2019/06/21 20:15:21 Destroy() [SB 4]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 4]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 4]", "2019/06/21 20:15:21 Destroy() [SB 5]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 5]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 5]", "2019/06/21 20:15:21 Destroy() [SB 6]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 6]", "2019/06/21 20:15:21 waiting for 1 procs in cg-5 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 6]", "2019/06/21 20:15:21 Destroy() [SB 7]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 7]", "2019/06/21 20:15:21 waiting for 1 procs in cg-6 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 7]", "2019/06/21 20:15:21 Destroy() [SB 8]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 8]", "2019/06/21 20:15:21 waiting for 1 procs in cg-7 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 8]", "2019/06/21 20:15:21 Destroy() [SB 9]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 9]", "2019/06/21 20:15:21 waiting for 1 procs in cg-8 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 9]", "2019/06/21 20:15:21 Destroy() [SB 10]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 10]", "2019/06/21 20:15:21 waiting for 1 procs in cg-9 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 10]", "2019/06/21 20:15:21 Destroy() [SB 11]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 11]", "2019/06/21 20:15:21 waiting for 1 procs in cg-10 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 11]", "2019/06/21 20:15:21 Destroy() [SB 12]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 12]", "2019/06/21 20:15:21 waiting for 1 procs in cg-11 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 12]", "2019/06/21 20:15:21 Destroy() [SB 13]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 13]", "2019/06/21 20:15:21 waiting for 1 procs in cg-12 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 13]", "2019/06/21 20:15:21 Destroy() [SB 14]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 14]", "2019/06/21 20:15:21 waiting for 1 procs in cg-13 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 14]", "2019/06/21 20:15:21 Destroy() [SB 15]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 15]", "2019/06/21 20:15:21 waiting for 1 procs in cg-14 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 15]", "2019/06/21 20:15:21 Destroy() [SB 16]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 16]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 16]", "2019/06/21 20:15:21 Destroy() [SB 17]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 17]", "2019/06/21 20:15:21 waiting for 1 procs in cg-16 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 17]", "2019/06/21 20:15:21 Destroy() [SB 18]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 18]", "2019/06/21 20:15:21 waiting for 1 procs in cg-17 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 18]", "2019/06/21 20:15:21 Destroy() [SB 19]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 19]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 19]", "2019/06/21 20:15:21 Destroy() [SB 20]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 20]", "2019/06/21 20:15:21 waiting for 1 procs in cg-19 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 20]", "2019/06/21 20:15:21 Destroy() [SB 21]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 21]", "2019/06/21 20:15:21 waiting for 1 procs in cg-20 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 21]", "2019/06/21 20:15:21 Destroy() [SB 22]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 22]", "2019/06/21 20:15:21 waiting for 1 procs in cg-21 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 22]", "2019/06/21 20:15:21 Destroy() [SB 23]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 23]", "2019/06/21 20:15:21 waiting for 1 procs in cg-22 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 23]", "2019/06/21 20:15:21 Destroy() [SB 24]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 24]", "2019/06/21 20:15:21 waiting for 1 procs in cg-23 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 24]", "2019/06/21 20:15:21 Destroy() [SB 25]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 25]", "2019/06/21 20:15:21 waiting for 1 procs in cg-24 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 25]", "2019/06/21 20:15:21 Destroy() [SB 26]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 26]", "2019/06/21 20:15:21 waiting for 1 procs in cg-25 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 26]", "2019/06/21 20:15:21 Destroy() [SB 27]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 27]", "2019/06/21 20:15:21 waiting for 1 procs in cg-26 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 27]", "2019/06/21 20:15:21 Destroy() [SB 28]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 28]", "2019/06/21 20:15:21 waiting for 1 procs in cg-27 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 28]", "2019/06/21 20:15:21 Destroy() [SB 29]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 29]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 29]", "2019/06/21 20:15:21 Destroy() [SB 30]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 30]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 30]", "2019/06/21 20:15:21 Destroy() [SB 31]", "2019/06/21 20:15:21 kill all procs in CG [SOCK 31]", "2019/06/21 20:15:21 unmount and remove dirs [SOCK 31]", "2019/06/21 20:15:21 received shutdown request [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 empty queues and release CGs [CGROUP POOL sock-handlers]", "2019/06/21 20:15:21 remove /sys/fs/cgroup/blkio/sock-handlers [CGROUP POOL sock-handlers]", "panic: Rmdir /sys/fs/cgroup/blkio/sock-handlers: device or resource busy", "", "goroutine 6 [running]:", "github.com/open-lambda/open-lambda/ol/sandbox.(*CgroupPool).cgTask(0xc0001a8180)", "\t/root/open-lambda/src/sandbox/cgroupFactory.go:119 +0xabb", "created by github.com/open-lambda/open-lambda/ol/sandbox.NewCgroupPool", "\t/root/open-lambda/src/sandbox/cgroupFactory.go:46 +0xfe", "" ] } POP conf: {'registry': '/tmp/tmp60hv8tqm'} PUSH conf: {'registry': '/tmp/tmp2ptw_9zp'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=10745, port=5000, log=/root/open-lambda/test-cluster/worker.out 2019/06/21 20:15:21 worker process 10745 does not a appear to be running, check worker.out { "test": "call_each_once_exec", "params": { "lambda_count": 1000, "alloc_mb": 10 }, "pass": false, "seconds": null, "total_seconds": 0.13235211372375488, "stats": null, "conf": { "startup_pkgs": null, "import_cache_mb": 256, "registry_cache_ms": 5000, "worker_dir": "/root/open-lambda/test-cluster/worker", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "benchmark_log": "", "pip_mirror": "", "sandbox_config": { "processes": 10 }, "cluster_name": "/root/open-lambda/test-cluster", "handler_cache_mb": 256, "Skip_pull_existing": false, "registry": "/tmp/tmp2ptw_9zp", "max_runners": 0, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sandbox": "sock", "docker_runtime": "", "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "server_mode": "lambda", "timing": false }, "exception": [ "Traceback (most recent call last):", " File \"test.py\", line 51, in wrapper", " run(['./ol', 'worker', '-p='+OLDIR, '--detach'])", " File \"test.py\", line 128, in run", " raise Exception(\"command failed: \" + \" \".join(cmd) + \", \")", "Exception: command failed: ./ol worker -p=test-cluster --detach, ", "" ], "worker_tail": [ "using existing OL directory at /root/open-lambda/test-cluster", "2019/06/21 20:15:21 previous working may be running, /root/open-lambda/test-cluster/worker/worker.pid already exists", "" ] } POP conf: {'registry': '/tmp/tmp2ptw_9zp'} POP conf: {'sandbox': 'sock', 'import_cache_mb': 256, 'handler_cache_mb': 256} PASSED: 13, FAILED: 2 Makefile:28: recipe for target 'test-all' failed make: *** [test-all] Error 2