python3 -u test.py RUN ./ol new -p=test-cluster Init OL dir at /root/open-lambda/test-cluster Create lambda base at /root/open-lambda/test-cluster/lambda (may take several minutes) Working Directory: /root/open-lambda/test-cluster Worker Defaults: { "registry": "/root/open-lambda/test-cluster/registry", "registry_cache_ms": 5000, "sandbox": "sock", "cluster_name": "/root/open-lambda/test-cluster", "pip_mirror": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "max_runners": 0, "handler_cache_mb": 239, "import_cache_mb": 239, "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-cluster/worker", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "worker_port": "5000", "Skip_pull_existing": false, "sandbox_config": { "processes": 10 }, "benchmark_log": "", "timing": false, "startup_pkgs": null, "docker_runtime": "", "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 } } You may modify the defaults here: /root/open-lambda/test-cluster/config.json You may now start a server using the "ol worker" command PUSH conf: {'server_mode': 'sock'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7311, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7311 { "test": "sock_churn", "params": { "procs": 1, "baseline": 0, "fork": true, "seconds": 15 }, "pass": true, "seconds": 16.478682041168213, "total_seconds": 16.74481964111328, "stats": { "sandboxes_per_sec": 6.266666666666667 }, "conf": { "sandbox": "sock", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "registry": "/root/open-lambda/test-cluster/registry", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "server_mode": "sock", "cluster_name": "/root/open-lambda/test-cluster", "timing": false, "import_cache_mb": 239, "max_runners": 0, "sandbox_config": { "processes": 10 }, "worker_port": "5000", "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "benchmark_log": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "docker_runtime": "", "startup_pkgs": null, "handler_cache_mb": 239, "Skip_pull_existing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:02:37 remove /sys/fs/cgroup/devices/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:02:37 remove /sys/fs/cgroup/freezer/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:02:37 remove /sys/fs/cgroup/hugetlb/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:02:37 remove /sys/fs/cgroup/memory/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:02:37 remove /sys/fs/cgroup/perf_event/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:02:37 remove /sys/fs/cgroup/systemd/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:02:37 remove /sys/fs/cgroup/pids/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:02:37 remove worker.pid", "2019/06/21 20:02:37 exiting", "" ] } POP conf: {'server_mode': 'sock'} PUSH conf: {'registry': '/root/open-lambda/test-registry', 'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson']} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7718, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7718 { "test": "ping_test", "params": {}, "pass": true, "seconds": 2.154463052749634, "total_seconds": 25.381184101104736, "stats": { "pings_per_sec": 464.15667660462464 }, "conf": { "sandbox": "sock", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "registry": "/root/open-lambda/test-registry", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "server_mode": "lambda", "cluster_name": "/root/open-lambda/test-cluster", "timing": false, "worker_port": "5000", "max_runners": 0, "sandbox_config": { "processes": 10 }, "import_cache_mb": 239, "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "benchmark_log": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "docker_runtime": "", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "handler_cache_mb": 239, "Skip_pull_existing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:03:02 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:03:02 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:03:02 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:03:02 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:03:02 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:03:02 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:03:02 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:03:02 remove worker.pid", "2019/06/21 20:03:02 exiting", "" ] } PUSH conf: {'import_cache_mb': 0, 'handler_cache_mb': 0} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7792, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7792 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 1.7052664756774902, "total_seconds": 9.727548360824585, "stats": null, "conf": { "sandbox": "sock", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "registry": "/root/open-lambda/test-registry", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "server_mode": "lambda", "cluster_name": "/root/open-lambda/test-cluster", "timing": false, "worker_port": "5000", "max_runners": 0, "sandbox_config": { "processes": 10 }, "import_cache_mb": 0, "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "benchmark_log": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "docker_runtime": "", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "handler_cache_mb": 0, "Skip_pull_existing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:03:12 remove /sys/fs/cgroup/devices/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:03:12 remove /sys/fs/cgroup/freezer/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:03:12 remove /sys/fs/cgroup/hugetlb/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:03:12 remove /sys/fs/cgroup/memory/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:03:12 remove /sys/fs/cgroup/perf_event/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:03:12 remove /sys/fs/cgroup/systemd/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:03:12 remove /sys/fs/cgroup/pids/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:03:12 remove worker.pid", "2019/06/21 20:03:12 exiting", "" ] } POP conf: {'import_cache_mb': 0, 'handler_cache_mb': 0} PUSH conf: {'import_cache_mb': 0, 'handler_cache_mb': 256} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7867, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7867 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 1.4030821323394775, "total_seconds": 9.473402261734009, "stats": null, "conf": { "sandbox": "sock", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "registry": "/root/open-lambda/test-registry", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "server_mode": "lambda", "cluster_name": "/root/open-lambda/test-cluster", "timing": false, "import_cache_mb": 0, "max_runners": 0, "sandbox_config": { "processes": 10 }, "worker_port": "5000", "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "benchmark_log": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "docker_runtime": "", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "handler_cache_mb": 256, "Skip_pull_existing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:03:21 remove /sys/fs/cgroup/devices/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:03:21 remove /sys/fs/cgroup/freezer/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:03:21 remove /sys/fs/cgroup/hugetlb/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:03:21 remove /sys/fs/cgroup/memory/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:03:21 remove /sys/fs/cgroup/perf_event/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:03:21 remove /sys/fs/cgroup/systemd/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:03:21 remove /sys/fs/cgroup/pids/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:03:21 remove worker.pid", "2019/06/21 20:03:21 exiting", "" ] } POP conf: {'import_cache_mb': 0, 'handler_cache_mb': 256} PUSH conf: {'import_cache_mb': 256, 'handler_cache_mb': 0} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7945, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7945 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 1.4751842021942139, "total_seconds": 9.641322135925293, "stats": null, "conf": { "sandbox": "sock", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "registry": "/root/open-lambda/test-registry", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "server_mode": "lambda", "cluster_name": "/root/open-lambda/test-cluster", "timing": false, "worker_port": "5000", "max_runners": 0, "sandbox_config": { "processes": 10 }, "import_cache_mb": 256, "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "benchmark_log": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "docker_runtime": "", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "handler_cache_mb": 0, "Skip_pull_existing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:03:31 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:03:31 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:03:31 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:03:31 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:03:31 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:03:31 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:03:31 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:03:31 remove worker.pid", "2019/06/21 20:03:31 exiting", "" ] } POP conf: {'import_cache_mb': 256, 'handler_cache_mb': 0} PUSH conf: {'import_cache_mb': 256, 'handler_cache_mb': 256} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=8029, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 8029 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 1.4544758796691895, "total_seconds": 9.727824449539185, "stats": null, "conf": { "sandbox": "sock", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "registry": "/root/open-lambda/test-registry", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "server_mode": "lambda", "cluster_name": "/root/open-lambda/test-cluster", "timing": false, "import_cache_mb": 256, "max_runners": 0, "sandbox_config": { "processes": 10 }, "worker_port": "5000", "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "benchmark_log": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "docker_runtime": "", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "handler_cache_mb": 256, "Skip_pull_existing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:03:41 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:03:41 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:03:41 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:03:41 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:03:41 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:03:41 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:03:41 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:03:41 remove worker.pid", "2019/06/21 20:03:41 exiting", "" ] } POP conf: {'import_cache_mb': 256, 'handler_cache_mb': 256} PUSH conf: {'sandbox': 'docker', 'import_cache_mb': 0, 'handler_cache_mb': 0} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=8114, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 8114 { "test": "smoke_tests", "params": {}, "pass": false, "seconds": 16.506208896636963, "total_seconds": 25.367936849594116, "stats": null, "conf": { "sandbox": "docker", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "registry": "/root/open-lambda/test-registry", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "server_mode": "lambda", "cluster_name": "/root/open-lambda/test-cluster", "timing": false, "worker_port": "5000", "max_runners": 0, "sandbox_config": { "processes": 10 }, "import_cache_mb": 0, "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "benchmark_log": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "docker_runtime": "", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "handler_cache_mb": 0, "Skip_pull_existing": false }, "exception": [ "Traceback (most recent call last):", " File \"test.py\", line 64, in wrapper", " raise Exception(\"mounts are leaking (%d before, %d after)\" % (mounts0, mounts1))", "Exception: mounts are leaking (29 before, 31 after)", "" ], "worker_tail": [ "using existing OL directory at /root/open-lambda/test-cluster", "2019/06/21 20:03:41 save PID 8114 to file /root/open-lambda/test-cluster/worker/worker.pid", "2019/06/21 20:03:41 Start Lambda Server", "2019/06/21 20:03:41 Create CodePuller", "2019/06/21 20:03:41 Initialized CodePuller (took 22.701\u00b5s)", "2019/06/21 20:03:41 Create InstallManager", "2019/06/21 20:03:49 Create InstallManager (took 8.124932656s)", "2019/06/21 20:03:49 Create ContainerFactory", "2019/06/21 20:03:49 Initialized handler container factory (took 14.795\u00b5s)", "2019/06/21 20:03:49 Create Benchmark", "2019/06/21 20:03:49 Setups Handlers", "2019/06/21 20:03:49 Execute handler by POSTing to localhost:5000/run/", "2019/06/21 20:03:49 Get status by sending request to localhost:5000/status", "2019/06/21 20:03:49 Receive request to /pid", "2019/06/21 20:03:49 Receive request to /run/echo", "2019/06/21 20:03:57 Channel() [SB 0]", "2019/06/21 20:03:57 Pause() [SB 0]", "2019/06/21 20:03:57 MemUsageKB() [SB 0]", "2019/06/21 20:03:57 EVICTING INSTANCE: 30552064 used / 0 limit", "2019/06/21 20:03:57 Destroy() [SB 0]", "2019/06/21 20:03:57 Receive request to /run/install", "2019/06/21 20:04:00 Channel() [SB 1]", "2019/06/21 20:04:00 Pause() [SB 1]", "2019/06/21 20:04:00 MemUsageKB() [SB 1]", "2019/06/21 20:04:00 EVICTING INSTANCE: 25141248 used / 0 limit", "2019/06/21 20:04:00 Destroy() [SB 1]", "2019/06/21 20:04:00 Receive request to /run/install2", "2019/06/21 20:04:02 Channel() [SB 2]", "2019/06/21 20:04:02 Pause() [SB 2]", "2019/06/21 20:04:02 MemUsageKB() [SB 2]", "2019/06/21 20:04:02 EVICTING INSTANCE: 23932928 used / 0 limit", "2019/06/21 20:04:02 Destroy() [SB 2]", "2019/06/21 20:04:02 Receive request to /run/install3", "2019/06/21 20:04:06 Channel() [SB 3]", "2019/06/21 20:04:06 Pause() [SB 3]", "2019/06/21 20:04:06 MemUsageKB() [SB 3]", "2019/06/21 20:04:06 EVICTING INSTANCE: 25169920 used / 0 limit", "2019/06/21 20:04:06 Destroy() [SB 3]", "2019/06/21 20:04:06 received kill signal, cleaning up", "2019/06/21 20:04:06 Destroy() [SB 0]", "2019/06/21 20:04:06 Destroy() [SB 1]", "2019/06/21 20:04:06 Destroy() [SB 2]", "2019/06/21 20:04:06 Destroy() [SB 3]", "2019/06/21 20:04:06 remove worker.pid", "2019/06/21 20:04:06 exiting", "" ] } POP conf: {'sandbox': 'docker', 'import_cache_mb': 0, 'handler_cache_mb': 0} PUSH conf: {'sandbox': 'docker', 'import_cache_mb': 0, 'handler_cache_mb': 256} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=9260, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 9260 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 9.60530686378479, "total_seconds": 23.743417024612427, "stats": null, "conf": { "sandbox": "docker", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "registry": "/root/open-lambda/test-registry", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "server_mode": "lambda", "cluster_name": "/root/open-lambda/test-cluster", "timing": false, "import_cache_mb": 0, "max_runners": 0, "sandbox_config": { "processes": 10 }, "worker_port": "5000", "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "benchmark_log": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "docker_runtime": "", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "handler_cache_mb": 256, "Skip_pull_existing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:04:29 Pause() [SB 3]", "2019/06/21 20:04:29 MemUsageKB() [SB 3]", "2019/06/21 20:04:29 received kill signal, cleaning up", "2019/06/21 20:04:29 Destroy() [SB 0]", "2019/06/21 20:04:29 Destroy() [SB 1]", "2019/06/21 20:04:30 Destroy() [SB 2]", "2019/06/21 20:04:30 Destroy() [SB 3]", "2019/06/21 20:04:30 remove worker.pid", "2019/06/21 20:04:30 exiting", "" ] } POP conf: {'sandbox': 'docker', 'import_cache_mb': 0, 'handler_cache_mb': 256} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=10309, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 10309 { "test": "fork_bomb", "params": {}, "pass": true, "seconds": 0.17214083671569824, "total_seconds": 8.437884092330933, "stats": null, "conf": { "sandbox": "sock", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "registry": "/root/open-lambda/test-registry", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "server_mode": "lambda", "cluster_name": "/root/open-lambda/test-cluster", "timing": false, "worker_port": "5000", "max_runners": 0, "sandbox_config": { "processes": 10 }, "import_cache_mb": 239, "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "benchmark_log": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "docker_runtime": "", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "handler_cache_mb": 239, "Skip_pull_existing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:04:38 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:04:38 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:04:38 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:04:38 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:04:38 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:04:38 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:04:38 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:04:38 remove worker.pid", "2019/06/21 20:04:38 exiting", "" ] } RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=10385, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 10385 { "test": "max_mem_alloc", "params": {}, "pass": true, "seconds": 0.9682149887084961, "total_seconds": 9.131910562515259, "stats": null, "conf": { "sandbox": "sock", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "registry": "/root/open-lambda/test-registry", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "server_mode": "lambda", "cluster_name": "/root/open-lambda/test-cluster", "timing": false, "worker_port": "5000", "max_runners": 0, "sandbox_config": { "processes": 10 }, "import_cache_mb": 239, "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "benchmark_log": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "docker_runtime": "", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "handler_cache_mb": 239, "Skip_pull_existing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:04:48 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:04:48 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:04:48 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:04:48 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:04:48 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:04:48 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:04:48 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:04:48 remove worker.pid", "2019/06/21 20:04:48 exiting", "" ] } POP conf: {'registry': '/root/open-lambda/test-registry', 'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson']} PUSH conf: {'sandbox': 'sock', 'import_cache_mb': 256, 'registry': '/root/open-lambda/test-registry', 'handler_cache_mb': 256} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=10502, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 10502 { "test": "stress_one_lambda", "params": { "procs": 1, "seconds": 15 }, "pass": true, "seconds": 15.048537015914917, "total_seconds": 15.384848594665527, "stats": { "reqs_per_sec": 277.53333333333336 }, "conf": { "sandbox": "sock", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "registry": "/root/open-lambda/test-registry", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "server_mode": "lambda", "cluster_name": "/root/open-lambda/test-cluster", "timing": false, "import_cache_mb": 256, "max_runners": 0, "sandbox_config": { "processes": 10 }, "worker_port": "5000", "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "benchmark_log": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "docker_runtime": "", "startup_pkgs": null, "handler_cache_mb": 256, "Skip_pull_existing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:05:03 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:03 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:03 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:03 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:03 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:03 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:03 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:03 remove worker.pid", "2019/06/21 20:05:03 exiting", "" ] } RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=10543, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 10543 { "test": "stress_one_lambda", "params": { "procs": 2, "seconds": 15 }, "pass": true, "seconds": 15.048274517059326, "total_seconds": 15.395285844802856, "stats": { "reqs_per_sec": 272.6666666666667 }, "conf": { "sandbox": "sock", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "registry": "/root/open-lambda/test-registry", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "server_mode": "lambda", "cluster_name": "/root/open-lambda/test-cluster", "timing": false, "import_cache_mb": 256, "max_runners": 0, "sandbox_config": { "processes": 10 }, "worker_port": "5000", "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "benchmark_log": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "docker_runtime": "", "startup_pkgs": null, "handler_cache_mb": 256, "Skip_pull_existing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:05:18 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:18 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:18 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:18 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:18 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:18 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:18 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:18 remove worker.pid", "2019/06/21 20:05:18 exiting", "" ] } RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=10589, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 10589 { "test": "stress_one_lambda", "params": { "procs": 8, "seconds": 15 }, "pass": true, "seconds": 15.0749831199646, "total_seconds": 15.41492772102356, "stats": { "reqs_per_sec": 232.53333333333333 }, "conf": { "sandbox": "sock", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "registry": "/root/open-lambda/test-registry", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "server_mode": "lambda", "cluster_name": "/root/open-lambda/test-cluster", "timing": false, "import_cache_mb": 256, "max_runners": 0, "sandbox_config": { "processes": 10 }, "worker_port": "5000", "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "benchmark_log": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "docker_runtime": "", "startup_pkgs": null, "handler_cache_mb": 256, "Skip_pull_existing": false }, "exception": null, "worker_tail": [ "2019/06/21 20:05:34 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:34 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:34 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:34 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:34 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:34 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:34 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:34 remove worker.pid", "2019/06/21 20:05:34 exiting", "" ] } POP conf: {'sandbox': 'sock', 'import_cache_mb': 256, 'registry': '/root/open-lambda/test-registry', 'handler_cache_mb': 256} PUSH conf: {'sandbox': 'sock', 'import_cache_mb': 256, 'handler_cache_mb': 256} PUSH conf: {'registry': '/tmp/tmp19gi_h7w'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=10671, port=5000, log=/root/open-lambda/test-cluster/worker.out ready out of memory, trying to kill OL Exception in thread Thread-1: Traceback (most recent call last): File "/usr/lib/python3.5/threading.py", line 914, in _bootstrap_inner self.run() File "/usr/lib/python3.5/threading.py", line 862, in run self._target(*self._args, **self._kwargs) File "test.py", line 28, in ol_oom_killer os.system(['pkill', 'ol']) TypeError: Can't convert 'list' object to str implicitly { "test": "call_each_once_exec", "params": { "lambda_count": 100, "alloc_mb": 1 }, "pass": false, "seconds": null, "total_seconds": 30.545791625976562, "stats": null, "conf": { "sandbox": "sock", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "registry": "/tmp/tmp19gi_h7w", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "server_mode": "lambda", "cluster_name": "/root/open-lambda/test-cluster", "timing": false, "worker_port": "5000", "max_runners": 0, "sandbox_config": { "processes": 10 }, "import_cache_mb": 256, "sock_cgroups": { "max_mem_mb": 50, "max_procs": 10 }, "benchmark_log": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "docker_runtime": "", "startup_pkgs": null, "handler_cache_mb": 256, "Skip_pull_existing": false }, "exception": [ "Traceback (most recent call last):", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py\", line 603, in urlopen", " chunked=chunked)", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py\", line 387, in _make_request", " six.raise_from(e, None)", " File \"\", line 2, in raise_from", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py\", line 383, in _make_request", " httplib_response = conn.getresponse()", " File \"/usr/lib/python3.5/http/client.py\", line 1197, in getresponse", " response.begin()", " File \"/usr/lib/python3.5/http/client.py\", line 297, in begin", " version, status, reason = self._read_status()", " File \"/usr/lib/python3.5/http/client.py\", line 266, in _read_status", " raise RemoteDisconnected(\"Remote end closed connection without\"", "http.client.RemoteDisconnected: Remote end closed connection without response", "", "During handling of the above exception, another exception occurred:", "", "Traceback (most recent call last):", " File \"/usr/lib/python3/dist-packages/requests/adapters.py\", line 376, in send", " timeout=timeout", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py\", line 641, in urlopen", " _stacktrace=sys.exc_info()[2])", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/util/retry.py\", line 368, in increment", " raise six.reraise(type(error), error, _stacktrace)", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/packages/six.py\", line 685, in reraise", " raise value.with_traceback(tb)", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py\", line 603, in urlopen", " chunked=chunked)", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py\", line 387, in _make_request", " six.raise_from(e, None)", " File \"\", line 2, in raise_from", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py\", line 383, in _make_request", " httplib_response = conn.getresponse()", " File \"/usr/lib/python3.5/http/client.py\", line 1197, in getresponse", " response.begin()", " File \"/usr/lib/python3.5/http/client.py\", line 297, in begin", " version, status, reason = self._read_status()", " File \"/usr/lib/python3.5/http/client.py\", line 266, in _read_status", " raise RemoteDisconnected(\"Remote end closed connection without\"", "requests.packages.urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))", "", "During handling of the above exception, another exception occurred:", "", "Traceback (most recent call last):", " File \"test.py\", line 55, in wrapper", " rv = fn(**kwargs)", " File \"test.py\", line 175, in call_each_once_exec", " r = requests.post(\"http://localhost:5000/run/L%d\"%i, data=json.dumps({\"alloc_mb\": alloc_mb}))", " File \"/usr/lib/python3/dist-packages/requests/api.py\", line 107, in post", " return request('post', url, data=data, json=json, **kwargs)", " File \"/usr/lib/python3/dist-packages/requests/api.py\", line 53, in request", " return session.request(method=method, url=url, **kwargs)", " File \"/usr/lib/python3/dist-packages/requests/sessions.py\", line 480, in request", " resp = self.send(prep, **send_kwargs)", " File \"/usr/lib/python3/dist-packages/requests/sessions.py\", line 588, in send", " r = adapter.send(request, **kwargs)", " File \"/usr/lib/python3/dist-packages/requests/adapters.py\", line 426, in send", " raise ConnectionError(err, request=request)", "requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))", "" ], "worker_tail": [ "using existing OL directory at /root/open-lambda/test-cluster", "2019/06/21 20:05:34 save PID 10671 to file /root/open-lambda/test-cluster/worker/worker.pid", "2019/06/21 20:05:34 Start Lambda Server", "2019/06/21 20:05:34 Create CodePuller", "2019/06/21 20:05:34 Initialized CodePuller (took 49.842\u00b5s)", "2019/06/21 20:05:34 Create InstallManager", "2019/06/21 20:05:34 Create InstallManager (took 2.046\u00b5s)", "2019/06/21 20:05:34 Create ContainerFactory", "2019/06/21 20:05:34 .Create(, /root/open-lambda/test-cluster/worker/import-cache, [], )", "2019/06/21 20:05:34 create /sys/fs/cgroup/blkio/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:34 create /sys/fs/cgroup/cpu/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:34 create /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:34 create /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:34 create /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:34 create /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:34 create /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:34 create /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:34 create /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]", "2019/06/21 20:05:34 start creating/serving CGs [CGROUP POOL sock-cache]", "2019/06/21 20:05:34 create /sys/fs/cgroup/blkio/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:05:34 create /sys/fs/cgroup/cpu/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:05:34 create /sys/fs/cgroup/devices/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:05:34 create /sys/fs/cgroup/freezer/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:05:34 create /sys/fs/cgroup/hugetlb/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:05:34 create /sys/fs/cgroup/memory/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:05:34 create /sys/fs/cgroup/perf_event/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:05:34 create /sys/fs/cgroup/systemd/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:05:34 create /sys/fs/cgroup/pids/sock-handlers [CGROUP POOL sock-handlers]", "2019/06/21 20:05:34 start creating/serving CGs [CGROUP POOL sock-handlers]", "2019/06/21 20:05:34 Initialized handler container factory (took 170.259289ms)", "2019/06/21 20:05:34 Create Benchmark", "2019/06/21 20:05:34 Setups Handlers", "2019/06/21 20:05:34 Execute handler by POSTing to localhost:5000/run/", "2019/06/21 20:05:34 Get status by sending request to localhost:5000/status", "2019/06/21 20:05:34 Receive request to /pid", "2019/06/21 20:05:34 Receive request to /run/L0", "2019/06/21 20:05:34 .Create(/root/open-lambda/test-cluster/worker/lambda_code/1-L0, /root/open-lambda/test-cluster/worker/handlers/L0, [], )", "2019/06/21 20:05:34 Channel() [SB 2]", "2019/06/21 20:05:34 Pause() [SB 2]", "2019/06/21 20:05:34 MemUsageKB() [SB 2]", "2019/06/21 20:05:34 Receive request to /run/L1", "2019/06/21 20:05:34 .Create(/root/open-lambda/test-cluster/worker/lambda_code/2-L1, /root/open-lambda/test-cluster/worker/handlers/L1, [], )", "2019/06/21 20:05:34 Channel() [SB 3]", "2019/06/21 20:05:34 Pause() [SB 3]", "2019/06/21 20:05:34 MemUsageKB() [SB 3]", "2019/06/21 20:05:34 Receive request to /run/L2", "2019/06/21 20:05:34 .Create(/root/open-lambda/test-cluster/worker/lambda_code/3-L2, /root/open-lambda/test-cluster/worker/handlers/L2, [], )", "2019/06/21 20:05:35 Channel() [SB 4]", "2019/06/21 20:05:35 Pause() [SB 4]", "2019/06/21 20:05:35 MemUsageKB() [SB 4]", "2019/06/21 20:05:35 Receive request to /run/L3", "2019/06/21 20:05:35 .Create(/root/open-lambda/test-cluster/worker/lambda_code/4-L3, /root/open-lambda/test-cluster/worker/handlers/L3, [], )", "2019/06/21 20:05:35 Channel() [SB 5]", "2019/06/21 20:05:35 Pause() [SB 5]", "2019/06/21 20:05:35 MemUsageKB() [SB 5]", "2019/06/21 20:05:35 Receive request to /run/L4", "2019/06/21 20:05:35 .Create(/root/open-lambda/test-cluster/worker/lambda_code/5-L4, /root/open-lambda/test-cluster/worker/handlers/L4, [], )", "2019/06/21 20:05:35 Channel() [SB 6]", "2019/06/21 20:05:35 Pause() [SB 6]", "2019/06/21 20:05:35 MemUsageKB() [SB 6]", "2019/06/21 20:05:35 Receive request to /run/L5", "2019/06/21 20:05:35 .Create(/root/open-lambda/test-cluster/worker/lambda_code/6-L5, /root/open-lambda/test-cluster/worker/handlers/L5, [], )", "2019/06/21 20:05:35 Channel() [SB 7]", "2019/06/21 20:05:35 Pause() [SB 7]", "2019/06/21 20:05:35 MemUsageKB() [SB 7]", "2019/06/21 20:05:35 Receive request to /run/L6", "2019/06/21 20:05:35 .Create(/root/open-lambda/test-cluster/worker/lambda_code/7-L6, /root/open-lambda/test-cluster/worker/handlers/L6, [], )", "2019/06/21 20:05:35 Channel() [SB 8]", "2019/06/21 20:05:35 Pause() [SB 8]", "2019/06/21 20:05:35 MemUsageKB() [SB 8]", "2019/06/21 20:05:35 Receive request to /run/L7", "2019/06/21 20:05:35 .Create(/root/open-lambda/test-cluster/worker/lambda_code/8-L7, /root/open-lambda/test-cluster/worker/handlers/L7, [], )", "2019/06/21 20:05:35 Channel() [SB 9]", "2019/06/21 20:05:35 Pause() [SB 9]", "2019/06/21 20:05:35 MemUsageKB() [SB 9]", "2019/06/21 20:05:35 Receive request to /run/L8", "2019/06/21 20:05:35 .Create(/root/open-lambda/test-cluster/worker/lambda_code/9-L8, /root/open-lambda/test-cluster/worker/handlers/L8, [], )", "2019/06/21 20:05:36 Channel() [SB 10]", "2019/06/21 20:05:36 Pause() [SB 10]", "2019/06/21 20:05:36 MemUsageKB() [SB 10]", "2019/06/21 20:05:36 Receive request to /run/L9", "2019/06/21 20:05:36 .Create(/root/open-lambda/test-cluster/worker/lambda_code/10-L9, /root/open-lambda/test-cluster/worker/handlers/L9, [], )", "2019/06/21 20:05:36 Channel() [SB 11]", "2019/06/21 20:05:36 Pause() [SB 11]", "2019/06/21 20:05:36 MemUsageKB() [SB 11]", "2019/06/21 20:05:36 Receive request to /run/L10", "2019/06/21 20:05:36 .Create(/root/open-lambda/test-cluster/worker/lambda_code/11-L10, /root/open-lambda/test-cluster/worker/handlers/L10, [], )", "2019/06/21 20:05:36 Channel() [SB 12]", "2019/06/21 20:05:36 Pause() [SB 12]", "2019/06/21 20:05:36 MemUsageKB() [SB 12]", "2019/06/21 20:05:36 Receive request to /run/L11", "2019/06/21 20:05:36 .Create(/root/open-lambda/test-cluster/worker/lambda_code/12-L11, /root/open-lambda/test-cluster/worker/handlers/L11, [], )", "2019/06/21 20:05:36 Channel() [SB 13]", "2019/06/21 20:05:36 Pause() [SB 13]", "2019/06/21 20:05:36 MemUsageKB() [SB 13]", "2019/06/21 20:05:36 Receive request to /run/L12", "2019/06/21 20:05:36 .Create(/root/open-lambda/test-cluster/worker/lambda_code/13-L12, /root/open-lambda/test-cluster/worker/handlers/L12, [], )", "2019/06/21 20:05:36 Channel() [SB 14]", "2019/06/21 20:05:36 Pause() [SB 14]", "2019/06/21 20:05:36 MemUsageKB() [SB 14]", "2019/06/21 20:05:36 Receive request to /run/L13", "2019/06/21 20:05:36 .Create(/root/open-lambda/test-cluster/worker/lambda_code/14-L13, /root/open-lambda/test-cluster/worker/handlers/L13, [], )", "2019/06/21 20:05:36 Channel() [SB 15]", "2019/06/21 20:05:37 Pause() [SB 15]", "2019/06/21 20:05:37 MemUsageKB() [SB 15]", "2019/06/21 20:05:37 Receive request to /run/L14", "2019/06/21 20:05:37 .Create(/root/open-lambda/test-cluster/worker/lambda_code/15-L14, /root/open-lambda/test-cluster/worker/handlers/L14, [], )", "2019/06/21 20:05:37 Channel() [SB 16]", "2019/06/21 20:05:37 Pause() [SB 16]", "2019/06/21 20:05:37 MemUsageKB() [SB 16]", "2019/06/21 20:05:37 Receive request to /run/L15", "2019/06/21 20:05:37 .Create(/root/open-lambda/test-cluster/worker/lambda_code/16-L15, /root/open-lambda/test-cluster/worker/handlers/L15, [], )", "2019/06/21 20:05:37 Channel() [SB 17]", "2019/06/21 20:05:37 Pause() [SB 17]", "2019/06/21 20:05:37 MemUsageKB() [SB 17]", "2019/06/21 20:05:37 Receive request to /run/L16", "2019/06/21 20:05:37 .Create(/root/open-lambda/test-cluster/worker/lambda_code/17-L16, /root/open-lambda/test-cluster/worker/handlers/L16, [], )", "2019/06/21 20:05:37 Channel() [SB 18]", "2019/06/21 20:05:37 Pause() [SB 18]", "2019/06/21 20:05:37 MemUsageKB() [SB 18]", "2019/06/21 20:05:37 Receive request to /run/L17", "2019/06/21 20:05:37 .Create(/root/open-lambda/test-cluster/worker/lambda_code/18-L17, /root/open-lambda/test-cluster/worker/handlers/L17, [], )", "2019/06/21 20:05:37 Channel() [SB 19]", "2019/06/21 20:05:37 Pause() [SB 19]", "2019/06/21 20:05:37 MemUsageKB() [SB 19]", "2019/06/21 20:05:37 Receive request to /run/L18", "2019/06/21 20:05:37 .Create(/root/open-lambda/test-cluster/worker/lambda_code/19-L18, /root/open-lambda/test-cluster/worker/handlers/L18, [], )", "2019/06/21 20:05:38 Channel() [SB 20]", "2019/06/21 20:05:38 Pause() [SB 20]", "2019/06/21 20:05:38 MemUsageKB() [SB 20]", "2019/06/21 20:05:38 Receive request to /run/L19", "2019/06/21 20:05:38 .Create(/root/open-lambda/test-cluster/worker/lambda_code/20-L19, /root/open-lambda/test-cluster/worker/handlers/L19, [], )", "2019/06/21 20:05:38 Channel() [SB 21]", "2019/06/21 20:05:38 Pause() [SB 21]", "2019/06/21 20:05:38 MemUsageKB() [SB 21]", "2019/06/21 20:05:38 Receive request to /run/L20", "2019/06/21 20:05:38 .Create(/root/open-lambda/test-cluster/worker/lambda_code/21-L20, /root/open-lambda/test-cluster/worker/handlers/L20, [], )", "2019/06/21 20:05:38 Channel() [SB 22]", "2019/06/21 20:05:38 Pause() [SB 22]", "2019/06/21 20:05:38 MemUsageKB() [SB 22]", "2019/06/21 20:05:38 Receive request to /run/L21", "2019/06/21 20:05:38 .Create(/root/open-lambda/test-cluster/worker/lambda_code/22-L21, /root/open-lambda/test-cluster/worker/handlers/L21, [], )", "2019/06/21 20:05:38 Channel() [SB 23]", "2019/06/21 20:05:38 Pause() [SB 23]", "2019/06/21 20:05:38 MemUsageKB() [SB 23]", "2019/06/21 20:05:38 Receive request to /run/L22", "2019/06/21 20:05:38 .Create(/root/open-lambda/test-cluster/worker/lambda_code/23-L22, /root/open-lambda/test-cluster/worker/handlers/L22, [], )", "2019/06/21 20:05:38 Channel() [SB 24]", "2019/06/21 20:05:38 Pause() [SB 24]", "2019/06/21 20:05:38 MemUsageKB() [SB 24]", "2019/06/21 20:05:38 Receive request to /run/L23", "2019/06/21 20:05:38 .Create(/root/open-lambda/test-cluster/worker/lambda_code/24-L23, /root/open-lambda/test-cluster/worker/handlers/L23, [], )", "2019/06/21 20:05:39 Channel() [SB 25]", "2019/06/21 20:05:39 Pause() [SB 25]", "2019/06/21 20:05:39 MemUsageKB() [SB 25]", "2019/06/21 20:05:39 Receive request to /run/L24", "2019/06/21 20:05:39 .Create(/root/open-lambda/test-cluster/worker/lambda_code/25-L24, /root/open-lambda/test-cluster/worker/handlers/L24, [], )", "2019/06/21 20:05:39 Channel() [SB 26]", "2019/06/21 20:05:39 Pause() [SB 26]", "2019/06/21 20:05:39 MemUsageKB() [SB 26]", "2019/06/21 20:05:39 Receive request to /run/L25", "2019/06/21 20:05:39 .Create(/root/open-lambda/test-cluster/worker/lambda_code/26-L25, /root/open-lambda/test-cluster/worker/handlers/L25, [], )", "2019/06/21 20:05:39 Channel() [SB 27]", "2019/06/21 20:05:39 Pause() [SB 27]", "2019/06/21 20:05:39 MemUsageKB() [SB 27]", "2019/06/21 20:05:39 Receive request to /run/L26", "2019/06/21 20:05:39 .Create(/root/open-lambda/test-cluster/worker/lambda_code/27-L26, /root/open-lambda/test-cluster/worker/handlers/L26, [], )", "2019/06/21 20:05:39 Channel() [SB 28]", "2019/06/21 20:05:39 Pause() [SB 28]", "2019/06/21 20:05:39 MemUsageKB() [SB 28]", "2019/06/21 20:05:39 Receive request to /run/L27", "2019/06/21 20:05:39 .Create(/root/open-lambda/test-cluster/worker/lambda_code/28-L27, /root/open-lambda/test-cluster/worker/handlers/L27, [], )", "2019/06/21 20:05:39 Channel() [SB 29]", "2019/06/21 20:05:39 Pause() [SB 29]", "2019/06/21 20:05:39 MemUsageKB() [SB 29]", "2019/06/21 20:05:39 Receive request to /run/L28", "2019/06/21 20:05:39 .Create(/root/open-lambda/test-cluster/worker/lambda_code/29-L28, /root/open-lambda/test-cluster/worker/handlers/L28, [], )", "2019/06/21 20:05:40 Channel() [SB 30]", "2019/06/21 20:05:40 Pause() [SB 30]", "2019/06/21 20:05:40 MemUsageKB() [SB 30]", "2019/06/21 20:05:40 Receive request to /run/L29", "2019/06/21 20:05:40 .Create(/root/open-lambda/test-cluster/worker/lambda_code/30-L29, /root/open-lambda/test-cluster/worker/handlers/L29, [], )", "2019/06/21 20:05:40 Channel() [SB 31]", "2019/06/21 20:05:40 Pause() [SB 31]", "2019/06/21 20:05:40 MemUsageKB() [SB 31]", "2019/06/21 20:05:40 Receive request to /run/L30", "2019/06/21 20:05:40 .Create(/root/open-lambda/test-cluster/worker/lambda_code/31-L30, /root/open-lambda/test-cluster/worker/handlers/L30, [], )", "2019/06/21 20:05:40 Channel() [SB 32]", "2019/06/21 20:05:40 Pause() [SB 32]", "2019/06/21 20:05:40 MemUsageKB() [SB 32]", "2019/06/21 20:05:40 Receive request to /run/L31", "2019/06/21 20:05:40 .Create(/root/open-lambda/test-cluster/worker/lambda_code/32-L31, /root/open-lambda/test-cluster/worker/handlers/L31, [], )", "2019/06/21 20:05:40 Channel() [SB 33]", "2019/06/21 20:05:40 Pause() [SB 33]", "2019/06/21 20:05:40 MemUsageKB() [SB 33]", "2019/06/21 20:05:40 Receive request to /run/L32", "2019/06/21 20:05:40 .Create(/root/open-lambda/test-cluster/worker/lambda_code/33-L32, /root/open-lambda/test-cluster/worker/handlers/L32, [], )", "2019/06/21 20:05:41 Channel() [SB 34]", "2019/06/21 20:05:41 Pause() [SB 34]", "2019/06/21 20:05:41 MemUsageKB() [SB 34]", "2019/06/21 20:05:41 Receive request to /run/L33", "2019/06/21 20:05:41 .Create(/root/open-lambda/test-cluster/worker/lambda_code/34-L33, /root/open-lambda/test-cluster/worker/handlers/L33, [], )", "2019/06/21 20:05:41 Channel() [SB 35]", "2019/06/21 20:05:41 Pause() [SB 35]", "2019/06/21 20:05:41 MemUsageKB() [SB 35]", "2019/06/21 20:05:41 Receive request to /run/L34", "2019/06/21 20:05:41 .Create(/root/open-lambda/test-cluster/worker/lambda_code/35-L34, /root/open-lambda/test-cluster/worker/handlers/L34, [], )", "2019/06/21 20:05:41 Channel() [SB 36]", "2019/06/21 20:05:41 Pause() [SB 36]", "2019/06/21 20:05:41 MemUsageKB() [SB 36]", "2019/06/21 20:05:41 Receive request to /run/L35", "2019/06/21 20:05:41 .Create(/root/open-lambda/test-cluster/worker/lambda_code/36-L35, /root/open-lambda/test-cluster/worker/handlers/L35, [], )", "2019/06/21 20:05:42 Channel() [SB 37]", "2019/06/21 20:05:42 Pause() [SB 37]", "2019/06/21 20:05:42 MemUsageKB() [SB 37]", "2019/06/21 20:05:42 Receive request to /run/L36", "2019/06/21 20:05:42 .Create(/root/open-lambda/test-cluster/worker/lambda_code/37-L36, /root/open-lambda/test-cluster/worker/handlers/L36, [], )", "2019/06/21 20:05:42 Channel() [SB 38]", "2019/06/21 20:05:42 Pause() [SB 38]", "2019/06/21 20:05:42 MemUsageKB() [SB 38]", "2019/06/21 20:05:42 Receive request to /run/L37", "2019/06/21 20:05:42 .Create(/root/open-lambda/test-cluster/worker/lambda_code/38-L37, /root/open-lambda/test-cluster/worker/handlers/L37, [], )", "2019/06/21 20:05:44 Channel() [SB 39]", "2019/06/21 20:05:44 Pause() [SB 39]", "2019/06/21 20:05:44 MemUsageKB() [SB 39]", "2019/06/21 20:05:44 Receive request to /run/L38", "2019/06/21 20:05:44 .Create(/root/open-lambda/test-cluster/worker/lambda_code/39-L38, /root/open-lambda/test-cluster/worker/handlers/L38, [], )", "2019/06/21 20:05:49 Channel() [SB 40]", "2019/06/21 20:05:51 Pause() [SB 40]", "2019/06/21 20:05:51 MemUsageKB() [SB 40]", "2019/06/21 20:05:53 Receive request to /run/L39", "2019/06/21 20:05:54 .Create(/root/open-lambda/test-cluster/worker/lambda_code/40-L39, /root/open-lambda/test-cluster/worker/handlers/L39, [], )", "2019/06/21 20:06:03 received kill signal, cleaning up", "2019/06/21 20:06:04 Destroy() [SB 2]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 2]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 2]", "2019/06/21 20:06:04 Destroy() [SB 3]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 3]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 3]", "2019/06/21 20:06:04 Destroy() [SB 4]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 4]", "2019/06/21 20:06:04 waiting for 1 procs in cg-3 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 4]", "2019/06/21 20:06:04 Destroy() [SB 5]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 5]", "2019/06/21 20:06:04 waiting for 1 procs in cg-4 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 5]", "2019/06/21 20:06:04 Destroy() [SB 6]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 6]", "2019/06/21 20:06:04 waiting for 1 procs in cg-5 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 6]", "2019/06/21 20:06:04 Destroy() [SB 7]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 7]", "2019/06/21 20:06:04 waiting for 1 procs in cg-6 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 7]", "2019/06/21 20:06:04 Destroy() [SB 8]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 8]", "2019/06/21 20:06:04 waiting for 1 procs in cg-7 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 8]", "2019/06/21 20:06:04 Destroy() [SB 9]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 9]", "2019/06/21 20:06:04 waiting for 1 procs in cg-8 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 9]", "2019/06/21 20:06:04 Destroy() [SB 10]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 10]", "2019/06/21 20:06:04 waiting for 1 procs in cg-9 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 10]", "2019/06/21 20:06:04 Destroy() [SB 11]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 11]", "2019/06/21 20:06:04 waiting for 1 procs in cg-10 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 11]", "2019/06/21 20:06:04 Destroy() [SB 12]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 12]", "2019/06/21 20:06:04 waiting for 1 procs in cg-11 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 12]", "2019/06/21 20:06:04 Destroy() [SB 13]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 13]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 13]", "2019/06/21 20:06:04 Destroy() [SB 14]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 14]", "2019/06/21 20:06:04 waiting for 1 procs in cg-13 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 14]", "2019/06/21 20:06:04 Destroy() [SB 15]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 15]", "2019/06/21 20:06:04 waiting for 1 procs in cg-14 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 15]", "2019/06/21 20:06:04 Destroy() [SB 16]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 16]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 16]", "2019/06/21 20:06:04 Destroy() [SB 17]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 17]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 17]", "2019/06/21 20:06:04 Destroy() [SB 18]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 18]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 18]", "2019/06/21 20:06:04 Destroy() [SB 19]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 19]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 19]", "2019/06/21 20:06:04 Destroy() [SB 20]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 20]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 20]", "2019/06/21 20:06:04 Destroy() [SB 21]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 21]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 21]", "2019/06/21 20:06:04 Destroy() [SB 22]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 22]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 22]", "2019/06/21 20:06:04 Destroy() [SB 23]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 23]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 23]", "2019/06/21 20:06:04 Destroy() [SB 24]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 24]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 24]", "2019/06/21 20:06:04 Destroy() [SB 25]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 25]", "2019/06/21 20:06:04 waiting for 1 procs in cg-24 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 25]", "2019/06/21 20:06:04 Destroy() [SB 26]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 26]", "2019/06/21 20:06:04 waiting for 1 procs in cg-25 to die [CGROUP POOL sock-handlers]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 26]", "2019/06/21 20:06:04 Destroy() [SB 27]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 27]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 27]", "2019/06/21 20:06:04 Destroy() [SB 28]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 28]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 28]", "2019/06/21 20:06:04 Destroy() [SB 29]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 29]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 29]", "2019/06/21 20:06:04 Destroy() [SB 30]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 30]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 30]", "2019/06/21 20:06:04 Destroy() [SB 31]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 31]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 31]", "2019/06/21 20:06:04 Destroy() [SB 32]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 32]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 32]", "2019/06/21 20:06:04 Destroy() [SB 33]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 33]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 33]", "2019/06/21 20:06:04 Destroy() [SB 34]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 34]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 34]", "2019/06/21 20:06:04 Destroy() [SB 35]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 35]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 35]", "2019/06/21 20:06:04 Destroy() [SB 36]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 36]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 36]", "2019/06/21 20:06:04 Destroy() [SB 37]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 37]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 37]", "2019/06/21 20:06:04 Destroy() [SB 38]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 38]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 38]", "2019/06/21 20:06:04 Destroy() [SB 39]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 39]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 39]", "2019/06/21 20:06:04 Destroy() [SB 40]", "2019/06/21 20:06:04 kill all procs in CG [SOCK 40]", "2019/06/21 20:06:04 unmount and remove dirs [SOCK 40]", "2019/06/21 20:06:04 received shutdown request [CGROUP POOL sock-handlers]", "2019/06/21 20:06:04 empty queues and release CGs [CGROUP POOL sock-handlers]", "2019/06/21 20:06:04 remove /sys/fs/cgroup/blkio/sock-handlers [CGROUP POOL sock-handlers]", "panic: Rmdir /sys/fs/cgroup/blkio/sock-handlers: device or resource busy", "", "goroutine 6 [running]:", "github.com/open-lambda/open-lambda/ol/sandbox.(*CgroupPool).cgTask(0xc0001a8180)", "\t/root/open-lambda/src/sandbox/cgroupFactory.go:119 +0xabb", "created by github.com/open-lambda/open-lambda/ol/sandbox.NewCgroupPool", "\t/root/open-lambda/src/sandbox/cgroupFactory.go:46 +0xfe", "" ] } POP conf: {'registry': '/tmp/tmp19gi_h7w'} PUSH conf: {'registry': '/tmp/tmpfsvhctbd'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=10895, port=5000, log=/root/open-lambda/test-cluster/worker.out 2019/06/21 20:06:05 worker process 10895 does not a appear to be running, check worker.out { "test": "call_each_once_exec", "params": { "lambda_count": 1000, "alloc_mb": 10 }, "pass": false, "seconds": null, "total_seconds": 0.7027819156646729, "stats": null, "conf": { "sandbox": "sock", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "registry": "/tmp/tmpfsvhctbd", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "server_mode": "lambda", "cluster_name": "/root/open-lambda/test-cluster", "timing": false, "import_cache_mb": 256, "max_runners": 0, "sandbox_config": { "processes": 10 }, "worker_port": "5000", "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "benchmark_log": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "docker_runtime": "", "startup_pkgs": null, "handler_cache_mb": 256, "Skip_pull_existing": false }, "exception": [ "Traceback (most recent call last):", " File \"test.py\", line 51, in wrapper", " run(['./ol', 'worker', '-p='+OLDIR, '--detach'])", " File \"test.py\", line 128, in run", " raise Exception(\"command failed: \" + \" \".join(cmd) + \", \")", "Exception: command failed: ./ol worker -p=test-cluster --detach, ", "" ], "worker_tail": [ "using existing OL directory at /root/open-lambda/test-cluster", "2019/06/21 20:06:05 previous working may be running, /root/open-lambda/test-cluster/worker/worker.pid already exists", "" ] } POP conf: {'registry': '/tmp/tmpfsvhctbd'} POP conf: {'sandbox': 'sock', 'import_cache_mb': 256, 'handler_cache_mb': 256} PASSED: 12, FAILED: 3 Makefile:28: recipe for target 'test-all' failed make: *** [test-all] Error 3