python3 -u test.py RUN ./ol new -p=test-dir b'Init OL dir at /root/open-lambda/test-dir\nCreate lambda base at /root/open-lambda/test-dir/lambda (may take several minutes)\nWorking Directory: /root/open-lambda/test-dir\n\nWorker Defaults: \n{\n\t"registry": "/root/open-lambda/test-dir/registry",\n\t"registry_cache_ms": 5000,\n\t"sandbox": "sock",\n\t"cluster_name": "/root/open-lambda/test-dir",\n\t"pip_mirror": "",\n\t"Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",\n\t"handler_cache_mb": 239,\n\t"import_cache_mb": 239,\n\t"server_mode": "lambda",\n\t"worker_dir": "/root/open-lambda/test-dir/worker",\n\t"SOCK_base_path": "/root/open-lambda/test-dir/lambda",\n\t"worker_port": "5000",\n\t"Skip_pull_existing": false,\n\t"sandbox_config": {},\n\t"startup_pkgs": null,\n\t"docker_runtime": "",\n\t"sock_cgroups": {\n\t\t"max_procs": 10,\n\t\t"max_mem_mb": 50\n\t}\n}\n\nYou may modify the defaults here: /root/open-lambda/test-dir/config.json\n\nYou may now start a server using the "ol worker" command\n' PUSH conf: {'registry': '/root/open-lambda/test-registry', 'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson']} ======================================== ping_test ======================================== RUN ./ol worker -p=test-dir --detach b'using existing OL directory at /root/open-lambda/test-dir\nStarting worker: pid=7295, port=5000, log=/root/open-lambda/test-dir/worker.out\nready\n' RUN ./ol kill -p=test-dir b'Kill worker process with PID 7295\n' { "test": "ping_test", "params": {}, "pass": true, "seconds": 2.249602794647217, "total_seconds": 22.277989387512207, "stats": { "pings_per_sec": 444.52734715472246 }, "ol-stats": { "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 1, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 1, "Create()/fresh-proc:ms.avg": 1898, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 4, "Create()/make-root-fs:ms.cnt": 1, "Create():ms.avg": 1903, "Create():ms.cnt": 1, "Destroy()/detach-root:ms.avg": 1, "Destroy()/detach-root:ms.cnt": 1, "Destroy()/kill-procs:ms.avg": 1, "Destroy()/kill-procs:ms.cnt": 1, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 1, "Destroy():ms.avg": 6, "Destroy():ms.cnt": 1 }, "conf": { "worker_port": "5000", "registry_cache_ms": 5000, "registry": "/root/open-lambda/test-registry", "import_cache_mb": 239, "sandbox_config": {}, "handler_cache_mb": 239, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-dir/worker", "pip_mirror": "", "Skip_pull_existing": false, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "cluster_name": "/root/open-lambda/test-dir", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "sandbox": "sock", "docker_runtime": "" }, "errors": [], "worker_tail": [ "2019/07/03 14:00:20 remove /sys/fs/cgroup/freezer/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:20 remove /sys/fs/cgroup/hugetlb/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:20 remove /sys/fs/cgroup/memory/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:20 remove /sys/fs/cgroup/perf_event/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:20 remove /sys/fs/cgroup/systemd/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:20 remove /sys/fs/cgroup/pids/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:20 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/03 14:00:20 remove worker.pid", "2019/07/03 14:00:20 exiting", "" ] } PUSH conf: {'handler_cache_mb': 100, 'import_cache_mb': 0} ======================================== smoke_tests ======================================== RUN ./ol worker -p=test-dir --detach b'using existing OL directory at /root/open-lambda/test-dir\nStarting worker: pid=7366, port=5000, log=/root/open-lambda/test-dir/worker.out\nready\n' RUN ./ol kill -p=test-dir b'Kill worker process with PID 7366\n' { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 1.8125736713409424, "total_seconds": 2.0487093925476074, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 4, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 4, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 4, "Create()/fresh-proc:ms.avg": 135, "Create()/fresh-proc:ms.cnt": 4, "Create()/make-root-fs:ms.avg": 3, "Create()/make-root-fs:ms.cnt": 4, "Create():ms.avg": 139, "Create():ms.cnt": 4, "Destroy()/detach-root:ms.avg": 1, "Destroy()/detach-root:ms.cnt": 4, "Destroy()/kill-procs:ms.avg": 2, "Destroy()/kill-procs:ms.cnt": 4, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 4, "Destroy():ms.avg": 3, "Destroy():ms.cnt": 5, "Pause():ms.avg": 3, "Pause():ms.cnt": 4, "evict:ms.avg": 6, "evict:ms.cnt": 1 }, "conf": { "worker_port": "5000", "registry_cache_ms": 5000, "registry": "/root/open-lambda/test-registry", "import_cache_mb": 0, "sandbox_config": {}, "handler_cache_mb": 100, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-dir/worker", "pip_mirror": "", "Skip_pull_existing": false, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "cluster_name": "/root/open-lambda/test-dir", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "sandbox": "sock", "docker_runtime": "" }, "errors": [], "worker_tail": [ "2019/07/03 14:00:22 remove /sys/fs/cgroup/freezer/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:00:22 remove /sys/fs/cgroup/hugetlb/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:00:22 remove /sys/fs/cgroup/memory/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:00:22 remove /sys/fs/cgroup/perf_event/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:00:22 remove /sys/fs/cgroup/systemd/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:00:22 remove /sys/fs/cgroup/pids/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:00:22 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/03 14:00:22 remove worker.pid", "2019/07/03 14:00:22 exiting", "" ] } POP conf: {'handler_cache_mb': 100, 'import_cache_mb': 0} PUSH conf: {'handler_cache_mb': 250, 'import_cache_mb': 0} ======================================== smoke_tests ======================================== RUN ./ol worker -p=test-dir --detach b'using existing OL directory at /root/open-lambda/test-dir\nStarting worker: pid=7416, port=5000, log=/root/open-lambda/test-dir/worker.out\nready\n' RUN ./ol kill -p=test-dir b'Kill worker process with PID 7416\n' { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 1.3491392135620117, "total_seconds": 1.5899579524993896, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 4, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 4, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 4, "Create()/fresh-proc:ms.avg": 135, "Create()/fresh-proc:ms.cnt": 4, "Create()/make-root-fs:ms.avg": 3, "Create()/make-root-fs:ms.cnt": 4, "Create():ms.avg": 139, "Create():ms.cnt": 4, "Destroy()/detach-root:ms.avg": 0, "Destroy()/detach-root:ms.cnt": 4, "Destroy()/kill-procs:ms.avg": 1, "Destroy()/kill-procs:ms.cnt": 4, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 4, "Destroy():ms.avg": 2, "Destroy():ms.cnt": 4, "Pause():ms.avg": 7, "Pause():ms.cnt": 4 }, "conf": { "worker_port": "5000", "registry_cache_ms": 5000, "server_mode": "lambda", "import_cache_mb": 0, "sandbox_config": {}, "handler_cache_mb": 250, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-registry", "worker_dir": "/root/open-lambda/test-dir/worker", "pip_mirror": "", "Skip_pull_existing": false, "cluster_name": "/root/open-lambda/test-dir", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "sandbox": "sock", "docker_runtime": "" }, "errors": [], "worker_tail": [ "2019/07/03 14:00:24 remove /sys/fs/cgroup/freezer/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:00:24 remove /sys/fs/cgroup/hugetlb/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:00:24 remove /sys/fs/cgroup/memory/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:00:24 remove /sys/fs/cgroup/perf_event/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:00:24 remove /sys/fs/cgroup/systemd/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:00:24 remove /sys/fs/cgroup/pids/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:00:24 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/03 14:00:24 remove worker.pid", "2019/07/03 14:00:24 exiting", "" ] } POP conf: {'handler_cache_mb': 250, 'import_cache_mb': 0} PUSH conf: {'handler_cache_mb': 100, 'import_cache_mb': 250} ======================================== smoke_tests ======================================== RUN ./ol worker -p=test-dir --detach b'using existing OL directory at /root/open-lambda/test-dir\nStarting worker: pid=7465, port=5000, log=/root/open-lambda/test-dir/worker.out\nready\n' RUN ./ol kill -p=test-dir b'Kill worker process with PID 7465\n' { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 0.8491294384002686, "total_seconds": 1.0900325775146484, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 4, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 5, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 5, "Create()/fork-proc:ms.avg": 7, "Create()/fork-proc:ms.cnt": 4, "Create()/fresh-proc:ms.avg": 138, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 3, "Create()/make-root-fs:ms.cnt": 5, "Create():ms.avg": 37, "Create():ms.cnt": 5, "Destroy()/detach-root:ms.avg": 0, "Destroy()/detach-root:ms.cnt": 5, "Destroy()/kill-procs:ms.avg": 1, "Destroy()/kill-procs:ms.cnt": 5, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 5, "Destroy():ms.avg": 1, "Destroy():ms.cnt": 6, "Pause():ms.avg": 7, "Pause():ms.cnt": 4, "evict:ms.avg": 0, "evict:ms.cnt": 1, "fork():ms.avg": 7, "fork():ms.cnt": 4, "forkRequest:ms.avg": 7, "forkRequest:ms.cnt": 4, "move-to-cg-after-fork:ms.avg": 0, "move-to-cg-after-fork:ms.cnt": 4 }, "conf": { "worker_port": "5000", "registry_cache_ms": 5000, "registry": "/root/open-lambda/test-registry", "import_cache_mb": 250, "sandbox_config": {}, "handler_cache_mb": 100, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-dir/worker", "pip_mirror": "", "Skip_pull_existing": false, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "cluster_name": "/root/open-lambda/test-dir", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "sandbox": "sock", "docker_runtime": "" }, "errors": [], "worker_tail": [ "2019/07/03 14:00:25 remove /sys/fs/cgroup/freezer/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:25 remove /sys/fs/cgroup/hugetlb/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:25 remove /sys/fs/cgroup/memory/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:25 remove /sys/fs/cgroup/perf_event/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:25 remove /sys/fs/cgroup/systemd/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:25 remove /sys/fs/cgroup/pids/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:25 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/03 14:00:25 remove worker.pid", "2019/07/03 14:00:25 exiting", "" ] } POP conf: {'handler_cache_mb': 100, 'import_cache_mb': 250} PUSH conf: {'handler_cache_mb': 250, 'import_cache_mb': 250} ======================================== smoke_tests ======================================== RUN ./ol worker -p=test-dir --detach b'using existing OL directory at /root/open-lambda/test-dir\nStarting worker: pid=7519, port=5000, log=/root/open-lambda/test-dir/worker.out\nready\n' RUN ./ol kill -p=test-dir b'Kill worker process with PID 7519\n' { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 0.824979305267334, "total_seconds": 1.0690109729766846, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 4, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 5, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 5, "Create()/fork-proc:ms.avg": 7, "Create()/fork-proc:ms.cnt": 4, "Create()/fresh-proc:ms.avg": 139, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 3, "Create()/make-root-fs:ms.cnt": 5, "Create():ms.avg": 37, "Create():ms.cnt": 5, "Destroy()/detach-root:ms.avg": 0, "Destroy()/detach-root:ms.cnt": 5, "Destroy()/kill-procs:ms.avg": 1, "Destroy()/kill-procs:ms.cnt": 5, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 5, "Destroy():ms.avg": 2, "Destroy():ms.cnt": 5, "Pause():ms.avg": 9, "Pause():ms.cnt": 4, "fork():ms.avg": 7, "fork():ms.cnt": 4, "forkRequest:ms.avg": 5, "forkRequest:ms.cnt": 4, "move-to-cg-after-fork:ms.avg": 1, "move-to-cg-after-fork:ms.cnt": 4 }, "conf": { "worker_port": "5000", "registry_cache_ms": 5000, "server_mode": "lambda", "import_cache_mb": 250, "sandbox_config": {}, "handler_cache_mb": 250, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-registry", "worker_dir": "/root/open-lambda/test-dir/worker", "pip_mirror": "", "Skip_pull_existing": false, "cluster_name": "/root/open-lambda/test-dir", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "sandbox": "sock", "docker_runtime": "" }, "errors": [], "worker_tail": [ "2019/07/03 14:00:26 remove /sys/fs/cgroup/freezer/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:26 remove /sys/fs/cgroup/hugetlb/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:26 remove /sys/fs/cgroup/memory/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:26 remove /sys/fs/cgroup/perf_event/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:26 remove /sys/fs/cgroup/systemd/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:26 remove /sys/fs/cgroup/pids/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:26 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/03 14:00:26 remove worker.pid", "2019/07/03 14:00:26 exiting", "" ] } POP conf: {'handler_cache_mb': 250, 'import_cache_mb': 250} PUSH conf: {'handler_cache_mb': 100, 'sandbox': 'docker', 'import_cache_mb': 0} ======================================== smoke_tests ======================================== RUN ./ol worker -p=test-dir --detach b'using existing OL directory at /root/open-lambda/test-dir\nStarting worker: pid=7574, port=5000, log=/root/open-lambda/test-dir/worker.out\nready\n' RUN ./ol kill -p=test-dir b'Kill worker process with PID 7574\n' { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 6.079941272735596, "total_seconds": 7.501704216003418, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 8, "Create():ms.avg": 1510, "Create():ms.cnt": 4, "Destroy():ms.avg": 306, "Destroy():ms.cnt": 8, "Pause():ms.avg": 28, "Pause():ms.cnt": 8 }, "conf": { "worker_port": "5000", "registry_cache_ms": 5000, "registry": "/root/open-lambda/test-registry", "import_cache_mb": 0, "sandbox_config": {}, "handler_cache_mb": 100, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-dir/worker", "pip_mirror": "", "Skip_pull_existing": false, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "cluster_name": "/root/open-lambda/test-dir", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "sandbox": "docker", "docker_runtime": "" }, "errors": [], "worker_tail": [ "Kill function: install2019/07/03 14:00:33 Destroy() [SB 1]", "2019/07/03 14:00:33 Destroy() [SB 1]", "Kill function: install22019/07/03 14:00:33 Destroy() [SB 2]", "2019/07/03 14:00:33 Destroy() [SB 2]", "Kill function: install32019/07/03 14:00:33 Destroy() [SB 3]", "2019/07/03 14:00:33 Destroy() [SB 3]", "2019/07/03 14:00:33 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/03 14:00:33 remove worker.pid", "2019/07/03 14:00:33 exiting", "" ] } POP conf: {'handler_cache_mb': 100, 'sandbox': 'docker', 'import_cache_mb': 0} PUSH conf: {'handler_cache_mb': 250, 'sandbox': 'docker', 'import_cache_mb': 0} ======================================== smoke_tests ======================================== RUN ./ol worker -p=test-dir --detach b'using existing OL directory at /root/open-lambda/test-dir\nStarting worker: pid=8583, port=5000, log=/root/open-lambda/test-dir/worker.out\nready\n' RUN ./ol kill -p=test-dir b'Kill worker process with PID 8583\n' { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 2.9514498710632324, "total_seconds": 3.9764175415039062, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 8, "Create():ms.avg": 728, "Create():ms.cnt": 4, "Destroy():ms.avg": 201, "Destroy():ms.cnt": 8, "Pause():ms.avg": 26, "Pause():ms.cnt": 8 }, "conf": { "worker_port": "5000", "registry_cache_ms": 5000, "server_mode": "lambda", "import_cache_mb": 0, "sandbox_config": {}, "handler_cache_mb": 250, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-registry", "worker_dir": "/root/open-lambda/test-dir/worker", "pip_mirror": "", "Skip_pull_existing": false, "cluster_name": "/root/open-lambda/test-dir", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "sandbox": "docker", "docker_runtime": "" }, "errors": [], "worker_tail": [ "Kill function: install2019/07/03 14:00:37 Destroy() [SB 1]", "2019/07/03 14:00:37 Destroy() [SB 1]", "Kill function: install22019/07/03 14:00:37 Destroy() [SB 2]", "2019/07/03 14:00:37 Destroy() [SB 2]", "Kill function: install32019/07/03 14:00:37 Destroy() [SB 3]", "2019/07/03 14:00:37 Destroy() [SB 3]", "2019/07/03 14:00:37 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/03 14:00:37 remove worker.pid", "2019/07/03 14:00:37 exiting", "" ] } POP conf: {'handler_cache_mb': 250, 'sandbox': 'docker', 'import_cache_mb': 0} ======================================== fork_bomb ======================================== RUN ./ol worker -p=test-dir --detach b'using existing OL directory at /root/open-lambda/test-dir\nStarting worker: pid=9591, port=5000, log=/root/open-lambda/test-dir/worker.out\nready\n' RUN ./ol kill -p=test-dir b'Kill worker process with PID 9591\n' { "test": "fork_bomb", "params": {}, "pass": true, "seconds": 0.07657599449157715, "total_seconds": 0.31199193000793457, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 1, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 2, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 2, "Create()/fork-proc:ms.avg": 7, "Create()/fork-proc:ms.cnt": 1, "Create()/fresh-proc:ms.avg": 140, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 4, "Create()/make-root-fs:ms.cnt": 2, "Create():ms.avg": 78, "Create():ms.cnt": 2, "Destroy()/detach-root:ms.avg": 1, "Destroy()/detach-root:ms.cnt": 2, "Destroy()/kill-procs:ms.avg": 0, "Destroy()/kill-procs:ms.cnt": 2, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 2, "Destroy():ms.avg": 2, "Destroy():ms.cnt": 2, "Pause():ms.avg": 4, "Pause():ms.cnt": 1, "fork():ms.avg": 7, "fork():ms.cnt": 1, "forkRequest:ms.avg": 7, "forkRequest:ms.cnt": 1, "move-to-cg-after-fork:ms.avg": 0, "move-to-cg-after-fork:ms.cnt": 1 }, "conf": { "worker_port": "5000", "registry_cache_ms": 5000, "registry": "/root/open-lambda/test-registry", "import_cache_mb": 239, "sandbox_config": {}, "handler_cache_mb": 239, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-dir/worker", "pip_mirror": "", "Skip_pull_existing": false, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "cluster_name": "/root/open-lambda/test-dir", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "sandbox": "sock", "docker_runtime": "" }, "errors": [], "worker_tail": [ "2019/07/03 14:00:38 remove /sys/fs/cgroup/freezer/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:38 remove /sys/fs/cgroup/hugetlb/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:38 remove /sys/fs/cgroup/memory/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:38 remove /sys/fs/cgroup/perf_event/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:38 remove /sys/fs/cgroup/systemd/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:38 remove /sys/fs/cgroup/pids/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:38 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/03 14:00:38 remove worker.pid", "2019/07/03 14:00:38 exiting", "" ] } ======================================== max_mem_alloc ======================================== RUN ./ol worker -p=test-dir --detach b'using existing OL directory at /root/open-lambda/test-dir\nStarting worker: pid=9634, port=5000, log=/root/open-lambda/test-dir/worker.out\nready\n' RUN ./ol kill -p=test-dir b'Kill worker process with PID 9634\n' { "test": "max_mem_alloc", "params": {}, "pass": true, "seconds": 0.7910726070404053, "total_seconds": 1.0261249542236328, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 1, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 2, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 2, "Create()/fork-proc:ms.avg": 8, "Create()/fork-proc:ms.cnt": 1, "Create()/fresh-proc:ms.avg": 141, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 4, "Create()/make-root-fs:ms.cnt": 2, "Create():ms.avg": 79, "Create():ms.cnt": 2, "Destroy()/detach-root:ms.avg": 1, "Destroy()/detach-root:ms.cnt": 2, "Destroy()/kill-procs:ms.avg": 0, "Destroy()/kill-procs:ms.cnt": 2, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 2, "Destroy():ms.avg": 2, "Destroy():ms.cnt": 2, "Pause():ms.avg": 4, "Pause():ms.cnt": 1, "fork():ms.avg": 8, "fork():ms.cnt": 1, "forkRequest:ms.avg": 7, "forkRequest:ms.cnt": 1, "move-to-cg-after-fork:ms.avg": 1, "move-to-cg-after-fork:ms.cnt": 1 }, "conf": { "worker_port": "5000", "registry_cache_ms": 5000, "registry": "/root/open-lambda/test-registry", "import_cache_mb": 239, "sandbox_config": {}, "handler_cache_mb": 239, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-dir/worker", "pip_mirror": "", "Skip_pull_existing": false, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "cluster_name": "/root/open-lambda/test-dir", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "sandbox": "sock", "docker_runtime": "" }, "errors": [], "worker_tail": [ "2019/07/03 14:00:39 remove /sys/fs/cgroup/freezer/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:39 remove /sys/fs/cgroup/hugetlb/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:39 remove /sys/fs/cgroup/memory/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:39 remove /sys/fs/cgroup/perf_event/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:39 remove /sys/fs/cgroup/systemd/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:39 remove /sys/fs/cgroup/pids/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:00:39 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/03 14:00:39 remove worker.pid", "2019/07/03 14:00:39 exiting", "" ] } POP conf: {'registry': '/root/open-lambda/test-registry', 'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson']} PUSH conf: {'server_mode': 'sock'} ======================================== sock_churn {'seconds': 15, 'baseline': 0, 'procs': 1, 'fork': True} ======================================== RUN ./ol worker -p=test-dir --detach b'using existing OL directory at /root/open-lambda/test-dir\nStarting worker: pid=9717, port=5000, log=/root/open-lambda/test-dir/worker.out\nready\n' RUN ./ol kill -p=test-dir b'Kill worker process with PID 9717\n' { "test": "sock_churn", "params": { "seconds": 15, "baseline": 0, "procs": 1, "fork": true }, "pass": true, "seconds": 15.21637487411499, "total_seconds": 15.437147378921509, "stats": { "sandboxes_per_sec": 46.733333333333334 }, "ol-stats": { "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 702, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 702, "Create()/fork-proc:ms.avg": 6, "Create()/fork-proc:ms.cnt": 701, "Create()/fresh-proc:ms.avg": 129, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 2, "Create()/make-root-fs:ms.cnt": 702, "Create():ms.avg": 9, "Create():ms.cnt": 702, "Destroy()/detach-root:ms.avg": 1, "Destroy()/detach-root:ms.cnt": 702, "Destroy()/kill-procs:ms.avg": 1, "Destroy()/kill-procs:ms.cnt": 702, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 702, "Destroy():ms.avg": 1, "Destroy():ms.cnt": 1403, "fork():ms.avg": 6, "fork():ms.cnt": 701, "forkRequest:ms.avg": 5, "forkRequest:ms.cnt": 701, "move-to-cg-after-fork:ms.avg": 0, "move-to-cg-after-fork:ms.cnt": 701 }, "conf": { "worker_port": "5000", "registry_cache_ms": 5000, "server_mode": "sock", "import_cache_mb": 239, "sandbox_config": {}, "handler_cache_mb": 239, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-dir/registry", "worker_dir": "/root/open-lambda/test-dir/worker", "pip_mirror": "", "Skip_pull_existing": false, "cluster_name": "/root/open-lambda/test-dir", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "startup_pkgs": null, "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "sandbox": "sock", "docker_runtime": "" }, "errors": [], "worker_tail": [ "2019/07/03 14:00:54 remove /sys/fs/cgroup/freezer/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:00:54 remove /sys/fs/cgroup/hugetlb/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:00:54 remove /sys/fs/cgroup/memory/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:00:54 remove /sys/fs/cgroup/perf_event/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:00:54 remove /sys/fs/cgroup/systemd/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:00:54 remove /sys/fs/cgroup/pids/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:00:54 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/03 14:00:54 remove worker.pid", "2019/07/03 14:00:54 exiting", "" ] } ======================================== sock_churn {'seconds': 15, 'baseline': 0, 'procs': 15, 'fork': True} ======================================== RUN ./ol worker -p=test-dir --detach b'using existing OL directory at /root/open-lambda/test-dir\nStarting worker: pid=12552, port=5000, log=/root/open-lambda/test-dir/worker.out\nready\n' RUN ./ol kill -p=test-dir b'Kill worker process with PID 12552\n' { "test": "sock_churn", "params": { "seconds": 15, "baseline": 0, "procs": 15, "fork": true }, "pass": true, "seconds": 15.402983903884888, "total_seconds": 15.652718782424927, "stats": { "sandboxes_per_sec": 45.6 }, "ol-stats": { "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 685, "Create()/acquire-mem:ms.avg": 105, "Create()/acquire-mem:ms.cnt": 685, "Create()/fork-proc:ms.avg": 29, "Create()/fork-proc:ms.cnt": 684, "Create()/fresh-proc:ms.avg": 135, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 1, "Create()/make-root-fs:ms.cnt": 685, "Create():ms.avg": 137, "Create():ms.cnt": 685, "Destroy()/detach-root:ms.avg": 1, "Destroy()/detach-root:ms.cnt": 685, "Destroy()/kill-procs:ms.avg": 4, "Destroy()/kill-procs:ms.cnt": 685, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 685, "Destroy():ms.avg": 11, "Destroy():ms.cnt": 1369, "fork():ms.avg": 29, "fork():ms.cnt": 684, "forkRequest:ms.avg": 8, "forkRequest:ms.cnt": 684, "move-to-cg-after-fork:ms.avg": 3, "move-to-cg-after-fork:ms.cnt": 684 }, "conf": { "worker_port": "5000", "registry_cache_ms": 5000, "server_mode": "sock", "import_cache_mb": 239, "sandbox_config": {}, "handler_cache_mb": 239, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-dir/registry", "worker_dir": "/root/open-lambda/test-dir/worker", "pip_mirror": "", "Skip_pull_existing": false, "cluster_name": "/root/open-lambda/test-dir", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "startup_pkgs": null, "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "sandbox": "sock", "docker_runtime": "" }, "errors": [], "worker_tail": [ "2019/07/03 14:01:10 remove /sys/fs/cgroup/freezer/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:01:10 remove /sys/fs/cgroup/hugetlb/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:01:10 remove /sys/fs/cgroup/memory/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:01:10 remove /sys/fs/cgroup/perf_event/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:01:10 remove /sys/fs/cgroup/systemd/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:01:10 remove /sys/fs/cgroup/pids/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:01:10 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/03 14:01:10 remove worker.pid", "2019/07/03 14:01:10 exiting", "" ] } POP conf: {'server_mode': 'sock'} PUSH conf: {'registry': '/tmp/tmpabvc8x9y', 'registry_cache_ms': 3000, 'sandbox': 'sock'} ======================================== update_code ======================================== RUN ./ol worker -p=test-dir --detach b'using existing OL directory at /root/open-lambda/test-dir\nStarting worker: pid=15339, port=5000, log=/root/open-lambda/test-dir/worker.out\nready\n' RUN ./ol kill -p=test-dir b'Kill worker process with PID 15339\n' { "test": "update_code", "params": {}, "pass": true, "seconds": 6.027464866638184, "total_seconds": 6.267958879470825, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 3, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 4, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 4, "Create()/fork-proc:ms.avg": 8, "Create()/fork-proc:ms.cnt": 3, "Create()/fresh-proc:ms.avg": 142, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 3, "Create()/make-root-fs:ms.cnt": 4, "Create():ms.avg": 46, "Create():ms.cnt": 4, "Destroy()/detach-root:ms.avg": 0, "Destroy()/detach-root:ms.cnt": 4, "Destroy()/kill-procs:ms.avg": 0, "Destroy()/kill-procs:ms.cnt": 4, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 4, "Destroy():ms.avg": 1, "Destroy():ms.cnt": 4, "Pause():ms.avg": 0, "Pause():ms.cnt": 1730, "Unpause():ms.avg": 0, "Unpause():ms.cnt": 1727, "fork():ms.avg": 8, "fork():ms.cnt": 3, "forkRequest:ms.avg": 6, "forkRequest:ms.cnt": 3, "move-to-cg-after-fork:ms.avg": 2, "move-to-cg-after-fork:ms.cnt": 3 }, "conf": { "worker_port": "5000", "registry_cache_ms": 3000, "registry": "/tmp/tmpabvc8x9y", "import_cache_mb": 239, "sandbox_config": {}, "handler_cache_mb": 239, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-dir/worker", "pip_mirror": "", "Skip_pull_existing": false, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "cluster_name": "/root/open-lambda/test-dir", "startup_pkgs": null, "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "sandbox": "sock", "docker_runtime": "" }, "errors": [], "worker_tail": [ "2019/07/03 14:01:16 remove /sys/fs/cgroup/freezer/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:01:16 remove /sys/fs/cgroup/hugetlb/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:01:16 remove /sys/fs/cgroup/memory/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:01:16 remove /sys/fs/cgroup/perf_event/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:01:16 remove /sys/fs/cgroup/systemd/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:01:16 remove /sys/fs/cgroup/pids/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:01:16 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/03 14:01:16 remove worker.pid", "2019/07/03 14:01:16 exiting", "" ] } POP conf: {'registry': '/tmp/tmpabvc8x9y', 'registry_cache_ms': 3000, 'sandbox': 'sock'} PUSH conf: {'handler_cache_mb': 250, 'registry': '/root/open-lambda/test-registry', 'sandbox': 'sock', 'import_cache_mb': 250} ======================================== stress_one_lambda {'seconds': 15, 'procs': 1} ======================================== RUN ./ol worker -p=test-dir --detach b'using existing OL directory at /root/open-lambda/test-dir\nStarting worker: pid=15382, port=5000, log=/root/open-lambda/test-dir/worker.out\nready\n' RUN ./ol kill -p=test-dir b'Kill worker process with PID 15382\n' { "test": "stress_one_lambda", "params": { "seconds": 15, "procs": 1 }, "pass": true, "seconds": 15.044407367706299, "total_seconds": 15.275239944458008, "stats": { "reqs_per_sec": 294.3333333333333 }, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 3, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 4, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 4, "Create()/fork-proc:ms.avg": 7, "Create()/fork-proc:ms.cnt": 3, "Create()/fresh-proc:ms.avg": 144, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 4, "Create()/make-root-fs:ms.cnt": 4, "Create():ms.avg": 46, "Create():ms.cnt": 4, "Destroy()/detach-root:ms.avg": 0, "Destroy()/detach-root:ms.cnt": 4, "Destroy()/kill-procs:ms.avg": 0, "Destroy()/kill-procs:ms.cnt": 4, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 4, "Destroy():ms.avg": 1, "Destroy():ms.cnt": 4, "Pause():ms.avg": 0, "Pause():ms.cnt": 4415, "Unpause():ms.avg": 0, "Unpause():ms.cnt": 4412, "fork():ms.avg": 7, "fork():ms.cnt": 3, "forkRequest:ms.avg": 6, "forkRequest:ms.cnt": 3, "move-to-cg-after-fork:ms.avg": 1, "move-to-cg-after-fork:ms.cnt": 3 }, "conf": { "worker_port": "5000", "registry_cache_ms": 5000, "server_mode": "lambda", "import_cache_mb": 250, "sandbox_config": {}, "handler_cache_mb": 250, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-registry", "worker_dir": "/root/open-lambda/test-dir/worker", "pip_mirror": "", "Skip_pull_existing": false, "cluster_name": "/root/open-lambda/test-dir", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "startup_pkgs": null, "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "sandbox": "sock", "docker_runtime": "" }, "errors": [], "worker_tail": [ "2019/07/03 14:01:31 remove /sys/fs/cgroup/freezer/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:01:31 remove /sys/fs/cgroup/hugetlb/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:01:31 remove /sys/fs/cgroup/memory/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:01:31 remove /sys/fs/cgroup/perf_event/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:01:31 remove /sys/fs/cgroup/systemd/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:01:31 remove /sys/fs/cgroup/pids/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:01:31 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/03 14:01:31 remove worker.pid", "2019/07/03 14:01:31 exiting", "" ] } ======================================== stress_one_lambda {'seconds': 15, 'procs': 2} ======================================== RUN ./ol worker -p=test-dir --detach b'using existing OL directory at /root/open-lambda/test-dir\nStarting worker: pid=15429, port=5000, log=/root/open-lambda/test-dir/worker.out\nready\n' RUN ./ol kill -p=test-dir b'Kill worker process with PID 15429\n' { "test": "stress_one_lambda", "params": { "seconds": 15, "procs": 2 }, "pass": true, "seconds": 15.046395063400269, "total_seconds": 15.279923439025879, "stats": { "reqs_per_sec": 294.1333333333333 }, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 3, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 4, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 4, "Create()/fork-proc:ms.avg": 8, "Create()/fork-proc:ms.cnt": 3, "Create()/fresh-proc:ms.avg": 153, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 3, "Create()/make-root-fs:ms.cnt": 4, "Create():ms.avg": 48, "Create():ms.cnt": 4, "Destroy()/detach-root:ms.avg": 0, "Destroy()/detach-root:ms.cnt": 4, "Destroy()/kill-procs:ms.avg": 0, "Destroy()/kill-procs:ms.cnt": 4, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 4, "Destroy():ms.avg": 1, "Destroy():ms.cnt": 4, "Pause():ms.avg": 0, "Pause():ms.cnt": 3890, "Unpause():ms.avg": 0, "Unpause():ms.cnt": 3887, "fork():ms.avg": 8, "fork():ms.cnt": 3, "forkRequest:ms.avg": 7, "forkRequest:ms.cnt": 3, "move-to-cg-after-fork:ms.avg": 0, "move-to-cg-after-fork:ms.cnt": 3 }, "conf": { "worker_port": "5000", "registry_cache_ms": 5000, "server_mode": "lambda", "import_cache_mb": 250, "sandbox_config": {}, "handler_cache_mb": 250, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-registry", "worker_dir": "/root/open-lambda/test-dir/worker", "pip_mirror": "", "Skip_pull_existing": false, "cluster_name": "/root/open-lambda/test-dir", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "startup_pkgs": null, "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "sandbox": "sock", "docker_runtime": "" }, "errors": [], "worker_tail": [ "2019/07/03 14:01:47 remove /sys/fs/cgroup/freezer/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:01:47 remove /sys/fs/cgroup/hugetlb/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:01:47 remove /sys/fs/cgroup/memory/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:01:47 remove /sys/fs/cgroup/perf_event/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:01:47 remove /sys/fs/cgroup/systemd/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:01:47 remove /sys/fs/cgroup/pids/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:01:47 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/03 14:01:47 remove worker.pid", "2019/07/03 14:01:47 exiting", "" ] } PUSH conf: {'handler_cache_mb': 500, 'import_cache_mb': 100} ======================================== stress_one_lambda {'seconds': 15, 'procs': 8} ======================================== RUN ./ol worker -p=test-dir --detach b'using existing OL directory at /root/open-lambda/test-dir\nStarting worker: pid=15477, port=5000, log=/root/open-lambda/test-dir/worker.out\nready\n' RUN ./ol kill -p=test-dir b'Kill worker process with PID 15477\n' { "test": "stress_one_lambda", "params": { "seconds": 15, "procs": 8 }, "pass": true, "seconds": 15.067277669906616, "total_seconds": 15.297974586486816, "stats": { "reqs_per_sec": 263.3333333333333 }, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 3, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 4, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 4, "Create()/fork-proc:ms.avg": 8, "Create()/fork-proc:ms.cnt": 3, "Create()/fresh-proc:ms.avg": 189, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 3, "Create()/make-root-fs:ms.cnt": 4, "Create():ms.avg": 57, "Create():ms.cnt": 4, "Destroy()/detach-root:ms.avg": 0, "Destroy()/detach-root:ms.cnt": 4, "Destroy()/kill-procs:ms.avg": 0, "Destroy()/kill-procs:ms.cnt": 4, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 4, "Destroy():ms.avg": 1, "Destroy():ms.cnt": 4, "Pause():ms.avg": 0, "Pause():ms.cnt": 3008, "Unpause():ms.avg": 0, "Unpause():ms.cnt": 3005, "fork():ms.avg": 8, "fork():ms.cnt": 3, "forkRequest:ms.avg": 6, "forkRequest:ms.cnt": 3, "move-to-cg-after-fork:ms.avg": 0, "move-to-cg-after-fork:ms.cnt": 3 }, "conf": { "worker_port": "5000", "registry_cache_ms": 5000, "server_mode": "lambda", "import_cache_mb": 100, "sandbox_config": {}, "handler_cache_mb": 500, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-registry", "worker_dir": "/root/open-lambda/test-dir/worker", "pip_mirror": "", "Skip_pull_existing": false, "cluster_name": "/root/open-lambda/test-dir", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "startup_pkgs": null, "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "sandbox": "sock", "docker_runtime": "" }, "errors": [], "worker_tail": [ "2019/07/03 14:02:02 remove /sys/fs/cgroup/freezer/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:02 remove /sys/fs/cgroup/hugetlb/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:02 remove /sys/fs/cgroup/memory/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:02 remove /sys/fs/cgroup/perf_event/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:02 remove /sys/fs/cgroup/systemd/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:02 remove /sys/fs/cgroup/pids/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:02 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/03 14:02:02 remove worker.pid", "2019/07/03 14:02:02 exiting", "" ] } POP conf: {'handler_cache_mb': 500, 'import_cache_mb': 100} POP conf: {'handler_cache_mb': 250, 'registry': '/root/open-lambda/test-registry', 'sandbox': 'sock', 'import_cache_mb': 250} PUSH conf: {'handler_cache_mb': 250, 'sandbox': 'sock', 'import_cache_mb': 250} PUSH conf: {'registry': '/tmp/tmpj332mnl1'} ======================================== call_each_once_exec {'alloc_mb': 1, 'lambda_count': 100} ======================================== RUN ./ol worker -p=test-dir --detach b'using existing OL directory at /root/open-lambda/test-dir\nStarting worker: pid=15532, port=5000, log=/root/open-lambda/test-dir/worker.out\nready\n' RUN ./ol kill -p=test-dir b'Kill worker process with PID 15532\n' { "test": "call_each_once_exec", "params": { "alloc_mb": 1, "lambda_count": 100 }, "pass": false, "seconds": null, "total_seconds": 0.8694307804107666, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 22, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 23, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 23, "Create()/fork-proc:ms.avg": 7, "Create()/fork-proc:ms.cnt": 22, "Create()/fresh-proc:ms.avg": 137, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 1, "Create()/make-root-fs:ms.cnt": 23, "Create():ms.avg": 14, "Create():ms.cnt": 23, "Destroy()/detach-root:ms.avg": 0, "Destroy()/detach-root:ms.cnt": 23, "Destroy()/kill-procs:ms.avg": 1, "Destroy()/kill-procs:ms.cnt": 23, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 23, "Destroy():ms.avg": 1, "Destroy():ms.cnt": 23, "Pause():ms.avg": 4, "Pause():ms.cnt": 22, "fork():ms.avg": 7, "fork():ms.cnt": 22, "forkRequest:ms.avg": 5, "forkRequest:ms.cnt": 22, "move-to-cg-after-fork:ms.avg": 1, "move-to-cg-after-fork:ms.cnt": 22 }, "conf": { "worker_port": "5000", "registry_cache_ms": 5000, "registry": "/tmp/tmpj332mnl1", "import_cache_mb": 250, "sandbox_config": {}, "handler_cache_mb": 250, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-dir/worker", "pip_mirror": "", "Skip_pull_existing": false, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "cluster_name": "/root/open-lambda/test-dir", "startup_pkgs": null, "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "sandbox": "sock", "docker_runtime": "" }, "errors": [ [ "Traceback (most recent call last):", " File \"test.py\", line 76, in wrapper", " rv = fn(**kwargs)", " File \"test.py\", line 216, in call_each_once_exec", " r.raise_for_status()", " File \"/usr/lib/python3/dist-packages/requests/models.py\", line 840, in raise_for_status", " raise HTTPError(http_error_msg, response=self)", "requests.exceptions.HTTPError: 502 Server Error: Bad Gateway for url: http://localhost:5000/run/L21", "" ] ], "worker_tail": [ "using existing OL directory at /root/open-lambda/test-dir", "2019/07/03 14:02:02 save PID 15532 to file /root/open-lambda/test-dir/worker/worker.pid", "2019/07/03 14:02:02 Start Lambda Server", "2019/07/03 14:02:02 Create HandlerPuller", "2019/07/03 14:02:02 Create ModulePuller", "2019/07/03 14:02:02 Create SandboxPool", "2019/07/03 14:02:02 create /sys/fs/cgroup/blkio/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:02 create /sys/fs/cgroup/cpu/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:02 create /sys/fs/cgroup/devices/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:02 create /sys/fs/cgroup/freezer/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:02 create /sys/fs/cgroup/hugetlb/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:02 create /sys/fs/cgroup/memory/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:02 create /sys/fs/cgroup/perf_event/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:02 create /sys/fs/cgroup/systemd/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:02 create /sys/fs/cgroup/pids/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:02 Create ImportCache", "2019/07/03 14:02:02 create /sys/fs/cgroup/blkio/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:02 create /sys/fs/cgroup/cpu/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:02 create /sys/fs/cgroup/devices/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:02 create /sys/fs/cgroup/freezer/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:02 create /sys/fs/cgroup/hugetlb/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:02 create /sys/fs/cgroup/memory/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:02 create /sys/fs/cgroup/perf_event/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:02 create /sys/fs/cgroup/systemd/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:02 create /sys/fs/cgroup/pids/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:02 Setups Handlers", "2019/07/03 14:02:02 Execute handler by POSTing to localhost:5000/run/", "2019/07/03 14:02:02 Get status by sending request to localhost:5000/status", "2019/07/03 14:02:02 start creating/serving CGs [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:02 start creating/serving CGs [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:02 .Create(, /root/open-lambda/test-dir/worker/scratch/import-cache-1001, [], )...", "2019/07/03 14:02:02 Receive request to /pid", "2019/07/03 14:02:02 Receive request to /run/L0", "2019/07/03 14:02:02 debug: LambdaFunc.Task() runs on goroutine 22 [FUNC L0]", "2019/07/03 14:02:02 Installs: [] [FUNC L0]", "2019/07/03 14:02:02 Imports: [] [FUNC L0]", "2019/07/03 14:02:02 increase instances to 1 [FUNC L0]", "2019/07/03 14:02:02 ...returns &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:02 .Create(/root/open-lambda/test-dir/worker/lambda_code/1-L0, /root/open-lambda/test-dir/worker/scratch/func-L0-1002, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:02 fork(&{0xc0000826e0 0xc00000f2c0 2 /root/open-lambda/test-dir/worker/sock-handlers/2 /root/open-lambda/test-dir/worker/lambda_code/1-L0 /root/open-lambda/test-dir/worker/scratch/func-L0-1002 }) [SB 1]", "2019/07/03 14:02:02 ...returns &{0xc0001b60e0 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:02 Channel() [SB 2]", "2019/07/03 14:02:02 Pause() [SB 2]", "2019/07/03 14:02:02 Receive request to /run/L1", "2019/07/03 14:02:02 debug: LambdaFunc.Task() runs on goroutine 33 [FUNC L1]", "2019/07/03 14:02:02 Installs: [] [FUNC L1]", "2019/07/03 14:02:02 Imports: [] [FUNC L1]", "2019/07/03 14:02:02 increase instances to 1 [FUNC L1]", "2019/07/03 14:02:02 .Create(/root/open-lambda/test-dir/worker/lambda_code/2-L1, /root/open-lambda/test-dir/worker/scratch/func-L1-1003, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:02 fork(&{0xc0000826e0 0xc00000f2e0 3 /root/open-lambda/test-dir/worker/sock-handlers/3 /root/open-lambda/test-dir/worker/lambda_code/2-L1 /root/open-lambda/test-dir/worker/scratch/func-L1-1003 }) [SB 1]", "2019/07/03 14:02:02 ...returns &{0xc0001b63f0 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:02 Channel() [SB 3]", "2019/07/03 14:02:02 Pause() [SB 3]", "2019/07/03 14:02:02 Receive request to /run/L2", "2019/07/03 14:02:02 debug: LambdaFunc.Task() runs on goroutine 43 [FUNC L2]", "2019/07/03 14:02:02 Installs: [] [FUNC L2]", "2019/07/03 14:02:02 Imports: [] [FUNC L2]", "2019/07/03 14:02:02 increase instances to 1 [FUNC L2]", "2019/07/03 14:02:02 .Create(/root/open-lambda/test-dir/worker/lambda_code/3-L2, /root/open-lambda/test-dir/worker/scratch/func-L2-1004, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:02 fork(&{0xc0000826e0 0xc00000f300 4 /root/open-lambda/test-dir/worker/sock-handlers/4 /root/open-lambda/test-dir/worker/lambda_code/3-L2 /root/open-lambda/test-dir/worker/scratch/func-L2-1004 }) [SB 1]", "2019/07/03 14:02:02 ...returns &{0xc0001b6700 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:02 Channel() [SB 4]", "2019/07/03 14:02:02 Pause() [SB 4]", "2019/07/03 14:02:02 Receive request to /run/L3", "2019/07/03 14:02:02 debug: LambdaFunc.Task() runs on goroutine 53 [FUNC L3]", "2019/07/03 14:02:02 Installs: [] [FUNC L3]", "2019/07/03 14:02:02 Imports: [] [FUNC L3]", "2019/07/03 14:02:02 increase instances to 1 [FUNC L3]", "2019/07/03 14:02:02 .Create(/root/open-lambda/test-dir/worker/lambda_code/4-L3, /root/open-lambda/test-dir/worker/scratch/func-L3-1005, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:02 fork(&{0xc0000826e0 0xc00000f320 5 /root/open-lambda/test-dir/worker/sock-handlers/5 /root/open-lambda/test-dir/worker/lambda_code/4-L3 /root/open-lambda/test-dir/worker/scratch/func-L3-1005 }) [SB 1]", "2019/07/03 14:02:02 ...returns &{0xc0001b6a10 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:02 Channel() [SB 5]", "2019/07/03 14:02:02 Pause() [SB 5]", "2019/07/03 14:02:02 Receive request to /run/L4", "2019/07/03 14:02:02 debug: LambdaFunc.Task() runs on goroutine 63 [FUNC L4]", "2019/07/03 14:02:02 Installs: [] [FUNC L4]", "2019/07/03 14:02:02 Imports: [] [FUNC L4]", "2019/07/03 14:02:02 increase instances to 1 [FUNC L4]", "2019/07/03 14:02:02 .Create(/root/open-lambda/test-dir/worker/lambda_code/5-L4, /root/open-lambda/test-dir/worker/scratch/func-L4-1006, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:02 fork(&{0xc0000826e0 0xc00000f340 6 /root/open-lambda/test-dir/worker/sock-handlers/6 /root/open-lambda/test-dir/worker/lambda_code/5-L4 /root/open-lambda/test-dir/worker/scratch/func-L4-1006 }) [SB 1]", "2019/07/03 14:02:02 ...returns &{0xc0001b6d20 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:02 Channel() [SB 6]", "2019/07/03 14:02:02 Pause() [SB 6]", "2019/07/03 14:02:02 Receive request to /run/L5", "2019/07/03 14:02:02 debug: LambdaFunc.Task() runs on goroutine 73 [FUNC L5]", "2019/07/03 14:02:02 Installs: [] [FUNC L5]", "2019/07/03 14:02:02 Imports: [] [FUNC L5]", "2019/07/03 14:02:02 increase instances to 1 [FUNC L5]", "2019/07/03 14:02:02 .Create(/root/open-lambda/test-dir/worker/lambda_code/6-L5, /root/open-lambda/test-dir/worker/scratch/func-L5-1007, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:02 fork(&{0xc0000826e0 0xc00000f360 7 /root/open-lambda/test-dir/worker/sock-handlers/7 /root/open-lambda/test-dir/worker/lambda_code/6-L5 /root/open-lambda/test-dir/worker/scratch/func-L5-1007 }) [SB 1]", "2019/07/03 14:02:02 ...returns &{0xc0001b7030 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:02 Channel() [SB 7]", "2019/07/03 14:02:02 Pause() [SB 7]", "2019/07/03 14:02:02 Receive request to /run/L6", "2019/07/03 14:02:02 debug: LambdaFunc.Task() runs on goroutine 83 [FUNC L6]", "2019/07/03 14:02:02 Installs: [] [FUNC L6]", "2019/07/03 14:02:02 Imports: [] [FUNC L6]", "2019/07/03 14:02:02 increase instances to 1 [FUNC L6]", "2019/07/03 14:02:02 .Create(/root/open-lambda/test-dir/worker/lambda_code/7-L6, /root/open-lambda/test-dir/worker/scratch/func-L6-1008, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:02 fork(&{0xc0000826e0 0xc00000f380 8 /root/open-lambda/test-dir/worker/sock-handlers/8 /root/open-lambda/test-dir/worker/lambda_code/7-L6 /root/open-lambda/test-dir/worker/scratch/func-L6-1008 }) [SB 1]", "2019/07/03 14:02:02 ...returns &{0xc0001b7340 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:02 Channel() [SB 8]", "2019/07/03 14:02:02 Pause() [SB 8]", "2019/07/03 14:02:02 Receive request to /run/L7", "2019/07/03 14:02:02 debug: LambdaFunc.Task() runs on goroutine 93 [FUNC L7]", "2019/07/03 14:02:02 Installs: [] [FUNC L7]", "2019/07/03 14:02:02 Imports: [] [FUNC L7]", "2019/07/03 14:02:02 increase instances to 1 [FUNC L7]", "2019/07/03 14:02:02 .Create(/root/open-lambda/test-dir/worker/lambda_code/8-L7, /root/open-lambda/test-dir/worker/scratch/func-L7-1009, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:02 fork(&{0xc0000826e0 0xc00000f3a0 9 /root/open-lambda/test-dir/worker/sock-handlers/9 /root/open-lambda/test-dir/worker/lambda_code/8-L7 /root/open-lambda/test-dir/worker/scratch/func-L7-1009 }) [SB 1]", "2019/07/03 14:02:02 ...returns &{0xc0001b7650 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:02 Channel() [SB 9]", "2019/07/03 14:02:02 Pause() [SB 9]", "2019/07/03 14:02:02 Receive request to /run/L8", "2019/07/03 14:02:02 debug: LambdaFunc.Task() runs on goroutine 103 [FUNC L8]", "2019/07/03 14:02:02 Installs: [] [FUNC L8]", "2019/07/03 14:02:02 Imports: [] [FUNC L8]", "2019/07/03 14:02:02 increase instances to 1 [FUNC L8]", "2019/07/03 14:02:02 .Create(/root/open-lambda/test-dir/worker/lambda_code/9-L8, /root/open-lambda/test-dir/worker/scratch/func-L8-1010, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:02 fork(&{0xc0000826e0 0xc00000f3c0 10 /root/open-lambda/test-dir/worker/sock-handlers/10 /root/open-lambda/test-dir/worker/lambda_code/9-L8 /root/open-lambda/test-dir/worker/scratch/func-L8-1010 }) [SB 1]", "2019/07/03 14:02:02 ...returns &{0xc0001b7960 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:02 Channel() [SB 10]", "2019/07/03 14:02:02 Pause() [SB 10]", "2019/07/03 14:02:02 Receive request to /run/L9", "2019/07/03 14:02:02 debug: LambdaFunc.Task() runs on goroutine 113 [FUNC L9]", "2019/07/03 14:02:02 Installs: [] [FUNC L9]", "2019/07/03 14:02:02 Imports: [] [FUNC L9]", "2019/07/03 14:02:02 increase instances to 1 [FUNC L9]", "2019/07/03 14:02:02 .Create(/root/open-lambda/test-dir/worker/lambda_code/10-L9, /root/open-lambda/test-dir/worker/scratch/func-L9-1011, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:02 fork(&{0xc0000826e0 0xc00000f3e0 11 /root/open-lambda/test-dir/worker/sock-handlers/11 /root/open-lambda/test-dir/worker/lambda_code/10-L9 /root/open-lambda/test-dir/worker/scratch/func-L9-1011 }) [SB 1]", "2019/07/03 14:02:02 ...returns &{0xc0001b7c70 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:02 Channel() [SB 11]", "2019/07/03 14:02:02 Pause() [SB 11]", "2019/07/03 14:02:02 Receive request to /run/L10", "2019/07/03 14:02:02 debug: LambdaFunc.Task() runs on goroutine 123 [FUNC L10]", "2019/07/03 14:02:02 Installs: [] [FUNC L10]", "2019/07/03 14:02:02 Imports: [] [FUNC L10]", "2019/07/03 14:02:02 increase instances to 1 [FUNC L10]", "2019/07/03 14:02:02 .Create(/root/open-lambda/test-dir/worker/lambda_code/11-L10, /root/open-lambda/test-dir/worker/scratch/func-L10-1012, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:02 fork(&{0xc0000826e0 0xc00000f400 12 /root/open-lambda/test-dir/worker/sock-handlers/12 /root/open-lambda/test-dir/worker/lambda_code/11-L10 /root/open-lambda/test-dir/worker/scratch/func-L10-1012 }) [SB 1]", "2019/07/03 14:02:02 ...returns &{0xc0001b7f80 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:02 Channel() [SB 12]", "2019/07/03 14:02:02 Pause() [SB 12]", "2019/07/03 14:02:02 Receive request to /run/L11", "2019/07/03 14:02:02 debug: LambdaFunc.Task() runs on goroutine 133 [FUNC L11]", "2019/07/03 14:02:02 Installs: [] [FUNC L11]", "2019/07/03 14:02:02 Imports: [] [FUNC L11]", "2019/07/03 14:02:02 increase instances to 1 [FUNC L11]", "2019/07/03 14:02:02 .Create(/root/open-lambda/test-dir/worker/lambda_code/12-L11, /root/open-lambda/test-dir/worker/scratch/func-L11-1013, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:02 fork(&{0xc0000826e0 0xc00000f420 13 /root/open-lambda/test-dir/worker/sock-handlers/13 /root/open-lambda/test-dir/worker/lambda_code/12-L11 /root/open-lambda/test-dir/worker/scratch/func-L11-1013 }) [SB 1]", "2019/07/03 14:02:02 ...returns &{0xc0003122a0 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:02 Channel() [SB 13]", "2019/07/03 14:02:02 Pause() [SB 13]", "2019/07/03 14:02:03 Receive request to /run/L12", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 143 [FUNC L12]", "2019/07/03 14:02:03 Installs: [] [FUNC L12]", "2019/07/03 14:02:03 Imports: [] [FUNC L12]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L12]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/13-L12, /root/open-lambda/test-dir/worker/scratch/func-L12-1014, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc00000f440 14 /root/open-lambda/test-dir/worker/sock-handlers/14 /root/open-lambda/test-dir/worker/lambda_code/13-L12 /root/open-lambda/test-dir/worker/scratch/func-L12-1014 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc0003125b0 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 14]", "2019/07/03 14:02:03 Pause() [SB 14]", "2019/07/03 14:02:03 Receive request to /run/L13", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 153 [FUNC L13]", "2019/07/03 14:02:03 Installs: [] [FUNC L13]", "2019/07/03 14:02:03 Imports: [] [FUNC L13]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L13]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/14-L13, /root/open-lambda/test-dir/worker/scratch/func-L13-1015, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc00000f460 15 /root/open-lambda/test-dir/worker/sock-handlers/15 /root/open-lambda/test-dir/worker/lambda_code/14-L13 /root/open-lambda/test-dir/worker/scratch/func-L13-1015 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc0003128c0 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 15]", "2019/07/03 14:02:03 Pause() [SB 15]", "2019/07/03 14:02:03 Receive request to /run/L14", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 163 [FUNC L14]", "2019/07/03 14:02:03 Installs: [] [FUNC L14]", "2019/07/03 14:02:03 Imports: [] [FUNC L14]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L14]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/15-L14, /root/open-lambda/test-dir/worker/scratch/func-L14-1016, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc00000f480 16 /root/open-lambda/test-dir/worker/sock-handlers/16 /root/open-lambda/test-dir/worker/lambda_code/15-L14 /root/open-lambda/test-dir/worker/scratch/func-L14-1016 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc000312bd0 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 16]", "2019/07/03 14:02:03 Pause() [SB 16]", "2019/07/03 14:02:03 Receive request to /run/L15", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 173 [FUNC L15]", "2019/07/03 14:02:03 Installs: [] [FUNC L15]", "2019/07/03 14:02:03 Imports: [] [FUNC L15]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L15]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/16-L15, /root/open-lambda/test-dir/worker/scratch/func-L15-1017, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc00000f4a0 17 /root/open-lambda/test-dir/worker/sock-handlers/17 /root/open-lambda/test-dir/worker/lambda_code/16-L15 /root/open-lambda/test-dir/worker/scratch/func-L15-1017 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc000312ee0 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 17]", "2019/07/03 14:02:03 Pause() [SB 17]", "2019/07/03 14:02:03 Receive request to /run/L16", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 183 [FUNC L16]", "2019/07/03 14:02:03 Installs: [] [FUNC L16]", "2019/07/03 14:02:03 Imports: [] [FUNC L16]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L16]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/17-L16, /root/open-lambda/test-dir/worker/scratch/func-L16-1018, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc00000f4c0 18 /root/open-lambda/test-dir/worker/sock-handlers/18 /root/open-lambda/test-dir/worker/lambda_code/17-L16 /root/open-lambda/test-dir/worker/scratch/func-L16-1018 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc0003131f0 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 18]", "2019/07/03 14:02:03 Pause() [SB 18]", "2019/07/03 14:02:03 Receive request to /run/L17", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 193 [FUNC L17]", "2019/07/03 14:02:03 Installs: [] [FUNC L17]", "2019/07/03 14:02:03 Imports: [] [FUNC L17]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L17]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/18-L17, /root/open-lambda/test-dir/worker/scratch/func-L17-1019, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc00000fc80 19 /root/open-lambda/test-dir/worker/sock-handlers/19 /root/open-lambda/test-dir/worker/lambda_code/18-L17 /root/open-lambda/test-dir/worker/scratch/func-L17-1019 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc000313500 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 19]", "2019/07/03 14:02:03 Pause() [SB 19]", "2019/07/03 14:02:03 Receive request to /run/L18", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 203 [FUNC L18]", "2019/07/03 14:02:03 Installs: [] [FUNC L18]", "2019/07/03 14:02:03 Imports: [] [FUNC L18]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L18]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/19-L18, /root/open-lambda/test-dir/worker/scratch/func-L18-1020, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc0001de3c0 20 /root/open-lambda/test-dir/worker/sock-handlers/20 /root/open-lambda/test-dir/worker/lambda_code/19-L18 /root/open-lambda/test-dir/worker/scratch/func-L18-1020 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc000313810 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 20]", "2019/07/03 14:02:03 Pause() [SB 20]", "2019/07/03 14:02:03 Receive request to /run/L19", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 213 [FUNC L19]", "2019/07/03 14:02:03 Installs: [] [FUNC L19]", "2019/07/03 14:02:03 Imports: [] [FUNC L19]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L19]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/20-L19, /root/open-lambda/test-dir/worker/scratch/func-L19-1021, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc0001deac0 21 /root/open-lambda/test-dir/worker/sock-handlers/21 /root/open-lambda/test-dir/worker/lambda_code/20-L19 /root/open-lambda/test-dir/worker/scratch/func-L19-1021 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc000313b20 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 21]", "2019/07/03 14:02:03 Pause() [SB 21]", "2019/07/03 14:02:03 Receive request to /run/L20", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 223 [FUNC L20]", "2019/07/03 14:02:03 Installs: [] [FUNC L20]", "2019/07/03 14:02:03 Imports: [] [FUNC L20]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L20]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/21-L20, /root/open-lambda/test-dir/worker/scratch/func-L20-1022, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc0001df1c0 22 /root/open-lambda/test-dir/worker/sock-handlers/22 /root/open-lambda/test-dir/worker/lambda_code/21-L20 /root/open-lambda/test-dir/worker/scratch/func-L20-1022 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc000313e30 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 22]", "2019/07/03 14:02:03 Pause() [SB 22]", "2019/07/03 14:02:03 Receive request to /run/L21", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 233 [FUNC L21]", "2019/07/03 14:02:03 Installs: [] [FUNC L21]", "2019/07/03 14:02:03 Imports: [] [FUNC L21]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L21]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/22-L21, /root/open-lambda/test-dir/worker/scratch/func-L21-1023, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc0001df8c0 23 /root/open-lambda/test-dir/worker/sock-handlers/23 /root/open-lambda/test-dir/worker/lambda_code/22-L21 /root/open-lambda/test-dir/worker/scratch/func-L21-1023 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc00046c150 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 23]", "2019/07/03 14:02:03 http: proxy error: read unix @->/host/ol.sock: read: connection reset by peer", "2019/07/03 14:02:03 Pause() [SB 23]", "2019/07/03 14:02:03 received kill signal, cleaning up", "Kill function: L132019/07/03 14:02:03 Destroy() [SB 15]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 15]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 15]", "Kill function: L102019/07/03 14:02:03 Destroy() [SB 12]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 12]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 12]", "Kill function: L142019/07/03 14:02:03 Destroy() [SB 16]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 16]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 16]", "Kill function: L42019/07/03 14:02:03 Destroy() [SB 6]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 6]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 6]", "Kill function: L52019/07/03 14:02:03 Destroy() [SB 7]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 7]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 7]", "Kill function: L62019/07/03 14:02:03 Destroy() [SB 8]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 8]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 8]", "Kill function: L72019/07/03 14:02:03 Destroy() [SB 9]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 9]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 9]", "Kill function: L82019/07/03 14:02:03 Destroy() [SB 10]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 10]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 10]", "Kill function: L92019/07/03 14:02:03 Destroy() [SB 11]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 11]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 11]", "Kill function: L152019/07/03 14:02:03 Destroy() [SB 17]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 17]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 17]", "Kill function: L162019/07/03 14:02:03 Destroy() [SB 18]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 18]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 18]", "Kill function: L212019/07/03 14:02:03 Destroy() [SB 23]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 23]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 23]", "Kill function: L02019/07/03 14:02:03 Destroy() [SB 2]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 2]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 2]", "Kill function: L112019/07/03 14:02:03 Destroy() [SB 13]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 13]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 13]", "Kill function: L122019/07/03 14:02:03 Destroy() [SB 14]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 14]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 14]", "Kill function: L172019/07/03 14:02:03 Destroy() [SB 19]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 19]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 19]", "Kill function: L182019/07/03 14:02:03 Destroy() [SB 20]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 20]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 20]", "Kill function: L192019/07/03 14:02:03 Destroy() [SB 21]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 21]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 21]", "Kill function: L12019/07/03 14:02:03 Destroy() [SB 3]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 3]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 3]", "Kill function: L22019/07/03 14:02:03 Destroy() [SB 4]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 4]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 4]", "Kill function: L32019/07/03 14:02:03 Destroy() [SB 5]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 5]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 5]", "Kill function: L202019/07/03 14:02:03 Destroy() [SB 22]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 22]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 22]", "2019/07/03 14:02:03 SOCKPool.Cleanup: make sure all memory is free", "2019/07/03 14:02:03 SOCKPool.Cleanup: memory pool emptied", "2019/07/03 14:02:03 received shutdown request [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:03 empty queues and release CGs [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:03 remove /sys/fs/cgroup/blkio/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:03 remove /sys/fs/cgroup/cpu/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:03 remove /sys/fs/cgroup/devices/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:03 remove /sys/fs/cgroup/freezer/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:03 remove /sys/fs/cgroup/hugetlb/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:03 remove /sys/fs/cgroup/memory/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:03 remove /sys/fs/cgroup/perf_event/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:03 remove /sys/fs/cgroup/systemd/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:03 remove /sys/fs/cgroup/pids/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:03 Destroy() [SB 1]", "2019/07/03 14:02:03 kill all procs in CG [SOCK 1]", "2019/07/03 14:02:03 unmount and remove dirs [SOCK 1]", "2019/07/03 14:02:03 SOCKPool.Cleanup: make sure all memory is free", "2019/07/03 14:02:03 SOCKPool.Cleanup: memory pool emptied", "2019/07/03 14:02:03 received shutdown request [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:03 empty queues and release CGs [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:03 remove /sys/fs/cgroup/blkio/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:03 remove /sys/fs/cgroup/cpu/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:03 remove /sys/fs/cgroup/devices/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:03 remove /sys/fs/cgroup/freezer/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:03 remove /sys/fs/cgroup/hugetlb/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:03 remove /sys/fs/cgroup/memory/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:03 remove /sys/fs/cgroup/perf_event/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:03 remove /sys/fs/cgroup/systemd/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:03 remove /sys/fs/cgroup/pids/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:03 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/03 14:02:03 remove worker.pid", "2019/07/03 14:02:03 exiting", "" ] } POP conf: {'registry': '/tmp/tmpj332mnl1'} PUSH conf: {'registry': '/tmp/tmpprxv11bq'} ======================================== call_each_once_exec {'alloc_mb': 10, 'lambda_count': 1000} ======================================== RUN ./ol worker -p=test-dir --detach b'using existing OL directory at /root/open-lambda/test-dir\nStarting worker: pid=15671, port=5000, log=/root/open-lambda/test-dir/worker.out\nready\n' RUN ./ol kill -p=test-dir b'Kill worker process with PID 15671\n' { "test": "call_each_once_exec", "params": { "alloc_mb": 10, "lambda_count": 1000 }, "pass": false, "seconds": null, "total_seconds": 1.1747605800628662, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 26, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 27, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 27, "Create()/fork-proc:ms.avg": 8, "Create()/fork-proc:ms.cnt": 26, "Create()/fresh-proc:ms.avg": 139, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 2, "Create()/make-root-fs:ms.cnt": 27, "Create():ms.avg": 15, "Create():ms.cnt": 27, "Destroy()/detach-root:ms.avg": 0, "Destroy()/detach-root:ms.cnt": 27, "Destroy()/kill-procs:ms.avg": 2, "Destroy()/kill-procs:ms.cnt": 27, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 27, "Destroy():ms.avg": 2, "Destroy():ms.cnt": 30, "Pause():ms.avg": 9, "Pause():ms.cnt": 26, "evict:ms.avg": 2, "evict:ms.cnt": 3, "fork():ms.avg": 8, "fork():ms.cnt": 26, "forkRequest:ms.avg": 6, "forkRequest:ms.cnt": 26, "move-to-cg-after-fork:ms.avg": 0, "move-to-cg-after-fork:ms.cnt": 26 }, "conf": { "worker_port": "5000", "registry_cache_ms": 5000, "server_mode": "lambda", "import_cache_mb": 250, "sandbox_config": {}, "handler_cache_mb": 250, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/tmp/tmpprxv11bq", "worker_dir": "/root/open-lambda/test-dir/worker", "pip_mirror": "", "Skip_pull_existing": false, "cluster_name": "/root/open-lambda/test-dir", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "startup_pkgs": null, "sock_cgroups": { "max_procs": 10, "max_mem_mb": 50 }, "sandbox": "sock", "docker_runtime": "" }, "errors": [ [ "Traceback (most recent call last):", " File \"test.py\", line 76, in wrapper", " rv = fn(**kwargs)", " File \"test.py\", line 216, in call_each_once_exec", " r.raise_for_status()", " File \"/usr/lib/python3/dist-packages/requests/models.py\", line 840, in raise_for_status", " raise HTTPError(http_error_msg, response=self)", "requests.exceptions.HTTPError: 502 Server Error: Bad Gateway for url: http://localhost:5000/run/L25", "" ] ], "worker_tail": [ "using existing OL directory at /root/open-lambda/test-dir", "2019/07/03 14:02:03 save PID 15671 to file /root/open-lambda/test-dir/worker/worker.pid", "2019/07/03 14:02:03 Start Lambda Server", "2019/07/03 14:02:03 Create HandlerPuller", "2019/07/03 14:02:03 Create ModulePuller", "2019/07/03 14:02:03 Create SandboxPool", "2019/07/03 14:02:03 create /sys/fs/cgroup/blkio/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:03 create /sys/fs/cgroup/cpu/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:03 create /sys/fs/cgroup/devices/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:03 create /sys/fs/cgroup/freezer/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:03 create /sys/fs/cgroup/hugetlb/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:03 create /sys/fs/cgroup/memory/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:03 create /sys/fs/cgroup/perf_event/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:03 create /sys/fs/cgroup/systemd/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:03 create /sys/fs/cgroup/pids/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:03 Create ImportCache", "2019/07/03 14:02:03 create /sys/fs/cgroup/blkio/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:03 create /sys/fs/cgroup/cpu/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:03 create /sys/fs/cgroup/devices/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:03 create /sys/fs/cgroup/freezer/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:03 create /sys/fs/cgroup/hugetlb/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:03 create /sys/fs/cgroup/memory/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:03 create /sys/fs/cgroup/perf_event/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:03 create /sys/fs/cgroup/systemd/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:03 create /sys/fs/cgroup/pids/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:03 Setups Handlers", "2019/07/03 14:02:03 Execute handler by POSTing to localhost:5000/run/", "2019/07/03 14:02:03 Get status by sending request to localhost:5000/status", "2019/07/03 14:02:03 start creating/serving CGs [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:03 start creating/serving CGs [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:03 .Create(, /root/open-lambda/test-dir/worker/scratch/import-cache-1001, [], )...", "2019/07/03 14:02:03 Receive request to /pid", "2019/07/03 14:02:03 Receive request to /run/L0", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 22 [FUNC L0]", "2019/07/03 14:02:03 Installs: [] [FUNC L0]", "2019/07/03 14:02:03 Imports: [] [FUNC L0]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L0]", "2019/07/03 14:02:03 ...returns &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/1-L0, /root/open-lambda/test-dir/worker/scratch/func-L0-1002, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc00000f2a0 2 /root/open-lambda/test-dir/worker/sock-handlers/2 /root/open-lambda/test-dir/worker/lambda_code/1-L0 /root/open-lambda/test-dir/worker/scratch/func-L0-1002 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc00029c0e0 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 2]", "2019/07/03 14:02:03 Pause() [SB 2]", "2019/07/03 14:02:03 Receive request to /run/L1", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 33 [FUNC L1]", "2019/07/03 14:02:03 Installs: [] [FUNC L1]", "2019/07/03 14:02:03 Imports: [] [FUNC L1]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L1]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/2-L1, /root/open-lambda/test-dir/worker/scratch/func-L1-1003, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc00000f2c0 3 /root/open-lambda/test-dir/worker/sock-handlers/3 /root/open-lambda/test-dir/worker/lambda_code/2-L1 /root/open-lambda/test-dir/worker/scratch/func-L1-1003 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc00029c3f0 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 3]", "2019/07/03 14:02:03 Pause() [SB 3]", "2019/07/03 14:02:03 Receive request to /run/L2", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 43 [FUNC L2]", "2019/07/03 14:02:03 Installs: [] [FUNC L2]", "2019/07/03 14:02:03 Imports: [] [FUNC L2]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L2]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/3-L2, /root/open-lambda/test-dir/worker/scratch/func-L2-1004, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc00000f2e0 4 /root/open-lambda/test-dir/worker/sock-handlers/4 /root/open-lambda/test-dir/worker/lambda_code/3-L2 /root/open-lambda/test-dir/worker/scratch/func-L2-1004 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc00029c700 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 4]", "2019/07/03 14:02:03 Pause() [SB 4]", "2019/07/03 14:02:03 Receive request to /run/L3", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 53 [FUNC L3]", "2019/07/03 14:02:03 Installs: [] [FUNC L3]", "2019/07/03 14:02:03 Imports: [] [FUNC L3]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L3]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/4-L3, /root/open-lambda/test-dir/worker/scratch/func-L3-1005, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc00000f300 5 /root/open-lambda/test-dir/worker/sock-handlers/5 /root/open-lambda/test-dir/worker/lambda_code/4-L3 /root/open-lambda/test-dir/worker/scratch/func-L3-1005 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc00029ca10 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 5]", "2019/07/03 14:02:03 Pause() [SB 5]", "2019/07/03 14:02:03 Receive request to /run/L4", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 63 [FUNC L4]", "2019/07/03 14:02:03 Installs: [] [FUNC L4]", "2019/07/03 14:02:03 Imports: [] [FUNC L4]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L4]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/5-L4, /root/open-lambda/test-dir/worker/scratch/func-L4-1006, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc00000f320 6 /root/open-lambda/test-dir/worker/sock-handlers/6 /root/open-lambda/test-dir/worker/lambda_code/5-L4 /root/open-lambda/test-dir/worker/scratch/func-L4-1006 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc00029cd20 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 6]", "2019/07/03 14:02:03 Pause() [SB 6]", "2019/07/03 14:02:03 Receive request to /run/L5", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 73 [FUNC L5]", "2019/07/03 14:02:03 Installs: [] [FUNC L5]", "2019/07/03 14:02:03 Imports: [] [FUNC L5]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L5]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/6-L5, /root/open-lambda/test-dir/worker/scratch/func-L5-1007, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc00000f340 7 /root/open-lambda/test-dir/worker/sock-handlers/7 /root/open-lambda/test-dir/worker/lambda_code/6-L5 /root/open-lambda/test-dir/worker/scratch/func-L5-1007 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc00029d030 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 7]", "2019/07/03 14:02:03 Pause() [SB 7]", "2019/07/03 14:02:03 Receive request to /run/L6", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 83 [FUNC L6]", "2019/07/03 14:02:03 Installs: [] [FUNC L6]", "2019/07/03 14:02:03 Imports: [] [FUNC L6]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L6]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/7-L6, /root/open-lambda/test-dir/worker/scratch/func-L6-1008, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc00000f360 8 /root/open-lambda/test-dir/worker/sock-handlers/8 /root/open-lambda/test-dir/worker/lambda_code/7-L6 /root/open-lambda/test-dir/worker/scratch/func-L6-1008 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc00029d340 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 8]", "2019/07/03 14:02:03 Pause() [SB 8]", "2019/07/03 14:02:03 Receive request to /run/L7", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 93 [FUNC L7]", "2019/07/03 14:02:03 Installs: [] [FUNC L7]", "2019/07/03 14:02:03 Imports: [] [FUNC L7]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L7]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/8-L7, /root/open-lambda/test-dir/worker/scratch/func-L7-1009, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc00000f380 9 /root/open-lambda/test-dir/worker/sock-handlers/9 /root/open-lambda/test-dir/worker/lambda_code/8-L7 /root/open-lambda/test-dir/worker/scratch/func-L7-1009 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc00029d650 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 9]", "2019/07/03 14:02:03 Pause() [SB 9]", "2019/07/03 14:02:03 Receive request to /run/L8", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 103 [FUNC L8]", "2019/07/03 14:02:03 Installs: [] [FUNC L8]", "2019/07/03 14:02:03 Imports: [] [FUNC L8]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L8]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/9-L8, /root/open-lambda/test-dir/worker/scratch/func-L8-1010, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc00000f3a0 10 /root/open-lambda/test-dir/worker/sock-handlers/10 /root/open-lambda/test-dir/worker/lambda_code/9-L8 /root/open-lambda/test-dir/worker/scratch/func-L8-1010 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc00029d960 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 10]", "2019/07/03 14:02:03 Pause() [SB 10]", "2019/07/03 14:02:03 Receive request to /run/L9", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 113 [FUNC L9]", "2019/07/03 14:02:03 Installs: [] [FUNC L9]", "2019/07/03 14:02:03 Imports: [] [FUNC L9]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L9]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/10-L9, /root/open-lambda/test-dir/worker/scratch/func-L9-1011, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc00000f3c0 11 /root/open-lambda/test-dir/worker/sock-handlers/11 /root/open-lambda/test-dir/worker/lambda_code/10-L9 /root/open-lambda/test-dir/worker/scratch/func-L9-1011 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc00029dc70 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 11]", "2019/07/03 14:02:03 Pause() [SB 11]", "2019/07/03 14:02:03 Receive request to /run/L10", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 123 [FUNC L10]", "2019/07/03 14:02:03 Installs: [] [FUNC L10]", "2019/07/03 14:02:03 Imports: [] [FUNC L10]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L10]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/11-L10, /root/open-lambda/test-dir/worker/scratch/func-L10-1012, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc00000f3e0 12 /root/open-lambda/test-dir/worker/sock-handlers/12 /root/open-lambda/test-dir/worker/lambda_code/11-L10 /root/open-lambda/test-dir/worker/scratch/func-L10-1012 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc00029df80 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 12]", "2019/07/03 14:02:03 Pause() [SB 12]", "2019/07/03 14:02:03 Receive request to /run/L11", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 133 [FUNC L11]", "2019/07/03 14:02:03 Installs: [] [FUNC L11]", "2019/07/03 14:02:03 Imports: [] [FUNC L11]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L11]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/12-L11, /root/open-lambda/test-dir/worker/scratch/func-L11-1013, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc00000f400 13 /root/open-lambda/test-dir/worker/sock-handlers/13 /root/open-lambda/test-dir/worker/lambda_code/12-L11 /root/open-lambda/test-dir/worker/scratch/func-L11-1013 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc0003fe2a0 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 13]", "2019/07/03 14:02:03 Pause() [SB 13]", "2019/07/03 14:02:03 Receive request to /run/L12", "2019/07/03 14:02:03 debug: LambdaFunc.Task() runs on goroutine 143 [FUNC L12]", "2019/07/03 14:02:03 Installs: [] [FUNC L12]", "2019/07/03 14:02:03 Imports: [] [FUNC L12]", "2019/07/03 14:02:03 increase instances to 1 [FUNC L12]", "2019/07/03 14:02:03 .Create(/root/open-lambda/test-dir/worker/lambda_code/13-L12, /root/open-lambda/test-dir/worker/scratch/func-L12-1014, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:03 fork(&{0xc0000826e0 0xc00000f420 14 /root/open-lambda/test-dir/worker/sock-handlers/14 /root/open-lambda/test-dir/worker/lambda_code/13-L12 /root/open-lambda/test-dir/worker/scratch/func-L12-1014 }) [SB 1]", "2019/07/03 14:02:03 ...returns &{0xc0003fe5b0 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:03 Channel() [SB 14]", "2019/07/03 14:02:04 Pause() [SB 14]", "2019/07/03 14:02:04 Receive request to /run/L13", "2019/07/03 14:02:04 debug: LambdaFunc.Task() runs on goroutine 153 [FUNC L13]", "2019/07/03 14:02:04 Installs: [] [FUNC L13]", "2019/07/03 14:02:04 Imports: [] [FUNC L13]", "2019/07/03 14:02:04 increase instances to 1 [FUNC L13]", "2019/07/03 14:02:04 .Create(/root/open-lambda/test-dir/worker/lambda_code/14-L13, /root/open-lambda/test-dir/worker/scratch/func-L13-1015, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:04 fork(&{0xc0000826e0 0xc00000f440 15 /root/open-lambda/test-dir/worker/sock-handlers/15 /root/open-lambda/test-dir/worker/lambda_code/14-L13 /root/open-lambda/test-dir/worker/scratch/func-L13-1015 }) [SB 1]", "2019/07/03 14:02:04 ...returns &{0xc0003fe8c0 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:04 Channel() [SB 15]", "2019/07/03 14:02:04 Evict Sandbox 2", "2019/07/03 14:02:04 Destroy() [SB 2]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 2]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 2]", "2019/07/03 14:02:04 Pause() [SB 15]", "2019/07/03 14:02:04 Receive request to /run/L14", "2019/07/03 14:02:04 debug: LambdaFunc.Task() runs on goroutine 164 [FUNC L14]", "2019/07/03 14:02:04 Installs: [] [FUNC L14]", "2019/07/03 14:02:04 Imports: [] [FUNC L14]", "2019/07/03 14:02:04 increase instances to 1 [FUNC L14]", "2019/07/03 14:02:04 .Create(/root/open-lambda/test-dir/worker/lambda_code/15-L14, /root/open-lambda/test-dir/worker/scratch/func-L14-1016, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:04 fork(&{0xc0000826e0 0xc00000f460 16 /root/open-lambda/test-dir/worker/sock-handlers/16 /root/open-lambda/test-dir/worker/lambda_code/15-L14 /root/open-lambda/test-dir/worker/scratch/func-L14-1016 }) [SB 1]", "2019/07/03 14:02:04 ...returns &{0xc0003febd0 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:04 Channel() [SB 16]", "2019/07/03 14:02:04 Pause() [SB 16]", "2019/07/03 14:02:04 Receive request to /run/L15", "2019/07/03 14:02:04 debug: LambdaFunc.Task() runs on goroutine 174 [FUNC L15]", "2019/07/03 14:02:04 Installs: [] [FUNC L15]", "2019/07/03 14:02:04 Imports: [] [FUNC L15]", "2019/07/03 14:02:04 increase instances to 1 [FUNC L15]", "2019/07/03 14:02:04 .Create(/root/open-lambda/test-dir/worker/lambda_code/16-L15, /root/open-lambda/test-dir/worker/scratch/func-L15-1017, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:04 fork(&{0xc0000826e0 0xc00000f480 17 /root/open-lambda/test-dir/worker/sock-handlers/17 /root/open-lambda/test-dir/worker/lambda_code/16-L15 /root/open-lambda/test-dir/worker/scratch/func-L15-1017 }) [SB 1]", "2019/07/03 14:02:04 ...returns &{0xc0003feee0 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:04 Channel() [SB 17]", "2019/07/03 14:02:04 Pause() [SB 17]", "2019/07/03 14:02:04 Receive request to /run/L16", "2019/07/03 14:02:04 debug: LambdaFunc.Task() runs on goroutine 184 [FUNC L16]", "2019/07/03 14:02:04 Installs: [] [FUNC L16]", "2019/07/03 14:02:04 Imports: [] [FUNC L16]", "2019/07/03 14:02:04 increase instances to 1 [FUNC L16]", "2019/07/03 14:02:04 .Create(/root/open-lambda/test-dir/worker/lambda_code/17-L16, /root/open-lambda/test-dir/worker/scratch/func-L16-1018, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:04 fork(&{0xc0000826e0 0xc00000f4a0 18 /root/open-lambda/test-dir/worker/sock-handlers/18 /root/open-lambda/test-dir/worker/lambda_code/17-L16 /root/open-lambda/test-dir/worker/scratch/func-L16-1018 }) [SB 1]", "2019/07/03 14:02:04 ...returns &{0xc0003ff1f0 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:04 Channel() [SB 18]", "2019/07/03 14:02:04 Pause() [SB 18]", "2019/07/03 14:02:04 Receive request to /run/L17", "2019/07/03 14:02:04 debug: LambdaFunc.Task() runs on goroutine 195 [FUNC L17]", "2019/07/03 14:02:04 Installs: [] [FUNC L17]", "2019/07/03 14:02:04 Imports: [] [FUNC L17]", "2019/07/03 14:02:04 increase instances to 1 [FUNC L17]", "2019/07/03 14:02:04 .Create(/root/open-lambda/test-dir/worker/lambda_code/18-L17, /root/open-lambda/test-dir/worker/scratch/func-L17-1019, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:04 fork(&{0xc0000826e0 0xc00000fc60 19 /root/open-lambda/test-dir/worker/sock-handlers/19 /root/open-lambda/test-dir/worker/lambda_code/18-L17 /root/open-lambda/test-dir/worker/scratch/func-L17-1019 }) [SB 1]", "2019/07/03 14:02:04 ...returns &{0xc0000d6380 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:04 Channel() [SB 19]", "2019/07/03 14:02:04 Evict Sandbox 3", "2019/07/03 14:02:04 Destroy() [SB 3]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 3]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 3]", "2019/07/03 14:02:04 Pause() [SB 19]", "2019/07/03 14:02:04 Receive request to /run/L18", "2019/07/03 14:02:04 debug: LambdaFunc.Task() runs on goroutine 206 [FUNC L18]", "2019/07/03 14:02:04 Installs: [] [FUNC L18]", "2019/07/03 14:02:04 Imports: [] [FUNC L18]", "2019/07/03 14:02:04 increase instances to 1 [FUNC L18]", "2019/07/03 14:02:04 .Create(/root/open-lambda/test-dir/worker/lambda_code/19-L18, /root/open-lambda/test-dir/worker/scratch/func-L18-1020, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:04 fork(&{0xc0000826e0 0xc0002c23a0 20 /root/open-lambda/test-dir/worker/sock-handlers/20 /root/open-lambda/test-dir/worker/lambda_code/19-L18 /root/open-lambda/test-dir/worker/scratch/func-L18-1020 }) [SB 1]", "2019/07/03 14:02:04 ...returns &{0xc0000d6700 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:04 Channel() [SB 20]", "2019/07/03 14:02:04 Pause() [SB 20]", "2019/07/03 14:02:04 Receive request to /run/L19", "2019/07/03 14:02:04 debug: LambdaFunc.Task() runs on goroutine 216 [FUNC L19]", "2019/07/03 14:02:04 Installs: [] [FUNC L19]", "2019/07/03 14:02:04 Imports: [] [FUNC L19]", "2019/07/03 14:02:04 increase instances to 1 [FUNC L19]", "2019/07/03 14:02:04 .Create(/root/open-lambda/test-dir/worker/lambda_code/20-L19, /root/open-lambda/test-dir/worker/scratch/func-L19-1021, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:04 fork(&{0xc0000826e0 0xc0002c2a40 21 /root/open-lambda/test-dir/worker/sock-handlers/21 /root/open-lambda/test-dir/worker/lambda_code/20-L19 /root/open-lambda/test-dir/worker/scratch/func-L19-1021 }) [SB 1]", "2019/07/03 14:02:04 ...returns &{0xc0000d6a10 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:04 Channel() [SB 21]", "2019/07/03 14:02:04 Pause() [SB 21]", "2019/07/03 14:02:04 Receive request to /run/L20", "2019/07/03 14:02:04 debug: LambdaFunc.Task() runs on goroutine 226 [FUNC L20]", "2019/07/03 14:02:04 Installs: [] [FUNC L20]", "2019/07/03 14:02:04 Imports: [] [FUNC L20]", "2019/07/03 14:02:04 increase instances to 1 [FUNC L20]", "2019/07/03 14:02:04 .Create(/root/open-lambda/test-dir/worker/lambda_code/21-L20, /root/open-lambda/test-dir/worker/scratch/func-L20-1022, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:04 fork(&{0xc0000826e0 0xc0002c31a0 22 /root/open-lambda/test-dir/worker/sock-handlers/22 /root/open-lambda/test-dir/worker/lambda_code/21-L20 /root/open-lambda/test-dir/worker/scratch/func-L20-1022 }) [SB 1]", "2019/07/03 14:02:04 ...returns &{0xc0000d6d20 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:04 Channel() [SB 22]", "2019/07/03 14:02:04 Pause() [SB 22]", "2019/07/03 14:02:04 Receive request to /run/L21", "2019/07/03 14:02:04 debug: LambdaFunc.Task() runs on goroutine 236 [FUNC L21]", "2019/07/03 14:02:04 Installs: [] [FUNC L21]", "2019/07/03 14:02:04 Imports: [] [FUNC L21]", "2019/07/03 14:02:04 increase instances to 1 [FUNC L21]", "2019/07/03 14:02:04 .Create(/root/open-lambda/test-dir/worker/lambda_code/22-L21, /root/open-lambda/test-dir/worker/scratch/func-L21-1023, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:04 fork(&{0xc0000826e0 0xc0002c3840 23 /root/open-lambda/test-dir/worker/sock-handlers/23 /root/open-lambda/test-dir/worker/lambda_code/22-L21 /root/open-lambda/test-dir/worker/scratch/func-L21-1023 }) [SB 1]", "2019/07/03 14:02:04 Evict Sandbox 4", "2019/07/03 14:02:04 Destroy() [SB 4]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 4]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 4]", "2019/07/03 14:02:04 ...returns &{0xc0000d7030 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:04 Channel() [SB 23]", "2019/07/03 14:02:04 Pause() [SB 23]", "2019/07/03 14:02:04 Receive request to /run/L22", "2019/07/03 14:02:04 debug: LambdaFunc.Task() runs on goroutine 247 [FUNC L22]", "2019/07/03 14:02:04 Installs: [] [FUNC L22]", "2019/07/03 14:02:04 Imports: [] [FUNC L22]", "2019/07/03 14:02:04 increase instances to 1 [FUNC L22]", "2019/07/03 14:02:04 .Create(/root/open-lambda/test-dir/worker/lambda_code/23-L22, /root/open-lambda/test-dir/worker/scratch/func-L22-1024, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:04 fork(&{0xc0000826e0 0xc0002c3fa0 24 /root/open-lambda/test-dir/worker/sock-handlers/24 /root/open-lambda/test-dir/worker/lambda_code/23-L22 /root/open-lambda/test-dir/worker/scratch/func-L22-1024 }) [SB 1]", "2019/07/03 14:02:04 ...returns &{0xc0000d7340 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:04 Channel() [SB 24]", "2019/07/03 14:02:04 Pause() [SB 24]", "2019/07/03 14:02:04 Receive request to /run/L23", "2019/07/03 14:02:04 debug: LambdaFunc.Task() runs on goroutine 257 [FUNC L23]", "2019/07/03 14:02:04 Installs: [] [FUNC L23]", "2019/07/03 14:02:04 Imports: [] [FUNC L23]", "2019/07/03 14:02:04 increase instances to 1 [FUNC L23]", "2019/07/03 14:02:04 .Create(/root/open-lambda/test-dir/worker/lambda_code/24-L23, /root/open-lambda/test-dir/worker/scratch/func-L23-1025, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:04 fork(&{0xc0000826e0 0xc00035c640 25 /root/open-lambda/test-dir/worker/sock-handlers/25 /root/open-lambda/test-dir/worker/lambda_code/24-L23 /root/open-lambda/test-dir/worker/scratch/func-L23-1025 }) [SB 1]", "2019/07/03 14:02:04 ...returns &{0xc0000d76c0 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:04 Channel() [SB 25]", "2019/07/03 14:02:04 Pause() [SB 25]", "2019/07/03 14:02:04 Receive request to /run/L24", "2019/07/03 14:02:04 debug: LambdaFunc.Task() runs on goroutine 267 [FUNC L24]", "2019/07/03 14:02:04 Installs: [] [FUNC L24]", "2019/07/03 14:02:04 Imports: [] [FUNC L24]", "2019/07/03 14:02:04 increase instances to 1 [FUNC L24]", "2019/07/03 14:02:04 .Create(/root/open-lambda/test-dir/worker/lambda_code/25-L24, /root/open-lambda/test-dir/worker/scratch/func-L24-1026, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:04 fork(&{0xc0000826e0 0xc00035cd40 26 /root/open-lambda/test-dir/worker/sock-handlers/26 /root/open-lambda/test-dir/worker/lambda_code/25-L24 /root/open-lambda/test-dir/worker/scratch/func-L24-1026 }) [SB 1]", "2019/07/03 14:02:04 ...returns &{0xc0000d79d0 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:04 Channel() [SB 26]", "2019/07/03 14:02:04 Pause() [SB 26]", "2019/07/03 14:02:04 Receive request to /run/L25", "2019/07/03 14:02:04 debug: LambdaFunc.Task() runs on goroutine 277 [FUNC L25]", "2019/07/03 14:02:04 Installs: [] [FUNC L25]", "2019/07/03 14:02:04 Imports: [] [FUNC L25]", "2019/07/03 14:02:04 increase instances to 1 [FUNC L25]", "2019/07/03 14:02:04 .Create(/root/open-lambda/test-dir/worker/lambda_code/26-L25, /root/open-lambda/test-dir/worker/scratch/func-L25-1027, [], &{0xc00011de30 {0 0} false [0x819070 0x81a3d0]})...", "2019/07/03 14:02:04 fork(&{0xc0000826e0 0xc00035d440 27 /root/open-lambda/test-dir/worker/sock-handlers/27 /root/open-lambda/test-dir/worker/lambda_code/26-L25 /root/open-lambda/test-dir/worker/scratch/func-L25-1027 }) [SB 1]", "2019/07/03 14:02:04 ...returns &{0xc0000d7d50 {0 0} false [0x819070 0x81a3d0]}, ", "2019/07/03 14:02:04 Channel() [SB 27]", "2019/07/03 14:02:04 http: proxy error: dial unix /root/open-lambda/test-dir/worker/scratch/func-L25-1027/ol.sock: connect: no such file or directory", "2019/07/03 14:02:04 Pause() [SB 27]", "2019/07/03 14:02:04 received kill signal, cleaning up", "Kill function: L62019/07/03 14:02:04 Destroy() [SB 8]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 8]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 8]", "Kill function: L82019/07/03 14:02:04 Destroy() [SB 10]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 10]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 10]", "Kill function: L102019/07/03 14:02:04 Destroy() [SB 12]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 12]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 12]", "Kill function: L142019/07/03 14:02:04 Destroy() [SB 16]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 16]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 16]", "Kill function: L232019/07/03 14:02:04 Destroy() [SB 25]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 25]", "2019/07/03 14:02:04 waiting for 1 procs in cg-24 to die [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 25]", "Kill function: L252019/07/03 14:02:04 Destroy() [SB 27]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 27]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 27]", "Kill function: L12019/07/03 14:02:04 Destroy() [SB 3]", "Kill function: L122019/07/03 14:02:04 Destroy() [SB 14]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 14]", "2019/07/03 14:02:04 waiting for 1 procs in cg-13 to die [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 14]", "Kill function: L152019/07/03 14:02:04 Destroy() [SB 17]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 17]", "2019/07/03 14:02:04 waiting for 1 procs in cg-16 to die [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 17]", "Kill function: L172019/07/03 14:02:04 Destroy() [SB 19]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 19]", "2019/07/03 14:02:04 waiting for 1 procs in cg-18 to die [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 19]", "Kill function: L182019/07/03 14:02:04 Destroy() [SB 20]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 20]", "2019/07/03 14:02:04 waiting for 1 procs in cg-19 to die [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 20]", "Kill function: L192019/07/03 14:02:04 Destroy() [SB 21]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 21]", "2019/07/03 14:02:04 waiting for 1 procs in cg-20 to die [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 21]", "Kill function: L212019/07/03 14:02:04 Destroy() [SB 23]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 23]", "2019/07/03 14:02:04 waiting for 1 procs in cg-22 to die [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 23]", "Kill function: L222019/07/03 14:02:04 Destroy() [SB 24]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 24]", "2019/07/03 14:02:04 waiting for 1 procs in cg-23 to die [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 24]", "Kill function: L02019/07/03 14:02:04 Destroy() [SB 2]", "Kill function: L22019/07/03 14:02:04 Destroy() [SB 4]", "Kill function: L42019/07/03 14:02:04 Destroy() [SB 6]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 6]", "2019/07/03 14:02:04 waiting for 1 procs in cg-5 to die [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 6]", "Kill function: L72019/07/03 14:02:04 Destroy() [SB 9]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 9]", "2019/07/03 14:02:04 waiting for 1 procs in cg-8 to die [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 9]", "Kill function: L92019/07/03 14:02:04 Destroy() [SB 11]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 11]", "2019/07/03 14:02:04 waiting for 1 procs in cg-10 to die [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 11]", "Kill function: L162019/07/03 14:02:04 Destroy() [SB 18]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 18]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 18]", "Kill function: L202019/07/03 14:02:04 Destroy() [SB 22]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 22]", "2019/07/03 14:02:04 waiting for 1 procs in cg-21 to die [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 22]", "Kill function: L242019/07/03 14:02:04 Destroy() [SB 26]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 26]", "2019/07/03 14:02:04 waiting for 1 procs in cg-25 to die [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 26]", "Kill function: L32019/07/03 14:02:04 Destroy() [SB 5]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 5]", "2019/07/03 14:02:04 waiting for 1 procs in cg-4 to die [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 5]", "Kill function: L52019/07/03 14:02:04 Destroy() [SB 7]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 7]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 7]", "Kill function: L112019/07/03 14:02:04 Destroy() [SB 13]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 13]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 13]", "Kill function: L132019/07/03 14:02:04 Destroy() [SB 15]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 15]", "2019/07/03 14:02:04 waiting for 1 procs in cg-14 to die [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 15]", "2019/07/03 14:02:04 SOCKPool.Cleanup: make sure all memory is free", "2019/07/03 14:02:04 SOCKPool.Cleanup: memory pool emptied", "2019/07/03 14:02:04 received shutdown request [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 empty queues and release CGs [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 remove /sys/fs/cgroup/blkio/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 remove /sys/fs/cgroup/cpu/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 remove /sys/fs/cgroup/devices/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 remove /sys/fs/cgroup/freezer/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 remove /sys/fs/cgroup/hugetlb/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 remove /sys/fs/cgroup/memory/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 remove /sys/fs/cgroup/perf_event/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 remove /sys/fs/cgroup/systemd/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 remove /sys/fs/cgroup/pids/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/03 14:02:04 Destroy() [SB 1]", "2019/07/03 14:02:04 kill all procs in CG [SOCK 1]", "2019/07/03 14:02:04 unmount and remove dirs [SOCK 1]", "2019/07/03 14:02:04 SOCKPool.Cleanup: make sure all memory is free", "2019/07/03 14:02:04 SOCKPool.Cleanup: memory pool emptied", "2019/07/03 14:02:04 received shutdown request [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:04 empty queues and release CGs [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:04 remove /sys/fs/cgroup/blkio/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:04 remove /sys/fs/cgroup/cpu/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:04 remove /sys/fs/cgroup/devices/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:04 remove /sys/fs/cgroup/freezer/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:04 remove /sys/fs/cgroup/hugetlb/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:04 remove /sys/fs/cgroup/memory/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:04 remove /sys/fs/cgroup/perf_event/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:04 remove /sys/fs/cgroup/systemd/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:04 remove /sys/fs/cgroup/pids/worker-sock-cache [CGROUP POOL worker-sock-cache]", "2019/07/03 14:02:04 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/03 14:02:04 remove worker.pid", "2019/07/03 14:02:04 exiting", "" ] } POP conf: {'registry': '/tmp/tmpprxv11bq'} POP conf: {'handler_cache_mb': 250, 'sandbox': 'sock', 'import_cache_mb': 250} PASSED: 15, FAILED: 2 Makefile:25: recipe for target 'test-all' failed make: *** [test-all] Error 2