python3 -u test.py RUN ./ol new -p=test-dir PUSH conf: {'registry': '/root/open-lambda/test-registry'} ======================================== ping_test ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "ping_test", "params": {}, "pass": true, "conf": { "worker_port": "5000", "registry": "/root/open-lambda/test-registry", "docker_runtime": "", "limits": { "installer_mem_mb": 200, "procs": 10, "mem_mb": 50 }, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "import_cache_mb": 250, "import_cache_tree": "", "pip_mirror": "", "registry_cache_ms": 5000, "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-dir/worker", "handler_cache_mb": 250, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "sandbox_config": {}, "sandbox": "sock" }, "seconds": 2.237283229827881, "total_seconds": 2.5285143852233887, "stats": { "pings_per_sec": 446.9799178400904 }, "ol-stats": {}, "errors": [], "worker_tail": [ "2019/07/12 19:17:55 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:17:55 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:17:55 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:17:55 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:17:55 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:17:55 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:17:55 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:17:55 remove worker.pid", "2019/07/12 19:17:55 exiting", "" ] } PUSH conf: {'import_cache_mb': 0, 'handler_cache_mb': 500} ======================================== install_tests ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "install_tests", "params": {}, "pass": true, "conf": { "worker_port": "5000", "registry": "/root/open-lambda/test-registry", "docker_runtime": "", "limits": { "installer_mem_mb": 200, "procs": 10, "mem_mb": 50 }, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "import_cache_mb": 0, "import_cache_tree": "", "pip_mirror": "", "registry_cache_ms": 5000, "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-dir/worker", "handler_cache_mb": 500, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "sandbox_config": {}, "sandbox": "sock" }, "seconds": 10.935606002807617, "total_seconds": 11.17734146118164, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 10, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 10, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 10, "Create()/fresh-proc:ms.avg": 280, "Create()/fresh-proc:ms.cnt": 10, "Create()/make-root-fs:ms.avg": 1, "Create()/make-root-fs:ms.cnt": 10, "Create():ms.avg": 282, "Create():ms.cnt": 10, "Destroy()/detach-root:ms.avg": 0, "Destroy()/detach-root:ms.cnt": 10, "Destroy()/kill-procs:ms.avg": 2, "Destroy()/kill-procs:ms.cnt": 10, "Destroy()/recursive-kill:ms.avg": 0, "Destroy()/recursive-kill:ms.cnt": 10, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 10, "Destroy():ms.avg": 3, "Destroy():ms.cnt": 10, "Pause():ms.avg": 4, "Pause():ms.cnt": 4, "Status():ms.avg": 0, "Status():ms.cnt": 6, "pip-install:ms.avg": 1440, "pip-install:ms.cnt": 6 }, "errors": [], "worker_tail": [ "2019/07/12 19:18:06 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:06 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:06 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:06 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:06 0 of 500 MB available [MEM POOL handler-sandboxes]", "2019/07/12 19:18:06 WARNING! Critically low on memory, so evicting an active Sandbox", "2019/07/12 19:18:06 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:18:06 remove worker.pid", "2019/07/12 19:18:06 exiting", "" ] } POP conf: {'import_cache_mb': 0, 'handler_cache_mb': 500} PUSH conf: {'import_cache_mb': 250, 'handler_cache_mb': 250} ======================================== install_tests ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "install_tests", "params": {}, "pass": true, "conf": { "worker_port": "5000", "registry": "/root/open-lambda/test-registry", "docker_runtime": "", "limits": { "installer_mem_mb": 200, "procs": 10, "mem_mb": 50 }, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "import_cache_mb": 250, "import_cache_tree": "", "pip_mirror": "", "registry_cache_ms": 5000, "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-dir/worker", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "sandbox_config": {}, "handler_cache_mb": 250 }, "seconds": 6.545871734619141, "total_seconds": 6.7778191566467285, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 10, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 11, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 11, "Create()/fork-proc:ms.avg": 7, "Create()/fork-proc:ms.cnt": 4, "Create()/fresh-proc:ms.avg": 145, "Create()/fresh-proc:ms.cnt": 7, "Create()/make-root-fs:ms.avg": 1, "Create()/make-root-fs:ms.cnt": 11, "Create():ms.avg": 97, "Create():ms.cnt": 11, "Destroy()/detach-root:ms.avg": 0, "Destroy()/detach-root:ms.cnt": 11, "Destroy()/kill-procs:ms.avg": 2, "Destroy()/kill-procs:ms.cnt": 11, "Destroy()/recursive-kill:ms.avg": 0, "Destroy()/recursive-kill:ms.cnt": 11, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 11, "Destroy():ms.avg": 1, "Destroy():ms.cnt": 18, "Pause():ms.avg": 3, "Pause():ms.cnt": 4, "Status():ms.avg": 0, "Status():ms.cnt": 6, "evict:ms.avg": 1, "evict:ms.cnt": 3, "fork():ms.avg": 7, "fork():ms.cnt": 4, "forkRequest:ms.avg": 4, "forkRequest:ms.cnt": 4, "move-to-cg-after-fork:ms.avg": 2, "move-to-cg-after-fork:ms.cnt": 4, "pip-install:ms.avg": 1005, "pip-install:ms.cnt": 6 }, "errors": [], "worker_tail": [ "2019/07/12 19:18:13 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:13 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:13 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:13 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:13 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:13 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:13 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:18:13 remove worker.pid", "2019/07/12 19:18:13 exiting", "" ] } POP conf: {'import_cache_mb': 250, 'handler_cache_mb': 250} PUSH conf: {'sandbox': 'docker', 'import_cache_mb': 0, 'handler_cache_mb': 500} ======================================== install_tests ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "install_tests", "params": {}, "pass": true, "conf": { "worker_port": "5000", "registry": "/root/open-lambda/test-registry", "docker_runtime": "", "limits": { "installer_mem_mb": 200, "procs": 10, "mem_mb": 50 }, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "import_cache_mb": 0, "import_cache_tree": "", "pip_mirror": "", "registry_cache_ms": 5000, "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-dir/worker", "handler_cache_mb": 500, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "sandbox_config": {}, "sandbox": "docker" }, "seconds": 17.730334758758545, "total_seconds": 18.99715232849121, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 10, "Create():ms.avg": 876, "Create():ms.cnt": 10, "Destroy():ms.avg": 268, "Destroy():ms.cnt": 10, "Pause():ms.avg": 30, "Pause():ms.cnt": 4, "Status():ms.avg": 0, "Status():ms.cnt": 6, "pip-install:ms.avg": 2038, "pip-install:ms.cnt": 6 }, "errors": [], "worker_tail": [ "2019/07/12 19:18:31 Receive request to /stats", "2019/07/12 19:18:31 received kill signal, cleaning up", "Kill function: echo2019/07/12 19:18:31 Destroy() [SB 0]", "Kill function: install2019/07/12 19:18:31 Destroy() [SB 6]", "Kill function: install22019/07/12 19:18:32 Destroy() [SB 7]", "Kill function: install32019/07/12 19:18:32 Destroy() [SB 9]", "2019/07/12 19:18:32 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:18:32 remove worker.pid", "2019/07/12 19:18:32 exiting", "" ] } POP conf: {'sandbox': 'docker', 'import_cache_mb': 0, 'handler_cache_mb': 500} ======================================== fork_bomb ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "fork_bomb", "params": {}, "pass": false, "conf": { "worker_port": "5000", "registry": "/root/open-lambda/test-registry", "docker_runtime": "", "limits": { "installer_mem_mb": 200, "procs": 10, "mem_mb": 50 }, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "import_cache_mb": 250, "import_cache_tree": "", "pip_mirror": "", "registry_cache_ms": 5000, "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-dir/worker", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "sandbox_config": {}, "handler_cache_mb": 250 }, "seconds": 0.35635828971862793, "total_seconds": 0.5872855186462402, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 1, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 2, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 2, "Create()/fork-proc:ms.avg": 7, "Create()/fork-proc:ms.cnt": 1, "Create()/fresh-proc:ms.avg": 315, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 1, "Create()/make-root-fs:ms.cnt": 2, "Create():ms.avg": 163, "Create():ms.cnt": 2, "Destroy()/detach-root:ms.avg": 1, "Destroy()/detach-root:ms.cnt": 2, "Destroy()/kill-procs:ms.avg": 0, "Destroy()/kill-procs:ms.cnt": 2, "Destroy()/recursive-kill:ms.avg": 1, "Destroy()/recursive-kill:ms.cnt": 2, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 2, "Destroy():ms.avg": 2, "Destroy():ms.cnt": 3, "Pause():ms.avg": 4, "Pause():ms.cnt": 1, "fork():ms.avg": 7, "fork():ms.cnt": 1, "forkRequest:ms.avg": 4, "forkRequest:ms.cnt": 1, "move-to-cg-after-fork:ms.avg": 3, "move-to-cg-after-fork:ms.cnt": 1 }, "errors": [ [ "mounts are leaking (29 before, 30 after), leaked: {'/dev/xvda1 on /root/open-lambda/test-dir/worker/cache-sandboxes type ext4 (rw,relatime,data=ordered)'}" ] ], "worker_tail": [ "using existing OL directory at /root/open-lambda/test-dir", "2019/07/12 19:18:32 save PID 10031 to file /root/open-lambda/test-dir/worker/worker.pid", "2019/07/12 19:18:32 Start Lambda Server", "2019/07/12 19:18:32 Create SandboxPool", "2019/07/12 19:18:32 create /sys/fs/cgroup/blkio/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:32 create /sys/fs/cgroup/cpu/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:32 create /sys/fs/cgroup/devices/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:32 create /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:32 create /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:32 create /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:32 create /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:32 create /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:32 create /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:32 Create DepTracer", "2019/07/12 19:18:32 Create PackagePuller", "2019/07/12 19:18:32 Create ImportCache", "2019/07/12 19:18:32 Import Cache Tree:", "2019/07/12 19:18:32 - ROOT", "2019/07/12 19:18:32 create /sys/fs/cgroup/blkio/worker-cache-sandboxes [CGROUP POOL worker-cache-sandboxes]", "2019/07/12 19:18:32 create /sys/fs/cgroup/cpu/worker-cache-sandboxes [CGROUP POOL worker-cache-sandboxes]", "2019/07/12 19:18:32 create /sys/fs/cgroup/devices/worker-cache-sandboxes [CGROUP POOL worker-cache-sandboxes]", "2019/07/12 19:18:32 create /sys/fs/cgroup/freezer/worker-cache-sandboxes [CGROUP POOL worker-cache-sandboxes]", "2019/07/12 19:18:32 create /sys/fs/cgroup/hugetlb/worker-cache-sandboxes [CGROUP POOL worker-cache-sandboxes]", "2019/07/12 19:18:32 create /sys/fs/cgroup/memory/worker-cache-sandboxes [CGROUP POOL worker-cache-sandboxes]", "2019/07/12 19:18:32 create /sys/fs/cgroup/perf_event/worker-cache-sandboxes [CGROUP POOL worker-cache-sandboxes]", "2019/07/12 19:18:32 create /sys/fs/cgroup/systemd/worker-cache-sandboxes [CGROUP POOL worker-cache-sandboxes]", "2019/07/12 19:18:32 create /sys/fs/cgroup/pids/worker-cache-sandboxes [CGROUP POOL worker-cache-sandboxes]", "2019/07/12 19:18:32 Create HandlerPuller", "2019/07/12 19:18:32 Setups Handlers", "2019/07/12 19:18:32 Execute handler by POSTing to localhost:5000/run/", "2019/07/12 19:18:32 Get status by sending request to localhost:5000/status", "2019/07/12 19:18:32 start creating/serving CGs [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:32 created [CGROUP worker-handler-sandboxes: cg-1]", "2019/07/12 19:18:32 created [CGROUP worker-handler-sandboxes: cg-2]", "2019/07/12 19:18:32 created [CGROUP worker-handler-sandboxes: cg-3]", "2019/07/12 19:18:32 created [CGROUP worker-handler-sandboxes: cg-4]", "2019/07/12 19:18:32 created [CGROUP worker-handler-sandboxes: cg-5]", "2019/07/12 19:18:32 created [CGROUP worker-handler-sandboxes: cg-6]", "2019/07/12 19:18:32 created [CGROUP worker-handler-sandboxes: cg-7]", "2019/07/12 19:18:32 created [CGROUP worker-handler-sandboxes: cg-8]", "2019/07/12 19:18:32 created [CGROUP worker-handler-sandboxes: cg-9]", "2019/07/12 19:18:32 created [CGROUP worker-handler-sandboxes: cg-10]", "2019/07/12 19:18:32 created [CGROUP worker-handler-sandboxes: cg-11]", "2019/07/12 19:18:32 created [CGROUP worker-handler-sandboxes: cg-12]", "2019/07/12 19:18:32 created [CGROUP worker-handler-sandboxes: cg-13]", "2019/07/12 19:18:32 created [CGROUP worker-handler-sandboxes: cg-14]", "2019/07/12 19:18:32 created [CGROUP worker-handler-sandboxes: cg-15]", "2019/07/12 19:18:32 created [CGROUP worker-handler-sandboxes: cg-16]", "2019/07/12 19:18:32 created [CGROUP worker-handler-sandboxes: cg-17]", "2019/07/12 19:18:32 start creating/serving CGs [CGROUP POOL worker-cache-sandboxes]", "2019/07/12 19:18:32 created [CGROUP worker-cache-sandboxes: cg-1]", "2019/07/12 19:18:32 created [CGROUP worker-cache-sandboxes: cg-2]", "2019/07/12 19:18:32 created [CGROUP worker-cache-sandboxes: cg-3]", "2019/07/12 19:18:32 created [CGROUP worker-cache-sandboxes: cg-4]", "2019/07/12 19:18:32 created [CGROUP worker-cache-sandboxes: cg-5]", "2019/07/12 19:18:32 created [CGROUP worker-cache-sandboxes: cg-6]", "2019/07/12 19:18:32 created [CGROUP worker-cache-sandboxes: cg-7]", "2019/07/12 19:18:32 created [CGROUP worker-cache-sandboxes: cg-8]", "2019/07/12 19:18:32 created [CGROUP worker-cache-sandboxes: cg-9]", "2019/07/12 19:18:32 created [CGROUP worker-cache-sandboxes: cg-10]", "2019/07/12 19:18:32 created [CGROUP worker-cache-sandboxes: cg-11]", "2019/07/12 19:18:32 created [CGROUP worker-cache-sandboxes: cg-12]", "2019/07/12 19:18:32 created [CGROUP worker-cache-sandboxes: cg-13]", "2019/07/12 19:18:32 created [CGROUP worker-cache-sandboxes: cg-14]", "2019/07/12 19:18:32 created [CGROUP worker-cache-sandboxes: cg-15]", "2019/07/12 19:18:32 created [CGROUP worker-cache-sandboxes: cg-16]", "2019/07/12 19:18:32 created [CGROUP worker-cache-sandboxes: cg-17]", "2019/07/12 19:18:32 Receive request to /pid", "2019/07/12 19:18:32 Receive request to /run/fbomb", "2019/07/12 19:18:32 debug: LambdaFunc.Task() runs on goroutine 22 [FUNC fbomb]", "2019/07/12 19:18:32 increase instances to 1 [FUNC fbomb]", "2019/07/12 19:18:32 Try using Zygote from ", "2019/07/12 19:18:32 .Create(, false, /root/open-lambda/test-dir/worker/cache-sandboxes/code-cache-sandboxes-1001, /root/open-lambda/test-dir/worker/cache-sandboxes/scratch-cache-sandboxes-1002, )=1... [SOCK POOL cache-sandboxes]", "2019/07/12 19:18:32 200 of 250 MB available [MEM POOL cache-sandboxes]", "2019/07/12 19:18:33 200 of 250 MB available [MEM POOL cache-sandboxes]", "2019/07/12 19:18:33 created [CGROUP worker-cache-sandboxes: cg-18]", "sock2.py: started new process with args: sock2.py /host/bootstrap.py 9", "sock2.py: joined cgroup, close FD 3", "sock2.py: joined cgroup, close FD 4", "sock2.py: joined cgroup, close FD 5", "sock2.py: joined cgroup, close FD 6", "sock2.py: joined cgroup, close FD 7", "sock2.py: joined cgroup, close FD 8", "sock2.py: joined cgroup, close FD 9", "sock2.py: joined cgroup, close FD 10", "sock2.py: joined cgroup, close FD 11", "sock2.py: start fork server on fd: 3", "2019/07/12 19:18:33 ...returns , [SOCK POOL cache-sandboxes]", "2019/07/12 19:18:33 .Create(, true, /root/open-lambda/test-dir/worker/lambda_code/1-fbomb, /root/open-lambda/test-dir/worker/scratch/func-fbomb-scratch-1003, )=2... [SOCK POOL handler-sandboxes]", "2019/07/12 19:18:33 200 of 250 MB available [MEM POOL handler-sandboxes]", "2019/07/12 19:18:33 fork(&{0xc0001d01c0 {0 0} false [0x81acd0 0x81c090]}) [SB 1]", "2019/07/12 19:18:33 200 of 250 MB available [MEM POOL handler-sandboxes]", "2019/07/12 19:18:33 created [CGROUP worker-handler-sandboxes: cg-18]", "sock2.py: start web server on fd: 3", "2019/07/12 19:18:33 ...returns , [SOCK POOL handler-sandboxes]", "2019/07/12 19:18:33 Channel() [SB 2]", "fork_times(20)", "fork_times(19)", "fork_times(18)", "fork_times(17)", "fork_times(16)", "fork_times(15)", "fork_times(14)", "fork_times(13)", "fork_times(12)", "fork_times(11)", "2019/07/12 19:18:33 Pause() [SB 2]", "2019/07/12 19:18:33 245 of 250 MB available [MEM POOL handler-sandboxes]", "2019/07/12 19:18:33 245 of 250 MB available [MEM POOL handler-sandboxes]", "2019/07/12 19:18:33 received kill signal, cleaning up", "2019/07/12 19:18:33 Destroy() [SB 1]", "2019/07/12 19:18:33 Destroy() [SB 2]", "2019/07/12 19:18:33 kill all procs in CG [SOCK 2]", "2019/07/12 19:18:33 release and recycle [CGROUP worker-handler-sandboxes: cg-1]", "2019/07/12 19:18:33 unmount and remove dirs [SOCK 2]", "2019/07/12 19:18:33 250 of 250 MB available [MEM POOL handler-sandboxes]", "2019/07/12 19:18:33 kill all procs in CG [SOCK 1]", "2019/07/12 19:18:33 release and recycle [CGROUP worker-cache-sandboxes: cg-1]", "2019/07/12 19:18:33 unmount and remove dirs [SOCK 1]", "2019/07/12 19:18:33 250 of 250 MB available [MEM POOL cache-sandboxes]", "2019/07/12 19:18:33 make sure all memory is free [SOCK POOL cache-sandboxes]", "2019/07/12 19:18:33 0 of 250 MB available [MEM POOL cache-sandboxes]", "2019/07/12 19:18:33 memory pool emptied [SOCK POOL cache-sandboxes]", "2019/07/12 19:18:33 received shutdown request [CGROUP POOL worker-cache-sandboxes]", "2019/07/12 19:18:33 empty queues and release CGs [CGROUP POOL worker-cache-sandboxes]", "2019/07/12 19:18:33 250 of 250 MB available [MEM POOL handler-sandboxes]", "2019/07/12 19:18:33 0 of 250 MB available [MEM POOL cache-sandboxes]", "2019/07/12 19:18:33 WARNING! Critically low on memory, so evicting an active Sandbox", "Kill function: fbomb2019/07/12 19:18:33 Destroy() [SB 2]", "2019/07/12 19:18:33 make sure all memory is free [SOCK POOL handler-sandboxes]", "2019/07/12 19:18:33 0 of 250 MB available [MEM POOL handler-sandboxes]", "2019/07/12 19:18:33 memory pool emptied [SOCK POOL handler-sandboxes]", "2019/07/12 19:18:33 received shutdown request [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:33 empty queues and release CGs [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:33 remove /sys/fs/cgroup/blkio/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:33 remove /sys/fs/cgroup/cpu/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:33 remove /sys/fs/cgroup/devices/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:33 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:33 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:33 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:33 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:33 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:33 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:18:33 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:18:33 remove worker.pid", "2019/07/12 19:18:33 exiting", "" ] } ======================================== max_mem_alloc ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "max_mem_alloc", "params": {}, "pass": false, "conf": { "worker_port": "5000", "registry": "/root/open-lambda/test-registry", "docker_runtime": "", "limits": { "installer_mem_mb": 200, "procs": 10, "mem_mb": 50 }, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "import_cache_mb": 250, "import_cache_tree": "", "pip_mirror": "", "registry_cache_ms": 5000, "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-dir/worker", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "sandbox_config": {}, "handler_cache_mb": 250 }, "seconds": null, "total_seconds": 0.11969280242919922, "stats": null, "ol-stats": null, "errors": [ [ "Traceback (most recent call last):", " File \"test.py\", line 77, in wrapper", " run(['./ol', 'worker', '-p='+OLDIR, '--detach'])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol worker -p=test-dir --detach) failed: using existing OL directory at /root/open-lambda/test-dir", "Starting worker: pid=10072, port=5000, log=/root/open-lambda/test-dir/worker.out", "2019/07/12 19:18:33 worker process 10072 does not a appear to be running, check worker.out", "", "" ], [ "Traceback (most recent call last):", " File \"test.py\", line 93, in wrapper", " run(['./ol', 'kill', '-p='+OLDIR])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol kill -p=test-dir) failed: 2019/07/12 19:18:33 open /root/open-lambda/test-dir/worker/worker.pid: no such file or directory", "", "" ] ], "worker_tail": [ "using existing OL directory at /root/open-lambda/test-dir", "2019/07/12 19:18:33 unlinkat /root/open-lambda/test-dir/worker/cache-sandboxes: device or resource busy", "" ] } PUSH conf: {'import_cache_mb': 250, 'handler_cache_mb': 250} ======================================== numpy_test ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "numpy_test", "params": {}, "pass": false, "conf": { "worker_port": "5000", "registry": "/root/open-lambda/test-registry", "docker_runtime": "", "limits": { "installer_mem_mb": 200, "procs": 10, "mem_mb": 50 }, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "import_cache_mb": 250, "import_cache_tree": "", "pip_mirror": "", "registry_cache_ms": 5000, "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-dir/worker", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "sandbox_config": {}, "handler_cache_mb": 250 }, "seconds": null, "total_seconds": 0.11654496192932129, "stats": null, "ol-stats": null, "errors": [ [ "Traceback (most recent call last):", " File \"test.py\", line 77, in wrapper", " run(['./ol', 'worker', '-p='+OLDIR, '--detach'])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol worker -p=test-dir --detach) failed: using existing OL directory at /root/open-lambda/test-dir", "Starting worker: pid=10088, port=5000, log=/root/open-lambda/test-dir/worker.out", "2019/07/12 19:18:33 worker process 10088 does not a appear to be running, check worker.out", "", "" ], [ "Traceback (most recent call last):", " File \"test.py\", line 93, in wrapper", " run(['./ol', 'kill', '-p='+OLDIR])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol kill -p=test-dir) failed: 2019/07/12 19:18:33 open /root/open-lambda/test-dir/worker/worker.pid: no such file or directory", "", "" ] ], "worker_tail": [ "using existing OL directory at /root/open-lambda/test-dir", "2019/07/12 19:18:33 unlinkat /root/open-lambda/test-dir/worker/cache-sandboxes: device or resource busy", "" ] } POP conf: {'import_cache_mb': 250, 'handler_cache_mb': 250} POP conf: {'registry': '/root/open-lambda/test-registry'} PUSH conf: {'server_mode': 'sock', 'import_cache_mb': 250, 'handler_cache_mb': 250} ======================================== sock_churn {'baseline': 0, 'fork': False, 'procs': 1, 'seconds': 5} ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "sock_churn", "params": { "baseline": 0, "fork": false, "procs": 1, "seconds": 5 }, "pass": false, "conf": { "worker_port": "5000", "registry": "/root/open-lambda/test-dir/registry", "docker_runtime": "", "limits": { "installer_mem_mb": 200, "procs": 10, "mem_mb": 50 }, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "import_cache_mb": 250, "import_cache_tree": "", "pip_mirror": "", "registry_cache_ms": 5000, "server_mode": "sock", "worker_dir": "/root/open-lambda/test-dir/worker", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "sandbox_config": {}, "handler_cache_mb": 250 }, "seconds": null, "total_seconds": 0.11634397506713867, "stats": null, "ol-stats": null, "errors": [ [ "Traceback (most recent call last):", " File \"test.py\", line 77, in wrapper", " run(['./ol', 'worker', '-p='+OLDIR, '--detach'])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol worker -p=test-dir --detach) failed: using existing OL directory at /root/open-lambda/test-dir", "Starting worker: pid=10104, port=5000, log=/root/open-lambda/test-dir/worker.out", "2019/07/12 19:18:33 worker process 10104 does not a appear to be running, check worker.out", "", "" ], [ "Traceback (most recent call last):", " File \"test.py\", line 93, in wrapper", " run(['./ol', 'kill', '-p='+OLDIR])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol kill -p=test-dir) failed: 2019/07/12 19:18:33 open /root/open-lambda/test-dir/worker/worker.pid: no such file or directory", "", "" ] ], "worker_tail": [ "using existing OL directory at /root/open-lambda/test-dir", "2019/07/12 19:18:33 unlinkat /root/open-lambda/test-dir/worker/cache-sandboxes: device or resource busy", "" ] } ======================================== sock_churn {'baseline': 0, 'fork': True, 'procs': 1, 'seconds': 15} ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "sock_churn", "params": { "baseline": 0, "fork": true, "procs": 1, "seconds": 15 }, "pass": false, "conf": { "worker_port": "5000", "registry": "/root/open-lambda/test-dir/registry", "docker_runtime": "", "limits": { "installer_mem_mb": 200, "procs": 10, "mem_mb": 50 }, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "import_cache_mb": 250, "import_cache_tree": "", "pip_mirror": "", "registry_cache_ms": 5000, "server_mode": "sock", "worker_dir": "/root/open-lambda/test-dir/worker", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "sandbox_config": {}, "handler_cache_mb": 250 }, "seconds": null, "total_seconds": 0.11674332618713379, "stats": null, "ol-stats": null, "errors": [ [ "Traceback (most recent call last):", " File \"test.py\", line 77, in wrapper", " run(['./ol', 'worker', '-p='+OLDIR, '--detach'])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol worker -p=test-dir --detach) failed: using existing OL directory at /root/open-lambda/test-dir", "Starting worker: pid=10120, port=5000, log=/root/open-lambda/test-dir/worker.out", "2019/07/12 19:18:33 worker process 10120 does not a appear to be running, check worker.out", "", "" ], [ "Traceback (most recent call last):", " File \"test.py\", line 93, in wrapper", " run(['./ol', 'kill', '-p='+OLDIR])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol kill -p=test-dir) failed: 2019/07/12 19:18:33 open /root/open-lambda/test-dir/worker/worker.pid: no such file or directory", "", "" ] ], "worker_tail": [ "using existing OL directory at /root/open-lambda/test-dir", "2019/07/12 19:18:33 unlinkat /root/open-lambda/test-dir/worker/cache-sandboxes: device or resource busy", "" ] } ======================================== sock_churn {'baseline': 0, 'fork': True, 'procs': 15, 'seconds': 15} ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "sock_churn", "params": { "baseline": 0, "fork": true, "procs": 15, "seconds": 15 }, "pass": false, "conf": { "worker_port": "5000", "registry": "/root/open-lambda/test-dir/registry", "docker_runtime": "", "limits": { "installer_mem_mb": 200, "procs": 10, "mem_mb": 50 }, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "import_cache_mb": 250, "import_cache_tree": "", "pip_mirror": "", "registry_cache_ms": 5000, "server_mode": "sock", "worker_dir": "/root/open-lambda/test-dir/worker", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "sandbox_config": {}, "handler_cache_mb": 250 }, "seconds": null, "total_seconds": 0.11681842803955078, "stats": null, "ol-stats": null, "errors": [ [ "Traceback (most recent call last):", " File \"test.py\", line 77, in wrapper", " run(['./ol', 'worker', '-p='+OLDIR, '--detach'])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol worker -p=test-dir --detach) failed: using existing OL directory at /root/open-lambda/test-dir", "Starting worker: pid=10136, port=5000, log=/root/open-lambda/test-dir/worker.out", "2019/07/12 19:18:34 worker process 10136 does not a appear to be running, check worker.out", "", "" ], [ "Traceback (most recent call last):", " File \"test.py\", line 93, in wrapper", " run(['./ol', 'kill', '-p='+OLDIR])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol kill -p=test-dir) failed: 2019/07/12 19:18:34 open /root/open-lambda/test-dir/worker/worker.pid: no such file or directory", "", "" ] ], "worker_tail": [ "using existing OL directory at /root/open-lambda/test-dir", "2019/07/12 19:18:33 unlinkat /root/open-lambda/test-dir/worker/cache-sandboxes: device or resource busy", "" ] } ======================================== recursive_kill {'depth': 2} ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "recursive_kill", "params": { "depth": 2 }, "pass": false, "conf": { "worker_port": "5000", "registry": "/root/open-lambda/test-dir/registry", "docker_runtime": "", "limits": { "installer_mem_mb": 200, "procs": 10, "mem_mb": 50 }, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "import_cache_mb": 250, "import_cache_tree": "", "pip_mirror": "", "registry_cache_ms": 5000, "server_mode": "sock", "worker_dir": "/root/open-lambda/test-dir/worker", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "sandbox_config": {}, "handler_cache_mb": 250 }, "seconds": null, "total_seconds": 0.11652541160583496, "stats": null, "ol-stats": null, "errors": [ [ "Traceback (most recent call last):", " File \"test.py\", line 77, in wrapper", " run(['./ol', 'worker', '-p='+OLDIR, '--detach'])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol worker -p=test-dir --detach) failed: using existing OL directory at /root/open-lambda/test-dir", "Starting worker: pid=10152, port=5000, log=/root/open-lambda/test-dir/worker.out", "2019/07/12 19:18:34 worker process 10152 does not a appear to be running, check worker.out", "", "" ], [ "Traceback (most recent call last):", " File \"test.py\", line 93, in wrapper", " run(['./ol', 'kill', '-p='+OLDIR])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol kill -p=test-dir) failed: 2019/07/12 19:18:34 open /root/open-lambda/test-dir/worker/worker.pid: no such file or directory", "", "" ] ], "worker_tail": [ "using existing OL directory at /root/open-lambda/test-dir", "2019/07/12 19:18:34 unlinkat /root/open-lambda/test-dir/worker/cache-sandboxes: device or resource busy", "" ] } ======================================== recursive_kill {'depth': 10} ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "recursive_kill", "params": { "depth": 10 }, "pass": false, "conf": { "worker_port": "5000", "registry": "/root/open-lambda/test-dir/registry", "docker_runtime": "", "limits": { "installer_mem_mb": 200, "procs": 10, "mem_mb": 50 }, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "import_cache_mb": 250, "import_cache_tree": "", "pip_mirror": "", "registry_cache_ms": 5000, "server_mode": "sock", "worker_dir": "/root/open-lambda/test-dir/worker", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "sandbox_config": {}, "handler_cache_mb": 250 }, "seconds": null, "total_seconds": 0.1166844367980957, "stats": null, "ol-stats": null, "errors": [ [ "Traceback (most recent call last):", " File \"test.py\", line 77, in wrapper", " run(['./ol', 'worker', '-p='+OLDIR, '--detach'])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol worker -p=test-dir --detach) failed: using existing OL directory at /root/open-lambda/test-dir", "Starting worker: pid=10168, port=5000, log=/root/open-lambda/test-dir/worker.out", "2019/07/12 19:18:34 worker process 10168 does not a appear to be running, check worker.out", "", "" ], [ "Traceback (most recent call last):", " File \"test.py\", line 93, in wrapper", " run(['./ol', 'kill', '-p='+OLDIR])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol kill -p=test-dir) failed: 2019/07/12 19:18:34 open /root/open-lambda/test-dir/worker/worker.pid: no such file or directory", "", "" ] ], "worker_tail": [ "using existing OL directory at /root/open-lambda/test-dir", "2019/07/12 19:18:34 unlinkat /root/open-lambda/test-dir/worker/cache-sandboxes: device or resource busy", "" ] } POP conf: {'server_mode': 'sock', 'import_cache_mb': 250, 'handler_cache_mb': 250} PUSH conf: {'registry': '/tmp/tmpk1ld3_r5', 'registry_cache_ms': 3000, 'sandbox': 'sock'} ======================================== update_code ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "update_code", "params": {}, "pass": false, "conf": { "worker_port": "5000", "registry": "/tmp/tmpk1ld3_r5", "docker_runtime": "", "limits": { "installer_mem_mb": 200, "procs": 10, "mem_mb": 50 }, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "import_cache_mb": 250, "import_cache_tree": "", "pip_mirror": "", "registry_cache_ms": 3000, "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-dir/worker", "handler_cache_mb": 250, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "sandbox_config": {}, "sandbox": "sock" }, "seconds": null, "total_seconds": 0.11638832092285156, "stats": null, "ol-stats": null, "errors": [ [ "Traceback (most recent call last):", " File \"test.py\", line 77, in wrapper", " run(['./ol', 'worker', '-p='+OLDIR, '--detach'])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol worker -p=test-dir --detach) failed: using existing OL directory at /root/open-lambda/test-dir", "Starting worker: pid=10184, port=5000, log=/root/open-lambda/test-dir/worker.out", "2019/07/12 19:18:34 worker process 10184 does not a appear to be running, check worker.out", "", "" ], [ "Traceback (most recent call last):", " File \"test.py\", line 93, in wrapper", " run(['./ol', 'kill', '-p='+OLDIR])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol kill -p=test-dir) failed: 2019/07/12 19:18:34 open /root/open-lambda/test-dir/worker/worker.pid: no such file or directory", "", "" ] ], "worker_tail": [ "using existing OL directory at /root/open-lambda/test-dir", "2019/07/12 19:18:34 unlinkat /root/open-lambda/test-dir/worker/cache-sandboxes: device or resource busy", "" ] } POP conf: {'registry': '/tmp/tmpk1ld3_r5', 'registry_cache_ms': 3000, 'sandbox': 'sock'} PUSH conf: {'registry': '/root/open-lambda/test-registry', 'sandbox': 'sock', 'import_cache_mb': 250, 'handler_cache_mb': 250} ======================================== stress_one_lambda {'seconds': 15, 'procs': 1} ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "stress_one_lambda", "params": { "seconds": 15, "procs": 1 }, "pass": false, "conf": { "worker_port": "5000", "registry": "/root/open-lambda/test-registry", "docker_runtime": "", "limits": { "installer_mem_mb": 200, "procs": 10, "mem_mb": 50 }, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "import_cache_mb": 250, "import_cache_tree": "", "pip_mirror": "", "registry_cache_ms": 5000, "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-dir/worker", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "sandbox_config": {}, "handler_cache_mb": 250 }, "seconds": null, "total_seconds": 0.11663365364074707, "stats": null, "ol-stats": null, "errors": [ [ "Traceback (most recent call last):", " File \"test.py\", line 77, in wrapper", " run(['./ol', 'worker', '-p='+OLDIR, '--detach'])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol worker -p=test-dir --detach) failed: using existing OL directory at /root/open-lambda/test-dir", "Starting worker: pid=10200, port=5000, log=/root/open-lambda/test-dir/worker.out", "2019/07/12 19:18:34 worker process 10200 does not a appear to be running, check worker.out", "", "" ], [ "Traceback (most recent call last):", " File \"test.py\", line 93, in wrapper", " run(['./ol', 'kill', '-p='+OLDIR])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol kill -p=test-dir) failed: 2019/07/12 19:18:34 open /root/open-lambda/test-dir/worker/worker.pid: no such file or directory", "", "" ] ], "worker_tail": [ "using existing OL directory at /root/open-lambda/test-dir", "2019/07/12 19:18:34 unlinkat /root/open-lambda/test-dir/worker/cache-sandboxes: device or resource busy", "" ] } ======================================== stress_one_lambda {'seconds': 15, 'procs': 2} ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "stress_one_lambda", "params": { "seconds": 15, "procs": 2 }, "pass": false, "conf": { "worker_port": "5000", "registry": "/root/open-lambda/test-registry", "docker_runtime": "", "limits": { "installer_mem_mb": 200, "procs": 10, "mem_mb": 50 }, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "import_cache_mb": 250, "import_cache_tree": "", "pip_mirror": "", "registry_cache_ms": 5000, "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-dir/worker", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "sandbox_config": {}, "handler_cache_mb": 250 }, "seconds": null, "total_seconds": 0.1167764663696289, "stats": null, "ol-stats": null, "errors": [ [ "Traceback (most recent call last):", " File \"test.py\", line 77, in wrapper", " run(['./ol', 'worker', '-p='+OLDIR, '--detach'])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol worker -p=test-dir --detach) failed: using existing OL directory at /root/open-lambda/test-dir", "Starting worker: pid=10216, port=5000, log=/root/open-lambda/test-dir/worker.out", "2019/07/12 19:18:34 worker process 10216 does not a appear to be running, check worker.out", "", "" ], [ "Traceback (most recent call last):", " File \"test.py\", line 93, in wrapper", " run(['./ol', 'kill', '-p='+OLDIR])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol kill -p=test-dir) failed: 2019/07/12 19:18:34 open /root/open-lambda/test-dir/worker/worker.pid: no such file or directory", "", "" ] ], "worker_tail": [ "using existing OL directory at /root/open-lambda/test-dir", "2019/07/12 19:18:34 unlinkat /root/open-lambda/test-dir/worker/cache-sandboxes: device or resource busy", "" ] } ======================================== stress_one_lambda {'seconds': 15, 'procs': 8} ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "stress_one_lambda", "params": { "seconds": 15, "procs": 8 }, "pass": false, "conf": { "worker_port": "5000", "registry": "/root/open-lambda/test-registry", "docker_runtime": "", "limits": { "installer_mem_mb": 200, "procs": 10, "mem_mb": 50 }, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "import_cache_mb": 250, "import_cache_tree": "", "pip_mirror": "", "registry_cache_ms": 5000, "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-dir/worker", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "sandbox_config": {}, "handler_cache_mb": 250 }, "seconds": null, "total_seconds": 0.11677265167236328, "stats": null, "ol-stats": null, "errors": [ [ "Traceback (most recent call last):", " File \"test.py\", line 77, in wrapper", " run(['./ol', 'worker', '-p='+OLDIR, '--detach'])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol worker -p=test-dir --detach) failed: using existing OL directory at /root/open-lambda/test-dir", "Starting worker: pid=10232, port=5000, log=/root/open-lambda/test-dir/worker.out", "2019/07/12 19:18:34 worker process 10232 does not a appear to be running, check worker.out", "", "" ], [ "Traceback (most recent call last):", " File \"test.py\", line 93, in wrapper", " run(['./ol', 'kill', '-p='+OLDIR])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol kill -p=test-dir) failed: 2019/07/12 19:18:34 open /root/open-lambda/test-dir/worker/worker.pid: no such file or directory", "", "" ] ], "worker_tail": [ "using existing OL directory at /root/open-lambda/test-dir", "2019/07/12 19:18:34 unlinkat /root/open-lambda/test-dir/worker/cache-sandboxes: device or resource busy", "" ] } POP conf: {'registry': '/root/open-lambda/test-registry', 'sandbox': 'sock', 'import_cache_mb': 250, 'handler_cache_mb': 250} PUSH conf: {'sandbox': 'sock', 'import_cache_mb': 250, 'handler_cache_mb': 250} PUSH conf: {'registry': '/tmp/tmptqepi31a'} ======================================== call_each_once_exec {'alloc_mb': 1, 'lambda_count': 100} ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "call_each_once_exec", "params": { "alloc_mb": 1, "lambda_count": 100 }, "pass": false, "conf": { "worker_port": "5000", "registry": "/tmp/tmptqepi31a", "docker_runtime": "", "limits": { "installer_mem_mb": 200, "procs": 10, "mem_mb": 50 }, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "import_cache_mb": 250, "import_cache_tree": "", "pip_mirror": "", "registry_cache_ms": 5000, "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-dir/worker", "handler_cache_mb": 250, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "sandbox_config": {}, "sandbox": "sock" }, "seconds": null, "total_seconds": 0.11760425567626953, "stats": null, "ol-stats": null, "errors": [ [ "Traceback (most recent call last):", " File \"test.py\", line 77, in wrapper", " run(['./ol', 'worker', '-p='+OLDIR, '--detach'])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol worker -p=test-dir --detach) failed: using existing OL directory at /root/open-lambda/test-dir", "Starting worker: pid=10248, port=5000, log=/root/open-lambda/test-dir/worker.out", "2019/07/12 19:18:34 worker process 10248 does not a appear to be running, check worker.out", "", "" ], [ "Traceback (most recent call last):", " File \"test.py\", line 93, in wrapper", " run(['./ol', 'kill', '-p='+OLDIR])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol kill -p=test-dir) failed: 2019/07/12 19:18:34 open /root/open-lambda/test-dir/worker/worker.pid: no such file or directory", "", "" ] ], "worker_tail": [ "using existing OL directory at /root/open-lambda/test-dir", "2019/07/12 19:18:34 unlinkat /root/open-lambda/test-dir/worker/cache-sandboxes: device or resource busy", "" ] } POP conf: {'registry': '/tmp/tmptqepi31a'} PUSH conf: {'registry': '/tmp/tmpyocv1w3m'} ======================================== call_each_once_exec {'alloc_mb': 10, 'lambda_count': 1000} ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "call_each_once_exec", "params": { "alloc_mb": 10, "lambda_count": 1000 }, "pass": false, "conf": { "worker_port": "5000", "registry": "/tmp/tmpyocv1w3m", "docker_runtime": "", "limits": { "installer_mem_mb": 200, "procs": 10, "mem_mb": 50 }, "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "import_cache_mb": 250, "import_cache_tree": "", "pip_mirror": "", "registry_cache_ms": 5000, "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-dir/worker", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "sandbox_config": {}, "handler_cache_mb": 250 }, "seconds": null, "total_seconds": 0.11864256858825684, "stats": null, "ol-stats": null, "errors": [ [ "Traceback (most recent call last):", " File \"test.py\", line 77, in wrapper", " run(['./ol', 'worker', '-p='+OLDIR, '--detach'])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol worker -p=test-dir --detach) failed: using existing OL directory at /root/open-lambda/test-dir", "Starting worker: pid=10264, port=5000, log=/root/open-lambda/test-dir/worker.out", "2019/07/12 19:18:35 worker process 10264 does not a appear to be running, check worker.out", "", "" ], [ "Traceback (most recent call last):", " File \"test.py\", line 93, in wrapper", " run(['./ol', 'kill', '-p='+OLDIR])", " File \"test.py\", line 179, in run", " raise Exception(\"command (%s) failed: %s\" % (\" \".join(cmd), out))", "Exception: command (./ol kill -p=test-dir) failed: 2019/07/12 19:18:35 open /root/open-lambda/test-dir/worker/worker.pid: no such file or directory", "", "" ] ], "worker_tail": [ "using existing OL directory at /root/open-lambda/test-dir", "2019/07/12 19:18:34 unlinkat /root/open-lambda/test-dir/worker/cache-sandboxes: device or resource busy", "" ] } POP conf: {'registry': '/tmp/tmpyocv1w3m'} POP conf: {'sandbox': 'sock', 'import_cache_mb': 250, 'handler_cache_mb': 250} PASSED: 4, FAILED: 14 Makefile:25: recipe for target 'test-all' failed make: *** [test-all] Error 14