python3 -u test.py RUN ./ol new -p=test-dir PUSH conf: {'registry': '/root/open-lambda/test-registry'} ======================================== ping_test ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "ping_test", "params": {}, "pass": true, "conf": { "worker_dir": "/root/open-lambda/test-dir/worker", "import_cache_tree": "", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "server_mode": "lambda", "handler_cache_mb": 250, "sandbox_config": {}, "docker_runtime": "", "worker_port": "5000", "limits": { "mem_mb": 50, "procs": 10, "installer_mem_mb": 200 }, "import_cache_mb": 250, "sandbox": "sock", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-registry", "pip_mirror": "" }, "seconds": 2.208021879196167, "total_seconds": 2.4334559440612793, "stats": { "pings_per_sec": 452.89846426286505 }, "ol-stats": {}, "errors": [], "worker_tail": [ "2019/07/12 19:51:38 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:51:38 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:51:38 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:51:38 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:51:38 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:51:38 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:51:38 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:51:38 remove worker.pid", "2019/07/12 19:51:38 exiting", "" ] } PUSH conf: {'import_cache_mb': 0, 'handler_cache_mb': 500} ======================================== install_tests ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "install_tests", "params": {}, "pass": true, "conf": { "worker_dir": "/root/open-lambda/test-dir/worker", "import_cache_tree": "", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "server_mode": "lambda", "handler_cache_mb": 500, "sandbox_config": {}, "docker_runtime": "", "worker_port": "5000", "limits": { "mem_mb": 50, "procs": 10, "installer_mem_mb": 200 }, "import_cache_mb": 0, "sandbox": "sock", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-registry", "pip_mirror": "" }, "seconds": 10.213173866271973, "total_seconds": 10.45229196548462, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 10, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 10, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 10, "Create()/fresh-proc:ms.avg": 258, "Create()/fresh-proc:ms.cnt": 10, "Create()/make-root-fs:ms.avg": 1, "Create()/make-root-fs:ms.cnt": 10, "Create():ms.avg": 260, "Create():ms.cnt": 10, "Destroy()/detach-root:ms.avg": 0, "Destroy()/detach-root:ms.cnt": 10, "Destroy()/kill-procs:ms.avg": 1, "Destroy()/kill-procs:ms.cnt": 10, "Destroy()/recursive-kill:ms.avg": 0, "Destroy()/recursive-kill:ms.cnt": 10, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 10, "Destroy():ms.avg": 2, "Destroy():ms.cnt": 10, "Pause():ms.avg": 4, "Pause():ms.cnt": 4, "Status():ms.avg": 0, "Status():ms.cnt": 6, "pip-install:ms.avg": 1356, "pip-install:ms.cnt": 6 }, "errors": [], "worker_tail": [ "2019/07/12 19:51:49 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:51:49 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:51:49 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:51:49 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:51:49 0 of 500 MB available [MEM POOL handler-sandboxes]", "2019/07/12 19:51:49 WARNING! Critically low on memory, so evicting an active Sandbox", "2019/07/12 19:51:49 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:51:49 remove worker.pid", "2019/07/12 19:51:49 exiting", "" ] } POP conf: {'import_cache_mb': 0, 'handler_cache_mb': 500} PUSH conf: {'import_cache_mb': 250, 'handler_cache_mb': 250} ======================================== install_tests ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "install_tests", "params": {}, "pass": true, "conf": { "worker_dir": "/root/open-lambda/test-dir/worker", "import_cache_tree": "", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "server_mode": "lambda", "handler_cache_mb": 250, "sandbox_config": {}, "docker_runtime": "", "worker_port": "5000", "limits": { "mem_mb": 50, "procs": 10, "installer_mem_mb": 200 }, "import_cache_mb": 250, "sandbox": "sock", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-registry", "pip_mirror": "" }, "seconds": 6.305354118347168, "total_seconds": 6.5371174812316895, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 10, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 11, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 11, "Create()/fork-proc:ms.avg": 6, "Create()/fork-proc:ms.cnt": 4, "Create()/fresh-proc:ms.avg": 139, "Create()/fresh-proc:ms.cnt": 7, "Create()/make-root-fs:ms.avg": 3, "Create()/make-root-fs:ms.cnt": 11, "Create():ms.avg": 95, "Create():ms.cnt": 11, "Destroy()/detach-root:ms.avg": 0, "Destroy()/detach-root:ms.cnt": 11, "Destroy()/kill-procs:ms.avg": 1, "Destroy()/kill-procs:ms.cnt": 11, "Destroy()/recursive-kill:ms.avg": 0, "Destroy()/recursive-kill:ms.cnt": 11, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 11, "Destroy():ms.avg": 1, "Destroy():ms.cnt": 18, "Pause():ms.avg": 5, "Pause():ms.cnt": 4, "Status():ms.avg": 0, "Status():ms.cnt": 6, "evict:ms.avg": 1, "evict:ms.cnt": 3, "fork():ms.avg": 6, "fork():ms.cnt": 4, "forkRequest:ms.avg": 6, "forkRequest:ms.cnt": 4, "move-to-cg-after-fork:ms.avg": 0, "move-to-cg-after-fork:ms.cnt": 4, "pip-install:ms.avg": 970, "pip-install:ms.cnt": 6 }, "errors": [], "worker_tail": [ "2019/07/12 19:51:55 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:51:55 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:51:55 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:51:55 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:51:55 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:51:55 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:51:55 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:51:55 remove worker.pid", "2019/07/12 19:51:55 exiting", "" ] } POP conf: {'import_cache_mb': 250, 'handler_cache_mb': 250} PUSH conf: {'sandbox': 'docker', 'import_cache_mb': 0, 'handler_cache_mb': 500} ======================================== install_tests ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "install_tests", "params": {}, "pass": true, "conf": { "worker_dir": "/root/open-lambda/test-dir/worker", "import_cache_tree": "", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "server_mode": "lambda", "handler_cache_mb": 500, "sandbox_config": {}, "docker_runtime": "", "worker_port": "5000", "limits": { "mem_mb": 50, "procs": 10, "installer_mem_mb": 200 }, "import_cache_mb": 0, "sandbox": "docker", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-registry", "pip_mirror": "" }, "seconds": 17.80065655708313, "total_seconds": 18.852253437042236, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 10, "Create():ms.avg": 834, "Create():ms.cnt": 10, "Destroy():ms.avg": 243, "Destroy():ms.cnt": 10, "Pause():ms.avg": 29, "Pause():ms.cnt": 4, "Status():ms.avg": 0, "Status():ms.cnt": 6, "pip-install:ms.avg": 2091, "pip-install:ms.cnt": 6 }, "errors": [], "worker_tail": [ "2019/07/12 19:52:14 Kill function: echo", "2019/07/12 19:52:14 Destroy() [SB 0]", "2019/07/12 19:52:14 Kill function: install", "2019/07/12 19:52:14 Destroy() [SB 6]", "2019/07/12 19:52:14 Kill function: install2", "2019/07/12 19:52:14 Destroy() [SB 7]", "2019/07/12 19:52:14 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:52:14 remove worker.pid", "2019/07/12 19:52:14 exiting", "" ] } POP conf: {'sandbox': 'docker', 'import_cache_mb': 0, 'handler_cache_mb': 500} ======================================== fork_bomb ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "fork_bomb", "params": {}, "pass": true, "conf": { "worker_dir": "/root/open-lambda/test-dir/worker", "import_cache_tree": "", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "server_mode": "lambda", "handler_cache_mb": 250, "sandbox_config": {}, "docker_runtime": "", "worker_port": "5000", "limits": { "mem_mb": 50, "procs": 10, "installer_mem_mb": 200 }, "import_cache_mb": 250, "sandbox": "sock", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-registry", "pip_mirror": "" }, "seconds": 0.17129206657409668, "total_seconds": 0.4020075798034668, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 1, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 2, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 2, "Create()/fork-proc:ms.avg": 7, "Create()/fork-proc:ms.cnt": 1, "Create()/fresh-proc:ms.avg": 137, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 1, "Create()/make-root-fs:ms.cnt": 2, "Create():ms.avg": 74, "Create():ms.cnt": 2, "Destroy()/detach-root:ms.avg": 1, "Destroy()/detach-root:ms.cnt": 2, "Destroy()/kill-procs:ms.avg": 0, "Destroy()/kill-procs:ms.cnt": 2, "Destroy()/recursive-kill:ms.avg": 1, "Destroy()/recursive-kill:ms.cnt": 2, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 2, "Destroy():ms.avg": 2, "Destroy():ms.cnt": 3, "Pause():ms.avg": 4, "Pause():ms.cnt": 1, "fork():ms.avg": 7, "fork():ms.cnt": 1, "forkRequest:ms.avg": 4, "forkRequest:ms.cnt": 1, "move-to-cg-after-fork:ms.avg": 2, "move-to-cg-after-fork:ms.cnt": 1 }, "errors": [], "worker_tail": [ "2019/07/12 19:52:15 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:52:15 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:52:15 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:52:15 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:52:15 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:52:15 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:52:15 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:52:15 remove worker.pid", "2019/07/12 19:52:15 exiting", "" ] } ======================================== max_mem_alloc ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "max_mem_alloc", "params": {}, "pass": true, "conf": { "worker_dir": "/root/open-lambda/test-dir/worker", "import_cache_tree": "", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "server_mode": "lambda", "handler_cache_mb": 250, "sandbox_config": {}, "docker_runtime": "", "worker_port": "5000", "limits": { "mem_mb": 50, "procs": 10, "installer_mem_mb": 200 }, "import_cache_mb": 250, "sandbox": "sock", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-registry", "pip_mirror": "" }, "seconds": 0.8219904899597168, "total_seconds": 1.0539875030517578, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 1, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 2, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 2, "Create()/fork-proc:ms.avg": 8, "Create()/fork-proc:ms.cnt": 1, "Create()/fresh-proc:ms.avg": 135, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 4, "Create()/make-root-fs:ms.cnt": 2, "Create():ms.avg": 76, "Create():ms.cnt": 2, "Destroy()/detach-root:ms.avg": 1, "Destroy()/detach-root:ms.cnt": 2, "Destroy()/kill-procs:ms.avg": 0, "Destroy()/kill-procs:ms.cnt": 2, "Destroy()/recursive-kill:ms.avg": 1, "Destroy()/recursive-kill:ms.cnt": 2, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 2, "Destroy():ms.avg": 2, "Destroy():ms.cnt": 3, "Pause():ms.avg": 4, "Pause():ms.cnt": 1, "fork():ms.avg": 8, "fork():ms.cnt": 1, "forkRequest:ms.avg": 5, "forkRequest:ms.cnt": 1, "move-to-cg-after-fork:ms.avg": 3, "move-to-cg-after-fork:ms.cnt": 1 }, "errors": [], "worker_tail": [ "2019/07/12 19:52:16 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:52:16 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:52:16 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:52:16 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:52:16 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:52:16 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:52:16 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:52:16 remove worker.pid", "2019/07/12 19:52:16 exiting", "" ] } PUSH conf: {'import_cache_mb': 250, 'handler_cache_mb': 250} ======================================== numpy_test ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "numpy_test", "params": {}, "pass": true, "conf": { "worker_dir": "/root/open-lambda/test-dir/worker", "import_cache_tree": "", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "server_mode": "lambda", "handler_cache_mb": 250, "sandbox_config": {}, "docker_runtime": "", "worker_port": "5000", "limits": { "mem_mb": 50, "procs": 10, "installer_mem_mb": 200 }, "import_cache_mb": 250, "sandbox": "sock", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-registry", "pip_mirror": "" }, "seconds": 25.659239053726196, "total_seconds": 25.89458656311035, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 12, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 13, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 13, "Create()/fork-proc:ms.avg": 5, "Create()/fork-proc:ms.cnt": 5, "Create()/fresh-proc:ms.avg": 141, "Create()/fresh-proc:ms.cnt": 8, "Create()/make-root-fs:ms.avg": 54, "Create()/make-root-fs:ms.cnt": 13, "Create():ms.avg": 144, "Create():ms.cnt": 13, "Destroy()/detach-root:ms.avg": 0, "Destroy()/detach-root:ms.cnt": 13, "Destroy()/kill-procs:ms.avg": 2, "Destroy()/kill-procs:ms.cnt": 13, "Destroy()/recursive-kill:ms.avg": 0, "Destroy()/recursive-kill:ms.cnt": 13, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 13, "Destroy():ms.avg": 1, "Destroy():ms.cnt": 22, "Pause():ms.avg": 3, "Pause():ms.cnt": 5, "Status():ms.avg": 0, "Status():ms.cnt": 7, "evict:ms.avg": 2, "evict:ms.cnt": 4, "fork():ms.avg": 5, "fork():ms.cnt": 5, "forkRequest:ms.avg": 5, "forkRequest:ms.cnt": 5, "move-to-cg-after-fork:ms.avg": 0, "move-to-cg-after-fork:ms.cnt": 5, "pip-install:ms.avg": 3360, "pip-install:ms.cnt": 7 }, "errors": [], "worker_tail": [ "2019/07/12 19:52:42 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:52:42 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:52:42 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:52:42 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:52:42 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:52:42 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:52:42 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:52:42 remove worker.pid", "2019/07/12 19:52:42 exiting", "" ] } POP conf: {'import_cache_mb': 250, 'handler_cache_mb': 250} POP conf: {'registry': '/root/open-lambda/test-registry'} PUSH conf: {'import_cache_mb': 250, 'handler_cache_mb': 250, 'server_mode': 'sock'} ======================================== sock_churn {'fork': False, 'baseline': 0, 'procs': 1, 'seconds': 5} ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "sock_churn", "params": { "fork": false, "baseline": 0, "procs": 1, "seconds": 5 }, "pass": true, "conf": { "worker_dir": "/root/open-lambda/test-dir/worker", "import_cache_tree": "", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "server_mode": "sock", "handler_cache_mb": 250, "sandbox_config": {}, "docker_runtime": "", "worker_port": "5000", "limits": { "mem_mb": 50, "procs": 10, "installer_mem_mb": 200 }, "import_cache_mb": 250, "sandbox": "sock", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-dir/registry", "pip_mirror": "" }, "seconds": 5.20707368850708, "total_seconds": 5.428040504455566, "stats": { "sandboxes_per_sec": 6.6 }, "ol-stats": { "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 33, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 33, "Create()/fresh-proc:ms.avg": 135, "Create()/fresh-proc:ms.cnt": 33, "Create()/make-root-fs:ms.avg": 3, "Create()/make-root-fs:ms.cnt": 33, "Create():ms.avg": 139, "Create():ms.cnt": 33, "Destroy()/detach-root:ms.avg": 1, "Destroy()/detach-root:ms.cnt": 33, "Destroy()/kill-procs:ms.avg": 0, "Destroy()/kill-procs:ms.cnt": 33, "Destroy()/recursive-kill:ms.avg": 0, "Destroy()/recursive-kill:ms.cnt": 33, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 33, "Destroy():ms.avg": 1, "Destroy():ms.cnt": 66 }, "errors": [], "worker_tail": [ "2019/07/12 19:52:47 remove /sys/fs/cgroup/freezer/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:52:47 remove /sys/fs/cgroup/hugetlb/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:52:47 remove /sys/fs/cgroup/memory/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:52:47 remove /sys/fs/cgroup/perf_event/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:52:47 remove /sys/fs/cgroup/systemd/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:52:47 remove /sys/fs/cgroup/pids/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:52:47 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:52:47 remove worker.pid", "2019/07/12 19:52:47 exiting", "" ] } ======================================== sock_churn {'fork': True, 'baseline': 0, 'procs': 1, 'seconds': 15} ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "sock_churn", "params": { "fork": true, "baseline": 0, "procs": 1, "seconds": 15 }, "pass": true, "conf": { "worker_dir": "/root/open-lambda/test-dir/worker", "import_cache_tree": "", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "server_mode": "sock", "handler_cache_mb": 250, "sandbox_config": {}, "docker_runtime": "", "worker_port": "5000", "limits": { "mem_mb": 50, "procs": 10, "installer_mem_mb": 200 }, "import_cache_mb": 250, "sandbox": "sock", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-dir/registry", "pip_mirror": "" }, "seconds": 15.193011283874512, "total_seconds": 15.413290977478027, "stats": { "sandboxes_per_sec": 49.2 }, "ol-stats": { "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 739, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 739, "Create()/fork-proc:ms.avg": 6, "Create()/fork-proc:ms.cnt": 738, "Create()/fresh-proc:ms.avg": 132, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 2, "Create()/make-root-fs:ms.cnt": 739, "Create():ms.avg": 9, "Create():ms.cnt": 739, "Destroy()/detach-root:ms.avg": 1, "Destroy()/detach-root:ms.cnt": 739, "Destroy()/kill-procs:ms.avg": 1, "Destroy()/kill-procs:ms.cnt": 739, "Destroy()/recursive-kill:ms.avg": 0, "Destroy()/recursive-kill:ms.cnt": 739, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 739, "Destroy():ms.avg": 0, "Destroy():ms.cnt": 2215, "fork():ms.avg": 6, "fork():ms.cnt": 738, "forkRequest:ms.avg": 5, "forkRequest:ms.cnt": 738, "move-to-cg-after-fork:ms.avg": 0, "move-to-cg-after-fork:ms.cnt": 738 }, "errors": [], "worker_tail": [ "2019/07/12 19:53:02 remove /sys/fs/cgroup/freezer/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:02 remove /sys/fs/cgroup/hugetlb/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:02 remove /sys/fs/cgroup/memory/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:02 remove /sys/fs/cgroup/perf_event/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:02 remove /sys/fs/cgroup/systemd/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:02 remove /sys/fs/cgroup/pids/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:02 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:53:02 remove worker.pid", "2019/07/12 19:53:02 exiting", "" ] } ======================================== sock_churn {'fork': True, 'baseline': 0, 'procs': 15, 'seconds': 15} ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "sock_churn", "params": { "fork": true, "baseline": 0, "procs": 15, "seconds": 15 }, "pass": true, "conf": { "worker_dir": "/root/open-lambda/test-dir/worker", "import_cache_tree": "", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "server_mode": "sock", "handler_cache_mb": 250, "sandbox_config": {}, "docker_runtime": "", "worker_port": "5000", "limits": { "mem_mb": 50, "procs": 10, "installer_mem_mb": 200 }, "import_cache_mb": 250, "sandbox": "sock", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-dir/registry", "pip_mirror": "" }, "seconds": 15.32593822479248, "total_seconds": 15.5534086227417, "stats": { "sandboxes_per_sec": 48.333333333333336 }, "ol-stats": { "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 726, "Create()/acquire-mem:ms.avg": 73, "Create()/acquire-mem:ms.cnt": 726, "Create()/fork-proc:ms.avg": 38, "Create()/fork-proc:ms.cnt": 725, "Create()/fresh-proc:ms.avg": 133, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 1, "Create()/make-root-fs:ms.cnt": 726, "Create():ms.avg": 114, "Create():ms.cnt": 726, "Destroy()/detach-root:ms.avg": 2, "Destroy()/detach-root:ms.cnt": 726, "Destroy()/kill-procs:ms.avg": 6, "Destroy()/kill-procs:ms.cnt": 726, "Destroy()/recursive-kill:ms.avg": 0, "Destroy()/recursive-kill:ms.cnt": 726, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 726, "Destroy():ms.avg": 7, "Destroy():ms.cnt": 2176, "fork():ms.avg": 38, "fork():ms.cnt": 725, "forkRequest:ms.avg": 8, "forkRequest:ms.cnt": 725, "move-to-cg-after-fork:ms.avg": 3, "move-to-cg-after-fork:ms.cnt": 725 }, "errors": [], "worker_tail": [ "2019/07/12 19:53:18 remove /sys/fs/cgroup/freezer/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:18 remove /sys/fs/cgroup/hugetlb/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:18 remove /sys/fs/cgroup/memory/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:18 remove /sys/fs/cgroup/perf_event/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:18 remove /sys/fs/cgroup/systemd/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:18 remove /sys/fs/cgroup/pids/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:18 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:53:18 remove worker.pid", "2019/07/12 19:53:18 exiting", "" ] } ======================================== recursive_kill {'depth': 2} ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "recursive_kill", "params": { "depth": 2 }, "pass": true, "conf": { "worker_dir": "/root/open-lambda/test-dir/worker", "import_cache_tree": "", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "server_mode": "sock", "handler_cache_mb": 250, "sandbox_config": {}, "docker_runtime": "", "worker_port": "5000", "limits": { "mem_mb": 50, "procs": 10, "installer_mem_mb": 200 }, "import_cache_mb": 250, "sandbox": "sock", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-dir/registry", "pip_mirror": "" }, "seconds": 0.17563104629516602, "total_seconds": 0.39800548553466797, "stats": null, "ol-stats": { "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 2, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 2, "Create()/fork-proc:ms.avg": 4, "Create()/fork-proc:ms.cnt": 1, "Create()/fresh-proc:ms.avg": 133, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 4, "Create()/make-root-fs:ms.cnt": 2, "Create():ms.avg": 74, "Create():ms.cnt": 2, "Destroy()/detach-root:ms.avg": 1, "Destroy()/detach-root:ms.cnt": 2, "Destroy()/kill-procs:ms.avg": 0, "Destroy()/kill-procs:ms.cnt": 2, "Destroy()/recursive-kill:ms.avg": 1, "Destroy()/recursive-kill:ms.cnt": 2, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 2, "Destroy():ms.avg": 1, "Destroy():ms.cnt": 4, "Pause():ms.avg": 1, "Pause():ms.cnt": 1, "fork():ms.avg": 4, "fork():ms.cnt": 1, "forkRequest:ms.avg": 4, "forkRequest:ms.cnt": 1, "move-to-cg-after-fork:ms.avg": 0, "move-to-cg-after-fork:ms.cnt": 1 }, "errors": [], "worker_tail": [ "2019/07/12 19:53:18 remove /sys/fs/cgroup/freezer/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:18 remove /sys/fs/cgroup/hugetlb/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:18 remove /sys/fs/cgroup/memory/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:18 remove /sys/fs/cgroup/perf_event/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:18 remove /sys/fs/cgroup/systemd/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:18 remove /sys/fs/cgroup/pids/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:18 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:53:18 remove worker.pid", "2019/07/12 19:53:18 exiting", "" ] } ======================================== recursive_kill {'depth': 10} ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "recursive_kill", "params": { "depth": 10 }, "pass": true, "conf": { "worker_dir": "/root/open-lambda/test-dir/worker", "import_cache_tree": "", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "server_mode": "sock", "handler_cache_mb": 250, "sandbox_config": {}, "docker_runtime": "", "worker_port": "5000", "limits": { "mem_mb": 50, "procs": 10, "installer_mem_mb": 200 }, "import_cache_mb": 250, "sandbox": "sock", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-dir/registry", "pip_mirror": "" }, "seconds": 0.324753999710083, "total_seconds": 0.5542593002319336, "stats": null, "ol-stats": { "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 10, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 10, "Create()/fork-proc:ms.avg": 4, "Create()/fork-proc:ms.cnt": 9, "Create()/fresh-proc:ms.avg": 135, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 3, "Create()/make-root-fs:ms.cnt": 10, "Create():ms.avg": 21, "Create():ms.cnt": 10, "Destroy()/detach-root:ms.avg": 0, "Destroy()/detach-root:ms.cnt": 10, "Destroy()/kill-procs:ms.avg": 0, "Destroy()/kill-procs:ms.cnt": 10, "Destroy()/recursive-kill:ms.avg": 5, "Destroy()/recursive-kill:ms.cnt": 10, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 10, "Destroy():ms.avg": 3, "Destroy():ms.cnt": 20, "Pause():ms.avg": 0, "Pause():ms.cnt": 9, "fork():ms.avg": 4, "fork():ms.cnt": 9, "forkRequest:ms.avg": 3, "forkRequest:ms.cnt": 9, "move-to-cg-after-fork:ms.avg": 0, "move-to-cg-after-fork:ms.cnt": 9 }, "errors": [], "worker_tail": [ "2019/07/12 19:53:19 remove /sys/fs/cgroup/freezer/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:19 remove /sys/fs/cgroup/hugetlb/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:19 remove /sys/fs/cgroup/memory/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:19 remove /sys/fs/cgroup/perf_event/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:19 remove /sys/fs/cgroup/systemd/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:19 remove /sys/fs/cgroup/pids/worker-sock-handlers [CGROUP POOL worker-sock-handlers]", "2019/07/12 19:53:19 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:53:19 remove worker.pid", "2019/07/12 19:53:19 exiting", "" ] } POP conf: {'import_cache_mb': 250, 'handler_cache_mb': 250, 'server_mode': 'sock'} PUSH conf: {'sandbox': 'sock', 'registry_cache_ms': 3000, 'registry': '/tmp/tmpodepsov4'} ======================================== update_code ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "update_code", "params": {}, "pass": true, "conf": { "worker_dir": "/root/open-lambda/test-dir/worker", "import_cache_tree": "", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "server_mode": "lambda", "handler_cache_mb": 250, "sandbox_config": {}, "docker_runtime": "", "worker_port": "5000", "limits": { "mem_mb": 50, "procs": 10, "installer_mem_mb": 200 }, "import_cache_mb": 250, "sandbox": "sock", "registry_cache_ms": 3000, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/tmp/tmpodepsov4", "pip_mirror": "" }, "seconds": 6.029616832733154, "total_seconds": 6.263854265213013, "stats": null, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 3, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 4, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 4, "Create()/fork-proc:ms.avg": 7, "Create()/fork-proc:ms.cnt": 3, "Create()/fresh-proc:ms.avg": 136, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 3, "Create()/make-root-fs:ms.cnt": 4, "Create():ms.avg": 44, "Create():ms.cnt": 4, "Destroy()/detach-root:ms.avg": 0, "Destroy()/detach-root:ms.cnt": 4, "Destroy()/kill-procs:ms.avg": 0, "Destroy()/kill-procs:ms.cnt": 4, "Destroy()/recursive-kill:ms.avg": 0, "Destroy()/recursive-kill:ms.cnt": 4, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 4, "Destroy():ms.avg": 1, "Destroy():ms.cnt": 7, "Pause():ms.avg": 0, "Pause():ms.cnt": 1664, "Unpause():ms.avg": 0, "Unpause():ms.cnt": 1661, "fork():ms.avg": 7, "fork():ms.cnt": 3, "forkRequest:ms.avg": 6, "forkRequest:ms.cnt": 3, "move-to-cg-after-fork:ms.avg": 0, "move-to-cg-after-fork:ms.cnt": 3 }, "errors": [], "worker_tail": [ "2019/07/12 19:53:25 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:53:25 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:53:25 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:53:25 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:53:25 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:53:25 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:53:25 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:53:25 remove worker.pid", "2019/07/12 19:53:25 exiting", "" ] } POP conf: {'sandbox': 'sock', 'registry_cache_ms': 3000, 'registry': '/tmp/tmpodepsov4'} PUSH conf: {'sandbox': 'sock', 'import_cache_mb': 250, 'registry': '/root/open-lambda/test-registry', 'handler_cache_mb': 250} ======================================== stress_one_lambda {'procs': 1, 'seconds': 15} ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "stress_one_lambda", "params": { "procs": 1, "seconds": 15 }, "pass": true, "conf": { "worker_dir": "/root/open-lambda/test-dir/worker", "import_cache_tree": "", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "server_mode": "lambda", "handler_cache_mb": 250, "sandbox_config": {}, "docker_runtime": "", "worker_port": "5000", "limits": { "mem_mb": 50, "procs": 10, "installer_mem_mb": 200 }, "import_cache_mb": 250, "sandbox": "sock", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-registry", "pip_mirror": "" }, "seconds": 15.042324542999268, "total_seconds": 15.273486852645874, "stats": { "reqs_per_sec": 283.2 }, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 3, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 4, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 4, "Create()/fork-proc:ms.avg": 8, "Create()/fork-proc:ms.cnt": 3, "Create()/fresh-proc:ms.avg": 138, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 3, "Create()/make-root-fs:ms.cnt": 4, "Create():ms.avg": 44, "Create():ms.cnt": 4, "Destroy()/detach-root:ms.avg": 0, "Destroy()/detach-root:ms.cnt": 4, "Destroy()/kill-procs:ms.avg": 0, "Destroy()/kill-procs:ms.cnt": 4, "Destroy()/recursive-kill:ms.avg": 0, "Destroy()/recursive-kill:ms.cnt": 4, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 4, "Destroy():ms.avg": 1, "Destroy():ms.cnt": 7, "Pause():ms.avg": 0, "Pause():ms.cnt": 4248, "Unpause():ms.avg": 0, "Unpause():ms.cnt": 4245, "fork():ms.avg": 8, "fork():ms.cnt": 3, "forkRequest:ms.avg": 6, "forkRequest:ms.cnt": 3, "move-to-cg-after-fork:ms.avg": 1, "move-to-cg-after-fork:ms.cnt": 3 }, "errors": [], "worker_tail": [ "2019/07/12 19:53:40 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:53:40 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:53:40 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:53:40 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:53:40 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:53:40 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:53:40 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:53:40 remove worker.pid", "2019/07/12 19:53:40 exiting", "" ] } ======================================== stress_one_lambda {'procs': 2, 'seconds': 15} ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "stress_one_lambda", "params": { "procs": 2, "seconds": 15 }, "pass": true, "conf": { "worker_dir": "/root/open-lambda/test-dir/worker", "import_cache_tree": "", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "server_mode": "lambda", "handler_cache_mb": 250, "sandbox_config": {}, "docker_runtime": "", "worker_port": "5000", "limits": { "mem_mb": 50, "procs": 10, "installer_mem_mb": 200 }, "import_cache_mb": 250, "sandbox": "sock", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-registry", "pip_mirror": "" }, "seconds": 15.046791553497314, "total_seconds": 15.278303146362305, "stats": { "reqs_per_sec": 280.1333333333333 }, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 3, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 4, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 4, "Create()/fork-proc:ms.avg": 7, "Create()/fork-proc:ms.cnt": 3, "Create()/fresh-proc:ms.avg": 137, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 3, "Create()/make-root-fs:ms.cnt": 4, "Create():ms.avg": 44, "Create():ms.cnt": 4, "Destroy()/detach-root:ms.avg": 0, "Destroy()/detach-root:ms.cnt": 4, "Destroy()/kill-procs:ms.avg": 0, "Destroy()/kill-procs:ms.cnt": 4, "Destroy()/recursive-kill:ms.avg": 0, "Destroy()/recursive-kill:ms.cnt": 4, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 4, "Destroy():ms.avg": 1, "Destroy():ms.cnt": 7, "Pause():ms.avg": 0, "Pause():ms.cnt": 3834, "Unpause():ms.avg": 0, "Unpause():ms.cnt": 3831, "fork():ms.avg": 7, "fork():ms.cnt": 3, "forkRequest:ms.avg": 6, "forkRequest:ms.cnt": 3, "move-to-cg-after-fork:ms.avg": 0, "move-to-cg-after-fork:ms.cnt": 3 }, "errors": [], "worker_tail": [ "2019/07/12 19:53:56 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:53:56 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:53:56 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:53:56 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:53:56 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:53:56 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:53:56 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:53:56 remove worker.pid", "2019/07/12 19:53:56 exiting", "" ] } ======================================== stress_one_lambda {'procs': 8, 'seconds': 15} ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "stress_one_lambda", "params": { "procs": 8, "seconds": 15 }, "pass": true, "conf": { "worker_dir": "/root/open-lambda/test-dir/worker", "import_cache_tree": "", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "server_mode": "lambda", "handler_cache_mb": 250, "sandbox_config": {}, "docker_runtime": "", "worker_port": "5000", "limits": { "mem_mb": 50, "procs": 10, "installer_mem_mb": 200 }, "import_cache_mb": 250, "sandbox": "sock", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/root/open-lambda/test-registry", "pip_mirror": "" }, "seconds": 15.063179969787598, "total_seconds": 15.293553352355957, "stats": { "reqs_per_sec": 258.6666666666667 }, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 3, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 4, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 4, "Create()/fork-proc:ms.avg": 8, "Create()/fork-proc:ms.cnt": 3, "Create()/fresh-proc:ms.avg": 136, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 3, "Create()/make-root-fs:ms.cnt": 4, "Create():ms.avg": 44, "Create():ms.cnt": 4, "Destroy()/detach-root:ms.avg": 0, "Destroy()/detach-root:ms.cnt": 4, "Destroy()/kill-procs:ms.avg": 6, "Destroy()/kill-procs:ms.cnt": 4, "Destroy()/recursive-kill:ms.avg": 0, "Destroy()/recursive-kill:ms.cnt": 4, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 4, "Destroy():ms.avg": 4, "Destroy():ms.cnt": 7, "Pause():ms.avg": 0, "Pause():ms.cnt": 2883, "Unpause():ms.avg": 0, "Unpause():ms.cnt": 2880, "fork():ms.avg": 8, "fork():ms.cnt": 3, "forkRequest:ms.avg": 7, "forkRequest:ms.cnt": 3, "move-to-cg-after-fork:ms.avg": 0, "move-to-cg-after-fork:ms.cnt": 3 }, "errors": [], "worker_tail": [ "2019/07/12 19:54:11 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:54:11 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:54:11 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:54:11 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:54:11 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:54:11 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:54:11 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:54:11 remove worker.pid", "2019/07/12 19:54:11 exiting", "" ] } POP conf: {'sandbox': 'sock', 'import_cache_mb': 250, 'registry': '/root/open-lambda/test-registry', 'handler_cache_mb': 250} PUSH conf: {'sandbox': 'sock', 'import_cache_mb': 250, 'handler_cache_mb': 250} PUSH conf: {'registry': '/tmp/tmp_pl_8jre'} ======================================== call_each_once_exec {'alloc_mb': 1, 'lambda_count': 100} ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "call_each_once_exec", "params": { "alloc_mb": 1, "lambda_count": 100 }, "pass": true, "conf": { "worker_dir": "/root/open-lambda/test-dir/worker", "import_cache_tree": "", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "server_mode": "lambda", "handler_cache_mb": 250, "sandbox_config": {}, "docker_runtime": "", "worker_port": "5000", "limits": { "mem_mb": 50, "procs": 10, "installer_mem_mb": 200 }, "import_cache_mb": 250, "sandbox": "sock", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/tmp/tmp_pl_8jre", "pip_mirror": "" }, "seconds": 2.4819533824920654, "total_seconds": 2.8236000537872314, "stats": { "reqs_per_sec": 40.291224690486594 }, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 100, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 101, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 101, "Create()/fork-proc:ms.avg": 7, "Create()/fork-proc:ms.cnt": 100, "Create()/fresh-proc:ms.avg": 134, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 0, "Create()/make-root-fs:ms.cnt": 101, "Create():ms.avg": 9, "Create():ms.cnt": 101, "Destroy()/detach-root:ms.avg": 0, "Destroy()/detach-root:ms.cnt": 101, "Destroy()/kill-procs:ms.avg": 0, "Destroy()/kill-procs:ms.cnt": 101, "Destroy()/recursive-kill:ms.avg": 0, "Destroy()/recursive-kill:ms.cnt": 101, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 101, "Destroy():ms.avg": 0, "Destroy():ms.cnt": 278, "Pause():ms.avg": 5, "Pause():ms.cnt": 100, "evict:ms.avg": 0, "evict:ms.cnt": 77, "fork():ms.avg": 7, "fork():ms.cnt": 100, "forkRequest:ms.avg": 6, "forkRequest:ms.cnt": 100, "move-to-cg-after-fork:ms.avg": 0, "move-to-cg-after-fork:ms.cnt": 100 }, "errors": [], "worker_tail": [ "2019/07/12 19:54:14 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:54:14 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:54:14 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:54:14 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:54:14 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:54:14 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:54:14 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:54:14 remove worker.pid", "2019/07/12 19:54:14 exiting", "" ] } POP conf: {'registry': '/tmp/tmp_pl_8jre'} PUSH conf: {'registry': '/tmp/tmpyqrr251a'} ======================================== call_each_once_exec {'alloc_mb': 10, 'lambda_count': 1000} ======================================== RUN ./ol worker -p=test-dir --detach RUN ./ol kill -p=test-dir { "test": "call_each_once_exec", "params": { "alloc_mb": 10, "lambda_count": 1000 }, "pass": true, "conf": { "worker_dir": "/root/open-lambda/test-dir/worker", "import_cache_tree": "", "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages", "server_mode": "lambda", "handler_cache_mb": 250, "sandbox_config": {}, "docker_runtime": "", "worker_port": "5000", "limits": { "mem_mb": 50, "procs": 10, "installer_mem_mb": 200 }, "import_cache_mb": 250, "sandbox": "sock", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-dir/lambda", "registry": "/tmp/tmpyqrr251a", "pip_mirror": "" }, "seconds": 36.56600260734558, "total_seconds": 36.90720891952515, "stats": { "reqs_per_sec": 27.347824786879453 }, "ol-stats": { "Channel():ms.avg": 0, "Channel():ms.cnt": 1000, "Create()/acquire-cgroup:ms.avg": 0, "Create()/acquire-cgroup:ms.cnt": 1001, "Create()/acquire-mem:ms.avg": 0, "Create()/acquire-mem:ms.cnt": 1001, "Create()/fork-proc:ms.avg": 7, "Create()/fork-proc:ms.cnt": 1000, "Create()/fresh-proc:ms.avg": 162, "Create()/fresh-proc:ms.cnt": 1, "Create()/make-root-fs:ms.avg": 1, "Create()/make-root-fs:ms.cnt": 1001, "Create():ms.avg": 10, "Create():ms.cnt": 1001, "Destroy()/detach-root:ms.avg": 0, "Destroy()/detach-root:ms.cnt": 1001, "Destroy()/kill-procs:ms.avg": 2, "Destroy()/kill-procs:ms.cnt": 1001, "Destroy()/recursive-kill:ms.avg": 0, "Destroy()/recursive-kill:ms.cnt": 1001, "Destroy()/remove-root:ms.avg": 0, "Destroy()/remove-root:ms.cnt": 1001, "Destroy():ms.avg": 0, "Destroy():ms.cnt": 2992, "Pause():ms.avg": 13, "Pause():ms.cnt": 1000, "evict:ms.avg": 2, "evict:ms.cnt": 991, "fork():ms.avg": 7, "fork():ms.cnt": 1000, "forkRequest:ms.avg": 6, "forkRequest:ms.cnt": 1000, "move-to-cg-after-fork:ms.avg": 1, "move-to-cg-after-fork:ms.cnt": 1000 }, "errors": [], "worker_tail": [ "2019/07/12 19:54:51 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:54:51 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:54:51 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:54:51 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:54:51 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:54:51 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]", "2019/07/12 19:54:51 save stats to /root/open-lambda/test-dir/worker/stats.json", "2019/07/12 19:54:51 remove worker.pid", "2019/07/12 19:54:51 exiting", "" ] } POP conf: {'registry': '/tmp/tmpyqrr251a'} POP conf: {'sandbox': 'sock', 'import_cache_mb': 250, 'handler_cache_mb': 250} PASSED: 18, FAILED: 0