python3 -u test.py RUN ./ol new -p=test-cluster Init OL dir at /root/open-lambda/test-cluster Create lambda base at /root/open-lambda/test-cluster/lambda (may take several minutes) Working Directory: /root/open-lambda/test-cluster Worker Defaults: { "registry": "/root/open-lambda/test-cluster/registry", "registry_cache_ms": 5000, "sandbox": "sock", "cluster_name": "/root/open-lambda/test-cluster", "pip_mirror": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "max_runners": 0, "handler_cache_mb": 239, "import_cache_mb": 239, "server_mode": "lambda", "worker_dir": "/root/open-lambda/test-cluster/worker", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "worker_port": "5000", "Skip_pull_existing": false, "sandbox_config": { "processes": 10 }, "benchmark_log": "", "timing": false, "startup_pkgs": null, "docker_runtime": "" } You may now start a server using the "worker" command PUSH conf: {'registry': '/root/open-lambda/test-registry', 'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson']} PUSH conf: {'import_cache_mb': 0, 'sandbox': 'sock', 'handler_cache_mb': 0} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7302, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7302 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 3.156620979309082, "total_seconds": 22.9092013835907, "stats": null, "conf": { "import_cache_mb": 0, "timing": false, "max_runners": 0, "server_mode": "lambda", "sandbox_config": { "processes": 10 }, "sandbox": "sock", "registry_cache_ms": 5000, "worker_dir": "/root/open-lambda/test-cluster/worker", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "pip_mirror": "", "docker_runtime": "", "worker_port": "5000", "benchmark_log": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "handler_cache_mb": 0, "Skip_pull_existing": false, "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry" }, "exception": null, "worker_tail": [ "2019/06/14 18:43:27 received kill signal, cleaning up", "2019/06/14 18:43:27 Cleanup Lambdas:", "2019/06/14 18:43:27 Function: echo", "2019/06/14 18:43:27 Function: install", "2019/06/14 18:43:27 Function: install2", "2019/06/14 18:43:27 Function: install3", "2019/06/14 18:43:27 Cleanup Container Factory", "2019/06/14 18:43:27 Finished Lambda Cleanup", "2019/06/14 18:43:27 exiting", "" ] } POP conf: {'import_cache_mb': 0, 'sandbox': 'sock', 'handler_cache_mb': 0} PUSH conf: {'import_cache_mb': 0, 'sandbox': 'sock', 'handler_cache_mb': 256} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7387, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7387 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 1.284043788909912, "total_seconds": 9.042889833450317, "stats": null, "conf": { "import_cache_mb": 0, "timing": false, "server_mode": "lambda", "sandbox": "sock", "benchmark_log": "", "docker_runtime": "", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "worker_dir": "/root/open-lambda/test-cluster/worker", "worker_port": "5000", "cluster_name": "/root/open-lambda/test-cluster", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sandbox_config": { "processes": 10 }, "max_runners": 0, "handler_cache_mb": 256, "Skip_pull_existing": false, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "registry": "/root/open-lambda/test-registry" }, "exception": null, "worker_tail": [ "2019/06/14 18:43:36 Function: install3", "2019/06/14 18:43:36 Instance: 3", "2019/06/14 18:43:36 destroy [SOCK 3]", "2019/06/14 18:43:36 Pause/KillAllProcs/Unpause [SOCK 3]", "2019/06/14 18:43:36 wait for init to die [SOCK 3]", "2019/06/14 18:43:36 unmount and remove dirs [SOCK 3]", "2019/06/14 18:43:36 Cleanup Container Factory", "2019/06/14 18:43:36 Finished Lambda Cleanup", "2019/06/14 18:43:36 exiting", "" ] } POP conf: {'import_cache_mb': 0, 'sandbox': 'sock', 'handler_cache_mb': 256} PUSH conf: {'import_cache_mb': 256, 'sandbox': 'sock', 'handler_cache_mb': 0} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7461, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7461 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 0.7511060237884521, "total_seconds": 8.601987361907959, "stats": null, "conf": { "import_cache_mb": 256, "timing": false, "server_mode": "lambda", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "sandbox_config": { "processes": 10 }, "sandbox": "sock", "registry_cache_ms": 5000, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "worker_dir": "/root/open-lambda/test-cluster/worker", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "pip_mirror": "", "docker_runtime": "", "worker_port": "5000", "benchmark_log": "", "max_runners": 0, "handler_cache_mb": 0, "Skip_pull_existing": false, "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry" }, "exception": null, "worker_tail": [ "2019/06/14 18:43:45 Function: echo", "2019/06/14 18:43:45 Function: install", "2019/06/14 18:43:45 Function: install2", "2019/06/14 18:43:45 Function: install3", "2019/06/14 18:43:45 Cleanup Container Factory", "2019/06/14 18:43:45 unmount and remove dirs [SOCK 3]", "2019/06/14 18:43:45 unmount root dir /root/open-lambda/test-cluster/worker/sock-handler-roots/3 failed :: invalid argument [SOCK 3]", "2019/06/14 18:43:45 Finished Lambda Cleanup", "2019/06/14 18:43:45 exiting", "" ] } POP conf: {'import_cache_mb': 256, 'sandbox': 'sock', 'handler_cache_mb': 0} PUSH conf: {'import_cache_mb': 256, 'sandbox': 'sock', 'handler_cache_mb': 256} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7542, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7542 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 0.7372169494628906, "total_seconds": 8.882110118865967, "stats": null, "conf": { "import_cache_mb": 256, "timing": false, "server_mode": "lambda", "sandbox": "sock", "benchmark_log": "", "docker_runtime": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "worker_port": "5000", "cluster_name": "/root/open-lambda/test-cluster", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sandbox_config": { "processes": 10 }, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "handler_cache_mb": 256, "Skip_pull_existing": false, "max_runners": 0, "registry": "/root/open-lambda/test-registry" }, "exception": null, "worker_tail": [ "2019/06/14 18:43:54 Function: install2", "2019/06/14 18:43:54 Instance: 2", "2019/06/14 18:43:54 destroy [SOCK 2]", "2019/06/14 18:43:54 Pause/KillAllProcs/Unpause [SOCK 2]", "2019/06/14 18:43:54 wait for init to die [SOCK 2]", "2019/06/14 18:43:54 unmount and remove dirs [SOCK 2]", "2019/06/14 18:43:54 Cleanup Container Factory", "2019/06/14 18:43:54 Finished Lambda Cleanup", "2019/06/14 18:43:54 exiting", "" ] } POP conf: {'import_cache_mb': 256, 'sandbox': 'sock', 'handler_cache_mb': 256} PUSH conf: {'import_cache_mb': 0, 'sandbox': 'docker', 'handler_cache_mb': 0} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7625, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7625 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 7.057769536972046, "total_seconds": 14.603512525558472, "stats": null, "conf": { "import_cache_mb": 0, "timing": false, "max_runners": 0, "server_mode": "lambda", "sandbox_config": { "processes": 10 }, "sandbox": "docker", "registry_cache_ms": 5000, "worker_dir": "/root/open-lambda/test-cluster/worker", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "pip_mirror": "", "docker_runtime": "", "worker_port": "5000", "benchmark_log": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "handler_cache_mb": 0, "Skip_pull_existing": false, "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry" }, "exception": null, "worker_tail": [ "2019/06/14 18:44:08 received kill signal, cleaning up", "2019/06/14 18:44:08 Cleanup Lambdas:", "2019/06/14 18:44:08 Function: echo", "2019/06/14 18:44:08 Function: install", "2019/06/14 18:44:08 Function: install2", "2019/06/14 18:44:08 Function: install3", "2019/06/14 18:44:08 Cleanup Container Factory", "2019/06/14 18:44:08 Finished Lambda Cleanup", "2019/06/14 18:44:08 exiting", "" ] } POP conf: {'import_cache_mb': 0, 'sandbox': 'docker', 'handler_cache_mb': 0} PUSH conf: {'import_cache_mb': 0, 'sandbox': 'docker', 'handler_cache_mb': 256} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=8569, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 8569 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 4.50018310546875, "total_seconds": 13.258925437927246, "stats": null, "conf": { "import_cache_mb": 0, "timing": false, "server_mode": "lambda", "sandbox": "docker", "benchmark_log": "", "docker_runtime": "", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "worker_dir": "/root/open-lambda/test-cluster/worker", "worker_port": "5000", "cluster_name": "/root/open-lambda/test-cluster", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sandbox_config": { "processes": 10 }, "max_runners": 0, "handler_cache_mb": 256, "Skip_pull_existing": false, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "registry": "/root/open-lambda/test-registry" }, "exception": null, "worker_tail": [ "2019/06/14 18:44:21 Function: install", "2019/06/14 18:44:21 Instance: 1", "2019/06/14 18:44:21 Function: install2", "2019/06/14 18:44:21 Instance: 2", "2019/06/14 18:44:21 Function: install3", "2019/06/14 18:44:21 Instance: 3", "2019/06/14 18:44:22 Cleanup Container Factory", "2019/06/14 18:44:22 Finished Lambda Cleanup", "2019/06/14 18:44:22 exiting", "" ] } POP conf: {'import_cache_mb': 0, 'sandbox': 'docker', 'handler_cache_mb': 256} POP conf: {'registry': '/root/open-lambda/test-registry', 'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson']} PUSH conf: {'import_cache_mb': 256, 'sandbox': 'sock', 'registry': '/root/open-lambda/test-registry', 'handler_cache_mb': 256} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=9612, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 9612 { "test": "stress_one_lambda", "params": { "procs": 1, "seconds": 15 }, "pass": true, "seconds": 15.010785818099976, "total_seconds": 15.329612970352173, "stats": { "reqs_per_sec": 278.6 }, "conf": { "import_cache_mb": 256, "timing": false, "server_mode": "lambda", "sandbox": "sock", "benchmark_log": "", "docker_runtime": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "startup_pkgs": null, "worker_port": "5000", "cluster_name": "/root/open-lambda/test-cluster", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sandbox_config": { "processes": 10 }, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "handler_cache_mb": 256, "Skip_pull_existing": false, "max_runners": 0, "registry": "/root/open-lambda/test-registry" }, "exception": null, "worker_tail": [ "2019/06/14 18:44:37 Function: echo", "2019/06/14 18:44:37 Instance: 0", "2019/06/14 18:44:37 destroy [SOCK 0]", "2019/06/14 18:44:37 Pause/KillAllProcs/Unpause [SOCK 0]", "2019/06/14 18:44:37 wait for init to die [SOCK 0]", "2019/06/14 18:44:37 unmount and remove dirs [SOCK 0]", "2019/06/14 18:44:37 Cleanup Container Factory", "2019/06/14 18:44:37 Finished Lambda Cleanup", "2019/06/14 18:44:37 exiting", "" ] } RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=9646, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 9646 { "test": "stress_one_lambda", "params": { "procs": 2, "seconds": 15 }, "pass": true, "seconds": 15.0483238697052, "total_seconds": 15.371538639068604, "stats": { "reqs_per_sec": 279.53333333333336 }, "conf": { "import_cache_mb": 256, "timing": false, "server_mode": "lambda", "sandbox": "sock", "benchmark_log": "", "docker_runtime": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "startup_pkgs": null, "worker_port": "5000", "cluster_name": "/root/open-lambda/test-cluster", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sandbox_config": { "processes": 10 }, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "handler_cache_mb": 256, "Skip_pull_existing": false, "max_runners": 0, "registry": "/root/open-lambda/test-registry" }, "exception": null, "worker_tail": [ "2019/06/14 18:44:52 unmount and remove dirs [SOCK 1]", "2019/06/14 18:44:52 Instance: 0", "2019/06/14 18:44:52 destroy [SOCK 0]", "2019/06/14 18:44:52 Pause/KillAllProcs/Unpause [SOCK 0]", "2019/06/14 18:44:52 wait for init to die [SOCK 0]", "2019/06/14 18:44:52 unmount and remove dirs [SOCK 0]", "2019/06/14 18:44:52 Cleanup Container Factory", "2019/06/14 18:44:52 Finished Lambda Cleanup", "2019/06/14 18:44:52 exiting", "" ] } RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=9688, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 9688 { "test": "stress_one_lambda", "params": { "procs": 8, "seconds": 15 }, "pass": true, "seconds": 15.06495189666748, "total_seconds": 15.511000394821167, "stats": { "reqs_per_sec": 250.2 }, "conf": { "import_cache_mb": 256, "timing": false, "server_mode": "lambda", "sandbox": "sock", "benchmark_log": "", "docker_runtime": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "startup_pkgs": null, "worker_port": "5000", "cluster_name": "/root/open-lambda/test-cluster", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sandbox_config": { "processes": 10 }, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "handler_cache_mb": 256, "Skip_pull_existing": false, "max_runners": 0, "registry": "/root/open-lambda/test-registry" }, "exception": null, "worker_tail": [ "2019/06/14 18:45:08 unmount and remove dirs [SOCK 3]", "2019/06/14 18:45:08 Instance: 0", "2019/06/14 18:45:08 destroy [SOCK 0]", "2019/06/14 18:45:08 Pause/KillAllProcs/Unpause [SOCK 0]", "2019/06/14 18:45:08 wait for init to die [SOCK 0]", "2019/06/14 18:45:08 unmount and remove dirs [SOCK 0]", "2019/06/14 18:45:08 Cleanup Container Factory", "2019/06/14 18:45:08 Finished Lambda Cleanup", "2019/06/14 18:45:08 exiting", "" ] } POP conf: {'import_cache_mb': 256, 'sandbox': 'sock', 'registry': '/root/open-lambda/test-registry', 'handler_cache_mb': 256} PUSH conf: {'import_cache_mb': 256, 'sandbox': 'sock', 'handler_cache_mb': 256} PUSH conf: {'registry': '/tmp/tmpa7cn2awg'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=9765, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 9765 { "test": "call_each_once_exec", "params": { "lambda_count": 100, "alloc_mb": 1 }, "pass": true, "seconds": 2.5744121074676514, "total_seconds": 3.093836545944214, "stats": { "reqs_per_sec": 38.84426228293614 }, "conf": { "import_cache_mb": 256, "timing": false, "server_mode": "lambda", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "sandbox_config": { "processes": 10 }, "sandbox": "sock", "registry_cache_ms": 5000, "startup_pkgs": null, "worker_dir": "/root/open-lambda/test-cluster/worker", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "pip_mirror": "", "docker_runtime": "", "worker_port": "5000", "benchmark_log": "", "max_runners": 0, "handler_cache_mb": 256, "Skip_pull_existing": false, "cluster_name": "/root/open-lambda/test-cluster", "registry": "/tmp/tmpa7cn2awg" }, "exception": null, "worker_tail": [ "2019/06/14 18:45:11 Function: L63", "2019/06/14 18:45:11 Instance: 63", "2019/06/14 18:45:11 destroy [SOCK 63]", "2019/06/14 18:45:11 Pause/KillAllProcs/Unpause [SOCK 63]", "2019/06/14 18:45:11 wait for init to die [SOCK 63]", "2019/06/14 18:45:11 unmount and remove dirs [SOCK 63]", "2019/06/14 18:45:11 Cleanup Container Factory", "2019/06/14 18:45:11 Finished Lambda Cleanup", "2019/06/14 18:45:11 exiting", "" ] } POP conf: {'registry': '/tmp/tmpa7cn2awg'} PUSH conf: {'registry': '/tmp/tmp1d9d6hcp'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=10389, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 10389 { "test": "call_each_once_exec", "params": { "lambda_count": 1000, "alloc_mb": 10 }, "pass": true, "seconds": 37.707555532455444, "total_seconds": 39.250096559524536, "stats": { "reqs_per_sec": 26.51990449302698 }, "conf": { "import_cache_mb": 256, "timing": false, "server_mode": "lambda", "sandbox": "sock", "benchmark_log": "", "docker_runtime": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "startup_pkgs": null, "worker_port": "5000", "cluster_name": "/root/open-lambda/test-cluster", "pip_mirror": "", "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "sandbox_config": { "processes": 10 }, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "handler_cache_mb": 256, "Skip_pull_existing": false, "max_runners": 0, "registry": "/tmp/tmp1d9d6hcp" }, "exception": null, "worker_tail": [ "2019/06/14 18:45:50 Function: L736", "2019/06/14 18:45:50 Function: L1", "2019/06/14 18:45:50 Function: L165", "2019/06/14 18:45:50 Function: L193", "2019/06/14 18:45:50 Function: L248", "2019/06/14 18:45:50 Function: L738", "2019/06/14 18:45:50 Cleanup Container Factory", "2019/06/14 18:45:50 Finished Lambda Cleanup", "2019/06/14 18:45:50 exiting", "" ] } POP conf: {'registry': '/tmp/tmp1d9d6hcp'} POP conf: {'import_cache_mb': 256, 'sandbox': 'sock', 'handler_cache_mb': 256} PASSED: 11, FAILED: 0