python3 -u test.py RUN ./ol new -p=test-cluster Init OL dir at /root/open-lambda/test-cluster Create lambda base at /root/open-lambda/test-cluster/lambda (may take several minutes) Working Directory: /root/open-lambda/test-cluster Worker Defaults: { "registry": "/root/open-lambda/test-cluster/registry", "registry_cache_ms": 5000, "sandbox": "sock", "cluster_name": "/root/open-lambda/test-cluster", "pip_mirror": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "max_runners": 0, "handler_cache_mb": 239, "import_cache_mb": 239, "worker_dir": "/root/open-lambda/test-cluster/worker", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "worker_port": "5000", "Skip_pull_existing": false, "sandbox_config": { "processes": 10 }, "benchmark_log": "", "timing": false, "startup_pkgs": null, "docker_runtime": "" } You may now start a server using the "worker" command PUSH conf: {'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson'], 'registry': '/root/open-lambda/test-registry'} PUSH conf: {'import_cache_mb': 0, 'handler_cache_mb': 0, 'sandbox': 'sock'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7289, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7289 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 5.106036186218262, "total_seconds": 29.61469054222107, "stats": null, "conf": { "max_runners": 0, "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "handler_cache_mb": 0, "sandbox": "sock", "timing": false, "sandbox_config": { "processes": 10 }, "worker_port": "5000", "worker_dir": "/root/open-lambda/test-cluster/worker", "registry": "/root/open-lambda/test-registry", "pip_mirror": "", "docker_runtime": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "registry_cache_ms": 5000, "import_cache_mb": 0, "cluster_name": "/root/open-lambda/test-cluster", "benchmark_log": "", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/12 17:27:51 received kill signal, cleaning up", "2019/06/12 17:27:51 Cleanup Lambdas:", "2019/06/12 17:27:51 Function: install3", "2019/06/12 17:27:51 Function: echo", "2019/06/12 17:27:51 Function: install", "2019/06/12 17:27:51 Function: install2", "2019/06/12 17:27:51 Cleanup Container Factory", "2019/06/12 17:27:51 Finished Lambda Cleanup", "2019/06/12 17:27:51 exiting", "" ] } POP conf: {'import_cache_mb': 0, 'handler_cache_mb': 0, 'sandbox': 'sock'} PUSH conf: {'import_cache_mb': 0, 'handler_cache_mb': 256, 'sandbox': 'sock'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7375, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7375 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 1.2511770725250244, "total_seconds": 8.907678842544556, "stats": null, "conf": { "max_runners": 0, "Skip_pull_existing": false, "handler_cache_mb": 256, "sandbox": "sock", "timing": false, "benchmark_log": "", "sandbox_config": { "processes": 10 }, "worker_port": "5000", "worker_dir": "/root/open-lambda/test-cluster/worker", "registry": "/root/open-lambda/test-registry", "pip_mirror": "", "docker_runtime": "", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "import_cache_mb": 0, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "cluster_name": "/root/open-lambda/test-cluster", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/12 17:28:00 Function: install3", "2019/06/12 17:28:00 Instance: 3", "2019/06/12 17:28:00 destroy [SOCK 3]", "2019/06/12 17:28:00 Pause/KillAllProcs/Unpause [SOCK 3]", "2019/06/12 17:28:00 wait for init to die [SOCK 3]", "2019/06/12 17:28:00 unmount and remove dirs [SOCK 3]", "2019/06/12 17:28:00 Cleanup Container Factory", "2019/06/12 17:28:00 Finished Lambda Cleanup", "2019/06/12 17:28:00 exiting", "" ] } POP conf: {'import_cache_mb': 0, 'handler_cache_mb': 256, 'sandbox': 'sock'} PUSH conf: {'import_cache_mb': 256, 'handler_cache_mb': 0, 'sandbox': 'sock'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7450, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7450 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 0.759650707244873, "total_seconds": 8.608296632766724, "stats": null, "conf": { "max_runners": 0, "Skip_pull_existing": false, "handler_cache_mb": 0, "sandbox": "sock", "timing": false, "sandbox_config": { "processes": 10 }, "worker_port": "5000", "worker_dir": "/root/open-lambda/test-cluster/worker", "registry": "/root/open-lambda/test-registry", "pip_mirror": "", "docker_runtime": "", "cluster_name": "/root/open-lambda/test-cluster", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "registry_cache_ms": 5000, "import_cache_mb": 256, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "benchmark_log": "", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/12 17:28:09 received kill signal, cleaning up", "2019/06/12 17:28:09 Cleanup Lambdas:", "2019/06/12 17:28:09 Function: install2", "2019/06/12 17:28:09 Function: install3", "2019/06/12 17:28:09 Function: echo", "2019/06/12 17:28:09 Function: install", "2019/06/12 17:28:09 Cleanup Container Factory", "2019/06/12 17:28:09 Finished Lambda Cleanup", "2019/06/12 17:28:09 exiting", "" ] } POP conf: {'import_cache_mb': 256, 'handler_cache_mb': 0, 'sandbox': 'sock'} PUSH conf: {'import_cache_mb': 256, 'handler_cache_mb': 256, 'sandbox': 'sock'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7533, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7533 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 0.7477719783782959, "total_seconds": 8.594419240951538, "stats": null, "conf": { "max_runners": 0, "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "handler_cache_mb": 256, "sandbox": "sock", "timing": false, "sandbox_config": { "processes": 10 }, "worker_port": "5000", "worker_dir": "/root/open-lambda/test-cluster/worker", "registry": "/root/open-lambda/test-registry", "pip_mirror": "", "docker_runtime": "", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "import_cache_mb": 256, "cluster_name": "/root/open-lambda/test-cluster", "benchmark_log": "", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/12 17:28:17 Function: install2", "2019/06/12 17:28:17 Instance: 2", "2019/06/12 17:28:17 destroy [SOCK 2]", "2019/06/12 17:28:17 Pause/KillAllProcs/Unpause [SOCK 2]", "2019/06/12 17:28:17 wait for init to die [SOCK 2]", "2019/06/12 17:28:17 unmount and remove dirs [SOCK 2]", "2019/06/12 17:28:17 Cleanup Container Factory", "2019/06/12 17:28:17 Finished Lambda Cleanup", "2019/06/12 17:28:17 exiting", "" ] } POP conf: {'import_cache_mb': 256, 'handler_cache_mb': 256, 'sandbox': 'sock'} PUSH conf: {'import_cache_mb': 0, 'handler_cache_mb': 0, 'sandbox': 'docker'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7614, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7614 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 8.831933736801147, "total_seconds": 16.481369495391846, "stats": null, "conf": { "max_runners": 0, "Skip_pull_existing": false, "handler_cache_mb": 0, "sandbox": "docker", "timing": false, "benchmark_log": "", "sandbox_config": { "processes": 10 }, "worker_port": "5000", "worker_dir": "/root/open-lambda/test-cluster/worker", "registry": "/root/open-lambda/test-registry", "pip_mirror": "", "docker_runtime": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "registry_cache_ms": 5000, "import_cache_mb": 0, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "cluster_name": "/root/open-lambda/test-cluster", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/12 17:28:34 received kill signal, cleaning up", "2019/06/12 17:28:34 Cleanup Lambdas:", "2019/06/12 17:28:34 Function: install3", "2019/06/12 17:28:34 Function: echo", "2019/06/12 17:28:34 Function: install", "2019/06/12 17:28:34 Function: install2", "2019/06/12 17:28:34 Cleanup Container Factory", "2019/06/12 17:28:34 Finished Lambda Cleanup", "2019/06/12 17:28:34 exiting", "" ] } POP conf: {'import_cache_mb': 0, 'handler_cache_mb': 0, 'sandbox': 'docker'} PUSH conf: {'import_cache_mb': 0, 'handler_cache_mb': 256, 'sandbox': 'docker'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=8558, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 8558 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 3.5621421337127686, "total_seconds": 12.118316888809204, "stats": null, "conf": { "max_runners": 0, "Skip_pull_existing": false, "handler_cache_mb": 256, "sandbox": "docker", "timing": false, "sandbox_config": { "processes": 10 }, "worker_port": "5000", "worker_dir": "/root/open-lambda/test-cluster/worker", "registry": "/root/open-lambda/test-registry", "pip_mirror": "", "docker_runtime": "", "cluster_name": "/root/open-lambda/test-cluster", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "import_cache_mb": 0, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "benchmark_log": "", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/12 17:28:46 Function: install", "2019/06/12 17:28:46 Instance: 1", "2019/06/12 17:28:46 Function: install2", "2019/06/12 17:28:46 Instance: 2", "2019/06/12 17:28:46 Function: install3", "2019/06/12 17:28:46 Instance: 3", "2019/06/12 17:28:46 Cleanup Container Factory", "2019/06/12 17:28:46 Finished Lambda Cleanup", "2019/06/12 17:28:46 exiting", "" ] } POP conf: {'import_cache_mb': 0, 'handler_cache_mb': 256, 'sandbox': 'docker'} POP conf: {'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson'], 'registry': '/root/open-lambda/test-registry'} PUSH conf: {'import_cache_mb': 256, 'registry': '/root/open-lambda/test-registry', 'handler_cache_mb': 256, 'sandbox': 'sock'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=9602, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 9602 { "test": "stress_one_lambda", "params": { "procs": 1, "seconds": 15 }, "pass": true, "seconds": 15.040589332580566, "total_seconds": 15.561613321304321, "stats": { "reqs_per_sec": 282.06666666666666 }, "conf": { "max_runners": 0, "Skip_pull_existing": false, "handler_cache_mb": 256, "sandbox": "sock", "timing": false, "sandbox_config": { "processes": 10 }, "worker_port": "5000", "worker_dir": "/root/open-lambda/test-cluster/worker", "registry": "/root/open-lambda/test-registry", "pip_mirror": "", "docker_runtime": "", "cluster_name": "/root/open-lambda/test-cluster", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "import_cache_mb": 256, "startup_pkgs": null, "benchmark_log": "", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/12 17:29:02 Function: echo", "2019/06/12 17:29:02 Instance: 0", "2019/06/12 17:29:02 destroy [SOCK 0]", "2019/06/12 17:29:02 Pause/KillAllProcs/Unpause [SOCK 0]", "2019/06/12 17:29:02 wait for init to die [SOCK 0]", "2019/06/12 17:29:02 unmount and remove dirs [SOCK 0]", "2019/06/12 17:29:02 Cleanup Container Factory", "2019/06/12 17:29:02 Finished Lambda Cleanup", "2019/06/12 17:29:02 exiting", "" ] } RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=9635, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 9635 { "test": "stress_one_lambda", "params": { "procs": 2, "seconds": 15 }, "pass": true, "seconds": 15.04959750175476, "total_seconds": 15.371678352355957, "stats": { "reqs_per_sec": 281.8 }, "conf": { "max_runners": 0, "Skip_pull_existing": false, "handler_cache_mb": 256, "sandbox": "sock", "timing": false, "sandbox_config": { "processes": 10 }, "worker_port": "5000", "worker_dir": "/root/open-lambda/test-cluster/worker", "registry": "/root/open-lambda/test-registry", "pip_mirror": "", "docker_runtime": "", "cluster_name": "/root/open-lambda/test-cluster", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "import_cache_mb": 256, "startup_pkgs": null, "benchmark_log": "", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/12 17:29:17 unmount and remove dirs [SOCK 1]", "2019/06/12 17:29:17 Instance: 0", "2019/06/12 17:29:17 destroy [SOCK 0]", "2019/06/12 17:29:17 Pause/KillAllProcs/Unpause [SOCK 0]", "2019/06/12 17:29:17 wait for init to die [SOCK 0]", "2019/06/12 17:29:17 unmount and remove dirs [SOCK 0]", "2019/06/12 17:29:17 Cleanup Container Factory", "2019/06/12 17:29:17 Finished Lambda Cleanup", "2019/06/12 17:29:17 exiting", "" ] } RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=9677, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 9677 { "test": "stress_one_lambda", "params": { "procs": 8, "seconds": 15 }, "pass": true, "seconds": 15.063488721847534, "total_seconds": 15.889222383499146, "stats": { "reqs_per_sec": 263.4 }, "conf": { "max_runners": 0, "Skip_pull_existing": false, "handler_cache_mb": 256, "sandbox": "sock", "timing": false, "sandbox_config": { "processes": 10 }, "worker_port": "5000", "worker_dir": "/root/open-lambda/test-cluster/worker", "registry": "/root/open-lambda/test-registry", "pip_mirror": "", "docker_runtime": "", "cluster_name": "/root/open-lambda/test-cluster", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "import_cache_mb": 256, "startup_pkgs": null, "benchmark_log": "", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/12 17:29:33 unmount and remove dirs [SOCK 2]", "2019/06/12 17:29:33 Instance: 0", "2019/06/12 17:29:33 destroy [SOCK 0]", "2019/06/12 17:29:33 Pause/KillAllProcs/Unpause [SOCK 0]", "2019/06/12 17:29:33 wait for init to die [SOCK 0]", "2019/06/12 17:29:33 unmount and remove dirs [SOCK 0]", "2019/06/12 17:29:33 Cleanup Container Factory", "2019/06/12 17:29:33 Finished Lambda Cleanup", "2019/06/12 17:29:33 exiting", "" ] } POP conf: {'import_cache_mb': 256, 'registry': '/root/open-lambda/test-registry', 'handler_cache_mb': 256, 'sandbox': 'sock'} PUSH conf: {'import_cache_mb': 256, 'handler_cache_mb': 256, 'sandbox': 'sock'} PUSH conf: {'registry': '/tmp/tmprygqlnl7'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=9753, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 9753 { "test": "call_each_once_exec", "params": { "lambda_count": 100, "alloc_mb": 1 }, "pass": true, "seconds": 2.5626258850097656, "total_seconds": 3.093510866165161, "stats": { "reqs_per_sec": 39.02289437712847 }, "conf": { "max_runners": 0, "Skip_pull_existing": false, "handler_cache_mb": 256, "sandbox": "sock", "timing": false, "sandbox_config": { "processes": 10 }, "worker_port": "5000", "worker_dir": "/root/open-lambda/test-cluster/worker", "registry": "/tmp/tmprygqlnl7", "pip_mirror": "", "docker_runtime": "", "cluster_name": "/root/open-lambda/test-cluster", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "registry_cache_ms": 5000, "import_cache_mb": 256, "startup_pkgs": null, "benchmark_log": "", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/12 17:29:36 Instance: 80", "2019/06/12 17:29:36 destroy [SOCK 80]", "2019/06/12 17:29:36 Pause/KillAllProcs/Unpause [SOCK 80]", "2019/06/12 17:29:36 wait for init to die [SOCK 80]", "2019/06/12 17:29:36 unmount and remove dirs [SOCK 80]", "2019/06/12 17:29:36 Function: L16", "2019/06/12 17:29:36 Cleanup Container Factory", "2019/06/12 17:29:36 Finished Lambda Cleanup", "2019/06/12 17:29:36 exiting", "" ] } POP conf: {'registry': '/tmp/tmprygqlnl7'} PUSH conf: {'registry': '/tmp/tmpa34rr998'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=10378, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 10378 { "test": "call_each_once_exec", "params": { "lambda_count": 1000, "alloc_mb": 10 }, "pass": true, "seconds": 39.74612879753113, "total_seconds": 41.187177896499634, "stats": { "reqs_per_sec": 25.159705159705158 }, "conf": { "max_runners": 0, "Skip_pull_existing": false, "startup_pkgs": null, "handler_cache_mb": 256, "sandbox": "sock", "timing": false, "sandbox_config": { "processes": 10 }, "worker_port": "5000", "worker_dir": "/root/open-lambda/test-cluster/worker", "registry": "/tmp/tmpa34rr998", "pip_mirror": "", "docker_runtime": "", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "import_cache_mb": 256, "cluster_name": "/root/open-lambda/test-cluster", "benchmark_log": "", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/12 17:30:17 destroy [SOCK 984]", "2019/06/12 17:30:17 Pause/KillAllProcs/Unpause [SOCK 984]", "2019/06/12 17:30:17 wait for init to die [SOCK 984]", "2019/06/12 17:30:17 unmount and remove dirs [SOCK 984]", "2019/06/12 17:30:17 Function: L131", "2019/06/12 17:30:17 Function: L660", "2019/06/12 17:30:17 Cleanup Container Factory", "2019/06/12 17:30:17 Finished Lambda Cleanup", "2019/06/12 17:30:17 exiting", "" ] } POP conf: {'registry': '/tmp/tmpa34rr998'} POP conf: {'import_cache_mb': 256, 'handler_cache_mb': 256, 'sandbox': 'sock'} PASSED: 11, FAILED: 0