python3 -u test.py RUN ./ol new -p=test-cluster Init OL dir at /root/open-lambda/test-cluster Create lambda base at /root/open-lambda/test-cluster/lambda (may take several minutes) Working Directory: /root/open-lambda/test-cluster Worker Defaults: { "registry": "/root/open-lambda/test-cluster/registry", "registry_cache_ms": 5000, "sandbox": "sock", "cluster_name": "/root/open-lambda/test-cluster", "pip_mirror": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "max_runners": 0, "handler_cache_mb": 239, "import_cache_mb": 239, "worker_dir": "/root/open-lambda/test-cluster/worker", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "worker_port": "5000", "Skip_pull_existing": false, "sandbox_config": { "processes": 10 }, "benchmark_log": "", "timing": false, "startup_pkgs": null, "docker_runtime": "" } You may now start a server using the "worker" command PUSH conf: {'registry': '/root/open-lambda/test-registry', 'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson']} PUSH conf: {'sandbox': 'sock', 'handler_cache_mb': 0, 'import_cache_mb': 0} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7261, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7261 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 3.0312418937683105, "total_seconds": 20.41003394126892, "stats": null, "conf": { "pip_mirror": "", "docker_runtime": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "benchmark_log": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "sandbox": "sock", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "worker_port": "5000", "import_cache_mb": 0, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "handler_cache_mb": 0, "sandbox_config": { "processes": 10 }, "max_runners": 0, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "Skip_pull_existing": false }, "exception": null, "worker_tail": [ "2019/06/12 21:43:16 Function: echo", "2019/06/12 21:43:16 Function: install", "2019/06/12 21:43:16 Function: install2", "2019/06/12 21:43:16 Function: install3", "2019/06/12 21:43:16 Cleanup Container Factory", "2019/06/12 21:43:16 unmount and remove dirs [SOCK 3]", "2019/06/12 21:43:16 unmount root dir /root/open-lambda/test-cluster/worker/sock-handler-roots/3 failed :: invalid argument [SOCK 3]", "2019/06/12 21:43:16 Finished Lambda Cleanup", "2019/06/12 21:43:16 exiting", "" ] } POP conf: {'sandbox': 'sock', 'handler_cache_mb': 0, 'import_cache_mb': 0} PUSH conf: {'sandbox': 'sock', 'handler_cache_mb': 256, 'import_cache_mb': 0} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7346, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7346 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 1.7811253070831299, "total_seconds": 9.843267679214478, "stats": null, "conf": { "pip_mirror": "", "docker_runtime": "", "benchmark_log": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "sandbox": "sock", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "import_cache_mb": 0, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "Skip_pull_existing": false, "handler_cache_mb": 256, "sandbox_config": { "processes": 10 }, "max_runners": 0, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "worker_port": "5000" }, "exception": null, "worker_tail": [ "2019/06/12 21:43:26 Function: install3", "2019/06/12 21:43:26 Instance: 3", "2019/06/12 21:43:26 destroy [SOCK 3]", "2019/06/12 21:43:26 Pause/KillAllProcs/Unpause [SOCK 3]", "2019/06/12 21:43:26 wait for init to die [SOCK 3]", "2019/06/12 21:43:26 unmount and remove dirs [SOCK 3]", "2019/06/12 21:43:26 Cleanup Container Factory", "2019/06/12 21:43:26 Finished Lambda Cleanup", "2019/06/12 21:43:26 exiting", "" ] } POP conf: {'sandbox': 'sock', 'handler_cache_mb': 256, 'import_cache_mb': 0} PUSH conf: {'sandbox': 'sock', 'handler_cache_mb': 0, 'import_cache_mb': 256} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7419, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7419 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 0.770606517791748, "total_seconds": 8.726459503173828, "stats": null, "conf": { "pip_mirror": "", "docker_runtime": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "benchmark_log": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "sandbox": "sock", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "worker_port": "5000", "import_cache_mb": 256, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "handler_cache_mb": 0, "sandbox_config": { "processes": 10 }, "max_runners": 0, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "Skip_pull_existing": false }, "exception": null, "worker_tail": [ "2019/06/12 21:43:35 Function: install2", "2019/06/12 21:43:35 Function: install3", "2019/06/12 21:43:35 Function: echo", "2019/06/12 21:43:35 Function: install", "2019/06/12 21:43:35 Cleanup Container Factory", "2019/06/12 21:43:35 unmount and remove dirs [SOCK 3]", "2019/06/12 21:43:35 unmount root dir /root/open-lambda/test-cluster/worker/sock-handler-roots/3 failed :: invalid argument [SOCK 3]", "2019/06/12 21:43:35 Finished Lambda Cleanup", "2019/06/12 21:43:35 exiting", "" ] } POP conf: {'sandbox': 'sock', 'handler_cache_mb': 0, 'import_cache_mb': 256} PUSH conf: {'sandbox': 'sock', 'handler_cache_mb': 256, 'import_cache_mb': 256} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7501, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7501 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 0.7435259819030762, "total_seconds": 8.606566667556763, "stats": null, "conf": { "pip_mirror": "", "docker_runtime": "", "benchmark_log": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "sandbox": "sock", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "import_cache_mb": 256, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "Skip_pull_existing": false, "handler_cache_mb": 256, "sandbox_config": { "processes": 10 }, "max_runners": 0, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "worker_port": "5000" }, "exception": null, "worker_tail": [ "2019/06/12 21:43:43 Function: install3", "2019/06/12 21:43:43 Instance: 3", "2019/06/12 21:43:43 destroy [SOCK 3]", "2019/06/12 21:43:43 Pause/KillAllProcs/Unpause [SOCK 3]", "2019/06/12 21:43:43 wait for init to die [SOCK 3]", "2019/06/12 21:43:43 unmount and remove dirs [SOCK 3]", "2019/06/12 21:43:43 Cleanup Container Factory", "2019/06/12 21:43:43 Finished Lambda Cleanup", "2019/06/12 21:43:43 exiting", "" ] } POP conf: {'sandbox': 'sock', 'handler_cache_mb': 256, 'import_cache_mb': 256} PUSH conf: {'sandbox': 'docker', 'handler_cache_mb': 0, 'import_cache_mb': 0} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7584, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7584 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 7.394480466842651, "total_seconds": 15.041639804840088, "stats": null, "conf": { "pip_mirror": "", "docker_runtime": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "benchmark_log": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "sandbox": "docker", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "worker_port": "5000", "import_cache_mb": 0, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "handler_cache_mb": 0, "sandbox_config": { "processes": 10 }, "max_runners": 0, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "Skip_pull_existing": false }, "exception": null, "worker_tail": [ "2019/06/12 21:43:58 received kill signal, cleaning up", "2019/06/12 21:43:58 Cleanup Lambdas:", "2019/06/12 21:43:58 Function: install", "2019/06/12 21:43:58 Function: install2", "2019/06/12 21:43:58 Function: install3", "2019/06/12 21:43:58 Function: echo", "2019/06/12 21:43:58 Cleanup Container Factory", "2019/06/12 21:43:58 Finished Lambda Cleanup", "2019/06/12 21:43:58 exiting", "" ] } POP conf: {'sandbox': 'docker', 'handler_cache_mb': 0, 'import_cache_mb': 0} PUSH conf: {'sandbox': 'docker', 'handler_cache_mb': 256, 'import_cache_mb': 0} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=8525, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 8525 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 3.3446362018585205, "total_seconds": 11.800937175750732, "stats": null, "conf": { "pip_mirror": "", "docker_runtime": "", "benchmark_log": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "sandbox": "docker", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "import_cache_mb": 0, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "Skip_pull_existing": false, "handler_cache_mb": 256, "sandbox_config": { "processes": 10 }, "max_runners": 0, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "worker_port": "5000" }, "exception": null, "worker_tail": [ "2019/06/12 21:44:09 Function: install", "2019/06/12 21:44:09 Instance: 1", "2019/06/12 21:44:10 Function: install2", "2019/06/12 21:44:10 Instance: 2", "2019/06/12 21:44:10 Function: install3", "2019/06/12 21:44:10 Instance: 3", "2019/06/12 21:44:10 Cleanup Container Factory", "2019/06/12 21:44:10 Finished Lambda Cleanup", "2019/06/12 21:44:10 exiting", "" ] } POP conf: {'sandbox': 'docker', 'handler_cache_mb': 256, 'import_cache_mb': 0} POP conf: {'registry': '/root/open-lambda/test-registry', 'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson']} PUSH conf: {'registry': '/root/open-lambda/test-registry', 'sandbox': 'sock', 'handler_cache_mb': 256, 'import_cache_mb': 256} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=9569, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 9569 { "test": "stress_one_lambda", "params": { "seconds": 15, "procs": 1 }, "pass": true, "seconds": 15.057425022125244, "total_seconds": 15.977617740631104, "stats": { "reqs_per_sec": 279.4 }, "conf": { "pip_mirror": "", "docker_runtime": "", "benchmark_log": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "sandbox": "sock", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "import_cache_mb": 256, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "Skip_pull_existing": false, "handler_cache_mb": 256, "sandbox_config": { "processes": 10 }, "max_runners": 0, "startup_pkgs": null, "worker_port": "5000" }, "exception": null, "worker_tail": [ "2019/06/12 21:44:26 Function: echo", "2019/06/12 21:44:26 Instance: 0", "2019/06/12 21:44:26 destroy [SOCK 0]", "2019/06/12 21:44:26 Pause/KillAllProcs/Unpause [SOCK 0]", "2019/06/12 21:44:26 wait for init to die [SOCK 0]", "2019/06/12 21:44:26 unmount and remove dirs [SOCK 0]", "2019/06/12 21:44:26 Cleanup Container Factory", "2019/06/12 21:44:26 Finished Lambda Cleanup", "2019/06/12 21:44:26 exiting", "" ] } RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=9602, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 9602 { "test": "stress_one_lambda", "params": { "seconds": 15, "procs": 2 }, "pass": true, "seconds": 15.048419952392578, "total_seconds": 15.371788263320923, "stats": { "reqs_per_sec": 274.73333333333335 }, "conf": { "pip_mirror": "", "docker_runtime": "", "benchmark_log": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "sandbox": "sock", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "import_cache_mb": 256, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "Skip_pull_existing": false, "handler_cache_mb": 256, "sandbox_config": { "processes": 10 }, "max_runners": 0, "startup_pkgs": null, "worker_port": "5000" }, "exception": null, "worker_tail": [ "2019/06/12 21:44:41 unmount and remove dirs [SOCK 1]", "2019/06/12 21:44:41 Instance: 0", "2019/06/12 21:44:41 destroy [SOCK 0]", "2019/06/12 21:44:41 Pause/KillAllProcs/Unpause [SOCK 0]", "2019/06/12 21:44:41 wait for init to die [SOCK 0]", "2019/06/12 21:44:41 unmount and remove dirs [SOCK 0]", "2019/06/12 21:44:41 Cleanup Container Factory", "2019/06/12 21:44:41 Finished Lambda Cleanup", "2019/06/12 21:44:41 exiting", "" ] } RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=9643, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 9643 { "test": "stress_one_lambda", "params": { "seconds": 15, "procs": 8 }, "pass": true, "seconds": 15.070520877838135, "total_seconds": 15.402940273284912, "stats": { "reqs_per_sec": 255.0 }, "conf": { "pip_mirror": "", "docker_runtime": "", "benchmark_log": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "sandbox": "sock", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "import_cache_mb": 256, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "Skip_pull_existing": false, "handler_cache_mb": 256, "sandbox_config": { "processes": 10 }, "max_runners": 0, "startup_pkgs": null, "worker_port": "5000" }, "exception": null, "worker_tail": [ "2019/06/12 21:44:57 unmount and remove dirs [SOCK 3]", "2019/06/12 21:44:57 Instance: 0", "2019/06/12 21:44:57 destroy [SOCK 0]", "2019/06/12 21:44:57 Pause/KillAllProcs/Unpause [SOCK 0]", "2019/06/12 21:44:57 wait for init to die [SOCK 0]", "2019/06/12 21:44:57 unmount and remove dirs [SOCK 0]", "2019/06/12 21:44:57 Cleanup Container Factory", "2019/06/12 21:44:57 Finished Lambda Cleanup", "2019/06/12 21:44:57 exiting", "" ] } POP conf: {'registry': '/root/open-lambda/test-registry', 'sandbox': 'sock', 'handler_cache_mb': 256, 'import_cache_mb': 256} PUSH conf: {'sandbox': 'sock', 'handler_cache_mb': 256, 'import_cache_mb': 256} PUSH conf: {'registry': '/tmp/tmpcy1123bv'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=9721, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 9721 { "test": "call_each_once_exec", "params": { "alloc_mb": 1, "lambda_count": 100 }, "pass": true, "seconds": 2.6118412017822266, "total_seconds": 3.1389267444610596, "stats": { "reqs_per_sec": 38.28763173655783 }, "conf": { "pip_mirror": "", "docker_runtime": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "benchmark_log": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/tmp/tmpcy1123bv", "sandbox": "sock", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "worker_port": "5000", "import_cache_mb": 256, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "handler_cache_mb": 256, "sandbox_config": { "processes": 10 }, "max_runners": 0, "startup_pkgs": null, "Skip_pull_existing": false }, "exception": null, "worker_tail": [ "2019/06/12 21:45:00 Function: L45", "2019/06/12 21:45:00 Instance: 45", "2019/06/12 21:45:00 destroy [SOCK 45]", "2019/06/12 21:45:00 Pause/KillAllProcs/Unpause [SOCK 45]", "2019/06/12 21:45:00 wait for init to die [SOCK 45]", "2019/06/12 21:45:00 unmount and remove dirs [SOCK 45]", "2019/06/12 21:45:00 Cleanup Container Factory", "2019/06/12 21:45:00 Finished Lambda Cleanup", "2019/06/12 21:45:00 exiting", "" ] } POP conf: {'registry': '/tmp/tmpcy1123bv'} PUSH conf: {'registry': '/tmp/tmpbfzsb5bq'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=10345, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 10345 { "test": "call_each_once_exec", "params": { "alloc_mb": 10, "lambda_count": 1000 }, "pass": true, "seconds": 40.263399839401245, "total_seconds": 41.70268249511719, "stats": { "reqs_per_sec": 24.836471917126048 }, "conf": { "pip_mirror": "", "docker_runtime": "", "benchmark_log": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/tmp/tmpbfzsb5bq", "sandbox": "sock", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "import_cache_mb": 256, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "Skip_pull_existing": false, "handler_cache_mb": 256, "sandbox_config": { "processes": 10 }, "max_runners": 0, "startup_pkgs": null, "worker_port": "5000" }, "exception": null, "worker_tail": [ "2019/06/12 21:45:42 Function: L782", "2019/06/12 21:45:42 Function: L458", "2019/06/12 21:45:42 Function: L549", "2019/06/12 21:45:42 Function: L896", "2019/06/12 21:45:42 Function: L900", "2019/06/12 21:45:42 Function: L869", "2019/06/12 21:45:42 Cleanup Container Factory", "2019/06/12 21:45:42 Finished Lambda Cleanup", "2019/06/12 21:45:42 exiting", "" ] } POP conf: {'registry': '/tmp/tmpbfzsb5bq'} POP conf: {'sandbox': 'sock', 'handler_cache_mb': 256, 'import_cache_mb': 256} PASSED: 11, FAILED: 0