python3 -u test.py RUN ./ol new -p=test-cluster Init OL dir at /root/open-lambda/test-cluster Create lambda base at /root/open-lambda/test-cluster/lambda (may take several minutes) Working Directory: /root/open-lambda/test-cluster Worker Defaults: { "registry": "/root/open-lambda/test-cluster/registry", "registry_cache_ms": 5000, "sandbox": "sock", "cluster_name": "/root/open-lambda/test-cluster", "pip_mirror": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "max_runners": 0, "handler_cache_mb": 239, "import_cache_mb": 239, "worker_dir": "/root/open-lambda/test-cluster/worker", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "worker_port": "5000", "Skip_pull_existing": false, "sandbox_config": { "processes": 10 }, "benchmark_log": "", "timing": false, "startup_pkgs": null, "docker_runtime": "" } You may now start a server using the "worker" command PUSH conf: {'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson'], 'registry': '/root/open-lambda/test-registry'} PUSH conf: {'sandbox': 'sock', 'handler_cache_mb': 0, 'import_cache_mb': 0} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7299, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7299 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 2.7586820125579834, "total_seconds": 18.117652893066406, "stats": null, "conf": { "benchmark_log": "", "Skip_pull_existing": false, "pip_mirror": "", "registry": "/root/open-lambda/test-registry", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "handler_cache_mb": 0, "max_runners": 0, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "docker_runtime": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "sandbox_config": { "processes": 10 }, "import_cache_mb": 0, "timing": false, "cluster_name": "/root/open-lambda/test-cluster", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "registry_cache_ms": 5000 }, "exception": null, "worker_tail": [ "2019/06/13 21:42:54 received kill signal, cleaning up", "2019/06/13 21:42:54 Cleanup Lambdas:", "2019/06/13 21:42:54 Function: echo", "2019/06/13 21:42:54 Function: install", "2019/06/13 21:42:54 Function: install2", "2019/06/13 21:42:54 Function: install3", "2019/06/13 21:42:54 Cleanup Container Factory", "2019/06/13 21:42:54 Finished Lambda Cleanup", "2019/06/13 21:42:54 exiting", "" ] } POP conf: {'sandbox': 'sock', 'handler_cache_mb': 0, 'import_cache_mb': 0} PUSH conf: {'sandbox': 'sock', 'handler_cache_mb': 256, 'import_cache_mb': 0} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7384, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7384 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 1.7260169982910156, "total_seconds": 9.592583179473877, "stats": null, "conf": { "benchmark_log": "", "Skip_pull_existing": false, "pip_mirror": "", "registry": "/root/open-lambda/test-registry", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "handler_cache_mb": 256, "max_runners": 0, "sandbox_config": { "processes": 10 }, "docker_runtime": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "import_cache_mb": 0, "timing": false, "cluster_name": "/root/open-lambda/test-cluster", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "registry_cache_ms": 5000 }, "exception": null, "worker_tail": [ "2019/06/13 21:43:03 Function: install3", "2019/06/13 21:43:03 Instance: 3", "2019/06/13 21:43:03 destroy [SOCK 3]", "2019/06/13 21:43:03 Pause/KillAllProcs/Unpause [SOCK 3]", "2019/06/13 21:43:03 wait for init to die [SOCK 3]", "2019/06/13 21:43:03 unmount and remove dirs [SOCK 3]", "2019/06/13 21:43:03 Cleanup Container Factory", "2019/06/13 21:43:03 Finished Lambda Cleanup", "2019/06/13 21:43:03 exiting", "" ] } POP conf: {'sandbox': 'sock', 'handler_cache_mb': 256, 'import_cache_mb': 0} PUSH conf: {'sandbox': 'sock', 'handler_cache_mb': 0, 'import_cache_mb': 256} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7457, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7457 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 0.7707171440124512, "total_seconds": 8.627728700637817, "stats": null, "conf": { "benchmark_log": "", "Skip_pull_existing": false, "pip_mirror": "", "registry": "/root/open-lambda/test-registry", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "handler_cache_mb": 0, "max_runners": 0, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "docker_runtime": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "sandbox_config": { "processes": 10 }, "import_cache_mb": 256, "timing": false, "cluster_name": "/root/open-lambda/test-cluster", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "registry_cache_ms": 5000 }, "exception": null, "worker_tail": [ "2019/06/13 21:43:12 Function: install", "2019/06/13 21:43:12 Function: install2", "2019/06/13 21:43:12 Function: install3", "2019/06/13 21:43:12 Function: echo", "2019/06/13 21:43:12 Cleanup Container Factory", "2019/06/13 21:43:12 unmount and remove dirs [SOCK 3]", "2019/06/13 21:43:12 unmount root dir /root/open-lambda/test-cluster/worker/sock-handler-roots/3 failed :: invalid argument [SOCK 3]", "2019/06/13 21:43:12 Finished Lambda Cleanup", "2019/06/13 21:43:12 exiting", "" ] } POP conf: {'sandbox': 'sock', 'handler_cache_mb': 0, 'import_cache_mb': 256} PUSH conf: {'sandbox': 'sock', 'handler_cache_mb': 256, 'import_cache_mb': 256} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7540, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7540 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 0.7560052871704102, "total_seconds": 8.722227334976196, "stats": null, "conf": { "benchmark_log": "", "Skip_pull_existing": false, "pip_mirror": "", "registry": "/root/open-lambda/test-registry", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "handler_cache_mb": 256, "max_runners": 0, "sandbox_config": { "processes": 10 }, "docker_runtime": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "import_cache_mb": 256, "timing": false, "cluster_name": "/root/open-lambda/test-cluster", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "registry_cache_ms": 5000 }, "exception": null, "worker_tail": [ "2019/06/13 21:43:21 Function: install3", "2019/06/13 21:43:21 Instance: 3", "2019/06/13 21:43:21 destroy [SOCK 3]", "2019/06/13 21:43:21 Pause/KillAllProcs/Unpause [SOCK 3]", "2019/06/13 21:43:21 wait for init to die [SOCK 3]", "2019/06/13 21:43:21 unmount and remove dirs [SOCK 3]", "2019/06/13 21:43:21 Cleanup Container Factory", "2019/06/13 21:43:21 Finished Lambda Cleanup", "2019/06/13 21:43:21 exiting", "" ] } POP conf: {'sandbox': 'sock', 'handler_cache_mb': 256, 'import_cache_mb': 256} PUSH conf: {'sandbox': 'docker', 'handler_cache_mb': 0, 'import_cache_mb': 0} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=7622, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 7622 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 7.482460260391235, "total_seconds": 15.23724913597107, "stats": null, "conf": { "benchmark_log": "", "Skip_pull_existing": false, "pip_mirror": "", "registry": "/root/open-lambda/test-registry", "sandbox": "docker", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "handler_cache_mb": 0, "max_runners": 0, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "docker_runtime": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "sandbox_config": { "processes": 10 }, "import_cache_mb": 0, "timing": false, "cluster_name": "/root/open-lambda/test-cluster", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "registry_cache_ms": 5000 }, "exception": null, "worker_tail": [ "2019/06/13 21:43:36 received kill signal, cleaning up", "2019/06/13 21:43:36 Cleanup Lambdas:", "2019/06/13 21:43:36 Function: install3", "2019/06/13 21:43:36 Function: echo", "2019/06/13 21:43:36 Function: install", "2019/06/13 21:43:36 Function: install2", "2019/06/13 21:43:36 Cleanup Container Factory", "2019/06/13 21:43:36 Finished Lambda Cleanup", "2019/06/13 21:43:36 exiting", "" ] } POP conf: {'sandbox': 'docker', 'handler_cache_mb': 0, 'import_cache_mb': 0} PUSH conf: {'sandbox': 'docker', 'handler_cache_mb': 256, 'import_cache_mb': 0} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=8559, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 8559 { "test": "smoke_tests", "params": {}, "pass": true, "seconds": 7.598080396652222, "total_seconds": 16.3803231716156, "stats": null, "conf": { "benchmark_log": "", "Skip_pull_existing": false, "pip_mirror": "", "registry": "/root/open-lambda/test-registry", "sandbox": "docker", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "handler_cache_mb": 256, "max_runners": 0, "sandbox_config": { "processes": 10 }, "docker_runtime": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "import_cache_mb": 0, "timing": false, "cluster_name": "/root/open-lambda/test-cluster", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "registry_cache_ms": 5000 }, "exception": null, "worker_tail": [ "2019/06/13 21:43:52 Function: install2", "2019/06/13 21:43:52 Instance: 2", "2019/06/13 21:43:52 Function: install3", "2019/06/13 21:43:52 Instance: 3", "2019/06/13 21:43:52 Function: echo", "2019/06/13 21:43:52 Instance: 0", "2019/06/13 21:43:52 Cleanup Container Factory", "2019/06/13 21:43:52 Finished Lambda Cleanup", "2019/06/13 21:43:52 exiting", "" ] } POP conf: {'sandbox': 'docker', 'handler_cache_mb': 256, 'import_cache_mb': 0} POP conf: {'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson'], 'registry': '/root/open-lambda/test-registry'} PUSH conf: {'sandbox': 'sock', 'handler_cache_mb': 256, 'import_cache_mb': 256, 'registry': '/root/open-lambda/test-registry'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=9606, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 9606 { "test": "stress_one_lambda", "params": { "procs": 1, "seconds": 15 }, "pass": true, "seconds": 15.014233112335205, "total_seconds": 15.936209917068481, "stats": { "reqs_per_sec": 284.53333333333336 }, "conf": { "benchmark_log": "", "Skip_pull_existing": false, "pip_mirror": "", "registry": "/root/open-lambda/test-registry", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "handler_cache_mb": 256, "max_runners": 0, "sandbox_config": { "processes": 10 }, "docker_runtime": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "startup_pkgs": null, "import_cache_mb": 256, "timing": false, "cluster_name": "/root/open-lambda/test-cluster", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "registry_cache_ms": 5000 }, "exception": null, "worker_tail": [ "2019/06/13 21:44:08 Function: echo", "2019/06/13 21:44:08 Instance: 0", "2019/06/13 21:44:08 destroy [SOCK 0]", "2019/06/13 21:44:08 Pause/KillAllProcs/Unpause [SOCK 0]", "2019/06/13 21:44:08 wait for init to die [SOCK 0]", "2019/06/13 21:44:08 unmount and remove dirs [SOCK 0]", "2019/06/13 21:44:08 Cleanup Container Factory", "2019/06/13 21:44:08 Finished Lambda Cleanup", "2019/06/13 21:44:08 exiting", "" ] } RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=9640, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 9640 { "test": "stress_one_lambda", "params": { "procs": 2, "seconds": 15 }, "pass": true, "seconds": 15.047487258911133, "total_seconds": 15.37291407585144, "stats": { "reqs_per_sec": 276.6 }, "conf": { "benchmark_log": "", "Skip_pull_existing": false, "pip_mirror": "", "registry": "/root/open-lambda/test-registry", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "handler_cache_mb": 256, "max_runners": 0, "sandbox_config": { "processes": 10 }, "docker_runtime": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "startup_pkgs": null, "import_cache_mb": 256, "timing": false, "cluster_name": "/root/open-lambda/test-cluster", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "registry_cache_ms": 5000 }, "exception": null, "worker_tail": [ "2019/06/13 21:44:24 unmount and remove dirs [SOCK 1]", "2019/06/13 21:44:24 Instance: 0", "2019/06/13 21:44:24 destroy [SOCK 0]", "2019/06/13 21:44:24 Pause/KillAllProcs/Unpause [SOCK 0]", "2019/06/13 21:44:24 wait for init to die [SOCK 0]", "2019/06/13 21:44:24 unmount and remove dirs [SOCK 0]", "2019/06/13 21:44:24 Cleanup Container Factory", "2019/06/13 21:44:24 Finished Lambda Cleanup", "2019/06/13 21:44:24 exiting", "" ] } RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=9682, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 9682 { "test": "stress_one_lambda", "params": { "procs": 8, "seconds": 15 }, "pass": true, "seconds": 15.070842266082764, "total_seconds": 15.402170419692993, "stats": { "reqs_per_sec": 243.53333333333333 }, "conf": { "benchmark_log": "", "Skip_pull_existing": false, "pip_mirror": "", "registry": "/root/open-lambda/test-registry", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "handler_cache_mb": 256, "max_runners": 0, "sandbox_config": { "processes": 10 }, "docker_runtime": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "startup_pkgs": null, "import_cache_mb": 256, "timing": false, "cluster_name": "/root/open-lambda/test-cluster", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "registry_cache_ms": 5000 }, "exception": null, "worker_tail": [ "2019/06/13 21:44:39 unmount and remove dirs [SOCK 5]", "2019/06/13 21:44:39 Instance: 0", "2019/06/13 21:44:39 destroy [SOCK 0]", "2019/06/13 21:44:39 Pause/KillAllProcs/Unpause [SOCK 0]", "2019/06/13 21:44:39 wait for init to die [SOCK 0]", "2019/06/13 21:44:39 unmount and remove dirs [SOCK 0]", "2019/06/13 21:44:39 Cleanup Container Factory", "2019/06/13 21:44:39 Finished Lambda Cleanup", "2019/06/13 21:44:39 exiting", "" ] } POP conf: {'sandbox': 'sock', 'handler_cache_mb': 256, 'import_cache_mb': 256, 'registry': '/root/open-lambda/test-registry'} PUSH conf: {'sandbox': 'sock', 'handler_cache_mb': 256, 'import_cache_mb': 256} PUSH conf: {'registry': '/tmp/tmppfa9igb9'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=9760, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 9760 { "test": "call_each_once_exec", "params": { "alloc_mb": 1, "lambda_count": 100 }, "pass": true, "seconds": 2.84086537361145, "total_seconds": 3.3798370361328125, "stats": { "reqs_per_sec": 35.2009523206858 }, "conf": { "benchmark_log": "", "Skip_pull_existing": false, "pip_mirror": "", "registry": "/tmp/tmppfa9igb9", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "handler_cache_mb": 256, "max_runners": 0, "startup_pkgs": null, "docker_runtime": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "sandbox_config": { "processes": 10 }, "import_cache_mb": 256, "timing": false, "cluster_name": "/root/open-lambda/test-cluster", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "registry_cache_ms": 5000 }, "exception": null, "worker_tail": [ "2019/06/13 21:44:42 Function: L72", "2019/06/13 21:44:42 Instance: 72", "2019/06/13 21:44:42 destroy [SOCK 72]", "2019/06/13 21:44:42 Pause/KillAllProcs/Unpause [SOCK 72]", "2019/06/13 21:44:42 wait for init to die [SOCK 72]", "2019/06/13 21:44:42 unmount and remove dirs [SOCK 72]", "2019/06/13 21:44:42 Cleanup Container Factory", "2019/06/13 21:44:42 Finished Lambda Cleanup", "2019/06/13 21:44:42 exiting", "" ] } POP conf: {'registry': '/tmp/tmppfa9igb9'} PUSH conf: {'registry': '/tmp/tmpcwjuk3rt'} RUN ./ol worker -p=test-cluster --detach using existing OL directory at /root/open-lambda/test-cluster Starting worker: pid=10386, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./ol kill -p=test-cluster Kill worker process with PID 10386 { "test": "call_each_once_exec", "params": { "alloc_mb": 10, "lambda_count": 1000 }, "pass": true, "seconds": 45.14904737472534, "total_seconds": 46.388774156570435, "stats": { "reqs_per_sec": 22.14889314255756 }, "conf": { "benchmark_log": "", "Skip_pull_existing": false, "pip_mirror": "", "registry": "/tmp/tmpcwjuk3rt", "sandbox": "sock", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "handler_cache_mb": 256, "max_runners": 0, "sandbox_config": { "processes": 10 }, "docker_runtime": "", "worker_dir": "/root/open-lambda/test-cluster/worker", "startup_pkgs": null, "import_cache_mb": 256, "timing": false, "cluster_name": "/root/open-lambda/test-cluster", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "worker_port": "5000", "registry_cache_ms": 5000 }, "exception": null, "worker_tail": [ "2019/06/13 21:45:29 Function: L955", "2019/06/13 21:45:29 Function: L104", "2019/06/13 21:45:29 Function: L624", "2019/06/13 21:45:29 Function: L232", "2019/06/13 21:45:29 Function: L462", "2019/06/13 21:45:29 Function: L922", "2019/06/13 21:45:29 Cleanup Container Factory", "2019/06/13 21:45:29 Finished Lambda Cleanup", "2019/06/13 21:45:29 exiting", "" ] } POP conf: {'registry': '/tmp/tmpcwjuk3rt'} POP conf: {'sandbox': 'sock', 'handler_cache_mb': 256, 'import_cache_mb': 256} PASSED: 11, FAILED: 0