python3 test.py Init OL dir at /root/open-lambda/test-cluster Create lambda base at /root/open-lambda/test-cluster/lambda (may take several minutes) Working Directory: /root/open-lambda/test-cluster Worker Defaults: { "registry": "/root/open-lambda/test-cluster/registry", "registry_cache_ms": 5000, "sandbox": "sock", "cluster_name": "/root/open-lambda/test-cluster", "pip_mirror": "", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "max_runners": 0, "handler_cache_size": 0, "import_cache_size": 0, "worker_dir": "/root/open-lambda/test-cluster/worker", "SOCK_base_path": "/root/open-lambda/test-cluster/lambda", "worker_port": "5000", "cg_pool_size": 0, "Skip_pull_existing": false, "sandbox_config": { "processes": 10 }, "benchmark_log": "", "timing": false, "startup_pkgs": null, "docker_runtime": "" } You may now start a server using the "worker" command Starting worker: pid=7299, port=5000, log=/root/open-lambda/test-cluster/worker.out ready RUN ./bin/ol new -p=test-cluster PUSH conf: {'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson'], 'registry': '/root/open-lambda/test-registry'} PUSH conf: {'cg_pool_size': 10, 'sandbox': 'sock', 'handler_cache_size': 0, 'import_cache_size': 0} RUN ./bin/ol worker -p=test-cluster --detach { "test": "test_smoke_echo", "params": {}, "pass": true, "seconds": 2.470813035964966, "stats": null, "conf": { "cg_pool_size": 10, "worker_port": "5000", "max_runners": 0, "sandbox": "sock", "docker_runtime": "", "import_cache_size": 0, "handler_cache_size": 0, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:44:26 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:0 Import_cache_size:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:44:26 Initialized registry manager (took 38.272\u00b5s)", "2019/06/03 21:44:43 Initialized installation manager (took 17.253180024s)", "2019/06/03 21:44:43 Initialized handler container factory (took 6.771171ms)", "2019/06/03 21:44:43 Initialized cache manager (took 1.325\u00b5s)", "2019/06/03 21:44:43 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:44:43 Get status by sending request to localhost:5000/status", "2019/06/03 21:44:43 Receive request to /pid", "2019/06/03 21:44:43 Receive request to /run/echo", "2019/06/03 21:44:46 handler hits: 0, import hits: 0, misses: 0", "" ] } { "test": "test_smoke_install", "params": { "num": null }, "pass": true, "seconds": 0.766277551651001, "stats": null, "conf": { "cg_pool_size": 10, "worker_port": "5000", "max_runners": 0, "sandbox": "sock", "docker_runtime": "", "import_cache_size": 0, "handler_cache_size": 0, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:44:26 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:0 Import_cache_size:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:44:26 Initialized registry manager (took 38.272\u00b5s)", "2019/06/03 21:44:43 Initialized installation manager (took 17.253180024s)", "2019/06/03 21:44:43 Initialized handler container factory (took 6.771171ms)", "2019/06/03 21:44:43 Initialized cache manager (took 1.325\u00b5s)", "2019/06/03 21:44:43 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:44:43 Get status by sending request to localhost:5000/status", "2019/06/03 21:44:43 Receive request to /pid", "2019/06/03 21:44:43 Receive request to /run/echo", "2019/06/03 21:44:46 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:44:46 Receive request to /run/install", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:44:46 handler hits: 0, import hits: 0, misses: 0", "" ] } { "test": "test_smoke_install", "params": { "num": 2 }, "pass": true, "seconds": 0.3641226291656494, "stats": null, "conf": { "cg_pool_size": 10, "worker_port": "5000", "max_runners": 0, "sandbox": "sock", "docker_runtime": "", "import_cache_size": 0, "handler_cache_size": 0, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:44:26 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:0 Import_cache_size:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:44:26 Initialized registry manager (took 38.272\u00b5s)", "2019/06/03 21:44:43 Initialized installation manager (took 17.253180024s)", "2019/06/03 21:44:43 Initialized handler container factory (took 6.771171ms)", "2019/06/03 21:44:43 Initialized cache manager (took 1.325\u00b5s)", "2019/06/03 21:44:43 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:44:43 Get status by sending request to localhost:5000/status", "2019/06/03 21:44:43 Receive request to /pid", "2019/06/03 21:44:43 Receive request to /run/echo", "2019/06/03 21:44:46 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:44:46 Receive request to /run/install", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:44:46 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:44:46 Receive request to /run/install2", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:44:47 handler hits: 0, import hits: 0, misses: 0", "" ] } { "test": "test_smoke_install", "params": { "num": 3 }, "pass": true, "seconds": 0.39139270782470703, "stats": null, "conf": { "cg_pool_size": 10, "worker_port": "5000", "max_runners": 0, "sandbox": "sock", "docker_runtime": "", "import_cache_size": 0, "handler_cache_size": 0, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:44:26 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:0 Import_cache_size:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:44:26 Initialized registry manager (took 38.272\u00b5s)", "2019/06/03 21:44:43 Initialized installation manager (took 17.253180024s)", "2019/06/03 21:44:43 Initialized handler container factory (took 6.771171ms)", "2019/06/03 21:44:43 Initialized cache manager (took 1.325\u00b5s)", "2019/06/03 21:44:43 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:44:43 Get status by sending request to localhost:5000/status", "2019/06/03 21:44:43 Receive request to /pid", "2019/06/03 21:44:43 Receive request to /run/echo", "2019/06/03 21:44:46 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:44:46 Receive request to /run/install", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:44:46 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:44:46 Receive request to /run/install2", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:44:47 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:44:47 Receive request to /run/install3", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:44:47 handler hits: 0, import hits: 0, misses: 0", "" ] }Kill worker process with PID 7299 Starting worker: pid=7387, port=5000, log=/root/open-lambda/test-cluster/worker.out ready POP conf: {'cg_pool_size': 10, 'sandbox': 'sock', 'handler_cache_size': 0, 'import_cache_size': 0} RUN ./bin/ol kill -p=test-cluster PUSH conf: {'cg_pool_size': 10, 'sandbox': 'sock', 'handler_cache_size': 10000000, 'import_cache_size': 0} RUN ./bin/ol worker -p=test-cluster --detach { "test": "test_smoke_echo", "params": {}, "pass": true, "seconds": 0.14528965950012207, "stats": null, "conf": { "cg_pool_size": 10, "worker_port": "5000", "max_runners": 0, "sandbox": "sock", "docker_runtime": "", "import_cache_size": 0, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "handler_cache_size": 10000000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:44:48 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:10000000 Import_cache_size:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:44:48 Initialized registry manager (took 16.132\u00b5s)", "2019/06/03 21:44:57 Initialized installation manager (took 8.290269992s)", "2019/06/03 21:44:57 Initialized handler container factory (took 3.338122ms)", "2019/06/03 21:44:57 Initialized cache manager (took 798ns)", "2019/06/03 21:44:57 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:44:57 Get status by sending request to localhost:5000/status", "2019/06/03 21:44:57 Receive request to /pid", "2019/06/03 21:44:57 Receive request to /run/echo", "2019/06/03 21:44:57 handler hits: 0, import hits: 0, misses: 0", "" ] } { "test": "test_smoke_install", "params": { "num": null }, "pass": true, "seconds": 0.35691070556640625, "stats": null, "conf": { "cg_pool_size": 10, "worker_port": "5000", "max_runners": 0, "sandbox": "sock", "docker_runtime": "", "import_cache_size": 0, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "handler_cache_size": 10000000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:44:48 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:10000000 Import_cache_size:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:44:48 Initialized registry manager (took 16.132\u00b5s)", "2019/06/03 21:44:57 Initialized installation manager (took 8.290269992s)", "2019/06/03 21:44:57 Initialized handler container factory (took 3.338122ms)", "2019/06/03 21:44:57 Initialized cache manager (took 798ns)", "2019/06/03 21:44:57 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:44:57 Get status by sending request to localhost:5000/status", "2019/06/03 21:44:57 Receive request to /pid", "2019/06/03 21:44:57 Receive request to /run/echo", "2019/06/03 21:44:57 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:44:57 Receive request to /run/install", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:44:57 handler hits: 0, import hits: 0, misses: 0", "" ] } { "test": "test_smoke_install", "params": { "num": 2 }, "pass": true, "seconds": 0.35764122009277344, "stats": null, "conf": { "cg_pool_size": 10, "worker_port": "5000", "max_runners": 0, "sandbox": "sock", "docker_runtime": "", "import_cache_size": 0, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "handler_cache_size": 10000000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:44:48 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:10000000 Import_cache_size:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:44:48 Initialized registry manager (took 16.132\u00b5s)", "2019/06/03 21:44:57 Initialized installation manager (took 8.290269992s)", "2019/06/03 21:44:57 Initialized handler container factory (took 3.338122ms)", "2019/06/03 21:44:57 Initialized cache manager (took 798ns)", "2019/06/03 21:44:57 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:44:57 Get status by sending request to localhost:5000/status", "2019/06/03 21:44:57 Receive request to /pid", "2019/06/03 21:44:57 Receive request to /run/echo", "2019/06/03 21:44:57 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:44:57 Receive request to /run/install", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:44:57 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:44:57 Receive request to /run/install2", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:44:58 handler hits: 0, import hits: 0, misses: 0", "" ] } { "test": "test_smoke_install", "params": { "num": 3 }, "pass": true, "seconds": 1.0005245208740234, "stats": null, "conf": { "cg_pool_size": 10, "worker_port": "5000", "max_runners": 0, "sandbox": "sock", "docker_runtime": "", "import_cache_size": 0, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "handler_cache_size": 10000000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:44:48 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:10000000 Import_cache_size:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:44:48 Initialized registry manager (took 16.132\u00b5s)", "2019/06/03 21:44:57 Initialized installation manager (took 8.290269992s)", "2019/06/03 21:44:57 Initialized handler container factory (took 3.338122ms)", "2019/06/03 21:44:57 Initialized cache manager (took 798ns)", "2019/06/03 21:44:57 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:44:57 Get status by sending request to localhost:5000/status", "2019/06/03 21:44:57 Receive request to /pid", "2019/06/03 21:44:57 Receive request to /run/echo", "2019/06/03 21:44:57 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:44:57 Receive request to /run/install", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:44:57 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:44:57 Receive request to /run/install2", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:44:58 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:44:58 Receive request to /run/install3", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:44:59 handler hits: 0, import hits: 0, misses: 0", "" ] }Kill worker process with PID 7387 Starting worker: pid=7463, port=5000, log=/root/open-lambda/test-cluster/worker.out ready POP conf: {'cg_pool_size': 10, 'sandbox': 'sock', 'handler_cache_size': 10000000, 'import_cache_size': 0} RUN ./bin/ol kill -p=test-cluster PUSH conf: {'cg_pool_size': 10, 'sandbox': 'sock', 'handler_cache_size': 0, 'import_cache_size': 10000000} RUN ./bin/ol worker -p=test-cluster --detach { "test": "test_smoke_echo", "params": {}, "pass": true, "seconds": 0.020372390747070312, "stats": null, "conf": { "cg_pool_size": 10, "worker_port": "5000", "max_runners": 0, "sandbox": "sock", "docker_runtime": "", "import_cache_size": 10000000, "handler_cache_size": 0, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:44:59 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:0 Import_cache_size:10000000 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:44:59 Initialized registry manager (took 18.533\u00b5s)", "2019/06/03 21:45:06 Initialized installation manager (took 7.527218415s)", "2019/06/03 21:45:06 Initialized handler container factory (took 3.684034ms)", "2019/06/03 21:45:06 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all", "2019/06/03 21:45:06 Initialized cache manager (took 173.66882ms)", "2019/06/03 21:45:06 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:45:06 Get status by sending request to localhost:5000/status", "2019/06/03 21:45:06 Receive request to /pid", "2019/06/03 21:45:06 Receive request to /run/echo", "2019/06/03 21:45:06 handler hits: 0, import hits: 0, misses: 1", "" ] } { "test": "test_smoke_install", "params": { "num": null }, "pass": true, "seconds": 0.32607221603393555, "stats": null, "conf": { "cg_pool_size": 10, "worker_port": "5000", "max_runners": 0, "sandbox": "sock", "docker_runtime": "", "import_cache_size": 10000000, "handler_cache_size": 0, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:44:59 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:0 Import_cache_size:10000000 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:44:59 Initialized registry manager (took 18.533\u00b5s)", "2019/06/03 21:45:06 Initialized installation manager (took 7.527218415s)", "2019/06/03 21:45:06 Initialized handler container factory (took 3.684034ms)", "2019/06/03 21:45:06 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all", "2019/06/03 21:45:06 Initialized cache manager (took 173.66882ms)", "2019/06/03 21:45:06 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:45:06 Get status by sending request to localhost:5000/status", "2019/06/03 21:45:06 Receive request to /pid", "2019/06/03 21:45:06 Receive request to /run/echo", "2019/06/03 21:45:06 handler hits: 0, import hits: 0, misses: 1", "2019/06/03 21:45:06 Receive request to /run/install", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:45:07 handler hits: 0, import hits: 0, misses: 2", "" ] } { "test": "test_smoke_install", "params": { "num": 2 }, "pass": true, "seconds": 0.2411785125732422, "stats": null, "conf": { "cg_pool_size": 10, "worker_port": "5000", "max_runners": 0, "sandbox": "sock", "docker_runtime": "", "import_cache_size": 10000000, "handler_cache_size": 0, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:44:59 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:0 Import_cache_size:10000000 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:44:59 Initialized registry manager (took 18.533\u00b5s)", "2019/06/03 21:45:06 Initialized installation manager (took 7.527218415s)", "2019/06/03 21:45:06 Initialized handler container factory (took 3.684034ms)", "2019/06/03 21:45:06 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all", "2019/06/03 21:45:06 Initialized cache manager (took 173.66882ms)", "2019/06/03 21:45:06 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:45:06 Get status by sending request to localhost:5000/status", "2019/06/03 21:45:06 Receive request to /pid", "2019/06/03 21:45:06 Receive request to /run/echo", "2019/06/03 21:45:06 handler hits: 0, import hits: 0, misses: 1", "2019/06/03 21:45:06 Receive request to /run/install", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:45:07 handler hits: 0, import hits: 0, misses: 2", "2019/06/03 21:45:07 Receive request to /run/install2", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:45:07 handler hits: 0, import hits: 0, misses: 3", "" ] }Kill worker process with PID 7463 Starting worker: pid=7547, port=5000, log=/root/open-lambda/test-cluster/worker.out ready { "test": "test_smoke_install", "params": { "num": 3 }, "pass": true, "seconds": 0.24205636978149414, "stats": null, "conf": { "cg_pool_size": 10, "worker_port": "5000", "max_runners": 0, "sandbox": "sock", "docker_runtime": "", "import_cache_size": 10000000, "handler_cache_size": 0, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:44:59 Initialized registry manager (took 18.533\u00b5s)", "2019/06/03 21:45:06 Initialized installation manager (took 7.527218415s)", "2019/06/03 21:45:06 Initialized handler container factory (took 3.684034ms)", "2019/06/03 21:45:06 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all", "2019/06/03 21:45:06 Initialized cache manager (took 173.66882ms)", "2019/06/03 21:45:06 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:45:06 Get status by sending request to localhost:5000/status", "2019/06/03 21:45:06 Receive request to /pid", "2019/06/03 21:45:06 Receive request to /run/echo", "2019/06/03 21:45:06 handler hits: 0, import hits: 0, misses: 1", "2019/06/03 21:45:06 Receive request to /run/install", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:45:07 handler hits: 0, import hits: 0, misses: 2", "2019/06/03 21:45:07 Receive request to /run/install2", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:45:07 handler hits: 0, import hits: 0, misses: 3", "2019/06/03 21:45:07 Receive request to /run/install3", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:45:07 handler hits: 0, import hits: 0, misses: 4", "" ] } POP conf: {'cg_pool_size': 10, 'sandbox': 'sock', 'handler_cache_size': 0, 'import_cache_size': 10000000} RUN ./bin/ol kill -p=test-cluster PUSH conf: {'cg_pool_size': 10, 'sandbox': 'sock', 'handler_cache_size': 10000000, 'import_cache_size': 10000000} RUN ./bin/ol worker -p=test-cluster --detach { "test": "test_smoke_echo", "params": {}, "pass": true, "seconds": 0.020642518997192383, "stats": null, "conf": { "cg_pool_size": 10, "worker_port": "5000", "max_runners": 0, "sandbox": "sock", "docker_runtime": "", "import_cache_size": 10000000, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "handler_cache_size": 10000000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:45:07 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:10000000 Import_cache_size:10000000 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:45:07 Initialized registry manager (took 17.197\u00b5s)", "2019/06/03 21:45:15 Initialized installation manager (took 7.542124963s)", "2019/06/03 21:45:15 Initialized handler container factory (took 3.345746ms)", "2019/06/03 21:45:15 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all", "2019/06/03 21:45:15 Initialized cache manager (took 142.713141ms)", "2019/06/03 21:45:15 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:45:15 Get status by sending request to localhost:5000/status", "2019/06/03 21:45:15 Receive request to /pid", "2019/06/03 21:45:15 Receive request to /run/echo", "2019/06/03 21:45:15 handler hits: 0, import hits: 0, misses: 1", "" ] } { "test": "test_smoke_install", "params": { "num": null }, "pass": true, "seconds": 0.23781514167785645, "stats": null, "conf": { "cg_pool_size": 10, "worker_port": "5000", "max_runners": 0, "sandbox": "sock", "docker_runtime": "", "import_cache_size": 10000000, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "handler_cache_size": 10000000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:45:07 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:10000000 Import_cache_size:10000000 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:45:07 Initialized registry manager (took 17.197\u00b5s)", "2019/06/03 21:45:15 Initialized installation manager (took 7.542124963s)", "2019/06/03 21:45:15 Initialized handler container factory (took 3.345746ms)", "2019/06/03 21:45:15 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all", "2019/06/03 21:45:15 Initialized cache manager (took 142.713141ms)", "2019/06/03 21:45:15 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:45:15 Get status by sending request to localhost:5000/status", "2019/06/03 21:45:15 Receive request to /pid", "2019/06/03 21:45:15 Receive request to /run/echo", "2019/06/03 21:45:15 handler hits: 0, import hits: 0, misses: 1", "2019/06/03 21:45:15 Receive request to /run/install", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:45:15 handler hits: 0, import hits: 0, misses: 2", "" ] } { "test": "test_smoke_install", "params": { "num": 2 }, "pass": true, "seconds": 0.23859477043151855, "stats": null, "conf": { "cg_pool_size": 10, "worker_port": "5000", "max_runners": 0, "sandbox": "sock", "docker_runtime": "", "import_cache_size": 10000000, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "handler_cache_size": 10000000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:45:07 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:10000000 Import_cache_size:10000000 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:45:07 Initialized registry manager (took 17.197\u00b5s)", "2019/06/03 21:45:15 Initialized installation manager (took 7.542124963s)", "2019/06/03 21:45:15 Initialized handler container factory (took 3.345746ms)", "2019/06/03 21:45:15 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all", "2019/06/03 21:45:15 Initialized cache manager (took 142.713141ms)", "2019/06/03 21:45:15 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:45:15 Get status by sending request to localhost:5000/status", "2019/06/03 21:45:15 Receive request to /pid", "2019/06/03 21:45:15 Receive request to /run/echo", "2019/06/03 21:45:15 handler hits: 0, import hits: 0, misses: 1", "2019/06/03 21:45:15 Receive request to /run/install", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:45:15 handler hits: 0, import hits: 0, misses: 2", "2019/06/03 21:45:15 Receive request to /run/install2", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:45:16 handler hits: 0, import hits: 0, misses: 3", "" ] }Kill worker process with PID 7547 Starting worker: pid=7631, port=5000, log=/root/open-lambda/test-cluster/worker.out ready { "test": "test_smoke_install", "params": { "num": 3 }, "pass": true, "seconds": 0.23825454711914062, "stats": null, "conf": { "cg_pool_size": 10, "worker_port": "5000", "max_runners": 0, "sandbox": "sock", "docker_runtime": "", "import_cache_size": 10000000, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "handler_cache_size": 10000000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:45:07 Initialized registry manager (took 17.197\u00b5s)", "2019/06/03 21:45:15 Initialized installation manager (took 7.542124963s)", "2019/06/03 21:45:15 Initialized handler container factory (took 3.345746ms)", "2019/06/03 21:45:15 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all", "2019/06/03 21:45:15 Initialized cache manager (took 142.713141ms)", "2019/06/03 21:45:15 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:45:15 Get status by sending request to localhost:5000/status", "2019/06/03 21:45:15 Receive request to /pid", "2019/06/03 21:45:15 Receive request to /run/echo", "2019/06/03 21:45:15 handler hits: 0, import hits: 0, misses: 1", "2019/06/03 21:45:15 Receive request to /run/install", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:45:15 handler hits: 0, import hits: 0, misses: 2", "2019/06/03 21:45:15 Receive request to /run/install2", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:45:16 handler hits: 0, import hits: 0, misses: 3", "2019/06/03 21:45:16 Receive request to /run/install3", "/bin/sh: 1: cannot create /dev/null: Read-only file system", "2019/06/03 21:45:16 handler hits: 0, import hits: 0, misses: 4", "" ] } POP conf: {'cg_pool_size': 10, 'sandbox': 'sock', 'handler_cache_size': 10000000, 'import_cache_size': 10000000} RUN ./bin/ol kill -p=test-cluster PUSH conf: {'cg_pool_size': 0, 'sandbox': 'docker', 'handler_cache_size': 0, 'import_cache_size': 0} RUN ./bin/ol worker -p=test-cluster --detach { "test": "test_smoke_echo", "params": {}, "pass": true, "seconds": 3.5824034214019775, "stats": null, "conf": { "cg_pool_size": 0, "worker_port": "5000", "max_runners": 0, "sandbox": "docker", "docker_runtime": "", "import_cache_size": 0, "handler_cache_size": 0, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:45:16 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:0 Import_cache_size:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:45:16 Initialized registry manager (took 17.535\u00b5s)", "2019/06/03 21:45:24 Initialized installation manager (took 7.525046459s)", "2019/06/03 21:45:24 Initialized handler container factory (took 26.289\u00b5s)", "2019/06/03 21:45:24 Initialized cache manager (took 712ns)", "2019/06/03 21:45:24 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:45:24 Get status by sending request to localhost:5000/status", "2019/06/03 21:45:24 Receive request to /pid", "2019/06/03 21:45:24 Receive request to /run/echo", "2019/06/03 21:45:27 handler hits: 0, import hits: 0, misses: 0", "" ] } { "test": "test_smoke_install", "params": { "num": null }, "pass": true, "seconds": 2.2834877967834473, "stats": null, "conf": { "cg_pool_size": 0, "worker_port": "5000", "max_runners": 0, "sandbox": "docker", "docker_runtime": "", "import_cache_size": 0, "handler_cache_size": 0, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:45:16 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:0 Import_cache_size:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:45:16 Initialized registry manager (took 17.535\u00b5s)", "2019/06/03 21:45:24 Initialized installation manager (took 7.525046459s)", "2019/06/03 21:45:24 Initialized handler container factory (took 26.289\u00b5s)", "2019/06/03 21:45:24 Initialized cache manager (took 712ns)", "2019/06/03 21:45:24 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:45:24 Get status by sending request to localhost:5000/status", "2019/06/03 21:45:24 Receive request to /pid", "2019/06/03 21:45:24 Receive request to /run/echo", "2019/06/03 21:45:27 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:45:27 Receive request to /run/install", "2019/06/03 21:45:29 handler hits: 0, import hits: 0, misses: 0", "" ] } { "test": "test_smoke_install", "params": { "num": 2 }, "pass": true, "seconds": 0.8841571807861328, "stats": null, "conf": { "cg_pool_size": 0, "worker_port": "5000", "max_runners": 0, "sandbox": "docker", "docker_runtime": "", "import_cache_size": 0, "handler_cache_size": 0, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:45:16 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:0 Import_cache_size:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:45:16 Initialized registry manager (took 17.535\u00b5s)", "2019/06/03 21:45:24 Initialized installation manager (took 7.525046459s)", "2019/06/03 21:45:24 Initialized handler container factory (took 26.289\u00b5s)", "2019/06/03 21:45:24 Initialized cache manager (took 712ns)", "2019/06/03 21:45:24 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:45:24 Get status by sending request to localhost:5000/status", "2019/06/03 21:45:24 Receive request to /pid", "2019/06/03 21:45:24 Receive request to /run/echo", "2019/06/03 21:45:27 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:45:27 Receive request to /run/install", "2019/06/03 21:45:29 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:45:29 Receive request to /run/install2", "2019/06/03 21:45:30 handler hits: 0, import hits: 0, misses: 0", "" ] }Kill worker process with PID 7631 Starting worker: pid=8568, port=5000, log=/root/open-lambda/test-cluster/worker.out ready { "test": "test_smoke_install", "params": { "num": 3 }, "pass": true, "seconds": 0.8820416927337646, "stats": null, "conf": { "cg_pool_size": 0, "worker_port": "5000", "max_runners": 0, "sandbox": "docker", "docker_runtime": "", "import_cache_size": 0, "handler_cache_size": 0, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:45:16 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:0 Import_cache_size:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:45:16 Initialized registry manager (took 17.535\u00b5s)", "2019/06/03 21:45:24 Initialized installation manager (took 7.525046459s)", "2019/06/03 21:45:24 Initialized handler container factory (took 26.289\u00b5s)", "2019/06/03 21:45:24 Initialized cache manager (took 712ns)", "2019/06/03 21:45:24 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:45:24 Get status by sending request to localhost:5000/status", "2019/06/03 21:45:24 Receive request to /pid", "2019/06/03 21:45:24 Receive request to /run/echo", "2019/06/03 21:45:27 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:45:27 Receive request to /run/install", "2019/06/03 21:45:29 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:45:29 Receive request to /run/install2", "2019/06/03 21:45:30 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:45:30 Receive request to /run/install3", "2019/06/03 21:45:31 handler hits: 0, import hits: 0, misses: 0", "" ] } POP conf: {'cg_pool_size': 0, 'sandbox': 'docker', 'handler_cache_size': 0, 'import_cache_size': 0} RUN ./bin/ol kill -p=test-cluster PUSH conf: {'cg_pool_size': 0, 'sandbox': 'docker', 'handler_cache_size': 10000000, 'import_cache_size': 0} RUN ./bin/ol worker -p=test-cluster --detach { "test": "test_smoke_echo", "params": {}, "pass": true, "seconds": 0.5792567729949951, "stats": null, "conf": { "cg_pool_size": 0, "worker_port": "5000", "max_runners": 0, "sandbox": "docker", "docker_runtime": "", "import_cache_size": 0, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "handler_cache_size": 10000000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:45:31 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:10000000 Import_cache_size:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:45:31 Initialized registry manager (took 20.584\u00b5s)", "2019/06/03 21:45:39 Initialized installation manager (took 7.474800345s)", "2019/06/03 21:45:39 Initialized handler container factory (took 24.015\u00b5s)", "2019/06/03 21:45:39 Initialized cache manager (took 675ns)", "2019/06/03 21:45:39 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:45:39 Get status by sending request to localhost:5000/status", "2019/06/03 21:45:39 Receive request to /pid", "2019/06/03 21:45:39 Receive request to /run/echo", "2019/06/03 21:45:39 handler hits: 0, import hits: 0, misses: 0", "" ] } { "test": "test_smoke_install", "params": { "num": null }, "pass": true, "seconds": 1.062171459197998, "stats": null, "conf": { "cg_pool_size": 0, "worker_port": "5000", "max_runners": 0, "sandbox": "docker", "docker_runtime": "", "import_cache_size": 0, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "handler_cache_size": 10000000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:45:31 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:10000000 Import_cache_size:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:45:31 Initialized registry manager (took 20.584\u00b5s)", "2019/06/03 21:45:39 Initialized installation manager (took 7.474800345s)", "2019/06/03 21:45:39 Initialized handler container factory (took 24.015\u00b5s)", "2019/06/03 21:45:39 Initialized cache manager (took 675ns)", "2019/06/03 21:45:39 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:45:39 Get status by sending request to localhost:5000/status", "2019/06/03 21:45:39 Receive request to /pid", "2019/06/03 21:45:39 Receive request to /run/echo", "2019/06/03 21:45:39 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:45:39 Receive request to /run/install", "2019/06/03 21:45:40 handler hits: 0, import hits: 0, misses: 0", "" ] } { "test": "test_smoke_install", "params": { "num": 2 }, "pass": true, "seconds": 1.4000530242919922, "stats": null, "conf": { "cg_pool_size": 0, "worker_port": "5000", "max_runners": 0, "sandbox": "docker", "docker_runtime": "", "import_cache_size": 0, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "handler_cache_size": 10000000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:45:31 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:10000000 Import_cache_size:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:45:31 Initialized registry manager (took 20.584\u00b5s)", "2019/06/03 21:45:39 Initialized installation manager (took 7.474800345s)", "2019/06/03 21:45:39 Initialized handler container factory (took 24.015\u00b5s)", "2019/06/03 21:45:39 Initialized cache manager (took 675ns)", "2019/06/03 21:45:39 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:45:39 Get status by sending request to localhost:5000/status", "2019/06/03 21:45:39 Receive request to /pid", "2019/06/03 21:45:39 Receive request to /run/echo", "2019/06/03 21:45:39 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:45:39 Receive request to /run/install", "2019/06/03 21:45:40 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:45:40 Receive request to /run/install2", "2019/06/03 21:45:42 handler hits: 0, import hits: 0, misses: 0", "" ] }Kill worker process with PID 8568 Starting worker: pid=9614, port=5000, log=/root/open-lambda/test-cluster/worker.out ready { "test": "test_smoke_install", "params": { "num": 3 }, "pass": true, "seconds": 0.7555079460144043, "stats": null, "conf": { "cg_pool_size": 0, "worker_port": "5000", "max_runners": 0, "sandbox": "docker", "docker_runtime": "", "import_cache_size": 0, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": [ "parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson" ], "sandbox_config": { "processes": 10 }, "handler_cache_size": 10000000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:45:31 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_size:10000000 Import_cache_size:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}", "2019/06/03 21:45:31 Initialized registry manager (took 20.584\u00b5s)", "2019/06/03 21:45:39 Initialized installation manager (took 7.474800345s)", "2019/06/03 21:45:39 Initialized handler container factory (took 24.015\u00b5s)", "2019/06/03 21:45:39 Initialized cache manager (took 675ns)", "2019/06/03 21:45:39 Execute handler by POSTing to localhost:5000/run/", "2019/06/03 21:45:39 Get status by sending request to localhost:5000/status", "2019/06/03 21:45:39 Receive request to /pid", "2019/06/03 21:45:39 Receive request to /run/echo", "2019/06/03 21:45:39 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:45:39 Receive request to /run/install", "2019/06/03 21:45:40 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:45:40 Receive request to /run/install2", "2019/06/03 21:45:42 handler hits: 0, import hits: 0, misses: 0", "2019/06/03 21:45:42 Receive request to /run/install3", "2019/06/03 21:45:42 handler hits: 0, import hits: 0, misses: 0", "" ] } POP conf: {'cg_pool_size': 0, 'sandbox': 'docker', 'handler_cache_size': 10000000, 'import_cache_size': 0} RUN ./bin/ol kill -p=test-cluster POP conf: {'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson'], 'registry': '/root/open-lambda/test-registry'} PUSH conf: {'cg_pool_size': 10, 'handler_cache_size': 10000000, 'sandbox': 'sock', 'import_cache_size': 10000000, 'registry': '/root/open-lambda/test-registry'} RUN ./bin/ol worker -p=test-cluster --detach { "test": "stress_one_lambda", "params": { "seconds": 15, "procs": 1 }, "pass": true, "seconds": 15.018732070922852, "stats": { "reqs_per_sec": 289.53333333333336 }, "conf": { "cg_pool_size": 10, "worker_port": "5000", "max_runners": 0, "sandbox": "sock", "docker_runtime": "", "import_cache_size": 10000000, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": null, "sandbox_config": { "processes": 10 }, "handler_cache_size": 10000000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:45:59 handler hits: 4333, import hits: 0, misses: 1", "2019/06/03 21:45:59 Receive request to /run/echo", "2019/06/03 21:45:59 handler hits: 4334, import hits: 0, misses: 1", "2019/06/03 21:45:59 Receive request to /run/echo", "2019/06/03 21:45:59 handler hits: 4335, import hits: 0, misses: 1", "2019/06/03 21:45:59 Receive request to /run/echo", "2019/06/03 21:45:59 handler hits: 4336, import hits: 0, misses: 1", "2019/06/03 21:45:59 Receive request to /run/echo", "2019/06/03 21:45:59 handler hits: 4337, import hits: 0, misses: 1", "2019/06/03 21:45:59 Receive request to /run/echo", "2019/06/03 21:45:59 handler hits: 4338, import hits: 0, misses: 1", "2019/06/03 21:45:59 Receive request to /run/echo", "2019/06/03 21:45:59 handler hits: 4339, import hits: 0, misses: 1", "2019/06/03 21:45:59 Receive request to /run/echo", "2019/06/03 21:45:59 handler hits: 4340, import hits: 0, misses: 1", "2019/06/03 21:45:59 Receive request to /run/echo", "2019/06/03 21:45:59 handler hits: 4341, import hits: 0, misses: 1", "2019/06/03 21:45:59 Receive request to /run/echo", "2019/06/03 21:45:59 handler hits: 4342, import hits: 0, misses: 1", "" ] } { "test": "stress_one_lambda", "params": { "seconds": 15, "procs": 2 }, "pass": true, "seconds": 15.04509162902832, "stats": { "reqs_per_sec": 287.0 }, "conf": { "cg_pool_size": 10, "worker_port": "5000", "max_runners": 0, "sandbox": "sock", "docker_runtime": "", "import_cache_size": 10000000, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": null, "sandbox_config": { "processes": 10 }, "handler_cache_size": 10000000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:46:14 handler hits: 8638, import hits: 0, misses: 1", "2019/06/03 21:46:14 Receive request to /run/echo", "2019/06/03 21:46:14 handler hits: 8639, import hits: 0, misses: 1", "2019/06/03 21:46:14 Receive request to /run/echo", "2019/06/03 21:46:14 handler hits: 8640, import hits: 0, misses: 1", "2019/06/03 21:46:14 Receive request to /run/echo", "2019/06/03 21:46:14 handler hits: 8641, import hits: 0, misses: 1", "2019/06/03 21:46:14 Receive request to /run/echo", "2019/06/03 21:46:14 handler hits: 8642, import hits: 0, misses: 1", "2019/06/03 21:46:14 Receive request to /run/echo", "2019/06/03 21:46:14 handler hits: 8643, import hits: 0, misses: 1", "2019/06/03 21:46:14 Receive request to /run/echo", "2019/06/03 21:46:14 handler hits: 8644, import hits: 0, misses: 1", "2019/06/03 21:46:14 Receive request to /run/echo", "2019/06/03 21:46:14 handler hits: 8645, import hits: 0, misses: 1", "2019/06/03 21:46:14 Receive request to /run/echo", "2019/06/03 21:46:14 handler hits: 8646, import hits: 0, misses: 1", "2019/06/03 21:46:14 Receive request to /run/echo", "2019/06/03 21:46:14 handler hits: 8647, import hits: 0, misses: 1", "" ] } Kill worker process with PID 9614 Starting worker: pid=9673, port=5000, log=/root/open-lambda/test-cluster/worker.out ready Kill worker process with PID 9673 Starting worker: pid=10000, port=5000, log=/root/open-lambda/test-cluster/worker.out ready { "test": "stress_one_lambda", "params": { "seconds": 15, "procs": 8 }, "pass": true, "seconds": 15.065735340118408, "stats": { "reqs_per_sec": 266.8666666666667 }, "conf": { "cg_pool_size": 10, "worker_port": "5000", "max_runners": 0, "sandbox": "sock", "docker_runtime": "", "import_cache_size": 10000000, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/root/open-lambda/test-registry", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": null, "sandbox_config": { "processes": 10 }, "handler_cache_size": 10000000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:46:29 handler hits: 12641, import hits: 0, misses: 1", "2019/06/03 21:46:29 Receive request to /run/echo", "2019/06/03 21:46:29 handler hits: 12642, import hits: 0, misses: 1", "2019/06/03 21:46:29 Receive request to /run/echo", "2019/06/03 21:46:29 handler hits: 12643, import hits: 0, misses: 1", "2019/06/03 21:46:29 Receive request to /run/echo", "2019/06/03 21:46:29 handler hits: 12644, import hits: 0, misses: 1", "2019/06/03 21:46:29 Receive request to /run/echo", "2019/06/03 21:46:29 handler hits: 12645, import hits: 0, misses: 1", "2019/06/03 21:46:29 Receive request to /run/echo", "2019/06/03 21:46:29 handler hits: 12646, import hits: 0, misses: 1", "2019/06/03 21:46:29 Receive request to /run/echo", "2019/06/03 21:46:29 Receive request to /run/echo", "2019/06/03 21:46:29 handler hits: 12647, import hits: 0, misses: 1", "2019/06/03 21:46:29 handler hits: 12648, import hits: 0, misses: 1", "2019/06/03 21:46:29 Receive request to /run/echo", "2019/06/03 21:46:29 handler hits: 12649, import hits: 0, misses: 1", "2019/06/03 21:46:29 Receive request to /run/echo", "2019/06/03 21:46:29 handler hits: 12650, import hits: 0, misses: 1", "" ] } POP conf: {'cg_pool_size': 10, 'handler_cache_size': 10000000, 'sandbox': 'sock', 'import_cache_size': 10000000, 'registry': '/root/open-lambda/test-registry'} RUN ./bin/ol kill -p=test-cluster PUSH conf: {'cg_pool_size': 10, 'sandbox': 'sock', 'import_cache_size': 10000000, 'handler_cache_size': 10000000} PUSH conf: {'registry': '/tmp/tmp0lr2ydap'} RUN ./bin/ol worker -p=test-cluster --detach { "test": "call_each_once_exec", "params": { "lambda_count": 50 }, "pass": true, "seconds": 1.159306287765503, "stats": { "reqs_per_sec": 43.13022533035317 }, "conf": { "cg_pool_size": 10, "worker_port": "5000", "max_runners": 0, "sandbox": "sock", "docker_runtime": "", "import_cache_size": 10000000, "handler_cache_size": 10000000, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/tmp/tmp0lr2ydap", "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": null, "sandbox_config": { "processes": 10 }, "registry_cache_ms": 5000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": null, "worker_tail": [ "2019/06/03 21:46:31 handler hits: 0, import hits: 0, misses: 41", "2019/06/03 21:46:31 Receive request to /run/L41", "2019/06/03 21:46:31 handler hits: 0, import hits: 0, misses: 42", "2019/06/03 21:46:31 Receive request to /run/L42", "2019/06/03 21:46:31 handler hits: 0, import hits: 0, misses: 43", "2019/06/03 21:46:31 Receive request to /run/L43", "2019/06/03 21:46:31 handler hits: 0, import hits: 0, misses: 44", "2019/06/03 21:46:31 Receive request to /run/L44", "2019/06/03 21:46:31 handler hits: 0, import hits: 0, misses: 45", "2019/06/03 21:46:31 Receive request to /run/L45", "2019/06/03 21:46:31 handler hits: 0, import hits: 0, misses: 46", "2019/06/03 21:46:31 Receive request to /run/L46", "2019/06/03 21:46:31 handler hits: 0, import hits: 0, misses: 47", "2019/06/03 21:46:31 Receive request to /run/L47", "2019/06/03 21:46:31 handler hits: 0, import hits: 0, misses: 48", "2019/06/03 21:46:31 Receive request to /run/L48", "2019/06/03 21:46:31 handler hits: 0, import hits: 0, misses: 49", "2019/06/03 21:46:31 Receive request to /run/L49", "2019/06/03 21:46:31 handler hits: 0, import hits: 0, misses: 50", "" ] } POP conf: {'registry': '/tmp/tmp0lr2ydap'} RUN ./bin/ol kill -p=test-cluster PUSH conf: {'registry': '/tmp/tmpbvtwadti'} RUN ./bin/ol worker -p=test-cluster --detach { "test": "call_each_once_exec", "params": { "lambda_count": 500 }, "pass": false, "seconds": 61.13347935676575, "stats": null, "conf": { "cg_pool_size": 10, "worker_port": "5000", "max_runners": 0, "sandbox": "sock", "docker_runtime": "", "import_cache_size": 10000000, "pip_mirror": "", "cluster_name": "/root/open-lambda/test-cluster", "registry": "/tmp/tmpbvtwadti", "registry_cache_ms": 5000, "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages", "timing": false, "worker_dir": "/root/open-lambda/test-cluster/worker", "benchmark_log": "", "Skip_pull_existing": false, "startup_pkgs": null, "sandbox_config": { "processes": 10 }, "handler_cache_size": 10000000, "SOCK_base_path": "/root/open-lambda/test-cluster/lambda" }, "exception": [ "Traceback (most recent call last):", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py\", line 603, in urlopen", " chunked=chunked)", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py\", line 387, in _make_request", " six.raise_from(e, None)", " File \"\", line 2, in raise_from", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py\", line 383, in _make_request", " httplib_response = conn.getresponse()", " File \"/usr/lib/python3.5/http/client.py\", line 1197, in getresponse", " response.begin()", " File \"/usr/lib/python3.5/http/client.py\", line 297, in begin", " version, status, reason = self._read_status()", " File \"/usr/lib/python3.5/http/client.py\", line 266, in _read_status", " raise RemoteDisconnected(\"Remote end closed connection without\"", "http.client.RemoteDisconnected: Remote end closed connection without response", "", "During handling of the above exception, another exception occurred:", "", "Traceback (most recent call last):", " File \"/usr/lib/python3/dist-packages/requests/adapters.py\", line 376, in send", " timeout=timeout", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py\", line 641, in urlopen", " _stacktrace=sys.exc_info()[2])", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/util/retry.py\", line 368, in increment", " raise six.reraise(type(error), error, _stacktrace)", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/packages/six.py\", line 685, in reraise", " raise value.with_traceback(tb)", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py\", line 603, in urlopen", " chunked=chunked)", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py\", line 387, in _make_request", " six.raise_from(e, None)", " File \"\", line 2, in raise_from", " File \"/usr/local/lib/python3.5/dist-packages/urllib3/connectionpool.py\", line 383, in _make_request", " httplib_response = conn.getresponse()", " File \"/usr/lib/python3.5/http/client.py\", line 1197, in getresponse", " response.begin()", " File \"/usr/lib/python3.5/http/client.py\", line 297, in begin", " version, status, reason = self._read_status()", " File \"/usr/lib/python3.5/http/client.py\", line 266, in _read_status", " raise RemoteDisconnected(\"Remote end closed connection without\"", "requests.packages.urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))", "", "During handling of the above exception, another exception occurred:", "", "Traceback (most recent call last):", " File \"test.py\", line 32, in wrapper", " result[\"stats\"] = fn(**kwargs)", " File \"test.py\", line 142, in call_each_once_exec", " r = requests.post(\"http://localhost:5000/run/L%d\"%i, data=\"null\")", " File \"/usr/lib/python3/dist-packages/requests/api.py\", line 107, in post", " return request('post', url, data=data, json=json, **kwargs)", " File \"/usr/lib/python3/dist-packages/requests/api.py\", line 53, in request", " return session.request(method=method, url=url, **kwargs)", " File \"/usr/lib/python3/dist-packages/requests/sessions.py\", line 480, in request", " resp = self.send(prep, **send_kwargs)", " File \"/usr/lib/python3/dist-packages/requests/sessions.py\", line 588, in send", " r = adapter.send(request, **kwargs)", " File \"/usr/lib/python3/dist-packages/requests/adapters.py\", line 426, in send", " raise ConnectionError(err, request=request)", "requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))", "" ], "worker_tail": [ "2019/06/03 21:46:36 Receive request to /run/L119", "2019/06/03 21:46:36 handler hits: 0, import hits: 0, misses: 120", "2019/06/03 21:46:36 Receive request to /run/L120", "2019/06/03 21:46:36 handler hits: 0, import hits: 0, misses: 121", "2019/06/03 21:46:36 Receive request to /run/L121", "2019/06/03 21:46:37 handler hits: 0, import hits: 0, misses: 122", "2019/06/03 21:46:37 Receive request to /run/L122", "2019/06/03 21:46:37 handler hits: 0, import hits: 0, misses: 123", "2019/06/03 21:46:37 Receive request to /run/L123", "2019/06/03 21:46:37 handler hits: 0, import hits: 0, misses: 124", "2019/06/03 21:46:59 Receive request to /run/L124", "2019/06/03 21:47:28 could not handle request: instance server failed to initialize after 20s", "panic: send on closed channel", "", "goroutine 1014 [running]:", "github.com/open-lambda/open-lambda/ol/handler.(*LambdaInstance).RunStart.func1(0xc000363570, 0xc0003ac460)", "\t/root/open-lambda/src/handler/handler.go:357 +0x1fd", "created by github.com/open-lambda/open-lambda/ol/handler.(*LambdaInstance).RunStart", "\t/root/open-lambda/src/handler/handler.go:340 +0x581", "" ] }fatal error: runtime: out of memory runtime stack: runtime.throw(0x9882d4, 0x16) /usr/local/go/src/runtime/panic.go:617 +0x72 fp=0x7ffd05c7e630 sp=0x7ffd05c7e600 pc=0x42e442 runtime.sysMap(0xc000000000, 0x4000000, 0xfc2558) /usr/local/go/src/runtime/mem_linux.go:170 +0xc7 fp=0x7ffd05c7e670 sp=0x7ffd05c7e630 pc=0x419ec7 runtime.(*mheap).sysAlloc(0xfa9b80, 0x2000, 0xfa9b90, 0x1) /usr/local/go/src/runtime/malloc.go:633 +0x1cd fp=0x7ffd05c7e718 sp=0x7ffd05c7e670 pc=0x40cfcd runtime.(*mheap).grow(0xfa9b80, 0x1, 0x0) /usr/local/go/src/runtime/mheap.go:1222 +0x42 fp=0x7ffd05c7e770 sp=0x7ffd05c7e718 pc=0x4271e2 runtime.(*mheap).allocSpanLocked(0xfa9b80, 0x1, 0xfc2568, 0x0) /usr/local/go/src/runtime/mheap.go:1150 +0x37f fp=0x7ffd05c7e7a8 sp=0x7ffd05c7e770 pc=0x4270cf runtime.(*mheap).alloc_m(0xfa9b80, 0x1, 0x2a, 0x6e43a318) /usr/local/go/src/runtime/mheap.go:977 +0xc2 fp=0x7ffd05c7e7f8 sp=0x7ffd05c7e7a8 pc=0x426722 runtime.(*mheap).alloc.func1() /usr/local/go/src/runtime/mheap.go:1048 +0x4c fp=0x7ffd05c7e830 sp=0x7ffd05c7e7f8 pc=0x4569ac runtime.(*mheap).alloc(0xfa9b80, 0x1, 0x1002a, 0x7ffd05c7e8d0) /usr/local/go/src/runtime/mheap.go:1047 +0x8a fp=0x7ffd05c7e880 sp=0x7ffd05c7e830 pc=0x4269fa runtime.(*mcentral).grow(0xfaa980, 0x0) /usr/local/go/src/runtime/mcentral.go:256 +0x95 fp=0x7ffd05c7e8c8 sp=0x7ffd05c7e880 pc=0x419945 runtime.(*mcentral).cacheSpan(0xfaa980, 0x7fa3bacb3000) /usr/local/go/src/runtime/mcentral.go:106 +0x2ff fp=0x7ffd05c7e928 sp=0x7ffd05c7e8c8 pc=0x41944f runtime.(*mcache).refill(0x7fa3bacb3008, 0x2a) /usr/local/go/src/runtime/mcache.go:135 +0x86 fp=0x7ffd05c7e948 sp=0x7ffd05c7e928 pc=0x418ee6 runtime.(*mcache).nextFree(0x7fa3bacb3008, 0xfa302a, 0x7fa3bacb3008, 0x7fa3bacb3000, 0x8) /usr/local/go/src/runtime/malloc.go:786 +0x88 fp=0x7ffd05c7e980 sp=0x7ffd05c7e948 pc=0x40d808 runtime.mallocgc(0x180, 0x978f00, 0x1, 0xfc25c0) /usr/local/go/src/runtime/malloc.go:939 +0x76e fp=0x7ffd05c7ea20 sp=0x7ffd05c7e980 pc=0x40e11e runtime.newobject(0x978f00, 0x4000) /usr/local/go/src/runtime/malloc.go:1068 +0x38 fp=0x7ffd05c7ea50 sp=0x7ffd05c7ea20 pc=0x40e528 runtime.malg(0x13600008000, 0xfac1f0) /usr/local/go/src/runtime/proc.go:3220 +0x31 fp=0x7ffd05c7ea90 sp=0x7ffd05c7ea50 pc=0x4377b1 runtime.mpreinit(...) /usr/local/go/src/runtime/os_linux.go:311 runtime.mcommoninit(0xfa4640) /usr/local/go/src/runtime/proc.go:618 +0xc2 fp=0x7ffd05c7eac8 sp=0x7ffd05c7ea90 pc=0x4311e2 runtime.schedinit() /usr/local/go/src/runtime/proc.go:540 +0x74 fp=0x7ffd05c7eb20 sp=0x7ffd05c7eac8 pc=0x430e74 runtime.rt0_go(0x7ffd05c7ec28, 0x3, 0x7ffd05c7ec28, 0x0, 0x7fa3ba512830, 0x0, 0x7ffd05c7ec28, 0x300000000, 0x458950, 0x0, ...) /usr/local/go/src/runtime/asm_amd64.s:195 +0x11a fp=0x7ffd05c7eb28 sp=0x7ffd05c7eb20 pc=0x458a7a POP conf: {'registry': '/tmp/tmpbvtwadti'} RUN ./bin/ol kill -p=test-cluster Traceback (most recent call last): File "test.py", line 217, in main() File "test.py", line 200, in main call_each_once(lambda_count=500) File "test.py", line 159, in call_each_once call_each_once_exec(lambda_count=lambda_count) File "/usr/lib/python3.5/contextlib.py", line 66, in __exit__ next(self.gen) File "test.py", line 77, in TestConf run(['./bin/ol', 'kill', '-p='+OLDIR]) File "test.py", line 86, in run raise Exception("command failed: " + " ".join(cmd)) Exception: command failed: ./bin/ol kill -p=test-cluster Makefile:29: recipe for target 'test-all' failed make: *** [test-all] Error 1