python3 test.py
Init OL dir at /root/open-lambda/test-cluster
Create lambda base at /root/open-lambda/test-cluster/lambda (may take several minutes)
Working Directory: /root/open-lambda/test-cluster

Worker Defaults: 
{
	"registry": "/root/open-lambda/test-cluster/registry",
	"registry_cache_ms": 5000,
	"sandbox": "sock",
	"cluster_name": "/root/open-lambda/test-cluster",
	"pip_mirror": "",
	"Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
	"max_runners": 0,
	"handler_cache_mb": 0,
	"import_cache_mb": 0,
	"worker_dir": "/root/open-lambda/test-cluster/worker",
	"SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
	"worker_port": "5000",
	"cg_pool_size": 0,
	"Skip_pull_existing": false,
	"sandbox_config": {
		"processes": 10
	},
	"benchmark_log": "",
	"timing": false,
	"startup_pkgs": null,
	"docker_runtime": ""
}

You may now start a server using the "worker" command
Starting worker: pid=7254, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
RUN ./bin/ol new -p=test-cluster
PUSH conf: {'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson'], 'registry': '/root/open-lambda/test-registry'}
PUSH conf: {'import_cache_mb': 0, 'sandbox': 'sock', 'handler_cache_mb': 0, 'cg_pool_size': 10}
RUN ./bin/ol worker -p=test-cluster --detach
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 1.3953948020935059,
  "stats": null,
  "conf": {
    "import_cache_mb": 0,
    "sandbox": "sock",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "Skip_pull_existing": false,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:51:35 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:51:35 Initialized registry manager (took 23.097\u00b5s)",
    "2019/06/04 15:51:52 Initialized installation manager (took 16.950672625s)",
    "2019/06/04 15:51:52 Initialized handler container factory (took 23.10947ms)",
    "2019/06/04 15:51:52 Initialized cache manager (took 1.127\u00b5s)",
    "2019/06/04 15:51:52 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:51:52 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:51:52 Receive request to /pid",
    "2019/06/04 15:51:52 Receive request to /run/echo",
    "2019/06/04 15:51:53 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 0.6034352779388428,
  "stats": null,
  "conf": {
    "import_cache_mb": 0,
    "sandbox": "sock",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "Skip_pull_existing": false,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:51:35 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:51:35 Initialized registry manager (took 23.097\u00b5s)",
    "2019/06/04 15:51:52 Initialized installation manager (took 16.950672625s)",
    "2019/06/04 15:51:52 Initialized handler container factory (took 23.10947ms)",
    "2019/06/04 15:51:52 Initialized cache manager (took 1.127\u00b5s)",
    "2019/06/04 15:51:52 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:51:52 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:51:52 Receive request to /pid",
    "2019/06/04 15:51:52 Receive request to /run/echo",
    "2019/06/04 15:51:53 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:51:53 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:51:54 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.3724038600921631,
  "stats": null,
  "conf": {
    "import_cache_mb": 0,
    "sandbox": "sock",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "Skip_pull_existing": false,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:51:35 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:51:35 Initialized registry manager (took 23.097\u00b5s)",
    "2019/06/04 15:51:52 Initialized installation manager (took 16.950672625s)",
    "2019/06/04 15:51:52 Initialized handler container factory (took 23.10947ms)",
    "2019/06/04 15:51:52 Initialized cache manager (took 1.127\u00b5s)",
    "2019/06/04 15:51:52 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:51:52 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:51:52 Receive request to /pid",
    "2019/06/04 15:51:52 Receive request to /run/echo",
    "2019/06/04 15:51:53 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:51:53 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:51:54 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:51:54 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:51:54 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.37532520294189453,
  "stats": null,
  "conf": {
    "import_cache_mb": 0,
    "sandbox": "sock",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "Skip_pull_existing": false,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:51:35 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:51:35 Initialized registry manager (took 23.097\u00b5s)",
    "2019/06/04 15:51:52 Initialized installation manager (took 16.950672625s)",
    "2019/06/04 15:51:52 Initialized handler container factory (took 23.10947ms)",
    "2019/06/04 15:51:52 Initialized cache manager (took 1.127\u00b5s)",
    "2019/06/04 15:51:52 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:51:52 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:51:52 Receive request to /pid",
    "2019/06/04 15:51:52 Receive request to /run/echo",
    "2019/06/04 15:51:53 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:51:53 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:51:54 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:51:54 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:51:54 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:51:54 Receive request to /run/install3",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:51:55 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}Kill worker process with PID 7254
Starting worker: pid=7341, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready

POP conf: {'import_cache_mb': 0, 'sandbox': 'sock', 'handler_cache_mb': 0, 'cg_pool_size': 10}
RUN ./bin/ol kill -p=test-cluster
PUSH conf: {'import_cache_mb': 0, 'sandbox': 'sock', 'handler_cache_mb': 256, 'cg_pool_size': 10}
RUN ./bin/ol worker -p=test-cluster --detach
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 0.2563941478729248,
  "stats": null,
  "conf": {
    "import_cache_mb": 0,
    "sandbox": "sock",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "Skip_pull_existing": false,
    "handler_cache_mb": 256,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "registry_cache_ms": 5000,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:51:55 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:51:55 Initialized registry manager (took 20.337\u00b5s)",
    "2019/06/04 15:52:03 Initialized installation manager (took 7.903501764s)",
    "2019/06/04 15:52:03 Initialized handler container factory (took 3.824769ms)",
    "2019/06/04 15:52:03 Initialized cache manager (took 824ns)",
    "2019/06/04 15:52:03 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:52:03 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:52:03 Receive request to /pid",
    "2019/06/04 15:52:03 Receive request to /run/echo",
    "2019/06/04 15:52:03 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 0.5436496734619141,
  "stats": null,
  "conf": {
    "import_cache_mb": 0,
    "sandbox": "sock",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "Skip_pull_existing": false,
    "handler_cache_mb": 256,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "registry_cache_ms": 5000,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:51:55 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:51:55 Initialized registry manager (took 20.337\u00b5s)",
    "2019/06/04 15:52:03 Initialized installation manager (took 7.903501764s)",
    "2019/06/04 15:52:03 Initialized handler container factory (took 3.824769ms)",
    "2019/06/04 15:52:03 Initialized cache manager (took 824ns)",
    "2019/06/04 15:52:03 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:52:03 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:52:03 Receive request to /pid",
    "2019/06/04 15:52:03 Receive request to /run/echo",
    "2019/06/04 15:52:03 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:52:03 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:52:04 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.5873055458068848,
  "stats": null,
  "conf": {
    "import_cache_mb": 0,
    "sandbox": "sock",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "Skip_pull_existing": false,
    "handler_cache_mb": 256,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "registry_cache_ms": 5000,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:51:55 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:51:55 Initialized registry manager (took 20.337\u00b5s)",
    "2019/06/04 15:52:03 Initialized installation manager (took 7.903501764s)",
    "2019/06/04 15:52:03 Initialized handler container factory (took 3.824769ms)",
    "2019/06/04 15:52:03 Initialized cache manager (took 824ns)",
    "2019/06/04 15:52:03 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:52:03 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:52:03 Receive request to /pid",
    "2019/06/04 15:52:03 Receive request to /run/echo",
    "2019/06/04 15:52:03 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:52:03 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:52:04 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:52:04 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:52:04 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.6626653671264648,
  "stats": null,
  "conf": {
    "import_cache_mb": 0,
    "sandbox": "sock",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "Skip_pull_existing": false,
    "handler_cache_mb": 256,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "registry_cache_ms": 5000,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:51:55 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:51:55 Initialized registry manager (took 20.337\u00b5s)",
    "2019/06/04 15:52:03 Initialized installation manager (took 7.903501764s)",
    "2019/06/04 15:52:03 Initialized handler container factory (took 3.824769ms)",
    "2019/06/04 15:52:03 Initialized cache manager (took 824ns)",
    "2019/06/04 15:52:03 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:52:03 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:52:03 Receive request to /pid",
    "2019/06/04 15:52:03 Receive request to /run/echo",
    "2019/06/04 15:52:03 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:52:03 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:52:04 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:52:04 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:52:04 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:52:04 Receive request to /run/install3",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:52:05 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}Kill worker process with PID 7341
Starting worker: pid=7416, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready

POP conf: {'import_cache_mb': 0, 'sandbox': 'sock', 'handler_cache_mb': 256, 'cg_pool_size': 10}
RUN ./bin/ol kill -p=test-cluster
PUSH conf: {'import_cache_mb': 256, 'sandbox': 'sock', 'handler_cache_mb': 0, 'cg_pool_size': 10}
RUN ./bin/ol worker -p=test-cluster --detach
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 0.023119688034057617,
  "stats": null,
  "conf": {
    "import_cache_mb": 256,
    "sandbox": "sock",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "Skip_pull_existing": false,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:52:05 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:52:05 Initialized registry manager (took 20.017\u00b5s)",
    "2019/06/04 15:52:13 Initialized installation manager (took 7.794314207s)",
    "2019/06/04 15:52:13 Initialized handler container factory (took 3.728964ms)",
    "2019/06/04 15:52:13 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/04 15:52:13 Initialized cache manager (took 336.415897ms)",
    "2019/06/04 15:52:13 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:52:13 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:52:13 Receive request to /pid",
    "2019/06/04 15:52:13 Receive request to /run/echo",
    "2019/06/04 15:52:13 handler hits: 0, import hits: 0, misses: 1",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 0.2434375286102295,
  "stats": null,
  "conf": {
    "import_cache_mb": 256,
    "sandbox": "sock",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "Skip_pull_existing": false,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:52:05 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:52:05 Initialized registry manager (took 20.017\u00b5s)",
    "2019/06/04 15:52:13 Initialized installation manager (took 7.794314207s)",
    "2019/06/04 15:52:13 Initialized handler container factory (took 3.728964ms)",
    "2019/06/04 15:52:13 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/04 15:52:13 Initialized cache manager (took 336.415897ms)",
    "2019/06/04 15:52:13 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:52:13 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:52:13 Receive request to /pid",
    "2019/06/04 15:52:13 Receive request to /run/echo",
    "2019/06/04 15:52:13 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/04 15:52:13 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:52:13 handler hits: 0, import hits: 0, misses: 2",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.2498641014099121,
  "stats": null,
  "conf": {
    "import_cache_mb": 256,
    "sandbox": "sock",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "Skip_pull_existing": false,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:52:05 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:52:05 Initialized registry manager (took 20.017\u00b5s)",
    "2019/06/04 15:52:13 Initialized installation manager (took 7.794314207s)",
    "2019/06/04 15:52:13 Initialized handler container factory (took 3.728964ms)",
    "2019/06/04 15:52:13 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/04 15:52:13 Initialized cache manager (took 336.415897ms)",
    "2019/06/04 15:52:13 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:52:13 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:52:13 Receive request to /pid",
    "2019/06/04 15:52:13 Receive request to /run/echo",
    "2019/06/04 15:52:13 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/04 15:52:13 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:52:13 handler hits: 0, import hits: 0, misses: 2",
    "2019/06/04 15:52:13 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:52:14 handler hits: 0, import hits: 0, misses: 3",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.251772403717041,
  "stats": null,
  "conf": {
    "import_cache_mb": 256,
    "sandbox": "sock",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "Skip_pull_existing": false,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:52:05 Initialized registry manager (took 20.017\u00b5s)",
    "2019/06/04 15:52:13 Initialized installation manager (took 7.794314207s)",
    "2019/06/04 15:52:13 Initialized handler container factory (took 3.728964ms)",
    "2019/06/04 15:52:13 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/04 15:52:13 Initialized cache manager (took 336.415897ms)",
    "2019/06/04 15:52:13 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:52:13 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:52:13 Receive request to /pid",
    "2019/06/04 15:52:13 Receive request to /run/echo",
    "2019/06/04 15:52:13 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/04 15:52:13 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:52:13 handler hits: 0, import hits: 0, misses: 2",
    "2019/06/04 15:52:13 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:52:14 handler hits: 0, import hits: 0, misses: 3",
    "2019/06/04 15:52:14 Receive request to /run/install3",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:52:14 handler hits: 0, import hits: 0, misses: 4",
    ""
  ]
}Kill worker process with PID 7416
Starting worker: pid=7501, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready

POP conf: {'import_cache_mb': 256, 'sandbox': 'sock', 'handler_cache_mb': 0, 'cg_pool_size': 10}
RUN ./bin/ol kill -p=test-cluster
PUSH conf: {'import_cache_mb': 256, 'sandbox': 'sock', 'handler_cache_mb': 256, 'cg_pool_size': 10}
RUN ./bin/ol worker -p=test-cluster --detach
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 0.022493600845336914,
  "stats": null,
  "conf": {
    "import_cache_mb": 256,
    "sandbox": "sock",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "Skip_pull_existing": false,
    "handler_cache_mb": 256,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "registry_cache_ms": 5000,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:52:14 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:52:14 Initialized registry manager (took 20.035\u00b5s)",
    "2019/06/04 15:52:22 Initialized installation manager (took 7.669845901s)",
    "2019/06/04 15:52:22 Initialized handler container factory (took 3.694174ms)",
    "2019/06/04 15:52:22 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/04 15:52:22 Initialized cache manager (took 144.617472ms)",
    "2019/06/04 15:52:22 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:52:22 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:52:22 Receive request to /pid",
    "2019/06/04 15:52:22 Receive request to /run/echo",
    "2019/06/04 15:52:22 handler hits: 0, import hits: 0, misses: 1",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 0.2446887493133545,
  "stats": null,
  "conf": {
    "import_cache_mb": 256,
    "sandbox": "sock",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "Skip_pull_existing": false,
    "handler_cache_mb": 256,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "registry_cache_ms": 5000,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:52:14 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:52:14 Initialized registry manager (took 20.035\u00b5s)",
    "2019/06/04 15:52:22 Initialized installation manager (took 7.669845901s)",
    "2019/06/04 15:52:22 Initialized handler container factory (took 3.694174ms)",
    "2019/06/04 15:52:22 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/04 15:52:22 Initialized cache manager (took 144.617472ms)",
    "2019/06/04 15:52:22 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:52:22 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:52:22 Receive request to /pid",
    "2019/06/04 15:52:22 Receive request to /run/echo",
    "2019/06/04 15:52:22 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/04 15:52:22 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:52:22 handler hits: 0, import hits: 0, misses: 2",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.25090765953063965,
  "stats": null,
  "conf": {
    "import_cache_mb": 256,
    "sandbox": "sock",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "Skip_pull_existing": false,
    "handler_cache_mb": 256,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "registry_cache_ms": 5000,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:52:14 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:52:14 Initialized registry manager (took 20.035\u00b5s)",
    "2019/06/04 15:52:22 Initialized installation manager (took 7.669845901s)",
    "2019/06/04 15:52:22 Initialized handler container factory (took 3.694174ms)",
    "2019/06/04 15:52:22 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/04 15:52:22 Initialized cache manager (took 144.617472ms)",
    "2019/06/04 15:52:22 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:52:22 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:52:22 Receive request to /pid",
    "2019/06/04 15:52:22 Receive request to /run/echo",
    "2019/06/04 15:52:22 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/04 15:52:22 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:52:22 handler hits: 0, import hits: 0, misses: 2",
    "2019/06/04 15:52:22 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:52:22 handler hits: 0, import hits: 0, misses: 3",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.24932575225830078,
  "stats": null,
  "conf": {
    "import_cache_mb": 256,
    "sandbox": "sock",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "Skip_pull_existing": false,
    "handler_cache_mb": 256,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "registry_cache_ms": 5000,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:52:14 Initialized registry manager (took 20.035\u00b5s)",
    "2019/06/04 15:52:22 Initialized installation manager (took 7.669845901s)",
    "2019/06/04 15:52:22 Initialized handler container factory (took 3.694174ms)",
    "2019/06/04 15:52:22 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/04 15:52:22 Initialized cache manager (took 144.617472ms)",
    "2019/06/04 15:52:22 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:52:22 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:52:22 Receive request to /pid",
    "2019/06/04 15:52:22 Receive request to /run/echo",
    "2019/06/04 15:52:22 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/04 15:52:22 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:52:22 handler hits: 0, import hits: 0, misses: 2",
    "2019/06/04 15:52:22 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:52:22 handler hits: 0, import hits: 0, misses: 3",
    "2019/06/04 15:52:22 Receive request to /run/install3",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/04 15:52:23 handler hits: 0, import hits: 0, misses: 4",
    ""
  ]
}Kill worker process with PID 7501
Starting worker: pid=7585, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready

POP conf: {'import_cache_mb': 256, 'sandbox': 'sock', 'handler_cache_mb': 256, 'cg_pool_size': 10}
RUN ./bin/ol kill -p=test-cluster
PUSH conf: {'import_cache_mb': 0, 'sandbox': 'docker', 'handler_cache_mb': 0, 'cg_pool_size': 0}
RUN ./bin/ol worker -p=test-cluster --detach
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 3.5171310901641846,
  "stats": null,
  "conf": {
    "import_cache_mb": 0,
    "sandbox": "docker",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "Skip_pull_existing": false,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 0,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:52:23 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:52:23 Initialized registry manager (took 21.218\u00b5s)",
    "2019/06/04 15:52:30 Initialized installation manager (took 7.647268039s)",
    "2019/06/04 15:52:30 Initialized handler container factory (took 32.212\u00b5s)",
    "2019/06/04 15:52:30 Initialized cache manager (took 865ns)",
    "2019/06/04 15:52:30 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:52:30 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:52:30 Receive request to /pid",
    "2019/06/04 15:52:30 Receive request to /run/echo",
    "2019/06/04 15:52:34 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 1.3513000011444092,
  "stats": null,
  "conf": {
    "import_cache_mb": 0,
    "sandbox": "docker",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "Skip_pull_existing": false,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 0,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:52:23 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:52:23 Initialized registry manager (took 21.218\u00b5s)",
    "2019/06/04 15:52:30 Initialized installation manager (took 7.647268039s)",
    "2019/06/04 15:52:30 Initialized handler container factory (took 32.212\u00b5s)",
    "2019/06/04 15:52:30 Initialized cache manager (took 865ns)",
    "2019/06/04 15:52:30 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:52:30 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:52:30 Receive request to /pid",
    "2019/06/04 15:52:30 Receive request to /run/echo",
    "2019/06/04 15:52:34 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:52:34 Receive request to /run/install",
    "2019/06/04 15:52:35 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 1.2146484851837158,
  "stats": null,
  "conf": {
    "import_cache_mb": 0,
    "sandbox": "docker",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "Skip_pull_existing": false,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 0,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:52:23 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:52:23 Initialized registry manager (took 21.218\u00b5s)",
    "2019/06/04 15:52:30 Initialized installation manager (took 7.647268039s)",
    "2019/06/04 15:52:30 Initialized handler container factory (took 32.212\u00b5s)",
    "2019/06/04 15:52:30 Initialized cache manager (took 865ns)",
    "2019/06/04 15:52:30 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:52:30 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:52:30 Receive request to /pid",
    "2019/06/04 15:52:30 Receive request to /run/echo",
    "2019/06/04 15:52:34 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:52:34 Receive request to /run/install",
    "2019/06/04 15:52:35 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:52:35 Receive request to /run/install2",
    "2019/06/04 15:52:36 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.9475195407867432,
  "stats": null,
  "conf": {
    "import_cache_mb": 0,
    "sandbox": "docker",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "Skip_pull_existing": false,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 0,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:52:23 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:52:23 Initialized registry manager (took 21.218\u00b5s)",
    "2019/06/04 15:52:30 Initialized installation manager (took 7.647268039s)",
    "2019/06/04 15:52:30 Initialized handler container factory (took 32.212\u00b5s)",
    "2019/06/04 15:52:30 Initialized cache manager (took 865ns)",
    "2019/06/04 15:52:30 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:52:30 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:52:30 Receive request to /pid",
    "2019/06/04 15:52:30 Receive request to /run/echo",
    "2019/06/04 15:52:34 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:52:34 Receive request to /run/install",
    "2019/06/04 15:52:35 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:52:35 Receive request to /run/install2",
    "2019/06/04 15:52:36 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:52:37 Receive request to /run/install3",
    "2019/06/04 15:52:37 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}Kill worker process with PID 7585
Starting worker: pid=8534, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready

POP conf: {'import_cache_mb': 0, 'sandbox': 'docker', 'handler_cache_mb': 0, 'cg_pool_size': 0}
RUN ./bin/ol kill -p=test-cluster
PUSH conf: {'import_cache_mb': 0, 'sandbox': 'docker', 'handler_cache_mb': 256, 'cg_pool_size': 0}
RUN ./bin/ol worker -p=test-cluster --detach
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 0.546891450881958,
  "stats": null,
  "conf": {
    "import_cache_mb": 0,
    "sandbox": "docker",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "Skip_pull_existing": false,
    "handler_cache_mb": 256,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "registry_cache_ms": 5000,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 0,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:52:37 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:52:37 Initialized registry manager (took 18.407\u00b5s)",
    "2019/06/04 15:52:45 Initialized installation manager (took 7.6191904s)",
    "2019/06/04 15:52:45 Initialized handler container factory (took 33.791\u00b5s)",
    "2019/06/04 15:52:45 Initialized cache manager (took 844ns)",
    "2019/06/04 15:52:45 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:52:45 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:52:45 Receive request to /pid",
    "2019/06/04 15:52:45 Receive request to /run/echo",
    "2019/06/04 15:52:46 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 1.161179780960083,
  "stats": null,
  "conf": {
    "import_cache_mb": 0,
    "sandbox": "docker",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "Skip_pull_existing": false,
    "handler_cache_mb": 256,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "registry_cache_ms": 5000,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 0,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:52:37 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:52:37 Initialized registry manager (took 18.407\u00b5s)",
    "2019/06/04 15:52:45 Initialized installation manager (took 7.6191904s)",
    "2019/06/04 15:52:45 Initialized handler container factory (took 33.791\u00b5s)",
    "2019/06/04 15:52:45 Initialized cache manager (took 844ns)",
    "2019/06/04 15:52:45 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:52:45 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:52:45 Receive request to /pid",
    "2019/06/04 15:52:45 Receive request to /run/echo",
    "2019/06/04 15:52:46 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:52:46 Receive request to /run/install",
    "2019/06/04 15:52:47 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.9666211605072021,
  "stats": null,
  "conf": {
    "import_cache_mb": 0,
    "sandbox": "docker",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "Skip_pull_existing": false,
    "handler_cache_mb": 256,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "registry_cache_ms": 5000,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 0,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:52:37 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:52:37 Initialized registry manager (took 18.407\u00b5s)",
    "2019/06/04 15:52:45 Initialized installation manager (took 7.6191904s)",
    "2019/06/04 15:52:45 Initialized handler container factory (took 33.791\u00b5s)",
    "2019/06/04 15:52:45 Initialized cache manager (took 844ns)",
    "2019/06/04 15:52:45 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:52:45 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:52:45 Receive request to /pid",
    "2019/06/04 15:52:45 Receive request to /run/echo",
    "2019/06/04 15:52:46 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:52:46 Receive request to /run/install",
    "2019/06/04 15:52:47 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:52:47 Receive request to /run/install2",
    "2019/06/04 15:52:48 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.7789099216461182,
  "stats": null,
  "conf": {
    "import_cache_mb": 0,
    "sandbox": "docker",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "Skip_pull_existing": false,
    "handler_cache_mb": 256,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cluster_name": "/root/open-lambda/test-cluster",
    "registry_cache_ms": 5000,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 0,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:52:37 Config: &{path:/root/open-lambda/test-cluster/config.json Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/04 15:52:37 Initialized registry manager (took 18.407\u00b5s)",
    "2019/06/04 15:52:45 Initialized installation manager (took 7.6191904s)",
    "2019/06/04 15:52:45 Initialized handler container factory (took 33.791\u00b5s)",
    "2019/06/04 15:52:45 Initialized cache manager (took 844ns)",
    "2019/06/04 15:52:45 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/04 15:52:45 Get status by sending request to localhost:5000/status",
    "2019/06/04 15:52:45 Receive request to /pid",
    "2019/06/04 15:52:45 Receive request to /run/echo",
    "2019/06/04 15:52:46 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:52:46 Receive request to /run/install",
    "2019/06/04 15:52:47 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:52:47 Receive request to /run/install2",
    "2019/06/04 15:52:48 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/04 15:52:48 Receive request to /run/install3",
    "2019/06/04 15:52:49 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}Kill worker process with PID 8534
Starting worker: pid=9573, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready

POP conf: {'import_cache_mb': 0, 'sandbox': 'docker', 'handler_cache_mb': 256, 'cg_pool_size': 0}
RUN ./bin/ol kill -p=test-cluster
POP conf: {'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson'], 'registry': '/root/open-lambda/test-registry'}
PUSH conf: {'import_cache_mb': 256, 'sandbox': 'sock', 'handler_cache_mb': 256, 'registry': '/root/open-lambda/test-registry', 'cg_pool_size': 10}
RUN ./bin/ol worker -p=test-cluster --detach
{
  "test": "stress_one_lambda",
  "params": {
    "seconds": 15,
    "procs": 1
  },
  "pass": true,
  "seconds": 15.10212755203247,
  "stats": {
    "reqs_per_sec": 277.26666666666665
  },
  "conf": {
    "import_cache_mb": 256,
    "sandbox": "sock",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "Skip_pull_existing": false,
    "handler_cache_mb": 256,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": null,
    "cluster_name": "/root/open-lambda/test-cluster",
    "registry_cache_ms": 5000,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:53:05 handler hits: 4149, import hits: 0, misses: 1",
    "2019/06/04 15:53:05 Receive request to /run/echo",
    "2019/06/04 15:53:05 handler hits: 4150, import hits: 0, misses: 1",
    "2019/06/04 15:53:05 Receive request to /run/echo",
    "2019/06/04 15:53:05 handler hits: 4151, import hits: 0, misses: 1",
    "2019/06/04 15:53:05 Receive request to /run/echo",
    "2019/06/04 15:53:05 handler hits: 4152, import hits: 0, misses: 1",
    "2019/06/04 15:53:05 Receive request to /run/echo",
    "2019/06/04 15:53:05 handler hits: 4153, import hits: 0, misses: 1",
    "2019/06/04 15:53:05 Receive request to /run/echo",
    "2019/06/04 15:53:05 handler hits: 4154, import hits: 0, misses: 1",
    "2019/06/04 15:53:05 Receive request to /run/echo",
    "2019/06/04 15:53:05 handler hits: 4155, import hits: 0, misses: 1",
    "2019/06/04 15:53:05 Receive request to /run/echo",
    "2019/06/04 15:53:05 handler hits: 4156, import hits: 0, misses: 1",
    "2019/06/04 15:53:05 Receive request to /run/echo",
    "2019/06/04 15:53:05 handler hits: 4157, import hits: 0, misses: 1",
    "2019/06/04 15:53:05 Receive request to /run/echo",
    "2019/06/04 15:53:05 handler hits: 4158, import hits: 0, misses: 1",
    ""
  ]
}
{
  "test": "stress_one_lambda",
  "params": {
    "seconds": 15,
    "procs": 2
  },
  "pass": true,
  "seconds": 15.047676801681519,
  "stats": {
    "reqs_per_sec": 276.53333333333336
  },
  "conf": {
    "import_cache_mb": 256,
    "sandbox": "sock",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "Skip_pull_existing": false,
    "handler_cache_mb": 256,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": null,
    "cluster_name": "/root/open-lambda/test-cluster",
    "registry_cache_ms": 5000,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:53:20 handler hits: 8297, import hits: 0, misses: 1",
    "2019/06/04 15:53:20 Receive request to /run/echo",
    "2019/06/04 15:53:20 handler hits: 8298, import hits: 0, misses: 1",
    "2019/06/04 15:53:20 Receive request to /run/echo",
    "2019/06/04 15:53:20 handler hits: 8299, import hits: 0, misses: 1",
    "2019/06/04 15:53:20 Receive request to /run/echo",
    "2019/06/04 15:53:20 handler hits: 8300, import hits: 0, misses: 1",
    "2019/06/04 15:53:20 Receive request to /run/echo",
    "2019/06/04 15:53:20 handler hits: 8301, import hits: 0, misses: 1",
    "2019/06/04 15:53:20 Receive request to /run/echo",
    "2019/06/04 15:53:20 handler hits: 8302, import hits: 0, misses: 1",
    "2019/06/04 15:53:20 Receive request to /run/echo",
    "2019/06/04 15:53:20 handler hits: 8303, import hits: 0, misses: 1",
    "2019/06/04 15:53:20 Receive request to /run/echo",
    "2019/06/04 15:53:20 handler hits: 8304, import hits: 0, misses: 1",
    "2019/06/04 15:53:20 Receive request to /run/echo",
    "2019/06/04 15:53:20 handler hits: 8305, import hits: 0, misses: 1",
    "2019/06/04 15:53:20 Receive request to /run/echo",
    "2019/06/04 15:53:20 handler hits: 8306, import hits: 0, misses: 1",
    ""
  ]
}
Kill worker process with PID 9573
Starting worker: pid=9632, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
Kill worker process with PID 9632
Starting worker: pid=10260, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
Kill worker process with PID 10260
{
  "test": "stress_one_lambda",
  "params": {
    "seconds": 15,
    "procs": 8
  },
  "pass": true,
  "seconds": 15.071202039718628,
  "stats": {
    "reqs_per_sec": 247.53333333333333
  },
  "conf": {
    "import_cache_mb": 256,
    "sandbox": "sock",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "Skip_pull_existing": false,
    "handler_cache_mb": 256,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": null,
    "cluster_name": "/root/open-lambda/test-cluster",
    "registry_cache_ms": 5000,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:53:35 handler hits: 12010, import hits: 0, misses: 1",
    "2019/06/04 15:53:35 Receive request to /run/echo",
    "2019/06/04 15:53:35 handler hits: 12011, import hits: 0, misses: 1",
    "2019/06/04 15:53:35 Receive request to /run/echo",
    "2019/06/04 15:53:35 handler hits: 12012, import hits: 0, misses: 1",
    "2019/06/04 15:53:35 Receive request to /run/echo",
    "2019/06/04 15:53:35 handler hits: 12013, import hits: 0, misses: 1",
    "2019/06/04 15:53:35 Receive request to /run/echo",
    "2019/06/04 15:53:35 handler hits: 12014, import hits: 0, misses: 1",
    "2019/06/04 15:53:35 Receive request to /run/echo",
    "2019/06/04 15:53:35 handler hits: 12015, import hits: 0, misses: 1",
    "2019/06/04 15:53:35 Receive request to /run/echo",
    "2019/06/04 15:53:35 handler hits: 12016, import hits: 0, misses: 1",
    "2019/06/04 15:53:35 Receive request to /run/echo",
    "2019/06/04 15:53:35 handler hits: 12017, import hits: 0, misses: 1",
    "2019/06/04 15:53:35 Receive request to /run/echo",
    "2019/06/04 15:53:35 handler hits: 12018, import hits: 0, misses: 1",
    "2019/06/04 15:53:35 Receive request to /run/echo",
    "2019/06/04 15:53:35 handler hits: 12019, import hits: 0, misses: 1",
    ""
  ]
}
POP conf: {'import_cache_mb': 256, 'sandbox': 'sock', 'handler_cache_mb': 256, 'registry': '/root/open-lambda/test-registry', 'cg_pool_size': 10}
RUN ./bin/ol kill -p=test-cluster
PUSH conf: {'import_cache_mb': 256, 'cg_pool_size': 10, 'handler_cache_mb': 256, 'sandbox': 'sock'}
PUSH conf: {'registry': '/tmp/tmpwhsu4998'}
RUN ./bin/ol worker -p=test-cluster --detach
{
  "test": "call_each_once_exec",
  "params": {
    "lambda_count": 100
  },
  "pass": true,
  "seconds": 2.830780029296875,
  "stats": {
    "reqs_per_sec": 35.32636874282617
  },
  "conf": {
    "import_cache_mb": 256,
    "sandbox": "sock",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/tmp/tmpwhsu4998",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 256,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": null,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Skip_pull_existing": false,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:53:39 EVICTING INSTANCE: 270581760 used / 268435456 limit",
    "2019/06/04 15:53:39 Receive request to /run/L94",
    "2019/06/04 15:53:39 handler hits: 0, import hits: 0, misses: 95",
    "2019/06/04 15:53:39 EVICTING INSTANCE: 269832192 used / 268435456 limit",
    "2019/06/04 15:53:39 Receive request to /run/L95",
    "2019/06/04 15:53:39 handler hits: 0, import hits: 0, misses: 96",
    "2019/06/04 15:53:39 EVICTING INSTANCE: 269164544 used / 268435456 limit",
    "2019/06/04 15:53:39 Receive request to /run/L96",
    "2019/06/04 15:53:39 handler hits: 0, import hits: 0, misses: 97",
    "2019/06/04 15:53:39 EVICTING INSTANCE: 270401536 used / 268435456 limit",
    "2019/06/04 15:53:39 Receive request to /run/L97",
    "2019/06/04 15:53:39 handler hits: 0, import hits: 0, misses: 98",
    "2019/06/04 15:53:39 EVICTING INSTANCE: 268705792 used / 268435456 limit",
    "2019/06/04 15:53:39 Receive request to /run/L98",
    "2019/06/04 15:53:39 handler hits: 0, import hits: 0, misses: 99",
    "2019/06/04 15:53:39 EVICTING INSTANCE: 269389824 used / 268435456 limit",
    "2019/06/04 15:53:39 Receive request to /run/L99",
    "2019/06/04 15:53:39 handler hits: 0, import hits: 0, misses: 100",
    "2019/06/04 15:53:39 EVICTING INSTANCE: 271269888 used / 268435456 limit",
    ""
  ]
}
POP conf: {'registry': '/tmp/tmpwhsu4998'}
RUN ./bin/ol kill -p=test-cluster
PUSH conf: {'registry': '/tmp/tmpb64ae_y9'}
RUN ./bin/ol worker -p=test-cluster --detach
{
  "test": "call_each_once_exec",
  "params": {
    "lambda_count": 1000
  },
  "pass": true,
  "seconds": 31.167791843414307,
  "stats": {
    "reqs_per_sec": 32.084438285079955
  },
  "conf": {
    "import_cache_mb": 256,
    "sandbox": "sock",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "registry": "/tmp/tmpb64ae_y9",
    "worker_port": "5000",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "benchmark_log": "",
    "Skip_pull_existing": false,
    "handler_cache_mb": 256,
    "pip_mirror": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": null,
    "cluster_name": "/root/open-lambda/test-cluster",
    "registry_cache_ms": 5000,
    "timing": false,
    "max_runners": 0,
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    }
  },
  "exception": null,
  "worker_tail": [
    "2019/06/04 15:54:11 handler hits: 0, import hits: 0, misses: 994",
    "2019/06/04 15:54:11 EVICTING INSTANCE: 270442496 used / 268435456 limit",
    "2019/06/04 15:54:11 Receive request to /run/L994",
    "2019/06/04 15:54:11 handler hits: 0, import hits: 0, misses: 995",
    "2019/06/04 15:54:11 EVICTING INSTANCE: 270819328 used / 268435456 limit",
    "2019/06/04 15:54:11 Receive request to /run/L995",
    "2019/06/04 15:54:11 handler hits: 0, import hits: 0, misses: 996",
    "2019/06/04 15:54:11 EVICTING INSTANCE: 269258752 used / 268435456 limit",
    "2019/06/04 15:54:11 Receive request to /run/L996",
    "2019/06/04 15:54:12 handler hits: 0, import hits: 0, misses: 997",
    "2019/06/04 15:54:12 EVICTING INSTANCE: 268869632 used / 268435456 limit",
    "2019/06/04 15:54:12 Receive request to /run/L997",
    "2019/06/04 15:54:12 handler hits: 0, import hits: 0, misses: 998",
    "2019/06/04 15:54:12 Receive request to /run/L998",
    "2019/06/04 15:54:12 handler hits: 0, import hits: 0, misses: 999",
    "2019/06/04 15:54:12 EVICTING INSTANCE: 270983168 used / 268435456 limit",
    "2019/06/04 15:54:12 Receive request to /run/L999",
    "2019/06/04 15:54:12 handler hits: 0, import hits: 0, misses: 1000",
    "2019/06/04 15:54:12 EVICTING INSTANCE: 271237120 used / 268435456 limit",
    ""
  ]
}
POP conf: {'registry': '/tmp/tmpb64ae_y9'}
RUN ./bin/ol kill -p=test-cluster
POP conf: {'import_cache_mb': 256, 'cg_pool_size': 10, 'handler_cache_mb': 256, 'sandbox': 'sock'}
PASSED: 29, FAILED: 0