python3 test.py
Init OL dir at /root/open-lambda/test-cluster
Create lambda base at /root/open-lambda/test-cluster/lambda (may take several minutes)
Working Directory: /root/open-lambda/test-cluster

Worker Defaults: 
{
	"registry": "/root/open-lambda/test-cluster/registry",
	"registry_cache_ms": 5000,
	"sandbox": "sock",
	"cluster_name": "/root/open-lambda/test-cluster",
	"pip_mirror": "",
	"Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
	"max_runners": 0,
	"handler_cache_mb": 239,
	"import_cache_mb": 239,
	"worker_dir": "/root/open-lambda/test-cluster/worker",
	"SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
	"worker_port": "5000",
	"cg_pool_size": 0,
	"Skip_pull_existing": false,
	"sandbox_config": {
		"processes": 10
	},
	"benchmark_log": "",
	"timing": false,
	"startup_pkgs": null,
	"docker_runtime": ""
}

You may now start a server using the "worker" command
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=7255, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
RUN ./ol new -p=test-cluster
PUSH conf: {'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson'], 'registry': '/root/open-lambda/test-registry'}
PUSH conf: {'handler_cache_mb': 0, 'import_cache_mb': 0, 'cg_pool_size': 10, 'sandbox': 'sock'}
RUN ./ol worker -p=test-cluster --detach
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 2.0436859130859375,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "import_cache_mb": 0,
    "handler_cache_mb": 0,
    "sandbox": "sock",
    "sandbox_config": {
      "processes": 10
    },
    "timing": false,
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 10,
    "docker_runtime": "",
    "cluster_name": "/root/open-lambda/test-cluster",
    "pip_mirror": "",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "registry_cache_ms": 5000,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 18:38:23 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:38:23 Initialized registry manager (took 57.241\u00b5s)",
    "2019/06/05 18:38:39 Initialized installation manager (took 15.529634846s)",
    "2019/06/05 18:38:39 Initialized handler container factory (took 5.573269ms)",
    "2019/06/05 18:38:39 Initialized cache manager (took 825ns)",
    "2019/06/05 18:38:39 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:38:39 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:38:39 Receive request to /pid",
    "2019/06/05 18:38:39 Receive request to /run/echo",
    "2019/06/05 18:38:41 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 0.6237919330596924,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "import_cache_mb": 0,
    "handler_cache_mb": 0,
    "sandbox": "sock",
    "sandbox_config": {
      "processes": 10
    },
    "timing": false,
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 10,
    "docker_runtime": "",
    "cluster_name": "/root/open-lambda/test-cluster",
    "pip_mirror": "",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "registry_cache_ms": 5000,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 18:38:23 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:38:23 Initialized registry manager (took 57.241\u00b5s)",
    "2019/06/05 18:38:39 Initialized installation manager (took 15.529634846s)",
    "2019/06/05 18:38:39 Initialized handler container factory (took 5.573269ms)",
    "2019/06/05 18:38:39 Initialized cache manager (took 825ns)",
    "2019/06/05 18:38:39 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:38:39 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:38:39 Receive request to /pid",
    "2019/06/05 18:38:39 Receive request to /run/echo",
    "2019/06/05 18:38:41 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:38:41 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:38:41 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.3603677749633789,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "import_cache_mb": 0,
    "handler_cache_mb": 0,
    "sandbox": "sock",
    "sandbox_config": {
      "processes": 10
    },
    "timing": false,
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 10,
    "docker_runtime": "",
    "cluster_name": "/root/open-lambda/test-cluster",
    "pip_mirror": "",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "registry_cache_ms": 5000,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 18:38:23 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:38:23 Initialized registry manager (took 57.241\u00b5s)",
    "2019/06/05 18:38:39 Initialized installation manager (took 15.529634846s)",
    "2019/06/05 18:38:39 Initialized handler container factory (took 5.573269ms)",
    "2019/06/05 18:38:39 Initialized cache manager (took 825ns)",
    "2019/06/05 18:38:39 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:38:39 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:38:39 Receive request to /pid",
    "2019/06/05 18:38:39 Receive request to /run/echo",
    "2019/06/05 18:38:41 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:38:41 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:38:41 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:38:41 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:38:42 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.36344480514526367,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "import_cache_mb": 0,
    "handler_cache_mb": 0,
    "sandbox": "sock",
    "sandbox_config": {
      "processes": 10
    },
    "timing": false,
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 10,
    "docker_runtime": "",
    "cluster_name": "/root/open-lambda/test-cluster",
    "pip_mirror": "",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "registry_cache_ms": 5000,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "2019/06/05 18:38:23 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:38:23 Initialized registry manager (took 57.241\u00b5s)",
    "2019/06/05 18:38:39 Initialized installation manager (took 15.529634846s)",
    "2019/06/05 18:38:39 Initialized handler container factory (took 5.573269ms)",
    "2019/06/05 18:38:39 Initialized cache manager (took 825ns)",
    "2019/06/05 18:38:39 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:38:39 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:38:39 Receive request to /pid",
    "2019/06/05 18:38:39 Receive request to /run/echo",
    "2019/06/05 18:38:41 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:38:41 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:38:41 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:38:41 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:38:42 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:38:42 Receive request to /run/install3",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:38:42 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}Kill worker process with PID 7255
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=7342, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready

POP conf: {'handler_cache_mb': 0, 'import_cache_mb': 0, 'cg_pool_size': 10, 'sandbox': 'sock'}
RUN ./ol kill -p=test-cluster
PUSH conf: {'handler_cache_mb': 256, 'import_cache_mb': 0, 'cg_pool_size': 10, 'sandbox': 'sock'}
RUN ./ol worker -p=test-cluster --detach
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 0.14576458930969238,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "pip_mirror": "",
    "import_cache_mb": 0,
    "sandbox": "sock",
    "timing": false,
    "registry_cache_ms": 5000,
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 10,
    "registry": "/root/open-lambda/test-registry",
    "docker_runtime": "",
    "sandbox_config": {
      "processes": 10
    },
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 18:38:42 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:38:42 Initialized registry manager (took 18.052\u00b5s)",
    "2019/06/05 18:38:50 Initialized installation manager (took 7.485635898s)",
    "2019/06/05 18:38:50 Initialized handler container factory (took 3.383719ms)",
    "2019/06/05 18:38:50 Initialized cache manager (took 782ns)",
    "2019/06/05 18:38:50 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:38:50 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:38:50 Receive request to /pid",
    "2019/06/05 18:38:50 Receive request to /run/echo",
    "2019/06/05 18:38:50 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 0.36123180389404297,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "pip_mirror": "",
    "import_cache_mb": 0,
    "sandbox": "sock",
    "timing": false,
    "registry_cache_ms": 5000,
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 10,
    "registry": "/root/open-lambda/test-registry",
    "docker_runtime": "",
    "sandbox_config": {
      "processes": 10
    },
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 18:38:42 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:38:42 Initialized registry manager (took 18.052\u00b5s)",
    "2019/06/05 18:38:50 Initialized installation manager (took 7.485635898s)",
    "2019/06/05 18:38:50 Initialized handler container factory (took 3.383719ms)",
    "2019/06/05 18:38:50 Initialized cache manager (took 782ns)",
    "2019/06/05 18:38:50 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:38:50 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:38:50 Receive request to /pid",
    "2019/06/05 18:38:50 Receive request to /run/echo",
    "2019/06/05 18:38:50 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:38:50 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:38:50 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.49398112297058105,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "pip_mirror": "",
    "import_cache_mb": 0,
    "sandbox": "sock",
    "timing": false,
    "registry_cache_ms": 5000,
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 10,
    "registry": "/root/open-lambda/test-registry",
    "docker_runtime": "",
    "sandbox_config": {
      "processes": 10
    },
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 18:38:42 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:38:42 Initialized registry manager (took 18.052\u00b5s)",
    "2019/06/05 18:38:50 Initialized installation manager (took 7.485635898s)",
    "2019/06/05 18:38:50 Initialized handler container factory (took 3.383719ms)",
    "2019/06/05 18:38:50 Initialized cache manager (took 782ns)",
    "2019/06/05 18:38:50 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:38:50 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:38:50 Receive request to /pid",
    "2019/06/05 18:38:50 Receive request to /run/echo",
    "2019/06/05 18:38:50 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:38:50 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:38:50 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:38:50 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:38:51 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.5621778964996338,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "pip_mirror": "",
    "import_cache_mb": 0,
    "sandbox": "sock",
    "timing": false,
    "registry_cache_ms": 5000,
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 10,
    "registry": "/root/open-lambda/test-registry",
    "docker_runtime": "",
    "sandbox_config": {
      "processes": 10
    },
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "2019/06/05 18:38:42 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:38:42 Initialized registry manager (took 18.052\u00b5s)",
    "2019/06/05 18:38:50 Initialized installation manager (took 7.485635898s)",
    "2019/06/05 18:38:50 Initialized handler container factory (took 3.383719ms)",
    "2019/06/05 18:38:50 Initialized cache manager (took 782ns)",
    "2019/06/05 18:38:50 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:38:50 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:38:50 Receive request to /pid",
    "2019/06/05 18:38:50 Receive request to /run/echo",
    "2019/06/05 18:38:50 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:38:50 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:38:50 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:38:50 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:38:51 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:38:51 Receive request to /run/install3",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:38:51 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}Kill worker process with PID 7342
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=7417, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready

POP conf: {'handler_cache_mb': 256, 'import_cache_mb': 0, 'cg_pool_size': 10, 'sandbox': 'sock'}
RUN ./ol kill -p=test-cluster
PUSH conf: {'handler_cache_mb': 0, 'import_cache_mb': 256, 'cg_pool_size': 10, 'sandbox': 'sock'}
RUN ./ol worker -p=test-cluster --detach
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 0.02187204360961914,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "handler_cache_mb": 0,
    "max_runners": 0,
    "sandbox_config": {
      "processes": 10
    },
    "timing": false,
    "registry_cache_ms": 5000,
    "sandbox": "sock",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 10,
    "docker_runtime": "",
    "cluster_name": "/root/open-lambda/test-cluster",
    "pip_mirror": "",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "import_cache_mb": 256,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 18:38:51 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:38:51 Initialized registry manager (took 17.623\u00b5s)",
    "2019/06/05 18:38:59 Initialized installation manager (took 7.533893825s)",
    "2019/06/05 18:38:59 Initialized handler container factory (took 3.344778ms)",
    "2019/06/05 18:38:59 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/05 18:38:59 Initialized cache manager (took 172.725136ms)",
    "2019/06/05 18:38:59 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:38:59 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:38:59 Receive request to /pid",
    "2019/06/05 18:38:59 Receive request to /run/echo",
    "2019/06/05 18:38:59 handler hits: 0, import hits: 0, misses: 1",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 0.24120616912841797,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "handler_cache_mb": 0,
    "max_runners": 0,
    "sandbox_config": {
      "processes": 10
    },
    "timing": false,
    "registry_cache_ms": 5000,
    "sandbox": "sock",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 10,
    "docker_runtime": "",
    "cluster_name": "/root/open-lambda/test-cluster",
    "pip_mirror": "",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "import_cache_mb": 256,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 18:38:51 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:38:51 Initialized registry manager (took 17.623\u00b5s)",
    "2019/06/05 18:38:59 Initialized installation manager (took 7.533893825s)",
    "2019/06/05 18:38:59 Initialized handler container factory (took 3.344778ms)",
    "2019/06/05 18:38:59 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/05 18:38:59 Initialized cache manager (took 172.725136ms)",
    "2019/06/05 18:38:59 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:38:59 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:38:59 Receive request to /pid",
    "2019/06/05 18:38:59 Receive request to /run/echo",
    "2019/06/05 18:38:59 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/05 18:38:59 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:38:59 handler hits: 0, import hits: 0, misses: 2",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.24951696395874023,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "handler_cache_mb": 0,
    "max_runners": 0,
    "sandbox_config": {
      "processes": 10
    },
    "timing": false,
    "registry_cache_ms": 5000,
    "sandbox": "sock",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 10,
    "docker_runtime": "",
    "cluster_name": "/root/open-lambda/test-cluster",
    "pip_mirror": "",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "import_cache_mb": 256,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 18:38:51 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:38:51 Initialized registry manager (took 17.623\u00b5s)",
    "2019/06/05 18:38:59 Initialized installation manager (took 7.533893825s)",
    "2019/06/05 18:38:59 Initialized handler container factory (took 3.344778ms)",
    "2019/06/05 18:38:59 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/05 18:38:59 Initialized cache manager (took 172.725136ms)",
    "2019/06/05 18:38:59 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:38:59 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:38:59 Receive request to /pid",
    "2019/06/05 18:38:59 Receive request to /run/echo",
    "2019/06/05 18:38:59 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/05 18:38:59 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:38:59 handler hits: 0, import hits: 0, misses: 2",
    "2019/06/05 18:38:59 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:39:00 handler hits: 0, import hits: 0, misses: 3",
    ""
  ]
}Kill worker process with PID 7417
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=7501, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready

{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.24572086334228516,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "handler_cache_mb": 0,
    "max_runners": 0,
    "sandbox_config": {
      "processes": 10
    },
    "timing": false,
    "registry_cache_ms": 5000,
    "sandbox": "sock",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 10,
    "docker_runtime": "",
    "cluster_name": "/root/open-lambda/test-cluster",
    "pip_mirror": "",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "import_cache_mb": 256,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "2019/06/05 18:38:51 Initialized registry manager (took 17.623\u00b5s)",
    "2019/06/05 18:38:59 Initialized installation manager (took 7.533893825s)",
    "2019/06/05 18:38:59 Initialized handler container factory (took 3.344778ms)",
    "2019/06/05 18:38:59 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/05 18:38:59 Initialized cache manager (took 172.725136ms)",
    "2019/06/05 18:38:59 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:38:59 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:38:59 Receive request to /pid",
    "2019/06/05 18:38:59 Receive request to /run/echo",
    "2019/06/05 18:38:59 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/05 18:38:59 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:38:59 handler hits: 0, import hits: 0, misses: 2",
    "2019/06/05 18:38:59 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:39:00 handler hits: 0, import hits: 0, misses: 3",
    "2019/06/05 18:39:00 Receive request to /run/install3",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:39:00 handler hits: 0, import hits: 0, misses: 4",
    ""
  ]
}
POP conf: {'handler_cache_mb': 0, 'import_cache_mb': 256, 'cg_pool_size': 10, 'sandbox': 'sock'}
RUN ./ol kill -p=test-cluster
PUSH conf: {'handler_cache_mb': 256, 'import_cache_mb': 256, 'cg_pool_size': 10, 'sandbox': 'sock'}
RUN ./ol worker -p=test-cluster --detach
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 0.021699905395507812,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "pip_mirror": "",
    "import_cache_mb": 256,
    "max_runners": 0,
    "timing": false,
    "registry_cache_ms": 5000,
    "sandbox": "sock",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 10,
    "registry": "/root/open-lambda/test-registry",
    "docker_runtime": "",
    "sandbox_config": {
      "processes": 10
    },
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 18:39:00 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:39:00 Initialized registry manager (took 16.877\u00b5s)",
    "2019/06/05 18:39:08 Initialized installation manager (took 7.61604811s)",
    "2019/06/05 18:39:08 Initialized handler container factory (took 3.274581ms)",
    "2019/06/05 18:39:08 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/05 18:39:08 Initialized cache manager (took 142.900702ms)",
    "2019/06/05 18:39:08 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:39:08 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:39:08 Receive request to /pid",
    "2019/06/05 18:39:08 Receive request to /run/echo",
    "2019/06/05 18:39:08 handler hits: 0, import hits: 0, misses: 1",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 0.24099135398864746,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "pip_mirror": "",
    "import_cache_mb": 256,
    "max_runners": 0,
    "timing": false,
    "registry_cache_ms": 5000,
    "sandbox": "sock",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 10,
    "registry": "/root/open-lambda/test-registry",
    "docker_runtime": "",
    "sandbox_config": {
      "processes": 10
    },
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 18:39:00 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:39:00 Initialized registry manager (took 16.877\u00b5s)",
    "2019/06/05 18:39:08 Initialized installation manager (took 7.61604811s)",
    "2019/06/05 18:39:08 Initialized handler container factory (took 3.274581ms)",
    "2019/06/05 18:39:08 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/05 18:39:08 Initialized cache manager (took 142.900702ms)",
    "2019/06/05 18:39:08 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:39:08 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:39:08 Receive request to /pid",
    "2019/06/05 18:39:08 Receive request to /run/echo",
    "2019/06/05 18:39:08 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/05 18:39:08 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:39:08 handler hits: 0, import hits: 0, misses: 2",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.23897051811218262,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "pip_mirror": "",
    "import_cache_mb": 256,
    "max_runners": 0,
    "timing": false,
    "registry_cache_ms": 5000,
    "sandbox": "sock",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 10,
    "registry": "/root/open-lambda/test-registry",
    "docker_runtime": "",
    "sandbox_config": {
      "processes": 10
    },
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 18:39:00 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:39:00 Initialized registry manager (took 16.877\u00b5s)",
    "2019/06/05 18:39:08 Initialized installation manager (took 7.61604811s)",
    "2019/06/05 18:39:08 Initialized handler container factory (took 3.274581ms)",
    "2019/06/05 18:39:08 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/05 18:39:08 Initialized cache manager (took 142.900702ms)",
    "2019/06/05 18:39:08 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:39:08 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:39:08 Receive request to /pid",
    "2019/06/05 18:39:08 Receive request to /run/echo",
    "2019/06/05 18:39:08 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/05 18:39:08 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:39:08 handler hits: 0, import hits: 0, misses: 2",
    "2019/06/05 18:39:08 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:39:08 handler hits: 0, import hits: 0, misses: 3",
    ""
  ]
}Kill worker process with PID 7501
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=7585, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready

{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.23944330215454102,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "pip_mirror": "",
    "import_cache_mb": 256,
    "max_runners": 0,
    "timing": false,
    "registry_cache_ms": 5000,
    "sandbox": "sock",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 10,
    "registry": "/root/open-lambda/test-registry",
    "docker_runtime": "",
    "sandbox_config": {
      "processes": 10
    },
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "2019/06/05 18:39:00 Initialized registry manager (took 16.877\u00b5s)",
    "2019/06/05 18:39:08 Initialized installation manager (took 7.61604811s)",
    "2019/06/05 18:39:08 Initialized handler container factory (took 3.274581ms)",
    "2019/06/05 18:39:08 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/05 18:39:08 Initialized cache manager (took 142.900702ms)",
    "2019/06/05 18:39:08 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:39:08 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:39:08 Receive request to /pid",
    "2019/06/05 18:39:08 Receive request to /run/echo",
    "2019/06/05 18:39:08 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/05 18:39:08 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:39:08 handler hits: 0, import hits: 0, misses: 2",
    "2019/06/05 18:39:08 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:39:08 handler hits: 0, import hits: 0, misses: 3",
    "2019/06/05 18:39:08 Receive request to /run/install3",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 18:39:09 handler hits: 0, import hits: 0, misses: 4",
    ""
  ]
}
POP conf: {'handler_cache_mb': 256, 'import_cache_mb': 256, 'cg_pool_size': 10, 'sandbox': 'sock'}
RUN ./ol kill -p=test-cluster
PUSH conf: {'handler_cache_mb': 0, 'import_cache_mb': 0, 'cg_pool_size': 0, 'sandbox': 'docker'}
RUN ./ol worker -p=test-cluster --detach
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 3.607926607131958,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "handler_cache_mb": 0,
    "sandbox": "docker",
    "sandbox_config": {
      "processes": 10
    },
    "timing": false,
    "registry_cache_ms": 5000,
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 0,
    "docker_runtime": "",
    "cluster_name": "/root/open-lambda/test-cluster",
    "pip_mirror": "",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "import_cache_mb": 0,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 18:39:09 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:39:09 Initialized registry manager (took 18.255\u00b5s)",
    "2019/06/05 18:39:16 Initialized installation manager (took 7.608910625s)",
    "2019/06/05 18:39:16 Initialized handler container factory (took 26.2\u00b5s)",
    "2019/06/05 18:39:16 Initialized cache manager (took 591ns)",
    "2019/06/05 18:39:16 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:39:16 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:39:16 Receive request to /pid",
    "2019/06/05 18:39:16 Receive request to /run/echo",
    "2019/06/05 18:39:20 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 1.253218412399292,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "handler_cache_mb": 0,
    "sandbox": "docker",
    "sandbox_config": {
      "processes": 10
    },
    "timing": false,
    "registry_cache_ms": 5000,
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 0,
    "docker_runtime": "",
    "cluster_name": "/root/open-lambda/test-cluster",
    "pip_mirror": "",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "import_cache_mb": 0,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 18:39:09 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:39:09 Initialized registry manager (took 18.255\u00b5s)",
    "2019/06/05 18:39:16 Initialized installation manager (took 7.608910625s)",
    "2019/06/05 18:39:16 Initialized handler container factory (took 26.2\u00b5s)",
    "2019/06/05 18:39:16 Initialized cache manager (took 591ns)",
    "2019/06/05 18:39:16 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:39:16 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:39:16 Receive request to /pid",
    "2019/06/05 18:39:16 Receive request to /run/echo",
    "2019/06/05 18:39:20 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:39:20 Receive request to /run/install",
    "2019/06/05 18:39:21 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.9310271739959717,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "handler_cache_mb": 0,
    "sandbox": "docker",
    "sandbox_config": {
      "processes": 10
    },
    "timing": false,
    "registry_cache_ms": 5000,
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 0,
    "docker_runtime": "",
    "cluster_name": "/root/open-lambda/test-cluster",
    "pip_mirror": "",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "import_cache_mb": 0,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 18:39:09 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:39:09 Initialized registry manager (took 18.255\u00b5s)",
    "2019/06/05 18:39:16 Initialized installation manager (took 7.608910625s)",
    "2019/06/05 18:39:16 Initialized handler container factory (took 26.2\u00b5s)",
    "2019/06/05 18:39:16 Initialized cache manager (took 591ns)",
    "2019/06/05 18:39:16 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:39:16 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:39:16 Receive request to /pid",
    "2019/06/05 18:39:16 Receive request to /run/echo",
    "2019/06/05 18:39:20 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:39:20 Receive request to /run/install",
    "2019/06/05 18:39:21 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:39:21 Receive request to /run/install2",
    "2019/06/05 18:39:22 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}Kill worker process with PID 7585
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=8523, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready

{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.9106338024139404,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "handler_cache_mb": 0,
    "sandbox": "docker",
    "sandbox_config": {
      "processes": 10
    },
    "timing": false,
    "registry_cache_ms": 5000,
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 0,
    "docker_runtime": "",
    "cluster_name": "/root/open-lambda/test-cluster",
    "pip_mirror": "",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "import_cache_mb": 0,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 18:39:09 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:39:09 Initialized registry manager (took 18.255\u00b5s)",
    "2019/06/05 18:39:16 Initialized installation manager (took 7.608910625s)",
    "2019/06/05 18:39:16 Initialized handler container factory (took 26.2\u00b5s)",
    "2019/06/05 18:39:16 Initialized cache manager (took 591ns)",
    "2019/06/05 18:39:16 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:39:16 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:39:16 Receive request to /pid",
    "2019/06/05 18:39:16 Receive request to /run/echo",
    "2019/06/05 18:39:20 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:39:20 Receive request to /run/install",
    "2019/06/05 18:39:21 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:39:21 Receive request to /run/install2",
    "2019/06/05 18:39:22 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:39:22 Receive request to /run/install3",
    "2019/06/05 18:39:23 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
POP conf: {'handler_cache_mb': 0, 'import_cache_mb': 0, 'cg_pool_size': 0, 'sandbox': 'docker'}
RUN ./ol kill -p=test-cluster
PUSH conf: {'handler_cache_mb': 256, 'import_cache_mb': 0, 'cg_pool_size': 0, 'sandbox': 'docker'}
RUN ./ol worker -p=test-cluster --detach
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 0.5683410167694092,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "pip_mirror": "",
    "import_cache_mb": 0,
    "sandbox": "docker",
    "timing": false,
    "registry_cache_ms": 5000,
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 0,
    "registry": "/root/open-lambda/test-registry",
    "docker_runtime": "",
    "sandbox_config": {
      "processes": 10
    },
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 18:39:23 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:39:23 Initialized registry manager (took 31.747\u00b5s)",
    "2019/06/05 18:39:31 Initialized installation manager (took 7.533014697s)",
    "2019/06/05 18:39:31 Initialized handler container factory (took 19.765\u00b5s)",
    "2019/06/05 18:39:31 Initialized cache manager (took 617ns)",
    "2019/06/05 18:39:31 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:39:31 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:39:31 Receive request to /pid",
    "2019/06/05 18:39:31 Receive request to /run/echo",
    "2019/06/05 18:39:31 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 1.1496152877807617,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "pip_mirror": "",
    "import_cache_mb": 0,
    "sandbox": "docker",
    "timing": false,
    "registry_cache_ms": 5000,
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 0,
    "registry": "/root/open-lambda/test-registry",
    "docker_runtime": "",
    "sandbox_config": {
      "processes": 10
    },
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 18:39:23 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:39:23 Initialized registry manager (took 31.747\u00b5s)",
    "2019/06/05 18:39:31 Initialized installation manager (took 7.533014697s)",
    "2019/06/05 18:39:31 Initialized handler container factory (took 19.765\u00b5s)",
    "2019/06/05 18:39:31 Initialized cache manager (took 617ns)",
    "2019/06/05 18:39:31 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:39:31 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:39:31 Receive request to /pid",
    "2019/06/05 18:39:31 Receive request to /run/echo",
    "2019/06/05 18:39:31 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:39:31 Receive request to /run/install",
    "2019/06/05 18:39:32 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.784156322479248,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "pip_mirror": "",
    "import_cache_mb": 0,
    "sandbox": "docker",
    "timing": false,
    "registry_cache_ms": 5000,
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 0,
    "registry": "/root/open-lambda/test-registry",
    "docker_runtime": "",
    "sandbox_config": {
      "processes": 10
    },
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 18:39:23 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:39:23 Initialized registry manager (took 31.747\u00b5s)",
    "2019/06/05 18:39:31 Initialized installation manager (took 7.533014697s)",
    "2019/06/05 18:39:31 Initialized handler container factory (took 19.765\u00b5s)",
    "2019/06/05 18:39:31 Initialized cache manager (took 617ns)",
    "2019/06/05 18:39:31 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:39:31 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:39:31 Receive request to /pid",
    "2019/06/05 18:39:31 Receive request to /run/echo",
    "2019/06/05 18:39:31 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:39:31 Receive request to /run/install",
    "2019/06/05 18:39:32 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:39:32 Receive request to /run/install2",
    "2019/06/05 18:39:33 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}Kill worker process with PID 8523
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=9559, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready

{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.7608442306518555,
  "stats": null,
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "pip_mirror": "",
    "import_cache_mb": 0,
    "sandbox": "docker",
    "timing": false,
    "registry_cache_ms": 5000,
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "cg_pool_size": 0,
    "registry": "/root/open-lambda/test-registry",
    "docker_runtime": "",
    "sandbox_config": {
      "processes": 10
    },
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 18:39:23 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 18:39:23 Initialized registry manager (took 31.747\u00b5s)",
    "2019/06/05 18:39:31 Initialized installation manager (took 7.533014697s)",
    "2019/06/05 18:39:31 Initialized handler container factory (took 19.765\u00b5s)",
    "2019/06/05 18:39:31 Initialized cache manager (took 617ns)",
    "2019/06/05 18:39:31 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 18:39:31 Get status by sending request to localhost:5000/status",
    "2019/06/05 18:39:31 Receive request to /pid",
    "2019/06/05 18:39:31 Receive request to /run/echo",
    "2019/06/05 18:39:31 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:39:31 Receive request to /run/install",
    "2019/06/05 18:39:32 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:39:32 Receive request to /run/install2",
    "2019/06/05 18:39:33 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 18:39:33 Receive request to /run/install3",
    "2019/06/05 18:39:34 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
POP conf: {'handler_cache_mb': 256, 'import_cache_mb': 0, 'cg_pool_size': 0, 'sandbox': 'docker'}
RUN ./ol kill -p=test-cluster
POP conf: {'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson'], 'registry': '/root/open-lambda/test-registry'}
PUSH conf: {'handler_cache_mb': 256, 'import_cache_mb': 256, 'registry': '/root/open-lambda/test-registry', 'cg_pool_size': 10, 'sandbox': 'sock'}
RUN ./ol worker -p=test-cluster --detach
{
  "test": "stress_one_lambda",
  "params": {
    "procs": 1,
    "seconds": 15
  },
  "pass": true,
  "seconds": 15.077876091003418,
  "stats": {
    "reqs_per_sec": 282.26666666666665
  },
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "pip_mirror": "",
    "handler_cache_mb": 256,
    "sandbox": "sock",
    "timing": false,
    "max_runners": 0,
    "startup_pkgs": null,
    "registry_cache_ms": 5000,
    "registry": "/root/open-lambda/test-registry",
    "docker_runtime": "",
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    },
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "import_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "2019/06/05 18:39:50 handler hits: 4224, import hits: 0, misses: 1",
    "2019/06/05 18:39:50 Receive request to /run/echo",
    "2019/06/05 18:39:50 handler hits: 4225, import hits: 0, misses: 1",
    "2019/06/05 18:39:50 Receive request to /run/echo",
    "2019/06/05 18:39:50 handler hits: 4226, import hits: 0, misses: 1",
    "2019/06/05 18:39:50 Receive request to /run/echo",
    "2019/06/05 18:39:50 handler hits: 4227, import hits: 0, misses: 1",
    "2019/06/05 18:39:50 Receive request to /run/echo",
    "2019/06/05 18:39:50 handler hits: 4228, import hits: 0, misses: 1",
    "2019/06/05 18:39:50 Receive request to /run/echo",
    "2019/06/05 18:39:50 handler hits: 4229, import hits: 0, misses: 1",
    "2019/06/05 18:39:50 Receive request to /run/echo",
    "2019/06/05 18:39:50 handler hits: 4230, import hits: 0, misses: 1",
    "2019/06/05 18:39:50 Receive request to /run/echo",
    "2019/06/05 18:39:50 handler hits: 4231, import hits: 0, misses: 1",
    "2019/06/05 18:39:50 Receive request to /run/echo",
    "2019/06/05 18:39:50 handler hits: 4232, import hits: 0, misses: 1",
    "2019/06/05 18:39:50 Receive request to /run/echo",
    "2019/06/05 18:39:50 handler hits: 4233, import hits: 0, misses: 1",
    ""
  ]
}
{
  "test": "stress_one_lambda",
  "params": {
    "procs": 2,
    "seconds": 15
  },
  "pass": true,
  "seconds": 15.049318790435791,
  "stats": {
    "reqs_per_sec": 279.8666666666667
  },
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "pip_mirror": "",
    "handler_cache_mb": 256,
    "sandbox": "sock",
    "timing": false,
    "max_runners": 0,
    "startup_pkgs": null,
    "registry_cache_ms": 5000,
    "registry": "/root/open-lambda/test-registry",
    "docker_runtime": "",
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    },
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "import_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "2019/06/05 18:40:05 handler hits: 8422, import hits: 0, misses: 1",
    "2019/06/05 18:40:05 Receive request to /run/echo",
    "2019/06/05 18:40:05 handler hits: 8423, import hits: 0, misses: 1",
    "2019/06/05 18:40:05 Receive request to /run/echo",
    "2019/06/05 18:40:05 handler hits: 8424, import hits: 0, misses: 1",
    "2019/06/05 18:40:05 Receive request to /run/echo",
    "2019/06/05 18:40:05 handler hits: 8425, import hits: 0, misses: 1",
    "2019/06/05 18:40:06 Receive request to /run/echo",
    "2019/06/05 18:40:06 Receive request to /run/echo",
    "2019/06/05 18:40:06 handler hits: 8426, import hits: 0, misses: 1",
    "2019/06/05 18:40:06 handler hits: 8427, import hits: 0, misses: 1",
    "2019/06/05 18:40:06 Receive request to /run/echo",
    "2019/06/05 18:40:06 handler hits: 8428, import hits: 0, misses: 1",
    "2019/06/05 18:40:06 Receive request to /run/echo",
    "2019/06/05 18:40:06 handler hits: 8429, import hits: 0, misses: 1",
    "2019/06/05 18:40:06 Receive request to /run/echo",
    "2019/06/05 18:40:06 handler hits: 8430, import hits: 0, misses: 1",
    "2019/06/05 18:40:06 Receive request to /run/echo",
    "2019/06/05 18:40:06 handler hits: 8431, import hits: 0, misses: 1",
    ""
  ]
}
Kill worker process with PID 9559
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=9619, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
Kill worker process with PID 9619
2019/06/05 18:45:24 worker didn't stop after 30s
{
  "test": "stress_one_lambda",
  "params": {
    "procs": 8,
    "seconds": 15
  },
  "pass": true,
  "seconds": 15.068202495574951,
  "stats": {
    "reqs_per_sec": 253.6
  },
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "pip_mirror": "",
    "handler_cache_mb": 256,
    "sandbox": "sock",
    "timing": false,
    "max_runners": 0,
    "startup_pkgs": null,
    "registry_cache_ms": 5000,
    "registry": "/root/open-lambda/test-registry",
    "docker_runtime": "",
    "cg_pool_size": 10,
    "sandbox_config": {
      "processes": 10
    },
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "import_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "2019/06/05 18:40:21 handler hits: 12226, import hits: 0, misses: 1",
    "2019/06/05 18:40:21 Receive request to /run/echo",
    "2019/06/05 18:40:21 handler hits: 12227, import hits: 0, misses: 1",
    "2019/06/05 18:40:21 Receive request to /run/echo",
    "2019/06/05 18:40:21 handler hits: 12228, import hits: 0, misses: 1",
    "2019/06/05 18:40:21 Receive request to /run/echo",
    "2019/06/05 18:40:21 handler hits: 12229, import hits: 0, misses: 1",
    "2019/06/05 18:40:21 Receive request to /run/echo",
    "2019/06/05 18:40:21 handler hits: 12230, import hits: 0, misses: 1",
    "2019/06/05 18:40:21 Receive request to /run/echo",
    "2019/06/05 18:40:21 handler hits: 12231, import hits: 0, misses: 1",
    "2019/06/05 18:40:21 Receive request to /run/echo",
    "2019/06/05 18:40:21 handler hits: 12232, import hits: 0, misses: 1",
    "2019/06/05 18:40:21 Receive request to /run/echo",
    "2019/06/05 18:40:21 handler hits: 12233, import hits: 0, misses: 1",
    "2019/06/05 18:40:21 Receive request to /run/echo",
    "2019/06/05 18:40:21 handler hits: 12234, import hits: 0, misses: 1",
    "2019/06/05 18:40:21 Receive request to /run/echo",
    "2019/06/05 18:40:21 handler hits: 12235, import hits: 0, misses: 1",
    ""
  ]
}
POP conf: {'handler_cache_mb': 256, 'import_cache_mb': 256, 'registry': '/root/open-lambda/test-registry', 'cg_pool_size': 10, 'sandbox': 'sock'}
RUN ./ol kill -p=test-cluster
PUSH conf: {'import_cache_mb': 256, 'handler_cache_mb': 256, 'cg_pool_size': 10, 'sandbox': 'sock'}
PUSH conf: {'registry': '/tmp/tmp16d0x7t7'}
RUN ./ol worker -p=test-cluster --detach
{
  "test": "call_each_once_exec",
  "params": {
    "lambda_count": 100
  },
  "pass": true,
  "seconds": 2.7982985973358154,
  "stats": {
    "reqs_per_sec": 35.73637802700926
  },
  "conf": {
    "Skip_pull_existing": false,
    "worker_port": "5000",
    "handler_cache_mb": 256,
    "max_runners": 0,
    "sandbox_config": {
      "processes": 10
    },
    "timing": false,
    "registry_cache_ms": 5000,
    "sandbox": "sock",
    "startup_pkgs": null,
    "cg_pool_size": 10,
    "docker_runtime": "",
    "cluster_name": "/root/open-lambda/test-cluster",
    "pip_mirror": "",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "benchmark_log": "",
    "import_cache_mb": 256,
    "registry": "/tmp/tmp16d0x7t7",
    "worker_dir": "/root/open-lambda/test-cluster/worker"
  },
  "exception": null,
  "worker_tail": [
    "2019/06/05 18:40:24 Receive request to /run/L93",
    "2019/06/05 18:40:24 handler hits: 0, import hits: 0, misses: 94",
    "2019/06/05 18:40:24 EVICTING INSTANCE: 268460032 used / 268435456 limit",
    "2019/06/05 18:40:24 Receive request to /run/L94",
    "2019/06/05 18:40:24 handler hits: 0, import hits: 0, misses: 95",
    "2019/06/05 18:40:24 EVICTING INSTANCE: 268615680 used / 268435456 limit",
    "2019/06/05 18:40:24 Receive request to /run/L95",
    "2019/06/05 18:40:24 handler hits: 0, import hits: 0, misses: 96",
    "2019/06/05 18:40:24 Receive request to /run/L96",
    "2019/06/05 18:40:24 handler hits: 0, import hits: 0, misses: 97",
    "2019/06/05 18:40:24 EVICTING INSTANCE: 270753792 used / 268435456 limit",
    "2019/06/05 18:40:24 Receive request to /run/L97",
    "2019/06/05 18:40:24 handler hits: 0, import hits: 0, misses: 98",
    "2019/06/05 18:40:24 EVICTING INSTANCE: 269479936 used / 268435456 limit",
    "2019/06/05 18:40:24 Receive request to /run/L98",
    "2019/06/05 18:40:24 handler hits: 0, import hits: 0, misses: 99",
    "2019/06/05 18:40:24 EVICTING INSTANCE: 270397440 used / 268435456 limit",
    "2019/06/05 18:40:24 Receive request to /run/L99",
    "2019/06/05 18:40:24 handler hits: 0, import hits: 0, misses: 100",
    ""
  ]
}
POP conf: {'registry': '/tmp/tmp16d0x7t7'}
RUN ./ol kill -p=test-cluster
Traceback (most recent call last):
  File "test.py", line 221, in <module>
    main()
  File "test.py", line 204, in main
    tests()
  File "test.py", line 187, in tests
    call_each_once(lambda_count=100)
  File "test.py", line 159, in call_each_once
    call_each_once_exec(lambda_count=lambda_count)
  File "/usr/lib/python3.5/contextlib.py", line 66, in __exit__
    next(self.gen)
  File "test.py", line 77, in TestConf
    run(['./ol', 'kill', '-p='+OLDIR])
  File "test.py", line 86, in run
    raise Exception("command failed: " + " ".join(cmd))
Exception: command failed: ./ol kill -p=test-cluster
Makefile:28: recipe for target 'test-all' failed
make: *** [test-all] Error 1