python3 -u test.py
RUN ./ol new -p=test-cluster
Init OL dir at /root/open-lambda/test-cluster
Create lambda base at /root/open-lambda/test-cluster/lambda (may take several minutes)
Working Directory: /root/open-lambda/test-cluster

Worker Defaults: 
{
	"registry": "/root/open-lambda/test-cluster/registry",
	"registry_cache_ms": 5000,
	"sandbox": "sock",
	"cluster_name": "/root/open-lambda/test-cluster",
	"pip_mirror": "",
	"Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
	"max_runners": 0,
	"handler_cache_mb": 239,
	"import_cache_mb": 239,
	"worker_dir": "/root/open-lambda/test-cluster/worker",
	"SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
	"worker_port": "5000",
	"cg_pool_size": 0,
	"Skip_pull_existing": false,
	"sandbox_config": {
		"processes": 10
	},
	"benchmark_log": "",
	"timing": false,
	"startup_pkgs": null,
	"docker_runtime": ""
}

You may now start a server using the "worker" command
PUSH conf: {'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson'], 'registry': '/root/open-lambda/test-registry'}
PUSH conf: {'import_cache_mb': 0, 'handler_cache_mb': 0, 'cg_pool_size': 10, 'sandbox': 'sock'}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=7372, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 1.5166890621185303,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "sandbox": "sock",
    "worker_port": "5000",
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "cg_pool_size": 10,
    "benchmark_log": "",
    "timing": false,
    "Skip_pull_existing": false
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 19:48:50 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:48:50 Initialized registry manager (took 20.013\u00b5s)",
    "2019/06/05 19:49:06 Initialized installation manager (took 16.620962581s)",
    "2019/06/05 19:49:06 Initialized handler container factory (took 8.739454ms)",
    "2019/06/05 19:49:06 Initialized cache manager (took 960ns)",
    "2019/06/05 19:49:06 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:49:06 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:49:06 Receive request to /pid",
    "2019/06/05 19:49:06 Receive request to /run/echo",
    "2019/06/05 19:49:08 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 1.045091152191162,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "sandbox": "sock",
    "worker_port": "5000",
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "cg_pool_size": 10,
    "benchmark_log": "",
    "timing": false,
    "Skip_pull_existing": false
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 19:48:50 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:48:50 Initialized registry manager (took 20.013\u00b5s)",
    "2019/06/05 19:49:06 Initialized installation manager (took 16.620962581s)",
    "2019/06/05 19:49:06 Initialized handler container factory (took 8.739454ms)",
    "2019/06/05 19:49:06 Initialized cache manager (took 960ns)",
    "2019/06/05 19:49:06 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:49:06 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:49:06 Receive request to /pid",
    "2019/06/05 19:49:06 Receive request to /run/echo",
    "2019/06/05 19:49:08 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:49:08 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:09 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.3816804885864258,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "sandbox": "sock",
    "worker_port": "5000",
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "cg_pool_size": 10,
    "benchmark_log": "",
    "timing": false,
    "Skip_pull_existing": false
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 19:48:50 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:48:50 Initialized registry manager (took 20.013\u00b5s)",
    "2019/06/05 19:49:06 Initialized installation manager (took 16.620962581s)",
    "2019/06/05 19:49:06 Initialized handler container factory (took 8.739454ms)",
    "2019/06/05 19:49:06 Initialized cache manager (took 960ns)",
    "2019/06/05 19:49:06 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:49:06 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:49:06 Receive request to /pid",
    "2019/06/05 19:49:06 Receive request to /run/echo",
    "2019/06/05 19:49:08 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:49:08 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:09 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:49:09 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:09 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.3717372417449951,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "sandbox": "sock",
    "worker_port": "5000",
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "cg_pool_size": 10,
    "benchmark_log": "",
    "timing": false,
    "Skip_pull_existing": false
  },
  "exception": null,
  "worker_tail": [
    "2019/06/05 19:48:50 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:48:50 Initialized registry manager (took 20.013\u00b5s)",
    "2019/06/05 19:49:06 Initialized installation manager (took 16.620962581s)",
    "2019/06/05 19:49:06 Initialized handler container factory (took 8.739454ms)",
    "2019/06/05 19:49:06 Initialized cache manager (took 960ns)",
    "2019/06/05 19:49:06 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:49:06 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:49:06 Receive request to /pid",
    "2019/06/05 19:49:06 Receive request to /run/echo",
    "2019/06/05 19:49:08 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:49:08 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:09 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:49:09 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:09 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:49:09 Receive request to /run/install3",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:10 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
POP conf: {'import_cache_mb': 0, 'handler_cache_mb': 0, 'cg_pool_size': 10, 'sandbox': 'sock'}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 7372
PUSH conf: {'import_cache_mb': 0, 'handler_cache_mb': 256, 'cg_pool_size': 10, 'sandbox': 'sock'}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=7697, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 0.15454530715942383,
  "stats": null,
  "conf": {
    "cg_pool_size": 10,
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "Skip_pull_existing": false,
    "import_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "handler_cache_mb": 256,
    "sandbox": "sock",
    "worker_port": "5000",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "benchmark_log": "",
    "timing": false,
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 19:49:10 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:49:10 Initialized registry manager (took 20.845\u00b5s)",
    "2019/06/05 19:49:17 Initialized installation manager (took 7.624326115s)",
    "2019/06/05 19:49:17 Initialized handler container factory (took 4.927461ms)",
    "2019/06/05 19:49:17 Initialized cache manager (took 666ns)",
    "2019/06/05 19:49:17 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:49:17 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:49:17 Receive request to /pid",
    "2019/06/05 19:49:17 Receive request to /run/echo",
    "2019/06/05 19:49:17 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 0.37699437141418457,
  "stats": null,
  "conf": {
    "cg_pool_size": 10,
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "Skip_pull_existing": false,
    "import_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "handler_cache_mb": 256,
    "sandbox": "sock",
    "worker_port": "5000",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "benchmark_log": "",
    "timing": false,
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 19:49:10 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:49:10 Initialized registry manager (took 20.845\u00b5s)",
    "2019/06/05 19:49:17 Initialized installation manager (took 7.624326115s)",
    "2019/06/05 19:49:17 Initialized handler container factory (took 4.927461ms)",
    "2019/06/05 19:49:17 Initialized cache manager (took 666ns)",
    "2019/06/05 19:49:17 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:49:17 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:49:17 Receive request to /pid",
    "2019/06/05 19:49:17 Receive request to /run/echo",
    "2019/06/05 19:49:17 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:49:17 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:18 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.3708462715148926,
  "stats": null,
  "conf": {
    "cg_pool_size": 10,
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "Skip_pull_existing": false,
    "import_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "handler_cache_mb": 256,
    "sandbox": "sock",
    "worker_port": "5000",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "benchmark_log": "",
    "timing": false,
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 19:49:10 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:49:10 Initialized registry manager (took 20.845\u00b5s)",
    "2019/06/05 19:49:17 Initialized installation manager (took 7.624326115s)",
    "2019/06/05 19:49:17 Initialized handler container factory (took 4.927461ms)",
    "2019/06/05 19:49:17 Initialized cache manager (took 666ns)",
    "2019/06/05 19:49:17 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:49:17 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:49:17 Receive request to /pid",
    "2019/06/05 19:49:17 Receive request to /run/echo",
    "2019/06/05 19:49:17 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:49:17 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:18 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:49:18 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:18 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 1.0309770107269287,
  "stats": null,
  "conf": {
    "cg_pool_size": 10,
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "Skip_pull_existing": false,
    "import_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "handler_cache_mb": 256,
    "sandbox": "sock",
    "worker_port": "5000",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "benchmark_log": "",
    "timing": false,
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "2019/06/05 19:49:10 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:49:10 Initialized registry manager (took 20.845\u00b5s)",
    "2019/06/05 19:49:17 Initialized installation manager (took 7.624326115s)",
    "2019/06/05 19:49:17 Initialized handler container factory (took 4.927461ms)",
    "2019/06/05 19:49:17 Initialized cache manager (took 666ns)",
    "2019/06/05 19:49:17 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:49:17 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:49:17 Receive request to /pid",
    "2019/06/05 19:49:17 Receive request to /run/echo",
    "2019/06/05 19:49:17 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:49:17 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:18 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:49:18 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:18 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:49:18 Receive request to /run/install3",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:19 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
POP conf: {'import_cache_mb': 0, 'handler_cache_mb': 256, 'cg_pool_size': 10, 'sandbox': 'sock'}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 7697
PUSH conf: {'import_cache_mb': 256, 'handler_cache_mb': 0, 'cg_pool_size': 10, 'sandbox': 'sock'}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=7772, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 0.03064703941345215,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "sandbox": "sock",
    "worker_port": "5000",
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "cg_pool_size": 10,
    "benchmark_log": "",
    "timing": false,
    "Skip_pull_existing": false
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 19:49:19 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:49:19 Initialized registry manager (took 17.457\u00b5s)",
    "2019/06/05 19:49:27 Initialized installation manager (took 7.488379961s)",
    "2019/06/05 19:49:27 Initialized handler container factory (took 4.863177ms)",
    "2019/06/05 19:49:27 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/05 19:49:27 Initialized cache manager (took 181.563172ms)",
    "2019/06/05 19:49:27 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:49:27 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:49:27 Receive request to /pid",
    "2019/06/05 19:49:27 Receive request to /run/echo",
    "2019/06/05 19:49:27 handler hits: 0, import hits: 0, misses: 1",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 0.2576873302459717,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "sandbox": "sock",
    "worker_port": "5000",
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "cg_pool_size": 10,
    "benchmark_log": "",
    "timing": false,
    "Skip_pull_existing": false
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 19:49:19 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:49:19 Initialized registry manager (took 17.457\u00b5s)",
    "2019/06/05 19:49:27 Initialized installation manager (took 7.488379961s)",
    "2019/06/05 19:49:27 Initialized handler container factory (took 4.863177ms)",
    "2019/06/05 19:49:27 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/05 19:49:27 Initialized cache manager (took 181.563172ms)",
    "2019/06/05 19:49:27 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:49:27 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:49:27 Receive request to /pid",
    "2019/06/05 19:49:27 Receive request to /run/echo",
    "2019/06/05 19:49:27 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/05 19:49:27 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:27 handler hits: 0, import hits: 0, misses: 2",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.2516934871673584,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "sandbox": "sock",
    "worker_port": "5000",
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "cg_pool_size": 10,
    "benchmark_log": "",
    "timing": false,
    "Skip_pull_existing": false
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 19:49:19 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:49:19 Initialized registry manager (took 17.457\u00b5s)",
    "2019/06/05 19:49:27 Initialized installation manager (took 7.488379961s)",
    "2019/06/05 19:49:27 Initialized handler container factory (took 4.863177ms)",
    "2019/06/05 19:49:27 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/05 19:49:27 Initialized cache manager (took 181.563172ms)",
    "2019/06/05 19:49:27 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:49:27 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:49:27 Receive request to /pid",
    "2019/06/05 19:49:27 Receive request to /run/echo",
    "2019/06/05 19:49:27 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/05 19:49:27 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:27 handler hits: 0, import hits: 0, misses: 2",
    "2019/06/05 19:49:27 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:28 handler hits: 0, import hits: 0, misses: 3",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.2531881332397461,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "sandbox": "sock",
    "worker_port": "5000",
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "cg_pool_size": 10,
    "benchmark_log": "",
    "timing": false,
    "Skip_pull_existing": false
  },
  "exception": null,
  "worker_tail": [
    "2019/06/05 19:49:19 Initialized registry manager (took 17.457\u00b5s)",
    "2019/06/05 19:49:27 Initialized installation manager (took 7.488379961s)",
    "2019/06/05 19:49:27 Initialized handler container factory (took 4.863177ms)",
    "2019/06/05 19:49:27 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/05 19:49:27 Initialized cache manager (took 181.563172ms)",
    "2019/06/05 19:49:27 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:49:27 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:49:27 Receive request to /pid",
    "2019/06/05 19:49:27 Receive request to /run/echo",
    "2019/06/05 19:49:27 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/05 19:49:27 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:27 handler hits: 0, import hits: 0, misses: 2",
    "2019/06/05 19:49:27 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:28 handler hits: 0, import hits: 0, misses: 3",
    "2019/06/05 19:49:28 Receive request to /run/install3",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:28 handler hits: 0, import hits: 0, misses: 4",
    ""
  ]
}
POP conf: {'import_cache_mb': 256, 'handler_cache_mb': 0, 'cg_pool_size': 10, 'sandbox': 'sock'}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 7772
PUSH conf: {'import_cache_mb': 256, 'handler_cache_mb': 256, 'cg_pool_size': 10, 'sandbox': 'sock'}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=7856, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 0.02843308448791504,
  "stats": null,
  "conf": {
    "cg_pool_size": 10,
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "Skip_pull_existing": false,
    "import_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "handler_cache_mb": 256,
    "sandbox": "sock",
    "worker_port": "5000",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "benchmark_log": "",
    "timing": false,
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 19:49:28 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:49:28 Initialized registry manager (took 20.231\u00b5s)",
    "2019/06/05 19:49:36 Initialized installation manager (took 7.465694607s)",
    "2019/06/05 19:49:36 Initialized handler container factory (took 4.734527ms)",
    "2019/06/05 19:49:36 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/05 19:49:36 Initialized cache manager (took 148.977075ms)",
    "2019/06/05 19:49:36 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:49:36 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:49:36 Receive request to /pid",
    "2019/06/05 19:49:36 Receive request to /run/echo",
    "2019/06/05 19:49:36 handler hits: 0, import hits: 0, misses: 1",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 0.2497549057006836,
  "stats": null,
  "conf": {
    "cg_pool_size": 10,
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "Skip_pull_existing": false,
    "import_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "handler_cache_mb": 256,
    "sandbox": "sock",
    "worker_port": "5000",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "benchmark_log": "",
    "timing": false,
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 19:49:28 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:49:28 Initialized registry manager (took 20.231\u00b5s)",
    "2019/06/05 19:49:36 Initialized installation manager (took 7.465694607s)",
    "2019/06/05 19:49:36 Initialized handler container factory (took 4.734527ms)",
    "2019/06/05 19:49:36 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/05 19:49:36 Initialized cache manager (took 148.977075ms)",
    "2019/06/05 19:49:36 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:49:36 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:49:36 Receive request to /pid",
    "2019/06/05 19:49:36 Receive request to /run/echo",
    "2019/06/05 19:49:36 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/05 19:49:36 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:36 handler hits: 0, import hits: 0, misses: 2",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.24963140487670898,
  "stats": null,
  "conf": {
    "cg_pool_size": 10,
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "Skip_pull_existing": false,
    "import_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "handler_cache_mb": 256,
    "sandbox": "sock",
    "worker_port": "5000",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "benchmark_log": "",
    "timing": false,
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 19:49:28 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:49:28 Initialized registry manager (took 20.231\u00b5s)",
    "2019/06/05 19:49:36 Initialized installation manager (took 7.465694607s)",
    "2019/06/05 19:49:36 Initialized handler container factory (took 4.734527ms)",
    "2019/06/05 19:49:36 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/05 19:49:36 Initialized cache manager (took 148.977075ms)",
    "2019/06/05 19:49:36 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:49:36 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:49:36 Receive request to /pid",
    "2019/06/05 19:49:36 Receive request to /run/echo",
    "2019/06/05 19:49:36 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/05 19:49:36 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:36 handler hits: 0, import hits: 0, misses: 2",
    "2019/06/05 19:49:36 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:36 handler hits: 0, import hits: 0, misses: 3",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.25514960289001465,
  "stats": null,
  "conf": {
    "cg_pool_size": 10,
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "Skip_pull_existing": false,
    "import_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "handler_cache_mb": 256,
    "sandbox": "sock",
    "worker_port": "5000",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "benchmark_log": "",
    "timing": false,
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "2019/06/05 19:49:28 Initialized registry manager (took 20.231\u00b5s)",
    "2019/06/05 19:49:36 Initialized installation manager (took 7.465694607s)",
    "2019/06/05 19:49:36 Initialized handler container factory (took 4.734527ms)",
    "2019/06/05 19:49:36 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/05 19:49:36 Initialized cache manager (took 148.977075ms)",
    "2019/06/05 19:49:36 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:49:36 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:49:36 Receive request to /pid",
    "2019/06/05 19:49:36 Receive request to /run/echo",
    "2019/06/05 19:49:36 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/05 19:49:36 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:36 handler hits: 0, import hits: 0, misses: 2",
    "2019/06/05 19:49:36 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:36 handler hits: 0, import hits: 0, misses: 3",
    "2019/06/05 19:49:36 Receive request to /run/install3",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/05 19:49:36 handler hits: 0, import hits: 0, misses: 4",
    ""
  ]
}
POP conf: {'import_cache_mb': 256, 'handler_cache_mb': 256, 'cg_pool_size': 10, 'sandbox': 'sock'}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 7856
PUSH conf: {'import_cache_mb': 0, 'handler_cache_mb': 0, 'cg_pool_size': 0, 'sandbox': 'docker'}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=7940, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 5.10062575340271,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "sandbox": "docker",
    "worker_port": "5000",
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "cg_pool_size": 0,
    "benchmark_log": "",
    "timing": false,
    "Skip_pull_existing": false
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 19:49:37 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:49:37 Initialized registry manager (took 18.674\u00b5s)",
    "2019/06/05 19:49:44 Initialized installation manager (took 7.455442121s)",
    "2019/06/05 19:49:44 Initialized handler container factory (took 39.648\u00b5s)",
    "2019/06/05 19:49:44 Initialized cache manager (took 633ns)",
    "2019/06/05 19:49:44 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:49:44 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:49:44 Receive request to /pid",
    "2019/06/05 19:49:44 Receive request to /run/echo",
    "2019/06/05 19:49:49 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 1.3298871517181396,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "sandbox": "docker",
    "worker_port": "5000",
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "cg_pool_size": 0,
    "benchmark_log": "",
    "timing": false,
    "Skip_pull_existing": false
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 19:49:37 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:49:37 Initialized registry manager (took 18.674\u00b5s)",
    "2019/06/05 19:49:44 Initialized installation manager (took 7.455442121s)",
    "2019/06/05 19:49:44 Initialized handler container factory (took 39.648\u00b5s)",
    "2019/06/05 19:49:44 Initialized cache manager (took 633ns)",
    "2019/06/05 19:49:44 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:49:44 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:49:44 Receive request to /pid",
    "2019/06/05 19:49:44 Receive request to /run/echo",
    "2019/06/05 19:49:49 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:49:49 Receive request to /run/install",
    "2019/06/05 19:49:51 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.9159324169158936,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "sandbox": "docker",
    "worker_port": "5000",
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "cg_pool_size": 0,
    "benchmark_log": "",
    "timing": false,
    "Skip_pull_existing": false
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 19:49:37 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:49:37 Initialized registry manager (took 18.674\u00b5s)",
    "2019/06/05 19:49:44 Initialized installation manager (took 7.455442121s)",
    "2019/06/05 19:49:44 Initialized handler container factory (took 39.648\u00b5s)",
    "2019/06/05 19:49:44 Initialized cache manager (took 633ns)",
    "2019/06/05 19:49:44 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:49:44 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:49:44 Receive request to /pid",
    "2019/06/05 19:49:44 Receive request to /run/echo",
    "2019/06/05 19:49:49 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:49:49 Receive request to /run/install",
    "2019/06/05 19:49:51 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:49:51 Receive request to /run/install2",
    "2019/06/05 19:49:51 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.9115808010101318,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 0,
    "sandbox": "docker",
    "worker_port": "5000",
    "max_runners": 0,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "cg_pool_size": 0,
    "benchmark_log": "",
    "timing": false,
    "Skip_pull_existing": false
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 19:49:37 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:49:37 Initialized registry manager (took 18.674\u00b5s)",
    "2019/06/05 19:49:44 Initialized installation manager (took 7.455442121s)",
    "2019/06/05 19:49:44 Initialized handler container factory (took 39.648\u00b5s)",
    "2019/06/05 19:49:44 Initialized cache manager (took 633ns)",
    "2019/06/05 19:49:44 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:49:44 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:49:44 Receive request to /pid",
    "2019/06/05 19:49:44 Receive request to /run/echo",
    "2019/06/05 19:49:49 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:49:49 Receive request to /run/install",
    "2019/06/05 19:49:51 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:49:51 Receive request to /run/install2",
    "2019/06/05 19:49:51 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:49:51 Receive request to /run/install3",
    "2019/06/05 19:49:52 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
POP conf: {'import_cache_mb': 0, 'handler_cache_mb': 0, 'cg_pool_size': 0, 'sandbox': 'docker'}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 7940
PUSH conf: {'import_cache_mb': 0, 'handler_cache_mb': 256, 'cg_pool_size': 0, 'sandbox': 'docker'}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=8882, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 0.5759029388427734,
  "stats": null,
  "conf": {
    "cg_pool_size": 0,
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "Skip_pull_existing": false,
    "import_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "handler_cache_mb": 256,
    "sandbox": "docker",
    "worker_port": "5000",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "benchmark_log": "",
    "timing": false,
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 19:49:52 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:49:52 Initialized registry manager (took 18.786\u00b5s)",
    "2019/06/05 19:50:00 Initialized installation manager (took 7.451525934s)",
    "2019/06/05 19:50:00 Initialized handler container factory (took 28.402\u00b5s)",
    "2019/06/05 19:50:00 Initialized cache manager (took 598ns)",
    "2019/06/05 19:50:00 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:50:00 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:50:00 Receive request to /pid",
    "2019/06/05 19:50:00 Receive request to /run/echo",
    "2019/06/05 19:50:00 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 1.0314252376556396,
  "stats": null,
  "conf": {
    "cg_pool_size": 0,
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "Skip_pull_existing": false,
    "import_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "handler_cache_mb": 256,
    "sandbox": "docker",
    "worker_port": "5000",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "benchmark_log": "",
    "timing": false,
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 19:49:52 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:49:52 Initialized registry manager (took 18.786\u00b5s)",
    "2019/06/05 19:50:00 Initialized installation manager (took 7.451525934s)",
    "2019/06/05 19:50:00 Initialized handler container factory (took 28.402\u00b5s)",
    "2019/06/05 19:50:00 Initialized cache manager (took 598ns)",
    "2019/06/05 19:50:00 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:50:00 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:50:00 Receive request to /pid",
    "2019/06/05 19:50:00 Receive request to /run/echo",
    "2019/06/05 19:50:00 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:50:01 Receive request to /run/install",
    "2019/06/05 19:50:02 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 1.594724416732788,
  "stats": null,
  "conf": {
    "cg_pool_size": 0,
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "Skip_pull_existing": false,
    "import_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "handler_cache_mb": 256,
    "sandbox": "docker",
    "worker_port": "5000",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "benchmark_log": "",
    "timing": false,
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 19:49:52 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:49:52 Initialized registry manager (took 18.786\u00b5s)",
    "2019/06/05 19:50:00 Initialized installation manager (took 7.451525934s)",
    "2019/06/05 19:50:00 Initialized handler container factory (took 28.402\u00b5s)",
    "2019/06/05 19:50:00 Initialized cache manager (took 598ns)",
    "2019/06/05 19:50:00 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:50:00 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:50:00 Receive request to /pid",
    "2019/06/05 19:50:00 Receive request to /run/echo",
    "2019/06/05 19:50:00 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:50:01 Receive request to /run/install",
    "2019/06/05 19:50:02 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:50:02 Receive request to /run/install2",
    "2019/06/05 19:50:03 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.879688024520874,
  "stats": null,
  "conf": {
    "cg_pool_size": 0,
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "Skip_pull_existing": false,
    "import_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "handler_cache_mb": 256,
    "sandbox": "docker",
    "worker_port": "5000",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "benchmark_log": "",
    "timing": false,
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/05 19:49:52 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/05 19:49:52 Initialized registry manager (took 18.786\u00b5s)",
    "2019/06/05 19:50:00 Initialized installation manager (took 7.451525934s)",
    "2019/06/05 19:50:00 Initialized handler container factory (took 28.402\u00b5s)",
    "2019/06/05 19:50:00 Initialized cache manager (took 598ns)",
    "2019/06/05 19:50:00 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/05 19:50:00 Get status by sending request to localhost:5000/status",
    "2019/06/05 19:50:00 Receive request to /pid",
    "2019/06/05 19:50:00 Receive request to /run/echo",
    "2019/06/05 19:50:00 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:50:01 Receive request to /run/install",
    "2019/06/05 19:50:02 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:50:02 Receive request to /run/install2",
    "2019/06/05 19:50:03 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/05 19:50:03 Receive request to /run/install3",
    "2019/06/05 19:50:04 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
POP conf: {'import_cache_mb': 0, 'handler_cache_mb': 256, 'cg_pool_size': 0, 'sandbox': 'docker'}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 8882
POP conf: {'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson'], 'registry': '/root/open-lambda/test-registry'}
PUSH conf: {'import_cache_mb': 256, 'handler_cache_mb': 256, 'cg_pool_size': 10, 'sandbox': 'sock', 'registry': '/root/open-lambda/test-registry'}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=9912, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "stress_one_lambda",
  "params": {
    "seconds": 15,
    "procs": 1
  },
  "pass": true,
  "seconds": 15.022011041641235,
  "stats": {
    "reqs_per_sec": 283.2
  },
  "conf": {
    "cg_pool_size": 10,
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "Skip_pull_existing": false,
    "import_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "handler_cache_mb": 256,
    "sandbox": "sock",
    "worker_port": "5000",
    "startup_pkgs": null,
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "benchmark_log": "",
    "timing": false,
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "2019/06/05 19:50:21 handler hits: 4238, import hits: 0, misses: 1",
    "2019/06/05 19:50:21 Receive request to /run/echo",
    "2019/06/05 19:50:21 handler hits: 4239, import hits: 0, misses: 1",
    "2019/06/05 19:50:21 Receive request to /run/echo",
    "2019/06/05 19:50:21 handler hits: 4240, import hits: 0, misses: 1",
    "2019/06/05 19:50:21 Receive request to /run/echo",
    "2019/06/05 19:50:21 handler hits: 4241, import hits: 0, misses: 1",
    "2019/06/05 19:50:21 Receive request to /run/echo",
    "2019/06/05 19:50:21 handler hits: 4242, import hits: 0, misses: 1",
    "2019/06/05 19:50:21 Receive request to /run/echo",
    "2019/06/05 19:50:21 handler hits: 4243, import hits: 0, misses: 1",
    "2019/06/05 19:50:21 Receive request to /run/echo",
    "2019/06/05 19:50:21 handler hits: 4244, import hits: 0, misses: 1",
    "2019/06/05 19:50:21 Receive request to /run/echo",
    "2019/06/05 19:50:21 handler hits: 4245, import hits: 0, misses: 1",
    "2019/06/05 19:50:21 Receive request to /run/echo",
    "2019/06/05 19:50:21 handler hits: 4246, import hits: 0, misses: 1",
    "2019/06/05 19:50:21 Receive request to /run/echo",
    "2019/06/05 19:50:21 handler hits: 4247, import hits: 0, misses: 1",
    ""
  ]
}
{
  "test": "stress_one_lambda",
  "params": {
    "seconds": 15,
    "procs": 2
  },
  "pass": true,
  "seconds": 15.05399227142334,
  "stats": {
    "reqs_per_sec": 285.8666666666667
  },
  "conf": {
    "cg_pool_size": 10,
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "Skip_pull_existing": false,
    "import_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "handler_cache_mb": 256,
    "sandbox": "sock",
    "worker_port": "5000",
    "startup_pkgs": null,
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "benchmark_log": "",
    "timing": false,
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "2019/06/05 19:50:36 handler hits: 8526, import hits: 0, misses: 1",
    "2019/06/05 19:50:36 Receive request to /run/echo",
    "2019/06/05 19:50:36 handler hits: 8527, import hits: 0, misses: 1",
    "2019/06/05 19:50:36 Receive request to /run/echo",
    "2019/06/05 19:50:36 handler hits: 8528, import hits: 0, misses: 1",
    "2019/06/05 19:50:36 Receive request to /run/echo",
    "2019/06/05 19:50:36 handler hits: 8529, import hits: 0, misses: 1",
    "2019/06/05 19:50:36 Receive request to /run/echo",
    "2019/06/05 19:50:36 handler hits: 8530, import hits: 0, misses: 1",
    "2019/06/05 19:50:36 Receive request to /run/echo",
    "2019/06/05 19:50:36 handler hits: 8531, import hits: 0, misses: 1",
    "2019/06/05 19:50:36 Receive request to /run/echo",
    "2019/06/05 19:50:36 handler hits: 8532, import hits: 0, misses: 1",
    "2019/06/05 19:50:36 Receive request to /run/echo",
    "2019/06/05 19:50:36 handler hits: 8533, import hits: 0, misses: 1",
    "2019/06/05 19:50:36 Receive request to /run/echo",
    "2019/06/05 19:50:36 handler hits: 8534, import hits: 0, misses: 1",
    "2019/06/05 19:50:36 Receive request to /run/echo",
    "2019/06/05 19:50:36 handler hits: 8535, import hits: 0, misses: 1",
    ""
  ]
}
{
  "test": "stress_one_lambda",
  "params": {
    "seconds": 15,
    "procs": 8
  },
  "pass": true,
  "seconds": 15.069449424743652,
  "stats": {
    "reqs_per_sec": 259.06666666666666
  },
  "conf": {
    "cg_pool_size": 10,
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "Skip_pull_existing": false,
    "import_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "handler_cache_mb": 256,
    "sandbox": "sock",
    "worker_port": "5000",
    "startup_pkgs": null,
    "registry": "/root/open-lambda/test-registry",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "benchmark_log": "",
    "timing": false,
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "2019/06/05 19:50:51 handler hits: 12412, import hits: 0, misses: 1",
    "2019/06/05 19:50:51 Receive request to /run/echo",
    "2019/06/05 19:50:51 handler hits: 12413, import hits: 0, misses: 1",
    "2019/06/05 19:50:51 Receive request to /run/echo",
    "2019/06/05 19:50:51 handler hits: 12414, import hits: 0, misses: 1",
    "2019/06/05 19:50:51 Receive request to /run/echo",
    "2019/06/05 19:50:51 handler hits: 12415, import hits: 0, misses: 1",
    "2019/06/05 19:50:51 Receive request to /run/echo",
    "2019/06/05 19:50:51 handler hits: 12416, import hits: 0, misses: 1",
    "2019/06/05 19:50:51 Receive request to /run/echo",
    "2019/06/05 19:50:51 handler hits: 12417, import hits: 0, misses: 1",
    "2019/06/05 19:50:51 Receive request to /run/echo",
    "2019/06/05 19:50:51 handler hits: 12418, import hits: 0, misses: 1",
    "2019/06/05 19:50:51 Receive request to /run/echo",
    "2019/06/05 19:50:51 handler hits: 12419, import hits: 0, misses: 1",
    "2019/06/05 19:50:51 Receive request to /run/echo",
    "2019/06/05 19:50:51 handler hits: 12420, import hits: 0, misses: 1",
    "2019/06/05 19:50:51 Receive request to /run/echo",
    "2019/06/05 19:50:51 handler hits: 12421, import hits: 0, misses: 1",
    ""
  ]
}
POP conf: {'import_cache_mb': 256, 'handler_cache_mb': 256, 'cg_pool_size': 10, 'sandbox': 'sock', 'registry': '/root/open-lambda/test-registry'}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 9912
PUSH conf: {'import_cache_mb': 256, 'handler_cache_mb': 256, 'cg_pool_size': 10, 'sandbox': 'sock'}
PUSH conf: {'registry': '/tmp/tmpgms8kfxa'}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=9972, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "call_each_once_exec",
  "params": {
    "lambda_count": 100,
    "alloc_mb": 1
  },
  "pass": true,
  "seconds": 4.013422012329102,
  "stats": {
    "reqs_per_sec": 24.916601671762596
  },
  "conf": {
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "registry_cache_ms": 5000,
    "handler_cache_mb": 256,
    "sandbox": "sock",
    "worker_port": "5000",
    "max_runners": 0,
    "startup_pkgs": null,
    "registry": "/tmp/tmpgms8kfxa",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "cg_pool_size": 10,
    "benchmark_log": "",
    "timing": false,
    "Skip_pull_existing": false
  },
  "exception": null,
  "worker_tail": [
    "2019/06/05 19:50:55 handler hits: 0, import hits: 0, misses: 94",
    "2019/06/05 19:50:55 EVICTING INSTANCE: 269565952 used / 268435456 limit",
    "2019/06/05 19:50:55 Receive request to /run/L94",
    "2019/06/05 19:50:55 handler hits: 0, import hits: 0, misses: 95",
    "2019/06/05 19:50:55 EVICTING INSTANCE: 271073280 used / 268435456 limit",
    "2019/06/05 19:50:55 Receive request to /run/L95",
    "2019/06/05 19:50:55 handler hits: 0, import hits: 0, misses: 96",
    "2019/06/05 19:50:55 EVICTING INSTANCE: 270127104 used / 268435456 limit",
    "2019/06/05 19:50:55 Receive request to /run/L96",
    "2019/06/05 19:50:55 handler hits: 0, import hits: 0, misses: 97",
    "2019/06/05 19:50:55 EVICTING INSTANCE: 269049856 used / 268435456 limit",
    "2019/06/05 19:50:55 Receive request to /run/L97",
    "2019/06/05 19:50:55 handler hits: 0, import hits: 0, misses: 98",
    "2019/06/05 19:50:55 EVICTING INSTANCE: 268443648 used / 268435456 limit",
    "2019/06/05 19:50:55 Receive request to /run/L98",
    "2019/06/05 19:50:55 handler hits: 0, import hits: 0, misses: 99",
    "2019/06/05 19:50:55 EVICTING INSTANCE: 268931072 used / 268435456 limit",
    "2019/06/05 19:50:55 Receive request to /run/L99",
    "2019/06/05 19:50:55 handler hits: 0, import hits: 0, misses: 100",
    ""
  ]
}
POP conf: {'registry': '/tmp/tmpgms8kfxa'}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 9972
PUSH conf: {'registry': '/tmp/tmpzrcl89mf'}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=10601, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "call_each_once_exec",
  "params": {
    "lambda_count": 1000,
    "alloc_mb": 10
  },
  "pass": true,
  "seconds": 62.532323598861694,
  "stats": {
    "reqs_per_sec": 15.991737545331517
  },
  "conf": {
    "cg_pool_size": 10,
    "pip_mirror": "",
    "sandbox_config": {
      "processes": 10
    },
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "Skip_pull_existing": false,
    "import_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "docker_runtime": "",
    "handler_cache_mb": 256,
    "sandbox": "sock",
    "worker_port": "5000",
    "startup_pkgs": null,
    "registry": "/tmp/tmpzrcl89mf",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "benchmark_log": "",
    "timing": false,
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "2019/06/05 19:51:59 handler hits: 0, import hits: 0, misses: 994",
    "2019/06/05 19:51:59 EVICTING INSTANCE: 270741504 used / 268435456 limit",
    "2019/06/05 19:51:59 Receive request to /run/L994",
    "2019/06/05 19:51:59 handler hits: 0, import hits: 0, misses: 995",
    "2019/06/05 19:51:59 EVICTING INSTANCE: 271396864 used / 268435456 limit",
    "2019/06/05 19:51:59 Receive request to /run/L995",
    "2019/06/05 19:51:59 handler hits: 0, import hits: 0, misses: 996",
    "2019/06/05 19:51:59 EVICTING INSTANCE: 269701120 used / 268435456 limit",
    "2019/06/05 19:51:59 Receive request to /run/L996",
    "2019/06/05 19:51:59 handler hits: 0, import hits: 0, misses: 997",
    "2019/06/05 19:51:59 Receive request to /run/L997",
    "2019/06/05 19:51:59 handler hits: 0, import hits: 0, misses: 998",
    "2019/06/05 19:51:59 EVICTING INSTANCE: 281591808 used / 268435456 limit",
    "2019/06/05 19:51:59 Receive request to /run/L998",
    "2019/06/05 19:51:59 handler hits: 0, import hits: 0, misses: 999",
    "2019/06/05 19:51:59 EVICTING INSTANCE: 281587712 used / 268435456 limit",
    "2019/06/05 19:51:59 EVICTING INSTANCE: 268918784 used / 268435456 limit",
    "2019/06/05 19:51:59 Receive request to /run/L999",
    "2019/06/05 19:51:59 handler hits: 0, import hits: 0, misses: 1000",
    ""
  ]
}
POP conf: {'registry': '/tmp/tmpzrcl89mf'}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 10601
POP conf: {'import_cache_mb': 256, 'handler_cache_mb': 256, 'cg_pool_size': 10, 'sandbox': 'sock'}
PASSED: 29, FAILED: 0