python3 -u test.py
RUN ./ol new -p=test-cluster
Init OL dir at /root/open-lambda/test-cluster
Create lambda base at /root/open-lambda/test-cluster/lambda (may take several minutes)
Working Directory: /root/open-lambda/test-cluster

Worker Defaults: 
{
	"registry": "/root/open-lambda/test-cluster/registry",
	"registry_cache_ms": 5000,
	"sandbox": "sock",
	"cluster_name": "/root/open-lambda/test-cluster",
	"pip_mirror": "",
	"Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
	"max_runners": 0,
	"handler_cache_mb": 239,
	"import_cache_mb": 239,
	"worker_dir": "/root/open-lambda/test-cluster/worker",
	"SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
	"worker_port": "5000",
	"cg_pool_size": 0,
	"Skip_pull_existing": false,
	"sandbox_config": {
		"processes": 10
	},
	"benchmark_log": "",
	"timing": false,
	"startup_pkgs": null,
	"docker_runtime": ""
}

You may now start a server using the "worker" command
PUSH conf: {'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson'], 'registry': '/root/open-lambda/test-registry'}
PUSH conf: {'sandbox': 'sock', 'import_cache_mb': 0, 'handler_cache_mb': 0, 'cg_pool_size': 10}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=7615, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 1.3675930500030518,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "sandbox_config": {
      "processes": 10
    },
    "cg_pool_size": 10,
    "benchmark_log": "",
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 0,
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "registry_cache_ms": 5000,
    "Skip_pull_existing": false,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "sandbox": "sock",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 15:08:50 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:08:50 Initialized registry manager (took 47.867\u00b5s)",
    "2019/06/06 15:09:06 Initialized installation manager (took 15.304733406s)",
    "2019/06/06 15:09:06 Initialized handler container factory (took 5.381631ms)",
    "2019/06/06 15:09:06 Initialized cache manager (took 949ns)",
    "2019/06/06 15:09:06 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:06 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:06 Receive request to /pid",
    "2019/06/06 15:09:06 Receive request to /run/echo",
    "2019/06/06 15:09:07 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 0.8050432205200195,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "sandbox_config": {
      "processes": 10
    },
    "cg_pool_size": 10,
    "benchmark_log": "",
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 0,
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "registry_cache_ms": 5000,
    "Skip_pull_existing": false,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "sandbox": "sock",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 15:08:50 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:08:50 Initialized registry manager (took 47.867\u00b5s)",
    "2019/06/06 15:09:06 Initialized installation manager (took 15.304733406s)",
    "2019/06/06 15:09:06 Initialized handler container factory (took 5.381631ms)",
    "2019/06/06 15:09:06 Initialized cache manager (took 949ns)",
    "2019/06/06 15:09:06 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:06 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:06 Receive request to /pid",
    "2019/06/06 15:09:06 Receive request to /run/echo",
    "2019/06/06 15:09:07 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:09:07 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:08 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.3744626045227051,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "sandbox_config": {
      "processes": 10
    },
    "cg_pool_size": 10,
    "benchmark_log": "",
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 0,
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "registry_cache_ms": 5000,
    "Skip_pull_existing": false,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "sandbox": "sock",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 15:08:50 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:08:50 Initialized registry manager (took 47.867\u00b5s)",
    "2019/06/06 15:09:06 Initialized installation manager (took 15.304733406s)",
    "2019/06/06 15:09:06 Initialized handler container factory (took 5.381631ms)",
    "2019/06/06 15:09:06 Initialized cache manager (took 949ns)",
    "2019/06/06 15:09:06 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:06 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:06 Receive request to /pid",
    "2019/06/06 15:09:06 Receive request to /run/echo",
    "2019/06/06 15:09:07 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:09:07 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:08 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:09:08 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:08 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.3687896728515625,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "sandbox_config": {
      "processes": 10
    },
    "cg_pool_size": 10,
    "benchmark_log": "",
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 0,
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "registry_cache_ms": 5000,
    "Skip_pull_existing": false,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "sandbox": "sock",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages"
  },
  "exception": null,
  "worker_tail": [
    "2019/06/06 15:08:50 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:08:50 Initialized registry manager (took 47.867\u00b5s)",
    "2019/06/06 15:09:06 Initialized installation manager (took 15.304733406s)",
    "2019/06/06 15:09:06 Initialized handler container factory (took 5.381631ms)",
    "2019/06/06 15:09:06 Initialized cache manager (took 949ns)",
    "2019/06/06 15:09:06 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:06 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:06 Receive request to /pid",
    "2019/06/06 15:09:06 Receive request to /run/echo",
    "2019/06/06 15:09:07 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:09:07 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:08 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:09:08 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:08 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:09:08 Receive request to /run/install3",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:09 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
POP conf: {'sandbox': 'sock', 'import_cache_mb': 0, 'handler_cache_mb': 0, 'cg_pool_size': 10}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 7615
PUSH conf: {'sandbox': 'sock', 'import_cache_mb': 0, 'handler_cache_mb': 256, 'cg_pool_size': 10}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=7705, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 0.1501600742340088,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "sandbox_config": {
      "processes": 10
    },
    "sandbox": "sock",
    "cg_pool_size": 10,
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "Skip_pull_existing": false,
    "import_cache_mb": 0,
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "benchmark_log": "",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "max_runners": 0,
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 15:09:09 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:09:09 Initialized registry manager (took 15.974\u00b5s)",
    "2019/06/06 15:09:16 Initialized installation manager (took 7.526904507s)",
    "2019/06/06 15:09:16 Initialized handler container factory (took 4.767697ms)",
    "2019/06/06 15:09:16 Initialized cache manager (took 798ns)",
    "2019/06/06 15:09:16 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:16 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:16 Receive request to /pid",
    "2019/06/06 15:09:16 Receive request to /run/echo",
    "2019/06/06 15:09:16 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 0.3675074577331543,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "sandbox_config": {
      "processes": 10
    },
    "sandbox": "sock",
    "cg_pool_size": 10,
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "Skip_pull_existing": false,
    "import_cache_mb": 0,
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "benchmark_log": "",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "max_runners": 0,
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 15:09:09 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:09:09 Initialized registry manager (took 15.974\u00b5s)",
    "2019/06/06 15:09:16 Initialized installation manager (took 7.526904507s)",
    "2019/06/06 15:09:16 Initialized handler container factory (took 4.767697ms)",
    "2019/06/06 15:09:16 Initialized cache manager (took 798ns)",
    "2019/06/06 15:09:16 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:16 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:16 Receive request to /pid",
    "2019/06/06 15:09:16 Receive request to /run/echo",
    "2019/06/06 15:09:16 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:09:16 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:17 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.3676929473876953,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "sandbox_config": {
      "processes": 10
    },
    "sandbox": "sock",
    "cg_pool_size": 10,
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "Skip_pull_existing": false,
    "import_cache_mb": 0,
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "benchmark_log": "",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "max_runners": 0,
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 15:09:09 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:09:09 Initialized registry manager (took 15.974\u00b5s)",
    "2019/06/06 15:09:16 Initialized installation manager (took 7.526904507s)",
    "2019/06/06 15:09:16 Initialized handler container factory (took 4.767697ms)",
    "2019/06/06 15:09:16 Initialized cache manager (took 798ns)",
    "2019/06/06 15:09:16 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:16 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:16 Receive request to /pid",
    "2019/06/06 15:09:16 Receive request to /run/echo",
    "2019/06/06 15:09:16 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:09:16 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:17 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:09:17 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:17 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 1.085789442062378,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "sandbox_config": {
      "processes": 10
    },
    "sandbox": "sock",
    "cg_pool_size": 10,
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "Skip_pull_existing": false,
    "import_cache_mb": 0,
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "benchmark_log": "",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "max_runners": 0,
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "2019/06/06 15:09:09 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:09:09 Initialized registry manager (took 15.974\u00b5s)",
    "2019/06/06 15:09:16 Initialized installation manager (took 7.526904507s)",
    "2019/06/06 15:09:16 Initialized handler container factory (took 4.767697ms)",
    "2019/06/06 15:09:16 Initialized cache manager (took 798ns)",
    "2019/06/06 15:09:16 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:16 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:16 Receive request to /pid",
    "2019/06/06 15:09:16 Receive request to /run/echo",
    "2019/06/06 15:09:16 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:09:16 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:17 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:09:17 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:17 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:09:17 Receive request to /run/install3",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:18 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
POP conf: {'sandbox': 'sock', 'import_cache_mb': 0, 'handler_cache_mb': 256, 'cg_pool_size': 10}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 7705
PUSH conf: {'sandbox': 'sock', 'import_cache_mb': 256, 'handler_cache_mb': 0, 'cg_pool_size': 10}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=7782, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 0.028993606567382812,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "sandbox_config": {
      "processes": 10
    },
    "cg_pool_size": 10,
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 256,
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "registry_cache_ms": 5000,
    "benchmark_log": "",
    "Skip_pull_existing": false,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "sandbox": "sock",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 15:09:18 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:09:18 Initialized registry manager (took 15.58\u00b5s)",
    "2019/06/06 15:09:26 Initialized installation manager (took 7.518178213s)",
    "2019/06/06 15:09:26 Initialized handler container factory (took 4.80123ms)",
    "2019/06/06 15:09:26 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/06 15:09:26 Initialized cache manager (took 187.168299ms)",
    "2019/06/06 15:09:26 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:26 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:26 Receive request to /pid",
    "2019/06/06 15:09:26 Receive request to /run/echo",
    "2019/06/06 15:09:26 handler hits: 0, import hits: 0, misses: 1",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 0.2525784969329834,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "sandbox_config": {
      "processes": 10
    },
    "cg_pool_size": 10,
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 256,
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "registry_cache_ms": 5000,
    "benchmark_log": "",
    "Skip_pull_existing": false,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "sandbox": "sock",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 15:09:18 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:09:18 Initialized registry manager (took 15.58\u00b5s)",
    "2019/06/06 15:09:26 Initialized installation manager (took 7.518178213s)",
    "2019/06/06 15:09:26 Initialized handler container factory (took 4.80123ms)",
    "2019/06/06 15:09:26 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/06 15:09:26 Initialized cache manager (took 187.168299ms)",
    "2019/06/06 15:09:26 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:26 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:26 Receive request to /pid",
    "2019/06/06 15:09:26 Receive request to /run/echo",
    "2019/06/06 15:09:26 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/06 15:09:26 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:26 handler hits: 0, import hits: 0, misses: 2",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.255814790725708,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "sandbox_config": {
      "processes": 10
    },
    "cg_pool_size": 10,
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 256,
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "registry_cache_ms": 5000,
    "benchmark_log": "",
    "Skip_pull_existing": false,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "sandbox": "sock",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 15:09:18 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:09:18 Initialized registry manager (took 15.58\u00b5s)",
    "2019/06/06 15:09:26 Initialized installation manager (took 7.518178213s)",
    "2019/06/06 15:09:26 Initialized handler container factory (took 4.80123ms)",
    "2019/06/06 15:09:26 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/06 15:09:26 Initialized cache manager (took 187.168299ms)",
    "2019/06/06 15:09:26 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:26 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:26 Receive request to /pid",
    "2019/06/06 15:09:26 Receive request to /run/echo",
    "2019/06/06 15:09:26 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/06 15:09:26 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:26 handler hits: 0, import hits: 0, misses: 2",
    "2019/06/06 15:09:26 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:27 handler hits: 0, import hits: 0, misses: 3",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.25731801986694336,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "sandbox_config": {
      "processes": 10
    },
    "cg_pool_size": 10,
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 256,
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "registry_cache_ms": 5000,
    "benchmark_log": "",
    "Skip_pull_existing": false,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "sandbox": "sock",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages"
  },
  "exception": null,
  "worker_tail": [
    "2019/06/06 15:09:18 Initialized registry manager (took 15.58\u00b5s)",
    "2019/06/06 15:09:26 Initialized installation manager (took 7.518178213s)",
    "2019/06/06 15:09:26 Initialized handler container factory (took 4.80123ms)",
    "2019/06/06 15:09:26 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/06 15:09:26 Initialized cache manager (took 187.168299ms)",
    "2019/06/06 15:09:26 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:26 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:26 Receive request to /pid",
    "2019/06/06 15:09:26 Receive request to /run/echo",
    "2019/06/06 15:09:26 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/06 15:09:26 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:26 handler hits: 0, import hits: 0, misses: 2",
    "2019/06/06 15:09:26 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:27 handler hits: 0, import hits: 0, misses: 3",
    "2019/06/06 15:09:27 Receive request to /run/install3",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:27 handler hits: 0, import hits: 0, misses: 4",
    ""
  ]
}
POP conf: {'sandbox': 'sock', 'import_cache_mb': 256, 'handler_cache_mb': 0, 'cg_pool_size': 10}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 7782
PUSH conf: {'sandbox': 'sock', 'import_cache_mb': 256, 'handler_cache_mb': 256, 'cg_pool_size': 10}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=7867, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 0.02881598472595215,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "sandbox_config": {
      "processes": 10
    },
    "sandbox": "sock",
    "cg_pool_size": 10,
    "benchmark_log": "",
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "Skip_pull_existing": false,
    "import_cache_mb": 256,
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "registry_cache_ms": 5000,
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "max_runners": 0,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 15:09:27 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:09:27 Initialized registry manager (took 18.329\u00b5s)",
    "2019/06/06 15:09:35 Initialized installation manager (took 7.567437088s)",
    "2019/06/06 15:09:35 Initialized handler container factory (took 5.150188ms)",
    "2019/06/06 15:09:35 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/06 15:09:35 Initialized cache manager (took 151.283993ms)",
    "2019/06/06 15:09:35 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:35 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:35 Receive request to /pid",
    "2019/06/06 15:09:35 Receive request to /run/echo",
    "2019/06/06 15:09:35 handler hits: 0, import hits: 0, misses: 1",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 0.25420308113098145,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "sandbox_config": {
      "processes": 10
    },
    "sandbox": "sock",
    "cg_pool_size": 10,
    "benchmark_log": "",
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "Skip_pull_existing": false,
    "import_cache_mb": 256,
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "registry_cache_ms": 5000,
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "max_runners": 0,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 15:09:27 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:09:27 Initialized registry manager (took 18.329\u00b5s)",
    "2019/06/06 15:09:35 Initialized installation manager (took 7.567437088s)",
    "2019/06/06 15:09:35 Initialized handler container factory (took 5.150188ms)",
    "2019/06/06 15:09:35 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/06 15:09:35 Initialized cache manager (took 151.283993ms)",
    "2019/06/06 15:09:35 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:35 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:35 Receive request to /pid",
    "2019/06/06 15:09:35 Receive request to /run/echo",
    "2019/06/06 15:09:35 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/06 15:09:35 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:35 handler hits: 0, import hits: 0, misses: 2",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.2503061294555664,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "sandbox_config": {
      "processes": 10
    },
    "sandbox": "sock",
    "cg_pool_size": 10,
    "benchmark_log": "",
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "Skip_pull_existing": false,
    "import_cache_mb": 256,
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "registry_cache_ms": 5000,
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "max_runners": 0,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 15:09:27 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:09:27 Initialized registry manager (took 18.329\u00b5s)",
    "2019/06/06 15:09:35 Initialized installation manager (took 7.567437088s)",
    "2019/06/06 15:09:35 Initialized handler container factory (took 5.150188ms)",
    "2019/06/06 15:09:35 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/06 15:09:35 Initialized cache manager (took 151.283993ms)",
    "2019/06/06 15:09:35 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:35 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:35 Receive request to /pid",
    "2019/06/06 15:09:35 Receive request to /run/echo",
    "2019/06/06 15:09:35 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/06 15:09:35 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:35 handler hits: 0, import hits: 0, misses: 2",
    "2019/06/06 15:09:35 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:35 handler hits: 0, import hits: 0, misses: 3",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.25509142875671387,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "sandbox_config": {
      "processes": 10
    },
    "sandbox": "sock",
    "cg_pool_size": 10,
    "benchmark_log": "",
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "Skip_pull_existing": false,
    "import_cache_mb": 256,
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "registry_cache_ms": 5000,
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "max_runners": 0,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages"
  },
  "exception": null,
  "worker_tail": [
    "2019/06/06 15:09:27 Initialized registry manager (took 18.329\u00b5s)",
    "2019/06/06 15:09:35 Initialized installation manager (took 7.567437088s)",
    "2019/06/06 15:09:35 Initialized handler container factory (took 5.150188ms)",
    "2019/06/06 15:09:35 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/06 15:09:35 Initialized cache manager (took 151.283993ms)",
    "2019/06/06 15:09:35 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:35 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:35 Receive request to /pid",
    "2019/06/06 15:09:35 Receive request to /run/echo",
    "2019/06/06 15:09:35 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/06 15:09:35 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:35 handler hits: 0, import hits: 0, misses: 2",
    "2019/06/06 15:09:35 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:35 handler hits: 0, import hits: 0, misses: 3",
    "2019/06/06 15:09:35 Receive request to /run/install3",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 15:09:36 handler hits: 0, import hits: 0, misses: 4",
    ""
  ]
}
POP conf: {'sandbox': 'sock', 'import_cache_mb': 256, 'handler_cache_mb': 256, 'cg_pool_size': 10}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 7867
PUSH conf: {'sandbox': 'docker', 'import_cache_mb': 0, 'handler_cache_mb': 0, 'cg_pool_size': 0}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=7951, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 4.5381035804748535,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "sandbox_config": {
      "processes": 10
    },
    "cg_pool_size": 0,
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 0,
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "benchmark_log": "",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "Skip_pull_existing": false,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "sandbox": "docker",
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 15:09:36 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:09:36 Initialized registry manager (took 17.301\u00b5s)",
    "2019/06/06 15:09:43 Initialized installation manager (took 7.554585478s)",
    "2019/06/06 15:09:43 Initialized handler container factory (took 38.572\u00b5s)",
    "2019/06/06 15:09:43 Initialized cache manager (took 607ns)",
    "2019/06/06 15:09:43 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:43 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:43 Receive request to /pid",
    "2019/06/06 15:09:43 Receive request to /run/echo",
    "2019/06/06 15:09:48 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 1.6463799476623535,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "sandbox_config": {
      "processes": 10
    },
    "cg_pool_size": 0,
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 0,
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "benchmark_log": "",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "Skip_pull_existing": false,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "sandbox": "docker",
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 15:09:36 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:09:36 Initialized registry manager (took 17.301\u00b5s)",
    "2019/06/06 15:09:43 Initialized installation manager (took 7.554585478s)",
    "2019/06/06 15:09:43 Initialized handler container factory (took 38.572\u00b5s)",
    "2019/06/06 15:09:43 Initialized cache manager (took 607ns)",
    "2019/06/06 15:09:43 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:43 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:43 Receive request to /pid",
    "2019/06/06 15:09:43 Receive request to /run/echo",
    "2019/06/06 15:09:48 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:09:48 Receive request to /run/install",
    "2019/06/06 15:09:49 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.9303979873657227,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "sandbox_config": {
      "processes": 10
    },
    "cg_pool_size": 0,
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 0,
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "benchmark_log": "",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "Skip_pull_existing": false,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "sandbox": "docker",
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 15:09:36 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:09:36 Initialized registry manager (took 17.301\u00b5s)",
    "2019/06/06 15:09:43 Initialized installation manager (took 7.554585478s)",
    "2019/06/06 15:09:43 Initialized handler container factory (took 38.572\u00b5s)",
    "2019/06/06 15:09:43 Initialized cache manager (took 607ns)",
    "2019/06/06 15:09:43 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:43 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:43 Receive request to /pid",
    "2019/06/06 15:09:43 Receive request to /run/echo",
    "2019/06/06 15:09:48 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:09:48 Receive request to /run/install",
    "2019/06/06 15:09:49 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:09:49 Receive request to /run/install2",
    "2019/06/06 15:09:50 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.9261195659637451,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "sandbox_config": {
      "processes": 10
    },
    "cg_pool_size": 0,
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 0,
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "benchmark_log": "",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "Skip_pull_existing": false,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "sandbox": "docker",
    "registry_cache_ms": 5000
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 15:09:36 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:09:36 Initialized registry manager (took 17.301\u00b5s)",
    "2019/06/06 15:09:43 Initialized installation manager (took 7.554585478s)",
    "2019/06/06 15:09:43 Initialized handler container factory (took 38.572\u00b5s)",
    "2019/06/06 15:09:43 Initialized cache manager (took 607ns)",
    "2019/06/06 15:09:43 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:43 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:43 Receive request to /pid",
    "2019/06/06 15:09:43 Receive request to /run/echo",
    "2019/06/06 15:09:48 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:09:48 Receive request to /run/install",
    "2019/06/06 15:09:49 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:09:49 Receive request to /run/install2",
    "2019/06/06 15:09:50 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:09:50 Receive request to /run/install3",
    "2019/06/06 15:09:51 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
POP conf: {'sandbox': 'docker', 'import_cache_mb': 0, 'handler_cache_mb': 0, 'cg_pool_size': 0}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 7951
PUSH conf: {'sandbox': 'docker', 'import_cache_mb': 0, 'handler_cache_mb': 256, 'cg_pool_size': 0}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=8887, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 0.5659711360931396,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "sandbox_config": {
      "processes": 10
    },
    "sandbox": "docker",
    "cg_pool_size": 0,
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "Skip_pull_existing": false,
    "import_cache_mb": 0,
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "registry_cache_ms": 5000,
    "benchmark_log": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "max_runners": 0,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 15:09:51 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:09:51 Initialized registry manager (took 28.958\u00b5s)",
    "2019/06/06 15:09:59 Initialized installation manager (took 7.563254471s)",
    "2019/06/06 15:09:59 Initialized handler container factory (took 26.383\u00b5s)",
    "2019/06/06 15:09:59 Initialized cache manager (took 662ns)",
    "2019/06/06 15:09:59 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:59 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:59 Receive request to /pid",
    "2019/06/06 15:09:59 Receive request to /run/echo",
    "2019/06/06 15:10:00 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 0.9163179397583008,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "sandbox_config": {
      "processes": 10
    },
    "sandbox": "docker",
    "cg_pool_size": 0,
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "Skip_pull_existing": false,
    "import_cache_mb": 0,
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "registry_cache_ms": 5000,
    "benchmark_log": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "max_runners": 0,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 15:09:51 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:09:51 Initialized registry manager (took 28.958\u00b5s)",
    "2019/06/06 15:09:59 Initialized installation manager (took 7.563254471s)",
    "2019/06/06 15:09:59 Initialized handler container factory (took 26.383\u00b5s)",
    "2019/06/06 15:09:59 Initialized cache manager (took 662ns)",
    "2019/06/06 15:09:59 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:59 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:59 Receive request to /pid",
    "2019/06/06 15:09:59 Receive request to /run/echo",
    "2019/06/06 15:10:00 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:10:00 Receive request to /run/install",
    "2019/06/06 15:10:00 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 1.5888073444366455,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "sandbox_config": {
      "processes": 10
    },
    "sandbox": "docker",
    "cg_pool_size": 0,
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "Skip_pull_existing": false,
    "import_cache_mb": 0,
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "registry_cache_ms": 5000,
    "benchmark_log": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "max_runners": 0,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 15:09:51 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:09:51 Initialized registry manager (took 28.958\u00b5s)",
    "2019/06/06 15:09:59 Initialized installation manager (took 7.563254471s)",
    "2019/06/06 15:09:59 Initialized handler container factory (took 26.383\u00b5s)",
    "2019/06/06 15:09:59 Initialized cache manager (took 662ns)",
    "2019/06/06 15:09:59 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:59 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:59 Receive request to /pid",
    "2019/06/06 15:09:59 Receive request to /run/echo",
    "2019/06/06 15:10:00 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:10:00 Receive request to /run/install",
    "2019/06/06 15:10:00 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:10:00 Receive request to /run/install2",
    "2019/06/06 15:10:02 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.7704489231109619,
  "stats": null,
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "sandbox_config": {
      "processes": 10
    },
    "sandbox": "docker",
    "cg_pool_size": 0,
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "Skip_pull_existing": false,
    "import_cache_mb": 0,
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "registry_cache_ms": 5000,
    "benchmark_log": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "docker_runtime": "",
    "max_runners": 0,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages"
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 15:09:51 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 15:09:51 Initialized registry manager (took 28.958\u00b5s)",
    "2019/06/06 15:09:59 Initialized installation manager (took 7.563254471s)",
    "2019/06/06 15:09:59 Initialized handler container factory (took 26.383\u00b5s)",
    "2019/06/06 15:09:59 Initialized cache manager (took 662ns)",
    "2019/06/06 15:09:59 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 15:09:59 Get status by sending request to localhost:5000/status",
    "2019/06/06 15:09:59 Receive request to /pid",
    "2019/06/06 15:09:59 Receive request to /run/echo",
    "2019/06/06 15:10:00 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:10:00 Receive request to /run/install",
    "2019/06/06 15:10:00 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:10:00 Receive request to /run/install2",
    "2019/06/06 15:10:02 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 15:10:02 Receive request to /run/install3",
    "2019/06/06 15:10:03 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
POP conf: {'sandbox': 'docker', 'import_cache_mb': 0, 'handler_cache_mb': 256, 'cg_pool_size': 0}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 8887
POP conf: {'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson'], 'registry': '/root/open-lambda/test-registry'}
PUSH conf: {'registry': '/root/open-lambda/test-registry', 'sandbox': 'sock', 'import_cache_mb': 256, 'handler_cache_mb': 256, 'cg_pool_size': 10}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=9925, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "stress_one_lambda",
  "params": {
    "seconds": 15,
    "procs": 1
  },
  "pass": true,
  "seconds": 15.083466053009033,
  "stats": {
    "reqs_per_sec": 284.06666666666666
  },
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "sandbox_config": {
      "processes": 10
    },
    "sandbox": "sock",
    "cg_pool_size": 10,
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "Skip_pull_existing": false,
    "import_cache_mb": 256,
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "registry_cache_ms": 5000,
    "benchmark_log": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": null,
    "docker_runtime": "",
    "max_runners": 0,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages"
  },
  "exception": null,
  "worker_tail": [
    "2019/06/06 15:10:20 handler hits: 4251, import hits: 0, misses: 1",
    "2019/06/06 15:10:20 Receive request to /run/echo",
    "2019/06/06 15:10:20 handler hits: 4252, import hits: 0, misses: 1",
    "2019/06/06 15:10:20 Receive request to /run/echo",
    "2019/06/06 15:10:20 handler hits: 4253, import hits: 0, misses: 1",
    "2019/06/06 15:10:20 Receive request to /run/echo",
    "2019/06/06 15:10:20 handler hits: 4254, import hits: 0, misses: 1",
    "2019/06/06 15:10:20 Receive request to /run/echo",
    "2019/06/06 15:10:20 handler hits: 4255, import hits: 0, misses: 1",
    "2019/06/06 15:10:20 Receive request to /run/echo",
    "2019/06/06 15:10:20 handler hits: 4256, import hits: 0, misses: 1",
    "2019/06/06 15:10:20 Receive request to /run/echo",
    "2019/06/06 15:10:20 handler hits: 4257, import hits: 0, misses: 1",
    "2019/06/06 15:10:20 Receive request to /run/echo",
    "2019/06/06 15:10:20 handler hits: 4258, import hits: 0, misses: 1",
    "2019/06/06 15:10:20 Receive request to /run/echo",
    "2019/06/06 15:10:20 handler hits: 4259, import hits: 0, misses: 1",
    "2019/06/06 15:10:20 Receive request to /run/echo",
    "2019/06/06 15:10:20 handler hits: 4260, import hits: 0, misses: 1",
    ""
  ]
}
{
  "test": "stress_one_lambda",
  "params": {
    "seconds": 15,
    "procs": 2
  },
  "pass": true,
  "seconds": 15.04795789718628,
  "stats": {
    "reqs_per_sec": 284.6
  },
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "sandbox_config": {
      "processes": 10
    },
    "sandbox": "sock",
    "cg_pool_size": 10,
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "Skip_pull_existing": false,
    "import_cache_mb": 256,
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "registry_cache_ms": 5000,
    "benchmark_log": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": null,
    "docker_runtime": "",
    "max_runners": 0,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages"
  },
  "exception": null,
  "worker_tail": [
    "2019/06/06 15:10:35 handler hits: 8520, import hits: 0, misses: 1",
    "2019/06/06 15:10:35 Receive request to /run/echo",
    "2019/06/06 15:10:35 handler hits: 8521, import hits: 0, misses: 1",
    "2019/06/06 15:10:35 Receive request to /run/echo",
    "2019/06/06 15:10:35 handler hits: 8522, import hits: 0, misses: 1",
    "2019/06/06 15:10:35 Receive request to /run/echo",
    "2019/06/06 15:10:35 handler hits: 8523, import hits: 0, misses: 1",
    "2019/06/06 15:10:35 Receive request to /run/echo",
    "2019/06/06 15:10:35 handler hits: 8524, import hits: 0, misses: 1",
    "2019/06/06 15:10:35 Receive request to /run/echo",
    "2019/06/06 15:10:35 Receive request to /run/echo",
    "2019/06/06 15:10:35 handler hits: 8525, import hits: 0, misses: 1",
    "2019/06/06 15:10:35 handler hits: 8526, import hits: 0, misses: 1",
    "2019/06/06 15:10:35 Receive request to /run/echo",
    "2019/06/06 15:10:35 handler hits: 8527, import hits: 0, misses: 1",
    "2019/06/06 15:10:35 Receive request to /run/echo",
    "2019/06/06 15:10:35 handler hits: 8528, import hits: 0, misses: 1",
    "2019/06/06 15:10:35 Receive request to /run/echo",
    "2019/06/06 15:10:35 handler hits: 8529, import hits: 0, misses: 1",
    ""
  ]
}
{
  "test": "stress_one_lambda",
  "params": {
    "seconds": 15,
    "procs": 8
  },
  "pass": true,
  "seconds": 15.06345534324646,
  "stats": {
    "reqs_per_sec": 261.0
  },
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "sandbox_config": {
      "processes": 10
    },
    "sandbox": "sock",
    "cg_pool_size": 10,
    "timing": false,
    "registry": "/root/open-lambda/test-registry",
    "Skip_pull_existing": false,
    "import_cache_mb": 256,
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "registry_cache_ms": 5000,
    "benchmark_log": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": null,
    "docker_runtime": "",
    "max_runners": 0,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages"
  },
  "exception": null,
  "worker_tail": [
    "2019/06/06 15:10:50 handler hits: 12435, import hits: 0, misses: 1",
    "2019/06/06 15:10:50 Receive request to /run/echo",
    "2019/06/06 15:10:50 handler hits: 12436, import hits: 0, misses: 1",
    "2019/06/06 15:10:50 Receive request to /run/echo",
    "2019/06/06 15:10:50 handler hits: 12437, import hits: 0, misses: 1",
    "2019/06/06 15:10:50 Receive request to /run/echo",
    "2019/06/06 15:10:50 handler hits: 12438, import hits: 0, misses: 1",
    "2019/06/06 15:10:50 Receive request to /run/echo",
    "2019/06/06 15:10:50 handler hits: 12439, import hits: 0, misses: 1",
    "2019/06/06 15:10:50 Receive request to /run/echo",
    "2019/06/06 15:10:50 handler hits: 12440, import hits: 0, misses: 1",
    "2019/06/06 15:10:50 Receive request to /run/echo",
    "2019/06/06 15:10:50 handler hits: 12441, import hits: 0, misses: 1",
    "2019/06/06 15:10:50 Receive request to /run/echo",
    "2019/06/06 15:10:50 handler hits: 12442, import hits: 0, misses: 1",
    "2019/06/06 15:10:50 Receive request to /run/echo",
    "2019/06/06 15:10:50 handler hits: 12443, import hits: 0, misses: 1",
    "2019/06/06 15:10:50 Receive request to /run/echo",
    "2019/06/06 15:10:50 handler hits: 12444, import hits: 0, misses: 1",
    ""
  ]
}
POP conf: {'registry': '/root/open-lambda/test-registry', 'sandbox': 'sock', 'import_cache_mb': 256, 'handler_cache_mb': 256, 'cg_pool_size': 10}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 9925
PUSH conf: {'sandbox': 'sock', 'import_cache_mb': 256, 'handler_cache_mb': 256, 'cg_pool_size': 10}
PUSH conf: {'registry': '/tmp/tmph9af5ebn'}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=9984, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "call_each_once_exec",
  "params": {
    "lambda_count": 100,
    "alloc_mb": 1
  },
  "pass": true,
  "seconds": 4.190245866775513,
  "stats": {
    "reqs_per_sec": 23.865124592923998
  },
  "conf": {
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "max_runners": 0,
    "sandbox_config": {
      "processes": 10
    },
    "cg_pool_size": 10,
    "timing": false,
    "registry": "/tmp/tmph9af5ebn",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "import_cache_mb": 256,
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000",
    "registry_cache_ms": 5000,
    "benchmark_log": "",
    "Skip_pull_existing": false,
    "startup_pkgs": null,
    "docker_runtime": "",
    "sandbox": "sock",
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages"
  },
  "exception": null,
  "worker_tail": [
    "2019/06/06 15:10:54 EVICTING INSTANCE: 272756736 used / 268435456 limit",
    "2019/06/06 15:10:54 EVICTING INSTANCE: 268869632 used / 268435456 limit",
    "2019/06/06 15:10:54 Receive request to /run/L94",
    "2019/06/06 15:10:54 handler hits: 0, import hits: 0, misses: 95",
    "2019/06/06 15:10:54 Receive request to /run/L95",
    "2019/06/06 15:10:54 handler hits: 0, import hits: 0, misses: 96",
    "2019/06/06 15:10:54 EVICTING INSTANCE: 272793600 used / 268435456 limit",
    "2019/06/06 15:10:54 Receive request to /run/L96",
    "2019/06/06 15:10:54 handler hits: 0, import hits: 0, misses: 97",
    "2019/06/06 15:10:54 EVICTING INSTANCE: 270802944 used / 268435456 limit",
    "2019/06/06 15:10:54 Receive request to /run/L97",
    "2019/06/06 15:10:54 handler hits: 0, import hits: 0, misses: 98",
    "2019/06/06 15:10:55 EVICTING INSTANCE: 271872000 used / 268435456 limit",
    "2019/06/06 15:10:55 Receive request to /run/L98",
    "2019/06/06 15:10:55 handler hits: 0, import hits: 0, misses: 99",
    "2019/06/06 15:10:55 EVICTING INSTANCE: 269877248 used / 268435456 limit",
    "2019/06/06 15:10:55 Receive request to /run/L99",
    "2019/06/06 15:10:55 handler hits: 0, import hits: 0, misses: 100",
    "2019/06/06 15:10:55 EVICTING INSTANCE: 269963264 used / 268435456 limit",
    ""
  ]
}
POP conf: {'registry': '/tmp/tmph9af5ebn'}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 9984
2019/06/06 15:15:55 worker didn't stop after 30s
PASSED: 28, FAILED: 0
Makefile:28: recipe for target 'test-all' failed
make: *** [test-all] Error 1