python3 -u test.py
RUN ./ol new -p=test-cluster
Init OL dir at /root/open-lambda/test-cluster
Create lambda base at /root/open-lambda/test-cluster/lambda (may take several minutes)
Working Directory: /root/open-lambda/test-cluster

Worker Defaults: 
{
	"registry": "/root/open-lambda/test-cluster/registry",
	"registry_cache_ms": 5000,
	"sandbox": "sock",
	"cluster_name": "/root/open-lambda/test-cluster",
	"pip_mirror": "",
	"Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
	"max_runners": 0,
	"handler_cache_mb": 239,
	"import_cache_mb": 239,
	"worker_dir": "/root/open-lambda/test-cluster/worker",
	"SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
	"worker_port": "5000",
	"cg_pool_size": 0,
	"Skip_pull_existing": false,
	"sandbox_config": {
		"processes": 10
	},
	"benchmark_log": "",
	"timing": false,
	"startup_pkgs": null,
	"docker_runtime": ""
}

You may now start a server using the "worker" command
PUSH conf: {'registry': '/root/open-lambda/test-registry', 'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson']}
PUSH conf: {'cg_pool_size': 10, 'sandbox': 'sock', 'import_cache_mb': 0, 'handler_cache_mb': 0}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=7387, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 2.3341994285583496,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "worker_port": "5000",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 10,
    "sandbox": "sock",
    "import_cache_mb": 0,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "benchmark_log": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 18:31:32 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 18:31:32 Initialized registry manager (took 21.886\u00b5s)",
    "2019/06/06 18:31:47 Initialized installation manager (took 15.003469432s)",
    "2019/06/06 18:31:47 Initialized handler container factory (took 8.052715ms)",
    "2019/06/06 18:31:47 Initialized cache manager (took 875ns)",
    "2019/06/06 18:31:47 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:31:47 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:31:47 Receive request to /pid",
    "2019/06/06 18:31:47 Receive request to /run/echo",
    "2019/06/06 18:31:49 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:31:49 EVICTING INSTANCE: 26779648 used / 0 limit",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 0.8525445461273193,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "worker_port": "5000",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 10,
    "sandbox": "sock",
    "import_cache_mb": 0,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "benchmark_log": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 18:31:32 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 18:31:32 Initialized registry manager (took 21.886\u00b5s)",
    "2019/06/06 18:31:47 Initialized installation manager (took 15.003469432s)",
    "2019/06/06 18:31:47 Initialized handler container factory (took 8.052715ms)",
    "2019/06/06 18:31:47 Initialized cache manager (took 875ns)",
    "2019/06/06 18:31:47 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:31:47 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:31:47 Receive request to /pid",
    "2019/06/06 18:31:47 Receive request to /run/echo",
    "2019/06/06 18:31:49 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:31:49 EVICTING INSTANCE: 26779648 used / 0 limit",
    "2019/06/06 18:31:49 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:31:50 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:31:50 EVICTING INSTANCE: 24780800 used / 0 limit",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.36435675621032715,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "worker_port": "5000",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 10,
    "sandbox": "sock",
    "import_cache_mb": 0,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "benchmark_log": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "2019/06/06 18:31:32 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 18:31:32 Initialized registry manager (took 21.886\u00b5s)",
    "2019/06/06 18:31:47 Initialized installation manager (took 15.003469432s)",
    "2019/06/06 18:31:47 Initialized handler container factory (took 8.052715ms)",
    "2019/06/06 18:31:47 Initialized cache manager (took 875ns)",
    "2019/06/06 18:31:47 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:31:47 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:31:47 Receive request to /pid",
    "2019/06/06 18:31:47 Receive request to /run/echo",
    "2019/06/06 18:31:49 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:31:49 EVICTING INSTANCE: 26779648 used / 0 limit",
    "2019/06/06 18:31:49 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:31:50 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:31:50 EVICTING INSTANCE: 24780800 used / 0 limit",
    "2019/06/06 18:31:50 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:31:50 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:31:50 EVICTING INSTANCE: 22884352 used / 0 limit",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.36307692527770996,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "worker_port": "5000",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 10,
    "sandbox": "sock",
    "import_cache_mb": 0,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "benchmark_log": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "2019/06/06 18:31:47 Initialized cache manager (took 875ns)",
    "2019/06/06 18:31:47 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:31:47 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:31:47 Receive request to /pid",
    "2019/06/06 18:31:47 Receive request to /run/echo",
    "2019/06/06 18:31:49 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:31:49 EVICTING INSTANCE: 26779648 used / 0 limit",
    "2019/06/06 18:31:49 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:31:50 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:31:50 EVICTING INSTANCE: 24780800 used / 0 limit",
    "2019/06/06 18:31:50 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:31:50 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:31:50 EVICTING INSTANCE: 22884352 used / 0 limit",
    "2019/06/06 18:31:50 Receive request to /run/install3",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:31:51 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:31:51 EVICTING INSTANCE: 22810624 used / 0 limit",
    ""
  ]
}
POP conf: {'cg_pool_size': 10, 'sandbox': 'sock', 'import_cache_mb': 0, 'handler_cache_mb': 0}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 7387
PUSH conf: {'cg_pool_size': 10, 'sandbox': 'sock', 'import_cache_mb': 0, 'handler_cache_mb': 256}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=7478, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 0.15159845352172852,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "benchmark_log": "",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 10,
    "sandbox": "sock",
    "import_cache_mb": 0,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "worker_port": "5000",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 18:31:51 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 18:31:51 Initialized registry manager (took 17.215\u00b5s)",
    "2019/06/06 18:31:58 Initialized installation manager (took 7.384915084s)",
    "2019/06/06 18:31:58 Initialized handler container factory (took 4.770091ms)",
    "2019/06/06 18:31:58 Initialized cache manager (took 874ns)",
    "2019/06/06 18:31:58 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:31:58 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:31:58 Receive request to /pid",
    "2019/06/06 18:31:58 Receive request to /run/echo",
    "2019/06/06 18:31:58 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 0.36692166328430176,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "benchmark_log": "",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 10,
    "sandbox": "sock",
    "import_cache_mb": 0,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "worker_port": "5000",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 18:31:51 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 18:31:51 Initialized registry manager (took 17.215\u00b5s)",
    "2019/06/06 18:31:58 Initialized installation manager (took 7.384915084s)",
    "2019/06/06 18:31:58 Initialized handler container factory (took 4.770091ms)",
    "2019/06/06 18:31:58 Initialized cache manager (took 874ns)",
    "2019/06/06 18:31:58 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:31:58 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:31:58 Receive request to /pid",
    "2019/06/06 18:31:58 Receive request to /run/echo",
    "2019/06/06 18:31:58 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:31:58 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:31:59 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.368880033493042,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "benchmark_log": "",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 10,
    "sandbox": "sock",
    "import_cache_mb": 0,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "worker_port": "5000",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 18:31:51 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 18:31:51 Initialized registry manager (took 17.215\u00b5s)",
    "2019/06/06 18:31:58 Initialized installation manager (took 7.384915084s)",
    "2019/06/06 18:31:58 Initialized handler container factory (took 4.770091ms)",
    "2019/06/06 18:31:58 Initialized cache manager (took 874ns)",
    "2019/06/06 18:31:58 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:31:58 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:31:58 Receive request to /pid",
    "2019/06/06 18:31:58 Receive request to /run/echo",
    "2019/06/06 18:31:58 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:31:58 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:31:59 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:31:59 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:31:59 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.5051007270812988,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "benchmark_log": "",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 10,
    "sandbox": "sock",
    "import_cache_mb": 0,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "worker_port": "5000",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "2019/06/06 18:31:51 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 18:31:51 Initialized registry manager (took 17.215\u00b5s)",
    "2019/06/06 18:31:58 Initialized installation manager (took 7.384915084s)",
    "2019/06/06 18:31:58 Initialized handler container factory (took 4.770091ms)",
    "2019/06/06 18:31:58 Initialized cache manager (took 874ns)",
    "2019/06/06 18:31:58 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:31:58 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:31:58 Receive request to /pid",
    "2019/06/06 18:31:58 Receive request to /run/echo",
    "2019/06/06 18:31:58 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:31:58 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:31:59 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:31:59 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:31:59 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:31:59 Receive request to /run/install3",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:32:00 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
POP conf: {'cg_pool_size': 10, 'sandbox': 'sock', 'import_cache_mb': 0, 'handler_cache_mb': 256}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 7478
PUSH conf: {'cg_pool_size': 10, 'sandbox': 'sock', 'import_cache_mb': 256, 'handler_cache_mb': 0}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=7554, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 0.027860641479492188,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "worker_port": "5000",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 10,
    "sandbox": "sock",
    "import_cache_mb": 256,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "benchmark_log": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 18:32:00 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 18:32:00 Initialized registry manager (took 16.212\u00b5s)",
    "2019/06/06 18:32:07 Initialized installation manager (took 7.574208668s)",
    "2019/06/06 18:32:07 Initialized handler container factory (took 4.677895ms)",
    "2019/06/06 18:32:07 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/06 18:32:08 Initialized cache manager (took 185.482555ms)",
    "2019/06/06 18:32:08 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:32:08 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:32:08 Receive request to /pid",
    "2019/06/06 18:32:08 Receive request to /run/echo",
    "2019/06/06 18:32:08 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/06 18:32:08 EVICTING INSTANCE: 2138112 used / 0 limit",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 0.4432973861694336,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "worker_port": "5000",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 10,
    "sandbox": "sock",
    "import_cache_mb": 256,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "benchmark_log": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 18:32:00 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 18:32:00 Initialized registry manager (took 16.212\u00b5s)",
    "2019/06/06 18:32:07 Initialized installation manager (took 7.574208668s)",
    "2019/06/06 18:32:07 Initialized handler container factory (took 4.677895ms)",
    "2019/06/06 18:32:07 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/06 18:32:08 Initialized cache manager (took 185.482555ms)",
    "2019/06/06 18:32:08 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:32:08 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:32:08 Receive request to /pid",
    "2019/06/06 18:32:08 Receive request to /run/echo",
    "2019/06/06 18:32:08 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/06 18:32:08 EVICTING INSTANCE: 2138112 used / 0 limit",
    "2019/06/06 18:32:08 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:32:08 handler hits: 0, import hits: 0, misses: 2",
    "2019/06/06 18:32:08 EVICTING INSTANCE: 14319616 used / 0 limit",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.2525513172149658,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "worker_port": "5000",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 10,
    "sandbox": "sock",
    "import_cache_mb": 256,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "benchmark_log": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "2019/06/06 18:32:00 Initialized registry manager (took 16.212\u00b5s)",
    "2019/06/06 18:32:07 Initialized installation manager (took 7.574208668s)",
    "2019/06/06 18:32:07 Initialized handler container factory (took 4.677895ms)",
    "2019/06/06 18:32:07 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/06 18:32:08 Initialized cache manager (took 185.482555ms)",
    "2019/06/06 18:32:08 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:32:08 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:32:08 Receive request to /pid",
    "2019/06/06 18:32:08 Receive request to /run/echo",
    "2019/06/06 18:32:08 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/06 18:32:08 EVICTING INSTANCE: 2138112 used / 0 limit",
    "2019/06/06 18:32:08 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:32:08 handler hits: 0, import hits: 0, misses: 2",
    "2019/06/06 18:32:08 EVICTING INSTANCE: 14319616 used / 0 limit",
    "2019/06/06 18:32:08 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:32:08 handler hits: 0, import hits: 0, misses: 3",
    "2019/06/06 18:32:08 EVICTING INSTANCE: 13320192 used / 0 limit",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.2512624263763428,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "worker_port": "5000",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 10,
    "sandbox": "sock",
    "import_cache_mb": 256,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "benchmark_log": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "2019/06/06 18:32:08 Initialized cache manager (took 185.482555ms)",
    "2019/06/06 18:32:08 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:32:08 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:32:08 Receive request to /pid",
    "2019/06/06 18:32:08 Receive request to /run/echo",
    "2019/06/06 18:32:08 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/06 18:32:08 EVICTING INSTANCE: 2138112 used / 0 limit",
    "2019/06/06 18:32:08 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:32:08 handler hits: 0, import hits: 0, misses: 2",
    "2019/06/06 18:32:08 EVICTING INSTANCE: 14319616 used / 0 limit",
    "2019/06/06 18:32:08 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:32:08 handler hits: 0, import hits: 0, misses: 3",
    "2019/06/06 18:32:08 EVICTING INSTANCE: 13320192 used / 0 limit",
    "2019/06/06 18:32:08 Receive request to /run/install3",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:32:09 handler hits: 0, import hits: 0, misses: 4",
    "2019/06/06 18:32:09 EVICTING INSTANCE: 13082624 used / 0 limit",
    ""
  ]
}
POP conf: {'cg_pool_size': 10, 'sandbox': 'sock', 'import_cache_mb': 256, 'handler_cache_mb': 0}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 7554
PUSH conf: {'cg_pool_size': 10, 'sandbox': 'sock', 'import_cache_mb': 256, 'handler_cache_mb': 256}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=7640, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 0.02719712257385254,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "benchmark_log": "",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 10,
    "sandbox": "sock",
    "import_cache_mb": 256,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "worker_port": "5000",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 18:32:09 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 18:32:09 Initialized registry manager (took 20.07\u00b5s)",
    "2019/06/06 18:32:16 Initialized installation manager (took 7.516025331s)",
    "2019/06/06 18:32:16 Initialized handler container factory (took 4.649349ms)",
    "2019/06/06 18:32:16 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/06 18:32:16 Initialized cache manager (took 149.002986ms)",
    "2019/06/06 18:32:16 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:32:16 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:32:16 Receive request to /pid",
    "2019/06/06 18:32:16 Receive request to /run/echo",
    "2019/06/06 18:32:16 handler hits: 0, import hits: 0, misses: 1",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 0.24563884735107422,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "benchmark_log": "",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 10,
    "sandbox": "sock",
    "import_cache_mb": 256,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "worker_port": "5000",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 18:32:09 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 18:32:09 Initialized registry manager (took 20.07\u00b5s)",
    "2019/06/06 18:32:16 Initialized installation manager (took 7.516025331s)",
    "2019/06/06 18:32:16 Initialized handler container factory (took 4.649349ms)",
    "2019/06/06 18:32:16 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/06 18:32:16 Initialized cache manager (took 149.002986ms)",
    "2019/06/06 18:32:16 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:32:16 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:32:16 Receive request to /pid",
    "2019/06/06 18:32:16 Receive request to /run/echo",
    "2019/06/06 18:32:16 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/06 18:32:17 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:32:17 handler hits: 0, import hits: 0, misses: 2",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.24608707427978516,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "benchmark_log": "",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 10,
    "sandbox": "sock",
    "import_cache_mb": 256,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "worker_port": "5000",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 18:32:09 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:sock Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:256 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:10 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 18:32:09 Initialized registry manager (took 20.07\u00b5s)",
    "2019/06/06 18:32:16 Initialized installation manager (took 7.516025331s)",
    "2019/06/06 18:32:16 Initialized handler container factory (took 4.649349ms)",
    "2019/06/06 18:32:16 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/06 18:32:16 Initialized cache manager (took 149.002986ms)",
    "2019/06/06 18:32:16 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:32:16 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:32:16 Receive request to /pid",
    "2019/06/06 18:32:16 Receive request to /run/echo",
    "2019/06/06 18:32:16 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/06 18:32:17 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:32:17 handler hits: 0, import hits: 0, misses: 2",
    "2019/06/06 18:32:17 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:32:17 handler hits: 0, import hits: 0, misses: 3",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.24536561965942383,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "benchmark_log": "",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 10,
    "sandbox": "sock",
    "import_cache_mb": 256,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "worker_port": "5000",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "2019/06/06 18:32:09 Initialized registry manager (took 20.07\u00b5s)",
    "2019/06/06 18:32:16 Initialized installation manager (took 7.516025331s)",
    "2019/06/06 18:32:16 Initialized handler container factory (took 4.649349ms)",
    "2019/06/06 18:32:16 invalid package sizes path /ol/open-lambda/worker/cache-manager/package_sizes.txt, using 0 for all",
    "2019/06/06 18:32:16 Initialized cache manager (took 149.002986ms)",
    "2019/06/06 18:32:16 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:32:16 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:32:16 Receive request to /pid",
    "2019/06/06 18:32:16 Receive request to /run/echo",
    "2019/06/06 18:32:16 handler hits: 0, import hits: 0, misses: 1",
    "2019/06/06 18:32:17 Receive request to /run/install",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:32:17 handler hits: 0, import hits: 0, misses: 2",
    "2019/06/06 18:32:17 Receive request to /run/install2",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:32:17 handler hits: 0, import hits: 0, misses: 3",
    "2019/06/06 18:32:17 Receive request to /run/install3",
    "/bin/sh: 1: cannot create /dev/null: Read-only file system",
    "2019/06/06 18:32:17 handler hits: 0, import hits: 0, misses: 4",
    ""
  ]
}
POP conf: {'cg_pool_size': 10, 'sandbox': 'sock', 'import_cache_mb': 256, 'handler_cache_mb': 256}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 7640
PUSH conf: {'cg_pool_size': 0, 'sandbox': 'docker', 'import_cache_mb': 0, 'handler_cache_mb': 0}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=7724, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 3.8159332275390625,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "worker_port": "5000",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 0,
    "sandbox": "docker",
    "import_cache_mb": 0,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "benchmark_log": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 18:32:17 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 18:32:17 Initialized registry manager (took 19.246\u00b5s)",
    "2019/06/06 18:32:25 Initialized installation manager (took 7.461574405s)",
    "2019/06/06 18:32:25 Initialized handler container factory (took 25.404\u00b5s)",
    "2019/06/06 18:32:25 Initialized cache manager (took 664ns)",
    "2019/06/06 18:32:25 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:32:25 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:32:25 Receive request to /pid",
    "2019/06/06 18:32:25 Receive request to /run/echo",
    "2019/06/06 18:32:29 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 1.2580938339233398,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "worker_port": "5000",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 0,
    "sandbox": "docker",
    "import_cache_mb": 0,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "benchmark_log": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 18:32:17 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 18:32:17 Initialized registry manager (took 19.246\u00b5s)",
    "2019/06/06 18:32:25 Initialized installation manager (took 7.461574405s)",
    "2019/06/06 18:32:25 Initialized handler container factory (took 25.404\u00b5s)",
    "2019/06/06 18:32:25 Initialized cache manager (took 664ns)",
    "2019/06/06 18:32:25 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:32:25 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:32:25 Receive request to /pid",
    "2019/06/06 18:32:25 Receive request to /run/echo",
    "2019/06/06 18:32:29 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:32:29 EVICTING INSTANCE: 31195136 used / 0 limit",
    "2019/06/06 18:32:29 Receive request to /run/install",
    "2019/06/06 18:32:30 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.9199371337890625,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "worker_port": "5000",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 0,
    "sandbox": "docker",
    "import_cache_mb": 0,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "benchmark_log": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 18:32:17 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:0 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 18:32:17 Initialized registry manager (took 19.246\u00b5s)",
    "2019/06/06 18:32:25 Initialized installation manager (took 7.461574405s)",
    "2019/06/06 18:32:25 Initialized handler container factory (took 25.404\u00b5s)",
    "2019/06/06 18:32:25 Initialized cache manager (took 664ns)",
    "2019/06/06 18:32:25 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:32:25 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:32:25 Receive request to /pid",
    "2019/06/06 18:32:25 Receive request to /run/echo",
    "2019/06/06 18:32:29 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:32:29 EVICTING INSTANCE: 31195136 used / 0 limit",
    "2019/06/06 18:32:29 Receive request to /run/install",
    "2019/06/06 18:32:30 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:32:30 EVICTING INSTANCE: 24903680 used / 0 limit",
    "2019/06/06 18:32:30 Receive request to /run/install2",
    "2019/06/06 18:32:31 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:32:31 EVICTING INSTANCE: 22749184 used / 0 limit",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.9087707996368408,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "worker_port": "5000",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 0,
    "sandbox": "docker",
    "import_cache_mb": 0,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "benchmark_log": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "2019/06/06 18:32:17 Initialized registry manager (took 19.246\u00b5s)",
    "2019/06/06 18:32:25 Initialized installation manager (took 7.461574405s)",
    "2019/06/06 18:32:25 Initialized handler container factory (took 25.404\u00b5s)",
    "2019/06/06 18:32:25 Initialized cache manager (took 664ns)",
    "2019/06/06 18:32:25 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:32:25 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:32:25 Receive request to /pid",
    "2019/06/06 18:32:25 Receive request to /run/echo",
    "2019/06/06 18:32:29 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:32:29 EVICTING INSTANCE: 31195136 used / 0 limit",
    "2019/06/06 18:32:29 Receive request to /run/install",
    "2019/06/06 18:32:30 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:32:30 EVICTING INSTANCE: 24903680 used / 0 limit",
    "2019/06/06 18:32:30 Receive request to /run/install2",
    "2019/06/06 18:32:31 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:32:31 EVICTING INSTANCE: 22749184 used / 0 limit",
    "2019/06/06 18:32:31 Receive request to /run/install3",
    "2019/06/06 18:32:32 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:32:32 EVICTING INSTANCE: 22843392 used / 0 limit",
    ""
  ]
}
POP conf: {'cg_pool_size': 0, 'sandbox': 'docker', 'import_cache_mb': 0, 'handler_cache_mb': 0}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 7724
PUSH conf: {'cg_pool_size': 0, 'sandbox': 'docker', 'import_cache_mb': 0, 'handler_cache_mb': 256}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=8669, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "test_smoke_echo",
  "params": {},
  "pass": true,
  "seconds": 0.5657773017883301,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "benchmark_log": "",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 0,
    "sandbox": "docker",
    "import_cache_mb": 0,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "worker_port": "5000",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 18:32:32 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 18:32:32 Initialized registry manager (took 18.281\u00b5s)",
    "2019/06/06 18:32:39 Initialized installation manager (took 7.49276265s)",
    "2019/06/06 18:32:39 Initialized handler container factory (took 25.98\u00b5s)",
    "2019/06/06 18:32:39 Initialized cache manager (took 730ns)",
    "2019/06/06 18:32:39 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:32:39 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:32:39 Receive request to /pid",
    "2019/06/06 18:32:39 Receive request to /run/echo",
    "2019/06/06 18:32:40 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": null
  },
  "pass": true,
  "seconds": 1.1177210807800293,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "benchmark_log": "",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 0,
    "sandbox": "docker",
    "import_cache_mb": 0,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "worker_port": "5000",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 18:32:32 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 18:32:32 Initialized registry manager (took 18.281\u00b5s)",
    "2019/06/06 18:32:39 Initialized installation manager (took 7.49276265s)",
    "2019/06/06 18:32:39 Initialized handler container factory (took 25.98\u00b5s)",
    "2019/06/06 18:32:39 Initialized cache manager (took 730ns)",
    "2019/06/06 18:32:39 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:32:39 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:32:39 Receive request to /pid",
    "2019/06/06 18:32:39 Receive request to /run/echo",
    "2019/06/06 18:32:40 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:32:40 Receive request to /run/install",
    "2019/06/06 18:32:41 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 2
  },
  "pass": true,
  "seconds": 0.882469654083252,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "benchmark_log": "",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 0,
    "sandbox": "docker",
    "import_cache_mb": 0,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "worker_port": "5000",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 18:32:32 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 18:32:32 Initialized registry manager (took 18.281\u00b5s)",
    "2019/06/06 18:32:39 Initialized installation manager (took 7.49276265s)",
    "2019/06/06 18:32:39 Initialized handler container factory (took 25.98\u00b5s)",
    "2019/06/06 18:32:39 Initialized cache manager (took 730ns)",
    "2019/06/06 18:32:39 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:32:39 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:32:39 Receive request to /pid",
    "2019/06/06 18:32:39 Receive request to /run/echo",
    "2019/06/06 18:32:40 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:32:40 Receive request to /run/install",
    "2019/06/06 18:32:41 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:32:41 Receive request to /run/install2",
    "2019/06/06 18:32:42 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
{
  "test": "test_smoke_install",
  "params": {
    "num": 3
  },
  "pass": true,
  "seconds": 0.7724616527557373,
  "stats": null,
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "benchmark_log": "",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 0,
    "sandbox": "docker",
    "import_cache_mb": 0,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "worker_port": "5000",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "using existing OL directory at /root/open-lambda/test-cluster",
    "2019/06/06 18:32:32 Config: &{Registry:/root/open-lambda/test-registry Registry_cache_ms:5000 Sandbox:docker Cluster_name:/root/open-lambda/test-cluster Pip_index: Pkgs_dir:/root/open-lambda/test-cluster/lambda/packages Max_runners:0 Handler_cache_mb:256 Import_cache_mb:0 Worker_dir:/root/open-lambda/test-cluster/worker SOCK_base_path:/root/open-lambda/test-cluster/lambda Worker_port:5000 Cg_pool_size:0 Skip_pull_existing:false Sandbox_config:map[processes:10] Benchmark_file: Timing:false Startup_pkgs:[parso jedi urllib3 idna chardet certifi requests simplejson] Docker_runtime:}",
    "2019/06/06 18:32:32 Initialized registry manager (took 18.281\u00b5s)",
    "2019/06/06 18:32:39 Initialized installation manager (took 7.49276265s)",
    "2019/06/06 18:32:39 Initialized handler container factory (took 25.98\u00b5s)",
    "2019/06/06 18:32:39 Initialized cache manager (took 730ns)",
    "2019/06/06 18:32:39 Execute handler by POSTing to localhost:5000/run/<lambda>",
    "2019/06/06 18:32:39 Get status by sending request to localhost:5000/status",
    "2019/06/06 18:32:39 Receive request to /pid",
    "2019/06/06 18:32:39 Receive request to /run/echo",
    "2019/06/06 18:32:40 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:32:40 Receive request to /run/install",
    "2019/06/06 18:32:41 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:32:41 Receive request to /run/install2",
    "2019/06/06 18:32:42 handler hits: 0, import hits: 0, misses: 0",
    "2019/06/06 18:32:42 Receive request to /run/install3",
    "2019/06/06 18:32:43 handler hits: 0, import hits: 0, misses: 0",
    ""
  ]
}
POP conf: {'cg_pool_size': 0, 'sandbox': 'docker', 'import_cache_mb': 0, 'handler_cache_mb': 256}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 8669
POP conf: {'registry': '/root/open-lambda/test-registry', 'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson']}
PUSH conf: {'registry': '/root/open-lambda/test-registry', 'cg_pool_size': 10, 'sandbox': 'sock', 'import_cache_mb': 256, 'handler_cache_mb': 256}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=9709, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "stress_one_lambda",
  "params": {
    "procs": 1,
    "seconds": 15
  },
  "pass": true,
  "seconds": 15.07648229598999,
  "stats": {
    "reqs_per_sec": 285.8
  },
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "benchmark_log": "",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 10,
    "sandbox": "sock",
    "import_cache_mb": 256,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "worker_port": "5000",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": null,
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "2019/06/06 18:32:59 handler hits: 4277, import hits: 0, misses: 1",
    "2019/06/06 18:32:59 Receive request to /run/echo",
    "2019/06/06 18:32:59 handler hits: 4278, import hits: 0, misses: 1",
    "2019/06/06 18:32:59 Receive request to /run/echo",
    "2019/06/06 18:32:59 handler hits: 4279, import hits: 0, misses: 1",
    "2019/06/06 18:32:59 Receive request to /run/echo",
    "2019/06/06 18:32:59 handler hits: 4280, import hits: 0, misses: 1",
    "2019/06/06 18:32:59 Receive request to /run/echo",
    "2019/06/06 18:32:59 handler hits: 4281, import hits: 0, misses: 1",
    "2019/06/06 18:32:59 Receive request to /run/echo",
    "2019/06/06 18:32:59 handler hits: 4282, import hits: 0, misses: 1",
    "2019/06/06 18:32:59 Receive request to /run/echo",
    "2019/06/06 18:32:59 handler hits: 4283, import hits: 0, misses: 1",
    "2019/06/06 18:32:59 Receive request to /run/echo",
    "2019/06/06 18:32:59 handler hits: 4284, import hits: 0, misses: 1",
    "2019/06/06 18:32:59 Receive request to /run/echo",
    "2019/06/06 18:32:59 handler hits: 4285, import hits: 0, misses: 1",
    "2019/06/06 18:33:00 Receive request to /run/echo",
    "2019/06/06 18:33:00 handler hits: 4286, import hits: 0, misses: 1",
    ""
  ]
}
{
  "test": "stress_one_lambda",
  "params": {
    "procs": 2,
    "seconds": 15
  },
  "pass": true,
  "seconds": 15.05068325996399,
  "stats": {
    "reqs_per_sec": 285.3333333333333
  },
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "benchmark_log": "",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 10,
    "sandbox": "sock",
    "import_cache_mb": 256,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "worker_port": "5000",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": null,
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "2019/06/06 18:33:15 handler hits: 8557, import hits: 0, misses: 1",
    "2019/06/06 18:33:15 Receive request to /run/echo",
    "2019/06/06 18:33:15 handler hits: 8558, import hits: 0, misses: 1",
    "2019/06/06 18:33:15 Receive request to /run/echo",
    "2019/06/06 18:33:15 handler hits: 8559, import hits: 0, misses: 1",
    "2019/06/06 18:33:15 Receive request to /run/echo",
    "2019/06/06 18:33:15 handler hits: 8560, import hits: 0, misses: 1",
    "2019/06/06 18:33:15 Receive request to /run/echo",
    "2019/06/06 18:33:15 handler hits: 8561, import hits: 0, misses: 1",
    "2019/06/06 18:33:15 Receive request to /run/echo",
    "2019/06/06 18:33:15 handler hits: 8562, import hits: 0, misses: 1",
    "2019/06/06 18:33:15 Receive request to /run/echo",
    "2019/06/06 18:33:15 Receive request to /run/echo",
    "2019/06/06 18:33:15 handler hits: 8563, import hits: 0, misses: 1",
    "2019/06/06 18:33:15 handler hits: 8564, import hits: 0, misses: 1",
    "2019/06/06 18:33:15 Receive request to /run/echo",
    "2019/06/06 18:33:15 handler hits: 8565, import hits: 0, misses: 1",
    "2019/06/06 18:33:15 Receive request to /run/echo",
    "2019/06/06 18:33:15 handler hits: 8566, import hits: 0, misses: 1",
    ""
  ]
}
{
  "test": "stress_one_lambda",
  "params": {
    "procs": 8,
    "seconds": 15
  },
  "pass": true,
  "seconds": 15.063989639282227,
  "stats": {
    "reqs_per_sec": 263.0
  },
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "benchmark_log": "",
    "registry": "/root/open-lambda/test-registry",
    "cg_pool_size": 10,
    "sandbox": "sock",
    "import_cache_mb": 256,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "worker_port": "5000",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": null,
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "2019/06/06 18:33:30 handler hits: 12502, import hits: 0, misses: 1",
    "2019/06/06 18:33:30 Receive request to /run/echo",
    "2019/06/06 18:33:30 handler hits: 12503, import hits: 0, misses: 1",
    "2019/06/06 18:33:30 Receive request to /run/echo",
    "2019/06/06 18:33:30 handler hits: 12504, import hits: 0, misses: 1",
    "2019/06/06 18:33:30 Receive request to /run/echo",
    "2019/06/06 18:33:30 handler hits: 12505, import hits: 0, misses: 1",
    "2019/06/06 18:33:30 Receive request to /run/echo",
    "2019/06/06 18:33:30 handler hits: 12506, import hits: 0, misses: 1",
    "2019/06/06 18:33:30 Receive request to /run/echo",
    "2019/06/06 18:33:30 handler hits: 12507, import hits: 0, misses: 1",
    "2019/06/06 18:33:30 Receive request to /run/echo",
    "2019/06/06 18:33:30 handler hits: 12508, import hits: 0, misses: 1",
    "2019/06/06 18:33:30 Receive request to /run/echo",
    "2019/06/06 18:33:30 handler hits: 12509, import hits: 0, misses: 1",
    "2019/06/06 18:33:30 Receive request to /run/echo",
    "2019/06/06 18:33:30 handler hits: 12510, import hits: 0, misses: 1",
    "2019/06/06 18:33:30 Receive request to /run/echo",
    "2019/06/06 18:33:30 handler hits: 12511, import hits: 0, misses: 1",
    ""
  ]
}
POP conf: {'registry': '/root/open-lambda/test-registry', 'cg_pool_size': 10, 'sandbox': 'sock', 'import_cache_mb': 256, 'handler_cache_mb': 256}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 9709
PUSH conf: {'cg_pool_size': 10, 'sandbox': 'sock', 'import_cache_mb': 256, 'handler_cache_mb': 256}
PUSH conf: {'registry': '/tmp/tmpjsuk6nrk'}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=9768, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "call_each_once_exec",
  "params": {
    "lambda_count": 100,
    "alloc_mb": 1
  },
  "pass": true,
  "seconds": 3.627307176589966,
  "stats": {
    "reqs_per_sec": 27.568888798620577
  },
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "worker_port": "5000",
    "registry": "/tmp/tmpjsuk6nrk",
    "cg_pool_size": 10,
    "sandbox": "sock",
    "import_cache_mb": 256,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "benchmark_log": "",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": null,
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "2019/06/06 18:33:33 EVICTING INSTANCE: 272281600 used / 268435456 limit",
    "2019/06/06 18:33:33 Receive request to /run/L94",
    "2019/06/06 18:33:33 handler hits: 0, import hits: 0, misses: 95",
    "2019/06/06 18:33:33 EVICTING INSTANCE: 273584128 used / 268435456 limit",
    "2019/06/06 18:33:33 Receive request to /run/L95",
    "2019/06/06 18:33:33 handler hits: 0, import hits: 0, misses: 96",
    "2019/06/06 18:33:34 EVICTING INSTANCE: 273362944 used / 268435456 limit",
    "2019/06/06 18:33:34 EVICTING INSTANCE: 270106624 used / 268435456 limit",
    "2019/06/06 18:33:34 Receive request to /run/L96",
    "2019/06/06 18:33:34 handler hits: 0, import hits: 0, misses: 97",
    "2019/06/06 18:33:34 EVICTING INSTANCE: 269033472 used / 268435456 limit",
    "2019/06/06 18:33:34 Receive request to /run/L97",
    "2019/06/06 18:33:34 handler hits: 0, import hits: 0, misses: 98",
    "2019/06/06 18:33:34 EVICTING INSTANCE: 270127104 used / 268435456 limit",
    "2019/06/06 18:33:34 Receive request to /run/L98",
    "2019/06/06 18:33:34 handler hits: 0, import hits: 0, misses: 99",
    "2019/06/06 18:33:34 EVICTING INSTANCE: 269557760 used / 268435456 limit",
    "2019/06/06 18:33:34 Receive request to /run/L99",
    "2019/06/06 18:33:34 handler hits: 0, import hits: 0, misses: 100",
    ""
  ]
}
POP conf: {'registry': '/tmp/tmpjsuk6nrk'}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 9768
PUSH conf: {'registry': '/tmp/tmpbx6zleb5'}
RUN ./ol worker -p=test-cluster --detach
using existing OL directory at /root/open-lambda/test-cluster
Starting worker: pid=10397, port=5000, log=/root/open-lambda/test-cluster/worker.out
ready
{
  "test": "call_each_once_exec",
  "params": {
    "lambda_count": 1000,
    "alloc_mb": 10
  },
  "pass": true,
  "seconds": 57.36969184875488,
  "stats": {
    "reqs_per_sec": 17.430815597112
  },
  "conf": {
    "sandbox_config": {
      "processes": 10
    },
    "benchmark_log": "",
    "registry": "/tmp/tmpbx6zleb5",
    "cg_pool_size": 10,
    "sandbox": "sock",
    "import_cache_mb": 256,
    "registry_cache_ms": 5000,
    "docker_runtime": "",
    "Skip_pull_existing": false,
    "max_runners": 0,
    "worker_port": "5000",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "startup_pkgs": null,
    "handler_cache_mb": 256,
    "cluster_name": "/root/open-lambda/test-cluster",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "timing": false,
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "pip_mirror": ""
  },
  "exception": null,
  "worker_tail": [
    "2019/06/06 18:34:32 EVICTING INSTANCE: 268570624 used / 268435456 limit",
    "2019/06/06 18:34:32 Receive request to /run/L994",
    "2019/06/06 18:34:32 handler hits: 0, import hits: 0, misses: 995",
    "2019/06/06 18:34:32 Receive request to /run/L995",
    "2019/06/06 18:34:32 handler hits: 0, import hits: 0, misses: 996",
    "2019/06/06 18:34:32 EVICTING INSTANCE: 281755648 used / 268435456 limit",
    "2019/06/06 18:34:32 Receive request to /run/L996",
    "2019/06/06 18:34:32 handler hits: 0, import hits: 0, misses: 997",
    "2019/06/06 18:34:32 EVICTING INSTANCE: 282021888 used / 268435456 limit",
    "2019/06/06 18:34:32 Receive request to /run/L997",
    "2019/06/06 18:34:32 handler hits: 0, import hits: 0, misses: 998",
    "2019/06/06 18:34:32 EVICTING INSTANCE: 283066368 used / 268435456 limit",
    "2019/06/06 18:34:32 EVICTING INSTANCE: 268832768 used / 268435456 limit",
    "2019/06/06 18:34:32 Receive request to /run/L998",
    "2019/06/06 18:34:32 handler hits: 0, import hits: 0, misses: 999",
    "2019/06/06 18:34:32 EVICTING INSTANCE: 268783616 used / 268435456 limit",
    "2019/06/06 18:34:32 Receive request to /run/L999",
    "2019/06/06 18:34:33 handler hits: 0, import hits: 0, misses: 1000",
    "2019/06/06 18:34:33 EVICTING INSTANCE: 268660736 used / 268435456 limit",
    ""
  ]
}
POP conf: {'registry': '/tmp/tmpbx6zleb5'}
RUN ./ol kill -p=test-cluster
Kill worker process with PID 10397
POP conf: {'cg_pool_size': 10, 'sandbox': 'sock', 'import_cache_mb': 256, 'handler_cache_mb': 256}
PASSED: 29, FAILED: 0