python3 -u test.py
RUN ./ol new -p=test-cluster
b'Init OL dir at /root/open-lambda/test-cluster\nCreate lambda base at /root/open-lambda/test-cluster/lambda (may take several minutes)\nWorking Directory: /root/open-lambda/test-cluster\n\nWorker Defaults: \n{\n\t"registry": "/root/open-lambda/test-cluster/registry",\n\t"registry_cache_ms": 5000,\n\t"sandbox": "sock",\n\t"cluster_name": "/root/open-lambda/test-cluster",\n\t"pip_mirror": "",\n\t"Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",\n\t"max_runners": 0,\n\t"handler_cache_mb": 239,\n\t"import_cache_mb": 239,\n\t"server_mode": "lambda",\n\t"worker_dir": "/root/open-lambda/test-cluster/worker",\n\t"SOCK_base_path": "/root/open-lambda/test-cluster/lambda",\n\t"worker_port": "5000",\n\t"Skip_pull_existing": false,\n\t"sandbox_config": {\n\t\t"processes": 10\n\t},\n\t"benchmark_log": "",\n\t"timing": false,\n\t"startup_pkgs": null,\n\t"docker_runtime": "",\n\t"sock_cgroups": {\n\t\t"max_procs": 10,\n\t\t"max_mem_mb": 50\n\t}\n}\n\nYou may modify the defaults here: /root/open-lambda/test-cluster/config.json\n\nYou may now start a server using the "ol worker" command\n'
PUSH conf: {'server_mode': 'sock'}
========================================
sock_churn
========================================
RUN ./ol worker -p=test-cluster --detach
b'using existing OL directory at /root/open-lambda/test-cluster\nStarting worker: pid=7307, port=5000, log=/root/open-lambda/test-cluster/worker.out\nready\n'
RUN ./ol kill -p=test-cluster
b'Kill worker process with PID 7307\n'
{
  "test": "sock_churn",
  "params": {
    "baseline": 0,
    "procs": 1,
    "fork": true,
    "seconds": 15
  },
  "pass": true,
  "seconds": 16.435333967208862,
  "total_seconds": 16.84422492980957,
  "stats": {
    "sandboxes_per_sec": 48.8
  },
  "ol-stats": {
    "Create().ms.avg": 10,
    "Create().ms.cnt": 733,
    "Create().ms.sum": 8029,
    "Destroy().ms.avg": 2,
    "Destroy().ms.cnt": 732,
    "Destroy().ms.sum": 2039,
    "acquire-cgroup.ms.avg": 0,
    "acquire-cgroup.ms.cnt": 733,
    "acquire-cgroup.ms.sum": 0,
    "acquire-mem.ms.avg": 0,
    "acquire-mem.ms.cnt": 733,
    "acquire-mem.ms.sum": 0,
    "fork().ms.avg": 6,
    "fork().ms.cnt": 732,
    "fork().ms.sum": 4589,
    "fork-proc.ms.avg": 6,
    "fork-proc.ms.cnt": 732,
    "fork-proc.ms.sum": 4604,
    "forkRequest.ms.avg": 5,
    "forkRequest.ms.cnt": 732,
    "forkRequest.ms.sum": 4178,
    "fresh-proc.ms.avg": 1250,
    "fresh-proc.ms.cnt": 1,
    "fresh-proc.ms.sum": 1250,
    "make-proc.ms.avg": 7,
    "make-proc.ms.cnt": 733,
    "make-proc.ms.sum": 5860,
    "make-root-fs.ms.avg": 2,
    "make-root-fs.ms.cnt": 733,
    "make-root-fs.ms.sum": 1716,
    "move-to-cg-after-fork.ms.avg": 0,
    "move-to-cg-after-fork.ms.cnt": 732,
    "move-to-cg-after-fork.ms.sum": 76
  },
  "conf": {
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "registry": "/root/open-lambda/test-cluster/registry",
    "worker_port": "5000",
    "timing": false,
    "sandbox_config": {
      "processes": 10
    },
    "server_mode": "sock",
    "sandbox": "sock",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "registry_cache_ms": 5000,
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "handler_cache_mb": 239,
    "startup_pkgs": null,
    "Skip_pull_existing": false,
    "sock_cgroups": {
      "max_procs": 10,
      "max_mem_mb": 50
    },
    "import_cache_mb": 239,
    "docker_runtime": "",
    "max_runners": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "benchmark_log": ""
  },
  "errors": [],
  "worker_tail": [
    "2019/06/27 22:26:01 remove /sys/fs/cgroup/devices/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:01 remove /sys/fs/cgroup/freezer/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:01 remove /sys/fs/cgroup/hugetlb/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:01 remove /sys/fs/cgroup/memory/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:01 remove /sys/fs/cgroup/perf_event/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:01 remove /sys/fs/cgroup/systemd/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:01 remove /sys/fs/cgroup/pids/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:01 remove worker.pid",
    "2019/06/27 22:26:01 exiting",
    ""
  ]
}
========================================
sock_churn
========================================
RUN ./ol worker -p=test-cluster --detach
b'using existing OL directory at /root/open-lambda/test-cluster\nStarting worker: pid=10266, port=5000, log=/root/open-lambda/test-cluster/worker.out\nready\n'
RUN ./ol kill -p=test-cluster
b'Kill worker process with PID 10266\n'
{
  "test": "sock_churn",
  "params": {
    "baseline": 0,
    "procs": 15,
    "fork": true,
    "seconds": 15
  },
  "pass": true,
  "seconds": 15.42742109298706,
  "total_seconds": 15.668722867965698,
  "stats": {
    "sandboxes_per_sec": 47.6
  },
  "ol-stats": {
    "Create().ms.avg": 142,
    "Create().ms.cnt": 715,
    "Create().ms.sum": 102238,
    "Destroy().ms.avg": 11,
    "Destroy().ms.cnt": 714,
    "Destroy().ms.sum": 8489,
    "acquire-cgroup.ms.avg": 0,
    "acquire-cgroup.ms.cnt": 715,
    "acquire-cgroup.ms.sum": 0,
    "acquire-mem.ms.avg": 98,
    "acquire-mem.ms.cnt": 715,
    "acquire-mem.ms.sum": 70208,
    "fork().ms.avg": 41,
    "fork().ms.cnt": 714,
    "fork().ms.sum": 29277,
    "fork-proc.ms.avg": 41,
    "fork-proc.ms.cnt": 714,
    "fork-proc.ms.sum": 29326,
    "forkRequest.ms.avg": 10,
    "forkRequest.ms.cnt": 714,
    "forkRequest.ms.sum": 7245,
    "fresh-proc.ms.avg": 144,
    "fresh-proc.ms.cnt": 1,
    "fresh-proc.ms.sum": 144,
    "make-proc.ms.avg": 41,
    "make-proc.ms.cnt": 715,
    "make-proc.ms.sum": 29472,
    "make-root-fs.ms.avg": 2,
    "make-root-fs.ms.cnt": 715,
    "make-root-fs.ms.sum": 1944,
    "move-to-cg-after-fork.ms.avg": 4,
    "move-to-cg-after-fork.ms.cnt": 714,
    "move-to-cg-after-fork.ms.sum": 2900
  },
  "conf": {
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "registry": "/root/open-lambda/test-cluster/registry",
    "worker_port": "5000",
    "timing": false,
    "sandbox_config": {
      "processes": 10
    },
    "server_mode": "sock",
    "sandbox": "sock",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "registry_cache_ms": 5000,
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "handler_cache_mb": 239,
    "startup_pkgs": null,
    "Skip_pull_existing": false,
    "sock_cgroups": {
      "max_procs": 10,
      "max_mem_mb": 50
    },
    "import_cache_mb": 239,
    "docker_runtime": "",
    "max_runners": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "benchmark_log": ""
  },
  "errors": [],
  "worker_tail": [
    "2019/06/27 22:26:17 remove /sys/fs/cgroup/devices/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:17 remove /sys/fs/cgroup/freezer/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:17 remove /sys/fs/cgroup/hugetlb/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:17 remove /sys/fs/cgroup/memory/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:17 remove /sys/fs/cgroup/perf_event/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:17 remove /sys/fs/cgroup/systemd/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:17 remove /sys/fs/cgroup/pids/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:17 remove worker.pid",
    "2019/06/27 22:26:17 exiting",
    ""
  ]
}
POP conf: {'server_mode': 'sock'}
PUSH conf: {'registry': '/root/open-lambda/test-registry', 'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson']}
========================================
ping_test
========================================
RUN ./ol worker -p=test-cluster --detach
b'using existing OL directory at /root/open-lambda/test-cluster\nStarting worker: pid=13172, port=5000, log=/root/open-lambda/test-cluster/worker.out\nready\n'
RUN ./ol kill -p=test-cluster
b'Kill worker process with PID 13172\n'
{
  "test": "ping_test",
  "params": {},
  "pass": true,
  "seconds": 2.10507869720459,
  "total_seconds": 17.70206904411316,
  "stats": {
    "pings_per_sec": 475.04598064231664
  },
  "ol-stats": {
    "Create().ms.avg": 150,
    "Create().ms.cnt": 1,
    "Create().ms.sum": 150,
    "acquire-cgroup.ms.avg": 3,
    "acquire-cgroup.ms.cnt": 1,
    "acquire-cgroup.ms.sum": 3,
    "acquire-mem.ms.avg": 3,
    "acquire-mem.ms.cnt": 1,
    "acquire-mem.ms.sum": 3,
    "fresh-proc.ms.avg": 137,
    "fresh-proc.ms.cnt": 1,
    "fresh-proc.ms.sum": 137,
    "make-proc.ms.avg": 137,
    "make-proc.ms.cnt": 1,
    "make-proc.ms.sum": 137,
    "make-root-fs.ms.avg": 4,
    "make-root-fs.ms.cnt": 1,
    "make-root-fs.ms.sum": 4
  },
  "conf": {
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "registry": "/root/open-lambda/test-registry",
    "pip_mirror": "",
    "docker_runtime": "",
    "server_mode": "lambda",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "registry_cache_ms": 5000,
    "timing": false,
    "sandbox": "sock",
    "handler_cache_mb": 239,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "Skip_pull_existing": false,
    "sock_cgroups": {
      "max_procs": 10,
      "max_mem_mb": 50
    },
    "import_cache_mb": 239,
    "benchmark_log": "",
    "sandbox_config": {
      "processes": 10
    },
    "max_runners": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000"
  },
  "errors": [],
  "worker_tail": [
    "2019/06/27 22:26:34 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:26:34 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:26:34 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:26:34 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:26:34 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:26:34 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:26:34 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:26:34 remove worker.pid",
    "2019/06/27 22:26:34 exiting",
    ""
  ]
}
PUSH conf: {'import_cache_mb': 0, 'handler_cache_mb': 100}
========================================
smoke_tests
========================================
RUN ./ol worker -p=test-cluster --detach
b'using existing OL directory at /root/open-lambda/test-cluster\nStarting worker: pid=13245, port=5000, log=/root/open-lambda/test-cluster/worker.out\nready\n'
RUN ./ol kill -p=test-cluster
b'Kill worker process with PID 13245\n'
{
  "test": "smoke_tests",
  "params": {},
  "pass": true,
  "seconds": 1.9432621002197266,
  "total_seconds": 9.906632423400879,
  "stats": null,
  "ol-stats": {
    "Channel().ms.avg": 0,
    "Channel().ms.cnt": 4,
    "Channel().ms.sum": 0,
    "Create().ms.avg": 140,
    "Create().ms.cnt": 4,
    "Create().ms.sum": 561,
    "Destroy().ms.avg": 6,
    "Destroy().ms.cnt": 3,
    "Destroy().ms.sum": 18,
    "Pause().ms.avg": 3,
    "Pause().ms.cnt": 3,
    "Pause().ms.sum": 10,
    "acquire-cgroup.ms.avg": 0,
    "acquire-cgroup.ms.cnt": 4,
    "acquire-cgroup.ms.sum": 0,
    "acquire-mem.ms.avg": 0,
    "acquire-mem.ms.cnt": 4,
    "acquire-mem.ms.sum": 0,
    "evict.ms.avg": 6,
    "evict.ms.cnt": 3,
    "evict.ms.sum": 18,
    "fresh-proc.ms.avg": 135,
    "fresh-proc.ms.cnt": 4,
    "fresh-proc.ms.sum": 542,
    "make-proc.ms.avg": 135,
    "make-proc.ms.cnt": 4,
    "make-proc.ms.sum": 542,
    "make-root-fs.ms.avg": 4,
    "make-root-fs.ms.cnt": 4,
    "make-root-fs.ms.sum": 16
  },
  "conf": {
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "registry": "/root/open-lambda/test-registry",
    "pip_mirror": "",
    "docker_runtime": "",
    "server_mode": "lambda",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "registry_cache_ms": 5000,
    "timing": false,
    "sandbox": "sock",
    "handler_cache_mb": 100,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "Skip_pull_existing": false,
    "sock_cgroups": {
      "max_procs": 10,
      "max_mem_mb": 50
    },
    "import_cache_mb": 0,
    "benchmark_log": "",
    "sandbox_config": {
      "processes": 10
    },
    "max_runners": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000"
  },
  "errors": [],
  "worker_tail": [
    "2019/06/27 22:26:44 remove /sys/fs/cgroup/devices/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:44 remove /sys/fs/cgroup/freezer/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:44 remove /sys/fs/cgroup/hugetlb/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:44 remove /sys/fs/cgroup/memory/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:44 remove /sys/fs/cgroup/perf_event/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:44 remove /sys/fs/cgroup/systemd/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:44 remove /sys/fs/cgroup/pids/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:44 remove worker.pid",
    "2019/06/27 22:26:44 exiting",
    ""
  ]
}
POP conf: {'import_cache_mb': 0, 'handler_cache_mb': 100}
PUSH conf: {'import_cache_mb': 0, 'handler_cache_mb': 250}
========================================
smoke_tests
========================================
RUN ./ol worker -p=test-cluster --detach
b'using existing OL directory at /root/open-lambda/test-cluster\nStarting worker: pid=13322, port=5000, log=/root/open-lambda/test-cluster/worker.out\nready\n'
RUN ./ol kill -p=test-cluster
b'Kill worker process with PID 13322\n'
{
  "test": "smoke_tests",
  "params": {},
  "pass": true,
  "seconds": 1.4040615558624268,
  "total_seconds": 9.378395080566406,
  "stats": null,
  "ol-stats": {
    "Channel().ms.avg": 0,
    "Channel().ms.cnt": 4,
    "Channel().ms.sum": 0,
    "Create().ms.avg": 145,
    "Create().ms.cnt": 4,
    "Create().ms.sum": 580,
    "Pause().ms.avg": 6,
    "Pause().ms.cnt": 3,
    "Pause().ms.sum": 18,
    "acquire-cgroup.ms.avg": 0,
    "acquire-cgroup.ms.cnt": 4,
    "acquire-cgroup.ms.sum": 0,
    "acquire-mem.ms.avg": 0,
    "acquire-mem.ms.cnt": 4,
    "acquire-mem.ms.sum": 0,
    "fresh-proc.ms.avg": 140,
    "fresh-proc.ms.cnt": 4,
    "fresh-proc.ms.sum": 563,
    "make-proc.ms.avg": 141,
    "make-proc.ms.cnt": 4,
    "make-proc.ms.sum": 564,
    "make-root-fs.ms.avg": 3,
    "make-root-fs.ms.cnt": 4,
    "make-root-fs.ms.sum": 15
  },
  "conf": {
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "timing": false,
    "sandbox_config": {
      "processes": 10
    },
    "server_mode": "lambda",
    "sandbox": "sock",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "registry_cache_ms": 5000,
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "handler_cache_mb": 250,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "Skip_pull_existing": false,
    "sock_cgroups": {
      "max_procs": 10,
      "max_mem_mb": 50
    },
    "import_cache_mb": 0,
    "docker_runtime": "",
    "max_runners": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "benchmark_log": ""
  },
  "errors": [],
  "worker_tail": [
    "2019/06/27 22:26:54 remove /sys/fs/cgroup/devices/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:54 remove /sys/fs/cgroup/freezer/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:54 remove /sys/fs/cgroup/hugetlb/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:54 remove /sys/fs/cgroup/memory/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:54 remove /sys/fs/cgroup/perf_event/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:54 remove /sys/fs/cgroup/systemd/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:54 remove /sys/fs/cgroup/pids/sock-handlers [CGROUP POOL sock-handlers]",
    "2019/06/27 22:26:54 remove worker.pid",
    "2019/06/27 22:26:54 exiting",
    ""
  ]
}
POP conf: {'import_cache_mb': 0, 'handler_cache_mb': 250}
PUSH conf: {'import_cache_mb': 250, 'handler_cache_mb': 100}
========================================
smoke_tests
========================================
RUN ./ol worker -p=test-cluster --detach
b'using existing OL directory at /root/open-lambda/test-cluster\nStarting worker: pid=13400, port=5000, log=/root/open-lambda/test-cluster/worker.out\nready\n'
RUN ./ol kill -p=test-cluster
b'Kill worker process with PID 13400\n'
{
  "test": "smoke_tests",
  "params": {},
  "pass": true,
  "seconds": 1.3259167671203613,
  "total_seconds": 9.490275621414185,
  "stats": null,
  "ol-stats": {
    "Channel().ms.avg": 0,
    "Channel().ms.cnt": 4,
    "Channel().ms.sum": 0,
    "Create().ms.avg": 144,
    "Create().ms.cnt": 5,
    "Create().ms.sum": 724,
    "Destroy().ms.avg": 3,
    "Destroy().ms.cnt": 3,
    "Destroy().ms.sum": 10,
    "Pause().ms.avg": 9,
    "Pause().ms.cnt": 3,
    "Pause().ms.sum": 27,
    "acquire-cgroup.ms.avg": 0,
    "acquire-cgroup.ms.cnt": 5,
    "acquire-cgroup.ms.sum": 3,
    "acquire-mem.ms.avg": 0,
    "acquire-mem.ms.cnt": 5,
    "acquire-mem.ms.sum": 3,
    "evict.ms.avg": 3,
    "evict.ms.cnt": 3,
    "evict.ms.sum": 10,
    "fresh-proc.ms.avg": 139,
    "fresh-proc.ms.cnt": 5,
    "fresh-proc.ms.sum": 696,
    "make-proc.ms.avg": 139,
    "make-proc.ms.cnt": 5,
    "make-proc.ms.sum": 696,
    "make-root-fs.ms.avg": 3,
    "make-root-fs.ms.cnt": 5,
    "make-root-fs.ms.sum": 17
  },
  "conf": {
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "registry": "/root/open-lambda/test-registry",
    "pip_mirror": "",
    "docker_runtime": "",
    "server_mode": "lambda",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "registry_cache_ms": 5000,
    "timing": false,
    "sandbox": "sock",
    "handler_cache_mb": 100,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "Skip_pull_existing": false,
    "sock_cgroups": {
      "max_procs": 10,
      "max_mem_mb": 50
    },
    "import_cache_mb": 250,
    "benchmark_log": "",
    "sandbox_config": {
      "processes": 10
    },
    "max_runners": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000"
  },
  "errors": [],
  "worker_tail": [
    "2019/06/27 22:27:03 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:03 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:03 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:03 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:03 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:03 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:03 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:03 remove worker.pid",
    "2019/06/27 22:27:03 exiting",
    ""
  ]
}
POP conf: {'import_cache_mb': 250, 'handler_cache_mb': 100}
PUSH conf: {'import_cache_mb': 250, 'handler_cache_mb': 250}
========================================
smoke_tests
========================================
RUN ./ol worker -p=test-cluster --detach
b'using existing OL directory at /root/open-lambda/test-cluster\nStarting worker: pid=13484, port=5000, log=/root/open-lambda/test-cluster/worker.out\nready\n'
RUN ./ol kill -p=test-cluster
b'Kill worker process with PID 13484\n'
{
  "test": "smoke_tests",
  "params": {},
  "pass": true,
  "seconds": 1.484497308731079,
  "total_seconds": 9.660788774490356,
  "stats": null,
  "ol-stats": {
    "Channel().ms.avg": 0,
    "Channel().ms.cnt": 4,
    "Channel().ms.sum": 0,
    "Create().ms.avg": 142,
    "Create().ms.cnt": 5,
    "Create().ms.sum": 711,
    "Pause().ms.avg": 5,
    "Pause().ms.cnt": 3,
    "Pause().ms.sum": 16,
    "acquire-cgroup.ms.avg": 0,
    "acquire-cgroup.ms.cnt": 5,
    "acquire-cgroup.ms.sum": 0,
    "acquire-mem.ms.avg": 0,
    "acquire-mem.ms.cnt": 5,
    "acquire-mem.ms.sum": 3,
    "fresh-proc.ms.avg": 137,
    "fresh-proc.ms.cnt": 5,
    "fresh-proc.ms.sum": 685,
    "make-proc.ms.avg": 137,
    "make-proc.ms.cnt": 5,
    "make-proc.ms.sum": 686,
    "make-root-fs.ms.avg": 3,
    "make-root-fs.ms.cnt": 5,
    "make-root-fs.ms.sum": 19
  },
  "conf": {
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "timing": false,
    "sandbox_config": {
      "processes": 10
    },
    "server_mode": "lambda",
    "sandbox": "sock",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "registry_cache_ms": 5000,
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "handler_cache_mb": 250,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "Skip_pull_existing": false,
    "sock_cgroups": {
      "max_procs": 10,
      "max_mem_mb": 50
    },
    "import_cache_mb": 250,
    "docker_runtime": "",
    "max_runners": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "benchmark_log": ""
  },
  "errors": [],
  "worker_tail": [
    "2019/06/27 22:27:13 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:13 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:13 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:13 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:13 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:13 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:13 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:13 remove worker.pid",
    "2019/06/27 22:27:13 exiting",
    ""
  ]
}
POP conf: {'import_cache_mb': 250, 'handler_cache_mb': 250}
PUSH conf: {'sandbox': 'docker', 'handler_cache_mb': 100, 'import_cache_mb': 0}
========================================
smoke_tests
========================================
RUN ./ol worker -p=test-cluster --detach
b'using existing OL directory at /root/open-lambda/test-cluster\nStarting worker: pid=13568, port=5000, log=/root/open-lambda/test-cluster/worker.out\nready\n'
RUN ./ol kill -p=test-cluster
b'Kill worker process with PID 13568\n'
{
  "test": "smoke_tests",
  "params": {},
  "pass": true,
  "seconds": 6.2217888832092285,
  "total_seconds": 15.80888295173645,
  "stats": null,
  "ol-stats": {
    "Channel().ms.avg": 0,
    "Channel().ms.cnt": 4,
    "Channel().ms.sum": 0,
    "Create().ms.avg": 1545,
    "Create().ms.cnt": 4,
    "Create().ms.sum": 6182,
    "Pause().ms.avg": 27,
    "Pause().ms.cnt": 3,
    "Pause().ms.sum": 83
  },
  "conf": {
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "registry": "/root/open-lambda/test-registry",
    "pip_mirror": "",
    "docker_runtime": "",
    "server_mode": "lambda",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "registry_cache_ms": 5000,
    "timing": false,
    "sandbox": "docker",
    "handler_cache_mb": 100,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "Skip_pull_existing": false,
    "sock_cgroups": {
      "max_procs": 10,
      "max_mem_mb": 50
    },
    "import_cache_mb": 0,
    "benchmark_log": "",
    "sandbox_config": {
      "processes": 10
    },
    "max_runners": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000"
  },
  "errors": [],
  "worker_tail": [
    "2019/06/27 22:27:27 Pause() [SB 3]",
    "2019/06/27 22:27:27 Receive request to /stats",
    "2019/06/27 22:27:27 received kill signal, cleaning up",
    "2019/06/27 22:27:27 Destroy() [SB 0]",
    "2019/06/27 22:27:28 Destroy() [SB 1]",
    "2019/06/27 22:27:28 Destroy() [SB 2]",
    "2019/06/27 22:27:28 Destroy() [SB 3]",
    "2019/06/27 22:27:29 remove worker.pid",
    "2019/06/27 22:27:29 exiting",
    ""
  ]
}
POP conf: {'sandbox': 'docker', 'handler_cache_mb': 100, 'import_cache_mb': 0}
PUSH conf: {'sandbox': 'docker', 'handler_cache_mb': 250, 'import_cache_mb': 0}
========================================
smoke_tests
========================================
RUN ./ol worker -p=test-cluster --detach
b'using existing OL directory at /root/open-lambda/test-cluster\nStarting worker: pid=14613, port=5000, log=/root/open-lambda/test-cluster/worker.out\nready\n'
RUN ./ol kill -p=test-cluster
b'Kill worker process with PID 14613\n'
{
  "test": "smoke_tests",
  "params": {},
  "pass": true,
  "seconds": 2.8188772201538086,
  "total_seconds": 11.7828209400177,
  "stats": null,
  "ol-stats": {
    "Channel().ms.avg": 0,
    "Channel().ms.cnt": 4,
    "Channel().ms.sum": 0,
    "Create().ms.avg": 694,
    "Create().ms.cnt": 4,
    "Create().ms.sum": 2778,
    "Pause().ms.avg": 28,
    "Pause().ms.cnt": 3,
    "Pause().ms.sum": 86
  },
  "conf": {
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "registry": "/root/open-lambda/test-registry",
    "worker_port": "5000",
    "timing": false,
    "sandbox_config": {
      "processes": 10
    },
    "server_mode": "lambda",
    "sandbox": "docker",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "registry_cache_ms": 5000,
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "handler_cache_mb": 250,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "Skip_pull_existing": false,
    "sock_cgroups": {
      "max_procs": 10,
      "max_mem_mb": 50
    },
    "import_cache_mb": 0,
    "docker_runtime": "",
    "max_runners": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "benchmark_log": ""
  },
  "errors": [],
  "worker_tail": [
    "2019/06/27 22:27:39 Pause() [SB 3]",
    "2019/06/27 22:27:39 Receive request to /stats",
    "2019/06/27 22:27:39 received kill signal, cleaning up",
    "2019/06/27 22:27:39 Destroy() [SB 0]",
    "2019/06/27 22:27:40 Destroy() [SB 1]",
    "2019/06/27 22:27:40 Destroy() [SB 2]",
    "2019/06/27 22:27:40 Destroy() [SB 3]",
    "2019/06/27 22:27:40 remove worker.pid",
    "2019/06/27 22:27:40 exiting",
    ""
  ]
}
POP conf: {'sandbox': 'docker', 'handler_cache_mb': 250, 'import_cache_mb': 0}
========================================
fork_bomb
========================================
RUN ./ol worker -p=test-cluster --detach
b'using existing OL directory at /root/open-lambda/test-cluster\nStarting worker: pid=15654, port=5000, log=/root/open-lambda/test-cluster/worker.out\nready\n'
RUN ./ol kill -p=test-cluster
b'Kill worker process with PID 15654\n'
{
  "test": "fork_bomb",
  "params": {},
  "pass": true,
  "seconds": 0.16465973854064941,
  "total_seconds": 8.529998540878296,
  "stats": null,
  "ol-stats": {
    "Channel().ms.avg": 0,
    "Channel().ms.cnt": 1,
    "Channel().ms.sum": 0,
    "Create().ms.avg": 241,
    "Create().ms.cnt": 2,
    "Create().ms.sum": 482,
    "acquire-cgroup.ms.avg": 1,
    "acquire-cgroup.ms.cnt": 2,
    "acquire-cgroup.ms.sum": 3,
    "acquire-mem.ms.avg": 1,
    "acquire-mem.ms.cnt": 2,
    "acquire-mem.ms.sum": 3,
    "fresh-proc.ms.avg": 231,
    "fresh-proc.ms.cnt": 2,
    "fresh-proc.ms.sum": 463,
    "make-proc.ms.avg": 231,
    "make-proc.ms.cnt": 2,
    "make-proc.ms.sum": 463,
    "make-root-fs.ms.avg": 5,
    "make-root-fs.ms.cnt": 2,
    "make-root-fs.ms.sum": 10
  },
  "conf": {
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "registry": "/root/open-lambda/test-registry",
    "pip_mirror": "",
    "docker_runtime": "",
    "server_mode": "lambda",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "registry_cache_ms": 5000,
    "timing": false,
    "sandbox": "sock",
    "handler_cache_mb": 239,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "Skip_pull_existing": false,
    "sock_cgroups": {
      "max_procs": 10,
      "max_mem_mb": 50
    },
    "import_cache_mb": 239,
    "benchmark_log": "",
    "sandbox_config": {
      "processes": 10
    },
    "max_runners": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000"
  },
  "errors": [],
  "worker_tail": [
    "2019/06/27 22:27:49 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:49 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:49 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:49 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:49 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:49 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:49 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:49 remove worker.pid",
    "2019/06/27 22:27:49 exiting",
    ""
  ]
}
========================================
max_mem_alloc
========================================
RUN ./ol worker -p=test-cluster --detach
b'using existing OL directory at /root/open-lambda/test-cluster\nStarting worker: pid=15730, port=5000, log=/root/open-lambda/test-cluster/worker.out\nready\n'
RUN ./ol kill -p=test-cluster
b'Kill worker process with PID 15730\n'
{
  "test": "max_mem_alloc",
  "params": {},
  "pass": true,
  "seconds": 0.9808769226074219,
  "total_seconds": 9.144822359085083,
  "stats": null,
  "ol-stats": {
    "Channel().ms.avg": 0,
    "Channel().ms.cnt": 1,
    "Channel().ms.sum": 0,
    "Create().ms.avg": 144,
    "Create().ms.cnt": 2,
    "Create().ms.sum": 289,
    "acquire-cgroup.ms.avg": 1,
    "acquire-cgroup.ms.cnt": 2,
    "acquire-cgroup.ms.sum": 3,
    "acquire-mem.ms.avg": 1,
    "acquire-mem.ms.cnt": 2,
    "acquire-mem.ms.sum": 3,
    "fresh-proc.ms.avg": 135,
    "fresh-proc.ms.cnt": 2,
    "fresh-proc.ms.sum": 271,
    "make-proc.ms.avg": 135,
    "make-proc.ms.cnt": 2,
    "make-proc.ms.sum": 271,
    "make-root-fs.ms.avg": 4,
    "make-root-fs.ms.cnt": 2,
    "make-root-fs.ms.sum": 8
  },
  "conf": {
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "registry": "/root/open-lambda/test-registry",
    "pip_mirror": "",
    "docker_runtime": "",
    "server_mode": "lambda",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "registry_cache_ms": 5000,
    "timing": false,
    "sandbox": "sock",
    "handler_cache_mb": 239,
    "startup_pkgs": [
      "parso",
      "jedi",
      "urllib3",
      "idna",
      "chardet",
      "certifi",
      "requests",
      "simplejson"
    ],
    "Skip_pull_existing": false,
    "sock_cgroups": {
      "max_procs": 10,
      "max_mem_mb": 50
    },
    "import_cache_mb": 239,
    "benchmark_log": "",
    "sandbox_config": {
      "processes": 10
    },
    "max_runners": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000"
  },
  "errors": [],
  "worker_tail": [
    "2019/06/27 22:27:58 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:58 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:58 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:58 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:58 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:58 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:58 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:27:58 remove worker.pid",
    "2019/06/27 22:27:58 exiting",
    ""
  ]
}
POP conf: {'registry': '/root/open-lambda/test-registry', 'startup_pkgs': ['parso', 'jedi', 'urllib3', 'idna', 'chardet', 'certifi', 'requests', 'simplejson']}
PUSH conf: {'sandbox': 'sock', 'registry': '/tmp/tmpn9st1__u', 'registry_cache_ms': 3000}
========================================
update_code
========================================
RUN ./ol worker -p=test-cluster --detach
b'using existing OL directory at /root/open-lambda/test-cluster\nStarting worker: pid=15847, port=5000, log=/root/open-lambda/test-cluster/worker.out\nready\n'
RUN ./ol kill -p=test-cluster
b'Kill worker process with PID 15847\n'
{
  "test": "update_code",
  "params": {},
  "pass": true,
  "seconds": 6.159935474395752,
  "total_seconds": 6.495927810668945,
  "stats": null,
  "ol-stats": {
    "Channel().ms.avg": 0,
    "Channel().ms.cnt": 3,
    "Channel().ms.sum": 0,
    "Create().ms.avg": 146,
    "Create().ms.cnt": 4,
    "Create().ms.sum": 584,
    "Destroy().ms.avg": 1,
    "Destroy().ms.cnt": 2,
    "Destroy().ms.sum": 2,
    "Pause().ms.avg": 0,
    "Pause().ms.cnt": 3414,
    "Pause().ms.sum": 8,
    "acquire-cgroup.ms.avg": 0,
    "acquire-cgroup.ms.cnt": 4,
    "acquire-cgroup.ms.sum": 3,
    "acquire-mem.ms.avg": 0,
    "acquire-mem.ms.cnt": 4,
    "acquire-mem.ms.sum": 3,
    "fresh-proc.ms.avg": 140,
    "fresh-proc.ms.cnt": 4,
    "fresh-proc.ms.sum": 561,
    "make-proc.ms.avg": 140,
    "make-proc.ms.cnt": 4,
    "make-proc.ms.sum": 561,
    "make-root-fs.ms.avg": 3,
    "make-root-fs.ms.cnt": 4,
    "make-root-fs.ms.sum": 14
  },
  "conf": {
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "registry": "/tmp/tmpn9st1__u",
    "worker_port": "5000",
    "timing": false,
    "sandbox_config": {
      "processes": 10
    },
    "server_mode": "lambda",
    "sandbox": "sock",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "registry_cache_ms": 3000,
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "handler_cache_mb": 239,
    "startup_pkgs": null,
    "Skip_pull_existing": false,
    "sock_cgroups": {
      "max_procs": 10,
      "max_mem_mb": 50
    },
    "import_cache_mb": 239,
    "docker_runtime": "",
    "max_runners": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "benchmark_log": ""
  },
  "errors": [],
  "worker_tail": [
    "2019/06/27 22:28:05 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:05 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:05 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:05 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:05 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:05 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:05 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:05 remove worker.pid",
    "2019/06/27 22:28:05 exiting",
    ""
  ]
}
POP conf: {'sandbox': 'sock', 'registry': '/tmp/tmpn9st1__u', 'registry_cache_ms': 3000}
PUSH conf: {'sandbox': 'sock', 'handler_cache_mb': 250, 'import_cache_mb': 250, 'registry': '/root/open-lambda/test-registry'}
========================================
stress_one_lambda
========================================
RUN ./ol worker -p=test-cluster --detach
b'using existing OL directory at /root/open-lambda/test-cluster\nStarting worker: pid=15892, port=5000, log=/root/open-lambda/test-cluster/worker.out\nready\n'
RUN ./ol kill -p=test-cluster
b'Kill worker process with PID 15892\n'
{
  "test": "stress_one_lambda",
  "params": {
    "procs": 1,
    "seconds": 15
  },
  "pass": true,
  "seconds": 15.046745300292969,
  "total_seconds": 15.380524635314941,
  "stats": {
    "reqs_per_sec": 292.1333333333333
  },
  "ol-stats": {
    "Channel().ms.avg": 0,
    "Channel().ms.cnt": 3,
    "Channel().ms.sum": 0,
    "Create().ms.avg": 146,
    "Create().ms.cnt": 4,
    "Create().ms.sum": 585,
    "Destroy().ms.avg": 1,
    "Destroy().ms.cnt": 2,
    "Destroy().ms.sum": 2,
    "Pause().ms.avg": 0,
    "Pause().ms.cnt": 8761,
    "Pause().ms.sum": 16,
    "acquire-cgroup.ms.avg": 0,
    "acquire-cgroup.ms.cnt": 4,
    "acquire-cgroup.ms.sum": 0,
    "acquire-mem.ms.avg": 0,
    "acquire-mem.ms.cnt": 4,
    "acquire-mem.ms.sum": 3,
    "fresh-proc.ms.avg": 141,
    "fresh-proc.ms.cnt": 4,
    "fresh-proc.ms.sum": 564,
    "make-proc.ms.avg": 141,
    "make-proc.ms.cnt": 4,
    "make-proc.ms.sum": 564,
    "make-root-fs.ms.avg": 3,
    "make-root-fs.ms.cnt": 4,
    "make-root-fs.ms.sum": 14
  },
  "conf": {
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "registry": "/root/open-lambda/test-registry",
    "pip_mirror": "",
    "docker_runtime": "",
    "server_mode": "lambda",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "registry_cache_ms": 5000,
    "timing": false,
    "sandbox": "sock",
    "handler_cache_mb": 250,
    "startup_pkgs": null,
    "Skip_pull_existing": false,
    "sock_cgroups": {
      "max_procs": 10,
      "max_mem_mb": 50
    },
    "import_cache_mb": 250,
    "benchmark_log": "",
    "sandbox_config": {
      "processes": 10
    },
    "max_runners": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000"
  },
  "errors": [],
  "worker_tail": [
    "2019/06/27 22:28:20 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:20 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:20 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:20 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:20 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:20 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:20 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:20 remove worker.pid",
    "2019/06/27 22:28:20 exiting",
    ""
  ]
}
========================================
stress_one_lambda
========================================
RUN ./ol worker -p=test-cluster --detach
b'using existing OL directory at /root/open-lambda/test-cluster\nStarting worker: pid=15940, port=5000, log=/root/open-lambda/test-cluster/worker.out\nready\n'
RUN ./ol kill -p=test-cluster
b'Kill worker process with PID 15940\n'
{
  "test": "stress_one_lambda",
  "params": {
    "procs": 2,
    "seconds": 15
  },
  "pass": true,
  "seconds": 15.047110319137573,
  "total_seconds": 15.378380060195923,
  "stats": {
    "reqs_per_sec": 292.26666666666665
  },
  "ol-stats": {
    "Channel().ms.avg": 0,
    "Channel().ms.cnt": 3,
    "Channel().ms.sum": 0,
    "Create().ms.avg": 145,
    "Create().ms.cnt": 4,
    "Create().ms.sum": 582,
    "Destroy().ms.avg": 1,
    "Destroy().ms.cnt": 2,
    "Destroy().ms.sum": 2,
    "Pause().ms.avg": 0,
    "Pause().ms.cnt": 8121,
    "Pause().ms.sum": 5,
    "acquire-cgroup.ms.avg": 0,
    "acquire-cgroup.ms.cnt": 4,
    "acquire-cgroup.ms.sum": 3,
    "acquire-mem.ms.avg": 0,
    "acquire-mem.ms.cnt": 4,
    "acquire-mem.ms.sum": 3,
    "fresh-proc.ms.avg": 139,
    "fresh-proc.ms.cnt": 4,
    "fresh-proc.ms.sum": 559,
    "make-proc.ms.avg": 139,
    "make-proc.ms.cnt": 4,
    "make-proc.ms.sum": 559,
    "make-root-fs.ms.avg": 3,
    "make-root-fs.ms.cnt": 4,
    "make-root-fs.ms.sum": 14
  },
  "conf": {
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "registry": "/root/open-lambda/test-registry",
    "pip_mirror": "",
    "docker_runtime": "",
    "server_mode": "lambda",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "registry_cache_ms": 5000,
    "timing": false,
    "sandbox": "sock",
    "handler_cache_mb": 250,
    "startup_pkgs": null,
    "Skip_pull_existing": false,
    "sock_cgroups": {
      "max_procs": 10,
      "max_mem_mb": 50
    },
    "import_cache_mb": 250,
    "benchmark_log": "",
    "sandbox_config": {
      "processes": 10
    },
    "max_runners": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000"
  },
  "errors": [],
  "worker_tail": [
    "2019/06/27 22:28:35 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:35 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:35 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:35 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:35 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:35 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:35 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:35 remove worker.pid",
    "2019/06/27 22:28:35 exiting",
    ""
  ]
}
PUSH conf: {'import_cache_mb': 100, 'handler_cache_mb': 500}
========================================
stress_one_lambda
========================================
RUN ./ol worker -p=test-cluster --detach
b'using existing OL directory at /root/open-lambda/test-cluster\nStarting worker: pid=15990, port=5000, log=/root/open-lambda/test-cluster/worker.out\nready\n'
RUN ./ol kill -p=test-cluster
b'Kill worker process with PID 15990\n'
{
  "test": "stress_one_lambda",
  "params": {
    "procs": 8,
    "seconds": 15
  },
  "pass": true,
  "seconds": 15.065525531768799,
  "total_seconds": 15.397970914840698,
  "stats": {
    "reqs_per_sec": 265.4
  },
  "ol-stats": {
    "Channel().ms.avg": 0,
    "Channel().ms.cnt": 3,
    "Channel().ms.sum": 0,
    "Create().ms.avg": 146,
    "Create().ms.cnt": 4,
    "Create().ms.sum": 585,
    "Destroy().ms.avg": 1,
    "Destroy().ms.cnt": 2,
    "Destroy().ms.sum": 2,
    "Pause().ms.avg": 0,
    "Pause().ms.cnt": 6673,
    "Pause().ms.sum": 923,
    "acquire-cgroup.ms.avg": 0,
    "acquire-cgroup.ms.cnt": 4,
    "acquire-cgroup.ms.sum": 3,
    "acquire-mem.ms.avg": 0,
    "acquire-mem.ms.cnt": 4,
    "acquire-mem.ms.sum": 3,
    "fresh-proc.ms.avg": 140,
    "fresh-proc.ms.cnt": 4,
    "fresh-proc.ms.sum": 560,
    "make-proc.ms.avg": 140,
    "make-proc.ms.cnt": 4,
    "make-proc.ms.sum": 560,
    "make-root-fs.ms.avg": 3,
    "make-root-fs.ms.cnt": 4,
    "make-root-fs.ms.sum": 15
  },
  "conf": {
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "registry": "/root/open-lambda/test-registry",
    "pip_mirror": "",
    "docker_runtime": "",
    "server_mode": "lambda",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "registry_cache_ms": 5000,
    "timing": false,
    "sandbox": "sock",
    "handler_cache_mb": 500,
    "startup_pkgs": null,
    "Skip_pull_existing": false,
    "sock_cgroups": {
      "max_procs": 10,
      "max_mem_mb": 50
    },
    "import_cache_mb": 100,
    "benchmark_log": "",
    "sandbox_config": {
      "processes": 10
    },
    "max_runners": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000"
  },
  "errors": [],
  "worker_tail": [
    "2019/06/27 22:28:51 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:51 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:51 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:51 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:51 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:51 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:51 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:28:51 remove worker.pid",
    "2019/06/27 22:28:51 exiting",
    ""
  ]
}
POP conf: {'import_cache_mb': 100, 'handler_cache_mb': 500}
POP conf: {'sandbox': 'sock', 'handler_cache_mb': 250, 'import_cache_mb': 250, 'registry': '/root/open-lambda/test-registry'}
PUSH conf: {'sandbox': 'sock', 'handler_cache_mb': 250, 'import_cache_mb': 250}
PUSH conf: {'registry': '/tmp/tmpquz_bdqp'}
========================================
call_each_once_exec
========================================
RUN ./ol worker -p=test-cluster --detach
b'using existing OL directory at /root/open-lambda/test-cluster\nStarting worker: pid=16046, port=5000, log=/root/open-lambda/test-cluster/worker.out\nready\n'
RUN ./ol kill -p=test-cluster
b'Kill worker process with PID 16046\n'
{
  "test": "call_each_once_exec",
  "params": {
    "alloc_mb": 1,
    "lambda_count": 100
  },
  "pass": true,
  "seconds": 15.741478204727173,
  "total_seconds": 16.085567235946655,
  "stats": {
    "reqs_per_sec": 6.35265432722459
  },
  "ol-stats": {
    "Channel().ms.avg": 0,
    "Channel().ms.cnt": 100,
    "Channel().ms.sum": 0,
    "Create().ms.avg": 141,
    "Create().ms.cnt": 101,
    "Create().ms.sum": 14258,
    "Destroy().ms.avg": 7,
    "Destroy().ms.cnt": 96,
    "Destroy().ms.sum": 716,
    "Pause().ms.avg": 5,
    "Pause().ms.cnt": 99,
    "Pause().ms.sum": 521,
    "acquire-cgroup.ms.avg": 0,
    "acquire-cgroup.ms.cnt": 101,
    "acquire-cgroup.ms.sum": 3,
    "acquire-mem.ms.avg": 0,
    "acquire-mem.ms.cnt": 101,
    "acquire-mem.ms.sum": 3,
    "evict.ms.avg": 7,
    "evict.ms.cnt": 96,
    "evict.ms.sum": 718,
    "fresh-proc.ms.avg": 138,
    "fresh-proc.ms.cnt": 101,
    "fresh-proc.ms.sum": 13944,
    "make-proc.ms.avg": 138,
    "make-proc.ms.cnt": 101,
    "make-proc.ms.sum": 13944,
    "make-root-fs.ms.avg": 2,
    "make-root-fs.ms.cnt": 101,
    "make-root-fs.ms.sum": 219
  },
  "conf": {
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "registry": "/tmp/tmpquz_bdqp",
    "worker_port": "5000",
    "timing": false,
    "sandbox_config": {
      "processes": 10
    },
    "server_mode": "lambda",
    "sandbox": "sock",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "registry_cache_ms": 5000,
    "pip_mirror": "",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "handler_cache_mb": 250,
    "startup_pkgs": null,
    "Skip_pull_existing": false,
    "sock_cgroups": {
      "max_procs": 10,
      "max_mem_mb": 50
    },
    "import_cache_mb": 250,
    "docker_runtime": "",
    "max_runners": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "benchmark_log": ""
  },
  "errors": [],
  "worker_tail": [
    "2019/06/27 22:29:07 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:29:07 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:29:07 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:29:07 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:29:07 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:29:07 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:29:07 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:29:07 remove worker.pid",
    "2019/06/27 22:29:07 exiting",
    ""
  ]
}
POP conf: {'registry': '/tmp/tmpquz_bdqp'}
PUSH conf: {'registry': '/tmp/tmpasjc9hwo'}
========================================
call_each_once_exec
========================================
RUN ./ol worker -p=test-cluster --detach
b'using existing OL directory at /root/open-lambda/test-cluster\nStarting worker: pid=16576, port=5000, log=/root/open-lambda/test-cluster/worker.out\nready\n'
RUN ./ol kill -p=test-cluster
b'Kill worker process with PID 16576\n'
{
  "test": "call_each_once_exec",
  "params": {
    "alloc_mb": 10,
    "lambda_count": 1000
  },
  "pass": true,
  "seconds": 164.81611251831055,
  "total_seconds": 165.14329648017883,
  "stats": {
    "reqs_per_sec": 6.067368914343323
  },
  "ol-stats": {
    "Channel().ms.avg": 0,
    "Channel().ms.cnt": 1000,
    "Channel().ms.sum": 0,
    "Create().ms.avg": 145,
    "Create().ms.cnt": 1001,
    "Create().ms.sum": 145707,
    "Destroy().ms.avg": 3,
    "Destroy().ms.cnt": 996,
    "Destroy().ms.sum": 3047,
    "Pause().ms.avg": 7,
    "Pause().ms.cnt": 999,
    "Pause().ms.sum": 7443,
    "acquire-cgroup.ms.avg": 0,
    "acquire-cgroup.ms.cnt": 1001,
    "acquire-cgroup.ms.sum": 3,
    "acquire-mem.ms.avg": 0,
    "acquire-mem.ms.cnt": 1001,
    "acquire-mem.ms.sum": 3,
    "evict.ms.avg": 3,
    "evict.ms.cnt": 996,
    "evict.ms.sum": 3059,
    "fresh-proc.ms.avg": 142,
    "fresh-proc.ms.cnt": 1001,
    "fresh-proc.ms.sum": 142488,
    "make-proc.ms.avg": 142,
    "make-proc.ms.cnt": 1001,
    "make-proc.ms.sum": 142489,
    "make-root-fs.ms.avg": 2,
    "make-root-fs.ms.cnt": 1001,
    "make-root-fs.ms.sum": 2734
  },
  "conf": {
    "Pkgs_dir": "/root/open-lambda/test-cluster/lambda/packages",
    "registry": "/tmp/tmpasjc9hwo",
    "pip_mirror": "",
    "docker_runtime": "",
    "server_mode": "lambda",
    "SOCK_base_path": "/root/open-lambda/test-cluster/lambda",
    "worker_dir": "/root/open-lambda/test-cluster/worker",
    "registry_cache_ms": 5000,
    "timing": false,
    "sandbox": "sock",
    "handler_cache_mb": 250,
    "startup_pkgs": null,
    "Skip_pull_existing": false,
    "sock_cgroups": {
      "max_procs": 10,
      "max_mem_mb": 50
    },
    "import_cache_mb": 250,
    "benchmark_log": "",
    "sandbox_config": {
      "processes": 10
    },
    "max_runners": 0,
    "cluster_name": "/root/open-lambda/test-cluster",
    "worker_port": "5000"
  },
  "errors": [],
  "worker_tail": [
    "2019/06/27 22:31:52 remove /sys/fs/cgroup/devices/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:31:52 remove /sys/fs/cgroup/freezer/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:31:52 remove /sys/fs/cgroup/hugetlb/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:31:52 remove /sys/fs/cgroup/memory/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:31:52 remove /sys/fs/cgroup/perf_event/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:31:52 remove /sys/fs/cgroup/systemd/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:31:52 remove /sys/fs/cgroup/pids/sock-cache [CGROUP POOL sock-cache]",
    "2019/06/27 22:31:52 remove worker.pid",
    "2019/06/27 22:31:52 exiting",
    ""
  ]
}
POP conf: {'registry': '/tmp/tmpasjc9hwo'}
POP conf: {'sandbox': 'sock', 'handler_cache_mb': 250, 'import_cache_mb': 250}
PASSED: 17, FAILED: 0