{
  "runs": [
    {
      "test": "ping_test",
      "params": {},
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "sandbox_config": {},
        "sandbox": "sock",
        "limits": {
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 200
        },
        "registry": "/root/open-lambda/test-registry",
        "import_cache_mb": 250,
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "registry_cache_ms": 5000,
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "import_cache_tree": "",
        "handler_cache_mb": 250,
        "docker_runtime": "",
        "pip_mirror": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "server_mode": "lambda"
      },
      "seconds": 2.1735942363739014,
      "total_seconds": 2.5009994506835938,
      "stats": {
        "pings_per_sec": 460.0721122283986
      },
      "ol-stats": {},
      "errors": [],
      "worker_tail": [
        "2019/07/15 15:34:20 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:20 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:20 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:20 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:20 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:20 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:20 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/07/15 15:34:20 remove worker.pid",
        "2019/07/15 15:34:20 exiting",
        ""
      ]
    },
    {
      "test": "install_tests",
      "params": {},
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "sandbox_config": {},
        "sandbox": "sock",
        "limits": {
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 200
        },
        "registry": "/root/open-lambda/test-registry",
        "import_cache_mb": 0,
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "registry_cache_ms": 5000,
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "import_cache_tree": "",
        "handler_cache_mb": 500,
        "docker_runtime": "",
        "pip_mirror": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "server_mode": "lambda"
      },
      "seconds": 11.574580430984497,
      "total_seconds": 11.812131404876709,
      "stats": null,
      "ol-stats": {
        "Channel():ms.avg": 0,
        "Channel():ms.cnt": 10,
        "Create()/acquire-cgroup:ms.avg": 0,
        "Create()/acquire-cgroup:ms.cnt": 10,
        "Create()/acquire-mem:ms.avg": 0,
        "Create()/acquire-mem:ms.cnt": 10,
        "Create()/fresh-proc:ms.avg": 280,
        "Create()/fresh-proc:ms.cnt": 10,
        "Create()/make-root-fs:ms.avg": 1,
        "Create()/make-root-fs:ms.cnt": 10,
        "Create():ms.avg": 282,
        "Create():ms.cnt": 10,
        "Destroy()/detach-root:ms.avg": 0,
        "Destroy()/detach-root:ms.cnt": 10,
        "Destroy()/kill-procs:ms.avg": 1,
        "Destroy()/kill-procs:ms.cnt": 10,
        "Destroy()/recursive-kill:ms.avg": 0,
        "Destroy()/recursive-kill:ms.cnt": 10,
        "Destroy()/remove-root:ms.avg": 0,
        "Destroy()/remove-root:ms.cnt": 10,
        "Destroy():ms.avg": 2,
        "Destroy():ms.cnt": 10,
        "Pause():ms.avg": 4,
        "Pause():ms.cnt": 4,
        "Status():ms.avg": 0,
        "Status():ms.cnt": 6,
        "pip-install:ms.avg": 1545,
        "pip-install:ms.cnt": 6
      },
      "errors": [],
      "worker_tail": [
        "2019/07/15 15:34:32 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:32 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:32 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:32 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:32 0 of 500 MB available [MEM POOL handler-sandboxes]",
        "2019/07/15 15:34:32 WARNING!  Critically low on memory, so evicting an active Sandbox",
        "2019/07/15 15:34:32 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/07/15 15:34:32 remove worker.pid",
        "2019/07/15 15:34:32 exiting",
        ""
      ]
    },
    {
      "test": "install_tests",
      "params": {},
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "limits": {
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 200
        },
        "registry": "/root/open-lambda/test-registry",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_mb": 250,
        "import_cache_tree": "",
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "server_mode": "lambda",
        "handler_cache_mb": 250,
        "pip_mirror": ""
      },
      "seconds": 6.24748682975769,
      "total_seconds": 6.465757369995117,
      "stats": null,
      "ol-stats": {
        "Channel():ms.avg": 0,
        "Channel():ms.cnt": 10,
        "Create()/acquire-cgroup:ms.avg": 0,
        "Create()/acquire-cgroup:ms.cnt": 11,
        "Create()/acquire-mem:ms.avg": 0,
        "Create()/acquire-mem:ms.cnt": 11,
        "Create()/fork-proc:ms.avg": 5,
        "Create()/fork-proc:ms.cnt": 4,
        "Create()/fresh-proc:ms.avg": 141,
        "Create()/fresh-proc:ms.cnt": 7,
        "Create()/make-root-fs:ms.avg": 1,
        "Create()/make-root-fs:ms.cnt": 11,
        "Create():ms.avg": 94,
        "Create():ms.cnt": 11,
        "Destroy()/detach-root:ms.avg": 0,
        "Destroy()/detach-root:ms.cnt": 11,
        "Destroy()/kill-procs:ms.avg": 1,
        "Destroy()/kill-procs:ms.cnt": 11,
        "Destroy()/recursive-kill:ms.avg": 0,
        "Destroy()/recursive-kill:ms.cnt": 11,
        "Destroy()/remove-root:ms.avg": 0,
        "Destroy()/remove-root:ms.cnt": 11,
        "Destroy():ms.avg": 1,
        "Destroy():ms.cnt": 18,
        "Pause():ms.avg": 4,
        "Pause():ms.cnt": 4,
        "Status():ms.avg": 0,
        "Status():ms.cnt": 6,
        "evict:ms.avg": 1,
        "evict:ms.cnt": 3,
        "fork():ms.avg": 5,
        "fork():ms.cnt": 4,
        "forkRequest:ms.avg": 5,
        "forkRequest:ms.cnt": 4,
        "move-to-cg-after-fork:ms.avg": 0,
        "move-to-cg-after-fork:ms.cnt": 4,
        "pip-install:ms.avg": 961,
        "pip-install:ms.cnt": 6
      },
      "errors": [],
      "worker_tail": [
        "2019/07/15 15:34:38 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:38 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:38 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:38 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:38 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:38 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:38 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/07/15 15:34:38 remove worker.pid",
        "2019/07/15 15:34:38 exiting",
        ""
      ]
    },
    {
      "test": "install_tests",
      "params": {},
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "sandbox_config": {},
        "sandbox": "docker",
        "limits": {
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 200
        },
        "registry": "/root/open-lambda/test-registry",
        "import_cache_mb": 0,
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "registry_cache_ms": 5000,
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "import_cache_tree": "",
        "handler_cache_mb": 500,
        "docker_runtime": "",
        "pip_mirror": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "server_mode": "lambda"
      },
      "seconds": 17.764436960220337,
      "total_seconds": 18.788320541381836,
      "stats": null,
      "ol-stats": {
        "Channel():ms.avg": 0,
        "Channel():ms.cnt": 10,
        "Create():ms.avg": 820,
        "Create():ms.cnt": 10,
        "Destroy():ms.avg": 245,
        "Destroy():ms.cnt": 10,
        "Pause():ms.avg": 26,
        "Pause():ms.cnt": 4,
        "Status():ms.avg": 0,
        "Status():ms.cnt": 6,
        "pip-install:ms.avg": 2090,
        "pip-install:ms.cnt": 6
      },
      "errors": [],
      "worker_tail": [
        "2019/07/15 15:34:56 Kill function: echo",
        "2019/07/15 15:34:56 Destroy() [SB 0]",
        "2019/07/15 15:34:57 Kill function: install",
        "2019/07/15 15:34:57 Destroy() [SB 6]",
        "2019/07/15 15:34:57 Kill function: install2",
        "2019/07/15 15:34:57 Destroy() [SB 7]",
        "2019/07/15 15:34:57 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/07/15 15:34:57 remove worker.pid",
        "2019/07/15 15:34:57 exiting",
        ""
      ]
    },
    {
      "test": "fork_bomb",
      "params": {},
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "limits": {
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 200
        },
        "registry": "/root/open-lambda/test-registry",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_mb": 250,
        "import_cache_tree": "",
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "server_mode": "lambda",
        "handler_cache_mb": 250,
        "pip_mirror": ""
      },
      "seconds": 0.1731700897216797,
      "total_seconds": 0.4024381637573242,
      "stats": null,
      "ol-stats": {
        "Channel():ms.avg": 0,
        "Channel():ms.cnt": 1,
        "Create()/acquire-cgroup:ms.avg": 0,
        "Create()/acquire-cgroup:ms.cnt": 2,
        "Create()/acquire-mem:ms.avg": 0,
        "Create()/acquire-mem:ms.cnt": 2,
        "Create()/fork-proc:ms.avg": 8,
        "Create()/fork-proc:ms.cnt": 1,
        "Create()/fresh-proc:ms.avg": 137,
        "Create()/fresh-proc:ms.cnt": 1,
        "Create()/make-root-fs:ms.avg": 2,
        "Create()/make-root-fs:ms.cnt": 2,
        "Create():ms.avg": 75,
        "Create():ms.cnt": 2,
        "Destroy()/detach-root:ms.avg": 1,
        "Destroy()/detach-root:ms.cnt": 2,
        "Destroy()/kill-procs:ms.avg": 0,
        "Destroy()/kill-procs:ms.cnt": 2,
        "Destroy()/recursive-kill:ms.avg": 1,
        "Destroy()/recursive-kill:ms.cnt": 2,
        "Destroy()/remove-root:ms.avg": 0,
        "Destroy()/remove-root:ms.cnt": 2,
        "Destroy():ms.avg": 2,
        "Destroy():ms.cnt": 3,
        "Pause():ms.avg": 3,
        "Pause():ms.cnt": 1,
        "fork():ms.avg": 8,
        "fork():ms.cnt": 1,
        "forkRequest:ms.avg": 4,
        "forkRequest:ms.cnt": 1,
        "move-to-cg-after-fork:ms.avg": 3,
        "move-to-cg-after-fork:ms.cnt": 1
      },
      "errors": [],
      "worker_tail": [
        "2019/07/15 15:34:57 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:57 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:57 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:57 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:57 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:57 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:57 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/07/15 15:34:57 remove worker.pid",
        "2019/07/15 15:34:57 exiting",
        ""
      ]
    },
    {
      "test": "max_mem_alloc",
      "params": {},
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "limits": {
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 200
        },
        "registry": "/root/open-lambda/test-registry",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_mb": 250,
        "import_cache_tree": "",
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "server_mode": "lambda",
        "handler_cache_mb": 250,
        "pip_mirror": ""
      },
      "seconds": 0.8610250949859619,
      "total_seconds": 1.0909488201141357,
      "stats": null,
      "ol-stats": {
        "Channel():ms.avg": 0,
        "Channel():ms.cnt": 1,
        "Create()/acquire-cgroup:ms.avg": 0,
        "Create()/acquire-cgroup:ms.cnt": 2,
        "Create()/acquire-mem:ms.avg": 0,
        "Create()/acquire-mem:ms.cnt": 2,
        "Create()/fork-proc:ms.avg": 6,
        "Create()/fork-proc:ms.cnt": 1,
        "Create()/fresh-proc:ms.avg": 138,
        "Create()/fresh-proc:ms.cnt": 1,
        "Create()/make-root-fs:ms.avg": 4,
        "Create()/make-root-fs:ms.cnt": 2,
        "Create():ms.avg": 77,
        "Create():ms.cnt": 2,
        "Destroy()/detach-root:ms.avg": 1,
        "Destroy()/detach-root:ms.cnt": 2,
        "Destroy()/kill-procs:ms.avg": 0,
        "Destroy()/kill-procs:ms.cnt": 2,
        "Destroy()/recursive-kill:ms.avg": 1,
        "Destroy()/recursive-kill:ms.cnt": 2,
        "Destroy()/remove-root:ms.avg": 0,
        "Destroy()/remove-root:ms.cnt": 2,
        "Destroy():ms.avg": 2,
        "Destroy():ms.cnt": 3,
        "Pause():ms.avg": 3,
        "Pause():ms.cnt": 1,
        "fork():ms.avg": 6,
        "fork():ms.cnt": 1,
        "forkRequest:ms.avg": 6,
        "forkRequest:ms.cnt": 1,
        "move-to-cg-after-fork:ms.avg": 0,
        "move-to-cg-after-fork:ms.cnt": 1
      },
      "errors": [],
      "worker_tail": [
        "2019/07/15 15:34:59 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:59 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:59 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:59 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:59 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:59 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:34:59 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/07/15 15:34:59 remove worker.pid",
        "2019/07/15 15:34:59 exiting",
        ""
      ]
    },
    {
      "test": "numpy_test",
      "params": {},
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "limits": {
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 200
        },
        "registry": "/root/open-lambda/test-registry",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_mb": 250,
        "import_cache_tree": "",
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "server_mode": "lambda",
        "handler_cache_mb": 250,
        "pip_mirror": ""
      },
      "seconds": 24.01884937286377,
      "total_seconds": 24.274686813354492,
      "stats": null,
      "ol-stats": {
        "Channel():ms.avg": 0,
        "Channel():ms.cnt": 12,
        "Create()/acquire-cgroup:ms.avg": 0,
        "Create()/acquire-cgroup:ms.cnt": 13,
        "Create()/acquire-mem:ms.avg": 0,
        "Create()/acquire-mem:ms.cnt": 13,
        "Create()/fork-proc:ms.avg": 5,
        "Create()/fork-proc:ms.cnt": 5,
        "Create()/fresh-proc:ms.avg": 148,
        "Create()/fresh-proc:ms.cnt": 8,
        "Create()/make-root-fs:ms.avg": 31,
        "Create()/make-root-fs:ms.cnt": 13,
        "Create():ms.avg": 125,
        "Create():ms.cnt": 13,
        "Destroy()/detach-root:ms.avg": 0,
        "Destroy()/detach-root:ms.cnt": 13,
        "Destroy()/kill-procs:ms.avg": 2,
        "Destroy()/kill-procs:ms.cnt": 13,
        "Destroy()/recursive-kill:ms.avg": 0,
        "Destroy()/recursive-kill:ms.cnt": 13,
        "Destroy()/remove-root:ms.avg": 0,
        "Destroy()/remove-root:ms.cnt": 13,
        "Destroy():ms.avg": 1,
        "Destroy():ms.cnt": 22,
        "Pause():ms.avg": 3,
        "Pause():ms.cnt": 5,
        "Status():ms.avg": 0,
        "Status():ms.cnt": 7,
        "evict:ms.avg": 2,
        "evict:ms.cnt": 4,
        "fork():ms.avg": 5,
        "fork():ms.cnt": 5,
        "forkRequest:ms.avg": 5,
        "forkRequest:ms.cnt": 5,
        "move-to-cg-after-fork:ms.avg": 0,
        "move-to-cg-after-fork:ms.cnt": 5,
        "pip-install:ms.avg": 3121,
        "pip-install:ms.cnt": 7
      },
      "errors": [],
      "worker_tail": [
        "2019/07/15 15:35:23 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:35:23 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:35:23 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:35:23 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:35:23 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:35:23 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:35:23 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/07/15 15:35:23 remove worker.pid",
        "2019/07/15 15:35:23 exiting",
        ""
      ]
    },
    {
      "test": "sock_churn",
      "params": {
        "procs": 1,
        "fork": false,
        "seconds": 5,
        "baseline": 0
      },
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "limits": {
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 200
        },
        "registry": "/root/open-lambda/test-dir/registry",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_mb": 250,
        "import_cache_tree": "",
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "server_mode": "sock",
        "handler_cache_mb": 250,
        "pip_mirror": ""
      },
      "seconds": 5.073418617248535,
      "total_seconds": 5.295649766921997,
      "stats": {
        "sandboxes_per_sec": 6.2
      },
      "ol-stats": {
        "Create()/acquire-cgroup:ms.avg": 0,
        "Create()/acquire-cgroup:ms.cnt": 31,
        "Create()/acquire-mem:ms.avg": 0,
        "Create()/acquire-mem:ms.cnt": 31,
        "Create()/fresh-proc:ms.avg": 137,
        "Create()/fresh-proc:ms.cnt": 31,
        "Create()/make-root-fs:ms.avg": 3,
        "Create()/make-root-fs:ms.cnt": 31,
        "Create():ms.avg": 141,
        "Create():ms.cnt": 31,
        "Destroy()/detach-root:ms.avg": 1,
        "Destroy()/detach-root:ms.cnt": 31,
        "Destroy()/kill-procs:ms.avg": 1,
        "Destroy()/kill-procs:ms.cnt": 31,
        "Destroy()/recursive-kill:ms.avg": 0,
        "Destroy()/recursive-kill:ms.cnt": 31,
        "Destroy()/remove-root:ms.avg": 0,
        "Destroy()/remove-root:ms.cnt": 31,
        "Destroy():ms.avg": 1,
        "Destroy():ms.cnt": 62
      },
      "errors": [],
      "worker_tail": [
        "2019/07/15 15:35:28 remove /sys/fs/cgroup/freezer/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:35:28 remove /sys/fs/cgroup/hugetlb/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:35:28 remove /sys/fs/cgroup/memory/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:35:28 remove /sys/fs/cgroup/perf_event/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:35:28 remove /sys/fs/cgroup/systemd/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:35:28 remove /sys/fs/cgroup/pids/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:35:28 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/07/15 15:35:28 remove worker.pid",
        "2019/07/15 15:35:28 exiting",
        ""
      ]
    },
    {
      "test": "sock_churn",
      "params": {
        "procs": 1,
        "fork": true,
        "seconds": 15,
        "baseline": 0
      },
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "limits": {
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 200
        },
        "registry": "/root/open-lambda/test-dir/registry",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_mb": 250,
        "import_cache_tree": "",
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "server_mode": "sock",
        "handler_cache_mb": 250,
        "pip_mirror": ""
      },
      "seconds": 15.171350479125977,
      "total_seconds": 15.393688201904297,
      "stats": {
        "sandboxes_per_sec": 47.06666666666667
      },
      "ol-stats": {
        "Create()/acquire-cgroup:ms.avg": 0,
        "Create()/acquire-cgroup:ms.cnt": 707,
        "Create()/acquire-mem:ms.avg": 0,
        "Create()/acquire-mem:ms.cnt": 707,
        "Create()/fork-proc:ms.avg": 6,
        "Create()/fork-proc:ms.cnt": 706,
        "Create()/fresh-proc:ms.avg": 137,
        "Create()/fresh-proc:ms.cnt": 1,
        "Create()/make-root-fs:ms.avg": 2,
        "Create()/make-root-fs:ms.cnt": 707,
        "Create():ms.avg": 10,
        "Create():ms.cnt": 707,
        "Destroy()/detach-root:ms.avg": 1,
        "Destroy()/detach-root:ms.cnt": 707,
        "Destroy()/kill-procs:ms.avg": 1,
        "Destroy()/kill-procs:ms.cnt": 707,
        "Destroy()/recursive-kill:ms.avg": 0,
        "Destroy()/recursive-kill:ms.cnt": 707,
        "Destroy()/remove-root:ms.avg": 0,
        "Destroy()/remove-root:ms.cnt": 707,
        "Destroy():ms.avg": 0,
        "Destroy():ms.cnt": 2119,
        "fork():ms.avg": 6,
        "fork():ms.cnt": 706,
        "forkRequest:ms.avg": 5,
        "forkRequest:ms.cnt": 706,
        "move-to-cg-after-fork:ms.avg": 0,
        "move-to-cg-after-fork:ms.cnt": 706
      },
      "errors": [],
      "worker_tail": [
        "2019/07/15 15:35:44 remove /sys/fs/cgroup/freezer/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:35:44 remove /sys/fs/cgroup/hugetlb/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:35:44 remove /sys/fs/cgroup/memory/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:35:44 remove /sys/fs/cgroup/perf_event/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:35:44 remove /sys/fs/cgroup/systemd/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:35:44 remove /sys/fs/cgroup/pids/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:35:44 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/07/15 15:35:44 remove worker.pid",
        "2019/07/15 15:35:44 exiting",
        ""
      ]
    },
    {
      "test": "sock_churn",
      "params": {
        "procs": 15,
        "fork": true,
        "seconds": 15,
        "baseline": 0
      },
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "limits": {
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 200
        },
        "registry": "/root/open-lambda/test-dir/registry",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_mb": 250,
        "import_cache_tree": "",
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "server_mode": "sock",
        "handler_cache_mb": 250,
        "pip_mirror": ""
      },
      "seconds": 15.422855138778687,
      "total_seconds": 15.648693084716797,
      "stats": {
        "sandboxes_per_sec": 47.53333333333333
      },
      "ol-stats": {
        "Create()/acquire-cgroup:ms.avg": 0,
        "Create()/acquire-cgroup:ms.cnt": 714,
        "Create()/acquire-mem:ms.avg": 81,
        "Create()/acquire-mem:ms.cnt": 714,
        "Create()/fork-proc:ms.avg": 55,
        "Create()/fork-proc:ms.cnt": 713,
        "Create()/fresh-proc:ms.avg": 137,
        "Create()/fresh-proc:ms.cnt": 1,
        "Create()/make-root-fs:ms.avg": 1,
        "Create()/make-root-fs:ms.cnt": 714,
        "Create():ms.avg": 140,
        "Create():ms.cnt": 714,
        "Destroy()/detach-root:ms.avg": 2,
        "Destroy()/detach-root:ms.cnt": 714,
        "Destroy()/kill-procs:ms.avg": 9,
        "Destroy()/kill-procs:ms.cnt": 714,
        "Destroy()/recursive-kill:ms.avg": 0,
        "Destroy()/recursive-kill:ms.cnt": 714,
        "Destroy()/remove-root:ms.avg": 0,
        "Destroy()/remove-root:ms.cnt": 714,
        "Destroy():ms.avg": 4,
        "Destroy():ms.cnt": 2140,
        "fork():ms.avg": 55,
        "fork():ms.cnt": 713,
        "forkRequest:ms.avg": 10,
        "forkRequest:ms.cnt": 713,
        "move-to-cg-after-fork:ms.avg": 3,
        "move-to-cg-after-fork:ms.cnt": 713
      },
      "errors": [],
      "worker_tail": [
        "2019/07/15 15:35:59 remove /sys/fs/cgroup/freezer/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:35:59 remove /sys/fs/cgroup/hugetlb/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:35:59 remove /sys/fs/cgroup/memory/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:35:59 remove /sys/fs/cgroup/perf_event/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:35:59 remove /sys/fs/cgroup/systemd/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:35:59 remove /sys/fs/cgroup/pids/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:35:59 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/07/15 15:35:59 remove worker.pid",
        "2019/07/15 15:35:59 exiting",
        ""
      ]
    },
    {
      "test": "recursive_kill",
      "params": {
        "depth": 2
      },
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "limits": {
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 200
        },
        "registry": "/root/open-lambda/test-dir/registry",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_mb": 250,
        "import_cache_tree": "",
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "server_mode": "sock",
        "handler_cache_mb": 250,
        "pip_mirror": ""
      },
      "seconds": 0.1839599609375,
      "total_seconds": 0.5067172050476074,
      "stats": null,
      "ol-stats": {
        "Create()/acquire-cgroup:ms.avg": 0,
        "Create()/acquire-cgroup:ms.cnt": 2,
        "Create()/acquire-mem:ms.avg": 0,
        "Create()/acquire-mem:ms.cnt": 2,
        "Create()/fork-proc:ms.avg": 4,
        "Create()/fork-proc:ms.cnt": 1,
        "Create()/fresh-proc:ms.avg": 142,
        "Create()/fresh-proc:ms.cnt": 1,
        "Create()/make-root-fs:ms.avg": 4,
        "Create()/make-root-fs:ms.cnt": 2,
        "Create():ms.avg": 78,
        "Create():ms.cnt": 2,
        "Destroy()/detach-root:ms.avg": 1,
        "Destroy()/detach-root:ms.cnt": 2,
        "Destroy()/kill-procs:ms.avg": 0,
        "Destroy()/kill-procs:ms.cnt": 2,
        "Destroy()/recursive-kill:ms.avg": 1,
        "Destroy()/recursive-kill:ms.cnt": 2,
        "Destroy()/remove-root:ms.avg": 0,
        "Destroy()/remove-root:ms.cnt": 2,
        "Destroy():ms.avg": 1,
        "Destroy():ms.cnt": 4,
        "Pause():ms.avg": 0,
        "Pause():ms.cnt": 1,
        "fork():ms.avg": 4,
        "fork():ms.cnt": 1,
        "forkRequest:ms.avg": 3,
        "forkRequest:ms.cnt": 1,
        "move-to-cg-after-fork:ms.avg": 1,
        "move-to-cg-after-fork:ms.cnt": 1
      },
      "errors": [],
      "worker_tail": [
        "2019/07/15 15:36:00 remove /sys/fs/cgroup/freezer/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:36:00 remove /sys/fs/cgroup/hugetlb/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:36:00 remove /sys/fs/cgroup/memory/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:36:00 remove /sys/fs/cgroup/perf_event/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:36:00 remove /sys/fs/cgroup/systemd/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:36:00 remove /sys/fs/cgroup/pids/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:36:00 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/07/15 15:36:00 remove worker.pid",
        "2019/07/15 15:36:00 exiting",
        ""
      ]
    },
    {
      "test": "recursive_kill",
      "params": {
        "depth": 10
      },
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "limits": {
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 200
        },
        "registry": "/root/open-lambda/test-dir/registry",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_mb": 250,
        "import_cache_tree": "",
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "server_mode": "sock",
        "handler_cache_mb": 250,
        "pip_mirror": ""
      },
      "seconds": 0.32729387283325195,
      "total_seconds": 0.557201623916626,
      "stats": null,
      "ol-stats": {
        "Create()/acquire-cgroup:ms.avg": 0,
        "Create()/acquire-cgroup:ms.cnt": 10,
        "Create()/acquire-mem:ms.avg": 0,
        "Create()/acquire-mem:ms.cnt": 10,
        "Create()/fork-proc:ms.avg": 4,
        "Create()/fork-proc:ms.cnt": 9,
        "Create()/fresh-proc:ms.avg": 135,
        "Create()/fresh-proc:ms.cnt": 1,
        "Create()/make-root-fs:ms.avg": 2,
        "Create()/make-root-fs:ms.cnt": 10,
        "Create():ms.avg": 21,
        "Create():ms.cnt": 10,
        "Destroy()/detach-root:ms.avg": 0,
        "Destroy()/detach-root:ms.cnt": 10,
        "Destroy()/kill-procs:ms.avg": 1,
        "Destroy()/kill-procs:ms.cnt": 10,
        "Destroy()/recursive-kill:ms.avg": 6,
        "Destroy()/recursive-kill:ms.cnt": 10,
        "Destroy()/remove-root:ms.avg": 0,
        "Destroy()/remove-root:ms.cnt": 10,
        "Destroy():ms.avg": 4,
        "Destroy():ms.cnt": 20,
        "Pause():ms.avg": 0,
        "Pause():ms.cnt": 9,
        "fork():ms.avg": 4,
        "fork():ms.cnt": 9,
        "forkRequest:ms.avg": 4,
        "forkRequest:ms.cnt": 9,
        "move-to-cg-after-fork:ms.avg": 0,
        "move-to-cg-after-fork:ms.cnt": 9
      },
      "errors": [],
      "worker_tail": [
        "2019/07/15 15:36:00 remove /sys/fs/cgroup/freezer/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:36:00 remove /sys/fs/cgroup/hugetlb/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:36:00 remove /sys/fs/cgroup/memory/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:36:00 remove /sys/fs/cgroup/perf_event/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:36:00 remove /sys/fs/cgroup/systemd/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:36:00 remove /sys/fs/cgroup/pids/worker-sock-handlers [CGROUP POOL worker-sock-handlers]",
        "2019/07/15 15:36:00 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/07/15 15:36:00 remove worker.pid",
        "2019/07/15 15:36:00 exiting",
        ""
      ]
    },
    {
      "test": "update_code",
      "params": {},
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "sandbox_config": {},
        "sandbox": "sock",
        "limits": {
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 200
        },
        "registry": "/tmp/tmppjm03ng7",
        "import_cache_mb": 250,
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "registry_cache_ms": 3000,
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "import_cache_tree": "",
        "handler_cache_mb": 250,
        "docker_runtime": "",
        "pip_mirror": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "server_mode": "lambda"
      },
      "seconds": 6.029643774032593,
      "total_seconds": 6.264220237731934,
      "stats": null,
      "ol-stats": {
        "Channel():ms.avg": 0,
        "Channel():ms.cnt": 3,
        "Create()/acquire-cgroup:ms.avg": 0,
        "Create()/acquire-cgroup:ms.cnt": 4,
        "Create()/acquire-mem:ms.avg": 0,
        "Create()/acquire-mem:ms.cnt": 4,
        "Create()/fork-proc:ms.avg": 8,
        "Create()/fork-proc:ms.cnt": 3,
        "Create()/fresh-proc:ms.avg": 135,
        "Create()/fresh-proc:ms.cnt": 1,
        "Create()/make-root-fs:ms.avg": 3,
        "Create()/make-root-fs:ms.cnt": 4,
        "Create():ms.avg": 44,
        "Create():ms.cnt": 4,
        "Destroy()/detach-root:ms.avg": 0,
        "Destroy()/detach-root:ms.cnt": 4,
        "Destroy()/kill-procs:ms.avg": 0,
        "Destroy()/kill-procs:ms.cnt": 4,
        "Destroy()/recursive-kill:ms.avg": 0,
        "Destroy()/recursive-kill:ms.cnt": 4,
        "Destroy()/remove-root:ms.avg": 0,
        "Destroy()/remove-root:ms.cnt": 4,
        "Destroy():ms.avg": 1,
        "Destroy():ms.cnt": 7,
        "Pause():ms.avg": 0,
        "Pause():ms.cnt": 1683,
        "Unpause():ms.avg": 0,
        "Unpause():ms.cnt": 1680,
        "fork():ms.avg": 8,
        "fork():ms.cnt": 3,
        "forkRequest:ms.avg": 6,
        "forkRequest:ms.cnt": 3,
        "move-to-cg-after-fork:ms.avg": 1,
        "move-to-cg-after-fork:ms.cnt": 3
      },
      "errors": [],
      "worker_tail": [
        "2019/07/15 15:36:07 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:07 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:07 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:07 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:07 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:07 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:07 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/07/15 15:36:07 remove worker.pid",
        "2019/07/15 15:36:07 exiting",
        ""
      ]
    },
    {
      "test": "stress_one_lambda",
      "params": {
        "procs": 1,
        "seconds": 15
      },
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "limits": {
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 200
        },
        "registry": "/root/open-lambda/test-registry",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_mb": 250,
        "import_cache_tree": "",
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "server_mode": "lambda",
        "handler_cache_mb": 250,
        "pip_mirror": ""
      },
      "seconds": 15.044552087783813,
      "total_seconds": 15.276179552078247,
      "stats": {
        "reqs_per_sec": 283.0
      },
      "ol-stats": {
        "Channel():ms.avg": 0,
        "Channel():ms.cnt": 3,
        "Create()/acquire-cgroup:ms.avg": 0,
        "Create()/acquire-cgroup:ms.cnt": 4,
        "Create()/acquire-mem:ms.avg": 0,
        "Create()/acquire-mem:ms.cnt": 4,
        "Create()/fork-proc:ms.avg": 7,
        "Create()/fork-proc:ms.cnt": 3,
        "Create()/fresh-proc:ms.avg": 137,
        "Create()/fresh-proc:ms.cnt": 1,
        "Create()/make-root-fs:ms.avg": 3,
        "Create()/make-root-fs:ms.cnt": 4,
        "Create():ms.avg": 43,
        "Create():ms.cnt": 4,
        "Destroy()/detach-root:ms.avg": 0,
        "Destroy()/detach-root:ms.cnt": 4,
        "Destroy()/kill-procs:ms.avg": 0,
        "Destroy()/kill-procs:ms.cnt": 4,
        "Destroy()/recursive-kill:ms.avg": 0,
        "Destroy()/recursive-kill:ms.cnt": 4,
        "Destroy()/remove-root:ms.avg": 0,
        "Destroy()/remove-root:ms.cnt": 4,
        "Destroy():ms.avg": 1,
        "Destroy():ms.cnt": 7,
        "Pause():ms.avg": 0,
        "Pause():ms.cnt": 4245,
        "Unpause():ms.avg": 0,
        "Unpause():ms.cnt": 4242,
        "fork():ms.avg": 7,
        "fork():ms.cnt": 3,
        "forkRequest:ms.avg": 6,
        "forkRequest:ms.cnt": 3,
        "move-to-cg-after-fork:ms.avg": 0,
        "move-to-cg-after-fork:ms.cnt": 3
      },
      "errors": [],
      "worker_tail": [
        "2019/07/15 15:36:22 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:22 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:22 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:22 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:22 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:22 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:22 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/07/15 15:36:22 remove worker.pid",
        "2019/07/15 15:36:22 exiting",
        ""
      ]
    },
    {
      "test": "stress_one_lambda",
      "params": {
        "procs": 2,
        "seconds": 15
      },
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "limits": {
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 200
        },
        "registry": "/root/open-lambda/test-registry",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_mb": 250,
        "import_cache_tree": "",
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "server_mode": "lambda",
        "handler_cache_mb": 250,
        "pip_mirror": ""
      },
      "seconds": 15.042874097824097,
      "total_seconds": 15.27322268486023,
      "stats": {
        "reqs_per_sec": 281.53333333333336
      },
      "ol-stats": {
        "Channel():ms.avg": 0,
        "Channel():ms.cnt": 3,
        "Create()/acquire-cgroup:ms.avg": 0,
        "Create()/acquire-cgroup:ms.cnt": 4,
        "Create()/acquire-mem:ms.avg": 0,
        "Create()/acquire-mem:ms.cnt": 4,
        "Create()/fork-proc:ms.avg": 7,
        "Create()/fork-proc:ms.cnt": 3,
        "Create()/fresh-proc:ms.avg": 137,
        "Create()/fresh-proc:ms.cnt": 1,
        "Create()/make-root-fs:ms.avg": 3,
        "Create()/make-root-fs:ms.cnt": 4,
        "Create():ms.avg": 44,
        "Create():ms.cnt": 4,
        "Destroy()/detach-root:ms.avg": 0,
        "Destroy()/detach-root:ms.cnt": 4,
        "Destroy()/kill-procs:ms.avg": 0,
        "Destroy()/kill-procs:ms.cnt": 4,
        "Destroy()/recursive-kill:ms.avg": 0,
        "Destroy()/recursive-kill:ms.cnt": 4,
        "Destroy()/remove-root:ms.avg": 0,
        "Destroy()/remove-root:ms.cnt": 4,
        "Destroy():ms.avg": 1,
        "Destroy():ms.cnt": 7,
        "Pause():ms.avg": 0,
        "Pause():ms.cnt": 3950,
        "Unpause():ms.avg": 0,
        "Unpause():ms.cnt": 3947,
        "fork():ms.avg": 7,
        "fork():ms.cnt": 3,
        "forkRequest:ms.avg": 6,
        "forkRequest:ms.cnt": 3,
        "move-to-cg-after-fork:ms.avg": 0,
        "move-to-cg-after-fork:ms.cnt": 3
      },
      "errors": [],
      "worker_tail": [
        "2019/07/15 15:36:37 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:37 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:37 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:37 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:37 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:37 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:37 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/07/15 15:36:37 remove worker.pid",
        "2019/07/15 15:36:37 exiting",
        ""
      ]
    },
    {
      "test": "stress_one_lambda",
      "params": {
        "procs": 8,
        "seconds": 15
      },
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "limits": {
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 200
        },
        "registry": "/root/open-lambda/test-registry",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_mb": 250,
        "import_cache_tree": "",
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "server_mode": "lambda",
        "handler_cache_mb": 250,
        "pip_mirror": ""
      },
      "seconds": 15.063709735870361,
      "total_seconds": 15.296294212341309,
      "stats": {
        "reqs_per_sec": 258.6
      },
      "ol-stats": {
        "Channel():ms.avg": 0,
        "Channel():ms.cnt": 3,
        "Create()/acquire-cgroup:ms.avg": 0,
        "Create()/acquire-cgroup:ms.cnt": 4,
        "Create()/acquire-mem:ms.avg": 0,
        "Create()/acquire-mem:ms.cnt": 4,
        "Create()/fork-proc:ms.avg": 6,
        "Create()/fork-proc:ms.cnt": 3,
        "Create()/fresh-proc:ms.avg": 138,
        "Create()/fresh-proc:ms.cnt": 1,
        "Create()/make-root-fs:ms.avg": 3,
        "Create()/make-root-fs:ms.cnt": 4,
        "Create():ms.avg": 44,
        "Create():ms.cnt": 4,
        "Destroy()/detach-root:ms.avg": 0,
        "Destroy()/detach-root:ms.cnt": 4,
        "Destroy()/kill-procs:ms.avg": 0,
        "Destroy()/kill-procs:ms.cnt": 4,
        "Destroy()/recursive-kill:ms.avg": 0,
        "Destroy()/recursive-kill:ms.cnt": 4,
        "Destroy()/remove-root:ms.avg": 0,
        "Destroy()/remove-root:ms.cnt": 4,
        "Destroy():ms.avg": 1,
        "Destroy():ms.cnt": 7,
        "Pause():ms.avg": 0,
        "Pause():ms.cnt": 2885,
        "Unpause():ms.avg": 0,
        "Unpause():ms.cnt": 2882,
        "fork():ms.avg": 6,
        "fork():ms.cnt": 3,
        "forkRequest:ms.avg": 5,
        "forkRequest:ms.cnt": 3,
        "move-to-cg-after-fork:ms.avg": 0,
        "move-to-cg-after-fork:ms.cnt": 3
      },
      "errors": [],
      "worker_tail": [
        "2019/07/15 15:36:52 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:52 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:52 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:52 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:52 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:52 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:52 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/07/15 15:36:52 remove worker.pid",
        "2019/07/15 15:36:52 exiting",
        ""
      ]
    },
    {
      "test": "call_each_once_exec",
      "params": {
        "lambda_count": 100,
        "alloc_mb": 1
      },
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "sandbox_config": {},
        "sandbox": "sock",
        "limits": {
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 200
        },
        "registry": "/tmp/tmp8m8hsdlh",
        "import_cache_mb": 250,
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "registry_cache_ms": 5000,
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "import_cache_tree": "",
        "handler_cache_mb": 250,
        "docker_runtime": "",
        "pip_mirror": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "server_mode": "lambda"
      },
      "seconds": 2.540613889694214,
      "total_seconds": 2.884375810623169,
      "stats": {
        "reqs_per_sec": 39.360949879612456
      },
      "ol-stats": {
        "Channel():ms.avg": 0,
        "Channel():ms.cnt": 100,
        "Create()/acquire-cgroup:ms.avg": 0,
        "Create()/acquire-cgroup:ms.cnt": 101,
        "Create()/acquire-mem:ms.avg": 0,
        "Create()/acquire-mem:ms.cnt": 101,
        "Create()/fork-proc:ms.avg": 7,
        "Create()/fork-proc:ms.cnt": 100,
        "Create()/fresh-proc:ms.avg": 136,
        "Create()/fresh-proc:ms.cnt": 1,
        "Create()/make-root-fs:ms.avg": 0,
        "Create()/make-root-fs:ms.cnt": 101,
        "Create():ms.avg": 9,
        "Create():ms.cnt": 101,
        "Destroy()/detach-root:ms.avg": 0,
        "Destroy()/detach-root:ms.cnt": 101,
        "Destroy()/kill-procs:ms.avg": 0,
        "Destroy()/kill-procs:ms.cnt": 101,
        "Destroy()/recursive-kill:ms.avg": 0,
        "Destroy()/recursive-kill:ms.cnt": 101,
        "Destroy()/remove-root:ms.avg": 0,
        "Destroy()/remove-root:ms.cnt": 101,
        "Destroy():ms.avg": 0,
        "Destroy():ms.cnt": 276,
        "Pause():ms.avg": 5,
        "Pause():ms.cnt": 100,
        "evict:ms.avg": 0,
        "evict:ms.cnt": 75,
        "fork():ms.avg": 7,
        "fork():ms.cnt": 100,
        "forkRequest:ms.avg": 6,
        "forkRequest:ms.cnt": 100,
        "move-to-cg-after-fork:ms.avg": 0,
        "move-to-cg-after-fork:ms.cnt": 100
      },
      "errors": [],
      "worker_tail": [
        "2019/07/15 15:36:55 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:55 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:55 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:55 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:55 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:55 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:36:55 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/07/15 15:36:55 remove worker.pid",
        "2019/07/15 15:36:55 exiting",
        ""
      ]
    },
    {
      "test": "call_each_once_exec",
      "params": {
        "lambda_count": 1000,
        "alloc_mb": 10
      },
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "limits": {
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 200
        },
        "registry": "/tmp/tmp5k0ez2im",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_mb": 250,
        "import_cache_tree": "",
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "server_mode": "lambda",
        "handler_cache_mb": 250,
        "pip_mirror": ""
      },
      "seconds": 37.252402782440186,
      "total_seconds": 37.49738049507141,
      "stats": {
        "reqs_per_sec": 26.84392356244376
      },
      "ol-stats": {
        "Channel():ms.avg": 0,
        "Channel():ms.cnt": 1000,
        "Create()/acquire-cgroup:ms.avg": 0,
        "Create()/acquire-cgroup:ms.cnt": 1001,
        "Create()/acquire-mem:ms.avg": 0,
        "Create()/acquire-mem:ms.cnt": 1001,
        "Create()/fork-proc:ms.avg": 8,
        "Create()/fork-proc:ms.cnt": 1000,
        "Create()/fresh-proc:ms.avg": 139,
        "Create()/fresh-proc:ms.cnt": 1,
        "Create()/make-root-fs:ms.avg": 1,
        "Create()/make-root-fs:ms.cnt": 1001,
        "Create():ms.avg": 9,
        "Create():ms.cnt": 1001,
        "Destroy()/detach-root:ms.avg": 0,
        "Destroy()/detach-root:ms.cnt": 1001,
        "Destroy()/kill-procs:ms.avg": 2,
        "Destroy()/kill-procs:ms.cnt": 1001,
        "Destroy()/recursive-kill:ms.avg": 0,
        "Destroy()/recursive-kill:ms.cnt": 1001,
        "Destroy()/remove-root:ms.avg": 0,
        "Destroy()/remove-root:ms.cnt": 1001,
        "Destroy():ms.avg": 0,
        "Destroy():ms.cnt": 2993,
        "Pause():ms.avg": 12,
        "Pause():ms.cnt": 1000,
        "evict:ms.avg": 2,
        "evict:ms.cnt": 992,
        "fork():ms.avg": 8,
        "fork():ms.cnt": 1000,
        "forkRequest:ms.avg": 6,
        "forkRequest:ms.cnt": 1000,
        "move-to-cg-after-fork:ms.avg": 1,
        "move-to-cg-after-fork:ms.cnt": 1000
      },
      "errors": [],
      "worker_tail": [
        "2019/07/15 15:37:33 remove /sys/fs/cgroup/freezer/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:37:33 remove /sys/fs/cgroup/hugetlb/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:37:33 remove /sys/fs/cgroup/memory/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:37:33 remove /sys/fs/cgroup/perf_event/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:37:33 remove /sys/fs/cgroup/systemd/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:37:33 remove /sys/fs/cgroup/pids/worker-handler-sandboxes [CGROUP POOL worker-handler-sandboxes]",
        "2019/07/15 15:37:33 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/07/15 15:37:33 remove worker.pid",
        "2019/07/15 15:37:33 exiting",
        ""
      ]
    }
  ],
  "passed": 18,
  "failed": 0,
  "seconds": 268.05992555618286
}