{
  "runs": [
    {
      "test": "ping_test",
      "params": {},
      "pass": true,
      "conf": {
        "sandbox_config": {},
        "mem_pool_mb": 500,
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "features": {
          "reuse_cgroups": false,
          "import_cache": true,
          "downsize_paused_mem": true
        },
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "trace": {
          "memory": false,
          "package": false,
          "cgroups": false,
          "evictor": false
        },
        "registry": "/root/open-lambda/test-registry",
        "pip_mirror": "",
        "limits": {
          "installer_mem_mb": 250,
          "procs": 10,
          "swappiness": 0,
          "mem_mb": 50
        },
        "storage": {
          "root": "private",
          "scratch": "",
          "code": ""
        },
        "worker_port": "5000",
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker"
      },
      "seconds": 2.204948902130127,
      "total_seconds": 2.5408589839935303,
      "stats": {
        "pings_per_sec": 453.52936838556326
      },
      "ol-stats": {
        "fresh-cgroup.cnt": 17,
        "fresh-cgroup.ms-avg": 0
      },
      "errors": [],
      "worker_tail": [
        "2019/10/01 21:51:39 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:51:39 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:51:39 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:51:39 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:51:39 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:51:39 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:51:39 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/10/01 21:51:39 remove worker.pid",
        "2019/10/01 21:51:39 exiting",
        ""
      ]
    },
    {
      "test": "install_tests",
      "params": {},
      "pass": true,
      "conf": {
        "sandbox_config": {},
        "mem_pool_mb": 500,
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "features": {
          "reuse_cgroups": false,
          "import_cache": false,
          "downsize_paused_mem": true
        },
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "trace": {
          "memory": false,
          "package": false,
          "cgroups": false,
          "evictor": false
        },
        "registry": "/root/open-lambda/test-registry",
        "pip_mirror": "",
        "limits": {
          "installer_mem_mb": 250,
          "procs": 10,
          "swappiness": 0,
          "mem_mb": 50
        },
        "storage": {
          "root": "private",
          "scratch": "",
          "code": ""
        },
        "worker_port": "5000",
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker"
      },
      "seconds": 11.349764823913574,
      "total_seconds": 11.588866949081421,
      "stats": null,
      "ol-stats": {
        "Channel().cnt": 10,
        "Channel().ms-avg": 0,
        "Create().cnt": 10,
        "Create().ms-avg": 294,
        "Create()/acquire-cgroup.cnt": 10,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 10,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fresh-proc.cnt": 10,
        "Create()/fresh-proc.ms-avg": 292,
        "Create()/make-root-fs.cnt": 10,
        "Create()/make-root-fs.ms-avg": 1,
        "Destroy().cnt": 10,
        "Destroy().ms-avg": 2,
        "Destroy()/detach-root.cnt": 10,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 10,
        "Destroy()/kill-procs.ms-avg": 1,
        "Destroy()/remove-root.cnt": 10,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 4,
        "LambdaFunc.Invoke.ms-avg": 2828,
        "Pause().cnt": 4,
        "Pause().ms-avg": 4,
        "ServeHTTP.cnt": 4,
        "ServeHTTP.ms-avg": 246,
        "fresh-cgroup.cnt": 27,
        "fresh-cgroup.ms-avg": 0,
        "pull-lambda.cnt": 4,
        "pull-lambda.ms-avg": 10,
        "pull-package.cnt": 6,
        "pull-package.ms-avg": 1368,
        "web-request.cnt": 4,
        "web-request.ms-avg": 2828
      },
      "errors": [],
      "worker_tail": [
        "2019/10/01 21:51:51 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:51:51 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:51:51 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:51:51 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:51:51 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:51:51 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:51:51 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/10/01 21:51:51 remove worker.pid",
        "2019/10/01 21:51:51 exiting",
        ""
      ]
    },
    {
      "test": "install_tests",
      "params": {},
      "pass": true,
      "conf": {
        "mem_pool_mb": 500,
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "features": {
          "import_cache": true,
          "reuse_cgroups": false,
          "downsize_paused_mem": true
        },
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry": "/root/open-lambda/test-registry",
        "pip_mirror": "",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "procs": 10,
          "mem_mb": 50
        },
        "worker_port": "5000",
        "storage": {
          "root": "private",
          "scratch": "",
          "code": ""
        },
        "sandbox_config": {},
        "docker_runtime": "",
        "trace": {
          "memory": false,
          "package": false,
          "cgroups": false,
          "evictor": false
        }
      },
      "seconds": 8.753864765167236,
      "total_seconds": 8.991661548614502,
      "stats": null,
      "ol-stats": {
        "Channel().cnt": 10,
        "Channel().ms-avg": 0,
        "Create().cnt": 11,
        "Create().ms-avg": 93,
        "Create()/acquire-cgroup.cnt": 11,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 11,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 4,
        "Create()/fork-proc.ms-avg": 7,
        "Create()/fresh-proc.cnt": 7,
        "Create()/fresh-proc.ms-avg": 139,
        "Create()/make-root-fs.cnt": 11,
        "Create()/make-root-fs.ms-avg": 1,
        "Destroy().cnt": 11,
        "Destroy().ms-avg": 2,
        "Destroy()/detach-root.cnt": 11,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 11,
        "Destroy()/kill-procs.ms-avg": 1,
        "Destroy()/remove-root.cnt": 11,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 4,
        "LambdaFunc.Invoke.ms-avg": 2180,
        "Pause().cnt": 8,
        "Pause().ms-avg": 4,
        "ServeHTTP.cnt": 4,
        "ServeHTTP.ms-avg": 43,
        "Unpause().cnt": 3,
        "Unpause().ms-avg": 0,
        "childExit().cnt": 4,
        "childExit().ms-avg": 0,
        "fork().cnt": 4,
        "fork().ms-avg": 7,
        "forkRequest.cnt": 4,
        "forkRequest.ms-avg": 6,
        "fresh-cgroup.cnt": 28,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 4,
        "move-to-cg-after-fork.ms-avg": 0,
        "pull-lambda.cnt": 4,
        "pull-lambda.ms-avg": 1,
        "pull-package.cnt": 6,
        "pull-package.ms-avg": 1389,
        "web-request.cnt": 4,
        "web-request.ms-avg": 2180
      },
      "errors": [],
      "worker_tail": [
        "2019/10/01 21:52:00 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:52:00 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:52:00 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:52:00 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:52:00 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:52:00 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:52:00 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/10/01 21:52:00 remove worker.pid",
        "2019/10/01 21:52:00 exiting",
        ""
      ]
    },
    {
      "test": "install_tests",
      "params": {},
      "pass": true,
      "conf": {
        "sandbox_config": {},
        "mem_pool_mb": 500,
        "registry_cache_ms": 5000,
        "sandbox": "docker",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "features": {
          "reuse_cgroups": false,
          "import_cache": false,
          "downsize_paused_mem": true
        },
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "trace": {
          "memory": false,
          "package": false,
          "cgroups": false,
          "evictor": false
        },
        "registry": "/root/open-lambda/test-registry",
        "pip_mirror": "",
        "limits": {
          "installer_mem_mb": 250,
          "procs": 10,
          "swappiness": 0,
          "mem_mb": 50
        },
        "storage": {
          "root": "private",
          "scratch": "",
          "code": ""
        },
        "worker_port": "5000",
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker"
      },
      "seconds": 36.58204913139343,
      "total_seconds": 39.513477087020874,
      "stats": null,
      "ol-stats": {
        "Channel().cnt": 10,
        "Channel().ms-avg": 0,
        "Create().cnt": 10,
        "Create().ms-avg": 1943,
        "Destroy().cnt": 10,
        "Destroy().ms-avg": 563,
        "LambdaFunc.Invoke.cnt": 4,
        "LambdaFunc.Invoke.ms-avg": 9134,
        "Pause().cnt": 4,
        "Pause().ms-avg": 237,
        "ServeHTTP.cnt": 4,
        "ServeHTTP.ms-avg": 1,
        "pull-lambda.cnt": 4,
        "pull-lambda.ms-avg": 1,
        "pull-package.cnt": 6,
        "pull-package.ms-avg": 3750,
        "web-request.cnt": 4,
        "web-request.ms-avg": 9134
      },
      "errors": [],
      "worker_tail": [
        "2019/10/01 21:52:38 Kill function: install",
        "2019/10/01 21:52:38 Destroy() [SB 6]",
        "2019/10/01 21:52:38 Kill function: install2",
        "2019/10/01 21:52:38 Destroy() [SB 7]",
        "2019/10/01 21:52:38 Kill function: install3",
        "2019/10/01 21:52:38 Destroy() [SB 9]",
        "2019/10/01 21:52:39 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/10/01 21:52:39 remove worker.pid",
        "2019/10/01 21:52:39 exiting",
        ""
      ]
    },
    {
      "test": "fork_bomb",
      "params": {},
      "pass": true,
      "conf": {
        "mem_pool_mb": 500,
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "features": {
          "import_cache": true,
          "reuse_cgroups": false,
          "downsize_paused_mem": true
        },
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry": "/root/open-lambda/test-registry",
        "pip_mirror": "",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "procs": 10,
          "mem_mb": 50
        },
        "worker_port": "5000",
        "storage": {
          "root": "private",
          "scratch": "",
          "code": ""
        },
        "sandbox_config": {},
        "docker_runtime": "",
        "trace": {
          "memory": false,
          "package": false,
          "cgroups": false,
          "evictor": false
        }
      },
      "seconds": 0.18112397193908691,
      "total_seconds": 0.6154627799987793,
      "stats": null,
      "ol-stats": {
        "Channel().cnt": 1,
        "Channel().ms-avg": 0,
        "Create().cnt": 2,
        "Create().ms-avg": 76,
        "Create()/acquire-cgroup.cnt": 2,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 2,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 1,
        "Create()/fork-proc.ms-avg": 6,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 136,
        "Create()/make-root-fs.cnt": 2,
        "Create()/make-root-fs.ms-avg": 4,
        "Destroy().cnt": 2,
        "Destroy().ms-avg": 2,
        "Destroy()/detach-root.cnt": 2,
        "Destroy()/detach-root.ms-avg": 1,
        "Destroy()/kill-procs.cnt": 2,
        "Destroy()/kill-procs.ms-avg": 0,
        "Destroy()/remove-root.cnt": 2,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 1,
        "LambdaFunc.Invoke.ms-avg": 177,
        "Pause().cnt": 2,
        "Pause().ms-avg": 5,
        "ServeHTTP.cnt": 1,
        "ServeHTTP.ms-avg": 12,
        "childExit().cnt": 1,
        "childExit().ms-avg": 0,
        "fork().cnt": 1,
        "fork().ms-avg": 6,
        "forkRequest.cnt": 1,
        "forkRequest.ms-avg": 4,
        "fresh-cgroup.cnt": 19,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 1,
        "move-to-cg-after-fork.ms-avg": 2,
        "pull-lambda.cnt": 1,
        "pull-lambda.ms-avg": 5,
        "web-request.cnt": 1,
        "web-request.ms-avg": 177
      },
      "errors": [],
      "worker_tail": [
        "2019/10/01 21:52:40 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:52:40 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:52:40 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:52:40 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:52:40 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:52:40 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:52:40 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/10/01 21:52:40 remove worker.pid",
        "2019/10/01 21:52:40 exiting",
        ""
      ]
    },
    {
      "test": "max_mem_alloc",
      "params": {},
      "pass": true,
      "conf": {
        "mem_pool_mb": 500,
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "features": {
          "import_cache": true,
          "reuse_cgroups": false,
          "downsize_paused_mem": true
        },
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry": "/root/open-lambda/test-registry",
        "pip_mirror": "",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "procs": 10,
          "mem_mb": 50
        },
        "worker_port": "5000",
        "storage": {
          "root": "private",
          "scratch": "",
          "code": ""
        },
        "sandbox_config": {},
        "docker_runtime": "",
        "trace": {
          "memory": false,
          "package": false,
          "cgroups": false,
          "evictor": false
        }
      },
      "seconds": 0.8415710926055908,
      "total_seconds": 1.0760047435760498,
      "stats": null,
      "ol-stats": {
        "Channel().cnt": 1,
        "Channel().ms-avg": 0,
        "Create().cnt": 2,
        "Create().ms-avg": 75,
        "Create()/acquire-cgroup.cnt": 2,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 2,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 1,
        "Create()/fork-proc.ms-avg": 5,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 136,
        "Create()/make-root-fs.cnt": 2,
        "Create()/make-root-fs.ms-avg": 4,
        "Destroy().cnt": 2,
        "Destroy().ms-avg": 2,
        "Destroy()/detach-root.cnt": 2,
        "Destroy()/detach-root.ms-avg": 1,
        "Destroy()/kill-procs.cnt": 2,
        "Destroy()/kill-procs.ms-avg": 0,
        "Destroy()/remove-root.cnt": 2,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 1,
        "LambdaFunc.Invoke.ms-avg": 837,
        "Pause().cnt": 2,
        "Pause().ms-avg": 4,
        "ServeHTTP.cnt": 1,
        "ServeHTTP.ms-avg": 672,
        "childExit().cnt": 1,
        "childExit().ms-avg": 0,
        "fork().cnt": 1,
        "fork().ms-avg": 5,
        "forkRequest.cnt": 1,
        "forkRequest.ms-avg": 5,
        "fresh-cgroup.cnt": 19,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 1,
        "move-to-cg-after-fork.ms-avg": 0,
        "pull-lambda.cnt": 1,
        "pull-lambda.ms-avg": 7,
        "web-request.cnt": 1,
        "web-request.ms-avg": 837
      },
      "errors": [],
      "worker_tail": [
        "2019/10/01 21:52:41 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:52:41 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:52:41 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:52:41 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:52:41 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:52:41 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:52:41 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/10/01 21:52:41 remove worker.pid",
        "2019/10/01 21:52:41 exiting",
        ""
      ]
    },
    {
      "test": "numpy_test",
      "params": {},
      "pass": true,
      "conf": {
        "mem_pool_mb": 500,
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "features": {
          "import_cache": true,
          "reuse_cgroups": false,
          "downsize_paused_mem": true
        },
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry": "/root/open-lambda/test-registry",
        "pip_mirror": "",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "procs": 10,
          "mem_mb": 50
        },
        "worker_port": "5000",
        "storage": {
          "root": "private",
          "scratch": "",
          "code": ""
        },
        "sandbox_config": {},
        "docker_runtime": "",
        "trace": {
          "memory": false,
          "package": false,
          "cgroups": false,
          "evictor": false
        }
      },
      "seconds": 115.38805913925171,
      "total_seconds": 115.63524866104126,
      "stats": null,
      "ol-stats": {
        "Channel().cnt": 12,
        "Channel().ms-avg": 0,
        "Create().cnt": 13,
        "Create().ms-avg": 297,
        "Create()/acquire-cgroup.cnt": 13,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 13,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 5,
        "Create()/fork-proc.ms-avg": 5,
        "Create()/fresh-proc.cnt": 8,
        "Create()/fresh-proc.ms-avg": 191,
        "Create()/make-root-fs.cnt": 13,
        "Create()/make-root-fs.ms-avg": 176,
        "Destroy().cnt": 13,
        "Destroy().ms-avg": 4,
        "Destroy()/detach-root.cnt": 13,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 13,
        "Destroy()/kill-procs.ms-avg": 2,
        "Destroy()/remove-root.cnt": 13,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 5,
        "LambdaFunc.Invoke.ms-avg": 23073,
        "Pause().cnt": 10,
        "Pause().ms-avg": 4,
        "ServeHTTP.cnt": 5,
        "ServeHTTP.ms-avg": 453,
        "Unpause().cnt": 4,
        "Unpause().ms-avg": 0,
        "childExit().cnt": 5,
        "childExit().ms-avg": 0,
        "fork().cnt": 5,
        "fork().ms-avg": 5,
        "forkRequest.cnt": 5,
        "forkRequest.ms-avg": 4,
        "fresh-cgroup.cnt": 30,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 5,
        "move-to-cg-after-fork.ms-avg": 0,
        "pull-lambda.cnt": 5,
        "pull-lambda.ms-avg": 7,
        "pull-package.cnt": 7,
        "pull-package.ms-avg": 16058,
        "web-request.cnt": 5,
        "web-request.ms-avg": 23073
      },
      "errors": [],
      "worker_tail": [
        "2019/10/01 21:54:37 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:54:37 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:54:37 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:54:37 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:54:37 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:54:37 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:54:37 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/10/01 21:54:37 remove worker.pid",
        "2019/10/01 21:54:37 exiting",
        ""
      ]
    },
    {
      "test": "sock_churn",
      "params": {
        "fork": false,
        "seconds": 5,
        "baseline": 0,
        "procs": 1
      },
      "pass": true,
      "conf": {
        "mem_pool_mb": 500,
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_tree": "",
        "server_mode": "sock",
        "features": {
          "import_cache": true,
          "reuse_cgroups": false,
          "downsize_paused_mem": true
        },
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry": "/root/open-lambda/test-dir/registry",
        "pip_mirror": "",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "procs": 10,
          "mem_mb": 50
        },
        "worker_port": "5000",
        "storage": {
          "root": "private",
          "scratch": "",
          "code": ""
        },
        "sandbox_config": {},
        "docker_runtime": "",
        "trace": {
          "memory": false,
          "package": false,
          "cgroups": false,
          "evictor": false
        }
      },
      "seconds": 5.147067546844482,
      "total_seconds": 5.269177675247192,
      "stats": {
        "sandboxes_per_sec": 2.8
      },
      "ol-stats": {
        "Create().cnt": 14,
        "Create().ms-avg": 258,
        "Create()/acquire-cgroup.cnt": 14,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 14,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fresh-proc.cnt": 14,
        "Create()/fresh-proc.ms-avg": 238,
        "Create()/make-root-fs.cnt": 14,
        "Create()/make-root-fs.ms-avg": 19,
        "Destroy().cnt": 28,
        "Destroy().ms-avg": 1,
        "Destroy()/detach-root.cnt": 14,
        "Destroy()/detach-root.ms-avg": 1,
        "Destroy()/kill-procs.cnt": 14,
        "Destroy()/kill-procs.ms-avg": 0,
        "Destroy()/remove-root.cnt": 14,
        "Destroy()/remove-root.ms-avg": 0,
        "fresh-cgroup.cnt": 31,
        "fresh-cgroup.ms-avg": 0
      },
      "errors": [],
      "worker_tail": [
        "2019/10/01 21:54:42 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:54:42 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:54:42 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:54:42 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:54:42 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:54:42 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:54:42 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/10/01 21:54:42 remove worker.pid",
        "2019/10/01 21:54:42 exiting",
        ""
      ]
    },
    {
      "test": "sock_churn",
      "params": {
        "fork": true,
        "seconds": 10,
        "baseline": 0,
        "procs": 1
      },
      "pass": true,
      "conf": {
        "mem_pool_mb": 500,
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_tree": "",
        "server_mode": "sock",
        "features": {
          "import_cache": true,
          "reuse_cgroups": false,
          "downsize_paused_mem": true
        },
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry": "/root/open-lambda/test-dir/registry",
        "pip_mirror": "",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "procs": 10,
          "mem_mb": 50
        },
        "worker_port": "5000",
        "storage": {
          "root": "private",
          "scratch": "",
          "code": ""
        },
        "sandbox_config": {},
        "docker_runtime": "",
        "trace": {
          "memory": false,
          "package": false,
          "cgroups": false,
          "evictor": false
        }
      },
      "seconds": 10.182140111923218,
      "total_seconds": 10.480207920074463,
      "stats": {
        "sandboxes_per_sec": 41.9
      },
      "ol-stats": {
        "Create().cnt": 420,
        "Create().ms-avg": 11,
        "Create()/acquire-cgroup.cnt": 420,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 420,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 419,
        "Create()/fork-proc.ms-avg": 6,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 135,
        "Create()/make-root-fs.cnt": 420,
        "Create()/make-root-fs.ms-avg": 3,
        "Destroy().cnt": 839,
        "Destroy().ms-avg": 1,
        "Destroy()/detach-root.cnt": 420,
        "Destroy()/detach-root.ms-avg": 1,
        "Destroy()/kill-procs.cnt": 420,
        "Destroy()/kill-procs.ms-avg": 0,
        "Destroy()/remove-root.cnt": 420,
        "Destroy()/remove-root.ms-avg": 0,
        "childExit().cnt": 419,
        "childExit().ms-avg": 0,
        "fork().cnt": 419,
        "fork().ms-avg": 6,
        "forkRequest.cnt": 419,
        "forkRequest.ms-avg": 6,
        "fresh-cgroup.cnt": 437,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 419,
        "move-to-cg-after-fork.ms-avg": 0
      },
      "errors": [],
      "worker_tail": [
        "2019/10/01 21:54:52 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:54:52 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:54:52 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:54:52 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:54:52 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:54:52 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:54:52 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/10/01 21:54:52 remove worker.pid",
        "2019/10/01 21:54:52 exiting",
        ""
      ]
    },
    {
      "test": "sock_churn",
      "params": {
        "fork": true,
        "seconds": 10,
        "baseline": 0,
        "procs": 15
      },
      "pass": true,
      "conf": {
        "mem_pool_mb": 500,
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_tree": "",
        "server_mode": "sock",
        "features": {
          "import_cache": true,
          "reuse_cgroups": false,
          "downsize_paused_mem": true
        },
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry": "/root/open-lambda/test-dir/registry",
        "pip_mirror": "",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "procs": 10,
          "mem_mb": 50
        },
        "worker_port": "5000",
        "storage": {
          "root": "private",
          "scratch": "",
          "code": ""
        },
        "sandbox_config": {},
        "docker_runtime": "",
        "trace": {
          "memory": false,
          "package": false,
          "cgroups": false,
          "evictor": false
        }
      },
      "seconds": 10.376093864440918,
      "total_seconds": 10.611222267150879,
      "stats": {
        "sandboxes_per_sec": 44.4
      },
      "ol-stats": {
        "Create().cnt": 445,
        "Create().ms-avg": 88,
        "Create()/acquire-cgroup.cnt": 445,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 445,
        "Create()/acquire-mem.ms-avg": 16,
        "Create()/fork-proc.cnt": 444,
        "Create()/fork-proc.ms-avg": 67,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 136,
        "Create()/make-root-fs.cnt": 445,
        "Create()/make-root-fs.ms-avg": 2,
        "Destroy().cnt": 889,
        "Destroy().ms-avg": 54,
        "Destroy()/detach-root.cnt": 445,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 445,
        "Destroy()/kill-procs.ms-avg": 14,
        "Destroy()/remove-root.cnt": 445,
        "Destroy()/remove-root.ms-avg": 0,
        "childExit().cnt": 444,
        "childExit().ms-avg": 68,
        "fork().cnt": 444,
        "fork().ms-avg": 67,
        "forkRequest.cnt": 444,
        "forkRequest.ms-avg": 11,
        "fresh-cgroup.cnt": 462,
        "fresh-cgroup.ms-avg": 3,
        "move-to-cg-after-fork.cnt": 444,
        "move-to-cg-after-fork.ms-avg": 2
      },
      "errors": [],
      "worker_tail": [
        "2019/10/01 21:55:03 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:03 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:03 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:03 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:03 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:03 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:03 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/10/01 21:55:03 remove worker.pid",
        "2019/10/01 21:55:03 exiting",
        ""
      ]
    },
    {
      "test": "sock_churn",
      "params": {
        "fork": true,
        "seconds": 10,
        "baseline": 32,
        "procs": 1
      },
      "pass": true,
      "conf": {
        "mem_pool_mb": 500,
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_tree": "",
        "server_mode": "sock",
        "features": {
          "import_cache": true,
          "reuse_cgroups": false,
          "downsize_paused_mem": true
        },
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry": "/root/open-lambda/test-dir/registry",
        "pip_mirror": "",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "procs": 10,
          "mem_mb": 50
        },
        "worker_port": "5000",
        "storage": {
          "root": "private",
          "scratch": "",
          "code": ""
        },
        "sandbox_config": {},
        "docker_runtime": "",
        "trace": {
          "memory": false,
          "package": false,
          "cgroups": false,
          "evictor": false
        }
      },
      "seconds": 10.829791784286499,
      "total_seconds": 11.764822244644165,
      "stats": {
        "sandboxes_per_sec": 41.5
      },
      "ol-stats": {
        "Create().cnt": 448,
        "Create().ms-avg": 7,
        "Create()/acquire-cgroup.cnt": 448,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 448,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 447,
        "Create()/fork-proc.ms-avg": 6,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 136,
        "Create()/make-root-fs.cnt": 448,
        "Create()/make-root-fs.ms-avg": 0,
        "Destroy().cnt": 863,
        "Destroy().ms-avg": 0,
        "Destroy()/detach-root.cnt": 448,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 448,
        "Destroy()/kill-procs.ms-avg": 0,
        "Destroy()/remove-root.cnt": 448,
        "Destroy()/remove-root.ms-avg": 0,
        "Pause().cnt": 32,
        "Pause().ms-avg": 0,
        "childExit().cnt": 447,
        "childExit().ms-avg": 0,
        "fork().cnt": 447,
        "fork().ms-avg": 6,
        "forkRequest.cnt": 447,
        "forkRequest.ms-avg": 5,
        "fresh-cgroup.cnt": 465,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 447,
        "move-to-cg-after-fork.ms-avg": 0
      },
      "errors": [],
      "worker_tail": [
        "2019/10/01 21:55:14 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:14 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:14 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:14 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:14 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:14 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:14 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/10/01 21:55:14 remove worker.pid",
        "2019/10/01 21:55:14 exiting",
        ""
      ]
    },
    {
      "test": "sock_churn",
      "params": {
        "fork": true,
        "seconds": 10,
        "baseline": 32,
        "procs": 15
      },
      "pass": true,
      "conf": {
        "mem_pool_mb": 500,
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_tree": "",
        "server_mode": "sock",
        "features": {
          "import_cache": true,
          "reuse_cgroups": false,
          "downsize_paused_mem": true
        },
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry": "/root/open-lambda/test-dir/registry",
        "pip_mirror": "",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "procs": 10,
          "mem_mb": 50
        },
        "worker_port": "5000",
        "storage": {
          "root": "private",
          "scratch": "",
          "code": ""
        },
        "sandbox_config": {},
        "docker_runtime": "",
        "trace": {
          "memory": false,
          "package": false,
          "cgroups": false,
          "evictor": false
        }
      },
      "seconds": 11.578738451004028,
      "total_seconds": 12.297508239746094,
      "stats": {
        "sandboxes_per_sec": 35.9
      },
      "ol-stats": {
        "Create().cnt": 392,
        "Create().ms-avg": 153,
        "Create()/acquire-cgroup.cnt": 392,
        "Create()/acquire-cgroup.ms-avg": 1,
        "Create()/acquire-mem.cnt": 392,
        "Create()/acquire-mem.ms-avg": 89,
        "Create()/fork-proc.cnt": 391,
        "Create()/fork-proc.ms-avg": 58,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 698,
        "Create()/make-root-fs.cnt": 392,
        "Create()/make-root-fs.ms-avg": 1,
        "Destroy().cnt": 751,
        "Destroy().ms-avg": 57,
        "Destroy()/detach-root.cnt": 392,
        "Destroy()/detach-root.ms-avg": 3,
        "Destroy()/kill-procs.cnt": 392,
        "Destroy()/kill-procs.ms-avg": 16,
        "Destroy()/remove-root.cnt": 392,
        "Destroy()/remove-root.ms-avg": 1,
        "Pause().cnt": 32,
        "Pause().ms-avg": 0,
        "childExit().cnt": 391,
        "childExit().ms-avg": 59,
        "fork().cnt": 391,
        "fork().ms-avg": 58,
        "forkRequest.cnt": 391,
        "forkRequest.ms-avg": 14,
        "fresh-cgroup.cnt": 409,
        "fresh-cgroup.ms-avg": 2,
        "move-to-cg-after-fork.cnt": 391,
        "move-to-cg-after-fork.ms-avg": 2
      },
      "errors": [],
      "worker_tail": [
        "2019/10/01 21:55:27 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:27 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:27 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:27 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:27 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:27 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:27 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/10/01 21:55:27 remove worker.pid",
        "2019/10/01 21:55:27 exiting",
        ""
      ]
    },
    {
      "test": "update_code",
      "params": {},
      "pass": false,
      "conf": {
        "sandbox_config": {},
        "mem_pool_mb": 500,
        "registry_cache_ms": 3000,
        "sandbox": "sock",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "features": {
          "reuse_cgroups": false,
          "import_cache": true,
          "downsize_paused_mem": true
        },
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "trace": {
          "memory": false,
          "package": false,
          "cgroups": false,
          "evictor": false
        },
        "registry": "/tmp/tmphhmpuo5a",
        "pip_mirror": "",
        "limits": {
          "installer_mem_mb": 250,
          "procs": 10,
          "swappiness": 0,
          "mem_mb": 50
        },
        "storage": {
          "root": "private",
          "scratch": "",
          "code": ""
        },
        "worker_port": "5000",
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker"
      },
      "seconds": null,
      "total_seconds": 8.396052837371826,
      "stats": null,
      "ol-stats": {
        "Channel().cnt": 1,
        "Channel().ms-avg": 0,
        "Create().cnt": 2,
        "Create().ms-avg": 3676,
        "Create()/acquire-cgroup.cnt": 2,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 2,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 1,
        "Create()/fork-proc.ms-avg": 8,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 7336,
        "Create()/make-root-fs.cnt": 2,
        "Create()/make-root-fs.ms-avg": 4,
        "Destroy().cnt": 2,
        "Destroy().ms-avg": 2,
        "Destroy()/detach-root.cnt": 2,
        "Destroy()/detach-root.ms-avg": 1,
        "Destroy()/kill-procs.cnt": 2,
        "Destroy()/kill-procs.ms-avg": 0,
        "Destroy()/remove-root.cnt": 2,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 1,
        "LambdaFunc.Invoke.ms-avg": 8029,
        "Pause().cnt": 2,
        "Pause().ms-avg": 3,
        "ServeHTTP.cnt": 1,
        "ServeHTTP.ms-avg": 4,
        "childExit().cnt": 1,
        "childExit().ms-avg": 0,
        "fork().cnt": 1,
        "fork().ms-avg": 8,
        "forkRequest.cnt": 1,
        "forkRequest.ms-avg": 7,
        "fresh-cgroup.cnt": 19,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 1,
        "move-to-cg-after-fork.ms-avg": 1,
        "pull-lambda.cnt": 1,
        "pull-lambda.ms-avg": 665,
        "web-request.cnt": 1,
        "web-request.ms-avg": 8029
      },
      "errors": [
        [
          "Traceback (most recent call last):",
          "  File \"test.py\", line 81, in wrapper",
          "    rv = fn(**kwargs)",
          "  File \"test.py\", line 403, in update_code",
          "    assert(t1 - t0 <= cache_seconds + 1)",
          "AssertionError",
          ""
        ]
      ],
      "worker_tail": [
        "using existing OL directory at /root/open-lambda/test-dir",
        "2019/10/01 21:55:27 save PID 17491 to file /root/open-lambda/test-dir/worker/worker.pid",
        "2019/10/01 21:55:27 Start Lambda Server",
        "2019/10/01 21:55:27 Storage dir at /root/open-lambda/test-dir/worker/code",
        "2019/10/01 21:55:27 Storage dir at /root/open-lambda/test-dir/worker/scratch",
        "2019/10/01 21:55:27 Create SandboxPool",
        "2019/10/01 21:55:27 create /sys/fs/cgroup/blkio/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:27 create /sys/fs/cgroup/cpu/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:27 create /sys/fs/cgroup/devices/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:27 create /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:27 create /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:27 create /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:27 create /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:27 create /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:27 create /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:27 Storage dir at /root/open-lambda/test-dir/worker/root-sandboxes",
        "2019/10/01 21:55:27 Create DepTracer",
        "2019/10/01 21:55:27 Create PackagePuller",
        "2019/10/01 21:55:27 Create ImportCache",
        "2019/10/01 21:55:27 Import Cache Tree:",
        "2019/10/01 21:55:27 0    - ROOT",
        "2019/10/01 21:55:27 Create HandlerPuller",
        "2019/10/01 21:55:27 Setups Handlers",
        "2019/10/01 21:55:27 Execute handler by POSTing to localhost:5000/run/<lambda>",
        "2019/10/01 21:55:27 Get status by sending request to localhost:5000/status",
        "2019/10/01 21:55:27 start creating/serving CGs [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:27 Receive request to /pid",
        "2019/10/01 21:55:27 Receive request to /run/version",
        "2019/10/01 21:55:27 debug: LambdaFunc.Task() runs on goroutine 19 [FUNC version]",
        "2019/10/01 21:55:28 increase instances to 1 [FUNC version]",
        "2019/10/01 21:55:28 Try using Zygote from <ROOT>",
        "2019/10/01 21:55:28 <sandboxes>.Create(<nil>, false, /root/open-lambda/test-dir/worker/code/1003-import-cache, /root/open-lambda/test-dir/worker/scratch/1004-import-cache, <installs=[], imports=[], mem-limit-mb=50>)=1... [SOCK POOL sandboxes]",
        "2019/10/01 21:55:28 use cgroup cg-1 [SOCK 1]",
        "sock2.py: started new process with args: sock2.py /host/bootstrap.py 9",
        "sock2.py: joined cgroup, close FD 3",
        "sock2.py: joined cgroup, close FD 4",
        "sock2.py: joined cgroup, close FD 5",
        "sock2.py: joined cgroup, close FD 6",
        "sock2.py: joined cgroup, close FD 7",
        "sock2.py: joined cgroup, close FD 8",
        "sock2.py: joined cgroup, close FD 9",
        "sock2.py: joined cgroup, close FD 10",
        "sock2.py: joined cgroup, close FD 11",
        "sock2.py: start fork server on fd: 3",
        "2019/10/01 21:55:35 ...returns <SB 1>, <nil> [SOCK POOL sandboxes]",
        "2019/10/01 21:55:35 <sandboxes>.Create(<SB 1>, true, /root/open-lambda/test-dir/worker/code/1001-version, /root/open-lambda/test-dir/worker/scratch/1002-version, <installs=[], imports=[], mem-limit-mb=50>)=2... [SOCK POOL sandboxes]",
        "2019/10/01 21:55:35 use cgroup cg-2 [SOCK 2]",
        "2019/10/01 21:55:35 fork(SB 2) [SB 1]",
        "sock2.py: start web server on fd: 3",
        "2019/10/01 21:55:35 move PID 17506 from CG cg-1 to CG cg-2 [SOCK 1]",
        "2019/10/01 21:55:35 ...returns <SB 2>, <nil> [SOCK POOL sandboxes]",
        "2019/10/01 21:55:35 Pause() [SB 1]",
        "2019/10/01 21:55:35 Channel() [SB 2]",
        "2019/10/01 21:55:35 Pause() [SB 2]",
        "2019/10/01 21:55:35 received kill signal, cleaning up",
        "2019/10/01 21:55:35 Kill function: version",
        "2019/10/01 21:55:35 Destroy() [SB 2]",
        "2019/10/01 21:55:35 CG ref count decremented to 0 [SOCK 2]",
        "2019/10/01 21:55:35 killed PIDs [17506] in CG [SOCK 2]",
        "2019/10/01 21:55:35 unmount and remove dirs [SOCK 2]",
        "2019/10/01 21:55:35 childExit(SB 2) [SB 1]",
        "2019/10/01 21:55:35 CG ref count decremented to 1 [SOCK 1]",
        "2019/10/01 21:55:35 Import Cache Tree:",
        "2019/10/01 21:55:35 1    - ROOT",
        "2019/10/01 21:55:35 Destroy() [SB 1]",
        "2019/10/01 21:55:35 CG ref count decremented to 0 [SOCK 1]",
        "2019/10/01 21:55:35 killed PIDs [17504] in CG [SOCK 1]",
        "2019/10/01 21:55:35 unmount and remove dirs [SOCK 1]",
        "2019/10/01 21:55:35 make sure all memory is free [SOCK POOL sandboxes]",
        "2019/10/01 21:55:35 memory pool emptied [SOCK POOL sandboxes]",
        "2019/10/01 21:55:35 received shutdown request [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:35 empty queues and release CGs [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:35 remove /sys/fs/cgroup/blkio/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:35 remove /sys/fs/cgroup/cpu/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:35 remove /sys/fs/cgroup/devices/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:35 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:35 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:35 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:35 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:35 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:35 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:35 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/10/01 21:55:35 remove worker.pid",
        "2019/10/01 21:55:35 exiting",
        ""
      ]
    },
    {
      "test": "stress_one_lambda",
      "params": {
        "seconds": 15,
        "procs": 1
      },
      "pass": true,
      "conf": {
        "mem_pool_mb": 500,
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "features": {
          "import_cache": true,
          "reuse_cgroups": false,
          "downsize_paused_mem": true
        },
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry": "/root/open-lambda/test-registry",
        "pip_mirror": "",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "procs": 10,
          "mem_mb": 50
        },
        "worker_port": "5000",
        "storage": {
          "root": "private",
          "scratch": "",
          "code": ""
        },
        "sandbox_config": {},
        "docker_runtime": "",
        "trace": {
          "memory": false,
          "package": false,
          "cgroups": false,
          "evictor": false
        }
      },
      "seconds": 15.047749996185303,
      "total_seconds": 15.277583122253418,
      "stats": {
        "reqs_per_sec": 281.6666666666667
      },
      "ol-stats": {
        "Channel().cnt": 3,
        "Channel().ms-avg": 0,
        "Create().cnt": 4,
        "Create().ms-avg": 44,
        "Create()/acquire-cgroup.cnt": 4,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 4,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 3,
        "Create()/fork-proc.ms-avg": 7,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 141,
        "Create()/make-root-fs.cnt": 4,
        "Create()/make-root-fs.ms-avg": 3,
        "Destroy().cnt": 4,
        "Destroy().ms-avg": 4,
        "Destroy()/detach-root.cnt": 4,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 4,
        "Destroy()/kill-procs.ms-avg": 2,
        "Destroy()/remove-root.cnt": 4,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 4225,
        "LambdaFunc.Invoke.ms-avg": 1,
        "Pause().cnt": 4228,
        "Pause().ms-avg": 0,
        "ServeHTTP.cnt": 4225,
        "ServeHTTP.ms-avg": 0,
        "Unpause().cnt": 4224,
        "Unpause().ms-avg": 0,
        "childExit().cnt": 3,
        "childExit().ms-avg": 0,
        "fork().cnt": 3,
        "fork().ms-avg": 7,
        "forkRequest.cnt": 3,
        "forkRequest.ms-avg": 5,
        "fresh-cgroup.cnt": 21,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 3,
        "move-to-cg-after-fork.ms-avg": 1,
        "pull-lambda.cnt": 3,
        "pull-lambda.ms-avg": 1,
        "web-request.cnt": 4225,
        "web-request.ms-avg": 1
      },
      "errors": [],
      "worker_tail": [
        "2019/10/01 21:55:51 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:51 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:51 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:51 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:51 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:51 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:55:51 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/10/01 21:55:51 remove worker.pid",
        "2019/10/01 21:55:51 exiting",
        ""
      ]
    },
    {
      "test": "stress_one_lambda",
      "params": {
        "seconds": 15,
        "procs": 2
      },
      "pass": true,
      "conf": {
        "mem_pool_mb": 500,
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "features": {
          "import_cache": true,
          "reuse_cgroups": false,
          "downsize_paused_mem": true
        },
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry": "/root/open-lambda/test-registry",
        "pip_mirror": "",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "procs": 10,
          "mem_mb": 50
        },
        "worker_port": "5000",
        "storage": {
          "root": "private",
          "scratch": "",
          "code": ""
        },
        "sandbox_config": {},
        "docker_runtime": "",
        "trace": {
          "memory": false,
          "package": false,
          "cgroups": false,
          "evictor": false
        }
      },
      "seconds": 15.043739795684814,
      "total_seconds": 15.275555849075317,
      "stats": {
        "reqs_per_sec": 279.8666666666667
      },
      "ol-stats": {
        "Channel().cnt": 3,
        "Channel().ms-avg": 0,
        "Create().cnt": 4,
        "Create().ms-avg": 44,
        "Create()/acquire-cgroup.cnt": 4,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 4,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 3,
        "Create()/fork-proc.ms-avg": 8,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 138,
        "Create()/make-root-fs.cnt": 4,
        "Create()/make-root-fs.ms-avg": 3,
        "Destroy().cnt": 4,
        "Destroy().ms-avg": 3,
        "Destroy()/detach-root.cnt": 4,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 4,
        "Destroy()/kill-procs.ms-avg": 2,
        "Destroy()/remove-root.cnt": 4,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 4198,
        "LambdaFunc.Invoke.ms-avg": 1,
        "Pause().cnt": 3687,
        "Pause().ms-avg": 0,
        "ServeHTTP.cnt": 4198,
        "ServeHTTP.ms-avg": 0,
        "Unpause().cnt": 3683,
        "Unpause().ms-avg": 0,
        "childExit().cnt": 3,
        "childExit().ms-avg": 0,
        "fork().cnt": 3,
        "fork().ms-avg": 8,
        "forkRequest.cnt": 3,
        "forkRequest.ms-avg": 5,
        "fresh-cgroup.cnt": 21,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 3,
        "move-to-cg-after-fork.ms-avg": 1,
        "pull-lambda.cnt": 3,
        "pull-lambda.ms-avg": 1,
        "web-request.cnt": 4198,
        "web-request.ms-avg": 1
      },
      "errors": [],
      "worker_tail": [
        "2019/10/01 21:56:06 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:56:06 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:56:06 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:56:06 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:56:06 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:56:06 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:56:06 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/10/01 21:56:06 remove worker.pid",
        "2019/10/01 21:56:06 exiting",
        ""
      ]
    },
    {
      "test": "stress_one_lambda",
      "params": {
        "seconds": 15,
        "procs": 8
      },
      "pass": true,
      "conf": {
        "mem_pool_mb": 500,
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "features": {
          "import_cache": true,
          "reuse_cgroups": false,
          "downsize_paused_mem": true
        },
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry": "/root/open-lambda/test-registry",
        "pip_mirror": "",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "procs": 10,
          "mem_mb": 50
        },
        "worker_port": "5000",
        "storage": {
          "root": "private",
          "scratch": "",
          "code": ""
        },
        "sandbox_config": {},
        "docker_runtime": "",
        "trace": {
          "memory": false,
          "package": false,
          "cgroups": false,
          "evictor": false
        }
      },
      "seconds": 15.070022821426392,
      "total_seconds": 15.5799081325531,
      "stats": {
        "reqs_per_sec": 260.8
      },
      "ol-stats": {
        "Channel().cnt": 3,
        "Channel().ms-avg": 0,
        "Create().cnt": 4,
        "Create().ms-avg": 44,
        "Create()/acquire-cgroup.cnt": 4,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 4,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 3,
        "Create()/fork-proc.ms-avg": 7,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 138,
        "Create()/make-root-fs.cnt": 4,
        "Create()/make-root-fs.ms-avg": 3,
        "Destroy().cnt": 4,
        "Destroy().ms-avg": 4,
        "Destroy()/detach-root.cnt": 4,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 4,
        "Destroy()/kill-procs.ms-avg": 2,
        "Destroy()/remove-root.cnt": 4,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 3912,
        "LambdaFunc.Invoke.ms-avg": 2,
        "Pause().cnt": 2936,
        "Pause().ms-avg": 0,
        "ServeHTTP.cnt": 3912,
        "ServeHTTP.ms-avg": 0,
        "Unpause().cnt": 2932,
        "Unpause().ms-avg": 0,
        "childExit().cnt": 3,
        "childExit().ms-avg": 0,
        "fork().cnt": 3,
        "fork().ms-avg": 7,
        "forkRequest.cnt": 3,
        "forkRequest.ms-avg": 5,
        "fresh-cgroup.cnt": 21,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 3,
        "move-to-cg-after-fork.ms-avg": 0,
        "pull-lambda.cnt": 3,
        "pull-lambda.ms-avg": 3,
        "web-request.cnt": 3912,
        "web-request.ms-avg": 2
      },
      "errors": [],
      "worker_tail": [
        "2019/10/01 21:56:21 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:56:21 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:56:21 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:56:21 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:56:21 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:56:21 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:56:21 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/10/01 21:56:21 remove worker.pid",
        "2019/10/01 21:56:21 exiting",
        ""
      ]
    },
    {
      "test": "call_each_once_exec",
      "params": {
        "lambda_count": 100,
        "alloc_mb": 1
      },
      "pass": true,
      "conf": {
        "sandbox_config": {},
        "mem_pool_mb": 500,
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "features": {
          "reuse_cgroups": true,
          "import_cache": true,
          "downsize_paused_mem": true
        },
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "trace": {
          "memory": false,
          "package": false,
          "cgroups": false,
          "evictor": false
        },
        "registry": "/tmp/tmputy36bjn",
        "pip_mirror": "",
        "limits": {
          "installer_mem_mb": 250,
          "procs": 10,
          "swappiness": 0,
          "mem_mb": 50
        },
        "storage": {
          "root": "private",
          "scratch": "",
          "code": ""
        },
        "worker_port": "5000",
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker"
      },
      "seconds": 2.916327953338623,
      "total_seconds": 3.3648836612701416,
      "stats": {
        "reqs_per_sec": 34.29002714880041
      },
      "ol-stats": {
        "Channel().cnt": 100,
        "Channel().ms-avg": 0,
        "Create().cnt": 101,
        "Create().ms-avg": 7,
        "Create()/acquire-cgroup.cnt": 101,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 101,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 100,
        "Create()/fork-proc.ms-avg": 5,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 137,
        "Create()/make-root-fs.cnt": 101,
        "Create()/make-root-fs.ms-avg": 0,
        "Destroy().cnt": 155,
        "Destroy().ms-avg": 4,
        "Destroy()/detach-root.cnt": 101,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 101,
        "Destroy()/kill-procs.ms-avg": 3,
        "Destroy()/remove-root.cnt": 101,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 100,
        "LambdaFunc.Invoke.ms-avg": 25,
        "Pause().cnt": 200,
        "Pause().ms-avg": 9,
        "ServeHTTP.cnt": 100,
        "ServeHTTP.ms-avg": 6,
        "Unpause().cnt": 99,
        "Unpause().ms-avg": 0,
        "childExit().cnt": 100,
        "childExit().ms-avg": 3,
        "evict.cnt": 54,
        "evict.ms-avg": 10,
        "fork().cnt": 100,
        "fork().ms-avg": 5,
        "forkRequest.cnt": 100,
        "forkRequest.ms-avg": 4,
        "fresh-cgroup.cnt": 69,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 100,
        "move-to-cg-after-fork.ms-avg": 0,
        "pull-lambda.cnt": 100,
        "pull-lambda.ms-avg": 3,
        "web-request.cnt": 100,
        "web-request.ms-avg": 25
      },
      "errors": [],
      "worker_tail": [
        "2019/10/01 21:56:25 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:56:25 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:56:25 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:56:25 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:56:25 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:56:25 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:56:25 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/10/01 21:56:25 remove worker.pid",
        "2019/10/01 21:56:25 exiting",
        ""
      ]
    },
    {
      "test": "call_each_once_exec",
      "params": {
        "lambda_count": 1000,
        "alloc_mb": 10
      },
      "pass": true,
      "conf": {
        "mem_pool_mb": 500,
        "registry_cache_ms": 5000,
        "sandbox": "sock",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "features": {
          "import_cache": true,
          "reuse_cgroups": true,
          "downsize_paused_mem": true
        },
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry": "/tmp/tmpkz2lrd_i",
        "pip_mirror": "",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "procs": 10,
          "mem_mb": 50
        },
        "worker_port": "5000",
        "storage": {
          "root": "private",
          "scratch": "",
          "code": ""
        },
        "sandbox_config": {},
        "docker_runtime": "",
        "trace": {
          "memory": false,
          "package": false,
          "cgroups": false,
          "evictor": false
        }
      },
      "seconds": 45.15898561477661,
      "total_seconds": 45.71528339385986,
      "stats": {
        "reqs_per_sec": 22.144047655520527
      },
      "ol-stats": {
        "Channel().cnt": 1000,
        "Channel().ms-avg": 0,
        "Create().cnt": 1001,
        "Create().ms-avg": 8,
        "Create()/acquire-cgroup.cnt": 1001,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 1001,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 1000,
        "Create()/fork-proc.ms-avg": 6,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 138,
        "Create()/make-root-fs.cnt": 1001,
        "Create()/make-root-fs.ms-avg": 1,
        "Destroy().cnt": 1980,
        "Destroy().ms-avg": 6,
        "Destroy()/detach-root.cnt": 1001,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 1001,
        "Destroy()/kill-procs.ms-avg": 7,
        "Destroy()/remove-root.cnt": 1001,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 1000,
        "LambdaFunc.Invoke.ms-avg": 41,
        "Pause().cnt": 2000,
        "Pause().ms-avg": 9,
        "ServeHTTP.cnt": 1000,
        "ServeHTTP.ms-avg": 19,
        "Unpause().cnt": 999,
        "Unpause().ms-avg": 0,
        "childExit().cnt": 1000,
        "childExit().ms-avg": 4,
        "evict.cnt": 979,
        "evict.ms-avg": 12,
        "fork().cnt": 1000,
        "fork().ms-avg": 6,
        "forkRequest.cnt": 1000,
        "forkRequest.ms-avg": 5,
        "fresh-cgroup.cnt": 40,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 1000,
        "move-to-cg-after-fork.ms-avg": 0,
        "pull-lambda.cnt": 1000,
        "pull-lambda.ms-avg": 3,
        "web-request.cnt": 1000,
        "web-request.ms-avg": 41
      },
      "errors": [],
      "worker_tail": [
        "2019/10/01 21:57:10 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:57:10 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:57:10 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:57:10 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:57:10 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:57:10 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2019/10/01 21:57:11 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2019/10/01 21:57:11 remove worker.pid",
        "2019/10/01 21:57:11 exiting",
        ""
      ]
    }
  ],
  "passed": 17,
  "failed": 1,
  "seconds": 508.2342493534088
}