{
  "runs": [
    {
      "test": "ping_test",
      "params": {},
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "limits": {
          "swappiness": 0,
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 250
        },
        "pip_mirror": "",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "sandbox": "sock",
        "features": {
          "reuse_cgroups": false,
          "import_cache": true,
          "downsize_paused_mem": true
        },
        "registry_cache_ms": 5000,
        "registry": "/root/open-lambda/test-registry",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "docker_runtime": "",
        "storage": {
          "scratch": "",
          "root": "private",
          "code": ""
        },
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "mem_pool_mb": 500,
        "sandbox_config": {},
        "trace": {
          "evictor": false,
          "cgroups": false,
          "memory": false,
          "package": false
        }
      },
      "seconds": 2.2027034759521484,
      "total_seconds": 2.3886899948120117,
      "stats": {
        "pings_per_sec": 453.99184584099896
      },
      "ol-stats": {
        "fresh-cgroup.cnt": 17,
        "fresh-cgroup.ms-avg": 0
      },
      "errors": [],
      "worker_tail": [
        "2020/10/01 21:43:08 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:08 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:08 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:08 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:08 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:08 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:08 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2020/10/01 21:43:08 remove worker.pid",
        "2020/10/01 21:43:08 exiting",
        ""
      ]
    },
    {
      "test": "install_tests",
      "params": {},
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "limits": {
          "swappiness": 0,
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 250
        },
        "pip_mirror": "",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "sandbox": "sock",
        "features": {
          "reuse_cgroups": false,
          "import_cache": false,
          "downsize_paused_mem": true
        },
        "registry_cache_ms": 5000,
        "registry": "/root/open-lambda/test-registry",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "docker_runtime": "",
        "storage": {
          "scratch": "",
          "root": "private",
          "code": ""
        },
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "mem_pool_mb": 500,
        "sandbox_config": {},
        "trace": {
          "evictor": false,
          "cgroups": false,
          "memory": false,
          "package": false
        }
      },
      "seconds": 8.900849103927612,
      "total_seconds": 9.14118766784668,
      "stats": null,
      "ol-stats": {
        "Channel().cnt": 10,
        "Channel().ms-avg": 0,
        "Create().cnt": 10,
        "Create().ms-avg": 268,
        "Create()/acquire-cgroup.cnt": 10,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 10,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fresh-proc.cnt": 10,
        "Create()/fresh-proc.ms-avg": 266,
        "Create()/make-root-fs.cnt": 10,
        "Create()/make-root-fs.ms-avg": 1,
        "Destroy().cnt": 10,
        "Destroy().ms-avg": 2,
        "Destroy()/detach-root.cnt": 10,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 10,
        "Destroy()/kill-procs.ms-avg": 1,
        "Destroy()/remove-root.cnt": 10,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 4,
        "LambdaFunc.Invoke.ms-avg": 2210,
        "Pause().cnt": 4,
        "Pause().ms-avg": 4,
        "ServeHTTP.cnt": 4,
        "ServeHTTP.ms-avg": 103,
        "fresh-cgroup.cnt": 27,
        "fresh-cgroup.ms-avg": 0,
        "pull-lambda.cnt": 4,
        "pull-lambda.ms-avg": 5,
        "pull-package.cnt": 6,
        "pull-package.ms-avg": 1095,
        "web-request.cnt": 4,
        "web-request.ms-avg": 2210
      },
      "errors": [],
      "worker_tail": [
        "2020/10/01 21:43:17 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:17 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:17 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:17 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:17 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:17 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:17 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2020/10/01 21:43:17 remove worker.pid",
        "2020/10/01 21:43:17 exiting",
        ""
      ]
    },
    {
      "test": "install_tests",
      "params": {},
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "mem_mb": 50,
          "procs": 10
        },
        "pip_mirror": "",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "storage": {
          "scratch": "",
          "root": "private",
          "code": ""
        },
        "features": {
          "reuse_cgroups": false,
          "import_cache": true,
          "downsize_paused_mem": true
        },
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry_cache_ms": 5000,
        "registry": "/root/open-lambda/test-registry",
        "sandbox": "sock",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "trace": {
          "evictor": false,
          "cgroups": false,
          "package": false,
          "memory": false
        },
        "mem_pool_mb": 500,
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda"
      },
      "seconds": 5.964884042739868,
      "total_seconds": 6.201556444168091,
      "stats": null,
      "ol-stats": {
        "Channel().cnt": 10,
        "Channel().ms-avg": 0,
        "Create().cnt": 11,
        "Create().ms-avg": 93,
        "Create()/acquire-cgroup.cnt": 11,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 11,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 4,
        "Create()/fork-proc.ms-avg": 7,
        "Create()/fresh-proc.cnt": 7,
        "Create()/fresh-proc.ms-avg": 140,
        "Create()/make-root-fs.cnt": 11,
        "Create()/make-root-fs.ms-avg": 1,
        "Destroy().cnt": 11,
        "Destroy().ms-avg": 2,
        "Destroy()/detach-root.cnt": 11,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 11,
        "Destroy()/kill-procs.ms-avg": 1,
        "Destroy()/remove-root.cnt": 11,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 4,
        "LambdaFunc.Invoke.ms-avg": 1483,
        "Pause().cnt": 8,
        "Pause().ms-avg": 4,
        "ServeHTTP.cnt": 4,
        "ServeHTTP.ms-avg": 43,
        "Unpause().cnt": 3,
        "Unpause().ms-avg": 0,
        "childExit().cnt": 4,
        "childExit().ms-avg": 0,
        "fork().cnt": 4,
        "fork().ms-avg": 7,
        "forkRequest.cnt": 4,
        "forkRequest.ms-avg": 6,
        "fresh-cgroup.cnt": 28,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 4,
        "move-to-cg-after-fork.ms-avg": 0,
        "pull-lambda.cnt": 4,
        "pull-lambda.ms-avg": 1,
        "pull-package.cnt": 6,
        "pull-package.ms-avg": 924,
        "web-request.cnt": 4,
        "web-request.ms-avg": 1483
      },
      "errors": [],
      "worker_tail": [
        "2020/10/01 21:43:23 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:23 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:23 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:23 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:23 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:23 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:23 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2020/10/01 21:43:23 remove worker.pid",
        "2020/10/01 21:43:23 exiting",
        ""
      ]
    },
    {
      "test": "install_tests",
      "params": {},
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "limits": {
          "swappiness": 0,
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 250
        },
        "pip_mirror": "",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "sandbox": "docker",
        "features": {
          "reuse_cgroups": false,
          "import_cache": false,
          "downsize_paused_mem": true
        },
        "registry_cache_ms": 5000,
        "registry": "/root/open-lambda/test-registry",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "docker_runtime": "",
        "storage": {
          "scratch": "",
          "root": "private",
          "code": ""
        },
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "mem_pool_mb": 500,
        "sandbox_config": {},
        "trace": {
          "evictor": false,
          "cgroups": false,
          "memory": false,
          "package": false
        }
      },
      "seconds": 15.677260160446167,
      "total_seconds": 16.705556869506836,
      "stats": null,
      "ol-stats": {
        "Channel().cnt": 10,
        "Channel().ms-avg": 0,
        "Create().cnt": 10,
        "Create().ms-avg": 838,
        "Destroy().cnt": 10,
        "Destroy().ms-avg": 262,
        "LambdaFunc.Invoke.cnt": 4,
        "LambdaFunc.Invoke.ms-avg": 3908,
        "Pause().cnt": 4,
        "Pause().ms-avg": 27,
        "ServeHTTP.cnt": 4,
        "ServeHTTP.ms-avg": 1,
        "pull-lambda.cnt": 4,
        "pull-lambda.ms-avg": 1,
        "pull-package.cnt": 6,
        "pull-package.ms-avg": 1716,
        "web-request.cnt": 4,
        "web-request.ms-avg": 3908
      },
      "errors": [],
      "worker_tail": [
        "2020/10/01 21:43:39 Kill function: install",
        "2020/10/01 21:43:39 Destroy() [SB 6]",
        "2020/10/01 21:43:40 Kill function: install2",
        "2020/10/01 21:43:40 Destroy() [SB 7]",
        "2020/10/01 21:43:40 Kill function: install3",
        "2020/10/01 21:43:40 Destroy() [SB 9]",
        "2020/10/01 21:43:40 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2020/10/01 21:43:40 remove worker.pid",
        "2020/10/01 21:43:40 exiting",
        ""
      ]
    },
    {
      "test": "fork_bomb",
      "params": {},
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "mem_mb": 50,
          "procs": 10
        },
        "pip_mirror": "",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "storage": {
          "scratch": "",
          "root": "private",
          "code": ""
        },
        "features": {
          "reuse_cgroups": false,
          "import_cache": true,
          "downsize_paused_mem": true
        },
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry_cache_ms": 5000,
        "registry": "/root/open-lambda/test-registry",
        "sandbox": "sock",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "trace": {
          "evictor": false,
          "cgroups": false,
          "package": false,
          "memory": false
        },
        "mem_pool_mb": 500,
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda"
      },
      "seconds": 0.17884159088134766,
      "total_seconds": 0.412320613861084,
      "stats": null,
      "ol-stats": {
        "Channel().cnt": 1,
        "Channel().ms-avg": 0,
        "Create().cnt": 2,
        "Create().ms-avg": 76,
        "Create()/acquire-cgroup.cnt": 2,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 2,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 1,
        "Create()/fork-proc.ms-avg": 6,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 138,
        "Create()/make-root-fs.cnt": 2,
        "Create()/make-root-fs.ms-avg": 4,
        "Destroy().cnt": 2,
        "Destroy().ms-avg": 2,
        "Destroy()/detach-root.cnt": 2,
        "Destroy()/detach-root.ms-avg": 1,
        "Destroy()/kill-procs.cnt": 2,
        "Destroy()/kill-procs.ms-avg": 0,
        "Destroy()/remove-root.cnt": 2,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 1,
        "LambdaFunc.Invoke.ms-avg": 175,
        "Pause().cnt": 2,
        "Pause().ms-avg": 4,
        "ServeHTTP.cnt": 1,
        "ServeHTTP.ms-avg": 12,
        "childExit().cnt": 1,
        "childExit().ms-avg": 0,
        "fork().cnt": 1,
        "fork().ms-avg": 6,
        "forkRequest.cnt": 1,
        "forkRequest.ms-avg": 5,
        "fresh-cgroup.cnt": 19,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 1,
        "move-to-cg-after-fork.ms-avg": 0,
        "pull-lambda.cnt": 1,
        "pull-lambda.ms-avg": 2,
        "web-request.cnt": 1,
        "web-request.ms-avg": 175
      },
      "errors": [],
      "worker_tail": [
        "2020/10/01 21:43:40 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:40 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:40 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:40 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:40 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:40 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:40 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2020/10/01 21:43:40 remove worker.pid",
        "2020/10/01 21:43:40 exiting",
        ""
      ]
    },
    {
      "test": "max_mem_alloc",
      "params": {},
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "mem_mb": 50,
          "procs": 10
        },
        "pip_mirror": "",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "storage": {
          "scratch": "",
          "root": "private",
          "code": ""
        },
        "features": {
          "reuse_cgroups": false,
          "import_cache": true,
          "downsize_paused_mem": true
        },
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry_cache_ms": 5000,
        "registry": "/root/open-lambda/test-registry",
        "sandbox": "sock",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "trace": {
          "evictor": false,
          "cgroups": false,
          "package": false,
          "memory": false
        },
        "mem_pool_mb": 500,
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda"
      },
      "seconds": 0.8603870868682861,
      "total_seconds": 1.0918772220611572,
      "stats": null,
      "ol-stats": {
        "Channel().cnt": 1,
        "Channel().ms-avg": 0,
        "Create().cnt": 2,
        "Create().ms-avg": 76,
        "Create()/acquire-cgroup.cnt": 2,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 2,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 1,
        "Create()/fork-proc.ms-avg": 5,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 138,
        "Create()/make-root-fs.cnt": 2,
        "Create()/make-root-fs.ms-avg": 4,
        "Destroy().cnt": 2,
        "Destroy().ms-avg": 2,
        "Destroy()/detach-root.cnt": 2,
        "Destroy()/detach-root.ms-avg": 1,
        "Destroy()/kill-procs.cnt": 2,
        "Destroy()/kill-procs.ms-avg": 0,
        "Destroy()/remove-root.cnt": 2,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 1,
        "LambdaFunc.Invoke.ms-avg": 856,
        "Pause().cnt": 2,
        "Pause().ms-avg": 4,
        "ServeHTTP.cnt": 1,
        "ServeHTTP.ms-avg": 690,
        "childExit().cnt": 1,
        "childExit().ms-avg": 0,
        "fork().cnt": 1,
        "fork().ms-avg": 5,
        "forkRequest.cnt": 1,
        "forkRequest.ms-avg": 2,
        "fresh-cgroup.cnt": 19,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 1,
        "move-to-cg-after-fork.ms-avg": 2,
        "pull-lambda.cnt": 1,
        "pull-lambda.ms-avg": 5,
        "web-request.cnt": 1,
        "web-request.ms-avg": 856
      },
      "errors": [],
      "worker_tail": [
        "2020/10/01 21:43:42 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:42 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:42 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:42 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:42 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:42 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:42 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2020/10/01 21:43:42 remove worker.pid",
        "2020/10/01 21:43:42 exiting",
        ""
      ]
    },
    {
      "test": "numpy_test",
      "params": {},
      "pass": false,
      "conf": {
        "worker_port": "5000",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "mem_mb": 50,
          "procs": 10
        },
        "pip_mirror": "",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "storage": {
          "scratch": "",
          "root": "private",
          "code": ""
        },
        "features": {
          "reuse_cgroups": false,
          "import_cache": true,
          "downsize_paused_mem": true
        },
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry_cache_ms": 5000,
        "registry": "/root/open-lambda/test-registry",
        "sandbox": "sock",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "trace": {
          "evictor": false,
          "cgroups": false,
          "package": false,
          "memory": false
        },
        "mem_pool_mb": 500,
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda"
      },
      "seconds": null,
      "total_seconds": 2.2393481731414795,
      "stats": null,
      "ol-stats": {
        "Channel().cnt": 1,
        "Channel().ms-avg": 0,
        "Create().cnt": 1,
        "Create().ms-avg": 143,
        "Create()/acquire-cgroup.cnt": 1,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 1,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 138,
        "Create()/make-root-fs.cnt": 1,
        "Create()/make-root-fs.ms-avg": 4,
        "Destroy().cnt": 1,
        "Destroy().ms-avg": 2,
        "Destroy()/detach-root.cnt": 1,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 1,
        "Destroy()/kill-procs.ms-avg": 1,
        "Destroy()/remove-root.cnt": 1,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 1,
        "LambdaFunc.Invoke.ms-avg": 2097,
        "Status().cnt": 1,
        "Status().ms-avg": 0,
        "fresh-cgroup.cnt": 18,
        "fresh-cgroup.ms-avg": 0,
        "pull-lambda.cnt": 1,
        "pull-lambda.ms-avg": 2,
        "pull-package.cnt": 1,
        "pull-package.ms-avg": 2094,
        "web-request.cnt": 1,
        "web-request.ms-avg": 2097
      },
      "errors": [
        [
          "Traceback (most recent call last):",
          "  File \"test.py\", line 81, in wrapper",
          "    rv = fn(**kwargs)",
          "  File \"test.py\", line 225, in numpy_test",
          "    raise Exception(\"STATUS %d: %s\" % (r.status_code, r.text))",
          "Exception: STATUS 500: install lambda returned status 500, body 'Traceback (most recent call last):",
          "  File \"sock2.py\", line 42, in post",
          "    self.write(json.dumps(f.f(event)))",
          "  File \"/handler/f.py\", line 72, in f",
          "    assert(rc == 0)",
          "AssertionError",
          "'",
          "",
          ""
        ]
      ],
      "worker_tail": [
        "using existing OL directory at /root/open-lambda/test-dir",
        "2020/10/01 21:43:42 save PID 10295 to file /root/open-lambda/test-dir/worker/worker.pid",
        "2020/10/01 21:43:42 Start Lambda Server",
        "2020/10/01 21:43:42 Storage dir at /root/open-lambda/test-dir/worker/code",
        "2020/10/01 21:43:42 Storage dir at /root/open-lambda/test-dir/worker/scratch",
        "2020/10/01 21:43:42 Create SandboxPool",
        "2020/10/01 21:43:42 create /sys/fs/cgroup/blkio/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:42 create /sys/fs/cgroup/cpu/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:42 create /sys/fs/cgroup/devices/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:42 create /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:42 create /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:42 create /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:42 create /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:42 create /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:42 create /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:42 Storage dir at /root/open-lambda/test-dir/worker/root-sandboxes",
        "2020/10/01 21:43:42 Create DepTracer",
        "2020/10/01 21:43:42 Create PackagePuller",
        "2020/10/01 21:43:42 Create ImportCache",
        "2020/10/01 21:43:42 Import Cache Tree:",
        "2020/10/01 21:43:42 0    - ROOT",
        "2020/10/01 21:43:42 Create HandlerPuller",
        "2020/10/01 21:43:42 Setups Handlers",
        "2020/10/01 21:43:42 Execute handler by POSTing to localhost:5000/run/<lambda>",
        "2020/10/01 21:43:42 Get status by sending request to localhost:5000/status",
        "2020/10/01 21:43:42 start creating/serving CGs [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:42 Receive request to /pid",
        "2020/10/01 21:43:42 Receive request to /run/numpy15",
        "2020/10/01 21:43:42 debug: LambdaFunc.Task() runs on goroutine 18 [FUNC numpy15]",
        "2020/10/01 21:43:42 do pip install, using scratchDir='/root/open-lambda/test-dir/lambda/packages/numpy==1.15'",
        "2020/10/01 21:43:42 run pip install numpy==1.15 from a new Sandbox to /root/open-lambda/test-dir/lambda/packages/numpy==1.15 on host",
        "2020/10/01 21:43:42 <sandboxes>.Create(<nil>, true, /root/open-lambda/test-dir/worker/admin-lambdas/pip-install, /root/open-lambda/test-dir/lambda/packages/numpy==1.15, <installs=[], imports=[], mem-limit-mb=250>)=1... [SOCK POOL sandboxes]",
        "2020/10/01 21:43:42 use cgroup cg-1 [SOCK 1]",
        "sock2.py: started new process with args: sock2.py /host/bootstrap.py 9",
        "sock2.py: joined cgroup, close FD 3",
        "sock2.py: joined cgroup, close FD 4",
        "sock2.py: joined cgroup, close FD 5",
        "sock2.py: joined cgroup, close FD 6",
        "sock2.py: joined cgroup, close FD 7",
        "sock2.py: joined cgroup, close FD 8",
        "sock2.py: joined cgroup, close FD 9",
        "sock2.py: joined cgroup, close FD 10",
        "sock2.py: joined cgroup, close FD 11",
        "sock2.py: start web server on fd: 3",
        "2020/10/01 21:43:42 ...returns <SB 1>, <nil> [SOCK POOL sandboxes]",
        "2020/10/01 21:43:42 Channel() [SB 1]",
        "Collecting numpy==1.15",
        "  Downloading numpy-1.15.0-cp36-cp36m-manylinux1_x86_64.whl (13.9 MB)",
        "ERROR: Exception:",
        "Traceback (most recent call last):",
        "  File \"/usr/local/lib/python3.6/dist-packages/pip/_vendor/urllib3/response.py\", line 437, in _error_catcher",
        "    yield",
        "  File \"/usr/local/lib/python3.6/dist-packages/pip/_vendor/urllib3/response.py\", line 519, in read",
        "    data = self._fp.read(amt) if not fp_closed else b\"\"",
        "  File \"/usr/local/lib/python3.6/dist-packages/pip/_vendor/cachecontrol/filewrapper.py\", line 62, in read",
        "    data = self.__fp.read(amt)",
        "  File \"/usr/lib/python3.6/http/client.py\", line 459, in read",
        "    n = self.readinto(b)",
        "  File \"/usr/lib/python3.6/http/client.py\", line 503, in readinto",
        "    n = self.fp.readinto(b)",
        "  File \"/usr/lib/python3.6/socket.py\", line 586, in readinto",
        "    return self._sock.recv_into(b)",
        "  File \"/usr/lib/python3.6/ssl.py\", line 1012, in recv_into",
        "    return self.read(nbytes, buffer)",
        "  File \"/usr/lib/python3.6/ssl.py\", line 874, in read",
        "    return self._sslobj.read(len, buffer)",
        "  File \"/usr/lib/python3.6/ssl.py\", line 631, in read",
        "    v = self._sslobj.read(len, buffer)",
        "ConnectionResetError: [Errno 104] Connection reset by peer",
        "",
        "During handling of the above exception, another exception occurred:",
        "",
        "Traceback (most recent call last):",
        "  File \"/usr/local/lib/python3.6/dist-packages/pip/_internal/cli/base_command.py\", line 228, in _main",
        "    status = self.run(options, args)",
        "  File \"/usr/local/lib/python3.6/dist-packages/pip/_internal/cli/req_command.py\", line 182, in wrapper",
        "    return func(self, options, args)",
        "  File \"/usr/local/lib/python3.6/dist-packages/pip/_internal/commands/install.py\", line 324, in run",
        "    reqs, check_supported_wheels=not options.target_dir",
        "  File \"/usr/local/lib/python3.6/dist-packages/pip/_internal/resolution/legacy/resolver.py\", line 183, in resolve",
        "    discovered_reqs.extend(self._resolve_one(requirement_set, req))",
        "  File \"/usr/local/lib/python3.6/dist-packages/pip/_internal/resolution/legacy/resolver.py\", line 388, in _resolve_one",
        "    abstract_dist = self._get_abstract_dist_for(req_to_install)",
        "  File \"/usr/local/lib/python3.6/dist-packages/pip/_internal/resolution/legacy/resolver.py\", line 340, in _get_abstract_dist_for",
        "    abstract_dist = self.preparer.prepare_linked_requirement(req)",
        "  File \"/usr/local/lib/python3.6/dist-packages/pip/_internal/operations/prepare.py\", line 469, in prepare_linked_requirement",
        "    hashes=self._get_linked_req_hashes(req)",
        "  File \"/usr/local/lib/python3.6/dist-packages/pip/_internal/operations/prepare.py\", line 259, in unpack_url",
        "    hashes=hashes,",
        "  File \"/usr/local/lib/python3.6/dist-packages/pip/_internal/operations/prepare.py\", line 130, in get_http_url",
        "    link, downloader, temp_dir.path, hashes",
        "  File \"/usr/local/lib/python3.6/dist-packages/pip/_internal/operations/prepare.py\", line 282, in _download_http_url",
        "    for chunk in download.chunks:",
        "  File \"/usr/local/lib/python3.6/dist-packages/pip/_internal/cli/progress_bars.py\", line 168, in iter",
        "    for x in it:",
        "  File \"/usr/local/lib/python3.6/dist-packages/pip/_internal/network/utils.py\", line 88, in response_chunks",
        "    decode_content=False,",
        "  File \"/usr/local/lib/python3.6/dist-packages/pip/_vendor/urllib3/response.py\", line 576, in stream",
        "    data = self.read(amt=amt, decode_content=decode_content)",
        "  File \"/usr/local/lib/python3.6/dist-packages/pip/_vendor/urllib3/response.py\", line 541, in read",
        "    raise IncompleteRead(self._fp_bytes_read, self.length_remaining)",
        "  File \"/usr/lib/python3.6/contextlib.py\", line 99, in __exit__",
        "    self.gen.throw(type, value, traceback)",
        "  File \"/usr/local/lib/python3.6/dist-packages/pip/_vendor/urllib3/response.py\", line 455, in _error_catcher",
        "    raise ProtocolError(\"Connection broken: %r\" % e, e)",
        "pip._vendor.urllib3.exceptions.ProtocolError: (\"Connection broken: ConnectionResetError(104, 'Connection reset by peer')\", ConnectionResetError(104, 'Connection reset by peer'))",
        "pip install returned code 512",
        "2020/10/01 21:43:44 Status(0) [SB 1]",
        "2020/10/01 21:43:44 Destroy() [SB 1]",
        "2020/10/01 21:43:44 CG ref count decremented to 0 [SOCK 1]",
        "2020/10/01 21:43:44 waiting for 1 procs in cg-1 to die [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:44 killed PIDs [10309] in CG [SOCK 1]",
        "2020/10/01 21:43:44 unmount and remove dirs [SOCK 1]",
        "2020/10/01 21:43:44 Error checking for new lambda code: install lambda returned status 500, body 'Traceback (most recent call last):",
        "  File \"sock2.py\", line 42, in post",
        "    self.write(json.dumps(f.f(event)))",
        "  File \"/handler/f.py\", line 72, in f",
        "    assert(rc == 0)",
        "AssertionError",
        "' [FUNC numpy15]",
        "2020/10/01 21:43:44 received kill signal, cleaning up",
        "2020/10/01 21:43:44 Kill function: numpy15",
        "2020/10/01 21:43:44 Import Cache Tree:",
        "2020/10/01 21:43:44 0    - ROOT",
        "2020/10/01 21:43:44 make sure all memory is free [SOCK POOL sandboxes]",
        "2020/10/01 21:43:44 memory pool emptied [SOCK POOL sandboxes]",
        "2020/10/01 21:43:44 received shutdown request [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:44 empty queues and release CGs [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:44 remove /sys/fs/cgroup/blkio/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:44 remove /sys/fs/cgroup/cpu/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:44 remove /sys/fs/cgroup/devices/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:44 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:44 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:44 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:44 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:44 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:44 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:44 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2020/10/01 21:43:44 remove worker.pid",
        "2020/10/01 21:43:44 exiting",
        ""
      ]
    },
    {
      "test": "sock_churn",
      "params": {
        "seconds": 5,
        "baseline": 0,
        "fork": false,
        "procs": 1
      },
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "mem_mb": 50,
          "procs": 10
        },
        "pip_mirror": "",
        "import_cache_tree": "",
        "server_mode": "sock",
        "storage": {
          "scratch": "",
          "root": "private",
          "code": ""
        },
        "features": {
          "reuse_cgroups": false,
          "import_cache": true,
          "downsize_paused_mem": true
        },
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry_cache_ms": 5000,
        "registry": "/root/open-lambda/test-dir/registry",
        "sandbox": "sock",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "trace": {
          "evictor": false,
          "cgroups": false,
          "package": false,
          "memory": false
        },
        "mem_pool_mb": 500,
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda"
      },
      "seconds": 5.2337706089019775,
      "total_seconds": 5.356930255889893,
      "stats": {
        "sandboxes_per_sec": 6.4
      },
      "ol-stats": {
        "Create().cnt": 32,
        "Create().ms-avg": 141,
        "Create()/acquire-cgroup.cnt": 32,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 32,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fresh-proc.cnt": 32,
        "Create()/fresh-proc.ms-avg": 136,
        "Create()/make-root-fs.cnt": 32,
        "Create()/make-root-fs.ms-avg": 4,
        "Destroy().cnt": 64,
        "Destroy().ms-avg": 1,
        "Destroy()/detach-root.cnt": 32,
        "Destroy()/detach-root.ms-avg": 1,
        "Destroy()/kill-procs.cnt": 32,
        "Destroy()/kill-procs.ms-avg": 0,
        "Destroy()/remove-root.cnt": 32,
        "Destroy()/remove-root.ms-avg": 0,
        "fresh-cgroup.cnt": 49,
        "fresh-cgroup.ms-avg": 0
      },
      "errors": [],
      "worker_tail": [
        "2020/10/01 21:43:49 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:49 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:49 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:49 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:49 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:49 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:43:49 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2020/10/01 21:43:49 remove worker.pid",
        "2020/10/01 21:43:49 exiting",
        ""
      ]
    },
    {
      "test": "sock_churn",
      "params": {
        "seconds": 10,
        "baseline": 0,
        "fork": true,
        "procs": 1
      },
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "mem_mb": 50,
          "procs": 10
        },
        "pip_mirror": "",
        "import_cache_tree": "",
        "server_mode": "sock",
        "storage": {
          "scratch": "",
          "root": "private",
          "code": ""
        },
        "features": {
          "reuse_cgroups": false,
          "import_cache": true,
          "downsize_paused_mem": true
        },
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry_cache_ms": 5000,
        "registry": "/root/open-lambda/test-dir/registry",
        "sandbox": "sock",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "trace": {
          "evictor": false,
          "cgroups": false,
          "package": false,
          "memory": false
        },
        "mem_pool_mb": 500,
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda"
      },
      "seconds": 10.17084813117981,
      "total_seconds": 10.405054330825806,
      "stats": {
        "sandboxes_per_sec": 42.9
      },
      "ol-stats": {
        "Create().cnt": 430,
        "Create().ms-avg": 11,
        "Create()/acquire-cgroup.cnt": 430,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 430,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 429,
        "Create()/fork-proc.ms-avg": 6,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 133,
        "Create()/make-root-fs.cnt": 430,
        "Create()/make-root-fs.ms-avg": 3,
        "Destroy().cnt": 859,
        "Destroy().ms-avg": 1,
        "Destroy()/detach-root.cnt": 430,
        "Destroy()/detach-root.ms-avg": 1,
        "Destroy()/kill-procs.cnt": 430,
        "Destroy()/kill-procs.ms-avg": 0,
        "Destroy()/remove-root.cnt": 430,
        "Destroy()/remove-root.ms-avg": 0,
        "childExit().cnt": 429,
        "childExit().ms-avg": 0,
        "fork().cnt": 429,
        "fork().ms-avg": 6,
        "forkRequest.cnt": 429,
        "forkRequest.ms-avg": 6,
        "fresh-cgroup.cnt": 447,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 429,
        "move-to-cg-after-fork.ms-avg": 0
      },
      "errors": [],
      "worker_tail": [
        "2020/10/01 21:44:00 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:00 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:00 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:00 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:00 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:00 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:00 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2020/10/01 21:44:00 remove worker.pid",
        "2020/10/01 21:44:00 exiting",
        ""
      ]
    },
    {
      "test": "sock_churn",
      "params": {
        "seconds": 10,
        "baseline": 0,
        "fork": true,
        "procs": 15
      },
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "mem_mb": 50,
          "procs": 10
        },
        "pip_mirror": "",
        "import_cache_tree": "",
        "server_mode": "sock",
        "storage": {
          "scratch": "",
          "root": "private",
          "code": ""
        },
        "features": {
          "reuse_cgroups": false,
          "import_cache": true,
          "downsize_paused_mem": true
        },
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry_cache_ms": 5000,
        "registry": "/root/open-lambda/test-dir/registry",
        "sandbox": "sock",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "trace": {
          "evictor": false,
          "cgroups": false,
          "package": false,
          "memory": false
        },
        "mem_pool_mb": 500,
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda"
      },
      "seconds": 10.37511396408081,
      "total_seconds": 10.630906820297241,
      "stats": {
        "sandboxes_per_sec": 44.3
      },
      "ol-stats": {
        "Create().cnt": 444,
        "Create().ms-avg": 95,
        "Create()/acquire-cgroup.cnt": 444,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 444,
        "Create()/acquire-mem.ms-avg": 16,
        "Create()/fork-proc.cnt": 443,
        "Create()/fork-proc.ms-avg": 75,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 133,
        "Create()/make-root-fs.cnt": 444,
        "Create()/make-root-fs.ms-avg": 1,
        "Destroy().cnt": 887,
        "Destroy().ms-avg": 59,
        "Destroy()/detach-root.cnt": 444,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 444,
        "Destroy()/kill-procs.ms-avg": 13,
        "Destroy()/remove-root.cnt": 444,
        "Destroy()/remove-root.ms-avg": 0,
        "childExit().cnt": 443,
        "childExit().ms-avg": 79,
        "fork().cnt": 443,
        "fork().ms-avg": 75,
        "forkRequest.cnt": 443,
        "forkRequest.ms-avg": 10,
        "fresh-cgroup.cnt": 461,
        "fresh-cgroup.ms-avg": 4,
        "move-to-cg-after-fork.cnt": 443,
        "move-to-cg-after-fork.ms-avg": 2
      },
      "errors": [],
      "worker_tail": [
        "2020/10/01 21:44:10 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:10 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:10 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:10 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:10 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:10 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:10 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2020/10/01 21:44:10 remove worker.pid",
        "2020/10/01 21:44:10 exiting",
        ""
      ]
    },
    {
      "test": "sock_churn",
      "params": {
        "seconds": 10,
        "baseline": 32,
        "fork": true,
        "procs": 1
      },
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "mem_mb": 50,
          "procs": 10
        },
        "pip_mirror": "",
        "import_cache_tree": "",
        "server_mode": "sock",
        "storage": {
          "scratch": "",
          "root": "private",
          "code": ""
        },
        "features": {
          "reuse_cgroups": false,
          "import_cache": true,
          "downsize_paused_mem": true
        },
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry_cache_ms": 5000,
        "registry": "/root/open-lambda/test-dir/registry",
        "sandbox": "sock",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "trace": {
          "evictor": false,
          "cgroups": false,
          "package": false,
          "memory": false
        },
        "mem_pool_mb": 500,
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda"
      },
      "seconds": 10.80896806716919,
      "total_seconds": 11.17996621131897,
      "stats": {
        "sandboxes_per_sec": 40.4
      },
      "ol-stats": {
        "Create().cnt": 437,
        "Create().ms-avg": 7,
        "Create()/acquire-cgroup.cnt": 437,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 437,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 436,
        "Create()/fork-proc.ms-avg": 7,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 133,
        "Create()/make-root-fs.cnt": 437,
        "Create()/make-root-fs.ms-avg": 0,
        "Destroy().cnt": 841,
        "Destroy().ms-avg": 0,
        "Destroy()/detach-root.cnt": 437,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 437,
        "Destroy()/kill-procs.ms-avg": 0,
        "Destroy()/remove-root.cnt": 437,
        "Destroy()/remove-root.ms-avg": 0,
        "Pause().cnt": 32,
        "Pause().ms-avg": 0,
        "childExit().cnt": 436,
        "childExit().ms-avg": 0,
        "fork().cnt": 436,
        "fork().ms-avg": 7,
        "forkRequest.cnt": 436,
        "forkRequest.ms-avg": 6,
        "fresh-cgroup.cnt": 454,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 436,
        "move-to-cg-after-fork.ms-avg": 0
      },
      "errors": [],
      "worker_tail": [
        "2020/10/01 21:44:21 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:21 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:21 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:21 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:21 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:21 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:21 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2020/10/01 21:44:21 remove worker.pid",
        "2020/10/01 21:44:21 exiting",
        ""
      ]
    },
    {
      "test": "sock_churn",
      "params": {
        "seconds": 10,
        "baseline": 32,
        "fork": true,
        "procs": 15
      },
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "mem_mb": 50,
          "procs": 10
        },
        "pip_mirror": "",
        "import_cache_tree": "",
        "server_mode": "sock",
        "storage": {
          "scratch": "",
          "root": "private",
          "code": ""
        },
        "features": {
          "reuse_cgroups": false,
          "import_cache": true,
          "downsize_paused_mem": true
        },
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry_cache_ms": 5000,
        "registry": "/root/open-lambda/test-dir/registry",
        "sandbox": "sock",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "trace": {
          "evictor": false,
          "cgroups": false,
          "package": false,
          "memory": false
        },
        "mem_pool_mb": 500,
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda"
      },
      "seconds": 11.101701259613037,
      "total_seconds": 11.457571506500244,
      "stats": {
        "sandboxes_per_sec": 38.7
      },
      "ol-stats": {
        "Create().cnt": 420,
        "Create().ms-avg": 134,
        "Create()/acquire-cgroup.cnt": 420,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 420,
        "Create()/acquire-mem.ms-avg": 75,
        "Create()/fork-proc.cnt": 419,
        "Create()/fork-proc.ms-avg": 55,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 161,
        "Create()/make-root-fs.cnt": 420,
        "Create()/make-root-fs.ms-avg": 1,
        "Destroy().cnt": 807,
        "Destroy().ms-avg": 49,
        "Destroy()/detach-root.cnt": 420,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 420,
        "Destroy()/kill-procs.ms-avg": 13,
        "Destroy()/remove-root.cnt": 420,
        "Destroy()/remove-root.ms-avg": 0,
        "Pause().cnt": 32,
        "Pause().ms-avg": 0,
        "childExit().cnt": 419,
        "childExit().ms-avg": 57,
        "fork().cnt": 419,
        "fork().ms-avg": 55,
        "forkRequest.cnt": 419,
        "forkRequest.ms-avg": 11,
        "fresh-cgroup.cnt": 437,
        "fresh-cgroup.ms-avg": 4,
        "move-to-cg-after-fork.cnt": 419,
        "move-to-cg-after-fork.ms-avg": 3
      },
      "errors": [],
      "worker_tail": [
        "2020/10/01 21:44:33 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:33 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:33 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:33 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:33 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:33 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:33 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2020/10/01 21:44:33 remove worker.pid",
        "2020/10/01 21:44:33 exiting",
        ""
      ]
    },
    {
      "test": "update_code",
      "params": {},
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "limits": {
          "swappiness": 0,
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 250
        },
        "pip_mirror": "",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "sandbox": "sock",
        "features": {
          "reuse_cgroups": false,
          "import_cache": true,
          "downsize_paused_mem": true
        },
        "registry_cache_ms": 3000,
        "registry": "/tmp/tmpvnk2n3i5",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "docker_runtime": "",
        "storage": {
          "scratch": "",
          "root": "private",
          "code": ""
        },
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "mem_pool_mb": 500,
        "sandbox_config": {},
        "trace": {
          "evictor": false,
          "cgroups": false,
          "memory": false,
          "package": false
        }
      },
      "seconds": 6.03501296043396,
      "total_seconds": 6.274040222167969,
      "stats": null,
      "ol-stats": {
        "Channel().cnt": 3,
        "Channel().ms-avg": 0,
        "Create().cnt": 4,
        "Create().ms-avg": 165,
        "Create()/acquire-cgroup.cnt": 4,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 4,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 3,
        "Create()/fork-proc.ms-avg": 8,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 626,
        "Create()/make-root-fs.cnt": 4,
        "Create()/make-root-fs.ms-avg": 2,
        "Destroy().cnt": 4,
        "Destroy().ms-avg": 3,
        "Destroy()/detach-root.cnt": 4,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 4,
        "Destroy()/kill-procs.ms-avg": 2,
        "Destroy()/remove-root.cnt": 4,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 1580,
        "LambdaFunc.Invoke.ms-avg": 1,
        "Pause().cnt": 1583,
        "Pause().ms-avg": 0,
        "ServeHTTP.cnt": 1580,
        "ServeHTTP.ms-avg": 0,
        "Unpause().cnt": 1579,
        "Unpause().ms-avg": 0,
        "childExit().cnt": 3,
        "childExit().ms-avg": 0,
        "fork().cnt": 3,
        "fork().ms-avg": 8,
        "forkRequest.cnt": 3,
        "forkRequest.ms-avg": 6,
        "fresh-cgroup.cnt": 21,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 3,
        "move-to-cg-after-fork.ms-avg": 1,
        "pull-lambda.cnt": 3,
        "pull-lambda.ms-avg": 2,
        "web-request.cnt": 1580,
        "web-request.ms-avg": 1
      },
      "errors": [],
      "worker_tail": [
        "2020/10/01 21:44:39 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:39 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:39 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:39 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:39 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:39 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:39 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2020/10/01 21:44:39 remove worker.pid",
        "2020/10/01 21:44:39 exiting",
        ""
      ]
    },
    {
      "test": "stress_one_lambda",
      "params": {
        "seconds": 15,
        "procs": 1
      },
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "mem_mb": 50,
          "procs": 10
        },
        "pip_mirror": "",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "storage": {
          "scratch": "",
          "root": "private",
          "code": ""
        },
        "features": {
          "reuse_cgroups": false,
          "import_cache": true,
          "downsize_paused_mem": true
        },
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry_cache_ms": 5000,
        "registry": "/root/open-lambda/test-registry",
        "sandbox": "sock",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "trace": {
          "evictor": false,
          "cgroups": false,
          "package": false,
          "memory": false
        },
        "mem_pool_mb": 500,
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda"
      },
      "seconds": 15.04340124130249,
      "total_seconds": 15.273571491241455,
      "stats": {
        "reqs_per_sec": 288.06666666666666
      },
      "ol-stats": {
        "Channel().cnt": 3,
        "Channel().ms-avg": 0,
        "Create().cnt": 4,
        "Create().ms-avg": 43,
        "Create()/acquire-cgroup.cnt": 4,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 4,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 3,
        "Create()/fork-proc.ms-avg": 7,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 135,
        "Create()/make-root-fs.cnt": 4,
        "Create()/make-root-fs.ms-avg": 3,
        "Destroy().cnt": 4,
        "Destroy().ms-avg": 3,
        "Destroy()/detach-root.cnt": 4,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 4,
        "Destroy()/kill-procs.ms-avg": 2,
        "Destroy()/remove-root.cnt": 4,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 4321,
        "LambdaFunc.Invoke.ms-avg": 1,
        "Pause().cnt": 4324,
        "Pause().ms-avg": 0,
        "ServeHTTP.cnt": 4321,
        "ServeHTTP.ms-avg": 0,
        "Unpause().cnt": 4320,
        "Unpause().ms-avg": 0,
        "childExit().cnt": 3,
        "childExit().ms-avg": 0,
        "fork().cnt": 3,
        "fork().ms-avg": 7,
        "forkRequest.cnt": 3,
        "forkRequest.ms-avg": 6,
        "fresh-cgroup.cnt": 21,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 3,
        "move-to-cg-after-fork.ms-avg": 0,
        "pull-lambda.cnt": 3,
        "pull-lambda.ms-avg": 1,
        "web-request.cnt": 4321,
        "web-request.ms-avg": 1
      },
      "errors": [],
      "worker_tail": [
        "2020/10/01 21:44:54 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:54 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:54 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:54 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:54 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:54 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:44:54 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2020/10/01 21:44:54 remove worker.pid",
        "2020/10/01 21:44:54 exiting",
        ""
      ]
    },
    {
      "test": "stress_one_lambda",
      "params": {
        "seconds": 15,
        "procs": 2
      },
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "mem_mb": 50,
          "procs": 10
        },
        "pip_mirror": "",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "storage": {
          "scratch": "",
          "root": "private",
          "code": ""
        },
        "features": {
          "reuse_cgroups": false,
          "import_cache": true,
          "downsize_paused_mem": true
        },
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry_cache_ms": 5000,
        "registry": "/root/open-lambda/test-registry",
        "sandbox": "sock",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "trace": {
          "evictor": false,
          "cgroups": false,
          "package": false,
          "memory": false
        },
        "mem_pool_mb": 500,
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda"
      },
      "seconds": 15.046984195709229,
      "total_seconds": 15.277143955230713,
      "stats": {
        "reqs_per_sec": 283.0
      },
      "ol-stats": {
        "Channel().cnt": 3,
        "Channel().ms-avg": 0,
        "Create().cnt": 4,
        "Create().ms-avg": 44,
        "Create()/acquire-cgroup.cnt": 4,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 4,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 3,
        "Create()/fork-proc.ms-avg": 8,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 135,
        "Create()/make-root-fs.cnt": 4,
        "Create()/make-root-fs.ms-avg": 3,
        "Destroy().cnt": 4,
        "Destroy().ms-avg": 3,
        "Destroy()/detach-root.cnt": 4,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 4,
        "Destroy()/kill-procs.ms-avg": 2,
        "Destroy()/remove-root.cnt": 4,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 4245,
        "LambdaFunc.Invoke.ms-avg": 1,
        "Pause().cnt": 3757,
        "Pause().ms-avg": 0,
        "ServeHTTP.cnt": 4245,
        "ServeHTTP.ms-avg": 0,
        "Unpause().cnt": 3753,
        "Unpause().ms-avg": 0,
        "childExit().cnt": 3,
        "childExit().ms-avg": 0,
        "fork().cnt": 3,
        "fork().ms-avg": 8,
        "forkRequest.cnt": 3,
        "forkRequest.ms-avg": 7,
        "fresh-cgroup.cnt": 21,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 3,
        "move-to-cg-after-fork.ms-avg": 0,
        "pull-lambda.cnt": 3,
        "pull-lambda.ms-avg": 1,
        "web-request.cnt": 4245,
        "web-request.ms-avg": 1
      },
      "errors": [],
      "worker_tail": [
        "2020/10/01 21:45:10 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:45:10 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:45:10 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:45:10 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:45:10 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:45:10 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:45:10 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2020/10/01 21:45:10 remove worker.pid",
        "2020/10/01 21:45:10 exiting",
        ""
      ]
    },
    {
      "test": "stress_one_lambda",
      "params": {
        "seconds": 15,
        "procs": 8
      },
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "mem_mb": 50,
          "procs": 10
        },
        "pip_mirror": "",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "storage": {
          "scratch": "",
          "root": "private",
          "code": ""
        },
        "features": {
          "reuse_cgroups": false,
          "import_cache": true,
          "downsize_paused_mem": true
        },
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry_cache_ms": 5000,
        "registry": "/root/open-lambda/test-registry",
        "sandbox": "sock",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "trace": {
          "evictor": false,
          "cgroups": false,
          "package": false,
          "memory": false
        },
        "mem_pool_mb": 500,
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda"
      },
      "seconds": 15.067862510681152,
      "total_seconds": 15.297250270843506,
      "stats": {
        "reqs_per_sec": 254.6
      },
      "ol-stats": {
        "Channel().cnt": 3,
        "Channel().ms-avg": 0,
        "Create().cnt": 4,
        "Create().ms-avg": 43,
        "Create()/acquire-cgroup.cnt": 4,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 4,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 3,
        "Create()/fork-proc.ms-avg": 7,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 138,
        "Create()/make-root-fs.cnt": 4,
        "Create()/make-root-fs.ms-avg": 3,
        "Destroy().cnt": 4,
        "Destroy().ms-avg": 4,
        "Destroy()/detach-root.cnt": 4,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 4,
        "Destroy()/kill-procs.ms-avg": 2,
        "Destroy()/remove-root.cnt": 4,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 3819,
        "LambdaFunc.Invoke.ms-avg": 2,
        "Pause().cnt": 2877,
        "Pause().ms-avg": 0,
        "ServeHTTP.cnt": 3819,
        "ServeHTTP.ms-avg": 0,
        "Unpause().cnt": 2873,
        "Unpause().ms-avg": 0,
        "childExit().cnt": 3,
        "childExit().ms-avg": 1,
        "fork().cnt": 3,
        "fork().ms-avg": 7,
        "forkRequest.cnt": 3,
        "forkRequest.ms-avg": 6,
        "fresh-cgroup.cnt": 21,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 3,
        "move-to-cg-after-fork.ms-avg": 0,
        "pull-lambda.cnt": 3,
        "pull-lambda.ms-avg": 3,
        "web-request.cnt": 3819,
        "web-request.ms-avg": 2
      },
      "errors": [],
      "worker_tail": [
        "2020/10/01 21:45:25 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:45:25 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:45:25 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:45:25 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:45:25 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:45:25 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:45:25 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2020/10/01 21:45:25 remove worker.pid",
        "2020/10/01 21:45:25 exiting",
        ""
      ]
    },
    {
      "test": "call_each_once_exec",
      "params": {
        "lambda_count": 100,
        "alloc_mb": 1
      },
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "limits": {
          "swappiness": 0,
          "mem_mb": 50,
          "procs": 10,
          "installer_mem_mb": 250
        },
        "pip_mirror": "",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "sandbox": "sock",
        "features": {
          "reuse_cgroups": true,
          "import_cache": true,
          "downsize_paused_mem": true
        },
        "registry_cache_ms": 5000,
        "registry": "/tmp/tmpi3j_k2z4",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "docker_runtime": "",
        "storage": {
          "scratch": "",
          "root": "private",
          "code": ""
        },
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda",
        "mem_pool_mb": 500,
        "sandbox_config": {},
        "trace": {
          "evictor": false,
          "cgroups": false,
          "memory": false,
          "package": false
        }
      },
      "seconds": 2.9594693183898926,
      "total_seconds": 3.402841567993164,
      "stats": {
        "reqs_per_sec": 33.79015482325944
      },
      "ol-stats": {
        "Channel().cnt": 100,
        "Channel().ms-avg": 0,
        "Create().cnt": 101,
        "Create().ms-avg": 7,
        "Create()/acquire-cgroup.cnt": 101,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 101,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 100,
        "Create()/fork-proc.ms-avg": 5,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 137,
        "Create()/make-root-fs.cnt": 101,
        "Create()/make-root-fs.ms-avg": 0,
        "Destroy().cnt": 156,
        "Destroy().ms-avg": 4,
        "Destroy()/detach-root.cnt": 101,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 101,
        "Destroy()/kill-procs.ms-avg": 3,
        "Destroy()/remove-root.cnt": 101,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 100,
        "LambdaFunc.Invoke.ms-avg": 25,
        "Pause().cnt": 200,
        "Pause().ms-avg": 10,
        "ServeHTTP.cnt": 100,
        "ServeHTTP.ms-avg": 6,
        "Unpause().cnt": 99,
        "Unpause().ms-avg": 0,
        "childExit().cnt": 100,
        "childExit().ms-avg": 3,
        "evict.cnt": 55,
        "evict.ms-avg": 11,
        "fork().cnt": 100,
        "fork().ms-avg": 5,
        "forkRequest.cnt": 100,
        "forkRequest.ms-avg": 4,
        "fresh-cgroup.cnt": 66,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 100,
        "move-to-cg-after-fork.ms-avg": 0,
        "pull-lambda.cnt": 100,
        "pull-lambda.ms-avg": 3,
        "web-request.cnt": 100,
        "web-request.ms-avg": 25
      },
      "errors": [],
      "worker_tail": [
        "2020/10/01 21:45:28 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:45:28 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:45:28 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:45:28 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:45:28 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:45:28 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:45:28 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2020/10/01 21:45:28 remove worker.pid",
        "2020/10/01 21:45:28 exiting",
        ""
      ]
    },
    {
      "test": "call_each_once_exec",
      "params": {
        "lambda_count": 1000,
        "alloc_mb": 10
      },
      "pass": true,
      "conf": {
        "worker_port": "5000",
        "limits": {
          "installer_mem_mb": 250,
          "swappiness": 0,
          "mem_mb": 50,
          "procs": 10
        },
        "pip_mirror": "",
        "import_cache_tree": "",
        "server_mode": "lambda",
        "storage": {
          "scratch": "",
          "root": "private",
          "code": ""
        },
        "features": {
          "reuse_cgroups": true,
          "import_cache": true,
          "downsize_paused_mem": true
        },
        "docker_runtime": "",
        "worker_dir": "/root/open-lambda/test-dir/worker",
        "registry_cache_ms": 5000,
        "registry": "/tmp/tmpxo3eu_mk",
        "sandbox": "sock",
        "Pkgs_dir": "/root/open-lambda/test-dir/lambda/packages",
        "trace": {
          "evictor": false,
          "cgroups": false,
          "package": false,
          "memory": false
        },
        "mem_pool_mb": 500,
        "sandbox_config": {},
        "SOCK_base_path": "/root/open-lambda/test-dir/lambda"
      },
      "seconds": 44.28638553619385,
      "total_seconds": 44.83769178390503,
      "stats": {
        "reqs_per_sec": 22.580315791199826
      },
      "ol-stats": {
        "Channel().cnt": 1000,
        "Channel().ms-avg": 0,
        "Create().cnt": 1001,
        "Create().ms-avg": 8,
        "Create()/acquire-cgroup.cnt": 1001,
        "Create()/acquire-cgroup.ms-avg": 0,
        "Create()/acquire-mem.cnt": 1001,
        "Create()/acquire-mem.ms-avg": 0,
        "Create()/fork-proc.cnt": 1000,
        "Create()/fork-proc.ms-avg": 6,
        "Create()/fresh-proc.cnt": 1,
        "Create()/fresh-proc.ms-avg": 143,
        "Create()/make-root-fs.cnt": 1001,
        "Create()/make-root-fs.ms-avg": 1,
        "Destroy().cnt": 1981,
        "Destroy().ms-avg": 5,
        "Destroy()/detach-root.cnt": 1001,
        "Destroy()/detach-root.ms-avg": 0,
        "Destroy()/kill-procs.cnt": 1001,
        "Destroy()/kill-procs.ms-avg": 7,
        "Destroy()/remove-root.cnt": 1001,
        "Destroy()/remove-root.ms-avg": 0,
        "LambdaFunc.Invoke.cnt": 1000,
        "LambdaFunc.Invoke.ms-avg": 40,
        "Pause().cnt": 2000,
        "Pause().ms-avg": 9,
        "ServeHTTP.cnt": 1000,
        "ServeHTTP.ms-avg": 19,
        "Unpause().cnt": 999,
        "Unpause().ms-avg": 0,
        "childExit().cnt": 1000,
        "childExit().ms-avg": 3,
        "evict.cnt": 980,
        "evict.ms-avg": 12,
        "fork().cnt": 1000,
        "fork().ms-avg": 6,
        "forkRequest.cnt": 1000,
        "forkRequest.ms-avg": 5,
        "fresh-cgroup.cnt": 40,
        "fresh-cgroup.ms-avg": 0,
        "move-to-cg-after-fork.cnt": 1000,
        "move-to-cg-after-fork.ms-avg": 0,
        "pull-lambda.cnt": 1000,
        "pull-lambda.ms-avg": 2,
        "web-request.cnt": 1000,
        "web-request.ms-avg": 40
      },
      "errors": [],
      "worker_tail": [
        "2020/10/01 21:46:13 remove /sys/fs/cgroup/freezer/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:46:13 remove /sys/fs/cgroup/hugetlb/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:46:13 remove /sys/fs/cgroup/memory/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:46:13 remove /sys/fs/cgroup/perf_event/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:46:13 remove /sys/fs/cgroup/systemd/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:46:13 remove /sys/fs/cgroup/pids/test-dir-sandboxes [CGROUP POOL test-dir-sandboxes]",
        "2020/10/01 21:46:13 save stats to /root/open-lambda/test-dir/worker/stats.json",
        "2020/10/01 21:46:13 remove worker.pid",
        "2020/10/01 21:46:13 exiting",
        ""
      ]
    }
  ],
  "passed": 17,
  "failed": 1,
  "seconds": 258.2047710418701
}