From 0356c7ef32dfcc05b1f5e7f242a022883a9ba816 Mon Sep 17 00:00:00 2001 From: Ayush Ranjan Date: Wed, 16 Sep 2020 13:46:32 -0700 Subject: [PATCH] [runtime tests] Add documentation. Added a README describing what these tests are, how they work and how to run them locally. Also reorganized the exclude files into a directory. PiperOrigin-RevId: 332079697 --- test/runtimes/BUILD | 10 +-- test/runtimes/README.md | 62 +++++++++++++++++++ .../go1.12.csv} | 0 .../java11.csv} | 0 .../nodejs12.4.0.csv} | 0 .../php7.3.6.csv} | 0 .../python3.7.3.csv} | 0 7 files changed, 67 insertions(+), 5 deletions(-) create mode 100644 test/runtimes/README.md rename test/runtimes/{exclude_go1.12.csv => exclude/go1.12.csv} (100%) rename test/runtimes/{exclude_java11.csv => exclude/java11.csv} (100%) rename test/runtimes/{exclude_nodejs12.4.0.csv => exclude/nodejs12.4.0.csv} (100%) rename test/runtimes/{exclude_php7.3.6.csv => exclude/php7.3.6.csv} (100%) rename test/runtimes/{exclude_python3.7.3.csv => exclude/python3.7.3.csv} (100%) diff --git a/test/runtimes/BUILD b/test/runtimes/BUILD index 066338ee3..22b526f59 100644 --- a/test/runtimes/BUILD +++ b/test/runtimes/BUILD @@ -5,7 +5,7 @@ package(licenses = ["notice"]) runtime_test( name = "go1.12", - exclude_file = "exclude_go1.12.csv", + exclude_file = "exclude/go1.12.csv", lang = "go", shard_count = 8, ) @@ -13,28 +13,28 @@ runtime_test( runtime_test( name = "java11", batch = 100, - exclude_file = "exclude_java11.csv", + exclude_file = "exclude/java11.csv", lang = "java", shard_count = 16, ) runtime_test( name = "nodejs12.4.0", - exclude_file = "exclude_nodejs12.4.0.csv", + exclude_file = "exclude/nodejs12.4.0.csv", lang = "nodejs", shard_count = 8, ) runtime_test( name = "php7.3.6", - exclude_file = "exclude_php7.3.6.csv", + exclude_file = "exclude/php7.3.6.csv", lang = "php", shard_count = 8, ) runtime_test( name = "python3.7.3", - exclude_file = "exclude_python3.7.3.csv", + exclude_file = "exclude/python3.7.3.csv", lang = "python", shard_count = 8, ) diff --git a/test/runtimes/README.md b/test/runtimes/README.md new file mode 100644 index 000000000..9dda1a728 --- /dev/null +++ b/test/runtimes/README.md @@ -0,0 +1,62 @@ +# gVisor Runtime Tests + +App Engine uses gvisor to sandbox application containers. The runtime tests aim +to test `runsc` compatibility with these +[standard runtimes](https://cloud.google.com/appengine/docs/standard/runtimes). +The test itself runs the language-defined tests inside the sandboxed standard +runtime container. + +Note: [Ruby runtime](https://cloud.google.com/appengine/docs/standard/ruby) is +currently in beta mode and so we do not run tests for it yet. + +### Testing Locally + +To run runtime tests individually from a given runtime, use the following table. + +Language | Version | Download Image | Run Test(s) +-------- | ------- | ------------------------------------------- | ----------- +Go | 1.12 | `make -C images load-runtimes_go1.12` | If the test name ends with `.go`, it is an on-disk test:
`docker run --runtime=runsc -it gvisor.dev/images/runtimes/go1.12 ( cd /usr/local/go/test ; go run run.go -v -- ... )`
Otherwise it is a tool test:
`docker run --runtime=runsc -it gvisor.dev/images/runtimes/go1.12 go tool dist test -v -no-rebuild ^TEST1$\|^TEST2$...` +Java | 11 | `make -C images load-runtimes_java11` | `docker run --runtime=runsc -it gvisor.dev/images/runtimes/java11 jtreg -agentvm -dir:/root/test/jdk -noreport -timeoutFactor:20 -verbose:summary ...` +NodeJS | 12.4.0 | `make -C images load-runtimes_nodejs12.4.0` | `docker run --runtime=runsc -it gvisor.dev/images/runtimes/nodejs12.4.0 python tools/test.py --timeout=180 ...` +Php | 7.3.6 | `make -C images load-runtimes_php7.3.6` | `docker run --runtime=runsc -it gvisor.dev/images/runtimes/php7.3.6 make test "TESTS=..."` +Python | 3.7.3 | `make -C images load-runtimes_python3.7.3` | `docker run --runtime=runsc -it gvisor.dev/images/runtimes/python3.7.3 ./python -m test ...` + +To run an entire runtime test locally, use the following table. + +Note: java runtime test take 1+ hours with 16 cores. + +Language | Version | Running the test suite +-------- | ------- | ---------------------------------------- +Go | 1.12 | `make go1.12-runtime-tests{_vfs2}` +Java | 11 | `make java11-runtime-tests{_vfs2}` +NodeJS | 12.4.0 | `make nodejs12.4.0-runtime-tests{_vfs2}` +Php | 7.3.6 | `make php7.3.6-runtime-tests{_vfs2}` +Python | 3.7.3 | `make python3.7.3-runtime-tests{_vfs2}` + +#### Clean Up + +Sometimes when runtime tests fail or when the testing container itself crashes +unexpectedly, the containers are not removed or sometimes do not even exit. This +can cause some docker commands like `docker system prune` to hang forever. + +Here are some helpful commands (should be executed in order): + +```bash +docker ps -a # Lists all docker processes; useful when investigating hanging containers. +docker kill $(docker ps -a -q) # Kills all running containers. +docker rm $(docker ps -a -q) # Removes all exited containers. +docker system prune # Remove unused data. +``` + +### Testing Infrastructure + +There are 3 components to this tests infrastructure: + +- [`runner`](runner) - This is the test entrypoint. This is the binary is + invoked by `bazel test`. The runner spawns the target runtime container + using `runsc` and then copies over the `proctor` binary into the container. +- [`proctor`](proctor) - This binary acts as our agent inside the container + which communicates with the runner and actually executes tests. +- [`exclude`](exclude) - Holds a CSV file for each language runtime containing + the full path of tests that should be excluded from running along with a + reason for exclusion. diff --git a/test/runtimes/exclude_go1.12.csv b/test/runtimes/exclude/go1.12.csv similarity index 100% rename from test/runtimes/exclude_go1.12.csv rename to test/runtimes/exclude/go1.12.csv diff --git a/test/runtimes/exclude_java11.csv b/test/runtimes/exclude/java11.csv similarity index 100% rename from test/runtimes/exclude_java11.csv rename to test/runtimes/exclude/java11.csv diff --git a/test/runtimes/exclude_nodejs12.4.0.csv b/test/runtimes/exclude/nodejs12.4.0.csv similarity index 100% rename from test/runtimes/exclude_nodejs12.4.0.csv rename to test/runtimes/exclude/nodejs12.4.0.csv diff --git a/test/runtimes/exclude_php7.3.6.csv b/test/runtimes/exclude/php7.3.6.csv similarity index 100% rename from test/runtimes/exclude_php7.3.6.csv rename to test/runtimes/exclude/php7.3.6.csv diff --git a/test/runtimes/exclude_python3.7.3.csv b/test/runtimes/exclude/python3.7.3.csv similarity index 100% rename from test/runtimes/exclude_python3.7.3.csv rename to test/runtimes/exclude/python3.7.3.csv