netbird/gvisor
0
0
Fork 0
mirror of https://github.com/netbirdio/gvisor.git synced 2026-05-22 17:12:49 -07:00
Code Issues Packages Projects Releases Wiki Activity

Runtime tests: Add flakiness detection and per-test timeout options.

Browse Source 
This adds flags and env variables for the following settings to runtime tests:

- `per_test_timeout`: A per-test timeout which can be shorter
  than the batch timeout. Useful to cap the duration of tests
  which flake by getting stuck (as is the case for
  `bug60120.phpt`)
- `runs_per_test`: Number of times to run each test (useful to
  detect flakes).
- `flaky_is_error`: Controls whether a flaky test is considered
  passing or failing for batch error code purposes. Useful when
  either diagnosing a flaky test, or diagnosing a
  consistently-failing test while bypassing flaky others.
- `flaky_short_circuit`: If a test is found to be flaky, declare
  it as such immediately, rather than waiting for the rest of the
  `--runs_per_test` to finish. Speeds up bisecting flaky test, at
  the cost of flakiness percentage accuracy.

PiperOrigin-RevId: 466791296
This commit is contained in:
Etienne Perot
2022-08-10 14:50:32 -07:00
committed by gVisor bot
parent 1faac756ee
commit c9cb22a16c
8 changed files with 244 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+9 -2
View File
@@ -236,13 +236,20 @@ packetimpact-tests:
@$(call test,--jobs=HOST_CPUS*3 --local_test_jobs=HOST_CPUS*3 //test/packetimpact/tests:all_tests)
.PHONY: packetimpact-tests
# Extra configuration options for runtime tests.
RUNTIME_TESTS_FILTER ?=
RUNTIME_TESTS_PER_TEST_TIMEOUT ?= 20m
RUNTIME_TESTS_RUNS_PER_TEST ?= 1
RUNTIME_TESTS_FLAKY_IS_ERROR ?= true
RUNTIME_TESTS_FLAKY_SHORT_CIRCUIT ?= true
%-runtime-tests: load-runtimes_% $(RUNTIME_BIN)
@$(call install_runtime,$(RUNTIME),--watchdog-action=panic)
@$(call test_runtime,$(RUNTIME),--test_timeout=1800 //test/runtimes:$*)
@$(call test_runtime,$(RUNTIME),--test_timeout=1800 --test_env=RUNTIME_TESTS_FILTER=$(RUNTIME_TESTS_FILTER) --test_env=RUNTIME_TESTS_PER_TEST_TIMEOUT=$(RUNTIME_TESTS_PER_TEST_TIMEOUT) --test_env=RUNTIME_TESTS_RUNS_PER_TEST=$(RUNTIME_TESTS_RUNS_PER_TEST) --test_env=RUNTIME_TESTS_FLAKY_IS_ERROR=$(RUNTIME_TESTS_FLAKY_IS_ERROR) --test_env=RUNTIME_TESTS_FLAKY_SHORT_CIRCUIT=$(RUNTIME_TESTS_FLAKY_SHORT_CIRCUIT) //test/runtimes:$*)
%-runtime-tests_lisafs: load-runtimes_% $(RUNTIME_BIN)
@$(call install_runtime,$(RUNTIME), --lisafs --watchdog-action=panic)
@$(call test_runtime,$(RUNTIME),--test_timeout=1800 //test/runtimes:$*)
@$(call test_runtime,$(RUNTIME),--test_timeout=1800 --test_env=RUNTIME_TESTS_FILTER=$(RUNTIME_TESTS_FILTER) --test_env=RUNTIME_TESTS_PER_TEST_TIMEOUT=$(RUNTIME_TESTS_PER_TEST_TIMEOUT) --test_env=RUNTIME_TESTS_RUNS_PER_TEST=$(RUNTIME_TESTS_RUNS_PER_TEST) --test_env=RUNTIME_TESTS_FLAKY_IS_ERROR=$(RUNTIME_TESTS_FLAKY_IS_ERROR) --test_env=RUNTIME_TESTS_FLAKY_SHORT_CIRCUIT=$(RUNTIME_TESTS_FLAKY_SHORT_CIRCUIT) //test/runtimes:$*)
do-tests: $(RUNTIME_BIN)
@$(RUNTIME_BIN) --rootless do true
pkg/test/testutil/testutil.go
+34 -6
View File
@@ -49,14 +49,26 @@ import (
)
var (
checkpoint = flag.Bool("checkpoint", boolFromEnv("CHECKPOINT", true), "control checkpoint/restore support")
partition = flag.Int("partition", intFromEnv("PARTITION", 1), "partition number, this is 1-indexed")
totalPartitions = flag.Int("total_partitions", intFromEnv("TOTAL_PARTITIONS", 1), "total number of partitions")
isRunningWithHostNet = flag.Bool("hostnet", boolFromEnv("HOSTNET", false), "whether test is running with hostnet")
checkpoint = flag.Bool("checkpoint", BoolFromEnv("CHECKPOINT", true), "control checkpoint/restore support")
partition = flag.Int("partition", IntFromEnv("PARTITION", 1), "partition number, this is 1-indexed")
totalPartitions = flag.Int("total_partitions", IntFromEnv("TOTAL_PARTITIONS", 1), "total number of partitions")
isRunningWithHostNet = flag.Bool("hostnet", BoolFromEnv("HOSTNET", false), "whether test is running with hostnet")
runscPath = flag.String("runsc", os.Getenv("RUNTIME"), "path to runsc binary")
)
func intFromEnv(name string, def int) int {
// StringFromEnv returns the value of the named environment variable, or `def` if unset/empty.
// It is useful for defining flags where the default value can be specified through the environment.
func StringFromEnv(name, def string) string {
str := os.Getenv(name)
if str == "" {
return def
}
return str
}
// IntFromEnv returns the integer value of the named environment variable, or `def` if unset/empty.
// It is useful for defining flags where the default value can be specified through the environment.
func IntFromEnv(name string, def int) int {
str := os.Getenv(name)
if str == "" {
return def
@@ -69,7 +81,9 @@ func intFromEnv(name string, def int) int {
return int(v)
}
func boolFromEnv(name string, def bool) bool {
// BoolFromEnv returns the boolean value of the named environment variable, or `def` if unset/empty.
// It is useful for defining flags where the default value can be specified through the environment.
func BoolFromEnv(name string, def bool) bool {
str := strings.ToLower(os.Getenv(name))
if str == "" {
return def
@@ -81,6 +95,20 @@ func boolFromEnv(name string, def bool) bool {
return v
}
// DurationFromEnv returns the duration of the named environment variable, or `def` if unset/empty.
// It is useful for defining flags where the default value can be specified through the environment.
func DurationFromEnv(name string, def time.Duration) time.Duration {
str := strings.ToLower(os.Getenv(name))
if str == "" {
return def
}
d, err := time.ParseDuration(str)
if err != nil {
panic(fmt.Errorf("invalid environment variable %q; got %q expected duration: %w", name, str, err))
}
return d
}
// IsCheckpointSupported returns the relevant command line flag.
func IsCheckpointSupported() bool {
return *checkpoint
test/runtimes/README.md
+25 -9
View File
@@ -34,16 +34,32 @@ NodeJS | 16.13.2 | `make nodejs16.13.2-runtime-tests`
Php | 8.1.1 | `make php8.1.1-runtime-tests`
Python | 3.10.2 | `make python3.10.2-runtime-tests`
To run runtime tests individually from a given runtime, you must build or
download the language image and call Docker directly with the test arguments.
You can modify the runtime test behaviors by passing in the following `make`
variables:
Language | Version | Download Image | Run Test(s)
-------- | ------- | ---------------------------------- | -----------
Go | 1.16 | `make load-runtimes_go1.16` | If the test name ends with `.go`, it is an on-disk test: <br> `docker run --runtime=runsc -it gvisor.dev/images/runtimes/go1.16 ( cd /usr/local/go/test ; go run run.go -v -- <TEST_NAME>... )` <br> Otherwise it is a tool test: <br> `docker run --runtime=runsc -it gvisor.dev/images/runtimes/go1.16 go tool dist test -v -no-rebuild ^TEST1$\|^TEST2$...`
Java | 17 | `make load-runtimes_java17` | `docker run --runtime=runsc -it gvisor.dev/images/runtimes/java17 jtreg -agentvm -dir:/root/test/jdk -noreport -timeoutFactor:5 -verbose:all -tl:200 <TEST_NAME>...`
NodeJS | 16.13.2 | `make load-runtimes_nodejs16.13.2` | `docker run --runtime=runsc -it gvisor.dev/images/runtimes/nodejs16.13.2 python tools/test.py --timeout=180 <TEST_NAME>...`
Php | 8.1.1 | `make load-runtimes_php8.1.1` | `docker run --runtime=runsc -it gvisor.dev/images/runtimes/php8.1.1 make test "TESTS=<TEST_NAME>..."`
Python | 3.10.2 | `make load-runtimes_python3.10.2` | `docker run --runtime=runsc -it gvisor.dev/images/runtimes/python3.10.2 ./python -m test <TEST_NAME>...`
* `RUNTIME_TESTS_FILTER`: Comma-separated list of tests to run, even if
otherwise excluded. Useful to debug single failing test cases.
* `RUNTIME_TESTS_PER_TEST_TIMEOUT`: Modify per-test timeout. Useful when
debugging a test that has a tendency to get stuck, in order to make it fail
faster.
* `RUNTIME_TESTS_RUNS_PER_TEST`: Number of times to run each test. Useful to
find flaky tests.
* `RUNTIME_TESTS_FLAKY_IS_ERROR`: Boolean indicating whether tests found flaky
(i.e. running them multiple times has sometimes succeeded, sometimes failed)
should be considered a test suite failure (`true`) or success (`false`).
* `RUNTIME_TESTS_FLAKY_SHORT_CIRCUIT`: If true, when running tests multiple
times, and a test has been found flaky (i.e. running it multiple times has
succeeded at least once and failed at least once), exit immediately, rather
than running all `RUNTIME_TESTS_RUNS_PER_TEST` attempts.
Example invocation:
```shell
$ make php8.1.1-runtime-tests \
RUNTIME_TESTS_FILTER=ext/standard/tests/file/bug60120.phpt \
RUNTIME_TESTS_PER_TEST_TIMEOUT=10s \
RUNTIME_TESTS_RUNS_PER_TEST=100
```
### Clean Up
test/runtimes/proctor/main.go
+74 -10
View File
@@ -29,11 +29,15 @@ import (
)
var (
runtime = flag.String("runtime", "", "name of runtime")
list = flag.Bool("list", false, "list all available tests")
testNames = flag.String("tests", "", "run a subset of the available tests")
pause = flag.Bool("pause", false, "cause container to pause indefinitely, reaping any zombie children")
timeout = flag.Duration("timeout", 90*time.Minute, "batch timeout")
runtime = flag.String("runtime", "", "name of runtime")
list = flag.Bool("list", false, "list all available tests")
testNames = flag.String("tests", "", "run a subset of the available tests")
pause = flag.Bool("pause", false, "cause container to pause indefinitely, reaping any zombie children")
timeout = flag.Duration("timeout", 90*time.Minute, "batch timeout")
perTestTimeout = flag.Duration("per_test_timeout", 20*time.Minute, "per-test timeout (a value of 0 disables per-test timeouts)")
runsPerTest = flag.Int("runs_per_test", 1, "number of times to run each test (a value of 0 is the same as a value of 1, i.e. running once)")
flakyIsError = flag.Bool("flaky_is_error", true, "if true, when running with multiple --runs_per_test, tests with inconsistent status will result in a failure status code for the batch; if false, they will be considered as passing")
flakyShortCircuit = flag.Bool("flaky_short_circuit", true, "if true, when running with multiple --runs_per_test and a test is detected as flaky, exit immediately rather than running all --runs_per_test")
)
// setNumFilesLimit changes the NOFILE soft rlimit if it is too high.
@@ -123,7 +127,7 @@ func main() {
case <-done:
return
case <-timer.C:
log.Println("The timeout duration is exceeded")
log.Println("The batch timeout duration is exceeded")
killed := false
for _, cmd := range cmds {
p := cmd.Process
@@ -140,13 +144,73 @@ func main() {
// Let tests to handle signals
time.Sleep(5 * time.Second)
}
panic("FAIL: The timeout duration is exceeded")
panic("FAIL: The batch timeout duration is exceeded")
}
}()
numIterations := *runsPerTest
if numIterations == 0 {
numIterations = 1
}
for _, cmd := range cmds {
cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
if err := cmd.Run(); err != nil {
log.Fatalf("FAIL: %v", err)
iterations := 0
successes := 0
var firstFailure error
for iteration := 1; iteration <= *runsPerTest; iteration++ {
// Make a copy of the command, as the same exec.Cmd object cannot be started multiple times.
cmdCopy := *cmd
// Handle test timeout.
testDone := make(chan struct{})
testTimedOutCh := make(chan bool, 1)
if *perTestTimeout != 0 {
go func() {
timer := time.NewTimer(*perTestTimeout)
defer timer.Stop()
select {
case <-timer.C:
testTimedOutCh <- true
cmdCopy.Process.Kill()
case <-done:
testTimedOutCh <- false
case <-testDone:
testTimedOutCh <- false
}
}()
}
// Run the test.
cmdCopy.Stdout, cmdCopy.Stderr = os.Stdout, os.Stderr
testErr := cmdCopy.Run()
close(testDone)
if <-testTimedOutCh {
testErr = fmt.Errorf("test timed out after %v", *perTestTimeout)
}
// Tally result.
iterations++
if testErr == nil {
successes++
} else if firstFailure == nil {
firstFailure = testErr
}
if *flakyShortCircuit && successes > 0 && firstFailure != nil {
break
}
}
if successes > 0 && firstFailure != nil {
// Test is flaky.
if *flakyIsError {
log.Fatalf("FLAKY: %v (%d failures out of %d)", firstFailure, iterations-successes, iterations)
} else {
log.Println(fmt.Sprintf("FLAKY: %v (%d failures out of %d)", firstFailure, iterations-successes, iterations))
}
} else if successes == 0 && firstFailure != nil {
// Test is 100% failing.
log.Fatalf("FAIL: %v", firstFailure)
} else if successes > 0 && firstFailure == nil {
// Test is 100% succeeding, do nothing.
} else {
log.Fatalf("Internal logic error")
}
}
}
test/runtimes/runner/BUILD
+4 -1
View File
@@ -7,5 +7,8 @@ go_binary(
testonly = 1,
srcs = ["main.go"],
visibility = ["//test/runtimes:__pkg__"],
deps = ["//test/runtimes/runner/lib"],
deps = [
"//pkg/test/testutil",
"//test/runtimes/runner/lib",
],
)
test/runtimes/runner/lib/exclude_test.go
+1 -4
View File
@@ -29,11 +29,8 @@ func TestMain(m *testing.M) {
// Test that the exclude file parses without error.
func TestExcludelist(t *testing.T) {
ex, err := getExcludes(*excludeFile)
_, err := ExcludeFilter(*excludeFile)
if err != nil {
t.Fatalf("error parsing exclude file: %v", err)
}
if *excludeFile != "" && len(ex) == 0 {
t.Errorf("got empty excludes for file %q", *excludeFile)
}
}
test/runtimes/runner/lib/lib.go
+60 -20
View File
@@ -32,20 +32,42 @@ import (
"gvisor.dev/gvisor/pkg/test/testutil"
)
// ProctorSettings contains settings passed directly to the proctor process.
type ProctorSettings struct {
// PerTestTimeout is the timeout for each individual test.
PerTestTimeout time.Duration
// RunsPerTest is the number of times to run each test.
// A value of 0 is the same as a value of 1, i.e. "run once".
RunsPerTest int
// If FlakyIsError is true, a flaky test will be considered as a failure.
// If it is false, a flaky test will be considered as passing.
FlakyIsError bool
// If FlakyShortCircuit is true, when runnins with RunsPerTest > 1 and a test is detected as
// flaky, exit immediately rather than running for all RunsPerTest attempts.
FlakyShortCircuit bool
}
// ToArgs converts these settings to command-line arguments to pass to the proctor binary.
func (p ProctorSettings) ToArgs() []string {
return []string{
fmt.Sprintf("--per_test_timeout=%v", p.PerTestTimeout),
fmt.Sprintf("--runs_per_test=%d", p.RunsPerTest),
fmt.Sprintf("--flaky_is_error=%v", p.FlakyIsError),
fmt.Sprintf("--flaky_short_circuit=%v", p.FlakyShortCircuit),
}
}
// Filter is a predicate function for filtering tests.
// It returns true if the given test name should be run.
type Filter func(test string) bool
// RunTests is a helper that is called by main. It exists so that we can run
// defered functions before exiting. It returns an exit code that should be
// passed to os.Exit.
func RunTests(lang, image, excludeFile string, batchSize int, timeout time.Duration) int {
func RunTests(lang, image string, filter Filter, batchSize int, timeout time.Duration, proctorSettings ProctorSettings) int {
// TODO(gvisor.dev/issue/1624): Remove those tests from all exclude lists
// that only fail with VFS1.
// Get tests to exclude.
excludes, err := getExcludes(excludeFile)
if err != nil {
fmt.Fprintf(os.Stderr, "Error getting exclude list: %s\n", err.Error())
return 1
}
// Construct the shared docker instance.
ctx := context.Background()
d := dockerutil.MakeContainer(ctx, testutil.DefaultLogger(lang))
@@ -63,7 +85,7 @@ func RunTests(lang, image, excludeFile string, batchSize int, timeout time.Durat
// Get a slice of tests to run. This will also start a single Docker
// container that will be used to run each test. The final test will
// stop the Docker container.
tests, err := getTests(ctx, d, lang, image, batchSize, timeoutChan, timeout, excludes)
tests, err := getTests(ctx, d, lang, image, batchSize, timeoutChan, timeout, filter, proctorSettings)
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err.Error())
return 1
@@ -73,7 +95,7 @@ func RunTests(lang, image, excludeFile string, batchSize int, timeout time.Durat
}
// getTests executes all tests as table tests.
func getTests(ctx context.Context, d *dockerutil.Container, lang, image string, batchSize int, timeoutChan chan struct{}, timeout time.Duration, excludes map[string]struct{}) ([]testing.InternalTest, error) {
func getTests(ctx context.Context, d *dockerutil.Container, lang, image string, batchSize int, timeoutChan chan struct{}, timeout time.Duration, filter Filter, proctorSettings ProctorSettings) ([]testing.InternalTest, error) {
startTime := time.Now()
// Start the container.
@@ -110,6 +132,19 @@ func getTests(ctx context.Context, d *dockerutil.Container, lang, image string,
if err != nil {
return nil, fmt.Errorf("TestsForShard() failed: %v", err)
}
indicesMap := make(map[int]struct{}, len(indices))
for _, i := range indices {
indicesMap[i] = struct{}{}
}
var testsNotInShard []string
for i, tc := range tests {
if _, found := indicesMap[i]; !found {
testsNotInShard = append(testsNotInShard, tc)
}
}
if len(testsNotInShard) > 0 {
log.Infof("Tests not in this shard: %s", strings.Join(testsNotInShard, ","))
}
var itests []testing.InternalTest
for i := 0; i < len(indices); i += batchSize {
@@ -119,8 +154,8 @@ func getTests(ctx context.Context, d *dockerutil.Container, lang, image string,
end = len(indices)
}
for _, tc := range indices[i:end] {
// Add test if not excluded.
if _, ok := excludes[tests[tc]]; ok {
// Add test if not filtered.
if filter != nil && !filter(tests[tc]) {
log.Infof("Skipping test case %s\n", tests[tc])
continue
}
@@ -147,14 +182,16 @@ func getTests(ctx context.Context, d *dockerutil.Container, lang, image string,
if !state.Running {
t.Fatalf("container is not running: state = %s", state.Status)
}
log.Infof("Running test case batch: %s", strings.Join(tcs, ","))
go func() {
output, err = d.Exec(
ctx, dockerutil.ExecOpts{},
argv := []string{
"/proctor/proctor", "--runtime", lang,
"--tests", strings.Join(tcs, ","),
fmt.Sprintf("--timeout=%s", timeout-time.Since(startTime)),
)
}
argv = append(argv, proctorSettings.ToArgs()...)
output, err = d.Exec(ctx, dockerutil.ExecOpts{}, argv...)
close(done)
}()
@@ -176,12 +213,12 @@ func getTests(ctx context.Context, d *dockerutil.Container, lang, image string,
return itests, nil
}
// getExcludes reads the exclude file and returns a set of test names to
// exclude.
func getExcludes(excludeFile string) (map[string]struct{}, error) {
// ExcludeFilter reads the exclude file and returns a filter that excludes the tests listed in
// the given CSV file.
func ExcludeFilter(excludeFile string) (Filter, error) {
excludes := make(map[string]struct{})
if excludeFile == "" {
return excludes, nil
return nil, nil
}
f, err := os.Open(excludeFile)
if err != nil {
@@ -206,7 +243,10 @@ func getExcludes(excludeFile string) (map[string]struct{}, error) {
}
excludes[record[0]] = struct{}{}
}
return excludes, nil
return func(test string) bool {
_, found := excludes[test]
return !found
}, nil
}
// testDeps implements testing.testDeps (an unexported interface), and is
test/runtimes/runner/main.go
+37 -6
View File
@@ -19,17 +19,24 @@ import (
"flag"
"fmt"
"os"
"strings"
"time"
"gvisor.dev/gvisor/pkg/test/testutil"
"gvisor.dev/gvisor/test/runtimes/runner/lib"
)
var (
lang = flag.String("lang", "", "language runtime to test")
image = flag.String("image", "", "docker image with runtime tests")
excludeFile = flag.String("exclude_file", "", "file containing list of tests to exclude, in CSV format with fields: test name, bug id, comment")
batchSize = flag.Int("batch", 50, "number of test cases run in one command")
timeout = flag.Duration("timeout", 20*time.Minute, "batch timeout")
lang = flag.String("lang", "", "language runtime to test")
image = flag.String("image", "", "docker image with runtime tests")
excludeFile = flag.String("exclude_file", "", "file containing list of tests to exclude, in CSV format with fields: test name, bug id, comment")
onlyTests = flag.String("tests", testutil.StringFromEnv("RUNTIME_TESTS_FILTER", ""), "if specified, runs only the given comma-separated list of test names, even those in --exclude_file")
batchSize = flag.Int("batch", 50, "number of test cases run in one command")
timeout = flag.Duration("timeout", 20*time.Minute, "batch timeout")
perTestTimeout = flag.Duration("per_test_timeout", testutil.DurationFromEnv("RUNTIME_TESTS_PER_TEST_TIMEOUT", 20*time.Minute), "per-test timeout (a value of 0 disables per-test timeouts)")
runsPerTest = flag.Int("runs_per_test", testutil.IntFromEnv("RUNTIME_TESTS_RUNS_PER_TEST", 1), "number of times to run each test (a value of 0 is the same as a value of 1, i.e. running once)")
flakyIsError = flag.Bool("flaky_is_error", testutil.BoolFromEnv("RUNTIME_TESTS_FLAKY_IS_ERROR", true), "if true, when running with multiple --runs_per_test, tests with inconsistent status will result in a failure status code for the batch; if false, they will be considered as passing")
flakyShortCircuit = flag.Bool("flaky_short_circuit", testutil.BoolFromEnv("RUNTIME_TESTS_FLAKY_SHORT_CIRCUIT", true), "if true, when running with multiple --runs_per_test and a test is detected as flaky, exit immediately rather than running all --runs_per_test")
)
func main() {
@@ -38,5 +45,29 @@ func main() {
fmt.Fprintf(os.Stderr, "lang and image flags must not be empty\n")
os.Exit(1)
}
os.Exit(lib.RunTests(*lang, *image, *excludeFile, *batchSize, *timeout))
proctorSettings := lib.ProctorSettings{
PerTestTimeout: *perTestTimeout,
RunsPerTest: *runsPerTest,
FlakyIsError: *flakyIsError,
FlakyShortCircuit: *flakyShortCircuit,
}
var filter lib.Filter
if *excludeFile != "" {
excludeFilter, err := lib.ExcludeFilter(*excludeFile)
if err != nil {
fmt.Fprintf(os.Stderr, "Error getting exclude list: %s\n", err.Error())
os.Exit(1)
}
filter = excludeFilter
}
if *onlyTests != "" {
tests := make(map[string]bool)
for _, test := range strings.Split(*onlyTests, ",") {
tests[test] = true
}
filter = func(test string) bool {
return tests[test]
}
}
os.Exit(lib.RunTests(*lang, *image, filter, *batchSize, *timeout, proctorSettings))
}