From c9cb22a16cb3bde375e57ccf444a1707c550dc8b Mon Sep 17 00:00:00 2001 From: Etienne Perot Date: Wed, 10 Aug 2022 14:48:15 -0700 Subject: [PATCH] Runtime tests: Add flakiness detection and per-test timeout options. This adds flags and env variables for the following settings to runtime tests: - `per_test_timeout`: A per-test timeout which can be shorter than the batch timeout. Useful to cap the duration of tests which flake by getting stuck (as is the case for `bug60120.phpt`) - `runs_per_test`: Number of times to run each test (useful to detect flakes). - `flaky_is_error`: Controls whether a flaky test is considered passing or failing for batch error code purposes. Useful when either diagnosing a flaky test, or diagnosing a consistently-failing test while bypassing flaky others. - `flaky_short_circuit`: If a test is found to be flaky, declare it as such immediately, rather than waiting for the rest of the `--runs_per_test` to finish. Speeds up bisecting flaky test, at the cost of flakiness percentage accuracy. PiperOrigin-RevId: 466791296 --- Makefile | 11 +++- pkg/test/testutil/testutil.go | 40 +++++++++-- test/runtimes/README.md | 34 +++++++--- test/runtimes/proctor/main.go | 84 +++++++++++++++++++++--- test/runtimes/runner/BUILD | 5 +- test/runtimes/runner/lib/exclude_test.go | 5 +- test/runtimes/runner/lib/lib.go | 80 ++++++++++++++++------ test/runtimes/runner/main.go | 43 ++++++++++-- 8 files changed, 244 insertions(+), 58 deletions(-) diff --git a/Makefile b/Makefile index 86168a0c9..21b19933d 100644 --- a/Makefile +++ b/Makefile @@ -236,13 +236,20 @@ packetimpact-tests: @$(call test,--jobs=HOST_CPUS*3 --local_test_jobs=HOST_CPUS*3 //test/packetimpact/tests:all_tests) .PHONY: packetimpact-tests +# Extra configuration options for runtime tests. +RUNTIME_TESTS_FILTER ?= +RUNTIME_TESTS_PER_TEST_TIMEOUT ?= 20m +RUNTIME_TESTS_RUNS_PER_TEST ?= 1 +RUNTIME_TESTS_FLAKY_IS_ERROR ?= true +RUNTIME_TESTS_FLAKY_SHORT_CIRCUIT ?= true + %-runtime-tests: load-runtimes_% $(RUNTIME_BIN) @$(call install_runtime,$(RUNTIME),--watchdog-action=panic) - @$(call test_runtime,$(RUNTIME),--test_timeout=1800 //test/runtimes:$*) + @$(call test_runtime,$(RUNTIME),--test_timeout=1800 --test_env=RUNTIME_TESTS_FILTER=$(RUNTIME_TESTS_FILTER) --test_env=RUNTIME_TESTS_PER_TEST_TIMEOUT=$(RUNTIME_TESTS_PER_TEST_TIMEOUT) --test_env=RUNTIME_TESTS_RUNS_PER_TEST=$(RUNTIME_TESTS_RUNS_PER_TEST) --test_env=RUNTIME_TESTS_FLAKY_IS_ERROR=$(RUNTIME_TESTS_FLAKY_IS_ERROR) --test_env=RUNTIME_TESTS_FLAKY_SHORT_CIRCUIT=$(RUNTIME_TESTS_FLAKY_SHORT_CIRCUIT) //test/runtimes:$*) %-runtime-tests_lisafs: load-runtimes_% $(RUNTIME_BIN) @$(call install_runtime,$(RUNTIME), --lisafs --watchdog-action=panic) - @$(call test_runtime,$(RUNTIME),--test_timeout=1800 //test/runtimes:$*) + @$(call test_runtime,$(RUNTIME),--test_timeout=1800 --test_env=RUNTIME_TESTS_FILTER=$(RUNTIME_TESTS_FILTER) --test_env=RUNTIME_TESTS_PER_TEST_TIMEOUT=$(RUNTIME_TESTS_PER_TEST_TIMEOUT) --test_env=RUNTIME_TESTS_RUNS_PER_TEST=$(RUNTIME_TESTS_RUNS_PER_TEST) --test_env=RUNTIME_TESTS_FLAKY_IS_ERROR=$(RUNTIME_TESTS_FLAKY_IS_ERROR) --test_env=RUNTIME_TESTS_FLAKY_SHORT_CIRCUIT=$(RUNTIME_TESTS_FLAKY_SHORT_CIRCUIT) //test/runtimes:$*) do-tests: $(RUNTIME_BIN) @$(RUNTIME_BIN) --rootless do true diff --git a/pkg/test/testutil/testutil.go b/pkg/test/testutil/testutil.go index 15d3e6ef2..745ba4026 100644 --- a/pkg/test/testutil/testutil.go +++ b/pkg/test/testutil/testutil.go @@ -49,14 +49,26 @@ import ( ) var ( - checkpoint = flag.Bool("checkpoint", boolFromEnv("CHECKPOINT", true), "control checkpoint/restore support") - partition = flag.Int("partition", intFromEnv("PARTITION", 1), "partition number, this is 1-indexed") - totalPartitions = flag.Int("total_partitions", intFromEnv("TOTAL_PARTITIONS", 1), "total number of partitions") - isRunningWithHostNet = flag.Bool("hostnet", boolFromEnv("HOSTNET", false), "whether test is running with hostnet") + checkpoint = flag.Bool("checkpoint", BoolFromEnv("CHECKPOINT", true), "control checkpoint/restore support") + partition = flag.Int("partition", IntFromEnv("PARTITION", 1), "partition number, this is 1-indexed") + totalPartitions = flag.Int("total_partitions", IntFromEnv("TOTAL_PARTITIONS", 1), "total number of partitions") + isRunningWithHostNet = flag.Bool("hostnet", BoolFromEnv("HOSTNET", false), "whether test is running with hostnet") runscPath = flag.String("runsc", os.Getenv("RUNTIME"), "path to runsc binary") ) -func intFromEnv(name string, def int) int { +// StringFromEnv returns the value of the named environment variable, or `def` if unset/empty. +// It is useful for defining flags where the default value can be specified through the environment. +func StringFromEnv(name, def string) string { + str := os.Getenv(name) + if str == "" { + return def + } + return str +} + +// IntFromEnv returns the integer value of the named environment variable, or `def` if unset/empty. +// It is useful for defining flags where the default value can be specified through the environment. +func IntFromEnv(name string, def int) int { str := os.Getenv(name) if str == "" { return def @@ -69,7 +81,9 @@ func intFromEnv(name string, def int) int { return int(v) } -func boolFromEnv(name string, def bool) bool { +// BoolFromEnv returns the boolean value of the named environment variable, or `def` if unset/empty. +// It is useful for defining flags where the default value can be specified through the environment. +func BoolFromEnv(name string, def bool) bool { str := strings.ToLower(os.Getenv(name)) if str == "" { return def @@ -81,6 +95,20 @@ func boolFromEnv(name string, def bool) bool { return v } +// DurationFromEnv returns the duration of the named environment variable, or `def` if unset/empty. +// It is useful for defining flags where the default value can be specified through the environment. +func DurationFromEnv(name string, def time.Duration) time.Duration { + str := strings.ToLower(os.Getenv(name)) + if str == "" { + return def + } + d, err := time.ParseDuration(str) + if err != nil { + panic(fmt.Errorf("invalid environment variable %q; got %q expected duration: %w", name, str, err)) + } + return d +} + // IsCheckpointSupported returns the relevant command line flag. func IsCheckpointSupported() bool { return *checkpoint diff --git a/test/runtimes/README.md b/test/runtimes/README.md index 165b27046..ad42d7a29 100644 --- a/test/runtimes/README.md +++ b/test/runtimes/README.md @@ -34,16 +34,32 @@ NodeJS | 16.13.2 | `make nodejs16.13.2-runtime-tests` Php | 8.1.1 | `make php8.1.1-runtime-tests` Python | 3.10.2 | `make python3.10.2-runtime-tests` -To run runtime tests individually from a given runtime, you must build or -download the language image and call Docker directly with the test arguments. +You can modify the runtime test behaviors by passing in the following `make` +variables: -Language | Version | Download Image | Run Test(s) --------- | ------- | ---------------------------------- | ----------- -Go | 1.16 | `make load-runtimes_go1.16` | If the test name ends with `.go`, it is an on-disk test:
`docker run --runtime=runsc -it gvisor.dev/images/runtimes/go1.16 ( cd /usr/local/go/test ; go run run.go -v -- ... )`
Otherwise it is a tool test:
`docker run --runtime=runsc -it gvisor.dev/images/runtimes/go1.16 go tool dist test -v -no-rebuild ^TEST1$\|^TEST2$...` -Java | 17 | `make load-runtimes_java17` | `docker run --runtime=runsc -it gvisor.dev/images/runtimes/java17 jtreg -agentvm -dir:/root/test/jdk -noreport -timeoutFactor:5 -verbose:all -tl:200 ...` -NodeJS | 16.13.2 | `make load-runtimes_nodejs16.13.2` | `docker run --runtime=runsc -it gvisor.dev/images/runtimes/nodejs16.13.2 python tools/test.py --timeout=180 ...` -Php | 8.1.1 | `make load-runtimes_php8.1.1` | `docker run --runtime=runsc -it gvisor.dev/images/runtimes/php8.1.1 make test "TESTS=..."` -Python | 3.10.2 | `make load-runtimes_python3.10.2` | `docker run --runtime=runsc -it gvisor.dev/images/runtimes/python3.10.2 ./python -m test ...` +* `RUNTIME_TESTS_FILTER`: Comma-separated list of tests to run, even if + otherwise excluded. Useful to debug single failing test cases. +* `RUNTIME_TESTS_PER_TEST_TIMEOUT`: Modify per-test timeout. Useful when + debugging a test that has a tendency to get stuck, in order to make it fail + faster. +* `RUNTIME_TESTS_RUNS_PER_TEST`: Number of times to run each test. Useful to + find flaky tests. +* `RUNTIME_TESTS_FLAKY_IS_ERROR`: Boolean indicating whether tests found flaky + (i.e. running them multiple times has sometimes succeeded, sometimes failed) + should be considered a test suite failure (`true`) or success (`false`). +* `RUNTIME_TESTS_FLAKY_SHORT_CIRCUIT`: If true, when running tests multiple + times, and a test has been found flaky (i.e. running it multiple times has + succeeded at least once and failed at least once), exit immediately, rather + than running all `RUNTIME_TESTS_RUNS_PER_TEST` attempts. + +Example invocation: + +```shell +$ make php8.1.1-runtime-tests \ + RUNTIME_TESTS_FILTER=ext/standard/tests/file/bug60120.phpt \ + RUNTIME_TESTS_PER_TEST_TIMEOUT=10s \ + RUNTIME_TESTS_RUNS_PER_TEST=100 +``` ### Clean Up diff --git a/test/runtimes/proctor/main.go b/test/runtimes/proctor/main.go index ea0b0ed4e..d721acb90 100644 --- a/test/runtimes/proctor/main.go +++ b/test/runtimes/proctor/main.go @@ -29,11 +29,15 @@ import ( ) var ( - runtime = flag.String("runtime", "", "name of runtime") - list = flag.Bool("list", false, "list all available tests") - testNames = flag.String("tests", "", "run a subset of the available tests") - pause = flag.Bool("pause", false, "cause container to pause indefinitely, reaping any zombie children") - timeout = flag.Duration("timeout", 90*time.Minute, "batch timeout") + runtime = flag.String("runtime", "", "name of runtime") + list = flag.Bool("list", false, "list all available tests") + testNames = flag.String("tests", "", "run a subset of the available tests") + pause = flag.Bool("pause", false, "cause container to pause indefinitely, reaping any zombie children") + timeout = flag.Duration("timeout", 90*time.Minute, "batch timeout") + perTestTimeout = flag.Duration("per_test_timeout", 20*time.Minute, "per-test timeout (a value of 0 disables per-test timeouts)") + runsPerTest = flag.Int("runs_per_test", 1, "number of times to run each test (a value of 0 is the same as a value of 1, i.e. running once)") + flakyIsError = flag.Bool("flaky_is_error", true, "if true, when running with multiple --runs_per_test, tests with inconsistent status will result in a failure status code for the batch; if false, they will be considered as passing") + flakyShortCircuit = flag.Bool("flaky_short_circuit", true, "if true, when running with multiple --runs_per_test and a test is detected as flaky, exit immediately rather than running all --runs_per_test") ) // setNumFilesLimit changes the NOFILE soft rlimit if it is too high. @@ -123,7 +127,7 @@ func main() { case <-done: return case <-timer.C: - log.Println("The timeout duration is exceeded") + log.Println("The batch timeout duration is exceeded") killed := false for _, cmd := range cmds { p := cmd.Process @@ -140,13 +144,73 @@ func main() { // Let tests to handle signals time.Sleep(5 * time.Second) } - panic("FAIL: The timeout duration is exceeded") + panic("FAIL: The batch timeout duration is exceeded") } }() + numIterations := *runsPerTest + if numIterations == 0 { + numIterations = 1 + } for _, cmd := range cmds { - cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr - if err := cmd.Run(); err != nil { - log.Fatalf("FAIL: %v", err) + iterations := 0 + successes := 0 + var firstFailure error + for iteration := 1; iteration <= *runsPerTest; iteration++ { + // Make a copy of the command, as the same exec.Cmd object cannot be started multiple times. + cmdCopy := *cmd + + // Handle test timeout. + testDone := make(chan struct{}) + testTimedOutCh := make(chan bool, 1) + if *perTestTimeout != 0 { + go func() { + timer := time.NewTimer(*perTestTimeout) + defer timer.Stop() + select { + case <-timer.C: + testTimedOutCh <- true + cmdCopy.Process.Kill() + case <-done: + testTimedOutCh <- false + case <-testDone: + testTimedOutCh <- false + } + }() + } + + // Run the test. + cmdCopy.Stdout, cmdCopy.Stderr = os.Stdout, os.Stderr + testErr := cmdCopy.Run() + close(testDone) + if <-testTimedOutCh { + testErr = fmt.Errorf("test timed out after %v", *perTestTimeout) + } + + // Tally result. + iterations++ + if testErr == nil { + successes++ + } else if firstFailure == nil { + firstFailure = testErr + } + if *flakyShortCircuit && successes > 0 && firstFailure != nil { + break + } + } + if successes > 0 && firstFailure != nil { + // Test is flaky. + if *flakyIsError { + log.Fatalf("FLAKY: %v (%d failures out of %d)", firstFailure, iterations-successes, iterations) + } else { + log.Println(fmt.Sprintf("FLAKY: %v (%d failures out of %d)", firstFailure, iterations-successes, iterations)) + } + } else if successes == 0 && firstFailure != nil { + // Test is 100% failing. + log.Fatalf("FAIL: %v", firstFailure) + } else if successes > 0 && firstFailure == nil { + // Test is 100% succeeding, do nothing. + } else { + log.Fatalf("Internal logic error") } } } diff --git a/test/runtimes/runner/BUILD b/test/runtimes/runner/BUILD index 70cc01594..091aff1f2 100644 --- a/test/runtimes/runner/BUILD +++ b/test/runtimes/runner/BUILD @@ -7,5 +7,8 @@ go_binary( testonly = 1, srcs = ["main.go"], visibility = ["//test/runtimes:__pkg__"], - deps = ["//test/runtimes/runner/lib"], + deps = [ + "//pkg/test/testutil", + "//test/runtimes/runner/lib", + ], ) diff --git a/test/runtimes/runner/lib/exclude_test.go b/test/runtimes/runner/lib/exclude_test.go index f996e895b..0f04460dd 100644 --- a/test/runtimes/runner/lib/exclude_test.go +++ b/test/runtimes/runner/lib/exclude_test.go @@ -29,11 +29,8 @@ func TestMain(m *testing.M) { // Test that the exclude file parses without error. func TestExcludelist(t *testing.T) { - ex, err := getExcludes(*excludeFile) + _, err := ExcludeFilter(*excludeFile) if err != nil { t.Fatalf("error parsing exclude file: %v", err) } - if *excludeFile != "" && len(ex) == 0 { - t.Errorf("got empty excludes for file %q", *excludeFile) - } } diff --git a/test/runtimes/runner/lib/lib.go b/test/runtimes/runner/lib/lib.go index 6c974cb3a..a25fc1dc6 100644 --- a/test/runtimes/runner/lib/lib.go +++ b/test/runtimes/runner/lib/lib.go @@ -32,20 +32,42 @@ import ( "gvisor.dev/gvisor/pkg/test/testutil" ) +// ProctorSettings contains settings passed directly to the proctor process. +type ProctorSettings struct { + // PerTestTimeout is the timeout for each individual test. + PerTestTimeout time.Duration + // RunsPerTest is the number of times to run each test. + // A value of 0 is the same as a value of 1, i.e. "run once". + RunsPerTest int + // If FlakyIsError is true, a flaky test will be considered as a failure. + // If it is false, a flaky test will be considered as passing. + FlakyIsError bool + // If FlakyShortCircuit is true, when runnins with RunsPerTest > 1 and a test is detected as + // flaky, exit immediately rather than running for all RunsPerTest attempts. + FlakyShortCircuit bool +} + +// ToArgs converts these settings to command-line arguments to pass to the proctor binary. +func (p ProctorSettings) ToArgs() []string { + return []string{ + fmt.Sprintf("--per_test_timeout=%v", p.PerTestTimeout), + fmt.Sprintf("--runs_per_test=%d", p.RunsPerTest), + fmt.Sprintf("--flaky_is_error=%v", p.FlakyIsError), + fmt.Sprintf("--flaky_short_circuit=%v", p.FlakyShortCircuit), + } +} + +// Filter is a predicate function for filtering tests. +// It returns true if the given test name should be run. +type Filter func(test string) bool + // RunTests is a helper that is called by main. It exists so that we can run // defered functions before exiting. It returns an exit code that should be // passed to os.Exit. -func RunTests(lang, image, excludeFile string, batchSize int, timeout time.Duration) int { +func RunTests(lang, image string, filter Filter, batchSize int, timeout time.Duration, proctorSettings ProctorSettings) int { // TODO(gvisor.dev/issue/1624): Remove those tests from all exclude lists // that only fail with VFS1. - // Get tests to exclude. - excludes, err := getExcludes(excludeFile) - if err != nil { - fmt.Fprintf(os.Stderr, "Error getting exclude list: %s\n", err.Error()) - return 1 - } - // Construct the shared docker instance. ctx := context.Background() d := dockerutil.MakeContainer(ctx, testutil.DefaultLogger(lang)) @@ -63,7 +85,7 @@ func RunTests(lang, image, excludeFile string, batchSize int, timeout time.Durat // Get a slice of tests to run. This will also start a single Docker // container that will be used to run each test. The final test will // stop the Docker container. - tests, err := getTests(ctx, d, lang, image, batchSize, timeoutChan, timeout, excludes) + tests, err := getTests(ctx, d, lang, image, batchSize, timeoutChan, timeout, filter, proctorSettings) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", err.Error()) return 1 @@ -73,7 +95,7 @@ func RunTests(lang, image, excludeFile string, batchSize int, timeout time.Durat } // getTests executes all tests as table tests. -func getTests(ctx context.Context, d *dockerutil.Container, lang, image string, batchSize int, timeoutChan chan struct{}, timeout time.Duration, excludes map[string]struct{}) ([]testing.InternalTest, error) { +func getTests(ctx context.Context, d *dockerutil.Container, lang, image string, batchSize int, timeoutChan chan struct{}, timeout time.Duration, filter Filter, proctorSettings ProctorSettings) ([]testing.InternalTest, error) { startTime := time.Now() // Start the container. @@ -110,6 +132,19 @@ func getTests(ctx context.Context, d *dockerutil.Container, lang, image string, if err != nil { return nil, fmt.Errorf("TestsForShard() failed: %v", err) } + indicesMap := make(map[int]struct{}, len(indices)) + for _, i := range indices { + indicesMap[i] = struct{}{} + } + var testsNotInShard []string + for i, tc := range tests { + if _, found := indicesMap[i]; !found { + testsNotInShard = append(testsNotInShard, tc) + } + } + if len(testsNotInShard) > 0 { + log.Infof("Tests not in this shard: %s", strings.Join(testsNotInShard, ",")) + } var itests []testing.InternalTest for i := 0; i < len(indices); i += batchSize { @@ -119,8 +154,8 @@ func getTests(ctx context.Context, d *dockerutil.Container, lang, image string, end = len(indices) } for _, tc := range indices[i:end] { - // Add test if not excluded. - if _, ok := excludes[tests[tc]]; ok { + // Add test if not filtered. + if filter != nil && !filter(tests[tc]) { log.Infof("Skipping test case %s\n", tests[tc]) continue } @@ -147,14 +182,16 @@ func getTests(ctx context.Context, d *dockerutil.Container, lang, image string, if !state.Running { t.Fatalf("container is not running: state = %s", state.Status) } + log.Infof("Running test case batch: %s", strings.Join(tcs, ",")) go func() { - output, err = d.Exec( - ctx, dockerutil.ExecOpts{}, + argv := []string{ "/proctor/proctor", "--runtime", lang, "--tests", strings.Join(tcs, ","), fmt.Sprintf("--timeout=%s", timeout-time.Since(startTime)), - ) + } + argv = append(argv, proctorSettings.ToArgs()...) + output, err = d.Exec(ctx, dockerutil.ExecOpts{}, argv...) close(done) }() @@ -176,12 +213,12 @@ func getTests(ctx context.Context, d *dockerutil.Container, lang, image string, return itests, nil } -// getExcludes reads the exclude file and returns a set of test names to -// exclude. -func getExcludes(excludeFile string) (map[string]struct{}, error) { +// ExcludeFilter reads the exclude file and returns a filter that excludes the tests listed in +// the given CSV file. +func ExcludeFilter(excludeFile string) (Filter, error) { excludes := make(map[string]struct{}) if excludeFile == "" { - return excludes, nil + return nil, nil } f, err := os.Open(excludeFile) if err != nil { @@ -206,7 +243,10 @@ func getExcludes(excludeFile string) (map[string]struct{}, error) { } excludes[record[0]] = struct{}{} } - return excludes, nil + return func(test string) bool { + _, found := excludes[test] + return !found + }, nil } // testDeps implements testing.testDeps (an unexported interface), and is diff --git a/test/runtimes/runner/main.go b/test/runtimes/runner/main.go index 6a2eb2859..675639082 100644 --- a/test/runtimes/runner/main.go +++ b/test/runtimes/runner/main.go @@ -19,17 +19,24 @@ import ( "flag" "fmt" "os" + "strings" "time" + "gvisor.dev/gvisor/pkg/test/testutil" "gvisor.dev/gvisor/test/runtimes/runner/lib" ) var ( - lang = flag.String("lang", "", "language runtime to test") - image = flag.String("image", "", "docker image with runtime tests") - excludeFile = flag.String("exclude_file", "", "file containing list of tests to exclude, in CSV format with fields: test name, bug id, comment") - batchSize = flag.Int("batch", 50, "number of test cases run in one command") - timeout = flag.Duration("timeout", 20*time.Minute, "batch timeout") + lang = flag.String("lang", "", "language runtime to test") + image = flag.String("image", "", "docker image with runtime tests") + excludeFile = flag.String("exclude_file", "", "file containing list of tests to exclude, in CSV format with fields: test name, bug id, comment") + onlyTests = flag.String("tests", testutil.StringFromEnv("RUNTIME_TESTS_FILTER", ""), "if specified, runs only the given comma-separated list of test names, even those in --exclude_file") + batchSize = flag.Int("batch", 50, "number of test cases run in one command") + timeout = flag.Duration("timeout", 20*time.Minute, "batch timeout") + perTestTimeout = flag.Duration("per_test_timeout", testutil.DurationFromEnv("RUNTIME_TESTS_PER_TEST_TIMEOUT", 20*time.Minute), "per-test timeout (a value of 0 disables per-test timeouts)") + runsPerTest = flag.Int("runs_per_test", testutil.IntFromEnv("RUNTIME_TESTS_RUNS_PER_TEST", 1), "number of times to run each test (a value of 0 is the same as a value of 1, i.e. running once)") + flakyIsError = flag.Bool("flaky_is_error", testutil.BoolFromEnv("RUNTIME_TESTS_FLAKY_IS_ERROR", true), "if true, when running with multiple --runs_per_test, tests with inconsistent status will result in a failure status code for the batch; if false, they will be considered as passing") + flakyShortCircuit = flag.Bool("flaky_short_circuit", testutil.BoolFromEnv("RUNTIME_TESTS_FLAKY_SHORT_CIRCUIT", true), "if true, when running with multiple --runs_per_test and a test is detected as flaky, exit immediately rather than running all --runs_per_test") ) func main() { @@ -38,5 +45,29 @@ func main() { fmt.Fprintf(os.Stderr, "lang and image flags must not be empty\n") os.Exit(1) } - os.Exit(lib.RunTests(*lang, *image, *excludeFile, *batchSize, *timeout)) + proctorSettings := lib.ProctorSettings{ + PerTestTimeout: *perTestTimeout, + RunsPerTest: *runsPerTest, + FlakyIsError: *flakyIsError, + FlakyShortCircuit: *flakyShortCircuit, + } + var filter lib.Filter + if *excludeFile != "" { + excludeFilter, err := lib.ExcludeFilter(*excludeFile) + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting exclude list: %s\n", err.Error()) + os.Exit(1) + } + filter = excludeFilter + } + if *onlyTests != "" { + tests := make(map[string]bool) + for _, test := range strings.Split(*onlyTests, ",") { + tests[test] = true + } + filter = func(test string) bool { + return tests[test] + } + } + os.Exit(lib.RunTests(*lang, *image, filter, *batchSize, *timeout, proctorSettings)) }