diff --git a/Makefile b/Makefile
index 86168a0c9..21b19933d 100644
--- a/Makefile
+++ b/Makefile
@@ -236,13 +236,20 @@ packetimpact-tests:
@$(call test,--jobs=HOST_CPUS*3 --local_test_jobs=HOST_CPUS*3 //test/packetimpact/tests:all_tests)
.PHONY: packetimpact-tests
+# Extra configuration options for runtime tests.
+RUNTIME_TESTS_FILTER ?=
+RUNTIME_TESTS_PER_TEST_TIMEOUT ?= 20m
+RUNTIME_TESTS_RUNS_PER_TEST ?= 1
+RUNTIME_TESTS_FLAKY_IS_ERROR ?= true
+RUNTIME_TESTS_FLAKY_SHORT_CIRCUIT ?= true
+
%-runtime-tests: load-runtimes_% $(RUNTIME_BIN)
@$(call install_runtime,$(RUNTIME),--watchdog-action=panic)
- @$(call test_runtime,$(RUNTIME),--test_timeout=1800 //test/runtimes:$*)
+ @$(call test_runtime,$(RUNTIME),--test_timeout=1800 --test_env=RUNTIME_TESTS_FILTER=$(RUNTIME_TESTS_FILTER) --test_env=RUNTIME_TESTS_PER_TEST_TIMEOUT=$(RUNTIME_TESTS_PER_TEST_TIMEOUT) --test_env=RUNTIME_TESTS_RUNS_PER_TEST=$(RUNTIME_TESTS_RUNS_PER_TEST) --test_env=RUNTIME_TESTS_FLAKY_IS_ERROR=$(RUNTIME_TESTS_FLAKY_IS_ERROR) --test_env=RUNTIME_TESTS_FLAKY_SHORT_CIRCUIT=$(RUNTIME_TESTS_FLAKY_SHORT_CIRCUIT) //test/runtimes:$*)
%-runtime-tests_lisafs: load-runtimes_% $(RUNTIME_BIN)
@$(call install_runtime,$(RUNTIME), --lisafs --watchdog-action=panic)
- @$(call test_runtime,$(RUNTIME),--test_timeout=1800 //test/runtimes:$*)
+ @$(call test_runtime,$(RUNTIME),--test_timeout=1800 --test_env=RUNTIME_TESTS_FILTER=$(RUNTIME_TESTS_FILTER) --test_env=RUNTIME_TESTS_PER_TEST_TIMEOUT=$(RUNTIME_TESTS_PER_TEST_TIMEOUT) --test_env=RUNTIME_TESTS_RUNS_PER_TEST=$(RUNTIME_TESTS_RUNS_PER_TEST) --test_env=RUNTIME_TESTS_FLAKY_IS_ERROR=$(RUNTIME_TESTS_FLAKY_IS_ERROR) --test_env=RUNTIME_TESTS_FLAKY_SHORT_CIRCUIT=$(RUNTIME_TESTS_FLAKY_SHORT_CIRCUIT) //test/runtimes:$*)
do-tests: $(RUNTIME_BIN)
@$(RUNTIME_BIN) --rootless do true
diff --git a/pkg/test/testutil/testutil.go b/pkg/test/testutil/testutil.go
index 15d3e6ef2..745ba4026 100644
--- a/pkg/test/testutil/testutil.go
+++ b/pkg/test/testutil/testutil.go
@@ -49,14 +49,26 @@ import (
)
var (
- checkpoint = flag.Bool("checkpoint", boolFromEnv("CHECKPOINT", true), "control checkpoint/restore support")
- partition = flag.Int("partition", intFromEnv("PARTITION", 1), "partition number, this is 1-indexed")
- totalPartitions = flag.Int("total_partitions", intFromEnv("TOTAL_PARTITIONS", 1), "total number of partitions")
- isRunningWithHostNet = flag.Bool("hostnet", boolFromEnv("HOSTNET", false), "whether test is running with hostnet")
+ checkpoint = flag.Bool("checkpoint", BoolFromEnv("CHECKPOINT", true), "control checkpoint/restore support")
+ partition = flag.Int("partition", IntFromEnv("PARTITION", 1), "partition number, this is 1-indexed")
+ totalPartitions = flag.Int("total_partitions", IntFromEnv("TOTAL_PARTITIONS", 1), "total number of partitions")
+ isRunningWithHostNet = flag.Bool("hostnet", BoolFromEnv("HOSTNET", false), "whether test is running with hostnet")
runscPath = flag.String("runsc", os.Getenv("RUNTIME"), "path to runsc binary")
)
-func intFromEnv(name string, def int) int {
+// StringFromEnv returns the value of the named environment variable, or `def` if unset/empty.
+// It is useful for defining flags where the default value can be specified through the environment.
+func StringFromEnv(name, def string) string {
+ str := os.Getenv(name)
+ if str == "" {
+ return def
+ }
+ return str
+}
+
+// IntFromEnv returns the integer value of the named environment variable, or `def` if unset/empty.
+// It is useful for defining flags where the default value can be specified through the environment.
+func IntFromEnv(name string, def int) int {
str := os.Getenv(name)
if str == "" {
return def
@@ -69,7 +81,9 @@ func intFromEnv(name string, def int) int {
return int(v)
}
-func boolFromEnv(name string, def bool) bool {
+// BoolFromEnv returns the boolean value of the named environment variable, or `def` if unset/empty.
+// It is useful for defining flags where the default value can be specified through the environment.
+func BoolFromEnv(name string, def bool) bool {
str := strings.ToLower(os.Getenv(name))
if str == "" {
return def
@@ -81,6 +95,20 @@ func boolFromEnv(name string, def bool) bool {
return v
}
+// DurationFromEnv returns the duration of the named environment variable, or `def` if unset/empty.
+// It is useful for defining flags where the default value can be specified through the environment.
+func DurationFromEnv(name string, def time.Duration) time.Duration {
+ str := strings.ToLower(os.Getenv(name))
+ if str == "" {
+ return def
+ }
+ d, err := time.ParseDuration(str)
+ if err != nil {
+ panic(fmt.Errorf("invalid environment variable %q; got %q expected duration: %w", name, str, err))
+ }
+ return d
+}
+
// IsCheckpointSupported returns the relevant command line flag.
func IsCheckpointSupported() bool {
return *checkpoint
diff --git a/test/runtimes/README.md b/test/runtimes/README.md
index 165b27046..ad42d7a29 100644
--- a/test/runtimes/README.md
+++ b/test/runtimes/README.md
@@ -34,16 +34,32 @@ NodeJS | 16.13.2 | `make nodejs16.13.2-runtime-tests`
Php | 8.1.1 | `make php8.1.1-runtime-tests`
Python | 3.10.2 | `make python3.10.2-runtime-tests`
-To run runtime tests individually from a given runtime, you must build or
-download the language image and call Docker directly with the test arguments.
+You can modify the runtime test behaviors by passing in the following `make`
+variables:
-Language | Version | Download Image | Run Test(s)
--------- | ------- | ---------------------------------- | -----------
-Go | 1.16 | `make load-runtimes_go1.16` | If the test name ends with `.go`, it is an on-disk test:
`docker run --runtime=runsc -it gvisor.dev/images/runtimes/go1.16 ( cd /usr/local/go/test ; go run run.go -v -- ... )`
Otherwise it is a tool test:
`docker run --runtime=runsc -it gvisor.dev/images/runtimes/go1.16 go tool dist test -v -no-rebuild ^TEST1$\|^TEST2$...`
-Java | 17 | `make load-runtimes_java17` | `docker run --runtime=runsc -it gvisor.dev/images/runtimes/java17 jtreg -agentvm -dir:/root/test/jdk -noreport -timeoutFactor:5 -verbose:all -tl:200 ...`
-NodeJS | 16.13.2 | `make load-runtimes_nodejs16.13.2` | `docker run --runtime=runsc -it gvisor.dev/images/runtimes/nodejs16.13.2 python tools/test.py --timeout=180 ...`
-Php | 8.1.1 | `make load-runtimes_php8.1.1` | `docker run --runtime=runsc -it gvisor.dev/images/runtimes/php8.1.1 make test "TESTS=..."`
-Python | 3.10.2 | `make load-runtimes_python3.10.2` | `docker run --runtime=runsc -it gvisor.dev/images/runtimes/python3.10.2 ./python -m test ...`
+* `RUNTIME_TESTS_FILTER`: Comma-separated list of tests to run, even if
+ otherwise excluded. Useful to debug single failing test cases.
+* `RUNTIME_TESTS_PER_TEST_TIMEOUT`: Modify per-test timeout. Useful when
+ debugging a test that has a tendency to get stuck, in order to make it fail
+ faster.
+* `RUNTIME_TESTS_RUNS_PER_TEST`: Number of times to run each test. Useful to
+ find flaky tests.
+* `RUNTIME_TESTS_FLAKY_IS_ERROR`: Boolean indicating whether tests found flaky
+ (i.e. running them multiple times has sometimes succeeded, sometimes failed)
+ should be considered a test suite failure (`true`) or success (`false`).
+* `RUNTIME_TESTS_FLAKY_SHORT_CIRCUIT`: If true, when running tests multiple
+ times, and a test has been found flaky (i.e. running it multiple times has
+ succeeded at least once and failed at least once), exit immediately, rather
+ than running all `RUNTIME_TESTS_RUNS_PER_TEST` attempts.
+
+Example invocation:
+
+```shell
+$ make php8.1.1-runtime-tests \
+ RUNTIME_TESTS_FILTER=ext/standard/tests/file/bug60120.phpt \
+ RUNTIME_TESTS_PER_TEST_TIMEOUT=10s \
+ RUNTIME_TESTS_RUNS_PER_TEST=100
+```
### Clean Up
diff --git a/test/runtimes/proctor/main.go b/test/runtimes/proctor/main.go
index ea0b0ed4e..d721acb90 100644
--- a/test/runtimes/proctor/main.go
+++ b/test/runtimes/proctor/main.go
@@ -29,11 +29,15 @@ import (
)
var (
- runtime = flag.String("runtime", "", "name of runtime")
- list = flag.Bool("list", false, "list all available tests")
- testNames = flag.String("tests", "", "run a subset of the available tests")
- pause = flag.Bool("pause", false, "cause container to pause indefinitely, reaping any zombie children")
- timeout = flag.Duration("timeout", 90*time.Minute, "batch timeout")
+ runtime = flag.String("runtime", "", "name of runtime")
+ list = flag.Bool("list", false, "list all available tests")
+ testNames = flag.String("tests", "", "run a subset of the available tests")
+ pause = flag.Bool("pause", false, "cause container to pause indefinitely, reaping any zombie children")
+ timeout = flag.Duration("timeout", 90*time.Minute, "batch timeout")
+ perTestTimeout = flag.Duration("per_test_timeout", 20*time.Minute, "per-test timeout (a value of 0 disables per-test timeouts)")
+ runsPerTest = flag.Int("runs_per_test", 1, "number of times to run each test (a value of 0 is the same as a value of 1, i.e. running once)")
+ flakyIsError = flag.Bool("flaky_is_error", true, "if true, when running with multiple --runs_per_test, tests with inconsistent status will result in a failure status code for the batch; if false, they will be considered as passing")
+ flakyShortCircuit = flag.Bool("flaky_short_circuit", true, "if true, when running with multiple --runs_per_test and a test is detected as flaky, exit immediately rather than running all --runs_per_test")
)
// setNumFilesLimit changes the NOFILE soft rlimit if it is too high.
@@ -123,7 +127,7 @@ func main() {
case <-done:
return
case <-timer.C:
- log.Println("The timeout duration is exceeded")
+ log.Println("The batch timeout duration is exceeded")
killed := false
for _, cmd := range cmds {
p := cmd.Process
@@ -140,13 +144,73 @@ func main() {
// Let tests to handle signals
time.Sleep(5 * time.Second)
}
- panic("FAIL: The timeout duration is exceeded")
+ panic("FAIL: The batch timeout duration is exceeded")
}
}()
+ numIterations := *runsPerTest
+ if numIterations == 0 {
+ numIterations = 1
+ }
for _, cmd := range cmds {
- cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
- if err := cmd.Run(); err != nil {
- log.Fatalf("FAIL: %v", err)
+ iterations := 0
+ successes := 0
+ var firstFailure error
+ for iteration := 1; iteration <= *runsPerTest; iteration++ {
+ // Make a copy of the command, as the same exec.Cmd object cannot be started multiple times.
+ cmdCopy := *cmd
+
+ // Handle test timeout.
+ testDone := make(chan struct{})
+ testTimedOutCh := make(chan bool, 1)
+ if *perTestTimeout != 0 {
+ go func() {
+ timer := time.NewTimer(*perTestTimeout)
+ defer timer.Stop()
+ select {
+ case <-timer.C:
+ testTimedOutCh <- true
+ cmdCopy.Process.Kill()
+ case <-done:
+ testTimedOutCh <- false
+ case <-testDone:
+ testTimedOutCh <- false
+ }
+ }()
+ }
+
+ // Run the test.
+ cmdCopy.Stdout, cmdCopy.Stderr = os.Stdout, os.Stderr
+ testErr := cmdCopy.Run()
+ close(testDone)
+ if <-testTimedOutCh {
+ testErr = fmt.Errorf("test timed out after %v", *perTestTimeout)
+ }
+
+ // Tally result.
+ iterations++
+ if testErr == nil {
+ successes++
+ } else if firstFailure == nil {
+ firstFailure = testErr
+ }
+ if *flakyShortCircuit && successes > 0 && firstFailure != nil {
+ break
+ }
+ }
+ if successes > 0 && firstFailure != nil {
+ // Test is flaky.
+ if *flakyIsError {
+ log.Fatalf("FLAKY: %v (%d failures out of %d)", firstFailure, iterations-successes, iterations)
+ } else {
+ log.Println(fmt.Sprintf("FLAKY: %v (%d failures out of %d)", firstFailure, iterations-successes, iterations))
+ }
+ } else if successes == 0 && firstFailure != nil {
+ // Test is 100% failing.
+ log.Fatalf("FAIL: %v", firstFailure)
+ } else if successes > 0 && firstFailure == nil {
+ // Test is 100% succeeding, do nothing.
+ } else {
+ log.Fatalf("Internal logic error")
}
}
}
diff --git a/test/runtimes/runner/BUILD b/test/runtimes/runner/BUILD
index 70cc01594..091aff1f2 100644
--- a/test/runtimes/runner/BUILD
+++ b/test/runtimes/runner/BUILD
@@ -7,5 +7,8 @@ go_binary(
testonly = 1,
srcs = ["main.go"],
visibility = ["//test/runtimes:__pkg__"],
- deps = ["//test/runtimes/runner/lib"],
+ deps = [
+ "//pkg/test/testutil",
+ "//test/runtimes/runner/lib",
+ ],
)
diff --git a/test/runtimes/runner/lib/exclude_test.go b/test/runtimes/runner/lib/exclude_test.go
index f996e895b..0f04460dd 100644
--- a/test/runtimes/runner/lib/exclude_test.go
+++ b/test/runtimes/runner/lib/exclude_test.go
@@ -29,11 +29,8 @@ func TestMain(m *testing.M) {
// Test that the exclude file parses without error.
func TestExcludelist(t *testing.T) {
- ex, err := getExcludes(*excludeFile)
+ _, err := ExcludeFilter(*excludeFile)
if err != nil {
t.Fatalf("error parsing exclude file: %v", err)
}
- if *excludeFile != "" && len(ex) == 0 {
- t.Errorf("got empty excludes for file %q", *excludeFile)
- }
}
diff --git a/test/runtimes/runner/lib/lib.go b/test/runtimes/runner/lib/lib.go
index 6c974cb3a..a25fc1dc6 100644
--- a/test/runtimes/runner/lib/lib.go
+++ b/test/runtimes/runner/lib/lib.go
@@ -32,20 +32,42 @@ import (
"gvisor.dev/gvisor/pkg/test/testutil"
)
+// ProctorSettings contains settings passed directly to the proctor process.
+type ProctorSettings struct {
+ // PerTestTimeout is the timeout for each individual test.
+ PerTestTimeout time.Duration
+ // RunsPerTest is the number of times to run each test.
+ // A value of 0 is the same as a value of 1, i.e. "run once".
+ RunsPerTest int
+ // If FlakyIsError is true, a flaky test will be considered as a failure.
+ // If it is false, a flaky test will be considered as passing.
+ FlakyIsError bool
+ // If FlakyShortCircuit is true, when runnins with RunsPerTest > 1 and a test is detected as
+ // flaky, exit immediately rather than running for all RunsPerTest attempts.
+ FlakyShortCircuit bool
+}
+
+// ToArgs converts these settings to command-line arguments to pass to the proctor binary.
+func (p ProctorSettings) ToArgs() []string {
+ return []string{
+ fmt.Sprintf("--per_test_timeout=%v", p.PerTestTimeout),
+ fmt.Sprintf("--runs_per_test=%d", p.RunsPerTest),
+ fmt.Sprintf("--flaky_is_error=%v", p.FlakyIsError),
+ fmt.Sprintf("--flaky_short_circuit=%v", p.FlakyShortCircuit),
+ }
+}
+
+// Filter is a predicate function for filtering tests.
+// It returns true if the given test name should be run.
+type Filter func(test string) bool
+
// RunTests is a helper that is called by main. It exists so that we can run
// defered functions before exiting. It returns an exit code that should be
// passed to os.Exit.
-func RunTests(lang, image, excludeFile string, batchSize int, timeout time.Duration) int {
+func RunTests(lang, image string, filter Filter, batchSize int, timeout time.Duration, proctorSettings ProctorSettings) int {
// TODO(gvisor.dev/issue/1624): Remove those tests from all exclude lists
// that only fail with VFS1.
- // Get tests to exclude.
- excludes, err := getExcludes(excludeFile)
- if err != nil {
- fmt.Fprintf(os.Stderr, "Error getting exclude list: %s\n", err.Error())
- return 1
- }
-
// Construct the shared docker instance.
ctx := context.Background()
d := dockerutil.MakeContainer(ctx, testutil.DefaultLogger(lang))
@@ -63,7 +85,7 @@ func RunTests(lang, image, excludeFile string, batchSize int, timeout time.Durat
// Get a slice of tests to run. This will also start a single Docker
// container that will be used to run each test. The final test will
// stop the Docker container.
- tests, err := getTests(ctx, d, lang, image, batchSize, timeoutChan, timeout, excludes)
+ tests, err := getTests(ctx, d, lang, image, batchSize, timeoutChan, timeout, filter, proctorSettings)
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err.Error())
return 1
@@ -73,7 +95,7 @@ func RunTests(lang, image, excludeFile string, batchSize int, timeout time.Durat
}
// getTests executes all tests as table tests.
-func getTests(ctx context.Context, d *dockerutil.Container, lang, image string, batchSize int, timeoutChan chan struct{}, timeout time.Duration, excludes map[string]struct{}) ([]testing.InternalTest, error) {
+func getTests(ctx context.Context, d *dockerutil.Container, lang, image string, batchSize int, timeoutChan chan struct{}, timeout time.Duration, filter Filter, proctorSettings ProctorSettings) ([]testing.InternalTest, error) {
startTime := time.Now()
// Start the container.
@@ -110,6 +132,19 @@ func getTests(ctx context.Context, d *dockerutil.Container, lang, image string,
if err != nil {
return nil, fmt.Errorf("TestsForShard() failed: %v", err)
}
+ indicesMap := make(map[int]struct{}, len(indices))
+ for _, i := range indices {
+ indicesMap[i] = struct{}{}
+ }
+ var testsNotInShard []string
+ for i, tc := range tests {
+ if _, found := indicesMap[i]; !found {
+ testsNotInShard = append(testsNotInShard, tc)
+ }
+ }
+ if len(testsNotInShard) > 0 {
+ log.Infof("Tests not in this shard: %s", strings.Join(testsNotInShard, ","))
+ }
var itests []testing.InternalTest
for i := 0; i < len(indices); i += batchSize {
@@ -119,8 +154,8 @@ func getTests(ctx context.Context, d *dockerutil.Container, lang, image string,
end = len(indices)
}
for _, tc := range indices[i:end] {
- // Add test if not excluded.
- if _, ok := excludes[tests[tc]]; ok {
+ // Add test if not filtered.
+ if filter != nil && !filter(tests[tc]) {
log.Infof("Skipping test case %s\n", tests[tc])
continue
}
@@ -147,14 +182,16 @@ func getTests(ctx context.Context, d *dockerutil.Container, lang, image string,
if !state.Running {
t.Fatalf("container is not running: state = %s", state.Status)
}
+ log.Infof("Running test case batch: %s", strings.Join(tcs, ","))
go func() {
- output, err = d.Exec(
- ctx, dockerutil.ExecOpts{},
+ argv := []string{
"/proctor/proctor", "--runtime", lang,
"--tests", strings.Join(tcs, ","),
fmt.Sprintf("--timeout=%s", timeout-time.Since(startTime)),
- )
+ }
+ argv = append(argv, proctorSettings.ToArgs()...)
+ output, err = d.Exec(ctx, dockerutil.ExecOpts{}, argv...)
close(done)
}()
@@ -176,12 +213,12 @@ func getTests(ctx context.Context, d *dockerutil.Container, lang, image string,
return itests, nil
}
-// getExcludes reads the exclude file and returns a set of test names to
-// exclude.
-func getExcludes(excludeFile string) (map[string]struct{}, error) {
+// ExcludeFilter reads the exclude file and returns a filter that excludes the tests listed in
+// the given CSV file.
+func ExcludeFilter(excludeFile string) (Filter, error) {
excludes := make(map[string]struct{})
if excludeFile == "" {
- return excludes, nil
+ return nil, nil
}
f, err := os.Open(excludeFile)
if err != nil {
@@ -206,7 +243,10 @@ func getExcludes(excludeFile string) (map[string]struct{}, error) {
}
excludes[record[0]] = struct{}{}
}
- return excludes, nil
+ return func(test string) bool {
+ _, found := excludes[test]
+ return !found
+ }, nil
}
// testDeps implements testing.testDeps (an unexported interface), and is
diff --git a/test/runtimes/runner/main.go b/test/runtimes/runner/main.go
index 6a2eb2859..675639082 100644
--- a/test/runtimes/runner/main.go
+++ b/test/runtimes/runner/main.go
@@ -19,17 +19,24 @@ import (
"flag"
"fmt"
"os"
+ "strings"
"time"
+ "gvisor.dev/gvisor/pkg/test/testutil"
"gvisor.dev/gvisor/test/runtimes/runner/lib"
)
var (
- lang = flag.String("lang", "", "language runtime to test")
- image = flag.String("image", "", "docker image with runtime tests")
- excludeFile = flag.String("exclude_file", "", "file containing list of tests to exclude, in CSV format with fields: test name, bug id, comment")
- batchSize = flag.Int("batch", 50, "number of test cases run in one command")
- timeout = flag.Duration("timeout", 20*time.Minute, "batch timeout")
+ lang = flag.String("lang", "", "language runtime to test")
+ image = flag.String("image", "", "docker image with runtime tests")
+ excludeFile = flag.String("exclude_file", "", "file containing list of tests to exclude, in CSV format with fields: test name, bug id, comment")
+ onlyTests = flag.String("tests", testutil.StringFromEnv("RUNTIME_TESTS_FILTER", ""), "if specified, runs only the given comma-separated list of test names, even those in --exclude_file")
+ batchSize = flag.Int("batch", 50, "number of test cases run in one command")
+ timeout = flag.Duration("timeout", 20*time.Minute, "batch timeout")
+ perTestTimeout = flag.Duration("per_test_timeout", testutil.DurationFromEnv("RUNTIME_TESTS_PER_TEST_TIMEOUT", 20*time.Minute), "per-test timeout (a value of 0 disables per-test timeouts)")
+ runsPerTest = flag.Int("runs_per_test", testutil.IntFromEnv("RUNTIME_TESTS_RUNS_PER_TEST", 1), "number of times to run each test (a value of 0 is the same as a value of 1, i.e. running once)")
+ flakyIsError = flag.Bool("flaky_is_error", testutil.BoolFromEnv("RUNTIME_TESTS_FLAKY_IS_ERROR", true), "if true, when running with multiple --runs_per_test, tests with inconsistent status will result in a failure status code for the batch; if false, they will be considered as passing")
+ flakyShortCircuit = flag.Bool("flaky_short_circuit", testutil.BoolFromEnv("RUNTIME_TESTS_FLAKY_SHORT_CIRCUIT", true), "if true, when running with multiple --runs_per_test and a test is detected as flaky, exit immediately rather than running all --runs_per_test")
)
func main() {
@@ -38,5 +45,29 @@ func main() {
fmt.Fprintf(os.Stderr, "lang and image flags must not be empty\n")
os.Exit(1)
}
- os.Exit(lib.RunTests(*lang, *image, *excludeFile, *batchSize, *timeout))
+ proctorSettings := lib.ProctorSettings{
+ PerTestTimeout: *perTestTimeout,
+ RunsPerTest: *runsPerTest,
+ FlakyIsError: *flakyIsError,
+ FlakyShortCircuit: *flakyShortCircuit,
+ }
+ var filter lib.Filter
+ if *excludeFile != "" {
+ excludeFilter, err := lib.ExcludeFilter(*excludeFile)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error getting exclude list: %s\n", err.Error())
+ os.Exit(1)
+ }
+ filter = excludeFilter
+ }
+ if *onlyTests != "" {
+ tests := make(map[string]bool)
+ for _, test := range strings.Split(*onlyTests, ",") {
+ tests[test] = true
+ }
+ filter = func(test string) bool {
+ return tests[test]
+ }
+ }
+ os.Exit(lib.RunTests(*lang, *image, filter, *batchSize, *timeout, proctorSettings))
}