Add blacklist support to the runtime test runner.

Tests in the blacklist will be explicitly skipped (with associated log line).

Checks in a blacklist for the nodejs tests.

PiperOrigin-RevId: 272272749

test/runtimes/BUILD

@@ -26,6 +26,7 @@ runtime_test(
 )
 
 runtime_test(
+    blacklist_file = "blacklist_nodejs12.4.0.csv",
     image = "gcr.io/gvisor-presubmit/nodejs12.4.0",
     lang = "nodejs",
 )

test/runtimes/blacklist_nodejs12.4.0.csv

@@ -0,0 +1,47 @@
+test name,bug id,comment
+benchmark/test-benchmark-fs.js,,
+benchmark/test-benchmark-module.js,,
+benchmark/test-benchmark-napi.js,,
+doctool/test-make-doc.js,b/68848110,Expected to fail.
+fixtures/test-error-first-line-offset.js,,
+fixtures/test-fs-readfile-error.js,,
+fixtures/test-fs-stat-sync-overflow.js,,
+internet/test-dgram-broadcast-multi-process.js,,
+internet/test-dgram-multicast-multi-process.js,,
+internet/test-dgram-multicast-set-interface-lo.js,,
+parallel/test-cluster-dgram-reuse.js,b/64024294,
+parallel/test-dgram-bind-fd.js,b/132447356,
+parallel/test-dgram-create-socket-handle-fd.js,b/132447238,
+parallel/test-dgram-createSocket-type.js,b/68847739,
+parallel/test-dgram-socket-buffer-size.js,b/68847921,
+parallel/test-fs-access.js,,
+parallel/test-fs-write-stream-double-close.js
+parallel/test-fs-write-stream-throw-type-error.js,b/110226209,
+parallel/test-fs-write-stream.js,,
+parallel/test-http2-respond-file-error-pipe-offset.js,,
+parallel/test-os.js,,
+parallel/test-process-uid-gid.js,,
+pseudo-tty/test-assert-colors.js,,
+pseudo-tty/test-assert-no-color.js,,
+pseudo-tty/test-assert-position-indicator.js,,
+pseudo-tty/test-async-wrap-getasyncid-tty.js,,
+pseudo-tty/test-fatal-error.js,,
+pseudo-tty/test-handle-wrap-isrefed-tty.js,,
+pseudo-tty/test-readable-tty-keepalive.js,,
+pseudo-tty/test-set-raw-mode-reset-process-exit.js,,
+pseudo-tty/test-set-raw-mode-reset-signal.js,,
+pseudo-tty/test-set-raw-mode-reset.js,,
+pseudo-tty/test-stderr-stdout-handle-sigwinch.js,,
+pseudo-tty/test-stdout-read.js,,
+pseudo-tty/test-tty-color-support.js,,
+pseudo-tty/test-tty-isatty.js,,
+pseudo-tty/test-tty-stdin-call-end.js,,
+pseudo-tty/test-tty-stdin-end.js,,
+pseudo-tty/test-stdin-write.js,,
+pseudo-tty/test-tty-stdout-end.js,,
+pseudo-tty/test-tty-stdout-resize.js,,
+pseudo-tty/test-tty-stream-constructors.js,,
+pseudo-tty/test-tty-window-size.js,,
+pseudo-tty/test-tty-wrap.js,,
+pummel/test-net-pingpong.js,,
+pummel/test-vm-memleak.js,,

test/runtimes/build_defs.bzl

@@ -6,19 +6,26 @@ def runtime_test(
         lang,
         image,
         shard_count = 50,
-        size = "enormous"):
+        size = "enormous",
+        blacklist_file = ""):
+    args = [
+        "--lang",
+        lang,
+        "--image",
+        image,
+    ]
+    data = [
+        ":runner",
+    ]
+    if blacklist_file != "":
+        args += ["--blacklist_file", "test/runtimes/" + blacklist_file]
+        data += [blacklist_file]
+
     sh_test(
         name = lang + "_test",
         srcs = ["runner.sh"],
-        args = [
-            "--lang",
-            lang,
-            "--image",
-            image,
-        ],
-        data = [
-            ":runner",
-        ],
+        args = args,
+        data = data,
         size = size,
         shard_count = shard_count,
         tags = [

test/runtimes/runner.go

@@ -16,6 +16,7 @@
 package main
 
 import (
+	"encoding/csv"
 	"flag"
 	"fmt"
 	"io"
@@ -31,8 +32,9 @@ import (
 )
 
 var (
-	lang  = flag.String("lang", "", "language runtime to test")
-	image = flag.String("image", "", "docker image with runtime tests")
+	lang          = flag.String("lang", "", "language runtime to test")
+	image         = flag.String("image", "", "docker image with runtime tests")
+	blacklistFile = flag.String("blacklist_file", "", "file containing blacklist of tests to exclude, in CSV format with fields: test name, bug id, comment")
 )
 
 // Wait time for each test to run.
@@ -52,6 +54,13 @@ func main() {
 // defered functions before exiting. It returns an exit code that should be
 // passed to os.Exit.
 func runTests() int {
+	// Get tests to blacklist.
+	blacklist, err := getBlacklist()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error getting blacklist: %s\n", err.Error())
+		return 1
+	}
+
 	// Create a single docker container that will be used for all tests.
 	d := dockerutil.MakeDocker("gvisor-" + *lang)
 	defer d.CleanUp()
@@ -59,7 +68,7 @@ func runTests() int {
 	// Get a slice of tests to run. This will also start a single Docker
 	// container that will be used to run each test. The final test will
 	// stop the Docker container.
-	tests, err := getTests(d)
+	tests, err := getTests(d, blacklist)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%s\n", err.Error())
 		return 1
@@ -71,7 +80,7 @@ func runTests() int {
 
 // getTests returns a slice of tests to run, subject to the shard size and
 // index.
-func getTests(d dockerutil.Docker) ([]testing.InternalTest, error) {
+func getTests(d dockerutil.Docker, blacklist map[string]struct{}) ([]testing.InternalTest, error) {
 	// Pull the image.
 	if err := dockerutil.Pull(*image); err != nil {
 		return nil, fmt.Errorf("docker pull %q failed: %v", *image, err)
@@ -106,12 +115,18 @@ func getTests(d dockerutil.Docker) ([]testing.InternalTest, error) {
 		itests = append(itests, testing.InternalTest{
 			Name: tc,
 			F: func(t *testing.T) {
+				// Is the test blacklisted?
+				if _, ok := blacklist[tc]; ok {
+					t.Skip("SKIP: blacklisted test %q", tc)
+				}
+
 				var (
 					now    = time.Now()
 					done   = make(chan struct{})
 					output string
 					err    error
 				)
+
 				go func() {
 					fmt.Printf("RUNNING %s...\n", tc)
 					output, err = d.Exec("/proctor", "--runtime", *lang, "--test", tc)
@@ -134,6 +149,43 @@ func getTests(d dockerutil.Docker) ([]testing.InternalTest, error) {
 	return itests, nil
 }
 
+// getBlacklist reads the blacklist file and returns a set of test names to
+// exclude.
+func getBlacklist() (map[string]struct{}, error) {
+	blacklist := make(map[string]struct{})
+	if *blacklistFile == "" {
+		return blacklist, nil
+	}
+	file, err := testutil.FindFile(*blacklistFile)
+	if err != nil {
+		return nil, err
+	}
+	f, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	r := csv.NewReader(f)
+
+	// First line is header. Skip it.
+	if _, err := r.Read(); err != nil {
+		return nil, err
+	}
+
+	for {
+		record, err := r.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		blacklist[record[0]] = struct{}{}
+	}
+	return blacklist, nil
+}
+
 // testDeps implements testing.testDeps (an unexported interface), and is
 // required to use testing.MainStart.
 type testDeps struct{}