Add support for excluding python runtime test cases.

Earlier entire test suites are excluded for python runtime tests. These suites
contain thousands of test cases. Only a handful fail but the entire suite is
excluded. For python we run tests on test suite granularity because running
them per test-case basis is too slow.

This change takes a hybrid approach. The test suites with no exclusions are run
together in one invocation (fast). The suites that have exclusions need more
work. We list all their test cases, extract un-excluded ones, stitch them
together and run them in one python invocation for each suite.

In effect, this change enables 3594 python runtime test cases!

PiperOrigin-RevId: 515404441

test/runtimes/exclude/python3.10.2.csv

@@ -1,11 +1 @@
-test name,bug id,comment
-test_asyncio,,Broken test
-test_asyncore,b/162973328,
-test_fcntl,b/162978767,fcntl invalid argument -- artificial test to make sure something works in 64 bit mode.
-test_posix,b/76174079,posix.sched_get_priority_min not implemented + posix.sched_rr_get_interval not permitted
-test_pty,b/162979921,
-test_readline,b/162980389,TestReadline hangs forever
-test_resource,b/76174079,
-test_signal,,Flaky - signal: alarm clock
-test_socket,,Broken test
-test_os,b/271473320,TestScandir.test_attributes fails with overlay
+Python excludes test cases differently. See exclude list and justification in proctor/lib/python.go.

test/runtimes/proctor/lib/python.go

@@ -18,9 +18,117 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"sort"
 	"strings"
 )
 
+// exclude holds test cases that fail in gVisor for various reasons and should
+// be excluded. The format of this map is [test library] -> [test cases...].
+// We need to handle test exclusion differently for python. Running python
+// tests on a test-case basis using `./python -m test --fromfile FILE` is too
+// slow. For example, it took 0.065s seconds to run
+// `./python -m test test_grammar` but it took 9.5 seconds to run the same set
+// of tests using `--fromfile`. The fast way of running a test library while
+// excluding a few test cases is to run `./python -m {lib} {test cases...}`. We
+// don't `--list-cases` + the CSV exclude file mechanism because tests are
+// selected from the list based on sharding/partition. We end up getting a
+// small list of tests from various test libraries. Tests from same library
+// are not grouped well together. So instead, we use `--list-tests` (which
+// lists test libraries). When running a test library, using this map we
+// generate a command to run all un-excluded tests from that library together.
+var exclude = map[string][]string{
+	// TODO(b/271473320): Un-exclude once this bug is fixed. Fails with overlay.
+	"test_os": []string{"TestScandir.test_attributes"},
+	// Broken test. Fails with runc too.
+	"test_asyncio.test_base_events": []string{
+		"BaseEventLoopWithSelectorTests.test_create_connection_service_name",
+	},
+	// TODO(b/271950879): Un-exclude once this bug is fixed.
+	"test_asyncio.test_events": []string{
+		"EPollEventLoopTests.test_bidirectional_pty",
+		"PollEventLoopTests.test_bidirectional_pty",
+		"SelectEventLoopTests.test_bidirectional_pty",
+	},
+	// TODO(b/162973328): Un-exclude once this bug is fixed.
+	"test_asyncore": []string{
+		"TestAPI_UseIPv4Poll.test_handle_expt",
+		"TestAPI_UseIPv4Select.test_handle_expt",
+	},
+	// TODO(b/162978767): Un-exclude once this bug is fixed.
+	"test_fcntl": []string{"TestFcntl.test_fcntl_64_bit"},
+	// TODO(b/76174079): Un-exclude once this bug is fixed.
+	"test_posix": []string{
+		"PosixTester.test_fs_holes",
+		"PosixTester.test_sched_priority",
+		"PosixTester.test_sched_rr_get_interval",
+		"TestPosixSpawn.test_setscheduler_only_param",
+		"TestPosixSpawnP.test_setscheduler_only_param",
+	},
+	// TODO(b/162979921): Un-exclude once this bug is fixed.
+	"test_pty": []string{
+		"PtyTest.test_fork",
+		"PtyTest.test_master_read",
+		"PtyTest.test_spawn_doesnt_hang",
+	},
+	// TODO(b/162980389): Un-exclude once this bug is fixed.
+	"test_readline": []string{"TestReadline.*"},
+	// TODO(b/76174079): Un-exclude once this bug is fixed.
+	"test_resource": []string{"ResourceTest.test_prlimit"},
+	// TODO(b/271949964): Un-exclude test cases as they are fixed.
+	"test_socket": []string{
+		"BasicUDPLITETest.testRecvFrom",
+		"BasicUDPLITETest.testRecvFromNegative",
+		"BasicUDPLITETest.testSendtoAndRecv",
+		"GeneralModuleTests.testGetServBy",
+		"GeneralModuleTests.testGetaddrinfo",
+		"RecvmsgIntoUDPLITETest.testRecvmsg",
+		"RecvmsgIntoUDPLITETest.testRecvmsgAfterClose",
+		"RecvmsgIntoUDPLITETest.testRecvmsgExplicitDefaults",
+		"RecvmsgIntoUDPLITETest.testRecvmsgFromSendmsg",
+		"RecvmsgIntoUDPLITETest.testRecvmsgIntoArray",
+		"RecvmsgIntoUDPLITETest.testRecvmsgIntoBadArgs",
+		"RecvmsgIntoUDPLITETest.testRecvmsgIntoGenerator",
+		"RecvmsgIntoUDPLITETest.testRecvmsgIntoScatter",
+		"RecvmsgIntoUDPLITETest.testRecvmsgLongAncillaryBuf",
+		"RecvmsgIntoUDPLITETest.testRecvmsgPeek",
+		"RecvmsgIntoUDPLITETest.testRecvmsgShortAncillaryBuf",
+		"RecvmsgIntoUDPLITETest.testRecvmsgShorter",
+		"RecvmsgIntoUDPLITETest.testRecvmsgTimeout",
+		"RecvmsgIntoUDPLITETest.testRecvmsgTrunc",
+		"RecvmsgUDPLITETest.testRecvmsg",
+		"RecvmsgUDPLITETest.testRecvmsgAfterClose",
+		"RecvmsgUDPLITETest.testRecvmsgBadArgs",
+		"RecvmsgUDPLITETest.testRecvmsgExplicitDefaults",
+		"RecvmsgUDPLITETest.testRecvmsgFromSendmsg",
+		"RecvmsgUDPLITETest.testRecvmsgLongAncillaryBuf",
+		"RecvmsgUDPLITETest.testRecvmsgPeek",
+		"RecvmsgUDPLITETest.testRecvmsgShortAncillaryBuf",
+		"RecvmsgUDPLITETest.testRecvmsgShorter",
+		"RecvmsgUDPLITETest.testRecvmsgTimeout",
+		"RecvmsgUDPLITETest.testRecvmsgTrunc",
+		"SendmsgUDPLITETest.testSendmsg",
+		"SendmsgUDPLITETest.testSendmsgAfterClose",
+		"SendmsgUDPLITETest.testSendmsgAncillaryGenerator",
+		"SendmsgUDPLITETest.testSendmsgArray",
+		"SendmsgUDPLITETest.testSendmsgBadArgs",
+		"SendmsgUDPLITETest.testSendmsgBadCmsg",
+		"SendmsgUDPLITETest.testSendmsgBadMultiCmsg",
+		"SendmsgUDPLITETest.testSendmsgDataGenerator",
+		"SendmsgUDPLITETest.testSendmsgExcessCmsgReject",
+		"SendmsgUDPLITETest.testSendmsgGather",
+		"SendmsgUDPLITETest.testSendmsgNoDestAddr",
+		"UDPLITETimeoutTest.testTimeoutZero",
+		"UDPLITETimeoutTest.testUDPLITETimeout",
+	},
+}
+
+// Some python test libraries contain other test libraries that have test cases
+// that need to be excluded. We need to expand such libraries so that the test
+// case exclusion can work correctly.
+var expand = []string{
+	"test_asyncio",
+}
+
 // pythonRunner implements TestRunner for Python.
 type pythonRunner struct{}
 
@@ -35,15 +143,92 @@ func (pythonRunner) ListTests() ([]string, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to list: %v", err)
 	}
-	var toolSlice []string
-	for _, test := range strings.Split(string(out), "\n") {
-		toolSlice = append(toolSlice, test)
+	testLibs := make(map[string]struct{})
+	for _, testLib := range strings.Split(string(out), "\n") {
+		if len(testLib) == 0 {
+			continue
+		}
+		testLibs[testLib] = struct{}{}
 	}
-	return toolSlice, nil
+	for _, libToExpand := range expand {
+		if _, ok := testLibs[libToExpand]; !ok {
+			return nil, fmt.Errorf("%s test library was not listed", libToExpand)
+		}
+		delete(testLibs, libToExpand)
+		subLibs, err := subTestLibs(libToExpand)
+		if err != nil {
+			return nil, err
+		}
+		for subLib := range subLibs {
+			testLibs[fmt.Sprintf("%s.%s", libToExpand, subLib)] = struct{}{}
+		}
+	}
+	res := make([]string, 0, len(testLibs))
+	for lib := range testLibs {
+		res = append(res, lib)
+	}
+	// Sort to have deterministic results across shards.
+	sort.Strings(res)
+	return res, nil
+}
+
+func subTestLibs(testLib string) (map[string]struct{}, error) {
+	// --list-{tests/cases} is only implemented by the 'test' library.
+	// Running './python -m test {X} --list-tests' does not list libraries inside
+	// X library. We need to list all test cases and extract sub libraries.
+	args := []string{"-m", "test", testLib, "--list-cases"}
+	cmd := exec.Command("./python", args...)
+	cmd.Stderr = os.Stderr
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("failed to list: %v", err)
+	}
+	subLibs := make(map[string]struct{})
+	for _, tc := range strings.Split(string(out), "\n") {
+		if len(tc) == 0 {
+			continue
+		}
+		idx := strings.Index(tc, testLib)
+		if idx < 0 {
+			return nil, fmt.Errorf("could not find %q library in test case %q", testLib, tc)
+		}
+		subLibs[strings.Split(tc[idx:], ".")[1]] = struct{}{}
+	}
+	return subLibs, nil
 }
 
 // TestCmds implements TestRunner.TestCmds.
 func (pythonRunner) TestCmds(tests []string) []*exec.Cmd {
-	args := append([]string{"-m", "test"}, tests...)
-	return []*exec.Cmd{exec.Command("./python", args...)}
+	var cmds []*exec.Cmd
+	var full []string
+	for _, testModule := range tests {
+		if excludeTCs, ok := exclude[testModule]; ok {
+			cmds = append(cmds, testCmdWithExclTCs(testModule, excludeTCs))
+		} else {
+			full = append(full, testModule)
+		}
+	}
+	if len(full) > 0 {
+		// Run all test modules (that have no excludes) together for speed.
+		// Running them individually with a new exec.Cmd takes longer.
+		args := append([]string{"-m", "test"}, full...)
+		cmds = append(cmds, exec.Command("./python", args...))
+	}
+	return cmds
+}
+
+func testCmdWithExclTCs(testModule string, excludeTCs []string) *exec.Cmd {
+	shellPipeline := []string{
+		// List all test cases in this module.
+		fmt.Sprintf("./python -m test %s --list-cases", testModule),
+		// Exclude the test cases. Note that '$' was added after each excluded test
+		// case to be exact about which test case to exclude.
+		fmt.Sprintf("grep -v \"%s$\"", strings.Join(excludeTCs, "$\\|")),
+		// Remove the "test.{testModule}." prefix.
+		fmt.Sprintf("sed -e \"s/^test.%s.//\"", testModule),
+		// Run all un-excluded test cases in one command.
+		fmt.Sprintf("xargs ./python -m test.%s", testModule),
+	}
+	args := []string{"-c", strings.Join(shellPipeline, " | ")}
+	return exec.Command("sh", args...)
 }