diff --git a/test/runtimes/exclude/python3.10.2.csv b/test/runtimes/exclude/python3.10.2.csv index 6ca462a12..5b80bc2c9 100644 --- a/test/runtimes/exclude/python3.10.2.csv +++ b/test/runtimes/exclude/python3.10.2.csv @@ -1,11 +1 @@ -test name,bug id,comment -test_asyncio,,Broken test -test_asyncore,b/162973328, -test_fcntl,b/162978767,fcntl invalid argument -- artificial test to make sure something works in 64 bit mode. -test_posix,b/76174079,posix.sched_get_priority_min not implemented + posix.sched_rr_get_interval not permitted -test_pty,b/162979921, -test_readline,b/162980389,TestReadline hangs forever -test_resource,b/76174079, -test_signal,,Flaky - signal: alarm clock -test_socket,,Broken test -test_os,b/271473320,TestScandir.test_attributes fails with overlay +Python excludes test cases differently. See exclude list and justification in proctor/lib/python.go. diff --git a/test/runtimes/proctor/lib/python.go b/test/runtimes/proctor/lib/python.go index 429bfd850..985420597 100644 --- a/test/runtimes/proctor/lib/python.go +++ b/test/runtimes/proctor/lib/python.go @@ -18,9 +18,117 @@ import ( "fmt" "os" "os/exec" + "sort" "strings" ) +// exclude holds test cases that fail in gVisor for various reasons and should +// be excluded. The format of this map is [test library] -> [test cases...]. +// We need to handle test exclusion differently for python. Running python +// tests on a test-case basis using `./python -m test --fromfile FILE` is too +// slow. For example, it took 0.065s seconds to run +// `./python -m test test_grammar` but it took 9.5 seconds to run the same set +// of tests using `--fromfile`. The fast way of running a test library while +// excluding a few test cases is to run `./python -m {lib} {test cases...}`. We +// don't `--list-cases` + the CSV exclude file mechanism because tests are +// selected from the list based on sharding/partition. We end up getting a +// small list of tests from various test libraries. Tests from same library +// are not grouped well together. So instead, we use `--list-tests` (which +// lists test libraries). When running a test library, using this map we +// generate a command to run all un-excluded tests from that library together. +var exclude = map[string][]string{ + // TODO(b/271473320): Un-exclude once this bug is fixed. Fails with overlay. + "test_os": []string{"TestScandir.test_attributes"}, + // Broken test. Fails with runc too. + "test_asyncio.test_base_events": []string{ + "BaseEventLoopWithSelectorTests.test_create_connection_service_name", + }, + // TODO(b/271950879): Un-exclude once this bug is fixed. + "test_asyncio.test_events": []string{ + "EPollEventLoopTests.test_bidirectional_pty", + "PollEventLoopTests.test_bidirectional_pty", + "SelectEventLoopTests.test_bidirectional_pty", + }, + // TODO(b/162973328): Un-exclude once this bug is fixed. + "test_asyncore": []string{ + "TestAPI_UseIPv4Poll.test_handle_expt", + "TestAPI_UseIPv4Select.test_handle_expt", + }, + // TODO(b/162978767): Un-exclude once this bug is fixed. + "test_fcntl": []string{"TestFcntl.test_fcntl_64_bit"}, + // TODO(b/76174079): Un-exclude once this bug is fixed. + "test_posix": []string{ + "PosixTester.test_fs_holes", + "PosixTester.test_sched_priority", + "PosixTester.test_sched_rr_get_interval", + "TestPosixSpawn.test_setscheduler_only_param", + "TestPosixSpawnP.test_setscheduler_only_param", + }, + // TODO(b/162979921): Un-exclude once this bug is fixed. + "test_pty": []string{ + "PtyTest.test_fork", + "PtyTest.test_master_read", + "PtyTest.test_spawn_doesnt_hang", + }, + // TODO(b/162980389): Un-exclude once this bug is fixed. + "test_readline": []string{"TestReadline.*"}, + // TODO(b/76174079): Un-exclude once this bug is fixed. + "test_resource": []string{"ResourceTest.test_prlimit"}, + // TODO(b/271949964): Un-exclude test cases as they are fixed. + "test_socket": []string{ + "BasicUDPLITETest.testRecvFrom", + "BasicUDPLITETest.testRecvFromNegative", + "BasicUDPLITETest.testSendtoAndRecv", + "GeneralModuleTests.testGetServBy", + "GeneralModuleTests.testGetaddrinfo", + "RecvmsgIntoUDPLITETest.testRecvmsg", + "RecvmsgIntoUDPLITETest.testRecvmsgAfterClose", + "RecvmsgIntoUDPLITETest.testRecvmsgExplicitDefaults", + "RecvmsgIntoUDPLITETest.testRecvmsgFromSendmsg", + "RecvmsgIntoUDPLITETest.testRecvmsgIntoArray", + "RecvmsgIntoUDPLITETest.testRecvmsgIntoBadArgs", + "RecvmsgIntoUDPLITETest.testRecvmsgIntoGenerator", + "RecvmsgIntoUDPLITETest.testRecvmsgIntoScatter", + "RecvmsgIntoUDPLITETest.testRecvmsgLongAncillaryBuf", + "RecvmsgIntoUDPLITETest.testRecvmsgPeek", + "RecvmsgIntoUDPLITETest.testRecvmsgShortAncillaryBuf", + "RecvmsgIntoUDPLITETest.testRecvmsgShorter", + "RecvmsgIntoUDPLITETest.testRecvmsgTimeout", + "RecvmsgIntoUDPLITETest.testRecvmsgTrunc", + "RecvmsgUDPLITETest.testRecvmsg", + "RecvmsgUDPLITETest.testRecvmsgAfterClose", + "RecvmsgUDPLITETest.testRecvmsgBadArgs", + "RecvmsgUDPLITETest.testRecvmsgExplicitDefaults", + "RecvmsgUDPLITETest.testRecvmsgFromSendmsg", + "RecvmsgUDPLITETest.testRecvmsgLongAncillaryBuf", + "RecvmsgUDPLITETest.testRecvmsgPeek", + "RecvmsgUDPLITETest.testRecvmsgShortAncillaryBuf", + "RecvmsgUDPLITETest.testRecvmsgShorter", + "RecvmsgUDPLITETest.testRecvmsgTimeout", + "RecvmsgUDPLITETest.testRecvmsgTrunc", + "SendmsgUDPLITETest.testSendmsg", + "SendmsgUDPLITETest.testSendmsgAfterClose", + "SendmsgUDPLITETest.testSendmsgAncillaryGenerator", + "SendmsgUDPLITETest.testSendmsgArray", + "SendmsgUDPLITETest.testSendmsgBadArgs", + "SendmsgUDPLITETest.testSendmsgBadCmsg", + "SendmsgUDPLITETest.testSendmsgBadMultiCmsg", + "SendmsgUDPLITETest.testSendmsgDataGenerator", + "SendmsgUDPLITETest.testSendmsgExcessCmsgReject", + "SendmsgUDPLITETest.testSendmsgGather", + "SendmsgUDPLITETest.testSendmsgNoDestAddr", + "UDPLITETimeoutTest.testTimeoutZero", + "UDPLITETimeoutTest.testUDPLITETimeout", + }, +} + +// Some python test libraries contain other test libraries that have test cases +// that need to be excluded. We need to expand such libraries so that the test +// case exclusion can work correctly. +var expand = []string{ + "test_asyncio", +} + // pythonRunner implements TestRunner for Python. type pythonRunner struct{} @@ -35,15 +143,92 @@ func (pythonRunner) ListTests() ([]string, error) { if err != nil { return nil, fmt.Errorf("failed to list: %v", err) } - var toolSlice []string - for _, test := range strings.Split(string(out), "\n") { - toolSlice = append(toolSlice, test) + testLibs := make(map[string]struct{}) + for _, testLib := range strings.Split(string(out), "\n") { + if len(testLib) == 0 { + continue + } + testLibs[testLib] = struct{}{} } - return toolSlice, nil + for _, libToExpand := range expand { + if _, ok := testLibs[libToExpand]; !ok { + return nil, fmt.Errorf("%s test library was not listed", libToExpand) + } + delete(testLibs, libToExpand) + subLibs, err := subTestLibs(libToExpand) + if err != nil { + return nil, err + } + for subLib := range subLibs { + testLibs[fmt.Sprintf("%s.%s", libToExpand, subLib)] = struct{}{} + } + } + res := make([]string, 0, len(testLibs)) + for lib := range testLibs { + res = append(res, lib) + } + // Sort to have deterministic results across shards. + sort.Strings(res) + return res, nil +} + +func subTestLibs(testLib string) (map[string]struct{}, error) { + // --list-{tests/cases} is only implemented by the 'test' library. + // Running './python -m test {X} --list-tests' does not list libraries inside + // X library. We need to list all test cases and extract sub libraries. + args := []string{"-m", "test", testLib, "--list-cases"} + cmd := exec.Command("./python", args...) + cmd.Stderr = os.Stderr + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("failed to list: %v", err) + } + subLibs := make(map[string]struct{}) + for _, tc := range strings.Split(string(out), "\n") { + if len(tc) == 0 { + continue + } + idx := strings.Index(tc, testLib) + if idx < 0 { + return nil, fmt.Errorf("could not find %q library in test case %q", testLib, tc) + } + subLibs[strings.Split(tc[idx:], ".")[1]] = struct{}{} + } + return subLibs, nil } // TestCmds implements TestRunner.TestCmds. func (pythonRunner) TestCmds(tests []string) []*exec.Cmd { - args := append([]string{"-m", "test"}, tests...) - return []*exec.Cmd{exec.Command("./python", args...)} + var cmds []*exec.Cmd + var full []string + for _, testModule := range tests { + if excludeTCs, ok := exclude[testModule]; ok { + cmds = append(cmds, testCmdWithExclTCs(testModule, excludeTCs)) + } else { + full = append(full, testModule) + } + } + if len(full) > 0 { + // Run all test modules (that have no excludes) together for speed. + // Running them individually with a new exec.Cmd takes longer. + args := append([]string{"-m", "test"}, full...) + cmds = append(cmds, exec.Command("./python", args...)) + } + return cmds +} + +func testCmdWithExclTCs(testModule string, excludeTCs []string) *exec.Cmd { + shellPipeline := []string{ + // List all test cases in this module. + fmt.Sprintf("./python -m test %s --list-cases", testModule), + // Exclude the test cases. Note that '$' was added after each excluded test + // case to be exact about which test case to exclude. + fmt.Sprintf("grep -v \"%s$\"", strings.Join(excludeTCs, "$\\|")), + // Remove the "test.{testModule}." prefix. + fmt.Sprintf("sed -e \"s/^test.%s.//\"", testModule), + // Run all un-excluded test cases in one command. + fmt.Sprintf("xargs ./python -m test.%s", testModule), + } + args := []string{"-c", strings.Join(shellPipeline, " | ")} + return exec.Command("sh", args...) }