From 1557eb9611b1b592797efaa265cd0e02bc6780aa Mon Sep 17 00:00:00 2001 From: Ayush Ranjan Date: Wed, 11 May 2022 13:38:07 -0700 Subject: [PATCH] Clean up runtime test exclude file. I followed the following procedure to generate this change: - Query the runtime test image to see if the excluded test even exists. Many tests were removed from upstream because they were broken or superseded by another test. No point in excluding them now, so removed all these entries from the exclude file. - Run the remaining tests with runc. If the test fails with runc (natively) then we can consider the test "broken". Such tests do not signal a compatibility gap in gVisor. Marked such tests with `Broken test`. - Run the non-broken tests with runsc (gVisor). If they are now passing, remove them from the exclude list. This effectively increases our testing surface. More importantly, these tests were once failing with gVisor. Now they pass. Something was fixed in between. These tests now act as regression tests for the fix. - Some tests were marked flaky. Before removing them from the exclude list, we need to ensure that they really don't flake. I ran some of these flaky tests 100 times to test for flakiness and only removed them if they passed 100 times. But maybe some test has a flake rate <1%. Or some flaky tests were not marked flaky. So we might end up re-enabling flaky tests. This may cause pain. We will need to investigate into such tests when this happens. PiperOrigin-RevId: 448073470 --- test/runtimes/exclude/go1.16.csv | 7 +- test/runtimes/exclude/java17.csv | 293 ++++++------------------ test/runtimes/exclude/nodejs16.13.2.csv | 47 +--- test/runtimes/exclude/php8.1.1.csv | 129 ++++------- test/runtimes/exclude/python3.10.2.csv | 13 +- 5 files changed, 120 insertions(+), 369 deletions(-) diff --git a/test/runtimes/exclude/go1.16.csv b/test/runtimes/exclude/go1.16.csv index 81e02cf64..1ff196813 100644 --- a/test/runtimes/exclude/go1.16.csv +++ b/test/runtimes/exclude/go1.16.csv @@ -1,13 +1,10 @@ test name,bug id,comment cgo_errors,,FLAKY -cgo_test,,FLAKY +cgo_test,, go_test:cmd/go,,FLAKY go_test:net,b/162473575,setsockopt: protocol not available. -go_test:os,b/118780122,we have a pollable filesystem but that's a surprise go_test:os/signal,b/118780860,/dev/pts not properly supported. Also being tracked in b/29356795. -go_test:runtime,b/118782341,sigtrap not reported or caught or something. Also being tracked in b/33003106. go_test:syscall,b/118781998,bad bytes -- bad mem addr; FcntlFlock(F_GETLK) not supported. -runtime:cpu124,b/118778254,segmentation fault test:0_1,,FLAKY -testcarchive,b/118782924,no sigpipe +testcarchive,b/118782924,FLAKY testshared,,FLAKY diff --git a/test/runtimes/exclude/java17.csv b/test/runtimes/exclude/java17.csv index f5b8ab2b0..6db5866f6 100644 --- a/test/runtimes/exclude/java17.csv +++ b/test/runtimes/exclude/java17.csv @@ -1,167 +1,56 @@ -com/sun/crypto/provider/Cipher/PBE/PKCS12Cipher.java,,Fails in Docker test name,bug id,comment -com/sun/jdi/InvokeHangTest.java,https://bugs.openjdk.java.net/browse/JDK-8218463, -com/sun/jdi/NashornPopFrameTest.java,, com/sun/jdi/OnJcmdTest.java,b/180542784, com/sun/jdi/ProcessAttachTest.java,, -com/sun/jndi/ldap/LdapPoolTimeoutTest.java,, -com/sun/jndi/ldap/LdapTimeoutTest.java,, -com/sun/management/HotSpotDiagnosticMXBean/CheckOrigin.java,,Fails in Docker -com/sun/management/OperatingSystemMXBean/GetCommittedVirtualMemorySize.java,, +com/sun/management/HotSpotDiagnosticMXBean/CheckOrigin.java,, com/sun/management/ThreadMXBean/ThreadCpuTimeArray.java,,Test assumes high CPU clock precision -com/sun/management/UnixOperatingSystemMXBean/GetMaxFileDescriptorCount.sh,, com/sun/tools/attach/AttachSelf.java,, com/sun/tools/attach/BasicTests.java,, com/sun/tools/attach/PermissionTest.java,, -com/sun/tools/attach/StartManagementAgent.java,, +com/sun/tools/attach/StartManagementAgent.java,,Broken test com/sun/tools/attach/TempDirTest.java,, com/sun/tools/attach/modules/Driver.java,, java/io/File/GetXSpace.java,, -java/lang/Character/CheckScript.java,,Fails in Docker -java/lang/Character/CheckUnicode.java,,Fails in Docker java/lang/Class/GetPackageBootLoaderChildLayer.java,, -java/lang/ClassLoader/nativeLibrary/NativeLibraryTest.java,,Fails in Docker -java/lang/ProcessBuilder/Basic.java#id0,, -java/lang/String/nativeEncoding/StringPlatformChars.java,, -java/lang/instrument/HiddenClass/HiddenClassAgent.java,, -java/lang/invoke/LFCaching/LFGarbageCollectedTest.java,, -java/lang/invoke/lambda/LambdaFileEncodingSerialization.java,, -java/lang/module/ModuleDescriptorTest.java,, -java/lang/reflect/exeCallerAccessTest/CallerAccessTest.java,, -java/net/CookieHandler/B6791927.java,,java.lang.RuntimeException: Expiration date shouldn't be 0 +java/lang/ClassLoader/nativeLibrary/NativeLibraryTest.java,,Broken test +java/lang/String/nativeEncoding/StringPlatformChars.java,,Broken test +java/lang/instrument/HiddenClass/HiddenClassAgent.java,,Broken test +java/lang/invoke/LFCaching/LFGarbageCollectedTest.java,,Broken test +java/lang/invoke/lambda/LambdaFileEncodingSerialization.java,,Broken test +java/lang/reflect/exeCallerAccessTest/CallerAccessTest.java,,Broken test java/net/DatagramSocket/AddressNotSet.java,, java/net/DatagramSocket/SetGetReceiveBufferSize.java,b/180507650, -java/net/Inet6Address/B6558853.java,,Times out -java/net/InetAddress/CheckJNI.java,,java.net.ConnectException: Connection timed out (Connection timed out) -java/net/InterfaceAddress/NetworkPrefixLength.java,b/78507103, -java/net/MulticastSocket/B6425815.java,,java.net.SocketException: Protocol not available (Error getting socket option) -java/net/MulticastSocket/B6427403.java,,java.net.SocketException: Protocol not available java/net/MulticastSocket/MulticastTTL.java,, -java/net/MulticastSocket/NetworkInterfaceEmptyGetInetAddressesTest.java,,java.net.SocketException: Protocol not available (Error getting socket option) -java/net/MulticastSocket/NoLoopbackPackets.java,,java.net.SocketException: Protocol not available java/net/MulticastSocket/Promiscuous.java,, java/net/MulticastSocket/SendPortZero.java,, java/net/MulticastSocket/SetLoopbackMode.java,, java/net/MulticastSocket/SetLoopbackModeIPv4.java,, java/net/MulticastSocket/SetLoopbackOption.java,, java/net/MulticastSocket/SetTTLAndGetTTL.java,, -java/net/MulticastSocket/Test.java,, java/net/MulticastSocket/TestDefaults.java,, java/net/MulticastSocket/TimeToLive.java,, java/net/NetworkInterface/NetworkInterfaceStreamTest.java,, -java/net/Socket/LinkLocal.java,,java.net.SocketTimeoutException: Receive timed out -java/net/Socket/ReadAfterReset.java,b/228068134,java.lang.RuntimeException: Client read 0 expected 1000 -java/net/Socket/SetSoLinger.java,b/78527327,SO_LINGER is not yet supported -java/net/Socket/SocketAcceptInterruptTest.java,, -java/net/Socket/SocketReadInterruptTest.java,, +java/net/Socket/SocketAcceptInterruptTest.java,,Broken test +java/net/Socket/SocketReadInterruptTest.java,,Broken test java/net/Socket/UrgentDataTest.java,b/111515323, -java/net/SocketOption/OptionsTest.java,,Fails in Docker +java/net/SocketOption/OptionsTest.java,, java/net/SocketPermission/SocketPermissionTest.java,, -java/net/URLConnection/6212146/TestDriver.java,,Fails in Docker -java/net/httpclient/RequestBuilderTest.java,,Fails in Docker -java/net/httpclient/websocket/WebSocketProxyTest.java,,Times out on runc too -java/net/ipv6tests/TcpTest.java,,java.net.ConnectException: Connection timed out (Connection timed out) -java/net/ipv6tests/UdpTest.java,,Times out -java/nio/channels/DatagramChannel/BasicMulticastTests.java,, java/nio/channels/DatagramChannel/SocketOptionTests.java,,java.net.SocketException: Invalid argument -java/nio/channels/DatagramChannel/UseDGWithIPv6.java,, -java/nio/channels/FileChannel/directio/DirectIOTest.java,,Fails in Docker -java/nio/channels/FileChannel/directio/PwriteDirect.java,,java.io.IOException: Invalid argument -java/nio/channels/Selector/OutOfBand.java#id0,, +java/nio/channels/FileChannel/directio/DirectIOTest.java,,Broken test java/nio/channels/Selector/OutOfBand.java,, +java/nio/channels/Selector/OutOfBand.java#id0,, java/nio/channels/Selector/SelectWithConsumer.java,,Flaky -java/nio/channels/ServerSocketChannel/SocketOptionTests.java,, -java/nio/channels/SocketChannel/LingerOnClose.java,, -java/nio/channels/SocketChannel/SocketOptionTests.java,b/77965901, java/nio/channels/etc/PrintSupportedOptions.java,, -java/nio/channels/spi/SelectorProvider/inheritedChannel/InheritedChannelTest.java,,Fails in Docker +java/nio/channels/spi/SelectorProvider/inheritedChannel/InheritedChannelTest.java,,Broken test java/nio/channels/unixdomain/SocketOptions.java,, -java/nio/file/Files/probeContentType/Basic.java,, -java/rmi/activation/Activatable/extLoadedImpl/ext.sh,, -java/rmi/transport/checkLeaseInfoLeak/CheckLeaseLeak.java,, -java/security/cert/PolicyNode/GetPolicyQualifiers.java,b/170263154,Kokoro executor cert expired -java/security/misc/Versions.java,, -java/text/Format/NumberFormat/CurrencyFormat.java,,Fails in Docker -java/util/Calendar/JapaneseEraNameTest.java,, -java/util/Currency/CurrencyTest.java,,Fails in Docker -java/util/Currency/ValidateISO4217.java,,Fails in Docker -java/util/EnumSet/BogusEnumSet.java,,"java.io.InvalidClassException: java.util.EnumSet; local class incompatible: stream classdesc serialVersionUID = -2409567991088730183, local class serialVersionUID = 1009687484059888093" -java/util/Locale/Bug8040211.java,,java.lang.RuntimeException: Failed. -java/util/Locale/LSRDataTest.java,, -java/util/Properties/CompatibilityTest.java,,"java.lang.RuntimeException: jdk.internal.org.xml.sax.SAXParseException; Internal DTD subset is not allowed. The Properties XML document must have the following DOCTYPE declaration: " -java/util/ResourceBundle/Control/MissingResourceCauseTestRun.java,, -java/util/ResourceBundle/Control/XMLResourceBundleTest.java,,java.util.MissingResourceException: Can't find bundle for base name XmlRB locale -java/util/ResourceBundle/modules/xmlformat/xmlformat.sh,,Timeout reached: 60000. Process is not alive! -java/util/TimeZone/TimeZoneTest.java,,Uncaught exception thrown in test method TestShortZoneIDs +java/rmi/transport/checkLeaseInfoLeak/CheckLeaseLeak.java,,Broken test +java/util/ResourceBundle/Control/MissingResourceCauseTestRun.java,,Broken test java/util/concurrent/locks/Lock/TimedAcquireLeak.java,, -java/util/jar/JarFile/mrjar/MultiReleaseJarAPI.java,,Fails in Docker -java/util/logging/LogManager/Configuration/updateConfiguration/SimpleUpdateConfigWithInputStreamTest.java,, +java/util/logging/LogManager/Configuration/updateConfiguration/SimpleUpdateConfigWithInputStreamTest.java,,Broken test java/util/logging/TestLoggerWeakRefLeak.java,, -java/util/spi/ResourceBundleControlProvider/UserDefaultControlTest.java,,java.util.MissingResourceException: Can't find bundle for base name com.foo.XmlRB locale -javax/imageio/AppletResourceTest.java,, -javax/imageio/plugins/jpeg/JPEGsNotAcceleratedTest.java,,java.awt.HeadlessException: No X11 DISPLAY variable was set but this program performed an operation which requires it. -javax/management/remote/mandatory/connection/RMIConnector_NPETest.java,, -javax/management/security/HashedPasswordFileTest.java,, -javax/net/ssl/DTLS/DTLSBufferOverflowUnderflowTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSDataExchangeTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSEnginesClosureTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSHandshakeTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSHandshakeWithReplicatedPacketsTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSIncorrectAppDataTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSMFLNTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSNotEnabledRC4Test.java,,Compilation failed -javax/net/ssl/DTLS/DTLSRehandshakeTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSRehandshakeWithCipherChangeTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSRehandshakeWithDataExTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSSequenceNumberTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSUnsupportedCiphersTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10BufferOverflowUnderflowTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10DataExchangeTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10EnginesClosureTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10HandshakeTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10HandshakeWithReplicatedPacketsTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10IncorrectAppDataTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10MFLNTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10NotEnabledRC4Test.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10RehandshakeTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10RehandshakeWithCipherChangeTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10RehandshakeWithDataExTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10SequenceNumberTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10UnsupportedCiphersTest.java,,Compilation failed -javax/net/ssl/SSLSession/JSSERenegotiate.java,,Fails in Docker -javax/net/ssl/Stapling/HttpsUrlConnClient.java,, -javax/net/ssl/Stapling/SSLSocketWithStapling.java,, -javax/net/ssl/TLS/TLSDataExchangeTest.java,,Compilation failed -javax/net/ssl/TLS/TLSEnginesClosureTest.java,,Compilation failed -javax/net/ssl/TLS/TLSHandshakeTest.java,,Compilation failed -javax/net/ssl/TLS/TLSMFLNTest.java,,Compilation failed -javax/net/ssl/TLS/TLSNotEnabledRC4Test.java,,Compilation failed -javax/net/ssl/TLS/TLSRehandshakeTest.java,,Compilation failed -javax/net/ssl/TLS/TLSRehandshakeWithCipherChangeTest.java,,Compilation failed -javax/net/ssl/TLS/TLSRehandshakeWithDataExTest.java,,Compilation failed -javax/net/ssl/TLS/TLSUnsupportedCiphersTest.java,,Compilation failed -javax/net/ssl/TLSv1/TLSDataExchangeTest.java,,Compilation failed -javax/net/ssl/TLSv1/TLSEnginesClosureTest.java,,Compilation failed -javax/net/ssl/TLSv1/TLSHandshakeTest.java,,Compilation failed -javax/net/ssl/TLSv1/TLSMFLNTest.java,,Compilation failed -javax/net/ssl/TLSv1/TLSNotEnabledRC4Test.java,,Compilation failed -javax/net/ssl/TLSv1/TLSRehandshakeTest.java,,Compilation failed -javax/net/ssl/TLSv1/TLSRehandshakeWithCipherChangeTest.java,,Compilation failed -javax/net/ssl/TLSv1/TLSRehandshakeWithDataExTest.java,,Compilation failed -javax/net/ssl/TLSv1/TLSUnsupportedCiphersTest.java,,Compilation failed -javax/net/ssl/TLSv11/TLSDataExchangeTest.java,,Compilation failed -javax/net/ssl/TLSv11/TLSEnginesClosureTest.java,,Compilation failed -javax/net/ssl/TLSv11/TLSHandshakeTest.java,,Compilation failed -javax/net/ssl/TLSv11/TLSMFLNTest.java,,Compilation failed -javax/net/ssl/TLSv11/TLSNotEnabledRC4Test.java,,Compilation failed -javax/net/ssl/TLSv11/TLSRehandshakeTest.java,,Compilation failed -javax/net/ssl/TLSv11/TLSRehandshakeWithCipherChangeTest.java,,Compilation failed -javax/net/ssl/TLSv11/TLSRehandshakeWithDataExTest.java,,Compilation failed -javax/net/ssl/TLSv11/TLSUnsupportedCiphersTest.java,,Compilation failed -javax/net/ssl/TLSv12/TLSEnginesClosureTest.java,,Compilation failed -javax/sound/sampled/AudioInputStream/FrameLengthAfterConversion.java,, -jdk/internal/jline/KeyConversionTest.java,, -jdk/internal/loader/NativeLibraries/Main.java,, +javax/imageio/plugins/jpeg/JPEGsNotAcceleratedTest.java,,Broken test +javax/management/remote/mandatory/connection/RMIConnector_NPETest.java,,Broken test +javax/management/security/HashedPasswordFileTest.java,,Broken test +jdk/internal/loader/NativeLibraries/Main.java,,Broken test jdk/jfr/api/consumer/log/TestDynamicStart.java,, jdk/jfr/api/consumer/streaming/TestBaseRepositoryAfterStart.java,,Times out jdk/jfr/api/consumer/streaming/TestBaseRepositoryLastModified.java,,Times out @@ -171,23 +60,9 @@ jdk/jfr/api/consumer/streaming/TestJVMCrash.java,, jdk/jfr/api/consumer/streaming/TestJVMExit.java,, jdk/jfr/api/consumer/streaming/TestOutOfProcessMigration.java,, jdk/jfr/api/consumer/streaming/TestRepositoryProperty.java,, -jdk/jfr/cmd/TestHelp.java,,java.lang.RuntimeException: 'Available commands are:' missing from stdout/stderr -jdk/jfr/cmd/TestPrint.java,,Missing file' missing from stdout/stderr -jdk/jfr/cmd/TestPrintDefault.java,,java.lang.RuntimeException: 'JVMInformation' missing from stdout/stderr -jdk/jfr/cmd/TestPrintJSON.java,,javax.script.ScriptException: :1:17 Expected an operand but found eof var jsonObject = ^ in at line number 1 at column number 17 -jdk/jfr/cmd/TestPrintXML.java,,org.xml.sax.SAXParseException; lineNumber: 1; columnNumber: 1; Premature end of file. -jdk/jfr/cmd/TestReconstruct.java,,java.lang.RuntimeException: 'Too few arguments' missing from stdout/stderr -jdk/jfr/cmd/TestSplit.java,,java.lang.RuntimeException: 'Missing file' missing from stdout/stderr -jdk/jfr/cmd/TestSummary.java,,java.lang.RuntimeException: 'Missing file' missing from stdout/stderr -jdk/jfr/event/compiler/TestCompilerStats.java,,java.lang.RuntimeException: Field nmetodsSize not in event jdk/jfr/event/gc/detailed/TestZUncommitEvent.java,, -jdk/jfr/event/metadata/TestDefaultConfigurations.java,,Setting 'threshold' in event 'jdk.SecurityPropertyModification' was not configured in the configuration 'default' jdk/jfr/event/oldobject/TestLargeRootSet.java,,Flaky - `main' threw exception: java.lang.RuntimeException: Could not find root object -jdk/jfr/event/runtime/TestActiveSettingEvent.java,,java.lang.Exception: Could not find setting with name jdk.X509Validation#threshold -jdk/jfr/event/runtime/TestModuleEvents.java,,java.lang.RuntimeException: assertEquals: expected jdk.proxy1 to equal java.base jdk/jfr/event/runtime/TestNetworkUtilizationEvent.java,, -jdk/jfr/event/runtime/TestThreadParkEvent.java,, -jdk/jfr/event/sampling/TestNative.java,, jdk/jfr/javaagent/TestLoadedAgent.java,b/180542638, jdk/jfr/jcmd/TestJcmdChangeLogLevel.java,, jdk/jfr/jcmd/TestJcmdConfigure.java,, @@ -207,98 +82,60 @@ jdk/jfr/jcmd/TestJcmdStartWithOptions.java,, jdk/jfr/jcmd/TestJcmdStartWithSettings.java,, jdk/jfr/jcmd/TestJcmdStopInvalidFile.java,, jdk/jfr/jmx/TestGetRecordings.java,b/180542639, -jdk/jfr/jvm/TestGetAllEventClasses.java,,Compilation failed jdk/jfr/jvm/TestJfrJavaBase.java,, jdk/jfr/startupargs/TestStartRecording.java,, -jdk/modules/incubator/ImageModules.java,, jdk/net/Sockets/ExtOptionTest.java,, -jdk/net/Sockets/QuickAckTest.java,, -lib/security/cacerts/VerifyCACerts.java,, -sun/management/jdp/JdpDefaultsTest.java,, -sun/management/jdp/JdpJmxRemoteDynamicPortTest.java,, -sun/management/jdp/JdpOffTest.java,, -sun/management/jdp/JdpSpecificAddressTest.java,, -sun/management/jmxremote/bootstrap/CustomLauncherTest.java,, -sun/management/jmxremote/bootstrap/JvmstatCountersTest.java,, -sun/management/jmxremote/bootstrap/LocalManagementTest.java,, -sun/management/jmxremote/bootstrap/RmiRegistrySslTest.java,, -sun/management/jmxremote/bootstrap/RmiSslBootstrapTest.sh,, -sun/management/jmxremote/startstop/JMXStartStopTest.java,, +sun/management/jmxremote/bootstrap/CustomLauncherTest.java,,Broken test +sun/management/jmxremote/bootstrap/JvmstatCountersTest.java,,Broken test +sun/management/jmxremote/bootstrap/LocalManagementTest.java,,Broken test +sun/management/jmxremote/bootstrap/RmiRegistrySslTest.java,,Broken test +sun/management/jmxremote/bootstrap/RmiSslBootstrapTest.sh,,Broken test +sun/management/jmxremote/startstop/JMXStartStopTest.java,,Broken test sun/management/jmxremote/startstop/JMXStatusPerfCountersTest.java,, sun/management/jmxremote/startstop/JMXStatusTest.java,, -sun/text/resources/LocaleDataTest.java,, sun/tools/jcmd/JcmdOutputEncodingTest.java,, sun/tools/jcmd/TestJcmdSanity.java,, -sun/tools/jhsdb/AlternateHashingTest.java,, -sun/tools/jhsdb/BasicLauncherTest.java,, -sun/tools/jhsdb/HeapDumpTest.java,, -sun/tools/jhsdb/HeapDumpTestWithActiveProcess.java,, -sun/tools/jhsdb/JShellHeapDumpTest.java,,Fails on runc too -sun/tools/jhsdb/JStackStressTest.java,, -sun/tools/jhsdb/heapconfig/JMapHeapConfigTest.java,, +sun/tools/jhsdb/BasicLauncherTest.java,,Broken test +sun/tools/jhsdb/HeapDumpTest.java,,Broken test +sun/tools/jhsdb/HeapDumpTestWithActiveProcess.java,,Broken test +sun/tools/jhsdb/JShellHeapDumpTest.java,,Broken test +sun/tools/jhsdb/JStackStressTest.java,,Broken test +sun/tools/jhsdb/heapconfig/JMapHeapConfigTest.java,,Broken test sun/tools/jinfo/BasicJInfoTest.java,, sun/tools/jinfo/JInfoTest.java,, -sun/tools/jmap/BasicJMapTest.java#G1,, -sun/tools/jmap/BasicJMapTest.java#Parallel,, -sun/tools/jmap/BasicJMapTest.java#Serial,, -sun/tools/jmap/BasicJMapTest.java#Shenandoah,, -sun/tools/jmap/BasicJMapTest.java#Z,, -sun/tools/jmap/BasicJMapTest.java,, +sun/tools/jmap/BasicJMapTest.java,,Broken test sun/tools/jstack/BasicJStackTest.java,, sun/tools/jstack/DeadlockDetectionTest.java,, -sun/tools/jstat/jstatClassloadOutput1.sh,, -sun/tools/jstatd/TestJstatdExternalRegistry.java,, +sun/tools/jstat/jstatClassloadOutput1.sh,,Broken test sun/tools/jstatd/TestJstatdPort.java,,Flaky sun/tools/jstatd/TestJstatdPortAndServer.java,,Flaky -sun/util/calendar/zi/TestZoneInfo310.java,, -tools/jar/ContentOrder.java,, -tools/jar/ReproducibleJar.java,, -tools/jar/modularJar/Basic.java,, -tools/jar/multiRelease/Basic.java,, -tools/jimage/JImageExtractTest.java,, -tools/jimage/JImageNonAsciiNameTest.java,, -tools/jimage/JImageTest.java,, -tools/jlink/JLinkReproducible3Test.java,, -tools/jlink/JLinkTest.java,, -tools/jlink/plugins/IncludeLocalesPluginTest.java,, -tools/jlink/plugins/StripNativeDebugSymbolsPlugin/StripNativeDebugSymbolsPluginTest.java,, -tools/jmod/JmodTest.java,, -tools/jmod/hashes/HashesTest.java,, -tools/jpackage/junit/junit.java,, -tools/jpackage/linux/AppAboutUrlTest.java#id0,, -tools/jpackage/linux/AppCategoryTest.java,, -tools/jpackage/linux/LinuxBundleNameTest.java,, -tools/jpackage/linux/MaintainerTest.java,, -tools/jpackage/linux/PackageDepsTest.java,, -tools/jpackage/linux/ReleaseTest.java,, -tools/jpackage/linux/ShortcutHintTest.java#id0,, -tools/jpackage/linux/jdk/jpackage/tests/UsrTreeTest.java,, -tools/jpackage/share/AddLauncherTest.java#id1,, -tools/jpackage/share/AppLauncherEnvTest.java,, -tools/jpackage/share/ArgumentsTest.java,, -tools/jpackage/share/EmptyFolderPackageTest.java,, -tools/jpackage/share/EmptyFolderTest.java,, -tools/jpackage/share/InstallDirTest.java#id0,, -tools/jpackage/share/LicenseTest.java#id0,, -tools/jpackage/share/MultiLauncherTwoPhaseTest.java,, -tools/jpackage/share/RuntimePackageTest.java#id0,, -tools/jpackage/share/SimplePackageTest.java,, -tools/jpackage/share/jdk/jpackage/tests/AppVersionTest.java,, -tools/jpackage/share/jdk/jpackage/tests/BasicTest.java,, -tools/jpackage/share/jdk/jpackage/tests/CookedRuntimeTest.java,, -tools/jpackage/share/jdk/jpackage/tests/DotInNameTest.java,, -tools/jpackage/share/jdk/jpackage/tests/JLinkOptionsTest.java,, -tools/jpackage/share/jdk/jpackage/tests/JavaOptionsEqualsTest.java#id0,, -tools/jpackage/share/jdk/jpackage/tests/JavaOptionsEqualsTest.java#id1,, -tools/jpackage/share/jdk/jpackage/tests/MainClassTest.java,, -tools/jpackage/share/jdk/jpackage/tests/ModulePathTest.java,, -tools/jpackage/share/jdk/jpackage/tests/ModulePathTest2.java,, -tools/jpackage/share/jdk/jpackage/tests/ModulePathTest3.java,, -tools/jpackage/share/jdk/jpackage/tests/MultipleJarAppTest.java,, -tools/jpackage/share/jdk/jpackage/tests/NoMPathRuntimeTest.java,, -tools/jpackage/share/jdk/jpackage/tests/VendorTest.java#id1,, -tools/launcher/BigJar.java,b/111611473, -tools/launcher/HelpFlagsTest.java,,java.lang.AssertionError: HelpFlagsTest failed: Tool jfr not covered by this test. Add specification to jdkTools array! -tools/launcher/JliLaunchTest.java,,Fails on runc too -tools/launcher/VersionCheck.java,,java.lang.AssertionError: VersionCheck failed: testToolVersion: [jfr]; -tools/launcher/modules/patch/systemmodules/PatchSystemModules.java,, +tools/jimage/JImageExtractTest.java,,Broken test +tools/jimage/JImageTest.java,,Broken test +tools/jlink/JLinkReproducible3Test.java,,Broken test +tools/jlink/plugins/StripNativeDebugSymbolsPlugin/StripNativeDebugSymbolsPluginTest.java,,Broken test +tools/jpackage/junit/junit.java,,Broken test +tools/jpackage/linux/AppCategoryTest.java,,Broken test +tools/jpackage/linux/LinuxBundleNameTest.java,,Broken test +tools/jpackage/linux/MaintainerTest.java,,Broken test +tools/jpackage/linux/PackageDepsTest.java,,Broken test +tools/jpackage/linux/ReleaseTest.java,,Broken test +tools/jpackage/linux/jdk/jpackage/tests/UsrTreeTest.java,,Broken test +tools/jpackage/share/AppLauncherEnvTest.java,,Broken test +tools/jpackage/share/ArgumentsTest.java,,Broken test +tools/jpackage/share/EmptyFolderPackageTest.java,,Broken test +tools/jpackage/share/EmptyFolderTest.java,,Broken test +tools/jpackage/share/MultiLauncherTwoPhaseTest.java,,Broken test +tools/jpackage/share/SimplePackageTest.java,,Broken test +tools/jpackage/share/jdk/jpackage/tests/AppVersionTest.java,,Broken test +tools/jpackage/share/jdk/jpackage/tests/BasicTest.java,,Broken test +tools/jpackage/share/jdk/jpackage/tests/CookedRuntimeTest.java,,Broken test +tools/jpackage/share/jdk/jpackage/tests/DotInNameTest.java,,Broken test +tools/jpackage/share/jdk/jpackage/tests/JLinkOptionsTest.java,,Broken test +tools/jpackage/share/jdk/jpackage/tests/MainClassTest.java,,Broken test +tools/jpackage/share/jdk/jpackage/tests/ModulePathTest.java,,Broken test +tools/jpackage/share/jdk/jpackage/tests/ModulePathTest2.java,,Broken test +tools/jpackage/share/jdk/jpackage/tests/ModulePathTest3.java,,Broken test +tools/jpackage/share/jdk/jpackage/tests/MultipleJarAppTest.java,,Broken test +tools/jpackage/share/jdk/jpackage/tests/NoMPathRuntimeTest.java,,Broken test +tools/launcher/BigJar.java,b/111611473,Broken test +tools/launcher/JliLaunchTest.java,,Broken test diff --git a/test/runtimes/exclude/nodejs16.13.2.csv b/test/runtimes/exclude/nodejs16.13.2.csv index 2a6cd980a..9f0e4939b 100644 --- a/test/runtimes/exclude/nodejs16.13.2.csv +++ b/test/runtimes/exclude/nodejs16.13.2.csv @@ -1,48 +1,11 @@ test name,bug id,comment -async-hooks/test-statwatcher.js,https://github.com/nodejs/node/issues/21425,Check for fix inclusion in nodejs releases after 2020-03-29 -benchmark/test-benchmark-fs.js,,Broken test benchmark/test-benchmark-napi.js,,Broken test -doctool/test-make-doc.js,b/68848110,Expected to fail. -internet/test-dgram-multicast-set-interface-lo.js,b/162798882, internet/test-dgram-multicast-ssm-multi-process.js,, -internet/test-dgram-multicast-ssmv6-multi-process.js,, -internet/test-dns.js,,Broken test -internet/test-doctool-versions.js,,Broken test -internet/test-uv-threadpool-schedule.js,,Broken test -parallel/test-dgram-bind-fd.js,b/132447356, -parallel/test-dgram-socket-buffer-size.js,b/68847921, -parallel/test-dns-channel-timeout.js,b/161893056, -parallel/test-dns-lookupService-promises.js,, -parallel/test-fs-access.js,,Broken test -parallel/test-http-writable-true-after-close.js,b/171301436,Flaky - Mismatched function calls. Expected exactly 1 actual 2 -parallel/test-https-selfsigned-no-keycertsign-no-crash.js,, -parallel/test-os.js,b/63997097, -parallel/test-process-uid-gid.js,,Does not work inside Docker with gid nobody -pseudo-tty/test-assert-colors.js,b/162801321, -pseudo-tty/test-assert-no-color.js,b/162801321, -pseudo-tty/test-assert-position-indicator.js,b/162801321, -pseudo-tty/test-async-wrap-getasyncid-tty.js,b/162801321, -pseudo-tty/test-fatal-error.js,b/162801321, -pseudo-tty/test-handle-wrap-isrefed-tty.js,b/162801321, -pseudo-tty/test-readable-tty-keepalive.js,b/162801321, -pseudo-tty/test-set-raw-mode-reset-process-exit.js,b/162801321, -pseudo-tty/test-set-raw-mode-reset-signal.js,b/162801321, -pseudo-tty/test-set-raw-mode-reset.js,b/162801321, -pseudo-tty/test-stderr-stdout-handle-sigwinch.js,b/162801321, -pseudo-tty/test-stdin-write.js,b/162801321, +internet/test-dgram-multicast-ssmv6-multi-process.js,,Broken test +parallel/test-dns-lookupService-promises.js,,Broken test +parallel/test-http-writable-true-after-close.js,b/171301436,Flaky +parallel/test-https-selfsigned-no-keycertsign-no-crash.js,,Broken test pseudo-tty/test-stdout-read.js,b/162801321, -pseudo-tty/test-tty-color-support.js,b/162801321, -pseudo-tty/test-tty-isatty.js,b/162801321, -pseudo-tty/test-tty-stdin-call-end.js,b/162801321, -pseudo-tty/test-tty-stdin-end.js,b/162801321, -pseudo-tty/test-tty-stdout-end.js,b/162801321, -pseudo-tty/test-tty-stdout-resize.js,b/162801321, -pseudo-tty/test-tty-stream-constructors.js,b/162801321, -pseudo-tty/test-tty-window-size.js,b/162801321, -pseudo-tty/test-tty-wrap.js,b/162801321, -pummel/test-net-pingpong.js,,Broken test -pummel/test-vm-memleak.js,b/162799436, -pummel/test-watch-file.js,,Flaky test which was fixed in https://github.com/nodejs/node/commit/b0b52b2023f5cd0df4ae921850815586b4313dca. Unexclude when bumping nodejs version. tick-processor/test-tick-processor-builtin.js,,Broken test -v8-updates/test-linux-perf.js,, +v8-updates/test-linux-perf.js,,Broken test wasi/test-wasi.js,, diff --git a/test/runtimes/exclude/php8.1.1.csv b/test/runtimes/exclude/php8.1.1.csv index 047ea31de..2d19a9495 100644 --- a/test/runtimes/exclude/php8.1.1.csv +++ b/test/runtimes/exclude/php8.1.1.csv @@ -1,94 +1,57 @@ test name,bug id,comment Zend/tests/concat_003.phpt,b/162896021, -ext/intl/tests/bug77895.phpt,, -ext/intl/tests/dateformat_bug65683_2.phpt,, -ext/mbstring/tests/bug76319.phpt,, -ext/mbstring/tests/bug76958.phpt,, -ext/mbstring/tests/bug77025.phpt,, -ext/mbstring/tests/bug77165.phpt,, -ext/mbstring/tests/bug77454.phpt,, -ext/mbstring/tests/mb_convert_encoding_leak.phpt,, -ext/mbstring/tests/mb_strrpos_encoding_3rd_param.phpt,, -ext/opcache/tests/bug78014.phpt,, -ext/opcache/tests/bug78175.phpt,, -ext/opcache/tests/bug78175_2.phpt,, -ext/opcache/tests/bug78376.phpt,, -ext/opcache/tests/bug78937_1.phpt,, -ext/opcache/tests/bug78937_2.phpt,, -ext/opcache/tests/bug78937_3.phpt,, -ext/opcache/tests/bug78937_4.phpt,, -ext/opcache/tests/bug78937_5.phpt,, -ext/opcache/tests/bug78937_6.phpt,, -ext/opcache/tests/jit/bug80634.phpt,, -ext/opcache/tests/jit/bug81256.phpt,, -ext/opcache/tests/opt/verify_return_type.phpt,, -ext/opcache/tests/preload_001.phpt,, -ext/opcache/tests/preload_002.phpt,, -ext/opcache/tests/preload_003.phpt,, -ext/opcache/tests/preload_004.phpt,, -ext/opcache/tests/preload_005.phpt,, -ext/opcache/tests/preload_006.phpt,, -ext/opcache/tests/preload_007.phpt,, -ext/opcache/tests/preload_008.phpt,, -ext/opcache/tests/preload_009.phpt,, -ext/opcache/tests/preload_010.phpt,, -ext/opcache/tests/preload_011.phpt,, -ext/opcache/tests/preload_012.phpt,, -ext/opcache/tests/preload_013.phpt,, -ext/opcache/tests/preload_bug79114.phpt,, -ext/opcache/tests/preload_class_alias.phpt,, -ext/opcache/tests/preload_class_alias_2.phpt,, -ext/opcache/tests/preload_dynamic_def_removal.phpt,, -ext/opcache/tests/preload_dynamic_function.phpt,, -ext/opcache/tests/preload_early_binding.phpt,, -ext/opcache/tests/preload_enum.phpt,, -ext/opcache/tests/preload_error_handler.phpt,, -ext/opcache/tests/preload_ind.phpt,, -ext/opcache/tests/preload_loadable_classes_1.phpt,, -ext/opcache/tests/preload_loadable_classes_2.phpt,, -ext/opcache/tests/preload_loadable_classes_3.phpt,, -ext/opcache/tests/preload_method_static_vars.phpt,, -ext/opcache/tests/preload_parse_error.phpt,, -ext/opcache/tests/preload_prop_info_table.phpt,, -ext/opcache/tests/preload_static_var_inheritance.phpt,, -ext/opcache/tests/preload_trait_alias.phpt,, -ext/opcache/tests/preload_trait_multiple_fixup.phpt,, -ext/opcache/tests/preload_trait_static.phpt,, -ext/opcache/tests/preload_unresolved_prop_type.phpt,, -ext/openssl/tests/openssl_dh_compute_key.phpt,, -ext/pcre/tests/cache_limit.phpt,,Broken test - Flaky -ext/session/tests/session_module_name_variation4.phpt,,Flaky -ext/session/tests/session_set_save_handler_class_018.phpt,, -ext/session/tests/session_set_save_handler_iface_003.phpt,, -ext/session/tests/session_set_save_handler_sid_001.phpt,, -ext/session/tests/session_set_save_handler_variation4.phpt,, +ext/opcache/tests/bug78014.phpt,,Broken test +ext/opcache/tests/bug78175.phpt,,Broken test +ext/opcache/tests/bug78175_2.phpt,,Broken test +ext/opcache/tests/bug78376.phpt,,Broken test +ext/opcache/tests/bug78937_1.phpt,,Broken test +ext/opcache/tests/bug78937_2.phpt,,Broken test +ext/opcache/tests/bug78937_3.phpt,,Broken test +ext/opcache/tests/bug78937_4.phpt,,Broken test +ext/opcache/tests/bug78937_5.phpt,,Broken test +ext/opcache/tests/bug78937_6.phpt,,Broken test +ext/opcache/tests/jit/bug80634.phpt,,Broken test +ext/opcache/tests/jit/bug81256.phpt,,Broken test +ext/opcache/tests/preload_001.phpt,,Broken test +ext/opcache/tests/preload_002.phpt,,Broken test +ext/opcache/tests/preload_003.phpt,,Broken test +ext/opcache/tests/preload_004.phpt,,Broken test +ext/opcache/tests/preload_005.phpt,,Broken test +ext/opcache/tests/preload_006.phpt,,Broken test +ext/opcache/tests/preload_007.phpt,,Broken test +ext/opcache/tests/preload_008.phpt,,Broken test +ext/opcache/tests/preload_009.phpt,,Broken test +ext/opcache/tests/preload_010.phpt,,Broken test +ext/opcache/tests/preload_011.phpt,,Broken test +ext/opcache/tests/preload_012.phpt,,Broken test +ext/opcache/tests/preload_013.phpt,,Broken test +ext/opcache/tests/preload_bug79114.phpt,,Broken test +ext/opcache/tests/preload_class_alias.phpt,,Broken test +ext/opcache/tests/preload_class_alias_2.phpt,,Broken test +ext/opcache/tests/preload_dynamic_def_removal.phpt,,Broken test +ext/opcache/tests/preload_dynamic_function.phpt,,Broken test +ext/opcache/tests/preload_early_binding.phpt,,Broken test +ext/opcache/tests/preload_enum.phpt,,Broken test +ext/opcache/tests/preload_error_handler.phpt,,Broken test +ext/opcache/tests/preload_ind.phpt,,Broken test +ext/opcache/tests/preload_loadable_classes_1.phpt,,Broken test +ext/opcache/tests/preload_loadable_classes_2.phpt,,Broken test +ext/opcache/tests/preload_loadable_classes_3.phpt,,Broken test +ext/opcache/tests/preload_method_static_vars.phpt,,Broken test +ext/opcache/tests/preload_parse_error.phpt,,Broken test +ext/opcache/tests/preload_prop_info_table.phpt,,Broken test +ext/opcache/tests/preload_static_var_inheritance.phpt,,Broken test +ext/opcache/tests/preload_trait_alias.phpt,,Broken test +ext/opcache/tests/preload_trait_multiple_fixup.phpt,,Broken test +ext/opcache/tests/preload_trait_static.phpt,,Broken test +ext/opcache/tests/preload_unresolved_prop_type.phpt,,Broken test +ext/openssl/tests/openssl_dh_compute_key.phpt,,Broken test ext/standard/tests/file/bug69442.phpt,, -ext/standard/tests/file/disk.phpt,https://bugs.php.net/bug.php?id=80018, -ext/standard/tests/file/disk_free_space_basic.phpt,https://bugs.php.net/bug.php?id=80018, -ext/standard/tests/file/disk_free_space_error.phpt,https://bugs.php.net/bug.php?id=80018, -ext/standard/tests/file/disk_free_space_variation.phpt,https://bugs.php.net/bug.php?id=80018, -ext/standard/tests/file/disk_total_space_basic.phpt,https://bugs.php.net/bug.php?id=80018, -ext/standard/tests/file/disk_total_space_error.phpt,https://bugs.php.net/bug.php?id=80018, -ext/standard/tests/file/disk_total_space_variation.phpt,https://bugs.php.net/bug.php?id=80018, ext/standard/tests/file/fopen_variation19.phpt,b/162894964, -ext/standard/tests/file/lstat_stat_variation14.phpt,,Flaky -ext/standard/tests/file/php_fd_wrapper_01.phpt,, -ext/standard/tests/file/php_fd_wrapper_02.phpt,, -ext/standard/tests/file/php_fd_wrapper_03.phpt,, -ext/standard/tests/file/php_fd_wrapper_04.phpt,, -ext/standard/tests/file/rename_variation.phpt,b/68717309, ext/standard/tests/file/symlink_link_linkinfo_is_link_variation4.phpt,b/162895341, -ext/standard/tests/general_functions/escapeshellarg_bug71270.phpt,, -ext/standard/tests/general_functions/escapeshellcmd_bug71270.phpt,, -ext/standard/tests/streams/proc_open_bug60120.phpt,,Flaky until php-src 3852a35fdbcb -ext/standard/tests/streams/proc_open_bug64438.phpt,,Flaky -ext/standard/tests/streams/proc_open_bug69900.phpt,,Flaky -ext/standard/tests/streams/stream_socket_sendto.phpt,, -ext/standard/tests/strings/007.phpt,, sapi/cli/tests/upload_2G.phpt,, tests/output/stream_isatty_err.phpt,b/68720279, tests/output/stream_isatty_in-err.phpt,b/68720282, -tests/output/stream_isatty_in-out-err.phpt,, tests/output/stream_isatty_in-out.phpt,b/68720299, tests/output/stream_isatty_out-err.phpt,b/68720311, tests/output/stream_isatty_out.phpt,b/68720325, diff --git a/test/runtimes/exclude/python3.10.2.csv b/test/runtimes/exclude/python3.10.2.csv index d90552706..f665b22c7 100644 --- a/test/runtimes/exclude/python3.10.2.csv +++ b/test/runtimes/exclude/python3.10.2.csv @@ -1,19 +1,10 @@ test name,bug id,comment -test_asyncio,,Fails on Docker. +test_asyncio,,Broken test test_asyncore,b/162973328, -test_epoll,b/162983393, test_fcntl,b/162978767,fcntl invalid argument -- artificial test to make sure something works in 64 bit mode. -test_httplib,b/163000009,OSError: [Errno 98] Address already in use -test_logging,b/162980079, -test_multiprocessing_fork,,Flaky. Sometimes times out. -test_multiprocessing_forkserver,,Flaky. Sometimes times out. -test_multiprocessing_main_handling,,Flaky. Sometimes times out. -test_multiprocessing_spawn,,Flaky. Sometimes times out. test_posix,b/76174079,posix.sched_get_priority_min not implemented + posix.sched_rr_get_interval not permitted test_pty,b/162979921, test_readline,b/162980389,TestReadline hangs forever test_resource,b/76174079, -test_selectors,b/76116849,OSError not raised with epoll test_signal,,Flaky - signal: alarm clock -test_smtplib,b/162980434,unclosed sockets -test_socket,b/75983380, +test_socket,,Broken test