Run failing packetimpact test and expect failure.

This will make it easier to notice if a code change causes an existing test to
pass.

PiperOrigin-RevId: 308057978

test/packetimpact/tests/defs.bzl

@@ -59,7 +59,11 @@ _packetimpact_test = rule(
 
 PACKETIMPACT_TAGS = ["local", "manual"]
 
-def packetimpact_linux_test(name, testbench_binary, **kwargs):
+def packetimpact_linux_test(
+        name,
+        testbench_binary,
+        expect_failure = False,
+        **kwargs):
     """Add a packetimpact test on linux.
 
     Args:
@@ -67,28 +71,37 @@ def packetimpact_linux_test(name, testbench_binary, **kwargs):
         testbench_binary: the testbench binary
         **kwargs: all the other args, forwarded to _packetimpact_test
     """
+    expect_failure_flag = ["--expect_failure"] if expect_failure else []
     _packetimpact_test(
         name = name + "_linux_test",
         testbench_binary = testbench_binary,
-        flags = ["--dut_platform", "linux"],
+        flags = ["--dut_platform", "linux"] + expect_failure_flag,
         tags = PACKETIMPACT_TAGS + ["packetimpact"],
         **kwargs
     )
 
-def packetimpact_netstack_test(name, testbench_binary, **kwargs):
+def packetimpact_netstack_test(
+        name,
+        testbench_binary,
+        expect_failure = False,
+        **kwargs):
     """Add a packetimpact test on netstack.
 
     Args:
         name: name of the test
         testbench_binary: the testbench binary
+        expect_failure: the test must fail
         **kwargs: all the other args, forwarded to _packetimpact_test
     """
+    expect_failure_flag = []
+    if expect_failure:
+        expect_failure_flag = ["--expect_failure"]
     _packetimpact_test(
         name = name + "_netstack_test",
         testbench_binary = testbench_binary,
         # This is the default runtime unless
         # "--test_arg=--runtime=OTHER_RUNTIME" is used to override the value.
-        flags = ["--dut_platform", "netstack", "--runtime=runsc-d"],
+        flags = ["--dut_platform", "netstack", "--runtime=runsc-d"] + expect_failure_flag,
         tags = PACKETIMPACT_TAGS + ["packetimpact"],
         **kwargs
     )
@@ -112,7 +125,13 @@ def packetimpact_go_test(name, size = "small", pure = True, linux = True, netsta
         tags = PACKETIMPACT_TAGS,
         **kwargs
     )
-    if linux:
-        packetimpact_linux_test(name = name, testbench_binary = testbench_binary)
-    if netstack:
-        packetimpact_netstack_test(name = name, testbench_binary = testbench_binary)
+    packetimpact_linux_test(
+        name = name,
+        expect_failure = not linux,
+        testbench_binary = testbench_binary,
+    )
+    packetimpact_netstack_test(
+        name = name,
+        expect_failure = not netstack,
+        testbench_binary = testbench_binary,
+    )

test/packetimpact/tests/test_runner.sh

@@ -29,7 +29,7 @@ function failure() {
 }
 trap 'failure ${LINENO} "$BASH_COMMAND"' ERR
 
-declare -r LONGOPTS="dut_platform:,posix_server_binary:,testbench_binary:,runtime:,tshark,extra_test_arg:"
+declare -r LONGOPTS="dut_platform:,posix_server_binary:,testbench_binary:,runtime:,tshark,extra_test_arg:,expect_failure"
 
 # Don't use declare below so that the error from getopt will end the script.
 PARSED=$(getopt --options "" --longoptions=$LONGOPTS --name "$0" -- "$@")
@@ -68,6 +68,10 @@ while true; do
       EXTRA_TEST_ARGS+="$2"
       shift 2
       ;;
+    --expect_failure)
+      declare -r EXPECT_FAILURE="1"
+      shift 1
+      ;;
     --)
       shift
       break
@@ -263,6 +267,15 @@ docker exec -t "${TESTBENCH}" \
   --local_ipv4=${TEST_NET_PREFIX}${TESTBENCH_NET_SUFFIX} \
   --remote_mac=${REMOTE_MAC} \
   --local_mac=${LOCAL_MAC} \
-  --device=${TEST_DEVICE}"
-
+  --device=${TEST_DEVICE}" && true
+declare -r TEST_RESULT="${?}"
+if [[ -z "${EXPECT_FAILURE-}" && "${TEST_RESULT}" != 0 ]]; then
+  echo 'FAIL: This test was expected to pass.'
+  exit ${TEST_RESULT}
+fi
+if [[ ! -z "${EXPECT_FAILURE-}" && "${TEST_RESULT}" == 0 ]]; then
+  echo 'FAIL: This test was expected to fail but passed.  Enable the test and' \
+    'mark the corresponding bug as fixed.'
+  exit 1
+fi
 echo PASS: No errors.