Syncing packetimpact tests in different directories

By exposing an ALL_TESTS list in defs.bzl we can make sure all packetimpact
users get to agree on the list of all tests. A defect in this approach is that
we have to keep a list of packetimpact_testbench rules in the BUILD file. An
helper validate_all_tests has been added to help keep BUILD and .bzl files in
sync.

PiperOrigin-RevId: 337411839

test/packetimpact/README.md

@@ -694,6 +694,13 @@ func TestMyTcpTest(t *testing.T) {
 }
 ```
 
+### Adding a new packetimpact test
+
+*   Create a go test in the [tests directory](tests/)
+*   Add a `packetimpact_testbench` rule in [BUILD](tests/BUILD)
+*   Add the test into the `ALL_TESTS` list in [defs.bzl](runner/defs.bzl),
+    otherwise you will see an error message complaining about a missing test.
+
 ## Other notes
 
 *   The time between receiving a SYN-ACK and replying with an ACK in `Handshake`

test/packetimpact/runner/defs.bzl

@@ -110,29 +110,15 @@ def packetimpact_netstack_test(
         **kwargs
     )
 
-def packetimpact_go_test(name, size = "small", pure = True, expect_native_failure = False, expect_netstack_failure = False, **kwargs):
+def packetimpact_go_test(name, expect_native_failure = False, expect_netstack_failure = False):
     """Add packetimpact tests written in go.
 
     Args:
         name: name of the test
-        size: size of the test
-        pure: make a static go binary
         expect_native_failure: the test must fail natively
         expect_netstack_failure: the test must fail for Netstack
-        **kwargs: all the other args, forwarded to go_test
     """
     testbench_binary = name + "_test"
-    go_test(
-        name = testbench_binary,
-        size = size,
-        pure = pure,
-        nogo = False,  # FIXME(gvisor.dev/issue/3374): Not working with all build systems.
-        tags = [
-            "local",
-            "manual",
-        ],
-        **kwargs
-    )
     packetimpact_native_test(
         name = name,
         expect_failure = expect_native_failure,
@@ -143,3 +129,156 @@ def packetimpact_go_test(name, size = "small", pure = True, expect_native_failur
         expect_failure = expect_netstack_failure,
         testbench_binary = testbench_binary,
     )
+
+def packetimpact_testbench(name, size = "small", pure = True, **kwargs):
+    """Build packetimpact testbench written in go.
+
+    Args:
+        name: name of the test
+        size: size of the test
+        pure: make a static go binary
+        **kwargs: all the other args, forwarded to go_test
+    """
+    go_test(
+        name = name + "_test",
+        size = size,
+        pure = pure,
+        nogo = False,  # FIXME(gvisor.dev/issue/3374): Not working with all build systems.
+        tags = [
+            "local",
+            "manual",
+        ],
+        **kwargs
+    )
+
+PacketimpactTestInfo = provider(
+    doc = "Provide information for packetimpact tests",
+    fields = ["name", "expect_netstack_failure"],
+)
+
+ALL_TESTS = [
+    PacketimpactTestInfo(
+        name = "fin_wait2_timeout",
+    ),
+    PacketimpactTestInfo(
+        name = "ipv4_id_uniqueness",
+    ),
+    PacketimpactTestInfo(
+        name = "udp_discard_mcast_source_addr",
+    ),
+    PacketimpactTestInfo(
+        name = "udp_recv_mcast_bcast",
+    ),
+    PacketimpactTestInfo(
+        name = "udp_any_addr_recv_unicast",
+    ),
+    PacketimpactTestInfo(
+        name = "udp_icmp_error_propagation",
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_reordering",
+        # TODO(b/139368047): Fix netstack then remove the line below.
+        expect_netstack_failure = True,
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_window_shrink",
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_zero_window_probe",
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_zero_window_probe_retransmit",
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_zero_window_probe_usertimeout",
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_retransmits",
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_outside_the_window",
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_noaccept_close_rst",
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_send_window_sizes_piggyback",
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_unacc_seq_ack",
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_paws_mechanism",
+        # TODO(b/156682000): Fix netstack then remove the line below.
+        expect_netstack_failure = True,
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_user_timeout",
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_queue_receive_in_syn_sent",
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_synsent_reset",
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_synrcvd_reset",
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_network_unreachable",
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_cork_mss",
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_handshake_window_size",
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_timewait_reset",
+        # TODO(b/168523247): Fix netstack then remove the line below.
+        expect_netstack_failure = True,
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_queue_send_in_syn_sent",
+    ),
+    PacketimpactTestInfo(
+        name = "icmpv6_param_problem",
+        # TODO(b/153485026): Fix netstack then remove the line below.
+        expect_netstack_failure = True,
+    ),
+    PacketimpactTestInfo(
+        name = "ipv6_unknown_options_action",
+        # TODO(b/159928940): Fix netstack then remove the line below.
+        expect_netstack_failure = True,
+    ),
+    PacketimpactTestInfo(
+        name = "ipv6_fragment_reassembly",
+    ),
+    PacketimpactTestInfo(
+        name = "udp_send_recv_dgram",
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_linger",
+    ),
+    PacketimpactTestInfo(
+        name = "tcp_rcv_buf_space",
+    ),
+]
+
+def validate_all_tests():
+    """
+    Make sure that ALL_TESTS list is in sync with the rules in BUILD.
+
+    This function is order-dependent, it is intended to be used after
+    all packetimpact_testbench rules and before using ALL_TESTS list
+    at the end of BUILD.
+    """
+    all_tests_dict = {}  # there is no set, using dict to approximate.
+    for test in ALL_TESTS:
+        rule_name = test.name + "_test"
+        all_tests_dict[rule_name] = True
+        if not native.existing_rule(rule_name):
+            fail("%s does not have a packetimpact_testbench rule in BUILD" % test.name)
+    for name in native.existing_rules():
+        if name.endswith("_test") and name not in all_tests_dict:
+            fail("%s is not declared in ALL_TESTS list in defs.bzl" % name[:-5])

test/packetimpact/tests/BUILD

@@ -1,11 +1,11 @@
-load("//test/packetimpact/runner:defs.bzl", "packetimpact_go_test")
+load("//test/packetimpact/runner:defs.bzl", "ALL_TESTS", "packetimpact_go_test", "packetimpact_testbench", "validate_all_tests")
 
 package(
     default_visibility = ["//test/packetimpact:__subpackages__"],
     licenses = ["notice"],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "fin_wait2_timeout",
     srcs = ["fin_wait2_timeout_test.go"],
     deps = [
@@ -15,7 +15,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "ipv4_id_uniqueness",
     srcs = ["ipv4_id_uniqueness_test.go"],
     deps = [
@@ -26,7 +26,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "udp_discard_mcast_source_addr",
     srcs = ["udp_discard_mcast_source_addr_test.go"],
     deps = [
@@ -37,7 +37,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "udp_recv_mcast_bcast",
     srcs = ["udp_recv_mcast_bcast_test.go"],
     deps = [
@@ -49,7 +49,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "udp_any_addr_recv_unicast",
     srcs = ["udp_any_addr_recv_unicast_test.go"],
     deps = [
@@ -60,7 +60,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "udp_icmp_error_propagation",
     srcs = ["udp_icmp_error_propagation_test.go"],
     deps = [
@@ -71,11 +71,9 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_reordering",
     srcs = ["tcp_reordering_test.go"],
-    # TODO(b/139368047): Fix netstack then remove the line below.
-    expect_netstack_failure = True,
     deps = [
         "//pkg/tcpip/header",
         "//pkg/tcpip/seqnum",
@@ -84,7 +82,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_window_shrink",
     srcs = ["tcp_window_shrink_test.go"],
     deps = [
@@ -94,7 +92,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_zero_window_probe",
     srcs = ["tcp_zero_window_probe_test.go"],
     deps = [
@@ -104,7 +102,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_zero_window_probe_retransmit",
     srcs = ["tcp_zero_window_probe_retransmit_test.go"],
     deps = [
@@ -114,7 +112,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_zero_window_probe_usertimeout",
     srcs = ["tcp_zero_window_probe_usertimeout_test.go"],
     deps = [
@@ -124,7 +122,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_retransmits",
     srcs = ["tcp_retransmits_test.go"],
     deps = [
@@ -134,7 +132,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_outside_the_window",
     srcs = ["tcp_outside_the_window_test.go"],
     deps = [
@@ -145,7 +143,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_noaccept_close_rst",
     srcs = ["tcp_noaccept_close_rst_test.go"],
     deps = [
@@ -155,7 +153,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_send_window_sizes_piggyback",
     srcs = ["tcp_send_window_sizes_piggyback_test.go"],
     deps = [
@@ -165,7 +163,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_unacc_seq_ack",
     srcs = ["tcp_unacc_seq_ack_test.go"],
     deps = [
@@ -176,11 +174,9 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_paws_mechanism",
     srcs = ["tcp_paws_mechanism_test.go"],
-    # TODO(b/156682000): Fix netstack then remove the line below.
-    expect_netstack_failure = True,
     deps = [
         "//pkg/tcpip/header",
         "//pkg/tcpip/seqnum",
@@ -189,7 +185,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_user_timeout",
     srcs = ["tcp_user_timeout_test.go"],
     deps = [
@@ -199,7 +195,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_queue_receive_in_syn_sent",
     srcs = ["tcp_queue_receive_in_syn_sent_test.go"],
     deps = [
@@ -209,7 +205,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_synsent_reset",
     srcs = ["tcp_synsent_reset_test.go"],
     deps = [
@@ -219,7 +215,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_synrcvd_reset",
     srcs = ["tcp_synrcvd_reset_test.go"],
     deps = [
@@ -229,7 +225,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_network_unreachable",
     srcs = ["tcp_network_unreachable_test.go"],
     deps = [
@@ -239,7 +235,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_cork_mss",
     srcs = ["tcp_cork_mss_test.go"],
     deps = [
@@ -249,7 +245,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_handshake_window_size",
     srcs = ["tcp_handshake_window_size_test.go"],
     deps = [
@@ -259,11 +255,9 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_timewait_reset",
     srcs = ["tcp_timewait_reset_test.go"],
-    # TODO(b/168523247): Fix netstack then remove the line below.
-    expect_netstack_failure = True,
     deps = [
         "//pkg/tcpip/header",
         "//test/packetimpact/testbench",
@@ -271,7 +265,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_queue_send_in_syn_sent",
     srcs = ["tcp_queue_send_in_syn_sent_test.go"],
     deps = [
@@ -281,11 +275,9 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "icmpv6_param_problem",
     srcs = ["icmpv6_param_problem_test.go"],
-    # TODO(b/153485026): Fix netstack then remove the line below.
-    expect_netstack_failure = True,
     deps = [
         "//pkg/tcpip",
         "//pkg/tcpip/header",
@@ -294,11 +286,9 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "ipv6_unknown_options_action",
     srcs = ["ipv6_unknown_options_action_test.go"],
-    # TODO(b/159928940): Fix netstack then remove the line below.
-    expect_netstack_failure = True,
     deps = [
         "//pkg/tcpip",
         "//pkg/tcpip/header",
@@ -307,7 +297,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "ipv6_fragment_reassembly",
     srcs = ["ipv6_fragment_reassembly_test.go"],
     deps = [
@@ -319,7 +309,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "udp_send_recv_dgram",
     srcs = ["udp_send_recv_dgram_test.go"],
     deps = [
@@ -329,7 +319,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_linger",
     srcs = ["tcp_linger_test.go"],
     deps = [
@@ -339,7 +329,7 @@ packetimpact_go_test(
     ],
 )
 
-packetimpact_go_test(
+packetimpact_testbench(
     name = "tcp_rcv_buf_space",
     srcs = ["tcp_rcv_buf_space_test.go"],
     deps = [
@@ -348,3 +338,10 @@ packetimpact_go_test(
         "@org_golang_x_sys//unix:go_default_library",
     ],
 )
+
+validate_all_tests()
+
+[packetimpact_go_test(
+    name = t.name,
+    expect_netstack_failure = hasattr(t, "expect_netstack_failure"),
+) for t in ALL_TESTS]