From b318556f83c06024c19088999ccfed4566ceb979 Mon Sep 17 00:00:00 2001 From: Zeling Feng Date: Mon, 15 Nov 2021 11:42:17 -0800 Subject: [PATCH] Dockerless PacketImpact - More hermetic test network setup - Easier to use/setup without docker Updates #6018, #6835. PiperOrigin-RevId: 410039248 --- WORKSPACE | 11 +- go.mod | 3 +- go.sum | 6 +- images/default/Dockerfile | 2 +- test/packetimpact/dut/BUILD | 13 +- test/packetimpact/dut/dut.go | 146 +++++ test/packetimpact/dut/linux/BUILD | 17 + test/packetimpact/dut/linux/linux.go | 95 ++++ test/packetimpact/dut/native/BUILD | 22 + test/packetimpact/dut/native/main.go | 95 ++++ test/packetimpact/dut/runsc/BUILD | 33 ++ test/packetimpact/dut/runsc/devid.go | 35 ++ test/packetimpact/dut/runsc/main.go | 191 +++++++ test/packetimpact/internal/testing/BUILD | 12 + test/packetimpact/internal/testing/testing.go | 31 ++ test/packetimpact/netdevs/netlink/BUILD | 18 + test/packetimpact/netdevs/netlink/netlink.go | 51 ++ test/packetimpact/runner/BUILD | 19 +- test/packetimpact/runner/defs.bzl | 91 +++- test/packetimpact/runner/main.go | 507 ++++++++++++++++++ test/packetimpact/tests/BUILD | 1 + 21 files changed, 1368 insertions(+), 31 deletions(-) create mode 100644 test/packetimpact/dut/dut.go create mode 100644 test/packetimpact/dut/linux/BUILD create mode 100644 test/packetimpact/dut/linux/linux.go create mode 100644 test/packetimpact/dut/native/BUILD create mode 100644 test/packetimpact/dut/native/main.go create mode 100644 test/packetimpact/dut/runsc/BUILD create mode 100644 test/packetimpact/dut/runsc/devid.go create mode 100644 test/packetimpact/dut/runsc/main.go create mode 100644 test/packetimpact/internal/testing/BUILD create mode 100644 test/packetimpact/internal/testing/testing.go create mode 100644 test/packetimpact/netdevs/netlink/BUILD create mode 100644 test/packetimpact/netdevs/netlink/netlink.go create mode 100644 test/packetimpact/runner/main.go diff --git a/WORKSPACE b/WORKSPACE index 48df49ca6..56728dc01 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -564,6 +564,13 @@ go_repository( version = "v0.27.0", ) +go_repository( + name = "com_github_google_gopacket", + importpath = "github.com/google/gopacket", + sum = "h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=", + version = "v1.1.19", +) + rules_proto_dependencies() rules_proto_toolchains() @@ -1439,8 +1446,8 @@ go_repository( go_repository( name = "com_github_vishvananda_netns", importpath = "github.com/vishvananda/netns", - sum = "h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=", - version = "v0.0.0-20210104183010-2eb08e3e575f", + sum = "h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg=", + version = "v0.0.0-20211101163701-50045581ed74", ) go_repository( diff --git a/go.mod b/go.mod index 9b72bb677..6423026e5 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( github.com/google/go-cmp v0.5.6 // indirect github.com/google/go-github/v35 v35.1.0 // indirect github.com/google/gofuzz v1.0.0 // indirect + github.com/google/gopacket v1.1.19 // indirect github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0 // indirect github.com/google/subcommands v1.0.2-0.20190508160503-636abe8753b8 // indirect github.com/google/uuid v1.1.2 // indirect @@ -59,7 +60,7 @@ require ( github.com/sirupsen/logrus v1.8.1 // indirect github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 // indirect github.com/vishvananda/netlink v1.0.1-0.20190930145447-2ec5bdc52b86 // indirect - github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect + github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect diff --git a/go.sum b/go.sum index c26d27a76..1d130353e 100644 --- a/go.sum +++ b/go.sum @@ -304,6 +304,8 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= +github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -533,8 +535,8 @@ github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vishvananda/netlink v1.0.1-0.20190930145447-2ec5bdc52b86 h1:7SWt9pGCMaw+N1ZhRsaLKaYNviFhxambdoaoYlDqz1w= github.com/vishvananda/netlink v1.0.1-0.20190930145447-2ec5bdc52b86/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= -github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA= -github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg= +github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= diff --git a/images/default/Dockerfile b/images/default/Dockerfile index 4384d6271..a9f54498e 100644 --- a/images/default/Dockerfile +++ b/images/default/Dockerfile @@ -7,7 +7,7 @@ RUN apt-get update && apt-get install -y curl gnupg2 git \ openjdk-11-jdk-headless zip unzip \ apt-transport-https ca-certificates gnupg-agent \ software-properties-common \ - pkg-config libffi-dev patch diffutils libssl-dev + pkg-config libffi-dev patch diffutils libssl-dev iptables # Install Docker client for the website build. RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - diff --git a/test/packetimpact/dut/BUILD b/test/packetimpact/dut/BUILD index 0be14ca3e..a21261a79 100644 --- a/test/packetimpact/dut/BUILD +++ b/test/packetimpact/dut/BUILD @@ -1,4 +1,4 @@ -load("//tools:defs.bzl", "cc_binary", "grpcpp") +load("//tools:defs.bzl", "cc_binary", "go_library", "grpcpp") package( default_visibility = ["//test/packetimpact:__subpackages__"], @@ -28,3 +28,14 @@ cc_binary( "@com_google_absl//absl/strings:str_format", ], ) + +go_library( + name = "dut", + testonly = True, + srcs = ["dut.go"], + deps = [ + "//test/packetimpact/testbench", + "@org_golang_x_sync//errgroup:go_default_library", + "@org_golang_x_sys//unix:go_default_library", + ], +) diff --git a/test/packetimpact/dut/dut.go b/test/packetimpact/dut/dut.go new file mode 100644 index 000000000..8f83f57c4 --- /dev/null +++ b/test/packetimpact/dut/dut.go @@ -0,0 +1,146 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +// Package dut provides common definitions and utilities to be shared by DUTs. +package dut + +import ( + "bufio" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "os" + "os/signal" + "strings" + + "golang.org/x/sync/errgroup" + "golang.org/x/sys/unix" + "gvisor.dev/gvisor/test/packetimpact/testbench" +) + +const ( + // completeFd is used for notifying the parent for the completion of setup. + completeFd = 3 + // PosixServerPort is the port the posix server should listen on. + PosixServerPort = 54321 +) + +// Ifaces describe the names of the interfaces on DUT. +type Ifaces struct { + // Ctrl is the name of the control interface. + Ctrl string + // Test is the name of the test interface. + Test string +} + +// Init puts the current process into the target network namespace, the user of +// this library should call this function in the beginning. +func Init() (Ifaces, error) { + // The DUT might create child processes, we don't want this fd to leak into + // those processes as it keeps the pipe open and the testbench will hang + // waiting for an EOF on the pipe. + unix.CloseOnExec(completeFd) + var ifaces Ifaces + // Parse command line flags. It is effectively the same as using top-level + // functions in flag package, but more explicit that we exit if the parsing + // failed. + fs := flag.NewFlagSet(os.Args[0], flag.ContinueOnError) + fs.StringVar(&ifaces.Ctrl, "ctrl_iface", "", "the name of the control interface") + fs.StringVar(&ifaces.Test, "test_iface", "", "the name of the test interface") + if err := fs.Parse(os.Args[1:]); err != nil { + return Ifaces{}, err + } + return ifaces, nil +} + +// DUT is an interface for different platforms of DUTs. +type DUT interface { + // Bootstrap starts a DUT and returns the collected DUTInfo and a function + // for the caller to call to wait for the completion of the DUT. + Bootstrap(ctx context.Context) (testbench.DUTInfo, func() error, error) + // Cleanup stops the DUT and cleans up the resources being used. + Cleanup() +} + +// Run is the provided function that calls dut's Bootstrap and Cleanup +// methods and returns the DUT information to the parent through the pipe. +func Run(dut DUT) error { + defer dut.Cleanup() + + // Register for cleanup signals. + stopSigs := make(chan os.Signal, 1) + signal.Notify(stopSigs, unix.SIGTERM, unix.SIGINT) + defer signal.Stop(stopSigs) + + // Start bootstrapping the DUT. + g, ctx := errgroup.WithContext(context.Background()) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + g.Go(func() error { + info, waitFn, err := dut.Bootstrap(ctx) + if err != nil { + return err + } + bytes, err := json.Marshal(info) + if err != nil { + return fmt.Errorf("failed to marshal DUT info into json: %w", err) + } + // Send the DUT information to the parent through the pipe. + completeFile := os.NewFile(completeFd, "complete") + for len(bytes) > 0 { + n, err := completeFile.Write(bytes) + if err != nil && err != io.ErrShortWrite { + return fmt.Errorf("write(%s) = %d, %w", completeFile.Name(), n, err) + } + bytes = bytes[n:] + } + if err := completeFile.Close(); err != nil { + return fmt.Errorf("close(%s) = %w", completeFile.Name(), err) + } + return waitFn() + }) + + select { + case <-ctx.Done(): + // The only reason for our context to be cancelled is the propagation of + // the cancellation from the errgroup g, which means DUT returned an error + // so we report it with g.Wait(). + if ctx.Err() == context.Canceled { + return fmt.Errorf("failed to bootstrap DUT: %w", g.Wait()) + } + panic(fmt.Sprintf("unknown reason for the context to be cancelled: %s, g.Wait() = %s", ctx.Err(), g.Wait())) + // An signal occurred, we should exit. + case <-stopSigs: + return nil + } +} + +// WaitForServer waits for a pattern to occur in posix_server's logs. +func WaitForServer(output io.Reader) error { + // TODO(gvisor.dev/issue/6835): waiting for the server via log output is + // fragile, a better way could be passing a file descriptor, which is not + // possible with docker, do that after the docker runner is removed. + scanner := bufio.NewScanner(output) + for scanner.Scan() { + if text := scanner.Text(); strings.HasPrefix(text, "Server listening on") { + return nil + } + } + return scanner.Err() +} diff --git a/test/packetimpact/dut/linux/BUILD b/test/packetimpact/dut/linux/BUILD new file mode 100644 index 000000000..609db7a54 --- /dev/null +++ b/test/packetimpact/dut/linux/BUILD @@ -0,0 +1,17 @@ +load("//tools:defs.bzl", "go_library") + +package( + default_visibility = ["//test/packetimpact/dut:__subpackages__"], + licenses = ["notice"], +) + +go_library( + name = "linux", + testonly = True, + srcs = ["linux.go"], + deps = [ + "//test/packetimpact/dut", + "//test/packetimpact/netdevs/netlink", + "//test/packetimpact/testbench", + ], +) diff --git a/test/packetimpact/dut/linux/linux.go b/test/packetimpact/dut/linux/linux.go new file mode 100644 index 000000000..1cdd16e38 --- /dev/null +++ b/test/packetimpact/dut/linux/linux.go @@ -0,0 +1,95 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +// Package linux provides utilities specific to bringing up linux DUTs. +package linux + +import ( + "os/exec" + "strings" + + "gvisor.dev/gvisor/test/packetimpact/dut" + netdevs "gvisor.dev/gvisor/test/packetimpact/netdevs/netlink" + "gvisor.dev/gvisor/test/packetimpact/testbench" +) + +const ( + // PosixServerPath is the path to the posix_server. + PosixServerPath = "test/packetimpact/dut/posix_server" +) + +func uname() (*testbench.DUTUname, error) { + machine, err := exec.Command("uname", "-m").Output() + if err != nil { + return nil, err + } + kernelRelease, err := exec.Command("uname", "-r").Output() + if err != nil { + return nil, err + } + kernelVersion, err := exec.Command("uname", "-v").Output() + if err != nil { + return nil, err + } + kernelName, err := exec.Command("uname", "-s").Output() + if err != nil { + return nil, err + } + operatingSystem, err := exec.Command("uname", "-o").Output() + if err != nil { + return nil, err + } + return &testbench.DUTUname{ + Machine: strings.TrimRight(string(machine), "\n"), + KernelName: strings.TrimRight(string(kernelName), "\n"), + KernelRelease: strings.TrimRight(string(kernelRelease), "\n"), + KernelVersion: strings.TrimRight(string(kernelVersion), "\n"), + OperatingSystem: strings.TrimRight(string(operatingSystem), "\n"), + }, nil +} + +// DUTInfo gatthers information about the linux DUT. +func DUTInfo(ifaces dut.Ifaces) (testbench.DUTInfo, error) { + _, ctrlIPv4, _, err := netdevs.IfaceInfo(ifaces.Ctrl) + if err != nil { + return testbench.DUTInfo{}, err + } + testLink, testIPv4, testIPv6, err := netdevs.IfaceInfo(ifaces.Test) + if err != nil { + return testbench.DUTInfo{}, err + } + dutUname, err := uname() + if err != nil { + return testbench.DUTInfo{}, err + } + + prefix, _ := testIPv4.Mask.Size() + + return testbench.DUTInfo{ + Net: &testbench.DUTTestNet{ + RemoteIPv6: testIPv6.IP, + RemoteIPv4: testIPv4.IP.To4(), + IPv4PrefixLength: prefix, + RemoteDevID: uint32(testLink.Attrs().Index), + RemoteDevName: ifaces.Test, + POSIXServerIP: ctrlIPv4.IP.To4(), + POSIXServerPort: dut.PosixServerPort, + RemoteMAC: testLink.Attrs().HardwareAddr, + }, + Uname: dutUname, + }, nil +} diff --git a/test/packetimpact/dut/native/BUILD b/test/packetimpact/dut/native/BUILD new file mode 100644 index 000000000..c9565ce2f --- /dev/null +++ b/test/packetimpact/dut/native/BUILD @@ -0,0 +1,22 @@ +load("//tools:defs.bzl", "go_binary") + +package( + default_visibility = ["//test/packetimpact:__subpackages__"], + licenses = ["notice"], +) + +go_binary( + name = "native", + testonly = True, + srcs = ["main.go"], + data = [ + "//test/packetimpact/dut:posix_server", + ], + deps = [ + "//pkg/test/testutil", + "//test/packetimpact/dut", + "//test/packetimpact/dut/linux", + "//test/packetimpact/testbench", + "@org_golang_x_sys//unix:go_default_library", + ], +) diff --git a/test/packetimpact/dut/native/main.go b/test/packetimpact/dut/native/main.go new file mode 100644 index 000000000..534b5c16e --- /dev/null +++ b/test/packetimpact/dut/native/main.go @@ -0,0 +1,95 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux && go1.10 +// +build linux,go1.10 + +// The native binary is used to bring up a native linux DUT. +package main + +import ( + "context" + "fmt" + "log" + "os" + "os/exec" + "strconv" + + "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/test/testutil" + "gvisor.dev/gvisor/test/packetimpact/dut" + "gvisor.dev/gvisor/test/packetimpact/dut/linux" + "gvisor.dev/gvisor/test/packetimpact/testbench" +) + +var _ dut.DUT = (*native)(nil) + +type native struct { + dut.Ifaces +} + +func main() { + ifaces, err := dut.Init() + if err != nil { + log.Fatal(err) + } + if err := dut.Run(&native{ + Ifaces: ifaces, + }); err != nil { + log.Fatal(err) + } +} + +// Bootstrap implements dut.DUT. +func (n *native) Bootstrap(ctx context.Context) (testbench.DUTInfo, func() error, error) { + // Enable ICMP sockets. + if err := os.WriteFile("/proc/sys/net/ipv4/ping_group_range", []byte("0 0"), 0); err != nil { + return testbench.DUTInfo{}, nil, fmt.Errorf("failed to enable icmp sockets: %w", err) + } + + // Find the posix_server binary. + path, err := testutil.FindFile(linux.PosixServerPath) + if err != nil { + return testbench.DUTInfo{}, nil, fmt.Errorf("failed to find the posix_server binary: %w", err) + } + + // Start the process. + cmd := exec.CommandContext(ctx, path, "--ip", "0.0.0.0", "--port", strconv.FormatUint(dut.PosixServerPort, 10)) + cmd.SysProcAttr = &unix.SysProcAttr{ + Pdeathsig: unix.SIGKILL, + } + errPipe, err := cmd.StderrPipe() + if err != nil { + return testbench.DUTInfo{}, nil, fmt.Errorf("failed to create stderr pipe to the posix server process: %w", err) + } + if err := cmd.Start(); err != nil { + return testbench.DUTInfo{}, nil, fmt.Errorf("failed to start the posix server process: %w", err) + } + if err := dut.WaitForServer(errPipe); err != nil { + return testbench.DUTInfo{}, nil, fmt.Errorf("failed to wait for the server to listen: %w", err) + } + + // Collect DUT information. + info, err := linux.DUTInfo(n.Ifaces) + if err != nil { + return testbench.DUTInfo{}, nil, fmt.Errorf("failed to collect information about the DUT: %w", err) + } + return info, cmd.Wait, nil +} + +// Bootstrap implements dut.DUT +func (*native) Cleanup() { + // For a native DUT case, we only need to cleanup the posix_server process which we set up to + // deliver a SIGKILL signal whenever we exit, so there is nothing to do here. +} diff --git a/test/packetimpact/dut/runsc/BUILD b/test/packetimpact/dut/runsc/BUILD new file mode 100644 index 000000000..9adcd7e70 --- /dev/null +++ b/test/packetimpact/dut/runsc/BUILD @@ -0,0 +1,33 @@ +load("//tools:defs.bzl", "go_binary") + +package( + default_visibility = ["//test/packetimpact:__subpackages__"], + licenses = ["notice"], +) + +go_binary( + name = "runsc", + testonly = True, + srcs = ["main.go"], + data = [ + ":devid", + "//runsc", + "//test/packetimpact/dut:posix_server", + ], + deps = [ + "//pkg/test/testutil", + "//test/packetimpact/dut", + "//test/packetimpact/dut/linux", + "//test/packetimpact/internal/testing", + "//test/packetimpact/testbench", + "@com_github_opencontainers_runtime_spec//specs-go:go_default_library", + "@org_golang_x_sys//unix:go_default_library", + ], +) + +go_binary( + name = "devid", + testonly = True, + srcs = ["devid.go"], + deps = ["@com_github_vishvananda_netlink//:go_default_library"], +) diff --git a/test/packetimpact/dut/runsc/devid.go b/test/packetimpact/dut/runsc/devid.go new file mode 100644 index 000000000..ada046454 --- /dev/null +++ b/test/packetimpact/dut/runsc/devid.go @@ -0,0 +1,35 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +// The devid binary is used to get the device ID in the runsc container. +package main + +import ( + "fmt" + "log" + "os" + + "github.com/vishvananda/netlink" +) + +func main() { + link, err := netlink.LinkByName(os.Args[1]) + if err != nil { + log.Fatalf("could not find the link: %s", err) + } + fmt.Printf("%d", link.Attrs().Index) +} diff --git a/test/packetimpact/dut/runsc/main.go b/test/packetimpact/dut/runsc/main.go new file mode 100644 index 000000000..8df6834f6 --- /dev/null +++ b/test/packetimpact/dut/runsc/main.go @@ -0,0 +1,191 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +// The runsc binary is used to bring up a gVisor DUT. +package main + +import ( + "context" + "fmt" + "log" + "os" + "os/exec" + "strconv" + + specs "github.com/opencontainers/runtime-spec/specs-go" + "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/test/testutil" + "gvisor.dev/gvisor/test/packetimpact/dut" + "gvisor.dev/gvisor/test/packetimpact/dut/linux" + "gvisor.dev/gvisor/test/packetimpact/internal/testing" + "gvisor.dev/gvisor/test/packetimpact/testbench" +) + +type runsc struct { + dut.Ifaces + containerID string + runscPath string + runscLogsPath string + bundleDir string + rootDir string + cleanupRootDir func() + cleanupBundleDir func() +} + +var _ dut.DUT = (*runsc)(nil) + +func main() { + ifaces, err := dut.Init() + if err != nil { + log.Fatal(err) + } + // Find the path to the binaries. + posixServerPath, err := testutil.FindFile(linux.PosixServerPath) + if err != nil { + log.Fatalf("failed to find posix_server binary: %s", err) + } + runscPath, err := testutil.FindFile("runsc/runsc") + if err != nil { + log.Fatalf("failed to find runsc binary: %s", err) + } + + // Create the OCI spec for the container with posix_server as the entrypoint. + spec := testutil.NewSpecWithArgs(posixServerPath, "--ip", "0.0.0.0", "--port", strconv.FormatUint(dut.PosixServerPort, 10)) + pwd, err := os.Getwd() + if err != nil { + log.Fatalf("failed to get the current working directory: %s", err) + } + spec.Process.Cwd = pwd + if spec.Linux == nil { + spec.Linux = &specs.Linux{} + } + + // Use the DUT namespace which is the current namespace's. + spec.Linux.Namespaces = append(spec.Linux.Namespaces, specs.LinuxNamespace{ + Type: "network", + Path: fmt.Sprintf("/proc/%d/ns/net", os.Getpid()), + }) + + // Prepare logs. + runscLogPath, err := testing.UndeclaredOutput("runsc.%%TIMESTAMP%%.%%COMMAND%%.log") + if err != nil { + log.Fatalf("failed to create runsc log file: %s", err) + } + + // Build the command to start runsc container. + bundleDir, cleanupBundleDir, err := testutil.SetupBundleDir(spec) + if err != nil { + log.Fatalf("failed to create bundle dir: %s", err) + } + rootDir, cleanupRootDir, err := testutil.SetupRootDir() + if err != nil { + cleanupBundleDir() + log.Fatalf("SetupRootDir failed: %v", err) + } + if err := dut.Run(&runsc{ + Ifaces: ifaces, + containerID: testutil.RandomContainerID(), + runscPath: runscPath, + runscLogsPath: runscLogPath, + bundleDir: bundleDir, + rootDir: rootDir, + cleanupRootDir: cleanupRootDir, + cleanupBundleDir: cleanupBundleDir, + }); err != nil { + log.Fatal(err) + } +} + +// Bootstrap implements dut.DUT. +func (r *runsc) Bootstrap(ctx context.Context) (testbench.DUTInfo, func() error, error) { + // runsc will flush the addresses so we collect the info before we start it. + info, err := linux.DUTInfo(r.Ifaces) + if err != nil { + return testbench.DUTInfo{}, nil, fmt.Errorf("failed to collect information about the DUT: %w", err) + } + + // Start posix_server inside a runsc container. + cmd := exec.CommandContext( + ctx, + r.runscPath, + "-root", r.rootDir, + "-network=sandbox", + "-debug", + "-debug-log", r.runscLogsPath, + "-log-format=text", + "-TESTONLY-unsafe-nonroot=true", + "-net-raw=true", + fmt.Sprintf("-panic-signal=%d", unix.SIGTERM), + "-watchdog-action=panic", + "run", + "-bundle", r.bundleDir, + r.containerID, + ) + errPipe, err := cmd.StderrPipe() + if err != nil { + return testbench.DUTInfo{}, nil, fmt.Errorf("failed to create stderr pipe to the posix server process: %w", err) + } + if err := cmd.Start(); err != nil { + return testbench.DUTInfo{}, nil, fmt.Errorf("failed to start the posix server process: %w", err) + } + if err := dut.WaitForServer(errPipe); err != nil { + return testbench.DUTInfo{}, nil, fmt.Errorf("failed to wait for the server to listen: %w", err) + } + + // runsc will keep using the assigned ip and mac addresses, but the device + // id could have changed, we need to figure it out. + remoteDevID, err := r.remoteDevID() + if err != nil { + return testbench.DUTInfo{}, nil, fmt.Errorf("failed to get test dev id: %w", err) + } + info.Net.RemoteDevID = remoteDevID + return info, cmd.Wait, nil +} + +// Cleanup implements dut.DUT. +func (r *runsc) Cleanup() { + r.cleanupRootDir() + r.cleanupBundleDir() +} + +// remoteDevID gets the id of the test interface inside the runsc container. +func (r *runsc) remoteDevID() (uint32, error) { + runscDevIDPath, err := testutil.FindFile("test/packetimpact/dut/runsc/devid") + if err != nil { + return 0, fmt.Errorf("failed to find binary runsc_devid: %w", err) + } + cmd := exec.Command( + r.runscPath, + "-root", + r.rootDir, + "-TESTONLY-unsafe-nonroot=true", + "exec", + r.containerID, + runscDevIDPath, + r.Ifaces.Test, + ) + bytes, err := cmd.CombinedOutput() + output := string(bytes) + if err != nil { + return 0, fmt.Errorf("failed to get the remote device id: %w, output: %s", err, output) + } + id, err := strconv.ParseUint(output, 10, 32) + if err != nil { + return 0, fmt.Errorf("%s is not a number: %w", output, err) + } + return uint32(id), nil +} diff --git a/test/packetimpact/internal/testing/BUILD b/test/packetimpact/internal/testing/BUILD new file mode 100644 index 000000000..41d4331eb --- /dev/null +++ b/test/packetimpact/internal/testing/BUILD @@ -0,0 +1,12 @@ +load("//tools:defs.bzl", "go_library") + +package( + default_visibility = ["//test/packetimpact:__subpackages__"], + licenses = ["notice"], +) + +go_library( + name = "testing", + testonly = True, + srcs = ["testing.go"], +) diff --git a/test/packetimpact/internal/testing/testing.go b/test/packetimpact/internal/testing/testing.go new file mode 100644 index 000000000..909fec84f --- /dev/null +++ b/test/packetimpact/internal/testing/testing.go @@ -0,0 +1,31 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package testing provides common testing functionalities. +package testing + +import ( + "fmt" + "os" + "path/filepath" +) + +// UndeclaredOutput creates a path under the undeclared outputs directory. +func UndeclaredOutput(name string) (string, error) { + const testUndeclaredOutputsDir = "TEST_UNDECLARED_OUTPUTS_DIR" + if dir, ok := os.LookupEnv(testUndeclaredOutputsDir); ok { + return filepath.Join(dir, name), nil + } + return "", fmt.Errorf("no %s env var", testUndeclaredOutputsDir) +} diff --git a/test/packetimpact/netdevs/netlink/BUILD b/test/packetimpact/netdevs/netlink/BUILD new file mode 100644 index 000000000..2317b4f4f --- /dev/null +++ b/test/packetimpact/netdevs/netlink/BUILD @@ -0,0 +1,18 @@ +load("//tools:defs.bzl", "go_library") + +package( + licenses = ["notice"], +) + +go_library( + name = "netlink", + testonly = True, + srcs = ["netlink.go"], + visibility = [ + "//test/packetimpact:__subpackages__", + ], + deps = [ + "@com_github_vishvananda_netlink//:go_default_library", + "@org_golang_x_sys//unix:go_default_library", + ], +) diff --git a/test/packetimpact/netdevs/netlink/netlink.go b/test/packetimpact/netdevs/netlink/netlink.go new file mode 100644 index 000000000..d1fd0d0c4 --- /dev/null +++ b/test/packetimpact/netdevs/netlink/netlink.go @@ -0,0 +1,51 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +// Package netlink has routines to get interfaces information through netlink. +package netlink + +import ( + "fmt" + + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" +) + +// IfaceInfo returns the device with its IPv4 and IPv6 addresses. An error is +// returned if the device is not present or there are no or more than 1 ip addr +// per address family. +func IfaceInfo(name string) (netlink.Link, netlink.Addr, netlink.Addr, error) { + link, err := netlink.LinkByName(name) + if err != nil { + return nil, netlink.Addr{}, netlink.Addr{}, fmt.Errorf("failed to get interface %s: %w", name, err) + } + ipv4Addrs, err := netlink.AddrList(link, unix.AF_INET) + if err != nil { + return nil, netlink.Addr{}, netlink.Addr{}, fmt.Errorf("failed to get ipv4 addrs: %w", err) + } + if len(ipv4Addrs) != 1 { + return nil, netlink.Addr{}, netlink.Addr{}, fmt.Errorf("expected 1 ipv4 addresses, got %d", len(ipv4Addrs)) + } + ipv6Addrs, err := netlink.AddrList(link, unix.AF_INET6) + if err != nil { + return nil, netlink.Addr{}, netlink.Addr{}, fmt.Errorf("failed to get ipv6 addrs: %w", err) + } + if len(ipv6Addrs) != 1 { + return nil, netlink.Addr{}, netlink.Addr{}, fmt.Errorf("expected 1 ipv6 addresses, got %d", len(ipv6Addrs)) + } + return link, ipv4Addrs[0], ipv6Addrs[0], nil +} diff --git a/test/packetimpact/runner/BUILD b/test/packetimpact/runner/BUILD index 888c44343..d1e56b340 100644 --- a/test/packetimpact/runner/BUILD +++ b/test/packetimpact/runner/BUILD @@ -1,4 +1,4 @@ -load("//tools:defs.bzl", "bzl_library", "go_library", "go_test") +load("//tools:defs.bzl", "bzl_library", "go_binary", "go_library", "go_test") package( default_visibility = ["//test/packetimpact:__subpackages__"], @@ -36,3 +36,20 @@ go_library( "@com_github_docker_docker//api/types/mount:go_default_library", ], ) + +go_binary( + name = "main", + testonly = True, + srcs = ["main.go"], + deps = [ + "//test/packetimpact/internal/testing", + "//test/packetimpact/netdevs/netlink", + "//test/packetimpact/testbench", + "@com_github_google_gopacket//:go_default_library", + "@com_github_google_gopacket//layers:go_default_library", + "@com_github_google_gopacket//pcapgo:go_default_library", + "@com_github_vishvananda_netlink//:go_default_library", + "@org_golang_x_sync//errgroup:go_default_library", + "@org_golang_x_sys//unix:go_default_library", + ], +) diff --git a/test/packetimpact/runner/defs.bzl b/test/packetimpact/runner/defs.bzl index 19f6cc0e0..d74ae9105 100644 --- a/test/packetimpact/runner/defs.bzl +++ b/test/packetimpact/runner/defs.bzl @@ -3,8 +3,12 @@ load("//tools:defs.bzl", "go_test") def _packetimpact_test_impl(ctx): - test_runner = ctx.executable._test_runner + test_runner = ctx.executable.test_runner bench = ctx.actions.declare_file("%s-bench" % ctx.label.name) + dut_binary_flag = [] if ctx.attr.dut_binary == None else [ + "--dut_binary", + ctx.file.dut_binary.short_path, + ] bench_content = "\n".join([ "#!/bin/bash", # This test will run part in a distinct user namespace. This can cause @@ -14,7 +18,7 @@ def _packetimpact_test_impl(ctx): "find . -type f -or -type d -exec chmod a+rx {} \\;", "%s %s --testbench_binary %s --num_duts %d $@\n" % ( test_runner.short_path, - " ".join(ctx.attr.flags), + " ".join(ctx.attr.flags + dut_binary_flag), ctx.files.testbench_binary[0].short_path, ctx.attr.num_duts, ), @@ -22,9 +26,11 @@ def _packetimpact_test_impl(ctx): ctx.actions.write(bench, bench_content, is_executable = True) transitive_files = [] - if hasattr(ctx.attr._test_runner, "data_runfiles"): - transitive_files.append(ctx.attr._test_runner.data_runfiles.files) - files = [test_runner] + ctx.files.testbench_binary + ctx.files._posix_server + if hasattr(ctx.attr.test_runner, "data_runfiles"): + transitive_files.append(ctx.attr.test_runner.data_runfiles.files) + if hasattr(ctx.attr.dut_binary, "data_runfiles"): + transitive_files.append(ctx.attr.dut_binary.data_runfiles.files) + files = [test_runner] + ctx.files.testbench_binary + ctx.files._posix_server + ctx.files.dut_binary runfiles = ctx.runfiles( files = files, transitive_files = depset(transitive = transitive_files), @@ -35,7 +41,7 @@ def _packetimpact_test_impl(ctx): _packetimpact_test = rule( attrs = { - "_test_runner": attr.label( + "test_runner": attr.label( executable = True, cfg = "target", default = ":packetimpact_test", @@ -56,6 +62,11 @@ _packetimpact_test = rule( mandatory = False, default = 1, ), + "dut_binary": attr.label( + executable = True, + cfg = "target", + allow_single_file = True, + ), }, test = True, implementation = _packetimpact_test_impl, @@ -71,6 +82,7 @@ def packetimpact_native_test( name, testbench_binary, expect_failure = False, + legacy_runner = False, **kwargs): """Add a native packetimpact test. @@ -78,21 +90,34 @@ def packetimpact_native_test( name: name of the test testbench_binary: the testbench binary expect_failure: the test must fail + legacy_runner: use the legacy docker runner **kwargs: all the other args, forwarded to _packetimpact_test """ expect_failure_flag = ["--expect_failure"] if expect_failure else [] - _packetimpact_test( - name = name + "_native_test", - testbench_binary = testbench_binary, - flags = ["--native"] + expect_failure_flag, - tags = PACKETIMPACT_TAGS, - **kwargs - ) + if legacy_runner: + _packetimpact_test( + name = name + "_native_test", + testbench_binary = testbench_binary, + flags = ["--native"] + expect_failure_flag, + tags = PACKETIMPACT_TAGS, + **kwargs + ) + else: + _packetimpact_test( + test_runner = "//test/packetimpact/runner:main", + name = name + "_native_test", + testbench_binary = testbench_binary, + flags = expect_failure_flag + ["--variant", "native"], + dut_binary = "//test/packetimpact/dut/native", + tags = PACKETIMPACT_TAGS, + **kwargs + ) def packetimpact_netstack_test( name, testbench_binary, expect_failure = False, + legacy_runner = False, **kwargs): """Add a packetimpact test on netstack. @@ -100,22 +125,34 @@ def packetimpact_netstack_test( name: name of the test testbench_binary: the testbench binary expect_failure: the test must fail + legacy_runner: use the legacy docker runner **kwargs: all the other args, forwarded to _packetimpact_test """ expect_failure_flag = [] if expect_failure: expect_failure_flag = ["--expect_failure"] - _packetimpact_test( - name = name + "_netstack_test", - testbench_binary = testbench_binary, - # Note that a distinct runtime must be provided in the form - # --test_arg=--runtime=other when invoking bazel. - flags = expect_failure_flag, - tags = PACKETIMPACT_TAGS, - **kwargs - ) + if legacy_runner: + _packetimpact_test( + name = name + "_netstack_test", + testbench_binary = testbench_binary, + # Note that a distinct runtime must be provided in the form + # --test_arg=--runtime=other when invoking bazel. + flags = expect_failure_flag, + tags = PACKETIMPACT_TAGS, + **kwargs + ) + else: + _packetimpact_test( + test_runner = "//test/packetimpact/runner:main", + name = name + "_netstack_test", + testbench_binary = testbench_binary, + flags = expect_failure_flag + ["--variant", "gvisor"], + dut_binary = "//test/packetimpact/dut/runsc", + tags = PACKETIMPACT_TAGS, + **kwargs + ) -def packetimpact_go_test(name, expect_native_failure = False, expect_netstack_failure = False, num_duts = 1, **kwargs): +def packetimpact_go_test(name, expect_native_failure = False, expect_netstack_failure = False, num_duts = 1, legacy_runner = False, **kwargs): """Add packetimpact tests written in go. Args: @@ -123,6 +160,7 @@ def packetimpact_go_test(name, expect_native_failure = False, expect_netstack_fa expect_native_failure: the test must fail natively expect_netstack_failure: the test must fail for Netstack num_duts: how many DUTs are needed for the test + legacy_runner: use the legacy docker runner **kwargs: all the other args, forwarded to packetimpact_native_test and packetimpact_netstack_test """ testbench_binary = name + "_test" @@ -131,6 +169,7 @@ def packetimpact_go_test(name, expect_native_failure = False, expect_netstack_fa expect_failure = expect_native_failure, num_duts = num_duts, testbench_binary = testbench_binary, + legacy_runner = legacy_runner, **kwargs ) packetimpact_netstack_test( @@ -138,6 +177,7 @@ def packetimpact_go_test(name, expect_native_failure = False, expect_netstack_fa expect_failure = expect_netstack_failure, num_duts = num_duts, testbench_binary = testbench_binary, + legacy_runner = legacy_runner, **kwargs ) @@ -169,6 +209,7 @@ PacketimpactTestInfo = provider( "timeout", "expect_netstack_failure", "num_duts", + "legacy_runner", ], ) @@ -293,6 +334,10 @@ ALL_TESTS = [ PacketimpactTestInfo( name = "generic_dgram_socket_send_recv", timeout = "long", + # This test has assumed the presense of the default interface and the + # default route installed by docker, using the docker until the test + # is migrated. + legacy_runner = True, ), ] diff --git a/test/packetimpact/runner/main.go b/test/packetimpact/runner/main.go new file mode 100644 index 000000000..4f13aa248 --- /dev/null +++ b/test/packetimpact/runner/main.go @@ -0,0 +1,507 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux && go1.10 +// +build linux,go1.10 + +// The runner binary is used as the test runner for PacketImpact tests. +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "runtime" + "syscall" + + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "github.com/google/gopacket/pcapgo" + "github.com/vishvananda/netlink" + "golang.org/x/sync/errgroup" + "golang.org/x/sys/unix" + "gvisor.dev/gvisor/test/packetimpact/internal/testing" + netdevs "gvisor.dev/gvisor/test/packetimpact/netdevs/netlink" + "gvisor.dev/gvisor/test/packetimpact/testbench" +) + +func main() { + const procSelfExe = "/proc/self/exe" + if os.Args[0] != procSelfExe { + // For the first time, re-execute in a new user name space and a new + // network namespace. + cmd := exec.Command(procSelfExe, os.Args[1:]...) + cmd.SysProcAttr = &unix.SysProcAttr{ + Cloneflags: unix.CLONE_NEWUSER | unix.CLONE_NEWNET, + Pdeathsig: unix.SIGTERM, + UidMappings: []syscall.SysProcIDMap{ + { + ContainerID: 0, + HostID: os.Getuid(), + Size: 1, + }, + }, + GidMappings: []syscall.SysProcIDMap{ + { + ContainerID: 0, + HostID: os.Getgid(), + Size: 1, + }, + }, + } + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + if exitStatus, ok := err.(*exec.ExitError); ok { + os.Exit(exitStatus.ExitCode()) + } else { + log.Fatalf("unknown failure: %s", err) + } + } + return + } + + var ( + dutBinary string + testBinary string + expectFailure bool + numDUTs int + variant string + runtime string + partition int + totalPartitions int + ) + fs := flag.NewFlagSet(os.Args[0], flag.ContinueOnError) + fs.StringVar(&dutBinary, "dut_binary", "", "path to the DUT binary") + fs.StringVar(&testBinary, "testbench_binary", "", "path to the test binary") + fs.BoolVar(&expectFailure, "expect_failure", false, "whether the test is expected to fail") + fs.IntVar(&numDUTs, "num_duts", 1, "number of DUTs to create") + fs.StringVar(&variant, "variant", "", "test variant could be native, gvisor or fuchsia") + // The following args are passed by CI environment which are not used by us. + fs.StringVar(&runtime, "runtime", "", "docker runtime to use (unused)") + fs.IntVar(&partition, "partition", 1, "1-indexed partition (unused)") + fs.IntVar(&totalPartitions, "total_partitions", 1, "total partitions (unused)") + if err := fs.Parse(os.Args[1:]); err != nil { + log.Fatal(err) + } + + g, ctx := errgroup.WithContext(context.Background()) + + // Create all the DUTs. + infoCh := make(chan testbench.DUTInfo, numDUTs) + var duts []*dut + for i := 0; i < numDUTs; i++ { + d, err := newDUT(ctx, i, dutBinary) + if err != nil { + log.Fatal(err) + } + duts = append(duts, d) + g.Go(func() error { + info, waitFn, err := d.bootstrap(ctx) + if err != nil { + return err + } + infoCh <- info + return waitFn() + }) + } + + // Wait for all the DUTs to bootstrap. + var infos []testbench.DUTInfo + for i := 0; i < numDUTs; i++ { + select { + case <-ctx.Done(): + log.Fatalf("failed to bootstrap dut: %s", g.Wait()) + case info := <-infoCh: + infos = append(infos, info) + } + } + + dutJSON, err := json.Marshal(&infos) + if err != nil { + log.Fatalf("failed to marshal json: %s", err) + } + + for _, d := range duts { + // When the Linux kernel receives a SYN-ACK for a SYN it didn't send, it + // will respond with an RST. In most packetimpact tests, the SYN is sent + // by the raw socket, the kernel knows nothing about the connection, this + // behavior will break lots of TCP related packetimpact tests. To prevent + // this, we can install the following iptables rules. The raw socket that + // packetimpact tests use will still be able to see everything. + for _, iptables := range []string{"/sbin/iptables-nft", "/sbin/ip6tables-nft"} { + cmd := exec.Command(iptables, "-A", "INPUT", "-i", d.peerIface(), "--proto", "tcp", "-j", "DROP") + if output, err := cmd.CombinedOutput(); err != nil { + log.Fatalf("failed to set iptables: %s, output: %s", err, string(output)) + } + } + // Start packet capture. + g.Go(func() error { + return d.writePcap(ctx, filepath.Base(testBinary)) + }) + } + + // Start the test itself. + testResult := make(chan error, 1) + go func() { + testArgs := []string{"--dut_infos_json", string(dutJSON)} + if variant == "native" { + testArgs = append(testArgs, "-native") + } + test := exec.CommandContext(ctx, testBinary, testArgs...) + test.SysProcAttr = &unix.SysProcAttr{ + Pdeathsig: unix.SIGTERM, + } + test.Stderr = os.Stderr + test.Stdout = os.Stdout + testResult <- test.Run() + }() + + select { + case <-ctx.Done(): + log.Fatalf("background tasks exited early: %s", g.Wait()) + case err := <-testResult: + switch { + case err != nil == expectFailure: + // Expected. + case expectFailure: + log.Fatalf("the test is expected to fail, but it succeeded") + case err != nil: + var exitStatus *exec.ExitError + if errors.As(err, &exitStatus) { + os.Exit(exitStatus.ExitCode()) + } + log.Fatalf("unknown error when executing test: %s", err) + } + } +} + +type dut struct { + cmd *exec.Cmd + id int + completeR *os.File + dutNetNS netNS +} + +func newDUT(ctx context.Context, id int, dutBinary string) (*dut, error) { + cmd := exec.CommandContext(ctx, dutBinary, "--ctrl_iface", dutSide.ifaceName(ctrlLink, id), "--test_iface", dutSide.ifaceName(testLink, id)) + + // Create the pipe for completion signal + completeR, completeW, err := os.Pipe() + if err != nil { + return nil, fmt.Errorf("failed to create pipe for completion signal: %w", err) + } + + // Create a new network namespace for the DUT. + dutNetNS, err := newNetNS() + if err != nil { + return nil, fmt.Errorf("failed to create a new namespace for DUT: %w", err) + } + + // Pass these two file descriptors to the DUT. + cmd.ExtraFiles = append(cmd.ExtraFiles, completeW) + + // Deliver SIGTERM to the child when the runner exits. + cmd.SysProcAttr = &unix.SysProcAttr{ + Pdeathsig: unix.SIGTERM, + } + + // Stream outputs from the DUT binary. + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + // Now create the veth pairs to connect the DUT and us. + for _, typ := range []linkType{ctrlLink, testLink} { + dutSideIfaceName := dutSide.ifaceName(typ, id) + tbSideIfaceName := tbSide.ifaceName(typ, id) + dutVeth := netlink.Veth{ + LinkAttrs: netlink.LinkAttrs{ + Name: dutSideIfaceName, + }, + PeerName: tbSideIfaceName, + } + tbVeth := netlink.Veth{ + LinkAttrs: netlink.LinkAttrs{ + Name: tbSideIfaceName, + }, + PeerName: dutSideIfaceName, + } + if err := netlink.LinkAdd(&dutVeth); err != nil { + return nil, fmt.Errorf("failed to add a %s veth pair for dut-%d: %w", typ, id, err) + } + + tbIPv4 := typ.ipv4(uint8(id), 1) + dutIPv4 := typ.ipv4(uint8(id), 2) + + // Move the DUT end into the created namespace. + if err := netlink.LinkSetNsFd(&dutVeth, int(dutNetNS)); err != nil { + return nil, fmt.Errorf("failed to move %s veth end to dut-%d: %w", typ, id, err) + } + + for _, conf := range []struct { + ns netNS + addr *netlink.Addr + veth *netlink.Veth + }{ + {ns: currentNetNS, addr: tbIPv4, veth: &tbVeth}, + {ns: dutNetNS, addr: dutIPv4, veth: &dutVeth}, + } { + if err := conf.ns.Do(func() error { + // Disable the DAD so that the generated IPv6 address can be used immediately. + if err := disableDad(conf.veth.Name); err != nil { + return fmt.Errorf("failed to disable DAD on %s: %w", conf.veth.Name, err) + } + // Manually add the IPv4 address. + if err := netlink.AddrAdd(conf.veth, conf.addr); err != nil { + return fmt.Errorf("failed to add addr %s to %s: %w", conf.addr, conf.veth.Name, err) + } + // Bring the link up. + if err := netlink.LinkSetUp(conf.veth); err != nil { + return fmt.Errorf("failed to set %s up: %w", conf.veth.Name, err) + } + return nil + }); err != nil { + return nil, err + } + } + } + + // Bring the loopback interface up in both namespaces. + for _, ns := range []netNS{currentNetNS, dutNetNS} { + if err := ns.Do(func() error { + return netlink.LinkSetUp(&netlink.Device{ + LinkAttrs: netlink.LinkAttrs{ + Name: "lo", + }, + }) + }); err != nil { + return nil, fmt.Errorf("failed to bring loopback up: %w", err) + } + } + + return &dut{cmd: cmd, id: id, completeR: completeR, dutNetNS: dutNetNS}, nil +} + +func (d *dut) bootstrap(ctx context.Context) (testbench.DUTInfo, func() error, error) { + if err := d.dutNetNS.Do(func() error { + return d.cmd.Start() + }); err != nil { + return testbench.DUTInfo{}, nil, fmt.Errorf("failed to start DUT %d: %w", d.id, err) + } + for _, file := range d.cmd.ExtraFiles { + if err := file.Close(); err != nil { + return testbench.DUTInfo{}, nil, fmt.Errorf("close(%d) = %w", file.Fd(), err) + } + } + + bytes, err := io.ReadAll(d.completeR) + if err != nil { + return testbench.DUTInfo{}, nil, fmt.Errorf("failed to read from %s complete pipe: %w", d.name(), err) + } + if err := d.completeR.Close(); err != nil { + return testbench.DUTInfo{}, nil, fmt.Errorf("failed to close the read end of completion pipe: %w", err) + } + var dutInfo testbench.DUTInfo + if err := json.Unmarshal(bytes, &dutInfo); err != nil { + return testbench.DUTInfo{}, nil, fmt.Errorf("invalid response from %s: %w, received: %s", d.name(), err, string(bytes)) + } + testIface, testIPv4, testIPv6, err := netdevs.IfaceInfo(d.peerIface()) + if err != nil { + return testbench.DUTInfo{}, nil, fmt.Errorf("failed to gather information about the testbench: %w", err) + } + dutInfo.Net.LocalMAC = testIface.Attrs().HardwareAddr + dutInfo.Net.LocalIPv4 = testIPv4.IP.To4() + dutInfo.Net.LocalIPv6 = testIPv6.IP + dutInfo.Net.LocalDevID = uint32(testIface.Attrs().Index) + dutInfo.Net.LocalDevName = testIface.Attrs().Name + return dutInfo, d.cmd.Wait, nil +} + +func (d *dut) name() string { + return fmt.Sprintf("dut-%d", d.id) +} + +func (d *dut) peerIface() string { + return tbSide.ifaceName(testLink, d.id) +} + +// writePcap creates the packet capture while the test is running. +func (d *dut) writePcap(ctx context.Context, testName string) error { + iface := d.peerIface() + // Create the pcap file. + fileName, err := testing.UndeclaredOutput(fmt.Sprintf("%s_%s.pcap", testName, iface)) + if err != nil { + return err + } + pcap, err := os.Create(fileName) + if err != nil { + return fmt.Errorf("open(%s) = %w", fileName, err) + } + defer func() { + if err := pcap.Close(); err != nil { + panic(fmt.Sprintf("close(%s) = %s", pcap.Name(), err)) + } + }() + + // Start the packet capture. + pcapw := pcapgo.NewWriter(pcap) + if err := pcapw.WriteFileHeader(1600, layers.LinkTypeEthernet); err != nil { + return fmt.Errorf("WriteFileHeader: %w", err) + } + handle, err := pcapgo.NewEthernetHandle(iface) + if err != nil { + return fmt.Errorf("pcapgo.NewEthernetHandle(%s): %w", iface, err) + } + source := gopacket.NewPacketSource(handle, layers.LayerTypeEthernet) + for { + select { + case packet := <-source.Packets(): + if err := pcapw.WritePacket(packet.Metadata().CaptureInfo, packet.Data()); err != nil { + return fmt.Errorf("pcapw.WritePacket(): %w", err) + } + case <-ctx.Done(): + return ctx.Err() + } + } +} + +// disableDad disables DAD on the iface when assigning IPv6 addrs. +func disableDad(iface string) error { + // DAD operation and mode on a given interface will be selected according to + // the maximum value of conf/{all,interface}/accept_dad. So we set it to 0 on + // both `iface` and `all`. + for _, name := range []string{iface, "all"} { + path := fmt.Sprintf("/proc/sys/net/ipv6/conf/%s/accept_dad", name) + if err := os.WriteFile(path, []byte("0"), 0); err != nil { + return err + } + } + return nil +} + +// netNS is a network namespace. +type netNS int + +const ( + currentNetNS netNS = -1 +) + +// newNetNS creates a new network namespace. +func newNetNS() (netNS, error) { + ns := currentNetNS + err := withSavedNetNS(func() error { + // Create the namespace via unshare(2). + if err := unix.Unshare(unix.CLONE_NEWNET); err != nil { + return err + } + // Return the created namespace. + fd, err := openNetNSFD() + if err != nil { + return err + } + ns = netNS(fd) + return nil + }) + return ns, err +} + +// Do calls the function in the given network namespace. +func (ns netNS) Do(f func() error) error { + if ns == currentNetNS { + // Simply call the function if we are already in the namespace. + return f() + } + return withSavedNetNS(func() error { + // Switch to the target namespace. + if err := unix.Setns(int(ns), unix.CLONE_NEWNET); err != nil { + return err + } + return f() + }) +} + +// linkType describes if the link is for ctrl or test. +type linkType string + +const ( + testLink linkType = "test" + ctrlLink linkType = "ctrl" +) + +// ipv4 creates an IPv4 address for the given network and host number. +func (l linkType) ipv4(network uint8, host uint8) *netlink.Addr { + const ( + testNetworkNumber uint8 = 172 + ctrlNetworkNumber uint8 = 192 + ) + var leadingByte uint8 + switch l { + case testLink: + leadingByte = testNetworkNumber + case ctrlLink: + leadingByte = ctrlNetworkNumber + default: + panic(fmt.Sprintf("unknown link type: %s", l)) + } + addr, err := netlink.ParseAddr(fmt.Sprintf("%d.0.%d.%d/24", leadingByte, network, host)) + if err != nil { + panic(fmt.Sprintf("failed to parse ip net: %s", err)) + } + return addr +} + +// side describes which side of the link (tb/dut). +type side string + +const ( + dutSide side = "dut" + tbSide side = "tb" +) + +func (s side) ifaceName(typ linkType, id int) string { + return fmt.Sprintf("%s-%d-%s", s, id, typ) +} + +// withSavedNetNS saves the current namespace and restores it after calling f. +func withSavedNetNS(f func() error) error { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + // Save the current namespace. + saved, err := openNetNSFD() + if err != nil { + return err + } + defer func() { + // Resotre the namespace when we return from f. + if err := unix.Setns(saved, unix.CLONE_NEWNET); err != nil { + panic(fmt.Sprintf("setns(%d, CLONE_NEWNET) = %s", saved, err)) + } + if err := unix.Close(saved); err != nil { + panic(fmt.Sprintf("close(%d) = %s", saved, err)) + } + }() + return f() +} + +func openNetNSFD() (int, error) { + nsPath := fmt.Sprintf("/proc/self/task/%d/ns/net", unix.Gettid()) + return unix.Open(nsPath, unix.O_RDONLY|unix.O_CLOEXEC, 0) +} diff --git a/test/packetimpact/tests/BUILD b/test/packetimpact/tests/BUILD index 4cff0cf4c..1d143809b 100644 --- a/test/packetimpact/tests/BUILD +++ b/test/packetimpact/tests/BUILD @@ -417,6 +417,7 @@ validate_all_tests() name = t.name, timeout = t.timeout if hasattr(t, "timeout") else "moderate", expect_netstack_failure = hasattr(t, "expect_netstack_failure"), + legacy_runner = t.legacy_runner if hasattr(t, "legacy_runner") else False, num_duts = t.num_duts if hasattr(t, "num_duts") else 1, ) for t in ALL_TESTS]