xdp: add flag for runsc to use an XDP socket on a host device

In combination with `xdp_loader redirect`, this allows `runsc` to receive
packets directly from any NIC. It is intended to be used with a machine's NIC
to avoid the Linux networking stack entirely.

PiperOrigin-RevId: 591090544

pkg/tcpip/link/xdp/endpoint.go

@@ -98,6 +98,10 @@ type Options struct {
 
 	// InterfaceIndex is the interface index of the underlying device.
 	InterfaceIndex int
+
+	// Bind is true when we're responsible for binding the AF_XDP socket to
+	// a device. When false, another process is expected to bind for us.
+	Bind bool
 }
 
 // New creates a new endpoint from an AF_XDP socket.
@@ -151,6 +155,7 @@ func New(opts *Options) (stack.LinkEndpoint, error) {
 		NFrames:      nFrames,
 		FrameSize:    frameSize,
 		NDescriptors: nFrames / 2,
+		Bind:         opts.Bind,
 	}
 	ep.control, err = xdp.ReadOnlyFromSocket(opts.FD, uint32(opts.InterfaceIndex), 0 /* queueID */, xdpOpts)
 	if err != nil {

pkg/xdp/xdp.go

@@ -70,6 +70,7 @@ type ReadOnlySocketOpts struct {
 	NFrames      uint32
 	FrameSize    uint32
 	NDescriptors uint32
+	Bind         bool
 }
 
 // DefaultReadOnlyOpts provides recommended default options for initializing a
@@ -282,26 +283,32 @@ func ReadOnlyFromSocket(sockfd int, ifaceIdx, queueID uint32, opts ReadOnlySocke
 	}
 	cb.TX.init(off, opts)
 
-	addr := unix.SockaddrXDP{
-		// XDP_USE_NEED_WAKEUP lets the driver sleep if there is no
-		// work to do. It will need to be woken by poll. It is expected
-		// that this improves performance by preventing the driver from
-		// burning cycles.
-		//
-		// By not setting either XDP_COPY or XDP_ZEROCOPY, we instruct
-		// the kernel to use zerocopy if available and then fallback to
-		// copy mode.
-		Flags:   unix.XDP_USE_NEED_WAKEUP,
-		Ifindex: ifaceIdx,
-		// AF_XDP sockets are per device RX queue, although multiple
-		// sockets on multiple queues (or devices) can share a single
-		// UMEM.
-		QueueID: queueID,
-		// We're not using shared mode, so the value here is irrelevant.
-		SharedUmemFD: 0,
-	}
-	if err := unix.Bind(sockfd, &addr); err != nil {
-		return nil, fmt.Errorf("failed to bind with addr %+v: %v", addr, err)
+	// In some cases we don't call bind, as we're not in the netns with the
+	// device. In those cases, another process with the same socket will
+	// bind for us.
+	if opts.Bind {
+		addr := unix.SockaddrXDP{
+			// XDP_USE_NEED_WAKEUP lets the driver sleep if there is no
+			// work to do. It will need to be woken by poll. It is expected
+			// that this improves performance by preventing the driver from
+			// burning cycles.
+			//
+			// By not setting either XDP_COPY or XDP_ZEROCOPY, we instruct
+			// the kernel to use zerocopy if available and then fallback to
+			// copy mode.
+			Flags:   unix.XDP_USE_NEED_WAKEUP,
+			Ifindex: ifaceIdx,
+			// AF_XDP sockets are per device RX queue, although multiple
+			// sockets on multiple queues (or devices) can share a single
+			// UMEM.
+			QueueID: queueID,
+			// We're not using shared mode, so the value here is irrelevant.
+			SharedUmemFD: 0,
+		}
+
+		if err := unix.Bind(sockfd, &addr); err != nil {
+			return nil, fmt.Errorf("failed to bind with addr %+v: %v", addr, err)
+		}
 	}
 
 	cleanup.Release()

runsc/boot/network.go

@@ -113,6 +113,18 @@ type FDBasedLink struct {
 	NumChannels int
 }
 
+// BindOpt indicates whether the sentry or runsc process is responsible for
+// binding the AF_XDP socket.
+type BindOpt int
+
+const (
+	// BindSentry indicates the sentry process must call bind.
+	BindSentry BindOpt = iota
+
+	// BindRunsc indicates the runsc process must call bind.
+	BindRunsc
+)
+
 // XDPLink configures an XDP link.
 type XDPLink struct {
 	Name              string
@@ -126,6 +138,7 @@ type XDPLink struct {
 	QDisc             config.QueueingDiscipline
 	Neighbors         []Neighbor
 	GvisorGROTimeout  time.Duration
+	Bind              BindOpt
 
 	// NumChannels controls how many underlying FDs are to be used to
 	// create this endpoint.
@@ -202,8 +215,18 @@ func (n *Network) CreateLinksAndRoutes(args *CreateLinksAndRoutesArgs, _ *struct
 	for _, l := range args.FDBasedLinks {
 		wantFDs += l.NumChannels
 	}
-	if len(args.XDPLinks) > 0 {
-		wantFDs += 4
+	for _, link := range args.XDPLinks {
+		// We have to keep several FDs alive when the sentry is
+		// responsible for binding, but when runsc binds we only expect
+		// the AF_XDP socket itself.
+		switch v := link.Bind; v {
+		case BindSentry:
+			wantFDs += 4
+		case BindRunsc:
+			wantFDs++
+		default:
+			return fmt.Errorf("unknown bind value: %d", v)
+		}
 	}
 	if args.PCAP {
 		wantFDs++
@@ -347,16 +370,18 @@ func (n *Network) CreateLinksAndRoutes(args *CreateLinksAndRoutesArgs, _ *struct
 		}
 		fdOffset++
 
-		// The parent process sends several other FDs in order
-		// to keep them open and alive. These are for BPF
-		// programs and maps that, if closed, will break the
-		// dispatcher.
-		for _, fdName := range []string{"program-fd", "sockmap-fd", "link-fd"} {
-			oldFD := args.FilePayload.Files[fdOffset].Fd()
-			if _, err := unix.Dup(int(oldFD)); err != nil {
-				return fmt.Errorf("failed to dup %s with FD %d: %v", fdName, oldFD, err)
+		// When the sentry is responsible for binding, the runsc
+		// process sends several other FDs in order to keep them open
+		// and alive. These are for BPF programs and maps that, if
+		// closed, will break the dispatcher.
+		if link.Bind == BindSentry {
+			for _, fdName := range []string{"program-fd", "sockmap-fd", "link-fd"} {
+				oldFD := args.FilePayload.Files[fdOffset].Fd()
+				if _, err := unix.Dup(int(oldFD)); err != nil {
+					return fmt.Errorf("failed to dup %s with FD %d: %v", fdName, oldFD, err)
+				}
+				fdOffset++
 			}
-			fdOffset++
 		}
 
 		mac := tcpip.LinkAddress(link.LinkAddress)
@@ -366,6 +391,7 @@ func (n *Network) CreateLinksAndRoutes(args *CreateLinksAndRoutesArgs, _ *struct
 			TXChecksumOffload: link.TXChecksumOffload,
 			RXChecksumOffload: link.RXChecksumOffload,
 			InterfaceIndex:    link.InterfaceIndex,
+			Bind:              link.Bind == BindSentry,
 		})
 		if err != nil {
 			return err

runsc/config/config.go

@@ -276,6 +276,14 @@ type Config struct {
 	// (rather than AF_PACKET). Enabling it disables RX checksum offload.
 	AFXDP bool `flag:"EXPERIMENTAL-afxdp"`
 
+	// AFXDPRedirectHost is the name of a network interface. runsc will
+	// scrape the address, routes, and neighbors of that interface, and
+	// send packets via an AF_XDP socket on that interface.
+	//
+	// Requires use of `xdp_loader redirect` to setup the XDP program and
+	// eBPF map that runsc hooks into.
+	AFXDPRedirectHost string `flag:"EXPERIMENTAL-xdp-redirect-host"`
+
 	// FDLimit specifies a limit on the number of host file descriptors that can
 	// be open simultaneously by the sentry and gofer. It applies separately to
 	// each.

runsc/config/flags.go

@@ -120,6 +120,7 @@ func RegisterFlags(flagSet *flag.FlagSet) {
 	flagSet.Int("num-network-channels", 1, "number of underlying channels(FDs) to use for network link endpoints.")
 	flagSet.Bool("buffer-pooling", true, "enable allocation of buffers from a shared pool instead of the heap.")
 	flagSet.Bool("EXPERIMENTAL-afxdp", false, "EXPERIMENTAL. Use an AF_XDP socket to receive packets.")
+	flagSet.String("EXPERIMENTAL-xdp-redirect-host", "", "EXPERIMENTAL. Use an AF_XDP socket attached to <interface name>. Use the IP of that interface.")
 	flagSet.Bool("reproduce-nat", false, "Scrape the host netns NAT table and reproduce it in the sandbox.")
 	flagSet.Bool("reproduce-nftables", false, "Attempt to scrape and reproduce nftable rules inside the sandbox. Overrides reproduce-nat when true.")
 

runsc/sandbox/BUILD

@@ -46,6 +46,7 @@ go_library(
         "//runsc/donation",
         "//runsc/sandbox/bpf",
         "//runsc/specutils",
+        "//tools/xdp/cmd",
         "@com_github_cenkalti_backoff//:go_default_library",
         "@com_github_cilium_ebpf//:go_default_library",
         "@com_github_cilium_ebpf//link:go_default_library",

runsc/sandbox/network.go

@@ -120,6 +120,13 @@ func isRootNS() (bool, error) {
 // net namespace with the given path, creates them in the sandbox, and removes
 // them from the host.
 func createInterfacesAndRoutesFromNS(conn *urpc.Client, nsPath string, conf *config.Config) error {
+	if conf.AFXDPRedirectHost != "" {
+		if err := createRedirectInterfacesAndRoutes(conn, conf); err != nil {
+			return fmt.Errorf("failed to create XDP redirect interface: %w", err)
+		}
+		return nil
+	}
+
 	// Join the network namespace that we will be copying.
 	restore, err := joinNetNS(nsPath)
 	if err != nil {

runsc/sandbox/xdp.go

@@ -1,4 +1,4 @@
-// Copyright 2018 The gVisor Authors.
+// Copyright 2023 The gVisor Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -22,10 +22,251 @@ import (
 
 	"github.com/cilium/ebpf"
 	"github.com/cilium/ebpf/link"
+	"github.com/vishvananda/netlink"
 	"golang.org/x/sys/unix"
+	"gvisor.dev/gvisor/pkg/log"
+	"gvisor.dev/gvisor/pkg/urpc"
+	"gvisor.dev/gvisor/runsc/boot"
+	"gvisor.dev/gvisor/runsc/config"
 	"gvisor.dev/gvisor/runsc/sandbox/bpf"
+	xdpcmd "gvisor.dev/gvisor/tools/xdp/cmd"
 )
 
+// createRedirectInterfacesAndRoutes initializes the network using an AF_XDP
+// socket on a *host* device, not a device in the container netns. It:
+//
+//   - scrapes the address, interface, and routes of the device and recreates
+//     them in the sandbox
+//   - does *not* remove them from the host device
+//   - creates an AF_XDP socket bound to the device
+//
+// In effect, this takes over the host device for the duration of the sentry's
+// lifetime. This also means only one container can run at a time, as it
+// monopolizes the device.
+//
+// TODO(b/240191988): Enbable device sharing via XDP_SHARED_UMEM.
+// TODO(b/240191988): IPv6 support.
+// TODO(b/240191988): Merge redundant code with CreateLinksAndRoutes once
+// features are finalized.
+func createRedirectInterfacesAndRoutes(conn *urpc.Client, conf *config.Config) error {
+	args, iface, err := prepareRedirectInterfaceArgs(conf)
+	if err != nil {
+		return fmt.Errorf("failed to generate redirect interface args: %w", err)
+	}
+
+	// Create an XDP socket. The sentry will mmap the rings.
+	xdpSockFD, err := unix.Socket(unix.AF_XDP, unix.SOCK_RAW, 0)
+	if err != nil {
+		return fmt.Errorf("unable to create AF_XDP socket: %w", err)
+	}
+	xdpSock := os.NewFile(uintptr(xdpSockFD), "xdp-sock-fd")
+
+	// Dup to ensure os.File doesn't close it prematurely.
+	if _, err := unix.Dup(xdpSockFD); err != nil {
+		return fmt.Errorf("failed to dup XDP sock: %w", err)
+	}
+	args.FilePayload.Files = append(args.FilePayload.Files, xdpSock)
+
+	// Pass PCAP log file if present.
+	if conf.PCAP != "" {
+		args.PCAP = true
+		pcap, err := os.OpenFile(conf.PCAP, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0664)
+		if err != nil {
+			return fmt.Errorf("failed to open PCAP file %s: %v", conf.PCAP, err)
+		}
+		args.FilePayload.Files = append(args.FilePayload.Files, pcap)
+	}
+
+	// Pass the host's NAT table if requested.
+	if conf.ReproduceNftables || conf.ReproduceNAT {
+		var f *os.File
+		if conf.ReproduceNftables {
+			log.Infof("reproing nftables")
+			f, err = checkNftables()
+		} else if conf.ReproduceNAT {
+			log.Infof("reproing legacy tables")
+			f, err = writeNATBlob()
+		}
+		if err != nil {
+			return fmt.Errorf("failed to write NAT blob: %v", err)
+		}
+		args.NATBlob = true
+		args.FilePayload.Files = append(args.FilePayload.Files, f)
+	}
+
+	log.Infof("Setting up network, config: %+v", args)
+	if err := conn.Call(boot.NetworkCreateLinksAndRoutes, &args, nil); err != nil {
+		return fmt.Errorf("creating links and routes: %w", err)
+	}
+
+	// Insert socket into eBPF map. Note that sockets are automatically
+	// removed from eBPF maps when released. See net/xdp/xsk.c:xsk_release
+	// and net/xdp/xsk.c:xsk_delete_from_maps.
+	mapPath := xdpcmd.RedirectMapPath(iface.Name)
+	pinnedMap, err := ebpf.LoadPinnedMap(mapPath, nil)
+	if err != nil {
+		return fmt.Errorf("failed to load pinned map %s: %w", mapPath, err)
+	}
+	mapKey := uint32(0)
+	mapVal := uint32(xdpSockFD)
+	if err := pinnedMap.Update(&mapKey, &mapVal, ebpf.UpdateAny); err != nil {
+		return fmt.Errorf("failed to insert socket into map %s: %w", mapPath, err)
+	}
+
+	// Bind to the device.
+	sockAddr := unix.SockaddrXDP{
+		// XDP_USE_NEED_WAKEUP lets the driver sleep if there is no
+		// work to do. It will need to be woken by poll. It is expected
+		// that this improves performance by preventing the driver from
+		// burning cycles.
+		//
+		// By not setting either XDP_COPY or XDP_ZEROCOPY, we instruct
+		// the kernel to use zerocopy if available and then fallback to
+		// copy mode.
+		Flags:   unix.XDP_USE_NEED_WAKEUP,
+		Ifindex: uint32(iface.Index),
+		// AF_XDP sockets are per device RX queue, although multiple
+		// sockets on multiple queues (or devices) can share a single
+		// UMEM.
+		//
+		// TODO(b/240191988): We can't assume there's only one queue,
+		// but this appears to be the case on gVNIC instances.
+		QueueID: 0,
+		// We're not using shared mode, so the value here is irrelevant.
+		SharedUmemFD: 0,
+	}
+	if err := unix.Bind(xdpSockFD, &sockAddr); err != nil {
+		return fmt.Errorf("failed to bind to interface %q with addr %+v: %v", iface.Name, sockAddr, err)
+	}
+
+	return nil
+}
+
+// Collect addresses, routes, and neighbors from the interfaces. We only
+// process two interfaces: the loopback and the interface we've been told to
+// bind to. This all takes place in the netns where the runsc binary is run,
+// *not* the netns passed to the container.
+func prepareRedirectInterfaceArgs(conf *config.Config) (boot.CreateLinksAndRoutesArgs, net.Interface, error) {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return boot.CreateLinksAndRoutesArgs{}, net.Interface{}, fmt.Errorf("querying interfaces: %w", err)
+	}
+
+	var args boot.CreateLinksAndRoutesArgs
+	var netIface net.Interface
+	for _, iface := range ifaces {
+		if iface.Flags&net.FlagUp == 0 {
+			log.Infof("Skipping down interface: %+v", iface)
+			continue
+		}
+
+		allAddrs, err := iface.Addrs()
+		if err != nil {
+			return boot.CreateLinksAndRoutesArgs{}, net.Interface{}, fmt.Errorf("fetching interface addresses for %q: %w", iface.Name, err)
+		}
+
+		// We build our own loopback device.
+		if iface.Flags&net.FlagLoopback != 0 {
+			link, err := loopbackLink(conf, iface, allAddrs)
+			if err != nil {
+				return boot.CreateLinksAndRoutesArgs{}, net.Interface{}, fmt.Errorf("getting loopback link for iface %q: %w", iface.Name, err)
+			}
+			args.LoopbackLinks = append(args.LoopbackLinks, link)
+			continue
+		}
+
+		if iface.Name != conf.AFXDPRedirectHost {
+			log.Infof("Skipping interface %q", iface.Name)
+			continue
+		}
+
+		var ipAddrs []*net.IPNet
+		for _, ifaddr := range allAddrs {
+			ipNet, ok := ifaddr.(*net.IPNet)
+			if !ok {
+				return boot.CreateLinksAndRoutesArgs{}, net.Interface{}, fmt.Errorf("address is not IPNet: %+v", ifaddr)
+			}
+			if ipNet.IP.To4() == nil {
+				log.Infof("Skipping non-IPv4 address %s", ipNet.IP)
+				continue
+			}
+			ipAddrs = append(ipAddrs, ipNet)
+		}
+		if len(ipAddrs) != 1 {
+			return boot.CreateLinksAndRoutesArgs{}, net.Interface{}, fmt.Errorf("we only handle a single IPv4 address, but interface %q has %d: %v", iface.Name, len(ipAddrs), ipAddrs)
+		}
+		prefix, _ := ipAddrs[0].Mask.Size()
+		addr := boot.IPWithPrefix{Address: ipAddrs[0].IP, PrefixLen: prefix}
+
+		// Collect data from the ARP table.
+		dump, err := netlink.NeighList(iface.Index, 0)
+		if err != nil {
+			return boot.CreateLinksAndRoutesArgs{}, net.Interface{}, fmt.Errorf("fetching ARP table for %q: %w", iface.Name, err)
+		}
+
+		var neighbors []boot.Neighbor
+		for _, n := range dump {
+			// There are only two "good" states NUD_PERMANENT and NUD_REACHABLE,
+			// but NUD_REACHABLE is fully dynamic and will be re-probed anyway.
+			if n.State == netlink.NUD_PERMANENT {
+				log.Debugf("Copying a static ARP entry: %+v %+v", n.IP, n.HardwareAddr)
+				// No flags are copied because Stack.AddStaticNeighbor does not support flags right now.
+				neighbors = append(neighbors, boot.Neighbor{IP: n.IP, HardwareAddr: n.HardwareAddr})
+			}
+		}
+
+		// Scrape routes.
+		routes, defv4, defv6, err := routesForIface(iface)
+		if err != nil {
+			return boot.CreateLinksAndRoutesArgs{}, net.Interface{}, fmt.Errorf("getting routes for interface %q: %v", iface.Name, err)
+		}
+		if defv4 != nil {
+			if !args.Defaultv4Gateway.Route.Empty() {
+				return boot.CreateLinksAndRoutesArgs{}, net.Interface{}, fmt.Errorf("more than one default route found, interface: %v, route: %v, default route: %+v", iface.Name, defv4, args.Defaultv4Gateway)
+			}
+			args.Defaultv4Gateway.Route = *defv4
+			args.Defaultv4Gateway.Name = iface.Name
+		}
+
+		if defv6 != nil {
+			if !args.Defaultv6Gateway.Route.Empty() {
+				return boot.CreateLinksAndRoutesArgs{}, net.Interface{}, fmt.Errorf("more than one default route found, interface: %v, route: %v, default route: %+v", iface.Name, defv6, args.Defaultv6Gateway)
+			}
+			args.Defaultv6Gateway.Route = *defv6
+			args.Defaultv6Gateway.Name = iface.Name
+		}
+
+		// Get the link address of the interface.
+		ifaceLink, err := netlink.LinkByName(iface.Name)
+		if err != nil {
+			return boot.CreateLinksAndRoutesArgs{}, net.Interface{}, fmt.Errorf("getting link for interface %q: %w", iface.Name, err)
+		}
+		linkAddress := ifaceLink.Attrs().HardwareAddr
+
+		xdplink := boot.XDPLink{
+			Name:              iface.Name,
+			InterfaceIndex:    iface.Index,
+			Routes:            routes,
+			TXChecksumOffload: conf.TXChecksumOffload,
+			RXChecksumOffload: conf.RXChecksumOffload,
+			NumChannels:       conf.NumNetworkChannels,
+			QDisc:             conf.QDisc,
+			Neighbors:         neighbors,
+			LinkAddress:       linkAddress,
+			Addresses:         []boot.IPWithPrefix{addr},
+			GvisorGROTimeout:  conf.GvisorGROTimeout,
+			Bind:              boot.BindRunsc,
+		}
+		args.XDPLinks = append(args.XDPLinks, xdplink)
+		netIface = iface
+	}
+
+	if len(args.XDPLinks) != 1 {
+		return boot.CreateLinksAndRoutesArgs{}, net.Interface{}, fmt.Errorf("expected 1 XDP link, but found %d", len(args.XDPLinks))
+	}
+	return args, netIface, nil
+}
+
 func createSocketXDP(iface net.Interface) ([]*os.File, error) {
 	// Create an XDP socket. The sentry will mmap memory for the various
 	// rings and bind to the device.

tools/xdp/cmd/redirect_host.go

@@ -31,6 +31,33 @@ import (
 	"gvisor.dev/gvisor/runsc/flag"
 )
 
+// bpffsDirName is the path at which BPFFS is expected to be mounted.
+const bpffsDirPath = "/sys/fs/bpf/"
+
+// RedirectPinDir returns the directory to which eBPF objects will be pinned
+// when xdp_loader is run against iface.
+func RedirectPinDir(iface string) string {
+	return filepath.Join(bpffsDirPath, iface)
+}
+
+// RedirectMapPath returns the path where the eBPF map will be pinned when
+// xdp_loader is run against iface.
+func RedirectMapPath(iface string) string {
+	return filepath.Join(RedirectPinDir(iface), "ip_map")
+}
+
+// RedirectProgramPath returns the path where the eBPF program will be pinned
+// when xdp_loader is run against iface.
+func RedirectProgramPath(iface string) string {
+	return filepath.Join(RedirectPinDir(iface), "program")
+}
+
+// RedirectLinkPath returns the path where the eBPF link will be pinned when
+// xdp_loader is run against iface.
+func RedirectLinkPath(iface string) string {
+	return filepath.Join(RedirectPinDir(iface), "link")
+}
+
 //go:embed bpf/redirect_host_ebpf.o
 var redirectProgram []byte
 
@@ -80,12 +107,11 @@ func (rc *RedirectHostCommand) execute() error {
 		return fmt.Errorf("%v", err)
 	}
 
-	const dirName = "/sys/fs/bpf/"
 	var (
-		pinDir      = filepath.Join(dirName, iface.Name)
-		mapPath     = filepath.Join(pinDir, "ip_map")
-		programPath = filepath.Join(pinDir, "program")
-		linkPath    = filepath.Join(pinDir, "link")
+		pinDir      = RedirectPinDir(iface.Name)
+		mapPath     = RedirectMapPath(iface.Name)
+		programPath = RedirectProgramPath(iface.Name)
+		linkPath    = RedirectLinkPath(iface.Name)
 	)
 
 	// User just wants to unpin things.