From 3a9134c2f04b2e21ce0cfc5b33633ce573aa9429 Mon Sep 17 00:00:00 2001 From: Kevin Krakauer Date: Wed, 30 Aug 2023 17:34:56 -0700 Subject: [PATCH] netstack: benchmark xdp endpoint This just swaps out the fdbased endpoint for an XDP one. Some related changes in here: - Moved the AF_XDP BPF program into its own Go package so it can be shared. - Added a flag to tcp_benchmark to disable user namespaces. This helps when running as root, which is required to install BPF programs. --- pkg/xdp/xdp.go | 2 + runsc/sandbox/BUILD | 4 +- runsc/sandbox/bpf/BUILD | 11 ++- runsc/sandbox/bpf/bpf.go | 24 ++++++ runsc/sandbox/network.go | 9 +-- test/benchmarks/tcp/BUILD | 9 ++- test/benchmarks/tcp/tcp_benchmark.sh | 21 +++++- test/benchmarks/tcp/tcp_proxy.go | 53 +++++++------ test/benchmarks/tcp/xdp.go | 107 +++++++++++++++++++++++++++ 9 files changed, 201 insertions(+), 39 deletions(-) create mode 100644 runsc/sandbox/bpf/bpf.go create mode 100644 test/benchmarks/tcp/xdp.go diff --git a/pkg/xdp/xdp.go b/pkg/xdp/xdp.go index 0cd6f13df..95eb85cbd 100644 --- a/pkg/xdp/xdp.go +++ b/pkg/xdp/xdp.go @@ -63,6 +63,8 @@ type ControlBlock struct { Completion CompletionQueue } +// TODO(b/240191988): None of this is read-only anymore. + // ReadOnlySocketOpts configure a read-only AF_XDP socket. type ReadOnlySocketOpts struct { NFrames uint32 diff --git a/runsc/sandbox/BUILD b/runsc/sandbox/BUILD index fdaff211e..23c2aee66 100644 --- a/runsc/sandbox/BUILD +++ b/runsc/sandbox/BUILD @@ -13,9 +13,6 @@ go_library( "network_unsafe.go", "sandbox.go", ], - embedsrcs = [ - "//runsc/sandbox/bpf:af_xdp_ebpf.o", # keep - ], visibility = [ "//runsc:__subpackages__", "//test:__subpackages__", @@ -43,6 +40,7 @@ go_library( "//runsc/config", "//runsc/console", "//runsc/donation", + "//runsc/sandbox/bpf", "//runsc/specutils", "@com_github_cenkalti_backoff//:go_default_library", "@com_github_cilium_ebpf//:go_default_library", diff --git a/runsc/sandbox/bpf/BUILD b/runsc/sandbox/bpf/BUILD index 5d88f0e95..acd0ce8e7 100644 --- a/runsc/sandbox/bpf/BUILD +++ b/runsc/sandbox/bpf/BUILD @@ -1,10 +1,19 @@ -load("//tools:defs.bzl", "bpf_program") +load("//tools:defs.bzl", "bpf_program", "go_library") package( default_applicable_licenses = ["//:license"], licenses = ["notice"], ) +go_library( + name = "bpf", + srcs = ["bpf.go"], + embedsrcs = [ + "af_xdp_ebpf.o", # keep + ], + visibility = ["//visibility:public"], +) + bpf_program( name = "af_xdp_ebpf", src = "af_xdp.ebpf.c", diff --git a/runsc/sandbox/bpf/bpf.go b/runsc/sandbox/bpf/bpf.go new file mode 100644 index 000000000..59ec6f00c --- /dev/null +++ b/runsc/sandbox/bpf/bpf.go @@ -0,0 +1,24 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package bpf provides compiled bpf programs as byte slices. +package bpf + +import _ "embed" + +// AFXDPProgram is a BPF program that, when attached to a device, redirects all +// packets to a single AF_XDP socket unconditionally. +// +//go:embed af_xdp_ebpf.o +var AFXDPProgram []byte diff --git a/runsc/sandbox/network.go b/runsc/sandbox/network.go index 8ed614449..49bdc9595 100644 --- a/runsc/sandbox/network.go +++ b/runsc/sandbox/network.go @@ -16,7 +16,6 @@ package sandbox import ( "bytes" - _ "embed" "fmt" "net" "os" @@ -35,6 +34,7 @@ import ( "gvisor.dev/gvisor/pkg/urpc" "gvisor.dev/gvisor/runsc/boot" "gvisor.dev/gvisor/runsc/config" + "gvisor.dev/gvisor/runsc/sandbox/bpf" "gvisor.dev/gvisor/runsc/specutils" ) @@ -417,11 +417,6 @@ func createSocket(iface net.Interface, ifaceLink netlink.Link, enableGSO bool) ( return &socketEntry{deviceFile, gsoMaxSize}, nil } -// program is the BPF program to attach to the socket. -// -//go:embed bpf/af_xdp_ebpf.o -var program []byte - func createSocketXDP(iface net.Interface) ([]*os.File, error) { // Create an XDP socket. The sentry will mmap memory for the various // rings and bind to the device. @@ -434,7 +429,7 @@ func createSocketXDP(iface net.Interface) ([]*os.File, error) { // device and insert our socket into its map. // Load into the kernel. - spec, err := ebpf.LoadCollectionSpecFromReader(bytes.NewReader(program)) + spec, err := ebpf.LoadCollectionSpecFromReader(bytes.NewReader(bpf.AFXDPProgram)) if err != nil { return nil, fmt.Errorf("failed to load spec: %v", err) } diff --git a/test/benchmarks/tcp/BUILD b/test/benchmarks/tcp/BUILD index adb1efca0..6e239ae2a 100644 --- a/test/benchmarks/tcp/BUILD +++ b/test/benchmarks/tcp/BUILD @@ -7,7 +7,10 @@ package( go_binary( name = "tcp_proxy", - srcs = ["tcp_proxy.go"], + srcs = [ + "tcp_proxy.go", + "xdp.go", + ], visibility = ["//:sandbox"], deps = [ "//pkg/tcpip", @@ -15,6 +18,7 @@ go_binary( "//pkg/tcpip/link/fdbased", "//pkg/tcpip/link/qdisc/fifo", "//pkg/tcpip/link/sniffer", + "//pkg/tcpip/link/xdp", "//pkg/tcpip/network/arp", "//pkg/tcpip/network/ipv4", "//pkg/tcpip/network/ipv6", @@ -22,6 +26,9 @@ go_binary( "//pkg/tcpip/transport/icmp", "//pkg/tcpip/transport/tcp", "//pkg/tcpip/transport/udp", + "//runsc/sandbox/bpf", + "@com_github_cilium_ebpf//:go_default_library", + "@com_github_cilium_ebpf//link:go_default_library", "@org_golang_x_sys//unix:go_default_library", ], ) diff --git a/test/benchmarks/tcp/tcp_benchmark.sh b/test/benchmarks/tcp/tcp_benchmark.sh index 2c1a8eaff..1c267f4a6 100755 --- a/test/benchmarks/tcp/tcp_benchmark.sh +++ b/test/benchmarks/tcp/tcp_benchmark.sh @@ -20,7 +20,6 @@ iperf_port=45201 # Not likely to be privileged. proxy_port=44000 # Ditto. mask=8 - client_addr=10.0.0.1 client_proxy_addr=10.0.0.2 server_proxy_addr=10.0.0.3 @@ -51,6 +50,8 @@ disable_linux_gro= gro=0 num_client_threads=1 sniff=false +xdp=false +declare -a unshare_opts=( -U -r ) # Check for netem support. lsmod_output=$(lsmod | grep sch_netem) @@ -203,6 +204,12 @@ while [[ $# -gt 0 ]]; do --sniff) netstack_opts="${netstack_opts} -sniff" ;; + --xdp) + xdp=true + ;; + --no-user-ns) + unshare_opts=() + ;; *) echo "unknown option: $1" echo "" @@ -229,6 +236,10 @@ while [[ $# -gt 0 ]]; do echo " --disable-linux-gro disable GRO in the Linux network stack" echo " --gro set gVisor GRO timeout" echo " --ipv6 use ipv6 for benchmarks" + echo " --iperf-binary name of the iperf binary to call" + echo " --sniff sniff and output packet logs" + echo " --xdp use AF_XDP socket instead of AF_PACKET" + echo " --no-user-ns don't run in a new user namespace. Useful for testing as root" echo "" echo "The output will of the script will be:" echo " " @@ -277,7 +288,8 @@ if ${client}; then # and forward traffic using netstack. client_args="${proxy_binary} ${netstack_opts} -port ${proxy_port} -client \\ -mtu ${mtu} -iface client.0 -addr ${client_proxy_addr} -mask ${mask} \\ - -forward ${full_server_proxy_addr} -gso=${gso} -swgso=${swgso} --gro=${gro}" + -forward ${full_server_proxy_addr} -gso=${gso} -swgso=${swgso} --gro=${gro} \\ + --xdp=${xdp}" fi # Server proxy that will listen on the proxy port and forward to the server's @@ -288,7 +300,8 @@ if ${server}; then # iperf server using netstack. server_args="${proxy_binary} ${netstack_opts} -port ${proxy_port} -server \\ -mtu ${mtu} -iface server.0 -addr ${server_proxy_addr} -mask ${mask} \\ - -forward ${full_server_addr} -gso=${gso} -swgso=${swgso} --gro=${gro}" + -forward ${full_server_addr} -gso=${gso} -swgso=${swgso} --gro=${gro} \\ + --xdp=${xdp}" fi # Specify loss and duplicate parameters only if they are non-zero @@ -301,7 +314,7 @@ if [[ "$(echo "$half_duplicate" | bc -q)" != "0" ]]; then duplicate_opt="duplicate ${half_duplicate}%" fi -exec unshare -U -m -n -r -f -p --mount-proc /bin/bash << EOF +exec unshare "${unshare_opts[@]}" -m -n -f -p --mount-proc /bin/bash << EOF set -e -m if [[ ${verbose} == "true" ]]; then diff --git a/test/benchmarks/tcp/tcp_proxy.go b/test/benchmarks/tcp/tcp_proxy.go index 75585ad8d..d6082a0aa 100644 --- a/test/benchmarks/tcp/tcp_proxy.go +++ b/test/benchmarks/tcp/tcp_proxy.go @@ -74,6 +74,7 @@ var ( traceprofile = flag.String("traceprofile", "", "write a 5s trace of the benchmark to the specified file.") useIpv6 = flag.Bool("ipv6", false, "use ipv6 instead of ipv4.") sniff = flag.Bool("sniff", false, "log sniffed packets") + useXDP = flag.Bool("xdp", false, "use AF_XDP as a link enpoint instead of fdbased") ) type impl interface { @@ -159,11 +160,6 @@ func setupNetwork(ifaceName string, numChannels int) (fds []int, err error) { } func newNetstackImpl(mode string) (impl, error) { - fds, err := setupNetwork(*iface, runtime.GOMAXPROCS(-1)) - if err != nil { - return nil, err - } - // Parse details. var parsedAddr tcpip.Address if *useIpv6 { @@ -210,25 +206,36 @@ func newNetstackImpl(mode string) (impl, error) { rand.Read(mac) // Fill with random data. mac[0] &^= 0x1 // Clear multicast bit. mac[0] |= 0x2 // Set local assignment bit (IEEE802). - ep, err := fdbased.New(&fdbased.Options{ - FDs: fds, - MTU: uint32(*mtu), - EthernetHeader: true, - Address: tcpip.LinkAddress(mac), - // Enable checksum generation as we need to generate valid - // checksums for the veth device to deliver our packets to the - // peer. But we do want to disable checksum verification as veth - // devices do perform GRO and the linux host kernel may not - // regenerate valid checksums after GRO. - TXChecksumOffload: false, - RXChecksumOffload: true, - //PacketDispatchMode: fdbased.RecvMMsg, - PacketDispatchMode: fdbased.PacketMMap, - GSOMaxSize: uint32(*gso), - GvisorGSOEnabled: *swgso, - }) + var ep stack.LinkEndpoint + var err error + if *useXDP { + ep, err = newXDPEndpoint(*iface, mac) + } else { + var fds []int + fds, err = setupNetwork(*iface, runtime.GOMAXPROCS(0)) + if err != nil { + return nil, err + } + ep, err = fdbased.New(&fdbased.Options{ + FDs: fds, + MTU: uint32(*mtu), + EthernetHeader: true, + Address: tcpip.LinkAddress(mac), + // Enable checksum generation as we need to generate valid + // checksums for the veth device to deliver our packets to the + // peer. But we do want to disable checksum verification as veth + // devices do perform GRO and the linux host kernel may not + // regenerate valid checksums after GRO. + TXChecksumOffload: false, + RXChecksumOffload: true, + //PacketDispatchMode: fdbased.RecvMMsg, + PacketDispatchMode: fdbased.PacketMMap, + GSOMaxSize: uint32(*gso), + GvisorGSOEnabled: *swgso, + }) + } if err != nil { - return nil, fmt.Errorf("failed to create FD endpoint: %v", err) + return nil, fmt.Errorf("failed to create endpoint: %v", err) } if *sniff { diff --git a/test/benchmarks/tcp/xdp.go b/test/benchmarks/tcp/xdp.go new file mode 100644 index 000000000..5260dd641 --- /dev/null +++ b/test/benchmarks/tcp/xdp.go @@ -0,0 +1,107 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "fmt" + "net" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/link" + "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/link/xdp" + "gvisor.dev/gvisor/pkg/tcpip/stack" + "gvisor.dev/gvisor/runsc/sandbox/bpf" +) + +func newXDPEndpoint(ifaceName string, mac net.HardwareAddr) (stack.LinkEndpoint, error) { + // Get all interfaces in the namespace. + ifaces, err := net.Interfaces() + if err != nil { + return nil, fmt.Errorf("querying interfaces: %v", err) + } + + // Find our specific interface. + var iface net.Interface + for _, netif := range ifaces { + if netif.Name == ifaceName { + iface = netif + break + } + } + // Zero is never used as an Index. Use that to determine whether an + // interface was found. + if iface.Index == 0 { + return nil, fmt.Errorf("failed to find interface: %v", ifaceName) + } + + // See sandbox.createSocketXDP. + + // Create an XDP socket. Later we'll mmap memory for the various rings + // and bind to the device. + fd, err := unix.Socket(unix.AF_XDP, unix.SOCK_RAW, 0) + if err != nil { + return nil, fmt.Errorf("unable to create AF_XDP socket: %v", err) + } + + // Attach a program to the device and insert our socket into its map. + + // Load into the kernel. + spec, err := ebpf.LoadCollectionSpecFromReader(bytes.NewReader(bpf.AFXDPProgram)) + if err != nil { + return nil, fmt.Errorf("failed to load spec: %v", err) + } + + var objects struct { + Program *ebpf.Program `ebpf:"xdp_prog"` + SockMap *ebpf.Map `ebpf:"sock_map"` + } + if err := spec.LoadAndAssign(&objects, nil); err != nil { + return nil, fmt.Errorf("failed to load program: %v", err) + } + + _, err = link.AttachRawLink(link.RawLinkOptions{ + Program: objects.Program, + Attach: ebpf.AttachXDP, + Target: iface.Index, + // By not setting the Flag field, the kernel will choose the + // fastest mode. In order those are: + // - Offloaded onto the NIC. + // - Running directly in the driver. + // - Generic mode, which works with any NIC/driver but lacks + // much of the XDP performance boost. + }) + if err != nil { + return nil, fmt.Errorf("failed to attach BPF program: %v", err) + } + + // Insert our AF_XDP socket into the BPF map that dictates where + // packets are redirected to. + key := uint32(0) + val := uint32(fd) + if err := objects.SockMap.Update(&key, &val, 0 /* flags */); err != nil { + return nil, fmt.Errorf("failed to insert socket into BPF map: %v", err) + } + + return xdp.New(&xdp.Options{ + FD: fd, + Address: tcpip.LinkAddress(mac), + TXChecksumOffload: false, + RXChecksumOffload: true, + InterfaceIndex: iface.Index, + }) +}