xdp_loader: add tcpdump-ish program

Uses an AF_XDP socket to log packets entering a device. Can be used for
testing, but mostly useful as a minimal example of receiving packets via
AF_XDP.

Note that this actually prevents the packets from reaching their intended
destination, as redirection does not also send the packet through Linux's
network stack.

PiperOrigin-RevId: 470073040

pkg/tcpip/link/sniffer/sniffer.go

@@ -55,11 +55,14 @@ var _ stack.GSOEndpoint = (*endpoint)(nil)
 var _ stack.LinkEndpoint = (*endpoint)(nil)
 var _ stack.NetworkDispatcher = (*endpoint)(nil)
 
-type direction int
+// A Direction indicates whether the packing is being sent or received.
+type Direction int
 
 const (
-	directionSend = iota
-	directionRecv
+	// DirectionSend indicates a sent packet.
+	DirectionSend = iota
+	// DirectionRecv indicates a received packet.
+	DirectionRecv
 )
 
 // New creates a new sniffer link-layer endpoint. It wraps around another
@@ -131,14 +134,14 @@ func NewWithWriter(lower stack.LinkEndpoint, writer io.Writer, snapLen uint32) (
 // called by the link-layer endpoint being wrapped when a packet arrives, and
 // logs the packet before forwarding to the actual dispatcher.
 func (e *endpoint) DeliverNetworkPacket(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) {
-	e.dumpPacket(directionRecv, protocol, pkt)
+	e.dumpPacket(DirectionRecv, protocol, pkt)
 	e.Endpoint.DeliverNetworkPacket(protocol, pkt)
 }
 
-func (e *endpoint) dumpPacket(dir direction, protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) {
+func (e *endpoint) dumpPacket(dir Direction, protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) {
 	writer := e.writer
 	if writer == nil && LogPackets.Load() == 1 {
-		logPacket(e.logPrefix, dir, protocol, pkt)
+		LogPacket(e.logPrefix, dir, protocol, pkt)
 	}
 	if writer != nil && LogPacketsToPCAP.Load() == 1 {
 		packet := pcapPacket{
@@ -161,12 +164,13 @@ func (e *endpoint) dumpPacket(dir direction, protocol tcpip.NetworkProtocolNumbe
 // forwards the request to the lower endpoint.
 func (e *endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) {
 	for _, pkt := range pkts.AsSlice() {
-		e.dumpPacket(directionSend, pkt.NetworkProtocolNumber, pkt)
+		e.dumpPacket(DirectionSend, pkt.NetworkProtocolNumber, pkt)
 	}
 	return e.Endpoint.WritePackets(pkts)
 }
 
-func logPacket(prefix string, dir direction, protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) {
+// LogPacket logs a packet to stdout.
+func LogPacket(prefix string, dir Direction, protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) {
 	// Figure out the network layer info.
 	var transProto uint8
 	src := tcpip.Address("unknown")
@@ -178,9 +182,9 @@ func logPacket(prefix string, dir direction, protocol tcpip.NetworkProtocolNumbe
 
 	var directionPrefix string
 	switch dir {
-	case directionSend:
+	case DirectionSend:
 		directionPrefix = "send"
-	case directionRecv:
+	case DirectionRecv:
 		directionPrefix = "recv"
 	default:
 		panic(fmt.Sprintf("unrecognized direction: %d", dir))

pkg/xdp/BUILD

@@ -0,0 +1,20 @@
+load("//tools:defs.bzl", "go_library")
+
+package(licenses = ["notice"])
+
+go_library(
+    name = "xdp",
+    srcs = [
+        "fillqueue.go",
+        "rxqueue.go",
+        "umem.go",
+        "xdp.go",
+        "xdp_unsafe.go",
+    ],
+    visibility = ["//:sandbox"],
+    deps = [
+        "//pkg/atomicbitops",
+        "//pkg/log",
+        "@org_golang_x_sys//unix:go_default_library",
+    ],
+)

pkg/xdp/fillqueue.go

@@ -0,0 +1,123 @@
+// Copyright 2022 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build amd64 || arm64
+// +build amd64 arm64
+
+package xdp
+
+import (
+	"fmt"
+
+	"gvisor.dev/gvisor/pkg/atomicbitops"
+)
+
+// The FillQueue is how a process tells the kernel which buffers are available
+// to be filled by incoming packets.
+type FillQueue struct {
+	// mem is the mmap'd area shared with the kernel. Many other fields of
+	// this struct point into mem.
+	mem []byte
+
+	// umem is the UMEM (i.e. shared buffer space) to which the queue's
+	// descriptors point.
+	umem *UMEM
+
+	// ring is the actual ring buffer. It is a list of frame addresses
+	// ready for incoming packets.
+	ring []uint64
+
+	// mask is used whenever indexing into ring. It prevents index out of
+	// bounds errors while allowing the producer and consumer pointers to
+	// repeatedly "overflow" and loop back around the ring.
+	mask uint32
+
+	// producer points to the shared atomic value that indicates the last
+	// produced descriptor. Only we update this value.
+	producer *atomicbitops.Uint32
+
+	// consumer points to the shared atomic value that indicates the last
+	// consumed descriptor. Only the kernel updates this value.
+	consumer *atomicbitops.Uint32
+
+	// flags points to the shared atomic value that holds flags for the
+	// queue.
+	flags *atomicbitops.Uint32
+
+	// Cached values are used to avoid relatively expensive atomic
+	// operations.
+	cachedProducer uint32
+	// cachedConsumer is actually len(ring) larger than the real consumer
+	// value. See Free() for details.
+	cachedConsumer uint32
+}
+
+// Reserve reserves descriptors in the fill queue. If toReserve descriptors
+// cannot be reserved, none are reserved.
+func (fq *FillQueue) Reserve(toReserve uint32) (nReserved, index uint32) {
+	if fq.free(toReserve) < toReserve {
+		// Unable to free the desired number of descriptors.
+		return 0, 0
+	}
+	idx := fq.cachedProducer
+	fq.cachedProducer += toReserve
+	return toReserve, idx
+}
+
+// free returns the number of free descriptors in the fill queue.
+func (fq *FillQueue) free(toReserve uint32) uint32 {
+	// cachedConsumer is always len(fq.ring) larger than the real consumer
+	// value. This lets us, in the common case, compute the number of free
+	// descriptors simply via fq.cachedConsumer - fq.cachedProducer.
+	if available := fq.cachedConsumer - fq.cachedProducer; available >= toReserve {
+		return available
+	}
+
+	// If we didn't already have enough descriptors available, check
+	// whether the kernel has returned some to us.
+	fq.cachedConsumer = fq.consumer.Load()
+	fq.cachedConsumer += uint32(len(fq.ring))
+	return fq.cachedConsumer - fq.cachedProducer
+}
+
+// Notify updates the prodcer such that it is visible to the kernel.
+func (fq *FillQueue) Notify() {
+	fq.producer.Store(fq.cachedProducer)
+}
+
+// Set sets the fill queue's descriptor at index to addr.
+func (fq *FillQueue) Set(index uint32, addr uint64) {
+	fq.ring[index&fq.mask] = addr
+}
+
+// FillAll fills the queue with as many buffers as possible from the UMEM, then
+// notifies the kernel.
+func (fq *FillQueue) FillAll() {
+	available := fq.free(fq.umem.nFreeFrames)
+	if available < 1 {
+		return
+	}
+	if available > fq.umem.nFreeFrames {
+		available = fq.umem.nFreeFrames
+	}
+	_, index := fq.Reserve(available)
+	for i := uint32(0); i < available; i++ {
+		addr, err := fq.umem.AllocFrame()
+		if err != nil {
+			panic(fmt.Sprintf("failed to alloc frame #%d: %v", index+i, err))
+		}
+		fq.Set(index+i, addr)
+	}
+	fq.Notify()
+}

pkg/xdp/rxqueue.go

@@ -0,0 +1,91 @@
+// Copyright 2022 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build amd64 || arm64
+// +build amd64 arm64
+
+package xdp
+
+import (
+	"golang.org/x/sys/unix"
+	"gvisor.dev/gvisor/pkg/atomicbitops"
+)
+
+// The RXQueue is how the kernel tells a process which buffers are full with
+// incoming packets.
+type RXQueue struct {
+	// mem is the mmap'd area shared with the kernel. Many other fields of
+	// this struct point into mem.
+	mem []byte
+
+	// ring is the list of XDP descriptors shared with the kernel.
+	// ring is the actual ring buffer. It is a list of XDP descriptors
+	// pointing to incoming packets.
+	ring []unix.XDPDesc
+
+	// mask is used whenever indexing into ring. It prevents index out of
+	// bounds errors while allowing the producer and consumer pointers to
+	// repeatedly "overflow" and loop back around the ring.
+	mask uint32
+
+	// producer points to the shared atomic value that indicates the last
+	// produced descriptor. Only the kernel updates this value.
+	producer *atomicbitops.Uint32
+
+	// consumer points to the shared atomic value that indicates the last
+	// consumed descriptor. Only we update this value.
+	consumer *atomicbitops.Uint32
+
+	// flags points to the shared atomic value that holds flags for the
+	// queue.
+	flags *atomicbitops.Uint32
+
+	// Cached values are used to avoid relatively expensive atomic
+	// operations.
+	cachedProducer uint32
+	cachedConsumer uint32
+}
+
+// Peek returns the number of packets available to read as well as the index at
+// which they start. Peek will only return a packet once, so callers must
+// process any received packets.
+func (rq *RXQueue) Peek() (nReceived, index uint32) {
+	entries := rq.free()
+	index = rq.cachedConsumer
+	rq.cachedConsumer += entries
+	return entries, index
+}
+
+func (rq *RXQueue) free() uint32 {
+	entries := rq.cachedProducer - rq.cachedConsumer
+	// If we're not aware of any RX'd packets, refresh the producer pointer
+	// to see whether the kernel enqueued anything.
+	if entries == 0 {
+		rq.cachedProducer = rq.producer.Load()
+		entries = rq.cachedProducer - rq.cachedConsumer
+	}
+	return entries
+}
+
+// Release notifies the kernel that we have consumed nDone packets.
+func (rq *RXQueue) Release(nDone uint32) {
+	// We don't have to use an atomic add becuase only we update this; the
+	// kernel just reads it.
+	rq.consumer.Store(rq.consumer.RacyLoad() + nDone)
+}
+
+// Get gets the descriptor at index.
+func (rq *RXQueue) Get(index uint32) unix.XDPDesc {
+	return rq.ring[index&rq.mask]
+}

pkg/xdp/umem.go

@@ -0,0 +1,61 @@
+// Copyright 2022 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build amd64 || arm64
+// +build amd64 arm64
+
+package xdp
+
+import (
+	"golang.org/x/sys/unix"
+)
+
+// UMEM is the shared memory area that the kernel and userspace put packets in.
+type UMEM struct {
+	// mem is the mmap'd area shared with the kernel.
+	mem []byte
+
+	// sockfd is the underlying AF_XDP socket.
+	sockfd uint32
+
+	// frameAddresses is a stack of available frame addresses.
+	frameAddresses []uint64
+
+	// nFreeFrames is the number of frames available and is used to index
+	// into frameAddresses.
+	nFreeFrames uint32
+}
+
+// SockFD returns the underlying AF_XDP socket FD.
+func (um *UMEM) SockFD() uint32 {
+	return um.sockfd
+}
+
+// FreeFrame returns the frame containing addr to the set of free frames.
+func (um *UMEM) FreeFrame(addr uint64) {
+	um.frameAddresses[um.nFreeFrames] = addr
+	um.nFreeFrames++
+}
+
+// AllocFrame returns the address of a frame that can be enqueued to the fill
+// or TX queue.
+func (um *UMEM) AllocFrame() (uint64, error) {
+	um.nFreeFrames--
+	return um.frameAddresses[um.nFreeFrames], nil
+}
+
+// Get gets the bytes of the packet pointed to by desc.
+func (um *UMEM) Get(desc unix.XDPDesc) []byte {
+	return um.mem[desc.Addr : desc.Addr+uint64(desc.Len)]
+}

pkg/xdp/xdp.go

@@ -0,0 +1,223 @@
+// Copyright 2022 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build amd64 || arm64
+// +build amd64 arm64
+
+// Package xdp provides tools for working with AF_XDP sockets.
+//
+// AF_XDP shares a memory area (UMEM) with the kernel to pass packets
+// back and forth. Communication is done via a number of queues.
+// Briefly, the queues work as follows:
+//
+//   - Receive: Userspace adds a descriptor to the fill queue. The
+//     descriptor points to an area of the UMEM that the kernel should fill
+//     with an incoming packet. The packet is filled by the kernel, which
+//     places a descriptor to the same UMEM area in the RX queue, signifying
+//     that userspace may read the packet.
+//   - Trasmit: Userspace adds a descriptor to TX queue. The kernel
+//     sends the packet (stored in UMEM) pointed to by the descriptor.
+//     Upon completion, the kernel places a desciptor in the completion
+//     queue to notify userspace that the packet is sent and the UMEM
+//     area can be reused.
+//
+// So in short: RX packets move from the fill to RX queue, and TX
+// packets move from the TX to completion queue.
+//
+// Note that the shared UMEM for RX and TX means that packet forwarding
+// can be done without copying; only the queues need to be updated to point to
+// the packet in UMEM.
+package xdp
+
+import (
+	"fmt"
+
+	"golang.org/x/sys/unix"
+	"gvisor.dev/gvisor/pkg/log"
+)
+
+// ReadOnlySocketOpts configure a read-only AF_XDP socket.
+type ReadOnlySocketOpts struct {
+	NFrames      uint32
+	FrameSize    uint32
+	NDescriptors uint32
+}
+
+// DefaultReadOnlyOpts provides recommended default options for initializing a
+// readonly AF_XDP socket. AF_XDP setup is extremely finnicky and can fail if
+// incorrect values are used.
+func DefaultReadOnlyOpts() ReadOnlySocketOpts {
+	return ReadOnlySocketOpts{
+		NFrames: 4096,
+		// Frames must be 2048 or 4096 bytes, although not all drivers support
+		// both.
+		FrameSize:    4096,
+		NDescriptors: 2048,
+	}
+}
+
+// ReadOnlySocket returns an initialized read-only AF_XDP socket bound to a
+// particular interface and queue.
+func ReadOnlySocket(ifaceIdx, queueID uint32, opts ReadOnlySocketOpts) (*UMEM, *FillQueue, *RXQueue, error) {
+	sockfd, err := unix.Socket(unix.AF_XDP, unix.SOCK_RAW, 0)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to create AF_XDP socket: %v", err)
+	}
+	return ReadOnlyFromSocket(sockfd, ifaceIdx, queueID, opts)
+}
+
+// ReadOnlyFromSocket takes an AF_XDP socket, initializes it, and binds it to a
+// particular interface and queue.
+func ReadOnlyFromSocket(sockfd int, ifaceIdx, queueID uint32, opts ReadOnlySocketOpts) (*UMEM, *FillQueue, *RXQueue, error) {
+	// Create the UMEM area. Use mmap instead of make([[]byte) to ensure
+	// that the UMEM is page-aligned. Aligning the UMEM keeps individual
+	// packets from spilling over between pages.
+	umemMemory, err := unix.Mmap(-1,
+		0,
+		int(opts.NFrames*opts.FrameSize),
+		unix.PROT_READ|unix.PROT_WRITE,
+		unix.MAP_PRIVATE|unix.MAP_ANONYMOUS)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to mmap umem: %v", err)
+	}
+	if sliceBackingPointer(umemMemory)%uintptr(unix.Getpagesize()) != 0 {
+		return nil, nil, nil, fmt.Errorf("UMEM is not page aligned (address 0x%x)", sliceBackingPointer(umemMemory))
+	}
+
+	umem := UMEM{
+		mem:            umemMemory,
+		sockfd:         uint32(sockfd),
+		frameAddresses: make([]uint64, opts.NFrames),
+		nFreeFrames:    opts.NFrames,
+	}
+
+	// Fill in each frame address.
+	for i := range umem.frameAddresses {
+		umem.frameAddresses[i] = uint64(i) * uint64(opts.FrameSize)
+	}
+
+	// Check whether we're likely to fail due to RLIMIT_MEMLOCK.
+	var rlimit unix.Rlimit
+	if err := unix.Getrlimit(unix.RLIMIT_MEMLOCK, &rlimit); err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to get rlimit for memlock: %v", err)
+	}
+	if rlimit.Cur < uint64(len(umem.mem)) {
+		log.Infof("UMEM size (%d) may exceed RLIMIT_MEMLOCK (%+v) and cause registration to fail", len(umem.mem), rlimit)
+	}
+
+	reg := unix.XDPUmemReg{
+		Addr: uint64(sliceBackingPointer(umemMemory)),
+		Len:  uint64(len(umemMemory)),
+		Size: opts.FrameSize,
+		// Not useful in the RX path.
+		Headroom: 0,
+		// TODO(b/240191988): Investigate use of SHARED flag.
+		Flags: 0,
+	}
+	if err := registerUMEM(sockfd, reg); err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to register UMEM: %v", err)
+	}
+
+	// Set the number of descriptors in the fill queue.
+	if err := unix.SetsockoptInt(sockfd, unix.SOL_XDP, unix.XDP_UMEM_FILL_RING, int(opts.NDescriptors)); err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to register fill ring: %v", err)
+	}
+
+	// Set the number of descriptors in the completion queue. Note: we
+	// don't actually use this (the completion queue is TX-specific), but
+	// bind() will fail if this is left unset.
+	if err := unix.SetsockoptInt(sockfd, unix.SOL_XDP, unix.XDP_UMEM_COMPLETION_RING, int(opts.NDescriptors)); err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to register fill ring: %v", err)
+	}
+
+	// Get offset information for the queues. Offsets indicate where, once
+	// we mmap space for each queue, values in the queue are. They give
+	// offsets for the shared pointers, a shared flags value, and the
+	// beginning of the ring of descriptors.
+	off, err := getOffsets(sockfd)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to get offsets: %v", err)
+	}
+
+	// Allocate space for the fill queue.
+	fillQueueMem, err := unix.Mmap(sockfd,
+		unix.XDP_UMEM_PGOFF_FILL_RING,
+		int(off.Fr.Desc+uint64(opts.NDescriptors)*sizeOfFillQueueDesc()),
+		unix.PROT_READ|unix.PROT_WRITE,
+		unix.MAP_SHARED|unix.MAP_POPULATE)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to mmap fill queue: %v", err)
+	}
+
+	// Setup the fillQueue with offsets into allocated memory.
+	fillQueue := FillQueue{
+		mem:            fillQueueMem,
+		mask:           opts.NDescriptors - 1,
+		umem:           &umem,
+		cachedConsumer: opts.NDescriptors,
+	}
+	fillQueue.init(off, opts)
+
+	// Allocate space for the (unused) completion queue.
+	_, err = unix.Mmap(sockfd,
+		unix.XDP_UMEM_PGOFF_COMPLETION_RING,
+		int(off.Cr.Desc+uint64(opts.NDescriptors)*sizeOfFillQueueDesc()),
+		unix.PROT_READ|unix.PROT_WRITE,
+		unix.MAP_SHARED|unix.MAP_POPULATE)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to mmap completion queue: %v", err)
+	}
+
+	// Set the number of descriptors in the RX queue.
+	if err := unix.SetsockoptInt(sockfd, unix.SOL_XDP, unix.XDP_RX_RING, int(opts.NDescriptors)); err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to register RX queue: %v", err)
+	}
+
+	// Allocate space for the RX queue.
+	rxQueueMem, err := unix.Mmap(sockfd,
+		unix.XDP_PGOFF_RX_RING,
+		int(off.Rx.Desc+uint64(opts.NDescriptors)*sizeOfRXQueueDesc()),
+		unix.PROT_READ|unix.PROT_WRITE,
+		unix.MAP_SHARED|unix.MAP_POPULATE)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to mmap fill queue: %v", err)
+	}
+
+	// Setup the rxQueue with offsets into allocated memory.
+	rxQueue := RXQueue{
+		mem:  rxQueueMem,
+		mask: opts.NDescriptors - 1,
+	}
+	rxQueue.init(off, opts)
+
+	addr := unix.SockaddrXDP{
+		// By not setting either XDP_COPY or XDP_ZEROCOPY, we instruct
+		// the kernel to use zerocopy if available and then fallback to
+		// copy mode.
+		// TODO(b/240191988): Look into whether to use XDP_USE_NEED_WAKEUP.
+		Flags:   0,
+		Ifindex: ifaceIdx,
+		// AF_XDP sockets are per device RX queue, although multiple
+		// sockets on multiple queues (or devices) can share a single
+		// UMEM.
+		QueueID: queueID,
+		// We're not using shared mode, so the value here is irrelevant.
+		SharedUmemFD: 0,
+	}
+	if err := unix.Bind(sockfd, &addr); err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to bind with addr %+v: %v", addr, err)
+	}
+
+	return &umem, &fillQueue, &rxQueue, nil
+}

pkg/xdp/xdp_unsafe.go

@@ -0,0 +1,78 @@
+// Copyright 2022 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package xdp
+
+import (
+	"fmt"
+	"reflect"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+	"gvisor.dev/gvisor/pkg/atomicbitops"
+)
+
+func registerUMEM(fd int, reg unix.XDPUmemReg) error {
+	if _, _, errno := unix.Syscall6(unix.SYS_SETSOCKOPT, uintptr(fd), unix.SOL_XDP, unix.XDP_UMEM_REG, uintptr(unsafe.Pointer(&reg)), unsafe.Sizeof(reg), 0); errno != 0 {
+		return fmt.Errorf("failed to setsockopt(XDP_UMEM_REG): errno %d", errno)
+	}
+	return nil
+}
+
+func getOffsets(fd int) (unix.XDPMmapOffsets, error) {
+	var off unix.XDPMmapOffsets
+	size := unsafe.Sizeof(off)
+	if _, _, errno := unix.Syscall6(unix.SYS_GETSOCKOPT, uintptr(fd), unix.SOL_XDP, unix.XDP_MMAP_OFFSETS, uintptr(unsafe.Pointer(&off)), uintptr(unsafe.Pointer(&size)), 0); errno != 0 {
+		return unix.XDPMmapOffsets{}, fmt.Errorf("failed to get offsets: %v", errno)
+	} else if unsafe.Sizeof(off) != size {
+		return unix.XDPMmapOffsets{}, fmt.Errorf("expected optlen of %d, but found %d", unsafe.Sizeof(off), size)
+	}
+	return off, nil
+}
+
+func sliceBackingPointer(slice []byte) uintptr {
+	return uintptr(unsafe.Pointer(&slice[0]))
+}
+
+func sizeOfFillQueueDesc() uint64 {
+	return uint64(unsafe.Sizeof(uint64(0)))
+}
+
+func sizeOfRXQueueDesc() uint64 {
+	return uint64(unsafe.Sizeof(unix.XDPDesc{}))
+}
+
+func (fq *FillQueue) init(off unix.XDPMmapOffsets, opts ReadOnlySocketOpts) {
+	fillQueueRingHdr := (*reflect.SliceHeader)(unsafe.Pointer(&fq.ring))
+	fillQueueRingHdr.Data = uintptr(unsafe.Pointer(&fq.mem[off.Fr.Desc]))
+	fillQueueRingHdr.Len = int(opts.NDescriptors)
+	fillQueueRingHdr.Cap = fillQueueRingHdr.Len
+	fq.producer = (*atomicbitops.Uint32)(unsafe.Pointer(&fq.mem[off.Fr.Producer]))
+	fq.consumer = (*atomicbitops.Uint32)(unsafe.Pointer(&fq.mem[off.Fr.Consumer]))
+	fq.flags = (*atomicbitops.Uint32)(unsafe.Pointer(&fq.mem[off.Fr.Flags]))
+}
+
+func (rq *RXQueue) init(off unix.XDPMmapOffsets, opts ReadOnlySocketOpts) {
+	rxQueueRingHdr := (*reflect.SliceHeader)(unsafe.Pointer(&rq.ring))
+	rxQueueRingHdr.Data = uintptr(unsafe.Pointer(&rq.mem[off.Rx.Desc]))
+	rxQueueRingHdr.Len = int(opts.NDescriptors)
+	rxQueueRingHdr.Cap = rxQueueRingHdr.Len
+	rq.producer = (*atomicbitops.Uint32)(unsafe.Pointer(&rq.mem[off.Rx.Producer]))
+	rq.consumer = (*atomicbitops.Uint32)(unsafe.Pointer(&rq.mem[off.Rx.Consumer]))
+	rq.flags = (*atomicbitops.Uint32)(unsafe.Pointer(&rq.mem[off.Rx.Flags]))
+	// These probably don't have to be atomic, but we're only loading once
+	// so better safe than sorry.
+	rq.cachedProducer = rq.producer.Load()
+	rq.cachedConsumer = rq.consumer.Load()
+}

tools/xdp/BUILD

@@ -10,9 +10,15 @@ go_binary(
     embedsrcs = [
         "//tools/xdp/bpf:drop_ebpf.o",  # keep
         "//tools/xdp/bpf:pass_ebpf.o",  # keep
+        "//tools/xdp/bpf:tcpdump_ebpf.o",  # keep
     ],
     visibility = ["//:sandbox"],
     deps = [
+        "//pkg/bufferv2",
+        "//pkg/tcpip/header",
+        "//pkg/tcpip/link/sniffer",
+        "//pkg/tcpip/stack",
+        "//pkg/xdp",
         "@com_github_cilium_ebpf//:go_default_library",
         "@com_github_cilium_ebpf//link:go_default_library",
         "@org_golang_x_sys//unix:go_default_library",

tools/xdp/README.md

@@ -0,0 +1,40 @@
+# XDP
+
+This directory contains tools for using XDP and, importantly, provides examples.
+
+The `xdp_loader` program can attach one of three programs to a network device.
+Those programs, specified via the `-program` flag, can be:
+
+-   `pass` - Allow all traffic, passing it on to the kernel network stack.
+-   `drop` - Drop all traffic before it hits the kernel network stack.
+-   `tcpdump` - Use an `AF_XDP` socket to print all network traffic. Unlike the
+    normal `tcpdump` tool, intercepted packets are not also passed to the kernel
+    network stack.
+
+# How do the examples work?
+
+## `XDP`
+
+The XDP pass and drop programs simply allow or drop all traffic on a given NIC.
+These examples give an idea of how to use the Cilium eBPF library and how to
+build eBPF programs within gVisor.
+
+## `AF_XDP`
+
+The code supporting `tcpdump` is a minimal example of using an `AF_XDP` socket
+to receive packets. There are very few other examples of `AF_XDP` floating
+around the internet. They all use the in-tree libbpf library
+unfortunately.[^libxdp]
+
+The XDP project has a useful [example][af_xdp_tutorial] that uses libbpf. One
+must also look at [libbpf itself][libbpf] to understand what's really going on.
+
+## TODO
+
+Kernel version < 5.4 has some weird offsets behavior. Just don't run on those
+machines.
+
+[af_xdp_tutorial]: https://github.com/xdp-project/xdp-tutorial/tree/master/advanced03-AF_XDP
+[libbpf]: https://github.com/torvalds/linux/tree/master/tools/testing/selftests/bpf/xsk.c
+[^libxdp]: XDP functionality has since moved to libxdp, but nobody seems to be
+    using it yet.

tools/xdp/bpf/BUILD

@@ -17,3 +17,11 @@ bpf_program(
     bpf_object = "drop_ebpf.o",
     visibility = ["//:sandbox"],
 )
+
+bpf_program(
+    name = "tcpdump_ebpf",
+    src = "tcpdump.ebpf.c",
+    hdrs = [],
+    bpf_object = "tcpdump_ebpf.o",
+    visibility = ["//:sandbox"],
+)

tools/xdp/bpf/tcpdump.ebpf.c

@@ -0,0 +1,49 @@
+// Copyright 2022 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <linux/bpf.h>
+
+#define section(secname) __attribute__((section(secname), used))
+
+char __license[] section("license") = "Apache-2.0";
+
+// Helper functions are defined positionally in <linux/bpf.h>, and their
+// signatures are scattered throughout the kernel. They can be found via the
+// defining macro BPF_CALL_[0-5].
+// TODO(b/240191988): Use vmlinux instead of this.
+static int (*bpf_redirect_map)(void *bpf_map, __u32 iface_index,
+                               __u64 flags) = (void *)51;
+
+struct bpf_map_def {
+  unsigned int type;
+  unsigned int key_size;
+  unsigned int value_size;
+  unsigned int max_entries;
+  unsigned int map_flags;
+};
+
+// A map of RX queue number to AF_XDP socket. We only ever use one key: 0.
+struct bpf_map_def section("maps") sock_map = {
+    .type = BPF_MAP_TYPE_XSKMAP,  // Note: "XSK" means AF_XDP socket.
+    .key_size = sizeof(int),
+    .value_size = sizeof(int),
+    .max_entries = 1,
+};
+
+section("xdp") int xdp_prog(struct xdp_md *ctx) {
+  // Lookup the socket for the current RX queue. Veth devices by default have
+  // only one RX queue. If one is found, redirect the packet to that socket.
+  // Otherwise pass it on to the kernel network stack.
+  return bpf_redirect_map(&sock_map, ctx->rx_queue_index, XDP_PASS);
+}

tools/xdp/main.go

@@ -12,6 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build amd64 || arm64
+// +build amd64 arm64
+
 // The xdp_loader tool is used to load compiled XDP object files into the XDP
 // hook of a net device. It is intended primarily for testing.
 package main
@@ -19,18 +22,27 @@ package main
 import (
 	"bytes"
 	_ "embed"
+	"errors"
 	"flag"
+	"fmt"
 	"log"
 	"net"
 
 	"github.com/cilium/ebpf"
 	"github.com/cilium/ebpf/link"
 	"golang.org/x/sys/unix"
+	"gvisor.dev/gvisor/pkg/bufferv2"
+	"gvisor.dev/gvisor/pkg/tcpip/header"
+	"gvisor.dev/gvisor/pkg/tcpip/link/sniffer"
+	"gvisor.dev/gvisor/pkg/tcpip/stack"
+	"gvisor.dev/gvisor/pkg/xdp"
 )
 
+// Flags.
 var (
-	device  = flag.String("device", "", "which device to attach to")
-	program = flag.String("program", "", "which program to install: one of [pass, drop]")
+	device      = flag.String("device", "", "which device to attach to")
+	deviceIndex = flag.Int("devidx", 0, "which device to attach to")
+	program     = flag.String("program", "", "which program to install: one of [pass, drop, tcpdump]")
 )
 
 // Builtin programs selectable by users.
@@ -40,61 +52,111 @@ var (
 
 	//go:embed bpf/drop_ebpf.o
 	drop []byte
+
+	//go:embed bpf/tcpdump_ebpf.o
+	tcpdump []byte
 )
 
 var programs = map[string][]byte{
-	"pass": pass,
-	"drop": drop,
+	"pass":    pass,
+	"drop":    drop,
+	"tcpdump": tcpdump,
 }
 
 func main() {
-	// Sanity check.
-	if len(pass) == 0 {
-		panic("the pass program failed to embed")
+	// log.Fatalf skips important defers, so put everythin in the run
+	// function where it can return errors instead.
+	if err := run(); err != nil {
+		log.Fatalf("%v", err)
 	}
-	if len(drop) == 0 {
-		panic("the drop program failed to embed")
+}
+
+func run() error {
+	// Sanity check.
+	for name, prog := range programs {
+		if len(prog) == 0 {
+			panic(fmt.Sprintf("the %s program failed to embed", name))
+		}
 	}
 
 	flag.Parse()
 
 	// Get a net device.
-	if *device == "" {
-		log.Fatalf("must specify -device")
-	}
-	iface, err := net.InterfaceByName(*device)
-	if err != nil {
-		log.Fatalf("unknown device %q: %v", *device, err)
+	var iface *net.Interface
+	var err error
+	switch {
+	case *device != "" && *deviceIndex != 0:
+		return fmt.Errorf("must specify exactly one of -device or -devidx")
+	case *device != "":
+		if iface, err = net.InterfaceByName(*device); err != nil {
+			return fmt.Errorf("unknown device %q: %v", *device, err)
+		}
+	case *deviceIndex != 0:
+		if iface, err = net.InterfaceByIndex(*deviceIndex); err != nil {
+			return fmt.Errorf("unknown device with index %d: %v", *deviceIndex, err)
+		}
+	default:
+		return fmt.Errorf("must specify -device or -devidx")
 	}
 
 	// Choose a program.
 	if *program == "" {
-		log.Fatalf("must specify -program")
+		return fmt.Errorf("must specify -program")
 	}
 	progData, ok := programs[*program]
 	if !ok {
-		log.Fatalf("unknown program %q", *program)
+		return fmt.Errorf("unknown program %q", *program)
 	}
 
 	// Load into the kernel. Note that this is usually done using bpf2go,
 	// but since we haven't set up that tool we do everything manually.
 	spec, err := ebpf.LoadCollectionSpecFromReader(bytes.NewReader(progData))
 	if err != nil {
-		log.Fatalf("failed to load spec: %v", err)
+		return fmt.Errorf("failed to load spec: %v", err)
 	}
 
 	// We need to pass a struct with a field of a specific type and tag.
-	var objects struct {
-		Program *ebpf.Program `ebpf:"xdp_prog"`
-	}
-	if err := spec.LoadAndAssign(&objects, nil); err != nil {
-		log.Fatalf("failed to load program: %v", err)
-	}
-	defer func() {
-		if err := objects.Program.Close(); err != nil {
-			log.Printf("failed to close program: %v", err)
+	var programObject *ebpf.Program
+	var sockmap *ebpf.Map
+	switch *program {
+	case "pass", "drop":
+		var objects struct {
+			Program *ebpf.Program `ebpf:"xdp_prog"`
 		}
-	}()
+		if err := spec.LoadAndAssign(&objects, nil); err != nil {
+			return fmt.Errorf("failed to load program: %v", err)
+		}
+		programObject = objects.Program
+		defer func() {
+			if err := objects.Program.Close(); err != nil {
+				log.Printf("failed to close program: %v", err)
+			}
+		}()
+	case "tcpdump":
+		var objects struct {
+			Program *ebpf.Program `ebpf:"xdp_prog"`
+			SockMap *ebpf.Map     `ebpf:"sock_map"`
+		}
+		if err := spec.LoadAndAssign(&objects, nil); err != nil {
+			return fmt.Errorf("failed to load program: %v", err)
+		}
+		programObject = objects.Program
+		sockmap = objects.SockMap
+		defer func() {
+			if err := objects.Program.Close(); err != nil {
+				log.Printf("failed to close program: %v", err)
+			}
+			if err := objects.SockMap.Close(); err != nil {
+				log.Printf("failed to close sock map: %v", err)
+			}
+		}()
+	default:
+		return fmt.Errorf("unknown program %q", *program)
+	}
+
+	// TODO(b/240191988): It would be nice to automatically detatch
+	// existing XDP programs, although this can be done with iproute2:
+	//   $ ip link set dev eth1 xdp off
 
 	// Attach the program to the XDP hook on the device. Fallback from best
 	// to worst mode.
@@ -109,7 +171,7 @@ func main() {
 	var attached link.Link
 	for _, mode := range modes {
 		attached, err = link.AttachXDP(link.XDPOptions{
-			Program:   objects.Program,
+			Program:   programObject,
 			Interface: iface.Index,
 			Flags:     mode.flag,
 		})
@@ -120,12 +182,88 @@ func main() {
 		log.Printf("failed to attach with mode %q: %v", mode.name, err)
 	}
 	if attached == nil {
-		log.Fatalf("failed to attach program")
+		return fmt.Errorf("failed to attach program")
 	}
 	defer attached.Close()
 
+	// tcpdump requires opening an AF_XDP socket and having a goroutine
+	// listen for packets.
+	if *program == "tcpdump" {
+		if err := startTcpdump(iface, sockmap); err != nil {
+			return fmt.Errorf("failed to create AF_XDP socket: %v", err)
+		}
+	}
+
 	log.Printf("Successfully attached! Press CTRL-C to quit and remove the program from the device.")
 	for {
 		unix.Pause()
 	}
 }
+
+func startTcpdump(iface *net.Interface, sockMap *ebpf.Map) error {
+	umem, fillQueue, rxQueue, err := xdp.ReadOnlySocket(
+		uint32(iface.Index), 0 /* queueID */, xdp.DefaultReadOnlyOpts())
+	if err != nil {
+		return fmt.Errorf("failed to create socket: %v", err)
+	}
+
+	// Insert our AF_XDP socket into the BPF map that dictates where
+	// packets are redirected to.
+	key := uint32(0)
+	val := umem.SockFD()
+	if err := sockMap.Update(&key, &val, 0 /* flags */); err != nil {
+		return fmt.Errorf("failed to insert socket into BPF map: %v", err)
+	}
+	log.Printf("updated key %d to value %d", key, val)
+
+	// Put as many UMEM buffers into the fill queue as possible.
+	fillQueue.FillAll()
+
+	go func() {
+		for {
+			pfds := []unix.PollFd{{Fd: int32(umem.SockFD()), Events: unix.POLLIN}}
+			_, err := unix.Poll(pfds, -1)
+			if err != nil {
+				if errors.Is(err, unix.EINTR) {
+					continue
+				}
+				panic(fmt.Sprintf("poll failed: %v", err))
+			}
+
+			// How many packets did we get?
+			nReceived, rxIndex := rxQueue.Peek()
+			if nReceived == 0 {
+				continue
+			}
+
+			// Keep the fill queue full.
+			fillQueue.FillAll()
+
+			// Read packets one-by-one and log them.
+			for i := uint32(0); i < nReceived; i++ {
+				// Wrap the packet in a PacketBuffer.
+				descriptor := rxQueue.Get(rxIndex + i)
+				data := umem.Get(descriptor)
+				pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{
+					Payload: bufferv2.MakeWithData(data[header.EthernetMinimumSize:]),
+				})
+
+				sniffer.LogPacket("",
+					sniffer.DirectionRecv, // XDP operates only on ingress.
+					header.Ethernet(data).Type(),
+					pkt)
+
+				// NOTE: the address is always 256 bytes offset
+				// from a page boundary. The kernel masks the
+				// address to the frame size, so this isn't a
+				// problem.
+				//
+				// Note that this limits MTU to 4096-256 bytes.
+				umem.FreeFrame(descriptor.Addr)
+			}
+			rxQueue.Release(nReceived)
+		}
+	}()
+
+	return nil
+}