From 9bf0b63a37b33186c060859699b7a6793afe7d6c Mon Sep 17 00:00:00 2001 From: Kevin Krakauer Date: Thu, 25 Aug 2022 13:44:49 -0700 Subject: [PATCH] xdp_loader: add tcpdump-ish program Uses an AF_XDP socket to log packets entering a device. Can be used for testing, but mostly useful as a minimal example of receiving packets via AF_XDP. Note that this actually prevents the packets from reaching their intended destination, as redirection does not also send the packet through Linux's network stack. PiperOrigin-RevId: 470073040 --- pkg/tcpip/link/sniffer/sniffer.go | 24 ++-- pkg/xdp/BUILD | 20 +++ pkg/xdp/fillqueue.go | 123 ++++++++++++++++ pkg/xdp/rxqueue.go | 91 ++++++++++++ pkg/xdp/umem.go | 61 ++++++++ pkg/xdp/xdp.go | 223 ++++++++++++++++++++++++++++++ pkg/xdp/xdp_unsafe.go | 78 +++++++++++ tools/xdp/BUILD | 6 + tools/xdp/README.md | 40 ++++++ tools/xdp/bpf/BUILD | 8 ++ tools/xdp/bpf/tcpdump.ebpf.c | 49 +++++++ tools/xdp/main.go | 198 ++++++++++++++++++++++---- 12 files changed, 881 insertions(+), 40 deletions(-) create mode 100644 pkg/xdp/BUILD create mode 100644 pkg/xdp/fillqueue.go create mode 100644 pkg/xdp/rxqueue.go create mode 100644 pkg/xdp/umem.go create mode 100644 pkg/xdp/xdp.go create mode 100644 pkg/xdp/xdp_unsafe.go create mode 100644 tools/xdp/README.md create mode 100644 tools/xdp/bpf/tcpdump.ebpf.c diff --git a/pkg/tcpip/link/sniffer/sniffer.go b/pkg/tcpip/link/sniffer/sniffer.go index 643a279a5..0a4f48d68 100644 --- a/pkg/tcpip/link/sniffer/sniffer.go +++ b/pkg/tcpip/link/sniffer/sniffer.go @@ -55,11 +55,14 @@ var _ stack.GSOEndpoint = (*endpoint)(nil) var _ stack.LinkEndpoint = (*endpoint)(nil) var _ stack.NetworkDispatcher = (*endpoint)(nil) -type direction int +// A Direction indicates whether the packing is being sent or received. +type Direction int const ( - directionSend = iota - directionRecv + // DirectionSend indicates a sent packet. + DirectionSend = iota + // DirectionRecv indicates a received packet. + DirectionRecv ) // New creates a new sniffer link-layer endpoint. It wraps around another @@ -131,14 +134,14 @@ func NewWithWriter(lower stack.LinkEndpoint, writer io.Writer, snapLen uint32) ( // called by the link-layer endpoint being wrapped when a packet arrives, and // logs the packet before forwarding to the actual dispatcher. func (e *endpoint) DeliverNetworkPacket(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) { - e.dumpPacket(directionRecv, protocol, pkt) + e.dumpPacket(DirectionRecv, protocol, pkt) e.Endpoint.DeliverNetworkPacket(protocol, pkt) } -func (e *endpoint) dumpPacket(dir direction, protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) { +func (e *endpoint) dumpPacket(dir Direction, protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) { writer := e.writer if writer == nil && LogPackets.Load() == 1 { - logPacket(e.logPrefix, dir, protocol, pkt) + LogPacket(e.logPrefix, dir, protocol, pkt) } if writer != nil && LogPacketsToPCAP.Load() == 1 { packet := pcapPacket{ @@ -161,12 +164,13 @@ func (e *endpoint) dumpPacket(dir direction, protocol tcpip.NetworkProtocolNumbe // forwards the request to the lower endpoint. func (e *endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) { for _, pkt := range pkts.AsSlice() { - e.dumpPacket(directionSend, pkt.NetworkProtocolNumber, pkt) + e.dumpPacket(DirectionSend, pkt.NetworkProtocolNumber, pkt) } return e.Endpoint.WritePackets(pkts) } -func logPacket(prefix string, dir direction, protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) { +// LogPacket logs a packet to stdout. +func LogPacket(prefix string, dir Direction, protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) { // Figure out the network layer info. var transProto uint8 src := tcpip.Address("unknown") @@ -178,9 +182,9 @@ func logPacket(prefix string, dir direction, protocol tcpip.NetworkProtocolNumbe var directionPrefix string switch dir { - case directionSend: + case DirectionSend: directionPrefix = "send" - case directionRecv: + case DirectionRecv: directionPrefix = "recv" default: panic(fmt.Sprintf("unrecognized direction: %d", dir)) diff --git a/pkg/xdp/BUILD b/pkg/xdp/BUILD new file mode 100644 index 000000000..b01c55cbe --- /dev/null +++ b/pkg/xdp/BUILD @@ -0,0 +1,20 @@ +load("//tools:defs.bzl", "go_library") + +package(licenses = ["notice"]) + +go_library( + name = "xdp", + srcs = [ + "fillqueue.go", + "rxqueue.go", + "umem.go", + "xdp.go", + "xdp_unsafe.go", + ], + visibility = ["//:sandbox"], + deps = [ + "//pkg/atomicbitops", + "//pkg/log", + "@org_golang_x_sys//unix:go_default_library", + ], +) diff --git a/pkg/xdp/fillqueue.go b/pkg/xdp/fillqueue.go new file mode 100644 index 000000000..0e196edab --- /dev/null +++ b/pkg/xdp/fillqueue.go @@ -0,0 +1,123 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 || arm64 +// +build amd64 arm64 + +package xdp + +import ( + "fmt" + + "gvisor.dev/gvisor/pkg/atomicbitops" +) + +// The FillQueue is how a process tells the kernel which buffers are available +// to be filled by incoming packets. +type FillQueue struct { + // mem is the mmap'd area shared with the kernel. Many other fields of + // this struct point into mem. + mem []byte + + // umem is the UMEM (i.e. shared buffer space) to which the queue's + // descriptors point. + umem *UMEM + + // ring is the actual ring buffer. It is a list of frame addresses + // ready for incoming packets. + ring []uint64 + + // mask is used whenever indexing into ring. It prevents index out of + // bounds errors while allowing the producer and consumer pointers to + // repeatedly "overflow" and loop back around the ring. + mask uint32 + + // producer points to the shared atomic value that indicates the last + // produced descriptor. Only we update this value. + producer *atomicbitops.Uint32 + + // consumer points to the shared atomic value that indicates the last + // consumed descriptor. Only the kernel updates this value. + consumer *atomicbitops.Uint32 + + // flags points to the shared atomic value that holds flags for the + // queue. + flags *atomicbitops.Uint32 + + // Cached values are used to avoid relatively expensive atomic + // operations. + cachedProducer uint32 + // cachedConsumer is actually len(ring) larger than the real consumer + // value. See Free() for details. + cachedConsumer uint32 +} + +// Reserve reserves descriptors in the fill queue. If toReserve descriptors +// cannot be reserved, none are reserved. +func (fq *FillQueue) Reserve(toReserve uint32) (nReserved, index uint32) { + if fq.free(toReserve) < toReserve { + // Unable to free the desired number of descriptors. + return 0, 0 + } + idx := fq.cachedProducer + fq.cachedProducer += toReserve + return toReserve, idx +} + +// free returns the number of free descriptors in the fill queue. +func (fq *FillQueue) free(toReserve uint32) uint32 { + // cachedConsumer is always len(fq.ring) larger than the real consumer + // value. This lets us, in the common case, compute the number of free + // descriptors simply via fq.cachedConsumer - fq.cachedProducer. + if available := fq.cachedConsumer - fq.cachedProducer; available >= toReserve { + return available + } + + // If we didn't already have enough descriptors available, check + // whether the kernel has returned some to us. + fq.cachedConsumer = fq.consumer.Load() + fq.cachedConsumer += uint32(len(fq.ring)) + return fq.cachedConsumer - fq.cachedProducer +} + +// Notify updates the prodcer such that it is visible to the kernel. +func (fq *FillQueue) Notify() { + fq.producer.Store(fq.cachedProducer) +} + +// Set sets the fill queue's descriptor at index to addr. +func (fq *FillQueue) Set(index uint32, addr uint64) { + fq.ring[index&fq.mask] = addr +} + +// FillAll fills the queue with as many buffers as possible from the UMEM, then +// notifies the kernel. +func (fq *FillQueue) FillAll() { + available := fq.free(fq.umem.nFreeFrames) + if available < 1 { + return + } + if available > fq.umem.nFreeFrames { + available = fq.umem.nFreeFrames + } + _, index := fq.Reserve(available) + for i := uint32(0); i < available; i++ { + addr, err := fq.umem.AllocFrame() + if err != nil { + panic(fmt.Sprintf("failed to alloc frame #%d: %v", index+i, err)) + } + fq.Set(index+i, addr) + } + fq.Notify() +} diff --git a/pkg/xdp/rxqueue.go b/pkg/xdp/rxqueue.go new file mode 100644 index 000000000..e36731d73 --- /dev/null +++ b/pkg/xdp/rxqueue.go @@ -0,0 +1,91 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 || arm64 +// +build amd64 arm64 + +package xdp + +import ( + "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/atomicbitops" +) + +// The RXQueue is how the kernel tells a process which buffers are full with +// incoming packets. +type RXQueue struct { + // mem is the mmap'd area shared with the kernel. Many other fields of + // this struct point into mem. + mem []byte + + // ring is the list of XDP descriptors shared with the kernel. + // ring is the actual ring buffer. It is a list of XDP descriptors + // pointing to incoming packets. + ring []unix.XDPDesc + + // mask is used whenever indexing into ring. It prevents index out of + // bounds errors while allowing the producer and consumer pointers to + // repeatedly "overflow" and loop back around the ring. + mask uint32 + + // producer points to the shared atomic value that indicates the last + // produced descriptor. Only the kernel updates this value. + producer *atomicbitops.Uint32 + + // consumer points to the shared atomic value that indicates the last + // consumed descriptor. Only we update this value. + consumer *atomicbitops.Uint32 + + // flags points to the shared atomic value that holds flags for the + // queue. + flags *atomicbitops.Uint32 + + // Cached values are used to avoid relatively expensive atomic + // operations. + cachedProducer uint32 + cachedConsumer uint32 +} + +// Peek returns the number of packets available to read as well as the index at +// which they start. Peek will only return a packet once, so callers must +// process any received packets. +func (rq *RXQueue) Peek() (nReceived, index uint32) { + entries := rq.free() + index = rq.cachedConsumer + rq.cachedConsumer += entries + return entries, index +} + +func (rq *RXQueue) free() uint32 { + entries := rq.cachedProducer - rq.cachedConsumer + // If we're not aware of any RX'd packets, refresh the producer pointer + // to see whether the kernel enqueued anything. + if entries == 0 { + rq.cachedProducer = rq.producer.Load() + entries = rq.cachedProducer - rq.cachedConsumer + } + return entries +} + +// Release notifies the kernel that we have consumed nDone packets. +func (rq *RXQueue) Release(nDone uint32) { + // We don't have to use an atomic add becuase only we update this; the + // kernel just reads it. + rq.consumer.Store(rq.consumer.RacyLoad() + nDone) +} + +// Get gets the descriptor at index. +func (rq *RXQueue) Get(index uint32) unix.XDPDesc { + return rq.ring[index&rq.mask] +} diff --git a/pkg/xdp/umem.go b/pkg/xdp/umem.go new file mode 100644 index 000000000..98fac4336 --- /dev/null +++ b/pkg/xdp/umem.go @@ -0,0 +1,61 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 || arm64 +// +build amd64 arm64 + +package xdp + +import ( + "golang.org/x/sys/unix" +) + +// UMEM is the shared memory area that the kernel and userspace put packets in. +type UMEM struct { + // mem is the mmap'd area shared with the kernel. + mem []byte + + // sockfd is the underlying AF_XDP socket. + sockfd uint32 + + // frameAddresses is a stack of available frame addresses. + frameAddresses []uint64 + + // nFreeFrames is the number of frames available and is used to index + // into frameAddresses. + nFreeFrames uint32 +} + +// SockFD returns the underlying AF_XDP socket FD. +func (um *UMEM) SockFD() uint32 { + return um.sockfd +} + +// FreeFrame returns the frame containing addr to the set of free frames. +func (um *UMEM) FreeFrame(addr uint64) { + um.frameAddresses[um.nFreeFrames] = addr + um.nFreeFrames++ +} + +// AllocFrame returns the address of a frame that can be enqueued to the fill +// or TX queue. +func (um *UMEM) AllocFrame() (uint64, error) { + um.nFreeFrames-- + return um.frameAddresses[um.nFreeFrames], nil +} + +// Get gets the bytes of the packet pointed to by desc. +func (um *UMEM) Get(desc unix.XDPDesc) []byte { + return um.mem[desc.Addr : desc.Addr+uint64(desc.Len)] +} diff --git a/pkg/xdp/xdp.go b/pkg/xdp/xdp.go new file mode 100644 index 000000000..3afc2eb6f --- /dev/null +++ b/pkg/xdp/xdp.go @@ -0,0 +1,223 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build amd64 || arm64 +// +build amd64 arm64 + +// Package xdp provides tools for working with AF_XDP sockets. +// +// AF_XDP shares a memory area (UMEM) with the kernel to pass packets +// back and forth. Communication is done via a number of queues. +// Briefly, the queues work as follows: +// +// - Receive: Userspace adds a descriptor to the fill queue. The +// descriptor points to an area of the UMEM that the kernel should fill +// with an incoming packet. The packet is filled by the kernel, which +// places a descriptor to the same UMEM area in the RX queue, signifying +// that userspace may read the packet. +// - Trasmit: Userspace adds a descriptor to TX queue. The kernel +// sends the packet (stored in UMEM) pointed to by the descriptor. +// Upon completion, the kernel places a desciptor in the completion +// queue to notify userspace that the packet is sent and the UMEM +// area can be reused. +// +// So in short: RX packets move from the fill to RX queue, and TX +// packets move from the TX to completion queue. +// +// Note that the shared UMEM for RX and TX means that packet forwarding +// can be done without copying; only the queues need to be updated to point to +// the packet in UMEM. +package xdp + +import ( + "fmt" + + "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/log" +) + +// ReadOnlySocketOpts configure a read-only AF_XDP socket. +type ReadOnlySocketOpts struct { + NFrames uint32 + FrameSize uint32 + NDescriptors uint32 +} + +// DefaultReadOnlyOpts provides recommended default options for initializing a +// readonly AF_XDP socket. AF_XDP setup is extremely finnicky and can fail if +// incorrect values are used. +func DefaultReadOnlyOpts() ReadOnlySocketOpts { + return ReadOnlySocketOpts{ + NFrames: 4096, + // Frames must be 2048 or 4096 bytes, although not all drivers support + // both. + FrameSize: 4096, + NDescriptors: 2048, + } +} + +// ReadOnlySocket returns an initialized read-only AF_XDP socket bound to a +// particular interface and queue. +func ReadOnlySocket(ifaceIdx, queueID uint32, opts ReadOnlySocketOpts) (*UMEM, *FillQueue, *RXQueue, error) { + sockfd, err := unix.Socket(unix.AF_XDP, unix.SOCK_RAW, 0) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to create AF_XDP socket: %v", err) + } + return ReadOnlyFromSocket(sockfd, ifaceIdx, queueID, opts) +} + +// ReadOnlyFromSocket takes an AF_XDP socket, initializes it, and binds it to a +// particular interface and queue. +func ReadOnlyFromSocket(sockfd int, ifaceIdx, queueID uint32, opts ReadOnlySocketOpts) (*UMEM, *FillQueue, *RXQueue, error) { + // Create the UMEM area. Use mmap instead of make([[]byte) to ensure + // that the UMEM is page-aligned. Aligning the UMEM keeps individual + // packets from spilling over between pages. + umemMemory, err := unix.Mmap(-1, + 0, + int(opts.NFrames*opts.FrameSize), + unix.PROT_READ|unix.PROT_WRITE, + unix.MAP_PRIVATE|unix.MAP_ANONYMOUS) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to mmap umem: %v", err) + } + if sliceBackingPointer(umemMemory)%uintptr(unix.Getpagesize()) != 0 { + return nil, nil, nil, fmt.Errorf("UMEM is not page aligned (address 0x%x)", sliceBackingPointer(umemMemory)) + } + + umem := UMEM{ + mem: umemMemory, + sockfd: uint32(sockfd), + frameAddresses: make([]uint64, opts.NFrames), + nFreeFrames: opts.NFrames, + } + + // Fill in each frame address. + for i := range umem.frameAddresses { + umem.frameAddresses[i] = uint64(i) * uint64(opts.FrameSize) + } + + // Check whether we're likely to fail due to RLIMIT_MEMLOCK. + var rlimit unix.Rlimit + if err := unix.Getrlimit(unix.RLIMIT_MEMLOCK, &rlimit); err != nil { + return nil, nil, nil, fmt.Errorf("failed to get rlimit for memlock: %v", err) + } + if rlimit.Cur < uint64(len(umem.mem)) { + log.Infof("UMEM size (%d) may exceed RLIMIT_MEMLOCK (%+v) and cause registration to fail", len(umem.mem), rlimit) + } + + reg := unix.XDPUmemReg{ + Addr: uint64(sliceBackingPointer(umemMemory)), + Len: uint64(len(umemMemory)), + Size: opts.FrameSize, + // Not useful in the RX path. + Headroom: 0, + // TODO(b/240191988): Investigate use of SHARED flag. + Flags: 0, + } + if err := registerUMEM(sockfd, reg); err != nil { + return nil, nil, nil, fmt.Errorf("failed to register UMEM: %v", err) + } + + // Set the number of descriptors in the fill queue. + if err := unix.SetsockoptInt(sockfd, unix.SOL_XDP, unix.XDP_UMEM_FILL_RING, int(opts.NDescriptors)); err != nil { + return nil, nil, nil, fmt.Errorf("failed to register fill ring: %v", err) + } + + // Set the number of descriptors in the completion queue. Note: we + // don't actually use this (the completion queue is TX-specific), but + // bind() will fail if this is left unset. + if err := unix.SetsockoptInt(sockfd, unix.SOL_XDP, unix.XDP_UMEM_COMPLETION_RING, int(opts.NDescriptors)); err != nil { + return nil, nil, nil, fmt.Errorf("failed to register fill ring: %v", err) + } + + // Get offset information for the queues. Offsets indicate where, once + // we mmap space for each queue, values in the queue are. They give + // offsets for the shared pointers, a shared flags value, and the + // beginning of the ring of descriptors. + off, err := getOffsets(sockfd) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get offsets: %v", err) + } + + // Allocate space for the fill queue. + fillQueueMem, err := unix.Mmap(sockfd, + unix.XDP_UMEM_PGOFF_FILL_RING, + int(off.Fr.Desc+uint64(opts.NDescriptors)*sizeOfFillQueueDesc()), + unix.PROT_READ|unix.PROT_WRITE, + unix.MAP_SHARED|unix.MAP_POPULATE) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to mmap fill queue: %v", err) + } + + // Setup the fillQueue with offsets into allocated memory. + fillQueue := FillQueue{ + mem: fillQueueMem, + mask: opts.NDescriptors - 1, + umem: &umem, + cachedConsumer: opts.NDescriptors, + } + fillQueue.init(off, opts) + + // Allocate space for the (unused) completion queue. + _, err = unix.Mmap(sockfd, + unix.XDP_UMEM_PGOFF_COMPLETION_RING, + int(off.Cr.Desc+uint64(opts.NDescriptors)*sizeOfFillQueueDesc()), + unix.PROT_READ|unix.PROT_WRITE, + unix.MAP_SHARED|unix.MAP_POPULATE) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to mmap completion queue: %v", err) + } + + // Set the number of descriptors in the RX queue. + if err := unix.SetsockoptInt(sockfd, unix.SOL_XDP, unix.XDP_RX_RING, int(opts.NDescriptors)); err != nil { + return nil, nil, nil, fmt.Errorf("failed to register RX queue: %v", err) + } + + // Allocate space for the RX queue. + rxQueueMem, err := unix.Mmap(sockfd, + unix.XDP_PGOFF_RX_RING, + int(off.Rx.Desc+uint64(opts.NDescriptors)*sizeOfRXQueueDesc()), + unix.PROT_READ|unix.PROT_WRITE, + unix.MAP_SHARED|unix.MAP_POPULATE) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to mmap fill queue: %v", err) + } + + // Setup the rxQueue with offsets into allocated memory. + rxQueue := RXQueue{ + mem: rxQueueMem, + mask: opts.NDescriptors - 1, + } + rxQueue.init(off, opts) + + addr := unix.SockaddrXDP{ + // By not setting either XDP_COPY or XDP_ZEROCOPY, we instruct + // the kernel to use zerocopy if available and then fallback to + // copy mode. + // TODO(b/240191988): Look into whether to use XDP_USE_NEED_WAKEUP. + Flags: 0, + Ifindex: ifaceIdx, + // AF_XDP sockets are per device RX queue, although multiple + // sockets on multiple queues (or devices) can share a single + // UMEM. + QueueID: queueID, + // We're not using shared mode, so the value here is irrelevant. + SharedUmemFD: 0, + } + if err := unix.Bind(sockfd, &addr); err != nil { + return nil, nil, nil, fmt.Errorf("failed to bind with addr %+v: %v", addr, err) + } + + return &umem, &fillQueue, &rxQueue, nil +} diff --git a/pkg/xdp/xdp_unsafe.go b/pkg/xdp/xdp_unsafe.go new file mode 100644 index 000000000..5c4ede51e --- /dev/null +++ b/pkg/xdp/xdp_unsafe.go @@ -0,0 +1,78 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package xdp + +import ( + "fmt" + "reflect" + "unsafe" + + "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/atomicbitops" +) + +func registerUMEM(fd int, reg unix.XDPUmemReg) error { + if _, _, errno := unix.Syscall6(unix.SYS_SETSOCKOPT, uintptr(fd), unix.SOL_XDP, unix.XDP_UMEM_REG, uintptr(unsafe.Pointer(®)), unsafe.Sizeof(reg), 0); errno != 0 { + return fmt.Errorf("failed to setsockopt(XDP_UMEM_REG): errno %d", errno) + } + return nil +} + +func getOffsets(fd int) (unix.XDPMmapOffsets, error) { + var off unix.XDPMmapOffsets + size := unsafe.Sizeof(off) + if _, _, errno := unix.Syscall6(unix.SYS_GETSOCKOPT, uintptr(fd), unix.SOL_XDP, unix.XDP_MMAP_OFFSETS, uintptr(unsafe.Pointer(&off)), uintptr(unsafe.Pointer(&size)), 0); errno != 0 { + return unix.XDPMmapOffsets{}, fmt.Errorf("failed to get offsets: %v", errno) + } else if unsafe.Sizeof(off) != size { + return unix.XDPMmapOffsets{}, fmt.Errorf("expected optlen of %d, but found %d", unsafe.Sizeof(off), size) + } + return off, nil +} + +func sliceBackingPointer(slice []byte) uintptr { + return uintptr(unsafe.Pointer(&slice[0])) +} + +func sizeOfFillQueueDesc() uint64 { + return uint64(unsafe.Sizeof(uint64(0))) +} + +func sizeOfRXQueueDesc() uint64 { + return uint64(unsafe.Sizeof(unix.XDPDesc{})) +} + +func (fq *FillQueue) init(off unix.XDPMmapOffsets, opts ReadOnlySocketOpts) { + fillQueueRingHdr := (*reflect.SliceHeader)(unsafe.Pointer(&fq.ring)) + fillQueueRingHdr.Data = uintptr(unsafe.Pointer(&fq.mem[off.Fr.Desc])) + fillQueueRingHdr.Len = int(opts.NDescriptors) + fillQueueRingHdr.Cap = fillQueueRingHdr.Len + fq.producer = (*atomicbitops.Uint32)(unsafe.Pointer(&fq.mem[off.Fr.Producer])) + fq.consumer = (*atomicbitops.Uint32)(unsafe.Pointer(&fq.mem[off.Fr.Consumer])) + fq.flags = (*atomicbitops.Uint32)(unsafe.Pointer(&fq.mem[off.Fr.Flags])) +} + +func (rq *RXQueue) init(off unix.XDPMmapOffsets, opts ReadOnlySocketOpts) { + rxQueueRingHdr := (*reflect.SliceHeader)(unsafe.Pointer(&rq.ring)) + rxQueueRingHdr.Data = uintptr(unsafe.Pointer(&rq.mem[off.Rx.Desc])) + rxQueueRingHdr.Len = int(opts.NDescriptors) + rxQueueRingHdr.Cap = rxQueueRingHdr.Len + rq.producer = (*atomicbitops.Uint32)(unsafe.Pointer(&rq.mem[off.Rx.Producer])) + rq.consumer = (*atomicbitops.Uint32)(unsafe.Pointer(&rq.mem[off.Rx.Consumer])) + rq.flags = (*atomicbitops.Uint32)(unsafe.Pointer(&rq.mem[off.Rx.Flags])) + // These probably don't have to be atomic, but we're only loading once + // so better safe than sorry. + rq.cachedProducer = rq.producer.Load() + rq.cachedConsumer = rq.consumer.Load() +} diff --git a/tools/xdp/BUILD b/tools/xdp/BUILD index 30a1fc97b..2a6062210 100644 --- a/tools/xdp/BUILD +++ b/tools/xdp/BUILD @@ -10,9 +10,15 @@ go_binary( embedsrcs = [ "//tools/xdp/bpf:drop_ebpf.o", # keep "//tools/xdp/bpf:pass_ebpf.o", # keep + "//tools/xdp/bpf:tcpdump_ebpf.o", # keep ], visibility = ["//:sandbox"], deps = [ + "//pkg/bufferv2", + "//pkg/tcpip/header", + "//pkg/tcpip/link/sniffer", + "//pkg/tcpip/stack", + "//pkg/xdp", "@com_github_cilium_ebpf//:go_default_library", "@com_github_cilium_ebpf//link:go_default_library", "@org_golang_x_sys//unix:go_default_library", diff --git a/tools/xdp/README.md b/tools/xdp/README.md new file mode 100644 index 000000000..85dabd1e8 --- /dev/null +++ b/tools/xdp/README.md @@ -0,0 +1,40 @@ +# XDP + +This directory contains tools for using XDP and, importantly, provides examples. + +The `xdp_loader` program can attach one of three programs to a network device. +Those programs, specified via the `-program` flag, can be: + +- `pass` - Allow all traffic, passing it on to the kernel network stack. +- `drop` - Drop all traffic before it hits the kernel network stack. +- `tcpdump` - Use an `AF_XDP` socket to print all network traffic. Unlike the + normal `tcpdump` tool, intercepted packets are not also passed to the kernel + network stack. + +# How do the examples work? + +## `XDP` + +The XDP pass and drop programs simply allow or drop all traffic on a given NIC. +These examples give an idea of how to use the Cilium eBPF library and how to +build eBPF programs within gVisor. + +## `AF_XDP` + +The code supporting `tcpdump` is a minimal example of using an `AF_XDP` socket +to receive packets. There are very few other examples of `AF_XDP` floating +around the internet. They all use the in-tree libbpf library +unfortunately.[^libxdp] + +The XDP project has a useful [example][af_xdp_tutorial] that uses libbpf. One +must also look at [libbpf itself][libbpf] to understand what's really going on. + +## TODO + +Kernel version < 5.4 has some weird offsets behavior. Just don't run on those +machines. + +[af_xdp_tutorial]: https://github.com/xdp-project/xdp-tutorial/tree/master/advanced03-AF_XDP +[libbpf]: https://github.com/torvalds/linux/tree/master/tools/testing/selftests/bpf/xsk.c +[^libxdp]: XDP functionality has since moved to libxdp, but nobody seems to be + using it yet. diff --git a/tools/xdp/bpf/BUILD b/tools/xdp/bpf/BUILD index 70f38c6d8..b2b2f9c70 100644 --- a/tools/xdp/bpf/BUILD +++ b/tools/xdp/bpf/BUILD @@ -17,3 +17,11 @@ bpf_program( bpf_object = "drop_ebpf.o", visibility = ["//:sandbox"], ) + +bpf_program( + name = "tcpdump_ebpf", + src = "tcpdump.ebpf.c", + hdrs = [], + bpf_object = "tcpdump_ebpf.o", + visibility = ["//:sandbox"], +) diff --git a/tools/xdp/bpf/tcpdump.ebpf.c b/tools/xdp/bpf/tcpdump.ebpf.c new file mode 100644 index 000000000..06ac23112 --- /dev/null +++ b/tools/xdp/bpf/tcpdump.ebpf.c @@ -0,0 +1,49 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#define section(secname) __attribute__((section(secname), used)) + +char __license[] section("license") = "Apache-2.0"; + +// Helper functions are defined positionally in , and their +// signatures are scattered throughout the kernel. They can be found via the +// defining macro BPF_CALL_[0-5]. +// TODO(b/240191988): Use vmlinux instead of this. +static int (*bpf_redirect_map)(void *bpf_map, __u32 iface_index, + __u64 flags) = (void *)51; + +struct bpf_map_def { + unsigned int type; + unsigned int key_size; + unsigned int value_size; + unsigned int max_entries; + unsigned int map_flags; +}; + +// A map of RX queue number to AF_XDP socket. We only ever use one key: 0. +struct bpf_map_def section("maps") sock_map = { + .type = BPF_MAP_TYPE_XSKMAP, // Note: "XSK" means AF_XDP socket. + .key_size = sizeof(int), + .value_size = sizeof(int), + .max_entries = 1, +}; + +section("xdp") int xdp_prog(struct xdp_md *ctx) { + // Lookup the socket for the current RX queue. Veth devices by default have + // only one RX queue. If one is found, redirect the packet to that socket. + // Otherwise pass it on to the kernel network stack. + return bpf_redirect_map(&sock_map, ctx->rx_queue_index, XDP_PASS); +} diff --git a/tools/xdp/main.go b/tools/xdp/main.go index 0cc2acc8d..ea68dd028 100644 --- a/tools/xdp/main.go +++ b/tools/xdp/main.go @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build amd64 || arm64 +// +build amd64 arm64 + // The xdp_loader tool is used to load compiled XDP object files into the XDP // hook of a net device. It is intended primarily for testing. package main @@ -19,18 +22,27 @@ package main import ( "bytes" _ "embed" + "errors" "flag" + "fmt" "log" "net" "github.com/cilium/ebpf" "github.com/cilium/ebpf/link" "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/bufferv2" + "gvisor.dev/gvisor/pkg/tcpip/header" + "gvisor.dev/gvisor/pkg/tcpip/link/sniffer" + "gvisor.dev/gvisor/pkg/tcpip/stack" + "gvisor.dev/gvisor/pkg/xdp" ) +// Flags. var ( - device = flag.String("device", "", "which device to attach to") - program = flag.String("program", "", "which program to install: one of [pass, drop]") + device = flag.String("device", "", "which device to attach to") + deviceIndex = flag.Int("devidx", 0, "which device to attach to") + program = flag.String("program", "", "which program to install: one of [pass, drop, tcpdump]") ) // Builtin programs selectable by users. @@ -40,61 +52,111 @@ var ( //go:embed bpf/drop_ebpf.o drop []byte + + //go:embed bpf/tcpdump_ebpf.o + tcpdump []byte ) var programs = map[string][]byte{ - "pass": pass, - "drop": drop, + "pass": pass, + "drop": drop, + "tcpdump": tcpdump, } func main() { - // Sanity check. - if len(pass) == 0 { - panic("the pass program failed to embed") + // log.Fatalf skips important defers, so put everythin in the run + // function where it can return errors instead. + if err := run(); err != nil { + log.Fatalf("%v", err) } - if len(drop) == 0 { - panic("the drop program failed to embed") +} + +func run() error { + // Sanity check. + for name, prog := range programs { + if len(prog) == 0 { + panic(fmt.Sprintf("the %s program failed to embed", name)) + } } flag.Parse() // Get a net device. - if *device == "" { - log.Fatalf("must specify -device") - } - iface, err := net.InterfaceByName(*device) - if err != nil { - log.Fatalf("unknown device %q: %v", *device, err) + var iface *net.Interface + var err error + switch { + case *device != "" && *deviceIndex != 0: + return fmt.Errorf("must specify exactly one of -device or -devidx") + case *device != "": + if iface, err = net.InterfaceByName(*device); err != nil { + return fmt.Errorf("unknown device %q: %v", *device, err) + } + case *deviceIndex != 0: + if iface, err = net.InterfaceByIndex(*deviceIndex); err != nil { + return fmt.Errorf("unknown device with index %d: %v", *deviceIndex, err) + } + default: + return fmt.Errorf("must specify -device or -devidx") } // Choose a program. if *program == "" { - log.Fatalf("must specify -program") + return fmt.Errorf("must specify -program") } progData, ok := programs[*program] if !ok { - log.Fatalf("unknown program %q", *program) + return fmt.Errorf("unknown program %q", *program) } // Load into the kernel. Note that this is usually done using bpf2go, // but since we haven't set up that tool we do everything manually. spec, err := ebpf.LoadCollectionSpecFromReader(bytes.NewReader(progData)) if err != nil { - log.Fatalf("failed to load spec: %v", err) + return fmt.Errorf("failed to load spec: %v", err) } // We need to pass a struct with a field of a specific type and tag. - var objects struct { - Program *ebpf.Program `ebpf:"xdp_prog"` - } - if err := spec.LoadAndAssign(&objects, nil); err != nil { - log.Fatalf("failed to load program: %v", err) - } - defer func() { - if err := objects.Program.Close(); err != nil { - log.Printf("failed to close program: %v", err) + var programObject *ebpf.Program + var sockmap *ebpf.Map + switch *program { + case "pass", "drop": + var objects struct { + Program *ebpf.Program `ebpf:"xdp_prog"` } - }() + if err := spec.LoadAndAssign(&objects, nil); err != nil { + return fmt.Errorf("failed to load program: %v", err) + } + programObject = objects.Program + defer func() { + if err := objects.Program.Close(); err != nil { + log.Printf("failed to close program: %v", err) + } + }() + case "tcpdump": + var objects struct { + Program *ebpf.Program `ebpf:"xdp_prog"` + SockMap *ebpf.Map `ebpf:"sock_map"` + } + if err := spec.LoadAndAssign(&objects, nil); err != nil { + return fmt.Errorf("failed to load program: %v", err) + } + programObject = objects.Program + sockmap = objects.SockMap + defer func() { + if err := objects.Program.Close(); err != nil { + log.Printf("failed to close program: %v", err) + } + if err := objects.SockMap.Close(); err != nil { + log.Printf("failed to close sock map: %v", err) + } + }() + default: + return fmt.Errorf("unknown program %q", *program) + } + + // TODO(b/240191988): It would be nice to automatically detatch + // existing XDP programs, although this can be done with iproute2: + // $ ip link set dev eth1 xdp off // Attach the program to the XDP hook on the device. Fallback from best // to worst mode. @@ -109,7 +171,7 @@ func main() { var attached link.Link for _, mode := range modes { attached, err = link.AttachXDP(link.XDPOptions{ - Program: objects.Program, + Program: programObject, Interface: iface.Index, Flags: mode.flag, }) @@ -120,12 +182,88 @@ func main() { log.Printf("failed to attach with mode %q: %v", mode.name, err) } if attached == nil { - log.Fatalf("failed to attach program") + return fmt.Errorf("failed to attach program") } defer attached.Close() + // tcpdump requires opening an AF_XDP socket and having a goroutine + // listen for packets. + if *program == "tcpdump" { + if err := startTcpdump(iface, sockmap); err != nil { + return fmt.Errorf("failed to create AF_XDP socket: %v", err) + } + } + log.Printf("Successfully attached! Press CTRL-C to quit and remove the program from the device.") for { unix.Pause() } } + +func startTcpdump(iface *net.Interface, sockMap *ebpf.Map) error { + umem, fillQueue, rxQueue, err := xdp.ReadOnlySocket( + uint32(iface.Index), 0 /* queueID */, xdp.DefaultReadOnlyOpts()) + if err != nil { + return fmt.Errorf("failed to create socket: %v", err) + } + + // Insert our AF_XDP socket into the BPF map that dictates where + // packets are redirected to. + key := uint32(0) + val := umem.SockFD() + if err := sockMap.Update(&key, &val, 0 /* flags */); err != nil { + return fmt.Errorf("failed to insert socket into BPF map: %v", err) + } + log.Printf("updated key %d to value %d", key, val) + + // Put as many UMEM buffers into the fill queue as possible. + fillQueue.FillAll() + + go func() { + for { + pfds := []unix.PollFd{{Fd: int32(umem.SockFD()), Events: unix.POLLIN}} + _, err := unix.Poll(pfds, -1) + if err != nil { + if errors.Is(err, unix.EINTR) { + continue + } + panic(fmt.Sprintf("poll failed: %v", err)) + } + + // How many packets did we get? + nReceived, rxIndex := rxQueue.Peek() + if nReceived == 0 { + continue + } + + // Keep the fill queue full. + fillQueue.FillAll() + + // Read packets one-by-one and log them. + for i := uint32(0); i < nReceived; i++ { + // Wrap the packet in a PacketBuffer. + descriptor := rxQueue.Get(rxIndex + i) + data := umem.Get(descriptor) + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + Payload: bufferv2.MakeWithData(data[header.EthernetMinimumSize:]), + }) + + sniffer.LogPacket("", + sniffer.DirectionRecv, // XDP operates only on ingress. + header.Ethernet(data).Type(), + pkt) + + // NOTE: the address is always 256 bytes offset + // from a page boundary. The kernel masks the + // address to the frame size, so this isn't a + // problem. + // + // Note that this limits MTU to 4096-256 bytes. + umem.FreeFrame(descriptor.Addr) + } + rxQueue.Release(nReceived) + } + }() + + return nil +}