runsc: add PCAP logging flag

Also: PCAP logging was broken, fixed that.

Fixes #5968.

PiperOrigin-RevId: 472812978

pkg/tcpip/link/sniffer/pcap.go

@@ -55,18 +55,20 @@ type pcapPacket struct {
 }
 
 func (p *pcapPacket) MarshalBinary() ([]byte, error) {
-	packetSize := p.packet.Size()
+	pkt := trimmedClone(p.packet)
+	defer pkt.DecRef()
+	packetSize := pkt.Size()
 	captureLen := p.maxCaptureLen
 	if packetSize < captureLen {
 		captureLen = packetSize
 	}
 	b := make([]byte, 16+captureLen)
-	binary.BigEndian.PutUint32(b[0:4], uint32(p.timestamp.Unix()))
-	binary.BigEndian.PutUint32(b[4:8], uint32(p.timestamp.Nanosecond()/1000))
-	binary.BigEndian.PutUint32(b[8:12], uint32(captureLen))
-	binary.BigEndian.PutUint32(b[12:16], uint32(packetSize))
+	binary.LittleEndian.PutUint32(b[0:4], uint32(p.timestamp.Unix()))
+	binary.LittleEndian.PutUint32(b[4:8], uint32(p.timestamp.Nanosecond()/1000))
+	binary.LittleEndian.PutUint32(b[8:12], uint32(captureLen))
+	binary.LittleEndian.PutUint32(b[12:16], uint32(packetSize))
 	w := tcpip.SliceWriter(b[16:])
-	for _, v := range p.packet.AsSlices() {
+	for _, v := range pkt.AsSlices() {
 		if captureLen == 0 {
 			break
 		}

pkg/tcpip/link/sniffer/sniffer.go

@@ -95,7 +95,7 @@ func writePCAPHeader(w io.Writer, maxLen uint32) error {
 	if err != nil {
 		return err
 	}
-	return binary.Write(w, binary.BigEndian, pcapHeader{
+	return binary.Write(w, binary.LittleEndian, pcapHeader{
 		// From https://wiki.wireshark.org/Development/LibpcapFileFormat
 		MagicNumber: 0xa1b2c3d4,
 
@@ -190,17 +190,7 @@ func LogPacket(prefix string, dir Direction, protocol tcpip.NetworkProtocolNumbe
 		panic(fmt.Sprintf("unrecognized direction: %d", dir))
 	}
 
-	// Clone the packet buffer to not modify the original.
-	//
-	// We don't clone the original packet buffer so that the new packet buffer
-	// does not have any of its headers set.
-	//
-	// We trim the link headers from the cloned buffer as the sniffer doesn't
-	// handle link headers.
-	buf := pkt.ToBuffer()
-	buf.TrimFront(int64(len(pkt.VirtioNetHeader().Slice())))
-	buf.TrimFront(int64(len(pkt.LinkHeader().Slice())))
-	pkt = stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: buf})
+	pkt = trimmedClone(pkt)
 	defer pkt.DecRef()
 	switch protocol {
 	case header.IPv4ProtocolNumber:
@@ -387,3 +377,17 @@ func LogPacket(prefix string, dir Direction, protocol tcpip.NetworkProtocolNumbe
 
 	log.Infof("%s%s %s %s:%d -> %s:%d len:%d id:%04x %s", prefix, directionPrefix, transName, src, srcPort, dst, dstPort, size, id, details)
 }
+
+// trimmedClone clones the packet buffer to not modify the original. It trims
+// anything before the network header.
+func trimmedClone(pkt *stack.PacketBuffer) *stack.PacketBuffer {
+	// We don't clone the original packet buffer so that the new packet buffer
+	// does not have any of its headers set.
+	//
+	// We trim the link headers from the cloned buffer as the sniffer doesn't
+	// handle link headers.
+	buf := pkt.ToBuffer()
+	buf.TrimFront(int64(len(pkt.VirtioNetHeader().Slice())))
+	buf.TrimFront(int64(len(pkt.LinkHeader().Slice())))
+	return stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: buf})
+}

runsc/boot/network.go

@@ -17,6 +17,7 @@ package boot
 import (
 	"fmt"
 	"net"
+	"os"
 	"runtime"
 	"strings"
 
@@ -126,6 +127,9 @@ type CreateLinksAndRoutesArgs struct {
 	Defaultv6Gateway DefaultRoute
 
 	AFXDP bool
+
+	// PCAP indicates that FilePayload also contains a PCAP log file.
+	PCAP bool
 }
 
 // IPWithPrefix is an address with its subnet prefix length.
@@ -168,8 +172,11 @@ func (n *Network) CreateLinksAndRoutes(args *CreateLinksAndRoutesArgs, _ *struct
 	if args.AFXDP {
 		wantFDs += 4
 	}
+	if args.PCAP {
+		wantFDs++
+	}
 	if got := len(args.FilePayload.Files); got != wantFDs {
-		return fmt.Errorf("args.FilePayload.Files has %d FDs but we need %d entries based on FDBasedLinks. AFXDP is %t", got, wantFDs, args.AFXDP)
+		return fmt.Errorf("args.FilePayload.Files has %d FDs but we need %d entries based on FDBasedLinks. AFXDP is %t, PCAP is %t", got, wantFDs, args.AFXDP, args.PCAP)
 	}
 
 	var nicID tcpip.NICID
@@ -280,7 +287,21 @@ func (n *Network) CreateLinksAndRoutes(args *CreateLinksAndRoutesArgs, _ *struct
 		}
 
 		// Wrap linkEP in a sniffer to enable packet logging.
-		sniffEP := sniffer.New(packetsocket.New(linkEP))
+		var sniffEP stack.LinkEndpoint
+		if args.PCAP {
+			newFD, err := unix.Dup(int(args.FilePayload.Files[fdOffset].Fd()))
+			if err != nil {
+				return fmt.Errorf("failed to dup pcap FD: %v", err)
+			}
+			const packetTruncateSize = 4096
+			sniffEP, err = sniffer.NewWithWriter(packetsocket.New(linkEP), os.NewFile(uintptr(newFD), "pcap-file"), packetTruncateSize)
+			if err != nil {
+				return fmt.Errorf("failed to create PCAP logger: %v", err)
+			}
+			fdOffset++
+		} else {
+			sniffEP = sniffer.New(packetsocket.New(linkEP))
+		}
 
 		var qDisc stack.QueueingDiscipline
 		switch link.QDisc {

runsc/config/config.go

@@ -106,6 +106,9 @@ type Config struct {
 	// LogPackets indicates that all network packets should be logged.
 	LogPackets bool `flag:"log-packets"`
 
+	// PCAP is a file to which network packets should be logged in PCAP format.
+	PCAP string `flag:"pcap-log"`
+
 	// Platform is the platform to run on.
 	Platform string `flag:"platform"`
 

runsc/config/flags.go

@@ -44,6 +44,7 @@ func RegisterFlags(flagSet *flag.FlagSet) {
 	flagSet.String("panic-log", "", "file path where panic reports and other Go's runtime messages are written.")
 	flagSet.String("coverage-report", "", "file path where Go coverage reports are written. Reports will only be generated if runsc is built with --collect_code_coverage and --instrumentation_filter Bazel flags.")
 	flagSet.Bool("log-packets", false, "enable network packet logging.")
+	flagSet.String("pcap-log", "", "location of PCAP log file.")
 	flagSet.String("debug-log-format", "text", "log format: text (default), json, or json-k8s.")
 	flagSet.Bool("alsologtostderr", false, "send log messages to stderr.")
 	flagSet.Bool("allow-flag-override", false, "allow OCI annotations (dev.gvisor.flag.<name>) to override flags for debugging.")

runsc/sandbox/network.go

@@ -294,6 +294,16 @@ func createInterfacesAndRoutesFromNS(conn *urpc.Client, nsPath string, conf *con
 		args.FDBasedLinks = append(args.FDBasedLinks, link)
 	}
 
+	// Pass PCAP log file if present.
+	if conf.PCAP != "" {
+		args.PCAP = true
+		pcap, err := os.OpenFile(conf.PCAP, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0664)
+		if err != nil {
+			return fmt.Errorf("failed to open PCAP file %s: %v", conf.PCAP, err)
+		}
+		args.FilePayload.Files = append(args.FilePayload.Files, pcap)
+	}
+
 	log.Debugf("Setting up network, config: %+v", args)
 	if err := conn.Call(boot.NetworkCreateLinksAndRoutes, &args, nil); err != nil {
 		return fmt.Errorf("creating links and routes: %w", err)