diff --git a/pkg/tcpip/link/sniffer/pcap.go b/pkg/tcpip/link/sniffer/pcap.go index 8cab65691..491957ac8 100644 --- a/pkg/tcpip/link/sniffer/pcap.go +++ b/pkg/tcpip/link/sniffer/pcap.go @@ -55,18 +55,20 @@ type pcapPacket struct { } func (p *pcapPacket) MarshalBinary() ([]byte, error) { - packetSize := p.packet.Size() + pkt := trimmedClone(p.packet) + defer pkt.DecRef() + packetSize := pkt.Size() captureLen := p.maxCaptureLen if packetSize < captureLen { captureLen = packetSize } b := make([]byte, 16+captureLen) - binary.BigEndian.PutUint32(b[0:4], uint32(p.timestamp.Unix())) - binary.BigEndian.PutUint32(b[4:8], uint32(p.timestamp.Nanosecond()/1000)) - binary.BigEndian.PutUint32(b[8:12], uint32(captureLen)) - binary.BigEndian.PutUint32(b[12:16], uint32(packetSize)) + binary.LittleEndian.PutUint32(b[0:4], uint32(p.timestamp.Unix())) + binary.LittleEndian.PutUint32(b[4:8], uint32(p.timestamp.Nanosecond()/1000)) + binary.LittleEndian.PutUint32(b[8:12], uint32(captureLen)) + binary.LittleEndian.PutUint32(b[12:16], uint32(packetSize)) w := tcpip.SliceWriter(b[16:]) - for _, v := range p.packet.AsSlices() { + for _, v := range pkt.AsSlices() { if captureLen == 0 { break } diff --git a/pkg/tcpip/link/sniffer/sniffer.go b/pkg/tcpip/link/sniffer/sniffer.go index 0a4f48d68..566ef5032 100644 --- a/pkg/tcpip/link/sniffer/sniffer.go +++ b/pkg/tcpip/link/sniffer/sniffer.go @@ -95,7 +95,7 @@ func writePCAPHeader(w io.Writer, maxLen uint32) error { if err != nil { return err } - return binary.Write(w, binary.BigEndian, pcapHeader{ + return binary.Write(w, binary.LittleEndian, pcapHeader{ // From https://wiki.wireshark.org/Development/LibpcapFileFormat MagicNumber: 0xa1b2c3d4, @@ -190,17 +190,7 @@ func LogPacket(prefix string, dir Direction, protocol tcpip.NetworkProtocolNumbe panic(fmt.Sprintf("unrecognized direction: %d", dir)) } - // Clone the packet buffer to not modify the original. - // - // We don't clone the original packet buffer so that the new packet buffer - // does not have any of its headers set. - // - // We trim the link headers from the cloned buffer as the sniffer doesn't - // handle link headers. - buf := pkt.ToBuffer() - buf.TrimFront(int64(len(pkt.VirtioNetHeader().Slice()))) - buf.TrimFront(int64(len(pkt.LinkHeader().Slice()))) - pkt = stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: buf}) + pkt = trimmedClone(pkt) defer pkt.DecRef() switch protocol { case header.IPv4ProtocolNumber: @@ -387,3 +377,17 @@ func LogPacket(prefix string, dir Direction, protocol tcpip.NetworkProtocolNumbe log.Infof("%s%s %s %s:%d -> %s:%d len:%d id:%04x %s", prefix, directionPrefix, transName, src, srcPort, dst, dstPort, size, id, details) } + +// trimmedClone clones the packet buffer to not modify the original. It trims +// anything before the network header. +func trimmedClone(pkt *stack.PacketBuffer) *stack.PacketBuffer { + // We don't clone the original packet buffer so that the new packet buffer + // does not have any of its headers set. + // + // We trim the link headers from the cloned buffer as the sniffer doesn't + // handle link headers. + buf := pkt.ToBuffer() + buf.TrimFront(int64(len(pkt.VirtioNetHeader().Slice()))) + buf.TrimFront(int64(len(pkt.LinkHeader().Slice()))) + return stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: buf}) +} diff --git a/runsc/boot/network.go b/runsc/boot/network.go index ffcc41680..18cb022cb 100644 --- a/runsc/boot/network.go +++ b/runsc/boot/network.go @@ -17,6 +17,7 @@ package boot import ( "fmt" "net" + "os" "runtime" "strings" @@ -126,6 +127,9 @@ type CreateLinksAndRoutesArgs struct { Defaultv6Gateway DefaultRoute AFXDP bool + + // PCAP indicates that FilePayload also contains a PCAP log file. + PCAP bool } // IPWithPrefix is an address with its subnet prefix length. @@ -168,8 +172,11 @@ func (n *Network) CreateLinksAndRoutes(args *CreateLinksAndRoutesArgs, _ *struct if args.AFXDP { wantFDs += 4 } + if args.PCAP { + wantFDs++ + } if got := len(args.FilePayload.Files); got != wantFDs { - return fmt.Errorf("args.FilePayload.Files has %d FDs but we need %d entries based on FDBasedLinks. AFXDP is %t", got, wantFDs, args.AFXDP) + return fmt.Errorf("args.FilePayload.Files has %d FDs but we need %d entries based on FDBasedLinks. AFXDP is %t, PCAP is %t", got, wantFDs, args.AFXDP, args.PCAP) } var nicID tcpip.NICID @@ -280,7 +287,21 @@ func (n *Network) CreateLinksAndRoutes(args *CreateLinksAndRoutesArgs, _ *struct } // Wrap linkEP in a sniffer to enable packet logging. - sniffEP := sniffer.New(packetsocket.New(linkEP)) + var sniffEP stack.LinkEndpoint + if args.PCAP { + newFD, err := unix.Dup(int(args.FilePayload.Files[fdOffset].Fd())) + if err != nil { + return fmt.Errorf("failed to dup pcap FD: %v", err) + } + const packetTruncateSize = 4096 + sniffEP, err = sniffer.NewWithWriter(packetsocket.New(linkEP), os.NewFile(uintptr(newFD), "pcap-file"), packetTruncateSize) + if err != nil { + return fmt.Errorf("failed to create PCAP logger: %v", err) + } + fdOffset++ + } else { + sniffEP = sniffer.New(packetsocket.New(linkEP)) + } var qDisc stack.QueueingDiscipline switch link.QDisc { diff --git a/runsc/config/config.go b/runsc/config/config.go index 3a6163138..5e11dcb91 100644 --- a/runsc/config/config.go +++ b/runsc/config/config.go @@ -106,6 +106,9 @@ type Config struct { // LogPackets indicates that all network packets should be logged. LogPackets bool `flag:"log-packets"` + // PCAP is a file to which network packets should be logged in PCAP format. + PCAP string `flag:"pcap-log"` + // Platform is the platform to run on. Platform string `flag:"platform"` diff --git a/runsc/config/flags.go b/runsc/config/flags.go index dee52a3ed..5fb611fe7 100644 --- a/runsc/config/flags.go +++ b/runsc/config/flags.go @@ -44,6 +44,7 @@ func RegisterFlags(flagSet *flag.FlagSet) { flagSet.String("panic-log", "", "file path where panic reports and other Go's runtime messages are written.") flagSet.String("coverage-report", "", "file path where Go coverage reports are written. Reports will only be generated if runsc is built with --collect_code_coverage and --instrumentation_filter Bazel flags.") flagSet.Bool("log-packets", false, "enable network packet logging.") + flagSet.String("pcap-log", "", "location of PCAP log file.") flagSet.String("debug-log-format", "text", "log format: text (default), json, or json-k8s.") flagSet.Bool("alsologtostderr", false, "send log messages to stderr.") flagSet.Bool("allow-flag-override", false, "allow OCI annotations (dev.gvisor.flag.) to override flags for debugging.") diff --git a/runsc/sandbox/network.go b/runsc/sandbox/network.go index 1081e9f5b..1168a3a63 100644 --- a/runsc/sandbox/network.go +++ b/runsc/sandbox/network.go @@ -294,6 +294,16 @@ func createInterfacesAndRoutesFromNS(conn *urpc.Client, nsPath string, conf *con args.FDBasedLinks = append(args.FDBasedLinks, link) } + // Pass PCAP log file if present. + if conf.PCAP != "" { + args.PCAP = true + pcap, err := os.OpenFile(conf.PCAP, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0664) + if err != nil { + return fmt.Errorf("failed to open PCAP file %s: %v", conf.PCAP, err) + } + args.FilePayload.Files = append(args.FilePayload.Files, pcap) + } + log.Debugf("Setting up network, config: %+v", args) if err := conn.Call(boot.NetworkCreateLinksAndRoutes, &args, nil); err != nil { return fmt.Errorf("creating links and routes: %w", err)