(Re)introduce packetsocket link endpoint

The packetsocket link endpoint is used to enable a packet endpoint to
receive packets right before they are sent to the driver for outgoing
packets or right after they are recieved from the driver for incoming
packets.

Before this change, only packets that are sent/received by the
`stack.nic` were delivered to packet endpoints. However, the packet
endpoint should also receive packets that don't reach `stack.nic`.
Such an example can be when the interface is bridged and an incoming
packet is sent out through a sibling bridge port instead of being
delivered to a `stack.nic`.

The packetsocket link endpoint must wrap a link endpoint that
populates the link headers for ingress packets. It should ideally
be placed as low in the link endpoint heirarchy as possible so that
packets are probed as close to device drivers as possible.

https://github.com/google/gvisor/commit/2d9b33c0fd7d812a7296cd71f14c03925f815f28
removed the original packetsocket link endpoint which only allowed
low-level capturing of outbound packets but not inbound packets.

The CL mentioned above is cl/395761629.

PiperOrigin-RevId: 424947620

pkg/tcpip/link/ethernet/ethernet_test.go

@@ -39,6 +39,10 @@ func (t *testNetworkDispatcher) DeliverNetworkPacket(tcpip.NetworkProtocolNumber
 	t.networkPackets++
 }
 
+func (*testNetworkDispatcher) DeliverLinkPacket(tcpip.NetworkProtocolNumber, *stack.PacketBuffer, bool) {
+	panic("not implemented")
+}
+
 func TestDeliverNetworkPacket(t *testing.T) {
 	const (
 		linkAddr       = tcpip.LinkAddress("\x02\x02\x03\x04\x05\x06")

pkg/tcpip/link/fdbased/endpoint_test.go

@@ -137,6 +137,10 @@ func (c *context) DeliverNetworkPacket(protocol tcpip.NetworkProtocolNumber, pkt
 	c.ch <- packetInfo{protocol, pkt}
 }
 
+func (c *context) DeliverLinkPacket(tcpip.NetworkProtocolNumber, *stack.PacketBuffer, bool) {
+	c.t.Fatal("DeliverLinkPacket not implemented")
+}
+
 func TestNoEthernetProperties(t *testing.T) {
 	c := newContext(t, &Options{MTU: mtu})
 	defer c.cleanup()
@@ -566,6 +570,10 @@ func (d *fakeNetworkDispatcher) DeliverNetworkPacket(_ tcpip.NetworkProtocolNumb
 	d.pkts = append(d.pkts, pkt)
 }
 
+func (*fakeNetworkDispatcher) DeliverLinkPacket(tcpip.NetworkProtocolNumber, *stack.PacketBuffer, bool) {
+	panic("not implemented")
+}
+
 func TestDispatchPacketFormat(t *testing.T) {
 	for _, test := range []struct {
 		name          string

pkg/tcpip/link/nested/nested.go

@@ -60,6 +60,16 @@ func (e *Endpoint) DeliverNetworkPacket(protocol tcpip.NetworkProtocolNumber, pk
 	}
 }
 
+// DeliverLinkPacket implements stack.NetworkDispatcher.
+func (e *Endpoint) DeliverLinkPacket(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer, incoming bool) {
+	e.mu.RLock()
+	d := e.dispatcher
+	e.mu.RUnlock()
+	if d != nil {
+		d.DeliverLinkPacket(protocol, pkt, incoming)
+	}
+}
+
 // Attach implements stack.LinkEndpoint.
 func (e *Endpoint) Attach(dispatcher stack.NetworkDispatcher) {
 	e.mu.Lock()

pkg/tcpip/link/nested/nested_test.go

@@ -58,6 +58,10 @@ func (d *counterDispatcher) DeliverNetworkPacket(tcpip.NetworkProtocolNumber, *s
 	d.count++
 }
 
+func (*counterDispatcher) DeliverLinkPacket(tcpip.NetworkProtocolNumber, *stack.PacketBuffer, bool) {
+	panic("not implemented")
+}
+
 func TestNestedLinkEndpoint(t *testing.T) {
 	var (
 		childEP  childEndpoint

pkg/tcpip/link/packetsocket/BUILD

@@ -0,0 +1,28 @@
+load("//tools:defs.bzl", "go_library", "go_test")
+
+package(licenses = ["notice"])
+
+go_library(
+    name = "packetsocket",
+    srcs = ["packetsocket.go"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/tcpip",
+        "//pkg/tcpip/link/nested",
+        "//pkg/tcpip/stack",
+    ],
+)
+
+go_test(
+    name = "packetsocket_x_test",
+    size = "small",
+    srcs = ["packetsocket_test.go"],
+    deps = [
+        ":packetsocket",
+        "//pkg/refs",
+        "//pkg/refsvfs2",
+        "//pkg/tcpip",
+        "//pkg/tcpip/header",
+        "//pkg/tcpip/stack",
+    ],
+)

pkg/tcpip/link/packetsocket/packetsocket.go

@@ -0,0 +1,56 @@
+// Copyright 2022 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package packetsocket provides a link endpoint that enables delivery of
+// incoming and outgoing packets to any interested packet sockets.
+package packetsocket
+
+import (
+	"gvisor.dev/gvisor/pkg/tcpip"
+	"gvisor.dev/gvisor/pkg/tcpip/link/nested"
+	"gvisor.dev/gvisor/pkg/tcpip/stack"
+)
+
+var _ stack.NetworkDispatcher = (*endpoint)(nil)
+var _ stack.LinkEndpoint = (*endpoint)(nil)
+
+type endpoint struct {
+	nested.Endpoint
+}
+
+// New creates a new packetsocket link endpoint wrapping a lower link endpoint.
+//
+// On ingress, the lower link endpoint must only deliver packets that have
+// a link-layer header set if one is required for the link.
+func New(lower stack.LinkEndpoint) stack.LinkEndpoint {
+	e := &endpoint{}
+	e.Endpoint.Init(lower, e)
+	return e
+}
+
+// DeliverNetworkPacket implements stack.NetworkDispatcher.
+func (e *endpoint) DeliverNetworkPacket(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) {
+	e.Endpoint.DeliverLinkPacket(protocol, pkt, true /* incoming */)
+
+	e.Endpoint.DeliverNetworkPacket(protocol, pkt)
+}
+
+// WritePackets implements stack.LinkEndpoint.
+func (e *endpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) {
+	for pkt := pkts.Front(); pkt != nil; pkt = pkt.Next() {
+		e.Endpoint.DeliverLinkPacket(pkt.NetworkProtocolNumber, pkt, false /* incoming */)
+	}
+
+	return e.Endpoint.WritePackets(pkts)
+}

pkg/tcpip/link/packetsocket/packetsocket_test.go

@@ -0,0 +1,169 @@
+// Copyright 2022 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package packetsocket_test
+
+import (
+	"math"
+	"os"
+	"testing"
+
+	"gvisor.dev/gvisor/pkg/refs"
+	"gvisor.dev/gvisor/pkg/refsvfs2"
+	"gvisor.dev/gvisor/pkg/tcpip"
+	"gvisor.dev/gvisor/pkg/tcpip/header"
+	"gvisor.dev/gvisor/pkg/tcpip/link/packetsocket"
+	"gvisor.dev/gvisor/pkg/tcpip/stack"
+)
+
+var _ stack.LinkEndpoint = (*nullEndpoint)(nil)
+
+type nullEndpoint struct {
+	disp stack.NetworkDispatcher
+}
+
+func (*nullEndpoint) MTU() uint32 {
+	return math.MaxUint32
+}
+func (*nullEndpoint) Capabilities() stack.LinkEndpointCapabilities {
+	return 0
+}
+func (*nullEndpoint) MaxHeaderLength() uint16 {
+	return 0
+}
+func (*nullEndpoint) LinkAddress() tcpip.LinkAddress {
+	var l tcpip.LinkAddress
+	return l
+}
+func (*nullEndpoint) WritePackets(pkts stack.PacketBufferList) (int, tcpip.Error) {
+	return pkts.Len(), nil
+}
+func (e *nullEndpoint) Attach(d stack.NetworkDispatcher)      { e.disp = d }
+func (e *nullEndpoint) IsAttached() bool                      { return e.disp != nil }
+func (*nullEndpoint) Wait()                                   {}
+func (*nullEndpoint) ARPHardwareType() header.ARPHardwareType { return header.ARPHardwareNone }
+func (*nullEndpoint) AddHeader(*stack.PacketBuffer)           {}
+
+var _ stack.NetworkDispatcher = (*testNetworkDispatcher)(nil)
+
+type linkPacketInfo struct {
+	pkt      *stack.PacketBuffer
+	protocol tcpip.NetworkProtocolNumber
+	incoming bool
+}
+
+type networkPacketInfo struct {
+	pkt      *stack.PacketBuffer
+	protocol tcpip.NetworkProtocolNumber
+}
+
+type testNetworkDispatcher struct {
+	t *testing.T
+
+	linkPacket linkPacketInfo
+
+	networkPacket networkPacketInfo
+}
+
+func (t *testNetworkDispatcher) reset() {
+	if pkt := t.linkPacket.pkt; pkt != nil {
+		pkt.DecRef()
+	}
+	if pkt := t.networkPacket.pkt; pkt != nil {
+		pkt.DecRef()
+	}
+
+	*t = testNetworkDispatcher{}
+}
+
+func (t *testNetworkDispatcher) DeliverNetworkPacket(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) {
+	networkPacket := networkPacketInfo{
+		pkt:      pkt,
+		protocol: protocol,
+	}
+
+	if t.networkPacket != (networkPacketInfo{}) {
+		t.t.Fatalf("already delivered network packet = %#v; new = %#v", t.networkPacket, networkPacket)
+	}
+
+	pkt.IncRef()
+
+	t.networkPacket = networkPacket
+}
+
+func (t *testNetworkDispatcher) DeliverLinkPacket(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer, incoming bool) {
+	linkPacket := linkPacketInfo{
+		pkt:      pkt,
+		protocol: protocol,
+		incoming: incoming,
+	}
+
+	if t.linkPacket != (linkPacketInfo{}) {
+		t.t.Fatalf("already delivered link packet = %#v; new = %#v", t.linkPacket, linkPacket)
+	}
+
+	pkt.IncRef()
+
+	t.linkPacket = linkPacket
+}
+
+func TestPacketDispatch(t *testing.T) {
+	const protocol = 5
+
+	var nullEP nullEndpoint
+	ep := packetsocket.New(&nullEP)
+
+	var d testNetworkDispatcher
+	defer d.reset()
+	ep.Attach(&d)
+
+	pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{})
+	defer pkt.DecRef()
+	pkt.NetworkProtocolNumber = protocol
+
+	{
+		var pkts stack.PacketBufferList
+		pkts.PushBack(pkt)
+		if n, err := ep.WritePackets(pkts); err != nil {
+			t.Fatalf("ep.WritePackets(_): %s", err)
+		} else if n != 1 {
+			t.Fatalf("got ep.WritePackets(_) = %d, want = 1", n)
+		}
+
+		if want := (networkPacketInfo{}); d.networkPacket != want {
+			t.Errorf("got d.networkPacket = %#v, want = %#v", d.networkPacket, want)
+		}
+		if want := (linkPacketInfo{pkt: pkt, protocol: protocol, incoming: false}); d.linkPacket != want {
+			t.Errorf("got d.linkPacket = %#v, want = %#v", d.linkPacket, want)
+		}
+	}
+
+	d.reset()
+	{
+		nullEP.disp.DeliverNetworkPacket(protocol, pkt)
+		if want := (networkPacketInfo{pkt: pkt, protocol: protocol}); d.networkPacket != want {
+			t.Errorf("got d.networkPacket = %#v, want = %#v", d.networkPacket, want)
+		}
+		if want := (linkPacketInfo{pkt: pkt, protocol: protocol, incoming: true}); d.linkPacket != want {
+			t.Errorf("got d.linkPacket = %#v, want = %#v", d.linkPacket, want)
+		}
+	}
+}
+
+func TestMain(m *testing.M) {
+	refs.SetLeakMode(refs.LeaksPanic)
+	code := m.Run()
+	refsvfs2.DoLeakCheck()
+	os.Exit(code)
+}

pkg/tcpip/link/sharedmem/sharedmem_test.go

@@ -155,6 +155,10 @@ func (c *testContext) DeliverNetworkPacket(proto tcpip.NetworkProtocolNumber, pk
 	c.packetCh <- struct{}{}
 }
 
+func (c *testContext) DeliverLinkPacket(tcpip.NetworkProtocolNumber, *stack.PacketBuffer, bool) {
+	c.t.Fatal("DeliverLinkPacket not implemented")
+}
+
 func (c *testContext) cleanup() {
 	c.ep.Close()
 	closeFDs(c.txCfg)

pkg/tcpip/link/tun/BUILD

@@ -35,6 +35,7 @@ go_library(
         "//pkg/tcpip/buffer",
         "//pkg/tcpip/header",
         "//pkg/tcpip/link/channel",
+        "//pkg/tcpip/link/packetsocket",
         "//pkg/tcpip/stack",
         "//pkg/waiter",
         "@org_golang_x_sys//unix:go_default_library",

pkg/tcpip/link/tun/device.go

@@ -24,6 +24,7 @@ import (
 	"gvisor.dev/gvisor/pkg/tcpip/buffer"
 	"gvisor.dev/gvisor/pkg/tcpip/header"
 	"gvisor.dev/gvisor/pkg/tcpip/link/channel"
+	"gvisor.dev/gvisor/pkg/tcpip/link/packetsocket"
 	"gvisor.dev/gvisor/pkg/tcpip/stack"
 	"gvisor.dev/gvisor/pkg/waiter"
 )
@@ -150,7 +151,7 @@ func attachOrCreateNIC(s *stack.Stack, name, prefix string, linkCaps stack.LinkE
 		if endpoint.name == "" {
 			endpoint.name = fmt.Sprintf("%s%d", prefix, id)
 		}
-		err := s.CreateNICWithOptions(endpoint.nicID, endpoint, stack.NICOptions{
+		err := s.CreateNICWithOptions(endpoint.nicID, packetsocket.New(endpoint), stack.NICOptions{
 			Name: endpoint.name,
 		})
 		switch err.(type) {

pkg/tcpip/link/waitable/waitable.go

@@ -28,6 +28,9 @@ import (
 	"gvisor.dev/gvisor/pkg/tcpip/stack"
 )
 
+var _ stack.NetworkDispatcher = (*Endpoint)(nil)
+var _ stack.LinkEndpoint = (*Endpoint)(nil)
+
 // Endpoint is a waitable link-layer endpoint.
 type Endpoint struct {
 	dispatchGate sync.Gate
@@ -59,6 +62,16 @@ func (e *Endpoint) DeliverNetworkPacket(protocol tcpip.NetworkProtocolNumber, pk
 	e.dispatchGate.Leave()
 }
 
+// DeliverLinkPacket implements stack.NetworkDispatcher.
+func (e *Endpoint) DeliverLinkPacket(protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer, incoming bool) {
+	if !e.dispatchGate.Enter() {
+		return
+	}
+
+	e.dispatcher.DeliverLinkPacket(protocol, pkt, incoming)
+	e.dispatchGate.Leave()
+}
+
 // Attach implements stack.LinkEndpoint.Attach. It saves the dispatcher and
 // registers with the lower endpoint as its dispatcher so that "e" is called
 // for inbound packets.

pkg/tcpip/link/waitable/waitable_test.go

@@ -44,6 +44,10 @@ func (e *countedEndpoint) DeliverNetworkPacket(protocol tcpip.NetworkProtocolNum
 	e.dispatchCount++
 }
 
+func (*countedEndpoint) DeliverLinkPacket(tcpip.NetworkProtocolNumber, *stack.PacketBuffer, bool) {
+	panic("not implemented")
+}
+
 func (e *countedEndpoint) Attach(dispatcher stack.NetworkDispatcher) {
 	e.attachCount++
 	e.dispatcher = dispatcher

pkg/tcpip/stack/nic.go

@@ -40,6 +40,7 @@ func (l *linkResolver) confirmReachable(addr tcpip.Address) {
 }
 
 var _ NetworkInterface = (*nic)(nil)
+var _ NetworkDispatcher = (*nic)(nil)
 
 // nic represents a "network interface card" to which the networking stack is
 // attached.
@@ -390,8 +391,6 @@ func (n *nic) writePacket(pkt *PacketBuffer) tcpip.Error {
 }
 
 func (n *nic) writeRawPacket(pkt *PacketBuffer) tcpip.Error {
-	n.deliverLinkPacket(pkt.NetworkProtocolNumber, pkt, false /* incoming */)
-
 	if err := n.qDisc.WritePacket(pkt); err != nil {
 		return err
 	}
@@ -723,12 +722,10 @@ func (n *nic) DeliverNetworkPacket(protocol tcpip.NetworkProtocolNumber, pkt *Pa
 
 	pkt.RXTransportChecksumValidated = n.NetworkLinkEndpoint.Capabilities()&CapabilityRXChecksumOffload != 0
 
-	n.deliverLinkPacket(protocol, pkt, true /* incoming */)
-
 	networkEndpoint.HandlePacket(pkt)
 }
 
-func (n *nic) deliverLinkPacket(protocol tcpip.NetworkProtocolNumber, pkt *PacketBuffer, incoming bool) {
+func (n *nic) DeliverLinkPacket(protocol tcpip.NetworkProtocolNumber, pkt *PacketBuffer, incoming bool) {
 	// Deliver to interested packet endpoints without holding NIC lock.
 	var packetEPPkt *PacketBuffer
 	defer func() {

pkg/tcpip/stack/registration.go

@@ -730,12 +730,18 @@ type NetworkDispatcher interface {
 	// DeliverNetworkPacket finds the appropriate network protocol endpoint
 	// and hands the packet over for further processing.
 	//
-	// pkt.LinkHeader may or may not be set before calling
-	// DeliverNetworkPacket. Some packets do not have link headers (e.g.
-	// packets sent via loopback), and won't have the field set.
+	//
+	// If the link-layer has a header, the packet's link header must be populated.
 	//
 	// DeliverNetworkPacket may modify pkt.
 	DeliverNetworkPacket(protocol tcpip.NetworkProtocolNumber, pkt *PacketBuffer)
+
+	// DeliverLinkPacket delivers a packet to any interested packet endpoints.
+	//
+	// This method should be called with both incoming and outgoing packets.
+	//
+	// If the link-layer has a header, the packet's link header must be populated.
+	DeliverLinkPacket(protocol tcpip.NetworkProtocolNumber, pkt *PacketBuffer, incoming bool)
 }
 
 // LinkEndpointCapabilities is the type associated with the capabilities

runsc/boot/BUILD

@@ -102,6 +102,7 @@ go_library(
         "//pkg/tcpip/link/ethernet",
         "//pkg/tcpip/link/fdbased",
         "//pkg/tcpip/link/loopback",
+        "//pkg/tcpip/link/packetsocket",
         "//pkg/tcpip/link/qdisc/fifo",
         "//pkg/tcpip/link/sniffer",
         "//pkg/tcpip/network/arp",

runsc/boot/loader.go

@@ -60,6 +60,7 @@ import (
 	"gvisor.dev/gvisor/pkg/tcpip"
 	"gvisor.dev/gvisor/pkg/tcpip/link/ethernet"
 	"gvisor.dev/gvisor/pkg/tcpip/link/loopback"
+	"gvisor.dev/gvisor/pkg/tcpip/link/packetsocket"
 	"gvisor.dev/gvisor/pkg/tcpip/link/sniffer"
 	"gvisor.dev/gvisor/pkg/tcpip/network/arp"
 	"gvisor.dev/gvisor/pkg/tcpip/network/ipv4"
@@ -1197,7 +1198,7 @@ func (f *sandboxNetstackCreator) CreateStack() (inet.Stack, error) {
 	n := &Network{Stack: s.(*netstack.Stack).Stack}
 	nicID := tcpip.NICID(f.uniqueID.UniqueID())
 	link := DefaultLoopbackLink
-	linkEP := ethernet.New(loopback.New())
+	linkEP := packetsocket.New(ethernet.New(loopback.New()))
 	opts := stack.NICOptions{Name: link.Name}
 
 	if err := n.createNICWithAddrs(nicID, linkEP, opts, link.Addresses); err != nil {

runsc/boot/network.go

@@ -26,6 +26,7 @@ import (
 	"gvisor.dev/gvisor/pkg/tcpip/link/ethernet"
 	"gvisor.dev/gvisor/pkg/tcpip/link/fdbased"
 	"gvisor.dev/gvisor/pkg/tcpip/link/loopback"
+	"gvisor.dev/gvisor/pkg/tcpip/link/packetsocket"
 	"gvisor.dev/gvisor/pkg/tcpip/link/qdisc/fifo"
 	"gvisor.dev/gvisor/pkg/tcpip/link/sniffer"
 	"gvisor.dev/gvisor/pkg/tcpip/network/ipv4"
@@ -175,7 +176,7 @@ func (n *Network) CreateLinksAndRoutes(args *CreateLinksAndRoutesArgs, _ *struct
 		nicID++
 		nicids[link.Name] = nicID
 
-		linkEP := ethernet.New(loopback.New())
+		linkEP := packetsocket.New(ethernet.New(loopback.New()))
 
 		log.Infof("Enabling loopback interface %q with id %d on addresses %+v", link.Name, nicID, link.Addresses)
 		opts := stack.NICOptions{Name: link.Name}
@@ -229,7 +230,7 @@ func (n *Network) CreateLinksAndRoutes(args *CreateLinksAndRoutesArgs, _ *struct
 		}
 
 		// Wrap linkEP in a sniffer to enable packet logging.
-		sniffEP := sniffer.New(linkEP)
+		sniffEP := sniffer.New(packetsocket.New(linkEP))
 
 		var qDisc stack.QueueingDiscipline
 		switch link.QDisc {