From 9b56ce185aa66926d6034fbb89e76f17b3ecd9c0 Mon Sep 17 00:00:00 2001
From: Bhasker Hariharan <bhaskerh@google.com>
Date: Mon, 13 Dec 2021 18:28:44 -0800
Subject: [PATCH] Hoist Fifo QDisc to NIC layer.

This will in future allow us to easily replace qdisc implementations
at runtime for any given NIC.

PiperOrigin-RevId: 416183274
---
 pkg/tcpip/link/qdisc/fifo/BUILD        |   3 +-
 pkg/tcpip/link/qdisc/fifo/endpoint.go  | 227 -------------------------
 pkg/tcpip/link/qdisc/fifo/fifo.go      | 139 +++++++++++++++
 pkg/tcpip/stack/neighbor_entry_test.go |   2 +-
 pkg/tcpip/stack/nic.go                 |  49 ++++--
 pkg/tcpip/stack/registration.go        |  86 ++++++----
 pkg/tcpip/stack/route.go               |  16 +-
 pkg/tcpip/stack/stack.go               |  19 ++-
 runsc/boot/loader.go                   |   4 +-
 runsc/boot/network.go                  |  20 ++-
 test/benchmarks/tcp/tcp_proxy.go       |   4 +-
 11 files changed, 265 insertions(+), 304 deletions(-)
 delete mode 100644 pkg/tcpip/link/qdisc/fifo/endpoint.go
 create mode 100644 pkg/tcpip/link/qdisc/fifo/fifo.go

diff --git a/pkg/tcpip/link/qdisc/fifo/BUILD b/pkg/tcpip/link/qdisc/fifo/BUILD
index 5bea598eb..6ec805dc4 100644
--- a/pkg/tcpip/link/qdisc/fifo/BUILD
+++ b/pkg/tcpip/link/qdisc/fifo/BUILD
@@ -5,7 +5,7 @@ package(licenses = ["notice"])
 go_library(
     name = "fifo",
     srcs = [
-        "endpoint.go",
+        "fifo.go",
         "packet_buffer_queue.go",
     ],
     visibility = ["//visibility:public"],
@@ -13,7 +13,6 @@ go_library(
         "//pkg/sleep",
         "//pkg/sync",
         "//pkg/tcpip",
-        "//pkg/tcpip/header",
         "//pkg/tcpip/stack",
     ],
 )
diff --git a/pkg/tcpip/link/qdisc/fifo/endpoint.go b/pkg/tcpip/link/qdisc/fifo/endpoint.go
deleted file mode 100644
index a68b274b2..000000000
--- a/pkg/tcpip/link/qdisc/fifo/endpoint.go
+++ /dev/null
@@ -1,227 +0,0 @@
-// Copyright 2020 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package fifo provides the implementation of data-link layer endpoints that
-// wrap another endpoint and queues all outbound packets and asynchronously
-// dispatches them to the lower endpoint.
-package fifo
-
-import (
-	"gvisor.dev/gvisor/pkg/sleep"
-	"gvisor.dev/gvisor/pkg/sync"
-	"gvisor.dev/gvisor/pkg/tcpip"
-	"gvisor.dev/gvisor/pkg/tcpip/header"
-	"gvisor.dev/gvisor/pkg/tcpip/stack"
-)
-
-var _ stack.LinkEndpoint = (*endpoint)(nil)
-var _ stack.GSOEndpoint = (*endpoint)(nil)
-
-// endpoint represents a LinkEndpoint which implements a FIFO queue for all
-// outgoing packets. endpoint can have 1 or more underlying queueDispatchers.
-// All outgoing packets are consistenly hashed to a single underlying queue
-// using the PacketBuffer.Hash if set, otherwise all packets are queued to the
-// first queue to avoid reordering in case of missing hash.
-type endpoint struct {
-	dispatcher  stack.NetworkDispatcher
-	lower       stack.LinkEndpoint
-	wg          sync.WaitGroup
-	dispatchers []*queueDispatcher
-}
-
-// queueDispatcher is responsible for dispatching all outbound packets in its
-// queue. It will also smartly batch packets when possible and write them
-// through the lower LinkEndpoint.
-type queueDispatcher struct {
-	lower          stack.LinkEndpoint
-	q              *packetBufferQueue
-	newPacketWaker sleep.Waker
-	closeWaker     sleep.Waker
-}
-
-// New creates a new fifo link endpoint with the n queues with maximum
-// capacity of queueLen.
-func New(lower stack.LinkEndpoint, n int, queueLen int) stack.LinkEndpoint {
-	e := &endpoint{
-		lower: lower,
-	}
-	// Create the required dispatchers
-	for i := 0; i < n; i++ {
-		qd := &queueDispatcher{
-			q:     &packetBufferQueue{limit: queueLen},
-			lower: lower,
-		}
-		e.dispatchers = append(e.dispatchers, qd)
-		e.wg.Add(1)
-		go func() {
-			defer e.wg.Done()
-			qd.dispatchLoop()
-		}()
-	}
-	return e
-}
-
-func (q *queueDispatcher) dispatchLoop() {
-	s := sleep.Sleeper{}
-	s.AddWaker(&q.newPacketWaker)
-	s.AddWaker(&q.closeWaker)
-	defer s.Done()
-
-	const batchSize = 32
-	var batch stack.PacketBufferList
-	for {
-		w := s.Fetch(true)
-		if w == &q.closeWaker {
-			return
-		}
-		// Must otherwise be the newPacketWaker.
-		for pkt := q.q.dequeue(); pkt != nil; pkt = q.q.dequeue() {
-			batch.PushBack(pkt)
-			if batch.Len() < batchSize && !q.q.empty() {
-				continue
-			}
-			// We pass a protocol of zero here because each packet carries its
-			// NetworkProtocol.
-			q.lower.WritePackets(stack.RouteInfo{}, batch, 0 /* protocol */)
-			batch.DecRef()
-			batch.Reset()
-		}
-	}
-}
-
-// DeliverNetworkPacket implements stack.NetworkDispatcher.DeliverNetworkPacket.
-func (e *endpoint) DeliverNetworkPacket(remote, local tcpip.LinkAddress, protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) {
-	e.dispatcher.DeliverNetworkPacket(remote, local, protocol, pkt)
-}
-
-// Attach implements stack.LinkEndpoint.Attach.
-func (e *endpoint) Attach(dispatcher stack.NetworkDispatcher) {
-	// nil means the NIC is being removed.
-	if dispatcher == nil {
-		e.lower.Attach(nil)
-		e.Wait()
-		e.dispatcher = nil
-		return
-	}
-	e.dispatcher = dispatcher
-	e.lower.Attach(e)
-}
-
-// IsAttached implements stack.LinkEndpoint.IsAttached.
-func (e *endpoint) IsAttached() bool {
-	return e.dispatcher != nil
-}
-
-// MTU implements stack.LinkEndpoint.MTU.
-func (e *endpoint) MTU() uint32 {
-	return e.lower.MTU()
-}
-
-// Capabilities implements stack.LinkEndpoint.Capabilities.
-func (e *endpoint) Capabilities() stack.LinkEndpointCapabilities {
-	return e.lower.Capabilities()
-}
-
-// MaxHeaderLength implements stack.LinkEndpoint.MaxHeaderLength.
-func (e *endpoint) MaxHeaderLength() uint16 {
-	return e.lower.MaxHeaderLength()
-}
-
-// LinkAddress implements stack.LinkEndpoint.LinkAddress.
-func (e *endpoint) LinkAddress() tcpip.LinkAddress {
-	return e.lower.LinkAddress()
-}
-
-// GSOMaxSize implements stack.GSOEndpoint.
-func (e *endpoint) GSOMaxSize() uint32 {
-	if gso, ok := e.lower.(stack.GSOEndpoint); ok {
-		return gso.GSOMaxSize()
-	}
-	return 0
-}
-
-// SupportedGSO implements stack.GSOEndpoint.
-func (e *endpoint) SupportedGSO() stack.SupportedGSO {
-	if gso, ok := e.lower.(stack.GSOEndpoint); ok {
-		return gso.SupportedGSO()
-	}
-	return stack.GSONotSupported
-}
-
-// WritePacket implements stack.LinkEndpoint.WritePacket.
-//
-// The packet must have the following fields populated:
-//  - pkt.EgressRoute
-//  - pkt.GSOOptions
-//  - pkt.NetworkProtocolNumber
-func (e *endpoint) WritePacket(r stack.RouteInfo, protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) tcpip.Error {
-	d := e.dispatchers[int(pkt.Hash)%len(e.dispatchers)]
-	if !d.q.enqueue(pkt) {
-		return &tcpip.ErrNoBufferSpace{}
-	}
-	d.newPacketWaker.Assert()
-	return nil
-}
-
-// WritePackets implements stack.LinkEndpoint.WritePackets.
-//
-// Each packet in the packet buffer list must have the following fields
-// populated:
-//  - pkt.EgressRoute
-//  - pkt.GSOOptions
-//  - pkt.NetworkProtocolNumber
-func (e *endpoint) WritePackets(r stack.RouteInfo, pkts stack.PacketBufferList, protocol tcpip.NetworkProtocolNumber) (int, tcpip.Error) {
-	enqueued := 0
-	for pkt := pkts.Front(); pkt != nil; {
-		d := e.dispatchers[int(pkt.Hash)%len(e.dispatchers)]
-		nxt := pkt.Next()
-		if !d.q.enqueue(pkt) {
-			if enqueued > 0 {
-				d.newPacketWaker.Assert()
-			}
-			return enqueued, &tcpip.ErrNoBufferSpace{}
-		}
-		pkt = nxt
-		enqueued++
-		d.newPacketWaker.Assert()
-	}
-	return enqueued, nil
-}
-
-// Wait implements stack.LinkEndpoint.Wait.
-func (e *endpoint) Wait() {
-	e.lower.Wait()
-
-	// The linkEP is gone. Teardown the outbound dispatcher goroutines.
-	for i := range e.dispatchers {
-		e.dispatchers[i].closeWaker.Assert()
-	}
-
-	e.wg.Wait()
-}
-
-// ARPHardwareType implements stack.LinkEndpoint.ARPHardwareType
-func (e *endpoint) ARPHardwareType() header.ARPHardwareType {
-	return e.lower.ARPHardwareType()
-}
-
-// AddHeader implements stack.LinkEndpoint.AddHeader.
-func (e *endpoint) AddHeader(local, remote tcpip.LinkAddress, protocol tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) {
-	e.lower.AddHeader(local, remote, protocol, pkt)
-}
-
-// WriteRawPacket implements stack.LinkEndpoint.
-func (e *endpoint) WriteRawPacket(pkt *stack.PacketBuffer) tcpip.Error {
-	return e.lower.WriteRawPacket(pkt)
-}
diff --git a/pkg/tcpip/link/qdisc/fifo/fifo.go b/pkg/tcpip/link/qdisc/fifo/fifo.go
new file mode 100644
index 000000000..d9dbfefbd
--- /dev/null
+++ b/pkg/tcpip/link/qdisc/fifo/fifo.go
@@ -0,0 +1,139 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package fifo provides the implementation of FIFO queuing discipline that
+// queues all outbound packets and asynchronously dispatches them to the
+// lower link endpoint in the order that they were queued.
+package fifo
+
+import (
+	"gvisor.dev/gvisor/pkg/sleep"
+	"gvisor.dev/gvisor/pkg/sync"
+	"gvisor.dev/gvisor/pkg/tcpip"
+	"gvisor.dev/gvisor/pkg/tcpip/stack"
+)
+
+var _ stack.QueueingDiscipline = (*discipline)(nil)
+
+// discipline represents a QueueingDiscipline which implements a FIFO queue for
+// all outgoing packets. discipline can have 1 or more underlying
+// queueDispatchers. All outgoing packets are consistenly hashed to a single
+// underlying queue using the PacketBuffer.Hash if set, otherwise all packets
+// are queued to the first queue to avoid reordering in case of missing hash.
+type discipline struct {
+	dispatcher  stack.NetworkDispatcher
+	lower       stack.LinkEndpoint
+	wg          sync.WaitGroup
+	dispatchers []*queueDispatcher
+}
+
+// queueDispatcher is responsible for dispatching all outbound packets in its
+// queue. It will also smartly batch packets when possible and write them
+// through the lower LinkEndpoint.
+type queueDispatcher struct {
+	lower          stack.LinkEndpoint
+	q              *packetBufferQueue
+	newPacketWaker sleep.Waker
+	closeWaker     sleep.Waker
+}
+
+// New creates a new fifo queuing discipline  with the n queues with maximum
+// capacity of queueLen.
+func New(lower stack.LinkEndpoint, n int, queueLen int) stack.QueueingDiscipline {
+	d := &discipline{
+		lower: lower,
+	}
+	// Create the required dispatchers
+	for i := 0; i < n; i++ {
+		qd := &queueDispatcher{
+			q:     &packetBufferQueue{limit: queueLen},
+			lower: lower,
+		}
+		d.dispatchers = append(d.dispatchers, qd)
+		d.wg.Add(1)
+		go func() {
+			defer d.wg.Done()
+			qd.dispatchLoop()
+		}()
+	}
+	return d
+}
+
+func (q *queueDispatcher) dispatchLoop() {
+	s := sleep.Sleeper{}
+	s.AddWaker(&q.newPacketWaker)
+	s.AddWaker(&q.closeWaker)
+	defer s.Done()
+
+	const batchSize = 32
+	var batch stack.PacketBufferList
+	for {
+		w := s.Fetch(true)
+		if w == &q.closeWaker {
+			return
+		}
+		// Must otherwise be the newPacketWaker.
+		for pkt := q.q.dequeue(); pkt != nil; pkt = q.q.dequeue() {
+			batch.PushBack(pkt)
+			if batch.Len() < batchSize && !q.q.empty() {
+				continue
+			}
+			// We pass a protocol of zero here because each packet carries its
+			// NetworkProtocol.
+			q.lower.WritePackets(stack.RouteInfo{}, batch, 0 /* protocol */)
+			batch.DecRef()
+			batch.Reset()
+		}
+	}
+}
+
+// WritePacket implements stack.QueueingDiscipline.WritePacket.
+//
+// The packet must have the following fields populated:
+//  - pkt.EgressRoute
+//  - pkt.GSOOptions
+//  - pkt.NetworkProtocolNumber
+func (d *discipline) WritePacket(_ stack.RouteInfo, _ tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) tcpip.Error {
+	qd := d.dispatchers[int(pkt.Hash)%len(d.dispatchers)]
+	if !qd.q.enqueue(pkt) {
+		return &tcpip.ErrNoBufferSpace{}
+	}
+	qd.newPacketWaker.Assert()
+	return nil
+}
+
+// WritePackets implements stack.QueueingDiscipline.WritePackets.
+//
+// Each packet in the packet buffer list must have the following fields
+// populated:
+//  - pkt.EgressRoute
+//  - pkt.GSOOptions
+//  - pkt.NetworkProtocolNumber
+func (d *discipline) WritePackets(_ stack.RouteInfo, pkts stack.PacketBufferList, _ tcpip.NetworkProtocolNumber) (int, tcpip.Error) {
+	enqueued := 0
+	for pkt := pkts.Front(); pkt != nil; {
+		qd := d.dispatchers[int(pkt.Hash)%len(d.dispatchers)]
+		nxt := pkt.Next()
+		if !qd.q.enqueue(pkt) {
+			if enqueued > 0 {
+				qd.newPacketWaker.Assert()
+			}
+			return enqueued, &tcpip.ErrNoBufferSpace{}
+		}
+		pkt = nxt
+		enqueued++
+		qd.newPacketWaker.Assert()
+	}
+	return enqueued, nil
+}
diff --git a/pkg/tcpip/stack/neighbor_entry_test.go b/pkg/tcpip/stack/neighbor_entry_test.go
index 59d86d6d4..7c84948f6 100644
--- a/pkg/tcpip/stack/neighbor_entry_test.go
+++ b/pkg/tcpip/stack/neighbor_entry_test.go
@@ -205,7 +205,7 @@ func entryTestSetup(c NUDConfigurations) (*neighborEntry, *testNUDDispatcher, *e
 	clock := faketime.NewManualClock()
 	disp := testNUDDispatcher{}
 	nic := nic{
-		LinkEndpoint: nil, // entryTestLinkResolver doesn't use a LinkEndpoint
+		NetworkLinkEndpoint: nil, // entryTestLinkResolver doesn't use this.
 
 		id: entryTestNICID,
 		stack: &Stack{
diff --git a/pkg/tcpip/stack/nic.go b/pkg/tcpip/stack/nic.go
index a79d8d07d..b4ac6ce86 100644
--- a/pkg/tcpip/stack/nic.go
+++ b/pkg/tcpip/stack/nic.go
@@ -44,7 +44,7 @@ var _ NetworkInterface = (*nic)(nil)
 // nic represents a "network interface card" to which the networking stack is
 // attached.
 type nic struct {
-	LinkEndpoint
+	NetworkLinkEndpoint
 
 	stack   *Stack
 	id      tcpip.NICID
@@ -82,6 +82,9 @@ type nic struct {
 		// +checklocks:mu
 		eps map[tcpip.NetworkProtocolNumber]*packetEndpointList
 	}
+
+	qDisc     QueueingDiscipline
+	rawLinkEP LinkRawWriter
 }
 
 // makeNICStats initializes the NIC statistics and associates them to the global
@@ -135,26 +138,33 @@ func (p *packetEndpointList) forEach(fn func(PacketEndpoint)) {
 }
 
 // newNIC returns a new NIC using the default NDP configurations from stack.
-func newNIC(stack *Stack, id tcpip.NICID, name string, ep LinkEndpoint, ctx NICContext) *nic {
+func newNIC(stack *Stack, id tcpip.NICID, ep LinkEndpoint, opts NICOptions) *nic {
 	// TODO(b/141011931): Validate a LinkEndpoint (ep) is valid. For
 	// example, make sure that the link address it provides is a valid
 	// unicast ethernet address.
 
+	// If no queueing discipline was specified provide a stub implementation that
+	// just delegates to the lower link endpoint.
+	qDisc := opts.QDisc
+	if qDisc == nil {
+		qDisc = ep
+	}
+
 	// TODO(b/143357959): RFC 8200 section 5 requires that IPv6 endpoints
 	// observe an MTU of at least 1280 bytes. Ensure that this requirement
 	// of IPv6 is supported on this endpoint's LinkEndpoint.
-
 	nic := &nic{
-		LinkEndpoint: ep,
-
+		NetworkLinkEndpoint:       ep,
 		stack:                     stack,
 		id:                        id,
-		name:                      name,
-		context:                   ctx,
+		name:                      opts.Name,
+		context:                   opts.Context,
 		stats:                     makeNICStats(stack.Stats().NICs),
 		networkEndpoints:          make(map[tcpip.NetworkProtocolNumber]NetworkEndpoint),
 		linkAddrResolvers:         make(map[tcpip.NetworkProtocolNumber]*linkResolver),
 		duplicateAddressDetectors: make(map[tcpip.NetworkProtocolNumber]DuplicateAddressDetector),
+		qDisc:                     qDisc,
+		rawLinkEP:                 ep,
 	}
 	nic.linkResQueue.init(nic)
 
@@ -183,7 +193,7 @@ func newNIC(stack *Stack, id tcpip.NICID, name string, ep LinkEndpoint, ctx NICC
 		}
 	}
 
-	nic.LinkEndpoint.Attach(nic)
+	nic.NetworkLinkEndpoint.Attach(nic)
 
 	return nic
 }
@@ -290,7 +300,7 @@ func (n *nic) remove() tcpip.Error {
 	}
 
 	// Detach from link endpoint, so no packet comes in.
-	n.LinkEndpoint.Attach(nil)
+	n.NetworkLinkEndpoint.Attach(nil)
 	return nil
 }
 
@@ -311,15 +321,20 @@ func (n *nic) Promiscuous() bool {
 
 // IsLoopback implements NetworkInterface.
 func (n *nic) IsLoopback() bool {
-	return n.LinkEndpoint.Capabilities()&CapabilityLoopback != 0
+	return n.NetworkLinkEndpoint.Capabilities()&CapabilityLoopback != 0
 }
 
-// WritePacket implements NetworkLinkEndpoint.
+// WritePacket implements LinkWriter.
 func (n *nic) WritePacket(r *Route, protocol tcpip.NetworkProtocolNumber, pkt *PacketBuffer) tcpip.Error {
 	_, err := n.enqueuePacketBuffer(r, protocol, pkt)
 	return err
 }
 
+// WriteRawPacket implements LinkRawWriter.
+func (n *nic) WriteRawPacket(pkt *PacketBuffer) tcpip.Error {
+	return n.rawLinkEP.WriteRawPacket(pkt)
+}
+
 func (n *nic) writePacketBuffer(r RouteInfo, protocol tcpip.NetworkProtocolNumber, pkt pendingPacketBuffer) (int, tcpip.Error) {
 	switch pkt := pkt.(type) {
 	case *PacketBuffer:
@@ -379,7 +394,7 @@ func (n *nic) writePacket(r RouteInfo, protocol tcpip.NetworkProtocolNumber, pkt
 	pkt.NetworkProtocolNumber = protocol
 	n.deliverOutboundPacket(r.RemoteLinkAddress, pkt)
 
-	if err := n.LinkEndpoint.WritePacket(r, protocol, pkt); err != nil {
+	if err := n.qDisc.WritePacket(r, protocol, pkt); err != nil {
 		return err
 	}
 
@@ -388,7 +403,7 @@ func (n *nic) writePacket(r RouteInfo, protocol tcpip.NetworkProtocolNumber, pkt
 	return nil
 }
 
-// WritePackets implements NetworkLinkEndpoint.
+// WritePackets implements LinkWriter..
 func (n *nic) WritePackets(r *Route, pkts PacketBufferList, protocol tcpip.NetworkProtocolNumber) (int, tcpip.Error) {
 	return n.enqueuePacketBuffer(r, protocol, &pkts)
 }
@@ -400,7 +415,7 @@ func (n *nic) writePackets(r RouteInfo, protocol tcpip.NetworkProtocolNumber, pk
 		n.deliverOutboundPacket(r.RemoteLinkAddress, pkt)
 	}
 
-	writtenPackets, err := n.LinkEndpoint.WritePackets(r, pkts, protocol)
+	writtenPackets, err := n.qDisc.WritePackets(r, pkts, protocol)
 	n.stats.tx.packets.IncrementBy(uint64(writtenPackets))
 	writtenBytes := 0
 	for i, pb := 0, pkts.Front(); i < writtenPackets && pb != nil; i, pb = i+1, pb.Next() {
@@ -734,9 +749,9 @@ func (n *nic) DeliverNetworkPacket(remote, local tcpip.LinkAddress, protocol tcp
 	// If no local link layer address is provided, assume it was sent
 	// directly to this NIC.
 	if local == "" {
-		local = n.LinkEndpoint.LinkAddress()
+		local = n.NetworkLinkEndpoint.LinkAddress()
 	}
-	pkt.RXTransportChecksumValidated = n.LinkEndpoint.Capabilities()&CapabilityRXChecksumOffload != 0
+	pkt.RXTransportChecksumValidated = n.NetworkLinkEndpoint.Capabilities()&CapabilityRXChecksumOffload != 0
 
 	// Deliver to interested packet endpoints without holding NIC lock.
 	var packetEPPkt *PacketBuffer
@@ -826,7 +841,7 @@ func (n *nic) deliverOutboundPacket(remote tcpip.LinkAddress, pkt *PacketBuffer)
 			// Add the link layer header as outgoing packets are intercepted before
 			// the link layer header is created and packet endpoints are interested
 			// in the link header.
-			n.LinkEndpoint.AddHeader(local, remote, pkt.NetworkProtocolNumber, packetEPPkt)
+			n.NetworkLinkEndpoint.AddHeader(local, remote, pkt.NetworkProtocolNumber, packetEPPkt)
 			packetEPPkt.PktType = tcpip.PacketOutgoing
 		}
 		clone := packetEPPkt.Clone()
diff --git a/pkg/tcpip/stack/registration.go b/pkg/tcpip/stack/registration.go
index 8a5e96784..af4884333 100644
--- a/pkg/tcpip/stack/registration.go
+++ b/pkg/tcpip/stack/registration.go
@@ -775,6 +775,42 @@ const (
 	CapabilityLoopback
 )
 
+// LinkWriter is an interface that supports sending packets via a data-link
+// layer endpoint.
+type LinkWriter interface {
+	// WritePacket writes a packet with the given protocol and route.
+	//
+	// WritePacket may modify the packet buffer. The packet buffer's
+	// network and transport header must be set.
+	//
+	// To participate in transparent bridging, a LinkEndpoint implementation
+	// should call eth.Encode with header.EthernetFields.SrcAddr set to
+	// r.LocalLinkAddress if it is provided.
+	WritePacket(RouteInfo, tcpip.NetworkProtocolNumber, *PacketBuffer) tcpip.Error
+
+	// WritePackets writes packets with the given protocol and route. Must not be
+	// called with an empty list of packet buffers.
+	//
+	// WritePackets may modify the packet buffers.
+	//
+	// Right now, WritePackets is used only when the software segmentation
+	// offload is enabled. If it will be used for something else, syscall filters
+	// may need to be updated.
+	WritePackets(RouteInfo, PacketBufferList, tcpip.NetworkProtocolNumber) (int, tcpip.Error)
+}
+
+// LinkRawWriter is an interface that must be implemented by all Link endpoints
+// to support emitting pre-formed packets which include the Link header.
+type LinkRawWriter interface {
+	// WriteRawPacket writes a packet directly to the link.
+	//
+	// If the link-layer has its own header, the payload must already include the
+	// header.
+	//
+	// WriteRawPacket may modify the packet.
+	WriteRawPacket(*PacketBuffer) tcpip.Error
+}
+
 // NetworkLinkEndpoint is a data-link layer that supports sending network
 // layer packets.
 type NetworkLinkEndpoint interface {
@@ -793,15 +829,6 @@ type NetworkLinkEndpoint interface {
 	// LinkAddress returns the link address (typically a MAC) of the
 	// endpoint.
 	LinkAddress() tcpip.LinkAddress
-}
-
-// LinkEndpoint is the interface implemented by data link layer protocols (e.g.,
-// ethernet, loopback, raw) and used by network layer protocols to send packets
-// out through the implementer's data link endpoint. When a link header exists,
-// it sets each PacketBuffer's LinkHeader field before passing it up the
-// stack.
-type LinkEndpoint interface {
-	NetworkLinkEndpoint
 
 	// Capabilities returns the set of capabilities supported by the
 	// endpoint.
@@ -835,34 +862,23 @@ type LinkEndpoint interface {
 
 	// AddHeader adds a link layer header to pkt if required.
 	AddHeader(local, remote tcpip.LinkAddress, protocol tcpip.NetworkProtocolNumber, pkt *PacketBuffer)
+}
 
-	// WritePacket writes a packet with the given protocol and route.
-	//
-	// WritePacket may modify the packet buffer. The packet buffer's
-	// network and transport header must be set.
-	//
-	// To participate in transparent bridging, a LinkEndpoint implementation
-	// should call eth.Encode with header.EthernetFields.SrcAddr set to
-	// r.LocalLinkAddress if it is provided.
-	WritePacket(RouteInfo, tcpip.NetworkProtocolNumber, *PacketBuffer) tcpip.Error
+// QueueingDiscipline provides a queueing strategy for outgoing packets (e.g
+// FIFO, LIFO, Random Early Drop etc).
+type QueueingDiscipline interface {
+	LinkWriter
+}
 
-	// WritePackets writes packets with the given protocol and route. Must not be
-	// called with an empty list of packet buffers.
-	//
-	// WritePackets may modify the packet buffers.
-	//
-	// Right now, WritePackets is used only when the software segmentation
-	// offload is enabled. If it will be used for something else, syscall filters
-	// may need to be updated.
-	WritePackets(RouteInfo, PacketBufferList, tcpip.NetworkProtocolNumber) (int, tcpip.Error)
-
-	// WriteRawPacket writes a packet directly to the link.
-	//
-	// If the link-layer has its own header, the payload must already include the
-	// header.
-	//
-	// WriteRawPacket may modify the packet.
-	WriteRawPacket(*PacketBuffer) tcpip.Error
+// LinkEndpoint is the interface implemented by data link layer protocols (e.g.,
+// ethernet, loopback, raw) and used by network layer protocols to send packets
+// out through the implementer's data link endpoint. When a link header exists,
+// it sets each PacketBuffer's LinkHeader field before passing it up the
+// stack.
+type LinkEndpoint interface {
+	NetworkLinkEndpoint
+	LinkWriter
+	LinkRawWriter
 }
 
 // InjectableLinkEndpoint is a LinkEndpoint where inbound packets are
diff --git a/pkg/tcpip/stack/route.go b/pkg/tcpip/stack/route.go
index 1abc4ecf9..388fc5b9a 100644
--- a/pkg/tcpip/stack/route.go
+++ b/pkg/tcpip/stack/route.go
@@ -195,7 +195,7 @@ func makeRoute(netProto tcpip.NetworkProtocolNumber, gateway, localAddr, remoteA
 		return r
 	}
 
-	if r.outgoingNIC.LinkEndpoint.Capabilities()&CapabilityResolutionRequired != 0 {
+	if r.outgoingNIC.NetworkLinkEndpoint.Capabilities()&CapabilityResolutionRequired != 0 {
 		if linkRes, ok := r.outgoingNIC.linkAddrResolvers[r.NetProto()]; ok {
 			r.linkRes = linkRes
 		}
@@ -233,7 +233,7 @@ func makeRouteInner(netProto tcpip.NetworkProtocolNumber, localAddr, remoteAddr
 		routeInfo: routeInfo{
 			NetProto:         netProto,
 			LocalAddress:     localAddr,
-			LocalLinkAddress: outgoingNIC.LinkEndpoint.LinkAddress(),
+			LocalLinkAddress: outgoingNIC.NetworkLinkEndpoint.LinkAddress(),
 			RemoteAddress:    remoteAddr,
 			Loop:             loop,
 		},
@@ -298,12 +298,12 @@ func (r *Route) RequiresTXTransportChecksum() bool {
 	if r.local() {
 		return false
 	}
-	return r.outgoingNIC.LinkEndpoint.Capabilities()&CapabilityTXChecksumOffload == 0
+	return r.outgoingNIC.NetworkLinkEndpoint.Capabilities()&CapabilityTXChecksumOffload == 0
 }
 
 // HasSoftwareGSOCapability returns true if the route supports software GSO.
 func (r *Route) HasSoftwareGSOCapability() bool {
-	if gso, ok := r.outgoingNIC.LinkEndpoint.(GSOEndpoint); ok {
+	if gso, ok := r.outgoingNIC.NetworkLinkEndpoint.(GSOEndpoint); ok {
 		return gso.SupportedGSO() == SWGSOSupported
 	}
 	return false
@@ -311,7 +311,7 @@ func (r *Route) HasSoftwareGSOCapability() bool {
 
 // HasHardwareGSOCapability returns true if the route supports hardware GSO.
 func (r *Route) HasHardwareGSOCapability() bool {
-	if gso, ok := r.outgoingNIC.LinkEndpoint.(GSOEndpoint); ok {
+	if gso, ok := r.outgoingNIC.NetworkLinkEndpoint.(GSOEndpoint); ok {
 		return gso.SupportedGSO() == HWGSOSupported
 	}
 	return false
@@ -319,17 +319,17 @@ func (r *Route) HasHardwareGSOCapability() bool {
 
 // HasSaveRestoreCapability returns true if the route supports save/restore.
 func (r *Route) HasSaveRestoreCapability() bool {
-	return r.outgoingNIC.LinkEndpoint.Capabilities()&CapabilitySaveRestore != 0
+	return r.outgoingNIC.NetworkLinkEndpoint.Capabilities()&CapabilitySaveRestore != 0
 }
 
 // HasDisconncetOkCapability returns true if the route supports disconnecting.
 func (r *Route) HasDisconncetOkCapability() bool {
-	return r.outgoingNIC.LinkEndpoint.Capabilities()&CapabilityDisconnectOk != 0
+	return r.outgoingNIC.NetworkLinkEndpoint.Capabilities()&CapabilityDisconnectOk != 0
 }
 
 // GSOMaxSize returns the maximum GSO packet size.
 func (r *Route) GSOMaxSize() uint32 {
-	if gso, ok := r.outgoingNIC.LinkEndpoint.(GSOEndpoint); ok {
+	if gso, ok := r.outgoingNIC.NetworkLinkEndpoint.(GSOEndpoint); ok {
 		return gso.GSOMaxSize()
 	}
 	return 0
diff --git a/pkg/tcpip/stack/stack.go b/pkg/tcpip/stack/stack.go
index 9c8376962..f751b484f 100644
--- a/pkg/tcpip/stack/stack.go
+++ b/pkg/tcpip/stack/stack.go
@@ -670,6 +670,9 @@ type NICOptions struct {
 	// should be tracked alongside a NIC, to avoid having to keep a
 	// map[tcpip.NICID]metadata mirroring stack.Stack's nic map.
 	Context NICContext
+
+	// QDisc is the queue discipline to use for this NIC.
+	QDisc QueueingDiscipline
 }
 
 // CreateNICWithOptions creates a NIC with the provided id, LinkEndpoint, and
@@ -695,7 +698,7 @@ func (s *Stack) CreateNICWithOptions(id tcpip.NICID, ep LinkEndpoint, opts NICOp
 		}
 	}
 
-	n := newNIC(s, id, opts.Name, ep, opts.Context)
+	n := newNIC(s, id, ep, opts)
 	for proto := range s.defaultForwardingEnabled {
 		if err := n.setForwarding(proto, true); err != nil {
 			panic(fmt.Sprintf("newNIC(%d, ...).setForwarding(%d, true): %s", id, proto, err))
@@ -721,7 +724,11 @@ func (s *Stack) GetLinkEndpointByName(name string) LinkEndpoint {
 	defer s.mu.RUnlock()
 	for _, nic := range s.nics {
 		if nic.Name() == name {
-			return nic.LinkEndpoint
+			linkEP, ok := nic.NetworkLinkEndpoint.(LinkEndpoint)
+			if !ok {
+				panic(fmt.Sprintf("unexpected NetworkLinkEndpoint(%#v) is not a LinkEndpoint", nic.NetworkLinkEndpoint))
+			}
+			return linkEP
 		}
 	}
 	return nil
@@ -866,14 +873,14 @@ func (s *Stack) NICInfo() map[tcpip.NICID]NICInfo {
 
 		info := NICInfo{
 			Name:              nic.name,
-			LinkAddress:       nic.LinkEndpoint.LinkAddress(),
+			LinkAddress:       nic.NetworkLinkEndpoint.LinkAddress(),
 			ProtocolAddresses: nic.primaryAddresses(),
 			Flags:             flags,
-			MTU:               nic.LinkEndpoint.MTU(),
+			MTU:               nic.NetworkLinkEndpoint.MTU(),
 			Stats:             nic.stats.local,
 			NetworkStats:      netStats,
 			Context:           nic.context,
-			ARPHardwareType:   nic.LinkEndpoint.ARPHardwareType(),
+			ARPHardwareType:   nic.NetworkLinkEndpoint.ARPHardwareType(),
 			Forwarding:        make(map[tcpip.NetworkProtocolNumber]bool),
 		}
 
@@ -1527,7 +1534,7 @@ func (s *Stack) Wait() {
 	s.mu.RLock()
 	defer s.mu.RUnlock()
 	for _, n := range s.nics {
-		n.LinkEndpoint.Wait()
+		n.NetworkLinkEndpoint.Wait()
 	}
 }
 
diff --git a/runsc/boot/loader.go b/runsc/boot/loader.go
index 247f0d54a..64f1eda8f 100644
--- a/runsc/boot/loader.go
+++ b/runsc/boot/loader.go
@@ -1198,7 +1198,9 @@ func (f *sandboxNetstackCreator) CreateStack() (inet.Stack, error) {
 	nicID := tcpip.NICID(f.uniqueID.UniqueID())
 	link := DefaultLoopbackLink
 	linkEP := ethernet.New(loopback.New())
-	if err := n.createNICWithAddrs(nicID, link.Name, linkEP, link.Addresses); err != nil {
+	opts := stack.NICOptions{Name: link.Name}
+
+	if err := n.createNICWithAddrs(nicID, linkEP, opts, link.Addresses); err != nil {
 		return nil, err
 	}
 
diff --git a/runsc/boot/network.go b/runsc/boot/network.go
index f819cf8fb..c48b502cb 100644
--- a/runsc/boot/network.go
+++ b/runsc/boot/network.go
@@ -178,7 +178,8 @@ func (n *Network) CreateLinksAndRoutes(args *CreateLinksAndRoutesArgs, _ *struct
 		linkEP := ethernet.New(loopback.New())
 
 		log.Infof("Enabling loopback interface %q with id %d on addresses %+v", link.Name, nicID, link.Addresses)
-		if err := n.createNICWithAddrs(nicID, link.Name, linkEP, link.Addresses); err != nil {
+		opts := stack.NICOptions{Name: link.Name}
+		if err := n.createNICWithAddrs(nicID, linkEP, opts, link.Addresses); err != nil {
 			return err
 		}
 
@@ -227,15 +228,23 @@ func (n *Network) CreateLinksAndRoutes(args *CreateLinksAndRoutesArgs, _ *struct
 			return err
 		}
 
+		// Wrap linkEP in a sniffer to enable packet logging.
+		sniffEP := sniffer.New(linkEP)
+
+		var qDisc stack.QueueingDiscipline
 		switch link.QDisc {
 		case config.QDiscNone:
 		case config.QDiscFIFO:
 			log.Infof("Enabling FIFO QDisc on %q", link.Name)
-			linkEP = fifo.New(linkEP, runtime.GOMAXPROCS(0), 1000)
+			qDisc = fifo.New(sniffEP, runtime.GOMAXPROCS(0), 1000)
 		}
 
 		log.Infof("Enabling interface %q with id %d on addresses %+v (%v) w/ %d channels", link.Name, nicID, link.Addresses, mac, link.NumChannels)
-		if err := n.createNICWithAddrs(nicID, link.Name, linkEP, link.Addresses); err != nil {
+		opts := stack.NICOptions{
+			Name:  link.Name,
+			QDisc: qDisc,
+		}
+		if err := n.createNICWithAddrs(nicID, sniffEP, opts, link.Addresses); err != nil {
 			return err
 		}
 
@@ -285,9 +294,8 @@ func (n *Network) CreateLinksAndRoutes(args *CreateLinksAndRoutesArgs, _ *struct
 
 // createNICWithAddrs creates a NIC in the network stack and adds the given
 // addresses.
-func (n *Network) createNICWithAddrs(id tcpip.NICID, name string, ep stack.LinkEndpoint, addrs []IPWithPrefix) error {
-	opts := stack.NICOptions{Name: name}
-	if err := n.Stack.CreateNICWithOptions(id, sniffer.New(ep), opts); err != nil {
+func (n *Network) createNICWithAddrs(id tcpip.NICID, ep stack.LinkEndpoint, opts stack.NICOptions, addrs []IPWithPrefix) error {
+	if err := n.Stack.CreateNICWithOptions(id, ep, opts); err != nil {
 		return fmt.Errorf("CreateNICWithOptions(%d, _, %+v) failed: %v", id, opts, err)
 	}
 
diff --git a/test/benchmarks/tcp/tcp_proxy.go b/test/benchmarks/tcp/tcp_proxy.go
index 8273b12a7..308a7c3af 100644
--- a/test/benchmarks/tcp/tcp_proxy.go
+++ b/test/benchmarks/tcp/tcp_proxy.go
@@ -205,7 +205,9 @@ func newNetstackImpl(mode string) (impl, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to create FD endpoint: %v", err)
 	}
-	if err := s.CreateNIC(nicID, fifo.New(ep, runtime.GOMAXPROCS(0), 1000)); err != nil {
+	qDisc := fifo.New(ep, runtime.GOMAXPROCS(0), 1000)
+	opts := stack.NICOptions{QDisc: qDisc}
+	if err := s.CreateNICWithOptions(nicID, ep, opts); err != nil {
 		return nil, fmt.Errorf("error creating NIC %q: %v", *iface, err)
 	}
 	protocolAddr := tcpip.ProtocolAddress{