Use PacketBuffers with GSO.

PiperOrigin-RevId: 282045221
This commit is contained in:
Kevin Krakauer
2019-11-22 14:52:35 -08:00
committed by gVisor bot
parent f27f38d137
commit 9db08c4e58
17 changed files with 95 additions and 90 deletions
+6 -6
View File
@@ -133,16 +133,16 @@ func (e *Endpoint) WritePacket(_ *stack.Route, gso *stack.GSO, protocol tcpip.Ne
}
// WritePackets stores outbound packets into the channel.
func (e *Endpoint) WritePackets(_ *stack.Route, gso *stack.GSO, hdrs []stack.PacketDescriptor, payload buffer.VectorisedView, protocol tcpip.NetworkProtocolNumber) (int, *tcpip.Error) {
payloadView := payload.ToView()
func (e *Endpoint) WritePackets(_ *stack.Route, gso *stack.GSO, pkts []tcpip.PacketBuffer, protocol tcpip.NetworkProtocolNumber) (int, *tcpip.Error) {
payloadView := pkts[0].Data.ToView()
n := 0
packetLoop:
for _, hdr := range hdrs {
off := hdr.Off
size := hdr.Size
for _, pkt := range pkts {
off := pkt.DataOffset
size := pkt.DataSize
p := PacketInfo{
Pkt: tcpip.PacketBuffer{
Header: hdr.Hdr,
Header: pkt.Header,
Data: buffer.NewViewFromBytes(payloadView[off : off+size]).ToVectorisedView(),
},
Proto: protocol,
+18 -7
View File
@@ -440,7 +440,7 @@ func (e *endpoint) WritePacket(r *stack.Route, gso *stack.GSO, protocol tcpip.Ne
// WritePackets writes outbound packets to the file descriptor. If it is not
// currently writable, the packet is dropped.
func (e *endpoint) WritePackets(r *stack.Route, gso *stack.GSO, hdrs []stack.PacketDescriptor, payload buffer.VectorisedView, protocol tcpip.NetworkProtocolNumber) (int, *tcpip.Error) {
func (e *endpoint) WritePackets(r *stack.Route, gso *stack.GSO, pkts []tcpip.PacketBuffer, protocol tcpip.NetworkProtocolNumber) (int, *tcpip.Error) {
var ethHdrBuf []byte
// hdr + data
iovLen := 2
@@ -463,9 +463,9 @@ func (e *endpoint) WritePackets(r *stack.Route, gso *stack.GSO, hdrs []stack.Pac
iovLen++
}
n := len(hdrs)
n := len(pkts)
views := payload.Views()
views := pkts[0].Data.Views()
/*
* Each bondary in views can add one more iovec.
*
@@ -483,14 +483,20 @@ func (e *endpoint) WritePackets(r *stack.Route, gso *stack.GSO, hdrs []stack.Pac
viewOff := 0
off := 0
nextOff := 0
for i := range hdrs {
for i := range pkts {
// TODO(b/134618279): Different packets may have different data
// in the future. We should handle this.
if !viewsEqual(pkts[i].Data.Views(), views) {
panic("All packets in pkts should have the same Data.")
}
prevIovecIdx := iovecIdx
mmsgHdr := &mmsgHdrs[i]
mmsgHdr.Msg.Iov = &iovec[iovecIdx]
packetSize := hdrs[i].Size
hdr := &hdrs[i].Hdr
packetSize := pkts[i].DataSize
hdr := &pkts[i].Header
off = hdrs[i].Off
off = pkts[i].DataOffset
if off != nextOff {
// We stop in a different point last time.
size := packetSize
@@ -555,6 +561,11 @@ func (e *endpoint) WritePackets(r *stack.Route, gso *stack.GSO, hdrs []stack.Pac
return packets, nil
}
// viewsEqual tests whether v1 and v2 refer to the same backing bytes.
func viewsEqual(vs1, vs2 []buffer.View) bool {
return len(vs1) == len(vs2) && (len(vs1) == 0 || &vs1[0] == &vs2[0])
}
// WriteRawPacket implements stack.LinkEndpoint.WriteRawPacket.
func (e *endpoint) WriteRawPacket(vv buffer.VectorisedView) *tcpip.Error {
return rawfile.NonBlockingWrite(e.fds[0], vv.ToView())
+1 -1
View File
@@ -92,7 +92,7 @@ func (e *endpoint) WritePacket(_ *stack.Route, _ *stack.GSO, protocol tcpip.Netw
}
// WritePackets implements stack.LinkEndpoint.WritePackets.
func (e *endpoint) WritePackets(_ *stack.Route, _ *stack.GSO, hdrs []stack.PacketDescriptor, payload buffer.VectorisedView, protocol tcpip.NetworkProtocolNumber) (int, *tcpip.Error) {
func (e *endpoint) WritePackets(*stack.Route, *stack.GSO, []tcpip.PacketBuffer, tcpip.NetworkProtocolNumber) (int, *tcpip.Error) {
panic("not implemented")
}
+2 -2
View File
@@ -87,12 +87,12 @@ func (m *InjectableEndpoint) InjectInbound(protocol tcpip.NetworkProtocolNumber,
// WritePackets writes outbound packets to the appropriate
// LinkInjectableEndpoint based on the RemoteAddress. HandleLocal only works if
// r.RemoteAddress has a route registered in this endpoint.
func (m *InjectableEndpoint) WritePackets(r *stack.Route, gso *stack.GSO, hdrs []stack.PacketDescriptor, payload buffer.VectorisedView, protocol tcpip.NetworkProtocolNumber) (int, *tcpip.Error) {
func (m *InjectableEndpoint) WritePackets(r *stack.Route, gso *stack.GSO, pkts []tcpip.PacketBuffer, protocol tcpip.NetworkProtocolNumber) (int, *tcpip.Error) {
endpoint, ok := m.routes[r.RemoteAddress]
if !ok {
return 0, tcpip.ErrNoRoute
}
return endpoint.WritePackets(r, gso, hdrs, payload, protocol)
return endpoint.WritePackets(r, gso, pkts, protocol)
}
// WritePacket writes outbound packets to the appropriate LinkInjectableEndpoint
+1 -1
View File
@@ -214,7 +214,7 @@ func (e *endpoint) WritePacket(r *stack.Route, _ *stack.GSO, protocol tcpip.Netw
}
// WritePackets implements stack.LinkEndpoint.WritePackets.
func (e *endpoint) WritePackets(r *stack.Route, _ *stack.GSO, hdrs []stack.PacketDescriptor, payload buffer.VectorisedView, protocol tcpip.NetworkProtocolNumber) (int, *tcpip.Error) {
func (e *endpoint) WritePackets(r *stack.Route, _ *stack.GSO, pkts []tcpip.PacketBuffer, protocol tcpip.NetworkProtocolNumber) (int, *tcpip.Error) {
panic("not implemented")
}
+6 -6
View File
@@ -233,15 +233,15 @@ func (e *endpoint) WritePacket(r *stack.Route, gso *stack.GSO, protocol tcpip.Ne
// WritePackets implements the stack.LinkEndpoint interface. It is called by
// higher-level protocols to write packets; it just logs the packet and
// forwards the request to the lower endpoint.
func (e *endpoint) WritePackets(r *stack.Route, gso *stack.GSO, hdrs []stack.PacketDescriptor, payload buffer.VectorisedView, protocol tcpip.NetworkProtocolNumber) (int, *tcpip.Error) {
view := payload.ToView()
for _, d := range hdrs {
func (e *endpoint) WritePackets(r *stack.Route, gso *stack.GSO, pkts []tcpip.PacketBuffer, protocol tcpip.NetworkProtocolNumber) (int, *tcpip.Error) {
view := pkts[0].Data.ToView()
for _, pkt := range pkts {
e.dumpPacket(gso, protocol, tcpip.PacketBuffer{
Header: d.Hdr,
Data: view[d.Off:][:d.Size].ToVectorisedView(),
Header: pkt.Header,
Data: view[pkt.DataOffset:][:pkt.DataSize].ToVectorisedView(),
})
}
return e.lower.WritePackets(r, gso, hdrs, payload, protocol)
return e.lower.WritePackets(r, gso, pkts, protocol)
}
// WriteRawPacket implements stack.LinkEndpoint.WriteRawPacket.
+3 -3
View File
@@ -112,12 +112,12 @@ func (e *Endpoint) WritePacket(r *stack.Route, gso *stack.GSO, protocol tcpip.Ne
// WritePackets implements stack.LinkEndpoint.WritePackets. It is called by
// higher-level protocols to write packets. It only forwards packets to the
// lower endpoint if Wait or WaitWrite haven't been called.
func (e *Endpoint) WritePackets(r *stack.Route, gso *stack.GSO, hdrs []stack.PacketDescriptor, payload buffer.VectorisedView, protocol tcpip.NetworkProtocolNumber) (int, *tcpip.Error) {
func (e *Endpoint) WritePackets(r *stack.Route, gso *stack.GSO, pkts []tcpip.PacketBuffer, protocol tcpip.NetworkProtocolNumber) (int, *tcpip.Error) {
if !e.writeGate.Enter() {
return len(hdrs), nil
return len(pkts), nil
}
n, err := e.lower.WritePackets(r, gso, hdrs, payload, protocol)
n, err := e.lower.WritePackets(r, gso, pkts, protocol)
e.writeGate.Leave()
return n, err
}
+3 -3
View File
@@ -71,9 +71,9 @@ func (e *countedEndpoint) WritePacket(r *stack.Route, _ *stack.GSO, protocol tcp
}
// WritePackets implements stack.LinkEndpoint.WritePackets.
func (e *countedEndpoint) WritePackets(r *stack.Route, _ *stack.GSO, hdrs []stack.PacketDescriptor, payload buffer.VectorisedView, protocol tcpip.NetworkProtocolNumber) (int, *tcpip.Error) {
e.writeCount += len(hdrs)
return len(hdrs), nil
func (e *countedEndpoint) WritePackets(r *stack.Route, _ *stack.GSO, pkts []tcpip.PacketBuffer, protocol tcpip.NetworkProtocolNumber) (int, *tcpip.Error) {
e.writeCount += len(pkts)
return len(pkts), nil
}
func (e *countedEndpoint) WriteRawPacket(buffer.VectorisedView) *tcpip.Error {
+1 -1
View File
@@ -84,7 +84,7 @@ func (e *endpoint) WritePacket(*stack.Route, *stack.GSO, stack.NetworkHeaderPara
}
// WritePackets implements stack.NetworkEndpoint.WritePackets.
func (e *endpoint) WritePackets(*stack.Route, *stack.GSO, []stack.PacketDescriptor, buffer.VectorisedView, stack.NetworkHeaderParams, stack.PacketLooping) (int, *tcpip.Error) {
func (e *endpoint) WritePackets(*stack.Route, *stack.GSO, []tcpip.PacketBuffer, stack.NetworkHeaderParams, stack.PacketLooping) (int, *tcpip.Error) {
return 0, tcpip.ErrNotSupported
}
+1 -1
View File
@@ -172,7 +172,7 @@ func (t *testObject) WritePacket(_ *stack.Route, _ *stack.GSO, protocol tcpip.Ne
}
// WritePackets implements stack.LinkEndpoint.WritePackets.
func (t *testObject) WritePackets(_ *stack.Route, _ *stack.GSO, hdr []stack.PacketDescriptor, payload buffer.VectorisedView, protocol tcpip.NetworkProtocolNumber) (int, *tcpip.Error) {
func (t *testObject) WritePackets(_ *stack.Route, _ *stack.GSO, pkt []tcpip.PacketBuffer, protocol tcpip.NetworkProtocolNumber) (int, *tcpip.Error) {
panic("not implemented")
}
+5 -5
View File
@@ -268,18 +268,18 @@ func (e *endpoint) WritePacket(r *stack.Route, gso *stack.GSO, params stack.Netw
}
// WritePackets implements stack.NetworkEndpoint.WritePackets.
func (e *endpoint) WritePackets(r *stack.Route, gso *stack.GSO, hdrs []stack.PacketDescriptor, payload buffer.VectorisedView, params stack.NetworkHeaderParams, loop stack.PacketLooping) (int, *tcpip.Error) {
func (e *endpoint) WritePackets(r *stack.Route, gso *stack.GSO, pkts []tcpip.PacketBuffer, params stack.NetworkHeaderParams, loop stack.PacketLooping) (int, *tcpip.Error) {
if loop&stack.PacketLoop != 0 {
panic("multiple packets in local loop")
}
if loop&stack.PacketOut == 0 {
return len(hdrs), nil
return len(pkts), nil
}
for i := range hdrs {
e.addIPHeader(r, &hdrs[i].Hdr, hdrs[i].Size, params)
for i := range pkts {
e.addIPHeader(r, &pkts[i].Header, pkts[i].DataSize, params)
}
n, err := e.linkEP.WritePackets(r, gso, hdrs, payload, ProtocolNumber)
n, err := e.linkEP.WritePackets(r, gso, pkts, ProtocolNumber)
r.Stats().IP.PacketsSent.IncrementBy(uint64(n))
return n, err
}
+6 -6
View File
@@ -137,21 +137,21 @@ func (e *endpoint) WritePacket(r *stack.Route, gso *stack.GSO, params stack.Netw
}
// WritePackets implements stack.LinkEndpoint.WritePackets.
func (e *endpoint) WritePackets(r *stack.Route, gso *stack.GSO, hdrs []stack.PacketDescriptor, payload buffer.VectorisedView, params stack.NetworkHeaderParams, loop stack.PacketLooping) (int, *tcpip.Error) {
func (e *endpoint) WritePackets(r *stack.Route, gso *stack.GSO, pkts []tcpip.PacketBuffer, params stack.NetworkHeaderParams, loop stack.PacketLooping) (int, *tcpip.Error) {
if loop&stack.PacketLoop != 0 {
panic("not implemented")
}
if loop&stack.PacketOut == 0 {
return len(hdrs), nil
return len(pkts), nil
}
for i := range hdrs {
hdr := &hdrs[i].Hdr
size := hdrs[i].Size
for i := range pkts {
hdr := &pkts[i].Header
size := pkts[i].DataSize
e.addIPHeader(r, hdr, size, params)
}
n, err := e.linkEP.WritePackets(r, gso, hdrs, payload, ProtocolNumber)
n, err := e.linkEP.WritePackets(r, gso, pkts, ProtocolNumber)
r.Stats().IP.PacketsSent.IncrementBy(uint64(n))
return n, err
}
+8
View File
@@ -31,6 +31,14 @@ type PacketBuffer struct {
// or otherwise modified.
Data buffer.VectorisedView
// DataOffset is used for GSO output. It is the offset into the Data
// field where the payload of this packet starts.
DataOffset int
// DataSize is used for GSO output. It is the size of this packet's
// payload.
DataSize int
// Header holds the headers of outbound packets. As a packet is passed
// down the stack, each layer adds to Header.
Header buffer.Prependable
+4 -4
View File
@@ -237,8 +237,8 @@ type NetworkEndpoint interface {
WritePacket(r *Route, gso *GSO, params NetworkHeaderParams, loop PacketLooping, pkt tcpip.PacketBuffer) *tcpip.Error
// WritePackets writes packets to the given destination address and
// protocol.
WritePackets(r *Route, gso *GSO, hdrs []PacketDescriptor, payload buffer.VectorisedView, params NetworkHeaderParams, loop PacketLooping) (int, *tcpip.Error)
// protocol. pkts must not be zero length.
WritePackets(r *Route, gso *GSO, pkts []tcpip.PacketBuffer, params NetworkHeaderParams, loop PacketLooping) (int, *tcpip.Error)
// WriteHeaderIncludedPacket writes a packet that includes a network
// header to the given destination address.
@@ -373,12 +373,12 @@ type LinkEndpoint interface {
WritePacket(r *Route, gso *GSO, protocol tcpip.NetworkProtocolNumber, pkt tcpip.PacketBuffer) *tcpip.Error
// WritePackets writes packets with the given protocol through the
// given route.
// given route. pkts must not be zero length.
//
// Right now, WritePackets is used only when the software segmentation
// offload is enabled. If it will be used for something else, it may
// require to change syscall filters.
WritePackets(r *Route, gso *GSO, hdrs []PacketDescriptor, payload buffer.VectorisedView, protocol tcpip.NetworkProtocolNumber) (int, *tcpip.Error)
WritePackets(r *Route, gso *GSO, pkts []tcpip.PacketBuffer, protocol tcpip.NetworkProtocolNumber) (int, *tcpip.Error)
// WriteRawPacket writes a packet directly to the link. The packet
// should already have an ethernet header.
+5 -24
View File
@@ -17,7 +17,6 @@ package stack
import (
"gvisor.dev/gvisor/pkg/sleep"
"gvisor.dev/gvisor/pkg/tcpip"
"gvisor.dev/gvisor/pkg/tcpip/buffer"
"gvisor.dev/gvisor/pkg/tcpip/header"
)
@@ -169,39 +168,21 @@ func (r *Route) WritePacket(gso *GSO, params NetworkHeaderParams, pkt tcpip.Pack
return err
}
// PacketDescriptor is a packet descriptor which contains a packet header and
// offset and size of packet data in a payload view.
type PacketDescriptor struct {
Hdr buffer.Prependable
Off int
Size int
}
// NewPacketDescriptors allocates a set of packet descriptors.
func NewPacketDescriptors(n int, hdrSize int) []PacketDescriptor {
buf := make([]byte, n*hdrSize)
hdrs := make([]PacketDescriptor, n)
for i := range hdrs {
hdrs[i].Hdr = buffer.NewEmptyPrependableFromView(buf[i*hdrSize:][:hdrSize])
}
return hdrs
}
// WritePackets writes the set of packets through the given route.
func (r *Route) WritePackets(gso *GSO, hdrs []PacketDescriptor, payload buffer.VectorisedView, params NetworkHeaderParams) (int, *tcpip.Error) {
func (r *Route) WritePackets(gso *GSO, pkts []tcpip.PacketBuffer, params NetworkHeaderParams) (int, *tcpip.Error) {
if !r.ref.isValidForOutgoing() {
return 0, tcpip.ErrInvalidEndpointState
}
n, err := r.ref.ep.WritePackets(r, gso, hdrs, payload, params, r.Loop)
n, err := r.ref.ep.WritePackets(r, gso, pkts, params, r.Loop)
if err != nil {
r.Stats().IP.OutgoingPacketErrors.IncrementBy(uint64(len(hdrs) - n))
r.Stats().IP.OutgoingPacketErrors.IncrementBy(uint64(len(pkts) - n))
}
r.ref.nic.stats.Tx.Packets.IncrementBy(uint64(n))
payloadSize := 0
for i := 0; i < n; i++ {
r.ref.nic.stats.Tx.Bytes.IncrementBy(uint64(hdrs[i].Hdr.UsedLength()))
payloadSize += hdrs[i].Size
r.ref.nic.stats.Tx.Bytes.IncrementBy(uint64(pkts[i].Header.UsedLength()))
payloadSize += pkts[i].DataSize
}
r.ref.nic.stats.Tx.Bytes.IncrementBy(uint64(payloadSize))
return n, err
+1 -1
View File
@@ -149,7 +149,7 @@ func (f *fakeNetworkEndpoint) WritePacket(r *stack.Route, gso *stack.GSO, params
}
// WritePackets implements stack.LinkEndpoint.WritePackets.
func (f *fakeNetworkEndpoint) WritePackets(r *stack.Route, gso *stack.GSO, hdrs []stack.PacketDescriptor, payload buffer.VectorisedView, params stack.NetworkHeaderParams, loop stack.PacketLooping) (int, *tcpip.Error) {
func (f *fakeNetworkEndpoint) WritePackets(r *stack.Route, gso *stack.GSO, pkts []tcpip.PacketBuffer, params stack.NetworkHeaderParams, loop stack.PacketLooping) (int, *tcpip.Error) {
panic("not implemented")
}
+24 -19
View File
@@ -631,11 +631,11 @@ func (e *endpoint) sendTCP(r *stack.Route, id stack.TransportEndpointID, data bu
return nil
}
func buildTCPHdr(r *stack.Route, id stack.TransportEndpointID, d *stack.PacketDescriptor, data buffer.VectorisedView, flags byte, seq, ack seqnum.Value, rcvWnd seqnum.Size, opts []byte, gso *stack.GSO) {
func buildTCPHdr(r *stack.Route, id stack.TransportEndpointID, pkt *tcpip.PacketBuffer, flags byte, seq, ack seqnum.Value, rcvWnd seqnum.Size, opts []byte, gso *stack.GSO) {
optLen := len(opts)
hdr := &d.Hdr
packetSize := d.Size
off := d.Off
hdr := &pkt.Header
packetSize := pkt.DataSize
off := pkt.DataOffset
// Initialize the header.
tcp := header.TCP(hdr.Prepend(header.TCPMinimumSize + optLen))
tcp.Encode(&header.TCPFields{
@@ -659,7 +659,7 @@ func buildTCPHdr(r *stack.Route, id stack.TransportEndpointID, d *stack.PacketDe
// header and data and get the right sum of the TCP packet.
tcp.SetChecksum(xsum)
} else if r.Capabilities()&stack.CapabilityTXChecksumOffload == 0 {
xsum = header.ChecksumVVWithOffset(data, xsum, off, packetSize)
xsum = header.ChecksumVVWithOffset(pkt.Data, xsum, off, packetSize)
tcp.SetChecksum(^tcp.CalculateChecksum(xsum))
}
@@ -674,7 +674,13 @@ func sendTCPBatch(r *stack.Route, id stack.TransportEndpointID, data buffer.Vect
mss := int(gso.MSS)
n := (data.Size() + mss - 1) / mss
hdrs := stack.NewPacketDescriptors(n, header.TCPMinimumSize+int(r.MaxHeaderLength())+optLen)
// Allocate one big slice for all the headers.
hdrSize := header.TCPMinimumSize + int(r.MaxHeaderLength()) + optLen
buf := make([]byte, n*hdrSize)
pkts := make([]tcpip.PacketBuffer, n)
for i := range pkts {
pkts[i].Header = buffer.NewEmptyPrependableFromView(buf[i*hdrSize:][:hdrSize])
}
size := data.Size()
off := 0
@@ -684,16 +690,17 @@ func sendTCPBatch(r *stack.Route, id stack.TransportEndpointID, data buffer.Vect
packetSize = size
}
size -= packetSize
hdrs[i].Off = off
hdrs[i].Size = packetSize
buildTCPHdr(r, id, &hdrs[i], data, flags, seq, ack, rcvWnd, opts, gso)
pkts[i].DataOffset = off
pkts[i].DataSize = packetSize
pkts[i].Data = data
buildTCPHdr(r, id, &pkts[i], flags, seq, ack, rcvWnd, opts, gso)
off += packetSize
seq = seq.Add(seqnum.Size(packetSize))
}
if ttl == 0 {
ttl = r.DefaultTTL()
}
sent, err := r.WritePackets(gso, hdrs, data, stack.NetworkHeaderParams{Protocol: ProtocolNumber, TTL: ttl, TOS: tos})
sent, err := r.WritePackets(gso, pkts, stack.NetworkHeaderParams{Protocol: ProtocolNumber, TTL: ttl, TOS: tos})
if err != nil {
r.Stats().TCP.SegmentSendErrors.IncrementBy(uint64(n - sent))
}
@@ -713,20 +720,18 @@ func sendTCP(r *stack.Route, id stack.TransportEndpointID, data buffer.Vectorise
return sendTCPBatch(r, id, data, ttl, tos, flags, seq, ack, rcvWnd, opts, gso)
}
d := &stack.PacketDescriptor{
Hdr: buffer.NewPrependable(header.TCPMinimumSize + int(r.MaxHeaderLength()) + optLen),
Off: 0,
Size: data.Size(),
pkt := tcpip.PacketBuffer{
Header: buffer.NewPrependable(header.TCPMinimumSize + int(r.MaxHeaderLength()) + optLen),
DataOffset: 0,
DataSize: data.Size(),
Data: data,
}
buildTCPHdr(r, id, d, data, flags, seq, ack, rcvWnd, opts, gso)
buildTCPHdr(r, id, &pkt, flags, seq, ack, rcvWnd, opts, gso)
if ttl == 0 {
ttl = r.DefaultTTL()
}
if err := r.WritePacket(gso, stack.NetworkHeaderParams{Protocol: ProtocolNumber, TTL: ttl, TOS: tos}, tcpip.PacketBuffer{
Header: d.Hdr,
Data: data,
}); err != nil {
if err := r.WritePacket(gso, stack.NetworkHeaderParams{Protocol: ProtocolNumber, TTL: ttl, TOS: tos}, pkt); err != nil {
r.Stats().TCP.SegmentSendErrors.Increment()
return err
}