diff --git a/pkg/sentry/fsimpl/sys/BUILD b/pkg/sentry/fsimpl/sys/BUILD index fa57122ec..2c6986357 100644 --- a/pkg/sentry/fsimpl/sys/BUILD +++ b/pkg/sentry/fsimpl/sys/BUILD @@ -21,7 +21,6 @@ go_library( srcs = [ "dir_refs.go", "kcov.go", - "net.go", "pci.go", "sys.go", ], diff --git a/pkg/sentry/fsimpl/sys/net.go b/pkg/sentry/fsimpl/sys/net.go deleted file mode 100644 index d48f0f31e..000000000 --- a/pkg/sentry/fsimpl/sys/net.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2022 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sys - -import ( - "bytes" - "fmt" - "time" - - "gvisor.dev/gvisor/pkg/abi/linux" - "gvisor.dev/gvisor/pkg/context" - "gvisor.dev/gvisor/pkg/sentry/fsimpl/kernfs" - "gvisor.dev/gvisor/pkg/sentry/inet" - "gvisor.dev/gvisor/pkg/sentry/kernel/auth" - "gvisor.dev/gvisor/pkg/sentry/vfs" - "gvisor.dev/gvisor/pkg/usermem" -) - -// newNetDir returns a directory containing a subdirectory for each network -// interface. -func (fs *filesystem) newNetDir(ctx context.Context, creds *auth.Credentials, mode linux.FileMode) map[string]kernfs.Inode { - // Get list of interfaces. - stk := inet.StackFromContext(ctx) - if stk == nil { - return map[string]kernfs.Inode{} - } - - subDirs := make(map[string]kernfs.Inode) - for idx, iface := range stk.Interfaces() { - subDirs[iface.Name] = fs.newIfaceDir(ctx, creds, mode, idx, stk) - } - return subDirs -} - -// newIfaceDir returns a directory containing per-interface files. -func (fs *filesystem) newIfaceDir(ctx context.Context, creds *auth.Credentials, mode linux.FileMode, idx int32, stk inet.Stack) kernfs.Inode { - files := map[string]kernfs.Inode{ - "gro_flush_timeout": fs.newGROTimeoutFile(ctx, creds, mode, idx, stk), - } - return fs.newDir(ctx, creds, mode, files) -} - -// groTimeoutFile enables the reading and writing of the GRO timeout. -// -// +stateify savable -type groTimeoutFile struct { - implStatFS - kernfs.DynamicBytesFile - - idx int32 - stk inet.Stack -} - -// newGROTimeoutFile returns a file that can be used to read and set the GRO -// timeout. -func (fs *filesystem) newGROTimeoutFile(ctx context.Context, creds *auth.Credentials, mode linux.FileMode, idx int32, stk inet.Stack) kernfs.Inode { - file := groTimeoutFile{idx: idx, stk: stk} - file.DynamicBytesFile.Init(ctx, creds, linux.UNNAMED_MAJOR, fs.devMinor, fs.NextIno(), &file, mode) - return &file -} - -// Generate implements vfs.DynamicBytesSource.Generate. -func (gf *groTimeoutFile) Generate(ctx context.Context, buf *bytes.Buffer) error { - timeout, err := gf.stk.GROTimeout(gf.idx) - if err != nil { - return err - } - fmt.Fprintf(buf, "%d\n", timeout.Nanoseconds()) - return nil -} - -// Write implements vfs.WritableDynamicBytesSource.Write. -func (gf *groTimeoutFile) Write(ctx context.Context, _ *vfs.FileDescription, src usermem.IOSequence, offset int64) (int64, error) { - val := []int32{0} - nRead, err := usermem.CopyInt32StringsInVec(ctx, src.IO, src.Addrs, val, src.Opts) - if err != nil { - return 0, err - } - gf.stk.SetGROTimeout(gf.idx, time.Duration(val[0])*time.Nanosecond) - return nRead, nil -} diff --git a/pkg/sentry/fsimpl/sys/sys.go b/pkg/sentry/fsimpl/sys/sys.go index dd9c1636d..161e282d4 100644 --- a/pkg/sentry/fsimpl/sys/sys.go +++ b/pkg/sentry/fsimpl/sys/sys.go @@ -118,7 +118,6 @@ func (fsType FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt classSub := map[string]kernfs.Inode{ "power_supply": fs.newDir(ctx, creds, defaultSysDirMode, nil), - "net": fs.newDir(ctx, creds, defaultSysDirMode, fs.newNetDir(ctx, creds, defaultSysDirMode)), } devicesSub := map[string]kernfs.Inode{ "system": fs.newDir(ctx, creds, defaultSysDirMode, map[string]kernfs.Inode{ diff --git a/pkg/sentry/inet/inet.go b/pkg/sentry/inet/inet.go index 9582ddca2..71909e761 100644 --- a/pkg/sentry/inet/inet.go +++ b/pkg/sentry/inet/inet.go @@ -16,8 +16,6 @@ package inet import ( - "time" - "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/tcpip" "gvisor.dev/gvisor/pkg/tcpip/stack" @@ -113,12 +111,6 @@ type Stack interface { // SetPortRange sets the UDP and TCP IPv4 and IPv6 ephemeral port range // (inclusive). SetPortRange(start uint16, end uint16) error - - // GROTimeout returns the GRO timeout. - GROTimeout(NICID int32) (time.Duration, error) - - // GROTimeout sets the GRO timeout. - SetGROTimeout(NICID int32, timeout time.Duration) error } // Interface contains information about a network interface. diff --git a/pkg/sentry/socket/hostinet/stack.go b/pkg/sentry/socket/hostinet/stack.go index f98150eca..028e507c8 100644 --- a/pkg/sentry/socket/hostinet/stack.go +++ b/pkg/sentry/socket/hostinet/stack.go @@ -22,7 +22,6 @@ import ( "reflect" "strconv" "strings" - "time" "gvisor.dev/gvisor/pkg/context" "gvisor.dev/gvisor/pkg/errors/linuxerr" @@ -356,14 +355,3 @@ func (*Stack) PortRange() (uint16, uint16) { func (*Stack) SetPortRange(uint16, uint16) error { return linuxerr.EACCES } - -// GROTimeout implements inet.Stack.GROTimeout. -func (s *Stack) GROTimeout(NICID int32) (time.Duration, error) { - return 0, nil -} - -// SetGROTimeout implements inet.Stack.SetGROTimeout. -func (s *Stack) SetGROTimeout(NICID int32, timeout time.Duration) error { - // We don't support setting the hostinet GRO timeout. - return linuxerr.EINVAL -} diff --git a/pkg/sentry/socket/netstack/stack.go b/pkg/sentry/socket/netstack/stack.go index a82b7b2a7..577b43263 100644 --- a/pkg/sentry/socket/netstack/stack.go +++ b/pkg/sentry/socket/netstack/stack.go @@ -16,7 +16,6 @@ package netstack import ( "fmt" - "time" "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/errors/linuxerr" @@ -516,14 +515,3 @@ func (s *Stack) PortRange() (uint16, uint16) { func (s *Stack) SetPortRange(start uint16, end uint16) error { return syserr.TranslateNetstackError(s.Stack.SetPortRange(start, end)).ToError() } - -// GROTimeout implements inet.Stack.GROTimeout. -func (s *Stack) GROTimeout(nicID int32) (time.Duration, error) { - timeout, err := s.Stack.GROTimeout(tcpip.NICID(nicID)) - return timeout, syserr.TranslateNetstackError(err).ToError() -} - -// SetGROTimeout implements inet.Stack.SetGROTimeout. -func (s *Stack) SetGROTimeout(nicID int32, timeout time.Duration) error { - return syserr.TranslateNetstackError(s.Stack.SetGROTimeout(tcpip.NICID(nicID), timeout)).ToError() -} diff --git a/pkg/tcpip/link/fdbased/endpoint.go b/pkg/tcpip/link/fdbased/endpoint.go index 9a51c8ef5..1e0b9f0cd 100644 --- a/pkg/tcpip/link/fdbased/endpoint.go +++ b/pkg/tcpip/link/fdbased/endpoint.go @@ -228,6 +228,9 @@ type Options struct { // InterfaceIndex is the interface index of the underlying device. InterfaceIndex int + + // GRO enables generic receive offload. + GRO bool } // fanoutID is used for AF_PACKET based endpoints to enable PACKET_FANOUT diff --git a/pkg/tcpip/link/xdp/endpoint.go b/pkg/tcpip/link/xdp/endpoint.go index d0fd88830..8a7891de5 100644 --- a/pkg/tcpip/link/xdp/endpoint.go +++ b/pkg/tcpip/link/xdp/endpoint.go @@ -102,6 +102,9 @@ type Options struct { // Bind is true when we're responsible for binding the AF_XDP socket to // a device. When false, another process is expected to bind for us. Bind bool + + // GRO enables generic receive offload. + GRO bool } // New creates a new endpoint from an AF_XDP socket. diff --git a/pkg/tcpip/stack/nic.go b/pkg/tcpip/stack/nic.go index 39028f9e8..55a4e76f4 100644 --- a/pkg/tcpip/stack/nic.go +++ b/pkg/tcpip/stack/nic.go @@ -78,8 +78,6 @@ type nic struct { qDisc QueueingDiscipline - gro groDispatcher - // deliverLinkPackets specifies whether this NIC delivers packets to // packet sockets. It is immutable. // @@ -210,7 +208,6 @@ func newNIC(stack *Stack, id tcpip.NICID, ep LinkEndpoint, opts NICOptions) *nic } } - nic.gro.init(opts.GROTimeout) nic.NetworkLinkEndpoint.Attach(nic) return nic @@ -317,9 +314,6 @@ func (n *nic) remove() tcpip.Error { n.enableDisableMu.Unlock() - // Shutdown GRO. - n.gro.close() - // Drain and drop any packets pending link resolution. // We must not hold n.enableDisableMu here. n.linkResQueue.cancel() @@ -752,7 +746,7 @@ func (n *nic) DeliverNetworkPacket(protocol tcpip.NetworkProtocolNumber, pkt *Pa n.DeliverLinkPacket(protocol, pkt) } - n.gro.dispatch(pkt, protocol, networkEndpoint) + networkEndpoint.HandlePacket(pkt) } func (n *nic) DeliverLinkPacket(protocol tcpip.NetworkProtocolNumber, pkt *PacketBuffer) { diff --git a/pkg/tcpip/stack/stack.go b/pkg/tcpip/stack/stack.go index ee8bca2db..825d0bff3 100644 --- a/pkg/tcpip/stack/stack.go +++ b/pkg/tcpip/stack/stack.go @@ -731,33 +731,6 @@ func (s *Stack) SetPortRange(start uint16, end uint16) tcpip.Error { return s.PortManager.SetPortRange(start, end) } -// GROTimeout returns the GRO timeout. -func (s *Stack) GROTimeout(nicID tcpip.NICID) (time.Duration, tcpip.Error) { - s.mu.RLock() - defer s.mu.RUnlock() - - nic, ok := s.nics[nicID] - if !ok { - return 0, &tcpip.ErrUnknownNICID{} - } - - return nic.gro.getInterval(), nil -} - -// SetGROTimeout sets the GRO timeout. -func (s *Stack) SetGROTimeout(nicID tcpip.NICID, timeout time.Duration) tcpip.Error { - s.mu.RLock() - defer s.mu.RUnlock() - - nic, ok := s.nics[nicID] - if !ok { - return &tcpip.ErrUnknownNICID{} - } - - nic.gro.setInterval(timeout) - return nil -} - // SetRouteTable assigns the route table to be used by this stack. It // specifies which NIC to use for given destination address ranges. // @@ -859,9 +832,6 @@ type NICOptions struct { // QDisc is the queue discipline to use for this NIC. QDisc QueueingDiscipline - // GROTimeout specifies the GRO timeout. Zero bypasses GRO. - GROTimeout time.Duration - // DeliverLinkPackets specifies whether the NIC is responsible for // delivering raw packets to packet sockets. DeliverLinkPackets bool diff --git a/runsc/boot/network.go b/runsc/boot/network.go index b60c7784e..46a8e04d3 100644 --- a/runsc/boot/network.go +++ b/runsc/boot/network.go @@ -21,7 +21,6 @@ import ( "os" "runtime" "strings" - "time" "golang.org/x/sys/unix" "gvisor.dev/gvisor/pkg/hostos" @@ -100,7 +99,7 @@ type FDBasedLink struct { Routes []Route GSOMaxSize uint32 GvisorGSOEnabled bool - GvisorGROTimeout time.Duration + GvisorGRO bool TXChecksumOffload bool RXChecksumOffload bool LinkAddress net.HardwareAddr @@ -136,7 +135,7 @@ type XDPLink struct { LinkAddress net.HardwareAddr QDisc config.QueueingDiscipline Neighbors []Neighbor - GvisorGROTimeout time.Duration + GvisorGRO bool Bind BindOpt // NumChannels controls how many underlying FDs are to be used to @@ -146,10 +145,10 @@ type XDPLink struct { // LoopbackLink configures a loopback link. type LoopbackLink struct { - Name string - Addresses []IPWithPrefix - Routes []Route - GvisorGROTimeout time.Duration + Name string + Addresses []IPWithPrefix + Routes []Route + GvisorGRO bool } // CreateLinksAndRoutesArgs are arguments to CreateLinkAndRoutes. @@ -256,7 +255,6 @@ func (n *Network) CreateLinksAndRoutes(args *CreateLinksAndRoutesArgs, _ *struct log.Infof("Enabling loopback interface %q with id %d on addresses %+v", link.Name, nicID, link.Addresses) opts := stack.NICOptions{ Name: link.Name, - GROTimeout: link.GvisorGROTimeout, DeliverLinkPackets: true, } if err := n.createNICWithAddrs(nicID, linkEP, opts, link.Addresses); err != nil { @@ -317,6 +315,7 @@ func (n *Network) CreateLinksAndRoutes(args *CreateLinksAndRoutesArgs, _ *struct GvisorGSOEnabled: link.GvisorGSOEnabled, TXChecksumOffload: link.TXChecksumOffload, RXChecksumOffload: link.RXChecksumOffload, + GRO: link.GvisorGRO, }) if err != nil { return err @@ -350,7 +349,6 @@ func (n *Network) CreateLinksAndRoutes(args *CreateLinksAndRoutesArgs, _ *struct opts := stack.NICOptions{ Name: link.Name, QDisc: qDisc, - GROTimeout: link.GvisorGROTimeout, DeliverLinkPackets: true, } if err := n.createNICWithAddrs(nicID, linkEP, opts, link.Addresses); err != nil { @@ -410,6 +408,7 @@ func (n *Network) CreateLinksAndRoutes(args *CreateLinksAndRoutesArgs, _ *struct RXChecksumOffload: link.RXChecksumOffload, InterfaceIndex: link.InterfaceIndex, Bind: link.Bind == BindSentry, + GRO: link.GvisorGRO, }) if err != nil { return err @@ -442,7 +441,6 @@ func (n *Network) CreateLinksAndRoutes(args *CreateLinksAndRoutesArgs, _ *struct opts := stack.NICOptions{ Name: link.Name, QDisc: qDisc, - GROTimeout: link.GvisorGROTimeout, DeliverLinkPackets: true, } if err := n.createNICWithAddrs(nicID, linkEP, opts, link.Addresses); err != nil { diff --git a/runsc/config/config.go b/runsc/config/config.go index d97ba3aab..c0cf4a02c 100644 --- a/runsc/config/config.go +++ b/runsc/config/config.go @@ -24,7 +24,6 @@ import ( "runtime" "strconv" "strings" - "time" "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/refs" @@ -126,9 +125,8 @@ type Config struct { // retains its old name of "software" GSO for API consistency. GvisorGSO bool `flag:"software-gso"` - // GvisorGROTimeout sets gVisor's generic receive offload timeout. Zero - // bypasses GRO. - GvisorGROTimeout time.Duration `flag:"gvisor-gro"` + // GvisorGRO enables gVisor's generic receive offload. + GvisorGRO bool `flag:"gvisor-gro"` // TXChecksumOffload indicates that TX Checksum Offload is enabled. TXChecksumOffload bool `flag:"tx-checksum-offload"` diff --git a/runsc/config/flags.go b/runsc/config/flags.go index fda4c44bc..e44d1d30c 100644 --- a/runsc/config/flags.go +++ b/runsc/config/flags.go @@ -114,7 +114,7 @@ func RegisterFlags(flagSet *flag.FlagSet) { flagSet.Bool("net-raw", false, "enable raw sockets. When false, raw sockets are disabled by removing CAP_NET_RAW from containers (`runsc exec` will still be able to utilize raw sockets). Raw sockets allow malicious containers to craft packets and potentially attack the network.") flagSet.Bool("gso", true, "enable host segmentation offload if it is supported by a network device.") flagSet.Bool("software-gso", true, "enable gVisor segmentation offload when host offload can't be enabled.") - flagSet.Duration("gvisor-gro", 0, "(e.g. \"20000ns\" or \"1ms\") sets gVisor's generic receive offload timeout. Zero bypasses GRO.") + flagSet.Bool("gvisor-gro", false, "enable gVisor generic receive offload") flagSet.Bool("tx-checksum-offload", false, "enable TX checksum offload.") flagSet.Bool("rx-checksum-offload", true, "enable RX checksum offload.") flagSet.Var(queueingDisciplinePtr(QDiscFIFO), "qdisc", "specifies which queueing discipline to apply by default to the non loopback nics used by the sandbox.") diff --git a/runsc/sandbox/network.go b/runsc/sandbox/network.go index 660acbecb..dfbe9322b 100644 --- a/runsc/sandbox/network.go +++ b/runsc/sandbox/network.go @@ -77,7 +77,7 @@ func setupNetwork(conn *urpc.Client, pid int, conf *config.Config) error { func createDefaultLoopbackInterface(conf *config.Config, conn *urpc.Client) error { link := boot.DefaultLoopbackLink - link.GvisorGROTimeout = conf.GvisorGROTimeout + link.GvisorGRO = conf.GvisorGRO if err := conn.Call(boot.NetworkCreateLinksAndRoutes, &boot.CreateLinksAndRoutesArgs{ LoopbackLinks: []boot.LoopbackLink{link}, }, nil); err != nil { @@ -277,7 +277,7 @@ func createInterfacesAndRoutesFromNS(conn *urpc.Client, nsPath string, conf *con Neighbors: neighbors, LinkAddress: linkAddress, Addresses: addresses, - GvisorGROTimeout: conf.GvisorGROTimeout, + GvisorGRO: conf.GvisorGRO, }) } else { link := boot.FDBasedLink{ @@ -317,7 +317,7 @@ func createInterfacesAndRoutesFromNS(conn *urpc.Client, nsPath string, conf *con link.GSOMaxSize = stack.GvisorGSOMaxSize link.GvisorGSOEnabled = true } - link.GvisorGROTimeout = conf.GvisorGROTimeout + link.GvisorGRO = conf.GvisorGRO args.FDBasedLinks = append(args.FDBasedLinks, link) } @@ -429,8 +429,8 @@ func createSocket(iface net.Interface, ifaceLink netlink.Link, enableGSO bool) ( // interface. func loopbackLink(conf *config.Config, iface net.Interface, addrs []net.Addr) (boot.LoopbackLink, error) { link := boot.LoopbackLink{ - Name: iface.Name, - GvisorGROTimeout: conf.GvisorGROTimeout, + Name: iface.Name, + GvisorGRO: conf.GvisorGRO, } for _, addr := range addrs { ipNet, ok := addr.(*net.IPNet) diff --git a/runsc/sandbox/xdp.go b/runsc/sandbox/xdp.go index 7b6eb7388..22898b5cc 100644 --- a/runsc/sandbox/xdp.go +++ b/runsc/sandbox/xdp.go @@ -216,7 +216,7 @@ func prepareRedirectInterfaceArgs(bind boot.BindOpt, conf *config.Config) (boot. Neighbors: neighbors, LinkAddress: linkAddress, Addresses: []boot.IPWithPrefix{addr}, - GvisorGROTimeout: conf.GvisorGROTimeout, + GvisorGRO: conf.GvisorGRO, Bind: bind, } args.XDPLinks = append(args.XDPLinks, xdplink) diff --git a/test/benchmarks/tcp/tcp_benchmark.sh b/test/benchmarks/tcp/tcp_benchmark.sh index 1c267f4a6..11225ccfe 100755 --- a/test/benchmarks/tcp/tcp_benchmark.sh +++ b/test/benchmarks/tcp/tcp_benchmark.sh @@ -47,7 +47,7 @@ helper_dir="$(dirname "$0")" netstack_opts= disable_linux_gso= disable_linux_gro= -gro=0 +gro=false num_client_threads=1 sniff=false xdp=false @@ -173,9 +173,7 @@ while [[ $# -gt 0 ]]; do disable_linux_gro=1 ;; --gro) - shift - [[ "$#" -le 0 ]] && echo "no GRO timeout provided" && exit 1 - gro=$1 + gro=true ;; --ipv6) client_addr=fd::1 @@ -234,7 +232,7 @@ while [[ $# -gt 0 ]]; do echo " --num-client-threads number of parallel client threads to run" echo " --disable-linux-gso disable segmentation offload (TSO, GSO, GRO) in the Linux network stack" echo " --disable-linux-gro disable GRO in the Linux network stack" - echo " --gro set gVisor GRO timeout" + echo " --gro enable gVisor GRO" echo " --ipv6 use ipv6 for benchmarks" echo " --iperf-binary name of the iperf binary to call" echo " --sniff sniff and output packet logs" diff --git a/test/benchmarks/tcp/tcp_proxy.go b/test/benchmarks/tcp/tcp_proxy.go index 11ff1473f..7bd04b1b4 100644 --- a/test/benchmarks/tcp/tcp_proxy.go +++ b/test/benchmarks/tcp/tcp_proxy.go @@ -64,7 +64,7 @@ var ( cubic = flag.Bool("cubic", false, "enable use of CUBIC congestion control for netstack") gso = flag.Int("gso", 0, "GSO maximum size") swgso = flag.Bool("swgso", false, "gVisor-level GSO") - gro = flag.Duration("gro", 0, "gVisor-level GRO timeout") + gro = flag.Bool("gro", false, "gVisor-level GRO") clientTCPProbeFile = flag.String("client_tcp_probe_file", "", "if specified, installs a tcp probe to dump endpoint state to the specified file.") serverTCPProbeFile = flag.String("server_tcp_probe_file", "", "if specified, installs a tcp probe to dump endpoint state to the specified file.") cpuprofile = flag.String("cpuprofile", "", "write cpu profile to the specified file.") @@ -228,7 +228,7 @@ func newNetstackImpl(mode string) (impl, error) { // regenerate valid checksums after GRO. TXChecksumOffload: false, RXChecksumOffload: true, - //PacketDispatchMode: fdbased.RecvMMsg, + // PacketDispatchMode: fdbased.RecvMMsg, PacketDispatchMode: fdbased.PacketMMap, GSOMaxSize: uint32(*gso), GvisorGSOEnabled: *swgso, @@ -243,7 +243,7 @@ func newNetstackImpl(mode string) (impl, error) { } qDisc := fifo.New(ep, runtime.GOMAXPROCS(0), 1000) - opts := stack.NICOptions{QDisc: qDisc, GROTimeout: *gro} + opts := stack.NICOptions{QDisc: qDisc} if err := s.CreateNICWithOptions(nicID, ep, opts); err != nil { return nil, fmt.Errorf("error creating NIC %q: %v", *iface, err) } diff --git a/test/runner/main.go b/test/runner/main.go index bacff47f8..3a1b50025 100644 --- a/test/runner/main.go +++ b/test/runner/main.go @@ -331,7 +331,7 @@ func runRunsc(tc *gtest.TestCase, spec *specs.Spec) error { "-watchdog-action=panic", "-platform", *platform, "-file-access", *fileAccess, - "-gvisor-gro=200000ns", + "-gvisor-gro", } if *network == "host" && !testutil.TestEnvSupportsNetAdmin {