diff --git a/pkg/tcpip/link/fdbased/endpoint.go b/pkg/tcpip/link/fdbased/endpoint.go index 314154445..fa64a703b 100644 --- a/pkg/tcpip/link/fdbased/endpoint.go +++ b/pkg/tcpip/link/fdbased/endpoint.go @@ -226,6 +226,10 @@ type Options struct { // GRO enables generic receive offload. GRO bool + + // ProcessorsPerChannel is the number of goroutines used to handle packets + // from each FD. + ProcessorsPerChannel int } // fanoutID is used for AF_PACKET based endpoints to enable PACKET_FANOUT diff --git a/runsc/boot/network.go b/runsc/boot/network.go index cc2b8b850..871b0f593 100644 --- a/runsc/boot/network.go +++ b/runsc/boot/network.go @@ -109,6 +109,10 @@ type FDBasedLink struct { // NumChannels controls how many underlying FDs are to be used to // create this endpoint. NumChannels int + + // ProcessorsPerChannel controls how many goroutines are used to handle + // packets on each channel. + ProcessorsPerChannel int } // BindOpt indicates whether the sentry or runsc process is responsible for @@ -309,16 +313,17 @@ func (n *Network) CreateLinksAndRoutes(args *CreateLinksAndRoutesArgs, _ *struct log.Infof("gso max size is: %d", link.GSOMaxSize) linkEP, err := fdbased.New(&fdbased.Options{ - FDs: FDs, - MTU: uint32(link.MTU), - EthernetHeader: mac != "", - Address: mac, - PacketDispatchMode: dispatchMode, - GSOMaxSize: link.GSOMaxSize, - GVisorGSOEnabled: link.GVisorGSOEnabled, - TXChecksumOffload: link.TXChecksumOffload, - RXChecksumOffload: link.RXChecksumOffload, - GRO: link.GVisorGRO, + FDs: FDs, + MTU: uint32(link.MTU), + EthernetHeader: mac != "", + Address: mac, + PacketDispatchMode: dispatchMode, + GSOMaxSize: link.GSOMaxSize, + GVisorGSOEnabled: link.GVisorGSOEnabled, + TXChecksumOffload: link.TXChecksumOffload, + RXChecksumOffload: link.RXChecksumOffload, + GRO: link.GVisorGRO, + ProcessorsPerChannel: link.ProcessorsPerChannel, }) if err != nil { return err diff --git a/runsc/config/config.go b/runsc/config/config.go index cc1c88890..8d41f1961 100644 --- a/runsc/config/config.go +++ b/runsc/config/config.go @@ -239,6 +239,12 @@ type Config struct { // scale for high throughput use cases. NumNetworkChannels int `flag:"num-network-channels"` + // NetworkProcessorsPerChannel controls the number of goroutines used to + // handle packets on a single network channel. A higher number can help handle + // many simultaneous connections. If this is 0, runsc will divide GOMAXPROCS + // evenly among each network channel. + NetworkProcessorsPerChannel int `flag:"network-processors-per-channel"` + // Rootless allows the sandbox to be started with a user that is not root. // Defense in depth measures are weaker in rootless mode. Specifically, the // sandbox and Gofer process run as root inside a user namespace with root diff --git a/runsc/config/flags.go b/runsc/config/flags.go index c800f1836..ae7a2546a 100644 --- a/runsc/config/flags.go +++ b/runsc/config/flags.go @@ -119,6 +119,7 @@ func RegisterFlags(flagSet *flag.FlagSet) { flagSet.Bool("rx-checksum-offload", true, "enable RX checksum offload.") flagSet.Var(queueingDisciplinePtr(QDiscFIFO), "qdisc", "specifies which queueing discipline to apply by default to the non loopback nics used by the sandbox.") flagSet.Int("num-network-channels", 1, "number of underlying channels(FDs) to use for network link endpoints.") + flagSet.Int("network-processors-per-channel", 1, "number of goroutines in each channel for processng inbound packets. If 0, the link endpoint will divide GOMAXPROCS evenly among the number of channels specified by num-network-channels.") flagSet.Bool("buffer-pooling", true, "DEPRECATED: this flag has no effect. Buffer pooling is always enabled.") flagSet.Var(&xdpConfig, "EXPERIMENTAL-xdp", `whether and how to use XDP. Can be one of: "off" (default), "ns", "redirect:", or "tunnel:"`) flagSet.Bool("EXPERIMENTAL-xdp-need-wakeup", true, "EXPERIMENTAL. Use XDP_USE_NEED_WAKEUP with XDP sockets.") // TODO(b/240191988): Figure out whether this helps and remove it as a flag. diff --git a/runsc/sandbox/network.go b/runsc/sandbox/network.go index b8f402597..c840a3c7a 100644 --- a/runsc/sandbox/network.go +++ b/runsc/sandbox/network.go @@ -281,16 +281,17 @@ func createInterfacesAndRoutesFromNS(conn *urpc.Client, nsPath string, conf *con }) } else { link := boot.FDBasedLink{ - Name: iface.Name, - MTU: iface.MTU, - Routes: routes, - TXChecksumOffload: conf.TXChecksumOffload, - RXChecksumOffload: conf.RXChecksumOffload, - NumChannels: conf.NumNetworkChannels, - QDisc: conf.QDisc, - Neighbors: neighbors, - LinkAddress: linkAddress, - Addresses: addresses, + Name: iface.Name, + MTU: iface.MTU, + Routes: routes, + TXChecksumOffload: conf.TXChecksumOffload, + RXChecksumOffload: conf.RXChecksumOffload, + NumChannels: conf.NumNetworkChannels, + ProcessorsPerChannel: conf.NetworkProcessorsPerChannel, + QDisc: conf.QDisc, + Neighbors: neighbors, + LinkAddress: linkAddress, + Addresses: addresses, } log.Debugf("Setting up network channels")