From fe88fe67683ad3516425e8111c3074184e370b82 Mon Sep 17 00:00:00 2001 From: Bhasker Hariharan Date: Wed, 15 Dec 2021 12:48:40 -0800 Subject: [PATCH] Support custom socket options in hostinet. PiperOrigin-RevId: 416625574 --- pkg/abi/linux/netdevice.go | 34 ++++++++ pkg/sentry/inet/BUILD | 1 + pkg/sentry/inet/inet.go | 5 ++ pkg/sentry/socket/hostinet/BUILD | 1 + pkg/sentry/socket/hostinet/socket.go | 21 ++++- pkg/sentry/socket/hostinet/socket_unsafe.go | 74 +++++++++++++++++- pkg/sentry/socket/hostinet/sockopt_impl.go | 8 +- pkg/sentry/socket/hostinet/stack.go | 21 +++-- pkg/sentry/socket/hostinet/stack_unsafe.go | 87 +++++++++++++++++++++ 9 files changed, 237 insertions(+), 15 deletions(-) create mode 100644 pkg/sentry/socket/hostinet/stack_unsafe.go diff --git a/pkg/abi/linux/netdevice.go b/pkg/abi/linux/netdevice.go index 51a39704b..e64d554df 100644 --- a/pkg/abi/linux/netdevice.go +++ b/pkg/abi/linux/netdevice.go @@ -86,3 +86,37 @@ type IFConf struct { _ [4]byte // Pad to sizeof(struct ifconf). Ptr uint64 } + +// EthtoolCmd is a marshallable type to be able to easily copyin the +// the command for an SIOCETHTOOL ioctl. +// +// +marshal +type EthtoolCmd uint32 + +const ( + // ETHTOOL_GFEATURES is the command to SIOCETHTOOL to query device + // features. + // See: + ETHTOOL_GFEATURES EthtoolCmd = 0x3a +) + +// EthtoolGFeatures is used to return a list of device features. +// See: +// +// +marshal +type EthtoolGFeatures struct { + Cmd uint32 + Size uint32 +} + +// EthtoolGetFeaturesBlock is used to return state of upto 32 device +// features. +// See: +// +// +marshal +type EthtoolGetFeaturesBlock struct { + Available uint32 + Requested uint32 + Active uint32 + NeverChanged uint32 +} diff --git a/pkg/sentry/inet/BUILD b/pkg/sentry/inet/BUILD index 2363cec5f..3bd141b89 100644 --- a/pkg/sentry/inet/BUILD +++ b/pkg/sentry/inet/BUILD @@ -27,6 +27,7 @@ go_library( "test_stack.go", ], deps = [ + "//pkg/abi/linux", "//pkg/context", "//pkg/tcpip", "//pkg/tcpip/stack", diff --git a/pkg/sentry/inet/inet.go b/pkg/sentry/inet/inet.go index b121fc1b4..a76ab1087 100644 --- a/pkg/sentry/inet/inet.go +++ b/pkg/sentry/inet/inet.go @@ -16,6 +16,7 @@ package inet import ( + "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/tcpip" "gvisor.dev/gvisor/pkg/tcpip/stack" ) @@ -120,6 +121,10 @@ type Interface struct { // MTU is the maximum transmission unit. MTU uint32 + + // Features are the device features queried from the host at + // stack creation time. These are immutable after startup. + Features []linux.EthtoolGetFeaturesBlock } // InterfaceAddr contains information about a network interface address. diff --git a/pkg/sentry/socket/hostinet/BUILD b/pkg/sentry/socket/hostinet/BUILD index 4ea89f9d0..e53321d82 100644 --- a/pkg/sentry/socket/hostinet/BUILD +++ b/pkg/sentry/socket/hostinet/BUILD @@ -13,6 +13,7 @@ go_library( "socket_vfs2.go", "sockopt_impl.go", "stack.go", + "stack_unsafe.go", ], visibility = ["//pkg/sentry:internal"], deps = [ diff --git a/pkg/sentry/socket/hostinet/socket.go b/pkg/sentry/socket/hostinet/socket.go index 3ac02e2c2..8b19a75f4 100644 --- a/pkg/sentry/socket/hostinet/socket.go +++ b/pkg/sentry/socket/hostinet/socket.go @@ -377,13 +377,13 @@ func (s *socketOpsCommon) Shutdown(_ *kernel.Task, how int) *syserr.Error { } // GetSockOpt implements socket.Socket.GetSockOpt. -func (s *socketOpsCommon) GetSockOpt(t *kernel.Task, level int, name int, _ hostarch.Addr, outLen int) (marshal.Marshallable, *syserr.Error) { +func (s *socketOpsCommon) GetSockOpt(t *kernel.Task, level int, name int, optValAddr hostarch.Addr, outLen int) (marshal.Marshallable, *syserr.Error) { if outLen < 0 { return nil, syserr.ErrInvalidArgument } // Only allow known and safe options. - optlen := getSockOptLen(t, level, name) + optlen, copyIn := getSockOptLen(t, level, name) switch level { case linux.SOL_IP: switch name { @@ -418,10 +418,21 @@ func (s *socketOpsCommon) GetSockOpt(t *kernel.Task, level int, name int, _ host return nil, syserr.ErrInvalidArgument } - opt, err := getsockopt(s.fd, level, name, optlen) + opt := make([]byte, optlen) + if copyIn { + // This is non-intuitive as normally in getsockopt one assumes that the + // parameter is purely an out parameter. But some custom options do require + // copying in the optVal so we do it here only for those custom options. + if _, err := t.CopyInBytes(optValAddr, opt); err != nil { + return nil, syserr.FromError(err) + } + } + var err error + opt, err = getsockopt(s.fd, level, name, opt) if err != nil { return nil, syserr.FromError(err) } + opt = postGetSockOpt(t, level, name, opt) optP := primitive.ByteSlice(opt) return &optP, nil } @@ -748,7 +759,9 @@ func translateIOSyscallError(err error) error { // State implements socket.Socket.State. func (s *socketOpsCommon) State() uint32 { info := linux.TCPInfo{} - buf, err := getsockopt(s.fd, unix.SOL_TCP, unix.TCP_INFO, linux.SizeOfTCPInfo) + buf := make([]byte, linux.SizeOfTCPInfo) + var err error + buf, err = getsockopt(s.fd, unix.SOL_TCP, unix.TCP_INFO, buf) if err != nil { if err != unix.ENOPROTOOPT { log.Warningf("Failed to get TCP socket info from %+v: %v", s, err) diff --git a/pkg/sentry/socket/hostinet/socket_unsafe.go b/pkg/sentry/socket/hostinet/socket_unsafe.go index d19a58736..af7327428 100644 --- a/pkg/sentry/socket/hostinet/socket_unsafe.go +++ b/pkg/sentry/socket/hostinet/socket_unsafe.go @@ -23,6 +23,7 @@ import ( "gvisor.dev/gvisor/pkg/errors/linuxerr" "gvisor.dev/gvisor/pkg/hostarch" "gvisor.dev/gvisor/pkg/sentry/arch" + "gvisor.dev/gvisor/pkg/sentry/inet" "gvisor.dev/gvisor/pkg/sentry/kernel" "gvisor.dev/gvisor/pkg/sentry/socket" "gvisor.dev/gvisor/pkg/syserr" @@ -102,6 +103,76 @@ func ioctl(ctx context.Context, fd int, io usermem.IO, args arch.SyscallArgument } _, err := ifc.CopyOut(cc, args[2].Pointer()) return 0, err + case linux.SIOCETHTOOL: + cc := &usermem.IOCopyContext{ + Ctx: ctx, + IO: io, + Opts: usermem.IOOpts{ + AddressSpaceActive: true, + }, + } + var ifr linux.IFReq + if _, err := ifr.CopyIn(cc, args[2].Pointer()); err != nil { + return 0, err + } + // SIOCETHTOOL commands specify the subcommand in the first 32 bytes pointed + // to by ifr.ifr_data. We need to copy it in first to understand the actual + // structure pointed by ifr.ifr_data. + ifrData := hostarch.Addr(hostarch.ByteOrder.Uint64(ifr.Data[:8])) + var ethtoolCmd linux.EthtoolCmd + if _, err := ethtoolCmd.CopyIn(cc, ifrData); err != nil { + return 0, err + } + // We only support ETHTOOL_GFEATURES. + if ethtoolCmd != linux.ETHTOOL_GFEATURES { + return 0, linuxerr.EOPNOTSUPP + } + var gfeatures linux.EthtoolGFeatures + if _, err := gfeatures.CopyIn(cc, ifrData); err != nil { + return 0, err + } + + // Find the requested device. + stk := inet.StackFromContext(ctx) + if stk == nil { + return 0, linuxerr.ENODEV + } + + var ( + iface inet.Interface + found bool + ) + for _, iface = range stk.Interfaces() { + if iface.Name == ifr.Name() { + found = true + break + } + } + if !found { + return 0, linuxerr.ENODEV + } + + // Copy out the feature blocks to the memory pointed to by ifrData. + blksToCopy := int(gfeatures.Size) + if blksToCopy > len(iface.Features) { + blksToCopy = len(iface.Features) + } + gfeatures.Size = uint32(blksToCopy) + if _, err := gfeatures.CopyOut(cc, ifrData); err != nil { + return 0, err + } + next, ok := ifrData.AddLength(uint64(unsafe.Sizeof(linux.EthtoolGFeatures{}))) + for i := 0; i < blksToCopy; i++ { + if !ok { + return 0, linuxerr.EFAULT + } + if _, err := iface.Features[i].CopyOut(cc, next); err != nil { + return 0, err + } + next, ok = next.AddLength(uint64(unsafe.Sizeof(linux.EthtoolGetFeaturesBlock{}))) + } + + return 0, nil default: return 0, linuxerr.ENOTTY } @@ -115,8 +186,7 @@ func accept4(fd int, addr *byte, addrlen *uint32, flags int) (int, error) { return int(afd), nil } -func getsockopt(fd int, level, name int, optlen int) ([]byte, error) { - opt := make([]byte, optlen) +func getsockopt(fd int, level, name int, opt []byte) ([]byte, error) { optlen32 := int32(len(opt)) _, _, errno := unix.Syscall6(unix.SYS_GETSOCKOPT, uintptr(fd), uintptr(level), uintptr(name), uintptr(firstBytePtr(opt)), uintptr(unsafe.Pointer(&optlen32)), 0) if errno != 0 { diff --git a/pkg/sentry/socket/hostinet/sockopt_impl.go b/pkg/sentry/socket/hostinet/sockopt_impl.go index 2397e04e7..66994e365 100644 --- a/pkg/sentry/socket/hostinet/sockopt_impl.go +++ b/pkg/sentry/socket/hostinet/sockopt_impl.go @@ -21,10 +21,14 @@ import ( "gvisor.dev/gvisor/pkg/sentry/kernel" ) -func getSockOptLen(t *kernel.Task, level, name int) int { - return 0 // No custom options. +func getSockOptLen(t *kernel.Task, level, name int) (len int, copyIn bool) { + return 0, false // No custom options. } func setSockOptLen(t *kernel.Task, level, name int) int { return 0 // No custom options. } + +func postGetSockOpt(t *kernel.Task, level, name int, opt []byte) []byte { + return opt // No custom changes to option value. +} diff --git a/pkg/sentry/socket/hostinet/stack.go b/pkg/sentry/socket/hostinet/stack.go index c84ab3fb7..e4598be82 100644 --- a/pkg/sentry/socket/hostinet/stack.go +++ b/pkg/sentry/socket/hostinet/stack.go @@ -23,7 +23,6 @@ import ( "reflect" "strconv" "strings" - "syscall" "golang.org/x/sys/unix" @@ -54,7 +53,7 @@ var defaultSendBufSize = inet.TCPBufferSize{ // Stack implements inet.Stack for host sockets. type Stack struct { // Stack is immutable. - interfaces map[int32]inet.Interface + interfaces map[int32]*inet.Interface interfaceAddrs map[int32][]inet.InterfaceAddr routes []inet.Route supportsIPv6 bool @@ -69,7 +68,7 @@ type Stack struct { // NewStack returns an empty Stack containing no configuration. func NewStack() *Stack { return &Stack{ - interfaces: make(map[int32]inet.Interface), + interfaces: make(map[int32]*inet.Interface), interfaceAddrs: make(map[int32][]inet.InterfaceAddr), } } @@ -129,7 +128,7 @@ func (s *Stack) Configure() error { // ExtractHostInterfaces will populate an interface map and // interfaceAddrs map with the results of the equivalent // netlink messages. -func ExtractHostInterfaces(links []syscall.NetlinkMessage, addrs []syscall.NetlinkMessage, interfaces map[int32]inet.Interface, interfaceAddrs map[int32][]inet.InterfaceAddr) error { +func ExtractHostInterfaces(links []syscall.NetlinkMessage, addrs []syscall.NetlinkMessage, interfaces map[int32]*inet.Interface, interfaceAddrs map[int32][]inet.InterfaceAddr) error { for _, link := range links { if link.Header.Type != unix.RTM_NEWLINK { continue @@ -158,7 +157,7 @@ func ExtractHostInterfaces(links []syscall.NetlinkMessage, addrs []syscall.Netli inetIF.Name = string(attr.Value[:len(attr.Value)-1]) } } - interfaces[ifinfo.Index] = inetIF + interfaces[ifinfo.Index] = &inetIF } for _, addr := range addrs { @@ -258,7 +257,15 @@ func addHostInterfaces(s *Stack) error { return fmt.Errorf("RTM_GETADDR failed: %v", err) } - return ExtractHostInterfaces(links, addrs, s.interfaces, s.interfaceAddrs) + if err := ExtractHostInterfaces(links, addrs, s.interfaces, s.interfaceAddrs); err != nil { + return err + } + + // query interface features for each of the host interfaces. + if err := queryInterfaceFeatures(s.interfaces); err != nil { + return err + } + return nil } func addHostRoutes(s *Stack) error { @@ -304,7 +311,7 @@ func readTCPBufferSizeFile(filename string) (inet.TCPBufferSize, error) { func (s *Stack) Interfaces() map[int32]inet.Interface { interfaces := make(map[int32]inet.Interface) for k, v := range s.interfaces { - interfaces[k] = v + interfaces[k] = *v } return interfaces } diff --git a/pkg/sentry/socket/hostinet/stack_unsafe.go b/pkg/sentry/socket/hostinet/stack_unsafe.go new file mode 100644 index 000000000..066c2d0ac --- /dev/null +++ b/pkg/sentry/socket/hostinet/stack_unsafe.go @@ -0,0 +1,87 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostinet + +import ( + "runtime" + "unsafe" + + "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/hostarch" + "gvisor.dev/gvisor/pkg/sentry/inet" +) + +func queryInterfaceFeatures(interfaces map[int32]*inet.Interface) error { + fd, err := unix.Socket(unix.AF_INET6, unix.SOCK_STREAM, 0) + if err != nil { + return err + } + defer unix.Close(fd) + for idx, nic := range interfaces { + var ifr linux.IFReq + copy(ifr.IFName[:], nic.Name) + var gfeatures linux.EthtoolGFeatures + // Each feature block is sufficient to query 32 features, the linux + // kernel today supports upto 64 features per device. Technically it + // can support more in the future but this is sufficient for our use + // right now. + const ( + numFeatureBlocks = 2 + ifrDataSz = unsafe.Sizeof(linux.EthtoolGFeatures{}) + numFeatureBlocks*unsafe.Sizeof(linux.EthtoolGetFeaturesBlock{}) + ) + featureBlocks := make([]linux.EthtoolGetFeaturesBlock, numFeatureBlocks) + b := make([]byte, ifrDataSz) + gfeatures.Cmd = uint32(linux.ETHTOOL_GFEATURES) + gfeatures.Size = numFeatureBlocks + gfeatures.MarshalBytes(b) + next := b[unsafe.Sizeof(linux.EthtoolGFeatures{}):] + for i := 0; i < numFeatureBlocks; i++ { + featureBlocks[i].MarshalBytes(next) + next = next[unsafe.Sizeof(linux.EthtoolGetFeaturesBlock{}):] + } + + // Technically the next two lines are not safe as Go GC can technically move + // b to a new location and the pointer value stored in ifr.Data could point + // to random memory. But the reality today is that Go GC is not a moving GC + // so this is essentially safe as of today. + // + // TODO(b/209014118): Use Pin API when available in Go runtime to make this + // safe. + dataPtr := unsafe.Pointer(&b[0]) + hostarch.ByteOrder.PutUint64(ifr.Data[:8], uint64(uintptr(dataPtr))) + + if _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd), unix.SIOCETHTOOL, uintptr(unsafe.Pointer(&ifr))); errno != 0 { + return errno + } + + // Unmarshall the features back. + gfeatures.UnmarshalBytes(b) + next = b[unsafe.Sizeof(linux.EthtoolGFeatures{}):] + for i := 0; i < int(gfeatures.Size); i++ { + featureBlocks[i].UnmarshalBytes(next) + next = next[unsafe.Sizeof(linux.EthtoolGetFeaturesBlock{}):] + } + // Store the queried features. + interfaces[idx].Features = make([]linux.EthtoolGetFeaturesBlock, gfeatures.Size) + copy(interfaces[idx].Features, featureBlocks) + + // This ensures b is not garbage collected before this point to ensure that + // the slice is not collected before the syscall returns and we copy out the + // data. + runtime.KeepAlive(b) + } + return nil +}