From 9e66f710de1af177b39cb3379d0765947ef9665b Mon Sep 17 00:00:00 2001 From: Ayush Ranjan Date: Thu, 2 Nov 2023 13:56:26 -0700 Subject: [PATCH] Delete devtmpfs and replace it with tmpfs. Our devtmpfs implementation uses the same tmpfs filesystem instance for all devtmpfs mounts in the sandbox. This would mean that devices mounted in a container are visible and accessible to all other containers in the sandbox. With GPU/TPU, the contents of devtmpfs can be different for different containers within the same sandbox. So it is important to not share the same devtmpfs contents. It is better to drop support for devtmpfs, than to implement it incorrectly. Instead, this change introduces a new dummy filesystem type named `dev`. This filesystem can not be mounted or listed by the application. This filesystem creates a new tmpfs instance on GetFilesystem() and populates it with all the device files. PiperOrigin-RevId: 578969556 --- pkg/sentry/devices/accel/BUILD | 1 - pkg/sentry/devices/accel/device.go | 6 - pkg/sentry/devices/memdev/BUILD | 1 - pkg/sentry/devices/memdev/memdev.go | 38 +--- pkg/sentry/devices/nvproxy/BUILD | 1 - pkg/sentry/devices/nvproxy/nvproxy.go | 20 -- pkg/sentry/devices/ttydev/BUILD | 1 - pkg/sentry/devices/ttydev/ttydev.go | 9 +- pkg/sentry/devices/tundev/BUILD | 1 - pkg/sentry/devices/tundev/tundev.go | 12 +- pkg/sentry/fsimpl/{devtmpfs => dev}/BUILD | 13 +- pkg/sentry/fsimpl/dev/dev.go | 189 ++++++++++++++++ pkg/sentry/fsimpl/dev/dev_test.go | 154 +++++++++++++ pkg/sentry/fsimpl/devtmpfs/devtmpfs.go | 231 -------------------- pkg/sentry/fsimpl/devtmpfs/devtmpfs_test.go | 229 ------------------- pkg/sentry/fsimpl/devtmpfs/save_restore.go | 23 -- pkg/sentry/fsimpl/fuse/BUILD | 1 - pkg/sentry/fsimpl/fuse/register.go | 21 +- pkg/sentry/vfs/device.go | 18 ++ runsc/boot/BUILD | 3 +- runsc/boot/loader.go | 9 + runsc/boot/vfs.go | 175 +++++---------- runsc/cmd/boot.go | 1 + 23 files changed, 459 insertions(+), 698 deletions(-) rename pkg/sentry/fsimpl/{devtmpfs => dev}/BUILD (78%) create mode 100644 pkg/sentry/fsimpl/dev/dev.go create mode 100644 pkg/sentry/fsimpl/dev/dev_test.go delete mode 100644 pkg/sentry/fsimpl/devtmpfs/devtmpfs.go delete mode 100644 pkg/sentry/fsimpl/devtmpfs/devtmpfs_test.go delete mode 100644 pkg/sentry/fsimpl/devtmpfs/save_restore.go diff --git a/pkg/sentry/devices/accel/BUILD b/pkg/sentry/devices/accel/BUILD index 5f9d97e63..617681d50 100644 --- a/pkg/sentry/devices/accel/BUILD +++ b/pkg/sentry/devices/accel/BUILD @@ -29,7 +29,6 @@ go_library( "//pkg/safemem", "//pkg/seccomp", "//pkg/sentry/arch", - "//pkg/sentry/fsimpl/devtmpfs", "//pkg/sentry/fsimpl/eventfd", "//pkg/sentry/kernel", "//pkg/sentry/memmap", diff --git a/pkg/sentry/devices/accel/device.go b/pkg/sentry/devices/accel/device.go index ec09dbfdf..a2d143e74 100644 --- a/pkg/sentry/devices/accel/device.go +++ b/pkg/sentry/devices/accel/device.go @@ -22,7 +22,6 @@ import ( "gvisor.dev/gvisor/pkg/context" "gvisor.dev/gvisor/pkg/errors/linuxerr" "gvisor.dev/gvisor/pkg/fdnotifier" - "gvisor.dev/gvisor/pkg/sentry/fsimpl/devtmpfs" "gvisor.dev/gvisor/pkg/sentry/kernel" "gvisor.dev/gvisor/pkg/sentry/vfs" "gvisor.dev/gvisor/pkg/sync" @@ -81,11 +80,6 @@ func (dev *tpuV4Device) Open(ctx context.Context, mnt *vfs.Mount, vfsd *vfs.Dent return &fd.vfsfd, nil } -// CreateDevtmpfsFile creates a /dev/accel[0-9]+ device file. -func CreateDevtmpfsFile(ctx context.Context, dev *devtmpfs.Accessor, num uint32) error { - return dev.CreateDeviceFile(ctx, fmt.Sprintf("accel%d", num), vfs.CharDevice, linux.ACCEL_MAJOR, num, 0666) -} - // RegisterTPUV4Device registers all devices implemented by this package in vfsObj. func RegisterTPUV4Device(vfsObj *vfs.VirtualFilesystem, minor uint32, lite bool) error { return vfsObj.RegisterDevice(vfs.CharDevice, linux.ACCEL_MAJOR, minor, &tpuV4Device{ diff --git a/pkg/sentry/devices/memdev/BUILD b/pkg/sentry/devices/memdev/BUILD index 7893fdd25..5e83fe3c8 100644 --- a/pkg/sentry/devices/memdev/BUILD +++ b/pkg/sentry/devices/memdev/BUILD @@ -21,7 +21,6 @@ go_library( "//pkg/errors/linuxerr", "//pkg/rand", "//pkg/safemem", - "//pkg/sentry/fsimpl/devtmpfs", "//pkg/sentry/fsimpl/tmpfs", "//pkg/sentry/kernel", "//pkg/sentry/kernel/auth", diff --git a/pkg/sentry/devices/memdev/memdev.go b/pkg/sentry/devices/memdev/memdev.go index 5759900c4..208cfc472 100644 --- a/pkg/sentry/devices/memdev/memdev.go +++ b/pkg/sentry/devices/memdev/memdev.go @@ -18,42 +18,28 @@ package memdev import ( "gvisor.dev/gvisor/pkg/abi/linux" - "gvisor.dev/gvisor/pkg/context" - "gvisor.dev/gvisor/pkg/sentry/fsimpl/devtmpfs" "gvisor.dev/gvisor/pkg/sentry/vfs" ) // Register registers all devices implemented by this package in vfsObj. func Register(vfsObj *vfs.VirtualFilesystem) error { - for minor, dev := range map[uint32]vfs.Device{ - nullDevMinor: nullDevice{}, - zeroDevMinor: zeroDevice{}, - fullDevMinor: fullDevice{}, - randomDevMinor: randomDevice{}, - urandomDevMinor: randomDevice{}, + for minor, spec := range map[uint32]struct { + dev vfs.Device + pathname string + }{ + nullDevMinor: {nullDevice{}, "null"}, + zeroDevMinor: {zeroDevice{}, "zero"}, + fullDevMinor: {fullDevice{}, "full"}, + randomDevMinor: {randomDevice{}, "random"}, + urandomDevMinor: {randomDevice{}, "urandom"}, } { - if err := vfsObj.RegisterDevice(vfs.CharDevice, linux.MEM_MAJOR, minor, dev, &vfs.RegisterDeviceOptions{ + if err := vfsObj.RegisterDevice(vfs.CharDevice, linux.MEM_MAJOR, minor, spec.dev, &vfs.RegisterDeviceOptions{ GroupName: "mem", + Pathname: spec.pathname, + FilePerms: 0666, }); err != nil { return err } } return nil } - -// CreateDevtmpfsFiles creates device special files in dev representing all -// devices implemented by this package. -func CreateDevtmpfsFiles(ctx context.Context, dev *devtmpfs.Accessor) error { - for minor, name := range map[uint32]string{ - nullDevMinor: "null", - zeroDevMinor: "zero", - fullDevMinor: "full", - randomDevMinor: "random", - urandomDevMinor: "urandom", - } { - if err := dev.CreateDeviceFile(ctx, name, vfs.CharDevice, linux.MEM_MAJOR, minor, 0666 /* mode */); err != nil { - return err - } - } - return nil -} diff --git a/pkg/sentry/devices/nvproxy/BUILD b/pkg/sentry/devices/nvproxy/BUILD index 4fc677f55..77de8af22 100644 --- a/pkg/sentry/devices/nvproxy/BUILD +++ b/pkg/sentry/devices/nvproxy/BUILD @@ -45,7 +45,6 @@ go_library( "//pkg/safemem", "//pkg/seccomp", "//pkg/sentry/arch", - "//pkg/sentry/fsimpl/devtmpfs", "//pkg/sentry/kernel", "//pkg/sentry/memmap", "//pkg/sentry/mm", diff --git a/pkg/sentry/devices/nvproxy/nvproxy.go b/pkg/sentry/devices/nvproxy/nvproxy.go index a92c7edd2..765f581d3 100644 --- a/pkg/sentry/devices/nvproxy/nvproxy.go +++ b/pkg/sentry/devices/nvproxy/nvproxy.go @@ -26,7 +26,6 @@ import ( "gvisor.dev/gvisor/pkg/hostarch" "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/marshal" - "gvisor.dev/gvisor/pkg/sentry/fsimpl/devtmpfs" "gvisor.dev/gvisor/pkg/sentry/mm" "gvisor.dev/gvisor/pkg/sentry/vfs" ) @@ -72,25 +71,6 @@ func Register(vfsObj *vfs.VirtualFilesystem, uvmDevMajor uint32) error { return nil } -// CreateDriverDevtmpfsFiles creates device special files in dev that should -// always exist when this package is enabled. It does not create per-device -// files in dev; see CreateIndexDevtmpfsFile. -func CreateDriverDevtmpfsFiles(ctx context.Context, dev *devtmpfs.Accessor, uvmDevMajor uint32) error { - if err := dev.CreateDeviceFile(ctx, "nvidiactl", vfs.CharDevice, nvgpu.NV_MAJOR_DEVICE_NUMBER, nvgpu.NV_CONTROL_DEVICE_MINOR, 0666); err != nil { - return err - } - if err := dev.CreateDeviceFile(ctx, "nvidia-uvm", vfs.CharDevice, uvmDevMajor, nvgpu.NVIDIA_UVM_PRIMARY_MINOR_NUMBER, 0666); err != nil { - return err - } - return nil -} - -// CreateIndexDevtmpfsFile creates the device special file in dev for the -// device with the given index. -func CreateIndexDevtmpfsFile(ctx context.Context, dev *devtmpfs.Accessor, minor uint32) error { - return dev.CreateDeviceFile(ctx, fmt.Sprintf("nvidia%d", minor), vfs.CharDevice, nvgpu.NV_MAJOR_DEVICE_NUMBER, minor, 0666) -} - // +stateify savable type nvproxy struct { objsMu objsMutex `state:"nosave"` diff --git a/pkg/sentry/devices/ttydev/BUILD b/pkg/sentry/devices/ttydev/BUILD index b93b08b29..609475ca0 100644 --- a/pkg/sentry/devices/ttydev/BUILD +++ b/pkg/sentry/devices/ttydev/BUILD @@ -12,7 +12,6 @@ go_library( "//pkg/abi/linux", "//pkg/context", "//pkg/errors/linuxerr", - "//pkg/sentry/fsimpl/devtmpfs", "//pkg/sentry/vfs", ], ) diff --git a/pkg/sentry/devices/ttydev/ttydev.go b/pkg/sentry/devices/ttydev/ttydev.go index 29b79b5d6..b8e525d79 100644 --- a/pkg/sentry/devices/ttydev/ttydev.go +++ b/pkg/sentry/devices/ttydev/ttydev.go @@ -19,7 +19,6 @@ import ( "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/context" "gvisor.dev/gvisor/pkg/errors/linuxerr" - "gvisor.dev/gvisor/pkg/sentry/fsimpl/devtmpfs" "gvisor.dev/gvisor/pkg/sentry/vfs" ) @@ -43,11 +42,7 @@ func (ttyDevice) Open(ctx context.Context, mnt *vfs.Mount, vfsd *vfs.Dentry, opt func Register(vfsObj *vfs.VirtualFilesystem) error { return vfsObj.RegisterDevice(vfs.CharDevice, linux.TTYAUX_MAJOR, ttyDevMinor, ttyDevice{}, &vfs.RegisterDeviceOptions{ GroupName: "tty", + Pathname: "tty", + FilePerms: 0666, }) } - -// CreateDevtmpfsFiles creates device special files in dev representing all -// devices implemented by this package. -func CreateDevtmpfsFiles(ctx context.Context, dev *devtmpfs.Accessor) error { - return dev.CreateDeviceFile(ctx, "tty", vfs.CharDevice, linux.TTYAUX_MAJOR, ttyDevMinor, 0666 /* mode */) -} diff --git a/pkg/sentry/devices/tundev/BUILD b/pkg/sentry/devices/tundev/BUILD index 993ddd79f..90019caa3 100644 --- a/pkg/sentry/devices/tundev/BUILD +++ b/pkg/sentry/devices/tundev/BUILD @@ -15,7 +15,6 @@ go_library( "//pkg/errors/linuxerr", "//pkg/hostarch", "//pkg/sentry/arch", - "//pkg/sentry/fsimpl/devtmpfs", "//pkg/sentry/inet", "//pkg/sentry/kernel", "//pkg/sentry/socket/netstack", diff --git a/pkg/sentry/devices/tundev/tundev.go b/pkg/sentry/devices/tundev/tundev.go index a0fcbdd8a..340f02fcc 100644 --- a/pkg/sentry/devices/tundev/tundev.go +++ b/pkg/sentry/devices/tundev/tundev.go @@ -25,7 +25,6 @@ import ( "gvisor.dev/gvisor/pkg/errors/linuxerr" "gvisor.dev/gvisor/pkg/hostarch" "gvisor.dev/gvisor/pkg/sentry/arch" - "gvisor.dev/gvisor/pkg/sentry/fsimpl/devtmpfs" "gvisor.dev/gvisor/pkg/sentry/inet" "gvisor.dev/gvisor/pkg/sentry/kernel" "gvisor.dev/gvisor/pkg/sentry/socket/netstack" @@ -193,11 +192,8 @@ func IsNetTunSupported(s inet.Stack) bool { // Register registers all devices implemented by this package in vfsObj. func Register(vfsObj *vfs.VirtualFilesystem) error { - return vfsObj.RegisterDevice(vfs.CharDevice, netTunDevMajor, netTunDevMinor, tunDevice{}, &vfs.RegisterDeviceOptions{}) -} - -// CreateDevtmpfsFiles creates device special files in dev representing all -// devices implemented by this package. -func CreateDevtmpfsFiles(ctx context.Context, dev *devtmpfs.Accessor) error { - return dev.CreateDeviceFile(ctx, "net/tun", vfs.CharDevice, netTunDevMajor, netTunDevMinor, 0666 /* mode */) + return vfsObj.RegisterDevice(vfs.CharDevice, netTunDevMajor, netTunDevMinor, tunDevice{}, &vfs.RegisterDeviceOptions{ + Pathname: "net/tun", + FilePerms: 0666, + }) } diff --git a/pkg/sentry/fsimpl/devtmpfs/BUILD b/pkg/sentry/fsimpl/dev/BUILD similarity index 78% rename from pkg/sentry/fsimpl/devtmpfs/BUILD rename to pkg/sentry/fsimpl/dev/BUILD index 8de87e60a..d596ca221 100644 --- a/pkg/sentry/fsimpl/devtmpfs/BUILD +++ b/pkg/sentry/fsimpl/dev/BUILD @@ -5,28 +5,27 @@ package(default_applicable_licenses = ["//:license"]) licenses(["notice"]) go_library( - name = "devtmpfs", + name = "dev", srcs = [ - "devtmpfs.go", - "save_restore.go", + "dev.go", ], visibility = ["//pkg/sentry:internal"], deps = [ "//pkg/abi/linux", "//pkg/context", + "//pkg/errors/linuxerr", "//pkg/fspath", "//pkg/sentry/fsimpl/tmpfs", "//pkg/sentry/kernel/auth", "//pkg/sentry/vfs", - "//pkg/sync", ], ) go_test( - name = "devtmpfs_test", + name = "dev_test", size = "small", - srcs = ["devtmpfs_test.go"], - library = ":devtmpfs", + srcs = ["dev_test.go"], + library = ":dev", deps = [ "//pkg/abi/linux", "//pkg/context", diff --git a/pkg/sentry/fsimpl/dev/dev.go b/pkg/sentry/fsimpl/dev/dev.go new file mode 100644 index 000000000..e7ca046b0 --- /dev/null +++ b/pkg/sentry/fsimpl/dev/dev.go @@ -0,0 +1,189 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package dev provides a filesystem implementation for /dev. +package dev + +import ( + "fmt" + "path" + + "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/context" + "gvisor.dev/gvisor/pkg/errors/linuxerr" + "gvisor.dev/gvisor/pkg/fspath" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs" + "gvisor.dev/gvisor/pkg/sentry/kernel/auth" + "gvisor.dev/gvisor/pkg/sentry/vfs" +) + +// Name is the dev filesystem name. +const Name = "dev" + +// FilesystemType implements vfs.FilesystemType. +// +// +stateify savable +type FilesystemType struct{} + +// Name implements vfs.FilesystemType.Name. +func (FilesystemType) Name() string { + return Name +} + +// GetFilesystem implements vfs.FilesystemType.GetFilesystem. +func (fst FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.VirtualFilesystem, creds *auth.Credentials, source string, opts vfs.GetFilesystemOptions) (*vfs.Filesystem, *vfs.Dentry, error) { + mntns, err := vfsObj.NewMountNamespace(ctx, creds, source /* source */, tmpfs.Name, &vfs.MountOptions{GetFilesystemOptions: vfs.GetFilesystemOptions{ + Data: "mode=0755", // opts from drivers/base/devtmpfs.c:devtmpfs_init() + }}, nil) + if err != nil { + return nil, nil, err + } + defer mntns.DecRef(ctx) + + root := mntns.Root(ctx) + defer root.DecRef(ctx) + + iopts, _ := opts.InternalData.(InternalData) // If not provided, zero value is OK. + + // Initialize contents. + if err := userspaceInit(ctx, vfsObj, creds, root, iopts.ShmMode); err != nil { + return nil, nil, err + } + if err := vfsObj.ForEachDevice(func(pathname string, kind vfs.DeviceKind, major, minor uint32, perms uint16) error { + if pathname == "" { + return nil + } + mode := linux.FileMode(perms) + switch kind { + case vfs.CharDevice: + mode |= linux.S_IFCHR + case vfs.BlockDevice: + mode |= linux.S_IFBLK + default: + panic(fmt.Sprintf("invalid DeviceKind: %v", kind)) + } + return CreateDeviceFile(ctx, vfsObj, creds, root, pathname, major, minor, mode, nil /* uid */, nil /* gid */) + }); err != nil { + return nil, nil, err + } + + root.Mount().Filesystem().IncRef() + root.Dentry().IncRef() + return root.Mount().Filesystem(), root.Dentry(), nil +} + +// Release implements vfs.FilesystemType.Release. +func (fst *FilesystemType) Release(ctx context.Context) {} + +// InternalData contains internal data passed in via vfs.GetFilesystemOptions. +type InternalData struct { + // ShmMode indicates the mode to create the /dev/shm dir with. + ShmMode *uint16 +} + +func pathOperationAt(root vfs.VirtualDentry, pathname string) *vfs.PathOperation { + return &vfs.PathOperation{ + Root: root, + Start: root, + Path: fspath.Parse(pathname), + } +} + +// CreateDeviceFile creates a device special file at the given pathname from root. +func CreateDeviceFile(ctx context.Context, vfsObj *vfs.VirtualFilesystem, creds *auth.Credentials, root vfs.VirtualDentry, pathname string, major, minor uint32, mode linux.FileMode, uid, gid *uint32) error { + // Create any parent directories. See + // devtmpfs.c:handle_create()=>create_path(). + parent := path.Dir(pathname) + if err := vfsObj.MkdirAllAt(ctx, parent, root, creds, &vfs.MkdirOptions{ + Mode: 0755, + }, true /* mustBeDir */); err != nil { + return fmt.Errorf("failed to create device parent directory %q: %v", parent, err) + } + created := true + pop := pathOperationAt(root, pathname) + if err := vfsObj.MknodAt(ctx, creds, pop, &vfs.MknodOptions{Mode: mode, DevMajor: major, DevMinor: minor}); err != nil { + if linuxerr.Equals(linuxerr.EEXIST, err) { + // EEXIST is silently ignored; compare + // opencontainers/runc:libcontainer/rootfs_linux.go:createDeviceNode(). + created = false + } else { + return fmt.Errorf("failed to create device file at %q: %w", pathname, err) + } + } + if created && (uid != nil || gid != nil) { + var opts vfs.SetStatOptions + if uid != nil { + opts.Stat.Mask |= linux.STATX_UID + opts.Stat.UID = *uid + } + if gid != nil { + opts.Stat.Mask |= linux.STATX_GID + opts.Stat.GID = *gid + } + if err := vfsObj.SetStatAt(ctx, creds, pop, &opts); err != nil { + return fmt.Errorf("failed to set UID/GID for device file %q: %w", pathname, err) + } + } + return nil +} + +// userspaceInit creates symbolic links and mount points in the devtmpfs +// instance that are created by userspace in Linux. It does not create mounts. +func userspaceInit(ctx context.Context, vfsObj *vfs.VirtualFilesystem, creds *auth.Credentials, root vfs.VirtualDentry, shmMode *uint16) error { + // Initialize symlinks. + for _, symlink := range []struct { + source string + target string + }{ + // systemd: src/shared/dev-setup.c:dev_setup() + {source: "fd", target: "/proc/self/fd"}, + {source: "stdin", target: "/proc/self/fd/0"}, + {source: "stdout", target: "/proc/self/fd/1"}, + {source: "stderr", target: "/proc/self/fd/2"}, + // /proc/kcore is not implemented. + + // Linux implements /dev/ptmx as a device node, but advises + // container implementations to create /dev/ptmx as a symlink + // to pts/ptmx (Documentation/filesystems/devpts.txt). Systemd + // follows this advice (src/nspawn/nspawn.c:setup_pts()), while + // LXC tries to create a bind mount and falls back to a symlink + // (src/lxc/conf.c:lxc_setup_devpts()). + {source: "ptmx", target: "pts/ptmx"}, + } { + if err := vfsObj.SymlinkAt(ctx, creds, pathOperationAt(root, symlink.source), symlink.target); err != nil { + return fmt.Errorf("failed to create symlink %q => %q: %v", symlink.source, symlink.target, err) + } + } + + // systemd: src/core/mount-setup.c:mount_table + for _, dir := range []string{ + "shm", + "pts", + } { + // "The access mode here doesn't really matter too much, since the + // mounted file system will take precedence anyway" + // - systemd: src/core/mount-setup.c:mount_one() + accessMode := linux.FileMode(0755) + if shmMode != nil && dir == "shm" { + accessMode = linux.FileMode(*shmMode) + } + if err := vfsObj.MkdirAt(ctx, creds, pathOperationAt(root, dir), &vfs.MkdirOptions{ + Mode: accessMode, + }); err != nil { + return fmt.Errorf("failed to create directory %q: %v", dir, err) + } + } + + return nil +} diff --git a/pkg/sentry/fsimpl/dev/dev_test.go b/pkg/sentry/fsimpl/dev/dev_test.go new file mode 100644 index 000000000..efedd2725 --- /dev/null +++ b/pkg/sentry/fsimpl/dev/dev_test.go @@ -0,0 +1,154 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev + +import ( + "testing" + + "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/context" + "gvisor.dev/gvisor/pkg/fspath" + "gvisor.dev/gvisor/pkg/sentry/contexttest" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs" + "gvisor.dev/gvisor/pkg/sentry/kernel/auth" + "gvisor.dev/gvisor/pkg/sentry/vfs" +) + +const ( + testDevMajor = 111 + testDevMinor = 11 + testDevPathname = "test" + testDevPerms = 0655 +) + +func setupDev(t *testing.T) (context.Context, *auth.Credentials, *vfs.VirtualFilesystem, vfs.VirtualDentry, func()) { + t.Helper() + + ctx := contexttest.Context(t) + creds := auth.CredentialsFromContext(ctx) + vfsObj := &vfs.VirtualFilesystem{} + if err := vfsObj.Init(ctx); err != nil { + t.Fatalf("VFS init: %v", err) + } + // Register tmpfs. + vfsObj.MustRegisterFilesystemType("tmpfs", tmpfs.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{ + AllowUserMount: true, + }) + vfsObj.MustRegisterFilesystemType(Name, &FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{}) + + vfsObj.RegisterDevice(vfs.CharDevice, testDevMajor, testDevMinor, nil, &vfs.RegisterDeviceOptions{ + GroupName: "test", + Pathname: testDevPathname, + FilePerms: testDevPerms, + }) + + // Create a test mount namespace with devfs mounted at root. + mntns, err := vfsObj.NewMountNamespace(ctx, creds, "dev" /* source */, Name /* fsTypeName */, &vfs.MountOptions{}, nil) + if err != nil { + t.Fatalf("failed to create tmpfs root mount: %v", err) + } + root := mntns.Root(ctx) + + return ctx, creds, vfsObj, root, func() { + root.DecRef(ctx) + mntns.DecRef(ctx) + } +} + +func TestUserspaceFiles(t *testing.T) { + ctx, creds, vfsObj, root, cleanup := setupDev(t) + defer cleanup() + + // Created files should be visible in the test mount namespace. + links := []struct { + source string + target string + }{ + { + source: "fd", + target: "/proc/self/fd", + }, + { + source: "stdin", + target: "/proc/self/fd/0", + }, + { + source: "stdout", + target: "/proc/self/fd/1", + }, + { + source: "stderr", + target: "/proc/self/fd/2", + }, + { + source: "ptmx", + target: "pts/ptmx", + }, + } + + for _, link := range links { + if gotTarget, err := vfsObj.ReadlinkAt(ctx, creds, &vfs.PathOperation{ + Root: root, + Start: root, + Path: fspath.Parse(link.source), + }); err != nil || gotTarget != link.target { + t.Errorf("readlink(%q): got (%q, %v), wanted (%q, nil)", link.source, gotTarget, err, link.target) + } + } + + dirs := []string{"shm", "pts"} + for _, dir := range dirs { + statx, err := vfsObj.StatAt(ctx, creds, &vfs.PathOperation{ + Root: root, + Start: root, + Path: fspath.Parse(dir), + }, &vfs.StatOptions{ + Mask: linux.STATX_MODE, + }) + if err != nil { + t.Errorf("stat(%q): got error %v ", dir, err) + continue + } + if want := uint16(0755) | linux.S_IFDIR; statx.Mode != want { + t.Errorf("stat(%q): got mode %x, want %x", dir, statx.Mode, want) + } + } +} + +func TestDeviceFile(t *testing.T) { + ctx, creds, vfsObj, root, cleanup := setupDev(t) + defer cleanup() + + // Test that the test device is created. + stat, err := vfsObj.StatAt(ctx, creds, &vfs.PathOperation{ + Root: root, + Start: root, + Path: fspath.Parse(testDevPathname), + }, &vfs.StatOptions{ + Mask: linux.STATX_TYPE | linux.STATX_MODE, + }) + if err != nil { + t.Fatalf("failed to stat device file at %q: %v", testDevPathname, err) + } + if stat.RdevMajor != testDevMajor { + t.Errorf("major device number: got %v, wanted %v", stat.RdevMajor, testDevMajor) + } + if stat.RdevMinor != testDevMinor { + t.Errorf("minor device number: got %v, wanted %v", stat.RdevMinor, testDevMinor) + } + if wantMode := uint16(linux.S_IFCHR | testDevPerms); stat.Mode != wantMode { + t.Errorf("device file mode: got %v, wanted %v", stat.Mode, wantMode) + } +} diff --git a/pkg/sentry/fsimpl/devtmpfs/devtmpfs.go b/pkg/sentry/fsimpl/devtmpfs/devtmpfs.go deleted file mode 100644 index 45f58861a..000000000 --- a/pkg/sentry/fsimpl/devtmpfs/devtmpfs.go +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package devtmpfs provides an implementation of /dev based on tmpfs, -// analogous to Linux's devtmpfs. -package devtmpfs - -import ( - "fmt" - "path" - - "gvisor.dev/gvisor/pkg/abi/linux" - "gvisor.dev/gvisor/pkg/context" - "gvisor.dev/gvisor/pkg/fspath" - "gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs" - "gvisor.dev/gvisor/pkg/sentry/kernel/auth" - "gvisor.dev/gvisor/pkg/sentry/vfs" - "gvisor.dev/gvisor/pkg/sync" -) - -// Name is the default filesystem name. -const Name = "devtmpfs" - -// FilesystemType implements vfs.FilesystemType. -// -// +stateify savable -type FilesystemType struct { - initOnce sync.Once `state:"nosave"` // FIXME(gvisor.dev/issue/1663): not yet supported. - initErr error - - // fs is the tmpfs filesystem that backs all mounts of this FilesystemType. - // root is fs' root. fs and root are immutable. - fs *vfs.Filesystem - root *vfs.Dentry -} - -// Name implements vfs.FilesystemType.Name. -func (*FilesystemType) Name() string { - return Name -} - -// GetFilesystem implements vfs.FilesystemType.GetFilesystem. -func (fst *FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.VirtualFilesystem, creds *auth.Credentials, source string, opts vfs.GetFilesystemOptions) (*vfs.Filesystem, *vfs.Dentry, error) { - fst.initOnce.Do(func() { - fs, root, err := tmpfs.FilesystemType{}.GetFilesystem(ctx, vfsObj, creds, "" /* source */, vfs.GetFilesystemOptions{ - Data: "mode=0755", // opts from drivers/base/devtmpfs.c:devtmpfs_init() - }) - if err != nil { - fst.initErr = err - return - } - fst.fs = fs - fst.root = root - }) - if fst.initErr != nil { - return nil, nil, fst.initErr - } - fst.fs.IncRef() - fst.root.IncRef() - return fst.fs, fst.root, nil -} - -// Release implements vfs.FilesystemType.Release. -func (fst *FilesystemType) Release(ctx context.Context) { - if fst.fs != nil { - // Release the original reference obtained when creating the filesystem. - fst.root.DecRef(ctx) - fst.fs.DecRef(ctx) - } -} - -// Accessor allows devices to create device special files in devtmpfs. -type Accessor struct { - vfsObj *vfs.VirtualFilesystem - mntns *vfs.MountNamespace - root vfs.VirtualDentry - creds *auth.Credentials -} - -// NewAccessor returns an Accessor that supports creation of device special -// files in the devtmpfs instance registered with name fsTypeName in vfsObj. -func NewAccessor(ctx context.Context, vfsObj *vfs.VirtualFilesystem, creds *auth.Credentials, fsTypeName string) (*Accessor, error) { - mntns, err := vfsObj.NewMountNamespace(ctx, creds, "devtmpfs" /* source */, fsTypeName, &vfs.MountOptions{}, nil) - if err != nil { - return nil, err - } - // Pass a reference on root to the Accessor. - root := mntns.Root(ctx) - return &Accessor{ - vfsObj: vfsObj, - mntns: mntns, - root: root, - creds: creds, - }, nil -} - -// Release must be called when a is no longer in use. -func (a *Accessor) Release(ctx context.Context) { - a.root.DecRef(ctx) - a.mntns.DecRef(ctx) -} - -// accessorContext implements context.Context by extending an existing -// context.Context with an Accessor's values for VFS-relevant state. -type accessorContext struct { - context.Context - a *Accessor -} - -func (a *Accessor) wrapContext(ctx context.Context) *accessorContext { - return &accessorContext{ - Context: ctx, - a: a, - } -} - -// Value implements context.Context.Value. -func (ac *accessorContext) Value(key any) any { - switch key { - case vfs.CtxMountNamespace: - ac.a.mntns.IncRef() - return ac.a.mntns - case vfs.CtxRoot: - ac.a.root.IncRef() - return ac.a.root - default: - return ac.Context.Value(key) - } -} - -func (a *Accessor) pathOperationAt(pathname string) *vfs.PathOperation { - return &vfs.PathOperation{ - Root: a.root, - Start: a.root, - Path: fspath.Parse(pathname), - } -} - -// CreateDeviceFile creates a device special file at the given pathname in the -// devtmpfs instance accessed by the Accessor. -func (a *Accessor) CreateDeviceFile(ctx context.Context, pathname string, kind vfs.DeviceKind, major, minor uint32, perms uint16) error { - actx := a.wrapContext(ctx) - - mode := (linux.FileMode)(perms) - switch kind { - case vfs.BlockDevice: - mode |= linux.S_IFBLK - case vfs.CharDevice: - mode |= linux.S_IFCHR - default: - panic(fmt.Sprintf("invalid vfs.DeviceKind: %v", kind)) - } - - // Create any parent directories. See - // devtmpfs.c:handle_create()=>path_create(). - parent := path.Dir(pathname) - if err := a.vfsObj.MkdirAllAt(ctx, parent, a.root, a.creds, &vfs.MkdirOptions{ - Mode: 0755, - }, true /* mustBeDir */); err != nil { - return fmt.Errorf("failed to create device parent directory %q: %v", parent, err) - } - - // NOTE: Linux's devtmpfs refuses to automatically delete files it didn't - // create, which it recognizes by storing a pointer to the kdevtmpfs struct - // thread in struct inode::i_private. Accessor doesn't yet support deletion - // of files at all, and probably won't as long as we don't need to support - // kernel modules, so this is moot for now. - return a.vfsObj.MknodAt(actx, a.creds, a.pathOperationAt(pathname), &vfs.MknodOptions{ - Mode: mode, - DevMajor: major, - DevMinor: minor, - }) -} - -// UserspaceInit creates symbolic links and mount points in the devtmpfs -// instance accessed by the Accessor that are created by userspace in Linux. It -// does not create mounts. -func (a *Accessor) UserspaceInit(ctx context.Context) error { - actx := a.wrapContext(ctx) - - // Initialize symlinks. - for _, symlink := range []struct { - source string - target string - }{ - // systemd: src/shared/dev-setup.c:dev_setup() - {source: "fd", target: "/proc/self/fd"}, - {source: "stdin", target: "/proc/self/fd/0"}, - {source: "stdout", target: "/proc/self/fd/1"}, - {source: "stderr", target: "/proc/self/fd/2"}, - // /proc/kcore is not implemented. - - // Linux implements /dev/ptmx as a device node, but advises - // container implementations to create /dev/ptmx as a symlink - // to pts/ptmx (Documentation/filesystems/devpts.txt). Systemd - // follows this advice (src/nspawn/nspawn.c:setup_pts()), while - // LXC tries to create a bind mount and falls back to a symlink - // (src/lxc/conf.c:lxc_setup_devpts()). - {source: "ptmx", target: "pts/ptmx"}, - } { - if err := a.vfsObj.SymlinkAt(actx, a.creds, a.pathOperationAt(symlink.source), symlink.target); err != nil { - return fmt.Errorf("failed to create symlink %q => %q: %v", symlink.source, symlink.target, err) - } - } - - // systemd: src/core/mount-setup.c:mount_table - for _, dir := range []string{ - "shm", - "pts", - } { - if err := a.vfsObj.MkdirAt(actx, a.creds, a.pathOperationAt(dir), &vfs.MkdirOptions{ - // systemd: src/core/mount-setup.c:mount_one() - Mode: 0755, - }); err != nil { - return fmt.Errorf("failed to create directory %q: %v", dir, err) - } - } - - return nil -} diff --git a/pkg/sentry/fsimpl/devtmpfs/devtmpfs_test.go b/pkg/sentry/fsimpl/devtmpfs/devtmpfs_test.go deleted file mode 100644 index 9fe4c56c9..000000000 --- a/pkg/sentry/fsimpl/devtmpfs/devtmpfs_test.go +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package devtmpfs - -import ( - "path" - "testing" - - "gvisor.dev/gvisor/pkg/abi/linux" - "gvisor.dev/gvisor/pkg/context" - "gvisor.dev/gvisor/pkg/fspath" - "gvisor.dev/gvisor/pkg/sentry/contexttest" - "gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs" - "gvisor.dev/gvisor/pkg/sentry/kernel/auth" - "gvisor.dev/gvisor/pkg/sentry/vfs" -) - -const devPath = "/dev" - -func setupDevtmpfs(t *testing.T) (context.Context, *auth.Credentials, *vfs.VirtualFilesystem, vfs.VirtualDentry, func()) { - t.Helper() - - ctx := contexttest.Context(t) - creds := auth.CredentialsFromContext(ctx) - vfsObj := &vfs.VirtualFilesystem{} - if err := vfsObj.Init(ctx); err != nil { - t.Fatalf("VFS init: %v", err) - } - // Register tmpfs just so that we can have a root filesystem that isn't - // devtmpfs. - vfsObj.MustRegisterFilesystemType("tmpfs", tmpfs.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{ - AllowUserMount: true, - }) - vfsObj.MustRegisterFilesystemType("devtmpfs", &FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{ - AllowUserMount: true, - }) - - // Create a test mount namespace with devtmpfs mounted at "/dev". - mntns, err := vfsObj.NewMountNamespace(ctx, creds, "tmpfs" /* source */, "tmpfs" /* fsTypeName */, &vfs.MountOptions{}, nil) - if err != nil { - t.Fatalf("failed to create tmpfs root mount: %v", err) - } - root := mntns.Root(ctx) - devpop := vfs.PathOperation{ - Root: root, - Start: root, - Path: fspath.Parse(devPath), - } - if err := vfsObj.MkdirAt(ctx, creds, &devpop, &vfs.MkdirOptions{ - Mode: 0755, - }); err != nil { - t.Fatalf("failed to create mount point: %v", err) - } - if _, err := vfsObj.MountAt(ctx, creds, "devtmpfs" /* source */, &devpop, "devtmpfs" /* fsTypeName */, &vfs.MountOptions{}); err != nil { - t.Fatalf("failed to mount devtmpfs: %v", err) - } - - return ctx, creds, vfsObj, root, func() { - root.DecRef(ctx) - mntns.DecRef(ctx) - } -} - -func TestUserspaceInit(t *testing.T) { - ctx, creds, vfsObj, root, cleanup := setupDevtmpfs(t) - defer cleanup() - - a, err := NewAccessor(ctx, vfsObj, creds, "devtmpfs") - if err != nil { - t.Fatalf("failed to create devtmpfs.Accessor: %v", err) - } - defer a.Release(ctx) - - // Create "userspace-initialized" files using a devtmpfs.Accessor. - if err := a.UserspaceInit(ctx); err != nil { - t.Fatalf("failed to userspace-initialize devtmpfs: %v", err) - } - - // Created files should be visible in the test mount namespace. - links := []struct { - source string - target string - }{ - { - source: "fd", - target: "/proc/self/fd", - }, - { - source: "stdin", - target: "/proc/self/fd/0", - }, - { - source: "stdout", - target: "/proc/self/fd/1", - }, - { - source: "stderr", - target: "/proc/self/fd/2", - }, - { - source: "ptmx", - target: "pts/ptmx", - }, - } - - for _, link := range links { - abspath := path.Join(devPath, link.source) - if gotTarget, err := vfsObj.ReadlinkAt(ctx, creds, &vfs.PathOperation{ - Root: root, - Start: root, - Path: fspath.Parse(abspath), - }); err != nil || gotTarget != link.target { - t.Errorf("readlink(%q): got (%q, %v), wanted (%q, nil)", abspath, gotTarget, err, link.target) - } - } - - dirs := []string{"shm", "pts"} - for _, dir := range dirs { - abspath := path.Join(devPath, dir) - statx, err := vfsObj.StatAt(ctx, creds, &vfs.PathOperation{ - Root: root, - Start: root, - Path: fspath.Parse(abspath), - }, &vfs.StatOptions{ - Mask: linux.STATX_MODE, - }) - if err != nil { - t.Errorf("stat(%q): got error %v ", abspath, err) - continue - } - if want := uint16(0755) | linux.S_IFDIR; statx.Mode != want { - t.Errorf("stat(%q): got mode %x, want %x", abspath, statx.Mode, want) - } - } -} - -func TestCreateDeviceFile(t *testing.T) { - ctx, creds, vfsObj, root, cleanup := setupDevtmpfs(t) - defer cleanup() - - a, err := NewAccessor(ctx, vfsObj, creds, "devtmpfs") - if err != nil { - t.Fatalf("failed to create devtmpfs.Accessor: %v", err) - } - defer a.Release(ctx) - - devFiles := []struct { - path string - kind vfs.DeviceKind - major uint32 - minor uint32 - perms uint16 - }{ - { - path: "dummy", - kind: vfs.CharDevice, - major: 12, - minor: 34, - perms: 0600, - }, - { - path: "foo/bar", - kind: vfs.BlockDevice, - major: 13, - minor: 35, - perms: 0660, - }, - { - path: "foo/baz", - kind: vfs.CharDevice, - major: 12, - minor: 40, - perms: 0666, - }, - { - path: "a/b/c/d/e", - kind: vfs.BlockDevice, - major: 12, - minor: 34, - perms: 0600, - }, - } - - for _, f := range devFiles { - if err := a.CreateDeviceFile(ctx, f.path, f.kind, f.major, f.minor, f.perms); err != nil { - t.Fatalf("failed to create device file: %v", err) - } - // The device special file should be visible in the test mount namespace. - abspath := path.Join(devPath, f.path) - stat, err := vfsObj.StatAt(ctx, creds, &vfs.PathOperation{ - Root: root, - Start: root, - Path: fspath.Parse(abspath), - }, &vfs.StatOptions{ - Mask: linux.STATX_TYPE | linux.STATX_MODE, - }) - if err != nil { - t.Fatalf("failed to stat device file at %q: %v", abspath, err) - } - if stat.RdevMajor != f.major { - t.Errorf("major device number: got %v, wanted %v", stat.RdevMajor, f.major) - } - if stat.RdevMinor != f.minor { - t.Errorf("minor device number: got %v, wanted %v", stat.RdevMinor, f.minor) - } - wantMode := f.perms - switch f.kind { - case vfs.CharDevice: - wantMode |= linux.S_IFCHR - case vfs.BlockDevice: - wantMode |= linux.S_IFBLK - } - if stat.Mode != wantMode { - t.Errorf("device file mode: got %v, wanted %v", stat.Mode, wantMode) - } - } -} diff --git a/pkg/sentry/fsimpl/devtmpfs/save_restore.go b/pkg/sentry/fsimpl/devtmpfs/save_restore.go deleted file mode 100644 index 28832d850..000000000 --- a/pkg/sentry/fsimpl/devtmpfs/save_restore.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package devtmpfs - -// afterLoad is invoked by stateify. -func (fst *FilesystemType) afterLoad() { - if fst.fs != nil { - // Ensure that we don't create another filesystem. - fst.initOnce.Do(func() {}) - } -} diff --git a/pkg/sentry/fsimpl/fuse/BUILD b/pkg/sentry/fsimpl/fuse/BUILD index f460d5ca4..ef61d7d24 100644 --- a/pkg/sentry/fsimpl/fuse/BUILD +++ b/pkg/sentry/fsimpl/fuse/BUILD @@ -59,7 +59,6 @@ go_library( "//pkg/marshal/primitive", "//pkg/refs", "//pkg/safemem", - "//pkg/sentry/fsimpl/devtmpfs", "//pkg/sentry/fsimpl/kernfs", "//pkg/sentry/fsutil", "//pkg/sentry/kernel", diff --git a/pkg/sentry/fsimpl/fuse/register.go b/pkg/sentry/fsimpl/fuse/register.go index b5b581152..d91c3811c 100644 --- a/pkg/sentry/fsimpl/fuse/register.go +++ b/pkg/sentry/fsimpl/fuse/register.go @@ -16,27 +16,14 @@ package fuse import ( "gvisor.dev/gvisor/pkg/abi/linux" - "gvisor.dev/gvisor/pkg/context" - "gvisor.dev/gvisor/pkg/sentry/fsimpl/devtmpfs" "gvisor.dev/gvisor/pkg/sentry/vfs" ) // Register registers the FUSE device with vfsObj. func Register(vfsObj *vfs.VirtualFilesystem) error { - if err := vfsObj.RegisterDevice(vfs.CharDevice, linux.MISC_MAJOR, fuseDevMinor, fuseDevice{}, &vfs.RegisterDeviceOptions{ + return vfsObj.RegisterDevice(vfs.CharDevice, linux.MISC_MAJOR, fuseDevMinor, fuseDevice{}, &vfs.RegisterDeviceOptions{ GroupName: "misc", - }); err != nil { - return err - } - - return nil -} - -// CreateDevtmpfsFile creates a device special file in devtmpfs. -func CreateDevtmpfsFile(ctx context.Context, dev *devtmpfs.Accessor) error { - if err := dev.CreateDeviceFile(ctx, "fuse", vfs.CharDevice, linux.MISC_MAJOR, fuseDevMinor, 0666 /* mode */); err != nil { - return err - } - - return nil + Pathname: "fuse", + FilePerms: 0666, + }) } diff --git a/pkg/sentry/vfs/device.go b/pkg/sentry/vfs/device.go index 52b6640aa..9ccb5796b 100644 --- a/pkg/sentry/vfs/device.go +++ b/pkg/sentry/vfs/device.go @@ -74,6 +74,12 @@ type RegisterDeviceOptions struct { // /proc/devices. If GroupName is empty, this registration will not be // shown in /proc/devices. GroupName string + // Pathname is the name for the device file of this device in /dev directory. + // If Pathname is empty, then no device file is created. + Pathname string + // FilePerms are the permission bits to create the device file with. Only + // used if Pathname is provided. + FilePerms uint16 } // RegisterDevice registers the given Device in vfs with the given major and @@ -92,6 +98,18 @@ func (vfs *VirtualFilesystem) RegisterDevice(kind DeviceKind, major, minor uint3 return nil } +// ForEachDevice calls the given callback for each registered device. +func (vfs *VirtualFilesystem) ForEachDevice(cb func(pathname string, kind DeviceKind, major, minor uint32, perms uint16) error) error { + vfs.devicesMu.Lock() + defer vfs.devicesMu.Unlock() + for tup, dev := range vfs.devices { + if err := cb(dev.opts.Pathname, tup.kind, tup.major, tup.minor, dev.opts.FilePerms); err != nil { + return err + } + } + return nil +} + // OpenDeviceSpecialFile returns a FileDescription representing the given // device. func (vfs *VirtualFilesystem) OpenDeviceSpecialFile(ctx context.Context, mnt *Mount, d *Dentry, kind DeviceKind, major, minor uint32, opts *OpenOptions) (*FileDescription, error) { diff --git a/runsc/boot/BUILD b/runsc/boot/BUILD index 227737057..3ffdf1d91 100644 --- a/runsc/boot/BUILD +++ b/runsc/boot/BUILD @@ -32,6 +32,7 @@ go_library( deps = [ "//pkg/abi", "//pkg/abi/linux", + "//pkg/abi/nvgpu", "//pkg/abi/tpu", "//pkg/bpf", "//pkg/cleanup", @@ -59,8 +60,8 @@ go_library( "//pkg/sentry/devices/tundev", "//pkg/sentry/fdimport", "//pkg/sentry/fsimpl/cgroupfs", + "//pkg/sentry/fsimpl/dev", "//pkg/sentry/fsimpl/devpts", - "//pkg/sentry/fsimpl/devtmpfs", "//pkg/sentry/fsimpl/erofs", "//pkg/sentry/fsimpl/fuse", "//pkg/sentry/fsimpl/gofer", diff --git a/runsc/boot/loader.go b/runsc/boot/loader.go index 47a9282e8..e680b02d5 100644 --- a/runsc/boot/loader.go +++ b/runsc/boot/loader.go @@ -123,6 +123,10 @@ type containerInfo struct { // nvidiaUVMDevMajor is the device major number used for nvidia-uvm. nvidiaUVMDevMajor uint32 + + // nvidiaDevMinors is a list of device minors for Nvidia GPU devices exposed + // to the sandbox. + nvidiaDevMinors NvidiaDevMinors } // Loader keeps state needed to start the kernel and run the container. @@ -282,6 +286,9 @@ type Args struct { // ProfileOpts contains the set of profiles to enable and the // corresponding FDs where profile data will be written. ProfileOpts profile.Opts + // NvidiaDevMinors is a list of device minors for Nvidia GPU devices exposed + // to the sandbox. + NvidiaDevMinors NvidiaDevMinors } // make sure stdioFDs are always the same on initial start and on restore @@ -315,6 +322,7 @@ func New(args Args) (*Loader, error) { conf: args.Conf, spec: args.Spec, goferMountConfs: args.GoferMountConfs, + nvidiaDevMinors: args.NvidiaDevMinors, } // Make host FDs stable between invocations. Host FDs must map to the exact @@ -857,6 +865,7 @@ func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid st goferFilestoreFDs: goferFilestoreFDs, goferMountConfs: goferMountConfs, nvidiaUVMDevMajor: l.nvidiaUVMDevMajor, + nvidiaDevMinors: l.root.nvidiaDevMinors, } info.procArgs, err = createProcessArgs(cid, spec, creds, l.k, pidns) if err != nil { diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index 323a242e7..8d777bdae 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -26,6 +26,7 @@ import ( specs "github.com/opencontainers/runtime-spec/specs-go" "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/abi/nvgpu" "gvisor.dev/gvisor/pkg/abi/tpu" "gvisor.dev/gvisor/pkg/cleanup" "gvisor.dev/gvisor/pkg/context" @@ -39,8 +40,8 @@ import ( "gvisor.dev/gvisor/pkg/sentry/devices/ttydev" "gvisor.dev/gvisor/pkg/sentry/devices/tundev" "gvisor.dev/gvisor/pkg/sentry/fsimpl/cgroupfs" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/dev" "gvisor.dev/gvisor/pkg/sentry/fsimpl/devpts" - "gvisor.dev/gvisor/pkg/sentry/fsimpl/devtmpfs" "gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs" "gvisor.dev/gvisor/pkg/sentry/fsimpl/fuse" "gvisor.dev/gvisor/pkg/sentry/fsimpl/gofer" @@ -86,7 +87,6 @@ var tmpfsAllowedData = []string{"mode", "size", "uid", "gid"} func registerFilesystems(k *kernel.Kernel, info *containerInfo) error { ctx := k.SupervisorContext() - creds := auth.NewRootCredentials(k.RootUserNamespace()) vfsObj := k.VFS() vfsObj.MustRegisterFilesystemType(cgroupfs.Name, &cgroupfs.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{ @@ -99,10 +99,7 @@ func registerFilesystems(k *kernel.Kernel, info *containerInfo) error { // usable state. AllowUserMount: true, }) - vfsObj.MustRegisterFilesystemType(devtmpfs.Name, &devtmpfs.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{ - AllowUserMount: true, - AllowUserList: true, - }) + vfsObj.MustRegisterFilesystemType(dev.Name, &dev.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{}) vfsObj.MustRegisterFilesystemType(erofs.Name, &erofs.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{ AllowUserList: true, }) @@ -151,36 +148,11 @@ func registerFilesystems(k *kernel.Kernel, info *containerInfo) error { return fmt.Errorf("registering fusedev: %w", err) } - // Setup files in devtmpfs. - a, err := devtmpfs.NewAccessor(ctx, vfsObj, creds, devtmpfs.Name) - if err != nil { - return fmt.Errorf("creating devtmpfs accessor: %w", err) - } - defer a.Release(ctx) - - if err := a.UserspaceInit(ctx); err != nil { - return fmt.Errorf("initializing userspace: %w", err) - } - if err := memdev.CreateDevtmpfsFiles(ctx, a); err != nil { - return fmt.Errorf("creating memdev devtmpfs files: %w", err) - } - if err := ttydev.CreateDevtmpfsFiles(ctx, a); err != nil { - return fmt.Errorf("creating ttydev devtmpfs files: %w", err) - } - if tunSupported { - if err := tundev.CreateDevtmpfsFiles(ctx, a); err != nil { - return fmt.Errorf("creating tundev devtmpfs files: %v", err) - } - } - if err := fuse.CreateDevtmpfsFile(ctx, a); err != nil { - return fmt.Errorf("creating fusedev devtmpfs files: %w", err) - } - - if err := nvproxyRegisterDevicesAndCreateFiles(ctx, info, k, vfsObj, a); err != nil { + if err := nvproxyRegisterDevices(info, vfsObj); err != nil { return err } - if err := tpuProxyRegisterDevicesAndCreateFiles(ctx, info, k, vfsObj, a); err != nil { + if err := tpuProxyRegisterDevices(info, vfsObj); err != nil { return err } @@ -246,7 +218,7 @@ func compileMounts(spec *specs.Spec, conf *config.Config) []specs.Mount { case "/sys": sysMounted = true case "/dev": - m.Type = devtmpfs.Name + m.Type = dev.Name devMounted = true case "/dev/pts": m.Type = devpts.Name @@ -290,7 +262,7 @@ func compileMounts(spec *specs.Spec, conf *config.Config) []specs.Mount { } if !devMounted { mandatoryMounts = append(mandatoryMounts, specs.Mount{ - Type: devtmpfs.Name, + Type: dev.Name, Destination: "/dev", }) } @@ -819,7 +791,7 @@ func getMountNameAndOptions(spec *specs.Spec, conf *config.Config, m *mountInfo, // Find filesystem name and FS specific data field. switch m.mount.Type { - case devpts.Name, devtmpfs.Name, proc.Name: + case devpts.Name, dev.Name, proc.Name: // Nothing to do. case Nonefs: @@ -1093,74 +1065,63 @@ func (c *containerMounter) configureRestore(ctx context.Context) (context.Contex } func createDeviceFiles(ctx context.Context, creds *auth.Credentials, info *containerInfo, vfsObj *vfs.VirtualFilesystem, root vfs.VirtualDentry) error { - if info.spec.Linux == nil { - return nil + if info.spec.Linux != nil { + // Create any device files specified in the spec. + for _, dev := range info.spec.Linux.Devices { + if err := createDeviceFile(ctx, creds, info, vfsObj, root, dev); err != nil { + return err + } + } } - for _, dev := range info.spec.Linux.Devices { - pop := vfs.PathOperation{ - Root: root, - Start: root, - Path: fspath.Parse(dev.Path), + if specutils.GPUFunctionalityRequested(info.spec, info.conf) && info.conf.NVProxyDocker { + // In Docker mode, devices are not injected into spec.Linux.Devices. So + // manually create appropriate device files. + mode := os.FileMode(0666) + nvidiaDevs := []specs.LinuxDevice{ + specs.LinuxDevice{Path: "/dev/nvidiactl", Type: "c", Major: nvgpu.NV_MAJOR_DEVICE_NUMBER, Minor: nvgpu.NV_CONTROL_DEVICE_MINOR, FileMode: &mode}, + specs.LinuxDevice{Path: "/dev/nvidia-uvm", Type: "c", Major: int64(info.nvidiaUVMDevMajor), Minor: nvgpu.NVIDIA_UVM_PRIMARY_MINOR_NUMBER, FileMode: &mode}, } - opts := vfs.MknodOptions{ - Mode: linux.FileMode(dev.FileMode.Perm()), + for _, minor := range info.nvidiaDevMinors { + nvidiaDevs = append(nvidiaDevs, specs.LinuxDevice{Path: fmt.Sprintf("/dev/nvidia%d", minor), Type: "c", Major: nvgpu.NV_MAJOR_DEVICE_NUMBER, Minor: int64(minor), FileMode: &mode}) } - // See https://github.com/opencontainers/runtime-spec/blob/main/config-linux.md#devices. - switch dev.Type { - case "b": - opts.Mode |= linux.S_IFBLK - opts.DevMajor = uint32(dev.Major) - opts.DevMinor = uint32(dev.Minor) - case "c", "u": - opts.Mode |= linux.S_IFCHR - opts.DevMajor = uint32(dev.Major) - opts.DevMinor = uint32(dev.Minor) - case "p": - opts.Mode |= linux.S_IFIFO - default: - return fmt.Errorf("specified device at %q has invalid type %q", dev.Path, dev.Type) - } - if dev.Path == "/dev/nvidia-uvm" && info.nvidiaUVMDevMajor != 0 && opts.DevMajor != info.nvidiaUVMDevMajor { - // nvidia-uvm's major device number is dynamically assigned, so the - // number that it has on the host may differ from the number that - // it has in sentry VFS; switch from the former to the latter. - log.Infof("Switching /dev/nvidia-uvm device major number from %d to %d", dev.Major, info.nvidiaUVMDevMajor) - opts.DevMajor = info.nvidiaUVMDevMajor - } - if err := vfsObj.MkdirAllAt(ctx, path.Dir(dev.Path), root, creds, &vfs.MkdirOptions{ - Mode: 0o755, - }, true /* mustBeDir */); err != nil { - return fmt.Errorf("failed to create ancestor directories of %q: %w", dev.Path, err) - } - // EEXIST is silently ignored; compare - // opencontainers/runc:libcontainer/rootfs_linux.go:createDeviceNode(). - created := true - if err := vfsObj.MknodAt(ctx, creds, &pop, &opts); err != nil && !linuxerr.Equals(linuxerr.EEXIST, err) { - if linuxerr.Equals(linuxerr.EEXIST, err) { - created = false - } else { - return fmt.Errorf("failed to create device file at %q: %w", dev.Path, err) - } - } - if created && (dev.UID != nil || dev.GID != nil) { - var opts vfs.SetStatOptions - if dev.UID != nil { - opts.Stat.Mask |= linux.STATX_UID - opts.Stat.UID = *dev.UID - } - if dev.GID != nil { - opts.Stat.Mask |= linux.STATX_GID - opts.Stat.GID = *dev.GID - } - if err := vfsObj.SetStatAt(ctx, creds, &pop, &opts); err != nil { - return fmt.Errorf("failed to set UID/GID for device file %q: %w", dev.Path, err) + for _, nvidiaDev := range nvidiaDevs { + if err := createDeviceFile(ctx, creds, info, vfsObj, root, nvidiaDev); err != nil { + return err } } } return nil } -func tpuProxyRegisterDevicesAndCreateFiles(ctx context.Context, info *containerInfo, k *kernel.Kernel, vfsObj *vfs.VirtualFilesystem, a *devtmpfs.Accessor) error { +func createDeviceFile(ctx context.Context, creds *auth.Credentials, info *containerInfo, vfsObj *vfs.VirtualFilesystem, root vfs.VirtualDentry, devSpec specs.LinuxDevice) error { + mode := linux.FileMode(devSpec.FileMode.Perm()) + var major, minor uint32 + // See https://github.com/opencontainers/runtime-spec/blob/main/config-linux.md#devices. + switch devSpec.Type { + case "b": + mode |= linux.S_IFBLK + major = uint32(devSpec.Major) + minor = uint32(devSpec.Minor) + case "c", "u": + mode |= linux.S_IFCHR + major = uint32(devSpec.Major) + minor = uint32(devSpec.Minor) + case "p": + mode |= linux.S_IFIFO + default: + return fmt.Errorf("specified device at %q has invalid type %q", devSpec.Path, devSpec.Type) + } + if devSpec.Path == "/dev/nvidia-uvm" && info.nvidiaUVMDevMajor != 0 && major != info.nvidiaUVMDevMajor { + // nvidia-uvm's major device number is dynamically assigned, so the + // number that it has on the host may differ from the number that + // it has in sentry VFS; switch from the former to the latter. + log.Infof("Switching /dev/nvidia-uvm device major number from %d to %d", devSpec.Major, info.nvidiaUVMDevMajor) + major = info.nvidiaUVMDevMajor + } + return dev.CreateDeviceFile(ctx, vfsObj, creds, root, devSpec.Path, major, minor, mode, devSpec.UID, devSpec.GID) +} + +func tpuProxyRegisterDevices(info *containerInfo, vfsObj *vfs.VirtualFilesystem) error { if !specutils.TPUProxyIsEnabled(info.spec, info.conf) { return nil } @@ -1198,19 +1159,16 @@ func tpuProxyRegisterDevicesAndCreateFiles(ctx context.Context, info *containerI if err := accel.RegisterTPUV4Device(vfsObj, uint32(deviceNum), deviceID == tpu.TPUV4liteDeviceID); err != nil { return fmt.Errorf("registering accel driver: %w", err) } - if err := accel.CreateDevtmpfsFile(ctx, a, uint32(deviceNum)); err != nil { - return fmt.Errorf("creating accel device file %q: %w", deviceNum, err) - } } } return nil } -func nvproxyRegisterDevicesAndCreateFiles(ctx context.Context, info *containerInfo, k *kernel.Kernel, vfsObj *vfs.VirtualFilesystem, a *devtmpfs.Accessor) error { +func nvproxyRegisterDevices(info *containerInfo, vfsObj *vfs.VirtualFilesystem) error { if !specutils.GPUFunctionalityRequested(info.spec, info.conf) { return nil } - uvmDevMajor, err := k.VFS().GetDynamicCharDevMajor() + uvmDevMajor, err := vfsObj.GetDynamicCharDevMajor() if err != nil { return fmt.Errorf("reserving device major number for nvidia-uvm: %w", err) } @@ -1218,22 +1176,5 @@ func nvproxyRegisterDevicesAndCreateFiles(ctx context.Context, info *containerIn return fmt.Errorf("registering nvproxy driver: %w", err) } info.nvidiaUVMDevMajor = uvmDevMajor - if info.conf.NVProxyDocker { - // In Docker mode, create all the device files now. - // In non-Docker mode, these are instead created as part of - // `createDeviceFiles`, using the spec's Device list. - minors, err := specutils.FindAllGPUDevices("/") - if err != nil { - return fmt.Errorf("getting nvidia devices: %w", err) - } - if err := nvproxy.CreateDriverDevtmpfsFiles(ctx, a, uvmDevMajor); err != nil { - return fmt.Errorf("creating nvproxy devtmpfs files: %w", err) - } - for _, minor := range minors { - if err := nvproxy.CreateIndexDevtmpfsFile(ctx, a, minor); err != nil { - return fmt.Errorf("creating nvproxy devtmpfs file for device minor %d: %w", minor, err) - } - } - } return nil } diff --git a/runsc/cmd/boot.go b/runsc/cmd/boot.go index ae218a58a..6691b0c10 100644 --- a/runsc/cmd/boot.go +++ b/runsc/cmd/boot.go @@ -433,6 +433,7 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma PodInitConfigFD: b.podInitConfigFD, SinkFDs: b.sinkFDs.GetArray(), ProfileOpts: b.profileFDs.ToOpts(), + NvidiaDevMinors: b.nvidiaDevMinors, } l, err := boot.New(bootArgs) if err != nil {