diff --git a/pkg/sentry/devices/accel/BUILD b/pkg/sentry/devices/accel/BUILD index 5f9d97e63..617681d50 100644 --- a/pkg/sentry/devices/accel/BUILD +++ b/pkg/sentry/devices/accel/BUILD @@ -29,7 +29,6 @@ go_library( "//pkg/safemem", "//pkg/seccomp", "//pkg/sentry/arch", - "//pkg/sentry/fsimpl/devtmpfs", "//pkg/sentry/fsimpl/eventfd", "//pkg/sentry/kernel", "//pkg/sentry/memmap", diff --git a/pkg/sentry/devices/accel/device.go b/pkg/sentry/devices/accel/device.go index ec09dbfdf..a2d143e74 100644 --- a/pkg/sentry/devices/accel/device.go +++ b/pkg/sentry/devices/accel/device.go @@ -22,7 +22,6 @@ import ( "gvisor.dev/gvisor/pkg/context" "gvisor.dev/gvisor/pkg/errors/linuxerr" "gvisor.dev/gvisor/pkg/fdnotifier" - "gvisor.dev/gvisor/pkg/sentry/fsimpl/devtmpfs" "gvisor.dev/gvisor/pkg/sentry/kernel" "gvisor.dev/gvisor/pkg/sentry/vfs" "gvisor.dev/gvisor/pkg/sync" @@ -81,11 +80,6 @@ func (dev *tpuV4Device) Open(ctx context.Context, mnt *vfs.Mount, vfsd *vfs.Dent return &fd.vfsfd, nil } -// CreateDevtmpfsFile creates a /dev/accel[0-9]+ device file. -func CreateDevtmpfsFile(ctx context.Context, dev *devtmpfs.Accessor, num uint32) error { - return dev.CreateDeviceFile(ctx, fmt.Sprintf("accel%d", num), vfs.CharDevice, linux.ACCEL_MAJOR, num, 0666) -} - // RegisterTPUV4Device registers all devices implemented by this package in vfsObj. func RegisterTPUV4Device(vfsObj *vfs.VirtualFilesystem, minor uint32, lite bool) error { return vfsObj.RegisterDevice(vfs.CharDevice, linux.ACCEL_MAJOR, minor, &tpuV4Device{ diff --git a/pkg/sentry/devices/memdev/BUILD b/pkg/sentry/devices/memdev/BUILD index 7893fdd25..5e83fe3c8 100644 --- a/pkg/sentry/devices/memdev/BUILD +++ b/pkg/sentry/devices/memdev/BUILD @@ -21,7 +21,6 @@ go_library( "//pkg/errors/linuxerr", "//pkg/rand", "//pkg/safemem", - "//pkg/sentry/fsimpl/devtmpfs", "//pkg/sentry/fsimpl/tmpfs", "//pkg/sentry/kernel", "//pkg/sentry/kernel/auth", diff --git a/pkg/sentry/devices/memdev/memdev.go b/pkg/sentry/devices/memdev/memdev.go index 5759900c4..208cfc472 100644 --- a/pkg/sentry/devices/memdev/memdev.go +++ b/pkg/sentry/devices/memdev/memdev.go @@ -18,42 +18,28 @@ package memdev import ( "gvisor.dev/gvisor/pkg/abi/linux" - "gvisor.dev/gvisor/pkg/context" - "gvisor.dev/gvisor/pkg/sentry/fsimpl/devtmpfs" "gvisor.dev/gvisor/pkg/sentry/vfs" ) // Register registers all devices implemented by this package in vfsObj. func Register(vfsObj *vfs.VirtualFilesystem) error { - for minor, dev := range map[uint32]vfs.Device{ - nullDevMinor: nullDevice{}, - zeroDevMinor: zeroDevice{}, - fullDevMinor: fullDevice{}, - randomDevMinor: randomDevice{}, - urandomDevMinor: randomDevice{}, + for minor, spec := range map[uint32]struct { + dev vfs.Device + pathname string + }{ + nullDevMinor: {nullDevice{}, "null"}, + zeroDevMinor: {zeroDevice{}, "zero"}, + fullDevMinor: {fullDevice{}, "full"}, + randomDevMinor: {randomDevice{}, "random"}, + urandomDevMinor: {randomDevice{}, "urandom"}, } { - if err := vfsObj.RegisterDevice(vfs.CharDevice, linux.MEM_MAJOR, minor, dev, &vfs.RegisterDeviceOptions{ + if err := vfsObj.RegisterDevice(vfs.CharDevice, linux.MEM_MAJOR, minor, spec.dev, &vfs.RegisterDeviceOptions{ GroupName: "mem", + Pathname: spec.pathname, + FilePerms: 0666, }); err != nil { return err } } return nil } - -// CreateDevtmpfsFiles creates device special files in dev representing all -// devices implemented by this package. -func CreateDevtmpfsFiles(ctx context.Context, dev *devtmpfs.Accessor) error { - for minor, name := range map[uint32]string{ - nullDevMinor: "null", - zeroDevMinor: "zero", - fullDevMinor: "full", - randomDevMinor: "random", - urandomDevMinor: "urandom", - } { - if err := dev.CreateDeviceFile(ctx, name, vfs.CharDevice, linux.MEM_MAJOR, minor, 0666 /* mode */); err != nil { - return err - } - } - return nil -} diff --git a/pkg/sentry/devices/nvproxy/BUILD b/pkg/sentry/devices/nvproxy/BUILD index 4fc677f55..77de8af22 100644 --- a/pkg/sentry/devices/nvproxy/BUILD +++ b/pkg/sentry/devices/nvproxy/BUILD @@ -45,7 +45,6 @@ go_library( "//pkg/safemem", "//pkg/seccomp", "//pkg/sentry/arch", - "//pkg/sentry/fsimpl/devtmpfs", "//pkg/sentry/kernel", "//pkg/sentry/memmap", "//pkg/sentry/mm", diff --git a/pkg/sentry/devices/nvproxy/nvproxy.go b/pkg/sentry/devices/nvproxy/nvproxy.go index a92c7edd2..765f581d3 100644 --- a/pkg/sentry/devices/nvproxy/nvproxy.go +++ b/pkg/sentry/devices/nvproxy/nvproxy.go @@ -26,7 +26,6 @@ import ( "gvisor.dev/gvisor/pkg/hostarch" "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/marshal" - "gvisor.dev/gvisor/pkg/sentry/fsimpl/devtmpfs" "gvisor.dev/gvisor/pkg/sentry/mm" "gvisor.dev/gvisor/pkg/sentry/vfs" ) @@ -72,25 +71,6 @@ func Register(vfsObj *vfs.VirtualFilesystem, uvmDevMajor uint32) error { return nil } -// CreateDriverDevtmpfsFiles creates device special files in dev that should -// always exist when this package is enabled. It does not create per-device -// files in dev; see CreateIndexDevtmpfsFile. -func CreateDriverDevtmpfsFiles(ctx context.Context, dev *devtmpfs.Accessor, uvmDevMajor uint32) error { - if err := dev.CreateDeviceFile(ctx, "nvidiactl", vfs.CharDevice, nvgpu.NV_MAJOR_DEVICE_NUMBER, nvgpu.NV_CONTROL_DEVICE_MINOR, 0666); err != nil { - return err - } - if err := dev.CreateDeviceFile(ctx, "nvidia-uvm", vfs.CharDevice, uvmDevMajor, nvgpu.NVIDIA_UVM_PRIMARY_MINOR_NUMBER, 0666); err != nil { - return err - } - return nil -} - -// CreateIndexDevtmpfsFile creates the device special file in dev for the -// device with the given index. -func CreateIndexDevtmpfsFile(ctx context.Context, dev *devtmpfs.Accessor, minor uint32) error { - return dev.CreateDeviceFile(ctx, fmt.Sprintf("nvidia%d", minor), vfs.CharDevice, nvgpu.NV_MAJOR_DEVICE_NUMBER, minor, 0666) -} - // +stateify savable type nvproxy struct { objsMu objsMutex `state:"nosave"` diff --git a/pkg/sentry/devices/ttydev/BUILD b/pkg/sentry/devices/ttydev/BUILD index b93b08b29..609475ca0 100644 --- a/pkg/sentry/devices/ttydev/BUILD +++ b/pkg/sentry/devices/ttydev/BUILD @@ -12,7 +12,6 @@ go_library( "//pkg/abi/linux", "//pkg/context", "//pkg/errors/linuxerr", - "//pkg/sentry/fsimpl/devtmpfs", "//pkg/sentry/vfs", ], ) diff --git a/pkg/sentry/devices/ttydev/ttydev.go b/pkg/sentry/devices/ttydev/ttydev.go index 29b79b5d6..b8e525d79 100644 --- a/pkg/sentry/devices/ttydev/ttydev.go +++ b/pkg/sentry/devices/ttydev/ttydev.go @@ -19,7 +19,6 @@ import ( "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/context" "gvisor.dev/gvisor/pkg/errors/linuxerr" - "gvisor.dev/gvisor/pkg/sentry/fsimpl/devtmpfs" "gvisor.dev/gvisor/pkg/sentry/vfs" ) @@ -43,11 +42,7 @@ func (ttyDevice) Open(ctx context.Context, mnt *vfs.Mount, vfsd *vfs.Dentry, opt func Register(vfsObj *vfs.VirtualFilesystem) error { return vfsObj.RegisterDevice(vfs.CharDevice, linux.TTYAUX_MAJOR, ttyDevMinor, ttyDevice{}, &vfs.RegisterDeviceOptions{ GroupName: "tty", + Pathname: "tty", + FilePerms: 0666, }) } - -// CreateDevtmpfsFiles creates device special files in dev representing all -// devices implemented by this package. -func CreateDevtmpfsFiles(ctx context.Context, dev *devtmpfs.Accessor) error { - return dev.CreateDeviceFile(ctx, "tty", vfs.CharDevice, linux.TTYAUX_MAJOR, ttyDevMinor, 0666 /* mode */) -} diff --git a/pkg/sentry/devices/tundev/BUILD b/pkg/sentry/devices/tundev/BUILD index 993ddd79f..90019caa3 100644 --- a/pkg/sentry/devices/tundev/BUILD +++ b/pkg/sentry/devices/tundev/BUILD @@ -15,7 +15,6 @@ go_library( "//pkg/errors/linuxerr", "//pkg/hostarch", "//pkg/sentry/arch", - "//pkg/sentry/fsimpl/devtmpfs", "//pkg/sentry/inet", "//pkg/sentry/kernel", "//pkg/sentry/socket/netstack", diff --git a/pkg/sentry/devices/tundev/tundev.go b/pkg/sentry/devices/tundev/tundev.go index a0fcbdd8a..340f02fcc 100644 --- a/pkg/sentry/devices/tundev/tundev.go +++ b/pkg/sentry/devices/tundev/tundev.go @@ -25,7 +25,6 @@ import ( "gvisor.dev/gvisor/pkg/errors/linuxerr" "gvisor.dev/gvisor/pkg/hostarch" "gvisor.dev/gvisor/pkg/sentry/arch" - "gvisor.dev/gvisor/pkg/sentry/fsimpl/devtmpfs" "gvisor.dev/gvisor/pkg/sentry/inet" "gvisor.dev/gvisor/pkg/sentry/kernel" "gvisor.dev/gvisor/pkg/sentry/socket/netstack" @@ -193,11 +192,8 @@ func IsNetTunSupported(s inet.Stack) bool { // Register registers all devices implemented by this package in vfsObj. func Register(vfsObj *vfs.VirtualFilesystem) error { - return vfsObj.RegisterDevice(vfs.CharDevice, netTunDevMajor, netTunDevMinor, tunDevice{}, &vfs.RegisterDeviceOptions{}) -} - -// CreateDevtmpfsFiles creates device special files in dev representing all -// devices implemented by this package. -func CreateDevtmpfsFiles(ctx context.Context, dev *devtmpfs.Accessor) error { - return dev.CreateDeviceFile(ctx, "net/tun", vfs.CharDevice, netTunDevMajor, netTunDevMinor, 0666 /* mode */) + return vfsObj.RegisterDevice(vfs.CharDevice, netTunDevMajor, netTunDevMinor, tunDevice{}, &vfs.RegisterDeviceOptions{ + Pathname: "net/tun", + FilePerms: 0666, + }) } diff --git a/pkg/sentry/fsimpl/devtmpfs/BUILD b/pkg/sentry/fsimpl/dev/BUILD similarity index 78% rename from pkg/sentry/fsimpl/devtmpfs/BUILD rename to pkg/sentry/fsimpl/dev/BUILD index 8de87e60a..d596ca221 100644 --- a/pkg/sentry/fsimpl/devtmpfs/BUILD +++ b/pkg/sentry/fsimpl/dev/BUILD @@ -5,28 +5,27 @@ package(default_applicable_licenses = ["//:license"]) licenses(["notice"]) go_library( - name = "devtmpfs", + name = "dev", srcs = [ - "devtmpfs.go", - "save_restore.go", + "dev.go", ], visibility = ["//pkg/sentry:internal"], deps = [ "//pkg/abi/linux", "//pkg/context", + "//pkg/errors/linuxerr", "//pkg/fspath", "//pkg/sentry/fsimpl/tmpfs", "//pkg/sentry/kernel/auth", "//pkg/sentry/vfs", - "//pkg/sync", ], ) go_test( - name = "devtmpfs_test", + name = "dev_test", size = "small", - srcs = ["devtmpfs_test.go"], - library = ":devtmpfs", + srcs = ["dev_test.go"], + library = ":dev", deps = [ "//pkg/abi/linux", "//pkg/context", diff --git a/pkg/sentry/fsimpl/dev/dev.go b/pkg/sentry/fsimpl/dev/dev.go new file mode 100644 index 000000000..e7ca046b0 --- /dev/null +++ b/pkg/sentry/fsimpl/dev/dev.go @@ -0,0 +1,189 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package dev provides a filesystem implementation for /dev. +package dev + +import ( + "fmt" + "path" + + "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/context" + "gvisor.dev/gvisor/pkg/errors/linuxerr" + "gvisor.dev/gvisor/pkg/fspath" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs" + "gvisor.dev/gvisor/pkg/sentry/kernel/auth" + "gvisor.dev/gvisor/pkg/sentry/vfs" +) + +// Name is the dev filesystem name. +const Name = "dev" + +// FilesystemType implements vfs.FilesystemType. +// +// +stateify savable +type FilesystemType struct{} + +// Name implements vfs.FilesystemType.Name. +func (FilesystemType) Name() string { + return Name +} + +// GetFilesystem implements vfs.FilesystemType.GetFilesystem. +func (fst FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.VirtualFilesystem, creds *auth.Credentials, source string, opts vfs.GetFilesystemOptions) (*vfs.Filesystem, *vfs.Dentry, error) { + mntns, err := vfsObj.NewMountNamespace(ctx, creds, source /* source */, tmpfs.Name, &vfs.MountOptions{GetFilesystemOptions: vfs.GetFilesystemOptions{ + Data: "mode=0755", // opts from drivers/base/devtmpfs.c:devtmpfs_init() + }}, nil) + if err != nil { + return nil, nil, err + } + defer mntns.DecRef(ctx) + + root := mntns.Root(ctx) + defer root.DecRef(ctx) + + iopts, _ := opts.InternalData.(InternalData) // If not provided, zero value is OK. + + // Initialize contents. + if err := userspaceInit(ctx, vfsObj, creds, root, iopts.ShmMode); err != nil { + return nil, nil, err + } + if err := vfsObj.ForEachDevice(func(pathname string, kind vfs.DeviceKind, major, minor uint32, perms uint16) error { + if pathname == "" { + return nil + } + mode := linux.FileMode(perms) + switch kind { + case vfs.CharDevice: + mode |= linux.S_IFCHR + case vfs.BlockDevice: + mode |= linux.S_IFBLK + default: + panic(fmt.Sprintf("invalid DeviceKind: %v", kind)) + } + return CreateDeviceFile(ctx, vfsObj, creds, root, pathname, major, minor, mode, nil /* uid */, nil /* gid */) + }); err != nil { + return nil, nil, err + } + + root.Mount().Filesystem().IncRef() + root.Dentry().IncRef() + return root.Mount().Filesystem(), root.Dentry(), nil +} + +// Release implements vfs.FilesystemType.Release. +func (fst *FilesystemType) Release(ctx context.Context) {} + +// InternalData contains internal data passed in via vfs.GetFilesystemOptions. +type InternalData struct { + // ShmMode indicates the mode to create the /dev/shm dir with. + ShmMode *uint16 +} + +func pathOperationAt(root vfs.VirtualDentry, pathname string) *vfs.PathOperation { + return &vfs.PathOperation{ + Root: root, + Start: root, + Path: fspath.Parse(pathname), + } +} + +// CreateDeviceFile creates a device special file at the given pathname from root. +func CreateDeviceFile(ctx context.Context, vfsObj *vfs.VirtualFilesystem, creds *auth.Credentials, root vfs.VirtualDentry, pathname string, major, minor uint32, mode linux.FileMode, uid, gid *uint32) error { + // Create any parent directories. See + // devtmpfs.c:handle_create()=>create_path(). + parent := path.Dir(pathname) + if err := vfsObj.MkdirAllAt(ctx, parent, root, creds, &vfs.MkdirOptions{ + Mode: 0755, + }, true /* mustBeDir */); err != nil { + return fmt.Errorf("failed to create device parent directory %q: %v", parent, err) + } + created := true + pop := pathOperationAt(root, pathname) + if err := vfsObj.MknodAt(ctx, creds, pop, &vfs.MknodOptions{Mode: mode, DevMajor: major, DevMinor: minor}); err != nil { + if linuxerr.Equals(linuxerr.EEXIST, err) { + // EEXIST is silently ignored; compare + // opencontainers/runc:libcontainer/rootfs_linux.go:createDeviceNode(). + created = false + } else { + return fmt.Errorf("failed to create device file at %q: %w", pathname, err) + } + } + if created && (uid != nil || gid != nil) { + var opts vfs.SetStatOptions + if uid != nil { + opts.Stat.Mask |= linux.STATX_UID + opts.Stat.UID = *uid + } + if gid != nil { + opts.Stat.Mask |= linux.STATX_GID + opts.Stat.GID = *gid + } + if err := vfsObj.SetStatAt(ctx, creds, pop, &opts); err != nil { + return fmt.Errorf("failed to set UID/GID for device file %q: %w", pathname, err) + } + } + return nil +} + +// userspaceInit creates symbolic links and mount points in the devtmpfs +// instance that are created by userspace in Linux. It does not create mounts. +func userspaceInit(ctx context.Context, vfsObj *vfs.VirtualFilesystem, creds *auth.Credentials, root vfs.VirtualDentry, shmMode *uint16) error { + // Initialize symlinks. + for _, symlink := range []struct { + source string + target string + }{ + // systemd: src/shared/dev-setup.c:dev_setup() + {source: "fd", target: "/proc/self/fd"}, + {source: "stdin", target: "/proc/self/fd/0"}, + {source: "stdout", target: "/proc/self/fd/1"}, + {source: "stderr", target: "/proc/self/fd/2"}, + // /proc/kcore is not implemented. + + // Linux implements /dev/ptmx as a device node, but advises + // container implementations to create /dev/ptmx as a symlink + // to pts/ptmx (Documentation/filesystems/devpts.txt). Systemd + // follows this advice (src/nspawn/nspawn.c:setup_pts()), while + // LXC tries to create a bind mount and falls back to a symlink + // (src/lxc/conf.c:lxc_setup_devpts()). + {source: "ptmx", target: "pts/ptmx"}, + } { + if err := vfsObj.SymlinkAt(ctx, creds, pathOperationAt(root, symlink.source), symlink.target); err != nil { + return fmt.Errorf("failed to create symlink %q => %q: %v", symlink.source, symlink.target, err) + } + } + + // systemd: src/core/mount-setup.c:mount_table + for _, dir := range []string{ + "shm", + "pts", + } { + // "The access mode here doesn't really matter too much, since the + // mounted file system will take precedence anyway" + // - systemd: src/core/mount-setup.c:mount_one() + accessMode := linux.FileMode(0755) + if shmMode != nil && dir == "shm" { + accessMode = linux.FileMode(*shmMode) + } + if err := vfsObj.MkdirAt(ctx, creds, pathOperationAt(root, dir), &vfs.MkdirOptions{ + Mode: accessMode, + }); err != nil { + return fmt.Errorf("failed to create directory %q: %v", dir, err) + } + } + + return nil +} diff --git a/pkg/sentry/fsimpl/dev/dev_test.go b/pkg/sentry/fsimpl/dev/dev_test.go new file mode 100644 index 000000000..efedd2725 --- /dev/null +++ b/pkg/sentry/fsimpl/dev/dev_test.go @@ -0,0 +1,154 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev + +import ( + "testing" + + "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/context" + "gvisor.dev/gvisor/pkg/fspath" + "gvisor.dev/gvisor/pkg/sentry/contexttest" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs" + "gvisor.dev/gvisor/pkg/sentry/kernel/auth" + "gvisor.dev/gvisor/pkg/sentry/vfs" +) + +const ( + testDevMajor = 111 + testDevMinor = 11 + testDevPathname = "test" + testDevPerms = 0655 +) + +func setupDev(t *testing.T) (context.Context, *auth.Credentials, *vfs.VirtualFilesystem, vfs.VirtualDentry, func()) { + t.Helper() + + ctx := contexttest.Context(t) + creds := auth.CredentialsFromContext(ctx) + vfsObj := &vfs.VirtualFilesystem{} + if err := vfsObj.Init(ctx); err != nil { + t.Fatalf("VFS init: %v", err) + } + // Register tmpfs. + vfsObj.MustRegisterFilesystemType("tmpfs", tmpfs.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{ + AllowUserMount: true, + }) + vfsObj.MustRegisterFilesystemType(Name, &FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{}) + + vfsObj.RegisterDevice(vfs.CharDevice, testDevMajor, testDevMinor, nil, &vfs.RegisterDeviceOptions{ + GroupName: "test", + Pathname: testDevPathname, + FilePerms: testDevPerms, + }) + + // Create a test mount namespace with devfs mounted at root. + mntns, err := vfsObj.NewMountNamespace(ctx, creds, "dev" /* source */, Name /* fsTypeName */, &vfs.MountOptions{}, nil) + if err != nil { + t.Fatalf("failed to create tmpfs root mount: %v", err) + } + root := mntns.Root(ctx) + + return ctx, creds, vfsObj, root, func() { + root.DecRef(ctx) + mntns.DecRef(ctx) + } +} + +func TestUserspaceFiles(t *testing.T) { + ctx, creds, vfsObj, root, cleanup := setupDev(t) + defer cleanup() + + // Created files should be visible in the test mount namespace. + links := []struct { + source string + target string + }{ + { + source: "fd", + target: "/proc/self/fd", + }, + { + source: "stdin", + target: "/proc/self/fd/0", + }, + { + source: "stdout", + target: "/proc/self/fd/1", + }, + { + source: "stderr", + target: "/proc/self/fd/2", + }, + { + source: "ptmx", + target: "pts/ptmx", + }, + } + + for _, link := range links { + if gotTarget, err := vfsObj.ReadlinkAt(ctx, creds, &vfs.PathOperation{ + Root: root, + Start: root, + Path: fspath.Parse(link.source), + }); err != nil || gotTarget != link.target { + t.Errorf("readlink(%q): got (%q, %v), wanted (%q, nil)", link.source, gotTarget, err, link.target) + } + } + + dirs := []string{"shm", "pts"} + for _, dir := range dirs { + statx, err := vfsObj.StatAt(ctx, creds, &vfs.PathOperation{ + Root: root, + Start: root, + Path: fspath.Parse(dir), + }, &vfs.StatOptions{ + Mask: linux.STATX_MODE, + }) + if err != nil { + t.Errorf("stat(%q): got error %v ", dir, err) + continue + } + if want := uint16(0755) | linux.S_IFDIR; statx.Mode != want { + t.Errorf("stat(%q): got mode %x, want %x", dir, statx.Mode, want) + } + } +} + +func TestDeviceFile(t *testing.T) { + ctx, creds, vfsObj, root, cleanup := setupDev(t) + defer cleanup() + + // Test that the test device is created. + stat, err := vfsObj.StatAt(ctx, creds, &vfs.PathOperation{ + Root: root, + Start: root, + Path: fspath.Parse(testDevPathname), + }, &vfs.StatOptions{ + Mask: linux.STATX_TYPE | linux.STATX_MODE, + }) + if err != nil { + t.Fatalf("failed to stat device file at %q: %v", testDevPathname, err) + } + if stat.RdevMajor != testDevMajor { + t.Errorf("major device number: got %v, wanted %v", stat.RdevMajor, testDevMajor) + } + if stat.RdevMinor != testDevMinor { + t.Errorf("minor device number: got %v, wanted %v", stat.RdevMinor, testDevMinor) + } + if wantMode := uint16(linux.S_IFCHR | testDevPerms); stat.Mode != wantMode { + t.Errorf("device file mode: got %v, wanted %v", stat.Mode, wantMode) + } +} diff --git a/pkg/sentry/fsimpl/devtmpfs/devtmpfs.go b/pkg/sentry/fsimpl/devtmpfs/devtmpfs.go deleted file mode 100644 index 45f58861a..000000000 --- a/pkg/sentry/fsimpl/devtmpfs/devtmpfs.go +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package devtmpfs provides an implementation of /dev based on tmpfs, -// analogous to Linux's devtmpfs. -package devtmpfs - -import ( - "fmt" - "path" - - "gvisor.dev/gvisor/pkg/abi/linux" - "gvisor.dev/gvisor/pkg/context" - "gvisor.dev/gvisor/pkg/fspath" - "gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs" - "gvisor.dev/gvisor/pkg/sentry/kernel/auth" - "gvisor.dev/gvisor/pkg/sentry/vfs" - "gvisor.dev/gvisor/pkg/sync" -) - -// Name is the default filesystem name. -const Name = "devtmpfs" - -// FilesystemType implements vfs.FilesystemType. -// -// +stateify savable -type FilesystemType struct { - initOnce sync.Once `state:"nosave"` // FIXME(gvisor.dev/issue/1663): not yet supported. - initErr error - - // fs is the tmpfs filesystem that backs all mounts of this FilesystemType. - // root is fs' root. fs and root are immutable. - fs *vfs.Filesystem - root *vfs.Dentry -} - -// Name implements vfs.FilesystemType.Name. -func (*FilesystemType) Name() string { - return Name -} - -// GetFilesystem implements vfs.FilesystemType.GetFilesystem. -func (fst *FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.VirtualFilesystem, creds *auth.Credentials, source string, opts vfs.GetFilesystemOptions) (*vfs.Filesystem, *vfs.Dentry, error) { - fst.initOnce.Do(func() { - fs, root, err := tmpfs.FilesystemType{}.GetFilesystem(ctx, vfsObj, creds, "" /* source */, vfs.GetFilesystemOptions{ - Data: "mode=0755", // opts from drivers/base/devtmpfs.c:devtmpfs_init() - }) - if err != nil { - fst.initErr = err - return - } - fst.fs = fs - fst.root = root - }) - if fst.initErr != nil { - return nil, nil, fst.initErr - } - fst.fs.IncRef() - fst.root.IncRef() - return fst.fs, fst.root, nil -} - -// Release implements vfs.FilesystemType.Release. -func (fst *FilesystemType) Release(ctx context.Context) { - if fst.fs != nil { - // Release the original reference obtained when creating the filesystem. - fst.root.DecRef(ctx) - fst.fs.DecRef(ctx) - } -} - -// Accessor allows devices to create device special files in devtmpfs. -type Accessor struct { - vfsObj *vfs.VirtualFilesystem - mntns *vfs.MountNamespace - root vfs.VirtualDentry - creds *auth.Credentials -} - -// NewAccessor returns an Accessor that supports creation of device special -// files in the devtmpfs instance registered with name fsTypeName in vfsObj. -func NewAccessor(ctx context.Context, vfsObj *vfs.VirtualFilesystem, creds *auth.Credentials, fsTypeName string) (*Accessor, error) { - mntns, err := vfsObj.NewMountNamespace(ctx, creds, "devtmpfs" /* source */, fsTypeName, &vfs.MountOptions{}, nil) - if err != nil { - return nil, err - } - // Pass a reference on root to the Accessor. - root := mntns.Root(ctx) - return &Accessor{ - vfsObj: vfsObj, - mntns: mntns, - root: root, - creds: creds, - }, nil -} - -// Release must be called when a is no longer in use. -func (a *Accessor) Release(ctx context.Context) { - a.root.DecRef(ctx) - a.mntns.DecRef(ctx) -} - -// accessorContext implements context.Context by extending an existing -// context.Context with an Accessor's values for VFS-relevant state. -type accessorContext struct { - context.Context - a *Accessor -} - -func (a *Accessor) wrapContext(ctx context.Context) *accessorContext { - return &accessorContext{ - Context: ctx, - a: a, - } -} - -// Value implements context.Context.Value. -func (ac *accessorContext) Value(key any) any { - switch key { - case vfs.CtxMountNamespace: - ac.a.mntns.IncRef() - return ac.a.mntns - case vfs.CtxRoot: - ac.a.root.IncRef() - return ac.a.root - default: - return ac.Context.Value(key) - } -} - -func (a *Accessor) pathOperationAt(pathname string) *vfs.PathOperation { - return &vfs.PathOperation{ - Root: a.root, - Start: a.root, - Path: fspath.Parse(pathname), - } -} - -// CreateDeviceFile creates a device special file at the given pathname in the -// devtmpfs instance accessed by the Accessor. -func (a *Accessor) CreateDeviceFile(ctx context.Context, pathname string, kind vfs.DeviceKind, major, minor uint32, perms uint16) error { - actx := a.wrapContext(ctx) - - mode := (linux.FileMode)(perms) - switch kind { - case vfs.BlockDevice: - mode |= linux.S_IFBLK - case vfs.CharDevice: - mode |= linux.S_IFCHR - default: - panic(fmt.Sprintf("invalid vfs.DeviceKind: %v", kind)) - } - - // Create any parent directories. See - // devtmpfs.c:handle_create()=>path_create(). - parent := path.Dir(pathname) - if err := a.vfsObj.MkdirAllAt(ctx, parent, a.root, a.creds, &vfs.MkdirOptions{ - Mode: 0755, - }, true /* mustBeDir */); err != nil { - return fmt.Errorf("failed to create device parent directory %q: %v", parent, err) - } - - // NOTE: Linux's devtmpfs refuses to automatically delete files it didn't - // create, which it recognizes by storing a pointer to the kdevtmpfs struct - // thread in struct inode::i_private. Accessor doesn't yet support deletion - // of files at all, and probably won't as long as we don't need to support - // kernel modules, so this is moot for now. - return a.vfsObj.MknodAt(actx, a.creds, a.pathOperationAt(pathname), &vfs.MknodOptions{ - Mode: mode, - DevMajor: major, - DevMinor: minor, - }) -} - -// UserspaceInit creates symbolic links and mount points in the devtmpfs -// instance accessed by the Accessor that are created by userspace in Linux. It -// does not create mounts. -func (a *Accessor) UserspaceInit(ctx context.Context) error { - actx := a.wrapContext(ctx) - - // Initialize symlinks. - for _, symlink := range []struct { - source string - target string - }{ - // systemd: src/shared/dev-setup.c:dev_setup() - {source: "fd", target: "/proc/self/fd"}, - {source: "stdin", target: "/proc/self/fd/0"}, - {source: "stdout", target: "/proc/self/fd/1"}, - {source: "stderr", target: "/proc/self/fd/2"}, - // /proc/kcore is not implemented. - - // Linux implements /dev/ptmx as a device node, but advises - // container implementations to create /dev/ptmx as a symlink - // to pts/ptmx (Documentation/filesystems/devpts.txt). Systemd - // follows this advice (src/nspawn/nspawn.c:setup_pts()), while - // LXC tries to create a bind mount and falls back to a symlink - // (src/lxc/conf.c:lxc_setup_devpts()). - {source: "ptmx", target: "pts/ptmx"}, - } { - if err := a.vfsObj.SymlinkAt(actx, a.creds, a.pathOperationAt(symlink.source), symlink.target); err != nil { - return fmt.Errorf("failed to create symlink %q => %q: %v", symlink.source, symlink.target, err) - } - } - - // systemd: src/core/mount-setup.c:mount_table - for _, dir := range []string{ - "shm", - "pts", - } { - if err := a.vfsObj.MkdirAt(actx, a.creds, a.pathOperationAt(dir), &vfs.MkdirOptions{ - // systemd: src/core/mount-setup.c:mount_one() - Mode: 0755, - }); err != nil { - return fmt.Errorf("failed to create directory %q: %v", dir, err) - } - } - - return nil -} diff --git a/pkg/sentry/fsimpl/devtmpfs/devtmpfs_test.go b/pkg/sentry/fsimpl/devtmpfs/devtmpfs_test.go deleted file mode 100644 index 9fe4c56c9..000000000 --- a/pkg/sentry/fsimpl/devtmpfs/devtmpfs_test.go +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package devtmpfs - -import ( - "path" - "testing" - - "gvisor.dev/gvisor/pkg/abi/linux" - "gvisor.dev/gvisor/pkg/context" - "gvisor.dev/gvisor/pkg/fspath" - "gvisor.dev/gvisor/pkg/sentry/contexttest" - "gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs" - "gvisor.dev/gvisor/pkg/sentry/kernel/auth" - "gvisor.dev/gvisor/pkg/sentry/vfs" -) - -const devPath = "/dev" - -func setupDevtmpfs(t *testing.T) (context.Context, *auth.Credentials, *vfs.VirtualFilesystem, vfs.VirtualDentry, func()) { - t.Helper() - - ctx := contexttest.Context(t) - creds := auth.CredentialsFromContext(ctx) - vfsObj := &vfs.VirtualFilesystem{} - if err := vfsObj.Init(ctx); err != nil { - t.Fatalf("VFS init: %v", err) - } - // Register tmpfs just so that we can have a root filesystem that isn't - // devtmpfs. - vfsObj.MustRegisterFilesystemType("tmpfs", tmpfs.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{ - AllowUserMount: true, - }) - vfsObj.MustRegisterFilesystemType("devtmpfs", &FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{ - AllowUserMount: true, - }) - - // Create a test mount namespace with devtmpfs mounted at "/dev". - mntns, err := vfsObj.NewMountNamespace(ctx, creds, "tmpfs" /* source */, "tmpfs" /* fsTypeName */, &vfs.MountOptions{}, nil) - if err != nil { - t.Fatalf("failed to create tmpfs root mount: %v", err) - } - root := mntns.Root(ctx) - devpop := vfs.PathOperation{ - Root: root, - Start: root, - Path: fspath.Parse(devPath), - } - if err := vfsObj.MkdirAt(ctx, creds, &devpop, &vfs.MkdirOptions{ - Mode: 0755, - }); err != nil { - t.Fatalf("failed to create mount point: %v", err) - } - if _, err := vfsObj.MountAt(ctx, creds, "devtmpfs" /* source */, &devpop, "devtmpfs" /* fsTypeName */, &vfs.MountOptions{}); err != nil { - t.Fatalf("failed to mount devtmpfs: %v", err) - } - - return ctx, creds, vfsObj, root, func() { - root.DecRef(ctx) - mntns.DecRef(ctx) - } -} - -func TestUserspaceInit(t *testing.T) { - ctx, creds, vfsObj, root, cleanup := setupDevtmpfs(t) - defer cleanup() - - a, err := NewAccessor(ctx, vfsObj, creds, "devtmpfs") - if err != nil { - t.Fatalf("failed to create devtmpfs.Accessor: %v", err) - } - defer a.Release(ctx) - - // Create "userspace-initialized" files using a devtmpfs.Accessor. - if err := a.UserspaceInit(ctx); err != nil { - t.Fatalf("failed to userspace-initialize devtmpfs: %v", err) - } - - // Created files should be visible in the test mount namespace. - links := []struct { - source string - target string - }{ - { - source: "fd", - target: "/proc/self/fd", - }, - { - source: "stdin", - target: "/proc/self/fd/0", - }, - { - source: "stdout", - target: "/proc/self/fd/1", - }, - { - source: "stderr", - target: "/proc/self/fd/2", - }, - { - source: "ptmx", - target: "pts/ptmx", - }, - } - - for _, link := range links { - abspath := path.Join(devPath, link.source) - if gotTarget, err := vfsObj.ReadlinkAt(ctx, creds, &vfs.PathOperation{ - Root: root, - Start: root, - Path: fspath.Parse(abspath), - }); err != nil || gotTarget != link.target { - t.Errorf("readlink(%q): got (%q, %v), wanted (%q, nil)", abspath, gotTarget, err, link.target) - } - } - - dirs := []string{"shm", "pts"} - for _, dir := range dirs { - abspath := path.Join(devPath, dir) - statx, err := vfsObj.StatAt(ctx, creds, &vfs.PathOperation{ - Root: root, - Start: root, - Path: fspath.Parse(abspath), - }, &vfs.StatOptions{ - Mask: linux.STATX_MODE, - }) - if err != nil { - t.Errorf("stat(%q): got error %v ", abspath, err) - continue - } - if want := uint16(0755) | linux.S_IFDIR; statx.Mode != want { - t.Errorf("stat(%q): got mode %x, want %x", abspath, statx.Mode, want) - } - } -} - -func TestCreateDeviceFile(t *testing.T) { - ctx, creds, vfsObj, root, cleanup := setupDevtmpfs(t) - defer cleanup() - - a, err := NewAccessor(ctx, vfsObj, creds, "devtmpfs") - if err != nil { - t.Fatalf("failed to create devtmpfs.Accessor: %v", err) - } - defer a.Release(ctx) - - devFiles := []struct { - path string - kind vfs.DeviceKind - major uint32 - minor uint32 - perms uint16 - }{ - { - path: "dummy", - kind: vfs.CharDevice, - major: 12, - minor: 34, - perms: 0600, - }, - { - path: "foo/bar", - kind: vfs.BlockDevice, - major: 13, - minor: 35, - perms: 0660, - }, - { - path: "foo/baz", - kind: vfs.CharDevice, - major: 12, - minor: 40, - perms: 0666, - }, - { - path: "a/b/c/d/e", - kind: vfs.BlockDevice, - major: 12, - minor: 34, - perms: 0600, - }, - } - - for _, f := range devFiles { - if err := a.CreateDeviceFile(ctx, f.path, f.kind, f.major, f.minor, f.perms); err != nil { - t.Fatalf("failed to create device file: %v", err) - } - // The device special file should be visible in the test mount namespace. - abspath := path.Join(devPath, f.path) - stat, err := vfsObj.StatAt(ctx, creds, &vfs.PathOperation{ - Root: root, - Start: root, - Path: fspath.Parse(abspath), - }, &vfs.StatOptions{ - Mask: linux.STATX_TYPE | linux.STATX_MODE, - }) - if err != nil { - t.Fatalf("failed to stat device file at %q: %v", abspath, err) - } - if stat.RdevMajor != f.major { - t.Errorf("major device number: got %v, wanted %v", stat.RdevMajor, f.major) - } - if stat.RdevMinor != f.minor { - t.Errorf("minor device number: got %v, wanted %v", stat.RdevMinor, f.minor) - } - wantMode := f.perms - switch f.kind { - case vfs.CharDevice: - wantMode |= linux.S_IFCHR - case vfs.BlockDevice: - wantMode |= linux.S_IFBLK - } - if stat.Mode != wantMode { - t.Errorf("device file mode: got %v, wanted %v", stat.Mode, wantMode) - } - } -} diff --git a/pkg/sentry/fsimpl/devtmpfs/save_restore.go b/pkg/sentry/fsimpl/devtmpfs/save_restore.go deleted file mode 100644 index 28832d850..000000000 --- a/pkg/sentry/fsimpl/devtmpfs/save_restore.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package devtmpfs - -// afterLoad is invoked by stateify. -func (fst *FilesystemType) afterLoad() { - if fst.fs != nil { - // Ensure that we don't create another filesystem. - fst.initOnce.Do(func() {}) - } -} diff --git a/pkg/sentry/fsimpl/fuse/BUILD b/pkg/sentry/fsimpl/fuse/BUILD index f460d5ca4..ef61d7d24 100644 --- a/pkg/sentry/fsimpl/fuse/BUILD +++ b/pkg/sentry/fsimpl/fuse/BUILD @@ -59,7 +59,6 @@ go_library( "//pkg/marshal/primitive", "//pkg/refs", "//pkg/safemem", - "//pkg/sentry/fsimpl/devtmpfs", "//pkg/sentry/fsimpl/kernfs", "//pkg/sentry/fsutil", "//pkg/sentry/kernel", diff --git a/pkg/sentry/fsimpl/fuse/register.go b/pkg/sentry/fsimpl/fuse/register.go index b5b581152..d91c3811c 100644 --- a/pkg/sentry/fsimpl/fuse/register.go +++ b/pkg/sentry/fsimpl/fuse/register.go @@ -16,27 +16,14 @@ package fuse import ( "gvisor.dev/gvisor/pkg/abi/linux" - "gvisor.dev/gvisor/pkg/context" - "gvisor.dev/gvisor/pkg/sentry/fsimpl/devtmpfs" "gvisor.dev/gvisor/pkg/sentry/vfs" ) // Register registers the FUSE device with vfsObj. func Register(vfsObj *vfs.VirtualFilesystem) error { - if err := vfsObj.RegisterDevice(vfs.CharDevice, linux.MISC_MAJOR, fuseDevMinor, fuseDevice{}, &vfs.RegisterDeviceOptions{ + return vfsObj.RegisterDevice(vfs.CharDevice, linux.MISC_MAJOR, fuseDevMinor, fuseDevice{}, &vfs.RegisterDeviceOptions{ GroupName: "misc", - }); err != nil { - return err - } - - return nil -} - -// CreateDevtmpfsFile creates a device special file in devtmpfs. -func CreateDevtmpfsFile(ctx context.Context, dev *devtmpfs.Accessor) error { - if err := dev.CreateDeviceFile(ctx, "fuse", vfs.CharDevice, linux.MISC_MAJOR, fuseDevMinor, 0666 /* mode */); err != nil { - return err - } - - return nil + Pathname: "fuse", + FilePerms: 0666, + }) } diff --git a/pkg/sentry/vfs/device.go b/pkg/sentry/vfs/device.go index 52b6640aa..9ccb5796b 100644 --- a/pkg/sentry/vfs/device.go +++ b/pkg/sentry/vfs/device.go @@ -74,6 +74,12 @@ type RegisterDeviceOptions struct { // /proc/devices. If GroupName is empty, this registration will not be // shown in /proc/devices. GroupName string + // Pathname is the name for the device file of this device in /dev directory. + // If Pathname is empty, then no device file is created. + Pathname string + // FilePerms are the permission bits to create the device file with. Only + // used if Pathname is provided. + FilePerms uint16 } // RegisterDevice registers the given Device in vfs with the given major and @@ -92,6 +98,18 @@ func (vfs *VirtualFilesystem) RegisterDevice(kind DeviceKind, major, minor uint3 return nil } +// ForEachDevice calls the given callback for each registered device. +func (vfs *VirtualFilesystem) ForEachDevice(cb func(pathname string, kind DeviceKind, major, minor uint32, perms uint16) error) error { + vfs.devicesMu.Lock() + defer vfs.devicesMu.Unlock() + for tup, dev := range vfs.devices { + if err := cb(dev.opts.Pathname, tup.kind, tup.major, tup.minor, dev.opts.FilePerms); err != nil { + return err + } + } + return nil +} + // OpenDeviceSpecialFile returns a FileDescription representing the given // device. func (vfs *VirtualFilesystem) OpenDeviceSpecialFile(ctx context.Context, mnt *Mount, d *Dentry, kind DeviceKind, major, minor uint32, opts *OpenOptions) (*FileDescription, error) { diff --git a/runsc/boot/BUILD b/runsc/boot/BUILD index 227737057..3ffdf1d91 100644 --- a/runsc/boot/BUILD +++ b/runsc/boot/BUILD @@ -32,6 +32,7 @@ go_library( deps = [ "//pkg/abi", "//pkg/abi/linux", + "//pkg/abi/nvgpu", "//pkg/abi/tpu", "//pkg/bpf", "//pkg/cleanup", @@ -59,8 +60,8 @@ go_library( "//pkg/sentry/devices/tundev", "//pkg/sentry/fdimport", "//pkg/sentry/fsimpl/cgroupfs", + "//pkg/sentry/fsimpl/dev", "//pkg/sentry/fsimpl/devpts", - "//pkg/sentry/fsimpl/devtmpfs", "//pkg/sentry/fsimpl/erofs", "//pkg/sentry/fsimpl/fuse", "//pkg/sentry/fsimpl/gofer", diff --git a/runsc/boot/loader.go b/runsc/boot/loader.go index 47a9282e8..e680b02d5 100644 --- a/runsc/boot/loader.go +++ b/runsc/boot/loader.go @@ -123,6 +123,10 @@ type containerInfo struct { // nvidiaUVMDevMajor is the device major number used for nvidia-uvm. nvidiaUVMDevMajor uint32 + + // nvidiaDevMinors is a list of device minors for Nvidia GPU devices exposed + // to the sandbox. + nvidiaDevMinors NvidiaDevMinors } // Loader keeps state needed to start the kernel and run the container. @@ -282,6 +286,9 @@ type Args struct { // ProfileOpts contains the set of profiles to enable and the // corresponding FDs where profile data will be written. ProfileOpts profile.Opts + // NvidiaDevMinors is a list of device minors for Nvidia GPU devices exposed + // to the sandbox. + NvidiaDevMinors NvidiaDevMinors } // make sure stdioFDs are always the same on initial start and on restore @@ -315,6 +322,7 @@ func New(args Args) (*Loader, error) { conf: args.Conf, spec: args.Spec, goferMountConfs: args.GoferMountConfs, + nvidiaDevMinors: args.NvidiaDevMinors, } // Make host FDs stable between invocations. Host FDs must map to the exact @@ -857,6 +865,7 @@ func (l *Loader) startSubcontainer(spec *specs.Spec, conf *config.Config, cid st goferFilestoreFDs: goferFilestoreFDs, goferMountConfs: goferMountConfs, nvidiaUVMDevMajor: l.nvidiaUVMDevMajor, + nvidiaDevMinors: l.root.nvidiaDevMinors, } info.procArgs, err = createProcessArgs(cid, spec, creds, l.k, pidns) if err != nil { diff --git a/runsc/boot/vfs.go b/runsc/boot/vfs.go index 323a242e7..8d777bdae 100644 --- a/runsc/boot/vfs.go +++ b/runsc/boot/vfs.go @@ -26,6 +26,7 @@ import ( specs "github.com/opencontainers/runtime-spec/specs-go" "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/abi/nvgpu" "gvisor.dev/gvisor/pkg/abi/tpu" "gvisor.dev/gvisor/pkg/cleanup" "gvisor.dev/gvisor/pkg/context" @@ -39,8 +40,8 @@ import ( "gvisor.dev/gvisor/pkg/sentry/devices/ttydev" "gvisor.dev/gvisor/pkg/sentry/devices/tundev" "gvisor.dev/gvisor/pkg/sentry/fsimpl/cgroupfs" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/dev" "gvisor.dev/gvisor/pkg/sentry/fsimpl/devpts" - "gvisor.dev/gvisor/pkg/sentry/fsimpl/devtmpfs" "gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs" "gvisor.dev/gvisor/pkg/sentry/fsimpl/fuse" "gvisor.dev/gvisor/pkg/sentry/fsimpl/gofer" @@ -86,7 +87,6 @@ var tmpfsAllowedData = []string{"mode", "size", "uid", "gid"} func registerFilesystems(k *kernel.Kernel, info *containerInfo) error { ctx := k.SupervisorContext() - creds := auth.NewRootCredentials(k.RootUserNamespace()) vfsObj := k.VFS() vfsObj.MustRegisterFilesystemType(cgroupfs.Name, &cgroupfs.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{ @@ -99,10 +99,7 @@ func registerFilesystems(k *kernel.Kernel, info *containerInfo) error { // usable state. AllowUserMount: true, }) - vfsObj.MustRegisterFilesystemType(devtmpfs.Name, &devtmpfs.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{ - AllowUserMount: true, - AllowUserList: true, - }) + vfsObj.MustRegisterFilesystemType(dev.Name, &dev.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{}) vfsObj.MustRegisterFilesystemType(erofs.Name, &erofs.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{ AllowUserList: true, }) @@ -151,36 +148,11 @@ func registerFilesystems(k *kernel.Kernel, info *containerInfo) error { return fmt.Errorf("registering fusedev: %w", err) } - // Setup files in devtmpfs. - a, err := devtmpfs.NewAccessor(ctx, vfsObj, creds, devtmpfs.Name) - if err != nil { - return fmt.Errorf("creating devtmpfs accessor: %w", err) - } - defer a.Release(ctx) - - if err := a.UserspaceInit(ctx); err != nil { - return fmt.Errorf("initializing userspace: %w", err) - } - if err := memdev.CreateDevtmpfsFiles(ctx, a); err != nil { - return fmt.Errorf("creating memdev devtmpfs files: %w", err) - } - if err := ttydev.CreateDevtmpfsFiles(ctx, a); err != nil { - return fmt.Errorf("creating ttydev devtmpfs files: %w", err) - } - if tunSupported { - if err := tundev.CreateDevtmpfsFiles(ctx, a); err != nil { - return fmt.Errorf("creating tundev devtmpfs files: %v", err) - } - } - if err := fuse.CreateDevtmpfsFile(ctx, a); err != nil { - return fmt.Errorf("creating fusedev devtmpfs files: %w", err) - } - - if err := nvproxyRegisterDevicesAndCreateFiles(ctx, info, k, vfsObj, a); err != nil { + if err := nvproxyRegisterDevices(info, vfsObj); err != nil { return err } - if err := tpuProxyRegisterDevicesAndCreateFiles(ctx, info, k, vfsObj, a); err != nil { + if err := tpuProxyRegisterDevices(info, vfsObj); err != nil { return err } @@ -246,7 +218,7 @@ func compileMounts(spec *specs.Spec, conf *config.Config) []specs.Mount { case "/sys": sysMounted = true case "/dev": - m.Type = devtmpfs.Name + m.Type = dev.Name devMounted = true case "/dev/pts": m.Type = devpts.Name @@ -290,7 +262,7 @@ func compileMounts(spec *specs.Spec, conf *config.Config) []specs.Mount { } if !devMounted { mandatoryMounts = append(mandatoryMounts, specs.Mount{ - Type: devtmpfs.Name, + Type: dev.Name, Destination: "/dev", }) } @@ -819,7 +791,7 @@ func getMountNameAndOptions(spec *specs.Spec, conf *config.Config, m *mountInfo, // Find filesystem name and FS specific data field. switch m.mount.Type { - case devpts.Name, devtmpfs.Name, proc.Name: + case devpts.Name, dev.Name, proc.Name: // Nothing to do. case Nonefs: @@ -1093,74 +1065,63 @@ func (c *containerMounter) configureRestore(ctx context.Context) (context.Contex } func createDeviceFiles(ctx context.Context, creds *auth.Credentials, info *containerInfo, vfsObj *vfs.VirtualFilesystem, root vfs.VirtualDentry) error { - if info.spec.Linux == nil { - return nil + if info.spec.Linux != nil { + // Create any device files specified in the spec. + for _, dev := range info.spec.Linux.Devices { + if err := createDeviceFile(ctx, creds, info, vfsObj, root, dev); err != nil { + return err + } + } } - for _, dev := range info.spec.Linux.Devices { - pop := vfs.PathOperation{ - Root: root, - Start: root, - Path: fspath.Parse(dev.Path), + if specutils.GPUFunctionalityRequested(info.spec, info.conf) && info.conf.NVProxyDocker { + // In Docker mode, devices are not injected into spec.Linux.Devices. So + // manually create appropriate device files. + mode := os.FileMode(0666) + nvidiaDevs := []specs.LinuxDevice{ + specs.LinuxDevice{Path: "/dev/nvidiactl", Type: "c", Major: nvgpu.NV_MAJOR_DEVICE_NUMBER, Minor: nvgpu.NV_CONTROL_DEVICE_MINOR, FileMode: &mode}, + specs.LinuxDevice{Path: "/dev/nvidia-uvm", Type: "c", Major: int64(info.nvidiaUVMDevMajor), Minor: nvgpu.NVIDIA_UVM_PRIMARY_MINOR_NUMBER, FileMode: &mode}, } - opts := vfs.MknodOptions{ - Mode: linux.FileMode(dev.FileMode.Perm()), + for _, minor := range info.nvidiaDevMinors { + nvidiaDevs = append(nvidiaDevs, specs.LinuxDevice{Path: fmt.Sprintf("/dev/nvidia%d", minor), Type: "c", Major: nvgpu.NV_MAJOR_DEVICE_NUMBER, Minor: int64(minor), FileMode: &mode}) } - // See https://github.com/opencontainers/runtime-spec/blob/main/config-linux.md#devices. - switch dev.Type { - case "b": - opts.Mode |= linux.S_IFBLK - opts.DevMajor = uint32(dev.Major) - opts.DevMinor = uint32(dev.Minor) - case "c", "u": - opts.Mode |= linux.S_IFCHR - opts.DevMajor = uint32(dev.Major) - opts.DevMinor = uint32(dev.Minor) - case "p": - opts.Mode |= linux.S_IFIFO - default: - return fmt.Errorf("specified device at %q has invalid type %q", dev.Path, dev.Type) - } - if dev.Path == "/dev/nvidia-uvm" && info.nvidiaUVMDevMajor != 0 && opts.DevMajor != info.nvidiaUVMDevMajor { - // nvidia-uvm's major device number is dynamically assigned, so the - // number that it has on the host may differ from the number that - // it has in sentry VFS; switch from the former to the latter. - log.Infof("Switching /dev/nvidia-uvm device major number from %d to %d", dev.Major, info.nvidiaUVMDevMajor) - opts.DevMajor = info.nvidiaUVMDevMajor - } - if err := vfsObj.MkdirAllAt(ctx, path.Dir(dev.Path), root, creds, &vfs.MkdirOptions{ - Mode: 0o755, - }, true /* mustBeDir */); err != nil { - return fmt.Errorf("failed to create ancestor directories of %q: %w", dev.Path, err) - } - // EEXIST is silently ignored; compare - // opencontainers/runc:libcontainer/rootfs_linux.go:createDeviceNode(). - created := true - if err := vfsObj.MknodAt(ctx, creds, &pop, &opts); err != nil && !linuxerr.Equals(linuxerr.EEXIST, err) { - if linuxerr.Equals(linuxerr.EEXIST, err) { - created = false - } else { - return fmt.Errorf("failed to create device file at %q: %w", dev.Path, err) - } - } - if created && (dev.UID != nil || dev.GID != nil) { - var opts vfs.SetStatOptions - if dev.UID != nil { - opts.Stat.Mask |= linux.STATX_UID - opts.Stat.UID = *dev.UID - } - if dev.GID != nil { - opts.Stat.Mask |= linux.STATX_GID - opts.Stat.GID = *dev.GID - } - if err := vfsObj.SetStatAt(ctx, creds, &pop, &opts); err != nil { - return fmt.Errorf("failed to set UID/GID for device file %q: %w", dev.Path, err) + for _, nvidiaDev := range nvidiaDevs { + if err := createDeviceFile(ctx, creds, info, vfsObj, root, nvidiaDev); err != nil { + return err } } } return nil } -func tpuProxyRegisterDevicesAndCreateFiles(ctx context.Context, info *containerInfo, k *kernel.Kernel, vfsObj *vfs.VirtualFilesystem, a *devtmpfs.Accessor) error { +func createDeviceFile(ctx context.Context, creds *auth.Credentials, info *containerInfo, vfsObj *vfs.VirtualFilesystem, root vfs.VirtualDentry, devSpec specs.LinuxDevice) error { + mode := linux.FileMode(devSpec.FileMode.Perm()) + var major, minor uint32 + // See https://github.com/opencontainers/runtime-spec/blob/main/config-linux.md#devices. + switch devSpec.Type { + case "b": + mode |= linux.S_IFBLK + major = uint32(devSpec.Major) + minor = uint32(devSpec.Minor) + case "c", "u": + mode |= linux.S_IFCHR + major = uint32(devSpec.Major) + minor = uint32(devSpec.Minor) + case "p": + mode |= linux.S_IFIFO + default: + return fmt.Errorf("specified device at %q has invalid type %q", devSpec.Path, devSpec.Type) + } + if devSpec.Path == "/dev/nvidia-uvm" && info.nvidiaUVMDevMajor != 0 && major != info.nvidiaUVMDevMajor { + // nvidia-uvm's major device number is dynamically assigned, so the + // number that it has on the host may differ from the number that + // it has in sentry VFS; switch from the former to the latter. + log.Infof("Switching /dev/nvidia-uvm device major number from %d to %d", devSpec.Major, info.nvidiaUVMDevMajor) + major = info.nvidiaUVMDevMajor + } + return dev.CreateDeviceFile(ctx, vfsObj, creds, root, devSpec.Path, major, minor, mode, devSpec.UID, devSpec.GID) +} + +func tpuProxyRegisterDevices(info *containerInfo, vfsObj *vfs.VirtualFilesystem) error { if !specutils.TPUProxyIsEnabled(info.spec, info.conf) { return nil } @@ -1198,19 +1159,16 @@ func tpuProxyRegisterDevicesAndCreateFiles(ctx context.Context, info *containerI if err := accel.RegisterTPUV4Device(vfsObj, uint32(deviceNum), deviceID == tpu.TPUV4liteDeviceID); err != nil { return fmt.Errorf("registering accel driver: %w", err) } - if err := accel.CreateDevtmpfsFile(ctx, a, uint32(deviceNum)); err != nil { - return fmt.Errorf("creating accel device file %q: %w", deviceNum, err) - } } } return nil } -func nvproxyRegisterDevicesAndCreateFiles(ctx context.Context, info *containerInfo, k *kernel.Kernel, vfsObj *vfs.VirtualFilesystem, a *devtmpfs.Accessor) error { +func nvproxyRegisterDevices(info *containerInfo, vfsObj *vfs.VirtualFilesystem) error { if !specutils.GPUFunctionalityRequested(info.spec, info.conf) { return nil } - uvmDevMajor, err := k.VFS().GetDynamicCharDevMajor() + uvmDevMajor, err := vfsObj.GetDynamicCharDevMajor() if err != nil { return fmt.Errorf("reserving device major number for nvidia-uvm: %w", err) } @@ -1218,22 +1176,5 @@ func nvproxyRegisterDevicesAndCreateFiles(ctx context.Context, info *containerIn return fmt.Errorf("registering nvproxy driver: %w", err) } info.nvidiaUVMDevMajor = uvmDevMajor - if info.conf.NVProxyDocker { - // In Docker mode, create all the device files now. - // In non-Docker mode, these are instead created as part of - // `createDeviceFiles`, using the spec's Device list. - minors, err := specutils.FindAllGPUDevices("/") - if err != nil { - return fmt.Errorf("getting nvidia devices: %w", err) - } - if err := nvproxy.CreateDriverDevtmpfsFiles(ctx, a, uvmDevMajor); err != nil { - return fmt.Errorf("creating nvproxy devtmpfs files: %w", err) - } - for _, minor := range minors { - if err := nvproxy.CreateIndexDevtmpfsFile(ctx, a, minor); err != nil { - return fmt.Errorf("creating nvproxy devtmpfs file for device minor %d: %w", minor, err) - } - } - } return nil } diff --git a/runsc/cmd/boot.go b/runsc/cmd/boot.go index ae218a58a..6691b0c10 100644 --- a/runsc/cmd/boot.go +++ b/runsc/cmd/boot.go @@ -433,6 +433,7 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma PodInitConfigFD: b.podInitConfigFD, SinkFDs: b.sinkFDs.GetArray(), ProfileOpts: b.profileFDs.ToOpts(), + NvidiaDevMinors: b.nvidiaDevMinors, } l, err := boot.New(bootArgs) if err != nil {