diff --git a/pkg/sentry/fsimpl/sys/BUILD b/pkg/sentry/fsimpl/sys/BUILD index 2c6986357..a3e733219 100644 --- a/pkg/sentry/fsimpl/sys/BUILD +++ b/pkg/sentry/fsimpl/sys/BUILD @@ -50,6 +50,12 @@ go_library( go_test( name = "sys_test", srcs = ["sys_test.go"], + library = ":sys", +) + +go_test( + name = "sys_integration_test", + srcs = ["sys_integration_test.go"], deps = [ ":sys", "//pkg/abi/linux", diff --git a/pkg/sentry/fsimpl/sys/sys.go b/pkg/sentry/fsimpl/sys/sys.go index 089e523ef..b493c1825 100644 --- a/pkg/sentry/fsimpl/sys/sys.go +++ b/pkg/sentry/fsimpl/sys/sys.go @@ -21,6 +21,7 @@ import ( "os" "path" "strconv" + "strings" "golang.org/x/sys/unix" "gvisor.dev/gvisor/pkg/abi/linux" @@ -203,16 +204,80 @@ func cpuDir(ctx context.Context, fs *filesystem, creds *auth.Credentials) kernfs k := kernel.KernelFromContext(ctx) maxCPUCores := k.ApplicationCores() children := map[string]kernfs.Inode{ - "online": fs.newCPUFile(ctx, creds, maxCPUCores, linux.FileMode(0444)), - "possible": fs.newCPUFile(ctx, creds, maxCPUCores, linux.FileMode(0444)), - "present": fs.newCPUFile(ctx, creds, maxCPUCores, linux.FileMode(0444)), + "online": fs.newCPUFile(ctx, creds, maxCPUCores, defaultSysMode), + "possible": fs.newCPUFile(ctx, creds, maxCPUCores, defaultSysMode), + "present": fs.newCPUFile(ctx, creds, maxCPUCores, defaultSysMode), } + // For consistency with /proc/cpuinfo, pretend all CPUs are in the same + // socket and each CPU is a distinct core. + fullMask := fullCPUMask(maxCPUCores) + "\n" for i := uint(0); i < maxCPUCores; i++ { - children[fmt.Sprintf("cpu%d", i)] = fs.newDir(ctx, creds, linux.FileMode(0555), nil) + oneMask := oneCPUMask(i, maxCPUCores) + "\n" + children[fmt.Sprintf("cpu%d", i)] = fs.newDir(ctx, creds, defaultSysDirMode, map[string]kernfs.Inode{ + "topology": fs.newDir(ctx, creds, defaultSysDirMode, map[string]kernfs.Inode{ + "core_cpus": fs.newStaticFile(ctx, creds, defaultSysMode, oneMask), + "core_siblings": fs.newStaticFile(ctx, creds, defaultSysMode, fullMask), + "package_cpus": fs.newStaticFile(ctx, creds, defaultSysMode, fullMask), + "thread_siblings": fs.newStaticFile(ctx, creds, defaultSysMode, oneMask), + }), + }) } return fs.newDir(ctx, creds, defaultSysDirMode, children) } +// fullCPUMask returns a "hex format ASCII string", consistent with Linux's +// include/linux/cpumask.h:cpumap_print_to_pagebuf(list=false) => +// lib/bitmap.c:bitmap_print_to_pagebuf(list=false), representing a CPU bitmask +// in which all `cores` CPUs are set. +func fullCPUMask(cores uint) string { + var ( + b strings.Builder + sep string + ) + if rem := cores % 32; rem != 0 { + cores -= rem + fmt.Fprintf(&b, "%x", (uint32(1)< +// lib/bitmap.c:bitmap_print_to_pagebuf(list=false), representing a CPU bitmask +// for `cores` CPUs in which only CPU `i` is set. +// +// Preconditions: i < cores. +func oneCPUMask(i, cores uint) string { + var ( + b strings.Builder + sep string + ) + word := func() (w uint32) { + if cores <= i { + w = uint32(1) << (i - cores) + } + return + } + if rem := cores % 32; rem != 0 { + cores -= rem + chars := (rem + 3) / 4 // 4 bits per hex character + fmt.Fprintf(&b, "%0*x", chars, word()) + sep = "," + } + for cores != 0 { + cores -= 32 + fmt.Fprintf(&b, "%s%08x", sep, word()) + sep = "," + } + return b.String() +} + // Returns a map from a PCI device name to its IOMMU group if available. func pciDeviceIOMMUGroups(iommuGroupsPath string) (map[string]string, error) { // IOMMU groups are organized as iommu_group_path/$GROUP, where $GROUP is diff --git a/pkg/sentry/fsimpl/sys/sys_integration_test.go b/pkg/sentry/fsimpl/sys/sys_integration_test.go new file mode 100644 index 000000000..4bc8da1cc --- /dev/null +++ b/pkg/sentry/fsimpl/sys/sys_integration_test.go @@ -0,0 +1,285 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sys_test + +import ( + "fmt" + "os" + "path" + "testing" + + "github.com/google/go-cmp/cmp" + "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/sys" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/testutil" + "gvisor.dev/gvisor/pkg/sentry/kernel" + "gvisor.dev/gvisor/pkg/sentry/kernel/auth" + "gvisor.dev/gvisor/pkg/sentry/vfs" +) + +const ( + vfioDev = "vfio-dev" +) + +func newTestSystem(t *testing.T, pciTestDir string) *testutil.System { + k, err := testutil.Boot() + if err != nil { + t.Fatalf("Failed to create test kernel: %v", err) + } + ctx := k.SupervisorContext() + creds := auth.CredentialsFromContext(ctx) + k.VFS().MustRegisterFilesystemType(sys.Name, sys.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{ + AllowUserMount: true, + }) + + mountOpts := &vfs.MountOptions{ + GetFilesystemOptions: vfs.GetFilesystemOptions{ + InternalData: &sys.InternalData{ + EnableTPUProxyPaths: pciTestDir != "", + TestSysfsPathPrefix: pciTestDir, + }, + }, + } + + mns, err := k.VFS().NewMountNamespace(ctx, creds, "", sys.Name, mountOpts, nil) + if err != nil { + t.Fatalf("Failed to create new mount namespace: %v", err) + } + return testutil.NewSystem(ctx, t, k.VFS(), mns) +} + +func TestReadCPUFile(t *testing.T) { + s := newTestSystem(t, "" /*pciTestDir*/) + defer s.Destroy() + k := kernel.KernelFromContext(s.Ctx) + maxCPUCores := k.ApplicationCores() + + expected := fmt.Sprintf("0-%d\n", maxCPUCores-1) + + for _, fname := range []string{"online", "possible", "present"} { + pop := s.PathOpAtRoot(fmt.Sprintf("devices/system/cpu/%s", fname)) + fd, err := s.VFS.OpenAt(s.Ctx, s.Creds, pop, &vfs.OpenOptions{}) + if err != nil { + t.Fatalf("OpenAt(pop:%+v) = %+v failed: %v", pop, fd, err) + } + defer fd.DecRef(s.Ctx) + content, err := s.ReadToEnd(fd) + if err != nil { + t.Fatalf("Read failed: %v", err) + } + if diff := cmp.Diff(expected, content); diff != "" { + t.Fatalf("Read returned unexpected data:\n--- want\n+++ got\n%v", diff) + } + } +} + +func TestSysRootContainsExpectedEntries(t *testing.T) { + s := newTestSystem(t, "" /*pciTestDir*/) + defer s.Destroy() + pop := s.PathOpAtRoot("/") + s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ + "block": linux.DT_DIR, + "bus": linux.DT_DIR, + "class": linux.DT_DIR, + "dev": linux.DT_DIR, + "devices": linux.DT_DIR, + "firmware": linux.DT_DIR, + "fs": linux.DT_DIR, + "kernel": linux.DT_DIR, + "module": linux.DT_DIR, + "power": linux.DT_DIR, + }) +} + +func TestCgroupMountpointExists(t *testing.T) { + // Note: The mountpoint is only created if cgroups are available. + s := newTestSystem(t, "" /*pciTestDir*/) + defer s.Destroy() + pop := s.PathOpAtRoot("/fs") + s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ + "cgroup": linux.DT_DIR, + }) + pop = s.PathOpAtRoot("/fs/cgroup") + s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ /*empty*/ }) +} + +// Check that sysfs creates the required PCI paths for V4 TPUs. +func TestEnableTPUProxyPathsV4(t *testing.T) { + // Set up the fs tree that will be mirrored in the sentry. + sysfsTestDir := t.TempDir() + busPath := path.Join(sysfsTestDir, "sys", "bus", "pci", "devices") + if err := os.MkdirAll(busPath, 0755); err != nil { + t.Fatalf("Failed to create bus directory: %v", err) + } + classAccelPath := path.Join(sysfsTestDir, "sys", "class", "accel") + if err := os.MkdirAll(classAccelPath, 0755); err != nil { + t.Fatalf("Failed to create accel directory: %v", err) + } + for i, pciAddress := range []string{"0000:00:04.0", "0000:00:05.0"} { + accelDev := fmt.Sprintf("accel%d", i) + accelPath := path.Join(sysfsTestDir, "sys", "devices", "pci0000:00", pciAddress, "accel", accelDev) + if err := os.MkdirAll(accelPath, 0755); err != nil { + t.Fatalf("Failed to create accel directory: %v", err) + } + if err := os.Symlink(path.Join("..", "..", "..", pciAddress), path.Join(accelPath, pciAddress)); err != nil { + t.Fatalf("Failed to symlink accel directory: %v", err) + } + if err := os.Symlink(path.Join("..", "..", "..", pciAddress), path.Join(accelPath, "device")); err != nil { + t.Fatalf("Failed to symlink accel device directory: %v", err) + } + if _, err := os.Create(path.Join(accelPath, "chip_model")); err != nil { + t.Fatalf("Failed to create chip_model: %v", err) + } + if _, err := os.Create(path.Join(accelPath, "device_owner")); err != nil { + t.Fatalf("Failed to create device_owner: %v", err) + } + if _, err := os.Create(path.Join(accelPath, "pci_address")); err != nil { + t.Fatalf("Failed to create pci_address: %v", err) + } + if err := os.Symlink(path.Join("..", "..", "..", "devices", "pci0000:00", pciAddress), path.Join(busPath, pciAddress)); err != nil { + t.Fatalf("Failed to symlink bus directory: %v", err) + } + if err := os.Symlink(path.Join("..", "..", "devices", "pci0000:00", pciAddress, "accel", accelDev), path.Join(classAccelPath, accelDev)); err != nil { + t.Fatalf("Failed to symlink accel directory: %v", err) + } + } + + s := newTestSystem(t, sysfsTestDir) + defer s.Destroy() + + pop := s.PathOpAtRoot("/devices/pci0000:00") + s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ + "0000:00:04.0": linux.DT_DIR, + "0000:00:05.0": linux.DT_DIR, + }) + pop = s.PathOpAtRoot("/devices/pci0000:00/0000:00:04.0/accel/accel0") + s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ + "0000:00:04.0": linux.DT_LNK, + "device": linux.DT_LNK, + "chip_model": linux.DT_REG, + "device_owner": linux.DT_REG, + "pci_address": linux.DT_REG, + }) + pop = s.PathOpAtRoot("/bus/pci/devices") + s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ + "0000:00:04.0": linux.DT_LNK, + "0000:00:05.0": linux.DT_LNK, + }) + pop = s.PathOpAtRoot("/class/accel") + s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ + "accel0": linux.DT_LNK, + "accel1": linux.DT_LNK, + }) +} + +type PCIDeviceInfo struct { + // IOMMU group. + group string + pciPath string + pciAddress string + name string +} + +func (dev PCIDeviceInfo) path() string { + return path.Join(dev.pciPath, dev.pciAddress, vfioDev, dev.name) +} + +func TestEnableTPUProxyPathsV5(t *testing.T) { + // Set up the fs tree that will be mirrored in the sentry. + sysfsTestDir := t.TempDir() + pciPath0 := path.Join(sysfsTestDir, "sys", "devices", "pci0000:00") + if err := os.MkdirAll(pciPath0, 0755); err != nil { + t.Fatalf("Failed to create PCI directory: %v", err) + } + pciPath1 := path.Join(sysfsTestDir, "sys", "devices", "pci0000:10") + if err := os.MkdirAll(pciPath1, 0755); err != nil { + t.Fatalf("Failed to create PCI directory: %v", err) + } + busPath := path.Join(sysfsTestDir, "sys", "bus", "pci", "devices") + if err := os.MkdirAll(busPath, 0755); err != nil { + t.Fatalf("Failed to create bus directory: %v", err) + } + sysClassPath := path.Join(sysfsTestDir, "sys", "class", vfioDev) + if err := os.MkdirAll(sysClassPath, 0755); err != nil { + t.Fatalf("Failed to create class directory: %v", err) + } + + devices := []PCIDeviceInfo{ + PCIDeviceInfo{ + group: "0", + pciPath: pciPath0, + pciAddress: "0000:00:04.0", + name: "vfio0", + }, + PCIDeviceInfo{ + group: "1", + pciPath: pciPath0, + pciAddress: "0000:00:05.0", + name: "vfio1", + }, + PCIDeviceInfo{ + group: "2", + pciPath: pciPath1, + pciAddress: "0000:10:05.0", + name: "vfio2", + }, + } + for _, device := range devices { + devicePath := device.path() + if err := os.MkdirAll(devicePath, 0755); err != nil { + t.Fatalf("Failed to create PCI device directory: %v", err) + } + if err := os.Symlink(path.Join("..", "..", "..", device.pciAddress), path.Join(devicePath, "device")); err != nil { + t.Fatalf("Failed to symlink device directory: %v", err) + } + if err := os.Symlink(path.Join("..", "..", "..", "devices", path.Base(device.pciPath), device.pciAddress), path.Join(busPath, device.pciAddress)); err != nil { + t.Fatalf("Failed to symlink bus directory: %v", err) + } + if err := os.Symlink(path.Join("..", "..", "devices", path.Base(device.pciPath), device.pciAddress, vfioDev, device.name), path.Join(sysClassPath, device.name)); err != nil { + t.Fatalf("Failed to symlink class directory: %v", err) + } + iommuPath := path.Join(sysfsTestDir, "sys", "kernel", "iommu_groups", device.group, "devices") + if err := os.MkdirAll(iommuPath, 0755); err != nil { + t.Fatalf("Failed to create iommu_groups directory: %v", err) + } + if err := os.Symlink(path.Join("..", "..", "..", "..", "devices", path.Base(device.pciPath), device.pciAddress), path.Join(iommuPath, device.pciAddress)); err != nil { + t.Fatalf("Failed to symlink iommu_group devices directory: %v", err) + } + if err := os.Symlink(path.Join("..", "..", "..", "kernel", "iommu_groups", device.group), path.Join(device.pciPath, device.pciAddress, "iommu_group")); err != nil { + t.Fatalf("Failed to symlink iommu_groups directory: %v", err) + } + } + s := newTestSystem(t, sysfsTestDir) + defer s.Destroy() + + for _, device := range devices { + // Validate PCI device symlinks. + pop := s.PathOpAtRoot(path.Join("devices", path.Base(device.pciPath), device.pciAddress)) + s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ + "iommu_group": linux.DT_LNK, + vfioDev: linux.DT_DIR, + }) + // Validate VFIO device symlinks. + pop = s.PathOpAtRoot(path.Join("devices", path.Base(device.pciPath), device.pciAddress, vfioDev, device.name)) + s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ + "device": linux.DT_LNK, + }) + // Validate $IOMMU_GROUP/devices. + pop = s.PathOpAtRoot(path.Join("kernel", "iommu_groups", string(device.group), "devices")) + s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ + device.pciAddress: linux.DT_LNK, + }) + } +} diff --git a/pkg/sentry/fsimpl/sys/sys_test.go b/pkg/sentry/fsimpl/sys/sys_test.go index 4bc8da1cc..885121aa8 100644 --- a/pkg/sentry/fsimpl/sys/sys_test.go +++ b/pkg/sentry/fsimpl/sys/sys_test.go @@ -1,4 +1,4 @@ -// Copyright 2019 The gVisor Authors. +// Copyright 2024 The gVisor Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,274 +12,65 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sys_test +package sys import ( - "fmt" - "os" - "path" "testing" - - "github.com/google/go-cmp/cmp" - "gvisor.dev/gvisor/pkg/abi/linux" - "gvisor.dev/gvisor/pkg/sentry/fsimpl/sys" - "gvisor.dev/gvisor/pkg/sentry/fsimpl/testutil" - "gvisor.dev/gvisor/pkg/sentry/kernel" - "gvisor.dev/gvisor/pkg/sentry/kernel/auth" - "gvisor.dev/gvisor/pkg/sentry/vfs" ) -const ( - vfioDev = "vfio-dev" -) - -func newTestSystem(t *testing.T, pciTestDir string) *testutil.System { - k, err := testutil.Boot() - if err != nil { - t.Fatalf("Failed to create test kernel: %v", err) - } - ctx := k.SupervisorContext() - creds := auth.CredentialsFromContext(ctx) - k.VFS().MustRegisterFilesystemType(sys.Name, sys.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{ - AllowUserMount: true, - }) - - mountOpts := &vfs.MountOptions{ - GetFilesystemOptions: vfs.GetFilesystemOptions{ - InternalData: &sys.InternalData{ - EnableTPUProxyPaths: pciTestDir != "", - TestSysfsPathPrefix: pciTestDir, - }, - }, - } - - mns, err := k.VFS().NewMountNamespace(ctx, creds, "", sys.Name, mountOpts, nil) - if err != nil { - t.Fatalf("Failed to create new mount namespace: %v", err) - } - return testutil.NewSystem(ctx, t, k.VFS(), mns) -} - -func TestReadCPUFile(t *testing.T) { - s := newTestSystem(t, "" /*pciTestDir*/) - defer s.Destroy() - k := kernel.KernelFromContext(s.Ctx) - maxCPUCores := k.ApplicationCores() - - expected := fmt.Sprintf("0-%d\n", maxCPUCores-1) - - for _, fname := range []string{"online", "possible", "present"} { - pop := s.PathOpAtRoot(fmt.Sprintf("devices/system/cpu/%s", fname)) - fd, err := s.VFS.OpenAt(s.Ctx, s.Creds, pop, &vfs.OpenOptions{}) - if err != nil { - t.Fatalf("OpenAt(pop:%+v) = %+v failed: %v", pop, fd, err) - } - defer fd.DecRef(s.Ctx) - content, err := s.ReadToEnd(fd) - if err != nil { - t.Fatalf("Read failed: %v", err) - } - if diff := cmp.Diff(expected, content); diff != "" { - t.Fatalf("Read returned unexpected data:\n--- want\n+++ got\n%v", diff) +func TestFullCPUMask(t *testing.T) { + for _, test := range []struct { + cores uint + want string + }{ + {1, "1"}, + {2, "3"}, + {3, "7"}, + {4, "f"}, + {5, "1f"}, + {32, "ffffffff"}, + {33, "1,ffffffff"}, + {36, "f,ffffffff"}, + {37, "1f,ffffffff"}, + {64, "ffffffff,ffffffff"}, + {65, "1,ffffffff,ffffffff"}, + } { + if got := fullCPUMask(test.cores); got != test.want { + t.Errorf("fullCPUMask(%d): got %s, want %s", test.cores, got, test.want) } } } -func TestSysRootContainsExpectedEntries(t *testing.T) { - s := newTestSystem(t, "" /*pciTestDir*/) - defer s.Destroy() - pop := s.PathOpAtRoot("/") - s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ - "block": linux.DT_DIR, - "bus": linux.DT_DIR, - "class": linux.DT_DIR, - "dev": linux.DT_DIR, - "devices": linux.DT_DIR, - "firmware": linux.DT_DIR, - "fs": linux.DT_DIR, - "kernel": linux.DT_DIR, - "module": linux.DT_DIR, - "power": linux.DT_DIR, - }) -} - -func TestCgroupMountpointExists(t *testing.T) { - // Note: The mountpoint is only created if cgroups are available. - s := newTestSystem(t, "" /*pciTestDir*/) - defer s.Destroy() - pop := s.PathOpAtRoot("/fs") - s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ - "cgroup": linux.DT_DIR, - }) - pop = s.PathOpAtRoot("/fs/cgroup") - s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ /*empty*/ }) -} - -// Check that sysfs creates the required PCI paths for V4 TPUs. -func TestEnableTPUProxyPathsV4(t *testing.T) { - // Set up the fs tree that will be mirrored in the sentry. - sysfsTestDir := t.TempDir() - busPath := path.Join(sysfsTestDir, "sys", "bus", "pci", "devices") - if err := os.MkdirAll(busPath, 0755); err != nil { - t.Fatalf("Failed to create bus directory: %v", err) - } - classAccelPath := path.Join(sysfsTestDir, "sys", "class", "accel") - if err := os.MkdirAll(classAccelPath, 0755); err != nil { - t.Fatalf("Failed to create accel directory: %v", err) - } - for i, pciAddress := range []string{"0000:00:04.0", "0000:00:05.0"} { - accelDev := fmt.Sprintf("accel%d", i) - accelPath := path.Join(sysfsTestDir, "sys", "devices", "pci0000:00", pciAddress, "accel", accelDev) - if err := os.MkdirAll(accelPath, 0755); err != nil { - t.Fatalf("Failed to create accel directory: %v", err) +func TestOneCPUMask(t *testing.T) { + for _, test := range []struct { + i uint + cores uint + want string + }{ + {0, 1, "1"}, + {0, 4, "1"}, + {1, 4, "2"}, + {2, 4, "4"}, + {3, 4, "8"}, + {0, 5, "01"}, + {4, 5, "10"}, + {0, 32, "00000001"}, + {26, 32, "04000000"}, + {0, 33, "0,00000001"}, + {31, 33, "0,80000000"}, + {32, 33, "1,00000000"}, + {0, 64, "00000000,00000001"}, + {31, 64, "00000000,80000000"}, + {32, 64, "00000001,00000000"}, + {63, 64, "80000000,00000000"}, + {0, 65, "0,00000000,00000001"}, + {31, 65, "0,00000000,80000000"}, + {32, 65, "0,00000001,00000000"}, + {63, 65, "0,80000000,00000000"}, + {64, 65, "1,00000000,00000000"}, + } { + if got := oneCPUMask(test.i, test.cores); got != test.want { + t.Errorf("oneCPUMask(%d, %d): got %s, want %s", test.i, test.cores, got, test.want) } - if err := os.Symlink(path.Join("..", "..", "..", pciAddress), path.Join(accelPath, pciAddress)); err != nil { - t.Fatalf("Failed to symlink accel directory: %v", err) - } - if err := os.Symlink(path.Join("..", "..", "..", pciAddress), path.Join(accelPath, "device")); err != nil { - t.Fatalf("Failed to symlink accel device directory: %v", err) - } - if _, err := os.Create(path.Join(accelPath, "chip_model")); err != nil { - t.Fatalf("Failed to create chip_model: %v", err) - } - if _, err := os.Create(path.Join(accelPath, "device_owner")); err != nil { - t.Fatalf("Failed to create device_owner: %v", err) - } - if _, err := os.Create(path.Join(accelPath, "pci_address")); err != nil { - t.Fatalf("Failed to create pci_address: %v", err) - } - if err := os.Symlink(path.Join("..", "..", "..", "devices", "pci0000:00", pciAddress), path.Join(busPath, pciAddress)); err != nil { - t.Fatalf("Failed to symlink bus directory: %v", err) - } - if err := os.Symlink(path.Join("..", "..", "devices", "pci0000:00", pciAddress, "accel", accelDev), path.Join(classAccelPath, accelDev)); err != nil { - t.Fatalf("Failed to symlink accel directory: %v", err) - } - } - - s := newTestSystem(t, sysfsTestDir) - defer s.Destroy() - - pop := s.PathOpAtRoot("/devices/pci0000:00") - s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ - "0000:00:04.0": linux.DT_DIR, - "0000:00:05.0": linux.DT_DIR, - }) - pop = s.PathOpAtRoot("/devices/pci0000:00/0000:00:04.0/accel/accel0") - s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ - "0000:00:04.0": linux.DT_LNK, - "device": linux.DT_LNK, - "chip_model": linux.DT_REG, - "device_owner": linux.DT_REG, - "pci_address": linux.DT_REG, - }) - pop = s.PathOpAtRoot("/bus/pci/devices") - s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ - "0000:00:04.0": linux.DT_LNK, - "0000:00:05.0": linux.DT_LNK, - }) - pop = s.PathOpAtRoot("/class/accel") - s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ - "accel0": linux.DT_LNK, - "accel1": linux.DT_LNK, - }) -} - -type PCIDeviceInfo struct { - // IOMMU group. - group string - pciPath string - pciAddress string - name string -} - -func (dev PCIDeviceInfo) path() string { - return path.Join(dev.pciPath, dev.pciAddress, vfioDev, dev.name) -} - -func TestEnableTPUProxyPathsV5(t *testing.T) { - // Set up the fs tree that will be mirrored in the sentry. - sysfsTestDir := t.TempDir() - pciPath0 := path.Join(sysfsTestDir, "sys", "devices", "pci0000:00") - if err := os.MkdirAll(pciPath0, 0755); err != nil { - t.Fatalf("Failed to create PCI directory: %v", err) - } - pciPath1 := path.Join(sysfsTestDir, "sys", "devices", "pci0000:10") - if err := os.MkdirAll(pciPath1, 0755); err != nil { - t.Fatalf("Failed to create PCI directory: %v", err) - } - busPath := path.Join(sysfsTestDir, "sys", "bus", "pci", "devices") - if err := os.MkdirAll(busPath, 0755); err != nil { - t.Fatalf("Failed to create bus directory: %v", err) - } - sysClassPath := path.Join(sysfsTestDir, "sys", "class", vfioDev) - if err := os.MkdirAll(sysClassPath, 0755); err != nil { - t.Fatalf("Failed to create class directory: %v", err) - } - - devices := []PCIDeviceInfo{ - PCIDeviceInfo{ - group: "0", - pciPath: pciPath0, - pciAddress: "0000:00:04.0", - name: "vfio0", - }, - PCIDeviceInfo{ - group: "1", - pciPath: pciPath0, - pciAddress: "0000:00:05.0", - name: "vfio1", - }, - PCIDeviceInfo{ - group: "2", - pciPath: pciPath1, - pciAddress: "0000:10:05.0", - name: "vfio2", - }, - } - for _, device := range devices { - devicePath := device.path() - if err := os.MkdirAll(devicePath, 0755); err != nil { - t.Fatalf("Failed to create PCI device directory: %v", err) - } - if err := os.Symlink(path.Join("..", "..", "..", device.pciAddress), path.Join(devicePath, "device")); err != nil { - t.Fatalf("Failed to symlink device directory: %v", err) - } - if err := os.Symlink(path.Join("..", "..", "..", "devices", path.Base(device.pciPath), device.pciAddress), path.Join(busPath, device.pciAddress)); err != nil { - t.Fatalf("Failed to symlink bus directory: %v", err) - } - if err := os.Symlink(path.Join("..", "..", "devices", path.Base(device.pciPath), device.pciAddress, vfioDev, device.name), path.Join(sysClassPath, device.name)); err != nil { - t.Fatalf("Failed to symlink class directory: %v", err) - } - iommuPath := path.Join(sysfsTestDir, "sys", "kernel", "iommu_groups", device.group, "devices") - if err := os.MkdirAll(iommuPath, 0755); err != nil { - t.Fatalf("Failed to create iommu_groups directory: %v", err) - } - if err := os.Symlink(path.Join("..", "..", "..", "..", "devices", path.Base(device.pciPath), device.pciAddress), path.Join(iommuPath, device.pciAddress)); err != nil { - t.Fatalf("Failed to symlink iommu_group devices directory: %v", err) - } - if err := os.Symlink(path.Join("..", "..", "..", "kernel", "iommu_groups", device.group), path.Join(device.pciPath, device.pciAddress, "iommu_group")); err != nil { - t.Fatalf("Failed to symlink iommu_groups directory: %v", err) - } - } - s := newTestSystem(t, sysfsTestDir) - defer s.Destroy() - - for _, device := range devices { - // Validate PCI device symlinks. - pop := s.PathOpAtRoot(path.Join("devices", path.Base(device.pciPath), device.pciAddress)) - s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ - "iommu_group": linux.DT_LNK, - vfioDev: linux.DT_DIR, - }) - // Validate VFIO device symlinks. - pop = s.PathOpAtRoot(path.Join("devices", path.Base(device.pciPath), device.pciAddress, vfioDev, device.name)) - s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ - "device": linux.DT_LNK, - }) - // Validate $IOMMU_GROUP/devices. - pop = s.PathOpAtRoot(path.Join("kernel", "iommu_groups", string(device.group), "devices")) - s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ - device.pciAddress: linux.DT_LNK, - }) } }