From 45962656054d10592cb6b913d743f4c022bfc8a9 Mon Sep 17 00:00:00 2001 From: Lucas Manning Date: Wed, 17 Jan 2024 14:52:02 -0800 Subject: [PATCH] Add unit test for v5 TPU sysfs proxying. PiperOrigin-RevId: 599297513 --- pkg/sentry/fsimpl/sys/sys.go | 15 +++---- pkg/sentry/fsimpl/sys/sys_test.go | 65 ++++++++++++++++++++++++++++--- 2 files changed, 67 insertions(+), 13 deletions(-) diff --git a/pkg/sentry/fsimpl/sys/sys.go b/pkg/sentry/fsimpl/sys/sys.go index c01329c12..281b8e80a 100644 --- a/pkg/sentry/fsimpl/sys/sys.go +++ b/pkg/sentry/fsimpl/sys/sys.go @@ -58,9 +58,9 @@ type InternalData struct { // EnableTPUProxyPaths is whether to populate sysfs paths used by hardware // accelerators. EnableTPUProxyPaths bool - // PCIDevicePathPrefix is a prefix for the PCI device paths. It is useful for + // TestSysfsPathPrefix is a prefix for the sysfs paths. It is useful for // unit testing. - PCIDevicePathPrefix string + TestSysfsPathPrefix string } // filesystem implements vfs.FilesystemImpl. @@ -133,11 +133,11 @@ func (fsType FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt idata := opts.InternalData.(*InternalData) productName = idata.ProductName if idata.EnableTPUProxyPaths { - deviceToIommuGroup, err := pciDeviceIOMMUGroups(path.Join(idata.PCIDevicePathPrefix, iommuGroupSysPath)) + deviceToIommuGroup, err := pciDeviceIOMMUGroups(path.Join(idata.TestSysfsPathPrefix, iommuGroupSysPath)) if err != nil { return nil, nil, err } - pciPath := path.Join(idata.PCIDevicePathPrefix, pciMainBusDevicePath) + pciPath := path.Join(idata.TestSysfsPathPrefix, pciMainBusDevicePath) pciMainBusSub, err := fs.mirrorPCIBusDeviceDir(ctx, creds, pciPath, deviceToIommuGroup) if err != nil { return nil, nil, err @@ -159,7 +159,8 @@ func (fsType FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt busSub["pci"] = fs.newDir(ctx, creds, defaultSysDirMode, map[string]kernfs.Inode{ "devices": fs.newDir(ctx, creds, defaultSysDirMode, pciDevicesSub), }) - iommuGroups, err := fs.mirrorIOMMUGroups(ctx, creds, iommuGroupSysPath) + iommuPath := path.Join(idata.TestSysfsPathPrefix, iommuGroupSysPath) + iommuGroups, err := fs.mirrorIOMMUGroups(ctx, creds, iommuPath) if err != nil { return nil, nil, err } @@ -213,8 +214,8 @@ func cpuDir(ctx context.Context, fs *filesystem, creds *auth.Credentials) kernfs // Returns a map from a PCI device name to its IOMMU group if available. func pciDeviceIOMMUGroups(iommuGroupsPath string) (map[string]string, error) { - // IOMMU groups are organizd as iommu_group_path/$GROUP, where $GROUP is - // the IOMMU group number of which the device is a memeber. + // IOMMU groups are organized as iommu_group_path/$GROUP, where $GROUP is + // the IOMMU group number of which the device is a member. iommuGroupNums, err := hostDirEntries(iommuGroupsPath) if err != nil { // When IOMMU is not enabled, skip the rest of the process. diff --git a/pkg/sentry/fsimpl/sys/sys_test.go b/pkg/sentry/fsimpl/sys/sys_test.go index 06f4fee5d..afd27c882 100644 --- a/pkg/sentry/fsimpl/sys/sys_test.go +++ b/pkg/sentry/fsimpl/sys/sys_test.go @@ -44,7 +44,7 @@ func newTestSystem(t *testing.T, pciTestDir string) *testutil.System { GetFilesystemOptions: vfs.GetFilesystemOptions{ InternalData: &sys.InternalData{ EnableTPUProxyPaths: pciTestDir != "", - PCIDevicePathPrefix: pciTestDir, + TestSysfsPathPrefix: pciTestDir, }, }, } @@ -114,8 +114,8 @@ func TestCgroupMountpointExists(t *testing.T) { // Check that sysfs creates the required PCI paths for V4 TPUs. func TestEnableTPUProxyPathsV4(t *testing.T) { // Set up the fs tree that will be mirrored in the sentry. - pciTestDir := t.TempDir() - accelPath := path.Join(pciTestDir, "sys", "devices", "pci0000:00", "0000:00:04.0", "accel", "accel0") + sysfsTestDir := t.TempDir() + accelPath := path.Join(sysfsTestDir, "sys", "devices", "pci0000:00", "0000:00:04.0", "accel", "accel0") if err := os.MkdirAll(accelPath, 0755); err != nil { t.Fatalf("Failed to create accel directory: %v", err) } @@ -134,14 +134,14 @@ func TestEnableTPUProxyPathsV4(t *testing.T) { if _, err := os.Create(path.Join(accelPath, "pci_address")); err != nil { t.Fatalf("Failed to create pci_address: %v", err) } - busPath := path.Join(pciTestDir, "sys", "bus", "pci", "devices") + busPath := path.Join(sysfsTestDir, "sys", "bus", "pci", "devices") if err := os.MkdirAll(busPath, 0755); err != nil { t.Fatalf("Failed to create bus directory: %v", err) } if err := os.Symlink(path.Join("..", "..", "..", "devices", "pci0000:00", "0000:00:04.0"), path.Join(busPath, "0000:00:04.0")); err != nil { t.Fatalf("Failed to symlink bus directory: %v", err) } - classAccelPath := path.Join(pciTestDir, "sys", "class", "accel") + classAccelPath := path.Join(sysfsTestDir, "sys", "class", "accel") if err := os.MkdirAll(classAccelPath, 0755); err != nil { t.Fatalf("Failed to create accel directory: %v", err) } @@ -149,7 +149,7 @@ func TestEnableTPUProxyPathsV4(t *testing.T) { t.Fatalf("Failed to symlink accel directory: %v", err) } - s := newTestSystem(t, pciTestDir) + s := newTestSystem(t, sysfsTestDir) defer s.Destroy() pop := s.PathOpAtRoot("/devices/pci0000:00") @@ -173,3 +173,56 @@ func TestEnableTPUProxyPathsV4(t *testing.T) { "accel0": linux.DT_LNK, }) } + +func TestEnableTPUProxyPathsV5(t *testing.T) { + // Set up the fs tree that will be mirrored in the sentry. + sysfsTestDir := t.TempDir() + accelPath := path.Join(sysfsTestDir, "sys", "devices", "pci0000:00", "0000:00:04.0", "accel", "accel0") + if err := os.MkdirAll(accelPath, 0755); err != nil { + t.Fatalf("Failed to create accel directory: %v", err) + } + if err := os.Symlink(path.Join("..", "..", "..", "0000:00:04.0"), path.Join(accelPath, "0000:00:04.0")); err != nil { + t.Fatalf("Failed to symlink accel directory: %v", err) + } + if err := os.Symlink(path.Join("..", "..", "..", "0000:00:04.0"), path.Join(accelPath, "device")); err != nil { + t.Fatalf("Failed to symlink accel device directory: %v", err) + } + busPath := path.Join(sysfsTestDir, "sys", "bus", "pci", "devices") + if err := os.MkdirAll(busPath, 0755); err != nil { + t.Fatalf("Failed to create bus directory: %v", err) + } + if err := os.Symlink(path.Join("..", "..", "..", "devices", "pci0000:00", "0000:00:04.0"), path.Join(busPath, "0000:00:04.0")); err != nil { + t.Fatalf("Failed to symlink bus directory: %v", err) + } + classAccelPath := path.Join(sysfsTestDir, "sys", "class", "accel") + if err := os.MkdirAll(classAccelPath, 0755); err != nil { + t.Fatalf("Failed to create accel directory: %v", err) + } + if err := os.Symlink(path.Join("..", "..", "devices", "pci0000:00", "0000:00:04.0", "accel", "accel0"), path.Join(classAccelPath, "accel0")); err != nil { + t.Fatalf("Failed to symlink accel directory: %v", err) + } + iommuPath := path.Join(sysfsTestDir, "sys", "kernel", "iommu_groups", "0", "devices") + if err := os.MkdirAll(iommuPath, 0755); err != nil { + t.Fatalf("Failed to create iommu_groups directory: %v", err) + } + if err := os.Symlink(path.Join("..", "..", "..", "devices", "pci0000:00", "0000:00:04.0"), path.Join(iommuPath, "0000:00:04.0")); err != nil { + t.Fatalf("Failed to symlink bus directory: %v", err) + } + if err := os.Symlink(path.Join("..", "..", "..", "kernel", "iommu_groups", "0000:00:04.0"), path.Join(accelPath, "iommu_group")); err != nil { + t.Fatalf("Failed to symlink iommu_groups directory: %v", err) + } + + s := newTestSystem(t, sysfsTestDir) + defer s.Destroy() + + pop := s.PathOpAtRoot("/devices/pci0000:00/0000:00:04.0/accel/accel0") + s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ + "0000:00:04.0": linux.DT_LNK, + "device": linux.DT_LNK, + "iommu_group": linux.DT_LNK, + }) + pop = s.PathOpAtRoot("/kernel/iommu_groups/0/devices") + s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ + "0000:00:04.0": linux.DT_LNK, + }) +}