mirror of
https://github.com/netbirdio/gvisor.git
synced 2026-05-22 17:12:49 -07:00
sysfs: implement some cpu topology files
This is required for Open MPI => hwloc to detect the number of CPUs and set the number of Open MPI slots accordingly. Before this CL, on amd64: ``` root@0496c77e84e9:/# ls /sys/devices/system/cpu/ | grep cpu | wc -l 96 root@0496c77e84e9:/# mpirun --allow-run-as-root -np 96 /bin/true [hwloc/linux] failed to find sysfs cpu topology directory, aborting linux discovery. [0496c77e84e9:00667] OPAL ERROR: Not supported in file ../../../../../opal/mca/hwloc/base/hwloc_base_util.c at line 418 -------------------------------------------------------------------------- It looks like orte_init failed for some reason; your parallel process is likely to abort. There are many reasons that a parallel process can fail during orte_init; some of which are due to configuration or environment problems. This failure appears to be an internal failure; here's some additional information (which may only be relevant to an Open MPI developer): topology discovery failed --> Returned value Not supported (-8) instead of ORTE_SUCCESS -------------------------------------------------------------------------- ``` Before this CL, on arm64: ``` root@bfad6b71e996:/# ls /sys/devices/system/cpu/ | grep cpu | wc -l 4 root@bfad6b71e996:/# mpirun --allow-run-as-root -np 4 /bin/true [hwloc/linux] failed to find sysfs cpu topology directory, aborting linux discovery. [hwloc/linux] failed to find sysfs cpu topology directory, aborting linux discovery. -------------------------------------------------------------------------- All nodes which are allocated for this job are already filled. -------------------------------------------------------------------------- ``` After this CL, on amd64: ``` root@0512ce557005:/# ls /sys/devices/system/cpu/ | grep cpu | wc -l 96 root@0512ce557005:/# mpirun --allow-run-as-root -np 96 /bin/true root@0512ce557005:/# mpirun --allow-run-as-root -np 97 /bin/true -------------------------------------------------------------------------- There are not enough slots available in the system to satisfy the 97 slots that were requested by the application: ... ``` After this CL, on arm64: ``` root@faf6ff491bc5:/# ls /sys/devices/system/cpu/ | grep cpu | wc -l 4 root@faf6ff491bc5:/# mpirun --allow-run-as-root -np 4 /bin/true root@faf6ff491bc5:/# mpirun --allow-run-as-root -np 5 /bin/true -------------------------------------------------------------------------- There are not enough slots available in the system to satisfy the 5 slots that were requested by the application: ... ``` Per #10484, this also lets `nvidia-smi topo` make more progress: ``` root@d5a696bb7e2c:/# nvidia-smi topo -m GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 CPU Affinity NUMA Affinity GPU NUMA ID Failed to run topology matrix ``` PiperOrigin-RevId: 663395602
This commit is contained in:
Jamie Liu
gVisor bot
@@ -50,6 +50,12 @@ go_library(
|
||||
go_test(
|
||||
name = "sys_test",
|
||||
srcs = ["sys_test.go"],
|
||||
library = ":sys",
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "sys_integration_test",
|
||||
srcs = ["sys_integration_test.go"],
|
||||
deps = [
|
||||
":sys",
|
||||
"//pkg/abi/linux",
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
"gvisor.dev/gvisor/pkg/abi/linux"
|
||||
@@ -203,16 +204,80 @@ func cpuDir(ctx context.Context, fs *filesystem, creds *auth.Credentials) kernfs
|
||||
k := kernel.KernelFromContext(ctx)
|
||||
maxCPUCores := k.ApplicationCores()
|
||||
children := map[string]kernfs.Inode{
|
||||
"online": fs.newCPUFile(ctx, creds, maxCPUCores, linux.FileMode(0444)),
|
||||
"possible": fs.newCPUFile(ctx, creds, maxCPUCores, linux.FileMode(0444)),
|
||||
"present": fs.newCPUFile(ctx, creds, maxCPUCores, linux.FileMode(0444)),
|
||||
"online": fs.newCPUFile(ctx, creds, maxCPUCores, defaultSysMode),
|
||||
"possible": fs.newCPUFile(ctx, creds, maxCPUCores, defaultSysMode),
|
||||
"present": fs.newCPUFile(ctx, creds, maxCPUCores, defaultSysMode),
|
||||
}
|
||||
// For consistency with /proc/cpuinfo, pretend all CPUs are in the same
|
||||
// socket and each CPU is a distinct core.
|
||||
fullMask := fullCPUMask(maxCPUCores) + "\n"
|
||||
for i := uint(0); i < maxCPUCores; i++ {
|
||||
children[fmt.Sprintf("cpu%d", i)] = fs.newDir(ctx, creds, linux.FileMode(0555), nil)
|
||||
oneMask := oneCPUMask(i, maxCPUCores) + "\n"
|
||||
children[fmt.Sprintf("cpu%d", i)] = fs.newDir(ctx, creds, defaultSysDirMode, map[string]kernfs.Inode{
|
||||
"topology": fs.newDir(ctx, creds, defaultSysDirMode, map[string]kernfs.Inode{
|
||||
"core_cpus": fs.newStaticFile(ctx, creds, defaultSysMode, oneMask),
|
||||
"core_siblings": fs.newStaticFile(ctx, creds, defaultSysMode, fullMask),
|
||||
"package_cpus": fs.newStaticFile(ctx, creds, defaultSysMode, fullMask),
|
||||
"thread_siblings": fs.newStaticFile(ctx, creds, defaultSysMode, oneMask),
|
||||
}),
|
||||
})
|
||||
}
|
||||
return fs.newDir(ctx, creds, defaultSysDirMode, children)
|
||||
}
|
||||
|
||||
// fullCPUMask returns a "hex format ASCII string", consistent with Linux's
|
||||
// include/linux/cpumask.h:cpumap_print_to_pagebuf(list=false) =>
|
||||
// lib/bitmap.c:bitmap_print_to_pagebuf(list=false), representing a CPU bitmask
|
||||
// in which all `cores` CPUs are set.
|
||||
func fullCPUMask(cores uint) string {
|
||||
var (
|
||||
b strings.Builder
|
||||
sep string
|
||||
)
|
||||
if rem := cores % 32; rem != 0 {
|
||||
cores -= rem
|
||||
fmt.Fprintf(&b, "%x", (uint32(1)<<rem)-1)
|
||||
sep = ","
|
||||
}
|
||||
for cores != 0 {
|
||||
cores -= 32
|
||||
fmt.Fprintf(&b, "%sffffffff", sep)
|
||||
sep = ","
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// oneCPUMask returns a "hex format ASCII string", consistent with Linux's
|
||||
// include/linux/cpumask.h:cpumap_print_to_pagebuf(list=false) =>
|
||||
// lib/bitmap.c:bitmap_print_to_pagebuf(list=false), representing a CPU bitmask
|
||||
// for `cores` CPUs in which only CPU `i` is set.
|
||||
//
|
||||
// Preconditions: i < cores.
|
||||
func oneCPUMask(i, cores uint) string {
|
||||
var (
|
||||
b strings.Builder
|
||||
sep string
|
||||
)
|
||||
word := func() (w uint32) {
|
||||
if cores <= i {
|
||||
w = uint32(1) << (i - cores)
|
||||
}
|
||||
return
|
||||
}
|
||||
if rem := cores % 32; rem != 0 {
|
||||
cores -= rem
|
||||
chars := (rem + 3) / 4 // 4 bits per hex character
|
||||
fmt.Fprintf(&b, "%0*x", chars, word())
|
||||
sep = ","
|
||||
}
|
||||
for cores != 0 {
|
||||
cores -= 32
|
||||
fmt.Fprintf(&b, "%s%08x", sep, word())
|
||||
sep = ","
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// Returns a map from a PCI device name to its IOMMU group if available.
|
||||
func pciDeviceIOMMUGroups(iommuGroupsPath string) (map[string]string, error) {
|
||||
// IOMMU groups are organized as iommu_group_path/$GROUP, where $GROUP is
|
||||
|
||||
@@ -0,0 +1,285 @@
|
||||
// Copyright 2019 The gVisor Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package sys_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"gvisor.dev/gvisor/pkg/abi/linux"
|
||||
"gvisor.dev/gvisor/pkg/sentry/fsimpl/sys"
|
||||
"gvisor.dev/gvisor/pkg/sentry/fsimpl/testutil"
|
||||
"gvisor.dev/gvisor/pkg/sentry/kernel"
|
||||
"gvisor.dev/gvisor/pkg/sentry/kernel/auth"
|
||||
"gvisor.dev/gvisor/pkg/sentry/vfs"
|
||||
)
|
||||
|
||||
const (
|
||||
vfioDev = "vfio-dev"
|
||||
)
|
||||
|
||||
func newTestSystem(t *testing.T, pciTestDir string) *testutil.System {
|
||||
k, err := testutil.Boot()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create test kernel: %v", err)
|
||||
}
|
||||
ctx := k.SupervisorContext()
|
||||
creds := auth.CredentialsFromContext(ctx)
|
||||
k.VFS().MustRegisterFilesystemType(sys.Name, sys.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
|
||||
AllowUserMount: true,
|
||||
})
|
||||
|
||||
mountOpts := &vfs.MountOptions{
|
||||
GetFilesystemOptions: vfs.GetFilesystemOptions{
|
||||
InternalData: &sys.InternalData{
|
||||
EnableTPUProxyPaths: pciTestDir != "",
|
||||
TestSysfsPathPrefix: pciTestDir,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
mns, err := k.VFS().NewMountNamespace(ctx, creds, "", sys.Name, mountOpts, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create new mount namespace: %v", err)
|
||||
}
|
||||
return testutil.NewSystem(ctx, t, k.VFS(), mns)
|
||||
}
|
||||
|
||||
func TestReadCPUFile(t *testing.T) {
|
||||
s := newTestSystem(t, "" /*pciTestDir*/)
|
||||
defer s.Destroy()
|
||||
k := kernel.KernelFromContext(s.Ctx)
|
||||
maxCPUCores := k.ApplicationCores()
|
||||
|
||||
expected := fmt.Sprintf("0-%d\n", maxCPUCores-1)
|
||||
|
||||
for _, fname := range []string{"online", "possible", "present"} {
|
||||
pop := s.PathOpAtRoot(fmt.Sprintf("devices/system/cpu/%s", fname))
|
||||
fd, err := s.VFS.OpenAt(s.Ctx, s.Creds, pop, &vfs.OpenOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("OpenAt(pop:%+v) = %+v failed: %v", pop, fd, err)
|
||||
}
|
||||
defer fd.DecRef(s.Ctx)
|
||||
content, err := s.ReadToEnd(fd)
|
||||
if err != nil {
|
||||
t.Fatalf("Read failed: %v", err)
|
||||
}
|
||||
if diff := cmp.Diff(expected, content); diff != "" {
|
||||
t.Fatalf("Read returned unexpected data:\n--- want\n+++ got\n%v", diff)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSysRootContainsExpectedEntries(t *testing.T) {
|
||||
s := newTestSystem(t, "" /*pciTestDir*/)
|
||||
defer s.Destroy()
|
||||
pop := s.PathOpAtRoot("/")
|
||||
s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{
|
||||
"block": linux.DT_DIR,
|
||||
"bus": linux.DT_DIR,
|
||||
"class": linux.DT_DIR,
|
||||
"dev": linux.DT_DIR,
|
||||
"devices": linux.DT_DIR,
|
||||
"firmware": linux.DT_DIR,
|
||||
"fs": linux.DT_DIR,
|
||||
"kernel": linux.DT_DIR,
|
||||
"module": linux.DT_DIR,
|
||||
"power": linux.DT_DIR,
|
||||
})
|
||||
}
|
||||
|
||||
func TestCgroupMountpointExists(t *testing.T) {
|
||||
// Note: The mountpoint is only created if cgroups are available.
|
||||
s := newTestSystem(t, "" /*pciTestDir*/)
|
||||
defer s.Destroy()
|
||||
pop := s.PathOpAtRoot("/fs")
|
||||
s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{
|
||||
"cgroup": linux.DT_DIR,
|
||||
})
|
||||
pop = s.PathOpAtRoot("/fs/cgroup")
|
||||
s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ /*empty*/ })
|
||||
}
|
||||
|
||||
// Check that sysfs creates the required PCI paths for V4 TPUs.
|
||||
func TestEnableTPUProxyPathsV4(t *testing.T) {
|
||||
// Set up the fs tree that will be mirrored in the sentry.
|
||||
sysfsTestDir := t.TempDir()
|
||||
busPath := path.Join(sysfsTestDir, "sys", "bus", "pci", "devices")
|
||||
if err := os.MkdirAll(busPath, 0755); err != nil {
|
||||
t.Fatalf("Failed to create bus directory: %v", err)
|
||||
}
|
||||
classAccelPath := path.Join(sysfsTestDir, "sys", "class", "accel")
|
||||
if err := os.MkdirAll(classAccelPath, 0755); err != nil {
|
||||
t.Fatalf("Failed to create accel directory: %v", err)
|
||||
}
|
||||
for i, pciAddress := range []string{"0000:00:04.0", "0000:00:05.0"} {
|
||||
accelDev := fmt.Sprintf("accel%d", i)
|
||||
accelPath := path.Join(sysfsTestDir, "sys", "devices", "pci0000:00", pciAddress, "accel", accelDev)
|
||||
if err := os.MkdirAll(accelPath, 0755); err != nil {
|
||||
t.Fatalf("Failed to create accel directory: %v", err)
|
||||
}
|
||||
if err := os.Symlink(path.Join("..", "..", "..", pciAddress), path.Join(accelPath, pciAddress)); err != nil {
|
||||
t.Fatalf("Failed to symlink accel directory: %v", err)
|
||||
}
|
||||
if err := os.Symlink(path.Join("..", "..", "..", pciAddress), path.Join(accelPath, "device")); err != nil {
|
||||
t.Fatalf("Failed to symlink accel device directory: %v", err)
|
||||
}
|
||||
if _, err := os.Create(path.Join(accelPath, "chip_model")); err != nil {
|
||||
t.Fatalf("Failed to create chip_model: %v", err)
|
||||
}
|
||||
if _, err := os.Create(path.Join(accelPath, "device_owner")); err != nil {
|
||||
t.Fatalf("Failed to create device_owner: %v", err)
|
||||
}
|
||||
if _, err := os.Create(path.Join(accelPath, "pci_address")); err != nil {
|
||||
t.Fatalf("Failed to create pci_address: %v", err)
|
||||
}
|
||||
if err := os.Symlink(path.Join("..", "..", "..", "devices", "pci0000:00", pciAddress), path.Join(busPath, pciAddress)); err != nil {
|
||||
t.Fatalf("Failed to symlink bus directory: %v", err)
|
||||
}
|
||||
if err := os.Symlink(path.Join("..", "..", "devices", "pci0000:00", pciAddress, "accel", accelDev), path.Join(classAccelPath, accelDev)); err != nil {
|
||||
t.Fatalf("Failed to symlink accel directory: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
s := newTestSystem(t, sysfsTestDir)
|
||||
defer s.Destroy()
|
||||
|
||||
pop := s.PathOpAtRoot("/devices/pci0000:00")
|
||||
s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{
|
||||
"0000:00:04.0": linux.DT_DIR,
|
||||
"0000:00:05.0": linux.DT_DIR,
|
||||
})
|
||||
pop = s.PathOpAtRoot("/devices/pci0000:00/0000:00:04.0/accel/accel0")
|
||||
s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{
|
||||
"0000:00:04.0": linux.DT_LNK,
|
||||
"device": linux.DT_LNK,
|
||||
"chip_model": linux.DT_REG,
|
||||
"device_owner": linux.DT_REG,
|
||||
"pci_address": linux.DT_REG,
|
||||
})
|
||||
pop = s.PathOpAtRoot("/bus/pci/devices")
|
||||
s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{
|
||||
"0000:00:04.0": linux.DT_LNK,
|
||||
"0000:00:05.0": linux.DT_LNK,
|
||||
})
|
||||
pop = s.PathOpAtRoot("/class/accel")
|
||||
s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{
|
||||
"accel0": linux.DT_LNK,
|
||||
"accel1": linux.DT_LNK,
|
||||
})
|
||||
}
|
||||
|
||||
type PCIDeviceInfo struct {
|
||||
// IOMMU group.
|
||||
group string
|
||||
pciPath string
|
||||
pciAddress string
|
||||
name string
|
||||
}
|
||||
|
||||
func (dev PCIDeviceInfo) path() string {
|
||||
return path.Join(dev.pciPath, dev.pciAddress, vfioDev, dev.name)
|
||||
}
|
||||
|
||||
func TestEnableTPUProxyPathsV5(t *testing.T) {
|
||||
// Set up the fs tree that will be mirrored in the sentry.
|
||||
sysfsTestDir := t.TempDir()
|
||||
pciPath0 := path.Join(sysfsTestDir, "sys", "devices", "pci0000:00")
|
||||
if err := os.MkdirAll(pciPath0, 0755); err != nil {
|
||||
t.Fatalf("Failed to create PCI directory: %v", err)
|
||||
}
|
||||
pciPath1 := path.Join(sysfsTestDir, "sys", "devices", "pci0000:10")
|
||||
if err := os.MkdirAll(pciPath1, 0755); err != nil {
|
||||
t.Fatalf("Failed to create PCI directory: %v", err)
|
||||
}
|
||||
busPath := path.Join(sysfsTestDir, "sys", "bus", "pci", "devices")
|
||||
if err := os.MkdirAll(busPath, 0755); err != nil {
|
||||
t.Fatalf("Failed to create bus directory: %v", err)
|
||||
}
|
||||
sysClassPath := path.Join(sysfsTestDir, "sys", "class", vfioDev)
|
||||
if err := os.MkdirAll(sysClassPath, 0755); err != nil {
|
||||
t.Fatalf("Failed to create class directory: %v", err)
|
||||
}
|
||||
|
||||
devices := []PCIDeviceInfo{
|
||||
PCIDeviceInfo{
|
||||
group: "0",
|
||||
pciPath: pciPath0,
|
||||
pciAddress: "0000:00:04.0",
|
||||
name: "vfio0",
|
||||
},
|
||||
PCIDeviceInfo{
|
||||
group: "1",
|
||||
pciPath: pciPath0,
|
||||
pciAddress: "0000:00:05.0",
|
||||
name: "vfio1",
|
||||
},
|
||||
PCIDeviceInfo{
|
||||
group: "2",
|
||||
pciPath: pciPath1,
|
||||
pciAddress: "0000:10:05.0",
|
||||
name: "vfio2",
|
||||
},
|
||||
}
|
||||
for _, device := range devices {
|
||||
devicePath := device.path()
|
||||
if err := os.MkdirAll(devicePath, 0755); err != nil {
|
||||
t.Fatalf("Failed to create PCI device directory: %v", err)
|
||||
}
|
||||
if err := os.Symlink(path.Join("..", "..", "..", device.pciAddress), path.Join(devicePath, "device")); err != nil {
|
||||
t.Fatalf("Failed to symlink device directory: %v", err)
|
||||
}
|
||||
if err := os.Symlink(path.Join("..", "..", "..", "devices", path.Base(device.pciPath), device.pciAddress), path.Join(busPath, device.pciAddress)); err != nil {
|
||||
t.Fatalf("Failed to symlink bus directory: %v", err)
|
||||
}
|
||||
if err := os.Symlink(path.Join("..", "..", "devices", path.Base(device.pciPath), device.pciAddress, vfioDev, device.name), path.Join(sysClassPath, device.name)); err != nil {
|
||||
t.Fatalf("Failed to symlink class directory: %v", err)
|
||||
}
|
||||
iommuPath := path.Join(sysfsTestDir, "sys", "kernel", "iommu_groups", device.group, "devices")
|
||||
if err := os.MkdirAll(iommuPath, 0755); err != nil {
|
||||
t.Fatalf("Failed to create iommu_groups directory: %v", err)
|
||||
}
|
||||
if err := os.Symlink(path.Join("..", "..", "..", "..", "devices", path.Base(device.pciPath), device.pciAddress), path.Join(iommuPath, device.pciAddress)); err != nil {
|
||||
t.Fatalf("Failed to symlink iommu_group devices directory: %v", err)
|
||||
}
|
||||
if err := os.Symlink(path.Join("..", "..", "..", "kernel", "iommu_groups", device.group), path.Join(device.pciPath, device.pciAddress, "iommu_group")); err != nil {
|
||||
t.Fatalf("Failed to symlink iommu_groups directory: %v", err)
|
||||
}
|
||||
}
|
||||
s := newTestSystem(t, sysfsTestDir)
|
||||
defer s.Destroy()
|
||||
|
||||
for _, device := range devices {
|
||||
// Validate PCI device symlinks.
|
||||
pop := s.PathOpAtRoot(path.Join("devices", path.Base(device.pciPath), device.pciAddress))
|
||||
s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{
|
||||
"iommu_group": linux.DT_LNK,
|
||||
vfioDev: linux.DT_DIR,
|
||||
})
|
||||
// Validate VFIO device symlinks.
|
||||
pop = s.PathOpAtRoot(path.Join("devices", path.Base(device.pciPath), device.pciAddress, vfioDev, device.name))
|
||||
s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{
|
||||
"device": linux.DT_LNK,
|
||||
})
|
||||
// Validate $IOMMU_GROUP/devices.
|
||||
pop = s.PathOpAtRoot(path.Join("kernel", "iommu_groups", string(device.group), "devices"))
|
||||
s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{
|
||||
device.pciAddress: linux.DT_LNK,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2019 The gVisor Authors.
|
||||
// Copyright 2024 The gVisor Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
@@ -12,274 +12,65 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package sys_test
|
||||
package sys
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"gvisor.dev/gvisor/pkg/abi/linux"
|
||||
"gvisor.dev/gvisor/pkg/sentry/fsimpl/sys"
|
||||
"gvisor.dev/gvisor/pkg/sentry/fsimpl/testutil"
|
||||
"gvisor.dev/gvisor/pkg/sentry/kernel"
|
||||
"gvisor.dev/gvisor/pkg/sentry/kernel/auth"
|
||||
"gvisor.dev/gvisor/pkg/sentry/vfs"
|
||||
)
|
||||
|
||||
const (
|
||||
vfioDev = "vfio-dev"
|
||||
)
|
||||
|
||||
func newTestSystem(t *testing.T, pciTestDir string) *testutil.System {
|
||||
k, err := testutil.Boot()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create test kernel: %v", err)
|
||||
}
|
||||
ctx := k.SupervisorContext()
|
||||
creds := auth.CredentialsFromContext(ctx)
|
||||
k.VFS().MustRegisterFilesystemType(sys.Name, sys.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
|
||||
AllowUserMount: true,
|
||||
})
|
||||
|
||||
mountOpts := &vfs.MountOptions{
|
||||
GetFilesystemOptions: vfs.GetFilesystemOptions{
|
||||
InternalData: &sys.InternalData{
|
||||
EnableTPUProxyPaths: pciTestDir != "",
|
||||
TestSysfsPathPrefix: pciTestDir,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
mns, err := k.VFS().NewMountNamespace(ctx, creds, "", sys.Name, mountOpts, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create new mount namespace: %v", err)
|
||||
}
|
||||
return testutil.NewSystem(ctx, t, k.VFS(), mns)
|
||||
}
|
||||
|
||||
func TestReadCPUFile(t *testing.T) {
|
||||
s := newTestSystem(t, "" /*pciTestDir*/)
|
||||
defer s.Destroy()
|
||||
k := kernel.KernelFromContext(s.Ctx)
|
||||
maxCPUCores := k.ApplicationCores()
|
||||
|
||||
expected := fmt.Sprintf("0-%d\n", maxCPUCores-1)
|
||||
|
||||
for _, fname := range []string{"online", "possible", "present"} {
|
||||
pop := s.PathOpAtRoot(fmt.Sprintf("devices/system/cpu/%s", fname))
|
||||
fd, err := s.VFS.OpenAt(s.Ctx, s.Creds, pop, &vfs.OpenOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("OpenAt(pop:%+v) = %+v failed: %v", pop, fd, err)
|
||||
}
|
||||
defer fd.DecRef(s.Ctx)
|
||||
content, err := s.ReadToEnd(fd)
|
||||
if err != nil {
|
||||
t.Fatalf("Read failed: %v", err)
|
||||
}
|
||||
if diff := cmp.Diff(expected, content); diff != "" {
|
||||
t.Fatalf("Read returned unexpected data:\n--- want\n+++ got\n%v", diff)
|
||||
func TestFullCPUMask(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
cores uint
|
||||
want string
|
||||
}{
|
||||
{1, "1"},
|
||||
{2, "3"},
|
||||
{3, "7"},
|
||||
{4, "f"},
|
||||
{5, "1f"},
|
||||
{32, "ffffffff"},
|
||||
{33, "1,ffffffff"},
|
||||
{36, "f,ffffffff"},
|
||||
{37, "1f,ffffffff"},
|
||||
{64, "ffffffff,ffffffff"},
|
||||
{65, "1,ffffffff,ffffffff"},
|
||||
} {
|
||||
if got := fullCPUMask(test.cores); got != test.want {
|
||||
t.Errorf("fullCPUMask(%d): got %s, want %s", test.cores, got, test.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSysRootContainsExpectedEntries(t *testing.T) {
|
||||
s := newTestSystem(t, "" /*pciTestDir*/)
|
||||
defer s.Destroy()
|
||||
pop := s.PathOpAtRoot("/")
|
||||
s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{
|
||||
"block": linux.DT_DIR,
|
||||
"bus": linux.DT_DIR,
|
||||
"class": linux.DT_DIR,
|
||||
"dev": linux.DT_DIR,
|
||||
"devices": linux.DT_DIR,
|
||||
"firmware": linux.DT_DIR,
|
||||
"fs": linux.DT_DIR,
|
||||
"kernel": linux.DT_DIR,
|
||||
"module": linux.DT_DIR,
|
||||
"power": linux.DT_DIR,
|
||||
})
|
||||
}
|
||||
|
||||
func TestCgroupMountpointExists(t *testing.T) {
|
||||
// Note: The mountpoint is only created if cgroups are available.
|
||||
s := newTestSystem(t, "" /*pciTestDir*/)
|
||||
defer s.Destroy()
|
||||
pop := s.PathOpAtRoot("/fs")
|
||||
s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{
|
||||
"cgroup": linux.DT_DIR,
|
||||
})
|
||||
pop = s.PathOpAtRoot("/fs/cgroup")
|
||||
s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{ /*empty*/ })
|
||||
}
|
||||
|
||||
// Check that sysfs creates the required PCI paths for V4 TPUs.
|
||||
func TestEnableTPUProxyPathsV4(t *testing.T) {
|
||||
// Set up the fs tree that will be mirrored in the sentry.
|
||||
sysfsTestDir := t.TempDir()
|
||||
busPath := path.Join(sysfsTestDir, "sys", "bus", "pci", "devices")
|
||||
if err := os.MkdirAll(busPath, 0755); err != nil {
|
||||
t.Fatalf("Failed to create bus directory: %v", err)
|
||||
}
|
||||
classAccelPath := path.Join(sysfsTestDir, "sys", "class", "accel")
|
||||
if err := os.MkdirAll(classAccelPath, 0755); err != nil {
|
||||
t.Fatalf("Failed to create accel directory: %v", err)
|
||||
}
|
||||
for i, pciAddress := range []string{"0000:00:04.0", "0000:00:05.0"} {
|
||||
accelDev := fmt.Sprintf("accel%d", i)
|
||||
accelPath := path.Join(sysfsTestDir, "sys", "devices", "pci0000:00", pciAddress, "accel", accelDev)
|
||||
if err := os.MkdirAll(accelPath, 0755); err != nil {
|
||||
t.Fatalf("Failed to create accel directory: %v", err)
|
||||
func TestOneCPUMask(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
i uint
|
||||
cores uint
|
||||
want string
|
||||
}{
|
||||
{0, 1, "1"},
|
||||
{0, 4, "1"},
|
||||
{1, 4, "2"},
|
||||
{2, 4, "4"},
|
||||
{3, 4, "8"},
|
||||
{0, 5, "01"},
|
||||
{4, 5, "10"},
|
||||
{0, 32, "00000001"},
|
||||
{26, 32, "04000000"},
|
||||
{0, 33, "0,00000001"},
|
||||
{31, 33, "0,80000000"},
|
||||
{32, 33, "1,00000000"},
|
||||
{0, 64, "00000000,00000001"},
|
||||
{31, 64, "00000000,80000000"},
|
||||
{32, 64, "00000001,00000000"},
|
||||
{63, 64, "80000000,00000000"},
|
||||
{0, 65, "0,00000000,00000001"},
|
||||
{31, 65, "0,00000000,80000000"},
|
||||
{32, 65, "0,00000001,00000000"},
|
||||
{63, 65, "0,80000000,00000000"},
|
||||
{64, 65, "1,00000000,00000000"},
|
||||
} {
|
||||
if got := oneCPUMask(test.i, test.cores); got != test.want {
|
||||
t.Errorf("oneCPUMask(%d, %d): got %s, want %s", test.i, test.cores, got, test.want)
|
||||
}
|
||||
if err := os.Symlink(path.Join("..", "..", "..", pciAddress), path.Join(accelPath, pciAddress)); err != nil {
|
||||
t.Fatalf("Failed to symlink accel directory: %v", err)
|
||||
}
|
||||
if err := os.Symlink(path.Join("..", "..", "..", pciAddress), path.Join(accelPath, "device")); err != nil {
|
||||
t.Fatalf("Failed to symlink accel device directory: %v", err)
|
||||
}
|
||||
if _, err := os.Create(path.Join(accelPath, "chip_model")); err != nil {
|
||||
t.Fatalf("Failed to create chip_model: %v", err)
|
||||
}
|
||||
if _, err := os.Create(path.Join(accelPath, "device_owner")); err != nil {
|
||||
t.Fatalf("Failed to create device_owner: %v", err)
|
||||
}
|
||||
if _, err := os.Create(path.Join(accelPath, "pci_address")); err != nil {
|
||||
t.Fatalf("Failed to create pci_address: %v", err)
|
||||
}
|
||||
if err := os.Symlink(path.Join("..", "..", "..", "devices", "pci0000:00", pciAddress), path.Join(busPath, pciAddress)); err != nil {
|
||||
t.Fatalf("Failed to symlink bus directory: %v", err)
|
||||
}
|
||||
if err := os.Symlink(path.Join("..", "..", "devices", "pci0000:00", pciAddress, "accel", accelDev), path.Join(classAccelPath, accelDev)); err != nil {
|
||||
t.Fatalf("Failed to symlink accel directory: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
s := newTestSystem(t, sysfsTestDir)
|
||||
defer s.Destroy()
|
||||
|
||||
pop := s.PathOpAtRoot("/devices/pci0000:00")
|
||||
s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{
|
||||
"0000:00:04.0": linux.DT_DIR,
|
||||
"0000:00:05.0": linux.DT_DIR,
|
||||
})
|
||||
pop = s.PathOpAtRoot("/devices/pci0000:00/0000:00:04.0/accel/accel0")
|
||||
s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{
|
||||
"0000:00:04.0": linux.DT_LNK,
|
||||
"device": linux.DT_LNK,
|
||||
"chip_model": linux.DT_REG,
|
||||
"device_owner": linux.DT_REG,
|
||||
"pci_address": linux.DT_REG,
|
||||
})
|
||||
pop = s.PathOpAtRoot("/bus/pci/devices")
|
||||
s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{
|
||||
"0000:00:04.0": linux.DT_LNK,
|
||||
"0000:00:05.0": linux.DT_LNK,
|
||||
})
|
||||
pop = s.PathOpAtRoot("/class/accel")
|
||||
s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{
|
||||
"accel0": linux.DT_LNK,
|
||||
"accel1": linux.DT_LNK,
|
||||
})
|
||||
}
|
||||
|
||||
type PCIDeviceInfo struct {
|
||||
// IOMMU group.
|
||||
group string
|
||||
pciPath string
|
||||
pciAddress string
|
||||
name string
|
||||
}
|
||||
|
||||
func (dev PCIDeviceInfo) path() string {
|
||||
return path.Join(dev.pciPath, dev.pciAddress, vfioDev, dev.name)
|
||||
}
|
||||
|
||||
func TestEnableTPUProxyPathsV5(t *testing.T) {
|
||||
// Set up the fs tree that will be mirrored in the sentry.
|
||||
sysfsTestDir := t.TempDir()
|
||||
pciPath0 := path.Join(sysfsTestDir, "sys", "devices", "pci0000:00")
|
||||
if err := os.MkdirAll(pciPath0, 0755); err != nil {
|
||||
t.Fatalf("Failed to create PCI directory: %v", err)
|
||||
}
|
||||
pciPath1 := path.Join(sysfsTestDir, "sys", "devices", "pci0000:10")
|
||||
if err := os.MkdirAll(pciPath1, 0755); err != nil {
|
||||
t.Fatalf("Failed to create PCI directory: %v", err)
|
||||
}
|
||||
busPath := path.Join(sysfsTestDir, "sys", "bus", "pci", "devices")
|
||||
if err := os.MkdirAll(busPath, 0755); err != nil {
|
||||
t.Fatalf("Failed to create bus directory: %v", err)
|
||||
}
|
||||
sysClassPath := path.Join(sysfsTestDir, "sys", "class", vfioDev)
|
||||
if err := os.MkdirAll(sysClassPath, 0755); err != nil {
|
||||
t.Fatalf("Failed to create class directory: %v", err)
|
||||
}
|
||||
|
||||
devices := []PCIDeviceInfo{
|
||||
PCIDeviceInfo{
|
||||
group: "0",
|
||||
pciPath: pciPath0,
|
||||
pciAddress: "0000:00:04.0",
|
||||
name: "vfio0",
|
||||
},
|
||||
PCIDeviceInfo{
|
||||
group: "1",
|
||||
pciPath: pciPath0,
|
||||
pciAddress: "0000:00:05.0",
|
||||
name: "vfio1",
|
||||
},
|
||||
PCIDeviceInfo{
|
||||
group: "2",
|
||||
pciPath: pciPath1,
|
||||
pciAddress: "0000:10:05.0",
|
||||
name: "vfio2",
|
||||
},
|
||||
}
|
||||
for _, device := range devices {
|
||||
devicePath := device.path()
|
||||
if err := os.MkdirAll(devicePath, 0755); err != nil {
|
||||
t.Fatalf("Failed to create PCI device directory: %v", err)
|
||||
}
|
||||
if err := os.Symlink(path.Join("..", "..", "..", device.pciAddress), path.Join(devicePath, "device")); err != nil {
|
||||
t.Fatalf("Failed to symlink device directory: %v", err)
|
||||
}
|
||||
if err := os.Symlink(path.Join("..", "..", "..", "devices", path.Base(device.pciPath), device.pciAddress), path.Join(busPath, device.pciAddress)); err != nil {
|
||||
t.Fatalf("Failed to symlink bus directory: %v", err)
|
||||
}
|
||||
if err := os.Symlink(path.Join("..", "..", "devices", path.Base(device.pciPath), device.pciAddress, vfioDev, device.name), path.Join(sysClassPath, device.name)); err != nil {
|
||||
t.Fatalf("Failed to symlink class directory: %v", err)
|
||||
}
|
||||
iommuPath := path.Join(sysfsTestDir, "sys", "kernel", "iommu_groups", device.group, "devices")
|
||||
if err := os.MkdirAll(iommuPath, 0755); err != nil {
|
||||
t.Fatalf("Failed to create iommu_groups directory: %v", err)
|
||||
}
|
||||
if err := os.Symlink(path.Join("..", "..", "..", "..", "devices", path.Base(device.pciPath), device.pciAddress), path.Join(iommuPath, device.pciAddress)); err != nil {
|
||||
t.Fatalf("Failed to symlink iommu_group devices directory: %v", err)
|
||||
}
|
||||
if err := os.Symlink(path.Join("..", "..", "..", "kernel", "iommu_groups", device.group), path.Join(device.pciPath, device.pciAddress, "iommu_group")); err != nil {
|
||||
t.Fatalf("Failed to symlink iommu_groups directory: %v", err)
|
||||
}
|
||||
}
|
||||
s := newTestSystem(t, sysfsTestDir)
|
||||
defer s.Destroy()
|
||||
|
||||
for _, device := range devices {
|
||||
// Validate PCI device symlinks.
|
||||
pop := s.PathOpAtRoot(path.Join("devices", path.Base(device.pciPath), device.pciAddress))
|
||||
s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{
|
||||
"iommu_group": linux.DT_LNK,
|
||||
vfioDev: linux.DT_DIR,
|
||||
})
|
||||
// Validate VFIO device symlinks.
|
||||
pop = s.PathOpAtRoot(path.Join("devices", path.Base(device.pciPath), device.pciAddress, vfioDev, device.name))
|
||||
s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{
|
||||
"device": linux.DT_LNK,
|
||||
})
|
||||
// Validate $IOMMU_GROUP/devices.
|
||||
pop = s.PathOpAtRoot(path.Join("kernel", "iommu_groups", string(device.group), "devices"))
|
||||
s.AssertAllDirentTypes(s.ListDirents(pop), map[string]testutil.DirentType{
|
||||
device.pciAddress: linux.DT_LNK,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user