From 004ed5316321bf9f8576cbbfe345ebad5a7e737e Mon Sep 17 00:00:00 2001 From: Etienne Perot Date: Wed, 20 Nov 2024 18:51:20 -0800 Subject: [PATCH] nvproxy: Do capability-based segmentation for seccomp filters. Updates #10856 PiperOrigin-RevId: 698599198 --- pkg/sentry/devices/nvproxy/BUILD | 2 + pkg/sentry/devices/nvproxy/nvconf/caps.go | 22 ++ pkg/sentry/devices/nvproxy/nvproxy_test.go | 128 ++++++++ pkg/sentry/devices/nvproxy/seccomp_filters.go | 300 ++++++------------ pkg/sentry/devices/nvproxy/version.go | 6 - runsc/boot/BUILD | 1 + runsc/boot/filter/BUILD | 1 + runsc/boot/filter/config/BUILD | 2 + runsc/boot/filter/config/config.go | 5 +- .../boot/filter/config/config_precompiled.go | 18 +- runsc/boot/filter/config/config_test.go | 14 +- runsc/boot/filter/dumpfilter/BUILD | 1 + runsc/boot/filter/dumpfilter/dumpfilter.go | 20 +- runsc/boot/filter/filter_bench_test.go | 6 +- runsc/boot/loader.go | 10 +- 15 files changed, 319 insertions(+), 217 deletions(-) diff --git a/pkg/sentry/devices/nvproxy/BUILD b/pkg/sentry/devices/nvproxy/BUILD index 561cb2b54..1551f9444 100644 --- a/pkg/sentry/devices/nvproxy/BUILD +++ b/pkg/sentry/devices/nvproxy/BUILD @@ -102,7 +102,9 @@ go_test( library = ":nvproxy", deps = [ "//pkg/abi/nvgpu", + "//pkg/seccomp", "//pkg/sentry/devices/nvproxy/nvconf", + "@org_golang_x_sys//unix:go_default_library", ], ) diff --git a/pkg/sentry/devices/nvproxy/nvconf/caps.go b/pkg/sentry/devices/nvproxy/nvconf/caps.go index d4f2452e3..fddbba60f 100644 --- a/pkg/sentry/devices/nvproxy/nvconf/caps.go +++ b/pkg/sentry/devices/nvproxy/nvconf/caps.go @@ -16,6 +16,8 @@ package nvconf import ( "fmt" + "maps" + "slices" "strings" ) @@ -162,3 +164,23 @@ func (c DriverCaps) NVIDIAFlags() []string { } return caps } + +// PopularCapabilitySets returns the most commonly used capability sets. +func PopularCapabilitySets() []DriverCaps { + capSets := make(map[DriverCaps]struct{}) + capSets[SupportedDriverCaps] = struct{}{} + capSets[DefaultDriverCaps] = struct{}{} + // Add every individual supported capability together with CapUtility. + for i := 0; i < numValidCaps; i++ { + cap := DriverCaps(1 << i) + if cap == CapUtility { + continue + } + if cap&SupportedDriverCaps == 0 { + continue + } + capSets[cap|CapUtility] = struct{}{} + } + // Return as a sorted list. + return slices.Sorted(maps.Keys(capSets)) +} diff --git a/pkg/sentry/devices/nvproxy/nvproxy_test.go b/pkg/sentry/devices/nvproxy/nvproxy_test.go index c6c2d3fb0..eaa75d4a9 100644 --- a/pkg/sentry/devices/nvproxy/nvproxy_test.go +++ b/pkg/sentry/devices/nvproxy/nvproxy_test.go @@ -15,9 +15,12 @@ package nvproxy import ( + "strings" "testing" + "golang.org/x/sys/unix" "gvisor.dev/gvisor/pkg/abi/nvgpu" + "gvisor.dev/gvisor/pkg/seccomp" "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf" ) @@ -210,3 +213,128 @@ func TestHandlers(t *testing.T) { ) }) } + +// TestFilterCapabilities loosely verifies that the seccomp filters have the +// expected number of entries relative to the capabilities that are enabled +// by comparing them against what the ABI handlers would suggest. +// This also acts as a useful reminder to keep the seccomp filters in sync +// with the ABI handers. +func TestFilterCapabilities(t *testing.T) { + var ( + // Set of frontend ioctls that nvproxy accepts but does not forward to the + // host. + nonForwardedFrontendIoctls = map[uint32]struct{}{ + nvgpu.NV_ESC_NUMA_INFO: struct{}{}, + } + + // Set of frontend ioctls that nvproxy makes but does not accept from the + // application. + nvproxyOnlyFrontendIoctls = map[uint32]struct{}{ /* Empty right now. */ } + + // Set of UVM ioctls that nvproxy accepts but does not forward to the host. + nonForwardedUVMIoctls = map[uint32]struct{}{ /* Empty right now. */ } + + // Set of UVM ioctls that nvproxy makes but does not accept from the + // application. + nvproxyOnlyUVMIoctls = map[uint32]struct{}{ + // UVM_TOOLS_READ_PROCESS_MEMORY is manually invoked by + // nvproxy when handling reads for UVM memory-mapped data. + nvgpu.UVM_TOOLS_READ_PROCESS_MEMORY: struct{}{}, + // Similar deal for writing to UVM memory-mapped data. + nvgpu.UVM_TOOLS_WRITE_PROCESS_MEMORY: struct{}{}, + } + ) + + // Build list of interesting capability sets. + capSets := []nvconf.DriverCaps{ + 0, + nvconf.ValidCapabilities, + nvconf.SupportedDriverCaps, + nvconf.DefaultDriverCaps, + } + for _, capName := range strings.Split(nvconf.ValidCapabilities.String(), ",") { + individualCap, _, err := nvconf.DriverCapsFromString(capName) + if err != nil { + t.Fatalf("nvconf.DriverCapsFromString(%q) failed: %v", capName, err) + } + capSets = append(capSets, individualCap) + } + for _, capSet := range nvconf.PopularCapabilitySets() { + capSets = append(capSets, capSet) + } + + // Build all the ABIs. + Init() + allAbis := make(map[string]*driverABI, len(abis)) + for version, abiCons := range abis { + allAbis[version.String()] = abiCons.cons() + } + + // Check that the filters are correct for each capability set. + // Dedupe the capability sets to avoid redundant tests. + tried := make(map[nvconf.DriverCaps]struct{}, len(capSets)) + for _, caps := range capSets { + if _, ok := tried[caps]; ok { + continue + } + tried[caps] = struct{}{} + testName := caps.String() + if testName == "" { + testName = "no_capabilities" + } + t.Run(testName, func(t *testing.T) { + frontendIoctls := map[uint32]struct{}{} + uvmIoctls := map[uint32]struct{}{} + if caps != 0 { + for frontendIoctl := range nvproxyOnlyFrontendIoctls { + frontendIoctls[frontendIoctl] = struct{}{} + } + for uvmIoctl := range nvproxyOnlyUVMIoctls { + uvmIoctls[uvmIoctl] = struct{}{} + } + for _, abi := range allAbis { + for ioctl, feHandler := range abi.frontendIoctl { + if _, nonForwarded := nonForwardedFrontendIoctls[ioctl]; nonForwarded { + continue + } + if feHandler.capSet&caps != 0 { + frontendIoctls[ioctl] = struct{}{} + } + } + for ioctl, uvmHandler := range abi.uvmIoctl { + if _, nonForwarded := nonForwardedUVMIoctls[ioctl]; nonForwarded { + continue + } + if uvmHandler.capSet&caps != 0 { + uvmIoctls[ioctl] = struct{}{} + } + } + } + } + wantFrontendIoctls := len(frontendIoctls) + if gotFrontendIoctls := len(frontendIoctlFilters(caps)); gotFrontendIoctls != wantFrontendIoctls { + t.Errorf("frontendIoctlFilters(%q) returned %d frontend ioctls, expected %d", caps.String(), gotFrontendIoctls, wantFrontendIoctls) + } + wantUvmIoctls := len(uvmIoctls) + if gotUvmIoctls := len(uvmIoctlFilters(caps)); gotUvmIoctls != wantUvmIoctls { + t.Errorf("uvmIoctlFilters(%q) returned %d UVM ioctls, expected %d", caps.String(), gotUvmIoctls, wantUvmIoctls) + } + if t.Failed() { + return + } + // Check that the total adds up too. + ioctlRules := Filters(caps).Get(unix.SYS_IOCTL) + if ioctlRules == nil { + t.Fatalf("Filters(%q) returned no SYS_IOCTL rules", caps.String()) + } + ioctlOr, isOr := ioctlRules.(seccomp.Or) + if !isOr { + t.Fatalf("Filters(%q) returned a non-Or rule for SYS_IOCTL: %v (type: %T)", caps.String(), ioctlRules, ioctlRules) + } + wantTotalIoctls := wantFrontendIoctls + wantUvmIoctls + if gotTotalIoctls := len(ioctlOr); gotTotalIoctls != wantTotalIoctls { + t.Errorf("Filters(%q) returned %d total ioctl rules, expected %d", caps.String(), gotTotalIoctls, wantTotalIoctls) + } + }) + } +} diff --git a/pkg/sentry/devices/nvproxy/seccomp_filters.go b/pkg/sentry/devices/nvproxy/seccomp_filters.go index f580918b3..695daf88a 100644 --- a/pkg/sentry/devices/nvproxy/seccomp_filters.go +++ b/pkg/sentry/devices/nvproxy/seccomp_filters.go @@ -19,206 +19,112 @@ import ( "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/abi/nvgpu" "gvisor.dev/gvisor/pkg/seccomp" + "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf" ) -// Filters returns seccomp-bpf filters for this package. -func Filters() seccomp.SyscallRules { - notIocSizeMask := ^(((uintptr(1) << linux.IOC_SIZEBITS) - 1) << linux.IOC_SIZESHIFT) // for ioctls taking arbitrary size +// Shorthands for NVIDIA driver capabilities. +const ( + // Shorthand for compute+utility capabilities. + // This is the default set of capabilities when capabilities are not + // explicitly specified, and using a shorthand for this makes ABI + // definitions in `version.go` more readable. + compUtil = nvconf.CapCompute | nvconf.CapUtility +) + +func frontendIoctlFilters(enabledCaps nvconf.DriverCaps) []seccomp.SyscallRule { + const ( + // for ioctls taking arbitrary size + notIocSizeMask = ^(((uintptr(1) << linux.IOC_SIZEBITS) - 1) << linux.IOC_SIZESHIFT) + ) + var ioctlRules []seccomp.SyscallRule + for _, feIoctl := range []struct { + arg1 seccomp.ValueMatcher + caps nvconf.DriverCaps + }{ + {seccomp.MaskedEqual(notIocSizeMask, frontendIoctlCmd(nvgpu.NV_ESC_CARD_INFO, 0)), compUtil}, + {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_CHECK_VERSION_STR, nvgpu.SizeofRMAPIVersion)), compUtil}, + {seccomp.MaskedEqual(notIocSizeMask, frontendIoctlCmd(nvgpu.NV_ESC_ATTACH_GPUS_TO_FD, 0)), compUtil}, + {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_REGISTER_FD, nvgpu.SizeofIoctlRegisterFD)), compUtil}, + {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_ALLOC_OS_EVENT, nvgpu.SizeofIoctlAllocOSEvent)), compUtil}, + {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_FREE_OS_EVENT, nvgpu.SizeofIoctlFreeOSEvent)), compUtil}, + {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_SYS_PARAMS, nvgpu.SizeofIoctlSysParams)), compUtil}, + {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_WAIT_OPEN_COMPLETE, nvgpu.SizeofIoctlWaitOpenComplete)), compUtil}, + {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_ALLOC_MEMORY, nvgpu.SizeofIoctlNVOS02ParametersWithFD)), compUtil}, + {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_FREE, nvgpu.SizeofNVOS00Parameters)), compUtil}, + {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_CONTROL, nvgpu.SizeofNVOS54Parameters)), compUtil}, + {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_ALLOC, nvgpu.SizeofNVOS64Parameters)), compUtil}, + {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_DUP_OBJECT, nvgpu.SizeofNVOS55Parameters)), compUtil}, + {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_SHARE, nvgpu.SizeofNVOS57Parameters)), compUtil}, + {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_VID_HEAP_CONTROL, nvgpu.SizeofNVOS32Parameters)), compUtil}, + {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_MAP_MEMORY, nvgpu.SizeofIoctlNVOS33ParametersWithFD)), compUtil}, + {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_UNMAP_MEMORY, nvgpu.SizeofNVOS34Parameters)), compUtil}, + {seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_UPDATE_DEVICE_MAPPING_INFO, nvgpu.SizeofNVOS56Parameters)), compUtil}, + } { + if feIoctl.caps&enabledCaps != 0 { + ioctlRules = append(ioctlRules, seccomp.PerArg{ + seccomp.NonNegativeFD{}, + feIoctl.arg1, + }) + } + } + return ioctlRules +} + +func uvmIoctlFilters(enabledCaps nvconf.DriverCaps) []seccomp.SyscallRule { + var ioctlRules []seccomp.SyscallRule + for _, uvmIoctl := range []struct { + arg1 seccomp.ValueMatcher + caps nvconf.DriverCaps + }{ + {seccomp.EqualTo(nvgpu.UVM_INITIALIZE), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_MM_INITIALIZE), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_DEINITIALIZE), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_CREATE_RANGE_GROUP), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_DESTROY_RANGE_GROUP), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_REGISTER_GPU_VASPACE), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_UNREGISTER_GPU_VASPACE), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_REGISTER_CHANNEL), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_UNREGISTER_CHANNEL), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_ENABLE_PEER_ACCESS), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_DISABLE_PEER_ACCESS), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_SET_RANGE_GROUP), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_MAP_EXTERNAL_ALLOCATION), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_FREE), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_REGISTER_GPU), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_UNREGISTER_GPU), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_PAGEABLE_MEM_ACCESS), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_SET_PREFERRED_LOCATION), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_UNSET_PREFERRED_LOCATION), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_DISABLE_READ_DUPLICATION), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_UNSET_ACCESSED_BY), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_MIGRATE), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_MIGRATE_RANGE_GROUP), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_TOOLS_READ_PROCESS_MEMORY), nvconf.ValidCapabilities}, + {seccomp.EqualTo(nvgpu.UVM_TOOLS_WRITE_PROCESS_MEMORY), nvconf.ValidCapabilities}, + {seccomp.EqualTo(nvgpu.UVM_MAP_DYNAMIC_PARALLELISM_REGION), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_UNMAP_EXTERNAL), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_ALLOC_SEMAPHORE_POOL), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_VALIDATE_VA_RANGE), compUtil}, + {seccomp.EqualTo(nvgpu.UVM_CREATE_EXTERNAL_RANGE), compUtil}, + } { + if uvmIoctl.caps&enabledCaps != 0 { + ioctlRules = append(ioctlRules, seccomp.PerArg{ + seccomp.NonNegativeFD{}, + uvmIoctl.arg1, + }) + } + } + return ioctlRules +} + +// Filters returns seccomp-bpf filters for this package when using the given +// set of capabilities. +func Filters(enabledCaps nvconf.DriverCaps) seccomp.SyscallRules { + var ioctlRules []seccomp.SyscallRule + ioctlRules = append(ioctlRules, frontendIoctlFilters(enabledCaps)...) + ioctlRules = append(ioctlRules, uvmIoctlFilters(enabledCaps)...) return seccomp.MakeSyscallRules(map[uintptr]seccomp.SyscallRule{ - unix.SYS_IOCTL: seccomp.Or{ - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.MaskedEqual(notIocSizeMask, frontendIoctlCmd(nvgpu.NV_ESC_CARD_INFO, 0)), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_CHECK_VERSION_STR, nvgpu.SizeofRMAPIVersion)), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.MaskedEqual(notIocSizeMask, frontendIoctlCmd(nvgpu.NV_ESC_ATTACH_GPUS_TO_FD, 0)), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_REGISTER_FD, nvgpu.SizeofIoctlRegisterFD)), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_ALLOC_OS_EVENT, nvgpu.SizeofIoctlAllocOSEvent)), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_FREE_OS_EVENT, nvgpu.SizeofIoctlFreeOSEvent)), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_SYS_PARAMS, nvgpu.SizeofIoctlSysParams)), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_WAIT_OPEN_COMPLETE, nvgpu.SizeofIoctlWaitOpenComplete)), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_ALLOC_MEMORY, nvgpu.SizeofIoctlNVOS02ParametersWithFD)), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_FREE, nvgpu.SizeofNVOS00Parameters)), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_CONTROL, nvgpu.SizeofNVOS54Parameters)), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_ALLOC, nvgpu.SizeofNVOS64Parameters)), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_DUP_OBJECT, nvgpu.SizeofNVOS55Parameters)), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_SHARE, nvgpu.SizeofNVOS57Parameters)), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_VID_HEAP_CONTROL, nvgpu.SizeofNVOS32Parameters)), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_MAP_MEMORY, nvgpu.SizeofIoctlNVOS33ParametersWithFD)), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_UNMAP_MEMORY, nvgpu.SizeofNVOS34Parameters)), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_UPDATE_DEVICE_MAPPING_INFO, nvgpu.SizeofNVOS56Parameters)), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_INITIALIZE), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_MM_INITIALIZE), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_DEINITIALIZE), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_CREATE_RANGE_GROUP), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_DESTROY_RANGE_GROUP), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_REGISTER_GPU_VASPACE), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_UNREGISTER_GPU_VASPACE), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_REGISTER_CHANNEL), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_UNREGISTER_CHANNEL), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_ENABLE_PEER_ACCESS), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_DISABLE_PEER_ACCESS), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_SET_RANGE_GROUP), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_MAP_EXTERNAL_ALLOCATION), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_FREE), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_REGISTER_GPU), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_UNREGISTER_GPU), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_PAGEABLE_MEM_ACCESS), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_SET_PREFERRED_LOCATION), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_UNSET_PREFERRED_LOCATION), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_DISABLE_READ_DUPLICATION), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_UNSET_ACCESSED_BY), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_MIGRATE), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_MIGRATE_RANGE_GROUP), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_TOOLS_READ_PROCESS_MEMORY), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_TOOLS_WRITE_PROCESS_MEMORY), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_MAP_DYNAMIC_PARALLELISM_REGION), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_UNMAP_EXTERNAL), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_ALLOC_SEMAPHORE_POOL), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_VALIDATE_VA_RANGE), - }, - seccomp.PerArg{ - seccomp.NonNegativeFD{}, - seccomp.EqualTo(nvgpu.UVM_CREATE_EXTERNAL_RANGE), - }, - }, + unix.SYS_IOCTL: seccomp.Or(ioctlRules), unix.SYS_MREMAP: seccomp.PerArg{ seccomp.AnyValue{}, seccomp.EqualTo(0), /* old_size */ diff --git a/pkg/sentry/devices/nvproxy/version.go b/pkg/sentry/devices/nvproxy/version.go index b015d7186..db5d578fe 100644 --- a/pkg/sentry/devices/nvproxy/version.go +++ b/pkg/sentry/devices/nvproxy/version.go @@ -22,7 +22,6 @@ import ( "strings" "gvisor.dev/gvisor/pkg/abi/nvgpu" - "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf" "gvisor.dev/gvisor/pkg/sync" ) @@ -172,11 +171,6 @@ func addDriverABI(major, minor, patch int, runfileChecksum string, cons driverAB // Init initializes abis global map. func Init() { abisOnce.Do(func() { - // Shorthands for capabilities, to keep the code below readable. - const ( - compUtil = nvconf.CapCompute | nvconf.CapUtility - ) - v535_104_05 := func() *driverABI { // Since there is no parent to inherit from, the driverABI needs to be // constructed with the entirety of the nvproxy functionality. diff --git a/runsc/boot/BUILD b/runsc/boot/BUILD index 2b7f293d1..7de9164f7 100644 --- a/runsc/boot/BUILD +++ b/runsc/boot/BUILD @@ -58,6 +58,7 @@ go_library( "//pkg/sentry/control", "//pkg/sentry/devices/memdev", "//pkg/sentry/devices/nvproxy", + "//pkg/sentry/devices/nvproxy/nvconf", "//pkg/sentry/devices/tpuproxy", "//pkg/sentry/devices/tpuproxy/vfio", "//pkg/sentry/devices/ttydev", diff --git a/runsc/boot/filter/BUILD b/runsc/boot/filter/BUILD index a40503a60..f432585ef 100644 --- a/runsc/boot/filter/BUILD +++ b/runsc/boot/filter/BUILD @@ -42,6 +42,7 @@ secbench_test( ":filter", "//pkg/abi/linux", "//pkg/seccomp", + "//pkg/sentry/devices/nvproxy/nvconf", "//pkg/sentry/platform/kvm", "//pkg/sentry/platform/systrap", "//runsc/boot/filter/config", diff --git a/runsc/boot/filter/config/BUILD b/runsc/boot/filter/config/BUILD index 8bd8343b7..eaa23ad9c 100644 --- a/runsc/boot/filter/config/BUILD +++ b/runsc/boot/filter/config/BUILD @@ -32,6 +32,7 @@ go_library( "//pkg/seccomp", "//pkg/seccomp/precompiledseccomp", "//pkg/sentry/devices/nvproxy", + "//pkg/sentry/devices/nvproxy/nvconf", "//pkg/sentry/devices/tpuproxy", "//pkg/sentry/platform", "//pkg/sentry/platform/platforms", @@ -49,6 +50,7 @@ go_test( library = ":config", deps = [ "//pkg/seccomp", + "//pkg/sentry/devices/nvproxy/nvconf", "//pkg/sentry/platform/kvm", "//pkg/sentry/platform/systrap", "@org_golang_x_sys//unix:go_default_library", diff --git a/runsc/boot/filter/config/config.go b/runsc/boot/filter/config/config.go index 579dd5de6..0fcf88d9a 100644 --- a/runsc/boot/filter/config/config.go +++ b/runsc/boot/filter/config/config.go @@ -26,6 +26,7 @@ import ( "gvisor.dev/gvisor/pkg/seccomp" "gvisor.dev/gvisor/pkg/seccomp/precompiledseccomp" "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy" + "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf" "gvisor.dev/gvisor/pkg/sentry/devices/tpuproxy" "gvisor.dev/gvisor/pkg/sentry/platform" "gvisor.dev/gvisor/pkg/sentry/socket/plugin" @@ -39,6 +40,7 @@ type Options struct { HostFilesystem bool ProfileEnable bool NVProxy bool + NVProxyCaps nvconf.DriverCaps TPUProxy bool ControllerFD uint32 CgoEnabled bool @@ -67,6 +69,7 @@ func (opt Options) ConfigKey() string { sb.WriteString(fmt.Sprintf("ProfileEnable=%t ", opt.ProfileEnable)) sb.WriteString(fmt.Sprintf("Instrumentation=%t ", isInstrumentationEnabled())) sb.WriteString(fmt.Sprintf("NVProxy=%t ", opt.NVProxy)) + sb.WriteString(fmt.Sprintf("NVProxyCaps=%v ", opt.NVProxyCaps)) sb.WriteString(fmt.Sprintf("TPUProxy=%t ", opt.TPUProxy)) sb.WriteString(fmt.Sprintf("CgoEnabled=%t ", opt.CgoEnabled)) sb.WriteString(fmt.Sprintf("PluginNetwork=%t ", opt.PluginNetwork)) @@ -147,7 +150,7 @@ func rules(opt Options, vars precompiledseccomp.Values) (seccomp.SyscallRules, s s.Merge(hostFilesystemFilters()) } if opt.NVProxy { - s.Merge(nvproxy.Filters()) + s.Merge(nvproxy.Filters(opt.NVProxyCaps)) } if opt.TPUProxy { s.Merge(tpuproxy.Filters()) diff --git a/runsc/boot/filter/config/config_precompiled.go b/runsc/boot/filter/config/config_precompiled.go index 273c0c9a7..db0b7fc54 100644 --- a/runsc/boot/filter/config/config_precompiled.go +++ b/runsc/boot/filter/config/config_precompiled.go @@ -21,6 +21,7 @@ import ( "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/seccomp" "gvisor.dev/gvisor/pkg/seccomp/precompiledseccomp" + "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf" "gvisor.dev/gvisor/pkg/sentry/platform" // Import platforms that we need to precompile filters for. @@ -86,13 +87,22 @@ func optionsToPrecompile() ([]Options, error) { return []Options{opt}, nil }, - // Expand NVProxy vs not. + // Expand NVProxy and its possible configurations. func(opt Options) ([]Options, error) { - nvProxyYes := opt - nvProxyYes.NVProxy = true + // Add the "NVProxy disabled" configuration. nvProxyNo := opt nvProxyNo.NVProxy = false - return []Options{nvProxyYes, nvProxyNo}, nil + opts := []Options{nvProxyNo} + + // Add a "yes NVProxy with this capability set" for each popular set + // of capabilities. + for _, caps := range nvconf.PopularCapabilitySets() { + optCopy := opt + optCopy.NVProxy = true + optCopy.NVProxyCaps = caps + opts = append(opts, optCopy) + } + return opts, nil }, // Expand TPUProxy vs not. diff --git a/runsc/boot/filter/config/config_test.go b/runsc/boot/filter/config/config_test.go index 5eb1ec50b..94347d160 100644 --- a/runsc/boot/filter/config/config_test.go +++ b/runsc/boot/filter/config/config_test.go @@ -21,6 +21,7 @@ import ( "golang.org/x/sys/unix" "gvisor.dev/gvisor/pkg/seccomp" + "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf" "gvisor.dev/gvisor/pkg/sentry/platform/kvm" "gvisor.dev/gvisor/pkg/sentry/platform/systrap" ) @@ -33,9 +34,15 @@ func TestIoctlFirstArgumentIsNonNegativeFD(t *testing.T) { "default kvm": { Platform: (&kvm.KVM{}).SeccompInfo(), }, - "nvproxy": { - Platform: (&systrap.Systrap{}).SeccompInfo(), - NVProxy: true, + "nvproxy default": { + Platform: (&systrap.Systrap{}).SeccompInfo(), + NVProxy: true, + NVProxyCaps: nvconf.DefaultDriverCaps, + }, + "nvproxy all": { + Platform: (&systrap.Systrap{}).SeccompInfo(), + NVProxy: true, + NVProxyCaps: nvconf.ValidCapabilities, }, "tpuproxy": { Platform: (&systrap.Systrap{}).SeccompInfo(), @@ -102,6 +109,7 @@ func TestOptionsConfigKey(t *testing.T) { "HostFilesystem": func(opt *Options) { opt.HostFilesystem = !opt.HostFilesystem }, "ProfileEnable": func(opt *Options) { opt.ProfileEnable = !opt.ProfileEnable }, "NVProxy": func(opt *Options) { opt.NVProxy = !opt.NVProxy }, + "NVProxyCaps": func(opt *Options) { opt.NVProxyCaps = ^opt.NVProxyCaps }, "TPUProxy": func(opt *Options) { opt.TPUProxy = !opt.TPUProxy }, "CgoEnabled": func(opt *Options) { opt.CgoEnabled = !opt.CgoEnabled }, "PluginNetwork": func(opt *Options) { opt.PluginNetwork = !opt.PluginNetwork }, diff --git a/runsc/boot/filter/dumpfilter/BUILD b/runsc/boot/filter/dumpfilter/BUILD index 8ff511d80..79803d9a3 100644 --- a/runsc/boot/filter/dumpfilter/BUILD +++ b/runsc/boot/filter/dumpfilter/BUILD @@ -16,6 +16,7 @@ go_binary( "//pkg/bpf", "//pkg/log", "//pkg/seccomp", + "//pkg/sentry/devices/nvproxy/nvconf", "//pkg/sentry/platform/systrap", "//runsc/boot/filter/config", "//runsc/flag", diff --git a/runsc/boot/filter/dumpfilter/dumpfilter.go b/runsc/boot/filter/dumpfilter/dumpfilter.go index 4facd43d4..d50c84457 100644 --- a/runsc/boot/filter/dumpfilter/dumpfilter.go +++ b/runsc/boot/filter/dumpfilter/dumpfilter.go @@ -23,6 +23,7 @@ import ( "gvisor.dev/gvisor/pkg/bpf" "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/seccomp" + "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf" "gvisor.dev/gvisor/pkg/sentry/platform/systrap" "gvisor.dev/gvisor/runsc/boot/filter/config" "gvisor.dev/gvisor/runsc/flag" @@ -31,7 +32,7 @@ import ( // Flags. var ( output = flag.String("output", "fancy", "Output type: 'fancy' (human-readable with line numbers resolved), 'plain' (diffable but still human-readable output), 'bytecode' (dump raw bytecode)") - nvproxy = flag.Bool("nvproxy", false, "Enable nvproxy in filter configuration") + nvproxyCaps = flag.String("nvproxy-caps", "", "If set, enable NVProxy with the given set of NVIDIA driver capabilities") optimize = flag.Bool("optimize", true, "Enable seccomp optimizations") denyAction = flag.String("deny-action", "default", "What to do if the syscall matches the 'deny' ruleset (one of: errno, kill_process, kill_thread)") defaultAction = flag.String("default-action", "default", "What to do if all the syscall rules fail to match (one of: errno, kill_process, kill_thread)") @@ -63,9 +64,22 @@ func action(s string) linux.BPFAction { func main() { flag.Parse() + var nvCaps nvconf.DriverCaps + if *nvproxyCaps != "" { + flagCaps, isAll, err := nvconf.DriverCapsFromString(*nvproxyCaps) + if err != nil { + log.Warningf("cannot parse NVProxy capabilities: %v", err) + os.Exit(1) + } + if isAll { + flagCaps |= nvconf.ValidCapabilities + } + nvCaps = flagCaps + } opt := config.Options{ - Platform: (&systrap.Systrap{}).SeccompInfo(), - NVProxy: *nvproxy, + Platform: (&systrap.Systrap{}).SeccompInfo(), + NVProxy: *nvproxyCaps != "", + NVProxyCaps: nvCaps, } rules, denyRules := config.Rules(opt) diff --git a/runsc/boot/filter/filter_bench_test.go b/runsc/boot/filter/filter_bench_test.go index a34af867f..efe749c96 100644 --- a/runsc/boot/filter/filter_bench_test.go +++ b/runsc/boot/filter/filter_bench_test.go @@ -22,6 +22,7 @@ import ( "golang.org/x/sys/unix" "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/seccomp" + "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf" "gvisor.dev/gvisor/pkg/sentry/platform/kvm" "gvisor.dev/gvisor/pkg/sentry/platform/systrap" "gvisor.dev/gvisor/runsc/boot/filter/config" @@ -102,8 +103,9 @@ func BenchmarkSentryKVM(b *testing.B) { func BenchmarkNVProxyIoctl(b *testing.B) { opts := config.Options{ - Platform: (&systrap.Systrap{}).SeccompInfo(), - NVProxy: true, + Platform: (&systrap.Systrap{}).SeccompInfo(), + NVProxy: true, + NVProxyCaps: nvconf.ValidCapabilities, } rules, denyRules := config.Rules(opts) var sequences []secbenchdef.Sequence diff --git a/runsc/boot/loader.go b/runsc/boot/loader.go index 2642cf871..6ad898bb8 100644 --- a/runsc/boot/loader.go +++ b/runsc/boot/loader.go @@ -41,6 +41,7 @@ import ( "gvisor.dev/gvisor/pkg/refs" "gvisor.dev/gvisor/pkg/sentry/control" "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy" + "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf" "gvisor.dev/gvisor/pkg/sentry/fdimport" "gvisor.dev/gvisor/pkg/sentry/fsimpl/host" "gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs" @@ -841,13 +842,20 @@ func (l *Loader) installSeccompFilters() error { log.Warningf("*** SECCOMP WARNING: syscall filter is DISABLED. Running in less secure mode.") } else { hostnet := l.root.conf.Network == config.NetworkHost + var nvproxyCaps nvconf.DriverCaps + nvproxyEnabled := specutils.NVProxyEnabled(l.root.spec, l.root.conf) + if nvproxyEnabled { + // TODO(gvisor.dev/issues/10856): Plumb capabilities here. + nvproxyCaps = nvconf.DefaultDriverCaps + } opts := filter.Options{ Platform: l.k.Platform.SeccompInfo(), HostNetwork: hostnet, HostNetworkRawSockets: hostnet && l.root.conf.EnableRaw, HostFilesystem: l.root.conf.DirectFS, ProfileEnable: l.root.conf.ProfileEnable, - NVProxy: specutils.NVProxyEnabled(l.root.spec, l.root.conf), // TODO(gvisor.dev/issues/10856): Plumb capabilities here. + NVProxy: nvproxyEnabled, + NVProxyCaps: nvproxyCaps, TPUProxy: specutils.TPUProxyIsEnabled(l.root.spec, l.root.conf), ControllerFD: uint32(l.ctrl.srv.FD()), CgoEnabled: config.CgoEnabled,