diff --git a/nogo.yaml b/nogo.yaml index 82c2264a4..27f5c4e2b 100644 --- a/nogo.yaml +++ b/nogo.yaml @@ -250,6 +250,8 @@ analyzers: - generated.gen.pb.go ST1021: # Doc should start with type name. internal: + exclude: + - "pkg/abi/nvgpu/.*.go" suppress: - "comment on exported type Translation" # Intentional. - "comment on exported type PinnedRange" # Intentional. diff --git a/pkg/abi/nvgpu/BUILD b/pkg/abi/nvgpu/BUILD new file mode 100644 index 000000000..c45c20f1b --- /dev/null +++ b/pkg/abi/nvgpu/BUILD @@ -0,0 +1,19 @@ +load("//tools:defs.bzl", "go_library") + +package(default_applicable_licenses = ["//:license"]) + +licenses(["notice"]) + +go_library( + name = "nvgpu", + srcs = [ + "classes.go", + "ctrl.go", + "frontend.go", + "nvgpu.go", + "status.go", + "uvm.go", + ], + marshal = True, + visibility = ["//pkg/sentry:internal"], +) diff --git a/pkg/abi/nvgpu/classes.go b/pkg/abi/nvgpu/classes.go new file mode 100644 index 000000000..a167384b6 --- /dev/null +++ b/pkg/abi/nvgpu/classes.go @@ -0,0 +1,199 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvgpu + +// Class handles, from src/nvidia/generated/g_allclasses.h. +const ( + NV01_ROOT = 0x00000000 + NV01_ROOT_NON_PRIV = 0x00000001 + NV01_MEMORY_SYSTEM = 0x0000003e + NV01_ROOT_CLIENT = 0x00000041 + NV01_MEMORY_SYSTEM_OS_DESCRIPTOR = 0x00000071 + NV01_EVENT_OS_EVENT = 0x00000079 + NV01_DEVICE_0 = 0x00000080 + NV20_SUBDEVICE_0 = 0x00002080 + NV50_THIRD_PARTY_P2P = 0x0000503c + GT200_DEBUGGER = 0x000083de + GF100_SUBDEVICE_MASTER = 0x000090e6 + FERMI_CONTEXT_SHARE_A = 0x00009067 + FERMI_VASPACE_A = 0x000090f1 + KEPLER_CHANNEL_GROUP_A = 0x0000a06c + TURING_USERMODE_A = 0x0000c461 + TURING_CHANNEL_GPFIFO_A = 0x0000c46f + AMPERE_CHANNEL_GPFIFO_A = 0x0000c56f + TURING_DMA_COPY_A = 0x0000c5b5 + AMPERE_DMA_COPY_A = 0x0000c6b5 + AMPERE_DMA_COPY_B = 0x0000c7b5 + TURING_COMPUTE_A = 0x0000c5c0 + AMPERE_COMPUTE_A = 0x0000c6c0 + ADA_COMPUTE_A = 0x0000c9c0 +) + +// NV0005_ALLOC_PARAMETERS is the alloc params type for NV01_EVENT_OS_EVENT, +// from src/common/sdk/nvidia/inc/class/cl0005.h. +// +// +marshal +type NV0005_ALLOC_PARAMETERS struct { + HParentClient Handle + HSrcResource Handle + HClass uint32 + NotifyIndex uint32 + Data P64 // actually FD for NV01_EVENT_OS_EVENT, see src/nvidia/src/kernel/rmapi/event.c:eventConstruct_IMPL() => src/nvidia/arch/nvalloc/unix/src/os.c:osUserHandleToKernelPtr() +} + +// NV0080_ALLOC_PARAMETERS is the alloc params type for NV01_DEVICE_0, from +// src/common/sdk/nvidia/inc/class/cl0080.h. +// +// +marshal +type NV0080_ALLOC_PARAMETERS struct { + DeviceID uint32 + HClientShare Handle + HTargetClient Handle + HTargetDevice Handle + Flags uint32 + Pad0 [4]byte + VASpaceSize uint64 + VAStartInternal uint64 + VALimitInternal uint64 + VAMode uint32 + Pad1 [4]byte +} + +// NV2080_ALLOC_PARAMETERS is the alloc params type for NV20_SUBDEVICE_0, from +// src/common/sdk/nvidia/inc/class/cl2080.h. +// +// +marshal +type NV2080_ALLOC_PARAMETERS struct { + SubDeviceID uint32 +} + +// NV503C_ALLOC_PARAMETERS is the alloc params type for NV50_THIRD_PARTY_P2P, +// from src/common/sdk/nvidia/inc/class/cl503c.h. +// +// +marshal +type NV503C_ALLOC_PARAMETERS struct { + Flags uint32 +} + +// NV83DE_ALLOC_PARAMETERS is the alloc params type for GT200_DEBUGGER, +// from src/common/sdk/nvidia/inc/class/cl83de.h. +// +// +marshal +type NV83DE_ALLOC_PARAMETERS struct { + HDebuggerClient_Obsolete Handle + HAppClient Handle + HClass3DObject Handle +} + +// NV_CTXSHARE_ALLOCATION_PARAMETERS is the alloc params type for +// FERMI_CONTEXT_SHARE_A, from src/common/sdk/nvidia/inc/nvos.h. +// +// +marshal +type NV_CTXSHARE_ALLOCATION_PARAMETERS struct { + HVASpace Handle + Flags uint32 + SubctxID uint32 +} + +// NV_VASPACE_ALLOCATION_PARAMETERS is the alloc params type for +// FERMI_VASPACE_A, from src/common/sdk/nvidia/inc/nvos.h. +// +// +marshal +type NV_VASPACE_ALLOCATION_PARAMETERS struct { + Index uint32 + Flags uint32 + VASize uint64 + VAStartInternal uint64 + VALimitInternal uint64 + BigPageSize uint32 + Pad0 [4]byte + VABase uint64 +} + +// NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS is the alloc params type for +// KEPLER_CHANNEL_GROUP_A, from src/common/sdk/nvidia/inc/nvos.h. +// +// +marshal +type NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS struct { + HObjectError Handle + HObjectECCError Handle + HVASpace Handle + EngineType uint32 + BIsCallingContextVgpuPlugin uint8 + Pad0 [3]byte +} + +// NV_MEMORY_DESC_PARAMS is from +// src/common/sdk/nvidia/inc/alloc/alloc_channel.h. +// +// +marshal +type NV_MEMORY_DESC_PARAMS struct { + Base uint64 + Size uint64 + AddressSpace uint32 + CacheAttrib uint32 +} + +// NV_CHANNEL_ALLOC_PARAMS is the alloc params type for TURING_CHANNEL_GPFIFO_A +// and AMPERE_CHANNEL_GPFIFO_A, from +// src/common/sdk/nvidia/inc/alloc/alloc_channel.h. +// +// +marshal +type NV_CHANNEL_ALLOC_PARAMS struct { + HObjectError Handle + HObjectBuffer Handle + GPFIFOOffset uint64 + GPFIFOEntries uint32 + Flags uint32 + HContextShare Handle + HVASpace Handle + HUserdMemory [NV_MAX_SUBDEVICES]Handle + UserdOffset [NV_MAX_SUBDEVICES]uint64 + EngineType uint32 + CID uint32 + SubDeviceID uint32 + HObjectECCError Handle + InstanceMem NV_MEMORY_DESC_PARAMS + UserdMem NV_MEMORY_DESC_PARAMS + RamfcMem NV_MEMORY_DESC_PARAMS + MthdbufMem NV_MEMORY_DESC_PARAMS + HPhysChannelGroup Handle + InternalFlags uint32 + ErrorNotifierMem NV_MEMORY_DESC_PARAMS + ECCErrorNotifierMem NV_MEMORY_DESC_PARAMS + ProcessID uint32 + SubProcessID uint32 +} + +// NVB0B5_ALLOCATION_PARAMETERS is the alloc param type for TURING_DMA_COPY_A, +// AMPERE_DMA_COPY_A, and AMPERE_DMA_COPY_B from +// src/common/sdk/nvidia/inc/class/clb0b5sw.h. +// +// +marshal +type NVB0B5_ALLOCATION_PARAMETERS struct { + Version uint32 + EngineType uint32 +} + +// NV_GR_ALLOCATION_PARAMETERS is the alloc param type for TURING_COMPUTE_A, +// AMPERE_COMPUTE_A, and ADA_COMPUTE_A, from src/common/sdk/nvidia/inc/nvos.h. +// +// +marshal +type NV_GR_ALLOCATION_PARAMETERS struct { + Version uint32 + Flags uint32 + Size uint32 + Caps uint32 +} diff --git a/pkg/abi/nvgpu/ctrl.go b/pkg/abi/nvgpu/ctrl.go new file mode 100644 index 000000000..69a4d0a4f --- /dev/null +++ b/pkg/abi/nvgpu/ctrl.go @@ -0,0 +1,251 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvgpu + +// From src/nvidia/interface/deprecated/rmapi_deprecated.h: +const ( + RM_GSS_LEGACY_MASK = 0x00008000 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrlxxxx.h: + +// +marshal +type NVXXXX_CTRL_XXX_INFO struct { + Index uint32 + Data uint32 +} + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0000/ctrl0000client.h: +const ( + NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE = 0xd01 + NV0000_CTRL_CMD_CLIENT_SET_INHERITED_SHARE_POLICY = 0xd04 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0000/ctrl0000gpu.h: +const ( + NV0000_CTRL_CMD_GPU_GET_ATTACHED_IDS = 0x201 + NV0000_CTRL_CMD_GPU_GET_ID_INFO = 0x202 + NV0000_CTRL_CMD_GPU_GET_ID_INFO_V2 = 0x205 + NV0000_CTRL_CMD_GPU_GET_PROBED_IDS = 0x214 + NV0000_CTRL_CMD_GPU_ATTACH_IDS = 0x215 + NV0000_CTRL_CMD_GPU_DETACH_IDS = 0x216 + NV0000_CTRL_CMD_GPU_GET_PCI_INFO = 0x21b + NV0000_CTRL_CMD_GPU_QUERY_DRAIN_STATE = 0x279 + NV0000_CTRL_CMD_GPU_GET_MEMOP_ENABLE = 0x27b +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0000/ctrl0000syncgpuboost.h: +const ( + NV0000_CTRL_CMD_SYNC_GPU_BOOST_GROUP_INFO = 0xa04 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0000/ctrl0000system.h: +const ( + NV0000_CTRL_CMD_SYSTEM_GET_BUILD_VERSION = 0x101 + NV0000_CTRL_CMD_SYSTEM_GET_FABRIC_STATUS = 0x136 +) + +// +marshal +type NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS struct { + SizeOfStrings uint32 + Pad [4]byte + PDriverVersionBuffer P64 + PVersionBuffer P64 + PTitleBuffer P64 + ChangelistNumber uint32 + OfficialChangelistNumber uint32 +} + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080fb.h: +const ( + NV0080_CTRL_CMD_FB_GET_CAPS_V2 = 0x801307 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080fifo.h: +const ( + NV0080_CTRL_CMD_FIFO_GET_CHANNELLIST = 0x80170d +) + +// +marshal +type NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS struct { + NumChannels uint32 + Pad [4]byte + PChannelHandleList P64 + PChannelList P64 +} + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080gpu.h: +const ( + NV0080_CTRL_CMD_GPU_GET_NUM_SUBDEVICES = 0x800280 + NV0080_CTRL_CMD_GPU_QUERY_SW_STATE_PERSISTENCE = 0x800288 + NV0080_CTRL_CMD_GPU_GET_VIRTUALIZATION_MODE = 0x800289 + NV0080_CTRL_CMD_GPU_GET_CLASSLIST_V2 = 0x800292 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080gr.h: + +// +marshal +type NV0080_CTRL_GR_ROUTE_INFO struct { + Flags uint32 + Pad [4]byte + Route uint64 +} + +// From src/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080host.h: +const ( + NV0080_CTRL_CMD_HOST_GET_CAPS_V2 = 0x801402 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080bus.h: +const ( + NV2080_CTRL_CMD_BUS_GET_PCI_INFO = 0x20801801 + NV2080_CTRL_CMD_BUS_GET_PCI_BAR_INFO = 0x20801803 + NV2080_CTRL_CMD_BUS_GET_INFO_V2 = 0x20801823 + NV2080_CTRL_CMD_BUS_GET_PCIE_SUPPORTED_GPU_ATOMICS = 0x2080182a +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080ce.h: +const ( + NV2080_CTRL_CMD_CE_GET_ALL_CAPS = 0x20802a0a +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080fb.h: +const ( + NV2080_CTRL_CMD_FB_GET_INFO_V2 = 0x20801303 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080fifo.h: +const ( + NV2080_CTRL_CMD_FIFO_DISABLE_CHANNELS = 0x2080110b + + NV2080_CTRL_FIFO_DISABLE_CHANNELS_MAX_ENTRIES = 64 +) + +// +marshal +type NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS struct { + BDisable uint8 + Pad1 [3]byte + NumChannels uint32 + BOnlyDisableScheduling uint8 + BRewindGpPut uint8 + Pad2 [6]byte + PRunlistPreemptEvent P64 + HClientList [NV2080_CTRL_FIFO_DISABLE_CHANNELS_MAX_ENTRIES]Handle + HChannelList [NV2080_CTRL_FIFO_DISABLE_CHANNELS_MAX_ENTRIES]Handle +} + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gpu.h: +const ( + NV2080_CTRL_CMD_GPU_GET_INFO_V2 = 0x20800102 + NV2080_CTRL_CMD_GPU_GET_NAME_STRING = 0x20800110 + NV2080_CTRL_CMD_GPU_GET_SHORT_NAME_STRING = 0x20800111 + NV2080_CTRL_CMD_GPU_GET_SIMULATION_INFO = 0x20800119 + NV2080_CTRL_CMD_GPU_QUERY_ECC_STATUS = 0x2080012f + NV2080_CTRL_CMD_GPU_QUERY_COMPUTE_MODE_RULES = 0x20800131 + NV2080_CTRL_CMD_GPU_ACQUIRE_COMPUTE_MODE_RESERVATION = 0x20800145 // undocumented; paramSize == 0 + NV2080_CTRL_CMD_GPU_RELEASE_COMPUTE_MODE_RESERVATION = 0x20800146 // undocumented; paramSize == 0 + NV2080_CTRL_CMD_GPU_GET_GID_INFO = 0x2080014a + NV2080_CTRL_CMD_GPU_GET_ENGINES_V2 = 0x20800170 + NV2080_CTRL_CMD_GPU_GET_ACTIVE_PARTITION_IDS = 0x2080018b + NV2080_CTRL_CMD_GPU_GET_COMPUTE_POLICY_CONFIG = 0x20800195 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gr.h: +const ( + NV2080_CTRL_CMD_GR_GET_INFO = 0x20801201 + NV2080_CTRL_CMD_GR_SET_CTXSW_PREEMPTION_MODE = 0x20801210 + NV2080_CTRL_CMD_GR_GET_CTX_BUFFER_SIZE = 0x20801218 + NV2080_CTRL_CMD_GR_GET_GLOBAL_SM_ORDER = 0x2080121b + NV2080_CTRL_CMD_GR_GET_CAPS_V2 = 0x20801227 + NV2080_CTRL_CMD_GR_GET_GPC_MASK = 0x2080122a + NV2080_CTRL_CMD_GR_GET_TPC_MASK = 0x2080122b +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gsp.h: +const ( + NV2080_CTRL_CMD_GSP_GET_FEATURES = 0x20803601 +) + +// +marshal +type NV2080_CTRL_GR_GET_INFO_PARAMS struct { + GRInfoListSize uint32 // in elements + Pad [4]byte + GRInfoList P64 + GRRouteInfo NV0080_CTRL_GR_ROUTE_INFO +} + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080mc.h: +const ( + NV2080_CTRL_CMD_MC_GET_ARCH_INFO = 0x20801701 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080nvlink.h: +const ( + NV2080_CTRL_CMD_NVLINK_GET_NVLINK_STATUS = 0x20803002 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080perf.h: +const ( + NV2080_CTRL_CMD_PERF_BOOST = 0x2080200a +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080rc.h: +const ( + NV2080_CTRL_CMD_RC_GET_WATCHDOG_INFO = 0x20802209 + NV2080_CTRL_CMD_RC_RELEASE_WATCHDOG_REQUESTS = 0x2080220c + NV2080_CTRL_CMD_RC_SOFT_DISABLE_WATCHDOG = 0x20802210 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080tmr.h: +const ( + NV2080_CTRL_CMD_TIMER_GET_GPU_CPU_TIME_CORRELATION_INFO = 0x20800406 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl503c.h: +const ( + NV503C_CTRL_CMD_REGISTER_VA_SPACE = 0x503c0102 + NV503C_CTRL_CMD_REGISTER_VIDMEM = 0x503c0104 + NV503C_CTRL_CMD_UNREGISTER_VIDMEM = 0x503c0105 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl83de/ctrl83dedebug.h: +const ( + NV83DE_CTRL_CMD_DEBUG_SET_EXCEPTION_MASK = 0x83de0309 + NV83DE_CTRL_CMD_DEBUG_READ_ALL_SM_ERROR_STATES = 0x83de030c + NV83DE_CTRL_CMD_DEBUG_CLEAR_ALL_SM_ERROR_STATES = 0x83de0310 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrlc36f.h: +const ( + NVC36F_CTRL_GET_CLASS_ENGINEID = 0xc36f0101 + NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN = 0xc36f0108 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl906f.h: +const ( + NV906F_CTRL_CMD_RESET_CHANNEL = 0x906f0102 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrl90e6.h: +const ( + NV90E6_CTRL_CMD_MASTER_GET_VIRTUAL_FUNCTION_ERROR_CONT_INTR_MASK = 0x90e60102 +) + +// From src/common/sdk/nvidia/inc/ctrl/ctrla06c.h: +const ( + NVA06C_CTRL_CMD_GPFIFO_SCHEDULE = 0xa06c0101 + NVA06C_CTRL_CMD_SET_TIMESLICE = 0xa06c0103 + NVA06C_CTRL_CMD_PREEMPT = 0xa06c0105 +) diff --git a/pkg/abi/nvgpu/frontend.go b/pkg/abi/nvgpu/frontend.go new file mode 100644 index 000000000..1e7e1a763 --- /dev/null +++ b/pkg/abi/nvgpu/frontend.go @@ -0,0 +1,322 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvgpu + +// NV_IOCTL_MAGIC is the "canonical" IOC_TYPE for frontend ioctls. +// The driver ignores IOC_TYPE, allowing any value to be passed. +const NV_IOCTL_MAGIC = uint32('F') + +// Frontend ioctl numbers. +// Note that these are only the IOC_NR part of the ioctl command. +const ( + // From kernel-open/common/inc/nv-ioctl-numbers.h: + NV_IOCTL_BASE = 200 + NV_ESC_CARD_INFO = NV_IOCTL_BASE + 0 + NV_ESC_REGISTER_FD = NV_IOCTL_BASE + 1 + NV_ESC_ALLOC_OS_EVENT = NV_IOCTL_BASE + 6 + NV_ESC_FREE_OS_EVENT = NV_IOCTL_BASE + 7 + NV_ESC_CHECK_VERSION_STR = NV_IOCTL_BASE + 10 + NV_ESC_SYS_PARAMS = NV_IOCTL_BASE + 14 + + // From kernel-open/common/inc/nv-ioctl-numa.h: + NV_ESC_NUMA_INFO = NV_IOCTL_BASE + 15 + + // From src/nvidia/arch/nvalloc/unix/include/nv_escape.h: + NV_ESC_RM_ALLOC_MEMORY = 0x27 + NV_ESC_RM_FREE = 0x29 + NV_ESC_RM_CONTROL = 0x2a + NV_ESC_RM_ALLOC = 0x2b + NV_ESC_RM_DUP_OBJECT = 0x34 + NV_ESC_RM_SHARE = 0x35 + NV_ESC_RM_VID_HEAP_CONTROL = 0x4a + NV_ESC_RM_MAP_MEMORY = 0x4e + NV_ESC_RM_UNMAP_MEMORY = 0x4f + NV_ESC_RM_UPDATE_DEVICE_MAPPING_INFO = 0x5e +) + +// Frontend ioctl parameter structs, from src/common/sdk/nvidia/inc/nvos.h or +// kernel-open/common/inc/nv-ioctl.h. + +// IoctlRegisterFD is nv_ioctl_register_fd_t, the parameter type for +// NV_ESC_REGISTER_FD. +// +// +marshal +type IoctlRegisterFD struct { + CtlFD int32 +} + +// IoctlAllocOSEvent is nv_ioctl_alloc_os_event_t, the parameter type for +// NV_ESC_ALLOC_OS_EVENT. +// +// +marshal +type IoctlAllocOSEvent struct { + HClient Handle + HDevice Handle + FD uint32 + Status uint32 +} + +// IoctlFreeOSEvent is nv_ioctl_free_os_event_t, the parameter type for +// NV_ESC_FREE_OS_EVENT. +// +// +marshal +type IoctlFreeOSEvent struct { + HClient Handle + HDevice Handle + FD uint32 + Status uint32 +} + +// RMAPIVersion is nv_rm_api_version_t, the parameter type for +// NV_ESC_CHECK_VERSION_STR. +// +// +marshal +type RMAPIVersion struct { + Cmd uint32 + Reply uint32 + VersionString [64]byte +} + +// IoctlSysParams is nv_ioctl_sys_params_t, the parameter type for +// NV_ESC_SYS_PARAMS. +// +// +marshal +type IoctlSysParams struct { + MemblockSize uint64 +} + +// IoctlNVOS02ParametersWithFD is nv_ioctl_nvos2_parameters_with_fd, the +// parameter type for NV_ESC_RM_ALLOC_MEMORY. +// +// +marshal +type IoctlNVOS02ParametersWithFD struct { + Params NVOS02Parameters + FD int32 + Pad0 [4]byte +} + +// +marshal +type NVOS02Parameters struct { + HRoot Handle + HObjectParent Handle + HObjectNew Handle + HClass uint32 + Flags uint32 + Pad0 [4]byte + PMemory P64 // address of application mapping, without indirection + Limit uint64 + Status uint32 + Pad1 [4]byte +} + +// NVOS00Parameters is NVOS00_PARAMETERS, the parameter type for +// NV_ESC_RM_FREE. +// +// +marshal +type NVOS00Parameters struct { + HRoot Handle + HObjectParent Handle + HObjectOld Handle + Status uint32 +} + +// NVOS21Parameters is NVOS21_PARAMETERS, one possible parameter type for +// NV_ESC_RM_ALLOC. +// +// +marshal +type NVOS21Parameters struct { + HRoot Handle + HObjectParent Handle + HObjectNew Handle + HClass uint32 + PAllocParms P64 + Status uint32 + Pad0 [4]byte +} + +// NVOS55Parameters is NVOS55_PARAMETERS, the parameter type for +// NV_ESC_RM_DUP_OBJECT. +// +// +marshal +type NVOS55Parameters struct { + HClient Handle + HParent Handle + HObject Handle + HClientSrc Handle + HObjectSrc Handle + Flags uint32 + Status uint32 +} + +// NVOS57Parameters is NVOS57_PARAMETERS, the parameter type for +// NV_ESC_RM_SHARE. +// +// +marshal +type NVOS57Parameters struct { + HClient Handle + HObject Handle + SharePolicy RS_SHARE_POLICY + Status uint32 +} + +// NVOS32Parameters is NVOS32_PARAMETERS, the parameter type for +// NV_ESC_RM_VID_HEAP_CONTROL. +// +// +marshal +type NVOS32Parameters struct { + HRoot Handle + HObjectParent Handle + Function uint32 + HVASpace Handle + IVCHeapNumber int16 + Pad [2]byte + Status uint32 + Total uint64 + Free uint64 + Data [144]byte // union +} + +// Possible values for NVOS32Parameters.Function: +const ( + NVOS32_FUNCTION_ALLOC_SIZE = 2 +) + +// NVOS32AllocSize is the type of NVOS32Parameters.Data for +// NVOS32_FUNCTION_ALLOC_SIZE. +type NVOS32AllocSize struct { + Owner uint32 + HMemory Handle + Type uint32 + Flags uint32 + Attr uint32 + Format uint32 + ComprCovg uint32 + ZcullCovg uint32 + PartitionStride uint32 + Width uint32 + Height uint32 + Pad0 [4]byte + Size uint64 + Alignment uint64 + Offset uint64 + Limit uint64 + Address P64 + RangeBegin uint64 + RangeEnd uint64 + Attr2 uint32 + CtagOffset uint32 +} + +// IoctlNVOS33ParametersWithFD is nv_ioctl_nvos33_parameters_with_fd, the +// parameter type for NV_ESC_RM_MAP_MEMORY, from +// src/nvidia/arch/nvalloc/unix/include/nv-unix-nvos-params-wrappers.h. +// +// +marshal +type IoctlNVOS33ParametersWithFD struct { + Params NVOS33Parameters + FD int32 + Pad0 [4]byte +} + +// +marshal +type NVOS33Parameters struct { + HClient Handle + HDevice Handle + HMemory Handle + Pad0 [4]byte + Offset uint64 + Length uint64 + PLinearAddress P64 // address of application mapping, without indirection + Status uint32 + Flags uint32 +} + +// NVOS34Parameters is NVOS34_PARAMETERS, the parameter type for +// NV_ESC_RM_UNMAP_MEMORY. +// +// +marshal +type NVOS34Parameters struct { + HClient Handle + HDevice Handle + HMemory Handle + Pad0 [4]byte + PLinearAddress P64 // address of application mapping, without indirection + Status uint32 + Flags uint32 +} + +// NVOS54Parameters is NVOS54_PARAMETERS, the parameter type for +// NV_ESC_RM_CONTROL. +// +// +marshal +type NVOS54Parameters struct { + HClient Handle + HObject Handle + Cmd uint32 + Flags uint32 + Params P64 + ParamsSize uint32 + Status uint32 +} + +// NVOS56Parameters is NVOS56_PARAMETERS, the parameter type for +// NV_ESC_RM_UPDATE_DEVICE_MAPPING_INFO. +// +// +marshal +type NVOS56Parameters struct { + HClient Handle + HDevice Handle + HMemory Handle + Pad0 [4]byte + POldCPUAddress P64 + PNewCPUAddress P64 + Status uint32 + Pad1 [4]byte +} + +// NVOS64Parameters is NVOS64_PARAMETERS, one possible parameter type for +// NV_ESC_RM_ALLOC. +// +// +marshal +type NVOS64Parameters struct { + HRoot Handle + HObjectParent Handle + HObjectNew Handle + HClass uint32 + PAllocParms P64 + PRightsRequested P64 + Flags uint32 + Status uint32 +} + +// Frontend ioctl parameter struct sizes. +var ( + SizeofIoctlRegisterFD = uint32((*IoctlRegisterFD)(nil).SizeBytes()) + SizeofIoctlAllocOSEvent = uint32((*IoctlAllocOSEvent)(nil).SizeBytes()) + SizeofIoctlFreeOSEvent = uint32((*IoctlFreeOSEvent)(nil).SizeBytes()) + SizeofRMAPIVersion = uint32((*RMAPIVersion)(nil).SizeBytes()) + SizeofIoctlSysParams = uint32((*IoctlSysParams)(nil).SizeBytes()) + SizeofIoctlNVOS02ParametersWithFD = uint32((*IoctlNVOS02ParametersWithFD)(nil).SizeBytes()) + SizeofNVOS00Parameters = uint32((*NVOS00Parameters)(nil).SizeBytes()) + SizeofNVOS21Parameters = uint32((*NVOS21Parameters)(nil).SizeBytes()) + SizeofIoctlNVOS33ParametersWithFD = uint32((*IoctlNVOS33ParametersWithFD)(nil).SizeBytes()) + SizeofNVOS55Parameters = uint32((*NVOS55Parameters)(nil).SizeBytes()) + SizeofNVOS57Parameters = uint32((*NVOS57Parameters)(nil).SizeBytes()) + SizeofNVOS32Parameters = uint32((*NVOS32Parameters)(nil).SizeBytes()) + SizeofNVOS34Parameters = uint32((*NVOS34Parameters)(nil).SizeBytes()) + SizeofNVOS54Parameters = uint32((*NVOS54Parameters)(nil).SizeBytes()) + SizeofNVOS56Parameters = uint32((*NVOS56Parameters)(nil).SizeBytes()) + SizeofNVOS64Parameters = uint32((*NVOS64Parameters)(nil).SizeBytes()) +) diff --git a/pkg/abi/nvgpu/nvgpu.go b/pkg/abi/nvgpu/nvgpu.go new file mode 100644 index 000000000..cb8b758d2 --- /dev/null +++ b/pkg/abi/nvgpu/nvgpu.go @@ -0,0 +1,64 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package nvgpu tracks the ABI of the Nvidia GPU Linux kernel driver: +// https://github.com/NVIDIA/open-gpu-kernel-modules +package nvgpu + +// Device numbers. +const ( + NV_MAJOR_DEVICE_NUMBER = 195 // from kernel-open/common/inc/nv.h + NV_CONTROL_DEVICE_MINOR = 255 // from kernel-open/common/inc/nv-linux.h + NVIDIA_UVM_PRIMARY_MINOR_NUMBER = 0 // from kernel-open/nvidia-uvm/uvm_common.h +) + +// Handle is NvHandle, from src/common/sdk/nvidia/inc/nvtypes.h. +// +// +marshal +type Handle struct { + Val uint32 +} + +// P64 is NvP64, from src/common/sdk/nvidia/inc/nvtypes.h. +// +// +marshal +type P64 uint64 + +// From src/common/sdk/nvidia/inc/nvlimits.h: +const ( + NV_MAX_DEVICES = 32 + NV_MAX_SUBDEVICES = 8 +) + +// RS_ACCESS_MASK is RS_ACCESS_MASK, from +// src/common/sdk/nvidia/inc/rs_access.h. +// +// +marshal +type RS_ACCESS_MASK struct { + Limbs [SDK_RS_ACCESS_MAX_LIMBS]uint32 // RsAccessLimb +} + +const SDK_RS_ACCESS_MAX_LIMBS = 1 + +// RS_SHARE_POLICY is RS_SHARE_POLICY, from +// src/common/sdk/nvidia/inc/rs_access.h. +// +// +marshal +type RS_SHARE_POLICY struct { + Target uint32 + AccessMask RS_ACCESS_MASK + Type uint16 + Action uint8 + Pad [1]byte +} diff --git a/pkg/abi/nvgpu/status.go b/pkg/abi/nvgpu/status.go new file mode 100644 index 000000000..189bc5db9 --- /dev/null +++ b/pkg/abi/nvgpu/status.go @@ -0,0 +1,22 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvgpu + +// Status codes, from src/common/sdk/nvidia/inc/nvstatuscodes.h. +const ( + NV_ERR_INVALID_ADDRESS = 0x0000001e + NV_ERR_INVALID_LIMIT = 0x0000002e + NV_ERR_NOT_SUPPORTED = 0x00000056 +) diff --git a/pkg/abi/nvgpu/uvm.go b/pkg/abi/nvgpu/uvm.go new file mode 100644 index 000000000..6ca579786 --- /dev/null +++ b/pkg/abi/nvgpu/uvm.go @@ -0,0 +1,236 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvgpu + +// HasRMCtrlFD is a type constraint for UVM parameter structs containing a +// RMCtrlFD field. This is necessary because, as of this writing (Go 1.20), +// there is no way to enable field access using a Go type constraint. +type HasRMCtrlFD interface { + GetRMCtrlFD() int32 + SetRMCtrlFD(int32) +} + +// UVM ioctl commands. +const ( + // From kernel-open/nvidia-uvm/uvm_linux_ioctl.h: + UVM_INITIALIZE = 0x30000001 + UVM_DEINITIALIZE = 0x30000002 + + // From kernel-open/nvidia-uvm/uvm_ioctl.h: + UVM_CREATE_RANGE_GROUP = 23 + UVM_DESTROY_RANGE_GROUP = 24 + UVM_REGISTER_GPU_VASPACE = 25 + UVM_UNREGISTER_GPU_VASPACE = 26 + UVM_REGISTER_CHANNEL = 27 + UVM_UNREGISTER_CHANNEL = 28 + UVM_MAP_EXTERNAL_ALLOCATION = 33 + UVM_FREE = 34 + UVM_REGISTER_GPU = 37 + UVM_UNREGISTER_GPU = 38 + UVM_PAGEABLE_MEM_ACCESS = 39 + UVM_MAP_DYNAMIC_PARALLELISM_REGION = 65 + UVM_ALLOC_SEMAPHORE_POOL = 68 + UVM_VALIDATE_VA_RANGE = 72 + UVM_CREATE_EXTERNAL_RANGE = 73 +) + +// +marshal +type UVM_INITIALIZE_PARAMS struct { + Flags uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// UVM_INITIALIZE_PARAMS flags, from kernel-open/nvidia-uvm/uvm_types.h. +const ( + UVM_INIT_FLAGS_MULTI_PROCESS_SHARING_MODE = 0x2 +) + +// +marshal +type UVM_CREATE_RANGE_GROUP_PARAMS struct { + RangeGroupID uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// +marshal +type UVM_DESTROY_RANGE_GROUP_PARAMS struct { + RangeGroupID uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// +marshal +type UVM_REGISTER_GPU_VASPACE_PARAMS struct { + GPUUUID [16]uint8 + RMCtrlFD int32 + HClient Handle + HVASpace Handle + RMStatus uint32 +} + +func (p *UVM_REGISTER_GPU_VASPACE_PARAMS) GetRMCtrlFD() int32 { + return p.RMCtrlFD +} + +func (p *UVM_REGISTER_GPU_VASPACE_PARAMS) SetRMCtrlFD(fd int32) { + p.RMCtrlFD = fd +} + +// +marshal +type UVM_UNREGISTER_GPU_VASPACE_PARAMS struct { + GPUUUID [16]uint8 + RMStatus uint32 +} + +// +marshal +type UVM_REGISTER_CHANNEL_PARAMS struct { + GPUUUID [16]uint8 + RMCtrlFD int32 + HClient Handle + HChannel Handle + Pad [4]byte + Base uint64 + Length uint64 + RMStatus uint32 + Pad0 [4]byte +} + +func (p *UVM_REGISTER_CHANNEL_PARAMS) GetRMCtrlFD() int32 { + return p.RMCtrlFD +} + +func (p *UVM_REGISTER_CHANNEL_PARAMS) SetRMCtrlFD(fd int32) { + p.RMCtrlFD = fd +} + +// +marshal +type UVM_UNREGISTER_CHANNEL_PARAMS struct { + GPUUUID [16]uint8 + HClient Handle + HChannel Handle + RMStatus uint32 +} + +// +marshal +type UVM_MAP_EXTERNAL_ALLOCATION_PARAMS struct { + Base uint64 + Length uint64 + Offset uint64 + PerGPUAttributes [UVM_MAX_GPUS]UvmGpuMappingAttributes + GPUAttributesCount uint64 + RMCtrlFD int32 + HClient Handle + HMemory Handle + RMStatus uint32 +} + +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS) GetRMCtrlFD() int32 { + return p.RMCtrlFD +} + +func (p *UVM_MAP_EXTERNAL_ALLOCATION_PARAMS) SetRMCtrlFD(fd int32) { + p.RMCtrlFD = fd +} + +// +marshal +type UVM_FREE_PARAMS struct { + Base uint64 + Length uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// +marshal +type UVM_REGISTER_GPU_PARAMS struct { + GPUUUID [16]uint8 + NumaEnabled uint8 + Pad [3]byte + NumaNodeID int32 + RMCtrlFD int32 + HClient Handle + HSMCPartRef Handle + RMStatus uint32 +} + +func (p *UVM_REGISTER_GPU_PARAMS) GetRMCtrlFD() int32 { + return p.RMCtrlFD +} + +func (p *UVM_REGISTER_GPU_PARAMS) SetRMCtrlFD(fd int32) { + p.RMCtrlFD = fd +} + +// +marshal +type UVM_UNREGISTER_GPU_PARAMS struct { + GPUUUID [16]uint8 + RMStatus uint32 +} + +// +marshal +type UVM_PAGEABLE_MEM_ACCESS_PARAMS struct { + PageableMemAccess uint8 + Pad [3]byte + RMStatus uint32 +} + +// +marshal +type UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS struct { + Base uint64 + Length uint64 + GPUUUID [16]uint8 + RMStatus uint32 + Pad0 [4]byte +} + +// +marshal +type UVM_ALLOC_SEMAPHORE_POOL_PARAMS struct { + Base uint64 + Length uint64 + PerGPUAttributes [UVM_MAX_GPUS]UvmGpuMappingAttributes + GPUAttributesCount uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// +marshal +type UVM_VALIDATE_VA_RANGE_PARAMS struct { + Base uint64 + Length uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// +marshal +type UVM_CREATE_EXTERNAL_RANGE_PARAMS struct { + Base uint64 + Length uint64 + RMStatus uint32 + Pad0 [4]byte +} + +// From kernel-open/nvidia-uvm/uvm_types.h: + +const UVM_MAX_GPUS = NV_MAX_DEVICES + +// +marshal +type UvmGpuMappingAttributes struct { + GPUUUID [16]byte + GPUMappingType uint32 + GPUCachingType uint32 + GPUFormatType uint32 + GPUElementBits uint32 + GPUCompressionType uint32 +} diff --git a/pkg/sentry/devices/nvproxy/BUILD b/pkg/sentry/devices/nvproxy/BUILD new file mode 100644 index 000000000..30d4a0290 --- /dev/null +++ b/pkg/sentry/devices/nvproxy/BUILD @@ -0,0 +1,55 @@ +load("//tools:defs.bzl", "go_library") +load("//pkg/sync/locking:locking.bzl", "declare_mutex") + +package(default_applicable_licenses = ["//:license"]) + +licenses(["notice"]) + +declare_mutex( + name = "objs_mutex", + out = "objs_mutex.go", + package = "nvproxy", + prefix = "objs", +) + +go_library( + name = "nvproxy", + srcs = [ + "frontend.go", + "frontend_mmap.go", + "frontend_unsafe.go", + "nvproxy.go", + "nvproxy_unsafe.go", + "objs_mutex.go", + "seccomp_filters.go", + "uvm.go", + "uvm_mmap.go", + "uvm_unsafe.go", + ], + visibility = ["//pkg/sentry:internal"], + deps = [ + "//pkg/abi/linux", + "//pkg/abi/nvgpu", + "//pkg/cleanup", + "//pkg/context", + "//pkg/errors/linuxerr", + "//pkg/fdnotifier", + "//pkg/hostarch", + "//pkg/log", + "//pkg/marshal", + "//pkg/marshal/primitive", + "//pkg/safemem", + "//pkg/seccomp", + "//pkg/sentry/arch", + "//pkg/sentry/fsimpl/devtmpfs", + "//pkg/sentry/kernel", + "//pkg/sentry/memmap", + "//pkg/sentry/mm", + "//pkg/sentry/vfs", + "//pkg/sync", + "//pkg/sync/locking", + "//pkg/usermem", + "//pkg/waiter", + "@org_golang_x_sys//unix:go_default_library", + ], +) diff --git a/pkg/sentry/devices/nvproxy/frontend.go b/pkg/sentry/devices/nvproxy/frontend.go new file mode 100644 index 000000000..65984a3c8 --- /dev/null +++ b/pkg/sentry/devices/nvproxy/frontend.go @@ -0,0 +1,902 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvproxy + +import ( + "fmt" + "sync/atomic" + + "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/abi/nvgpu" + "gvisor.dev/gvisor/pkg/cleanup" + "gvisor.dev/gvisor/pkg/context" + "gvisor.dev/gvisor/pkg/errors/linuxerr" + "gvisor.dev/gvisor/pkg/fdnotifier" + "gvisor.dev/gvisor/pkg/hostarch" + "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/sentry/arch" + "gvisor.dev/gvisor/pkg/sentry/kernel" + "gvisor.dev/gvisor/pkg/sentry/memmap" + "gvisor.dev/gvisor/pkg/sentry/mm" + "gvisor.dev/gvisor/pkg/sentry/vfs" + "gvisor.dev/gvisor/pkg/usermem" + "gvisor.dev/gvisor/pkg/waiter" +) + +// frontendDevice implements vfs.Device for /dev/nvidia# and /dev/nvidiactl. +// +// +stateify savable +type frontendDevice struct { + nvp *nvproxy + minor uint32 +} + +// Open implements vfs.Device.Open. +func (dev *frontendDevice) Open(ctx context.Context, mnt *vfs.Mount, vfsd *vfs.Dentry, opts vfs.OpenOptions) (*vfs.FileDescription, error) { + var hostPath string + if dev.minor == nvgpu.NV_CONTROL_DEVICE_MINOR { + hostPath = "/dev/nvidiactl" + } else { + hostPath = fmt.Sprintf("/dev/nvidia%d", dev.minor) + } + hostFD, err := unix.Openat(-1, hostPath, int((opts.Flags&unix.O_ACCMODE)|unix.O_NOFOLLOW), 0) + if err != nil { + ctx.Warningf("nvproxy: failed to open host %s: %v", hostPath, err) + return nil, err + } + fd := &frontendFD{ + nvp: dev.nvp, + hostFD: int32(hostFD), + isControl: dev.minor == nvgpu.NV_CONTROL_DEVICE_MINOR, + } + if err := fd.vfsfd.Init(fd, opts.Flags, mnt, vfsd, &vfs.FileDescriptionOptions{ + UseDentryMetadata: true, + }); err != nil { + unix.Close(hostFD) + return nil, err + } + if err := fdnotifier.AddFD(int32(hostFD), &fd.queue); err != nil { + unix.Close(hostFD) + return nil, err + } + fd.memmapFile.fd = fd + return &fd.vfsfd, nil +} + +// frontendFD implements vfs.FileDescriptionImpl for /dev/nvidia# and +// /dev/nvidiactl. +// +// frontendFD is not savable; we do not implement save/restore of host GPU +// state. +type frontendFD struct { + vfsfd vfs.FileDescription + vfs.FileDescriptionDefaultImpl + vfs.DentryMetadataFileDescriptionImpl + vfs.NoLockFD + + nvp *nvproxy + hostFD int32 + isControl bool + memmapFile frontendFDMemmapFile + + queue waiter.Queue + + haveMmapContext atomic.Bool +} + +// Release implements vfs.FileDescriptionImpl.Release. +func (fd *frontendFD) Release(context.Context) { + fdnotifier.RemoveFD(fd.hostFD) + fd.queue.Notify(waiter.EventHUp) + unix.Close(int(fd.hostFD)) +} + +// EventRegister implements waiter.Waitable.EventRegister. +func (fd *frontendFD) EventRegister(e *waiter.Entry) error { + fd.queue.EventRegister(e) + if err := fdnotifier.UpdateFD(fd.hostFD); err != nil { + fd.queue.EventUnregister(e) + return err + } + return nil +} + +// EventUnregister implements waiter.Waitable.EventUnregister. +func (fd *frontendFD) EventUnregister(e *waiter.Entry) { + fd.queue.EventUnregister(e) + if err := fdnotifier.UpdateFD(fd.hostFD); err != nil { + panic(fmt.Sprint("UpdateFD:", err)) + } +} + +// Readiness implements waiter.Waitable.Readiness. +func (fd *frontendFD) Readiness(mask waiter.EventMask) waiter.EventMask { + return fdnotifier.NonBlockingPoll(fd.hostFD, mask) +} + +// Epollable implements vfs.FileDescriptionImpl.Epollable. +func (fd *frontendFD) Epollable() bool { + return true +} + +// Ioctl implements vfs.FileDescriptionImpl.Ioctl. +func (fd *frontendFD) Ioctl(ctx context.Context, uio usermem.IO, sysno uintptr, args arch.SyscallArguments) (uintptr, error) { + cmd := args[1].Uint() + nr := linux.IOC_NR(cmd) + argPtr := args[2].Pointer() + argSize := linux.IOC_SIZE(cmd) + + t := kernel.TaskFromContext(ctx) + if t == nil { + panic("Ioctl should be called from a task context") + } + + fi := frontendIoctlState{ + fd: fd, + ctx: ctx, + t: t, + nr: nr, + ioctlParamsAddr: argPtr, + ioctlParamsSize: argSize, + } + + // nr determines the argument type. + // Don't log nr since it's already visible as the last byte of cmd in + // strace logging. + // Implementors: + // - To map nr to a symbol, look in + // src/nvidia/arch/nvalloc/unix/include/nv_escape.h, + // kernel-open/common/inc/nv-ioctl-numbers.h, and + // kernel-open/common/inc/nv-ioctl-numa.h. + // - To determine the parameter type, find the implementation in + // kernel-open/nvidia/nv.c:nvidia_ioctl() or + // src/nvidia/arch/nvalloc/unix/src/escape.c:RmIoctl(). + // - Add symbol and parameter type definitions to //pkg/abi/nvgpu. + // - Add filter to seccomp_filters.go. + // - Add handling below. + switch nr { + case + nvgpu.NV_ESC_CARD_INFO, // nv_ioctl_card_info_t + nvgpu.NV_ESC_CHECK_VERSION_STR, // nv_rm_api_version_t + nvgpu.NV_ESC_SYS_PARAMS, // nv_ioctl_sys_params_t + nvgpu.NV_ESC_RM_DUP_OBJECT, // NVOS55_PARAMETERS + nvgpu.NV_ESC_RM_SHARE, // NVOS57_PARAMETERS + nvgpu.NV_ESC_RM_UNMAP_MEMORY, // NVOS34_PARAMETERS + nvgpu.NV_ESC_RM_UPDATE_DEVICE_MAPPING_INFO: // NVOS56_PARAMETERS + return frontendIoctlSimple(&fi) + case nvgpu.NV_ESC_REGISTER_FD: + return frontendRegisterFD(&fi) + case nvgpu.NV_ESC_ALLOC_OS_EVENT: + return rmAllocOSEvent(&fi) + case nvgpu.NV_ESC_FREE_OS_EVENT: + return rmFreeOSEvent(&fi) + case nvgpu.NV_ESC_NUMA_INFO: + // The CPU topology seen by the host driver differs from the CPU + // topology presented by the sentry to the application, so reject this + // ioctl; doing so is non-fatal. + ctx.Debugf("nvproxy: ignoring NV_ESC_NUMA_INFO") + return 0, linuxerr.EINVAL + case nvgpu.NV_ESC_RM_ALLOC_MEMORY: + return rmAllocMemory(&fi) + case nvgpu.NV_ESC_RM_FREE: + return rmFree(&fi) + case nvgpu.NV_ESC_RM_CONTROL: + return rmControl(&fi) + case nvgpu.NV_ESC_RM_ALLOC: + return rmAlloc(&fi) + case nvgpu.NV_ESC_RM_VID_HEAP_CONTROL: + return rmVidHeapControl(&fi) + case nvgpu.NV_ESC_RM_MAP_MEMORY: + return rmMapMemory(&fi) + default: + ctx.Warningf("nvproxy: unknown frontend ioctl %d == %#x (argSize=%d, cmd=%#x)", nr, nr, argSize, cmd) + return 0, linuxerr.EINVAL + } +} + +func frontendIoctlCmd(nr, argSize uint32) uintptr { + return uintptr(linux.IOWR(nvgpu.NV_IOCTL_MAGIC, nr, argSize)) +} + +// frontendIoctlState holds the state of a call to frontendFD.Ioctl(). +type frontendIoctlState struct { + fd *frontendFD + ctx context.Context + t *kernel.Task + nr uint32 + ioctlParamsAddr hostarch.Addr + ioctlParamsSize uint32 +} + +// frontendIoctlSimple implements a frontend ioctl whose parameters don't +// contain any pointers requiring translation, file descriptors, or special +// cases or effects, and consequently don't need to be typed by the sentry. +func frontendIoctlSimple(fi *frontendIoctlState) (uintptr, error) { + if fi.ioctlParamsSize == 0 { + return frontendIoctlInvoke[byte](fi, nil) + } + + ioctlParams := make([]byte, fi.ioctlParamsSize) + if _, err := fi.t.CopyInBytes(fi.ioctlParamsAddr, ioctlParams); err != nil { + return 0, err + } + n, err := frontendIoctlInvoke(fi, &ioctlParams[0]) + if err != nil { + return n, err + } + if _, err := fi.t.CopyOutBytes(fi.ioctlParamsAddr, ioctlParams); err != nil { + return n, err + } + return n, nil +} + +func frontendRegisterFD(fi *frontendIoctlState) (uintptr, error) { + var ioctlParams nvgpu.IoctlRegisterFD + if fi.ioctlParamsSize != nvgpu.SizeofIoctlRegisterFD { + return 0, linuxerr.EINVAL + } + if _, err := ioctlParams.CopyIn(fi.t, fi.ioctlParamsAddr); err != nil { + return 0, err + } + ctlFileGeneric, _ := fi.t.FDTable().Get(ioctlParams.CtlFD) + if ctlFileGeneric == nil { + return 0, linuxerr.EINVAL + } + defer ctlFileGeneric.DecRef(fi.ctx) + ctlFile, ok := ctlFileGeneric.Impl().(*frontendFD) + if !ok { + return 0, linuxerr.EINVAL + } + sentryIoctlParams := nvgpu.IoctlRegisterFD{ + CtlFD: ctlFile.hostFD, + } + // The returned ctl_fd can't change, so skip copying out. + return frontendIoctlInvoke(fi, &sentryIoctlParams) +} + +func rmAllocOSEvent(fi *frontendIoctlState) (uintptr, error) { + var ioctlParams nvgpu.IoctlAllocOSEvent + if fi.ioctlParamsSize != nvgpu.SizeofIoctlAllocOSEvent { + return 0, linuxerr.EINVAL + } + if _, err := ioctlParams.CopyIn(fi.t, fi.ioctlParamsAddr); err != nil { + return 0, err + } + eventFileGeneric, _ := fi.t.FDTable().Get(int32(ioctlParams.FD)) + if eventFileGeneric == nil { + return 0, linuxerr.EINVAL + } + defer eventFileGeneric.DecRef(fi.ctx) + eventFile, ok := eventFileGeneric.Impl().(*frontendFD) + if !ok { + return 0, linuxerr.EINVAL + } + sentryIoctlParams := ioctlParams + sentryIoctlParams.FD = uint32(eventFile.hostFD) + + n, err := frontendIoctlInvoke(fi, &sentryIoctlParams) + if err != nil { + return n, err + } + + outIoctlParams := sentryIoctlParams + outIoctlParams.FD = ioctlParams.FD + if _, err := outIoctlParams.CopyOut(fi.t, fi.ioctlParamsAddr); err != nil { + return n, err + } + + return n, nil +} + +func rmFreeOSEvent(fi *frontendIoctlState) (uintptr, error) { + var ioctlParams nvgpu.IoctlFreeOSEvent + if fi.ioctlParamsSize != nvgpu.SizeofIoctlFreeOSEvent { + return 0, linuxerr.EINVAL + } + if _, err := ioctlParams.CopyIn(fi.t, fi.ioctlParamsAddr); err != nil { + return 0, err + } + eventFileGeneric, _ := fi.t.FDTable().Get(int32(ioctlParams.FD)) + if eventFileGeneric == nil { + return 0, linuxerr.EINVAL + } + defer eventFileGeneric.DecRef(fi.ctx) + eventFile, ok := eventFileGeneric.Impl().(*frontendFD) + if !ok { + return 0, linuxerr.EINVAL + } + sentryIoctlParams := ioctlParams + sentryIoctlParams.FD = uint32(eventFile.hostFD) + + n, err := frontendIoctlInvoke(fi, &sentryIoctlParams) + if err != nil { + return n, err + } + + outIoctlParams := sentryIoctlParams + outIoctlParams.FD = ioctlParams.FD + if _, err := outIoctlParams.CopyOut(fi.t, fi.ioctlParamsAddr); err != nil { + return n, err + } + + return n, nil +} + +func rmAllocMemory(fi *frontendIoctlState) (uintptr, error) { + var ioctlParams nvgpu.IoctlNVOS02ParametersWithFD + if fi.ioctlParamsSize != nvgpu.SizeofIoctlNVOS02ParametersWithFD { + return 0, linuxerr.EINVAL + } + if _, err := ioctlParams.CopyIn(fi.t, fi.ioctlParamsAddr); err != nil { + return 0, err + } + + if log.IsLogging(log.Debug) { + fi.ctx.Debugf("nvproxy: NV_ESC_RM_ALLOC_MEMORY class %#08x", ioctlParams.Params.HClass) + } + // See src/nvidia/arch/nvalloc/unix/src/escape.c:RmIoctl() and + // src/nvidia/interface/deprecated/rmapi_deprecated_allocmemory.c:rmAllocMemoryTable + // for implementation. + switch ioctlParams.Params.HClass { + case nvgpu.NV01_MEMORY_SYSTEM_OS_DESCRIPTOR: + return rmAllocOSDescriptor(fi, &ioctlParams) + default: + fi.ctx.Warningf("nvproxy: unknown NV_ESC_RM_ALLOC_MEMORY class %#08x", ioctlParams.Params.HClass) + return 0, linuxerr.EINVAL + } +} + +func rmAllocOSDescriptor(fi *frontendIoctlState, ioctlParams *nvgpu.IoctlNVOS02ParametersWithFD) (uintptr, error) { + // Compare src/nvidia/arch/nvalloc/unix/src/escape.c:RmAllocOsDescriptor() + // => RmCreateOsDescriptor(). + failWithStatus := func(status uint32) error { + outIoctlParams := *ioctlParams + outIoctlParams.Params.Status = status + _, err := outIoctlParams.CopyOut(fi.t, fi.ioctlParamsAddr) + return err + } + appAddr := addrFromP64(ioctlParams.Params.PMemory) + if !appAddr.IsPageAligned() { + return 0, failWithStatus(nvgpu.NV_ERR_NOT_SUPPORTED) + } + arLen := ioctlParams.Params.Limit + 1 + if arLen == 0 { // integer overflow + return 0, failWithStatus(nvgpu.NV_ERR_INVALID_LIMIT) + } + var ok bool + arLen, ok = hostarch.PageRoundUp(arLen) + if !ok { + return 0, failWithStatus(nvgpu.NV_ERR_INVALID_ADDRESS) + } + appAR, ok := appAddr.ToRange(arLen) + if !ok { + return 0, failWithStatus(nvgpu.NV_ERR_INVALID_ADDRESS) + } + + // The host driver will collect pages from our address space starting at + // PMemory, so we must assemble a contiguous mapping equivalent to the + // application's. + at := hostarch.Read + if ((ioctlParams.Params.Flags >> 21) & 0x1) == 0 /* NVOS02_FLAGS_ALLOC_USER_READ_ONLY_NO */ { + at.Write = true + } + // Reserve a range in our address space. + m, _, errno := unix.RawSyscall6(unix.SYS_MMAP, 0 /* addr */, uintptr(arLen), unix.PROT_NONE, unix.MAP_PRIVATE|unix.MAP_ANONYMOUS, ^uintptr(0) /* fd */, 0 /* offset */) + if errno != 0 { + return 0, errno + } + cu := cleanup.Make(func() { + unix.RawSyscall(unix.SYS_MUNMAP, m, uintptr(arLen), 0) + }) + defer cu.Clean() + // Mirror application mappings into the reserved range. + prs, err := fi.t.MemoryManager().Pin(fi.ctx, appAR, at, false /* ignorePermissions */) + cu.Add(func() { + mm.Unpin(prs) + }) + if err != nil { + return 0, err + } + sentryAddr := uintptr(m) + for _, pr := range prs { + ims, err := pr.File.MapInternal(memmap.FileRange{pr.Offset, pr.Offset + uint64(pr.Source.Length())}, at) + if err != nil { + return 0, err + } + for !ims.IsEmpty() { + im := ims.Head() + if _, _, errno := unix.RawSyscall6(unix.SYS_MREMAP, im.Addr(), 0 /* old_size */, uintptr(im.Len()), linux.MREMAP_MAYMOVE|linux.MREMAP_FIXED, sentryAddr, 0); errno != 0 { + return 0, errno + } + sentryAddr += uintptr(im.Len()) + ims = ims.Tail() + } + } + sentryIoctlParams := *ioctlParams + sentryIoctlParams.Params.PMemory = nvgpu.P64(uint64(m)) + // NV01_MEMORY_SYSTEM_OS_DESCRIPTOR shouldn't use ioctlParams.FD; clobber + // it to be sure. + sentryIoctlParams.FD = -1 + + fi.fd.nvp.objsMu.Lock() + n, err := frontendIoctlInvoke(fi, &sentryIoctlParams) + if err != nil { + fi.fd.nvp.objsMu.Unlock() + return n, err + } + // Transfer ownership of pinned pages to an osDescMem object, to be + // unpinned when the driver OsDescMem is freed. + o := &osDescMem{ + pinnedRanges: prs, + } + o.object.init(o) + fi.fd.nvp.objsLive[sentryIoctlParams.Params.HObjectNew] = &o.object + fi.fd.nvp.objsMu.Unlock() + cu.Release() + fi.ctx.Infof("nvproxy: pinned pages for OS descriptor with handle %#x", sentryIoctlParams.Params.HObjectNew) + // Unmap the reserved range, which is no longer required. + unix.RawSyscall(unix.SYS_MUNMAP, m, uintptr(arLen), 0) + + outIoctlParams := sentryIoctlParams + outIoctlParams.Params.PMemory = ioctlParams.Params.PMemory + outIoctlParams.FD = ioctlParams.FD + if _, err := outIoctlParams.CopyOut(fi.t, fi.ioctlParamsAddr); err != nil { + return n, err + } + + return n, nil +} + +func rmFree(fi *frontendIoctlState) (uintptr, error) { + var ioctlParams nvgpu.NVOS00Parameters + if fi.ioctlParamsSize != nvgpu.SizeofNVOS00Parameters { + return 0, linuxerr.EINVAL + } + if _, err := ioctlParams.CopyIn(fi.t, fi.ioctlParamsAddr); err != nil { + return 0, err + } + + fi.fd.nvp.objsMu.Lock() + n, err := frontendIoctlInvoke(fi, &ioctlParams) + if err != nil { + fi.fd.nvp.objsMu.Unlock() + return n, err + } + o, ok := fi.fd.nvp.objsLive[ioctlParams.HObjectOld] + if ok { + delete(fi.fd.nvp.objsLive, ioctlParams.HObjectOld) + } + fi.fd.nvp.objsMu.Unlock() + if ok { + o.Release(fi.ctx) + } + + if _, err := ioctlParams.CopyOut(fi.t, fi.ioctlParamsAddr); err != nil { + return n, err + } + return n, nil +} + +func rmControl(fi *frontendIoctlState) (uintptr, error) { + var ioctlParams nvgpu.NVOS54Parameters + if fi.ioctlParamsSize != nvgpu.SizeofNVOS54Parameters { + return 0, linuxerr.EINVAL + } + if _, err := ioctlParams.CopyIn(fi.t, fi.ioctlParamsAddr); err != nil { + return 0, err + } + + // Cmd determines the type of Params. + if log.IsLogging(log.Debug) { + fi.ctx.Debugf("nvproxy: control command %#x", ioctlParams.Cmd) + } + if ioctlParams.Cmd&nvgpu.RM_GSS_LEGACY_MASK != 0 { + // This is a "legacy GSS control" that is implemented by the GPU System + // Processor (GSP). Conseqeuently, its parameters cannot reasonably + // contain application pointers, and the control is in any case + // undocumented. + // See + // src/nvidia/src/kernel/rmapi/entry_points.c:_nv04ControlWithSecInfo() + // => + // src/nvidia/interface/deprecated/rmapi_deprecated_control.c:RmDeprecatedGetControlHandler() + // => + // src/nvidia/interface/deprecated/rmapi_gss_legacy_control.c:RmGssLegacyRpcCmd(). + return rmControlSimple(fi, &ioctlParams) + } + // Implementors: + // - Top two bytes of Cmd specifies class; third byte specifies category; + // fourth byte specifies "message ID" (command within class/category). + // e.g. 0x800288: + // - Class 0x0080 => look in + // src/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080base.h for categories. + // - Category 0x02 => NV0080_CTRL_GPU => look in + // src/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080gpu.h for + // `#define NV0080_CTRL_CMD_GPU_QUERY_SW_STATE_PERSISTENCE (0x800288)` + // and accompanying documentation, parameter type. + // - If this fails, or to find implementation, grep for `methodId=.*0x` to find entry in g_*_nvoc.c; + // implementing function is is "pFunc". + // - Add symbol definition to //pkg/abi/nvgpu. Parameter type definition is + // only required for non-simple commands. + // - Add handling below. + switch ioctlParams.Cmd { + case + nvgpu.NV0000_CTRL_CMD_CLIENT_GET_ADDR_SPACE_TYPE, + nvgpu.NV0000_CTRL_CMD_CLIENT_SET_INHERITED_SHARE_POLICY, + nvgpu.NV0000_CTRL_CMD_GPU_GET_ATTACHED_IDS, + nvgpu.NV0000_CTRL_CMD_GPU_GET_ID_INFO, + nvgpu.NV0000_CTRL_CMD_GPU_GET_ID_INFO_V2, + nvgpu.NV0000_CTRL_CMD_GPU_GET_PROBED_IDS, + nvgpu.NV0000_CTRL_CMD_GPU_ATTACH_IDS, + nvgpu.NV0000_CTRL_CMD_GPU_DETACH_IDS, + nvgpu.NV0000_CTRL_CMD_GPU_GET_PCI_INFO, + nvgpu.NV0000_CTRL_CMD_GPU_QUERY_DRAIN_STATE, + nvgpu.NV0000_CTRL_CMD_GPU_GET_MEMOP_ENABLE, + nvgpu.NV0000_CTRL_CMD_SYNC_GPU_BOOST_GROUP_INFO, + nvgpu.NV0000_CTRL_CMD_SYSTEM_GET_FABRIC_STATUS, + nvgpu.NV0080_CTRL_CMD_FB_GET_CAPS_V2, + nvgpu.NV0080_CTRL_CMD_GPU_GET_NUM_SUBDEVICES, + nvgpu.NV0080_CTRL_CMD_GPU_QUERY_SW_STATE_PERSISTENCE, + nvgpu.NV0080_CTRL_CMD_GPU_GET_VIRTUALIZATION_MODE, + 0x80028b, // unknown, paramsSize == 1 + nvgpu.NV0080_CTRL_CMD_GPU_GET_CLASSLIST_V2, + nvgpu.NV0080_CTRL_CMD_HOST_GET_CAPS_V2, + nvgpu.NV2080_CTRL_CMD_BUS_GET_PCI_INFO, + nvgpu.NV2080_CTRL_CMD_BUS_GET_PCI_BAR_INFO, + nvgpu.NV2080_CTRL_CMD_BUS_GET_INFO_V2, + nvgpu.NV2080_CTRL_CMD_BUS_GET_PCIE_SUPPORTED_GPU_ATOMICS, + nvgpu.NV2080_CTRL_CMD_CE_GET_ALL_CAPS, + nvgpu.NV2080_CTRL_CMD_FB_GET_INFO_V2, + nvgpu.NV2080_CTRL_CMD_GPU_GET_INFO_V2, + nvgpu.NV2080_CTRL_CMD_GPU_GET_NAME_STRING, + nvgpu.NV2080_CTRL_CMD_GPU_GET_SHORT_NAME_STRING, + nvgpu.NV2080_CTRL_CMD_GPU_GET_SIMULATION_INFO, + nvgpu.NV2080_CTRL_CMD_GPU_QUERY_ECC_STATUS, + nvgpu.NV2080_CTRL_CMD_GPU_QUERY_COMPUTE_MODE_RULES, + nvgpu.NV2080_CTRL_CMD_GPU_ACQUIRE_COMPUTE_MODE_RESERVATION, + nvgpu.NV2080_CTRL_CMD_GPU_RELEASE_COMPUTE_MODE_RESERVATION, + nvgpu.NV2080_CTRL_CMD_GPU_GET_GID_INFO, + nvgpu.NV2080_CTRL_CMD_GPU_GET_ENGINES_V2, + nvgpu.NV2080_CTRL_CMD_GPU_GET_ACTIVE_PARTITION_IDS, + nvgpu.NV2080_CTRL_CMD_GPU_GET_COMPUTE_POLICY_CONFIG, + nvgpu.NV2080_CTRL_CMD_GR_SET_CTXSW_PREEMPTION_MODE, + nvgpu.NV2080_CTRL_CMD_GR_GET_CTX_BUFFER_SIZE, + nvgpu.NV2080_CTRL_CMD_GR_GET_GLOBAL_SM_ORDER, + nvgpu.NV2080_CTRL_CMD_GR_GET_CAPS_V2, + nvgpu.NV2080_CTRL_CMD_GR_GET_GPC_MASK, + nvgpu.NV2080_CTRL_CMD_GR_GET_TPC_MASK, + nvgpu.NV2080_CTRL_CMD_GSP_GET_FEATURES, + nvgpu.NV2080_CTRL_CMD_MC_GET_ARCH_INFO, + nvgpu.NV2080_CTRL_CMD_NVLINK_GET_NVLINK_STATUS, + nvgpu.NV2080_CTRL_CMD_PERF_BOOST, + nvgpu.NV2080_CTRL_CMD_RC_GET_WATCHDOG_INFO, + nvgpu.NV2080_CTRL_CMD_RC_RELEASE_WATCHDOG_REQUESTS, + nvgpu.NV2080_CTRL_CMD_RC_SOFT_DISABLE_WATCHDOG, + nvgpu.NV2080_CTRL_CMD_TIMER_GET_GPU_CPU_TIME_CORRELATION_INFO, + nvgpu.NV503C_CTRL_CMD_REGISTER_VA_SPACE, + nvgpu.NV503C_CTRL_CMD_REGISTER_VIDMEM, + nvgpu.NV503C_CTRL_CMD_UNREGISTER_VIDMEM, + nvgpu.NV83DE_CTRL_CMD_DEBUG_SET_EXCEPTION_MASK, + nvgpu.NV83DE_CTRL_CMD_DEBUG_READ_ALL_SM_ERROR_STATES, + nvgpu.NV83DE_CTRL_CMD_DEBUG_CLEAR_ALL_SM_ERROR_STATES, + nvgpu.NV906F_CTRL_CMD_RESET_CHANNEL, + nvgpu.NV90E6_CTRL_CMD_MASTER_GET_VIRTUAL_FUNCTION_ERROR_CONT_INTR_MASK, + nvgpu.NVC36F_CTRL_GET_CLASS_ENGINEID, + nvgpu.NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN, + nvgpu.NVA06C_CTRL_CMD_GPFIFO_SCHEDULE, + nvgpu.NVA06C_CTRL_CMD_SET_TIMESLICE, + nvgpu.NVA06C_CTRL_CMD_PREEMPT: + return rmControlSimple(fi, &ioctlParams) + + case nvgpu.NV0000_CTRL_CMD_SYSTEM_GET_BUILD_VERSION: + return ctrlClientSystemGetBuildVersion(fi, &ioctlParams) + + case nvgpu.NV0080_CTRL_CMD_FIFO_GET_CHANNELLIST: + return ctrlDevFIFOGetChannelList(fi, &ioctlParams) + + case nvgpu.NV2080_CTRL_CMD_FIFO_DISABLE_CHANNELS: + return ctrlSubdevFIFODisableChannels(fi, &ioctlParams) + + case nvgpu.NV2080_CTRL_CMD_GR_GET_INFO: + return ctrlSubdevGRGetInfo(fi, &ioctlParams) + + default: + fi.ctx.Warningf("nvproxy: unknown control command %#x (paramsSize=%d)", ioctlParams.Cmd, ioctlParams.ParamsSize) + return 0, linuxerr.EINVAL + } +} + +func rmControlSimple(fi *frontendIoctlState, ioctlParams *nvgpu.NVOS54Parameters) (uintptr, error) { + if ioctlParams.ParamsSize == 0 { + if ioctlParams.Params != 0 { + return 0, linuxerr.EINVAL + } + return rmControlInvoke[byte](fi, ioctlParams, nil) + } + if ioctlParams.Params == 0 { + return 0, linuxerr.EINVAL + } + + ctrlParams := make([]byte, ioctlParams.ParamsSize) + if _, err := fi.t.CopyInBytes(addrFromP64(ioctlParams.Params), ctrlParams); err != nil { + return 0, err + } + n, err := rmControlInvoke(fi, ioctlParams, &ctrlParams[0]) + if err != nil { + return n, err + } + if _, err := fi.t.CopyOutBytes(addrFromP64(ioctlParams.Params), ctrlParams); err != nil { + return n, err + } + return n, nil +} + +func ctrlClientSystemGetBuildVersion(fi *frontendIoctlState, ioctlParams *nvgpu.NVOS54Parameters) (uintptr, error) { + var ctrlParams nvgpu.NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS + if ctrlParams.SizeBytes() != int(ioctlParams.ParamsSize) { + return 0, linuxerr.EINVAL + } + if _, err := ctrlParams.CopyIn(fi.t, addrFromP64(ioctlParams.Params)); err != nil { + return 0, err + } + + if ctrlParams.PDriverVersionBuffer == 0 || ctrlParams.PVersionBuffer == 0 || ctrlParams.PTitleBuffer == 0 { + // No strings are written if any are null. See + // src/nvidia/interface/deprecated/rmapi_deprecated_control.c:V2_CONVERTER(_NV0000_CTRL_CMD_SYSTEM_GET_BUILD_VERSION). + return ctrlClientSystemGetBuildVersionInvoke(fi, ioctlParams, &ctrlParams, nil, nil, nil) + } + + // Need to buffer strings for copy-out. + if ctrlParams.SizeOfStrings == 0 { + return 0, linuxerr.EINVAL + } + driverVersionBuf := make([]byte, ctrlParams.SizeOfStrings) + versionBuf := make([]byte, ctrlParams.SizeOfStrings) + titleBuf := make([]byte, ctrlParams.SizeOfStrings) + n, err := ctrlClientSystemGetBuildVersionInvoke(fi, ioctlParams, &ctrlParams, &driverVersionBuf[0], &versionBuf[0], &titleBuf[0]) + if err != nil { + return n, err + } + if _, err := fi.t.CopyOutBytes(addrFromP64(ctrlParams.PDriverVersionBuffer), driverVersionBuf); err != nil { + return n, err + } + if _, err := fi.t.CopyOutBytes(addrFromP64(ctrlParams.PVersionBuffer), versionBuf); err != nil { + return n, err + } + if _, err := fi.t.CopyOutBytes(addrFromP64(ctrlParams.PTitleBuffer), titleBuf); err != nil { + return n, err + } + return n, nil +} + +func ctrlSubdevFIFODisableChannels(fi *frontendIoctlState, ioctlParams *nvgpu.NVOS54Parameters) (uintptr, error) { + var ctrlParams nvgpu.NV2080_CTRL_FIFO_DISABLE_CHANNELS_PARAMS + if ctrlParams.SizeBytes() != int(ioctlParams.ParamsSize) { + return 0, linuxerr.EINVAL + } + if _, err := ctrlParams.CopyIn(fi.t, addrFromP64(ioctlParams.Params)); err != nil { + return 0, err + } + // This pointer must be NULL; see + // src/nvidia/src/kernel/gpu/fifo/kernel_fifo_ctrl.c:subdeviceCtrlCmdFifoDisableChannels_IMPL(). + // Consequently, we don't need to translate it, but we do want to ensure + // that it actually is NULL. + if ctrlParams.PRunlistPreemptEvent != 0 { + return 0, linuxerr.EINVAL + } + n, err := rmControlInvoke(fi, ioctlParams, &ctrlParams) + if err != nil { + return n, err + } + if _, err := ctrlParams.CopyOut(fi.t, addrFromP64(ioctlParams.Params)); err != nil { + return n, err + } + return n, nil +} + +func rmAlloc(fi *frontendIoctlState) (uintptr, error) { + // Copy in parameters and convert to NVOS64Parameters. + var ( + ioctlParams nvgpu.NVOS64Parameters + isNVOS64 bool + ) + switch fi.ioctlParamsSize { + case nvgpu.SizeofNVOS21Parameters: + var buf nvgpu.NVOS21Parameters + if _, err := buf.CopyIn(fi.t, fi.ioctlParamsAddr); err != nil { + return 0, err + } + ioctlParams = nvgpu.NVOS64Parameters{ + HRoot: buf.HRoot, + HObjectParent: buf.HObjectParent, + HObjectNew: buf.HObjectNew, + HClass: buf.HClass, + PAllocParms: buf.PAllocParms, + Status: buf.Status, + } + case nvgpu.SizeofNVOS64Parameters: + if _, err := ioctlParams.CopyIn(fi.t, fi.ioctlParamsAddr); err != nil { + return 0, err + } + isNVOS64 = true + default: + return 0, linuxerr.EINVAL + } + + // hClass determines the type of pAllocParms. + if log.IsLogging(log.Debug) { + fi.ctx.Debugf("nvproxy: allocation class %#08x", ioctlParams.HClass) + } + // Implementors: + // - To map hClass to a symbol, look in + // src/nvidia/generated/g_allclasses.h. + // - See src/nvidia/src/kernel/rmapi/resource_list.h for table mapping class + // ("External Class") to the type of pAllocParms ("Alloc Param Info") and + // the class whose constructor interprets it ("Internal Class"). + // - Add symbol and parameter type definitions to //pkg/abi/nvgpu. + // - Add handling below. + switch ioctlParams.HClass { + case nvgpu.NV01_ROOT, nvgpu.NV01_ROOT_NON_PRIV, nvgpu.NV01_ROOT_CLIENT: + return rmAllocSimple[nvgpu.Handle](fi, &ioctlParams, isNVOS64) + case nvgpu.NV01_EVENT_OS_EVENT: + return rmAllocEventOSEvent(fi, &ioctlParams, isNVOS64) + case nvgpu.NV01_DEVICE_0: + return rmAllocSimple[nvgpu.NV0080_ALLOC_PARAMETERS](fi, &ioctlParams, isNVOS64) + case nvgpu.NV20_SUBDEVICE_0: + return rmAllocSimple[nvgpu.NV2080_ALLOC_PARAMETERS](fi, &ioctlParams, isNVOS64) + case nvgpu.NV50_THIRD_PARTY_P2P: + return rmAllocSimple[nvgpu.NV503C_ALLOC_PARAMETERS](fi, &ioctlParams, isNVOS64) + case nvgpu.GT200_DEBUGGER: + return rmAllocSimple[nvgpu.NV83DE_ALLOC_PARAMETERS](fi, &ioctlParams, isNVOS64) + case nvgpu.FERMI_CONTEXT_SHARE_A: + return rmAllocSimple[nvgpu.NV_CTXSHARE_ALLOCATION_PARAMETERS](fi, &ioctlParams, isNVOS64) + case nvgpu.FERMI_VASPACE_A: + return rmAllocSimple[nvgpu.NV_VASPACE_ALLOCATION_PARAMETERS](fi, &ioctlParams, isNVOS64) + case nvgpu.KEPLER_CHANNEL_GROUP_A: + return rmAllocSimple[nvgpu.NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS](fi, &ioctlParams, isNVOS64) + case nvgpu.TURING_CHANNEL_GPFIFO_A, nvgpu.AMPERE_CHANNEL_GPFIFO_A: + return rmAllocSimple[nvgpu.NV_CHANNEL_ALLOC_PARAMS](fi, &ioctlParams, isNVOS64) + case nvgpu.TURING_DMA_COPY_A, nvgpu.AMPERE_DMA_COPY_A, nvgpu.AMPERE_DMA_COPY_B: + return rmAllocSimple[nvgpu.NVB0B5_ALLOCATION_PARAMETERS](fi, &ioctlParams, isNVOS64) + case nvgpu.TURING_COMPUTE_A, nvgpu.AMPERE_COMPUTE_A, nvgpu.ADA_COMPUTE_A: + return rmAllocSimple[nvgpu.NV_GR_ALLOCATION_PARAMETERS](fi, &ioctlParams, isNVOS64) + case + nvgpu.GF100_SUBDEVICE_MASTER, + nvgpu.TURING_USERMODE_A: + return rmAllocNoParams(fi, &ioctlParams, isNVOS64) + default: + fi.ctx.Warningf("nvproxy: unknown allocation class %#08x", ioctlParams.HClass) + return 0, linuxerr.EINVAL + } +} + +// Unlike frontendIoctlSimple and rmControlSimple, rmAllocSimple requires the +// parameter type since the parameter's size is otherwise unknown. +func rmAllocSimple[Params any, PParams marshalPtr[Params]](fi *frontendIoctlState, ioctlParams *nvgpu.NVOS64Parameters, isNVOS64 bool) (uintptr, error) { + if ioctlParams.PAllocParms == 0 { + return rmAllocInvoke[byte](fi, ioctlParams, nil, isNVOS64) + } + + var allocParams Params + if _, err := (PParams)(&allocParams).CopyIn(fi.t, addrFromP64(ioctlParams.PAllocParms)); err != nil { + return 0, err + } + n, err := rmAllocInvoke(fi, ioctlParams, &allocParams, isNVOS64) + if err != nil { + return n, err + } + if _, err := (PParams)(&allocParams).CopyOut(fi.t, addrFromP64(ioctlParams.PAllocParms)); err != nil { + return n, err + } + return n, nil +} + +func rmAllocNoParams(fi *frontendIoctlState, ioctlParams *nvgpu.NVOS64Parameters, isNVOS64 bool) (uintptr, error) { + return rmAllocInvoke[byte](fi, ioctlParams, nil, isNVOS64) +} + +func rmAllocEventOSEvent(fi *frontendIoctlState, ioctlParams *nvgpu.NVOS64Parameters, isNVOS64 bool) (uintptr, error) { + var allocParams nvgpu.NV0005_ALLOC_PARAMETERS + if _, err := allocParams.CopyIn(fi.t, addrFromP64(ioctlParams.PAllocParms)); err != nil { + return 0, err + } + eventFileGeneric, _ := fi.t.FDTable().Get(int32(allocParams.Data)) + if eventFileGeneric == nil { + return 0, linuxerr.EINVAL + } + defer eventFileGeneric.DecRef(fi.ctx) + eventFile, ok := eventFileGeneric.Impl().(*frontendFD) + if !ok { + return 0, linuxerr.EINVAL + } + sentryAllocParams := allocParams + sentryAllocParams.Data = nvgpu.P64(uint64(eventFile.hostFD)) + + n, err := rmAllocInvoke(fi, ioctlParams, &sentryAllocParams, isNVOS64) + if err != nil { + return n, err + } + + outAllocParams := sentryAllocParams + outAllocParams.Data = allocParams.Data + if _, err := outAllocParams.CopyOut(fi.t, addrFromP64(ioctlParams.PAllocParms)); err != nil { + return n, err + } + return n, nil +} + +func rmVidHeapControl(fi *frontendIoctlState) (uintptr, error) { + var ioctlParams nvgpu.NVOS32Parameters + if fi.ioctlParamsSize != nvgpu.SizeofNVOS32Parameters { + return 0, linuxerr.EINVAL + } + if _, err := ioctlParams.CopyIn(fi.t, fi.ioctlParamsAddr); err != nil { + return 0, err + } + + // Function determines the type of Data. + if log.IsLogging(log.Debug) { + fi.ctx.Debugf("nvproxy: VID_HEAP_CONTROL function %d", ioctlParams.Function) + } + // See + // src/nvidia/interface/deprecated/rmapi_deprecated_vidheapctrl.c:rmVidHeapControlTable + // for implementation. + switch ioctlParams.Function { + case nvgpu.NVOS32_FUNCTION_ALLOC_SIZE: + return rmVidHeapControlAllocSize(fi, &ioctlParams) + default: + fi.ctx.Warningf("nvproxy: unknown VID_HEAP_CONTROL function %d", ioctlParams.Function) + return 0, linuxerr.EINVAL + } +} + +func rmMapMemory(fi *frontendIoctlState) (uintptr, error) { + var ioctlParams nvgpu.IoctlNVOS33ParametersWithFD + if fi.ioctlParamsSize != nvgpu.SizeofIoctlNVOS33ParametersWithFD { + return 0, linuxerr.EINVAL + } + if _, err := ioctlParams.CopyIn(fi.t, fi.ioctlParamsAddr); err != nil { + return 0, err + } + mapFileGeneric, _ := fi.t.FDTable().Get(ioctlParams.FD) + if mapFileGeneric == nil { + return 0, linuxerr.EINVAL + } + defer mapFileGeneric.DecRef(fi.ctx) + mapFile, ok := mapFileGeneric.Impl().(*frontendFD) + if !ok { + return 0, linuxerr.EINVAL + } + if mapFile.haveMmapContext.Load() || !mapFile.haveMmapContext.CompareAndSwap(false, true) { + fi.ctx.Warningf("nvproxy: attempted to reuse FD %d for NV_ESC_RM_MAP_MEMORY", ioctlParams.FD) + return 0, linuxerr.EINVAL + } + sentryIoctlParams := ioctlParams + sentryIoctlParams.FD = mapFile.hostFD + + n, err := frontendIoctlInvoke(fi, &sentryIoctlParams) + if err != nil { + return n, err + } + + outIoctlParams := sentryIoctlParams + outIoctlParams.FD = ioctlParams.FD + if _, err := outIoctlParams.CopyOut(fi.t, fi.ioctlParamsAddr); err != nil { + return n, err + } + + return n, nil +} diff --git a/pkg/sentry/devices/nvproxy/frontend_mmap.go b/pkg/sentry/devices/nvproxy/frontend_mmap.go new file mode 100644 index 000000000..b57f13924 --- /dev/null +++ b/pkg/sentry/devices/nvproxy/frontend_mmap.go @@ -0,0 +1,85 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvproxy + +import ( + "gvisor.dev/gvisor/pkg/context" + "gvisor.dev/gvisor/pkg/errors/linuxerr" + "gvisor.dev/gvisor/pkg/hostarch" + "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/safemem" + "gvisor.dev/gvisor/pkg/sentry/memmap" + "gvisor.dev/gvisor/pkg/sentry/vfs" +) + +// ConfigureMMap implements vfs.FileDescriptionImpl.ConfigureMMap. +func (fd *frontendFD) ConfigureMMap(ctx context.Context, opts *memmap.MMapOpts) error { + return vfs.GenericConfigureMMap(&fd.vfsfd, fd, opts) +} + +// AddMapping implements memmap.Mappable.AddMapping. +func (fd *frontendFD) AddMapping(ctx context.Context, ms memmap.MappingSpace, ar hostarch.AddrRange, offset uint64, writable bool) error { + return nil +} + +// RemoveMapping implements memmap.Mappable.RemoveMapping. +func (fd *frontendFD) RemoveMapping(ctx context.Context, ms memmap.MappingSpace, ar hostarch.AddrRange, offset uint64, writable bool) { +} + +// CopyMapping implements memmap.Mappable.CopyMapping. +func (fd *frontendFD) CopyMapping(ctx context.Context, ms memmap.MappingSpace, srcAR, dstAR hostarch.AddrRange, offset uint64, writable bool) error { + return nil +} + +// Translate implements memmap.Mappable.Translate. +func (fd *frontendFD) Translate(ctx context.Context, required, optional memmap.MappableRange, at hostarch.AccessType) ([]memmap.Translation, error) { + return []memmap.Translation{ + { + Source: optional, + File: &fd.memmapFile, + Offset: optional.Start, + Perms: at, + }, + }, nil +} + +// InvalidateUnsavable implements memmap.Mappable.InvalidateUnsavable. +func (fd *frontendFD) InvalidateUnsavable(ctx context.Context) error { + return nil +} + +type frontendFDMemmapFile struct { + fd *frontendFD +} + +// IncRef implements memmap.File.IncRef. +func (mf *frontendFDMemmapFile) IncRef(fr memmap.FileRange) { +} + +// DecRef implements memmap.File.DecRef. +func (mf *frontendFDMemmapFile) DecRef(fr memmap.FileRange) { +} + +// MapInternal implements memmap.File.MapInternal. +func (mf *frontendFDMemmapFile) MapInternal(fr memmap.FileRange, at hostarch.AccessType) (safemem.BlockSeq, error) { + // FIXME(jamieliu): determine if this is safe + log.Traceback("nvproxy: rejecting frontendFDMemmapFile.MapInternal") + return safemem.BlockSeq{}, linuxerr.EINVAL +} + +// FD implements memmap.File.FD. +func (mf *frontendFDMemmapFile) FD() int { + return int(mf.fd.hostFD) +} diff --git a/pkg/sentry/devices/nvproxy/frontend_unsafe.go b/pkg/sentry/devices/nvproxy/frontend_unsafe.go new file mode 100644 index 000000000..8f0ef51f1 --- /dev/null +++ b/pkg/sentry/devices/nvproxy/frontend_unsafe.go @@ -0,0 +1,241 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvproxy + +import ( + "runtime" + "unsafe" + + "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/abi/nvgpu" + "gvisor.dev/gvisor/pkg/errors/linuxerr" + "gvisor.dev/gvisor/pkg/marshal/primitive" +) + +func frontendIoctlInvoke[Params any](fi *frontendIoctlState, sentryParams *Params) (uintptr, error) { + n, _, errno := unix.RawSyscall(unix.SYS_IOCTL, uintptr(fi.fd.hostFD), frontendIoctlCmd(fi.nr, fi.ioctlParamsSize), uintptr(unsafe.Pointer(sentryParams))) + if errno != 0 { + return n, errno + } + return n, nil +} + +func rmControlInvoke[Params any](fi *frontendIoctlState, ioctlParams *nvgpu.NVOS54Parameters, ctrlParams *Params) (uintptr, error) { + defer runtime.KeepAlive(ctrlParams) // since we convert to non-pointer-typed P64 + sentryIoctlParams := *ioctlParams + sentryIoctlParams.Params = p64FromPtr(unsafe.Pointer(ctrlParams)) + n, err := frontendIoctlInvoke(fi, &sentryIoctlParams) + if err != nil { + return n, err + } + outIoctlParams := sentryIoctlParams + outIoctlParams.Params = ioctlParams.Params + if _, err := outIoctlParams.CopyOut(fi.t, fi.ioctlParamsAddr); err != nil { + return n, err + } + return n, nil +} + +func ctrlClientSystemGetBuildVersionInvoke(fi *frontendIoctlState, ioctlParams *nvgpu.NVOS54Parameters, ctrlParams *nvgpu.NV0000_CTRL_SYSTEM_GET_BUILD_VERSION_PARAMS, driverVersionBuf, versionBuf, titleBuf *byte) (uintptr, error) { + sentryCtrlParams := *ctrlParams + sentryCtrlParams.PDriverVersionBuffer = p64FromPtr(unsafe.Pointer(driverVersionBuf)) + sentryCtrlParams.PVersionBuffer = p64FromPtr(unsafe.Pointer(versionBuf)) + sentryCtrlParams.PTitleBuffer = p64FromPtr(unsafe.Pointer(titleBuf)) + n, err := rmControlInvoke(fi, ioctlParams, &sentryCtrlParams) + if err != nil { + return n, err + } + outCtrlParams := sentryCtrlParams + outCtrlParams.PDriverVersionBuffer = ctrlParams.PDriverVersionBuffer + outCtrlParams.PVersionBuffer = ctrlParams.PVersionBuffer + outCtrlParams.PTitleBuffer = ctrlParams.PTitleBuffer + if _, err := outCtrlParams.CopyOut(fi.t, addrFromP64(ioctlParams.Params)); err != nil { + return n, err + } + return n, nil +} + +func ctrlDevFIFOGetChannelList(fi *frontendIoctlState, ioctlParams *nvgpu.NVOS54Parameters) (uintptr, error) { + var ctrlParams nvgpu.NV0080_CTRL_FIFO_GET_CHANNELLIST_PARAMS + if ctrlParams.SizeBytes() != int(ioctlParams.ParamsSize) { + return 0, linuxerr.EINVAL + } + if _, err := ctrlParams.CopyIn(fi.t, addrFromP64(ioctlParams.Params)); err != nil { + return 0, err + } + if ctrlParams.NumChannels == 0 { + // Compare + // src/nvidia/src/kernel/gpu/fifo/kernel_fifo_ctrl.c:deviceCtrlCmdFifoGetChannelList_IMPL(). + return 0, linuxerr.EINVAL + } + channelHandleList := make([]uint32, ctrlParams.NumChannels) + if _, err := primitive.CopyUint32SliceIn(fi.t, addrFromP64(ctrlParams.PChannelHandleList), channelHandleList); err != nil { + return 0, err + } + channelList := make([]uint32, ctrlParams.NumChannels) + if _, err := primitive.CopyUint32SliceIn(fi.t, addrFromP64(ctrlParams.PChannelList), channelList); err != nil { + return 0, err + } + sentryCtrlParams := ctrlParams + sentryCtrlParams.PChannelHandleList = p64FromPtr(unsafe.Pointer(&channelHandleList[0])) + sentryCtrlParams.PChannelList = p64FromPtr(unsafe.Pointer(&channelList[0])) + + n, err := rmControlInvoke(fi, ioctlParams, &sentryCtrlParams) + if err != nil { + return n, err + } + + if _, err := primitive.CopyUint32SliceOut(fi.t, addrFromP64(ctrlParams.PChannelHandleList), channelHandleList); err != nil { + return 0, err + } + if _, err := primitive.CopyUint32SliceOut(fi.t, addrFromP64(ctrlParams.PChannelList), channelList); err != nil { + return 0, err + } + outCtrlParams := sentryCtrlParams + outCtrlParams.PChannelHandleList = ctrlParams.PChannelHandleList + outCtrlParams.PChannelList = ctrlParams.PChannelList + if _, err := outCtrlParams.CopyOut(fi.t, addrFromP64(ioctlParams.Params)); err != nil { + return n, err + } + + return n, nil +} + +func ctrlSubdevGRGetInfo(fi *frontendIoctlState, ioctlParams *nvgpu.NVOS54Parameters) (uintptr, error) { + var ctrlParams nvgpu.NV2080_CTRL_GR_GET_INFO_PARAMS + if ctrlParams.SizeBytes() != int(ioctlParams.ParamsSize) { + return 0, linuxerr.EINVAL + } + if _, err := ctrlParams.CopyIn(fi.t, addrFromP64(ioctlParams.Params)); err != nil { + return 0, err + } + if ctrlParams.GRInfoListSize == 0 { + // Compare + // src/nvidia/src/kernel/gpu/gr/kernel_graphics.c:_kgraphicsCtrlCmdGrGetInfoV2(). + return 0, linuxerr.EINVAL + } + infoList := make([]byte, int(ctrlParams.GRInfoListSize)*(*nvgpu.NVXXXX_CTRL_XXX_INFO)(nil).SizeBytes()) + if _, err := fi.t.CopyInBytes(addrFromP64(ctrlParams.GRInfoList), infoList); err != nil { + return 0, err + } + sentryCtrlParams := ctrlParams + sentryCtrlParams.GRInfoList = p64FromPtr(unsafe.Pointer(&infoList[0])) + + n, err := rmControlInvoke(fi, ioctlParams, &sentryCtrlParams) + if err != nil { + return n, err + } + + if _, err := fi.t.CopyOutBytes(addrFromP64(ctrlParams.GRInfoList), infoList); err != nil { + return n, err + } + outCtrlParams := sentryCtrlParams + outCtrlParams.GRInfoList = ctrlParams.GRInfoList + if _, err := outCtrlParams.CopyOut(fi.t, addrFromP64(ioctlParams.Params)); err != nil { + return n, err + } + + return n, nil +} + +func rmAllocInvoke[Params any](fi *frontendIoctlState, ioctlParams *nvgpu.NVOS64Parameters, allocParams *Params, isNVOS64 bool) (uintptr, error) { + defer runtime.KeepAlive(allocParams) // since we convert to non-pointer-typed P64 + + if isNVOS64 { + sentryIoctlParams := *ioctlParams + sentryIoctlParams.PAllocParms = p64FromPtr(unsafe.Pointer(allocParams)) + var rightsRequested nvgpu.RS_ACCESS_MASK + if ioctlParams.PRightsRequested != 0 { + if _, err := rightsRequested.CopyIn(fi.t, addrFromP64(ioctlParams.PRightsRequested)); err != nil { + return 0, err + } + sentryIoctlParams.PRightsRequested = p64FromPtr(unsafe.Pointer(&rightsRequested)) + } + n, err := frontendIoctlInvoke(fi, &sentryIoctlParams) + if err != nil { + return n, err + } + if ioctlParams.PRightsRequested != 0 { + if _, err := rightsRequested.CopyOut(fi.t, addrFromP64(ioctlParams.PRightsRequested)); err != nil { + return n, err + } + } + outIoctlParams := sentryIoctlParams + outIoctlParams.PAllocParms = ioctlParams.PAllocParms + outIoctlParams.PRightsRequested = ioctlParams.PRightsRequested + if _, err := outIoctlParams.CopyOut(fi.t, fi.ioctlParamsAddr); err != nil { + return n, err + } + return n, nil + } + + sentryIoctlParams := nvgpu.NVOS21Parameters{ + HRoot: ioctlParams.HRoot, + HObjectParent: ioctlParams.HObjectParent, + HObjectNew: ioctlParams.HObjectNew, + HClass: ioctlParams.HClass, + PAllocParms: p64FromPtr(unsafe.Pointer(allocParams)), + Status: ioctlParams.Status, + } + n, err := frontendIoctlInvoke(fi, &sentryIoctlParams) + if err != nil { + return n, err + } + outIoctlParams := nvgpu.NVOS21Parameters{ + HRoot: sentryIoctlParams.HRoot, + HObjectParent: sentryIoctlParams.HObjectParent, + HObjectNew: sentryIoctlParams.HObjectNew, + HClass: sentryIoctlParams.HClass, + PAllocParms: ioctlParams.PAllocParms, + Status: sentryIoctlParams.Status, + } + if _, err := outIoctlParams.CopyOut(fi.t, fi.ioctlParamsAddr); err != nil { + return n, err + } + return n, nil +} + +func rmVidHeapControlAllocSize(fi *frontendIoctlState, ioctlParams *nvgpu.NVOS32Parameters) (uintptr, error) { + allocSizeParams := (*nvgpu.NVOS32AllocSize)(unsafe.Pointer(&ioctlParams.Data)) + + sentryIoctlParams := *ioctlParams + sentryAllocSizeParams := (*nvgpu.NVOS32AllocSize)(unsafe.Pointer(&sentryIoctlParams.Data)) + var addr uint64 + if allocSizeParams.Address != 0 { + if _, err := primitive.CopyUint64In(fi.t, addrFromP64(allocSizeParams.Address), &addr); err != nil { + return 0, err + } + sentryAllocSizeParams.Address = p64FromPtr(unsafe.Pointer(&addr)) + } + + n, err := frontendIoctlInvoke(fi, &sentryIoctlParams) + if err != nil { + return n, err + } + + outIoctlParams := sentryIoctlParams + outAllocSizeParams := (*nvgpu.NVOS32AllocSize)(unsafe.Pointer(&outIoctlParams.Data)) + if allocSizeParams.Address != 0 { + if _, err := primitive.CopyUint64Out(fi.t, addrFromP64(allocSizeParams.Address), addr); err != nil { + return n, err + } + outAllocSizeParams.Address = allocSizeParams.Address + } + if _, err := outIoctlParams.CopyOut(fi.t, fi.ioctlParamsAddr); err != nil { + return n, err + } + + return n, nil +} diff --git a/pkg/sentry/devices/nvproxy/nvproxy.go b/pkg/sentry/devices/nvproxy/nvproxy.go new file mode 100644 index 000000000..9897931e2 --- /dev/null +++ b/pkg/sentry/devices/nvproxy/nvproxy.go @@ -0,0 +1,136 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package nvproxy implements proxying for the Nvidia GPU Linux kernel driver: +// https://github.com/NVIDIA/open-gpu-kernel-modules +package nvproxy + +import ( + "fmt" + + "gvisor.dev/gvisor/pkg/abi/nvgpu" + "gvisor.dev/gvisor/pkg/context" + "gvisor.dev/gvisor/pkg/hostarch" + "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/marshal" + "gvisor.dev/gvisor/pkg/sentry/fsimpl/devtmpfs" + "gvisor.dev/gvisor/pkg/sentry/mm" + "gvisor.dev/gvisor/pkg/sentry/vfs" +) + +// Register registers all devices implemented by this package in vfsObj. +func Register(vfsObj *vfs.VirtualFilesystem, uvmDevMajor uint32) error { + // The kernel driver's interface is unstable, so only allow versions of the + // driver that are known to be supported. + version, err := hostDriverVersion() + if err != nil { + return fmt.Errorf("failed to get Nvidia driver version: %w", err) + } + switch version { + case "525.60.13": + log.Infof("Nvidia driver version: %s", version) + default: + return fmt.Errorf("unknown Nvidia driver version: %s", version) + } + + nvp := &nvproxy{ + objsLive: make(map[nvgpu.Handle]*object), + } + for minor := uint32(0); minor <= nvgpu.NV_CONTROL_DEVICE_MINOR; minor++ { + if err := vfsObj.RegisterDevice(vfs.CharDevice, nvgpu.NV_MAJOR_DEVICE_NUMBER, minor, &frontendDevice{ + nvp: nvp, + minor: minor, + }, &vfs.RegisterDeviceOptions{ + GroupName: "nvidia-frontend", + }); err != nil { + return err + } + } + if err := vfsObj.RegisterDevice(vfs.CharDevice, uvmDevMajor, nvgpu.NVIDIA_UVM_PRIMARY_MINOR_NUMBER, &uvmDevice{ + nvp: nvp, + }, &vfs.RegisterDeviceOptions{ + GroupName: "nvidia-uvm", + }); err != nil { + return err + } + return nil +} + +// CreateDriverDevtmpfsFiles creates device special files in dev that should +// always exist when this package is enabled. It does not create per-device +// files in dev; see CreateIndexDevtmpfsFile. +func CreateDriverDevtmpfsFiles(ctx context.Context, dev *devtmpfs.Accessor, uvmDevMajor uint32) error { + if err := dev.CreateDeviceFile(ctx, "nvidiactl", vfs.CharDevice, nvgpu.NV_MAJOR_DEVICE_NUMBER, nvgpu.NV_CONTROL_DEVICE_MINOR, 0666); err != nil { + return err + } + if err := dev.CreateDeviceFile(ctx, "nvidia-uvm", vfs.CharDevice, uvmDevMajor, nvgpu.NVIDIA_UVM_PRIMARY_MINOR_NUMBER, 0666); err != nil { + return err + } + return nil +} + +// CreateIndexDevtmpfsFile creates the device special file in dev for the +// device with the given index. +func CreateIndexDevtmpfsFile(ctx context.Context, dev *devtmpfs.Accessor, index uint32) error { + return dev.CreateDeviceFile(ctx, fmt.Sprintf("nvidia%d", index), vfs.CharDevice, nvgpu.NV_MAJOR_DEVICE_NUMBER, index, 0666) +} + +// +stateify savable +type nvproxy struct { + objsMu objsMutex + objsLive map[nvgpu.Handle]*object +} + +// object tracks an object allocated through the driver. +// +// +stateify savable +type object struct { + impl objectImpl +} + +func (o *object) init(impl objectImpl) { + o.impl = impl +} + +// Release is called after the represented object is freed. +func (o *object) Release(ctx context.Context) { + o.impl.Release(ctx) +} + +type objectImpl interface { + Release(ctx context.Context) +} + +// osDescMem is an objectImpl tracking an OS descriptor. +// +// +stateify savable +type osDescMem struct { + object + pinnedRanges []mm.PinnedRange +} + +// Release implements objectImpl.Release. +func (o *osDescMem) Release(ctx context.Context) { + ctx.Infof("nvproxy: unpinning pages for released OS descriptor") + mm.Unpin(o.pinnedRanges) +} + +type marshalPtr[T any] interface { + *T + marshal.Marshallable +} + +func addrFromP64(p nvgpu.P64) hostarch.Addr { + return hostarch.Addr(uintptr(uint64(p))) +} diff --git a/pkg/sentry/devices/nvproxy/nvproxy_unsafe.go b/pkg/sentry/devices/nvproxy/nvproxy_unsafe.go new file mode 100644 index 000000000..18d00ace0 --- /dev/null +++ b/pkg/sentry/devices/nvproxy/nvproxy_unsafe.go @@ -0,0 +1,74 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvproxy + +import ( + "bytes" + "fmt" + "unsafe" + + "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/abi/nvgpu" +) + +func hostDriverVersion() (string, error) { + ctlFD, err := unix.Openat(-1, "/dev/nvidiactl", unix.O_RDONLY|unix.O_NOFOLLOW, 0) + if err != nil { + return "", fmt.Errorf("failed to open /dev/nvidiactl: %w", err) + } + defer unix.Close(ctlFD) + + // From src/nvidia/arch/nvalloc/unix/include/nv-ioctl.h: + const NV_RM_API_VERSION_REPLY_RECOGNIZED = 1 + + // 530.30.02 and later versions of the host driver `#define + // NV_RM_API_VERSION_CMD_QUERY '2'`, which causes this ioctl to return the + // driver version without performing a check. Earlier versions of the + // driver `#define NV_RM_API_VERSION_CMD_OVERRIDE '2'`, which causes the + // ioctl to no-op. Try with Cmd '2' first, hoping that the driver + // interprets it as _QUERY; if the returned string is empty, then it was + // interpreted as _OVERRIDE and we need to perform an actual check (Cmd 0), + // which has the downside of logging an error message. + ioctlParams := nvgpu.RMAPIVersion{ + Cmd: '2', + } + if _, _, errno := unix.RawSyscall(unix.SYS_IOCTL, uintptr(ctlFD), frontendIoctlCmd(nvgpu.NV_ESC_CHECK_VERSION_STR, uint32(unsafe.Sizeof(ioctlParams))), uintptr(unsafe.Pointer(&ioctlParams))); errno != 0 { + return "", fmt.Errorf("NV_ESC_CHECK_VERSION_STR ioctl error: %w", errno) + } + if ioctlParams.Reply != NV_RM_API_VERSION_REPLY_RECOGNIZED { + return "", fmt.Errorf("unknown NV_ESC_CHECK_VERSION_STR reply: %d", ioctlParams.Reply) + } + if ioctlParams.VersionString[0] == '\x00' { + ioctlParams.Cmd = 0 + ioctlParams.Reply = 0 + // We expect the check to fail on our empty version string, so tolerate + // EINVAL. + if _, _, errno := unix.RawSyscall(unix.SYS_IOCTL, uintptr(ctlFD), frontendIoctlCmd(nvgpu.NV_ESC_CHECK_VERSION_STR, uint32(unsafe.Sizeof(ioctlParams))), uintptr(unsafe.Pointer(&ioctlParams))); errno != 0 && errno != unix.EINVAL { + return "", fmt.Errorf("fallback NV_ESC_CHECK_VERSION_STR ioctl error: %w", errno) + } + if ioctlParams.Reply != NV_RM_API_VERSION_REPLY_RECOGNIZED { + return "", fmt.Errorf("unknown fallback NV_ESC_CHECK_VERSION_STR reply: %d", ioctlParams.Reply) + } + } + + if i := bytes.IndexByte(ioctlParams.VersionString[:], '\x00'); i >= 0 { + return string(ioctlParams.VersionString[:i]), nil + } + return string(ioctlParams.VersionString[:]), nil +} + +func p64FromPtr(ptr unsafe.Pointer) nvgpu.P64 { + return nvgpu.P64(uint64(uintptr(ptr))) +} diff --git a/pkg/sentry/devices/nvproxy/seccomp_filters.go b/pkg/sentry/devices/nvproxy/seccomp_filters.go new file mode 100644 index 000000000..fe440350e --- /dev/null +++ b/pkg/sentry/devices/nvproxy/seccomp_filters.go @@ -0,0 +1,190 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvproxy + +import ( + "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/abi/nvgpu" + "gvisor.dev/gvisor/pkg/seccomp" +) + +// Filters returns seccomp-bpf filters for this package. +func Filters() seccomp.SyscallRules { + nonNegativeFD := seccomp.LessThanOrEqual(0x7fff_ffff /* max int32 */) + notIocSizeMask := ^(((uintptr(1) << linux.IOC_SIZEBITS) - 1) << linux.IOC_SIZESHIFT) // for ioctls taking arbitrary size + return seccomp.SyscallRules{ + unix.SYS_OPENAT: []seccomp.Rule{ + { + // All paths that we openat() are absolute, so we pass a dirfd + // of -1 (which is invalid for relative paths, but ignored for + // absolute paths) to hedge against bugs involving AT_FDCWD or + // real dirfds. + seccomp.EqualTo(^uintptr(0)), + seccomp.MatchAny{}, + seccomp.MaskedEqual(unix.O_NOFOLLOW|unix.O_CREAT, unix.O_NOFOLLOW), + seccomp.MatchAny{}, + }, + }, + unix.SYS_IOCTL: []seccomp.Rule{ + { + nonNegativeFD, + seccomp.MaskedEqual(notIocSizeMask, frontendIoctlCmd(nvgpu.NV_ESC_CARD_INFO, 0)), + }, + { + nonNegativeFD, + seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_CHECK_VERSION_STR, nvgpu.SizeofRMAPIVersion)), + }, + { + nonNegativeFD, + seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_REGISTER_FD, nvgpu.SizeofIoctlRegisterFD)), + }, + { + nonNegativeFD, + seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_ALLOC_OS_EVENT, nvgpu.SizeofIoctlAllocOSEvent)), + }, + { + nonNegativeFD, + seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_FREE_OS_EVENT, nvgpu.SizeofIoctlFreeOSEvent)), + }, + { + nonNegativeFD, + seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_SYS_PARAMS, nvgpu.SizeofIoctlSysParams)), + }, + { + nonNegativeFD, + seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_ALLOC_MEMORY, nvgpu.SizeofIoctlNVOS02ParametersWithFD)), + }, + { + nonNegativeFD, + seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_FREE, nvgpu.SizeofNVOS00Parameters)), + }, + { + nonNegativeFD, + seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_CONTROL, nvgpu.SizeofNVOS54Parameters)), + }, + { + nonNegativeFD, + seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_ALLOC, nvgpu.SizeofNVOS21Parameters)), + }, + { + nonNegativeFD, + seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_ALLOC, nvgpu.SizeofNVOS64Parameters)), + }, + { + nonNegativeFD, + seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_DUP_OBJECT, nvgpu.SizeofNVOS55Parameters)), + }, + { + nonNegativeFD, + seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_SHARE, nvgpu.SizeofNVOS57Parameters)), + }, + { + nonNegativeFD, + seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_VID_HEAP_CONTROL, nvgpu.SizeofNVOS32Parameters)), + }, + { + nonNegativeFD, + seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_MAP_MEMORY, nvgpu.SizeofIoctlNVOS33ParametersWithFD)), + }, + { + nonNegativeFD, + seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_UNMAP_MEMORY, nvgpu.SizeofNVOS34Parameters)), + }, + { + nonNegativeFD, + seccomp.EqualTo(frontendIoctlCmd(nvgpu.NV_ESC_RM_UPDATE_DEVICE_MAPPING_INFO, nvgpu.SizeofNVOS56Parameters)), + }, + { + nonNegativeFD, + seccomp.EqualTo(nvgpu.UVM_INITIALIZE), + }, + { + nonNegativeFD, + seccomp.EqualTo(nvgpu.UVM_DEINITIALIZE), + }, + { + nonNegativeFD, + seccomp.EqualTo(nvgpu.UVM_CREATE_RANGE_GROUP), + }, + { + nonNegativeFD, + seccomp.EqualTo(nvgpu.UVM_DESTROY_RANGE_GROUP), + }, + { + nonNegativeFD, + seccomp.EqualTo(nvgpu.UVM_REGISTER_GPU_VASPACE), + }, + { + nonNegativeFD, + seccomp.EqualTo(nvgpu.UVM_UNREGISTER_GPU_VASPACE), + }, + { + nonNegativeFD, + seccomp.EqualTo(nvgpu.UVM_REGISTER_CHANNEL), + }, + { + nonNegativeFD, + seccomp.EqualTo(nvgpu.UVM_UNREGISTER_CHANNEL), + }, + { + nonNegativeFD, + seccomp.EqualTo(nvgpu.UVM_MAP_EXTERNAL_ALLOCATION), + }, + { + nonNegativeFD, + seccomp.EqualTo(nvgpu.UVM_FREE), + }, + { + nonNegativeFD, + seccomp.EqualTo(nvgpu.UVM_REGISTER_GPU), + }, + { + nonNegativeFD, + seccomp.EqualTo(nvgpu.UVM_UNREGISTER_GPU), + }, + { + nonNegativeFD, + seccomp.EqualTo(nvgpu.UVM_PAGEABLE_MEM_ACCESS), + }, + { + nonNegativeFD, + seccomp.EqualTo(nvgpu.UVM_MAP_DYNAMIC_PARALLELISM_REGION), + }, + { + nonNegativeFD, + seccomp.EqualTo(nvgpu.UVM_ALLOC_SEMAPHORE_POOL), + }, + { + nonNegativeFD, + seccomp.EqualTo(nvgpu.UVM_VALIDATE_VA_RANGE), + }, + { + nonNegativeFD, + seccomp.EqualTo(nvgpu.UVM_CREATE_EXTERNAL_RANGE), + }, + }, + unix.SYS_MREMAP: []seccomp.Rule{ + { + seccomp.MatchAny{}, + seccomp.EqualTo(0), /* old_size */ + seccomp.MatchAny{}, + seccomp.EqualTo(linux.MREMAP_MAYMOVE | linux.MREMAP_FIXED), + seccomp.MatchAny{}, + seccomp.EqualTo(0), + }, + }, + } +} diff --git a/pkg/sentry/devices/nvproxy/uvm.go b/pkg/sentry/devices/nvproxy/uvm.go new file mode 100644 index 000000000..108091e4a --- /dev/null +++ b/pkg/sentry/devices/nvproxy/uvm.go @@ -0,0 +1,271 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvproxy + +import ( + "fmt" + + "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/abi/nvgpu" + "gvisor.dev/gvisor/pkg/context" + "gvisor.dev/gvisor/pkg/errors/linuxerr" + "gvisor.dev/gvisor/pkg/fdnotifier" + "gvisor.dev/gvisor/pkg/hostarch" + "gvisor.dev/gvisor/pkg/marshal" + "gvisor.dev/gvisor/pkg/sentry/arch" + "gvisor.dev/gvisor/pkg/sentry/kernel" + "gvisor.dev/gvisor/pkg/sentry/vfs" + "gvisor.dev/gvisor/pkg/usermem" + "gvisor.dev/gvisor/pkg/waiter" +) + +// uvmDevice implements vfs.Device for /dev/nvidia-uvm. +// +// +stateify savable +type uvmDevice struct { + nvp *nvproxy +} + +// Open implements vfs.Device.Open. +func (dev *uvmDevice) Open(ctx context.Context, mnt *vfs.Mount, vfsd *vfs.Dentry, opts vfs.OpenOptions) (*vfs.FileDescription, error) { + hostFD, err := unix.Openat(-1, "/dev/nvidia-uvm", int((opts.Flags&unix.O_ACCMODE)|unix.O_NOFOLLOW), 0) + if err != nil { + ctx.Warningf("nvproxy: failed to open host /dev/nvidia-uvm: %v", err) + return nil, err + } + fd := &uvmFD{ + nvp: dev.nvp, + hostFD: int32(hostFD), + } + if err := fd.vfsfd.Init(fd, opts.Flags, mnt, vfsd, &vfs.FileDescriptionOptions{ + UseDentryMetadata: true, + }); err != nil { + unix.Close(hostFD) + return nil, err + } + if err := fdnotifier.AddFD(int32(hostFD), &fd.queue); err != nil { + unix.Close(hostFD) + return nil, err + } + fd.memmapFile.fd = fd + return &fd.vfsfd, nil +} + +// uvmFD implements vfs.FileDescriptionImpl for /dev/nvidia-uvm. +// +// uvmFD is not savable; we do not implement save/restore of host GPU state. +type uvmFD struct { + vfsfd vfs.FileDescription + vfs.FileDescriptionDefaultImpl + vfs.DentryMetadataFileDescriptionImpl + vfs.NoLockFD + + nvp *nvproxy + hostFD int32 + memmapFile uvmFDMemmapFile + + queue waiter.Queue +} + +// Release implements vfs.FileDescriptionImpl.Release. +func (fd *uvmFD) Release(context.Context) { + fdnotifier.RemoveFD(fd.hostFD) + fd.queue.Notify(waiter.EventHUp) + unix.Close(int(fd.hostFD)) +} + +// EventRegister implements waiter.Waitable.EventRegister. +func (fd *uvmFD) EventRegister(e *waiter.Entry) error { + fd.queue.EventRegister(e) + if err := fdnotifier.UpdateFD(fd.hostFD); err != nil { + fd.queue.EventUnregister(e) + return err + } + return nil +} + +// EventUnregister implements waiter.Waitable.EventUnregister. +func (fd *uvmFD) EventUnregister(e *waiter.Entry) { + fd.queue.EventUnregister(e) + if err := fdnotifier.UpdateFD(fd.hostFD); err != nil { + panic(fmt.Sprint("UpdateFD:", err)) + } +} + +// Readiness implements waiter.Waitable.Readiness. +func (fd *uvmFD) Readiness(mask waiter.EventMask) waiter.EventMask { + return fdnotifier.NonBlockingPoll(fd.hostFD, mask) +} + +// Epollable implements vfs.FileDescriptionImpl.Epollable. +func (fd *uvmFD) Epollable() bool { + return true +} + +// Ioctl implements vfs.FileDescriptionImpl.Ioctl. +func (fd *uvmFD) Ioctl(ctx context.Context, uio usermem.IO, sysno uintptr, args arch.SyscallArguments) (uintptr, error) { + cmd := args[1].Uint() + argPtr := args[2].Pointer() + + t := kernel.TaskFromContext(ctx) + if t == nil { + panic("Ioctl should be called from a task context") + } + + ui := uvmIoctlState{ + fd: fd, + ctx: ctx, + t: t, + cmd: cmd, + ioctlParamsAddr: argPtr, + } + + switch cmd { + case nvgpu.UVM_INITIALIZE: + return uvmInitialize(&ui) + case nvgpu.UVM_DEINITIALIZE: + return uvmIoctlInvoke[byte](&ui, nil) + case nvgpu.UVM_CREATE_RANGE_GROUP: + return uvmIoctlSimple[nvgpu.UVM_CREATE_RANGE_GROUP_PARAMS](&ui) + case nvgpu.UVM_DESTROY_RANGE_GROUP: + return uvmIoctlSimple[nvgpu.UVM_DESTROY_RANGE_GROUP_PARAMS](&ui) + case nvgpu.UVM_REGISTER_GPU_VASPACE: + return uvmIoctlHasRMCtrlFD[nvgpu.UVM_REGISTER_GPU_VASPACE_PARAMS](&ui) + case nvgpu.UVM_UNREGISTER_GPU_VASPACE: + return uvmIoctlSimple[nvgpu.UVM_UNREGISTER_GPU_VASPACE_PARAMS](&ui) + case nvgpu.UVM_REGISTER_CHANNEL: + return uvmIoctlHasRMCtrlFD[nvgpu.UVM_REGISTER_CHANNEL_PARAMS](&ui) + case nvgpu.UVM_UNREGISTER_CHANNEL: + return uvmIoctlSimple[nvgpu.UVM_UNREGISTER_CHANNEL_PARAMS](&ui) + case nvgpu.UVM_MAP_EXTERNAL_ALLOCATION: + return uvmIoctlHasRMCtrlFD[nvgpu.UVM_MAP_EXTERNAL_ALLOCATION_PARAMS](&ui) + case nvgpu.UVM_FREE: + return uvmIoctlSimple[nvgpu.UVM_FREE_PARAMS](&ui) + case nvgpu.UVM_REGISTER_GPU: + return uvmIoctlHasRMCtrlFD[nvgpu.UVM_REGISTER_GPU_PARAMS](&ui) + case nvgpu.UVM_UNREGISTER_GPU: + return uvmIoctlSimple[nvgpu.UVM_UNREGISTER_GPU_PARAMS](&ui) + case nvgpu.UVM_PAGEABLE_MEM_ACCESS: + return uvmIoctlSimple[nvgpu.UVM_PAGEABLE_MEM_ACCESS_PARAMS](&ui) + case nvgpu.UVM_MAP_DYNAMIC_PARALLELISM_REGION: + return uvmIoctlSimple[nvgpu.UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS](&ui) + case nvgpu.UVM_ALLOC_SEMAPHORE_POOL: + return uvmIoctlSimple[nvgpu.UVM_ALLOC_SEMAPHORE_POOL_PARAMS](&ui) + case nvgpu.UVM_VALIDATE_VA_RANGE: + return uvmIoctlSimple[nvgpu.UVM_VALIDATE_VA_RANGE_PARAMS](&ui) + case nvgpu.UVM_CREATE_EXTERNAL_RANGE: + return uvmIoctlSimple[nvgpu.UVM_CREATE_EXTERNAL_RANGE_PARAMS](&ui) + default: + ctx.Warningf("nvproxy: unknown uvm ioctl %d", cmd) + return 0, linuxerr.EINVAL + } +} + +// uvmIoctlState holds the state of a call to uvmFD.Ioctl(). +type uvmIoctlState struct { + fd *uvmFD + ctx context.Context + t *kernel.Task + cmd uint32 + ioctlParamsAddr hostarch.Addr +} + +func uvmIoctlSimple[Params any, PParams marshalPtr[Params]](ui *uvmIoctlState) (uintptr, error) { + var ioctlParams Params + if _, err := (PParams)(&ioctlParams).CopyIn(ui.t, ui.ioctlParamsAddr); err != nil { + return 0, err + } + n, err := uvmIoctlInvoke(ui, &ioctlParams) + if err != nil { + return n, err + } + if _, err := (PParams)(&ioctlParams).CopyOut(ui.t, ui.ioctlParamsAddr); err != nil { + return n, err + } + return n, nil +} + +func uvmInitialize(ui *uvmIoctlState) (uintptr, error) { + var ioctlParams nvgpu.UVM_INITIALIZE_PARAMS + if _, err := ioctlParams.CopyIn(ui.t, ui.ioctlParamsAddr); err != nil { + return 0, err + } + sentryIoctlParams := ioctlParams + // This is necessary to share the host UVM FD between sentry and + // application processes. + sentryIoctlParams.Flags = ioctlParams.Flags | nvgpu.UVM_INIT_FLAGS_MULTI_PROCESS_SHARING_MODE + n, err := uvmIoctlInvoke(ui, &sentryIoctlParams) + if err != nil { + return n, err + } + outIoctlParams := sentryIoctlParams + // Only expose the MULTI_PROCESS_SHARING_MODE flag if it was present in + // ioctlParams. + outIoctlParams.Flags &^= ^ioctlParams.Flags & nvgpu.UVM_INIT_FLAGS_MULTI_PROCESS_SHARING_MODE + if _, err := outIoctlParams.CopyOut(ui.t, ui.ioctlParamsAddr); err != nil { + return n, err + } + return n, nil +} + +type hasRMCtrlFDPtr[T any] interface { + *T + marshal.Marshallable + nvgpu.HasRMCtrlFD +} + +func uvmIoctlHasRMCtrlFD[Params any, PParams hasRMCtrlFDPtr[Params]](ui *uvmIoctlState) (uintptr, error) { + var ioctlParams Params + if _, err := (PParams)(&ioctlParams).CopyIn(ui.t, ui.ioctlParamsAddr); err != nil { + return 0, err + } + + rmCtrlFD := (PParams)(&ioctlParams).GetRMCtrlFD() + if rmCtrlFD < 0 { + n, err := uvmIoctlInvoke(ui, &ioctlParams) + if err != nil { + return n, err + } + if _, err := (PParams)(&ioctlParams).CopyOut(ui.t, ui.ioctlParamsAddr); err != nil { + return n, err + } + return n, nil + } + + ctlFileGeneric, _ := ui.t.FDTable().Get(rmCtrlFD) + if ctlFileGeneric == nil { + return 0, linuxerr.EINVAL + } + defer ctlFileGeneric.DecRef(ui.ctx) + ctlFile, ok := ctlFileGeneric.Impl().(*frontendFD) + if !ok { + return 0, linuxerr.EINVAL + } + + sentryIoctlParams := ioctlParams + (PParams)(&sentryIoctlParams).SetRMCtrlFD(ctlFile.hostFD) + n, err := uvmIoctlInvoke(ui, &sentryIoctlParams) + if err != nil { + return n, err + } + + outIoctlParams := sentryIoctlParams + (PParams)(&outIoctlParams).SetRMCtrlFD(rmCtrlFD) + if _, err := (PParams)(&outIoctlParams).CopyOut(ui.t, ui.ioctlParamsAddr); err != nil { + return n, err + } + + return n, nil +} diff --git a/pkg/sentry/devices/nvproxy/uvm_mmap.go b/pkg/sentry/devices/nvproxy/uvm_mmap.go new file mode 100644 index 000000000..dad7c9ff9 --- /dev/null +++ b/pkg/sentry/devices/nvproxy/uvm_mmap.go @@ -0,0 +1,87 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvproxy + +import ( + "gvisor.dev/gvisor/pkg/context" + "gvisor.dev/gvisor/pkg/errors/linuxerr" + "gvisor.dev/gvisor/pkg/hostarch" + "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/safemem" + "gvisor.dev/gvisor/pkg/sentry/memmap" + "gvisor.dev/gvisor/pkg/sentry/vfs" +) + +// ConfigureMMap implements vfs.FileDescriptionImpl.ConfigureMMap. +func (fd *uvmFD) ConfigureMMap(ctx context.Context, opts *memmap.MMapOpts) error { + return vfs.GenericConfigureMMap(&fd.vfsfd, fd, opts) +} + +// AddMapping implements memmap.Mappable.AddMapping. +func (fd *uvmFD) AddMapping(ctx context.Context, ms memmap.MappingSpace, ar hostarch.AddrRange, offset uint64, writable bool) error { + return nil +} + +// RemoveMapping implements memmap.Mappable.RemoveMapping. +func (fd *uvmFD) RemoveMapping(ctx context.Context, ms memmap.MappingSpace, ar hostarch.AddrRange, offset uint64, writable bool) { +} + +// CopyMapping implements memmap.Mappable.CopyMapping. +func (fd *uvmFD) CopyMapping(ctx context.Context, ms memmap.MappingSpace, srcAR, dstAR hostarch.AddrRange, offset uint64, writable bool) error { + return nil +} + +// Translate implements memmap.Mappable.Translate. +func (fd *uvmFD) Translate(ctx context.Context, required, optional memmap.MappableRange, at hostarch.AccessType) ([]memmap.Translation, error) { + return []memmap.Translation{ + { + Source: optional, + File: &fd.memmapFile, + Offset: optional.Start, + // kernel-open/nvidia-uvm/uvm.c:uvm_mmap() requires mappings to be + // PROT_READ|PROT_WRITE. + Perms: hostarch.ReadWrite, + }, + }, nil +} + +// InvalidateUnsavable implements memmap.Mappable.InvalidateUnsavable. +func (fd *uvmFD) InvalidateUnsavable(ctx context.Context) error { + return nil +} + +type uvmFDMemmapFile struct { + fd *uvmFD +} + +// IncRef implements memmap.File.IncRef. +func (mf *uvmFDMemmapFile) IncRef(fr memmap.FileRange) { +} + +// DecRef implements memmap.File.DecRef. +func (mf *uvmFDMemmapFile) DecRef(fr memmap.FileRange) { +} + +// MapInternal implements memmap.File.MapInternal. +func (mf *uvmFDMemmapFile) MapInternal(fr memmap.FileRange, at hostarch.AccessType) (safemem.BlockSeq, error) { + // TODO(jamieliu): make an attempt with MAP_FIXED_NOREPLACE? + log.Traceback("nvproxy: rejecting uvmFDMemmapFile.MapInternal") + return safemem.BlockSeq{}, linuxerr.EINVAL +} + +// FD implements memmap.File.FD. +func (mf *uvmFDMemmapFile) FD() int { + return int(mf.fd.hostFD) +} diff --git a/pkg/sentry/devices/nvproxy/uvm_unsafe.go b/pkg/sentry/devices/nvproxy/uvm_unsafe.go new file mode 100644 index 000000000..2acaf1ca8 --- /dev/null +++ b/pkg/sentry/devices/nvproxy/uvm_unsafe.go @@ -0,0 +1,29 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvproxy + +import ( + "unsafe" + + "golang.org/x/sys/unix" +) + +func uvmIoctlInvoke[Params any](ui *uvmIoctlState, ioctlParams *Params) (uintptr, error) { + n, _, errno := unix.RawSyscall(unix.SYS_IOCTL, uintptr(ui.fd.hostFD), uintptr(ui.cmd), uintptr(unsafe.Pointer(ioctlParams))) + if errno != 0 { + return n, errno + } + return n, nil +}