From 41ec0d4189e316e3d17d89a320242399229d02d5 Mon Sep 17 00:00:00 2001
From: Ayush Ranjan <ayushranjan@google.com>
Date: Thu, 20 Jul 2023 22:07:35 -0700
Subject: [PATCH] Add nvproxy support for V100 Nvidia GPUs.

Tested on 1 V100 GPU:
```
$ docker run --runtime=runsc --rm --gpus all nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda11.7.1-ubi8
[Vector addition of 50000 elements]
Copy input data from the host memory to the CUDA device
CUDA kernel launch with 196 blocks of 256 threads
Copy output data from the CUDA device to the host memory
Test PASSED
Done
```

PiperOrigin-RevId: 549837326
---
 pkg/abi/nvgpu/classes.go               | 10 ++++++++++
 pkg/abi/nvgpu/ctrl.go                  |  1 +
 pkg/sentry/devices/nvproxy/frontend.go |  9 +++++----
 pkg/sentry/devices/nvproxy/nvproxy.go  |  4 +++-
 4 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/pkg/abi/nvgpu/classes.go b/pkg/abi/nvgpu/classes.go
index 4dba91c9a..d1bdf09ed 100644
--- a/pkg/abi/nvgpu/classes.go
+++ b/pkg/abi/nvgpu/classes.go
@@ -30,6 +30,8 @@ const (
 	FERMI_CONTEXT_SHARE_A            = 0x00009067
 	FERMI_VASPACE_A                  = 0x000090f1
 	KEPLER_CHANNEL_GROUP_A           = 0x0000a06c
+	VOLTA_USERMODE_A                 = 0x0000c361
+	VOLTA_CHANNEL_GPFIFO_A           = 0x0000c36f
 	TURING_USERMODE_A                = 0x0000c461
 	TURING_CHANNEL_GPFIFO_A          = 0x0000c46f
 	AMPERE_CHANNEL_GPFIFO_A          = 0x0000c56f
@@ -45,6 +47,14 @@ const (
 	HOPPER_COMPUTE_A                 = 0x0000cbc0
 )
 
+// Class handles for older generations that are not supported by the open source
+// driver. Volta was the last such generation. These are defined in files under
+// src/common/sdk/nvidia/inc/class/.
+const (
+	VOLTA_COMPUTE_A  = 0x0000c3c0
+	VOLTA_DMA_COPY_A = 0x0000c3b5
+)
+
 // NV0005_ALLOC_PARAMETERS is the alloc params type for NV01_EVENT_OS_EVENT,
 // from src/common/sdk/nvidia/inc/class/cl0005.h.
 //
diff --git a/pkg/abi/nvgpu/ctrl.go b/pkg/abi/nvgpu/ctrl.go
index 30938b692..ded0cf2c8 100644
--- a/pkg/abi/nvgpu/ctrl.go
+++ b/pkg/abi/nvgpu/ctrl.go
@@ -243,6 +243,7 @@ const (
 
 // From src/common/sdk/nvidia/inc/ctrl/ctrl90e6.h:
 const (
+	NV90E6_CTRL_CMD_MASTER_GET_ERROR_INTR_OFFSET_MASK                = 0x90e60101
 	NV90E6_CTRL_CMD_MASTER_GET_VIRTUAL_FUNCTION_ERROR_CONT_INTR_MASK = 0x90e60102
 )
 
diff --git a/pkg/sentry/devices/nvproxy/frontend.go b/pkg/sentry/devices/nvproxy/frontend.go
index 2e4747512..8baeff550 100644
--- a/pkg/sentry/devices/nvproxy/frontend.go
+++ b/pkg/sentry/devices/nvproxy/frontend.go
@@ -596,6 +596,7 @@ func rmControl(fi *frontendIoctlState) (uintptr, error) {
 		nvgpu.NV83DE_CTRL_CMD_DEBUG_READ_ALL_SM_ERROR_STATES,
 		nvgpu.NV83DE_CTRL_CMD_DEBUG_CLEAR_ALL_SM_ERROR_STATES,
 		nvgpu.NV906F_CTRL_CMD_RESET_CHANNEL,
+		nvgpu.NV90E6_CTRL_CMD_MASTER_GET_ERROR_INTR_OFFSET_MASK,
 		nvgpu.NV90E6_CTRL_CMD_MASTER_GET_VIRTUAL_FUNCTION_ERROR_CONT_INTR_MASK,
 		nvgpu.NVC36F_CTRL_GET_CLASS_ENGINEID,
 		nvgpu.NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN,
@@ -770,15 +771,15 @@ func rmAlloc(fi *frontendIoctlState) (uintptr, error) {
 		return rmAllocSimple[nvgpu.NV_VASPACE_ALLOCATION_PARAMETERS](fi, &ioctlParams, isNVOS64)
 	case nvgpu.KEPLER_CHANNEL_GROUP_A:
 		return rmAllocSimple[nvgpu.NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS](fi, &ioctlParams, isNVOS64)
-	case nvgpu.TURING_CHANNEL_GPFIFO_A, nvgpu.AMPERE_CHANNEL_GPFIFO_A:
+	case nvgpu.VOLTA_CHANNEL_GPFIFO_A, nvgpu.TURING_CHANNEL_GPFIFO_A, nvgpu.AMPERE_CHANNEL_GPFIFO_A:
 		return rmAllocSimple[nvgpu.NV_CHANNEL_ALLOC_PARAMS](fi, &ioctlParams, isNVOS64)
-	case nvgpu.TURING_DMA_COPY_A, nvgpu.AMPERE_DMA_COPY_A, nvgpu.AMPERE_DMA_COPY_B, nvgpu.HOPPER_DMA_COPY_A:
+	case nvgpu.VOLTA_DMA_COPY_A, nvgpu.TURING_DMA_COPY_A, nvgpu.AMPERE_DMA_COPY_A, nvgpu.AMPERE_DMA_COPY_B, nvgpu.HOPPER_DMA_COPY_A:
 		return rmAllocSimple[nvgpu.NVB0B5_ALLOCATION_PARAMETERS](fi, &ioctlParams, isNVOS64)
-	case nvgpu.TURING_COMPUTE_A, nvgpu.AMPERE_COMPUTE_A, nvgpu.AMPERE_COMPUTE_B, nvgpu.ADA_COMPUTE_A, nvgpu.HOPPER_COMPUTE_A:
+	case nvgpu.VOLTA_COMPUTE_A, nvgpu.TURING_COMPUTE_A, nvgpu.AMPERE_COMPUTE_A, nvgpu.AMPERE_COMPUTE_B, nvgpu.ADA_COMPUTE_A, nvgpu.HOPPER_COMPUTE_A:
 		return rmAllocSimple[nvgpu.NV_GR_ALLOCATION_PARAMETERS](fi, &ioctlParams, isNVOS64)
 	case nvgpu.HOPPER_USERMODE_A:
 		return rmAllocSimple[nvgpu.NV_HOPPER_USERMODE_A_PARAMS](fi, &ioctlParams, isNVOS64)
-	case nvgpu.GF100_SUBDEVICE_MASTER, nvgpu.TURING_USERMODE_A:
+	case nvgpu.GF100_SUBDEVICE_MASTER, nvgpu.VOLTA_USERMODE_A, nvgpu.TURING_USERMODE_A:
 		return rmAllocNoParams(fi, &ioctlParams, isNVOS64)
 	default:
 		fi.ctx.Warningf("nvproxy: unknown allocation class %#08x", ioctlParams.HClass)
diff --git a/pkg/sentry/devices/nvproxy/nvproxy.go b/pkg/sentry/devices/nvproxy/nvproxy.go
index 626345197..0fa16f700 100644
--- a/pkg/sentry/devices/nvproxy/nvproxy.go
+++ b/pkg/sentry/devices/nvproxy/nvproxy.go
@@ -13,7 +13,9 @@
 // limitations under the License.
 
 // Package nvproxy implements proxying for the Nvidia GPU Linux kernel driver:
-// https://github.com/NVIDIA/open-gpu-kernel-modules
+// https://github.com/NVIDIA/open-gpu-kernel-modules.
+//
+// Supported Nvidia GPUs: T4, L4, A100, A10G, V100 and H100.
 package nvproxy
 
 import (