netbird/gvisor
0
0
Fork 0
mirror of https://github.com/netbirdio/gvisor.git synced 2026-05-22 17:12:49 -07:00
Code Issues Packages Projects Releases Wiki Activity

Add VFIO device for VFIO container.

Browse Source 
PiperOrigin-RevId: 612606195
This commit is contained in:
Jing Chen
2024-03-04 15:28:38 -08:00
committed by gVisor bot
parent c087777e37
commit eb62843a8d
9 changed files with 181 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/sentry/devices/tpuproxy/BUILD
+2
View File
@@ -10,6 +10,7 @@ go_library(
"device.go",
"seccomp_filter.go",
"tpu.go",
"vfio.go",
],
visibility = [
"//pkg/sentry:internal",
@@ -19,6 +20,7 @@ go_library(
"//pkg/context",
"//pkg/devutil",
"//pkg/errors/linuxerr",
"//pkg/fdnotifier",
"//pkg/log",
"//pkg/seccomp",
"//pkg/sentry/arch",
pkg/sentry/devices/tpuproxy/device.go
+61 -7
View File
@@ -16,32 +16,41 @@ package tpuproxy
import (
"fmt"
"path/filepath"
"golang.org/x/sys/unix"
"gvisor.dev/gvisor/pkg/abi/linux"
"gvisor.dev/gvisor/pkg/context"
"gvisor.dev/gvisor/pkg/devutil"
"gvisor.dev/gvisor/pkg/errors/linuxerr"
"gvisor.dev/gvisor/pkg/fdnotifier"
"gvisor.dev/gvisor/pkg/log"
"gvisor.dev/gvisor/pkg/sentry/vfs"
"gvisor.dev/gvisor/pkg/sync"
)
const (
tpuDeviceGroupName = "vfio"
// VFIO_MINOR is the VFIO minor number from include/linux/miscdevice.h.
VFIO_MINOR = 196
// VFIOPath is the path to a VFIO device, it is ususally used to
// construct a VFIO container.
VFIOPath = "/dev/vfio/vfio"
tpuDeviceGroupName = "vfio"
vfioDeviceGroupName = "vfio"
)
// vfioDevice implements vfs.Device for /dev/vfio/[0-9]+
// device implements TPU's vfs.Device for /dev/vfio/[0-9]+
//
// +stateify savable
type vfioDevice struct {
type tpuDevice struct {
mu sync.Mutex
minor uint32
}
// Open implememnts vfs.Device.Open.
func (dev *vfioDevice) Open(ctx context.Context, mnt *vfs.Mount, d *vfs.Dentry, opts vfs.OpenOptions) (*vfs.FileDescription, error) {
func (dev *tpuDevice) Open(ctx context.Context, mnt *vfs.Mount, d *vfs.Dentry, opts vfs.OpenOptions) (*vfs.FileDescription, error) {
devClient := devutil.GoferClientFromContext(ctx)
if devClient == nil {
log.Warningf("devutil.CtxDevGoferClient is not set")
@@ -49,13 +58,13 @@ func (dev *vfioDevice) Open(ctx context.Context, mnt *vfs.Mount, d *vfs.Dentry,
}
dev.mu.Lock()
defer dev.mu.Unlock()
devName := fmt.Sprintf("/dev/vfio/%d", dev.minor)
devName := fmt.Sprintf("vfio/%d", dev.minor)
hostFD, err := devClient.OpenAt(ctx, devName, opts.Flags)
if err != nil {
ctx.Warningf("vfioDevice: failed to open host %s: %v", devName, err)
return nil, err
}
fd := &vfioFD{
fd := &tpuFD{
hostFD: int32(hostFD),
device: dev,
}
@@ -68,11 +77,56 @@ func (dev *vfioDevice) Open(ctx context.Context, mnt *vfs.Mount, d *vfs.Dentry,
return &fd.vfsfd, nil
}
// device implements vfs.Device for /dev/vfio/vfio.
type vfioDevice struct {
mu sync.Mutex
}
// Open implements vfs.Device.Open.
func (dev *vfioDevice) Open(ctx context.Context, mnt *vfs.Mount, d *vfs.Dentry, opts vfs.OpenOptions) (*vfs.FileDescription, error) {
client := devutil.GoferClientFromContext(ctx)
if client == nil {
log.Warningf("devutil.CtxDevGoferClient is not set")
return nil, linuxerr.ENOENT
}
dev.mu.Lock()
defer dev.mu.Unlock()
name := fmt.Sprintf("vfio/%s", filepath.Base(VFIOPath))
hostFd, err := client.OpenAt(ctx, name, opts.Flags)
if err != nil {
ctx.Warningf("failed to open host file %s: %v", name, err)
return nil, err
}
fd := &vfioFd{
hostFd: int32(hostFd),
device: dev,
}
if err := fd.vfsfd.Init(fd, opts.Flags, mnt, d, &vfs.FileDescriptionOptions{
UseDentryMetadata: true,
}); err != nil {
unix.Close(hostFd)
return nil, err
}
if err := fdnotifier.AddFD(int32(hostFd), &fd.queue); err != nil {
unix.Close(hostFd)
return nil, err
}
return &fd.vfsfd, nil
}
// RegisterTPUDevice registers devices implemented by this package in vfsObj.
func RegisterTPUDevice(vfsObj *vfs.VirtualFilesystem, minor uint32) error {
return vfsObj.RegisterDevice(vfs.CharDevice, linux.VFIO_MAJOR, minor, &vfioDevice{
return vfsObj.RegisterDevice(vfs.CharDevice, linux.VFIO_MAJOR, minor, &tpuDevice{
minor: minor,
}, &vfs.RegisterDeviceOptions{
GroupName: tpuDeviceGroupName,
})
}
// RegisterVfioDevice registers VFIO devices that are implemented by this pacakge in vfsObj.
func RegisterVfioDevice(vfsObj *vfs.VirtualFilesystem) error {
return vfsObj.RegisterDevice(vfs.CharDevice, linux.MISC_MAJOR, VFIO_MINOR, &vfioDevice{}, &vfs.RegisterDeviceOptions{
GroupName: vfioDeviceGroupName,
})
}
pkg/sentry/devices/tpuproxy/tpu.go
+10 -10
View File
@@ -24,43 +24,43 @@ import (
"gvisor.dev/gvisor/pkg/waiter"
)
// vfioFD implements vfs.FileDescriptionImpl for /dev/vfio/[0-9]+
// tpuFD implements vfs.FileDescriptionImpl for /dev/vfio/[0-9]+
//
// vfioFD is not savable until TPU save/restore is needed.
type vfioFD struct {
// tpuFD is not savable until TPU save/restore is needed.
type tpuFD struct {
vfsfd vfs.FileDescription
vfs.FileDescriptionDefaultImpl
vfs.DentryMetadataFileDescriptionImpl
vfs.NoLockFD
hostFD int32
device *vfioDevice
device *tpuDevice
}
// Release implements vfs.FileDescriptionImpl.Release.
func (fd *vfioFD) Release(context.Context) {
func (fd *tpuFD) Release(context.Context) {
}
// EventRegister implements waiter.Waitable.EventRegister.
func (fd *vfioFD) EventRegister(e *waiter.Entry) error {
func (fd *tpuFD) EventRegister(e *waiter.Entry) error {
return nil
}
// EventUnregister implements waiter.Waitable.EventUnregister.
func (fd *vfioFD) EventUnregister(e *waiter.Entry) {
func (fd *tpuFD) EventUnregister(e *waiter.Entry) {
}
// Readiness implements waiter.Waitable.Readiness.
func (fd *vfioFD) Readiness(mask waiter.EventMask) waiter.EventMask {
func (fd *tpuFD) Readiness(mask waiter.EventMask) waiter.EventMask {
return waiter.EventErr
}
// Epollable implements vfs.FileDescriptionImpl.Epollable.
func (fd *vfioFD) Epollable() bool {
func (fd *tpuFD) Epollable() bool {
return true
}
// Ioctl implements vfs.FileDescriptionImpl.Ioctl.
func (fd *vfioFD) Ioctl(ctx context.Context, uio usermem.IO, sysno uintptr, args arch.SyscallArguments) (uintptr, error) {
func (fd *tpuFD) Ioctl(ctx context.Context, uio usermem.IO, sysno uintptr, args arch.SyscallArguments) (uintptr, error) {
return 0, linuxerr.ENOSYS
}
pkg/sentry/devices/tpuproxy/vfio.go
+80
View File
@@ -0,0 +1,80 @@
// Copyright 2024 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tpuproxy
import (
"fmt"
"golang.org/x/sys/unix"
"gvisor.dev/gvisor/pkg/context"
"gvisor.dev/gvisor/pkg/errors/linuxerr"
"gvisor.dev/gvisor/pkg/fdnotifier"
"gvisor.dev/gvisor/pkg/sentry/arch"
"gvisor.dev/gvisor/pkg/sentry/vfs"
"gvisor.dev/gvisor/pkg/usermem"
"gvisor.dev/gvisor/pkg/waiter"
)
// deviceFD implements vfs.FileDescriptionImpl for /dev/vfio/vfio.
type vfioFd struct {
vfsfd vfs.FileDescription
vfs.FileDescriptionDefaultImpl
vfs.DentryMetadataFileDescriptionImpl
vfs.NoLockFD
hostFd int32
device *vfioDevice
queue waiter.Queue
}
// Release implements vfs.FileDescriptionImpl.Release.
func (fd *vfioFd) Release(context.Context) {
fdnotifier.RemoveFD(fd.hostFd)
fd.queue.Notify(waiter.EventHUp)
unix.Close(int(fd.hostFd))
}
// EventRegister implements waiter.Waitable.EventRegister.
func (fd *vfioFd) EventRegister(e *waiter.Entry) error {
fd.queue.EventRegister(e)
if err := fdnotifier.UpdateFD(fd.hostFd); err != nil {
fd.queue.EventUnregister(e)
return err
}
return nil
}
// EventUnregister implements waiter.Waitable.EventUnregister.
func (fd *vfioFd) EventUnregister(e *waiter.Entry) {
fd.queue.EventUnregister(e)
if err := fdnotifier.UpdateFD(fd.hostFd); err != nil {
panic(fmt.Sprint("UpdateFD:", err))
}
}
// Readiness implements waiter.Waitable.Readiness.
func (fd *vfioFd) Readiness(mask waiter.EventMask) waiter.EventMask {
return fdnotifier.NonBlockingPoll(fd.hostFd, mask)
}
// Epollable implements vfs.FileDescriptionImpl.Epollable.
func (fd *vfioFd) Epollable() bool {
return true
}
// Ioctl implements vfs.FileDescriptionImpl.Ioctl.
func (fd *vfioFd) Ioctl(ctx context.Context, uio usermem.IO, sysno uintptr, args arch.SyscallArguments) (uintptr, error) {
return 0, linuxerr.ENOSYS
}
runsc/boot/vfs.go
+3
View File
@@ -1433,6 +1433,9 @@ func tpuProxyRegisterDevices(info *containerInfo, vfsObj *vfs.VirtualFilesystem)
}
}
}
if err := tpuproxy.RegisterVfioDevice(vfsObj); err != nil {
return fmt.Errorf("registering vfio driver: %w", err)
}
return nil
}
runsc/cmd/BUILD
+1
View File
@@ -88,6 +88,7 @@ go_library(
"//pkg/prometheus",
"//pkg/ring0",
"//pkg/sentry/control",
"//pkg/sentry/devices/tpuproxy",
"//pkg/sentry/kernel",
"//pkg/sentry/kernel/auth",
"//pkg/sentry/platform",
runsc/cmd/gofer.go
+8 -1
View File
@@ -30,6 +30,7 @@ import (
specs "github.com/opencontainers/runtime-spec/specs-go"
"golang.org/x/sys/unix"
"gvisor.dev/gvisor/pkg/log"
"gvisor.dev/gvisor/pkg/sentry/devices/tpuproxy"
"gvisor.dev/gvisor/pkg/unet"
"gvisor.dev/gvisor/runsc/boot"
"gvisor.dev/gvisor/runsc/cmd/util"
@@ -538,13 +539,19 @@ func shouldExposeNvidiaDevice(path string) bool {
return nvidiaDevPathReg.MatchString(path)
}
// shouldExposeVfioDevice returns true if path refers to an VFIO device
// which shuold be exposed to the container.
func shouldExposeVFIODevice(path string) bool {
return strings.HasPrefix(path, filepath.Dir(tpuproxy.VFIOPath))
}
// shouldExposeTpuDevice returns true if path refers to a TPU device which
// should be exposed to the container.
//
// Precondition: tpuproxy is enabled.
func shouldExposeTpuDevice(path string) bool {
_, valid, _ := util.ExtractTpuDeviceMinor(path)
return valid
return valid || shouldExposeVFIODevice(path)
}
func (g *Gofer) setupDev(spec *specs.Spec, conf *config.Config, root, procPath string) error {
runsc/specutils/BUILD
+1
View File
@@ -19,6 +19,7 @@ go_library(
"//pkg/abi/linux",
"//pkg/bits",
"//pkg/log",
"//pkg/sentry/devices/tpuproxy",
"//pkg/sentry/kernel/auth",
"//runsc/config",
"//runsc/flag",
runsc/specutils/specutils.go
+15 -1
View File
@@ -35,6 +35,7 @@ import (
"gvisor.dev/gvisor/pkg/abi/linux"
"gvisor.dev/gvisor/pkg/bits"
"gvisor.dev/gvisor/pkg/log"
"gvisor.dev/gvisor/pkg/sentry/devices/tpuproxy"
"gvisor.dev/gvisor/pkg/sentry/kernel/auth"
"gvisor.dev/gvisor/runsc/config"
"gvisor.dev/gvisor/runsc/flag"
@@ -577,6 +578,19 @@ func TPUProxyIsEnabled(spec *specs.Spec, conf *config.Config) bool {
return ret
}
// VFIOFunctionalityRequested returns true if the container should have access
// to VFIO functionality.
func VFIOFunctionalityRequested(dev *specs.LinuxDevice) bool {
return strings.HasPrefix(dev.Path, filepath.Dir(tpuproxy.VFIOPath))
}
// AcceleratorFunctionalityRequested returns true if the container should have
// access to compute accelerators. Compute accelerators are different from GPUs
// by using a different major number and different device char files.
func AcceleratorFunctionalityRequested(dev *specs.LinuxDevice) bool {
return strings.HasPrefix(dev.Path, "/dev/accel")
}
// TPUFunctionalityRequested returns true if the container should have access
// to TPU functionality.
func TPUFunctionalityRequested(spec *specs.Spec, conf *config.Config) bool {
@@ -585,7 +599,7 @@ func TPUFunctionalityRequested(spec *specs.Spec, conf *config.Config) bool {
}
if spec.Linux != nil {
for _, dev := range spec.Linux.Devices {
if strings.HasPrefix(dev.Path, "/dev/accel") {
if AcceleratorFunctionalityRequested(&dev) || VFIOFunctionalityRequested(&dev) {
return true
}
}