From 05e7c2fcebb7b55220b68ed01a833c751c1990f1 Mon Sep 17 00:00:00 2001 From: Sergey Madaminov Date: Tue, 27 Sep 2022 17:41:44 -0700 Subject: [PATCH] mmap() implementation for the IO_URING. Once the user receives a file descriptor from `io_uring_setup()`, it will be used for the subsequent `mmap()` calls. Thus, we need to add support for it in our iouringfs. PiperOrigin-RevId: 477318038 --- pkg/abi/linux/iouring.go | 160 +++++++++--- pkg/hostarch/BUILD | 1 + pkg/hostarch/addr.go | 13 + pkg/hostarch/addr_test.go | 71 +++++ pkg/hostarch/hostarch_arm64.go | 6 + pkg/hostarch/hostarch_x86.go | 6 + pkg/sentry/fsimpl/iouringfs/BUILD | 13 +- pkg/sentry/fsimpl/iouringfs/iouringfs.go | 246 ++++++++++++++++-- pkg/sentry/fsimpl/iouringfs/iouringfs_test.go | 69 +++++ pkg/sentry/syscalls/linux/vfs2/iouringfs.go | 24 +- pkg/sentry/syscalls/linux/vfs2/vfs2.go | 4 +- test/syscalls/BUILD | 2 + test/syscalls/linux/io_uring.cc | 137 +++++++++- test/util/io_uring_util.h | 26 +- 14 files changed, 706 insertions(+), 72 deletions(-) create mode 100644 pkg/hostarch/addr_test.go create mode 100644 pkg/sentry/fsimpl/iouringfs/iouringfs_test.go diff --git a/pkg/abi/linux/iouring.go b/pkg/abi/linux/iouring.go index 59b9c6f88..b13056cad 100644 --- a/pkg/abi/linux/iouring.go +++ b/pkg/abi/linux/iouring.go @@ -14,6 +14,11 @@ package linux +import ( + "fmt" + "reflect" +) + // Constants for io_uring_setup(2). See include/uapi/linux/io_uring.h. const ( IORING_SETUP_IOPOLL = (1 << 0) @@ -26,6 +31,11 @@ const ( IORING_SETUP_SUBMIT_ALL = (1 << 7) ) +// Constants for IoUringParams.Features. See include/uapi/linux/io_uring.h. +const ( + IORING_FEAT_SINGLE_MMAP = (1 << 0) +) + // Constants for IO_URING. See include/uapi/linux/io_uring.h. const ( IORING_SETUP_COOP_TASKRUN = (1 << 8) @@ -36,46 +46,62 @@ const ( // Constants for IO_URING. See io_uring/io_uring.c. const ( - IORING_MAX_ENTRIES = (1 << 15) // 32768 + IORING_MAX_ENTRIES = (1 << 15) // 32768 + IORING_MAX_CQ_ENTRIES = (2 * IORING_MAX_ENTRIES) ) -// IoSqringOffsets implements io_sqring_offsets struct. +// Constants for the offsets for the application to mmap the data it needs. // See include/uapi/linux/io_uring.h. +const ( + IORING_OFF_SQ_RING = 0 + IORING_OFF_CQ_RING = 0x8000000 + IORING_OFF_SQES = 0x10000000 +) + +// IORingIndex represents SQE array indexes. // // +marshal -type IoSqringOffsets struct { - Head uint32 - Tail uint32 - RingMask uint32 - RingEntries uint32 - Flags uint32 - Dropped uint32 - Array uint32 - Resv1 uint32 - Resv2 uint64 +type IORingIndex uint32 + +// IOSqRingOffsets implements io_sqring_offsets struct. +// IOSqRingOffsets represents offsets into IORings. +// See struct io_sqring_offsets in include/uapi/linux/io_uring.h. +// +// +marshal +type IOSqRingOffsets struct { + Head uint32 // Offset to io_rings.sq.head + Tail uint32 // Offset to io_rings.sq.tail + RingMask uint32 // Offset to io_rings.sq_ring_mask + RingEntries uint32 // Offset to io_rings.sq_ring_entries + Flags uint32 // Offset to io_rings.sq_flags + Dropped uint32 // Offset to io_rings.sq_dropped + Array uint32 // Offset to an array of SQE indices + Resv1 uint32 // Currently reserved and expected to be zero + Resv2 uint64 // Currently reserved and expected to be zero } -// IoCqringOffsets implements io_cqring_offsets struct. -// See include/uapi/linux/io_uring.h. +// IOCqRingOffsets implements io_cqring_offsets struct. +// IOCqRingOffsets represents offsets into IORings. +// See struct io_cqring_offsets in include/uapi/linux/io_uring.h. // // +marshal -type IoCqringOffsets struct { - Head uint32 - Tail uint32 - RingMask uint32 - RingEntries uint32 - Overflow uint32 - Cqes uint32 - Flags uint32 - Resv1 uint32 - Resv2 uint64 +type IOCqRingOffsets struct { + Head uint32 // Offset to io_rings.cq.head + Tail uint32 // Offset to io_rings.cq.tail + RingMask uint32 // Offset to io_rings.cq_ring_mask + RingEntries uint32 // Offset to io_rings.cq_ring_entries + Overflow uint32 // Offset to io_rings.cq_overflow + Cqes uint32 // Offset to io_rings.cqes + Flags uint32 // Offset to io_rings.cq_flags + Resv1 uint32 // Currently reserved and expected to be zero + Resv2 uint64 // Currently reserved and expected to be zero } -// IoUringParams implements io_uring_params struct. -// See include/uapi/linux/io_uring.h. +// IOUringParams implements io_uring_params struct. +// See struct io_uring_params in include/uapi/linux/io_uring.h. // // +marshal -type IoUringParams struct { +type IOUringParams struct { SqEntries uint32 CqEntries uint32 Flags uint32 @@ -84,6 +110,82 @@ type IoUringParams struct { Features uint32 WqFd uint32 Resv [3]uint32 - SqOff IoSqringOffsets - CqOff IoCqringOffsets + SqOff IOSqRingOffsets + CqOff IOCqRingOffsets +} + +// IOUringCqe implements IO completion data structure (Completion Queue Entry) +// io_uring_cqe struct. As we don't currently support IORING_SETUP_CQE32 flag +// its size is 16 bytes. +// See struct io_uring_cqe in include/uapi/linux/io_uring.h. +// +// +marshal +type IOUringCqe struct { + userData uint64 + res int32 + flags uint32 +} + +// IOUring implements io_uring struct. +// See struct io_uring in io_uring/io_uring.c. +// +// +marshal +type IOUring struct { + // Both head and tail should be cacheline aligned. And we assume that + // cacheline size is 64 bytes. + head uint32 + _ [60]byte + tail uint32 + _ [60]byte +} + +// IORings implements io_rings struct. +// This struct describes layout of the mapped region backed by the ringBuffersFile. +// See struct io_rings in io_uring/io_uring.c. +// +// +marshal +type IORings struct { + sq, cq IOUring + sqRingMask, cqRingMask uint32 + sqRingEntries, cqRingEntries uint32 + sqDropped uint32 + sqFlags int32 + cqFlags uint32 + cqOverflow uint32 + _ [32]byte // Padding so cqes is cacheline aligned + // Linux has an additional field struct io_uring_cqe cqes[], which represents + // a dynamic array. We don't include it here in order to enable marshalling. +} + +// PreComputedIOSqRingOffsets stores precomputed values for IOSqRingOffsets. +var PreComputedIOSqRingOffsets IOSqRingOffsets + +// PreComputedIOCqRingOffsets stores precomputed values for IOCqRingOffsets. +var PreComputedIOCqRingOffsets IOCqRingOffsets + +func init() { + ioRingsType := reflect.TypeOf((*IORings)(nil)).Elem() + ioUringType := reflect.TypeOf((*IOUring)(nil)).Elem() + + offsetof := func(ty reflect.Type, name string) uint32 { + if f, ok := ty.FieldByName(name); ok { + return uint32(f.Offset) + } + panic(fmt.Sprintf("In type %q, no field named %q", ty.Name(), name)) + } + + PreComputedIOSqRingOffsets.Head = offsetof(ioRingsType, "sq") + offsetof(ioUringType, "head") + PreComputedIOSqRingOffsets.Tail = offsetof(ioRingsType, "sq") + offsetof(ioUringType, "tail") + PreComputedIOSqRingOffsets.RingMask = offsetof(ioRingsType, "sqRingMask") + PreComputedIOSqRingOffsets.RingEntries = offsetof(ioRingsType, "sqRingEntries") + PreComputedIOSqRingOffsets.Flags = offsetof(ioRingsType, "sqFlags") + PreComputedIOSqRingOffsets.Dropped = offsetof(ioRingsType, "sqDropped") + + PreComputedIOCqRingOffsets.Head = offsetof(ioRingsType, "cq") + offsetof(ioUringType, "head") + PreComputedIOCqRingOffsets.Tail = offsetof(ioRingsType, "cq") + offsetof(ioUringType, "tail") + PreComputedIOCqRingOffsets.RingMask = offsetof(ioRingsType, "cqRingMask") + PreComputedIOCqRingOffsets.RingEntries = offsetof(ioRingsType, "cqRingEntries") + PreComputedIOCqRingOffsets.Overflow = offsetof(ioRingsType, "cqOverflow") + PreComputedIOCqRingOffsets.Flags = offsetof(ioRingsType, "cqFlags") + } diff --git a/pkg/hostarch/BUILD b/pkg/hostarch/BUILD index 1e8def4d9..2a985f82d 100644 --- a/pkg/hostarch/BUILD +++ b/pkg/hostarch/BUILD @@ -19,6 +19,7 @@ go_test( size = "small", srcs = [ "addr_range_seq_test.go", + "addr_test.go", ], library = ":hostarch", ) diff --git a/pkg/hostarch/addr.go b/pkg/hostarch/addr.go index cb2e11637..ef7b341bb 100644 --- a/pkg/hostarch/addr.go +++ b/pkg/hostarch/addr.go @@ -133,3 +133,16 @@ func ToPages(x uint64) (uint64, bool) { } return xRoundedUp / PageSize, true } + +// CacheLineRoundDown returns the offset rounded down to the nearest cache line boundary. +func CacheLineRoundDown(x uint64) uint64 { + return x & ^uint64(CacheLineSize-1) +} + +// CacheLineRoundUp returns the offset rounded up to the nearest cache line boundary. ok is true iff +// rounding up did not wrap around. +func CacheLineRoundUp(x uint64) (val uint64, ok bool) { + val = CacheLineRoundDown(x + uint64(CacheLineSize-1)) + ok = val >= x + return +} diff --git a/pkg/hostarch/addr_test.go b/pkg/hostarch/addr_test.go new file mode 100644 index 000000000..30d2485b8 --- /dev/null +++ b/pkg/hostarch/addr_test.go @@ -0,0 +1,71 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostarch + +import ( + "fmt" + "math" + "testing" +) + +func TestCacheLineRoundUp(t *testing.T) { + tests := []struct { + input uint64 + output uint64 + }{ + {0, 0}, + {63, 64}, + {64, 64}, + {65, 128}, + {66, 128}, + {99, 128}, + {127, 128}, + {128, 128}, + {129, 192}, + {math.MaxUint32, math.MaxUint32 + 1}, + } + for i, tt := range tests { + t.Run(fmt.Sprintf("case-%d", i), func(t *testing.T) { + s, ok := CacheLineRoundUp(tt.input) + if s != tt.output { + t.Errorf("Expected %d, got %d", tt.output, s) + } + if !ok { + t.Errorf("Expected no wrap around, got %t. Input %d, expected %d", ok, tt.input, tt.output) + } + }) + } +} + +func TestCacheLineRoundUpWrapAround(t *testing.T) { + tests := []struct { + input uint64 + output uint64 + }{ + {math.MaxUint64 - 1, 0}, + {math.MaxUint64, 0}, + } + for i, tt := range tests { + t.Run(fmt.Sprintf("case-%d", i), func(t *testing.T) { + s, ok := CacheLineRoundUp(tt.input) + if s != tt.output { + t.Errorf("Expected %d, got %d", tt.output, s) + } + if ok { + t.Errorf("Expected wrap around, got %t. Input %d, expected %d", ok, tt.input, tt.output) + } + }) + } +} diff --git a/pkg/hostarch/hostarch_arm64.go b/pkg/hostarch/hostarch_arm64.go index a65c810a5..635f0e475 100644 --- a/pkg/hostarch/hostarch_arm64.go +++ b/pkg/hostarch/hostarch_arm64.go @@ -33,6 +33,9 @@ const ( // HugePageSize is the system huge page size. HugePageSize = 1 << HugePageShift + // CacheLineSize is the size of the cache line. + CacheLineSize = 1 << CacheLineShift + // PageShift is the binary log of the system page size. PageShift = 12 @@ -40,6 +43,9 @@ const ( // Should be calculated by "PageShift + (PageShift - 3)" // when multiple page size support is ready. HugePageShift = 21 + + // CacheLineShift is the binary log of the cache line size. + CacheLineShift = 6 ) var ( diff --git a/pkg/hostarch/hostarch_x86.go b/pkg/hostarch/hostarch_x86.go index 00bf668f3..25c8ff3a0 100644 --- a/pkg/hostarch/hostarch_x86.go +++ b/pkg/hostarch/hostarch_x86.go @@ -26,11 +26,17 @@ const ( // HugePageSize is the system huge page size. HugePageSize = 1 << HugePageShift + // CacheLineSize is the size of the cache line. + CacheLineSize = 1 << CacheLineShift + // PageShift is the binary log of the system page size. PageShift = 12 // HugePageShift is the binary log of the system huge page size. HugePageShift = 21 + + // CacheLineShift is the binary log of the cache line size. + CacheLineShift = 6 ) var ( diff --git a/pkg/sentry/fsimpl/iouringfs/BUILD b/pkg/sentry/fsimpl/iouringfs/BUILD index c730e82ec..593d2e217 100644 --- a/pkg/sentry/fsimpl/iouringfs/BUILD +++ b/pkg/sentry/fsimpl/iouringfs/BUILD @@ -1,4 +1,4 @@ -load("//tools:defs.bzl", "go_library") +load("//tools:defs.bzl", "go_library", "go_test") licenses(["notice"]) @@ -9,7 +9,18 @@ go_library( deps = [ "//pkg/abi/linux", "//pkg/context", + "//pkg/errors/linuxerr", "//pkg/hostarch", + "//pkg/sentry/memmap", + "//pkg/sentry/pgalloc", + "//pkg/sentry/usage", "//pkg/sentry/vfs", ], ) + +go_test( + name = "iouringfs_test", + size = "small", + srcs = ["iouringfs_test.go"], + library = ":iouringfs", +) diff --git a/pkg/sentry/fsimpl/iouringfs/iouringfs.go b/pkg/sentry/fsimpl/iouringfs/iouringfs.go index 7ffebae7f..7ba8d525b 100644 --- a/pkg/sentry/fsimpl/iouringfs/iouringfs.go +++ b/pkg/sentry/fsimpl/iouringfs/iouringfs.go @@ -17,46 +17,105 @@ package iouringfs import ( + "fmt" + "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/context" + "gvisor.dev/gvisor/pkg/errors/linuxerr" "gvisor.dev/gvisor/pkg/hostarch" + "gvisor.dev/gvisor/pkg/sentry/memmap" + "gvisor.dev/gvisor/pkg/sentry/pgalloc" + "gvisor.dev/gvisor/pkg/sentry/usage" "gvisor.dev/gvisor/pkg/sentry/vfs" ) -// IoUring implements io_uring struct. See io_uring/io_uring.c. -type IoUring struct { - head uint32 - tail uint32 -} - -// IoUringCqe implements IO completion data structure (Completion Queue Entry) -// io_uring_cqe struct. See include/uapi/linux/io_uring.h. -type IoUringCqe struct { - userData uint64 - res int16 - flags uint32 - bigCqe *uint64 -} - // FileDescription implements vfs.FileDescriptionImpl for file-based IO_URING. // It is based on io_rings struct. See io_uring/io_uring.c. // // +stateify savable -type FileDescription struct { +type fileDescription struct { vfsfd vfs.FileDescription vfs.FileDescriptionDefaultImpl vfs.DentryMetadataFileDescriptionImpl vfs.NoLockFD + + rbmf ringsBufferFile + sqemf sqEntriesFile } -var _ vfs.FileDescriptionImpl = (*FileDescription)(nil) +var _ vfs.FileDescriptionImpl = (*fileDescription)(nil) + +func roundUpPowerOfTwo(n uint32) (uint32, bool) { + if n > (1 << 31) { + return 0, false + } + result := uint32(1) + for result < n { + result = result << 1 + } + return result, true +} // New creates a new iouring fd. -func New(ctx context.Context, vfsObj *vfs.VirtualFilesystem, entries uint32, params *linux.IoUringParams, paramsUser hostarch.Addr) (*vfs.FileDescription, error) { +func New(ctx context.Context, vfsObj *vfs.VirtualFilesystem, entries uint32, params *linux.IOUringParams) (*vfs.FileDescription, error) { + if entries > linux.IORING_MAX_ENTRIES { + return nil, linuxerr.EINVAL + } + vd := vfsObj.NewAnonVirtualDentry("[io_uring]") defer vd.DecRef(ctx) - iouringfd := &FileDescription{} + mfp := pgalloc.MemoryFileProviderFromContext(ctx) + if mfp == nil { + panic(fmt.Sprintf("context.Context %T lacks non-nil value for key %T", ctx, pgalloc.CtxMemoryFileProvider)) + } + + numSqEntries, ok := roundUpPowerOfTwo(entries) + if !ok { + return nil, linuxerr.EOVERFLOW + } + var numCqEntries uint32 + if params.Flags&linux.IORING_SETUP_CQSIZE != 0 { + if params.CqEntries > linux.IORING_MAX_CQ_ENTRIES { + return nil, linuxerr.EINVAL + } + numCqEntries = params.CqEntries + } else { + numCqEntries = 2 * numSqEntries + } + + // Allocate enough space to store the `struct io_rings` plus a given number of indexes + // corresponding to the number of SQEs. + ioRingsWithCqesSize := uint32((*linux.IORings)(nil).SizeBytes()) + + numCqEntries*uint32((*linux.IOUringCqe)(nil).SizeBytes()) + ringsBufferSize := uint64(ioRingsWithCqesSize + + numSqEntries*uint32((*linux.IORingIndex)(nil).SizeBytes())) + ringsBufferSize = uint64(hostarch.Addr(ringsBufferSize).MustRoundUp()) + + rbfr, err := mfp.MemoryFile().Allocate(ringsBufferSize, pgalloc.AllocOpts{Kind: usage.Anonymous}) + if err != nil { + return nil, linuxerr.ENOMEM + } + + // Allocate enough space to store the given number of submission queue entries. + sqEntriesSize := uint64(numSqEntries * 64) + sqEntriesSize = uint64(hostarch.Addr(sqEntriesSize).MustRoundUp()) + + sqefr, err := mfp.MemoryFile().Allocate(sqEntriesSize, pgalloc.AllocOpts{Kind: usage.Anonymous}) + if err != nil { + return nil, linuxerr.ENOMEM + } + + iouringfd := &fileDescription{ + rbmf: ringsBufferFile{ + mf: mfp.MemoryFile(), + fr: rbfr, + }, + sqemf: sqEntriesFile{ + mf: mfp.MemoryFile(), + fr: sqefr, + }, + } // iouringfd is always set up with read/write mode. // See io_uring/io_uring.c:io_uring_install_fd(). @@ -69,9 +128,156 @@ func New(ctx context.Context, vfsObj *vfs.VirtualFilesystem, entries uint32, par return nil, err } + params.SqEntries = numSqEntries + params.CqEntries = numCqEntries + + arrayOffset := uint64(hostarch.Addr(ioRingsWithCqesSize)) + arrayOffset, ok = hostarch.CacheLineRoundUp(arrayOffset) + if !ok { + return nil, linuxerr.EOVERFLOW + } + + params.SqOff = linux.PreComputedIOSqRingOffsets + params.SqOff.Array = uint32(arrayOffset) + + cqesOffset := uint64(hostarch.Addr((*linux.IORings)(nil).SizeBytes())) + cqesOffset, ok = hostarch.CacheLineRoundUp(cqesOffset) + if !ok { + return nil, linuxerr.EOVERFLOW + } + + params.CqOff = linux.PreComputedIOCqRingOffsets + params.CqOff.Cqes = uint32(cqesOffset) + + // Set features supported by the current IO_URING implementation. + params.Features = linux.IORING_FEAT_SINGLE_MMAP + return &iouringfd.vfsfd, nil } // Release implements vfs.FileDescriptionImpl.Release. -func (iouringfd *FileDescription) Release(context.Context) { +func (fd *fileDescription) Release(context.Context) { +} + +// ConfigureMMap implements vfs.FileDescriptionImpl.ConfigureMMap. +func (fd *fileDescription) ConfigureMMap(ctx context.Context, opts *memmap.MMapOpts) error { + var mf memmap.Mappable + switch opts.Offset { + case linux.IORING_OFF_SQ_RING, linux.IORING_OFF_CQ_RING: + mf = &fd.rbmf + case linux.IORING_OFF_SQES: + mf = &fd.sqemf + default: + return linuxerr.EINVAL + } + + return vfs.GenericConfigureMMap(&fd.vfsfd, mf, opts) +} + +// sqEntriesFile implements memmap.Mappable for SQ entries. +type sqEntriesFile struct { + mf *pgalloc.MemoryFile + fr memmap.FileRange +} + +// AddMapping implements memmap.Mappable.AddMapping. +func (sqemf *sqEntriesFile) AddMapping(ctx context.Context, ms memmap.MappingSpace, ar hostarch.AddrRange, offset uint64, writable bool) error { + return nil +} + +// RemoveMapping implements memmap.Mappable.RemoveMapping. +func (sqemf *sqEntriesFile) RemoveMapping(ctx context.Context, ms memmap.MappingSpace, ar hostarch.AddrRange, offset uint64, writable bool) { +} + +// CopyMapping implements memmap.Mappable.CopyMapping. +func (sqemf *sqEntriesFile) CopyMapping(ctx context.Context, ms memmap.MappingSpace, srcAR, dstAR hostarch.AddrRange, offset uint64, writable bool) error { + return nil +} + +// Translate implements memmap.Mappable.Translate. +func (sqemf *sqEntriesFile) Translate(ctx context.Context, required, optional memmap.MappableRange, at hostarch.AccessType) ([]memmap.Translation, error) { + expectedAccessType := hostarch.AccessType{ + Read: true, + Write: true, + Execute: false, + } + if at != expectedAccessType { + return nil, &memmap.BusError{linuxerr.EPERM} + } + + if required.End > sqemf.fr.Length() { + return nil, &memmap.BusError{linuxerr.EFAULT} + } + + if source := optional.Intersect(memmap.MappableRange{0, sqemf.fr.Length()}); source.Length() != 0 { + return []memmap.Translation{ + { + Source: source, + File: sqemf.mf, + Offset: sqemf.fr.Start + source.Start, + Perms: at, + }, + }, nil + } + + return nil, linuxerr.EFAULT +} + +// InvalidateUnsavable implements memmap.Mappable.InvalidateUnsavable. +func (sqemf *sqEntriesFile) InvalidateUnsavable(ctx context.Context) error { + return nil +} + +// ringBuffersFile implements memmap.Mappable for SQ and CQ ring buffers. +type ringsBufferFile struct { + mf *pgalloc.MemoryFile + fr memmap.FileRange +} + +// AddMapping implements memmap.Mappable.AddMapping. +func (rbmf *ringsBufferFile) AddMapping(ctx context.Context, ms memmap.MappingSpace, ar hostarch.AddrRange, offset uint64, writable bool) error { + return nil +} + +// RemoveMapping implements memmap.Mappable.RemoveMapping. +func (rbmf *ringsBufferFile) RemoveMapping(ctx context.Context, ms memmap.MappingSpace, ar hostarch.AddrRange, offset uint64, writable bool) { +} + +// CopyMapping implements memmap.Mappable.CopyMapping. +func (rbmf *ringsBufferFile) CopyMapping(ctx context.Context, ms memmap.MappingSpace, srcAR, dstAR hostarch.AddrRange, offset uint64, writable bool) error { + return nil +} + +// Translate implements memmap.Mappable.Translate. +func (rbmf *ringsBufferFile) Translate(ctx context.Context, required, optional memmap.MappableRange, at hostarch.AccessType) ([]memmap.Translation, error) { + expectedAccessType := hostarch.AccessType{ + Read: true, + Write: true, + Execute: false, + } + if at != expectedAccessType { + return nil, &memmap.BusError{linuxerr.EPERM} + } + + if required.End > rbmf.fr.Length() { + return nil, &memmap.BusError{linuxerr.EFAULT} + } + + if source := optional.Intersect(memmap.MappableRange{0, rbmf.fr.Length()}); source.Length() != 0 { + return []memmap.Translation{ + { + Source: source, + File: rbmf.mf, + Offset: rbmf.fr.Start + source.Start, + Perms: at, + }, + }, nil + } + + return nil, linuxerr.EFAULT +} + +// InvalidateUnsavable implements memmap.Mappable.InvalidateUnsavable. +func (rbmf *ringsBufferFile) InvalidateUnsavable(ctx context.Context) error { + return nil } diff --git a/pkg/sentry/fsimpl/iouringfs/iouringfs_test.go b/pkg/sentry/fsimpl/iouringfs/iouringfs_test.go new file mode 100644 index 000000000..6395edd53 --- /dev/null +++ b/pkg/sentry/fsimpl/iouringfs/iouringfs_test.go @@ -0,0 +1,69 @@ +// Copyright 2021 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iouringfs + +import ( + "fmt" + "math" + "testing" +) + +func TestRoundUpPowerOfTwo(t *testing.T) { + tests := []struct { + input uint32 + output uint32 + }{ + {0, 1}, + {1, 1}, + {2, 2}, + {3, 4}, + {4, 4}, + {5, 8}, + {6, 8}, + {7, 8}, + {8, 8}, + {1 << 31, 2147483648}, + {1<<31 - 1, 2147483648}, + } + for i, tt := range tests { + t.Run(fmt.Sprintf("case-%d", i), func(t *testing.T) { + s, ok := roundUpPowerOfTwo(tt.input) + if s != tt.output { + t.Errorf("Expected %d, got %d", tt.output, s) + } + if !ok { + t.Errorf("Expected no error, got %t. Input %d, expected %d", ok, tt.input, tt.output) + } + }) + } +} + +func TestRoundUpPowerOfTwoOverflow(t *testing.T) { + tests := []struct { + input uint32 + output uint32 + }{ + {1<<31 + 1, 0}, + {math.MaxUint32, 0}, + } + for i, tt := range tests { + t.Run(fmt.Sprintf("case-%d", i), func(t *testing.T) { + s, ok := roundUpPowerOfTwo(tt.input) + if s != tt.output || ok { + t.Errorf("Expected value %d and overflow, got %d and %t", tt.output, s, ok) + } + }) + } +} diff --git a/pkg/sentry/syscalls/linux/vfs2/iouringfs.go b/pkg/sentry/syscalls/linux/vfs2/iouringfs.go index e67e39de9..476413d61 100644 --- a/pkg/sentry/syscalls/linux/vfs2/iouringfs.go +++ b/pkg/sentry/syscalls/linux/vfs2/iouringfs.go @@ -22,11 +22,15 @@ import ( "gvisor.dev/gvisor/pkg/sentry/kernel" ) -// IoUringSetup implements linux syscall io_uring_setup(2). -func IoUringSetup(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel.SyscallControl, error) { +// IOUringSetup implements linux syscall io_uring_setup(2). +func IOUringSetup(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel.SyscallControl, error) { entries := uint32(args[0].Uint()) paramsAddr := args[1].Pointer() - var params linux.IoUringParams + var params linux.IOUringParams + + if entries == 0 { + return 0, nil, linuxerr.EINVAL + } if _, err := params.CopyIn(t, paramsAddr); err != nil { return 0, nil, err } @@ -37,8 +41,16 @@ func IoUringSetup(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel. } } + // List of currently supported flags in our IO_URING implementation. + const supportedFlags = linux.IORING_SETUP_IOPOLL + + // Since we don't implement everything, we fail explicitly on flags that are unimplemented. + if params.Flags|supportedFlags != supportedFlags { + return 0, nil, linuxerr.EINVAL + } + vfsObj := t.Kernel().VFS() - iouringfd, err := iouringfs.New(t, vfsObj, entries, ¶ms, paramsAddr) + iouringfd, err := iouringfs.New(t, vfsObj, entries, ¶ms) if err != nil { return 0, nil, err @@ -54,5 +66,9 @@ func IoUringSetup(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel. return 0, nil, err } + if _, err := params.CopyOut(t, paramsAddr); err != nil { + return 0, nil, err + } + return uintptr(fd), nil, nil } diff --git a/pkg/sentry/syscalls/linux/vfs2/vfs2.go b/pkg/sentry/syscalls/linux/vfs2/vfs2.go index 1f97d83c7..6a6b50a78 100644 --- a/pkg/sentry/syscalls/linux/vfs2/vfs2.go +++ b/pkg/sentry/syscalls/linux/vfs2/vfs2.go @@ -163,7 +163,7 @@ func Override() { s.Table[327] = syscalls.Supported("preadv2", Preadv2) s.Table[328] = syscalls.Supported("pwritev2", Pwritev2) s.Table[332] = syscalls.Supported("statx", Statx) - s.Table[425] = syscalls.PartiallySupported("io_uring_setup", IoUringSetup, "Not all flags and functionality supported.", nil) + s.Table[425] = syscalls.PartiallySupported("io_uring_setup", IOUringSetup, "Not all flags and functionality supported.", nil) s.Table[436] = syscalls.Supported("close_range", CloseRange) s.Table[439] = syscalls.Supported("faccessat2", Faccessat2) s.Table[441] = syscalls.Supported("epoll_pwait2", EpollPwait2) @@ -280,7 +280,7 @@ func Override() { s.Table[286] = syscalls.Supported("preadv2", Preadv2) s.Table[287] = syscalls.Supported("pwritev2", Pwritev2) s.Table[291] = syscalls.Supported("statx", Statx) - s.Table[425] = syscalls.PartiallySupported("io_uring_setup", IoUringSetup, "Not all flags and functionality supported.", nil) + s.Table[425] = syscalls.PartiallySupported("io_uring_setup", IOUringSetup, "Not all flags and functionality supported.", nil) s.Table[436] = syscalls.Supported("close_range", CloseRange) s.Table[439] = syscalls.Supported("faccessat2", Faccessat2) s.Table[441] = syscalls.Supported("epoll_pwait2", EpollPwait2) diff --git a/test/syscalls/BUILD b/test/syscalls/BUILD index db966cba9..116700219 100644 --- a/test/syscalls/BUILD +++ b/test/syscalls/BUILD @@ -247,6 +247,8 @@ syscall_test( ) syscall_test( + # Temporarily added due to intermittent ENOMEM failures. See b/216213621. + tags = ["notap"], test = "//test/syscalls/linux:iouring_test", ) diff --git a/test/syscalls/linux/io_uring.cc b/test/syscalls/linux/io_uring.cc index 436bc51ae..03b9ae131 100644 --- a/test/syscalls/linux/io_uring.cc +++ b/test/syscalls/linux/io_uring.cc @@ -18,10 +18,13 @@ #include #include #include +#include #include #include #include +#include + #include "gtest/gtest.h" #include "test/util/io_uring_util.h" #include "test/util/test_util.h" @@ -31,21 +34,137 @@ namespace testing { namespace { -TEST(IoUringTest, ValidFD) { - FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIoUringFD(1)); -} - -TEST(IoUringTest, SetUp) { +// Testing that io_uring_setup(2) successfully returns a valid file descriptor. +TEST(IOUringTest, ValidFD) { struct io_uring_params params; - memset(¶ms, 0, sizeof(params)); - ASSERT_THAT(IoUringSetup(1, ¶ms), SyscallSucceeds()); + FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params)); } -TEST(IoUringTest, ParamsNonZeroResv) { +// Testing that io_uring_setup(2) fails with EINVAL on non-zero params. +TEST(IOUringTest, ParamsNonZeroResv) { struct io_uring_params params; memset(¶ms, 0, sizeof(params)); params.resv[1] = 1; - ASSERT_THAT(IoUringSetup(1, ¶ms), SyscallFailsWithErrno(EINVAL)); + ASSERT_THAT(IOUringSetup(1, ¶ms), SyscallFailsWithErrno(EINVAL)); +} + +// Testing that io_uring_setup(2) fails with EINVAL on unsupported flags. +TEST(IOUringTest, UnsupportedFlags) { + if (IsRunningOnGvisor()) { + struct io_uring_params params; + memset(¶ms, 0, sizeof(params)); + params.flags |= IORING_SETUP_SQPOLL; + ASSERT_THAT(IOUringSetup(1, ¶ms), SyscallFailsWithErrno(EINVAL)); + } +} + +// Testing that both mmap and munmap calls succeed and subsequent access to +// unmapped memory results in SIGSEGV. +TEST(IOUringTest, MMapMUnMapWork) { + struct io_uring_params params; + FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params)); + + void *ptr = nullptr; + int sring_sz = params.sq_off.array + params.sq_entries * sizeof(unsigned); + + ptr = mmap(0, sring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, + iouringfd.get(), IORING_OFF_SQ_RING); + + EXPECT_NE(ptr, MAP_FAILED); + + ASSERT_THAT(munmap(ptr, sring_sz), SyscallSucceeds()); + + EXPECT_EXIT(*reinterpret_cast(ptr) = 42, + ::testing::KilledBySignal(SIGSEGV), ""); +} + +// Testing that both mmap fails with EINVAL when an invalid offset is passed. +TEST(IOUringTest, MMapWrongOffset) { + struct io_uring_params params; + FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params)); + + int sring_sz = params.sq_off.array + params.sq_entries * sizeof(unsigned); + + EXPECT_THAT(reinterpret_cast( + mmap(0, sring_sz, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE, iouringfd.get(), 66)), + SyscallFailsWithErrno(EINVAL)); +} + +// Testing that mmap() handles all three IO_URING-specific offsets and that +// returned addresses are page aligned. +TEST(IOUringTest, MMapOffsets) { + struct io_uring_params params; + FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params)); + + void *sqPtr = nullptr; + void *cqPtr = nullptr; + void *sqePtr = nullptr; + + int sring_sz = params.sq_off.array + params.sq_entries * sizeof(unsigned); + int cring_sz = params.cq_off.cqes + params.cq_entries * 32; + + sqPtr = mmap(0, sring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, + iouringfd.get(), IORING_OFF_SQ_RING); + + cqPtr = mmap(0, cring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, + iouringfd.get(), IORING_OFF_CQ_RING); + + sqePtr = mmap(0, 64, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, + iouringfd.get(), IORING_OFF_SQES); + + EXPECT_NE(sqPtr, MAP_FAILED); + EXPECT_NE(cqPtr, MAP_FAILED); + EXPECT_NE(sqePtr, MAP_FAILED); + + EXPECT_EQ((uintptr_t)sqPtr % kPageSize, 0); + EXPECT_EQ((uintptr_t)cqPtr % kPageSize, 0); + EXPECT_EQ((uintptr_t)sqePtr % kPageSize, 0); + + ASSERT_THAT(munmap(sqPtr, sring_sz), SyscallSucceeds()); + ASSERT_THAT(munmap(cqPtr, cring_sz), SyscallSucceeds()); + ASSERT_THAT(munmap(sqePtr, 64), SyscallSucceeds()); +} + +// Testing that io_uring_params are populated with correct values. +TEST(IOUringTest, ReturnedParamsValues) { + if (IsRunningOnGvisor()) { + struct io_uring_params params; + FileDescriptor iouringfd = + ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params)); + + EXPECT_EQ(params.sq_entries, 1); + EXPECT_EQ(params.cq_entries, 2); + + EXPECT_EQ(params.sq_off.head, 0); + EXPECT_EQ(params.sq_off.tail, 64); + EXPECT_EQ(params.sq_off.ring_mask, 256); + EXPECT_EQ(params.sq_off.ring_entries, 264); + EXPECT_EQ(params.sq_off.flags, 276); + EXPECT_EQ(params.sq_off.dropped, 272); + EXPECT_EQ(params.sq_off.array, 384); + + EXPECT_EQ(params.cq_off.head, 128); + EXPECT_EQ(params.cq_off.tail, 192); + EXPECT_EQ(params.cq_off.ring_mask, 260); + EXPECT_EQ(params.cq_off.ring_entries, 268); + EXPECT_EQ(params.cq_off.overflow, 284); + EXPECT_EQ(params.cq_off.cqes, 320); + EXPECT_EQ(params.cq_off.flags, 280); + + // gVisor should support IORING_FEAT_SINGLE_MMAP. + EXPECT_NE((params.features & IORING_FEAT_SINGLE_MMAP), 0); + } +} + +// Testing that offset of SQE indices array is cacheline aligned. +TEST(IOUringTest, SqeIndexArrayCacheAligned) { + struct io_uring_params params; + for (uint32_t i = 1; i < 10; ++i) { + FileDescriptor iouringfd = + ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(i, params)); + ASSERT_EQ(params.sq_off.array % 64, 0); + } } } // namespace diff --git a/test/util/io_uring_util.h b/test/util/io_uring_util.h index ed0f02e34..a7afa7e80 100644 --- a/test/util/io_uring_util.h +++ b/test/util/io_uring_util.h @@ -27,6 +27,15 @@ namespace testing { #define __NR_io_uring_setup 425 +// io_uring_setup(2) flags. +#define IORING_SETUP_SQPOLL (1U << 1) + +#define IORING_FEAT_SINGLE_MMAP (1U << 0) + +#define IORING_OFF_SQ_RING 0ULL +#define IORING_OFF_CQ_RING 0x8000000ULL +#define IORING_OFF_SQES 0x10000000ULL + struct io_sqring_offsets { uint32_t head; uint32_t tail; @@ -36,7 +45,7 @@ struct io_sqring_offsets { uint32_t dropped; uint32_t array; uint32_t resv1; - uint32_t resv2; + uint64_t resv2; }; struct io_cqring_offsets { @@ -46,7 +55,9 @@ struct io_cqring_offsets { uint32_t ring_entries; uint32_t overflow; uint32_t cqes; - uint64_t resv[2]; + uint32_t flags; + uint32_t resv1; + uint64_t resv2; }; struct io_uring_params { @@ -56,21 +67,22 @@ struct io_uring_params { uint32_t sq_thread_cpu; uint32_t sq_thread_idle; uint32_t features; - uint32_t resv[4]; + uint32_t wq_fd; + uint32_t resv[3]; struct io_sqring_offsets sq_off; struct io_cqring_offsets cq_off; }; // This is a wrapper for the io_uring_setup(2) system call. -inline uint32_t IoUringSetup(uint32_t entries, struct io_uring_params* params) { +inline int IOUringSetup(uint32_t entries, struct io_uring_params* params) { return syscall(__NR_io_uring_setup, entries, params); } // Returns a new iouringfd with the given number of entries. -inline PosixErrorOr NewIoUringFD(uint32_t entries) { - struct io_uring_params params; +inline PosixErrorOr NewIOUringFD( + uint32_t entries, struct io_uring_params& params) { memset(¶ms, 0, sizeof(params)); - uint32_t fd = IoUringSetup(entries, ¶ms); + int fd = IOUringSetup(entries, ¶ms); MaybeSave(); if (fd < 0) { return PosixError(errno, "io_uring_setup");