mmap() implementation for the IO_URING.

Once the user receives a file descriptor from `io_uring_setup()`, it will be
used for the subsequent `mmap()` calls. Thus, we need to add support for it in
our iouringfs.

PiperOrigin-RevId: 477318038

pkg/abi/linux/iouring.go

@@ -14,6 +14,11 @@
 
 package linux
 
+import (
+	"fmt"
+	"reflect"
+)
+
 // Constants for io_uring_setup(2). See include/uapi/linux/io_uring.h.
 const (
 	IORING_SETUP_IOPOLL     = (1 << 0)
@@ -26,6 +31,11 @@ const (
 	IORING_SETUP_SUBMIT_ALL = (1 << 7)
 )
 
+// Constants for IoUringParams.Features. See include/uapi/linux/io_uring.h.
+const (
+	IORING_FEAT_SINGLE_MMAP = (1 << 0)
+)
+
 // Constants for IO_URING. See include/uapi/linux/io_uring.h.
 const (
 	IORING_SETUP_COOP_TASKRUN = (1 << 8)
@@ -36,46 +46,62 @@ const (
 
 // Constants for IO_URING. See io_uring/io_uring.c.
 const (
-	IORING_MAX_ENTRIES = (1 << 15) // 32768
+	IORING_MAX_ENTRIES    = (1 << 15) // 32768
+	IORING_MAX_CQ_ENTRIES = (2 * IORING_MAX_ENTRIES)
 )
 
-// IoSqringOffsets implements io_sqring_offsets struct.
+// Constants for the offsets for the application to mmap the data it needs.
 // See include/uapi/linux/io_uring.h.
+const (
+	IORING_OFF_SQ_RING = 0
+	IORING_OFF_CQ_RING = 0x8000000
+	IORING_OFF_SQES    = 0x10000000
+)
+
+// IORingIndex represents SQE array indexes.
 //
 // +marshal
-type IoSqringOffsets struct {
-	Head        uint32
-	Tail        uint32
-	RingMask    uint32
-	RingEntries uint32
-	Flags       uint32
-	Dropped     uint32
-	Array       uint32
-	Resv1       uint32
-	Resv2       uint64
+type IORingIndex uint32
+
+// IOSqRingOffsets implements io_sqring_offsets struct.
+// IOSqRingOffsets represents offsets into IORings.
+// See struct io_sqring_offsets in include/uapi/linux/io_uring.h.
+//
+// +marshal
+type IOSqRingOffsets struct {
+	Head        uint32 // Offset to io_rings.sq.head
+	Tail        uint32 // Offset to io_rings.sq.tail
+	RingMask    uint32 // Offset to io_rings.sq_ring_mask
+	RingEntries uint32 // Offset to io_rings.sq_ring_entries
+	Flags       uint32 // Offset to io_rings.sq_flags
+	Dropped     uint32 // Offset to io_rings.sq_dropped
+	Array       uint32 // Offset to an array of SQE indices
+	Resv1       uint32 // Currently reserved and expected to be zero
+	Resv2       uint64 // Currently reserved and expected to be zero
 }
 
-// IoCqringOffsets implements io_cqring_offsets struct.
-// See include/uapi/linux/io_uring.h.
+// IOCqRingOffsets implements io_cqring_offsets struct.
+// IOCqRingOffsets represents offsets into IORings.
+// See struct io_cqring_offsets in include/uapi/linux/io_uring.h.
 //
 // +marshal
-type IoCqringOffsets struct {
-	Head        uint32
-	Tail        uint32
-	RingMask    uint32
-	RingEntries uint32
-	Overflow    uint32
-	Cqes        uint32
-	Flags       uint32
-	Resv1       uint32
-	Resv2       uint64
+type IOCqRingOffsets struct {
+	Head        uint32 // Offset to io_rings.cq.head
+	Tail        uint32 // Offset to io_rings.cq.tail
+	RingMask    uint32 // Offset to io_rings.cq_ring_mask
+	RingEntries uint32 // Offset to io_rings.cq_ring_entries
+	Overflow    uint32 // Offset to io_rings.cq_overflow
+	Cqes        uint32 // Offset to io_rings.cqes
+	Flags       uint32 // Offset to io_rings.cq_flags
+	Resv1       uint32 // Currently reserved and expected to be zero
+	Resv2       uint64 // Currently reserved and expected to be zero
 }
 
-// IoUringParams implements io_uring_params struct.
-// See include/uapi/linux/io_uring.h.
+// IOUringParams implements io_uring_params struct.
+// See struct io_uring_params in include/uapi/linux/io_uring.h.
 //
 // +marshal
-type IoUringParams struct {
+type IOUringParams struct {
 	SqEntries    uint32
 	CqEntries    uint32
 	Flags        uint32
@@ -84,6 +110,82 @@ type IoUringParams struct {
 	Features     uint32
 	WqFd         uint32
 	Resv         [3]uint32
-	SqOff        IoSqringOffsets
-	CqOff        IoCqringOffsets
+	SqOff        IOSqRingOffsets
+	CqOff        IOCqRingOffsets
+}
+
+// IOUringCqe implements IO completion data structure (Completion Queue Entry)
+// io_uring_cqe struct. As we don't currently support IORING_SETUP_CQE32 flag
+// its size is 16 bytes.
+// See struct io_uring_cqe in include/uapi/linux/io_uring.h.
+//
+// +marshal
+type IOUringCqe struct {
+	userData uint64
+	res      int32
+	flags    uint32
+}
+
+// IOUring implements io_uring struct.
+// See struct io_uring in io_uring/io_uring.c.
+//
+// +marshal
+type IOUring struct {
+	// Both head and tail should be cacheline aligned. And we assume that
+	// cacheline size is 64 bytes.
+	head uint32
+	_    [60]byte
+	tail uint32
+	_    [60]byte
+}
+
+// IORings implements io_rings struct.
+// This struct describes layout of the mapped region backed by the ringBuffersFile.
+// See struct io_rings in io_uring/io_uring.c.
+//
+// +marshal
+type IORings struct {
+	sq, cq                       IOUring
+	sqRingMask, cqRingMask       uint32
+	sqRingEntries, cqRingEntries uint32
+	sqDropped                    uint32
+	sqFlags                      int32
+	cqFlags                      uint32
+	cqOverflow                   uint32
+	_                            [32]byte // Padding so cqes is cacheline aligned
+	// Linux has an additional field struct io_uring_cqe cqes[], which represents
+	// a dynamic array. We don't include it here in order to enable marshalling.
+}
+
+// PreComputedIOSqRingOffsets stores precomputed values for IOSqRingOffsets.
+var PreComputedIOSqRingOffsets IOSqRingOffsets
+
+// PreComputedIOCqRingOffsets stores precomputed values for IOCqRingOffsets.
+var PreComputedIOCqRingOffsets IOCqRingOffsets
+
+func init() {
+	ioRingsType := reflect.TypeOf((*IORings)(nil)).Elem()
+	ioUringType := reflect.TypeOf((*IOUring)(nil)).Elem()
+
+	offsetof := func(ty reflect.Type, name string) uint32 {
+		if f, ok := ty.FieldByName(name); ok {
+			return uint32(f.Offset)
+		}
+		panic(fmt.Sprintf("In type %q, no field named %q", ty.Name(), name))
+	}
+
+	PreComputedIOSqRingOffsets.Head = offsetof(ioRingsType, "sq") + offsetof(ioUringType, "head")
+	PreComputedIOSqRingOffsets.Tail = offsetof(ioRingsType, "sq") + offsetof(ioUringType, "tail")
+	PreComputedIOSqRingOffsets.RingMask = offsetof(ioRingsType, "sqRingMask")
+	PreComputedIOSqRingOffsets.RingEntries = offsetof(ioRingsType, "sqRingEntries")
+	PreComputedIOSqRingOffsets.Flags = offsetof(ioRingsType, "sqFlags")
+	PreComputedIOSqRingOffsets.Dropped = offsetof(ioRingsType, "sqDropped")
+
+	PreComputedIOCqRingOffsets.Head = offsetof(ioRingsType, "cq") + offsetof(ioUringType, "head")
+	PreComputedIOCqRingOffsets.Tail = offsetof(ioRingsType, "cq") + offsetof(ioUringType, "tail")
+	PreComputedIOCqRingOffsets.RingMask = offsetof(ioRingsType, "cqRingMask")
+	PreComputedIOCqRingOffsets.RingEntries = offsetof(ioRingsType, "cqRingEntries")
+	PreComputedIOCqRingOffsets.Overflow = offsetof(ioRingsType, "cqOverflow")
+	PreComputedIOCqRingOffsets.Flags = offsetof(ioRingsType, "cqFlags")
+
 }

pkg/hostarch/BUILD

@@ -19,6 +19,7 @@ go_test(
     size = "small",
     srcs = [
         "addr_range_seq_test.go",
+        "addr_test.go",
     ],
     library = ":hostarch",
 )

pkg/hostarch/addr.go

@@ -133,3 +133,16 @@ func ToPages(x uint64) (uint64, bool) {
 	}
 	return xRoundedUp / PageSize, true
 }
+
+// CacheLineRoundDown returns the offset rounded down to the nearest cache line boundary.
+func CacheLineRoundDown(x uint64) uint64 {
+	return x & ^uint64(CacheLineSize-1)
+}
+
+// CacheLineRoundUp returns the offset rounded up to the nearest cache line boundary. ok is true iff
+// rounding up did not wrap around.
+func CacheLineRoundUp(x uint64) (val uint64, ok bool) {
+	val = CacheLineRoundDown(x + uint64(CacheLineSize-1))
+	ok = val >= x
+	return
+}

pkg/hostarch/addr_test.go

@@ -0,0 +1,71 @@
+// Copyright 2022 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hostarch
+
+import (
+	"fmt"
+	"math"
+	"testing"
+)
+
+func TestCacheLineRoundUp(t *testing.T) {
+	tests := []struct {
+		input  uint64
+		output uint64
+	}{
+		{0, 0},
+		{63, 64},
+		{64, 64},
+		{65, 128},
+		{66, 128},
+		{99, 128},
+		{127, 128},
+		{128, 128},
+		{129, 192},
+		{math.MaxUint32, math.MaxUint32 + 1},
+	}
+	for i, tt := range tests {
+		t.Run(fmt.Sprintf("case-%d", i), func(t *testing.T) {
+			s, ok := CacheLineRoundUp(tt.input)
+			if s != tt.output {
+				t.Errorf("Expected %d, got %d", tt.output, s)
+			}
+			if !ok {
+				t.Errorf("Expected no wrap around, got %t. Input %d, expected %d", ok, tt.input, tt.output)
+			}
+		})
+	}
+}
+
+func TestCacheLineRoundUpWrapAround(t *testing.T) {
+	tests := []struct {
+		input  uint64
+		output uint64
+	}{
+		{math.MaxUint64 - 1, 0},
+		{math.MaxUint64, 0},
+	}
+	for i, tt := range tests {
+		t.Run(fmt.Sprintf("case-%d", i), func(t *testing.T) {
+			s, ok := CacheLineRoundUp(tt.input)
+			if s != tt.output {
+				t.Errorf("Expected %d, got %d", tt.output, s)
+			}
+			if ok {
+				t.Errorf("Expected wrap around, got %t. Input %d, expected %d", ok, tt.input, tt.output)
+			}
+		})
+	}
+}

pkg/hostarch/hostarch_arm64.go

@@ -33,6 +33,9 @@ const (
 	// HugePageSize is the system huge page size.
 	HugePageSize = 1 << HugePageShift
 
+	// CacheLineSize is the size of the cache line.
+	CacheLineSize = 1 << CacheLineShift
+
 	// PageShift is the binary log of the system page size.
 	PageShift = 12
 
@@ -40,6 +43,9 @@ const (
 	// Should be calculated by "PageShift + (PageShift - 3)"
 	// when multiple page size support is ready.
 	HugePageShift = 21
+
+	// CacheLineShift is the binary log of the cache line size.
+	CacheLineShift = 6
 )
 
 var (

pkg/hostarch/hostarch_x86.go

@@ -26,11 +26,17 @@ const (
 	// HugePageSize is the system huge page size.
 	HugePageSize = 1 << HugePageShift
 
+	// CacheLineSize is the size of the cache line.
+	CacheLineSize = 1 << CacheLineShift
+
 	// PageShift is the binary log of the system page size.
 	PageShift = 12
 
 	// HugePageShift is the binary log of the system huge page size.
 	HugePageShift = 21
+
+	// CacheLineShift is the binary log of the cache line size.
+	CacheLineShift = 6
 )
 
 var (

pkg/sentry/fsimpl/iouringfs/BUILD

@@ -1,4 +1,4 @@
-load("//tools:defs.bzl", "go_library")
+load("//tools:defs.bzl", "go_library", "go_test")
 
 licenses(["notice"])
 
@@ -9,7 +9,18 @@ go_library(
     deps = [
         "//pkg/abi/linux",
         "//pkg/context",
+        "//pkg/errors/linuxerr",
         "//pkg/hostarch",
+        "//pkg/sentry/memmap",
+        "//pkg/sentry/pgalloc",
+        "//pkg/sentry/usage",
         "//pkg/sentry/vfs",
     ],
 )
+
+go_test(
+    name = "iouringfs_test",
+    size = "small",
+    srcs = ["iouringfs_test.go"],
+    library = ":iouringfs",
+)

pkg/sentry/fsimpl/iouringfs/iouringfs.go

@@ -17,46 +17,105 @@
 package iouringfs
 
 import (
+	"fmt"
+
 	"gvisor.dev/gvisor/pkg/abi/linux"
 	"gvisor.dev/gvisor/pkg/context"
+	"gvisor.dev/gvisor/pkg/errors/linuxerr"
 	"gvisor.dev/gvisor/pkg/hostarch"
+	"gvisor.dev/gvisor/pkg/sentry/memmap"
+	"gvisor.dev/gvisor/pkg/sentry/pgalloc"
+	"gvisor.dev/gvisor/pkg/sentry/usage"
 	"gvisor.dev/gvisor/pkg/sentry/vfs"
 )
 
-// IoUring  implements io_uring struct. See io_uring/io_uring.c.
-type IoUring struct {
-	head uint32
-	tail uint32
-}
-
-// IoUringCqe implements IO completion data structure (Completion Queue Entry)
-// io_uring_cqe struct. See include/uapi/linux/io_uring.h.
-type IoUringCqe struct {
-	userData uint64
-	res      int16
-	flags    uint32
-	bigCqe   *uint64
-}
-
 // FileDescription implements vfs.FileDescriptionImpl for file-based IO_URING.
 // It is based on io_rings struct. See io_uring/io_uring.c.
 //
 // +stateify savable
-type FileDescription struct {
+type fileDescription struct {
 	vfsfd vfs.FileDescription
 	vfs.FileDescriptionDefaultImpl
 	vfs.DentryMetadataFileDescriptionImpl
 	vfs.NoLockFD
+
+	rbmf  ringsBufferFile
+	sqemf sqEntriesFile
 }
 
-var _ vfs.FileDescriptionImpl = (*FileDescription)(nil)
+var _ vfs.FileDescriptionImpl = (*fileDescription)(nil)
+
+func roundUpPowerOfTwo(n uint32) (uint32, bool) {
+	if n > (1 << 31) {
+		return 0, false
+	}
+	result := uint32(1)
+	for result < n {
+		result = result << 1
+	}
+	return result, true
+}
 
 // New creates a new iouring fd.
-func New(ctx context.Context, vfsObj *vfs.VirtualFilesystem, entries uint32, params *linux.IoUringParams, paramsUser hostarch.Addr) (*vfs.FileDescription, error) {
+func New(ctx context.Context, vfsObj *vfs.VirtualFilesystem, entries uint32, params *linux.IOUringParams) (*vfs.FileDescription, error) {
+	if entries > linux.IORING_MAX_ENTRIES {
+		return nil, linuxerr.EINVAL
+	}
+
 	vd := vfsObj.NewAnonVirtualDentry("[io_uring]")
 	defer vd.DecRef(ctx)
 
-	iouringfd := &FileDescription{}
+	mfp := pgalloc.MemoryFileProviderFromContext(ctx)
+	if mfp == nil {
+		panic(fmt.Sprintf("context.Context %T lacks non-nil value for key %T", ctx, pgalloc.CtxMemoryFileProvider))
+	}
+
+	numSqEntries, ok := roundUpPowerOfTwo(entries)
+	if !ok {
+		return nil, linuxerr.EOVERFLOW
+	}
+	var numCqEntries uint32
+	if params.Flags&linux.IORING_SETUP_CQSIZE != 0 {
+		if params.CqEntries > linux.IORING_MAX_CQ_ENTRIES {
+			return nil, linuxerr.EINVAL
+		}
+		numCqEntries = params.CqEntries
+	} else {
+		numCqEntries = 2 * numSqEntries
+	}
+
+	// Allocate enough space to store the `struct io_rings` plus a given number of indexes
+	// corresponding to the number of SQEs.
+	ioRingsWithCqesSize := uint32((*linux.IORings)(nil).SizeBytes()) +
+		numCqEntries*uint32((*linux.IOUringCqe)(nil).SizeBytes())
+	ringsBufferSize := uint64(ioRingsWithCqesSize +
+		numSqEntries*uint32((*linux.IORingIndex)(nil).SizeBytes()))
+	ringsBufferSize = uint64(hostarch.Addr(ringsBufferSize).MustRoundUp())
+
+	rbfr, err := mfp.MemoryFile().Allocate(ringsBufferSize, pgalloc.AllocOpts{Kind: usage.Anonymous})
+	if err != nil {
+		return nil, linuxerr.ENOMEM
+	}
+
+	// Allocate enough space to store the given number of submission queue entries.
+	sqEntriesSize := uint64(numSqEntries * 64)
+	sqEntriesSize = uint64(hostarch.Addr(sqEntriesSize).MustRoundUp())
+
+	sqefr, err := mfp.MemoryFile().Allocate(sqEntriesSize, pgalloc.AllocOpts{Kind: usage.Anonymous})
+	if err != nil {
+		return nil, linuxerr.ENOMEM
+	}
+
+	iouringfd := &fileDescription{
+		rbmf: ringsBufferFile{
+			mf: mfp.MemoryFile(),
+			fr: rbfr,
+		},
+		sqemf: sqEntriesFile{
+			mf: mfp.MemoryFile(),
+			fr: sqefr,
+		},
+	}
 
 	// iouringfd is always set up with read/write mode.
 	// See io_uring/io_uring.c:io_uring_install_fd().
@@ -69,9 +128,156 @@ func New(ctx context.Context, vfsObj *vfs.VirtualFilesystem, entries uint32, par
 		return nil, err
 	}
 
+	params.SqEntries = numSqEntries
+	params.CqEntries = numCqEntries
+
+	arrayOffset := uint64(hostarch.Addr(ioRingsWithCqesSize))
+	arrayOffset, ok = hostarch.CacheLineRoundUp(arrayOffset)
+	if !ok {
+		return nil, linuxerr.EOVERFLOW
+	}
+
+	params.SqOff = linux.PreComputedIOSqRingOffsets
+	params.SqOff.Array = uint32(arrayOffset)
+
+	cqesOffset := uint64(hostarch.Addr((*linux.IORings)(nil).SizeBytes()))
+	cqesOffset, ok = hostarch.CacheLineRoundUp(cqesOffset)
+	if !ok {
+		return nil, linuxerr.EOVERFLOW
+	}
+
+	params.CqOff = linux.PreComputedIOCqRingOffsets
+	params.CqOff.Cqes = uint32(cqesOffset)
+
+	// Set features supported by the current IO_URING implementation.
+	params.Features = linux.IORING_FEAT_SINGLE_MMAP
+
 	return &iouringfd.vfsfd, nil
 }
 
 // Release implements vfs.FileDescriptionImpl.Release.
-func (iouringfd *FileDescription) Release(context.Context) {
+func (fd *fileDescription) Release(context.Context) {
+}
+
+// ConfigureMMap implements vfs.FileDescriptionImpl.ConfigureMMap.
+func (fd *fileDescription) ConfigureMMap(ctx context.Context, opts *memmap.MMapOpts) error {
+	var mf memmap.Mappable
+	switch opts.Offset {
+	case linux.IORING_OFF_SQ_RING, linux.IORING_OFF_CQ_RING:
+		mf = &fd.rbmf
+	case linux.IORING_OFF_SQES:
+		mf = &fd.sqemf
+	default:
+		return linuxerr.EINVAL
+	}
+
+	return vfs.GenericConfigureMMap(&fd.vfsfd, mf, opts)
+}
+
+// sqEntriesFile implements memmap.Mappable for SQ entries.
+type sqEntriesFile struct {
+	mf *pgalloc.MemoryFile
+	fr memmap.FileRange
+}
+
+// AddMapping implements memmap.Mappable.AddMapping.
+func (sqemf *sqEntriesFile) AddMapping(ctx context.Context, ms memmap.MappingSpace, ar hostarch.AddrRange, offset uint64, writable bool) error {
+	return nil
+}
+
+// RemoveMapping implements memmap.Mappable.RemoveMapping.
+func (sqemf *sqEntriesFile) RemoveMapping(ctx context.Context, ms memmap.MappingSpace, ar hostarch.AddrRange, offset uint64, writable bool) {
+}
+
+// CopyMapping implements memmap.Mappable.CopyMapping.
+func (sqemf *sqEntriesFile) CopyMapping(ctx context.Context, ms memmap.MappingSpace, srcAR, dstAR hostarch.AddrRange, offset uint64, writable bool) error {
+	return nil
+}
+
+// Translate implements memmap.Mappable.Translate.
+func (sqemf *sqEntriesFile) Translate(ctx context.Context, required, optional memmap.MappableRange, at hostarch.AccessType) ([]memmap.Translation, error) {
+	expectedAccessType := hostarch.AccessType{
+		Read:    true,
+		Write:   true,
+		Execute: false,
+	}
+	if at != expectedAccessType {
+		return nil, &memmap.BusError{linuxerr.EPERM}
+	}
+
+	if required.End > sqemf.fr.Length() {
+		return nil, &memmap.BusError{linuxerr.EFAULT}
+	}
+
+	if source := optional.Intersect(memmap.MappableRange{0, sqemf.fr.Length()}); source.Length() != 0 {
+		return []memmap.Translation{
+			{
+				Source: source,
+				File:   sqemf.mf,
+				Offset: sqemf.fr.Start + source.Start,
+				Perms:  at,
+			},
+		}, nil
+	}
+
+	return nil, linuxerr.EFAULT
+}
+
+// InvalidateUnsavable implements memmap.Mappable.InvalidateUnsavable.
+func (sqemf *sqEntriesFile) InvalidateUnsavable(ctx context.Context) error {
+	return nil
+}
+
+// ringBuffersFile implements memmap.Mappable for SQ and CQ ring buffers.
+type ringsBufferFile struct {
+	mf *pgalloc.MemoryFile
+	fr memmap.FileRange
+}
+
+// AddMapping implements memmap.Mappable.AddMapping.
+func (rbmf *ringsBufferFile) AddMapping(ctx context.Context, ms memmap.MappingSpace, ar hostarch.AddrRange, offset uint64, writable bool) error {
+	return nil
+}
+
+// RemoveMapping implements memmap.Mappable.RemoveMapping.
+func (rbmf *ringsBufferFile) RemoveMapping(ctx context.Context, ms memmap.MappingSpace, ar hostarch.AddrRange, offset uint64, writable bool) {
+}
+
+// CopyMapping implements memmap.Mappable.CopyMapping.
+func (rbmf *ringsBufferFile) CopyMapping(ctx context.Context, ms memmap.MappingSpace, srcAR, dstAR hostarch.AddrRange, offset uint64, writable bool) error {
+	return nil
+}
+
+// Translate implements memmap.Mappable.Translate.
+func (rbmf *ringsBufferFile) Translate(ctx context.Context, required, optional memmap.MappableRange, at hostarch.AccessType) ([]memmap.Translation, error) {
+	expectedAccessType := hostarch.AccessType{
+		Read:    true,
+		Write:   true,
+		Execute: false,
+	}
+	if at != expectedAccessType {
+		return nil, &memmap.BusError{linuxerr.EPERM}
+	}
+
+	if required.End > rbmf.fr.Length() {
+		return nil, &memmap.BusError{linuxerr.EFAULT}
+	}
+
+	if source := optional.Intersect(memmap.MappableRange{0, rbmf.fr.Length()}); source.Length() != 0 {
+		return []memmap.Translation{
+			{
+				Source: source,
+				File:   rbmf.mf,
+				Offset: rbmf.fr.Start + source.Start,
+				Perms:  at,
+			},
+		}, nil
+	}
+
+	return nil, linuxerr.EFAULT
+}
+
+// InvalidateUnsavable implements memmap.Mappable.InvalidateUnsavable.
+func (rbmf *ringsBufferFile) InvalidateUnsavable(ctx context.Context) error {
+	return nil
 }

pkg/sentry/fsimpl/iouringfs/iouringfs_test.go

@@ -0,0 +1,69 @@
+// Copyright 2021 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package iouringfs
+
+import (
+	"fmt"
+	"math"
+	"testing"
+)
+
+func TestRoundUpPowerOfTwo(t *testing.T) {
+	tests := []struct {
+		input  uint32
+		output uint32
+	}{
+		{0, 1},
+		{1, 1},
+		{2, 2},
+		{3, 4},
+		{4, 4},
+		{5, 8},
+		{6, 8},
+		{7, 8},
+		{8, 8},
+		{1 << 31, 2147483648},
+		{1<<31 - 1, 2147483648},
+	}
+	for i, tt := range tests {
+		t.Run(fmt.Sprintf("case-%d", i), func(t *testing.T) {
+			s, ok := roundUpPowerOfTwo(tt.input)
+			if s != tt.output {
+				t.Errorf("Expected %d, got %d", tt.output, s)
+			}
+			if !ok {
+				t.Errorf("Expected no error, got %t. Input %d, expected %d", ok, tt.input, tt.output)
+			}
+		})
+	}
+}
+
+func TestRoundUpPowerOfTwoOverflow(t *testing.T) {
+	tests := []struct {
+		input  uint32
+		output uint32
+	}{
+		{1<<31 + 1, 0},
+		{math.MaxUint32, 0},
+	}
+	for i, tt := range tests {
+		t.Run(fmt.Sprintf("case-%d", i), func(t *testing.T) {
+			s, ok := roundUpPowerOfTwo(tt.input)
+			if s != tt.output || ok {
+				t.Errorf("Expected value %d and overflow, got %d and %t", tt.output, s, ok)
+			}
+		})
+	}
+}

pkg/sentry/syscalls/linux/vfs2/iouringfs.go

@@ -22,11 +22,15 @@ import (
 	"gvisor.dev/gvisor/pkg/sentry/kernel"
 )
 
-// IoUringSetup implements linux syscall io_uring_setup(2).
-func IoUringSetup(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel.SyscallControl, error) {
+// IOUringSetup implements linux syscall io_uring_setup(2).
+func IOUringSetup(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel.SyscallControl, error) {
 	entries := uint32(args[0].Uint())
 	paramsAddr := args[1].Pointer()
-	var params linux.IoUringParams
+	var params linux.IOUringParams
+
+	if entries == 0 {
+		return 0, nil, linuxerr.EINVAL
+	}
 	if _, err := params.CopyIn(t, paramsAddr); err != nil {
 		return 0, nil, err
 	}
@@ -37,8 +41,16 @@ func IoUringSetup(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel.
 		}
 	}
 
+	// List of currently supported flags in our IO_URING implementation.
+	const supportedFlags = linux.IORING_SETUP_IOPOLL
+
+	// Since we don't implement everything, we fail explicitly on flags that are unimplemented.
+	if params.Flags|supportedFlags != supportedFlags {
+		return 0, nil, linuxerr.EINVAL
+	}
+
 	vfsObj := t.Kernel().VFS()
-	iouringfd, err := iouringfs.New(t, vfsObj, entries, &params, paramsAddr)
+	iouringfd, err := iouringfs.New(t, vfsObj, entries, &params)
 
 	if err != nil {
 		return 0, nil, err
@@ -54,5 +66,9 @@ func IoUringSetup(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel.
 		return 0, nil, err
 	}
 
+	if _, err := params.CopyOut(t, paramsAddr); err != nil {
+		return 0, nil, err
+	}
+
 	return uintptr(fd), nil, nil
 }

pkg/sentry/syscalls/linux/vfs2/vfs2.go

@@ -163,7 +163,7 @@ func Override() {
 	s.Table[327] = syscalls.Supported("preadv2", Preadv2)
 	s.Table[328] = syscalls.Supported("pwritev2", Pwritev2)
 	s.Table[332] = syscalls.Supported("statx", Statx)
-	s.Table[425] = syscalls.PartiallySupported("io_uring_setup", IoUringSetup, "Not all flags and functionality supported.", nil)
+	s.Table[425] = syscalls.PartiallySupported("io_uring_setup", IOUringSetup, "Not all flags and functionality supported.", nil)
 	s.Table[436] = syscalls.Supported("close_range", CloseRange)
 	s.Table[439] = syscalls.Supported("faccessat2", Faccessat2)
 	s.Table[441] = syscalls.Supported("epoll_pwait2", EpollPwait2)
@@ -280,7 +280,7 @@ func Override() {
 	s.Table[286] = syscalls.Supported("preadv2", Preadv2)
 	s.Table[287] = syscalls.Supported("pwritev2", Pwritev2)
 	s.Table[291] = syscalls.Supported("statx", Statx)
-	s.Table[425] = syscalls.PartiallySupported("io_uring_setup", IoUringSetup, "Not all flags and functionality supported.", nil)
+	s.Table[425] = syscalls.PartiallySupported("io_uring_setup", IOUringSetup, "Not all flags and functionality supported.", nil)
 	s.Table[436] = syscalls.Supported("close_range", CloseRange)
 	s.Table[439] = syscalls.Supported("faccessat2", Faccessat2)
 	s.Table[441] = syscalls.Supported("epoll_pwait2", EpollPwait2)

test/syscalls/BUILD

@@ -247,6 +247,8 @@ syscall_test(
 )
 
 syscall_test(
+    # Temporarily added due to intermittent ENOMEM failures. See b/216213621.
+    tags = ["notap"],
     test = "//test/syscalls/linux:iouring_test",
 )
 

test/syscalls/linux/io_uring.cc

@@ -18,10 +18,13 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/epoll.h>
+#include <sys/mman.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
 
+#include <cstdint>
+
 #include "gtest/gtest.h"
 #include "test/util/io_uring_util.h"
 #include "test/util/test_util.h"
@@ -31,21 +34,137 @@ namespace testing {
 
 namespace {
 
-TEST(IoUringTest, ValidFD) {
-  FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIoUringFD(1));
-}
-
-TEST(IoUringTest, SetUp) {
+// Testing that io_uring_setup(2) successfully returns a valid file descriptor.
+TEST(IOUringTest, ValidFD) {
   struct io_uring_params params;
-  memset(&params, 0, sizeof(params));
-  ASSERT_THAT(IoUringSetup(1, &params), SyscallSucceeds());
+  FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params));
 }
 
-TEST(IoUringTest, ParamsNonZeroResv) {
+// Testing that io_uring_setup(2) fails with EINVAL on non-zero params.
+TEST(IOUringTest, ParamsNonZeroResv) {
   struct io_uring_params params;
   memset(&params, 0, sizeof(params));
   params.resv[1] = 1;
-  ASSERT_THAT(IoUringSetup(1, &params), SyscallFailsWithErrno(EINVAL));
+  ASSERT_THAT(IOUringSetup(1, &params), SyscallFailsWithErrno(EINVAL));
+}
+
+// Testing that io_uring_setup(2) fails with EINVAL on unsupported flags.
+TEST(IOUringTest, UnsupportedFlags) {
+  if (IsRunningOnGvisor()) {
+    struct io_uring_params params;
+    memset(&params, 0, sizeof(params));
+    params.flags |= IORING_SETUP_SQPOLL;
+    ASSERT_THAT(IOUringSetup(1, &params), SyscallFailsWithErrno(EINVAL));
+  }
+}
+
+// Testing that both mmap and munmap calls succeed and subsequent access to
+// unmapped memory results in SIGSEGV.
+TEST(IOUringTest, MMapMUnMapWork) {
+  struct io_uring_params params;
+  FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params));
+
+  void *ptr = nullptr;
+  int sring_sz = params.sq_off.array + params.sq_entries * sizeof(unsigned);
+
+  ptr = mmap(0, sring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
+             iouringfd.get(), IORING_OFF_SQ_RING);
+
+  EXPECT_NE(ptr, MAP_FAILED);
+
+  ASSERT_THAT(munmap(ptr, sring_sz), SyscallSucceeds());
+
+  EXPECT_EXIT(*reinterpret_cast<volatile int *>(ptr) = 42,
+              ::testing::KilledBySignal(SIGSEGV), "");
+}
+
+// Testing that both mmap fails with EINVAL when an invalid offset is passed.
+TEST(IOUringTest, MMapWrongOffset) {
+  struct io_uring_params params;
+  FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params));
+
+  int sring_sz = params.sq_off.array + params.sq_entries * sizeof(unsigned);
+
+  EXPECT_THAT(reinterpret_cast<uintptr_t>(
+                  mmap(0, sring_sz, PROT_READ | PROT_WRITE,
+                       MAP_SHARED | MAP_POPULATE, iouringfd.get(), 66)),
+              SyscallFailsWithErrno(EINVAL));
+}
+
+// Testing that mmap() handles all three IO_URING-specific offsets and that
+// returned addresses are page aligned.
+TEST(IOUringTest, MMapOffsets) {
+  struct io_uring_params params;
+  FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params));
+
+  void *sqPtr = nullptr;
+  void *cqPtr = nullptr;
+  void *sqePtr = nullptr;
+
+  int sring_sz = params.sq_off.array + params.sq_entries * sizeof(unsigned);
+  int cring_sz = params.cq_off.cqes + params.cq_entries * 32;
+
+  sqPtr = mmap(0, sring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
+               iouringfd.get(), IORING_OFF_SQ_RING);
+
+  cqPtr = mmap(0, cring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
+               iouringfd.get(), IORING_OFF_CQ_RING);
+
+  sqePtr = mmap(0, 64, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
+                iouringfd.get(), IORING_OFF_SQES);
+
+  EXPECT_NE(sqPtr, MAP_FAILED);
+  EXPECT_NE(cqPtr, MAP_FAILED);
+  EXPECT_NE(sqePtr, MAP_FAILED);
+
+  EXPECT_EQ((uintptr_t)sqPtr % kPageSize, 0);
+  EXPECT_EQ((uintptr_t)cqPtr % kPageSize, 0);
+  EXPECT_EQ((uintptr_t)sqePtr % kPageSize, 0);
+
+  ASSERT_THAT(munmap(sqPtr, sring_sz), SyscallSucceeds());
+  ASSERT_THAT(munmap(cqPtr, cring_sz), SyscallSucceeds());
+  ASSERT_THAT(munmap(sqePtr, 64), SyscallSucceeds());
+}
+
+// Testing that io_uring_params are populated with correct values.
+TEST(IOUringTest, ReturnedParamsValues) {
+  if (IsRunningOnGvisor()) {
+    struct io_uring_params params;
+    FileDescriptor iouringfd =
+        ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params));
+
+    EXPECT_EQ(params.sq_entries, 1);
+    EXPECT_EQ(params.cq_entries, 2);
+
+    EXPECT_EQ(params.sq_off.head, 0);
+    EXPECT_EQ(params.sq_off.tail, 64);
+    EXPECT_EQ(params.sq_off.ring_mask, 256);
+    EXPECT_EQ(params.sq_off.ring_entries, 264);
+    EXPECT_EQ(params.sq_off.flags, 276);
+    EXPECT_EQ(params.sq_off.dropped, 272);
+    EXPECT_EQ(params.sq_off.array, 384);
+
+    EXPECT_EQ(params.cq_off.head, 128);
+    EXPECT_EQ(params.cq_off.tail, 192);
+    EXPECT_EQ(params.cq_off.ring_mask, 260);
+    EXPECT_EQ(params.cq_off.ring_entries, 268);
+    EXPECT_EQ(params.cq_off.overflow, 284);
+    EXPECT_EQ(params.cq_off.cqes, 320);
+    EXPECT_EQ(params.cq_off.flags, 280);
+
+    // gVisor should support IORING_FEAT_SINGLE_MMAP.
+    EXPECT_NE((params.features & IORING_FEAT_SINGLE_MMAP), 0);
+  }
+}
+
+// Testing that offset of SQE indices array is cacheline aligned.
+TEST(IOUringTest, SqeIndexArrayCacheAligned) {
+  struct io_uring_params params;
+  for (uint32_t i = 1; i < 10; ++i) {
+    FileDescriptor iouringfd =
+        ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(i, params));
+    ASSERT_EQ(params.sq_off.array % 64, 0);
+  }
 }
 
 }  // namespace

test/util/io_uring_util.h

@@ -27,6 +27,15 @@ namespace testing {
 
 #define __NR_io_uring_setup 425
 
+// io_uring_setup(2) flags.
+#define IORING_SETUP_SQPOLL (1U << 1)
+
+#define IORING_FEAT_SINGLE_MMAP (1U << 0)
+
+#define IORING_OFF_SQ_RING 0ULL
+#define IORING_OFF_CQ_RING 0x8000000ULL
+#define IORING_OFF_SQES 0x10000000ULL
+
 struct io_sqring_offsets {
   uint32_t head;
   uint32_t tail;
@@ -36,7 +45,7 @@ struct io_sqring_offsets {
   uint32_t dropped;
   uint32_t array;
   uint32_t resv1;
-  uint32_t resv2;
+  uint64_t resv2;
 };
 
 struct io_cqring_offsets {
@@ -46,7 +55,9 @@ struct io_cqring_offsets {
   uint32_t ring_entries;
   uint32_t overflow;
   uint32_t cqes;
-  uint64_t resv[2];
+  uint32_t flags;
+  uint32_t resv1;
+  uint64_t resv2;
 };
 
 struct io_uring_params {
@@ -56,21 +67,22 @@ struct io_uring_params {
   uint32_t sq_thread_cpu;
   uint32_t sq_thread_idle;
   uint32_t features;
-  uint32_t resv[4];
+  uint32_t wq_fd;
+  uint32_t resv[3];
   struct io_sqring_offsets sq_off;
   struct io_cqring_offsets cq_off;
 };
 
 // This is a wrapper for the io_uring_setup(2) system call.
-inline uint32_t IoUringSetup(uint32_t entries, struct io_uring_params* params) {
+inline int IOUringSetup(uint32_t entries, struct io_uring_params* params) {
   return syscall(__NR_io_uring_setup, entries, params);
 }
 
 // Returns a new iouringfd with the given number of entries.
-inline PosixErrorOr<FileDescriptor> NewIoUringFD(uint32_t entries) {
-  struct io_uring_params params;
+inline PosixErrorOr<FileDescriptor> NewIOUringFD(
+    uint32_t entries, struct io_uring_params& params) {
   memset(&params, 0, sizeof(params));
-  uint32_t fd = IoUringSetup(entries, &params);
+  int fd = IOUringSetup(entries, &params);
   MaybeSave();
   if (fd < 0) {
     return PosixError(errno, "io_uring_setup");