io_uring_enter with NOP implementation for the IO_URING.

With io_uring_setup() and mmap() we can proceed with implementing a simplest
operation, namely, NOP. Once we make sure that we can send this command and
receive a response from the gVisor, we can move forward to read operation.

PiperOrigin-RevId: 485106019

pkg/abi/linux/iouring.go

@@ -116,9 +116,9 @@ type IOUringParams struct {
 //
 // +marshal
 type IOUringCqe struct {
-	userData uint64
-	res      int32
-	flags    uint32
+	UserData uint64
+	Res      int32
+	Flags    uint32
 }
 
 // IOUring implements io_uring struct.
@@ -128,9 +128,9 @@ type IOUringCqe struct {
 type IOUring struct {
 	// Both head and tail should be cacheline aligned. And we assume that
 	// cacheline size is 64 bytes.
-	head uint32
+	Head uint32
 	_    [60]byte
-	tail uint32
+	Tail uint32
 	_    [60]byte
 }
 
@@ -140,14 +140,37 @@ type IOUring struct {
 //
 // +marshal
 type IORings struct {
-	sq, cq                       IOUring
-	sqRingMask, cqRingMask       uint32
-	sqRingEntries, cqRingEntries uint32
+	Sq, Cq                       IOUring
+	SqRingMask, CqRingMask       uint32
+	SqRingEntries, CqRingEntries uint32
 	sqDropped                    uint32
 	sqFlags                      int32
 	cqFlags                      uint32
-	cqOverflow                   uint32
+	CqOverflow                   uint32
 	_                            [32]byte // Padding so cqes is cacheline aligned
 	// Linux has an additional field struct io_uring_cqe cqes[], which represents
 	// a dynamic array. We don't include it here in order to enable marshalling.
 }
+
+// IOUringSqe implements io_uring_sqe struct.
+// This struct represents IO submission data structure (Submission Queue Entry). As we don't yet
+// support IORING_SETUP_SQE128 flag, its size is 64 bytes with no extra padding at the end.
+// See include/uapi/linux/io_uring.h.
+//
+// +marshal
+type IOUringSqe struct {
+	Opcode              uint8
+	Flags               uint8
+	ioPrio              uint16
+	fd                  int32
+	offOrAddrOrCmdOp    uint64
+	addrOrSpliceOff     uint64
+	len                 uint32
+	specialFlags        uint32
+	UserData            uint64
+	bufIndexOrGroup     uint16
+	personality         uint16
+	spliceFDOrFileIndex int32
+	addr3               uint64
+	_                   uint64
+}

pkg/abi/linux/iouring_offsets.tmpl

@@ -17,10 +17,10 @@ package linux
 // PreComputedIOSqRingOffsets returns precomputed values for IOSqRingOffsets.
 func PreComputedIOSqRingOffsets() IOSqRingOffsets {
 	return IOSqRingOffsets{
-		Head: {{ .IORings.sq.Offset }} + {{ .IOUring.head.Offset }},
-		Tail: {{ .IORings.sq.Offset }} + {{ .IOUring.tail.Offset }},
-		RingMask: {{ .IORings.sqRingMask.Offset }},
-		RingEntries: {{ .IORings.sqRingEntries.Offset }},
+		Head: {{ .IORings.Sq.Offset }} + {{ .IOUring.Head.Offset }},
+		Tail: {{ .IORings.Sq.Offset }} + {{ .IOUring.Tail.Offset }},
+		RingMask: {{ .IORings.SqRingMask.Offset }},
+		RingEntries: {{ .IORings.SqRingEntries.Offset }},
 		Flags: {{ .IORings.sqFlags.Offset }},
 		Dropped: {{ .IORings.sqDropped.Offset }},
 	}
@@ -29,11 +29,11 @@ func PreComputedIOSqRingOffsets() IOSqRingOffsets {
 // PreComputedIOCqRingOffsets returns precomputed values for IOCqRingOffsets.
 func PreComputedIOCqRingOffsets() IOCqRingOffsets {
 	return IOCqRingOffsets {
-		Head: {{ .IORings.cq.Offset }} + {{ .IOUring.head.Offset }},
-		Tail: {{ .IORings.cq.Offset }} + {{ .IOUring.tail.Offset }},
-		RingMask: {{ .IORings.cqRingMask.Offset }},
-		RingEntries: {{ .IORings.cqRingEntries.Offset }},
-		Overflow: {{ .IORings.cqOverflow.Offset }},
+		Head: {{ .IORings.Cq.Offset }} + {{ .IOUring.Head.Offset }},
+		Tail: {{ .IORings.Cq.Offset }} + {{ .IOUring.Tail.Offset }},
+		RingMask: {{ .IORings.CqRingMask.Offset }},
+		RingEntries: {{ .IORings.CqRingEntries.Offset }},
+		Overflow: {{ .IORings.CqOverflow.Offset }},
 		Flags: {{ .IORings.cqFlags.Offset }},
 	}
 }

pkg/sentry/fsimpl/iouringfs/BUILD

@@ -8,9 +8,11 @@ go_library(
     visibility = ["//pkg/sentry:internal"],
     deps = [
         "//pkg/abi/linux",
+        "//pkg/atomicbitops",
         "//pkg/context",
         "//pkg/errors/linuxerr",
         "//pkg/hostarch",
+        "//pkg/safemem",
         "//pkg/sentry/memmap",
         "//pkg/sentry/pgalloc",
         "//pkg/sentry/usage",

pkg/sentry/fsimpl/iouringfs/iouringfs.go

@@ -13,16 +13,25 @@
 // limitations under the License.
 
 // Package iouringfs provides a filesystem implementation for IO_URING basing
-// it on anonfs.
+// it on anonfs. Currently, we don't support neither IOPOLL nor SQPOLL modes.
+// Thus, user needs to set up IO_URING first with io_uring_setup(2) syscall and
+// then issue submission request using io_uring_enter(2).
+//
+// Another important note, as of now, we don't support deferred CQE. In other
+// words, the size of the backlogged set of CQE is zero. Whenever, completion
+// queue ring buffer is full, we drop the subsequent completion queue entries.
 package iouringfs
 
 import (
 	"fmt"
+	"sync"
 
 	"gvisor.dev/gvisor/pkg/abi/linux"
+	"gvisor.dev/gvisor/pkg/atomicbitops"
 	"gvisor.dev/gvisor/pkg/context"
 	"gvisor.dev/gvisor/pkg/errors/linuxerr"
 	"gvisor.dev/gvisor/pkg/hostarch"
+	"gvisor.dev/gvisor/pkg/safemem"
 	"gvisor.dev/gvisor/pkg/sentry/memmap"
 	"gvisor.dev/gvisor/pkg/sentry/pgalloc"
 	"gvisor.dev/gvisor/pkg/sentry/usage"
@@ -33,7 +42,7 @@ import (
 // It is based on io_rings struct. See io_uring/io_uring.c.
 //
 // +stateify savable
-type fileDescription struct {
+type FileDescription struct {
 	vfsfd vfs.FileDescription
 	vfs.FileDescriptionDefaultImpl
 	vfs.DentryMetadataFileDescriptionImpl
@@ -41,9 +50,16 @@ type fileDescription struct {
 
 	rbmf  ringsBufferFile
 	sqemf sqEntriesFile
+
+	// mu protects the fields below.
+	mu sync.Mutex `state:"nosave"`
+
+	ioRings *safemem.BlockSeq
+	sqes    *safemem.BlockSeq
+	cqes    *safemem.BlockSeq
 }
 
-var _ vfs.FileDescriptionImpl = (*fileDescription)(nil)
+var _ vfs.FileDescriptionImpl = (*FileDescription)(nil)
 
 func roundUpPowerOfTwo(n uint32) (uint32, bool) {
 	if n > (1 << 31) {
@@ -98,15 +114,14 @@ func New(ctx context.Context, vfsObj *vfs.VirtualFilesystem, entries uint32, par
 	}
 
 	// Allocate enough space to store the given number of submission queue entries.
-	sqEntriesSize := uint64(numSqEntries * 64)
+	sqEntriesSize := uint64(numSqEntries * uint32((*linux.IOUringSqe)(nil).SizeBytes()))
 	sqEntriesSize = uint64(hostarch.Addr(sqEntriesSize).MustRoundUp())
-
 	sqefr, err := mfp.MemoryFile().Allocate(sqEntriesSize, pgalloc.AllocOpts{Kind: usage.Anonymous})
 	if err != nil {
 		return nil, linuxerr.ENOMEM
 	}
 
-	iouringfd := &fileDescription{
+	iouringfd := &FileDescription{
 		rbmf: ringsBufferFile{
 			mf: mfp.MemoryFile(),
 			fr: rbfr,
@@ -152,15 +167,135 @@ func New(ctx context.Context, vfsObj *vfs.VirtualFilesystem, entries uint32, par
 	// Set features supported by the current IO_URING implementation.
 	params.Features = linux.IORING_FEAT_SINGLE_MMAP
 
+	if err := iouringfd.populateIORings(params); err != nil {
+		return nil, err
+	}
+
+	if err := iouringfd.cacheSqesMapping(); err != nil {
+		return nil, err
+	}
+	if err := iouringfd.cacheCqesMapping(); err != nil {
+		return nil, err
+	}
+
 	return &iouringfd.vfsfd, nil
 }
 
 // Release implements vfs.FileDescriptionImpl.Release.
-func (fd *fileDescription) Release(context.Context) {
+func (fd *FileDescription) Release(context.Context) {
+	fd.rbmf.mf.DecRef(fd.rbmf.fr)
+	fd.sqemf.mf.DecRef(fd.sqemf.fr)
+}
+
+// unmarshalIORings handles unmarshalling IORings struct considering that there could be more than
+// one block in the BlockSeq.
+func unmarshalIORings(ioRings *linux.IORings, bs *safemem.BlockSeq) error {
+	if bs.NumBlocks() == 1 && !bs.Head().NeedSafecopy() {
+		ioRings.UnmarshalBytes(bs.Head().TakeFirst((*linux.IORings)(nil).SizeBytes()).ToSlice())
+
+		return nil
+	}
+
+	buf := make([]byte, (*linux.IORings)(nil).SizeBytes())
+	cp, cperr := safemem.CopySeq(safemem.BlockSeqOf(safemem.BlockFromSafeSlice(buf)), *bs)
+	if cp == 0 {
+		return cperr
+	}
+	ioRings.UnmarshalBytes(buf)
+
+	return nil
+}
+
+// marshalIORings handles marshalling IORings struct considering that there could be more than one
+// BlockSeq.
+func marshalIORings(ioRings *linux.IORings, bs *safemem.BlockSeq) error {
+	if bs.NumBlocks() == 1 && !bs.Head().NeedSafecopy() {
+		ioRings.MarshalBytes(bs.Head().TakeFirst((*linux.IORings)(nil).SizeBytes()).ToSlice())
+	}
+
+	buf := make([]byte, (*linux.IORings)(nil).SizeBytes())
+	ioRings.MarshalBytes(buf)
+	cp, cperr := safemem.CopySeq(*bs, safemem.BlockSeqOf(safemem.BlockFromSafeSlice(buf)))
+	if cp == 0 {
+		return cperr
+	}
+
+	return nil
+}
+
+// unmarshalSqe handles unmarshalling SQE struct considering that there could be more than one block
+// in the BlockSeq.
+func unmarshalSqe(sqe *linux.IOUringSqe, sqes *safemem.BlockSeq, sqHead uint32) error {
+	sqeSize := uint32((*linux.IOUringSqe)(nil).SizeBytes())
+	if sqes.NumBlocks() == 1 && !sqes.Head().NeedSafecopy() {
+		sqe.UnmarshalBytes(sqes.Head().ToSlice()[sqHead*sqeSize : (sqHead+1)*sqeSize])
+
+		return nil
+	}
+
+	buf := make([]byte, sqes.NumBytes())
+	cp, cperr := safemem.CopySeq(safemem.BlockSeqOf(safemem.BlockFromSafeSlice(buf[sqHead*sqeSize:(sqHead+1)*sqeSize])), *sqes)
+	if cp == 0 {
+		return cperr
+	}
+	sqe.UnmarshalBytes(buf)
+
+	return nil
+}
+
+// populateIORings populates IORings struct backed by the allocated memory.
+func (fd *FileDescription) populateIORings(params *linux.IOUringParams) error {
+	bs, err := fd.rbmf.mf.MapInternal(fd.rbmf.fr, hostarch.ReadWrite)
+	if err != nil {
+		return err
+	}
+
+	fd.ioRings = &bs
+
+	var ioRings linux.IORings
+	if err = unmarshalIORings(&ioRings, &bs); err != nil {
+		return err
+	}
+
+	ioRings.SqRingMask = params.SqEntries - 1
+	ioRings.CqRingMask = params.CqEntries - 1
+	ioRings.SqRingEntries = params.SqEntries
+	ioRings.CqRingEntries = params.CqEntries
+
+	if err = marshalIORings(&ioRings, &bs); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// cacheSqesMapping caches the beginning of an area for the SQEs backed by the allocated memory.
+func (fd *FileDescription) cacheSqesMapping() error {
+	bs, err := fd.sqemf.mf.MapInternal(fd.sqemf.fr, hostarch.ReadWrite)
+	if err != nil {
+		return err
+	}
+	fd.sqes = &bs
+
+	return nil
+}
+
+// cacheCqesMapping caches the beginning of an area for the CQEs backed by the allocated memory.
+func (fd *FileDescription) cacheCqesMapping() error {
+	bs := *fd.ioRings
+	cqesOffset := uint64(hostarch.Addr((*linux.IORings)(nil).SizeBytes()))
+	cqesOffset, ok := hostarch.CacheLineRoundUp(cqesOffset)
+	if !ok {
+		return linuxerr.EOVERFLOW
+	}
+	bs = bs.DropFirst(int(cqesOffset))
+	fd.cqes = &bs
+
+	return nil
 }
 
 // ConfigureMMap implements vfs.FileDescriptionImpl.ConfigureMMap.
-func (fd *fileDescription) ConfigureMMap(ctx context.Context, opts *memmap.MMapOpts) error {
+func (fd *FileDescription) ConfigureMMap(ctx context.Context, opts *memmap.MMapOpts) error {
 	var mf memmap.Mappable
 	switch opts.Offset {
 	case linux.IORING_OFF_SQ_RING, linux.IORING_OFF_CQ_RING:
@@ -171,9 +306,115 @@ func (fd *fileDescription) ConfigureMMap(ctx context.Context, opts *memmap.MMapO
 		return linuxerr.EINVAL
 	}
 
+	opts.Offset = 0
+
 	return vfs.GenericConfigureMMap(&fd.vfsfd, mf, opts)
 }
 
+// ProcessSubmissions processes submission requests.
+func (fd *FileDescription) ProcessSubmissions(toSubmit uint32, minComplete uint32, flags uint32) (int, error) {
+	fd.mu.Lock()
+	defer fd.mu.Unlock()
+
+	var ioRings linux.IORings
+	err := fd.getIORings(&ioRings)
+	if err != nil {
+		return -1, err
+	}
+
+	sqes := fd.sqes
+	cqes := fd.cqes
+
+	var sqe linux.IOUringSqe
+	sqHead := atomicbitops.FromUint32(ioRings.Sq.Head)
+	sqTail := atomicbitops.FromUint32(ioRings.Sq.Tail)
+	cqHead := atomicbitops.FromUint32(ioRings.Cq.Head)
+	cqTail := atomicbitops.FromUint32(ioRings.Cq.Tail)
+
+	submitted := uint32(0)
+	for toSubmit > submitted {
+		sqHeadMasked := sqHead.Load() & ioRings.SqRingMask
+		cqTailMasked := cqTail.Load() & ioRings.CqRingMask
+		// This means that the submission queue is empty.
+		if sqHead == sqTail {
+			return int(submitted), nil
+		}
+
+		if err = unmarshalSqe(&sqe, sqes, sqHeadMasked); err != nil {
+			return -1, err
+		}
+
+		cqe, err := fd.ProcessSubmission(&sqe, flags)
+		if err != nil {
+			return -1, err
+		}
+		sqHead.Add(1)
+		if (cqTail.Load()-cqHead.Load())/ioRings.CqRingEntries == 1 {
+			ioRings.CqOverflow++
+		} else {
+			if err = fd.updateCq(cqes, cqe, cqTailMasked); err != nil {
+				return -1, err
+			}
+			cqTail.Add(1)
+		}
+		submitted++
+	}
+	ioRings.Sq.Head = sqHead.Load()
+	ioRings.Cq.Tail = cqTail.Load()
+
+	if err = marshalIORings(&ioRings, fd.ioRings); err != nil {
+		return -1, err
+	}
+
+	return int(submitted), nil
+}
+
+// ProcessSubmission processes a single submission request.
+func (fd *FileDescription) ProcessSubmission(sqe *linux.IOUringSqe, flags uint32) (*linux.IOUringCqe, error) {
+	switch op := sqe.Opcode; op {
+	case 0: // NOP
+		return &linux.IOUringCqe{
+			UserData: sqe.UserData,
+			Res:      0,
+			Flags:    0,
+		}, nil
+	default: // Unsupported operation
+		return &linux.IOUringCqe{
+			UserData: sqe.UserData,
+			Res:      -int32(linuxerr.EINVAL.Errno()),
+			Flags:    0,
+		}, nil
+	}
+}
+
+// updateCq updates a completion queue by adding a given completion queue entry.
+func (fd *FileDescription) updateCq(cqes *safemem.BlockSeq, cqe *linux.IOUringCqe, cqTail uint32) error {
+	cqeSize := uint32((*linux.IOUringCqe)(nil).SizeBytes())
+	if cqes.NumBlocks() == 1 && !cqes.Head().NeedSafecopy() {
+		cqe.MarshalBytes(cqes.Head().ToSlice()[cqTail*cqeSize : (cqTail+1)*cqeSize])
+
+		return nil
+	}
+
+	buf := make([]byte, cqes.NumBytes())
+	cqe.MarshalBytes(buf)
+	cp, cperr := safemem.CopySeq(cqes.DropFirst64(uint64(cqTail*cqeSize)), safemem.BlockSeqOf(safemem.BlockFromSafeSlice(buf)))
+	if cp == 0 {
+		return cperr
+	}
+
+	return nil
+}
+
+// getIORings unmarshalls IORings struct backed by the allocated memory.
+func (fd *FileDescription) getIORings(ioRings *linux.IORings) error {
+	if err := unmarshalIORings(ioRings, fd.ioRings); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // sqEntriesFile implements memmap.Mappable for SQ entries.
 type sqEntriesFile struct {
 	mf *pgalloc.MemoryFile
@@ -196,15 +437,6 @@ func (sqemf *sqEntriesFile) CopyMapping(ctx context.Context, ms memmap.MappingSp
 
 // Translate implements memmap.Mappable.Translate.
 func (sqemf *sqEntriesFile) Translate(ctx context.Context, required, optional memmap.MappableRange, at hostarch.AccessType) ([]memmap.Translation, error) {
-	expectedAccessType := hostarch.AccessType{
-		Read:    true,
-		Write:   true,
-		Execute: false,
-	}
-	if at != expectedAccessType {
-		return nil, &memmap.BusError{linuxerr.EPERM}
-	}
-
 	if required.End > sqemf.fr.Length() {
 		return nil, &memmap.BusError{linuxerr.EFAULT}
 	}
@@ -250,15 +482,6 @@ func (rbmf *ringsBufferFile) CopyMapping(ctx context.Context, ms memmap.MappingS
 
 // Translate implements memmap.Mappable.Translate.
 func (rbmf *ringsBufferFile) Translate(ctx context.Context, required, optional memmap.MappableRange, at hostarch.AccessType) ([]memmap.Translation, error) {
-	expectedAccessType := hostarch.AccessType{
-		Read:    true,
-		Write:   true,
-		Execute: false,
-	}
-	if at != expectedAccessType {
-		return nil, &memmap.BusError{linuxerr.EPERM}
-	}
-
 	if required.End > rbmf.fr.Length() {
 		return nil, &memmap.BusError{linuxerr.EFAULT}
 	}

pkg/sentry/syscalls/linux/vfs2/iouringfs.go

@@ -17,6 +17,7 @@ package vfs2
 import (
 	"gvisor.dev/gvisor/pkg/abi/linux"
 	"gvisor.dev/gvisor/pkg/errors/linuxerr"
+	"gvisor.dev/gvisor/pkg/hostarch"
 	"gvisor.dev/gvisor/pkg/sentry/arch"
 	"gvisor.dev/gvisor/pkg/sentry/fsimpl/iouringfs"
 	"gvisor.dev/gvisor/pkg/sentry/kernel"
@@ -42,7 +43,7 @@ func IOUringSetup(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel.
 	}
 
 	// List of currently supported flags in our IO_URING implementation.
-	const supportedFlags = linux.IORING_SETUP_IOPOLL
+	const supportedFlags = 0 // Currently support none
 
 	// Since we don't implement everything, we fail explicitly on flags that are unimplemented.
 	if params.Flags|supportedFlags != supportedFlags {
@@ -53,7 +54,8 @@ func IOUringSetup(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel.
 	iouringfd, err := iouringfs.New(t, vfsObj, entries, &params)
 
 	if err != nil {
-		return 0, nil, err
+		// return 0, nil, err
+		return 0, nil, linuxerr.EPERM
 	}
 	defer iouringfd.DecRef(t)
 
@@ -72,3 +74,48 @@ func IOUringSetup(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel.
 
 	return uintptr(fd), nil, nil
 }
+
+// IOUringEnter implements linux syscall io_uring_enter(2).
+func IOUringEnter(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel.SyscallControl, error) {
+	fd := int32(args[0].Int())
+	toSubmit := uint32(args[1].Uint())
+	minComplete := uint32(args[2].Uint())
+	flags := uint32(args[3].Uint())
+	sigSet := args[4].Pointer()
+
+	ret := -1
+
+	// List of currently supported flags for io_uring_enter(2).
+	const supportedFlags = 0 // Currently support none
+
+	// Since we don't implement everything, we fail explicitly on flags that are unimplemented.
+	if flags|supportedFlags != supportedFlags {
+		return uintptr(ret), nil, linuxerr.EINVAL
+	}
+
+	// Currently don't support replacing an existing signal mask.
+	if sigSet != hostarch.Addr(0) {
+		return uintptr(ret), nil, linuxerr.EFAULT
+	}
+
+	// If a user requested to submit zero SQEs, then we don't process any and return right away.
+	if toSubmit == 0 {
+		return uintptr(ret), nil, nil
+	}
+
+	file := t.GetFileVFS2(fd)
+	if file == nil {
+		return uintptr(ret), nil, linuxerr.EBADF
+	}
+	defer file.DecRef(t)
+	iouringfd, ok := file.Impl().(*iouringfs.FileDescription)
+	if !ok {
+		return uintptr(ret), nil, linuxerr.EBADF
+	}
+	ret, err := iouringfd.ProcessSubmissions(toSubmit, minComplete, flags)
+	if err != nil {
+		return uintptr(ret), nil, err
+	}
+
+	return uintptr(ret), nil, nil
+}

pkg/sentry/syscalls/linux/vfs2/vfs2.go

@@ -164,6 +164,7 @@ func Override() {
 	s.Table[328] = syscalls.Supported("pwritev2", Pwritev2)
 	s.Table[332] = syscalls.Supported("statx", Statx)
 	s.Table[425] = syscalls.PartiallySupported("io_uring_setup", IOUringSetup, "Not all flags and functionality supported.", nil)
+	s.Table[426] = syscalls.PartiallySupported("io_uring_enter", IOUringEnter, "Not all flags and functionality supported.", nil)
 	s.Table[436] = syscalls.Supported("close_range", CloseRange)
 	s.Table[439] = syscalls.Supported("faccessat2", Faccessat2)
 	s.Table[441] = syscalls.Supported("epoll_pwait2", EpollPwait2)
@@ -281,6 +282,7 @@ func Override() {
 	s.Table[287] = syscalls.Supported("pwritev2", Pwritev2)
 	s.Table[291] = syscalls.Supported("statx", Statx)
 	s.Table[425] = syscalls.PartiallySupported("io_uring_setup", IOUringSetup, "Not all flags and functionality supported.", nil)
+	s.Table[426] = syscalls.PartiallySupported("io_uring_enter", IOUringEnter, "Not all flags and functionality supported.", nil)
 	s.Table[436] = syscalls.Supported("close_range", CloseRange)
 	s.Table[439] = syscalls.Supported("faccessat2", Faccessat2)
 	s.Table[441] = syscalls.Supported("epoll_pwait2", EpollPwait2)

test/syscalls/linux/BUILD

@@ -1076,6 +1076,7 @@ cc_binary(
         "//test/util:temp_path",
         "//test/util:test_main",
         "//test/util:test_util",
+        "//test/util:thread_util",
     ],
 )
 

test/util/BUILD

@@ -62,6 +62,7 @@ cc_library(
 cc_library(
     name = "io_uring_util",
     testonly = 1,
+    srcs = ["io_uring_util.cc"],
     hdrs = ["io_uring_util.h"],
     deps = [
         ":file_descriptor",

test/util/io_uring_util.cc

@@ -0,0 +1,116 @@
+// Copyright 2022 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test/util/io_uring_util.h"
+
+#include <memory>
+
+namespace gvisor {
+namespace testing {
+
+PosixErrorOr<std::unique_ptr<IOUring>> IOUring::InitIOUring(
+    unsigned int entries, IOUringParams &params) {
+  PosixErrorOr<FileDescriptor> fd = NewIOUringFD(entries, params);
+  if (!fd.ok()) {
+    return fd.error();
+  }
+
+  return std::make_unique<IOUring>(std::move(fd.ValueOrDie()), entries, params);
+}
+
+IOUring::IOUring(FileDescriptor &&fd, unsigned int entries,
+                 IOUringParams &params)
+    : iouringfd_(std::move(fd)) {
+  cring_sz_ = params.cq_off.cqes + params.cq_entries * sizeof(IOUringCqe);
+  sring_sz_ = params.sq_off.array + params.sq_entries * sizeof(unsigned);
+  sqes_sz_ = params.sq_entries * sizeof(IOUringSqe);
+
+  cq_ptr_ =
+      mmap(0, cring_sz_, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
+           iouringfd_.get(), IORING_OFF_SQ_RING);
+  sq_ptr_ =
+      mmap(0, sring_sz_, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
+           iouringfd_.get(), IORING_OFF_SQ_RING);
+  sqe_ptr_ = mmap(0, sqes_sz_, PROT_READ | PROT_WRITE,
+                  MAP_SHARED | MAP_POPULATE, iouringfd_.get(), IORING_OFF_SQES);
+
+  cqes_ = reinterpret_cast<IOUringCqe *>(reinterpret_cast<char *>(cq_ptr_) +
+                                         params.cq_off.cqes);
+
+  cq_head_ptr_ = reinterpret_cast<uint32_t *>(
+      reinterpret_cast<char *>(cq_ptr_) + params.cq_off.head);
+  cq_tail_ptr_ = reinterpret_cast<uint32_t *>(
+      reinterpret_cast<char *>(cq_ptr_) + params.cq_off.tail);
+  sq_head_ptr_ = reinterpret_cast<uint32_t *>(
+      reinterpret_cast<char *>(sq_ptr_) + params.sq_off.head);
+  sq_tail_ptr_ = reinterpret_cast<uint32_t *>(
+      reinterpret_cast<char *>(sq_ptr_) + params.sq_off.tail);
+  cq_overflow_ptr_ = reinterpret_cast<uint32_t *>(
+      reinterpret_cast<char *>(cq_ptr_) + params.cq_off.overflow);
+  sq_dropped_ptr_ = reinterpret_cast<uint32_t *>(
+      reinterpret_cast<char *>(sq_ptr_) + params.sq_off.dropped);
+
+  sq_mask_ = *(reinterpret_cast<uint32_t *>(reinterpret_cast<char *>(sq_ptr_) +
+                                            params.sq_off.ring_mask));
+  sq_array_ = reinterpret_cast<unsigned *>(reinterpret_cast<char *>(sq_ptr_) +
+                                           params.sq_off.array);
+}
+
+IOUring::~IOUring() {
+  munmap(cq_ptr_, cring_sz_);
+  munmap(sq_ptr_, sring_sz_);
+  munmap(sqe_ptr_, sqes_sz_);
+}
+
+uint32_t IOUring::load_cq_head() { return io_uring_atomic_read(cq_head_ptr_); }
+
+uint32_t IOUring::load_cq_tail() { return io_uring_atomic_read(cq_tail_ptr_); }
+
+uint32_t IOUring::load_sq_head() { return io_uring_atomic_read(sq_head_ptr_); }
+
+uint32_t IOUring::load_sq_tail() { return io_uring_atomic_read(sq_tail_ptr_); }
+
+uint32_t IOUring::load_cq_overflow() {
+  return io_uring_atomic_read(cq_overflow_ptr_);
+}
+
+uint32_t IOUring::load_sq_dropped() {
+  return io_uring_atomic_read(sq_dropped_ptr_);
+}
+
+void IOUring::store_cq_head(uint32_t cq_head_val) {
+  io_uring_atomic_write(cq_head_ptr_, cq_head_val);
+}
+
+void IOUring::store_sq_tail(uint32_t sq_tail_val) {
+  io_uring_atomic_write(sq_tail_ptr_, sq_tail_val);
+}
+
+int IOUring::Enter(unsigned int to_submit, unsigned int min_complete,
+                   unsigned int flags, sigset_t *sig) {
+  return IOUringEnter(iouringfd_.get(), to_submit, min_complete, flags, sig);
+}
+
+IOUringCqe *IOUring::get_cqes() { return cqes_; }
+
+IOUringSqe *IOUring::get_sqes() {
+  return reinterpret_cast<IOUringSqe *>(sqe_ptr_);
+}
+
+uint32_t IOUring::get_sq_mask() { return sq_mask_; }
+
+unsigned *IOUring::get_sq_array() { return sq_array_; }
+
+}  // namespace testing
+}  // namespace gvisor

test/util/io_uring_util.h

@@ -15,6 +15,10 @@
 #ifndef GVISOR_TEST_UTIL_IOURING_UTIL_H_
 #define GVISOR_TEST_UTIL_IOURING_UTIL_H_
 
+#include <linux/fs.h>
+#include <sys/mman.h>
+
+#include <atomic>
 #include <cerrno>
 #include <cstdint>
 
@@ -26,6 +30,7 @@ namespace gvisor {
 namespace testing {
 
 #define __NR_io_uring_setup 425
+#define __NR_io_uring_enter 426
 
 // io_uring_setup(2) flags.
 #define IORING_SETUP_SQPOLL (1U << 1)
@@ -36,6 +41,9 @@ namespace testing {
 #define IORING_OFF_CQ_RING 0x8000000ULL
 #define IORING_OFF_SQES 0x10000000ULL
 
+// IO_URING operation codes.
+#define IORING_OP_NOP 0
+
 struct io_sqring_offsets {
   uint32_t head;
   uint32_t tail;
@@ -73,14 +81,136 @@ struct io_uring_params {
   struct io_cqring_offsets cq_off;
 };
 
+struct io_uring_cqe {
+  uint64_t user_data;
+  int32_t res;
+  uint32_t flags;
+};
+
+struct io_uring_sqe {
+  uint8_t opcode;
+  uint8_t flags;
+  uint16_t ioprio;
+  int32_t fd;
+  union {
+    uint64_t off;
+    uint64_t addr2;
+    struct {
+      uint32_t cmd_op;
+      uint32_t __pad1;
+    };
+  };
+  union {
+    uint64_t addr;
+    uint64_t splice_off_in;
+  };
+  uint32_t len;
+  union {
+    __kernel_rwf_t rw_flags;
+    uint32_t fsync_flags;
+    uint16_t poll_events;
+    uint32_t poll32_events;
+    uint32_t sync_range_flags;
+    uint32_t msg_flags;
+    uint32_t timeout_flags;
+    uint32_t accept_flags;
+    uint32_t cancel_flags;
+    uint32_t open_flags;
+    uint32_t statx_flags;
+    uint32_t fadvise_advice;
+    uint32_t splice_flags;
+    uint32_t rename_flags;
+    uint32_t unlink_flags;
+    uint32_t hardlink_flags;
+    uint32_t xattr_flags;
+  };
+  uint64_t user_data;
+  union {
+    uint16_t buf_index;
+    uint16_t buf_group;
+  } __attribute__((packed));
+  uint16_t personality;
+  union {
+    int32_t splice_fd_in;
+    uint32_t file_index;
+  };
+  union {
+    struct {
+      uint64_t addr3;
+      uint64_t __pad2[1];
+    };
+    uint8_t cmd[0];
+  };
+};
+
+using IOSqringOffsets = struct io_sqring_offsets;
+using ICqringOffsets = struct io_cqring_offsets;
+using IOUringCqe = struct io_uring_cqe;
+using IOUringParams = struct io_uring_params;
+using IOUringSqe = struct io_uring_sqe;
+
+// Helper class for IO_URING
+class IOUring {
+ public:
+  IOUring() = delete;
+  IOUring(FileDescriptor &&fd, unsigned int entries, IOUringParams &params);
+  ~IOUring();
+
+  static PosixErrorOr<std::unique_ptr<IOUring>> InitIOUring(
+      unsigned int entries, IOUringParams &params);
+
+  uint32_t load_cq_head();
+  uint32_t load_cq_tail();
+  uint32_t load_sq_head();
+  uint32_t load_sq_tail();
+  uint32_t load_cq_overflow();
+  uint32_t load_sq_dropped();
+  void store_cq_head(uint32_t cq_head_val);
+  void store_sq_tail(uint32_t sq_tail_val);
+  int Enter(unsigned int to_submit, unsigned int min_complete,
+            unsigned int flags, sigset_t *sig);
+
+  IOUringCqe *get_cqes();
+  IOUringSqe *get_sqes();
+  uint32_t get_sq_mask();
+  unsigned *get_sq_array();
+
+  int Fd() { return iouringfd_.get(); }
+
+ private:
+  IOUringCqe *cqes_ = nullptr;
+  FileDescriptor iouringfd_;
+  size_t cring_sz_;
+  size_t sring_sz_;
+  size_t sqes_sz_;
+  uint32_t sq_mask_;
+  unsigned *sq_array_ = nullptr;
+  uint32_t *cq_head_ptr_ = nullptr;
+  uint32_t *cq_tail_ptr_ = nullptr;
+  uint32_t *sq_head_ptr_ = nullptr;
+  uint32_t *sq_tail_ptr_ = nullptr;
+  uint32_t *cq_overflow_ptr_ = nullptr;
+  uint32_t *sq_dropped_ptr_ = nullptr;
+  void *sq_ptr_ = nullptr;
+  void *cq_ptr_ = nullptr;
+  void *sqe_ptr_ = nullptr;
+};
+
 // This is a wrapper for the io_uring_setup(2) system call.
-inline int IOUringSetup(uint32_t entries, struct io_uring_params* params) {
+inline int IOUringSetup(uint32_t entries, IOUringParams *params) {
   return syscall(__NR_io_uring_setup, entries, params);
 }
 
+// This is a wrapper for the io_uring_enter(2) system call.
+inline int IOUringEnter(unsigned int fd, unsigned int to_submit,
+                        unsigned int min_complete, unsigned int flags,
+                        sigset_t *sig) {
+  return syscall(__NR_io_uring_enter, fd, to_submit, min_complete, flags, sig);
+}
+
 // Returns a new iouringfd with the given number of entries.
-inline PosixErrorOr<FileDescriptor> NewIOUringFD(
-    uint32_t entries, struct io_uring_params& params) {
+inline PosixErrorOr<FileDescriptor> NewIOUringFD(uint32_t entries,
+                                                 IOUringParams &params) {
   memset(&params, 0, sizeof(params));
   int fd = IOUringSetup(entries, &params);
   MaybeSave();
@@ -90,6 +220,18 @@ inline PosixErrorOr<FileDescriptor> NewIOUringFD(
   return FileDescriptor(fd);
 }
 
+template <typename T>
+static inline void io_uring_atomic_write(T *p, T v) {
+  std::atomic_store_explicit(reinterpret_cast<std::atomic<T> *>(p), v,
+                             std::memory_order_release);
+}
+
+template <typename T>
+static inline T io_uring_atomic_read(const T *p) {
+  return std::atomic_load_explicit(reinterpret_cast<const std::atomic<T> *>(p),
+                                   std::memory_order_acquire);
+}
+
 }  // namespace testing
 }  // namespace gvisor