diff --git a/pkg/abi/linux/iouring.go b/pkg/abi/linux/iouring.go index 691701af2..df27f9b18 100644 --- a/pkg/abi/linux/iouring.go +++ b/pkg/abi/linux/iouring.go @@ -116,9 +116,9 @@ type IOUringParams struct { // // +marshal type IOUringCqe struct { - userData uint64 - res int32 - flags uint32 + UserData uint64 + Res int32 + Flags uint32 } // IOUring implements io_uring struct. @@ -128,9 +128,9 @@ type IOUringCqe struct { type IOUring struct { // Both head and tail should be cacheline aligned. And we assume that // cacheline size is 64 bytes. - head uint32 + Head uint32 _ [60]byte - tail uint32 + Tail uint32 _ [60]byte } @@ -140,14 +140,37 @@ type IOUring struct { // // +marshal type IORings struct { - sq, cq IOUring - sqRingMask, cqRingMask uint32 - sqRingEntries, cqRingEntries uint32 + Sq, Cq IOUring + SqRingMask, CqRingMask uint32 + SqRingEntries, CqRingEntries uint32 sqDropped uint32 sqFlags int32 cqFlags uint32 - cqOverflow uint32 + CqOverflow uint32 _ [32]byte // Padding so cqes is cacheline aligned // Linux has an additional field struct io_uring_cqe cqes[], which represents // a dynamic array. We don't include it here in order to enable marshalling. } + +// IOUringSqe implements io_uring_sqe struct. +// This struct represents IO submission data structure (Submission Queue Entry). As we don't yet +// support IORING_SETUP_SQE128 flag, its size is 64 bytes with no extra padding at the end. +// See include/uapi/linux/io_uring.h. +// +// +marshal +type IOUringSqe struct { + Opcode uint8 + Flags uint8 + ioPrio uint16 + fd int32 + offOrAddrOrCmdOp uint64 + addrOrSpliceOff uint64 + len uint32 + specialFlags uint32 + UserData uint64 + bufIndexOrGroup uint16 + personality uint16 + spliceFDOrFileIndex int32 + addr3 uint64 + _ uint64 +} diff --git a/pkg/abi/linux/iouring_offsets.tmpl b/pkg/abi/linux/iouring_offsets.tmpl index 61f581c85..7632dd617 100644 --- a/pkg/abi/linux/iouring_offsets.tmpl +++ b/pkg/abi/linux/iouring_offsets.tmpl @@ -17,10 +17,10 @@ package linux // PreComputedIOSqRingOffsets returns precomputed values for IOSqRingOffsets. func PreComputedIOSqRingOffsets() IOSqRingOffsets { return IOSqRingOffsets{ - Head: {{ .IORings.sq.Offset }} + {{ .IOUring.head.Offset }}, - Tail: {{ .IORings.sq.Offset }} + {{ .IOUring.tail.Offset }}, - RingMask: {{ .IORings.sqRingMask.Offset }}, - RingEntries: {{ .IORings.sqRingEntries.Offset }}, + Head: {{ .IORings.Sq.Offset }} + {{ .IOUring.Head.Offset }}, + Tail: {{ .IORings.Sq.Offset }} + {{ .IOUring.Tail.Offset }}, + RingMask: {{ .IORings.SqRingMask.Offset }}, + RingEntries: {{ .IORings.SqRingEntries.Offset }}, Flags: {{ .IORings.sqFlags.Offset }}, Dropped: {{ .IORings.sqDropped.Offset }}, } @@ -29,11 +29,11 @@ func PreComputedIOSqRingOffsets() IOSqRingOffsets { // PreComputedIOCqRingOffsets returns precomputed values for IOCqRingOffsets. func PreComputedIOCqRingOffsets() IOCqRingOffsets { return IOCqRingOffsets { - Head: {{ .IORings.cq.Offset }} + {{ .IOUring.head.Offset }}, - Tail: {{ .IORings.cq.Offset }} + {{ .IOUring.tail.Offset }}, - RingMask: {{ .IORings.cqRingMask.Offset }}, - RingEntries: {{ .IORings.cqRingEntries.Offset }}, - Overflow: {{ .IORings.cqOverflow.Offset }}, + Head: {{ .IORings.Cq.Offset }} + {{ .IOUring.Head.Offset }}, + Tail: {{ .IORings.Cq.Offset }} + {{ .IOUring.Tail.Offset }}, + RingMask: {{ .IORings.CqRingMask.Offset }}, + RingEntries: {{ .IORings.CqRingEntries.Offset }}, + Overflow: {{ .IORings.CqOverflow.Offset }}, Flags: {{ .IORings.cqFlags.Offset }}, } } diff --git a/pkg/sentry/fsimpl/iouringfs/BUILD b/pkg/sentry/fsimpl/iouringfs/BUILD index 593d2e217..817f8a278 100644 --- a/pkg/sentry/fsimpl/iouringfs/BUILD +++ b/pkg/sentry/fsimpl/iouringfs/BUILD @@ -8,9 +8,11 @@ go_library( visibility = ["//pkg/sentry:internal"], deps = [ "//pkg/abi/linux", + "//pkg/atomicbitops", "//pkg/context", "//pkg/errors/linuxerr", "//pkg/hostarch", + "//pkg/safemem", "//pkg/sentry/memmap", "//pkg/sentry/pgalloc", "//pkg/sentry/usage", diff --git a/pkg/sentry/fsimpl/iouringfs/iouringfs.go b/pkg/sentry/fsimpl/iouringfs/iouringfs.go index 1c870fbdd..6125384f4 100644 --- a/pkg/sentry/fsimpl/iouringfs/iouringfs.go +++ b/pkg/sentry/fsimpl/iouringfs/iouringfs.go @@ -13,16 +13,25 @@ // limitations under the License. // Package iouringfs provides a filesystem implementation for IO_URING basing -// it on anonfs. +// it on anonfs. Currently, we don't support neither IOPOLL nor SQPOLL modes. +// Thus, user needs to set up IO_URING first with io_uring_setup(2) syscall and +// then issue submission request using io_uring_enter(2). +// +// Another important note, as of now, we don't support deferred CQE. In other +// words, the size of the backlogged set of CQE is zero. Whenever, completion +// queue ring buffer is full, we drop the subsequent completion queue entries. package iouringfs import ( "fmt" + "sync" "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/atomicbitops" "gvisor.dev/gvisor/pkg/context" "gvisor.dev/gvisor/pkg/errors/linuxerr" "gvisor.dev/gvisor/pkg/hostarch" + "gvisor.dev/gvisor/pkg/safemem" "gvisor.dev/gvisor/pkg/sentry/memmap" "gvisor.dev/gvisor/pkg/sentry/pgalloc" "gvisor.dev/gvisor/pkg/sentry/usage" @@ -33,7 +42,7 @@ import ( // It is based on io_rings struct. See io_uring/io_uring.c. // // +stateify savable -type fileDescription struct { +type FileDescription struct { vfsfd vfs.FileDescription vfs.FileDescriptionDefaultImpl vfs.DentryMetadataFileDescriptionImpl @@ -41,9 +50,16 @@ type fileDescription struct { rbmf ringsBufferFile sqemf sqEntriesFile + + // mu protects the fields below. + mu sync.Mutex `state:"nosave"` + + ioRings *safemem.BlockSeq + sqes *safemem.BlockSeq + cqes *safemem.BlockSeq } -var _ vfs.FileDescriptionImpl = (*fileDescription)(nil) +var _ vfs.FileDescriptionImpl = (*FileDescription)(nil) func roundUpPowerOfTwo(n uint32) (uint32, bool) { if n > (1 << 31) { @@ -98,15 +114,14 @@ func New(ctx context.Context, vfsObj *vfs.VirtualFilesystem, entries uint32, par } // Allocate enough space to store the given number of submission queue entries. - sqEntriesSize := uint64(numSqEntries * 64) + sqEntriesSize := uint64(numSqEntries * uint32((*linux.IOUringSqe)(nil).SizeBytes())) sqEntriesSize = uint64(hostarch.Addr(sqEntriesSize).MustRoundUp()) - sqefr, err := mfp.MemoryFile().Allocate(sqEntriesSize, pgalloc.AllocOpts{Kind: usage.Anonymous}) if err != nil { return nil, linuxerr.ENOMEM } - iouringfd := &fileDescription{ + iouringfd := &FileDescription{ rbmf: ringsBufferFile{ mf: mfp.MemoryFile(), fr: rbfr, @@ -152,15 +167,135 @@ func New(ctx context.Context, vfsObj *vfs.VirtualFilesystem, entries uint32, par // Set features supported by the current IO_URING implementation. params.Features = linux.IORING_FEAT_SINGLE_MMAP + if err := iouringfd.populateIORings(params); err != nil { + return nil, err + } + + if err := iouringfd.cacheSqesMapping(); err != nil { + return nil, err + } + if err := iouringfd.cacheCqesMapping(); err != nil { + return nil, err + } + return &iouringfd.vfsfd, nil } // Release implements vfs.FileDescriptionImpl.Release. -func (fd *fileDescription) Release(context.Context) { +func (fd *FileDescription) Release(context.Context) { + fd.rbmf.mf.DecRef(fd.rbmf.fr) + fd.sqemf.mf.DecRef(fd.sqemf.fr) +} + +// unmarshalIORings handles unmarshalling IORings struct considering that there could be more than +// one block in the BlockSeq. +func unmarshalIORings(ioRings *linux.IORings, bs *safemem.BlockSeq) error { + if bs.NumBlocks() == 1 && !bs.Head().NeedSafecopy() { + ioRings.UnmarshalBytes(bs.Head().TakeFirst((*linux.IORings)(nil).SizeBytes()).ToSlice()) + + return nil + } + + buf := make([]byte, (*linux.IORings)(nil).SizeBytes()) + cp, cperr := safemem.CopySeq(safemem.BlockSeqOf(safemem.BlockFromSafeSlice(buf)), *bs) + if cp == 0 { + return cperr + } + ioRings.UnmarshalBytes(buf) + + return nil +} + +// marshalIORings handles marshalling IORings struct considering that there could be more than one +// BlockSeq. +func marshalIORings(ioRings *linux.IORings, bs *safemem.BlockSeq) error { + if bs.NumBlocks() == 1 && !bs.Head().NeedSafecopy() { + ioRings.MarshalBytes(bs.Head().TakeFirst((*linux.IORings)(nil).SizeBytes()).ToSlice()) + } + + buf := make([]byte, (*linux.IORings)(nil).SizeBytes()) + ioRings.MarshalBytes(buf) + cp, cperr := safemem.CopySeq(*bs, safemem.BlockSeqOf(safemem.BlockFromSafeSlice(buf))) + if cp == 0 { + return cperr + } + + return nil +} + +// unmarshalSqe handles unmarshalling SQE struct considering that there could be more than one block +// in the BlockSeq. +func unmarshalSqe(sqe *linux.IOUringSqe, sqes *safemem.BlockSeq, sqHead uint32) error { + sqeSize := uint32((*linux.IOUringSqe)(nil).SizeBytes()) + if sqes.NumBlocks() == 1 && !sqes.Head().NeedSafecopy() { + sqe.UnmarshalBytes(sqes.Head().ToSlice()[sqHead*sqeSize : (sqHead+1)*sqeSize]) + + return nil + } + + buf := make([]byte, sqes.NumBytes()) + cp, cperr := safemem.CopySeq(safemem.BlockSeqOf(safemem.BlockFromSafeSlice(buf[sqHead*sqeSize:(sqHead+1)*sqeSize])), *sqes) + if cp == 0 { + return cperr + } + sqe.UnmarshalBytes(buf) + + return nil +} + +// populateIORings populates IORings struct backed by the allocated memory. +func (fd *FileDescription) populateIORings(params *linux.IOUringParams) error { + bs, err := fd.rbmf.mf.MapInternal(fd.rbmf.fr, hostarch.ReadWrite) + if err != nil { + return err + } + + fd.ioRings = &bs + + var ioRings linux.IORings + if err = unmarshalIORings(&ioRings, &bs); err != nil { + return err + } + + ioRings.SqRingMask = params.SqEntries - 1 + ioRings.CqRingMask = params.CqEntries - 1 + ioRings.SqRingEntries = params.SqEntries + ioRings.CqRingEntries = params.CqEntries + + if err = marshalIORings(&ioRings, &bs); err != nil { + return err + } + + return nil +} + +// cacheSqesMapping caches the beginning of an area for the SQEs backed by the allocated memory. +func (fd *FileDescription) cacheSqesMapping() error { + bs, err := fd.sqemf.mf.MapInternal(fd.sqemf.fr, hostarch.ReadWrite) + if err != nil { + return err + } + fd.sqes = &bs + + return nil +} + +// cacheCqesMapping caches the beginning of an area for the CQEs backed by the allocated memory. +func (fd *FileDescription) cacheCqesMapping() error { + bs := *fd.ioRings + cqesOffset := uint64(hostarch.Addr((*linux.IORings)(nil).SizeBytes())) + cqesOffset, ok := hostarch.CacheLineRoundUp(cqesOffset) + if !ok { + return linuxerr.EOVERFLOW + } + bs = bs.DropFirst(int(cqesOffset)) + fd.cqes = &bs + + return nil } // ConfigureMMap implements vfs.FileDescriptionImpl.ConfigureMMap. -func (fd *fileDescription) ConfigureMMap(ctx context.Context, opts *memmap.MMapOpts) error { +func (fd *FileDescription) ConfigureMMap(ctx context.Context, opts *memmap.MMapOpts) error { var mf memmap.Mappable switch opts.Offset { case linux.IORING_OFF_SQ_RING, linux.IORING_OFF_CQ_RING: @@ -171,9 +306,115 @@ func (fd *fileDescription) ConfigureMMap(ctx context.Context, opts *memmap.MMapO return linuxerr.EINVAL } + opts.Offset = 0 + return vfs.GenericConfigureMMap(&fd.vfsfd, mf, opts) } +// ProcessSubmissions processes submission requests. +func (fd *FileDescription) ProcessSubmissions(toSubmit uint32, minComplete uint32, flags uint32) (int, error) { + fd.mu.Lock() + defer fd.mu.Unlock() + + var ioRings linux.IORings + err := fd.getIORings(&ioRings) + if err != nil { + return -1, err + } + + sqes := fd.sqes + cqes := fd.cqes + + var sqe linux.IOUringSqe + sqHead := atomicbitops.FromUint32(ioRings.Sq.Head) + sqTail := atomicbitops.FromUint32(ioRings.Sq.Tail) + cqHead := atomicbitops.FromUint32(ioRings.Cq.Head) + cqTail := atomicbitops.FromUint32(ioRings.Cq.Tail) + + submitted := uint32(0) + for toSubmit > submitted { + sqHeadMasked := sqHead.Load() & ioRings.SqRingMask + cqTailMasked := cqTail.Load() & ioRings.CqRingMask + // This means that the submission queue is empty. + if sqHead == sqTail { + return int(submitted), nil + } + + if err = unmarshalSqe(&sqe, sqes, sqHeadMasked); err != nil { + return -1, err + } + + cqe, err := fd.ProcessSubmission(&sqe, flags) + if err != nil { + return -1, err + } + sqHead.Add(1) + if (cqTail.Load()-cqHead.Load())/ioRings.CqRingEntries == 1 { + ioRings.CqOverflow++ + } else { + if err = fd.updateCq(cqes, cqe, cqTailMasked); err != nil { + return -1, err + } + cqTail.Add(1) + } + submitted++ + } + ioRings.Sq.Head = sqHead.Load() + ioRings.Cq.Tail = cqTail.Load() + + if err = marshalIORings(&ioRings, fd.ioRings); err != nil { + return -1, err + } + + return int(submitted), nil +} + +// ProcessSubmission processes a single submission request. +func (fd *FileDescription) ProcessSubmission(sqe *linux.IOUringSqe, flags uint32) (*linux.IOUringCqe, error) { + switch op := sqe.Opcode; op { + case 0: // NOP + return &linux.IOUringCqe{ + UserData: sqe.UserData, + Res: 0, + Flags: 0, + }, nil + default: // Unsupported operation + return &linux.IOUringCqe{ + UserData: sqe.UserData, + Res: -int32(linuxerr.EINVAL.Errno()), + Flags: 0, + }, nil + } +} + +// updateCq updates a completion queue by adding a given completion queue entry. +func (fd *FileDescription) updateCq(cqes *safemem.BlockSeq, cqe *linux.IOUringCqe, cqTail uint32) error { + cqeSize := uint32((*linux.IOUringCqe)(nil).SizeBytes()) + if cqes.NumBlocks() == 1 && !cqes.Head().NeedSafecopy() { + cqe.MarshalBytes(cqes.Head().ToSlice()[cqTail*cqeSize : (cqTail+1)*cqeSize]) + + return nil + } + + buf := make([]byte, cqes.NumBytes()) + cqe.MarshalBytes(buf) + cp, cperr := safemem.CopySeq(cqes.DropFirst64(uint64(cqTail*cqeSize)), safemem.BlockSeqOf(safemem.BlockFromSafeSlice(buf))) + if cp == 0 { + return cperr + } + + return nil +} + +// getIORings unmarshalls IORings struct backed by the allocated memory. +func (fd *FileDescription) getIORings(ioRings *linux.IORings) error { + if err := unmarshalIORings(ioRings, fd.ioRings); err != nil { + return err + } + + return nil +} + // sqEntriesFile implements memmap.Mappable for SQ entries. type sqEntriesFile struct { mf *pgalloc.MemoryFile @@ -196,15 +437,6 @@ func (sqemf *sqEntriesFile) CopyMapping(ctx context.Context, ms memmap.MappingSp // Translate implements memmap.Mappable.Translate. func (sqemf *sqEntriesFile) Translate(ctx context.Context, required, optional memmap.MappableRange, at hostarch.AccessType) ([]memmap.Translation, error) { - expectedAccessType := hostarch.AccessType{ - Read: true, - Write: true, - Execute: false, - } - if at != expectedAccessType { - return nil, &memmap.BusError{linuxerr.EPERM} - } - if required.End > sqemf.fr.Length() { return nil, &memmap.BusError{linuxerr.EFAULT} } @@ -250,15 +482,6 @@ func (rbmf *ringsBufferFile) CopyMapping(ctx context.Context, ms memmap.MappingS // Translate implements memmap.Mappable.Translate. func (rbmf *ringsBufferFile) Translate(ctx context.Context, required, optional memmap.MappableRange, at hostarch.AccessType) ([]memmap.Translation, error) { - expectedAccessType := hostarch.AccessType{ - Read: true, - Write: true, - Execute: false, - } - if at != expectedAccessType { - return nil, &memmap.BusError{linuxerr.EPERM} - } - if required.End > rbmf.fr.Length() { return nil, &memmap.BusError{linuxerr.EFAULT} } diff --git a/pkg/sentry/syscalls/linux/vfs2/iouringfs.go b/pkg/sentry/syscalls/linux/vfs2/iouringfs.go index 476413d61..50727aec0 100644 --- a/pkg/sentry/syscalls/linux/vfs2/iouringfs.go +++ b/pkg/sentry/syscalls/linux/vfs2/iouringfs.go @@ -17,6 +17,7 @@ package vfs2 import ( "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/errors/linuxerr" + "gvisor.dev/gvisor/pkg/hostarch" "gvisor.dev/gvisor/pkg/sentry/arch" "gvisor.dev/gvisor/pkg/sentry/fsimpl/iouringfs" "gvisor.dev/gvisor/pkg/sentry/kernel" @@ -42,7 +43,7 @@ func IOUringSetup(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel. } // List of currently supported flags in our IO_URING implementation. - const supportedFlags = linux.IORING_SETUP_IOPOLL + const supportedFlags = 0 // Currently support none // Since we don't implement everything, we fail explicitly on flags that are unimplemented. if params.Flags|supportedFlags != supportedFlags { @@ -53,7 +54,8 @@ func IOUringSetup(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel. iouringfd, err := iouringfs.New(t, vfsObj, entries, ¶ms) if err != nil { - return 0, nil, err + // return 0, nil, err + return 0, nil, linuxerr.EPERM } defer iouringfd.DecRef(t) @@ -72,3 +74,48 @@ func IOUringSetup(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel. return uintptr(fd), nil, nil } + +// IOUringEnter implements linux syscall io_uring_enter(2). +func IOUringEnter(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel.SyscallControl, error) { + fd := int32(args[0].Int()) + toSubmit := uint32(args[1].Uint()) + minComplete := uint32(args[2].Uint()) + flags := uint32(args[3].Uint()) + sigSet := args[4].Pointer() + + ret := -1 + + // List of currently supported flags for io_uring_enter(2). + const supportedFlags = 0 // Currently support none + + // Since we don't implement everything, we fail explicitly on flags that are unimplemented. + if flags|supportedFlags != supportedFlags { + return uintptr(ret), nil, linuxerr.EINVAL + } + + // Currently don't support replacing an existing signal mask. + if sigSet != hostarch.Addr(0) { + return uintptr(ret), nil, linuxerr.EFAULT + } + + // If a user requested to submit zero SQEs, then we don't process any and return right away. + if toSubmit == 0 { + return uintptr(ret), nil, nil + } + + file := t.GetFileVFS2(fd) + if file == nil { + return uintptr(ret), nil, linuxerr.EBADF + } + defer file.DecRef(t) + iouringfd, ok := file.Impl().(*iouringfs.FileDescription) + if !ok { + return uintptr(ret), nil, linuxerr.EBADF + } + ret, err := iouringfd.ProcessSubmissions(toSubmit, minComplete, flags) + if err != nil { + return uintptr(ret), nil, err + } + + return uintptr(ret), nil, nil +} diff --git a/pkg/sentry/syscalls/linux/vfs2/vfs2.go b/pkg/sentry/syscalls/linux/vfs2/vfs2.go index 6a6b50a78..3a9ac74b0 100644 --- a/pkg/sentry/syscalls/linux/vfs2/vfs2.go +++ b/pkg/sentry/syscalls/linux/vfs2/vfs2.go @@ -164,6 +164,7 @@ func Override() { s.Table[328] = syscalls.Supported("pwritev2", Pwritev2) s.Table[332] = syscalls.Supported("statx", Statx) s.Table[425] = syscalls.PartiallySupported("io_uring_setup", IOUringSetup, "Not all flags and functionality supported.", nil) + s.Table[426] = syscalls.PartiallySupported("io_uring_enter", IOUringEnter, "Not all flags and functionality supported.", nil) s.Table[436] = syscalls.Supported("close_range", CloseRange) s.Table[439] = syscalls.Supported("faccessat2", Faccessat2) s.Table[441] = syscalls.Supported("epoll_pwait2", EpollPwait2) @@ -281,6 +282,7 @@ func Override() { s.Table[287] = syscalls.Supported("pwritev2", Pwritev2) s.Table[291] = syscalls.Supported("statx", Statx) s.Table[425] = syscalls.PartiallySupported("io_uring_setup", IOUringSetup, "Not all flags and functionality supported.", nil) + s.Table[426] = syscalls.PartiallySupported("io_uring_enter", IOUringEnter, "Not all flags and functionality supported.", nil) s.Table[436] = syscalls.Supported("close_range", CloseRange) s.Table[439] = syscalls.Supported("faccessat2", Faccessat2) s.Table[441] = syscalls.Supported("epoll_pwait2", EpollPwait2) diff --git a/test/syscalls/linux/BUILD b/test/syscalls/linux/BUILD index 14cd74885..4522b5786 100644 --- a/test/syscalls/linux/BUILD +++ b/test/syscalls/linux/BUILD @@ -1076,6 +1076,7 @@ cc_binary( "//test/util:temp_path", "//test/util:test_main", "//test/util:test_util", + "//test/util:thread_util", ], ) diff --git a/test/syscalls/linux/io_uring.cc b/test/syscalls/linux/io_uring.cc index 8002f5dfd..0d9534cc5 100644 --- a/test/syscalls/linux/io_uring.cc +++ b/test/syscalls/linux/io_uring.cc @@ -23,6 +23,7 @@ #include #include +#include #include #include "gtest/gtest.h" @@ -30,6 +31,7 @@ #include "test/util/memory_util.h" #include "test/util/multiprocess_util.h" #include "test/util/test_util.h" +#include "test/util/thread_util.h" namespace gvisor { namespace testing { @@ -38,13 +40,13 @@ namespace { // Testing that io_uring_setup(2) successfully returns a valid file descriptor. TEST(IOUringTest, ValidFD) { - struct io_uring_params params; + IOUringParams params; FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params)); } // Testing that io_uring_setup(2) fails with EINVAL on non-zero params. TEST(IOUringTest, ParamsNonZeroResv) { - struct io_uring_params params; + IOUringParams params; memset(¶ms, 0, sizeof(params)); params.resv[1] = 1; ASSERT_THAT(IOUringSetup(1, ¶ms), SyscallFailsWithErrno(EINVAL)); @@ -53,7 +55,7 @@ TEST(IOUringTest, ParamsNonZeroResv) { // Testing that io_uring_setup(2) fails with EINVAL on unsupported flags. TEST(IOUringTest, UnsupportedFlags) { if (IsRunningOnGvisor()) { - struct io_uring_params params; + IOUringParams params; memset(¶ms, 0, sizeof(params)); params.flags |= IORING_SETUP_SQPOLL; ASSERT_THAT(IOUringSetup(1, ¶ms), SyscallFailsWithErrno(EINVAL)); @@ -63,7 +65,7 @@ TEST(IOUringTest, UnsupportedFlags) { // Testing that both mmap and munmap calls succeed and subsequent access to // unmapped memory results in SIGSEGV. TEST(IOUringTest, MMapMUnMapWork) { - struct io_uring_params params; + IOUringParams params; FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params)); void *ptr = nullptr; @@ -90,7 +92,7 @@ TEST(IOUringTest, MMapMUnMapWork) { // Testing that both mmap fails with EINVAL when an invalid offset is passed. TEST(IOUringTest, MMapWrongOffset) { - struct io_uring_params params; + IOUringParams params; FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params)); int sring_sz = params.sq_off.array + params.sq_entries * sizeof(unsigned); @@ -104,42 +106,42 @@ TEST(IOUringTest, MMapWrongOffset) { // Testing that mmap() handles all three IO_URING-specific offsets and that // returned addresses are page aligned. TEST(IOUringTest, MMapOffsets) { - struct io_uring_params params; + IOUringParams params; FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params)); - void *sqPtr = nullptr; - void *cqPtr = nullptr; - void *sqePtr = nullptr; + void *sq_ptr = nullptr; + void *cq_ptr = nullptr; + void *sqe_ptr = nullptr; int sring_sz = params.sq_off.array + params.sq_entries * sizeof(unsigned); - int cring_sz = params.cq_off.cqes + params.cq_entries * 32; + int cring_sz = params.cq_off.cqes + params.cq_entries * sizeof(IOUringCqe); - sqPtr = mmap(0, sring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, - iouringfd.get(), IORING_OFF_SQ_RING); + sq_ptr = mmap(0, sring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, + iouringfd.get(), IORING_OFF_SQ_RING); - cqPtr = mmap(0, cring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, - iouringfd.get(), IORING_OFF_CQ_RING); + cq_ptr = mmap(0, cring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, + iouringfd.get(), IORING_OFF_CQ_RING); - sqePtr = mmap(0, 64, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, - iouringfd.get(), IORING_OFF_SQES); + sqe_ptr = mmap(0, sizeof(IOUringSqe), PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE, iouringfd.get(), IORING_OFF_SQES); - EXPECT_NE(sqPtr, MAP_FAILED); - EXPECT_NE(cqPtr, MAP_FAILED); - EXPECT_NE(sqePtr, MAP_FAILED); + EXPECT_NE(sq_ptr, MAP_FAILED); + EXPECT_NE(cq_ptr, MAP_FAILED); + EXPECT_NE(sqe_ptr, MAP_FAILED); - EXPECT_EQ((uintptr_t)sqPtr % kPageSize, 0); - EXPECT_EQ((uintptr_t)cqPtr % kPageSize, 0); - EXPECT_EQ((uintptr_t)sqePtr % kPageSize, 0); + EXPECT_EQ(reinterpret_cast(sq_ptr) % kPageSize, 0); + EXPECT_EQ(reinterpret_cast(cq_ptr) % kPageSize, 0); + EXPECT_EQ(reinterpret_cast(sqe_ptr) % kPageSize, 0); - ASSERT_THAT(munmap(sqPtr, sring_sz), SyscallSucceeds()); - ASSERT_THAT(munmap(cqPtr, cring_sz), SyscallSucceeds()); - ASSERT_THAT(munmap(sqePtr, 64), SyscallSucceeds()); + ASSERT_THAT(munmap(sq_ptr, sring_sz), SyscallSucceeds()); + ASSERT_THAT(munmap(cq_ptr, cring_sz), SyscallSucceeds()); + ASSERT_THAT(munmap(sqe_ptr, sizeof(IOUringSqe)), SyscallSucceeds()); } -// Testing that io_uring_params are populated with correct values. +// Testing that IOUringParams are populated with correct values. TEST(IOUringTest, ReturnedParamsValues) { if (IsRunningOnGvisor()) { - struct io_uring_params params; + IOUringParams params; FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params)); @@ -169,7 +171,7 @@ TEST(IOUringTest, ReturnedParamsValues) { // Testing that offset of SQE indices array is cacheline aligned. TEST(IOUringTest, SqeIndexArrayCacheAligned) { - struct io_uring_params params; + IOUringParams params; for (uint32_t i = 1; i < 10; ++i) { FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(i, params)); @@ -177,6 +179,402 @@ TEST(IOUringTest, SqeIndexArrayCacheAligned) { } } +// Testing that io_uring_enter(2) successfully handles a single NOP operation. +TEST(IOUringTest, SingleNOPTest) { + IOUringParams params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params)); + + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); + + IOUringSqe *sqe = io_uring->get_sqes(); + sqe->opcode = IORING_OP_NOP; + sqe->user_data = 42; + + uint32_t sq_tail = io_uring->load_sq_tail(); + io_uring->store_sq_tail(sq_tail + 1); + + int ret = io_uring->Enter(1, 1, 0, nullptr); + ASSERT_EQ(ret, 1); + + IOUringCqe *cqe = io_uring->get_cqes(); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 1); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 1); + + ASSERT_EQ(cqe->user_data, 42); + ASSERT_EQ(cqe->res, 0); + + uint32_t cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 1); +} + +// Testing that io_uring_enter(2) successfully queueing NOP operations. +TEST(IOUringTest, QueueingNOPTest) { + IOUringParams params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(4, params)); + + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); + + unsigned *sq_array = io_uring->get_sq_array(); + unsigned index = 0; + IOUringSqe *sqe = io_uring->get_sqes(); + for (size_t i = 0; i < 4; ++i) { + sqe[i].opcode = IORING_OP_NOP; + sqe[i].user_data = 42 + i; + index = i & io_uring->get_sq_mask(); + sq_array[index] = index; + } + + uint32_t sq_tail = io_uring->load_sq_tail(); + ASSERT_EQ(sq_tail, 0); + io_uring->store_sq_tail(sq_tail + 4); + + int ret = io_uring->Enter(2, 2, 0, nullptr); + ASSERT_EQ(ret, 2); + + IOUringCqe *cqe = io_uring->get_cqes(); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 2); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 2); + + for (size_t i = 0; i < 2; ++i) { + ASSERT_EQ(cqe[i].res, 0); + ASSERT_EQ(cqe[i].user_data, 42 + i); + } + + uint32_t cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 2); + + ret = io_uring->Enter(2, 2, 0, nullptr); + ASSERT_EQ(ret, 2); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 4); + + cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 4); + + for (size_t i = 2; i < 4; ++i) { + ASSERT_EQ(cqe[i].res, 0); + ASSERT_EQ(cqe[i].user_data, 42 + i); + } + + cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 2); +} + +// Testing that io_uring_enter(2) successfully multiple NOP operations. +TEST(IOUringTest, MultipleNOPTest) { + IOUringParams params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(4, params)); + + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); + + unsigned *sq_array = io_uring->get_sq_array(); + unsigned index = 0; + IOUringSqe *sqe = io_uring->get_sqes(); + for (size_t i = 0; i < 3; ++i) { + sqe[i].opcode = IORING_OP_NOP; + sqe[i].user_data = 42 + i; + index = i & io_uring->get_sq_mask(); + sq_array[index] = index; + } + + uint32_t sq_tail = io_uring->load_sq_tail(); + ASSERT_EQ(sq_tail, 0); + io_uring->store_sq_tail(sq_tail + 3); + + int ret = io_uring->Enter(3, 3, 0, nullptr); + ASSERT_EQ(ret, 3); + + IOUringCqe *cqe = io_uring->get_cqes(); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 3); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 3); + + for (size_t i = 0; i < 3; ++i) { + ASSERT_EQ(cqe[i].res, 0); + ASSERT_EQ(cqe[i].user_data, 42 + i); + } + + uint32_t cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 3); +} + +// Testing that io_uring_enter(2) successfully handles multiple threads +// submitting NOP operations. +TEST(IOUringTest, MultiThreadedNOPTest) { + IOUringParams params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(4, params)); + + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); + + unsigned *sq_array = io_uring->get_sq_array(); + unsigned index = 0; + IOUringSqe *sqe = io_uring->get_sqes(); + for (size_t i = 0; i < 4; ++i) { + sqe[i].opcode = IORING_OP_NOP; + sqe[i].user_data = 42 + i; + index = i & io_uring->get_sq_mask(); + sq_array[index] = index; + } + + uint32_t sq_tail = io_uring->load_sq_tail(); + io_uring->store_sq_tail(sq_tail + 4); + + for (int i = 0; i < 4; i++) { + ScopedThread t([&] { + IOUring *io_uring_ptr = io_uring.get(); + int ret = io_uring_ptr->Enter(1, 1, 0, nullptr); + ASSERT_EQ(ret, 1); + }); + } + + IOUringCqe *cqe = io_uring->get_cqes(); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 4); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 4); + + for (size_t i = 0; i < 4; ++i) { + ASSERT_EQ(cqe[i].res, 0); + ASSERT_EQ(cqe[i].user_data, 42 + i); + } + + uint32_t cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 4); +} + +// Testing that io_uring_enter(2) successfully consumes submission with an +// invalid opcode and returned CQE contains EINVAL in its result field. +TEST(IOUringTest, InvalidOpCodeTest) { + IOUringParams params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params)); + + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); + + IOUringSqe *sqe = io_uring->get_sqes(); + sqe->opcode = 255; // maximum value for one-byte unsigned integer + sqe->user_data = 42; + + uint32_t sq_tail = io_uring->load_sq_tail(); + io_uring->store_sq_tail(sq_tail + 1); + + int ret = io_uring->Enter(1, 1, 0, nullptr); + ASSERT_EQ(ret, 1); + + IOUringCqe *cqe = io_uring->get_cqes(); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 1); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 1); + + ASSERT_EQ(cqe->user_data, 42); + ASSERT_EQ(cqe->res, -EINVAL); + + uint32_t cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 1); +} + +// Testing that io_uring_enter(2) successfully consumes submission and SQE ring +// buffers wrap around. +TEST(IOUringTest, SQERingBuffersWrapAroundTest) { + IOUringParams params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(4, params)); + + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); + + unsigned *sq_array = io_uring->get_sq_array(); + unsigned index = 0; + IOUringSqe *sqe = io_uring->get_sqes(); + for (size_t i = 0; i < 4; ++i) { + sqe[i].opcode = IORING_OP_NOP; + sqe[i].user_data = 42 + i; + index = i & io_uring->get_sq_mask(); + sq_array[index] = index; + } + + uint32_t sq_tail = io_uring->load_sq_tail(); + io_uring->store_sq_tail(sq_tail + 4); + + int ret = io_uring->Enter(4, 4, 0, nullptr); + ASSERT_EQ(ret, 4); + + IOUringCqe *cqe = io_uring->get_cqes(); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 4); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 4); + + for (size_t i = 0; i < 4; ++i) { + ASSERT_EQ(cqe[i].res, 0); + ASSERT_EQ(cqe[i].user_data, 42 + i); + } + + uint32_t cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 4); + + for (size_t i = 0; i < 4; ++i) { + sqe[i].user_data = 42 + 2 * (i + 1); + } + + sq_tail = io_uring->load_sq_tail(); + io_uring->store_sq_tail(sq_tail + 4); + + ret = io_uring->Enter(4, 4, 0, nullptr); + ASSERT_EQ(ret, 4); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 8); + + cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 8); + + for (size_t i = 0; i < 4; ++i) { + ASSERT_EQ(cqe[4 + i].res, 0); + ASSERT_EQ(cqe[4 + i].user_data, 42 + 2 * (i + 1)); + } + + cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 4); +} + +// Testing that io_uring_enter(2) fails with EFAULT when non-null sigset_t has +// been passed as we currently don't support replacing signal mask. +TEST(IOUringTest, NonNullSigsetTest) { + if (IsRunningOnGvisor()) { + IOUringParams params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params)); + + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); + + IOUringSqe *sqe = io_uring->get_sqes(); + sqe->opcode = IORING_OP_NOP; + sqe->user_data = 42; + + uint32_t sq_tail = io_uring->load_sq_tail(); + io_uring->store_sq_tail(sq_tail + 1); + + sigset_t non_null_sigset; + EXPECT_THAT(io_uring->Enter(1, 1, 0, &non_null_sigset), + SyscallFailsWithErrno(EFAULT)); + } +} + +// Testing that completion queue overflow counter is incremented when the +// completion queue is not drained by the user and completion queue entries are +// not overwritten. +TEST(IOUringTest, OverflowCQTest) { + if (IsRunningOnGvisor()) { + IOUringParams params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(4, params)); + + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); + + unsigned *sq_array = io_uring->get_sq_array(); + unsigned index = 0; + IOUringSqe *sqe = io_uring->get_sqes(); + IOUringCqe *cqe = io_uring->get_cqes(); + + for (size_t submission_round = 0; submission_round < 2; + ++submission_round) { + for (size_t i = 0; i < 4; ++i) { + sqe[i].opcode = IORING_OP_NOP; + sqe[i].user_data = 42 + i + submission_round; + index = i & io_uring->get_sq_mask(); + sq_array[index] = index; + } + + uint32_t sq_tail = io_uring->load_sq_tail(); + ASSERT_EQ(sq_tail, 4 * submission_round); + io_uring->store_sq_tail(sq_tail + 4); + + int ret = io_uring->Enter(4, 4, 0, nullptr); + ASSERT_EQ(ret, 4); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 4 * (submission_round + 1)); + + uint32_t dropped = io_uring->load_sq_dropped(); + ASSERT_EQ(dropped, 0); + + uint32_t cq_overflow_counter = io_uring->load_cq_overflow(); + ASSERT_EQ(cq_overflow_counter, 0); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 4 * (submission_round + 1)); + + for (size_t i = 0; i < 4; ++i) { + ASSERT_EQ(cqe[i + 4 * submission_round].res, 0); + ASSERT_EQ(cqe[i + 4 * submission_round].user_data, + 42 + i + submission_round); + } + } + + for (size_t i = 0; i < 2; ++i) { + sqe[i].opcode = IORING_OP_NOP; + sqe[i].user_data = 52 + i; + index = i & io_uring->get_sq_mask(); + sq_array[index] = index; + } + + uint32_t sq_tail = io_uring->load_sq_tail(); + ASSERT_EQ(sq_tail, 8); + io_uring->store_sq_tail(sq_tail + 2); + + int ret = io_uring->Enter(2, 2, 0, nullptr); + ASSERT_EQ(ret, 2); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 10); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 8); + + ASSERT_EQ(cqe[0].res, 0); + ASSERT_EQ(cqe[0].user_data, 42); + ASSERT_EQ(cqe[1].res, 0); + ASSERT_EQ(cqe[1].user_data, 43); + + uint32_t dropped = io_uring->load_sq_dropped(); + ASSERT_EQ(dropped, 0); + + uint32_t cq_overflow_counter = io_uring->load_cq_overflow(); + ASSERT_EQ(cq_overflow_counter, 2); + } +} + } // namespace } // namespace testing diff --git a/test/util/BUILD b/test/util/BUILD index e309b50e0..657ca0899 100644 --- a/test/util/BUILD +++ b/test/util/BUILD @@ -62,6 +62,7 @@ cc_library( cc_library( name = "io_uring_util", testonly = 1, + srcs = ["io_uring_util.cc"], hdrs = ["io_uring_util.h"], deps = [ ":file_descriptor", diff --git a/test/util/io_uring_util.cc b/test/util/io_uring_util.cc new file mode 100644 index 000000000..f71a4c567 --- /dev/null +++ b/test/util/io_uring_util.cc @@ -0,0 +1,116 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "test/util/io_uring_util.h" + +#include + +namespace gvisor { +namespace testing { + +PosixErrorOr> IOUring::InitIOUring( + unsigned int entries, IOUringParams ¶ms) { + PosixErrorOr fd = NewIOUringFD(entries, params); + if (!fd.ok()) { + return fd.error(); + } + + return std::make_unique(std::move(fd.ValueOrDie()), entries, params); +} + +IOUring::IOUring(FileDescriptor &&fd, unsigned int entries, + IOUringParams ¶ms) + : iouringfd_(std::move(fd)) { + cring_sz_ = params.cq_off.cqes + params.cq_entries * sizeof(IOUringCqe); + sring_sz_ = params.sq_off.array + params.sq_entries * sizeof(unsigned); + sqes_sz_ = params.sq_entries * sizeof(IOUringSqe); + + cq_ptr_ = + mmap(0, cring_sz_, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, + iouringfd_.get(), IORING_OFF_SQ_RING); + sq_ptr_ = + mmap(0, sring_sz_, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, + iouringfd_.get(), IORING_OFF_SQ_RING); + sqe_ptr_ = mmap(0, sqes_sz_, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE, iouringfd_.get(), IORING_OFF_SQES); + + cqes_ = reinterpret_cast(reinterpret_cast(cq_ptr_) + + params.cq_off.cqes); + + cq_head_ptr_ = reinterpret_cast( + reinterpret_cast(cq_ptr_) + params.cq_off.head); + cq_tail_ptr_ = reinterpret_cast( + reinterpret_cast(cq_ptr_) + params.cq_off.tail); + sq_head_ptr_ = reinterpret_cast( + reinterpret_cast(sq_ptr_) + params.sq_off.head); + sq_tail_ptr_ = reinterpret_cast( + reinterpret_cast(sq_ptr_) + params.sq_off.tail); + cq_overflow_ptr_ = reinterpret_cast( + reinterpret_cast(cq_ptr_) + params.cq_off.overflow); + sq_dropped_ptr_ = reinterpret_cast( + reinterpret_cast(sq_ptr_) + params.sq_off.dropped); + + sq_mask_ = *(reinterpret_cast(reinterpret_cast(sq_ptr_) + + params.sq_off.ring_mask)); + sq_array_ = reinterpret_cast(reinterpret_cast(sq_ptr_) + + params.sq_off.array); +} + +IOUring::~IOUring() { + munmap(cq_ptr_, cring_sz_); + munmap(sq_ptr_, sring_sz_); + munmap(sqe_ptr_, sqes_sz_); +} + +uint32_t IOUring::load_cq_head() { return io_uring_atomic_read(cq_head_ptr_); } + +uint32_t IOUring::load_cq_tail() { return io_uring_atomic_read(cq_tail_ptr_); } + +uint32_t IOUring::load_sq_head() { return io_uring_atomic_read(sq_head_ptr_); } + +uint32_t IOUring::load_sq_tail() { return io_uring_atomic_read(sq_tail_ptr_); } + +uint32_t IOUring::load_cq_overflow() { + return io_uring_atomic_read(cq_overflow_ptr_); +} + +uint32_t IOUring::load_sq_dropped() { + return io_uring_atomic_read(sq_dropped_ptr_); +} + +void IOUring::store_cq_head(uint32_t cq_head_val) { + io_uring_atomic_write(cq_head_ptr_, cq_head_val); +} + +void IOUring::store_sq_tail(uint32_t sq_tail_val) { + io_uring_atomic_write(sq_tail_ptr_, sq_tail_val); +} + +int IOUring::Enter(unsigned int to_submit, unsigned int min_complete, + unsigned int flags, sigset_t *sig) { + return IOUringEnter(iouringfd_.get(), to_submit, min_complete, flags, sig); +} + +IOUringCqe *IOUring::get_cqes() { return cqes_; } + +IOUringSqe *IOUring::get_sqes() { + return reinterpret_cast(sqe_ptr_); +} + +uint32_t IOUring::get_sq_mask() { return sq_mask_; } + +unsigned *IOUring::get_sq_array() { return sq_array_; } + +} // namespace testing +} // namespace gvisor diff --git a/test/util/io_uring_util.h b/test/util/io_uring_util.h index a7afa7e80..0b93badec 100644 --- a/test/util/io_uring_util.h +++ b/test/util/io_uring_util.h @@ -15,6 +15,10 @@ #ifndef GVISOR_TEST_UTIL_IOURING_UTIL_H_ #define GVISOR_TEST_UTIL_IOURING_UTIL_H_ +#include +#include + +#include #include #include @@ -26,6 +30,7 @@ namespace gvisor { namespace testing { #define __NR_io_uring_setup 425 +#define __NR_io_uring_enter 426 // io_uring_setup(2) flags. #define IORING_SETUP_SQPOLL (1U << 1) @@ -36,6 +41,9 @@ namespace testing { #define IORING_OFF_CQ_RING 0x8000000ULL #define IORING_OFF_SQES 0x10000000ULL +// IO_URING operation codes. +#define IORING_OP_NOP 0 + struct io_sqring_offsets { uint32_t head; uint32_t tail; @@ -73,14 +81,136 @@ struct io_uring_params { struct io_cqring_offsets cq_off; }; +struct io_uring_cqe { + uint64_t user_data; + int32_t res; + uint32_t flags; +}; + +struct io_uring_sqe { + uint8_t opcode; + uint8_t flags; + uint16_t ioprio; + int32_t fd; + union { + uint64_t off; + uint64_t addr2; + struct { + uint32_t cmd_op; + uint32_t __pad1; + }; + }; + union { + uint64_t addr; + uint64_t splice_off_in; + }; + uint32_t len; + union { + __kernel_rwf_t rw_flags; + uint32_t fsync_flags; + uint16_t poll_events; + uint32_t poll32_events; + uint32_t sync_range_flags; + uint32_t msg_flags; + uint32_t timeout_flags; + uint32_t accept_flags; + uint32_t cancel_flags; + uint32_t open_flags; + uint32_t statx_flags; + uint32_t fadvise_advice; + uint32_t splice_flags; + uint32_t rename_flags; + uint32_t unlink_flags; + uint32_t hardlink_flags; + uint32_t xattr_flags; + }; + uint64_t user_data; + union { + uint16_t buf_index; + uint16_t buf_group; + } __attribute__((packed)); + uint16_t personality; + union { + int32_t splice_fd_in; + uint32_t file_index; + }; + union { + struct { + uint64_t addr3; + uint64_t __pad2[1]; + }; + uint8_t cmd[0]; + }; +}; + +using IOSqringOffsets = struct io_sqring_offsets; +using ICqringOffsets = struct io_cqring_offsets; +using IOUringCqe = struct io_uring_cqe; +using IOUringParams = struct io_uring_params; +using IOUringSqe = struct io_uring_sqe; + +// Helper class for IO_URING +class IOUring { + public: + IOUring() = delete; + IOUring(FileDescriptor &&fd, unsigned int entries, IOUringParams ¶ms); + ~IOUring(); + + static PosixErrorOr> InitIOUring( + unsigned int entries, IOUringParams ¶ms); + + uint32_t load_cq_head(); + uint32_t load_cq_tail(); + uint32_t load_sq_head(); + uint32_t load_sq_tail(); + uint32_t load_cq_overflow(); + uint32_t load_sq_dropped(); + void store_cq_head(uint32_t cq_head_val); + void store_sq_tail(uint32_t sq_tail_val); + int Enter(unsigned int to_submit, unsigned int min_complete, + unsigned int flags, sigset_t *sig); + + IOUringCqe *get_cqes(); + IOUringSqe *get_sqes(); + uint32_t get_sq_mask(); + unsigned *get_sq_array(); + + int Fd() { return iouringfd_.get(); } + + private: + IOUringCqe *cqes_ = nullptr; + FileDescriptor iouringfd_; + size_t cring_sz_; + size_t sring_sz_; + size_t sqes_sz_; + uint32_t sq_mask_; + unsigned *sq_array_ = nullptr; + uint32_t *cq_head_ptr_ = nullptr; + uint32_t *cq_tail_ptr_ = nullptr; + uint32_t *sq_head_ptr_ = nullptr; + uint32_t *sq_tail_ptr_ = nullptr; + uint32_t *cq_overflow_ptr_ = nullptr; + uint32_t *sq_dropped_ptr_ = nullptr; + void *sq_ptr_ = nullptr; + void *cq_ptr_ = nullptr; + void *sqe_ptr_ = nullptr; +}; + // This is a wrapper for the io_uring_setup(2) system call. -inline int IOUringSetup(uint32_t entries, struct io_uring_params* params) { +inline int IOUringSetup(uint32_t entries, IOUringParams *params) { return syscall(__NR_io_uring_setup, entries, params); } +// This is a wrapper for the io_uring_enter(2) system call. +inline int IOUringEnter(unsigned int fd, unsigned int to_submit, + unsigned int min_complete, unsigned int flags, + sigset_t *sig) { + return syscall(__NR_io_uring_enter, fd, to_submit, min_complete, flags, sig); +} + // Returns a new iouringfd with the given number of entries. -inline PosixErrorOr NewIOUringFD( - uint32_t entries, struct io_uring_params& params) { +inline PosixErrorOr NewIOUringFD(uint32_t entries, + IOUringParams ¶ms) { memset(¶ms, 0, sizeof(params)); int fd = IOUringSetup(entries, ¶ms); MaybeSave(); @@ -90,6 +220,18 @@ inline PosixErrorOr NewIOUringFD( return FileDescriptor(fd); } +template +static inline void io_uring_atomic_write(T *p, T v) { + std::atomic_store_explicit(reinterpret_cast *>(p), v, + std::memory_order_release); +} + +template +static inline T io_uring_atomic_read(const T *p) { + return std::atomic_load_explicit(reinterpret_cast *>(p), + std::memory_order_acquire); +} + } // namespace testing } // namespace gvisor