diff --git a/pkg/sentry/fsimpl/iouringfs/BUILD b/pkg/sentry/fsimpl/iouringfs/BUILD index 9ed27add1..770462080 100644 --- a/pkg/sentry/fsimpl/iouringfs/BUILD +++ b/pkg/sentry/fsimpl/iouringfs/BUILD @@ -4,13 +4,18 @@ licenses(["notice"]) go_library( name = "iouringfs", - srcs = ["iouringfs.go"], + srcs = [ + "buffer.go", + "iouringfs.go", + "iouringfs_unsafe.go", + ], visibility = ["//pkg/sentry:internal"], deps = [ "//pkg/abi/linux", "//pkg/atomicbitops", "//pkg/context", "//pkg/errors/linuxerr", + "//pkg/gohacks", "//pkg/hostarch", "//pkg/safemem", "//pkg/sentry/kernel", @@ -27,4 +32,5 @@ go_test( size = "small", srcs = ["iouringfs_test.go"], library = ":iouringfs", + deps = ["//pkg/hostarch"], ) diff --git a/pkg/sentry/fsimpl/iouringfs/buffer.go b/pkg/sentry/fsimpl/iouringfs/buffer.go new file mode 100644 index 000000000..b7b2e14e3 --- /dev/null +++ b/pkg/sentry/fsimpl/iouringfs/buffer.go @@ -0,0 +1,173 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iouringfs + +import ( + "fmt" + + "gvisor.dev/gvisor/pkg/safemem" +) + +// sharedBuffer represents a memory buffer shared between the sentry and +// userspace. In many cases, this is simply an internal mmap on the underlying +// memory (aka fast mode). However in some cases the mapped region may lie +// across multiple blocks and we need to copy the region into a contiguous +// buffer (aka slow mode). The goal in either case is to present a contiguous +// slice for easy access. +// +// sharedBuffer must be initialized with init before first use. +// +// Example +// ======= +/* +var sb sharedBuffer +bs := MapInternal(...) +sb.init(bs) + +fetch := true + +for !done { + var err error + + // (Re-)Fetch the view. + var view []byte + if fetch { + view, err = sb.view(128) + } + + // Use the view slice to access the region, both for read or write. + someState := dosomething(view[10]) + view[20] = someState & mask + + // Write back the changes. + fetch, err = sb.writeback(128) +} +*/ +// In the above example, in fast mode view returns a slice that points directly +// to the underlying memory and requires no copying. Writeback is a no-op, and +// the view can be reused on subsequent loop iterations (writeback will return +// refetch == false). +// +// In slow mode, view will copy disjoint parts of the region from different +// blocks to a single contiguous slice. Writeback will also required a copy, and +// a new view will have to be fetched on every loop iteration (writeback will +// return refetch == true). +// +// sharedBuffer is *not* thread safe. +type sharedBuffer struct { + bs safemem.BlockSeq + + // copy is allocated once and reused on subsequent calls to view. We don't + // use the Task's copy scratch buffer because these buffers may be accessed + // from a background context. + copy []byte + + // needsWriteback indicates whether we need to copy out back data from the + // slice returned by the last view() call. + needsWriteback bool +} + +// init initializes the sharedBuffer, and must be called before first use. +func (b *sharedBuffer) init(bs safemem.BlockSeq) { + b.bs = bs +} + +func (b *sharedBuffer) valid() bool { + return !b.bs.IsEmpty() +} + +// view returns a slice representing the shared buffer. When done, view must be +// released with either writeback{,Window} or drop. +func (b *sharedBuffer) view(n int) ([]byte, error) { + if uint64(n) > b.bs.NumBytes() { + // Mapping too short? This is a bug. + panic(fmt.Sprintf("iouringfs: mapping too short for requested len: mapping length %v, requested %d", b.bs.NumBytes(), n)) + } + + // Fast path: use mapping directly, no copies required. + h := b.bs.Head() + if h.Len() <= n && !h.NeedSafecopy() { + b.needsWriteback = false + return h.ToSlice()[:n], nil + } + + // Buffer mapped across multiple blocks, or requires safe copy. + if len(b.copy) < n { + b.copy = make([]byte, n) + } + dst := safemem.BlockSeqOf(safemem.BlockFromSafeSlice(b.copy[:n])) + copyN, err := safemem.CopySeq(dst, b.bs) + if err != nil { + return nil, err + } + if copyN != uint64(n) { + // Short copy risks exposing stale data from view buffer. This should never happen. + panic(fmt.Sprintf("iouringfs: short copy for shared buffer view: want %d, got %d", n, copyN)) + } + b.needsWriteback = true + return b.copy, nil +} + +// writeback writes back the changes to the slice returned by the previous view +// call. On return, writeback indicates if the previous view may be reused, or +// needs to be refetched with a new call to view. +// +// Precondition: Must follow a call to view. n must match the value pased to +// view. +// +// Postcondition: Previous view is invalidated whether writeback is successful +// or not. To attempt another modification, a new view may need to be obtained, +// according to refetch. +func (b *sharedBuffer) writeback(n int) (refetch bool, err error) { + return b.writebackWindow(0, n) +} + +// writebackWindow is like writeback, but only writes back a subregion. Useful +// if the caller knows only a small region has been updated, as it reduces how +// much data need to be copied. writebackWindow still potentially invalidates +// the entire view, caller must check refetch to determine if the view needs to +// be refreshed. +func (b *sharedBuffer) writebackWindow(off, len int) (refetch bool, err error) { + if uint64(off+len) > b.bs.NumBytes() { + panic(fmt.Sprintf("iouringfs: requested writeback to shared buffer from offset %d for %d bytes would overflow underlying region of size %d", off, len, b.bs.NumBytes())) + } + + if !b.needsWriteback { + return false, nil + } + + // Existing view invalid after this point. + b.needsWriteback = false + + src := safemem.BlockSeqOf(safemem.BlockFromSafeSlice(b.copy[off : off+len])) + dst := b.bs.DropFirst(off) + copyN, err := safemem.CopySeq(dst, src) + if err != nil { + return true, err + } + if copyN != uint64(len) { + panic(fmt.Sprintf("iouringfs: short copy for shared buffer writeback: want %d, got %d", len, copyN)) + } + return true, nil +} + +// drop releases a view without writeback. Returns whether any existing views +// need to be refetched. Useful when caller is done with a view that doesn't +// need to be modified. +func (b *sharedBuffer) drop() bool { + wb := b.needsWriteback + b.needsWriteback = false + return wb +} diff --git a/pkg/sentry/fsimpl/iouringfs/iouringfs.go b/pkg/sentry/fsimpl/iouringfs/iouringfs.go index 1d549212f..2588b56a8 100644 --- a/pkg/sentry/fsimpl/iouringfs/iouringfs.go +++ b/pkg/sentry/fsimpl/iouringfs/iouringfs.go @@ -28,7 +28,6 @@ import ( "sync" "gvisor.dev/gvisor/pkg/abi/linux" - "gvisor.dev/gvisor/pkg/atomicbitops" "gvisor.dev/gvisor/pkg/context" "gvisor.dev/gvisor/pkg/errors/linuxerr" "gvisor.dev/gvisor/pkg/hostarch" @@ -57,9 +56,11 @@ type FileDescription struct { // mu protects the fields below. mu sync.Mutex `state:"nosave"` - ioRings *safemem.BlockSeq - sqes *safemem.BlockSeq - cqes *safemem.BlockSeq + ioRings linux.IORings + + ioRingsBuf sharedBuffer + sqesBuf sharedBuffer + cqesBuf sharedBuffer } var _ vfs.FileDescriptionImpl = (*FileDescription)(nil) @@ -171,14 +172,24 @@ func New(ctx context.Context, vfsObj *vfs.VirtualFilesystem, entries uint32, par // Set features supported by the current IO_URING implementation. params.Features = linux.IORING_FEAT_SINGLE_MMAP - if err := iouringfd.populateIORings(params); err != nil { + // Map all shared buffers. + if err := iouringfd.mapSharedBuffers(); err != nil { return nil, err } - if err := iouringfd.cacheSqesMapping(); err != nil { + // Initialize IORings struct from params. + iouringfd.ioRings.SqRingMask = params.SqEntries - 1 + iouringfd.ioRings.CqRingMask = params.CqEntries - 1 + iouringfd.ioRings.SqRingEntries = params.SqEntries + iouringfd.ioRings.CqRingEntries = params.CqEntries + + // Write IORings out to shared buffer. + view, err := iouringfd.ioRingsBuf.view(iouringfd.ioRings.SizeBytes()) + if err != nil { return nil, err } - if err := iouringfd.cacheCqesMapping(); err != nil { + iouringfd.ioRings.MarshalUnsafe(view) + if _, err := iouringfd.ioRingsBuf.writeback(iouringfd.ioRings.SizeBytes()); err != nil { return nil, err } @@ -191,111 +202,34 @@ func (fd *FileDescription) Release(context.Context) { fd.sqemf.mf.DecRef(fd.sqemf.fr) } -// unmarshalIORings handles unmarshalling IORings struct considering that there could be more than -// one block in the BlockSeq. -func unmarshalIORings(ioRings *linux.IORings, bs *safemem.BlockSeq) error { - if bs.NumBlocks() == 1 && !bs.Head().NeedSafecopy() { - ioRings.UnmarshalBytes(bs.Head().TakeFirst((*linux.IORings)(nil).SizeBytes()).ToSlice()) - - return nil - } - - buf := make([]byte, (*linux.IORings)(nil).SizeBytes()) - cp, cperr := safemem.CopySeq(safemem.BlockSeqOf(safemem.BlockFromSafeSlice(buf)), *bs) - if cp == 0 { - return cperr - } - ioRings.UnmarshalBytes(buf) - - return nil -} - -// marshalIORings handles marshalling IORings struct considering that there could be more than one -// BlockSeq. -func marshalIORings(ioRings *linux.IORings, bs *safemem.BlockSeq) error { - if bs.NumBlocks() == 1 && !bs.Head().NeedSafecopy() { - ioRings.MarshalBytes(bs.Head().TakeFirst((*linux.IORings)(nil).SizeBytes()).ToSlice()) - } - - buf := make([]byte, (*linux.IORings)(nil).SizeBytes()) - ioRings.MarshalBytes(buf) - cp, cperr := safemem.CopySeq(*bs, safemem.BlockSeqOf(safemem.BlockFromSafeSlice(buf))) - if cp == 0 { - return cperr - } - - return nil -} - -// unmarshalSqe handles unmarshalling SQE struct considering that there could be more than one block -// in the BlockSeq. -func unmarshalSqe(sqe *linux.IOUringSqe, sqes *safemem.BlockSeq, sqHead uint32) error { - sqeSize := uint32((*linux.IOUringSqe)(nil).SizeBytes()) - if sqes.NumBlocks() == 1 && !sqes.Head().NeedSafecopy() { - sqe.UnmarshalBytes(sqes.Head().ToSlice()[sqHead*sqeSize : (sqHead+1)*sqeSize]) - - return nil - } - - buf := make([]byte, sqes.NumBytes()) - cp, cperr := safemem.CopySeq(safemem.BlockSeqOf(safemem.BlockFromSafeSlice(buf[sqHead*sqeSize:(sqHead+1)*sqeSize])), *sqes) - if cp == 0 { - return cperr - } - sqe.UnmarshalBytes(buf) - - return nil -} - -// populateIORings populates IORings struct backed by the allocated memory. -func (fd *FileDescription) populateIORings(params *linux.IOUringParams) error { - bs, err := fd.rbmf.mf.MapInternal(fd.rbmf.fr, hostarch.ReadWrite) +// mapSharedBuffers caches internal mappings for the ring's shared memory +// regions. +func (fd *FileDescription) mapSharedBuffers() error { + // Mapping for the IORings header struct. + rb, err := fd.rbmf.mf.MapInternal(fd.rbmf.fr, hostarch.ReadWrite) if err != nil { return err } + fd.ioRingsBuf.init(rb) - fd.ioRings = &bs - - var ioRings linux.IORings - if err = unmarshalIORings(&ioRings, &bs); err != nil { - return err - } - - ioRings.SqRingMask = params.SqEntries - 1 - ioRings.CqRingMask = params.CqEntries - 1 - ioRings.SqRingEntries = params.SqEntries - ioRings.CqRingEntries = params.CqEntries - - if err = marshalIORings(&ioRings, &bs); err != nil { - return err - } - - return nil -} - -// cacheSqesMapping caches the beginning of an area for the SQEs backed by the allocated memory. -func (fd *FileDescription) cacheSqesMapping() error { - bs, err := fd.sqemf.mf.MapInternal(fd.sqemf.fr, hostarch.ReadWrite) - if err != nil { - return err - } - fd.sqes = &bs - - return nil -} - -// cacheCqesMapping caches the beginning of an area for the CQEs backed by the allocated memory. -func (fd *FileDescription) cacheCqesMapping() error { - bs := *fd.ioRings - cqesOffset := uint64(hostarch.Addr((*linux.IORings)(nil).SizeBytes())) + // Mapping for the CQEs array. This is contiguous to the header struct. + cqesOffset := uint64(fd.ioRings.SizeBytes()) cqesOffset, ok := hostarch.CacheLineRoundUp(cqesOffset) if !ok { return linuxerr.EOVERFLOW } - bs = bs.DropFirst(int(cqesOffset)) - fd.cqes = &bs + cqes := rb.DropFirst(int(cqesOffset)) + fd.cqesBuf.init(cqes) + + // Mapping for the SQEs array. + sqes, err := fd.sqemf.mf.MapInternal(fd.sqemf.fr, hostarch.ReadWrite) + if err != nil { + return err + } + fd.sqesBuf.init(sqes) return nil + } // ConfigureMMap implements vfs.FileDescriptionImpl.ConfigureMMap. @@ -320,52 +254,103 @@ func (fd *FileDescription) ProcessSubmissions(t *kernel.Task, toSubmit uint32, m fd.mu.Lock() defer fd.mu.Unlock() - var ioRings linux.IORings - err := fd.getIORings(&ioRings) - if err != nil { - return -1, err - } - - sqes := fd.sqes - cqes := fd.cqes - + var err error var sqe linux.IOUringSqe - sqHead := atomicbitops.FromUint32(ioRings.Sq.Head) - sqTail := atomicbitops.FromUint32(ioRings.Sq.Tail) - cqHead := atomicbitops.FromUint32(ioRings.Cq.Head) - cqTail := atomicbitops.FromUint32(ioRings.Cq.Tail) + sqOff := linux.PreComputedIOSqRingOffsets() + cqOff := linux.PreComputedIOCqRingOffsets() + sqArraySize := sqe.SizeBytes() * int(fd.ioRings.SqRingEntries) + cqArraySize := (*linux.IOUringCqe)(nil).SizeBytes() * int(fd.ioRings.CqRingEntries) + + // Fetch all buffers initially. + fetchRB := true + fetchSQA := true + fetchCQA := true + + var view, sqaView, cqaView []byte submitted := uint32(0) + for toSubmit > submitted { - sqHeadMasked := sqHead.Load() & ioRings.SqRingMask - cqTailMasked := cqTail.Load() & ioRings.CqRingMask - // This means that the submission queue is empty. + if fetchRB { + view, err = fd.ioRingsBuf.view(fd.ioRings.SizeBytes()) + if err != nil { + return -1, err + } + } + + // Note: The kernel uses sqHead as a cursor and writes cqTail. Userspace + // uses cqHead as a cursor and writes sqTail. + + sqHeadPtr := atomicUint32AtOffset(view, int(sqOff.Head)) + sqTailPtr := atomicUint32AtOffset(view, int(sqOff.Tail)) + cqHeadPtr := atomicUint32AtOffset(view, int(cqOff.Head)) + cqTailPtr := atomicUint32AtOffset(view, int(cqOff.Tail)) + overflowPtr := atomicUint32AtOffset(view, int(cqOff.Overflow)) + + // Load the pointers once, so we work with a stable value. Particularly, + // usersapce can update the SQ tail at any time. + sqHead := sqHeadPtr.Load() + sqTail := sqTailPtr.Load() + + // Is the submission queue is empty? if sqHead == sqTail { return int(submitted), nil } - if err = unmarshalSqe(&sqe, sqes, sqHeadMasked); err != nil { + // We have at least one pending sqe, unmarshal the first from the + // submission queue. + if fetchSQA { + sqaView, err = fd.sqesBuf.view(sqArraySize) + if err != nil { + return -1, err + } + } + sqaOff := int(sqHead&fd.ioRings.SqRingMask) * sqe.SizeBytes() + sqe.UnmarshalUnsafe(sqaView[sqaOff : sqaOff+sqe.SizeBytes()]) + fetchSQA = fd.sqesBuf.drop() + + // Dispatch request from unmarshalled entry. + cqe := fd.ProcessSubmission(t, &sqe, flags) + + // Advance sq head. + sqHeadPtr.Add(1) + + // Load once so we have stable values. Particularly, userspace can + // update the CQ head at any time. + cqHead := cqHeadPtr.Load() + cqTail := cqTailPtr.Load() + + // Marshal response to completion queue. + if (cqTail - cqHead) >= fd.ioRings.CqRingEntries { + // CQ ring full. + fd.ioRings.CqOverflow++ + overflowPtr.Store(fd.ioRings.CqOverflow) + } else { + // Have room in CQ, marshal CQE. + if fetchCQA { + cqaView, err = fd.cqesBuf.view(cqArraySize) + if err != nil { + return -1, err + } + } + cqaOff := int(cqTail&fd.ioRings.CqRingMask) * cqe.SizeBytes() + cqe.MarshalUnsafe(cqaView[cqaOff : cqaOff+cqe.SizeBytes()]) + fetchCQA, err = fd.cqesBuf.writebackWindow(cqaOff, cqe.SizeBytes()) + if err != nil { + return -1, err + } + + // Advance cq tail. + cqTailPtr.Add(1) + } + + fetchRB, err = fd.ioRingsBuf.writeback(fd.ioRings.SizeBytes()) + if err != nil { return -1, err } - cqe := fd.ProcessSubmission(t, &sqe, flags) - sqHead.Add(1) - if (cqTail.Load()-cqHead.Load())/ioRings.CqRingEntries == 1 { - ioRings.CqOverflow++ - } else { - if err = fd.updateCq(cqes, cqe, cqTailMasked); err != nil { - return -1, err - } - cqTail.Add(1) - } submitted++ } - ioRings.Sq.Head = sqHead.Load() - ioRings.Cq.Tail = cqTail.Load() - - if err = marshalIORings(&ioRings, fd.ioRings); err != nil { - return -1, err - } return int(submitted), nil } @@ -466,15 +451,6 @@ func (fd *FileDescription) updateCq(cqes *safemem.BlockSeq, cqe *linux.IOUringCq return nil } -// getIORings unmarshalls IORings struct backed by the allocated memory. -func (fd *FileDescription) getIORings(ioRings *linux.IORings) error { - if err := unmarshalIORings(ioRings, fd.ioRings); err != nil { - return err - } - - return nil -} - // sqEntriesFile implements memmap.Mappable for SQ entries. type sqEntriesFile struct { mf *pgalloc.MemoryFile diff --git a/pkg/sentry/fsimpl/iouringfs/iouringfs_test.go b/pkg/sentry/fsimpl/iouringfs/iouringfs_test.go index 6395edd53..681a830fc 100644 --- a/pkg/sentry/fsimpl/iouringfs/iouringfs_test.go +++ b/pkg/sentry/fsimpl/iouringfs/iouringfs_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 The gVisor Authors. +// Copyright 2022 The gVisor Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,7 +17,10 @@ package iouringfs import ( "fmt" "math" + "strings" "testing" + + "gvisor.dev/gvisor/pkg/hostarch" ) func TestRoundUpPowerOfTwo(t *testing.T) { @@ -67,3 +70,74 @@ func TestRoundUpPowerOfTwoOverflow(t *testing.T) { }) } } + +func TestAtomicUint32AtOffset(t *testing.T) { + buf := make([]byte, 4096) + a := atomicUint32AtOffset(buf, 512) + + want := uint32(123456) + hostarch.ByteOrder.PutUint32(buf[512:], want) + if a.Load() != want { + t.Errorf("Expected %d, got %d", want, a.Load()) + } + + // Update value through slice. + want = 654321 + hostarch.ByteOrder.PutUint32(buf[512:], want) + if a.Load() != want { + t.Errorf("Expected %d, got %d", want, a.Load()) + } + + // Update value through pointer. + want = 789012 + a.Store(want) + if got := hostarch.ByteOrder.Uint32(buf[512:]); got != want { + t.Errorf("Expected %d, got %d", want, got) + } +} + +func TestUint32PtrAtOffsetEndOfSlice(t *testing.T) { + const sizeOfUint32 int = 4 + buf := make([]byte, 4096) + + // Cast successful at end of slice + _ = atomicUint32AtOffset(buf, 4096-sizeOfUint32) +} + +func TestUint32PtrAtOffsetInvalidOffsets(t *testing.T) { + tests := []struct { + offset int + panicSubstr string + }{ + {1, "unaligned"}, + {511, "unaligned"}, + {-1, "overrun"}, + {4093, "overrun"}, + {4094, "overrun"}, + {4095, "overrun"}, + {4096, "overrun"}, + {5000, "overrun"}, + } + const sizeOfUint32 int = 4 + + for i, tt := range tests { + t.Run(fmt.Sprintf("case-%d", i), func(t *testing.T) { + buf := make([]byte, 4096) + + defer func() { + if r := recover(); r != nil { + if strings.Contains(fmt.Sprintf("%s", r), tt.panicSubstr) { + t.Logf("Got expected panic: %v", r) + return + } + + t.Errorf("Unexpected panic: %v", r) + } + }() + + _ = atomicUint32AtOffset(buf, tt.offset) + + t.Errorf("Didn't get expected panic") + }) + } +} diff --git a/pkg/sentry/fsimpl/iouringfs/iouringfs_unsafe.go b/pkg/sentry/fsimpl/iouringfs/iouringfs_unsafe.go new file mode 100644 index 000000000..715ace5d6 --- /dev/null +++ b/pkg/sentry/fsimpl/iouringfs/iouringfs_unsafe.go @@ -0,0 +1,35 @@ +// Copyright 2022 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iouringfs + +import ( + "fmt" + "unsafe" + + "gvisor.dev/gvisor/pkg/atomicbitops" + "gvisor.dev/gvisor/pkg/gohacks" +) + +func atomicUint32AtOffset(buf []byte, offset int) *atomicbitops.Uint32 { + const sizeOfUint32 int = 4 + if offset+sizeOfUint32 > len(buf) || offset < 0 { + panic(fmt.Sprintf("cast at offset %d for slice of len %d would result in overrun", offset, len(buf))) + } + if offset%sizeOfUint32 != 0 { + panic(fmt.Sprintf("cast at offset %d would produce unaligned pointer", offset)) + } + hdr := (*gohacks.SliceHeader)(unsafe.Pointer(&buf)) + return (*atomicbitops.Uint32)(unsafe.Add(hdr.Data, offset)) +} diff --git a/test/syscalls/linux/io_uring.cc b/test/syscalls/linux/io_uring.cc index 54585c8f0..a52ae9fee 100644 --- a/test/syscalls/linux/io_uring.cc +++ b/test/syscalls/linux/io_uring.cc @@ -447,6 +447,49 @@ TEST(IOUringTest, InvalidOpCodeTest) { io_uring->store_cq_head(cq_head + 1); } +// Tests that filling the shared memory region with garbage data doesn't cause a +// kernel panic. +TEST(IOUringTest, CorruptRingHeader) { + const int kEntries = 64; + + IOUringParams params; + FileDescriptor iouringfd = + ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(kEntries, params)); + + int sring_sz = params.sq_off.array + params.sq_entries * sizeof(unsigned); + int cring_sz = params.cq_off.cqes + params.cq_entries * sizeof(IOUringCqe); + int sqes_sz = params.sq_entries * sizeof(IOUringSqe); + + void *sq_ptr = + mmap(0, sring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, + iouringfd.get(), IORING_OFF_SQ_RING); + + void *cq_ptr = + mmap(0, cring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, + iouringfd.get(), IORING_OFF_CQ_RING); + + void *sqe_ptr = + mmap(0, sqes_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, + iouringfd.get(), IORING_OFF_SQES); + + EXPECT_NE(sq_ptr, MAP_FAILED); + EXPECT_NE(cq_ptr, MAP_FAILED); + EXPECT_NE(sqe_ptr, MAP_FAILED); + + // Corrupt all the buffers. + memset(sq_ptr, 0xff, sring_sz); + memset(cq_ptr, 0xff, cring_sz); + memset(sqe_ptr, 0xff, sqes_sz); + + IOUringEnter(iouringfd.get(), 1, 0, IORING_ENTER_GETEVENTS, nullptr); + + // If kernel hasn't panicked, the test succeeds. + + EXPECT_THAT(munmap(sq_ptr, sring_sz), SyscallSucceeds()); + EXPECT_THAT(munmap(cq_ptr, cring_sz), SyscallSucceeds()); + EXPECT_THAT(munmap(sqe_ptr, sizeof(IOUringSqe)), SyscallSucceeds()); +} + // Testing that io_uring_enter(2) successfully consumes submission and SQE ring // buffers wrap around. TEST(IOUringTest, SQERingBuffersWrapAroundTest) {