Disable io_uring syscalls by default.

The current io_uring support is very limited and experimental. Disable
it by default, and add a flag to enable it for testing.

PiperOrigin-RevId: 500760451

pkg/abi/linux/iouring.go

@@ -126,6 +126,7 @@ type IOUringParams struct {
 // See struct io_uring_cqe in include/uapi/linux/io_uring.h.
 //
 // +marshal
+// +stateify savable
 type IOUringCqe struct {
 	UserData uint64
 	Res      int32
@@ -136,6 +137,7 @@ type IOUringCqe struct {
 // See struct io_uring in io_uring/io_uring.c.
 //
 // +marshal
+// +stateify savable
 type IOUring struct {
 	// Both head and tail should be cacheline aligned. And we assume that
 	// cacheline size is 64 bytes.
@@ -150,15 +152,19 @@ type IOUring struct {
 // See struct io_rings in io_uring/io_uring.c.
 //
 // +marshal
+// +stateify savable
 type IORings struct {
-	Sq, Cq                       IOUring
-	SqRingMask, CqRingMask       uint32
-	SqRingEntries, CqRingEntries uint32
-	sqDropped                    uint32
-	sqFlags                      int32
-	cqFlags                      uint32
-	CqOverflow                   uint32
-	_                            [32]byte // Padding so cqes is cacheline aligned
+	Sq            IOUring
+	Cq            IOUring
+	SqRingMask    uint32
+	CqRingMask    uint32
+	SqRingEntries uint32
+	CqRingEntries uint32
+	sqDropped     uint32
+	sqFlags       int32
+	cqFlags       uint32
+	CqOverflow    uint32
+	_             [32]byte // Padding so cqes is cacheline aligned
 	// Linux has an additional field struct io_uring_cqe cqes[], which represents
 	// a dynamic array. We don't include it here in order to enable marshalling.
 }
@@ -169,6 +175,7 @@ type IORings struct {
 // See include/uapi/linux/io_uring.h.
 //
 // +marshal
+// +stateify savable
 type IOUringSqe struct {
 	Opcode              uint8
 	Flags               uint8

pkg/sentry/fsimpl/iouringfs/iouringfs.go

@@ -50,6 +50,8 @@ type FileDescription struct {
 	vfs.DentryMetadataFileDescriptionImpl
 	vfs.NoLockFD
 
+	mfp pgalloc.MemoryFileProvider
+
 	rbmf  ringsBufferFile
 	sqemf sqEntriesFile
 
@@ -62,9 +64,13 @@ type FileDescription struct {
 
 	ioRings linux.IORings
 
-	ioRingsBuf sharedBuffer
-	sqesBuf    sharedBuffer
-	cqesBuf    sharedBuffer
+	ioRingsBuf sharedBuffer `state:"nosave"`
+	sqesBuf    sharedBuffer `state:"nosave"`
+	cqesBuf    sharedBuffer `state:"nosave"`
+
+	// remap indicates whether the shared buffers need to be remapped
+	// due to a S/R. Protected by ProcessSubmissions critical section.
+	remap bool
 }
 
 var _ vfs.FileDescriptionImpl = (*FileDescription)(nil)
@@ -117,7 +123,9 @@ func New(ctx context.Context, vfsObj *vfs.VirtualFilesystem, entries uint32, par
 		numSqEntries*uint32((*linux.IORingIndex)(nil).SizeBytes()))
 	ringsBufferSize = uint64(hostarch.Addr(ringsBufferSize).MustRoundUp())
 
-	rbfr, err := mfp.MemoryFile().Allocate(ringsBufferSize, pgalloc.AllocOpts{Kind: usage.Anonymous})
+	mf := mfp.MemoryFile()
+
+	rbfr, err := mf.Allocate(ringsBufferSize, pgalloc.AllocOpts{Kind: usage.Anonymous})
 	if err != nil {
 		return nil, linuxerr.ENOMEM
 	}
@@ -125,18 +133,17 @@ func New(ctx context.Context, vfsObj *vfs.VirtualFilesystem, entries uint32, par
 	// Allocate enough space to store the given number of submission queue entries.
 	sqEntriesSize := uint64(numSqEntries * uint32((*linux.IOUringSqe)(nil).SizeBytes()))
 	sqEntriesSize = uint64(hostarch.Addr(sqEntriesSize).MustRoundUp())
-	sqefr, err := mfp.MemoryFile().Allocate(sqEntriesSize, pgalloc.AllocOpts{Kind: usage.Anonymous})
+	sqefr, err := mf.Allocate(sqEntriesSize, pgalloc.AllocOpts{Kind: usage.Anonymous})
 	if err != nil {
 		return nil, linuxerr.ENOMEM
 	}
 
 	iouringfd := &FileDescription{
+		mfp: mfp,
 		rbmf: ringsBufferFile{
-			mf: mfp.MemoryFile(),
 			fr: rbfr,
 		},
 		sqemf: sqEntriesFile{
-			mf: mfp.MemoryFile(),
 			fr: sqefr,
 		},
 		// See ProcessSubmissions for why the capacity is 1.
@@ -195,6 +202,10 @@ func New(ctx context.Context, vfsObj *vfs.VirtualFilesystem, entries uint32, par
 		return nil, err
 	}
 	iouringfd.ioRings.MarshalUnsafe(view)
+
+	buf := make([]byte, iouringfd.ioRings.SizeBytes())
+	iouringfd.ioRings.MarshalUnsafe(buf)
+
 	if _, err := iouringfd.ioRingsBuf.writeback(iouringfd.ioRings.SizeBytes()); err != nil {
 		return nil, err
 	}
@@ -203,16 +214,19 @@ func New(ctx context.Context, vfsObj *vfs.VirtualFilesystem, entries uint32, par
 }
 
 // Release implements vfs.FileDescriptionImpl.Release.
-func (fd *FileDescription) Release(context.Context) {
-	fd.rbmf.mf.DecRef(fd.rbmf.fr)
-	fd.sqemf.mf.DecRef(fd.sqemf.fr)
+func (fd *FileDescription) Release(ctx context.Context) {
+	mf := pgalloc.MemoryFileProviderFromContext(ctx).MemoryFile()
+	mf.DecRef(fd.rbmf.fr)
+	mf.DecRef(fd.sqemf.fr)
 }
 
 // mapSharedBuffers caches internal mappings for the ring's shared memory
 // regions.
 func (fd *FileDescription) mapSharedBuffers() error {
+	mf := fd.mfp.MemoryFile()
+
 	// Mapping for the IORings header struct.
-	rb, err := fd.rbmf.mf.MapInternal(fd.rbmf.fr, hostarch.ReadWrite)
+	rb, err := mf.MapInternal(fd.rbmf.fr, hostarch.ReadWrite)
 	if err != nil {
 		return err
 	}
@@ -228,7 +242,7 @@ func (fd *FileDescription) mapSharedBuffers() error {
 	fd.cqesBuf.init(cqes)
 
 	// Mapping for the SQEs array.
-	sqes, err := fd.sqemf.mf.MapInternal(fd.sqemf.fr, hostarch.ReadWrite)
+	sqes, err := mf.MapInternal(fd.sqemf.fr, hostarch.ReadWrite)
 	if err != nil {
 		return err
 	}
@@ -320,6 +334,14 @@ func (fd *FileDescription) ProcessSubmissions(t *kernel.Task, toSubmit uint32, m
 		}
 	}()
 
+	// The rest of this function is a critical section with respect to
+	// concurrent callers.
+
+	if fd.remap {
+		fd.mapSharedBuffers()
+		fd.remap = false
+	}
+
 	var err error
 	var sqe linux.IOUringSqe
 
@@ -524,8 +546,9 @@ func (fd *FileDescription) updateCq(cqes *safemem.BlockSeq, cqe *linux.IOUringCq
 }
 
 // sqEntriesFile implements memmap.Mappable for SQ entries.
+//
+// +stateify savable
 type sqEntriesFile struct {
-	mf *pgalloc.MemoryFile
 	fr memmap.FileRange
 }
 
@@ -553,7 +576,7 @@ func (sqemf *sqEntriesFile) Translate(ctx context.Context, required, optional me
 		return []memmap.Translation{
 			{
 				Source: source,
-				File:   sqemf.mf,
+				File:   pgalloc.MemoryFileProviderFromContext(ctx).MemoryFile(),
 				Offset: sqemf.fr.Start + source.Start,
 				Perms:  at,
 			},
@@ -569,8 +592,9 @@ func (sqemf *sqEntriesFile) InvalidateUnsavable(ctx context.Context) error {
 }
 
 // ringBuffersFile implements memmap.Mappable for SQ and CQ ring buffers.
+//
+// +stateify savable
 type ringsBufferFile struct {
-	mf *pgalloc.MemoryFile
 	fr memmap.FileRange
 }
 
@@ -598,7 +622,7 @@ func (rbmf *ringsBufferFile) Translate(ctx context.Context, required, optional m
 		return []memmap.Translation{
 			{
 				Source: source,
-				File:   rbmf.mf,
+				File:   pgalloc.MemoryFileProviderFromContext(ctx).MemoryFile(),
 				Offset: rbmf.fr.Start + source.Start,
 				Perms:  at,
 			},

pkg/sentry/fsimpl/iouringfs/iouringfs_state.go

@@ -23,8 +23,7 @@ func (fd *FileDescription) beforeSave() {
 
 // afterLoad is invoked by stateify.
 func (fd *FileDescription) afterLoad() {
+	// Remap shared buffers.
+	fd.remap = true
 	fd.runC = make(chan struct{}, 1)
-	// Wake up any potential sleepers from before the Save. The Pause for Save
-	// ensured there were no active tasks at save time.
-	fd.runC <- struct{}{}
 }

pkg/sentry/kernel/kernel.go

@@ -75,6 +75,10 @@ import (
 	"gvisor.dev/gvisor/pkg/tcpip"
 )
 
+// IOUringEnabled is set to true when IO_URING is enabled. Added as a global to
+// allow easy access everywhere.
+var IOUringEnabled = false
+
 // userCounters is a set of user counters.
 //
 // +stateify savable

pkg/sentry/syscalls/linux/sys_iouring.go

@@ -25,6 +25,10 @@ import (
 
 // IOUringSetup implements linux syscall io_uring_setup(2).
 func IOUringSetup(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel.SyscallControl, error) {
+	if !kernel.IOUringEnabled {
+		return 0, nil, linuxerr.ENOSYS
+	}
+
 	entries := uint32(args[0].Uint())
 	paramsAddr := args[1].Pointer()
 	var params linux.IOUringParams
@@ -77,6 +81,10 @@ func IOUringSetup(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel.
 
 // IOUringEnter implements linux syscall io_uring_enter(2).
 func IOUringEnter(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel.SyscallControl, error) {
+	if !kernel.IOUringEnabled {
+		return 0, nil, linuxerr.ENOSYS
+	}
+
 	fd := int32(args[0].Int())
 	toSubmit := uint32(args[1].Uint())
 	minComplete := uint32(args[2].Uint())

runsc/boot/loader.go

@@ -245,6 +245,8 @@ func New(args Args) (*Loader, error) {
 		return nil, fmt.Errorf("setting up memory usage: %w", err)
 	}
 
+	kernel.IOUringEnabled = args.Conf.IOUring
+
 	// Make host FDs stable between invocations. Host FDs must map to the exact
 	// same number when the sandbox is restored. Otherwise the wrong FD will be
 	// used.

runsc/cli/main.go

@@ -229,6 +229,7 @@ func Main(version string) {
 	log.Infof("\t\tOverlay: Root=%t, SubMounts=%t, FilestoreDir=%q", overlay2.RootMount, overlay2.SubMounts, overlay2.FilestoreDir)
 	log.Infof("\t\tNetwork: %v, logging: %t", conf.Network, conf.LogPackets)
 	log.Infof("\t\tStrace: %t, max size: %d, syscalls: %s", conf.Strace, conf.StraceLogSize, conf.StraceSyscalls)
+	log.Infof("\t\tIOURING: %t", conf.IOUring)
 	log.Infof("\t\tDebug: %v", conf.Debug)
 	log.Infof("\t\tSystemd: %v", conf.SystemdCgroup)
 	log.Infof("***************************")

runsc/config/config.go

@@ -262,6 +262,10 @@ type Config struct {
 	// used.
 	DCache int `flag:"dcache"`
 
+	// IOUring enables support for the IO_URING API calls to perform
+	// asynchronous I/O operations.
+	IOUring bool `flag:"iouring"`
+
 	// TestOnlyAllowRunAsCurrentUserWithoutChroot should only be used in
 	// tests. It allows runsc to start the sandbox process as the current
 	// user, and without chrooting the sandbox process. This can be

runsc/config/flags.go

@@ -94,6 +94,7 @@ func RegisterFlags(flagSet *flag.FlagSet) {
 	flagSet.Bool("ignore-cgroups", false, "don't configure cgroups.")
 	flagSet.Int("fdlimit", -1, "Specifies a limit on the number of host file descriptors that can be open. Applies separately to the sentry and gofer. Note: each file in the sandbox holds more than one host FD open.")
 	flagSet.Int("dcache", -1, "Set the global dentry cache size. This acts as a coarse-grained control on the number of host FDs simultaneously open by the sentry. If negative, per-mount caches are used.")
+	flagSet.Bool("iouring", false, "TEST ONLY; Enables io_uring syscalls in the sentry. Support is experimental and very limited.")
 
 	// Flags that control sandbox runtime behavior: network related.
 	flagSet.Var(networkTypePtr(NetworkSandbox), "network", "specifies which network to use: sandbox (default), host, none. Using network inside the sandbox is more secure because it's isolated from the host network.")

test/runner/defs.bzl

@@ -69,6 +69,7 @@ def _syscall_test(
         file_access = "exclusive",
         overlay = False,
         add_host_communication = False,
+        iouring = False,
         container = None,
         one_sandbox = True,
         **kwargs):
@@ -137,6 +138,7 @@ def _syscall_test(
         "--debug=" + str(debug),
         "--container=" + str(container),
         "--one-sandbox=" + str(one_sandbox),
+        "--iouring=" + str(iouring),
     ]
 
     # Trace points are platform agnostic, so enable them for ptrace only.
@@ -165,6 +167,7 @@ def syscall_test(
         add_host_communication = False,
         add_hostinet = False,
         one_sandbox = True,
+        iouring = False,
         allow_native = True,
         debug = True,
         container = None,
@@ -179,6 +182,7 @@ def syscall_test(
       add_host_communication: setup UDS and pipe external communication for tests.
       add_hostinet: add a hostinet test.
       one_sandbox: runs each unit test in a new sandbox instance.
+      iouring: enable IO_URING support.
       allow_native: generate a native test variant.
       debug: enable debug output.
       container: Run the test in a container. If None, determined from other information.
@@ -195,6 +199,7 @@ def syscall_test(
             use_tmpfs = False,
             add_host_communication = add_host_communication,
             tags = tags,
+            iouring = iouring,
             debug = debug,
             container = container,
             one_sandbox = one_sandbox,
@@ -208,6 +213,7 @@ def syscall_test(
             use_tmpfs = use_tmpfs,
             add_host_communication = add_host_communication,
             tags = platform_tags + tags,
+            iouring = iouring,
             debug = debug,
             container = container,
             one_sandbox = one_sandbox,
@@ -222,6 +228,7 @@ def syscall_test(
             add_host_communication = add_host_communication,
             tags = platforms.get(default_platform, []) + tags,
             debug = debug,
+            iouring = iouring,
             container = container,
             one_sandbox = one_sandbox,
             overlay = True,
@@ -236,6 +243,7 @@ def syscall_test(
             add_host_communication = add_host_communication,
             tags = platforms.get(default_platform, []) + tags,
             debug = debug,
+            iouring = iouring,
             container = container,
             one_sandbox = one_sandbox,
             **kwargs
@@ -248,6 +256,7 @@ def syscall_test(
             use_tmpfs = use_tmpfs,
             add_host_communication = add_host_communication,
             tags = platforms.get(default_platform, []) + tags,
+            iouring = iouring,
             debug = debug,
             container = container,
             one_sandbox = one_sandbox,

test/runner/main.go

@@ -56,6 +56,7 @@ var (
 	trace              = flag.Bool("trace", false, "enables all trace points")
 
 	addUDSTree = flag.Bool("add-host-communication", false, "expose a tree of UDS and pipe utilities to test communication with the host")
+	ioUring    = flag.Bool("iouring", false, "Enables IO_URING API for asynchronous I/O")
 	// TODO(gvisor.dev/issue/4572): properly support leak checking for runsc, and
 	// set to true as the default for the test runner.
 	leakCheck = flag.Bool("leak-check", false, "check for reference leaks")
@@ -198,6 +199,7 @@ func runRunsc(tc *gtest.TestCase, spec *specs.Spec) error {
 		"-TESTONLY-allow-packet-endpoint-write=true",
 		"-net-raw=true",
 		fmt.Sprintf("-panic-signal=%d", unix.SIGTERM),
+		fmt.Sprintf("-iouring=%t", *ioUring),
 		"-watchdog-action=panic",
 		"-platform", *platform,
 		"-file-access", *fileAccess,
@@ -428,11 +430,17 @@ func runTestCaseRunsc(testBin string, tc *gtest.TestCase, args []string, t *test
 	const (
 		platformVar = "TEST_ON_GVISOR"
 		networkVar  = "GVISOR_NETWORK"
+		ioUringVar  = "IOURING_ENABLED"
 	)
 	env := append(os.Environ(), platformVar+"="+*platform, networkVar+"="+*network)
 	if *platformSupport != "" {
 		env = append(env, fmt.Sprintf("%s=%s", platformSupportEnvVar, *platformSupport))
 	}
+	if *ioUring {
+		env = append(env, ioUringVar+"=TRUE")
+	} else {
+		env = append(env, ioUringVar+"=FALSE")
+	}
 
 	// Remove shard env variables so that the gunit binary does not try to
 	// interpret them.

test/syscalls/BUILD

@@ -260,6 +260,7 @@ syscall_test(
 )
 
 syscall_test(
+    iouring = True,
     # Temporarily added due to intermittent ENOMEM failures. See b/216213621.
     tags = ["notap"],
     test = "//test/syscalls/linux:iouring_test",

test/syscalls/linux/io_uring.cc

@@ -25,6 +25,7 @@
 #include <sys/types.h>
 #include <unistd.h>
 
+#include <cerrno>
 #include <cstddef>
 #include <cstdint>
 
@@ -41,6 +42,24 @@ namespace testing {
 
 namespace {
 
+bool IOUringAvailable() {
+  if (IsRunningOnGvisor()) {
+    return true;
+  }
+
+  // io_uring is relatively new and may not be available on all kernels. Probe
+  // using an intentionally invalid call to io_uring_enter.
+  errno = 0;
+  int rc = syscall(__NR_io_uring_enter, -1, 1, 1, 0, nullptr);
+  if (rc != -1) {
+    // How did this succeed?
+    std::cerr << "Probe io_uring_enter(2) with invalid FD somehow succeeded..."
+              << std::endl;
+    return false;
+  }
+  return errno != ENOSYS;
+}
+
 // IOVecContainsString checks that a tuple argument of (struct iovec *, int)
 // corresponding to an iovec array and its length, contains data that matches
 // the string length strlen and the string value str.
@@ -74,12 +93,16 @@ MATCHER_P(IOVecContainsString, str, "") {
 
 // Testing that io_uring_setup(2) successfully returns a valid file descriptor.
 TEST(IOUringTest, ValidFD) {
+  SKIP_IF(!IOUringAvailable());
+
   IOUringParams params;
   FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params));
 }
 
 // Testing that io_uring_setup(2) fails with EINVAL on non-zero params.
 TEST(IOUringTest, ParamsNonZeroResv) {
+  SKIP_IF(!IOUringAvailable());
+
   IOUringParams params;
   memset(&params, 0, sizeof(params));
   params.resv[1] = 1;
@@ -87,6 +110,8 @@ TEST(IOUringTest, ParamsNonZeroResv) {
 }
 
 TEST(IOUringTest, ZeroCQEntries) {
+  SKIP_IF(!IOUringAvailable());
+
   IOUringParams params;
   params.cq_entries = 0;
   params.flags = IORING_SETUP_CQSIZE;
@@ -94,6 +119,8 @@ TEST(IOUringTest, ZeroCQEntries) {
 }
 
 TEST(IOUringTest, ZeroCQEntriesLessThanSQEntries) {
+  SKIP_IF(!IOUringAvailable());
+
   IOUringParams params;
   params.cq_entries = 16;
   params.flags = IORING_SETUP_CQSIZE;
@@ -102,17 +129,20 @@ TEST(IOUringTest, ZeroCQEntriesLessThanSQEntries) {
 
 // Testing that io_uring_setup(2) fails with EINVAL on unsupported flags.
 TEST(IOUringTest, UnsupportedFlags) {
-  if (IsRunningOnGvisor()) {
-    IOUringParams params;
-    memset(&params, 0, sizeof(params));
-    params.flags |= IORING_SETUP_SQPOLL;
-    ASSERT_THAT(IOUringSetup(1, &params), SyscallFailsWithErrno(EINVAL));
-  }
+  // Gvisor only test, since linux supports all flags.
+  SKIP_IF(!IsRunningOnGvisor());
+
+  IOUringParams params;
+  memset(&params, 0, sizeof(params));
+  params.flags |= IORING_SETUP_SQPOLL;
+  ASSERT_THAT(IOUringSetup(1, &params), SyscallFailsWithErrno(EINVAL));
 }
 
 // Testing that both mmap and munmap calls succeed and subsequent access to
 // unmapped memory results in SIGSEGV.
 TEST(IOUringTest, MMapMUnMapWork) {
+  SKIP_IF(!IOUringAvailable());
+
   IOUringParams params;
   FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params));
 
@@ -140,6 +170,8 @@ TEST(IOUringTest, MMapMUnMapWork) {
 
 // Testing that both mmap fails with EINVAL when an invalid offset is passed.
 TEST(IOUringTest, MMapWrongOffset) {
+  SKIP_IF(!IOUringAvailable());
+
   IOUringParams params;
   FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params));
 
@@ -154,6 +186,8 @@ TEST(IOUringTest, MMapWrongOffset) {
 // Testing that mmap() handles all three IO_URING-specific offsets and that
 // returned addresses are page aligned.
 TEST(IOUringTest, MMapOffsets) {
+  SKIP_IF(!IOUringAvailable());
+
   IOUringParams params;
   FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params));
 
@@ -188,37 +222,38 @@ TEST(IOUringTest, MMapOffsets) {
 
 // Testing that IOUringParams are populated with correct values.
 TEST(IOUringTest, ReturnedParamsValues) {
-  if (IsRunningOnGvisor()) {
-    IOUringParams params;
-    FileDescriptor iouringfd =
-        ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params));
+  SKIP_IF(!IsRunningOnGvisor());
 
-    EXPECT_EQ(params.sq_entries, 1);
-    EXPECT_EQ(params.cq_entries, 2);
+  IOUringParams params;
+  FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params));
 
-    EXPECT_EQ(params.sq_off.head, 0);
-    EXPECT_EQ(params.sq_off.tail, 64);
-    EXPECT_EQ(params.sq_off.ring_mask, 256);
-    EXPECT_EQ(params.sq_off.ring_entries, 264);
-    EXPECT_EQ(params.sq_off.flags, 276);
-    EXPECT_EQ(params.sq_off.dropped, 272);
-    EXPECT_EQ(params.sq_off.array, 384);
+  EXPECT_EQ(params.sq_entries, 1);
+  EXPECT_EQ(params.cq_entries, 2);
 
-    EXPECT_EQ(params.cq_off.head, 128);
-    EXPECT_EQ(params.cq_off.tail, 192);
-    EXPECT_EQ(params.cq_off.ring_mask, 260);
-    EXPECT_EQ(params.cq_off.ring_entries, 268);
-    EXPECT_EQ(params.cq_off.overflow, 284);
-    EXPECT_EQ(params.cq_off.cqes, 320);
-    EXPECT_EQ(params.cq_off.flags, 280);
+  EXPECT_EQ(params.sq_off.head, 0);
+  EXPECT_EQ(params.sq_off.tail, 64);
+  EXPECT_EQ(params.sq_off.ring_mask, 256);
+  EXPECT_EQ(params.sq_off.ring_entries, 264);
+  EXPECT_EQ(params.sq_off.flags, 276);
+  EXPECT_EQ(params.sq_off.dropped, 272);
+  EXPECT_EQ(params.sq_off.array, 384);
 
-    // gVisor should support IORING_FEAT_SINGLE_MMAP.
-    EXPECT_NE((params.features & IORING_FEAT_SINGLE_MMAP), 0);
-  }
+  EXPECT_EQ(params.cq_off.head, 128);
+  EXPECT_EQ(params.cq_off.tail, 192);
+  EXPECT_EQ(params.cq_off.ring_mask, 260);
+  EXPECT_EQ(params.cq_off.ring_entries, 268);
+  EXPECT_EQ(params.cq_off.overflow, 284);
+  EXPECT_EQ(params.cq_off.cqes, 320);
+  EXPECT_EQ(params.cq_off.flags, 280);
+
+  // gVisor should support IORING_FEAT_SINGLE_MMAP.
+  EXPECT_NE((params.features & IORING_FEAT_SINGLE_MMAP), 0);
 }
 
 // Testing that offset of SQE indices array is cacheline aligned.
 TEST(IOUringTest, SqeIndexArrayCacheAligned) {
+  SKIP_IF(!IOUringAvailable());
+
   IOUringParams params;
   for (uint32_t i = 1; i < 10; ++i) {
     FileDescriptor iouringfd =
@@ -229,10 +264,15 @@ TEST(IOUringTest, SqeIndexArrayCacheAligned) {
 
 // Testing that io_uring_enter(2) successfully handles a single NOP operation.
 TEST(IOUringTest, SingleNOPTest) {
+  SKIP_IF(!IOUringAvailable());
+
   IOUringParams params;
   std::unique_ptr<IOUring> io_uring =
       ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params));
 
+  ASSERT_EQ(params.sq_entries, 1);
+  ASSERT_EQ(params.cq_entries, 2);
+
   uint32_t sq_head = io_uring->load_sq_head();
   ASSERT_EQ(sq_head, 0);
 
@@ -263,6 +303,8 @@ TEST(IOUringTest, SingleNOPTest) {
 
 // Testing that io_uring_enter(2) successfully queueing NOP operations.
 TEST(IOUringTest, QueueingNOPTest) {
+  SKIP_IF(!IOUringAvailable());
+
   IOUringParams params;
   std::unique_ptr<IOUring> io_uring =
       ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(4, params));
@@ -323,6 +365,8 @@ TEST(IOUringTest, QueueingNOPTest) {
 
 // Testing that io_uring_enter(2) successfully multiple NOP operations.
 TEST(IOUringTest, MultipleNOPTest) {
+  SKIP_IF(!IOUringAvailable());
+
   IOUringParams params;
   std::unique_ptr<IOUring> io_uring =
       ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(4, params));
@@ -367,6 +411,8 @@ TEST(IOUringTest, MultipleNOPTest) {
 // Testing that io_uring_enter(2) successfully handles multiple threads
 // submitting NOP operations.
 TEST(IOUringTest, MultiThreadedNOPTest) {
+  SKIP_IF(!IOUringAvailable());
+
   IOUringParams params;
   std::unique_ptr<IOUring> io_uring =
       ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(4, params));
@@ -415,6 +461,8 @@ TEST(IOUringTest, MultiThreadedNOPTest) {
 // Testing that io_uring_enter(2) successfully consumes submission with an
 // invalid opcode and returned CQE contains EINVAL in its result field.
 TEST(IOUringTest, InvalidOpCodeTest) {
+  SKIP_IF(!IOUringAvailable());
+
   IOUringParams params;
   std::unique_ptr<IOUring> io_uring =
       ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params));
@@ -450,6 +498,8 @@ TEST(IOUringTest, InvalidOpCodeTest) {
 // Tests that filling the shared memory region with garbage data doesn't cause a
 // kernel panic.
 TEST(IOUringTest, CorruptRingHeader) {
+  SKIP_IF(!IOUringAvailable());
+
   const int kEntries = 64;
 
   IOUringParams params;
@@ -493,6 +543,8 @@ TEST(IOUringTest, CorruptRingHeader) {
 // Testing that io_uring_enter(2) successfully consumes submission and SQE ring
 // buffers wrap around.
 TEST(IOUringTest, SQERingBuffersWrapAroundTest) {
+  SKIP_IF(!IOUringAvailable());
+
   IOUringParams params;
   std::unique_ptr<IOUring> io_uring =
       ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(4, params));
@@ -560,114 +612,116 @@ TEST(IOUringTest, SQERingBuffersWrapAroundTest) {
 // Testing that io_uring_enter(2) fails with EFAULT when non-null sigset_t has
 // been passed as we currently don't support replacing signal mask.
 TEST(IOUringTest, NonNullSigsetTest) {
-  if (IsRunningOnGvisor()) {
-    IOUringParams params;
-    std::unique_ptr<IOUring> io_uring =
-        ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params));
+  SKIP_IF(!IsRunningOnGvisor());
 
-    uint32_t sq_head = io_uring->load_sq_head();
-    ASSERT_EQ(sq_head, 0);
+  IOUringParams params;
+  std::unique_ptr<IOUring> io_uring =
+      ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params));
 
-    IOUringSqe *sqe = io_uring->get_sqes();
-    sqe->opcode = IORING_OP_NOP;
-    sqe->user_data = 42;
+  uint32_t sq_head = io_uring->load_sq_head();
+  ASSERT_EQ(sq_head, 0);
 
-    uint32_t sq_tail = io_uring->load_sq_tail();
-    io_uring->store_sq_tail(sq_tail + 1);
+  IOUringSqe *sqe = io_uring->get_sqes();
+  sqe->opcode = IORING_OP_NOP;
+  sqe->user_data = 42;
 
-    sigset_t non_null_sigset;
-    EXPECT_THAT(io_uring->Enter(1, 1, IORING_ENTER_GETEVENTS, &non_null_sigset),
-                SyscallFailsWithErrno(EFAULT));
-  }
+  uint32_t sq_tail = io_uring->load_sq_tail();
+  io_uring->store_sq_tail(sq_tail + 1);
+
+  sigset_t non_null_sigset;
+  EXPECT_THAT(io_uring->Enter(1, 1, IORING_ENTER_GETEVENTS, &non_null_sigset),
+              SyscallFailsWithErrno(EFAULT));
 }
 
 // Testing that completion queue overflow counter is incremented when the
 // completion queue is not drained by the user and completion queue entries are
 // not overwritten.
 TEST(IOUringTest, OverflowCQTest) {
-  if (IsRunningOnGvisor()) {
-    IOUringParams params;
-    std::unique_ptr<IOUring> io_uring =
-        ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(4, params));
+  // Gvisor's completion queue overflow behaviour is different from Linux.
+  SKIP_IF(!IsRunningOnGvisor());
 
-    uint32_t sq_head = io_uring->load_sq_head();
-    ASSERT_EQ(sq_head, 0);
+  IOUringParams params;
+  std::unique_ptr<IOUring> io_uring =
+      ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(4, params));
 
-    unsigned *sq_array = io_uring->get_sq_array();
-    unsigned index = 0;
-    IOUringSqe *sqe = io_uring->get_sqes();
-    IOUringCqe *cqe = io_uring->get_cqes();
+  uint32_t sq_head = io_uring->load_sq_head();
+  ASSERT_EQ(sq_head, 0);
 
-    for (size_t submission_round = 0; submission_round < 2;
-         ++submission_round) {
-      for (size_t i = 0; i < 4; ++i) {
-        sqe[i].opcode = IORING_OP_NOP;
-        sqe[i].user_data = 42 + i + submission_round;
-        index = i & io_uring->get_sq_mask();
-        sq_array[index] = index;
-      }
+  unsigned *sq_array = io_uring->get_sq_array();
+  unsigned index = 0;
+  IOUringSqe *sqe = io_uring->get_sqes();
+  IOUringCqe *cqe = io_uring->get_cqes();
 
-      uint32_t sq_tail = io_uring->load_sq_tail();
-      ASSERT_EQ(sq_tail, 4 * submission_round);
-      io_uring->store_sq_tail(sq_tail + 4);
-
-      int ret = io_uring->Enter(4, 4, IORING_ENTER_GETEVENTS, nullptr);
-      ASSERT_EQ(ret, 4);
-
-      sq_head = io_uring->load_sq_head();
-      ASSERT_EQ(sq_head, 4 * (submission_round + 1));
-
-      uint32_t dropped = io_uring->load_sq_dropped();
-      ASSERT_EQ(dropped, 0);
-
-      uint32_t cq_overflow_counter = io_uring->load_cq_overflow();
-      ASSERT_EQ(cq_overflow_counter, 0);
-
-      uint32_t cq_tail = io_uring->load_cq_tail();
-      ASSERT_EQ(cq_tail, 4 * (submission_round + 1));
-
-      for (size_t i = 0; i < 4; ++i) {
-        ASSERT_EQ(cqe[i + 4 * submission_round].res, 0);
-        ASSERT_EQ(cqe[i + 4 * submission_round].user_data,
-                  42 + i + submission_round);
-      }
-    }
-
-    for (size_t i = 0; i < 2; ++i) {
+  for (size_t submission_round = 0; submission_round < 2; ++submission_round) {
+    for (size_t i = 0; i < 4; ++i) {
       sqe[i].opcode = IORING_OP_NOP;
-      sqe[i].user_data = 52 + i;
+      sqe[i].user_data = 42 + i + submission_round;
       index = i & io_uring->get_sq_mask();
       sq_array[index] = index;
     }
 
     uint32_t sq_tail = io_uring->load_sq_tail();
-    ASSERT_EQ(sq_tail, 8);
-    io_uring->store_sq_tail(sq_tail + 2);
+    ASSERT_EQ(sq_tail, 4 * submission_round);
+    io_uring->store_sq_tail(sq_tail + 4);
 
-    int ret = io_uring->Enter(2, 2, IORING_ENTER_GETEVENTS, nullptr);
-    ASSERT_EQ(ret, 2);
+    int ret = io_uring->Enter(4, 4, IORING_ENTER_GETEVENTS, nullptr);
+    ASSERT_EQ(ret, 4);
 
     sq_head = io_uring->load_sq_head();
-    ASSERT_EQ(sq_head, 10);
-
-    uint32_t cq_tail = io_uring->load_cq_tail();
-    ASSERT_EQ(cq_tail, 8);
-
-    ASSERT_EQ(cqe[0].res, 0);
-    ASSERT_EQ(cqe[0].user_data, 42);
-    ASSERT_EQ(cqe[1].res, 0);
-    ASSERT_EQ(cqe[1].user_data, 43);
+    ASSERT_EQ(sq_head, 4 * (submission_round + 1));
 
     uint32_t dropped = io_uring->load_sq_dropped();
     ASSERT_EQ(dropped, 0);
 
     uint32_t cq_overflow_counter = io_uring->load_cq_overflow();
-    ASSERT_EQ(cq_overflow_counter, 2);
+    ASSERT_EQ(cq_overflow_counter, 0);
+
+    uint32_t cq_tail = io_uring->load_cq_tail();
+    ASSERT_EQ(cq_tail, 4 * (submission_round + 1));
+
+    for (size_t i = 0; i < 4; ++i) {
+      ASSERT_EQ(cqe[i + 4 * submission_round].res, 0);
+      ASSERT_EQ(cqe[i + 4 * submission_round].user_data,
+                42 + i + submission_round);
+    }
   }
+
+  for (size_t i = 0; i < 2; ++i) {
+    sqe[i].opcode = IORING_OP_NOP;
+    sqe[i].user_data = 52 + i;
+    index = i & io_uring->get_sq_mask();
+    sq_array[index] = index;
+  }
+
+  uint32_t sq_tail = io_uring->load_sq_tail();
+  ASSERT_EQ(sq_tail, 8);
+  io_uring->store_sq_tail(sq_tail + 2);
+
+  int ret = io_uring->Enter(2, 2, IORING_ENTER_GETEVENTS, nullptr);
+  ASSERT_EQ(ret, 2);
+
+  sq_head = io_uring->load_sq_head();
+  ASSERT_EQ(sq_head, 10);
+
+  uint32_t cq_tail = io_uring->load_cq_tail();
+  ASSERT_EQ(cq_tail, 8);
+
+  ASSERT_EQ(cqe[0].res, 0);
+  ASSERT_EQ(cqe[0].user_data, 42);
+  ASSERT_EQ(cqe[1].res, 0);
+  ASSERT_EQ(cqe[1].user_data, 43);
+
+  uint32_t dropped = io_uring->load_sq_dropped();
+  ASSERT_EQ(dropped, 0);
+
+  uint32_t cq_overflow_counter = io_uring->load_cq_overflow();
+  ASSERT_EQ(cq_overflow_counter, 2);
 }
 
 // Testing that io_uring_enter(2) successfully handles single READV operation.
 TEST(IOUringTest, SingleREADVTest) {
+  SKIP_IF(!IOUringAvailable());
+
   struct io_uring_params params;
   std::unique_ptr<IOUring> io_uring =
       ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params));
@@ -737,6 +791,8 @@ TEST(IOUringTest, SingleREADVTest) {
 
 // Tests that IORING_OP_READV handles EOF on an empty file correctly.
 TEST(IOUringTest, ReadvEmptyFile) {
+  SKIP_IF(!IOUringAvailable());
+
   struct io_uring_params params;
   std::unique_ptr<IOUring> io_uring =
       ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params));
@@ -790,6 +846,8 @@ TEST(IOUringTest, ReadvEmptyFile) {
 // Testing that io_uring_enter(2) successfully handles three READV operations
 // from three different files submitted through a single invocation.
 TEST(IOUringTest, ThreeREADVSingleEnterTest) {
+  SKIP_IF(!IOUringAvailable());
+
   struct io_uring_params params;
   std::unique_ptr<IOUring> io_uring =
       ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(4, params));
@@ -881,6 +939,8 @@ TEST(IOUringTest, ThreeREADVSingleEnterTest) {
 // Testing that io_uring_enter(2) successfully handles READV operation, which is
 // racing with deletion of the same file.
 TEST(IOUringTest, READVRaceWithDeleteTest) {
+  SKIP_IF(!IOUringAvailable());
+
   struct io_uring_params params;
   std::unique_ptr<IOUring> io_uring =
       ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(2, params));
@@ -986,6 +1046,8 @@ TEST(IOUringTest, READVRaceWithDeleteTest) {
 // Testing that io_uring_enter(2) successfully handles single READV operation
 // with short read situation.
 TEST(IOUringTest, ShortReadREADVTest) {
+  SKIP_IF(!IOUringAvailable());
+
   struct io_uring_params params;
   std::unique_ptr<IOUring> io_uring =
       ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params));
@@ -1058,6 +1120,8 @@ TEST(IOUringTest, ShortReadREADVTest) {
 // Testing that io_uring_enter(2) successfully handles single READV operation
 // when there file does not have read permissions.
 TEST(IOUringTest, NoReadPermissionsREADVTest) {
+  SKIP_IF(!IOUringAvailable());
+
   struct io_uring_params params;
   std::unique_ptr<IOUring> io_uring =
       ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params));
@@ -1132,6 +1196,8 @@ class IOUringSqeFieldsTest : public ::testing::Test,
 // Testing that io_uring_enter(2) successfully handles single READV operation
 // and returns EINVAL error in the CQE when either ioprio or buf_index is set.
 TEST_P(IOUringSqeFieldsTest, READVWithInvalidSqeFieldValue) {
+  SKIP_IF(!IOUringAvailable());
+
   const SqeFieldsUT p = GetParam();
 
   struct io_uring_params params;

test/util/test_util.cc

@@ -41,6 +41,7 @@ namespace gvisor {
 namespace testing {
 
 constexpr char kGvisorNetwork[] = "GVISOR_NETWORK";
+constexpr char kIOUringEnabled[] = "IOURING_ENABLED";
 
 bool IsRunningOnGvisor() { return GvisorPlatform() != Platform::kNative; }
 
@@ -58,6 +59,11 @@ bool IsRunningWithHostinet() {
   return env && strcmp(env, "host") == 0;
 }
 
+bool IsIOUringEnabled() {
+  const char* env = getenv(kIOUringEnabled);
+  return env && strcmp(env, "TRUE") == 0;
+}
+
 // Inline cpuid instruction.  Preserve %ebx/%rbx register. In PIC compilations
 // %ebx contains the address of the global offset table. %rbx is occasionally
 // used to address stack variables in presence of dynamic allocas.

test/util/test_util.h

@@ -224,6 +224,7 @@ constexpr char kFuchsia[] = "fuchsia";
 bool IsRunningOnGvisor();
 const std::string GvisorPlatform();
 bool IsRunningWithHostinet();
+bool IsIOUringEnabled();
 
 #ifdef __linux__
 void SetupGvisorDeathTest();