From ef96e9328ea5bf92aa24875a3c3537b8edf04d81 Mon Sep 17 00:00:00 2001 From: Rahat Mahmood Date: Mon, 9 Jan 2023 11:09:37 -0800 Subject: [PATCH] Disable io_uring syscalls by default. The current io_uring support is very limited and experimental. Disable it by default, and add a flag to enable it for testing. PiperOrigin-RevId: 500760451 --- pkg/abi/linux/iouring.go | 23 +- pkg/sentry/fsimpl/iouringfs/iouringfs.go | 56 +++- .../fsimpl/iouringfs/iouringfs_state.go | 5 +- pkg/sentry/kernel/kernel.go | 4 + pkg/sentry/syscalls/linux/sys_iouring.go | 8 + runsc/boot/loader.go | 2 + runsc/cli/main.go | 1 + runsc/config/config.go | 4 + runsc/config/flags.go | 1 + test/runner/defs.bzl | 9 + test/runner/main.go | 8 + test/syscalls/BUILD | 1 + test/syscalls/linux/io_uring.cc | 274 +++++++++++------- test/util/test_util.cc | 6 + test/util/test_util.h | 1 + 15 files changed, 272 insertions(+), 131 deletions(-) diff --git a/pkg/abi/linux/iouring.go b/pkg/abi/linux/iouring.go index 4a5e2e062..f208d6b7b 100644 --- a/pkg/abi/linux/iouring.go +++ b/pkg/abi/linux/iouring.go @@ -126,6 +126,7 @@ type IOUringParams struct { // See struct io_uring_cqe in include/uapi/linux/io_uring.h. // // +marshal +// +stateify savable type IOUringCqe struct { UserData uint64 Res int32 @@ -136,6 +137,7 @@ type IOUringCqe struct { // See struct io_uring in io_uring/io_uring.c. // // +marshal +// +stateify savable type IOUring struct { // Both head and tail should be cacheline aligned. And we assume that // cacheline size is 64 bytes. @@ -150,15 +152,19 @@ type IOUring struct { // See struct io_rings in io_uring/io_uring.c. // // +marshal +// +stateify savable type IORings struct { - Sq, Cq IOUring - SqRingMask, CqRingMask uint32 - SqRingEntries, CqRingEntries uint32 - sqDropped uint32 - sqFlags int32 - cqFlags uint32 - CqOverflow uint32 - _ [32]byte // Padding so cqes is cacheline aligned + Sq IOUring + Cq IOUring + SqRingMask uint32 + CqRingMask uint32 + SqRingEntries uint32 + CqRingEntries uint32 + sqDropped uint32 + sqFlags int32 + cqFlags uint32 + CqOverflow uint32 + _ [32]byte // Padding so cqes is cacheline aligned // Linux has an additional field struct io_uring_cqe cqes[], which represents // a dynamic array. We don't include it here in order to enable marshalling. } @@ -169,6 +175,7 @@ type IORings struct { // See include/uapi/linux/io_uring.h. // // +marshal +// +stateify savable type IOUringSqe struct { Opcode uint8 Flags uint8 diff --git a/pkg/sentry/fsimpl/iouringfs/iouringfs.go b/pkg/sentry/fsimpl/iouringfs/iouringfs.go index 45c3a1c14..e294a33c7 100644 --- a/pkg/sentry/fsimpl/iouringfs/iouringfs.go +++ b/pkg/sentry/fsimpl/iouringfs/iouringfs.go @@ -50,6 +50,8 @@ type FileDescription struct { vfs.DentryMetadataFileDescriptionImpl vfs.NoLockFD + mfp pgalloc.MemoryFileProvider + rbmf ringsBufferFile sqemf sqEntriesFile @@ -62,9 +64,13 @@ type FileDescription struct { ioRings linux.IORings - ioRingsBuf sharedBuffer - sqesBuf sharedBuffer - cqesBuf sharedBuffer + ioRingsBuf sharedBuffer `state:"nosave"` + sqesBuf sharedBuffer `state:"nosave"` + cqesBuf sharedBuffer `state:"nosave"` + + // remap indicates whether the shared buffers need to be remapped + // due to a S/R. Protected by ProcessSubmissions critical section. + remap bool } var _ vfs.FileDescriptionImpl = (*FileDescription)(nil) @@ -117,7 +123,9 @@ func New(ctx context.Context, vfsObj *vfs.VirtualFilesystem, entries uint32, par numSqEntries*uint32((*linux.IORingIndex)(nil).SizeBytes())) ringsBufferSize = uint64(hostarch.Addr(ringsBufferSize).MustRoundUp()) - rbfr, err := mfp.MemoryFile().Allocate(ringsBufferSize, pgalloc.AllocOpts{Kind: usage.Anonymous}) + mf := mfp.MemoryFile() + + rbfr, err := mf.Allocate(ringsBufferSize, pgalloc.AllocOpts{Kind: usage.Anonymous}) if err != nil { return nil, linuxerr.ENOMEM } @@ -125,18 +133,17 @@ func New(ctx context.Context, vfsObj *vfs.VirtualFilesystem, entries uint32, par // Allocate enough space to store the given number of submission queue entries. sqEntriesSize := uint64(numSqEntries * uint32((*linux.IOUringSqe)(nil).SizeBytes())) sqEntriesSize = uint64(hostarch.Addr(sqEntriesSize).MustRoundUp()) - sqefr, err := mfp.MemoryFile().Allocate(sqEntriesSize, pgalloc.AllocOpts{Kind: usage.Anonymous}) + sqefr, err := mf.Allocate(sqEntriesSize, pgalloc.AllocOpts{Kind: usage.Anonymous}) if err != nil { return nil, linuxerr.ENOMEM } iouringfd := &FileDescription{ + mfp: mfp, rbmf: ringsBufferFile{ - mf: mfp.MemoryFile(), fr: rbfr, }, sqemf: sqEntriesFile{ - mf: mfp.MemoryFile(), fr: sqefr, }, // See ProcessSubmissions for why the capacity is 1. @@ -195,6 +202,10 @@ func New(ctx context.Context, vfsObj *vfs.VirtualFilesystem, entries uint32, par return nil, err } iouringfd.ioRings.MarshalUnsafe(view) + + buf := make([]byte, iouringfd.ioRings.SizeBytes()) + iouringfd.ioRings.MarshalUnsafe(buf) + if _, err := iouringfd.ioRingsBuf.writeback(iouringfd.ioRings.SizeBytes()); err != nil { return nil, err } @@ -203,16 +214,19 @@ func New(ctx context.Context, vfsObj *vfs.VirtualFilesystem, entries uint32, par } // Release implements vfs.FileDescriptionImpl.Release. -func (fd *FileDescription) Release(context.Context) { - fd.rbmf.mf.DecRef(fd.rbmf.fr) - fd.sqemf.mf.DecRef(fd.sqemf.fr) +func (fd *FileDescription) Release(ctx context.Context) { + mf := pgalloc.MemoryFileProviderFromContext(ctx).MemoryFile() + mf.DecRef(fd.rbmf.fr) + mf.DecRef(fd.sqemf.fr) } // mapSharedBuffers caches internal mappings for the ring's shared memory // regions. func (fd *FileDescription) mapSharedBuffers() error { + mf := fd.mfp.MemoryFile() + // Mapping for the IORings header struct. - rb, err := fd.rbmf.mf.MapInternal(fd.rbmf.fr, hostarch.ReadWrite) + rb, err := mf.MapInternal(fd.rbmf.fr, hostarch.ReadWrite) if err != nil { return err } @@ -228,7 +242,7 @@ func (fd *FileDescription) mapSharedBuffers() error { fd.cqesBuf.init(cqes) // Mapping for the SQEs array. - sqes, err := fd.sqemf.mf.MapInternal(fd.sqemf.fr, hostarch.ReadWrite) + sqes, err := mf.MapInternal(fd.sqemf.fr, hostarch.ReadWrite) if err != nil { return err } @@ -320,6 +334,14 @@ func (fd *FileDescription) ProcessSubmissions(t *kernel.Task, toSubmit uint32, m } }() + // The rest of this function is a critical section with respect to + // concurrent callers. + + if fd.remap { + fd.mapSharedBuffers() + fd.remap = false + } + var err error var sqe linux.IOUringSqe @@ -524,8 +546,9 @@ func (fd *FileDescription) updateCq(cqes *safemem.BlockSeq, cqe *linux.IOUringCq } // sqEntriesFile implements memmap.Mappable for SQ entries. +// +// +stateify savable type sqEntriesFile struct { - mf *pgalloc.MemoryFile fr memmap.FileRange } @@ -553,7 +576,7 @@ func (sqemf *sqEntriesFile) Translate(ctx context.Context, required, optional me return []memmap.Translation{ { Source: source, - File: sqemf.mf, + File: pgalloc.MemoryFileProviderFromContext(ctx).MemoryFile(), Offset: sqemf.fr.Start + source.Start, Perms: at, }, @@ -569,8 +592,9 @@ func (sqemf *sqEntriesFile) InvalidateUnsavable(ctx context.Context) error { } // ringBuffersFile implements memmap.Mappable for SQ and CQ ring buffers. +// +// +stateify savable type ringsBufferFile struct { - mf *pgalloc.MemoryFile fr memmap.FileRange } @@ -598,7 +622,7 @@ func (rbmf *ringsBufferFile) Translate(ctx context.Context, required, optional m return []memmap.Translation{ { Source: source, - File: rbmf.mf, + File: pgalloc.MemoryFileProviderFromContext(ctx).MemoryFile(), Offset: rbmf.fr.Start + source.Start, Perms: at, }, diff --git a/pkg/sentry/fsimpl/iouringfs/iouringfs_state.go b/pkg/sentry/fsimpl/iouringfs/iouringfs_state.go index 77c485c59..94d9d5163 100644 --- a/pkg/sentry/fsimpl/iouringfs/iouringfs_state.go +++ b/pkg/sentry/fsimpl/iouringfs/iouringfs_state.go @@ -23,8 +23,7 @@ func (fd *FileDescription) beforeSave() { // afterLoad is invoked by stateify. func (fd *FileDescription) afterLoad() { + // Remap shared buffers. + fd.remap = true fd.runC = make(chan struct{}, 1) - // Wake up any potential sleepers from before the Save. The Pause for Save - // ensured there were no active tasks at save time. - fd.runC <- struct{}{} } diff --git a/pkg/sentry/kernel/kernel.go b/pkg/sentry/kernel/kernel.go index 1d9b24348..260fcc919 100644 --- a/pkg/sentry/kernel/kernel.go +++ b/pkg/sentry/kernel/kernel.go @@ -75,6 +75,10 @@ import ( "gvisor.dev/gvisor/pkg/tcpip" ) +// IOUringEnabled is set to true when IO_URING is enabled. Added as a global to +// allow easy access everywhere. +var IOUringEnabled = false + // userCounters is a set of user counters. // // +stateify savable diff --git a/pkg/sentry/syscalls/linux/sys_iouring.go b/pkg/sentry/syscalls/linux/sys_iouring.go index 5be795ecd..8e356bdb2 100644 --- a/pkg/sentry/syscalls/linux/sys_iouring.go +++ b/pkg/sentry/syscalls/linux/sys_iouring.go @@ -25,6 +25,10 @@ import ( // IOUringSetup implements linux syscall io_uring_setup(2). func IOUringSetup(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel.SyscallControl, error) { + if !kernel.IOUringEnabled { + return 0, nil, linuxerr.ENOSYS + } + entries := uint32(args[0].Uint()) paramsAddr := args[1].Pointer() var params linux.IOUringParams @@ -77,6 +81,10 @@ func IOUringSetup(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel. // IOUringEnter implements linux syscall io_uring_enter(2). func IOUringEnter(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel.SyscallControl, error) { + if !kernel.IOUringEnabled { + return 0, nil, linuxerr.ENOSYS + } + fd := int32(args[0].Int()) toSubmit := uint32(args[1].Uint()) minComplete := uint32(args[2].Uint()) diff --git a/runsc/boot/loader.go b/runsc/boot/loader.go index dd5c9372e..2f183894e 100644 --- a/runsc/boot/loader.go +++ b/runsc/boot/loader.go @@ -245,6 +245,8 @@ func New(args Args) (*Loader, error) { return nil, fmt.Errorf("setting up memory usage: %w", err) } + kernel.IOUringEnabled = args.Conf.IOUring + // Make host FDs stable between invocations. Host FDs must map to the exact // same number when the sandbox is restored. Otherwise the wrong FD will be // used. diff --git a/runsc/cli/main.go b/runsc/cli/main.go index bb8f161d5..198f41958 100644 --- a/runsc/cli/main.go +++ b/runsc/cli/main.go @@ -229,6 +229,7 @@ func Main(version string) { log.Infof("\t\tOverlay: Root=%t, SubMounts=%t, FilestoreDir=%q", overlay2.RootMount, overlay2.SubMounts, overlay2.FilestoreDir) log.Infof("\t\tNetwork: %v, logging: %t", conf.Network, conf.LogPackets) log.Infof("\t\tStrace: %t, max size: %d, syscalls: %s", conf.Strace, conf.StraceLogSize, conf.StraceSyscalls) + log.Infof("\t\tIOURING: %t", conf.IOUring) log.Infof("\t\tDebug: %v", conf.Debug) log.Infof("\t\tSystemd: %v", conf.SystemdCgroup) log.Infof("***************************") diff --git a/runsc/config/config.go b/runsc/config/config.go index c43dbe7c9..f8023c63b 100644 --- a/runsc/config/config.go +++ b/runsc/config/config.go @@ -262,6 +262,10 @@ type Config struct { // used. DCache int `flag:"dcache"` + // IOUring enables support for the IO_URING API calls to perform + // asynchronous I/O operations. + IOUring bool `flag:"iouring"` + // TestOnlyAllowRunAsCurrentUserWithoutChroot should only be used in // tests. It allows runsc to start the sandbox process as the current // user, and without chrooting the sandbox process. This can be diff --git a/runsc/config/flags.go b/runsc/config/flags.go index 3df557f3b..3000b1c5b 100644 --- a/runsc/config/flags.go +++ b/runsc/config/flags.go @@ -94,6 +94,7 @@ func RegisterFlags(flagSet *flag.FlagSet) { flagSet.Bool("ignore-cgroups", false, "don't configure cgroups.") flagSet.Int("fdlimit", -1, "Specifies a limit on the number of host file descriptors that can be open. Applies separately to the sentry and gofer. Note: each file in the sandbox holds more than one host FD open.") flagSet.Int("dcache", -1, "Set the global dentry cache size. This acts as a coarse-grained control on the number of host FDs simultaneously open by the sentry. If negative, per-mount caches are used.") + flagSet.Bool("iouring", false, "TEST ONLY; Enables io_uring syscalls in the sentry. Support is experimental and very limited.") // Flags that control sandbox runtime behavior: network related. flagSet.Var(networkTypePtr(NetworkSandbox), "network", "specifies which network to use: sandbox (default), host, none. Using network inside the sandbox is more secure because it's isolated from the host network.") diff --git a/test/runner/defs.bzl b/test/runner/defs.bzl index 8fcfa438a..1277a441b 100644 --- a/test/runner/defs.bzl +++ b/test/runner/defs.bzl @@ -69,6 +69,7 @@ def _syscall_test( file_access = "exclusive", overlay = False, add_host_communication = False, + iouring = False, container = None, one_sandbox = True, **kwargs): @@ -137,6 +138,7 @@ def _syscall_test( "--debug=" + str(debug), "--container=" + str(container), "--one-sandbox=" + str(one_sandbox), + "--iouring=" + str(iouring), ] # Trace points are platform agnostic, so enable them for ptrace only. @@ -165,6 +167,7 @@ def syscall_test( add_host_communication = False, add_hostinet = False, one_sandbox = True, + iouring = False, allow_native = True, debug = True, container = None, @@ -179,6 +182,7 @@ def syscall_test( add_host_communication: setup UDS and pipe external communication for tests. add_hostinet: add a hostinet test. one_sandbox: runs each unit test in a new sandbox instance. + iouring: enable IO_URING support. allow_native: generate a native test variant. debug: enable debug output. container: Run the test in a container. If None, determined from other information. @@ -195,6 +199,7 @@ def syscall_test( use_tmpfs = False, add_host_communication = add_host_communication, tags = tags, + iouring = iouring, debug = debug, container = container, one_sandbox = one_sandbox, @@ -208,6 +213,7 @@ def syscall_test( use_tmpfs = use_tmpfs, add_host_communication = add_host_communication, tags = platform_tags + tags, + iouring = iouring, debug = debug, container = container, one_sandbox = one_sandbox, @@ -222,6 +228,7 @@ def syscall_test( add_host_communication = add_host_communication, tags = platforms.get(default_platform, []) + tags, debug = debug, + iouring = iouring, container = container, one_sandbox = one_sandbox, overlay = True, @@ -236,6 +243,7 @@ def syscall_test( add_host_communication = add_host_communication, tags = platforms.get(default_platform, []) + tags, debug = debug, + iouring = iouring, container = container, one_sandbox = one_sandbox, **kwargs @@ -248,6 +256,7 @@ def syscall_test( use_tmpfs = use_tmpfs, add_host_communication = add_host_communication, tags = platforms.get(default_platform, []) + tags, + iouring = iouring, debug = debug, container = container, one_sandbox = one_sandbox, diff --git a/test/runner/main.go b/test/runner/main.go index c2901c204..caedde091 100644 --- a/test/runner/main.go +++ b/test/runner/main.go @@ -56,6 +56,7 @@ var ( trace = flag.Bool("trace", false, "enables all trace points") addUDSTree = flag.Bool("add-host-communication", false, "expose a tree of UDS and pipe utilities to test communication with the host") + ioUring = flag.Bool("iouring", false, "Enables IO_URING API for asynchronous I/O") // TODO(gvisor.dev/issue/4572): properly support leak checking for runsc, and // set to true as the default for the test runner. leakCheck = flag.Bool("leak-check", false, "check for reference leaks") @@ -198,6 +199,7 @@ func runRunsc(tc *gtest.TestCase, spec *specs.Spec) error { "-TESTONLY-allow-packet-endpoint-write=true", "-net-raw=true", fmt.Sprintf("-panic-signal=%d", unix.SIGTERM), + fmt.Sprintf("-iouring=%t", *ioUring), "-watchdog-action=panic", "-platform", *platform, "-file-access", *fileAccess, @@ -428,11 +430,17 @@ func runTestCaseRunsc(testBin string, tc *gtest.TestCase, args []string, t *test const ( platformVar = "TEST_ON_GVISOR" networkVar = "GVISOR_NETWORK" + ioUringVar = "IOURING_ENABLED" ) env := append(os.Environ(), platformVar+"="+*platform, networkVar+"="+*network) if *platformSupport != "" { env = append(env, fmt.Sprintf("%s=%s", platformSupportEnvVar, *platformSupport)) } + if *ioUring { + env = append(env, ioUringVar+"=TRUE") + } else { + env = append(env, ioUringVar+"=FALSE") + } // Remove shard env variables so that the gunit binary does not try to // interpret them. diff --git a/test/syscalls/BUILD b/test/syscalls/BUILD index 334119814..1cfd7c363 100644 --- a/test/syscalls/BUILD +++ b/test/syscalls/BUILD @@ -260,6 +260,7 @@ syscall_test( ) syscall_test( + iouring = True, # Temporarily added due to intermittent ENOMEM failures. See b/216213621. tags = ["notap"], test = "//test/syscalls/linux:iouring_test", diff --git a/test/syscalls/linux/io_uring.cc b/test/syscalls/linux/io_uring.cc index a52ae9fee..8e37d2391 100644 --- a/test/syscalls/linux/io_uring.cc +++ b/test/syscalls/linux/io_uring.cc @@ -25,6 +25,7 @@ #include #include +#include #include #include @@ -41,6 +42,24 @@ namespace testing { namespace { +bool IOUringAvailable() { + if (IsRunningOnGvisor()) { + return true; + } + + // io_uring is relatively new and may not be available on all kernels. Probe + // using an intentionally invalid call to io_uring_enter. + errno = 0; + int rc = syscall(__NR_io_uring_enter, -1, 1, 1, 0, nullptr); + if (rc != -1) { + // How did this succeed? + std::cerr << "Probe io_uring_enter(2) with invalid FD somehow succeeded..." + << std::endl; + return false; + } + return errno != ENOSYS; +} + // IOVecContainsString checks that a tuple argument of (struct iovec *, int) // corresponding to an iovec array and its length, contains data that matches // the string length strlen and the string value str. @@ -74,12 +93,16 @@ MATCHER_P(IOVecContainsString, str, "") { // Testing that io_uring_setup(2) successfully returns a valid file descriptor. TEST(IOUringTest, ValidFD) { + SKIP_IF(!IOUringAvailable()); + IOUringParams params; FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params)); } // Testing that io_uring_setup(2) fails with EINVAL on non-zero params. TEST(IOUringTest, ParamsNonZeroResv) { + SKIP_IF(!IOUringAvailable()); + IOUringParams params; memset(¶ms, 0, sizeof(params)); params.resv[1] = 1; @@ -87,6 +110,8 @@ TEST(IOUringTest, ParamsNonZeroResv) { } TEST(IOUringTest, ZeroCQEntries) { + SKIP_IF(!IOUringAvailable()); + IOUringParams params; params.cq_entries = 0; params.flags = IORING_SETUP_CQSIZE; @@ -94,6 +119,8 @@ TEST(IOUringTest, ZeroCQEntries) { } TEST(IOUringTest, ZeroCQEntriesLessThanSQEntries) { + SKIP_IF(!IOUringAvailable()); + IOUringParams params; params.cq_entries = 16; params.flags = IORING_SETUP_CQSIZE; @@ -102,17 +129,20 @@ TEST(IOUringTest, ZeroCQEntriesLessThanSQEntries) { // Testing that io_uring_setup(2) fails with EINVAL on unsupported flags. TEST(IOUringTest, UnsupportedFlags) { - if (IsRunningOnGvisor()) { - IOUringParams params; - memset(¶ms, 0, sizeof(params)); - params.flags |= IORING_SETUP_SQPOLL; - ASSERT_THAT(IOUringSetup(1, ¶ms), SyscallFailsWithErrno(EINVAL)); - } + // Gvisor only test, since linux supports all flags. + SKIP_IF(!IsRunningOnGvisor()); + + IOUringParams params; + memset(¶ms, 0, sizeof(params)); + params.flags |= IORING_SETUP_SQPOLL; + ASSERT_THAT(IOUringSetup(1, ¶ms), SyscallFailsWithErrno(EINVAL)); } // Testing that both mmap and munmap calls succeed and subsequent access to // unmapped memory results in SIGSEGV. TEST(IOUringTest, MMapMUnMapWork) { + SKIP_IF(!IOUringAvailable()); + IOUringParams params; FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params)); @@ -140,6 +170,8 @@ TEST(IOUringTest, MMapMUnMapWork) { // Testing that both mmap fails with EINVAL when an invalid offset is passed. TEST(IOUringTest, MMapWrongOffset) { + SKIP_IF(!IOUringAvailable()); + IOUringParams params; FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params)); @@ -154,6 +186,8 @@ TEST(IOUringTest, MMapWrongOffset) { // Testing that mmap() handles all three IO_URING-specific offsets and that // returned addresses are page aligned. TEST(IOUringTest, MMapOffsets) { + SKIP_IF(!IOUringAvailable()); + IOUringParams params; FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params)); @@ -188,37 +222,38 @@ TEST(IOUringTest, MMapOffsets) { // Testing that IOUringParams are populated with correct values. TEST(IOUringTest, ReturnedParamsValues) { - if (IsRunningOnGvisor()) { - IOUringParams params; - FileDescriptor iouringfd = - ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params)); + SKIP_IF(!IsRunningOnGvisor()); - EXPECT_EQ(params.sq_entries, 1); - EXPECT_EQ(params.cq_entries, 2); + IOUringParams params; + FileDescriptor iouringfd = ASSERT_NO_ERRNO_AND_VALUE(NewIOUringFD(1, params)); - EXPECT_EQ(params.sq_off.head, 0); - EXPECT_EQ(params.sq_off.tail, 64); - EXPECT_EQ(params.sq_off.ring_mask, 256); - EXPECT_EQ(params.sq_off.ring_entries, 264); - EXPECT_EQ(params.sq_off.flags, 276); - EXPECT_EQ(params.sq_off.dropped, 272); - EXPECT_EQ(params.sq_off.array, 384); + EXPECT_EQ(params.sq_entries, 1); + EXPECT_EQ(params.cq_entries, 2); - EXPECT_EQ(params.cq_off.head, 128); - EXPECT_EQ(params.cq_off.tail, 192); - EXPECT_EQ(params.cq_off.ring_mask, 260); - EXPECT_EQ(params.cq_off.ring_entries, 268); - EXPECT_EQ(params.cq_off.overflow, 284); - EXPECT_EQ(params.cq_off.cqes, 320); - EXPECT_EQ(params.cq_off.flags, 280); + EXPECT_EQ(params.sq_off.head, 0); + EXPECT_EQ(params.sq_off.tail, 64); + EXPECT_EQ(params.sq_off.ring_mask, 256); + EXPECT_EQ(params.sq_off.ring_entries, 264); + EXPECT_EQ(params.sq_off.flags, 276); + EXPECT_EQ(params.sq_off.dropped, 272); + EXPECT_EQ(params.sq_off.array, 384); - // gVisor should support IORING_FEAT_SINGLE_MMAP. - EXPECT_NE((params.features & IORING_FEAT_SINGLE_MMAP), 0); - } + EXPECT_EQ(params.cq_off.head, 128); + EXPECT_EQ(params.cq_off.tail, 192); + EXPECT_EQ(params.cq_off.ring_mask, 260); + EXPECT_EQ(params.cq_off.ring_entries, 268); + EXPECT_EQ(params.cq_off.overflow, 284); + EXPECT_EQ(params.cq_off.cqes, 320); + EXPECT_EQ(params.cq_off.flags, 280); + + // gVisor should support IORING_FEAT_SINGLE_MMAP. + EXPECT_NE((params.features & IORING_FEAT_SINGLE_MMAP), 0); } // Testing that offset of SQE indices array is cacheline aligned. TEST(IOUringTest, SqeIndexArrayCacheAligned) { + SKIP_IF(!IOUringAvailable()); + IOUringParams params; for (uint32_t i = 1; i < 10; ++i) { FileDescriptor iouringfd = @@ -229,10 +264,15 @@ TEST(IOUringTest, SqeIndexArrayCacheAligned) { // Testing that io_uring_enter(2) successfully handles a single NOP operation. TEST(IOUringTest, SingleNOPTest) { + SKIP_IF(!IOUringAvailable()); + IOUringParams params; std::unique_ptr io_uring = ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params)); + ASSERT_EQ(params.sq_entries, 1); + ASSERT_EQ(params.cq_entries, 2); + uint32_t sq_head = io_uring->load_sq_head(); ASSERT_EQ(sq_head, 0); @@ -263,6 +303,8 @@ TEST(IOUringTest, SingleNOPTest) { // Testing that io_uring_enter(2) successfully queueing NOP operations. TEST(IOUringTest, QueueingNOPTest) { + SKIP_IF(!IOUringAvailable()); + IOUringParams params; std::unique_ptr io_uring = ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(4, params)); @@ -323,6 +365,8 @@ TEST(IOUringTest, QueueingNOPTest) { // Testing that io_uring_enter(2) successfully multiple NOP operations. TEST(IOUringTest, MultipleNOPTest) { + SKIP_IF(!IOUringAvailable()); + IOUringParams params; std::unique_ptr io_uring = ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(4, params)); @@ -367,6 +411,8 @@ TEST(IOUringTest, MultipleNOPTest) { // Testing that io_uring_enter(2) successfully handles multiple threads // submitting NOP operations. TEST(IOUringTest, MultiThreadedNOPTest) { + SKIP_IF(!IOUringAvailable()); + IOUringParams params; std::unique_ptr io_uring = ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(4, params)); @@ -415,6 +461,8 @@ TEST(IOUringTest, MultiThreadedNOPTest) { // Testing that io_uring_enter(2) successfully consumes submission with an // invalid opcode and returned CQE contains EINVAL in its result field. TEST(IOUringTest, InvalidOpCodeTest) { + SKIP_IF(!IOUringAvailable()); + IOUringParams params; std::unique_ptr io_uring = ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params)); @@ -450,6 +498,8 @@ TEST(IOUringTest, InvalidOpCodeTest) { // Tests that filling the shared memory region with garbage data doesn't cause a // kernel panic. TEST(IOUringTest, CorruptRingHeader) { + SKIP_IF(!IOUringAvailable()); + const int kEntries = 64; IOUringParams params; @@ -493,6 +543,8 @@ TEST(IOUringTest, CorruptRingHeader) { // Testing that io_uring_enter(2) successfully consumes submission and SQE ring // buffers wrap around. TEST(IOUringTest, SQERingBuffersWrapAroundTest) { + SKIP_IF(!IOUringAvailable()); + IOUringParams params; std::unique_ptr io_uring = ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(4, params)); @@ -560,114 +612,116 @@ TEST(IOUringTest, SQERingBuffersWrapAroundTest) { // Testing that io_uring_enter(2) fails with EFAULT when non-null sigset_t has // been passed as we currently don't support replacing signal mask. TEST(IOUringTest, NonNullSigsetTest) { - if (IsRunningOnGvisor()) { - IOUringParams params; - std::unique_ptr io_uring = - ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params)); + SKIP_IF(!IsRunningOnGvisor()); - uint32_t sq_head = io_uring->load_sq_head(); - ASSERT_EQ(sq_head, 0); + IOUringParams params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params)); - IOUringSqe *sqe = io_uring->get_sqes(); - sqe->opcode = IORING_OP_NOP; - sqe->user_data = 42; + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); - uint32_t sq_tail = io_uring->load_sq_tail(); - io_uring->store_sq_tail(sq_tail + 1); + IOUringSqe *sqe = io_uring->get_sqes(); + sqe->opcode = IORING_OP_NOP; + sqe->user_data = 42; - sigset_t non_null_sigset; - EXPECT_THAT(io_uring->Enter(1, 1, IORING_ENTER_GETEVENTS, &non_null_sigset), - SyscallFailsWithErrno(EFAULT)); - } + uint32_t sq_tail = io_uring->load_sq_tail(); + io_uring->store_sq_tail(sq_tail + 1); + + sigset_t non_null_sigset; + EXPECT_THAT(io_uring->Enter(1, 1, IORING_ENTER_GETEVENTS, &non_null_sigset), + SyscallFailsWithErrno(EFAULT)); } // Testing that completion queue overflow counter is incremented when the // completion queue is not drained by the user and completion queue entries are // not overwritten. TEST(IOUringTest, OverflowCQTest) { - if (IsRunningOnGvisor()) { - IOUringParams params; - std::unique_ptr io_uring = - ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(4, params)); + // Gvisor's completion queue overflow behaviour is different from Linux. + SKIP_IF(!IsRunningOnGvisor()); - uint32_t sq_head = io_uring->load_sq_head(); - ASSERT_EQ(sq_head, 0); + IOUringParams params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(4, params)); - unsigned *sq_array = io_uring->get_sq_array(); - unsigned index = 0; - IOUringSqe *sqe = io_uring->get_sqes(); - IOUringCqe *cqe = io_uring->get_cqes(); + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); - for (size_t submission_round = 0; submission_round < 2; - ++submission_round) { - for (size_t i = 0; i < 4; ++i) { - sqe[i].opcode = IORING_OP_NOP; - sqe[i].user_data = 42 + i + submission_round; - index = i & io_uring->get_sq_mask(); - sq_array[index] = index; - } + unsigned *sq_array = io_uring->get_sq_array(); + unsigned index = 0; + IOUringSqe *sqe = io_uring->get_sqes(); + IOUringCqe *cqe = io_uring->get_cqes(); - uint32_t sq_tail = io_uring->load_sq_tail(); - ASSERT_EQ(sq_tail, 4 * submission_round); - io_uring->store_sq_tail(sq_tail + 4); - - int ret = io_uring->Enter(4, 4, IORING_ENTER_GETEVENTS, nullptr); - ASSERT_EQ(ret, 4); - - sq_head = io_uring->load_sq_head(); - ASSERT_EQ(sq_head, 4 * (submission_round + 1)); - - uint32_t dropped = io_uring->load_sq_dropped(); - ASSERT_EQ(dropped, 0); - - uint32_t cq_overflow_counter = io_uring->load_cq_overflow(); - ASSERT_EQ(cq_overflow_counter, 0); - - uint32_t cq_tail = io_uring->load_cq_tail(); - ASSERT_EQ(cq_tail, 4 * (submission_round + 1)); - - for (size_t i = 0; i < 4; ++i) { - ASSERT_EQ(cqe[i + 4 * submission_round].res, 0); - ASSERT_EQ(cqe[i + 4 * submission_round].user_data, - 42 + i + submission_round); - } - } - - for (size_t i = 0; i < 2; ++i) { + for (size_t submission_round = 0; submission_round < 2; ++submission_round) { + for (size_t i = 0; i < 4; ++i) { sqe[i].opcode = IORING_OP_NOP; - sqe[i].user_data = 52 + i; + sqe[i].user_data = 42 + i + submission_round; index = i & io_uring->get_sq_mask(); sq_array[index] = index; } uint32_t sq_tail = io_uring->load_sq_tail(); - ASSERT_EQ(sq_tail, 8); - io_uring->store_sq_tail(sq_tail + 2); + ASSERT_EQ(sq_tail, 4 * submission_round); + io_uring->store_sq_tail(sq_tail + 4); - int ret = io_uring->Enter(2, 2, IORING_ENTER_GETEVENTS, nullptr); - ASSERT_EQ(ret, 2); + int ret = io_uring->Enter(4, 4, IORING_ENTER_GETEVENTS, nullptr); + ASSERT_EQ(ret, 4); sq_head = io_uring->load_sq_head(); - ASSERT_EQ(sq_head, 10); - - uint32_t cq_tail = io_uring->load_cq_tail(); - ASSERT_EQ(cq_tail, 8); - - ASSERT_EQ(cqe[0].res, 0); - ASSERT_EQ(cqe[0].user_data, 42); - ASSERT_EQ(cqe[1].res, 0); - ASSERT_EQ(cqe[1].user_data, 43); + ASSERT_EQ(sq_head, 4 * (submission_round + 1)); uint32_t dropped = io_uring->load_sq_dropped(); ASSERT_EQ(dropped, 0); uint32_t cq_overflow_counter = io_uring->load_cq_overflow(); - ASSERT_EQ(cq_overflow_counter, 2); + ASSERT_EQ(cq_overflow_counter, 0); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 4 * (submission_round + 1)); + + for (size_t i = 0; i < 4; ++i) { + ASSERT_EQ(cqe[i + 4 * submission_round].res, 0); + ASSERT_EQ(cqe[i + 4 * submission_round].user_data, + 42 + i + submission_round); + } } + + for (size_t i = 0; i < 2; ++i) { + sqe[i].opcode = IORING_OP_NOP; + sqe[i].user_data = 52 + i; + index = i & io_uring->get_sq_mask(); + sq_array[index] = index; + } + + uint32_t sq_tail = io_uring->load_sq_tail(); + ASSERT_EQ(sq_tail, 8); + io_uring->store_sq_tail(sq_tail + 2); + + int ret = io_uring->Enter(2, 2, IORING_ENTER_GETEVENTS, nullptr); + ASSERT_EQ(ret, 2); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 10); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 8); + + ASSERT_EQ(cqe[0].res, 0); + ASSERT_EQ(cqe[0].user_data, 42); + ASSERT_EQ(cqe[1].res, 0); + ASSERT_EQ(cqe[1].user_data, 43); + + uint32_t dropped = io_uring->load_sq_dropped(); + ASSERT_EQ(dropped, 0); + + uint32_t cq_overflow_counter = io_uring->load_cq_overflow(); + ASSERT_EQ(cq_overflow_counter, 2); } // Testing that io_uring_enter(2) successfully handles single READV operation. TEST(IOUringTest, SingleREADVTest) { + SKIP_IF(!IOUringAvailable()); + struct io_uring_params params; std::unique_ptr io_uring = ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params)); @@ -737,6 +791,8 @@ TEST(IOUringTest, SingleREADVTest) { // Tests that IORING_OP_READV handles EOF on an empty file correctly. TEST(IOUringTest, ReadvEmptyFile) { + SKIP_IF(!IOUringAvailable()); + struct io_uring_params params; std::unique_ptr io_uring = ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params)); @@ -790,6 +846,8 @@ TEST(IOUringTest, ReadvEmptyFile) { // Testing that io_uring_enter(2) successfully handles three READV operations // from three different files submitted through a single invocation. TEST(IOUringTest, ThreeREADVSingleEnterTest) { + SKIP_IF(!IOUringAvailable()); + struct io_uring_params params; std::unique_ptr io_uring = ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(4, params)); @@ -881,6 +939,8 @@ TEST(IOUringTest, ThreeREADVSingleEnterTest) { // Testing that io_uring_enter(2) successfully handles READV operation, which is // racing with deletion of the same file. TEST(IOUringTest, READVRaceWithDeleteTest) { + SKIP_IF(!IOUringAvailable()); + struct io_uring_params params; std::unique_ptr io_uring = ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(2, params)); @@ -986,6 +1046,8 @@ TEST(IOUringTest, READVRaceWithDeleteTest) { // Testing that io_uring_enter(2) successfully handles single READV operation // with short read situation. TEST(IOUringTest, ShortReadREADVTest) { + SKIP_IF(!IOUringAvailable()); + struct io_uring_params params; std::unique_ptr io_uring = ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params)); @@ -1058,6 +1120,8 @@ TEST(IOUringTest, ShortReadREADVTest) { // Testing that io_uring_enter(2) successfully handles single READV operation // when there file does not have read permissions. TEST(IOUringTest, NoReadPermissionsREADVTest) { + SKIP_IF(!IOUringAvailable()); + struct io_uring_params params; std::unique_ptr io_uring = ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params)); @@ -1132,6 +1196,8 @@ class IOUringSqeFieldsTest : public ::testing::Test, // Testing that io_uring_enter(2) successfully handles single READV operation // and returns EINVAL error in the CQE when either ioprio or buf_index is set. TEST_P(IOUringSqeFieldsTest, READVWithInvalidSqeFieldValue) { + SKIP_IF(!IOUringAvailable()); + const SqeFieldsUT p = GetParam(); struct io_uring_params params; diff --git a/test/util/test_util.cc b/test/util/test_util.cc index 4d11fd9c3..1c55eeb15 100644 --- a/test/util/test_util.cc +++ b/test/util/test_util.cc @@ -41,6 +41,7 @@ namespace gvisor { namespace testing { constexpr char kGvisorNetwork[] = "GVISOR_NETWORK"; +constexpr char kIOUringEnabled[] = "IOURING_ENABLED"; bool IsRunningOnGvisor() { return GvisorPlatform() != Platform::kNative; } @@ -58,6 +59,11 @@ bool IsRunningWithHostinet() { return env && strcmp(env, "host") == 0; } +bool IsIOUringEnabled() { + const char* env = getenv(kIOUringEnabled); + return env && strcmp(env, "TRUE") == 0; +} + // Inline cpuid instruction. Preserve %ebx/%rbx register. In PIC compilations // %ebx contains the address of the global offset table. %rbx is occasionally // used to address stack variables in presence of dynamic allocas. diff --git a/test/util/test_util.h b/test/util/test_util.h index b15c80c8a..449f85241 100644 --- a/test/util/test_util.h +++ b/test/util/test_util.h @@ -224,6 +224,7 @@ constexpr char kFuchsia[] = "fuchsia"; bool IsRunningOnGvisor(); const std::string GvisorPlatform(); bool IsRunningWithHostinet(); +bool IsIOUringEnabled(); #ifdef __linux__ void SetupGvisorDeathTest();