From 60f83d99ea1df148ea9ed6e1055f8fe28ca25780 Mon Sep 17 00:00:00 2001 From: Sergey Madaminov Date: Thu, 10 Nov 2022 12:52:40 -0800 Subject: [PATCH] io_uring_enter: adding support for READV operation. Handling NOP operations helped us to introduce facilities to process submission requests. Now we can proceed to adding more meaningful operations. The logical choice for the first one is the READV operation as it is a next operation after NOP and it is also used for the cat-like application example based of IO_URING. PiperOrigin-RevId: 487615618 --- pkg/abi/linux/iouring.go | 23 +- pkg/sentry/fsimpl/iouringfs/BUILD | 2 + pkg/sentry/fsimpl/iouringfs/iouringfs.go | 87 +++- pkg/sentry/syscalls/linux/vfs2/iouringfs.go | 4 +- test/syscalls/linux/io_uring.cc | 541 +++++++++++++++++++- test/util/BUILD | 2 + test/util/io_uring_util.h | 7 + 7 files changed, 630 insertions(+), 36 deletions(-) diff --git a/pkg/abi/linux/iouring.go b/pkg/abi/linux/iouring.go index df27f9b18..4a5e2e062 100644 --- a/pkg/abi/linux/iouring.go +++ b/pkg/abi/linux/iouring.go @@ -26,6 +26,11 @@ const ( IORING_SETUP_SUBMIT_ALL = (1 << 7) ) +// Constants for io_uring_enter(2). See include/uapi/linux/io_uring.h. +const ( + IORING_ENTER_GETEVENTS = (1 << 0) +) + // Constants for IoUringParams.Features. See include/uapi/linux/io_uring.h. const ( IORING_FEAT_SINGLE_MMAP = (1 << 0) @@ -53,6 +58,12 @@ const ( IORING_OFF_SQES = 0x10000000 ) +// Constants for the IO_URING opcodes. See include/uapi/linux/io_uring.h. +const ( + IORING_OP_NOP = 0 + IORING_OP_READV = 1 +) + // IORingIndex represents SQE array indexes. // // +marshal @@ -161,14 +172,14 @@ type IORings struct { type IOUringSqe struct { Opcode uint8 Flags uint8 - ioPrio uint16 - fd int32 - offOrAddrOrCmdOp uint64 - addrOrSpliceOff uint64 - len uint32 + IoPrio uint16 + Fd int32 + OffOrAddrOrCmdOp uint64 + AddrOrSpliceOff uint64 + Len uint32 specialFlags uint32 UserData uint64 - bufIndexOrGroup uint16 + BufIndexOrGroup uint16 personality uint16 spliceFDOrFileIndex int32 addr3 uint64 diff --git a/pkg/sentry/fsimpl/iouringfs/BUILD b/pkg/sentry/fsimpl/iouringfs/BUILD index 817f8a278..9ed27add1 100644 --- a/pkg/sentry/fsimpl/iouringfs/BUILD +++ b/pkg/sentry/fsimpl/iouringfs/BUILD @@ -13,10 +13,12 @@ go_library( "//pkg/errors/linuxerr", "//pkg/hostarch", "//pkg/safemem", + "//pkg/sentry/kernel", "//pkg/sentry/memmap", "//pkg/sentry/pgalloc", "//pkg/sentry/usage", "//pkg/sentry/vfs", + "//pkg/usermem", ], ) diff --git a/pkg/sentry/fsimpl/iouringfs/iouringfs.go b/pkg/sentry/fsimpl/iouringfs/iouringfs.go index a29e23732..735f78357 100644 --- a/pkg/sentry/fsimpl/iouringfs/iouringfs.go +++ b/pkg/sentry/fsimpl/iouringfs/iouringfs.go @@ -32,10 +32,12 @@ import ( "gvisor.dev/gvisor/pkg/errors/linuxerr" "gvisor.dev/gvisor/pkg/hostarch" "gvisor.dev/gvisor/pkg/safemem" + "gvisor.dev/gvisor/pkg/sentry/kernel" "gvisor.dev/gvisor/pkg/sentry/memmap" "gvisor.dev/gvisor/pkg/sentry/pgalloc" "gvisor.dev/gvisor/pkg/sentry/usage" "gvisor.dev/gvisor/pkg/sentry/vfs" + "gvisor.dev/gvisor/pkg/usermem" ) // FileDescription implements vfs.FileDescriptionImpl for file-based IO_URING. @@ -313,7 +315,7 @@ func (fd *FileDescription) ConfigureMMap(ctx context.Context, opts *memmap.MMapO } // ProcessSubmissions processes submission requests. -func (fd *FileDescription) ProcessSubmissions(toSubmit uint32, minComplete uint32, flags uint32) (int, error) { +func (fd *FileDescription) ProcessSubmissions(t *kernel.Task, toSubmit uint32, minComplete uint32, flags uint32) (int, error) { fd.mu.Lock() defer fd.mu.Unlock() @@ -345,10 +347,7 @@ func (fd *FileDescription) ProcessSubmissions(toSubmit uint32, minComplete uint3 return -1, err } - cqe, err := fd.ProcessSubmission(&sqe, flags) - if err != nil { - return -1, err - } + cqe := fd.ProcessSubmission(t, &sqe, flags) sqHead.Add(1) if (cqTail.Load()-cqHead.Load())/ioRings.CqRingEntries == 1 { ioRings.CqOverflow++ @@ -371,21 +370,75 @@ func (fd *FileDescription) ProcessSubmissions(toSubmit uint32, minComplete uint3 } // ProcessSubmission processes a single submission request. -func (fd *FileDescription) ProcessSubmission(sqe *linux.IOUringSqe, flags uint32) (*linux.IOUringCqe, error) { +func (fd *FileDescription) ProcessSubmission(t *kernel.Task, sqe *linux.IOUringSqe, flags uint32) *linux.IOUringCqe { + var ( + cqeErr error + cqeFlags uint32 + retValue int32 + ) + switch op := sqe.Opcode; op { - case 0: // NOP - return &linux.IOUringCqe{ - UserData: sqe.UserData, - Res: 0, - Flags: 0, - }, nil + case linux.IORING_OP_NOP: + // For the NOP operation, we don't do anything special. + case linux.IORING_OP_READV: + retValue, cqeErr = fd.handleReadv(t, sqe, flags) default: // Unsupported operation - return &linux.IOUringCqe{ - UserData: sqe.UserData, - Res: -int32(linuxerr.EINVAL.Errno()), - Flags: 0, - }, nil + retValue = -int32(linuxerr.EINVAL.Errno()) } + + if cqeErr != nil { + retValue = -int32(kernel.ExtractErrno(cqeErr, -1)) + } + + return &linux.IOUringCqe{ + UserData: sqe.UserData, + Res: retValue, + Flags: cqeFlags, + } +} + +// handleReadv handles IORING_OP_READV. +func (fd *FileDescription) handleReadv(t *kernel.Task, sqe *linux.IOUringSqe, flags uint32) (int32, error) { + // Check that a file descriptor is valid. + if sqe.Fd < 0 { + return 0, linuxerr.EBADF + } + // Currently we don't support any flags for the SQEs. + if sqe.Flags != 0 { + return 0, linuxerr.EINVAL + } + // If the file is not seekable then offset must be zero. And currently, we don't support them. + if sqe.OffOrAddrOrCmdOp != 0 { + return 0, linuxerr.EINVAL + } + // ioprio should not be set for the READV operation. + if sqe.IoPrio != 0 { + return 0, linuxerr.EINVAL + } + // buf_index should not be set for the READV operation. + if sqe.BufIndexOrGroup != 0 { + return 0, linuxerr.EINVAL + } + + // AddressSpaceActive is set to true as we are doing this from the task goroutine.And this is a + // case as we currently don't support neither IOPOLL nor SQPOLL modes. + dst, err := t.IovecsIOSequence(hostarch.Addr(sqe.AddrOrSpliceOff), int(sqe.Len), usermem.IOOpts{ + AddressSpaceActive: true, + }) + if err != nil { + return 0, err + } + file := t.GetFileVFS2(sqe.Fd) + if file == nil { + return 0, linuxerr.EBADF + } + defer file.DecRef(t) + n, err := file.PRead(t, dst, 0, vfs.ReadOptions{}) + if err != nil { + return 0, err + } + + return int32(n), nil } // updateCq updates a completion queue by adding a given completion queue entry. diff --git a/pkg/sentry/syscalls/linux/vfs2/iouringfs.go b/pkg/sentry/syscalls/linux/vfs2/iouringfs.go index 50727aec0..b61da35f1 100644 --- a/pkg/sentry/syscalls/linux/vfs2/iouringfs.go +++ b/pkg/sentry/syscalls/linux/vfs2/iouringfs.go @@ -86,7 +86,7 @@ func IOUringEnter(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel. ret := -1 // List of currently supported flags for io_uring_enter(2). - const supportedFlags = 0 // Currently support none + const supportedFlags = linux.IORING_ENTER_GETEVENTS // Since we don't implement everything, we fail explicitly on flags that are unimplemented. if flags|supportedFlags != supportedFlags { @@ -112,7 +112,7 @@ func IOUringEnter(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel. if !ok { return uintptr(ret), nil, linuxerr.EBADF } - ret, err := iouringfd.ProcessSubmissions(toSubmit, minComplete, flags) + ret, err := iouringfd.ProcessSubmissions(t, toSubmit, minComplete, flags) if err != nil { return uintptr(ret), nil, err } diff --git a/test/syscalls/linux/io_uring.cc b/test/syscalls/linux/io_uring.cc index ead75c9e0..47bc4119e 100644 --- a/test/syscalls/linux/io_uring.cc +++ b/test/syscalls/linux/io_uring.cc @@ -12,7 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include +#include #include #include #include @@ -30,6 +32,7 @@ #include "test/util/io_uring_util.h" #include "test/util/memory_util.h" #include "test/util/multiprocess_util.h" +#include "test/util/temp_path.h" #include "test/util/test_util.h" #include "test/util/thread_util.h" @@ -38,6 +41,37 @@ namespace testing { namespace { +// IOVecContainsString checks that a tuple argument of (struct iovec *, int) +// corresponding to an iovec array and its length, contains data that matches +// the string length strlen and the string value str. +MATCHER_P(IOVecContainsString, str, "") { + struct iovec *iovs = arg.first; + int len = strlen(str); + int niov = arg.second; + int offset = 0; + + for (int i = 0; i < niov; i++) { + struct iovec iov = iovs[i]; + if (len < offset) { + *result_listener << "strlen " << len << " < offset " << offset; + return false; + } + if (strncmp(static_cast(iov.iov_base), &str[offset], iov.iov_len)) { + absl::string_view iovec_string(static_cast(iov.iov_base), + iov.iov_len); + *result_listener << iovec_string << " @offset " << offset; + return false; + } + offset += iov.iov_len; + } + if (offset != len) { + *result_listener << offset; + return false; + } + + return true; +} + // Testing that io_uring_setup(2) successfully returns a valid file descriptor. TEST(IOUringTest, ValidFD) { IOUringParams params; @@ -209,7 +243,7 @@ TEST(IOUringTest, SingleNOPTest) { uint32_t sq_tail = io_uring->load_sq_tail(); io_uring->store_sq_tail(sq_tail + 1); - int ret = io_uring->Enter(1, 1, 0, nullptr); + int ret = io_uring->Enter(1, 1, IORING_ENTER_GETEVENTS, nullptr); ASSERT_EQ(ret, 1); IOUringCqe *cqe = io_uring->get_cqes(); @@ -250,7 +284,7 @@ TEST(IOUringTest, QueueingNOPTest) { ASSERT_EQ(sq_tail, 0); io_uring->store_sq_tail(sq_tail + 4); - int ret = io_uring->Enter(2, 2, 0, nullptr); + int ret = io_uring->Enter(2, 2, IORING_ENTER_GETEVENTS, nullptr); ASSERT_EQ(ret, 2); IOUringCqe *cqe = io_uring->get_cqes(); @@ -269,7 +303,7 @@ TEST(IOUringTest, QueueingNOPTest) { uint32_t cq_head = io_uring->load_cq_head(); io_uring->store_cq_head(cq_head + 2); - ret = io_uring->Enter(2, 2, 0, nullptr); + ret = io_uring->Enter(2, 2, IORING_ENTER_GETEVENTS, nullptr); ASSERT_EQ(ret, 2); sq_head = io_uring->load_sq_head(); @@ -310,7 +344,7 @@ TEST(IOUringTest, MultipleNOPTest) { ASSERT_EQ(sq_tail, 0); io_uring->store_sq_tail(sq_tail + 3); - int ret = io_uring->Enter(3, 3, 0, nullptr); + int ret = io_uring->Enter(3, 3, IORING_ENTER_GETEVENTS, nullptr); ASSERT_EQ(ret, 3); IOUringCqe *cqe = io_uring->get_cqes(); @@ -356,7 +390,7 @@ TEST(IOUringTest, MultiThreadedNOPTest) { for (int i = 0; i < 4; i++) { ScopedThread t([&] { IOUring *io_uring_ptr = io_uring.get(); - int ret = io_uring_ptr->Enter(1, 1, 0, nullptr); + int ret = io_uring_ptr->Enter(1, 1, IORING_ENTER_GETEVENTS, nullptr); ASSERT_EQ(ret, 1); }); } @@ -395,7 +429,7 @@ TEST(IOUringTest, InvalidOpCodeTest) { uint32_t sq_tail = io_uring->load_sq_tail(); io_uring->store_sq_tail(sq_tail + 1); - int ret = io_uring->Enter(1, 1, 0, nullptr); + int ret = io_uring->Enter(1, 1, IORING_ENTER_GETEVENTS, nullptr); ASSERT_EQ(ret, 1); IOUringCqe *cqe = io_uring->get_cqes(); @@ -436,7 +470,7 @@ TEST(IOUringTest, SQERingBuffersWrapAroundTest) { uint32_t sq_tail = io_uring->load_sq_tail(); io_uring->store_sq_tail(sq_tail + 4); - int ret = io_uring->Enter(4, 4, 0, nullptr); + int ret = io_uring->Enter(4, 4, IORING_ENTER_GETEVENTS, nullptr); ASSERT_EQ(ret, 4); IOUringCqe *cqe = io_uring->get_cqes(); @@ -462,7 +496,7 @@ TEST(IOUringTest, SQERingBuffersWrapAroundTest) { sq_tail = io_uring->load_sq_tail(); io_uring->store_sq_tail(sq_tail + 4); - ret = io_uring->Enter(4, 4, 0, nullptr); + ret = io_uring->Enter(4, 4, IORING_ENTER_GETEVENTS, nullptr); ASSERT_EQ(ret, 4); sq_head = io_uring->load_sq_head(); @@ -499,7 +533,7 @@ TEST(IOUringTest, NonNullSigsetTest) { io_uring->store_sq_tail(sq_tail + 1); sigset_t non_null_sigset; - EXPECT_THAT(io_uring->Enter(1, 1, 0, &non_null_sigset), + EXPECT_THAT(io_uring->Enter(1, 1, IORING_ENTER_GETEVENTS, &non_null_sigset), SyscallFailsWithErrno(EFAULT)); } } @@ -534,7 +568,7 @@ TEST(IOUringTest, OverflowCQTest) { ASSERT_EQ(sq_tail, 4 * submission_round); io_uring->store_sq_tail(sq_tail + 4); - int ret = io_uring->Enter(4, 4, 0, nullptr); + int ret = io_uring->Enter(4, 4, IORING_ENTER_GETEVENTS, nullptr); ASSERT_EQ(ret, 4); sq_head = io_uring->load_sq_head(); @@ -567,7 +601,7 @@ TEST(IOUringTest, OverflowCQTest) { ASSERT_EQ(sq_tail, 8); io_uring->store_sq_tail(sq_tail + 2); - int ret = io_uring->Enter(2, 2, 0, nullptr); + int ret = io_uring->Enter(2, 2, IORING_ENTER_GETEVENTS, nullptr); ASSERT_EQ(ret, 2); sq_head = io_uring->load_sq_head(); @@ -589,6 +623,491 @@ TEST(IOUringTest, OverflowCQTest) { } } +// Testing that io_uring_enter(2) successfully handles single READV operation. +TEST(IOUringTest, SingleREADVTest) { + struct io_uring_params params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params)); + + ASSERT_EQ(params.sq_entries, 1); + ASSERT_EQ(params.cq_entries, 2); + + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); + + std::string file_name = NewTempAbsPath(); + std::string contents("DEADBEEF"); + ASSERT_NO_ERRNO(CreateWithContents(file_name, contents, 0666)); + + FileDescriptor filefd = ASSERT_NO_ERRNO_AND_VALUE(Open(file_name, O_RDONLY)); + ASSERT_GE(filefd.get(), 0); + + struct stat st = ASSERT_NO_ERRNO_AND_VALUE(Stat(file_name)); + off_t file_sz = st.st_size; + ASSERT_GT(file_sz, 0); + + int num_blocks = (file_sz + BLOCK_SZ - 1) / BLOCK_SZ; + ASSERT_EQ(num_blocks, 1); + + unsigned *sq_array = io_uring->get_sq_array(); + struct io_uring_sqe *sqe = io_uring->get_sqes(); + + struct iovec iov; + iov.iov_len = file_sz; + void *buf; + ASSERT_THAT(posix_memalign(&buf, BLOCK_SZ, BLOCK_SZ), SyscallSucceeds()); + iov.iov_base = buf; + + sqe->flags = 0; + sqe->fd = filefd.get(); + sqe->opcode = IORING_OP_READV; + sqe->addr = reinterpret_cast(&iov); + sqe->len = num_blocks; + sqe->off = 0; + sqe->user_data = reinterpret_cast(&iov); + sq_array[0] = 0; + + uint32_t sq_tail = io_uring->load_sq_tail(); + io_uring->store_sq_tail(sq_tail + 1); + + int ret = io_uring->Enter(1, 1, 0, nullptr); + ASSERT_EQ(ret, 1); + + struct io_uring_cqe *cqe = io_uring->get_cqes(); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 1); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 1); + + ASSERT_EQ(cqe->res, file_sz); + + struct iovec *fi = reinterpret_cast(cqe->user_data); + + std::pair iovec_desc(fi, num_blocks); + EXPECT_THAT(iovec_desc, IOVecContainsString(contents.c_str())); + + uint32_t cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 1); +} + +// Testing that io_uring_enter(2) successfully handles three READV operations +// from three different files submitted through a single invocation. +TEST(IOUringTest, ThreeREADVSingleEnterTest) { + struct io_uring_params params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(4, params)); + + ASSERT_EQ(params.sq_entries, 4); + ASSERT_EQ(params.cq_entries, 8); + + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); + + FileDescriptor filefd[3]; + unsigned *sq_array = io_uring->get_sq_array(); + struct io_uring_sqe *sqe = io_uring->get_sqes(); + off_t file_sz[3]; + int num_blocks[3]; + struct iovec iov[3]; + + for (size_t i = 0; i < 3; i++) { + std::string file_name = NewTempAbsPath(); + std::string contents("DEADBEEF"); + for (size_t j = 0; j < i; ++j) { + contents.append(" DEADBEEF"); + } + ASSERT_NO_ERRNO(CreateWithContents(file_name, contents, 0666)); + + filefd[i] = ASSERT_NO_ERRNO_AND_VALUE(Open(file_name, O_RDONLY)); + ASSERT_GE(filefd[i].get(), 0); + + struct stat st = ASSERT_NO_ERRNO_AND_VALUE(Stat(file_name)); + file_sz[i] = st.st_size; + ASSERT_GT(file_sz[i], 0); + + num_blocks[i] = (file_sz[i] + BLOCK_SZ - 1) / BLOCK_SZ; + ASSERT_EQ(num_blocks[i], 1); + + iov[i].iov_len = file_sz[i]; + void *buf; + ASSERT_THAT(posix_memalign(&buf, BLOCK_SZ, BLOCK_SZ), SyscallSucceeds()); + iov[i].iov_base = buf; + + sqe[i].flags = 0; + sqe[i].fd = filefd[i].get(); + sqe[i].opcode = IORING_OP_READV; + sqe[i].addr = reinterpret_cast(&iov[i]); + sqe[i].len = num_blocks[i]; + sqe[i].off = 0; + sqe[i].user_data = reinterpret_cast(&iov[i]); + sq_array[i] = i; + + uint32_t sq_tail = io_uring->load_sq_tail(); + io_uring->store_sq_tail(sq_tail + 1); + } + + ASSERT_EQ(file_sz[0], 8); + ASSERT_EQ(file_sz[1], 17); + ASSERT_EQ(file_sz[2], 26); + + int ret = io_uring->Enter(3, 3, 0, nullptr); + ASSERT_EQ(ret, 3); + + struct io_uring_cqe *cqe = io_uring->get_cqes(); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 3); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 3); + + ASSERT_EQ(cqe[0].res, file_sz[0]); + ASSERT_EQ(cqe[1].res, file_sz[1]); + ASSERT_EQ(cqe[2].res, file_sz[2]); + + for (size_t i = 0; i < 3; i++) { + struct iovec *fi = reinterpret_cast(cqe->user_data); + + std::string contents("DEADBEEF"); + for (size_t j = 0; j < i; ++j) { + contents.append(" DEADBEEF"); + } + + std::pair iovec_desc(&fi[i], num_blocks[i]); + EXPECT_THAT(iovec_desc, IOVecContainsString(contents.c_str())); + + uint32_t cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 1); + } +} + +// Testing that io_uring_enter(2) successfully handles READV operation, which is +// racing with deletion of the same file. +TEST(IOUringTest, READVRaceWithDeleteTest) { + struct io_uring_params params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(2, params)); + + ASSERT_EQ(params.sq_entries, 2); + ASSERT_EQ(params.cq_entries, 4); + + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); + + std::string file_name[2]; + FileDescriptor filefd[2]; + unsigned *sq_array = io_uring->get_sq_array(); + struct io_uring_sqe *sqe = io_uring->get_sqes(); + off_t file_sz[2]; + int num_blocks[2]; + struct iovec iov[2]; + + for (size_t i = 0; i < 2; i++) { + file_name[i] = NewTempAbsPath(); + std::string contents("DEADBEEF"); + for (size_t j = 0; j < i; ++j) { + contents.append(" DEADBEEF"); + } + ASSERT_NO_ERRNO(CreateWithContents(file_name[i], contents, 0666)); + + filefd[i] = ASSERT_NO_ERRNO_AND_VALUE(Open(file_name[i], O_RDONLY)); + ASSERT_GE(filefd[i].get(), 0); + + struct stat st = ASSERT_NO_ERRNO_AND_VALUE(Stat(file_name[i])); + file_sz[i] = st.st_size; + ASSERT_GT(file_sz[i], 0); + + num_blocks[i] = (file_sz[i] + BLOCK_SZ - 1) / BLOCK_SZ; + ASSERT_EQ(num_blocks[i], 1); + + iov[i].iov_len = file_sz[i]; + void *buf; + ASSERT_THAT(posix_memalign(&buf, BLOCK_SZ, BLOCK_SZ), SyscallSucceeds()); + iov[i].iov_base = buf; + + sqe[i].flags = 0; + sqe[i].fd = filefd[i].get(); + sqe[i].opcode = IORING_OP_READV; + sqe[i].addr = reinterpret_cast(&iov[i]); + sqe[i].len = num_blocks[i]; + sqe[i].off = 0; + sqe[i].user_data = reinterpret_cast(&iov[i]); + sq_array[i] = i; + + uint32_t sq_tail = io_uring->load_sq_tail(); + io_uring->store_sq_tail(sq_tail + 1); + } + + ASSERT_EQ(file_sz[0], 8); + ASSERT_EQ(file_sz[1], 17); + + ScopedThread t1([&] { + IOUring *io_uring_ptr = io_uring.get(); + int ret = io_uring_ptr->Enter(2, 2, IORING_ENTER_GETEVENTS, nullptr); + ASSERT_EQ(ret, 2); + }); + + ScopedThread t2([&] { filefd[0].reset(); }); + + t1.Join(); + t2.Join(); + + struct io_uring_cqe *cqe = io_uring->get_cqes(); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 2); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 2); + + ASSERT_TRUE(cqe[0].res == -EBADF || cqe[0].res == 8); + ASSERT_EQ(cqe[1].res, file_sz[1]); + + for (size_t i = 0; i < 2; i++) { + if (cqe[i].res == -EBADF) { + uint32_t cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 1); + + continue; + } + + struct iovec *fi = reinterpret_cast(cqe->user_data); + + std::string contents("DEADBEEF"); + for (size_t j = 0; j < i; ++j) { + contents.append(" DEADBEEF"); + } + + std::pair iovec_desc(&fi[i], num_blocks[i]); + EXPECT_THAT(iovec_desc, IOVecContainsString(contents.c_str())); + + uint32_t cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 1); + } +} + +// Testing that io_uring_enter(2) successfully handles single READV operation +// with short read situation. +TEST(IOUringTest, ShortReadREADVTest) { + struct io_uring_params params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params)); + + ASSERT_EQ(params.sq_entries, 1); + ASSERT_EQ(params.cq_entries, 2); + + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); + + std::string file_name = NewTempAbsPath(); + std::string contents("DEADBEEF"); + ASSERT_NO_ERRNO(CreateWithContents(file_name, contents, 0666)); + + FileDescriptor filefd = ASSERT_NO_ERRNO_AND_VALUE(Open(file_name, O_RDONLY)); + ASSERT_GE(filefd.get(), 0); + + struct stat st = ASSERT_NO_ERRNO_AND_VALUE(Stat(file_name)); + // Set file size to be twice of its actual size to mimic the short read. + off_t file_sz = 2 * st.st_size; + ASSERT_GT(file_sz, 0); + + int num_blocks = (file_sz + BLOCK_SZ - 1) / BLOCK_SZ; + ASSERT_EQ(num_blocks, 1); + + unsigned *sq_array = io_uring->get_sq_array(); + struct io_uring_sqe *sqe = io_uring->get_sqes(); + + struct iovec iov; + iov.iov_len = file_sz; + void *buf; + ASSERT_THAT(posix_memalign(&buf, BLOCK_SZ, BLOCK_SZ), SyscallSucceeds()); + iov.iov_base = buf; + + sqe->flags = 0; + sqe->fd = filefd.get(); + sqe->opcode = IORING_OP_READV; + sqe->addr = reinterpret_cast(&iov); + sqe->len = num_blocks; + sqe->off = 0; + sqe->user_data = reinterpret_cast(&iov); + sq_array[0] = 0; + + uint32_t sq_tail = io_uring->load_sq_tail(); + io_uring->store_sq_tail(sq_tail + 1); + + int ret = io_uring->Enter(1, 1, 0, nullptr); + ASSERT_EQ(ret, 1); + + struct io_uring_cqe *cqe = io_uring->get_cqes(); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 1); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 1); + + ASSERT_EQ(cqe->res, file_sz / 2); + + struct iovec *fi = reinterpret_cast(cqe->user_data); + fi->iov_len = file_sz / 2; + + std::pair iovec_desc(fi, num_blocks); + EXPECT_THAT(iovec_desc, IOVecContainsString(contents.c_str())); + + uint32_t cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 1); +} + +// Testing that io_uring_enter(2) successfully handles single READV operation +// when there file does not have read permissions. +TEST(IOUringTest, NoReadPermissionsREADVTest) { + struct io_uring_params params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params)); + + ASSERT_EQ(params.sq_entries, 1); + ASSERT_EQ(params.cq_entries, 2); + + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); + + std::string file_name = NewTempAbsPath(); + std::string contents("DEADBEEF"); + ASSERT_NO_ERRNO(CreateWithContents(file_name, contents, 0666)); + + FileDescriptor filefd = ASSERT_NO_ERRNO_AND_VALUE(Open(file_name, O_WRONLY)); + ASSERT_GE(filefd.get(), 0); + + struct stat st = ASSERT_NO_ERRNO_AND_VALUE(Stat(file_name)); + off_t file_sz = st.st_size; + ASSERT_GT(file_sz, 0); + + int num_blocks = (file_sz + BLOCK_SZ - 1) / BLOCK_SZ; + ASSERT_EQ(num_blocks, 1); + + unsigned *sq_array = io_uring->get_sq_array(); + struct io_uring_sqe *sqe = io_uring->get_sqes(); + + struct iovec iov; + iov.iov_len = file_sz; + void *buf; + ASSERT_THAT(posix_memalign(&buf, BLOCK_SZ, BLOCK_SZ), SyscallSucceeds()); + iov.iov_base = buf; + + sqe->flags = 0; + sqe->fd = filefd.get(); + sqe->opcode = IORING_OP_READV; + sqe->addr = reinterpret_cast(&iov); + sqe->len = num_blocks; + sqe->off = 0; + sqe->user_data = reinterpret_cast(&iov); + sq_array[0] = 0; + + uint32_t sq_tail = io_uring->load_sq_tail(); + io_uring->store_sq_tail(sq_tail + 1); + + int ret = io_uring->Enter(1, 1, 0, nullptr); + ASSERT_EQ(ret, 1); + + struct io_uring_cqe *cqe = io_uring->get_cqes(); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 1); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 1); + + ASSERT_EQ(cqe->res, -EBADF); + + uint32_t cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 1); +} + +struct SqeFieldsUT { + uint16_t ioprio; + uint16_t buf_index; +}; + +class IOUringSqeFieldsTest : public ::testing::Test, + public ::testing::WithParamInterface { +}; + +// Testing that io_uring_enter(2) successfully handles single READV operation +// and returns EINVAL error in the CQE when either ioprio or buf_index is set. +TEST_P(IOUringSqeFieldsTest, READVWithInvalidSqeFieldValue) { + const SqeFieldsUT p = GetParam(); + + struct io_uring_params params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params)); + + ASSERT_EQ(params.sq_entries, 1); + ASSERT_EQ(params.cq_entries, 2); + + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); + + std::string file_name = NewTempAbsPath(); + std::string contents("DEADBEEF"); + ASSERT_NO_ERRNO(CreateWithContents(file_name, contents, 0666)); + + FileDescriptor filefd = ASSERT_NO_ERRNO_AND_VALUE(Open(file_name, O_RDONLY)); + ASSERT_GE(filefd.get(), 0); + + struct stat st = ASSERT_NO_ERRNO_AND_VALUE(Stat(file_name)); + off_t file_sz = st.st_size; + ASSERT_GT(file_sz, 0); + + int num_blocks = (file_sz + BLOCK_SZ - 1) / BLOCK_SZ; + ASSERT_EQ(num_blocks, 1); + + unsigned *sq_array = io_uring->get_sq_array(); + struct io_uring_sqe *sqe = io_uring->get_sqes(); + + struct iovec iov; + iov.iov_len = file_sz; + void *buf; + ASSERT_THAT(posix_memalign(&buf, BLOCK_SZ, BLOCK_SZ), SyscallSucceeds()); + iov.iov_base = buf; + + sqe->flags = 0; + sqe->fd = filefd.get(); + sqe->opcode = IORING_OP_READV; + sqe->addr = reinterpret_cast(&iov); + sqe->len = num_blocks; + sqe->off = 0; + sqe->user_data = reinterpret_cast(&iov); + sqe->ioprio = p.ioprio; + sqe->buf_index = p.buf_index; + sq_array[0] = 0; + + uint32_t sq_tail = io_uring->load_sq_tail(); + io_uring->store_sq_tail(sq_tail + 1); + + int ret = io_uring->Enter(1, 1, 0, nullptr); + ASSERT_EQ(ret, 1); + + struct io_uring_cqe *cqe = io_uring->get_cqes(); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 1); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 1); + + ASSERT_EQ(cqe->res, -EINVAL); + + uint32_t cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 1); +} + +INSTANTIATE_TEST_SUITE_P( + IOUringSqeFields, IOUringSqeFieldsTest, + ::testing::Values(SqeFieldsUT{.ioprio = 0, .buf_index = 1}, + SqeFieldsUT{.ioprio = 1, .buf_index = 0})); + } // namespace } // namespace testing diff --git a/test/util/BUILD b/test/util/BUILD index 657ca0899..c9387a49b 100644 --- a/test/util/BUILD +++ b/test/util/BUILD @@ -68,6 +68,8 @@ cc_library( ":file_descriptor", ":posix_error", ":save_util", + "//test/util:temp_path", + "//test/util:test_util", ], ) diff --git a/test/util/io_uring_util.h b/test/util/io_uring_util.h index 5dea5b340..8012071ec 100644 --- a/test/util/io_uring_util.h +++ b/test/util/io_uring_util.h @@ -16,6 +16,7 @@ #define GVISOR_TEST_UTIL_IOURING_UTIL_H_ #include +#include #include #include @@ -36,6 +37,9 @@ namespace testing { #define IORING_SETUP_SQPOLL (1U << 1) #define IORING_SETUP_CQSIZE (1U << 3) +// io_uring_enter(2) flags +#define IORING_ENTER_GETEVENTS (1U << 0) + #define IORING_FEAT_SINGLE_MMAP (1U << 0) #define IORING_OFF_SQ_RING 0ULL @@ -44,6 +48,9 @@ namespace testing { // IO_URING operation codes. #define IORING_OP_NOP 0 +#define IORING_OP_READV 1 + +#define BLOCK_SZ kPageSize struct io_sqring_offsets { uint32_t head;