diff --git a/pkg/abi/linux/iouring.go b/pkg/abi/linux/iouring.go index df27f9b18..4a5e2e062 100644 --- a/pkg/abi/linux/iouring.go +++ b/pkg/abi/linux/iouring.go @@ -26,6 +26,11 @@ const ( IORING_SETUP_SUBMIT_ALL = (1 << 7) ) +// Constants for io_uring_enter(2). See include/uapi/linux/io_uring.h. +const ( + IORING_ENTER_GETEVENTS = (1 << 0) +) + // Constants for IoUringParams.Features. See include/uapi/linux/io_uring.h. const ( IORING_FEAT_SINGLE_MMAP = (1 << 0) @@ -53,6 +58,12 @@ const ( IORING_OFF_SQES = 0x10000000 ) +// Constants for the IO_URING opcodes. See include/uapi/linux/io_uring.h. +const ( + IORING_OP_NOP = 0 + IORING_OP_READV = 1 +) + // IORingIndex represents SQE array indexes. // // +marshal @@ -161,14 +172,14 @@ type IORings struct { type IOUringSqe struct { Opcode uint8 Flags uint8 - ioPrio uint16 - fd int32 - offOrAddrOrCmdOp uint64 - addrOrSpliceOff uint64 - len uint32 + IoPrio uint16 + Fd int32 + OffOrAddrOrCmdOp uint64 + AddrOrSpliceOff uint64 + Len uint32 specialFlags uint32 UserData uint64 - bufIndexOrGroup uint16 + BufIndexOrGroup uint16 personality uint16 spliceFDOrFileIndex int32 addr3 uint64 diff --git a/pkg/sentry/fsimpl/iouringfs/BUILD b/pkg/sentry/fsimpl/iouringfs/BUILD index 817f8a278..9ed27add1 100644 --- a/pkg/sentry/fsimpl/iouringfs/BUILD +++ b/pkg/sentry/fsimpl/iouringfs/BUILD @@ -13,10 +13,12 @@ go_library( "//pkg/errors/linuxerr", "//pkg/hostarch", "//pkg/safemem", + "//pkg/sentry/kernel", "//pkg/sentry/memmap", "//pkg/sentry/pgalloc", "//pkg/sentry/usage", "//pkg/sentry/vfs", + "//pkg/usermem", ], ) diff --git a/pkg/sentry/fsimpl/iouringfs/iouringfs.go b/pkg/sentry/fsimpl/iouringfs/iouringfs.go index a29e23732..735f78357 100644 --- a/pkg/sentry/fsimpl/iouringfs/iouringfs.go +++ b/pkg/sentry/fsimpl/iouringfs/iouringfs.go @@ -32,10 +32,12 @@ import ( "gvisor.dev/gvisor/pkg/errors/linuxerr" "gvisor.dev/gvisor/pkg/hostarch" "gvisor.dev/gvisor/pkg/safemem" + "gvisor.dev/gvisor/pkg/sentry/kernel" "gvisor.dev/gvisor/pkg/sentry/memmap" "gvisor.dev/gvisor/pkg/sentry/pgalloc" "gvisor.dev/gvisor/pkg/sentry/usage" "gvisor.dev/gvisor/pkg/sentry/vfs" + "gvisor.dev/gvisor/pkg/usermem" ) // FileDescription implements vfs.FileDescriptionImpl for file-based IO_URING. @@ -313,7 +315,7 @@ func (fd *FileDescription) ConfigureMMap(ctx context.Context, opts *memmap.MMapO } // ProcessSubmissions processes submission requests. -func (fd *FileDescription) ProcessSubmissions(toSubmit uint32, minComplete uint32, flags uint32) (int, error) { +func (fd *FileDescription) ProcessSubmissions(t *kernel.Task, toSubmit uint32, minComplete uint32, flags uint32) (int, error) { fd.mu.Lock() defer fd.mu.Unlock() @@ -345,10 +347,7 @@ func (fd *FileDescription) ProcessSubmissions(toSubmit uint32, minComplete uint3 return -1, err } - cqe, err := fd.ProcessSubmission(&sqe, flags) - if err != nil { - return -1, err - } + cqe := fd.ProcessSubmission(t, &sqe, flags) sqHead.Add(1) if (cqTail.Load()-cqHead.Load())/ioRings.CqRingEntries == 1 { ioRings.CqOverflow++ @@ -371,21 +370,75 @@ func (fd *FileDescription) ProcessSubmissions(toSubmit uint32, minComplete uint3 } // ProcessSubmission processes a single submission request. -func (fd *FileDescription) ProcessSubmission(sqe *linux.IOUringSqe, flags uint32) (*linux.IOUringCqe, error) { +func (fd *FileDescription) ProcessSubmission(t *kernel.Task, sqe *linux.IOUringSqe, flags uint32) *linux.IOUringCqe { + var ( + cqeErr error + cqeFlags uint32 + retValue int32 + ) + switch op := sqe.Opcode; op { - case 0: // NOP - return &linux.IOUringCqe{ - UserData: sqe.UserData, - Res: 0, - Flags: 0, - }, nil + case linux.IORING_OP_NOP: + // For the NOP operation, we don't do anything special. + case linux.IORING_OP_READV: + retValue, cqeErr = fd.handleReadv(t, sqe, flags) default: // Unsupported operation - return &linux.IOUringCqe{ - UserData: sqe.UserData, - Res: -int32(linuxerr.EINVAL.Errno()), - Flags: 0, - }, nil + retValue = -int32(linuxerr.EINVAL.Errno()) } + + if cqeErr != nil { + retValue = -int32(kernel.ExtractErrno(cqeErr, -1)) + } + + return &linux.IOUringCqe{ + UserData: sqe.UserData, + Res: retValue, + Flags: cqeFlags, + } +} + +// handleReadv handles IORING_OP_READV. +func (fd *FileDescription) handleReadv(t *kernel.Task, sqe *linux.IOUringSqe, flags uint32) (int32, error) { + // Check that a file descriptor is valid. + if sqe.Fd < 0 { + return 0, linuxerr.EBADF + } + // Currently we don't support any flags for the SQEs. + if sqe.Flags != 0 { + return 0, linuxerr.EINVAL + } + // If the file is not seekable then offset must be zero. And currently, we don't support them. + if sqe.OffOrAddrOrCmdOp != 0 { + return 0, linuxerr.EINVAL + } + // ioprio should not be set for the READV operation. + if sqe.IoPrio != 0 { + return 0, linuxerr.EINVAL + } + // buf_index should not be set for the READV operation. + if sqe.BufIndexOrGroup != 0 { + return 0, linuxerr.EINVAL + } + + // AddressSpaceActive is set to true as we are doing this from the task goroutine.And this is a + // case as we currently don't support neither IOPOLL nor SQPOLL modes. + dst, err := t.IovecsIOSequence(hostarch.Addr(sqe.AddrOrSpliceOff), int(sqe.Len), usermem.IOOpts{ + AddressSpaceActive: true, + }) + if err != nil { + return 0, err + } + file := t.GetFileVFS2(sqe.Fd) + if file == nil { + return 0, linuxerr.EBADF + } + defer file.DecRef(t) + n, err := file.PRead(t, dst, 0, vfs.ReadOptions{}) + if err != nil { + return 0, err + } + + return int32(n), nil } // updateCq updates a completion queue by adding a given completion queue entry. diff --git a/pkg/sentry/syscalls/linux/vfs2/iouringfs.go b/pkg/sentry/syscalls/linux/vfs2/iouringfs.go index 50727aec0..b61da35f1 100644 --- a/pkg/sentry/syscalls/linux/vfs2/iouringfs.go +++ b/pkg/sentry/syscalls/linux/vfs2/iouringfs.go @@ -86,7 +86,7 @@ func IOUringEnter(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel. ret := -1 // List of currently supported flags for io_uring_enter(2). - const supportedFlags = 0 // Currently support none + const supportedFlags = linux.IORING_ENTER_GETEVENTS // Since we don't implement everything, we fail explicitly on flags that are unimplemented. if flags|supportedFlags != supportedFlags { @@ -112,7 +112,7 @@ func IOUringEnter(t *kernel.Task, args arch.SyscallArguments) (uintptr, *kernel. if !ok { return uintptr(ret), nil, linuxerr.EBADF } - ret, err := iouringfd.ProcessSubmissions(toSubmit, minComplete, flags) + ret, err := iouringfd.ProcessSubmissions(t, toSubmit, minComplete, flags) if err != nil { return uintptr(ret), nil, err } diff --git a/test/syscalls/linux/io_uring.cc b/test/syscalls/linux/io_uring.cc index ead75c9e0..47bc4119e 100644 --- a/test/syscalls/linux/io_uring.cc +++ b/test/syscalls/linux/io_uring.cc @@ -12,7 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include +#include #include #include #include @@ -30,6 +32,7 @@ #include "test/util/io_uring_util.h" #include "test/util/memory_util.h" #include "test/util/multiprocess_util.h" +#include "test/util/temp_path.h" #include "test/util/test_util.h" #include "test/util/thread_util.h" @@ -38,6 +41,37 @@ namespace testing { namespace { +// IOVecContainsString checks that a tuple argument of (struct iovec *, int) +// corresponding to an iovec array and its length, contains data that matches +// the string length strlen and the string value str. +MATCHER_P(IOVecContainsString, str, "") { + struct iovec *iovs = arg.first; + int len = strlen(str); + int niov = arg.second; + int offset = 0; + + for (int i = 0; i < niov; i++) { + struct iovec iov = iovs[i]; + if (len < offset) { + *result_listener << "strlen " << len << " < offset " << offset; + return false; + } + if (strncmp(static_cast(iov.iov_base), &str[offset], iov.iov_len)) { + absl::string_view iovec_string(static_cast(iov.iov_base), + iov.iov_len); + *result_listener << iovec_string << " @offset " << offset; + return false; + } + offset += iov.iov_len; + } + if (offset != len) { + *result_listener << offset; + return false; + } + + return true; +} + // Testing that io_uring_setup(2) successfully returns a valid file descriptor. TEST(IOUringTest, ValidFD) { IOUringParams params; @@ -209,7 +243,7 @@ TEST(IOUringTest, SingleNOPTest) { uint32_t sq_tail = io_uring->load_sq_tail(); io_uring->store_sq_tail(sq_tail + 1); - int ret = io_uring->Enter(1, 1, 0, nullptr); + int ret = io_uring->Enter(1, 1, IORING_ENTER_GETEVENTS, nullptr); ASSERT_EQ(ret, 1); IOUringCqe *cqe = io_uring->get_cqes(); @@ -250,7 +284,7 @@ TEST(IOUringTest, QueueingNOPTest) { ASSERT_EQ(sq_tail, 0); io_uring->store_sq_tail(sq_tail + 4); - int ret = io_uring->Enter(2, 2, 0, nullptr); + int ret = io_uring->Enter(2, 2, IORING_ENTER_GETEVENTS, nullptr); ASSERT_EQ(ret, 2); IOUringCqe *cqe = io_uring->get_cqes(); @@ -269,7 +303,7 @@ TEST(IOUringTest, QueueingNOPTest) { uint32_t cq_head = io_uring->load_cq_head(); io_uring->store_cq_head(cq_head + 2); - ret = io_uring->Enter(2, 2, 0, nullptr); + ret = io_uring->Enter(2, 2, IORING_ENTER_GETEVENTS, nullptr); ASSERT_EQ(ret, 2); sq_head = io_uring->load_sq_head(); @@ -310,7 +344,7 @@ TEST(IOUringTest, MultipleNOPTest) { ASSERT_EQ(sq_tail, 0); io_uring->store_sq_tail(sq_tail + 3); - int ret = io_uring->Enter(3, 3, 0, nullptr); + int ret = io_uring->Enter(3, 3, IORING_ENTER_GETEVENTS, nullptr); ASSERT_EQ(ret, 3); IOUringCqe *cqe = io_uring->get_cqes(); @@ -356,7 +390,7 @@ TEST(IOUringTest, MultiThreadedNOPTest) { for (int i = 0; i < 4; i++) { ScopedThread t([&] { IOUring *io_uring_ptr = io_uring.get(); - int ret = io_uring_ptr->Enter(1, 1, 0, nullptr); + int ret = io_uring_ptr->Enter(1, 1, IORING_ENTER_GETEVENTS, nullptr); ASSERT_EQ(ret, 1); }); } @@ -395,7 +429,7 @@ TEST(IOUringTest, InvalidOpCodeTest) { uint32_t sq_tail = io_uring->load_sq_tail(); io_uring->store_sq_tail(sq_tail + 1); - int ret = io_uring->Enter(1, 1, 0, nullptr); + int ret = io_uring->Enter(1, 1, IORING_ENTER_GETEVENTS, nullptr); ASSERT_EQ(ret, 1); IOUringCqe *cqe = io_uring->get_cqes(); @@ -436,7 +470,7 @@ TEST(IOUringTest, SQERingBuffersWrapAroundTest) { uint32_t sq_tail = io_uring->load_sq_tail(); io_uring->store_sq_tail(sq_tail + 4); - int ret = io_uring->Enter(4, 4, 0, nullptr); + int ret = io_uring->Enter(4, 4, IORING_ENTER_GETEVENTS, nullptr); ASSERT_EQ(ret, 4); IOUringCqe *cqe = io_uring->get_cqes(); @@ -462,7 +496,7 @@ TEST(IOUringTest, SQERingBuffersWrapAroundTest) { sq_tail = io_uring->load_sq_tail(); io_uring->store_sq_tail(sq_tail + 4); - ret = io_uring->Enter(4, 4, 0, nullptr); + ret = io_uring->Enter(4, 4, IORING_ENTER_GETEVENTS, nullptr); ASSERT_EQ(ret, 4); sq_head = io_uring->load_sq_head(); @@ -499,7 +533,7 @@ TEST(IOUringTest, NonNullSigsetTest) { io_uring->store_sq_tail(sq_tail + 1); sigset_t non_null_sigset; - EXPECT_THAT(io_uring->Enter(1, 1, 0, &non_null_sigset), + EXPECT_THAT(io_uring->Enter(1, 1, IORING_ENTER_GETEVENTS, &non_null_sigset), SyscallFailsWithErrno(EFAULT)); } } @@ -534,7 +568,7 @@ TEST(IOUringTest, OverflowCQTest) { ASSERT_EQ(sq_tail, 4 * submission_round); io_uring->store_sq_tail(sq_tail + 4); - int ret = io_uring->Enter(4, 4, 0, nullptr); + int ret = io_uring->Enter(4, 4, IORING_ENTER_GETEVENTS, nullptr); ASSERT_EQ(ret, 4); sq_head = io_uring->load_sq_head(); @@ -567,7 +601,7 @@ TEST(IOUringTest, OverflowCQTest) { ASSERT_EQ(sq_tail, 8); io_uring->store_sq_tail(sq_tail + 2); - int ret = io_uring->Enter(2, 2, 0, nullptr); + int ret = io_uring->Enter(2, 2, IORING_ENTER_GETEVENTS, nullptr); ASSERT_EQ(ret, 2); sq_head = io_uring->load_sq_head(); @@ -589,6 +623,491 @@ TEST(IOUringTest, OverflowCQTest) { } } +// Testing that io_uring_enter(2) successfully handles single READV operation. +TEST(IOUringTest, SingleREADVTest) { + struct io_uring_params params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params)); + + ASSERT_EQ(params.sq_entries, 1); + ASSERT_EQ(params.cq_entries, 2); + + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); + + std::string file_name = NewTempAbsPath(); + std::string contents("DEADBEEF"); + ASSERT_NO_ERRNO(CreateWithContents(file_name, contents, 0666)); + + FileDescriptor filefd = ASSERT_NO_ERRNO_AND_VALUE(Open(file_name, O_RDONLY)); + ASSERT_GE(filefd.get(), 0); + + struct stat st = ASSERT_NO_ERRNO_AND_VALUE(Stat(file_name)); + off_t file_sz = st.st_size; + ASSERT_GT(file_sz, 0); + + int num_blocks = (file_sz + BLOCK_SZ - 1) / BLOCK_SZ; + ASSERT_EQ(num_blocks, 1); + + unsigned *sq_array = io_uring->get_sq_array(); + struct io_uring_sqe *sqe = io_uring->get_sqes(); + + struct iovec iov; + iov.iov_len = file_sz; + void *buf; + ASSERT_THAT(posix_memalign(&buf, BLOCK_SZ, BLOCK_SZ), SyscallSucceeds()); + iov.iov_base = buf; + + sqe->flags = 0; + sqe->fd = filefd.get(); + sqe->opcode = IORING_OP_READV; + sqe->addr = reinterpret_cast(&iov); + sqe->len = num_blocks; + sqe->off = 0; + sqe->user_data = reinterpret_cast(&iov); + sq_array[0] = 0; + + uint32_t sq_tail = io_uring->load_sq_tail(); + io_uring->store_sq_tail(sq_tail + 1); + + int ret = io_uring->Enter(1, 1, 0, nullptr); + ASSERT_EQ(ret, 1); + + struct io_uring_cqe *cqe = io_uring->get_cqes(); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 1); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 1); + + ASSERT_EQ(cqe->res, file_sz); + + struct iovec *fi = reinterpret_cast(cqe->user_data); + + std::pair iovec_desc(fi, num_blocks); + EXPECT_THAT(iovec_desc, IOVecContainsString(contents.c_str())); + + uint32_t cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 1); +} + +// Testing that io_uring_enter(2) successfully handles three READV operations +// from three different files submitted through a single invocation. +TEST(IOUringTest, ThreeREADVSingleEnterTest) { + struct io_uring_params params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(4, params)); + + ASSERT_EQ(params.sq_entries, 4); + ASSERT_EQ(params.cq_entries, 8); + + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); + + FileDescriptor filefd[3]; + unsigned *sq_array = io_uring->get_sq_array(); + struct io_uring_sqe *sqe = io_uring->get_sqes(); + off_t file_sz[3]; + int num_blocks[3]; + struct iovec iov[3]; + + for (size_t i = 0; i < 3; i++) { + std::string file_name = NewTempAbsPath(); + std::string contents("DEADBEEF"); + for (size_t j = 0; j < i; ++j) { + contents.append(" DEADBEEF"); + } + ASSERT_NO_ERRNO(CreateWithContents(file_name, contents, 0666)); + + filefd[i] = ASSERT_NO_ERRNO_AND_VALUE(Open(file_name, O_RDONLY)); + ASSERT_GE(filefd[i].get(), 0); + + struct stat st = ASSERT_NO_ERRNO_AND_VALUE(Stat(file_name)); + file_sz[i] = st.st_size; + ASSERT_GT(file_sz[i], 0); + + num_blocks[i] = (file_sz[i] + BLOCK_SZ - 1) / BLOCK_SZ; + ASSERT_EQ(num_blocks[i], 1); + + iov[i].iov_len = file_sz[i]; + void *buf; + ASSERT_THAT(posix_memalign(&buf, BLOCK_SZ, BLOCK_SZ), SyscallSucceeds()); + iov[i].iov_base = buf; + + sqe[i].flags = 0; + sqe[i].fd = filefd[i].get(); + sqe[i].opcode = IORING_OP_READV; + sqe[i].addr = reinterpret_cast(&iov[i]); + sqe[i].len = num_blocks[i]; + sqe[i].off = 0; + sqe[i].user_data = reinterpret_cast(&iov[i]); + sq_array[i] = i; + + uint32_t sq_tail = io_uring->load_sq_tail(); + io_uring->store_sq_tail(sq_tail + 1); + } + + ASSERT_EQ(file_sz[0], 8); + ASSERT_EQ(file_sz[1], 17); + ASSERT_EQ(file_sz[2], 26); + + int ret = io_uring->Enter(3, 3, 0, nullptr); + ASSERT_EQ(ret, 3); + + struct io_uring_cqe *cqe = io_uring->get_cqes(); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 3); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 3); + + ASSERT_EQ(cqe[0].res, file_sz[0]); + ASSERT_EQ(cqe[1].res, file_sz[1]); + ASSERT_EQ(cqe[2].res, file_sz[2]); + + for (size_t i = 0; i < 3; i++) { + struct iovec *fi = reinterpret_cast(cqe->user_data); + + std::string contents("DEADBEEF"); + for (size_t j = 0; j < i; ++j) { + contents.append(" DEADBEEF"); + } + + std::pair iovec_desc(&fi[i], num_blocks[i]); + EXPECT_THAT(iovec_desc, IOVecContainsString(contents.c_str())); + + uint32_t cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 1); + } +} + +// Testing that io_uring_enter(2) successfully handles READV operation, which is +// racing with deletion of the same file. +TEST(IOUringTest, READVRaceWithDeleteTest) { + struct io_uring_params params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(2, params)); + + ASSERT_EQ(params.sq_entries, 2); + ASSERT_EQ(params.cq_entries, 4); + + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); + + std::string file_name[2]; + FileDescriptor filefd[2]; + unsigned *sq_array = io_uring->get_sq_array(); + struct io_uring_sqe *sqe = io_uring->get_sqes(); + off_t file_sz[2]; + int num_blocks[2]; + struct iovec iov[2]; + + for (size_t i = 0; i < 2; i++) { + file_name[i] = NewTempAbsPath(); + std::string contents("DEADBEEF"); + for (size_t j = 0; j < i; ++j) { + contents.append(" DEADBEEF"); + } + ASSERT_NO_ERRNO(CreateWithContents(file_name[i], contents, 0666)); + + filefd[i] = ASSERT_NO_ERRNO_AND_VALUE(Open(file_name[i], O_RDONLY)); + ASSERT_GE(filefd[i].get(), 0); + + struct stat st = ASSERT_NO_ERRNO_AND_VALUE(Stat(file_name[i])); + file_sz[i] = st.st_size; + ASSERT_GT(file_sz[i], 0); + + num_blocks[i] = (file_sz[i] + BLOCK_SZ - 1) / BLOCK_SZ; + ASSERT_EQ(num_blocks[i], 1); + + iov[i].iov_len = file_sz[i]; + void *buf; + ASSERT_THAT(posix_memalign(&buf, BLOCK_SZ, BLOCK_SZ), SyscallSucceeds()); + iov[i].iov_base = buf; + + sqe[i].flags = 0; + sqe[i].fd = filefd[i].get(); + sqe[i].opcode = IORING_OP_READV; + sqe[i].addr = reinterpret_cast(&iov[i]); + sqe[i].len = num_blocks[i]; + sqe[i].off = 0; + sqe[i].user_data = reinterpret_cast(&iov[i]); + sq_array[i] = i; + + uint32_t sq_tail = io_uring->load_sq_tail(); + io_uring->store_sq_tail(sq_tail + 1); + } + + ASSERT_EQ(file_sz[0], 8); + ASSERT_EQ(file_sz[1], 17); + + ScopedThread t1([&] { + IOUring *io_uring_ptr = io_uring.get(); + int ret = io_uring_ptr->Enter(2, 2, IORING_ENTER_GETEVENTS, nullptr); + ASSERT_EQ(ret, 2); + }); + + ScopedThread t2([&] { filefd[0].reset(); }); + + t1.Join(); + t2.Join(); + + struct io_uring_cqe *cqe = io_uring->get_cqes(); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 2); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 2); + + ASSERT_TRUE(cqe[0].res == -EBADF || cqe[0].res == 8); + ASSERT_EQ(cqe[1].res, file_sz[1]); + + for (size_t i = 0; i < 2; i++) { + if (cqe[i].res == -EBADF) { + uint32_t cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 1); + + continue; + } + + struct iovec *fi = reinterpret_cast(cqe->user_data); + + std::string contents("DEADBEEF"); + for (size_t j = 0; j < i; ++j) { + contents.append(" DEADBEEF"); + } + + std::pair iovec_desc(&fi[i], num_blocks[i]); + EXPECT_THAT(iovec_desc, IOVecContainsString(contents.c_str())); + + uint32_t cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 1); + } +} + +// Testing that io_uring_enter(2) successfully handles single READV operation +// with short read situation. +TEST(IOUringTest, ShortReadREADVTest) { + struct io_uring_params params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params)); + + ASSERT_EQ(params.sq_entries, 1); + ASSERT_EQ(params.cq_entries, 2); + + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); + + std::string file_name = NewTempAbsPath(); + std::string contents("DEADBEEF"); + ASSERT_NO_ERRNO(CreateWithContents(file_name, contents, 0666)); + + FileDescriptor filefd = ASSERT_NO_ERRNO_AND_VALUE(Open(file_name, O_RDONLY)); + ASSERT_GE(filefd.get(), 0); + + struct stat st = ASSERT_NO_ERRNO_AND_VALUE(Stat(file_name)); + // Set file size to be twice of its actual size to mimic the short read. + off_t file_sz = 2 * st.st_size; + ASSERT_GT(file_sz, 0); + + int num_blocks = (file_sz + BLOCK_SZ - 1) / BLOCK_SZ; + ASSERT_EQ(num_blocks, 1); + + unsigned *sq_array = io_uring->get_sq_array(); + struct io_uring_sqe *sqe = io_uring->get_sqes(); + + struct iovec iov; + iov.iov_len = file_sz; + void *buf; + ASSERT_THAT(posix_memalign(&buf, BLOCK_SZ, BLOCK_SZ), SyscallSucceeds()); + iov.iov_base = buf; + + sqe->flags = 0; + sqe->fd = filefd.get(); + sqe->opcode = IORING_OP_READV; + sqe->addr = reinterpret_cast(&iov); + sqe->len = num_blocks; + sqe->off = 0; + sqe->user_data = reinterpret_cast(&iov); + sq_array[0] = 0; + + uint32_t sq_tail = io_uring->load_sq_tail(); + io_uring->store_sq_tail(sq_tail + 1); + + int ret = io_uring->Enter(1, 1, 0, nullptr); + ASSERT_EQ(ret, 1); + + struct io_uring_cqe *cqe = io_uring->get_cqes(); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 1); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 1); + + ASSERT_EQ(cqe->res, file_sz / 2); + + struct iovec *fi = reinterpret_cast(cqe->user_data); + fi->iov_len = file_sz / 2; + + std::pair iovec_desc(fi, num_blocks); + EXPECT_THAT(iovec_desc, IOVecContainsString(contents.c_str())); + + uint32_t cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 1); +} + +// Testing that io_uring_enter(2) successfully handles single READV operation +// when there file does not have read permissions. +TEST(IOUringTest, NoReadPermissionsREADVTest) { + struct io_uring_params params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params)); + + ASSERT_EQ(params.sq_entries, 1); + ASSERT_EQ(params.cq_entries, 2); + + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); + + std::string file_name = NewTempAbsPath(); + std::string contents("DEADBEEF"); + ASSERT_NO_ERRNO(CreateWithContents(file_name, contents, 0666)); + + FileDescriptor filefd = ASSERT_NO_ERRNO_AND_VALUE(Open(file_name, O_WRONLY)); + ASSERT_GE(filefd.get(), 0); + + struct stat st = ASSERT_NO_ERRNO_AND_VALUE(Stat(file_name)); + off_t file_sz = st.st_size; + ASSERT_GT(file_sz, 0); + + int num_blocks = (file_sz + BLOCK_SZ - 1) / BLOCK_SZ; + ASSERT_EQ(num_blocks, 1); + + unsigned *sq_array = io_uring->get_sq_array(); + struct io_uring_sqe *sqe = io_uring->get_sqes(); + + struct iovec iov; + iov.iov_len = file_sz; + void *buf; + ASSERT_THAT(posix_memalign(&buf, BLOCK_SZ, BLOCK_SZ), SyscallSucceeds()); + iov.iov_base = buf; + + sqe->flags = 0; + sqe->fd = filefd.get(); + sqe->opcode = IORING_OP_READV; + sqe->addr = reinterpret_cast(&iov); + sqe->len = num_blocks; + sqe->off = 0; + sqe->user_data = reinterpret_cast(&iov); + sq_array[0] = 0; + + uint32_t sq_tail = io_uring->load_sq_tail(); + io_uring->store_sq_tail(sq_tail + 1); + + int ret = io_uring->Enter(1, 1, 0, nullptr); + ASSERT_EQ(ret, 1); + + struct io_uring_cqe *cqe = io_uring->get_cqes(); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 1); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 1); + + ASSERT_EQ(cqe->res, -EBADF); + + uint32_t cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 1); +} + +struct SqeFieldsUT { + uint16_t ioprio; + uint16_t buf_index; +}; + +class IOUringSqeFieldsTest : public ::testing::Test, + public ::testing::WithParamInterface { +}; + +// Testing that io_uring_enter(2) successfully handles single READV operation +// and returns EINVAL error in the CQE when either ioprio or buf_index is set. +TEST_P(IOUringSqeFieldsTest, READVWithInvalidSqeFieldValue) { + const SqeFieldsUT p = GetParam(); + + struct io_uring_params params; + std::unique_ptr io_uring = + ASSERT_NO_ERRNO_AND_VALUE(IOUring::InitIOUring(1, params)); + + ASSERT_EQ(params.sq_entries, 1); + ASSERT_EQ(params.cq_entries, 2); + + uint32_t sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 0); + + std::string file_name = NewTempAbsPath(); + std::string contents("DEADBEEF"); + ASSERT_NO_ERRNO(CreateWithContents(file_name, contents, 0666)); + + FileDescriptor filefd = ASSERT_NO_ERRNO_AND_VALUE(Open(file_name, O_RDONLY)); + ASSERT_GE(filefd.get(), 0); + + struct stat st = ASSERT_NO_ERRNO_AND_VALUE(Stat(file_name)); + off_t file_sz = st.st_size; + ASSERT_GT(file_sz, 0); + + int num_blocks = (file_sz + BLOCK_SZ - 1) / BLOCK_SZ; + ASSERT_EQ(num_blocks, 1); + + unsigned *sq_array = io_uring->get_sq_array(); + struct io_uring_sqe *sqe = io_uring->get_sqes(); + + struct iovec iov; + iov.iov_len = file_sz; + void *buf; + ASSERT_THAT(posix_memalign(&buf, BLOCK_SZ, BLOCK_SZ), SyscallSucceeds()); + iov.iov_base = buf; + + sqe->flags = 0; + sqe->fd = filefd.get(); + sqe->opcode = IORING_OP_READV; + sqe->addr = reinterpret_cast(&iov); + sqe->len = num_blocks; + sqe->off = 0; + sqe->user_data = reinterpret_cast(&iov); + sqe->ioprio = p.ioprio; + sqe->buf_index = p.buf_index; + sq_array[0] = 0; + + uint32_t sq_tail = io_uring->load_sq_tail(); + io_uring->store_sq_tail(sq_tail + 1); + + int ret = io_uring->Enter(1, 1, 0, nullptr); + ASSERT_EQ(ret, 1); + + struct io_uring_cqe *cqe = io_uring->get_cqes(); + + sq_head = io_uring->load_sq_head(); + ASSERT_EQ(sq_head, 1); + + uint32_t cq_tail = io_uring->load_cq_tail(); + ASSERT_EQ(cq_tail, 1); + + ASSERT_EQ(cqe->res, -EINVAL); + + uint32_t cq_head = io_uring->load_cq_head(); + io_uring->store_cq_head(cq_head + 1); +} + +INSTANTIATE_TEST_SUITE_P( + IOUringSqeFields, IOUringSqeFieldsTest, + ::testing::Values(SqeFieldsUT{.ioprio = 0, .buf_index = 1}, + SqeFieldsUT{.ioprio = 1, .buf_index = 0})); + } // namespace } // namespace testing diff --git a/test/util/BUILD b/test/util/BUILD index 657ca0899..c9387a49b 100644 --- a/test/util/BUILD +++ b/test/util/BUILD @@ -68,6 +68,8 @@ cc_library( ":file_descriptor", ":posix_error", ":save_util", + "//test/util:temp_path", + "//test/util:test_util", ], ) diff --git a/test/util/io_uring_util.h b/test/util/io_uring_util.h index 5dea5b340..8012071ec 100644 --- a/test/util/io_uring_util.h +++ b/test/util/io_uring_util.h @@ -16,6 +16,7 @@ #define GVISOR_TEST_UTIL_IOURING_UTIL_H_ #include +#include #include #include @@ -36,6 +37,9 @@ namespace testing { #define IORING_SETUP_SQPOLL (1U << 1) #define IORING_SETUP_CQSIZE (1U << 3) +// io_uring_enter(2) flags +#define IORING_ENTER_GETEVENTS (1U << 0) + #define IORING_FEAT_SINGLE_MMAP (1U << 0) #define IORING_OFF_SQ_RING 0ULL @@ -44,6 +48,9 @@ namespace testing { // IO_URING operation codes. #define IORING_OP_NOP 0 +#define IORING_OP_READV 1 + +#define BLOCK_SZ kPageSize struct io_sqring_offsets { uint32_t head;