From f52a0f6272698622839ed134a080fef2e4813d70 Mon Sep 17 00:00:00 2001 From: Ayush Ranjan Date: Wed, 28 Jun 2023 07:50:55 -0700 Subject: [PATCH] Prepopulate pages for large writes in tmpfs. When an application makes a huge write, it is better to pre-fault all the pages that will later be touched by runtime.memmove() via safemem.CopySeq(). Earlier we were calling MemoryFile.Allocate(), which does not populate the pages. Otherwise, these pages fault one at a time as they are touched, which is really slow due to context switching. Note that this optimization only works for shmem-backed MemoryFiles. This means that writes to unallocated regions of tmpfs mounts (like /tmp) will be faster. rootfs (overlay) does not benefit because even though it has an upper tmpfs layer, the MemoryFile used is disk-backed. We only prepopulate if more than 1 page is being written to. This is because the syscall path is susceptible to certain overheads that the page fault path does not have. For example seccomp-bpf and syscall auditing. The improved large writes in tmpfs is evidenced by (on Linux >=5.14): - Before ``` $ docker run --runtime=runsc --rm ubuntu bash -c "dd if=/dev/zero of=/tmp/file.txt bs=40960 count=100000" 100000+0 records in 100000+0 records out 4096000000 bytes (4.1 GB, 3.8 GiB) copied, 3.10212 s, 1.3 GB/s ``` - After ``` docker run --runtime=runsc --rm ubuntu bash -c "dd if=/dev/zero of=/tmp/file.txt bs=40960 count=100000" 100000+0 records in 100000+0 records out 4096000000 bytes (4.1 GB, 3.8 GiB) copied, 2.38347 s, 1.7 GB/s ``` In this workload, this change reduces latency by 23% (3.10212s -> 2.38347s). PiperOrigin-RevId: 544056038 --- pkg/sentry/fsimpl/tmpfs/regular_file.go | 5 ++++- pkg/sentry/pgalloc/pgalloc.go | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/sentry/fsimpl/tmpfs/regular_file.go b/pkg/sentry/fsimpl/tmpfs/regular_file.go index c090b00e0..dc50af169 100644 --- a/pkg/sentry/fsimpl/tmpfs/regular_file.go +++ b/pkg/sentry/fsimpl/tmpfs/regular_file.go @@ -728,7 +728,10 @@ func (rw *regularFileReadWriter) WriteFromBlocks(srcs safemem.BlockSeq) (uint64, goto exitLoop } gapMR.End = gapMR.Start + (hostarch.PageSize * pagesReserved) - fr, err := rw.file.inode.fs.mf.Allocate(gapMR.Length(), pgalloc.AllocOpts{Kind: rw.file.memoryUsageKind}) + fr, err := rw.file.inode.fs.mf.AllocateAndFill(gapMR.Length(), rw.file.memoryUsageKind, pgalloc.AllocateAndWritePopulate, safemem.ReaderFunc(func(dsts safemem.BlockSeq) (uint64, error) { + // No-op here. The write to dsts will happen in the next iteration. + return dsts.NumBytes(), nil + })) if err != nil { retErr = err rw.file.inode.fs.unaccountPages(pagesReserved) diff --git a/pkg/sentry/pgalloc/pgalloc.go b/pkg/sentry/pgalloc/pgalloc.go index e567caf47..732804800 100644 --- a/pkg/sentry/pgalloc/pgalloc.go +++ b/pkg/sentry/pgalloc/pgalloc.go @@ -697,10 +697,14 @@ func tryPopulateMadv(b safemem.Block) bool { return true } end := hostarch.Addr(b.Addr() + uintptr(b.Len())).RoundDown() - if start >= end { + bLen := end - start + // Only call madvise(MADV_POPULATE_WRITE) if >=2 pages are being populated. + // 1 syscall overhead >= 1 page fault overhead. This is because syscalls are + // susceptible to additional overheads like seccomp-bpf filters and auditing. + if start >= end || bLen <= hostarch.PageSize { return true } - _, _, errno := unix.RawSyscall(unix.SYS_MADVISE, uintptr(start), uintptr(end-start), unix.MADV_POPULATE_WRITE) + _, _, errno := unix.RawSyscall(unix.SYS_MADVISE, uintptr(start), uintptr(bLen), unix.MADV_POPULATE_WRITE) if errno != 0 { if errno == unix.EINVAL { // EINVAL is expected if MADV_POPULATE_WRITE is not supported (Linux <5.14).