diff --git a/pkg/sentry/devices/accel/tpu_v4_mmap.go b/pkg/sentry/devices/accel/tpu_v4_mmap.go index 77f222e66..cda00e7e3 100644 --- a/pkg/sentry/devices/accel/tpu_v4_mmap.go +++ b/pkg/sentry/devices/accel/tpu_v4_mmap.go @@ -61,6 +61,8 @@ func (fd *tpuV4FD) InvalidateUnsavable(ctx context.Context) error { } type accelFDMemmapFile struct { + memmap.NoBufferedIOFallback + fd *tpuV4FD } diff --git a/pkg/sentry/devices/nvproxy/frontend_mmap.go b/pkg/sentry/devices/nvproxy/frontend_mmap.go index 95b8ad6cb..94457424b 100644 --- a/pkg/sentry/devices/nvproxy/frontend_mmap.go +++ b/pkg/sentry/devices/nvproxy/frontend_mmap.go @@ -61,6 +61,8 @@ func (fd *frontendFD) InvalidateUnsavable(ctx context.Context) error { } type frontendFDMemmapFile struct { + memmap.NoBufferedIOFallback + fd *frontendFD } diff --git a/pkg/sentry/devices/nvproxy/uvm_mmap.go b/pkg/sentry/devices/nvproxy/uvm_mmap.go index 7235d8aad..9f81081f3 100644 --- a/pkg/sentry/devices/nvproxy/uvm_mmap.go +++ b/pkg/sentry/devices/nvproxy/uvm_mmap.go @@ -69,6 +69,8 @@ func (fd *uvmFD) InvalidateUnsavable(ctx context.Context) error { } type uvmFDMemmapFile struct { + memmap.NoBufferedIOFallback + fd *uvmFD } diff --git a/pkg/sentry/devices/tpuproxy/tpu_mmap.go b/pkg/sentry/devices/tpuproxy/tpu_mmap.go index 553b9c09a..0d42cbea4 100644 --- a/pkg/sentry/devices/tpuproxy/tpu_mmap.go +++ b/pkg/sentry/devices/tpuproxy/tpu_mmap.go @@ -61,6 +61,8 @@ func (fd *tpuFD) InvalidateUnsavable(ctx context.Context) error { } type tpuFDMemmapFile struct { + memmap.NoBufferedIOFallback + fd *tpuFD } @@ -120,6 +122,8 @@ func (fd *pciDeviceFD) InvalidateUnsavable(ctx context.Context) error { } type pciDeviceFdMemmapFile struct { + memmap.NoBufferedIOFallback + fd *pciDeviceFD } diff --git a/pkg/sentry/devices/tpuproxy/vfio_mmap.go b/pkg/sentry/devices/tpuproxy/vfio_mmap.go index 6ad8fa083..daf08d817 100644 --- a/pkg/sentry/devices/tpuproxy/vfio_mmap.go +++ b/pkg/sentry/devices/tpuproxy/vfio_mmap.go @@ -61,6 +61,8 @@ func (fd *vfioFD) InvalidateUnsavable(ctx context.Context) error { } type vfioFDMemmapFile struct { + memmap.NoBufferedIOFallback + fd *vfioFD } diff --git a/pkg/sentry/fsimpl/erofs/regular_file.go b/pkg/sentry/fsimpl/erofs/regular_file.go index 088dae932..da758eb1c 100644 --- a/pkg/sentry/fsimpl/erofs/regular_file.go +++ b/pkg/sentry/fsimpl/erofs/regular_file.go @@ -190,6 +190,8 @@ func (i *inode) InvalidateUnsavable(ctx context.Context) error { // +stateify savable type imageMemmapFile struct { + memmap.NoBufferedIOFallback + image *erofs.Image } diff --git a/pkg/sentry/fsimpl/gofer/regular_file.go b/pkg/sentry/fsimpl/gofer/regular_file.go index 6c1ab5179..f23540696 100644 --- a/pkg/sentry/fsimpl/gofer/regular_file.go +++ b/pkg/sentry/fsimpl/gofer/regular_file.go @@ -916,6 +916,8 @@ func (d *dentry) Evict(ctx context.Context, er pgalloc.EvictableRange) { // // +stateify savable type dentryPlatformFile struct { + memmap.NoBufferedIOFallback + *dentry // fdRefs counts references on memmap.File offsets. fdRefs is protected diff --git a/pkg/sentry/fsimpl/gofer/special_file.go b/pkg/sentry/fsimpl/gofer/special_file.go index ebdecbfbc..5936bd508 100644 --- a/pkg/sentry/fsimpl/gofer/special_file.go +++ b/pkg/sentry/fsimpl/gofer/special_file.go @@ -43,6 +43,7 @@ import ( type specialFileFD struct { fileDescription specialFDEntry + memmap.NoBufferedIOFallback // releaseMu synchronizes the closing of fd.handle with fd.sync(). It's safe // to access fd.handle without locking for operations that require a ref to diff --git a/pkg/sentry/fsimpl/kernfs/mmap_util.go b/pkg/sentry/fsimpl/kernfs/mmap_util.go index acb3991b7..0f07a9406 100644 --- a/pkg/sentry/fsimpl/kernfs/mmap_util.go +++ b/pkg/sentry/fsimpl/kernfs/mmap_util.go @@ -28,6 +28,8 @@ import ( // // +stateify savable type inodePlatformFile struct { + memmap.NoBufferedIOFallback + // hostFD contains the host fd that this file was originally created from, // which must be available at time of restore. // diff --git a/pkg/sentry/memmap/memmap.go b/pkg/sentry/memmap/memmap.go index adca2d8ef..cb57079d2 100644 --- a/pkg/sentry/memmap/memmap.go +++ b/pkg/sentry/memmap/memmap.go @@ -436,6 +436,32 @@ type File interface { // reference is held on the mapped pages. MapInternal(fr FileRange, at hostarch.AccessType) (safemem.BlockSeq, error) + // BufferReadAt reads len(dst) bytes from the file into dst, starting at + // file offset off. It returns the number of bytes read. Like + // io.ReaderAt.ReadAt(), it never returns a short read with a nil error. + // + // Implementations of File for which MapInternal() never returns + // BufferedIOFallbackErr can embed NoBufferedIOFallback to obtain an + // appropriate implementation of BufferReadAt. + // + // Preconditions: + // * MapInternal() returned a BufferedIOFallbackErr. + // * At least one reference must be held on all read pages. + BufferReadAt(off uint64, dst []byte) (uint64, error) + + // BufferWriteAt writes len(src) bytes src to the file, starting at file + // offset off. It returns the number of bytes written. Like + // io.WriterAt.WriteAt(), it never returns a short write with a nil error. + // + // Implementations of File for which MapInternal() never returns + // BufferedIOFallbackErr can embed NoBufferedIOFallback to obtain an + // appropriate implementation of BufferWriteAt. + // + // Preconditions: + // * MapInternal() returned a BufferedIOFallbackErr. + // * At least one reference must be held on all written pages. + BufferWriteAt(off uint64, src []byte) (uint64, error) + // FD returns the file descriptor represented by the File. // // The only permitted operation on the returned file descriptor is to map @@ -443,6 +469,31 @@ type File interface { FD() int } +// BufferedIOFallbackErr is returned (by value) by implementations of +// File.MapInternal() that cannot succeed, but can still support memory-mapped +// I/O by falling back to buffered reads and writes. +type BufferedIOFallbackErr struct{} + +// Error implements error.Error. +func (BufferedIOFallbackErr) Error() string { + return "memmap.File.MapInternal() is unsupported, fall back to buffered R/W for internally-mapped I/O" +} + +// NoBufferedIOFallback implements File.BufferReadAt() and BufferWriteAt() for +// implementations of File for which MapInternal() never returns +// BufferedIOFallbackErr. +type NoBufferedIOFallback struct{} + +// BufferReadAt implements File.BufferReadAt. +func (NoBufferedIOFallback) BufferReadAt(off uint64, dst []byte) (uint64, error) { + panic("unimplemented: memmap.File.MapInternal() should not have returned BufferedIOFallbackErr") +} + +// BufferWriteAt implements File.BufferWriteAt. +func (NoBufferedIOFallback) BufferWriteAt(off uint64, src []byte) (uint64, error) { + panic("unimplemented: memmap.File.MapInternal() should not have returned BufferedIOFallbackErr") +} + // FileRange represents a range of uint64 offsets into a File. // // type FileRange diff --git a/pkg/sentry/mm/io.go b/pkg/sentry/mm/io.go index ea3a03a3e..0ac1469b1 100644 --- a/pkg/sentry/mm/io.go +++ b/pkg/sentry/mm/io.go @@ -15,12 +15,15 @@ package mm import ( + "fmt" + "gvisor.dev/gvisor/pkg/context" "gvisor.dev/gvisor/pkg/errors/linuxerr" "gvisor.dev/gvisor/pkg/hostarch" "gvisor.dev/gvisor/pkg/safemem" "gvisor.dev/gvisor/pkg/sentry/memmap" "gvisor.dev/gvisor/pkg/sentry/platform" + "gvisor.dev/gvisor/pkg/sync" "gvisor.dev/gvisor/pkg/usermem" ) @@ -118,6 +121,11 @@ func (mm *MemoryManager) CopyOut(ctx context.Context, addr hostarch.Addr, src [] } // Go through internal mappings. + // NOTE(gvisor.dev/issue/10331): Using mm.withInternalMappings() here means + // that if we encounter any memmap.BufferedIOFallbackErrs, this copy will + // traverse an unnecessary layer of buffering. This can be fixed by + // inlining mm.withInternalMappings() and passing src subslices directly to + // memmap.File.BufferWriteAt(). n64, err := mm.withInternalMappings(ctx, ar, hostarch.Write, opts.IgnorePermissions, func(ims safemem.BlockSeq) (uint64, error) { n, err := safemem.CopySeq(ims, safemem.BlockSeqOf(safemem.BlockFromSafeSlice(src))) return n, translateIOError(ctx, err) @@ -161,6 +169,11 @@ func (mm *MemoryManager) CopyIn(ctx context.Context, addr hostarch.Addr, dst []b } // Go through internal mappings. + // NOTE(gvisor.dev/issue/10331): Using mm.withInternalMappings() here means + // that if we encounter any memmap.BufferedIOFallbackErrs, this copy will + // traverse an unnecessary layer of buffering. This can be fixed by + // inlining mm.withInternalMappings() and passing dst subslices directly to + // memmap.File.BufferReadAt(). n64, err := mm.withInternalMappings(ctx, ar, hostarch.Read, opts.IgnorePermissions, func(ims safemem.BlockSeq) (uint64, error) { n, err := safemem.CopySeq(safemem.BlockSeqOf(safemem.BlockFromSafeSlice(dst)), ims) return n, translateIOError(ctx, err) @@ -549,18 +562,19 @@ func (mm *MemoryManager) withInternalMappings(ctx context.Context, ar hostarch.A } ar.End = pendaddr } - imend, imerr := mm.getPMAInternalMappingsLocked(pseg, ar) + imbs, t, imerr := mm.getIOMappingsLocked(pseg, ar, at) mm.activeMu.DowngradeLock() - if imendaddr := imend.Start(); imendaddr < ar.End { - if imendaddr <= ar.Start { + if imlen := imbs.NumBytes(); imlen < uint64(ar.Length()) { + if imlen == 0 { + t.flush(0, nil) mm.activeMu.RUnlock() return 0, translateIOError(ctx, imerr) } - ar.End = imendaddr + ar.End = ar.Start + hostarch.Addr(imlen) } // Do I/O. - un, err := f(mm.internalMappingsLocked(pseg, ar)) + un, err := t.flush(f(imbs)) mm.activeMu.RUnlock() n := int64(un) @@ -619,15 +633,16 @@ func (mm *MemoryManager) withVecInternalMappings(ctx context.Context, ars hostar mm.activeMu.Unlock() return 0, translateIOError(ctx, perr) } - imars, imerr := mm.getVecPMAInternalMappingsLocked(pars) + imbs, t, imerr := mm.getVecIOMappingsLocked(pars, at) mm.activeMu.DowngradeLock() - if imars.NumBytes() == 0 { + if imbs.NumBytes() == 0 { + t.flush(0, nil) mm.activeMu.RUnlock() return 0, translateIOError(ctx, imerr) } // Do I/O. - un, err := f(mm.vecInternalMappingsLocked(imars)) + un, err := t.flush(f(imbs)) mm.activeMu.RUnlock() n := int64(un) @@ -645,6 +660,263 @@ func (mm *MemoryManager) withVecInternalMappings(ctx context.Context, ars hostar return n, translateIOError(ctx, verr) } +// getIOMappingsLocked returns internal mappings appropriate for I/O for +// addresses in ar. If mappings are only available for a strict subset of ar, +// the returned error is non-nil. +// +// ioBufTracker.flush() must be called on the returned ioBufTracker when the +// returned mappings are no longer in use, and its return value indicates the +// number of bytes actually completed after buffer flushing. Returned mappings +// are valid until either mm.activeMu is unlocked or ioBufTracker.flush() is +// called. +// +// Preconditions: +// - mm.activeMu must be locked for writing. +// - pseg.Range().Contains(ar.Start). +// - pmas must exist for all addresses in ar. +// - ar.Length() != 0. +// +// Postconditions: getIOMappingsLocked does not invalidate iterators into mm.pmas. +func (mm *MemoryManager) getIOMappingsLocked(pseg pmaIterator, ar hostarch.AddrRange, at hostarch.AccessType) (safemem.BlockSeq, *ioBufTracker, error) { + if checkInvariants { + if !ar.WellFormed() || ar.Length() == 0 { + panic(fmt.Sprintf("invalid ar: %v", ar)) + } + if !pseg.Range().Contains(ar.Start) { + panic(fmt.Sprintf("initial pma %v does not cover start of ar %v", pseg.Range(), ar)) + } + } + + if ar.End <= pseg.End() { + // Since only one pma is involved, we can use pma.internalMappings + // directly, avoiding a slice allocation. + if err := pseg.getInternalMappingsLocked(); err != nil { + if _, ok := err.(memmap.BufferedIOFallbackErr); ok { + goto slowPath + } + return safemem.BlockSeq{}, nil, err + } + offset := uint64(ar.Start - pseg.Start()) + return pseg.ValuePtr().internalMappings.DropFirst64(offset).TakeFirst64(uint64(ar.Length())), nil, nil + } + +slowPath: + ims, t, _, err := mm.getIOMappingsTrackedLocked(pseg, ar, at, nil, nil, 0) + return safemem.BlockSeqFromSlice(ims), t, err +} + +// getVecIOMappingsLocked returns internal mappings appropriate for I/O for +// addresses in ars. If mappings are only available for a strict subset of ar, +// the returned error is non-nil. +// +// ioBufTracker.flush() must be called on the returned ioBufTracker when the +// returned mappings are no longer in use, and its return value indicates the +// number of bytes actually completed after buffer flushing. Returned mappings +// are valid until either mm.activeMu is unlocked or ioBufTracker.flush() is +// called. +// +// Preconditions: +// - mm.activeMu must be locked for writing. +// - pmas must exist for all addresses in ar. +// +// Postconditions: getVecIOMappingsLocked does not invalidate iterators into +// mm.pmas +func (mm *MemoryManager) getVecIOMappingsLocked(ars hostarch.AddrRangeSeq, at hostarch.AccessType) (safemem.BlockSeq, *ioBufTracker, error) { + if ars.NumRanges() == 1 { + ar := ars.Head() + return mm.getIOMappingsLocked(mm.pmas.FindSegment(ar.Start), ar, at) + } + + var ims []safemem.Block + var t *ioBufTracker + unbufBytes := uint64(0) + for arsit := ars; !arsit.IsEmpty(); arsit = arsit.Tail() { + ar := arsit.Head() + if ar.Length() == 0 { + continue + } + var err error + ims, t, unbufBytes, err = mm.getIOMappingsTrackedLocked(mm.pmas.FindSegment(ar.Start), ar, at, ims, t, unbufBytes) + if err != nil { + return safemem.BlockSeqFromSlice(ims), t, err + } + } + return safemem.BlockSeqFromSlice(ims), t, nil +} + +// getIOMappingsTrackedLocked collects internal mappings appropriate for I/O +// for addresses in ar, appends them to ims, and returns an updated slice. If +// mappings are only available for a strict subset of ar, the returned error is +// non-nil. +// +// If any iterated memmap.Files require buffering for I/O, they are recorded in +// an ioBufTracker. Since the ioBufTracker pointer is initially nil (to +// minimize overhead for the common case where no memmap.files require +// buffering for I/O), getIOMappingsTrackedLocked returns an updated +// ioBufTracker pointer. +// +// unbufBytes is the number of bytes of unbuffered mappings that have been +// appended to ims since the last buffered mapping; getIOMappingsTrackedLocked +// also returns an updated value for unbufBytes. +// +// Returned mappings are valid until either mm.activeMu is unlocked or +// ioBufTracker.flush() is called. +// +// Preconditions: +// - mm.activeMu must be locked for writing. +// - pseg.Range().Contains(ar.Start). +// - pmas must exist for all addresses in ar. +// - ar.Length() != 0. +// +// Postconditions: getIOMappingsTrackedLocked does not invalidate iterators +// into mm.pmas. +func (mm *MemoryManager) getIOMappingsTrackedLocked(pseg pmaIterator, ar hostarch.AddrRange, at hostarch.AccessType, ims []safemem.Block, t *ioBufTracker, unbufBytes uint64) ([]safemem.Block, *ioBufTracker, uint64, error) { + for { + pmaAR := ar.Intersect(pseg.Range()) + if err := pseg.getInternalMappingsLocked(); err == nil { + // Iterate the subset of the PMA's cached internal mappings that + // correspond to pmaAR, and append them to ims. + for pims := pseg.ValuePtr().internalMappings.DropFirst64(uint64(pmaAR.Start - pseg.Start())).TakeFirst64(uint64(pmaAR.Length())); !pims.IsEmpty(); pims = pims.Tail() { + ims = append(ims, pims.Head()) + } + unbufBytes += uint64(pmaAR.Length()) + } else if _, ok := err.(memmap.BufferedIOFallbackErr); !ok { + return ims, t, unbufBytes, err + } else { + // Fall back to buffered I/O as instructed. + if t == nil { + t = getIOBufTracker(at.Write) + } + buf := getByteSlicePtr(int(pmaAR.Length())) + pma := pseg.ValuePtr() + off := pseg.fileRangeOf(pmaAR).Start + // If the caller will read from the buffer, fill it from the file; + // otherwise leave it zeroed. + if at.Read || at.Execute { + var n uint64 + n, err = pma.file.BufferReadAt(off, *buf) + *buf = (*buf)[:n] + } else { + err = nil + } + if len(*buf) != 0 { + ims = append(ims, safemem.BlockFromSafeSlice(*buf)) + t.bufs = append(t.bufs, ioBuf{ + unbufBytesBefore: unbufBytes, + file: pma.file, + off: off, + buf: buf, + }) + unbufBytes = 0 + } + if err != nil { + return ims, t, unbufBytes, err + } + } + if ar.End <= pseg.End() { + return ims, t, unbufBytes, nil + } + pseg, _ = pseg.NextNonEmpty() + } +} + +type ioBuf struct { + unbufBytesBefore uint64 + file memmap.File + off uint64 + buf *[]byte +} + +type ioBufTracker struct { + write bool + bufs []ioBuf +} + +var ioBufTrackerPool = sync.Pool{ + New: func() any { + return &ioBufTracker{} + }, +} + +func getIOBufTracker(write bool) *ioBufTracker { + t := ioBufTrackerPool.Get().(*ioBufTracker) + t.write = write + return t +} + +func putIOBufTracker(t *ioBufTracker) { + for i := range t.bufs { + t.bufs[i].file = nil + putByteSlicePtr(t.bufs[i].buf) + t.bufs[i].buf = nil + } + t.bufs = t.bufs[:0] + ioBufTrackerPool.Put(t) +} + +func (t *ioBufTracker) flush(prevN uint64, prevErr error) (uint64, error) { + if t == nil { + return prevN, prevErr + } + return t.flushSlow(prevN, prevErr) +} + +func (t *ioBufTracker) flushSlow(prevN uint64, prevErr error) (uint64, error) { + defer putIOBufTracker(t) + if !t.write { + return prevN, prevErr + } + // Flush dirty buffers to underlying memmap.Files. + rem := prevN + done := uint64(0) + for i := range t.bufs { + buf := &t.bufs[i] + if rem <= buf.unbufBytesBefore { + // The write ended before reaching buf.buf. + break + } + rem -= buf.unbufBytesBefore + done += buf.unbufBytesBefore + n, err := buf.file.BufferWriteAt(buf.off, (*buf.buf)[:min(len(*buf.buf), int(rem))]) + rem -= n + done += n + if err != nil { + return done, err + } + } + // All buffers covered by prevN were written back successfully. + return prevN, prevErr +} + +var byteSlicePtrPool sync.Pool + +// getByteSlicePtr returns a pointer to a byte slice with the given length. The +// slice is either newly-allocated or recycled from a previous call to +// putByteSlicePtr. The pointer should be passed to putByteSlicePtr when the +// slice is no longer in use. +func getByteSlicePtr(l int) *[]byte { + a := byteSlicePtrPool.Get() + if a == nil { + s := make([]byte, l) + return &s + } + sp := a.(*[]byte) + s := *sp + if l <= cap(s) { + s = s[:l] + } else { + s = make([]byte, l) + } + *sp = s + return sp +} + +// putByteSlicePtr marks all of the given's slice capacity reusable by a future +// call to getByteSlicePtr. +func putByteSlicePtr(s *[]byte) { + byteSlicePtrPool.Put(s) +} + // truncatedAddrRangeSeq returns a copy of ars, but with the end truncated to // at most address end on AddrRange arsit.Head(). It is used in vector I/O paths to // truncate hostarch.AddrRangeSeq when errors occur. diff --git a/pkg/sentry/mm/pma.go b/pkg/sentry/mm/pma.go index dd9831b46..00ea047ea 100644 --- a/pkg/sentry/mm/pma.go +++ b/pkg/sentry/mm/pma.go @@ -796,75 +796,13 @@ func (mm *MemoryManager) movePMAsLocked(oldAR, newAR hostarch.AddrRange) { mm.unmapASLocked(oldAR) } -// getPMAInternalMappingsLocked ensures that pmas for all addresses in ar have -// cached internal mappings. It returns: -// -// - An iterator to the gap after the last pma with internal mappings -// containing an address in ar. If internal mappings exist for no addresses in -// ar, the iterator is to a gap that begins before ar.Start. -// -// - An error that is non-nil if internal mappings exist for only a subset of -// ar. -// -// Preconditions: -// - mm.activeMu must be locked for writing. -// - pseg.Range().Contains(ar.Start). -// - pmas must exist for all addresses in ar. -// - ar.Length() != 0. -// -// Postconditions: getPMAInternalMappingsLocked does not invalidate iterators -// into mm.pmas. -func (mm *MemoryManager) getPMAInternalMappingsLocked(pseg pmaIterator, ar hostarch.AddrRange) (pmaGapIterator, error) { - if checkInvariants { - if !ar.WellFormed() || ar.Length() == 0 { - panic(fmt.Sprintf("invalid ar: %v", ar)) - } - if !pseg.Range().Contains(ar.Start) { - panic(fmt.Sprintf("initial pma %v does not cover start of ar %v", pseg.Range(), ar)) - } - } - - for { - if err := pseg.getInternalMappingsLocked(); err != nil { - return pseg.PrevGap(), err - } - if ar.End <= pseg.End() { - return pseg.NextGap(), nil - } - pseg, _ = pseg.NextNonEmpty() - } -} - -// getVecPMAInternalMappingsLocked ensures that pmas for all addresses in ars -// have cached internal mappings. It returns the subset of ars for which -// internal mappings exist. If this is not equal to ars, it returns a non-nil -// error explaining why. -// -// Preconditions: -// - mm.activeMu must be locked for writing. -// - pmas must exist for all addresses in ar. -// -// Postconditions: getVecPMAInternalMappingsLocked does not invalidate iterators -// into mm.pmas. -func (mm *MemoryManager) getVecPMAInternalMappingsLocked(ars hostarch.AddrRangeSeq) (hostarch.AddrRangeSeq, error) { - for arsit := ars; !arsit.IsEmpty(); arsit = arsit.Tail() { - ar := arsit.Head() - if ar.Length() == 0 { - continue - } - if pend, err := mm.getPMAInternalMappingsLocked(mm.pmas.FindSegment(ar.Start), ar); err != nil { - return truncatedAddrRangeSeq(ars, arsit, pend.Start()), err - } - } - return ars, nil -} - -// internalMappingsLocked returns internal mappings for addresses in ar. +// internalMappingsLocked returns cached internal mappings for addresses in ar. // // Preconditions: // - mm.activeMu must be locked. -// - Internal mappings must have been previously established for all addresses -// in ar. +// - While mm.activeMu was locked, a call to +// existingPMAsLocked(needInternalMappings=true) succeeded for all +// addresses in ar. // - ar.Length() != 0. // - pseg.Range().Contains(ar.Start). func (mm *MemoryManager) internalMappingsLocked(pseg pmaIterator, ar hostarch.AddrRange) safemem.BlockSeq { @@ -898,12 +836,14 @@ func (mm *MemoryManager) internalMappingsLocked(pseg pmaIterator, ar hostarch.Ad return safemem.BlockSeqFromSlice(ims) } -// vecInternalMappingsLocked returns internal mappings for addresses in ars. +// vecInternalMappingsLocked returns cached internal mappings for addresses in +// ars. // // Preconditions: // - mm.activeMu must be locked. -// - Internal mappings must have been previously established for all addresses -// in ars. +// - While mm.activeMu was locked, a call to +// existingVecPMAsLocked(needInternalMappings=true) succeeded for all +// addresses in ars. func (mm *MemoryManager) vecInternalMappingsLocked(ars hostarch.AddrRangeSeq) safemem.BlockSeq { var ims []safemem.Block for ; !ars.IsEmpty(); ars = ars.Tail() { diff --git a/pkg/sentry/pgalloc/pgalloc.go b/pkg/sentry/pgalloc/pgalloc.go index f01c8573b..e7fe18096 100644 --- a/pkg/sentry/pgalloc/pgalloc.go +++ b/pkg/sentry/pgalloc/pgalloc.go @@ -66,6 +66,8 @@ func (d Direction) String() string { // MemoryFile is a memmap.File whose pages may be allocated to arbitrary // users. type MemoryFile struct { + memmap.NoBufferedIOFallback + // opts holds options passed to NewMemoryFile. opts is immutable. opts MemoryFileOpts