From 56ce1247561eeac1f3555d1a34cf1cc38b7cd120 Mon Sep 17 00:00:00 2001 From: Tiwei Bie Date: Mon, 9 Oct 2023 21:09:17 +0800 Subject: [PATCH] erofs: eliminate unnecessary escapes and memory copies Currently, when we access the on-disk filesystem objects, the objects will be copied to temporary buffers and will escape to heap. This patch gets rid of these unnecessary copies and escapes by casting and using the memory backed by the image file directly. Signed-off-by: Tiwei Bie --- pkg/erofs/BUILD | 6 +- pkg/erofs/erofs.go | 137 +++++++++++++++++++++++++++++--------- pkg/erofs/erofs_test.go | 16 ++--- pkg/erofs/erofs_unsafe.go | 21 ++++++ 4 files changed, 137 insertions(+), 43 deletions(-) create mode 100644 pkg/erofs/erofs_unsafe.go diff --git a/pkg/erofs/BUILD b/pkg/erofs/BUILD index 8c033adf6..b517ee3ed 100644 --- a/pkg/erofs/BUILD +++ b/pkg/erofs/BUILD @@ -7,7 +7,10 @@ package( go_library( name = "erofs", - srcs = ["erofs.go"], + srcs = [ + "erofs.go", + "erofs_unsafe.go", + ], marshal = True, visibility = ["//visibility:public"], deps = [ @@ -17,7 +20,6 @@ go_library( "//pkg/hostarch", "//pkg/log", "//pkg/marshal", - "//pkg/marshal/primitive", "//pkg/safemem", "@org_golang_x_sys//unix:go_default_library", ], diff --git a/pkg/erofs/erofs.go b/pkg/erofs/erofs.go index 9ab5b595c..f0ac37166 100644 --- a/pkg/erofs/erofs.go +++ b/pkg/erofs/erofs.go @@ -35,7 +35,6 @@ import ( "gvisor.dev/gvisor/pkg/hostarch" "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/marshal" - "gvisor.dev/gvisor/pkg/marshal/primitive" "gvisor.dev/gvisor/pkg/safemem" ) @@ -92,6 +91,14 @@ const ( FeatureIncompatSupported = 0x0 ) +// Sizes of on-disk structures in bytes. +const ( + SuperBlockSize = 128 + InodeCompactSize = 32 + InodeExtendedSize = 64 + DirentSize = 12 +) + // SuperBlock represents on-disk super block. // // +marshal @@ -176,16 +183,20 @@ type InodeExtended struct { // Dirent represents on-disk directory entry. // -// This struct is misaligned according to go_marshal, as it is only 12 bytes in size. The -// last field needs to be marked unaligned so the struct is marked unpacked and the -// generated methods behave correctly. -// // +marshal type Dirent struct { - Nid uint64 + NidLow uint32 + NidHigh uint32 NameOff uint16 FileType uint8 - Reserved uint8 `marshal:"unaligned"` + Reserved uint8 +} + +// Nid returns the inode number of the inode referenced by this dirent. +func (d *Dirent) Nid() uint64 { + // EROFS on-disk structures are always in little endian. + // TODO: This implementation does not support big endian yet. + return (uint64(d.NidHigh) << 32) | uint64(d.NidLow) } // Image represents an open EROFS image. @@ -251,7 +262,8 @@ func (i *Image) RootNid() uint64 { // initSuperBlock initializes the super block of this image. func (i *Image) initSuperBlock() error { - if err := i.UnmarshalAt(&i.sb, SuperBlockOffset); err != nil { + // i.sb is used in the hot path. Let's save a copy of it. + if err := i.unmarshalAt(&i.sb, SuperBlockOffset); err != nil { return fmt.Errorf("image size is too small") } @@ -303,19 +315,79 @@ func (i *Image) FD() int { return int(i.src.Fd()) } -// BytesAt returns the bytes at [off, off+n) of the image. -func (i *Image) BytesAt(off, n uint64) ([]byte, error) { +// checkRange checks whether the range [off, off+n) is valid. +func (i *Image) checkRange(off, n uint64) bool { size := uint64(len(i.bytes)) end := off + n - if off >= size || off > end || end > size { - log.Warningf("Invalid range (off: 0x%x, n: 0x%x) for image (size: 0x%x)", off, n, size) - return nil, linuxerr.EFAULT - } - return i.bytes[off:end], nil + return off < size && off <= end && end <= size } -// UnmarshalAt deserializes data from the bytes at [off, off+n) of the image. -func (i *Image) UnmarshalAt(data marshal.Marshallable, off uint64) error { +// BytesAt returns the bytes at [off, off+n) of the image. +func (i *Image) BytesAt(off, n uint64) ([]byte, error) { + if ok := i.checkRange(off, n); !ok { + log.Warningf("Invalid byte range (off: 0x%x, n: 0x%x) for image (size: 0x%x)", off, n, len(i.bytes)) + return nil, linuxerr.EFAULT + } + return i.bytes[off : off+n], nil +} + +// checkInodeAlignment checks whether off matches inode's alignment requirement. +func checkInodeAlignment(off uint64) bool { + // Each valid inode should be aligned with an inode slot, which is + // a fixed value (32 bytes). + return off&((1< 0 { - // Unmarshal the first dirent in the current block. + // Get the first dirent in the current block. direntOff := start - d := &Dirent{} - if err := i.image.UnmarshalAt(d, direntOff); err != nil { + d, err := i.image.direntAt(direntOff) + if err != nil { return err } // Apart from the offset of the first filename, nameOff0 also indicates @@ -663,9 +734,8 @@ func (i *Inode) IterDirents(cb func(name string, typ uint8, nid uint64) error) e next = nil nameLen = maxSize - uint32(d.NameOff) } else { - // Unmarshal the next adjacent dirent. - next = &Dirent{} - if err := i.image.UnmarshalAt(next, direntOff); err != nil { + // Get the next adjacent dirent. + if next, err = i.image.direntAt(direntOff); err != nil { return err } nameLen = uint32(next.NameOff - d.NameOff) @@ -685,10 +755,11 @@ func (i *Inode) IterDirents(cb func(name string, typ uint8, nid uint64) error) e return linuxerr.EUCLEAN } name := string(buf[:nameLen]) - if err := cb(name, d.FileType, d.Nid); err != nil { + if err := cb(name, d.FileType, d.Nid()); err != nil { return err } + // Go on to process the next adjacent dirent. d = next } diff --git a/pkg/erofs/erofs_test.go b/pkg/erofs/erofs_test.go index 28bb0e8fd..d29b3b87e 100644 --- a/pkg/erofs/erofs_test.go +++ b/pkg/erofs/erofs_test.go @@ -19,19 +19,19 @@ import ( ) func TestOnDiskStructureSizes(t *testing.T) { - if sb := new(SuperBlock); sb.SizeBytes() != 128 { - t.Errorf("wrong super block size: want 128, got %d", sb.SizeBytes()) + if sb := new(SuperBlock); sb.SizeBytes() != SuperBlockSize { + t.Errorf("wrong super block size: want %d, got %d", SuperBlockSize, sb.SizeBytes()) } - if i := new(InodeCompact); i.SizeBytes() != 32 { - t.Errorf("wrong compact inode size: want 32, got %d", i.SizeBytes()) + if i := new(InodeCompact); i.SizeBytes() != InodeCompactSize { + t.Errorf("wrong compact inode size: want %d, got %d", InodeCompactSize, i.SizeBytes()) } - if i := new(InodeExtended); i.SizeBytes() != 64 { - t.Errorf("wrong extended inode size: want 64, got %d", i.SizeBytes()) + if i := new(InodeExtended); i.SizeBytes() != InodeExtendedSize { + t.Errorf("wrong extended inode size: want %d, got %d", InodeExtendedSize, i.SizeBytes()) } - if d := new(Dirent); d.SizeBytes() != 12 { - t.Errorf("wrong dirent size: want 12, got %d", d.SizeBytes()) + if d := new(Dirent); d.SizeBytes() != DirentSize { + t.Errorf("wrong dirent size: want %d, got %d", DirentSize, d.SizeBytes()) } } diff --git a/pkg/erofs/erofs_unsafe.go b/pkg/erofs/erofs_unsafe.go new file mode 100644 index 000000000..c1c37dbc3 --- /dev/null +++ b/pkg/erofs/erofs_unsafe.go @@ -0,0 +1,21 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package erofs + +import "unsafe" + +func (i *Image) pointerAt(off uint64) unsafe.Pointer { + return unsafe.Pointer(&i.bytes[off]) +}