From ccc5642b1d5b222377e18e69596aa30d533caab1 Mon Sep 17 00:00:00 2001 From: Andrei Vagin Date: Mon, 26 Aug 2024 14:53:14 -0700 Subject: [PATCH] kvm: map entrie sentry address space if the kernel is 6.9 or higher MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before th 6.9 kernel we couldn't map the entire sentry address space into VM, because there were a two-byte overhead per page in the kernel. This issue was fixed by a364c014a2c1 ("kvm/x86: allocate the write-tracking metadata on-demand"). KVM benchmark results: cpu: 11th Gen Intel(R) Core(TM) i7-1185G7 @ 3.00GHz │ before │ after │ │ sec/op │ sec/op vs base │ ApplicationSyscall-4 336.5n ± ∞ ¹ 337.9n ± ∞ ¹ ~ (p=0.786 n=5) KernelSyscall-4 3.400µ ± ∞ ¹ 3.185µ ± ∞ ¹ -6.32% (p=0.008 n=5) SentrySyscall-4 324.4n ± ∞ ¹ 206.1n ± ∞ ¹ -36.47% (p=0.008 n=5) HostMMap-4 6.703µ ± ∞ ¹ 2.765µ ± ∞ ¹ -58.75% (p=0.008 n=5) KernelVDSO-4 34.59n ± ∞ ¹ 33.61n ± ∞ ¹ ~ (p=0.056 n=5) WorldSwitchToUserRoundtrip-4 5.093µ ± ∞ ¹ 3.565µ ± ∞ ¹ -30.00% (p=0.008 n=5) geomean 871.5n 647.2n -25.74% ¹ need >= 6 samples for confidence interval at level 0.95 PiperOrigin-RevId: 667725254 --- pkg/sentry/platform/kvm/BUILD | 1 + pkg/sentry/platform/kvm/bluepill_fault.go | 4 +-- pkg/sentry/platform/kvm/kvm_safecopy_test.go | 6 ++-- pkg/sentry/platform/kvm/kvm_test.go | 28 ++++++++++++++++++ pkg/sentry/platform/kvm/machine.go | 30 +++++++++++++++++--- 5 files changed, 60 insertions(+), 9 deletions(-) diff --git a/pkg/sentry/platform/kvm/BUILD b/pkg/sentry/platform/kvm/BUILD index 72acd018d..f153ab489 100644 --- a/pkg/sentry/platform/kvm/BUILD +++ b/pkg/sentry/platform/kvm/BUILD @@ -67,6 +67,7 @@ go_library( "//pkg/cpuid", "//pkg/fd", "//pkg/hostarch", + "//pkg/hostos", "//pkg/hosttid", "//pkg/log", "//pkg/metric", diff --git a/pkg/sentry/platform/kvm/bluepill_fault.go b/pkg/sentry/platform/kvm/bluepill_fault.go index c13ebd8a2..6955864a4 100644 --- a/pkg/sentry/platform/kvm/bluepill_fault.go +++ b/pkg/sentry/platform/kvm/bluepill_fault.go @@ -21,14 +21,14 @@ import ( "gvisor.dev/gvisor/pkg/hostarch" ) -const ( +var ( // faultBlockSize is the size used for servicing memory faults. // // This should be large enough to avoid frequent faults and avoid using // all available KVM slots (~512), but small enough that KVM does not // complain about slot sizes (~4GB). See handleBluepillFault for how // this block is used. - faultBlockSize = 2 << 30 + faultBlockSize = uintptr(2 << 30) // faultBlockMask is the mask for the fault blocks. // diff --git a/pkg/sentry/platform/kvm/kvm_safecopy_test.go b/pkg/sentry/platform/kvm/kvm_safecopy_test.go index fe488e707..1a9732e17 100644 --- a/pkg/sentry/platform/kvm/kvm_safecopy_test.go +++ b/pkg/sentry/platform/kvm/kvm_safecopy_test.go @@ -59,7 +59,7 @@ func testSafecopy(t *testing.T, mapSize uintptr, fileSize uintptr, testFunc func uintptr(memfile.Fd()), 0) if errno != 0 { - t.Errorf("error mapping file: %v", errno) + t.Fatalf("error mapping file: %v", errno) } mappings[i] = addr testFunc(t, c, addr) @@ -68,8 +68,9 @@ func testSafecopy(t *testing.T, mapSize uintptr, fileSize uintptr, testFunc func }) } +var mapSize = faultBlockSize + func TestSafecopySigbus(t *testing.T) { - mapSize := uintptr(faultBlockSize) fileSize := mapSize - hostarch.PageSize buf := make([]byte, hostarch.PageSize) testSafecopy(t, mapSize, fileSize, func(t *testing.T, c *vCPU, addr uintptr) { @@ -83,7 +84,6 @@ func TestSafecopySigbus(t *testing.T) { } func TestSafecopy(t *testing.T) { - mapSize := uintptr(faultBlockSize) fileSize := mapSize testSafecopy(t, mapSize, fileSize, func(t *testing.T, c *vCPU, addr uintptr) { want := uint32(0x12345678) diff --git a/pkg/sentry/platform/kvm/kvm_test.go b/pkg/sentry/platform/kvm/kvm_test.go index 29cf18df7..4874185ec 100644 --- a/pkg/sentry/platform/kvm/kvm_test.go +++ b/pkg/sentry/platform/kvm/kvm_test.go @@ -517,6 +517,34 @@ func BenchmarkKernelSyscall(b *testing.B) { }) } +func BenchmarkSentrySyscall(b *testing.B) { + // Note that the target passed here is irrelevant, we never execute SwitchToUser. + applicationTest(b, true, testutil.AddrOfGetpid(), func(c *vCPU, regs *arch.Registers, pt *pagetables.PageTables) bool { + // iteration does not include machine.Get() / machine.Put(). + for i := 0; i < b.N; i++ { + testutil.Getpid() + } + return false + }) +} + +func BenchmarkHostMMap(b *testing.B) { + kvmTest(b, nil, func(c *vCPU) bool { + // iteration does not include machine.Get() / machine.Put(). + for i := 0; i < b.N; i++ { + addr, _, errno := unix.Syscall6(unix.SYS_MMAP, 0, hostarch.PageSize, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_ANONYMOUS|unix.MAP_PRIVATE, 0, 0) + if errno != 0 { + b.Fatalf("mmap failed: %s", errno) + } + _, _, errno = unix.Syscall(unix.SYS_MUNMAP, addr, hostarch.PageSize, 0) + if errno != 0 { + b.Fatalf("munmap failed: %s", errno) + } + } + return false + }) +} + func BenchmarkKernelVDSO(b *testing.B) { // Note that the target passed here is irrelevant, we never execute SwitchToUser. applicationTest(b, true, testutil.AddrOfGetpid(), func(c *vCPU, regs *arch.Registers, pt *pagetables.PageTables) bool { diff --git a/pkg/sentry/platform/kvm/machine.go b/pkg/sentry/platform/kvm/machine.go index b427caf21..278694d79 100644 --- a/pkg/sentry/platform/kvm/machine.go +++ b/pkg/sentry/platform/kvm/machine.go @@ -25,6 +25,7 @@ import ( "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/atomicbitops" "gvisor.dev/gvisor/pkg/hostarch" + "gvisor.dev/gvisor/pkg/hostos" "gvisor.dev/gvisor/pkg/hosttid" "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/metric" @@ -296,9 +297,25 @@ func newMachine(vm int) (*machine, error) { m.upperSharedPageTables.MarkReadOnlyShared() m.kernel.PageTables = pagetables.NewWithUpper(newAllocator(), m.upperSharedPageTables, ring0.KernelStartAddress) - // Install seccomp rules to trap runtime mmap system calls. They will - // be handled by seccompMmapHandler. - seccompMmapRules(m) + // Before the 6.9 kernel we couldn't map the entire sentry + // address space into VM, because there were a two-byte overhead + // per page in the kernel. This issue was fixed by a364c014a2c1 + // ("kvm/x86: allocate the write-tracking metadata on-demand"). + kernelVersion, err := hostos.KernelVersion() + if err != nil { + return nil, err + } + mapEntireAddressSpace := runtime.GOARCH != "amd64" || kernelVersion.AtLeast(6, 9) + if mapEntireAddressSpace { + // Increase faultBlockSize to be sure that we will not reach the limit. + // faultBlockSize has to equal or less than KVM_MEM_MAX_NR_PAGES. + faultBlockSize = uintptr(1) << 42 + faultBlockMask = ^uintptr(faultBlockSize - 1) + } else { + // Install seccomp rules to trap runtime mmap system calls. They will + // be handled by seccompMmapHandler. + seccompMmapRules(m) + } // Apply the physical mappings. Note that these mappings may point to // guest physical addresses that are not actually available. These @@ -352,6 +369,7 @@ func newMachine(vm int) (*machine, error) { // seccompMmapHandler, so here we have to guarantee that mmap is not // called while we hold the slot spinlock. disableAsyncPreemption() + applyVirtualRegions(func(vr virtualRegion) { if excludeVirtualRegion(vr) { return // skip region. @@ -365,8 +383,12 @@ func newMachine(vm int) (*machine, error) { mapRegion(vr, 0) }) + if mapEntireAddressSpace { + for _, r := range physicalRegions { + m.mapPhysical(r.physical, r.length, physicalRegions) + } + } enableAsyncPreemption() - // Initialize architecture state. if err := m.initArchState(); err != nil { m.Destroy()