kvm: map entrie sentry address space if the kernel is 6.9 or higher

Before th 6.9 kernel we couldn't map the entire sentry address space into VM,
because there were a two-byte overhead per page in the kernel. This issue was
fixed by a364c014a2c1 ("kvm/x86: allocate the write-tracking metadata
on-demand").

KVM benchmark results:
cpu: 11th Gen Intel(R) Core(TM) i7-1185G7 @ 3.00GHz
                             │    before    │                after                │
                             │    sec/op    │    sec/op     vs base               │
ApplicationSyscall-4           336.5n ± ∞ ¹   337.9n ± ∞ ¹        ~ (p=0.786 n=5)
KernelSyscall-4                3.400µ ± ∞ ¹   3.185µ ± ∞ ¹   -6.32% (p=0.008 n=5)
SentrySyscall-4                324.4n ± ∞ ¹   206.1n ± ∞ ¹  -36.47% (p=0.008 n=5)
HostMMap-4                     6.703µ ± ∞ ¹   2.765µ ± ∞ ¹  -58.75% (p=0.008 n=5)
KernelVDSO-4                   34.59n ± ∞ ¹   33.61n ± ∞ ¹        ~ (p=0.056 n=5)
WorldSwitchToUserRoundtrip-4   5.093µ ± ∞ ¹   3.565µ ± ∞ ¹  -30.00% (p=0.008 n=5)
geomean                        871.5n         647.2n        -25.74%
¹ need >= 6 samples for confidence interval at level 0.95
PiperOrigin-RevId: 667725254

pkg/sentry/platform/kvm/BUILD

@@ -67,6 +67,7 @@ go_library(
         "//pkg/cpuid",
         "//pkg/fd",
         "//pkg/hostarch",
+        "//pkg/hostos",
         "//pkg/hosttid",
         "//pkg/log",
         "//pkg/metric",

pkg/sentry/platform/kvm/bluepill_fault.go

@@ -21,14 +21,14 @@ import (
 	"gvisor.dev/gvisor/pkg/hostarch"
 )
 
-const (
+var (
 	// faultBlockSize is the size used for servicing memory faults.
 	//
 	// This should be large enough to avoid frequent faults and avoid using
 	// all available KVM slots (~512), but small enough that KVM does not
 	// complain about slot sizes (~4GB). See handleBluepillFault for how
 	// this block is used.
-	faultBlockSize = 2 << 30
+	faultBlockSize = uintptr(2 << 30)
 
 	// faultBlockMask is the mask for the fault blocks.
 	//

pkg/sentry/platform/kvm/kvm_safecopy_test.go

@@ -59,7 +59,7 @@ func testSafecopy(t *testing.T, mapSize uintptr, fileSize uintptr, testFunc func
 				uintptr(memfile.Fd()),
 				0)
 			if errno != 0 {
-				t.Errorf("error mapping file: %v", errno)
+				t.Fatalf("error mapping file: %v", errno)
 			}
 			mappings[i] = addr
 			testFunc(t, c, addr)
@@ -68,8 +68,9 @@ func testSafecopy(t *testing.T, mapSize uintptr, fileSize uintptr, testFunc func
 	})
 }
 
+var mapSize = faultBlockSize
+
 func TestSafecopySigbus(t *testing.T) {
-	mapSize := uintptr(faultBlockSize)
 	fileSize := mapSize - hostarch.PageSize
 	buf := make([]byte, hostarch.PageSize)
 	testSafecopy(t, mapSize, fileSize, func(t *testing.T, c *vCPU, addr uintptr) {
@@ -83,7 +84,6 @@ func TestSafecopySigbus(t *testing.T) {
 }
 
 func TestSafecopy(t *testing.T) {
-	mapSize := uintptr(faultBlockSize)
 	fileSize := mapSize
 	testSafecopy(t, mapSize, fileSize, func(t *testing.T, c *vCPU, addr uintptr) {
 		want := uint32(0x12345678)

pkg/sentry/platform/kvm/kvm_test.go

@@ -517,6 +517,34 @@ func BenchmarkKernelSyscall(b *testing.B) {
 	})
 }
 
+func BenchmarkSentrySyscall(b *testing.B) {
+	// Note that the target passed here is irrelevant, we never execute SwitchToUser.
+	applicationTest(b, true, testutil.AddrOfGetpid(), func(c *vCPU, regs *arch.Registers, pt *pagetables.PageTables) bool {
+		// iteration does not include machine.Get() / machine.Put().
+		for i := 0; i < b.N; i++ {
+			testutil.Getpid()
+		}
+		return false
+	})
+}
+
+func BenchmarkHostMMap(b *testing.B) {
+	kvmTest(b, nil, func(c *vCPU) bool {
+		// iteration does not include machine.Get() / machine.Put().
+		for i := 0; i < b.N; i++ {
+			addr, _, errno := unix.Syscall6(unix.SYS_MMAP, 0, hostarch.PageSize, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_ANONYMOUS|unix.MAP_PRIVATE, 0, 0)
+			if errno != 0 {
+				b.Fatalf("mmap failed: %s", errno)
+			}
+			_, _, errno = unix.Syscall(unix.SYS_MUNMAP, addr, hostarch.PageSize, 0)
+			if errno != 0 {
+				b.Fatalf("munmap failed: %s", errno)
+			}
+		}
+		return false
+	})
+}
+
 func BenchmarkKernelVDSO(b *testing.B) {
 	// Note that the target passed here is irrelevant, we never execute SwitchToUser.
 	applicationTest(b, true, testutil.AddrOfGetpid(), func(c *vCPU, regs *arch.Registers, pt *pagetables.PageTables) bool {

pkg/sentry/platform/kvm/machine.go

@@ -25,6 +25,7 @@ import (
 	"gvisor.dev/gvisor/pkg/abi/linux"
 	"gvisor.dev/gvisor/pkg/atomicbitops"
 	"gvisor.dev/gvisor/pkg/hostarch"
+	"gvisor.dev/gvisor/pkg/hostos"
 	"gvisor.dev/gvisor/pkg/hosttid"
 	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/pkg/metric"
@@ -296,9 +297,25 @@ func newMachine(vm int) (*machine, error) {
 	m.upperSharedPageTables.MarkReadOnlyShared()
 	m.kernel.PageTables = pagetables.NewWithUpper(newAllocator(), m.upperSharedPageTables, ring0.KernelStartAddress)
 
-	// Install seccomp rules to trap runtime mmap system calls. They will
-	// be handled by seccompMmapHandler.
-	seccompMmapRules(m)
+	// Before the 6.9 kernel we couldn't map the entire sentry
+	// address space into VM, because there were a two-byte overhead
+	// per page in the kernel. This issue was fixed by a364c014a2c1
+	// ("kvm/x86: allocate the write-tracking metadata on-demand").
+	kernelVersion, err := hostos.KernelVersion()
+	if err != nil {
+		return nil, err
+	}
+	mapEntireAddressSpace := runtime.GOARCH != "amd64" || kernelVersion.AtLeast(6, 9)
+	if mapEntireAddressSpace {
+		// Increase faultBlockSize to be sure that we will not reach the limit.
+		// faultBlockSize has to equal or less than KVM_MEM_MAX_NR_PAGES.
+		faultBlockSize = uintptr(1) << 42
+		faultBlockMask = ^uintptr(faultBlockSize - 1)
+	} else {
+		// Install seccomp rules to trap runtime mmap system calls. They will
+		// be handled by seccompMmapHandler.
+		seccompMmapRules(m)
+	}
 
 	// Apply the physical mappings. Note that these mappings may point to
 	// guest physical addresses that are not actually available. These
@@ -352,6 +369,7 @@ func newMachine(vm int) (*machine, error) {
 	// seccompMmapHandler, so here we have to guarantee that mmap is not
 	// called while we hold the slot spinlock.
 	disableAsyncPreemption()
+
 	applyVirtualRegions(func(vr virtualRegion) {
 		if excludeVirtualRegion(vr) {
 			return // skip region.
@@ -365,8 +383,12 @@ func newMachine(vm int) (*machine, error) {
 		mapRegion(vr, 0)
 
 	})
+	if mapEntireAddressSpace {
+		for _, r := range physicalRegions {
+			m.mapPhysical(r.physical, r.length, physicalRegions)
+		}
+	}
 	enableAsyncPreemption()
-
 	// Initialize architecture state.
 	if err := m.initArchState(); err != nil {
 		m.Destroy()