From fbca0560dd1f9fc240d18137b5a2ba4d43e25f7f Mon Sep 17 00:00:00 2001 From: Jamie Liu Date: Mon, 17 Mar 2025 11:30:46 -0700 Subject: [PATCH] kvm: honor memmap.File.MemoryType() Updates #11436 PiperOrigin-RevId: 737689743 --- pkg/ring0/pagetables/pagetables_aarch64.go | 59 ++++++++++--------- pkg/ring0/pagetables/pagetables_amd64_test.go | 10 ++++ pkg/ring0/pagetables/pagetables_arm64_test.go | 7 +++ pkg/ring0/pagetables/pagetables_x86.go | 32 ++++++---- pkg/sentry/platform/kvm/address_space.go | 12 ++-- pkg/sentry/platform/kvm/kvm_const_arm64.go | 7 --- pkg/sentry/platform/kvm/machine_amd64.go | 5 ++ .../platform/kvm/machine_amd64_unsafe.go | 31 ++++++++++ .../platform/kvm/machine_arm64_unsafe.go | 7 ++- 9 files changed, 116 insertions(+), 54 deletions(-) diff --git a/pkg/ring0/pagetables/pagetables_aarch64.go b/pkg/ring0/pagetables/pagetables_aarch64.go index 6c2fe2a70..97ce934e0 100644 --- a/pkg/ring0/pagetables/pagetables_aarch64.go +++ b/pkg/ring0/pagetables/pagetables_aarch64.go @@ -52,29 +52,26 @@ func (p *PageTables) TTBR1_EL1(noFlush bool, asid uint16) uint64 { // Bits in page table entries. const ( - typeTable = 0x3 << 0 - typeSect = 0x1 << 0 - typePage = 0x3 << 0 - pteValid = 0x1 << 0 - pteTableBit = 0x1 << 1 - pteTypeMask = 0x3 << 0 - present = pteValid | pteTableBit - user = 0x1 << 6 /* AP[1] */ - readOnly = 0x1 << 7 /* AP[2] */ - accessed = 0x1 << 10 - dbm = 0x1 << 51 - writable = dbm - cont = 0x1 << 52 - pxn = 0x1 << 53 - xn = 0x1 << 54 - dirty = 0x1 << 55 - nG = 0x1 << 11 - shared = 0x3 << 8 -) - -const ( - mtDevicenGnRE = 0x1 << 2 - mtNormal = 0x4 << 2 + typeTable = 0x3 << 0 + typeSect = 0x1 << 0 + typePage = 0x3 << 0 + pteValid = 0x1 << 0 + pteTableBit = 0x1 << 1 + pteTypeMask = 0x3 << 0 + present = pteValid | pteTableBit + attrIndxShift = 2 + attrIndxMask = 0x7 + user = 0x1 << 6 /* AP[1] */ + readOnly = 0x1 << 7 /* AP[2] */ + accessed = 0x1 << 10 + dbm = 0x1 << 51 + writable = dbm + cont = 0x1 << 52 + pxn = 0x1 << 53 + xn = 0x1 << 54 + dirty = 0x1 << 55 + nG = 0x1 << 11 + shared = 0x3 << 8 ) const ( @@ -93,6 +90,9 @@ type MapOpts struct { // User indicates the page is a user page. User bool + + // MemoryType is the memory type. + MemoryType hostarch.MemoryType } // PTE is a page table entry. @@ -119,15 +119,15 @@ func (p *PTE) Valid() bool { //go:nosplit func (p *PTE) Opts() MapOpts { v := atomic.LoadUintptr((*uintptr)(p)) - return MapOpts{ AccessType: hostarch.AccessType{ Read: true, Write: v&readOnly == 0, Execute: v&xn == 0, }, - Global: v&nG == 0, - User: v&user != 0, + Global: v&nG == 0, + User: v&user != 0, + MemoryType: hostarch.MemoryType((v >> attrIndxShift) & attrIndxMask), } } @@ -191,11 +191,12 @@ func (p *PTE) Set(addr uintptr, opts MapOpts) { if opts.User { v |= user - v |= mtNormal } else { v = v &^ user - v |= mtNormal } + + v |= uintptr(opts.MemoryType&attrIndxMask) << attrIndxShift + atomic.StoreUintptr((*uintptr)(p), v) } @@ -209,7 +210,7 @@ func (p *PTE) setPageTable(pt *PageTables, ptes *PTEs) { // This should never happen. panic("unaligned physical address!") } - v := addr | typeTable | protDefault | mtNormal + v := addr | typeTable | protDefault | (uintptr(hostarch.MemoryTypeWriteBack) << attrIndxShift) atomic.StoreUintptr((*uintptr)(p), v) } diff --git a/pkg/ring0/pagetables/pagetables_amd64_test.go b/pkg/ring0/pagetables/pagetables_amd64_test.go index c27b3b10a..2c08cf8f4 100644 --- a/pkg/ring0/pagetables/pagetables_amd64_test.go +++ b/pkg/ring0/pagetables/pagetables_amd64_test.go @@ -74,3 +74,13 @@ func TestSplit2MPage(t *testing.T) { {0x00007f0000000000 + pmdSize - pteSize, pteSize, pmdSize*42 + pmdSize - pteSize, MapOpts{AccessType: hostarch.Read}}, }) } + +func TestNumMemoryTypes(t *testing.T) { + // The PAT accommodates up to 8 entries. However, PTE.Set() currently + // assumes that NumMemoryTypes <= 4, since the location of the most + // significant bit of the PAT index in page table entries varies depending + // on page size (and is never bit 5 == writeThroughShift + 2). + if hostarch.NumMemoryTypes > 4 { + t.Errorf("PTE.Set() and PTE.Opts() must be altered to handle %d MemoryTypes", hostarch.NumMemoryTypes) + } +} diff --git a/pkg/ring0/pagetables/pagetables_arm64_test.go b/pkg/ring0/pagetables/pagetables_arm64_test.go index 1c919ec7d..0c73e0f72 100644 --- a/pkg/ring0/pagetables/pagetables_arm64_test.go +++ b/pkg/ring0/pagetables/pagetables_arm64_test.go @@ -79,3 +79,10 @@ func TestSplit2MPage(t *testing.T) { {0x0000ff0000000000 + pmdSize - pteSize, pteSize, pmdSize*42 + pmdSize - pteSize, MapOpts{AccessType: hostarch.Read, User: true}}, }) } + +func TestNumMemoryTypes(t *testing.T) { + // MAIR accommodates up to 8 entries. + if hostarch.NumMemoryTypes > 8 { + t.Errorf("PTE.Set() and PTE.Opts() must be altered to map %d MemoryTypes to a smaller set of MAIR entries", hostarch.NumMemoryTypes) + } +} diff --git a/pkg/ring0/pagetables/pagetables_x86.go b/pkg/ring0/pagetables/pagetables_x86.go index dc98d8452..2109ccdf3 100644 --- a/pkg/ring0/pagetables/pagetables_x86.go +++ b/pkg/ring0/pagetables/pagetables_x86.go @@ -49,16 +49,17 @@ func (p *PageTables) CR3(noFlush bool, pcid uint16) uint64 { // Bits in page table entries. const ( - present = 0x001 - writable = 0x002 - user = 0x004 - writeThrough = 0x008 - cacheDisable = 0x010 - accessed = 0x020 - dirty = 0x040 - super = 0x080 - global = 0x100 - optionMask = executeDisable | 0xfff + present = 0x001 + writable = 0x002 + user = 0x004 + accessed = 0x020 + dirty = 0x040 + super = 0x080 + global = 0x100 + optionMask = executeDisable | 0xfff + + writeThroughShift = 3 + patIndexMask = 0x3 ) // MapOpts are x86 options. @@ -71,6 +72,9 @@ type MapOpts struct { // User indicates the page is a user page. User bool + + // MemoryType is the memory type. + MemoryType hostarch.MemoryType } // PTE is a page table entry. @@ -103,8 +107,9 @@ func (p *PTE) Opts() MapOpts { Write: v&writable != 0, Execute: v&executeDisable == 0, }, - Global: v&global != 0, - User: v&user != 0, + Global: v&global != 0, + User: v&user != 0, + MemoryType: hostarch.MemoryType((v >> writeThroughShift) & patIndexMask), } } @@ -154,6 +159,7 @@ func (p *PTE) Set(addr uintptr, opts MapOpts) { if opts.AccessType.Write { v |= writable | dirty } + v |= uintptr(opts.MemoryType&patIndexMask) << writeThroughShift if p.IsSuper() { // Note that this is inherited from the previous instance. Set // does not change the value of Super. See above. @@ -172,7 +178,7 @@ func (p *PTE) setPageTable(pt *PageTables, ptes *PTEs) { // This should never happen. panic("unaligned physical address!") } - v := addr | present | user | writable | accessed | dirty + v := addr | present | user | writable | accessed | dirty | (uintptr(hostarch.MemoryTypeWriteBack) << writeThroughShift) atomic.StoreUintptr((*uintptr)(p), v) } diff --git a/pkg/sentry/platform/kvm/address_space.go b/pkg/sentry/platform/kvm/address_space.go index 1b16dcb3c..41dc9e6a2 100644 --- a/pkg/sentry/platform/kvm/address_space.go +++ b/pkg/sentry/platform/kvm/address_space.go @@ -98,8 +98,9 @@ func (as *addressSpace) Touch(c *vCPU) bool { } type hostMapEntry struct { - addr uintptr - length uintptr + addr uintptr + length uintptr + memType hostarch.MemoryType } // mapLocked maps the given host entry. @@ -130,6 +131,7 @@ func (as *addressSpace) mapLocked(addr hostarch.Addr, m hostMapEntry, at hostarc inv = as.pageTables.Map(addr, length, pagetables.MapOpts{ AccessType: at, User: true, + MemoryType: m.memType, }, physical) || inv m.addr += length m.length -= length @@ -161,6 +163,7 @@ func (as *addressSpace) MapFile(addr hostarch.Addr, f memmap.File, fr memmap.Fil if err != nil { return err } + mt := f.MemoryType() // See block in mapLocked. as.pageTables.Allocator.(*allocator).cpu = as.machine.Get() @@ -186,8 +189,9 @@ func (as *addressSpace) MapFile(addr hostarch.Addr, f memmap.File, fr memmap.Fil // Perform the mapping. prev := as.mapLocked(addr, hostMapEntry{ - addr: b.Addr(), - length: uintptr(b.Len()), + addr: b.Addr(), + length: uintptr(b.Len()), + memType: mt, }, at) inv = inv || prev addr += hostarch.Addr(b.Len()) diff --git a/pkg/sentry/platform/kvm/kvm_const_arm64.go b/pkg/sentry/platform/kvm/kvm_const_arm64.go index fa51e9180..de40b68de 100644 --- a/pkg/sentry/platform/kvm/kvm_const_arm64.go +++ b/pkg/sentry/platform/kvm/kvm_const_arm64.go @@ -119,12 +119,6 @@ const ( // Arm64: Memory Attribute Indirection Register EL1. const ( - _MT_DEVICE_nGnRnE = 0 - _MT_DEVICE_nGnRE = 1 - _MT_DEVICE_GRE = 2 - _MT_NORMAL_NC = 3 - _MT_NORMAL = 4 - _MT_NORMAL_WT = 5 _MT_ATTR_DEVICE_nGnRnE = 0x00 _MT_ATTR_DEVICE_nGnRE = 0x04 _MT_ATTR_DEVICE_GRE = 0x0c @@ -132,7 +126,6 @@ const ( _MT_ATTR_NORMAL_WT = 0xbb _MT_ATTR_NORMAL = 0xff _MT_ATTR_MASK = 0xff - _MT_EL1_INIT = (_MT_ATTR_DEVICE_nGnRnE << (_MT_DEVICE_nGnRnE * 8)) | (_MT_ATTR_DEVICE_nGnRE << (_MT_DEVICE_nGnRE * 8)) | (_MT_ATTR_DEVICE_GRE << (_MT_DEVICE_GRE * 8)) | (_MT_ATTR_NORMAL_NC << (_MT_NORMAL_NC * 8)) | (_MT_ATTR_NORMAL << (_MT_NORMAL * 8)) | (_MT_ATTR_NORMAL_WT << (_MT_NORMAL_WT * 8)) ) const ( diff --git a/pkg/sentry/platform/kvm/machine_amd64.go b/pkg/sentry/platform/kvm/machine_amd64.go index 5aecfc670..342013d90 100644 --- a/pkg/sentry/platform/kvm/machine_amd64.go +++ b/pkg/sentry/platform/kvm/machine_amd64.go @@ -128,6 +128,11 @@ func (c *vCPU) initArchState() error { return err } + // Set up the PAT as required by ring0/pagetables. + if err := c.setPAT(); err != nil { + return err + } + // Set the entrypoint for the kernel. kernelUserRegs.RIP = uint64(ring0.AddrOfStart()) kernelUserRegs.RAX = uint64(reflect.ValueOf(&c.CPU).Pointer()) diff --git a/pkg/sentry/platform/kvm/machine_amd64_unsafe.go b/pkg/sentry/platform/kvm/machine_amd64_unsafe.go index e37bd6d3f..32a79939a 100644 --- a/pkg/sentry/platform/kvm/machine_amd64_unsafe.go +++ b/pkg/sentry/platform/kvm/machine_amd64_unsafe.go @@ -23,6 +23,7 @@ import ( "golang.org/x/sys/unix" "gvisor.dev/gvisor/pkg/abi/linux" + "gvisor.dev/gvisor/pkg/hostarch" "gvisor.dev/gvisor/pkg/hostsyscall" ) @@ -72,6 +73,36 @@ func (c *vCPU) setCPUID() error { return nil } +func (c *vCPU) setPAT() error { + // See Intel SDM Vol. 3, Sec. 13.12.2 "IA32_PAT MSR", or AMD64 APM Vol. 2, + // Sec. 7.8.1 "PAT Register". + const ( + _MSR_IA32_PAT = 0x277 + + _PAT_UC = 0x00 + _PAT_WC = 0x01 + _PAT_WB = 0x06 + ) + registers := modelControlRegisters{ + nmsrs: 1, + } + registers.entries[0].index = _MSR_IA32_PAT + if hostarch.NumMemoryTypes != 3 { + panic("additional memory types must be configured in PAT") + } + registers.entries[0].data = (_PAT_WB << (hostarch.MemoryTypeWriteBack * 8)) | + (_PAT_WC << (hostarch.MemoryTypeWriteCombine * 8)) | + (_PAT_UC << (hostarch.MemoryTypeUncached * 8)) + if errno := hostsyscall.RawSyscallErrno( + unix.SYS_IOCTL, + uintptr(c.fd), + KVM_SET_MSRS, + uintptr(unsafe.Pointer(®isters))); errno != 0 { + return fmt.Errorf("error setting PAT: %v", errno) + } + return nil +} + // getTSCFreq gets the TSC frequency. // // If mustSucceed is true, then this function panics on error. diff --git a/pkg/sentry/platform/kvm/machine_arm64_unsafe.go b/pkg/sentry/platform/kvm/machine_arm64_unsafe.go index 4420e6fb8..946129da8 100644 --- a/pkg/sentry/platform/kvm/machine_arm64_unsafe.go +++ b/pkg/sentry/platform/kvm/machine_arm64_unsafe.go @@ -91,7 +91,12 @@ func (c *vCPU) initArchState() error { } // mair_el1 - data = _MT_EL1_INIT + if hostarch.NumMemoryTypes != 3 { + panic("additional memory types must be configured in MAIR") + } + data = (_MT_ATTR_NORMAL << (hostarch.MemoryTypeWriteBack * 8)) | + (_MT_ATTR_NORMAL_NC << (hostarch.MemoryTypeWriteCombine * 8)) | + (_MT_ATTR_DEVICE_nGnRnE << (hostarch.MemoryTypeUncached * 8)) reg.id = _KVM_ARM64_REGS_MAIR_EL1 if err := c.setOneRegister(®); err != nil { return err