kvm/arm64: enable guest mmu from the host

Right now, the start function is executed with the disabled MMU and it means
every access the PA is equal to the MVA [1]. This is known as a flat address
mapping.

By coincidence, the code segment is mapped in lower addresses and we have the
flat address mapping for it. But it isn't the case if we set buildmode=pie to
enable ASLR.

[1] https://developer.arm.com/documentation/ddi0406/b/System-Level-Architecture/Virtual-Memory-System-Architecture--VMSA-/Memory-access-sequence/Enabling-and-disabling-the-MMU

PiperOrigin-RevId: 523745143

pkg/ring0/entry_arm64.s

@@ -126,22 +126,6 @@
 
 #define FPEN_ENABLE (FPEN_NOTRAP << FPEN_SHIFT)
 
-// sctlr_el1: system control register el1.
-#define SCTLR_M         1 << 0
-#define SCTLR_C         1 << 2
-#define SCTLR_I         1 << 12
-#define SCTLR_DZE       1 << 14
-#define SCTLR_UCT       1 << 15
-#define SCTLR_UCI       1 << 26
-
-#define SCTLR_EL1_DEFAULT       (SCTLR_M | SCTLR_C | SCTLR_I | SCTLR_UCT | SCTLR_UCI | SCTLR_DZE)
-
-// cntkctl_el1: counter-timer kernel control register el1.
-#define CNTKCTL_EL0PCTEN 	1 << 0
-#define CNTKCTL_EL0VCTEN 	1 << 1
-
-#define CNTKCTL_EL1_DEFAULT 	(CNTKCTL_EL0PCTEN | CNTKCTL_EL0VCTEN)
-
 // Saves a register set.
 //
 // This is a macro because it may need to executed in contents where a stack is
@@ -604,39 +588,9 @@ TEXT ·kernelExitToEl1(SB),NOSPLIT,$0
 
 	ERET()
 
-// start is the CPU entrypoint.
 TEXT ·start(SB),NOSPLIT,$0
-	// Init.
-	WORD $0xd508871f    // __tlbi(vmalle1)
-	DSB $7          // dsb(nsh)
-
-	MOVD $1<<12, R1         // Reset mdscr_el1 and disable
-	MSR R1, MDSCR_EL1       // access to the DCC from EL0
-	ISB $15
-
-	MRS TTBR1_EL1, R1
-	MSR R1, TTBR0_EL1
-	ISB $15
-
-	MOVD $CNTKCTL_EL1_DEFAULT, R1
-	MSR R1, CNTKCTL_EL1
-
-	MOVD R8, RSV_REG
-	ORR $0xffff000000000000, RSV_REG, RSV_REG
-	WORD $0xd518d092        //MSR R18, TPIDR_EL1
-
-	// Enable trap for accessing fpsimd.
-	MSR $0, CPACR_EL1
-
-	// Init.
-	MOVD $SCTLR_EL1_DEFAULT, R1 // re-enable the mmu.
-	MSR R1, SCTLR_EL1
-	ISB $15
-	WORD $0xd508751f // ic iallu
-
 	DSB $7          // dsb(nsh)
 	ISB $15
-
 	B ·kernelExitToEl1(SB)
 
 // func AddrOfStart() uintptr

pkg/sentry/platform/kvm/kvm_const_arm64.go

@@ -40,6 +40,10 @@ const (
 	_KVM_ARM64_REGS_VBAR_EL1   = 0x603000000013c600
 	_KVM_ARM64_REGS_TIMER_CNT  = 0x603000000013df1a
 	_KVM_ARM64_REGS_CNTFRQ_EL0 = 0x603000000013df00
+
+	_KVM_ARM64_REGS_MDSCR_EL1   = 0x6030000000138012
+	_KVM_ARM64_REGS_CNTKCTL_EL1 = 0x603000000013c708
+	_KVM_ARM64_REGS_TPIDR_EL1   = 0x603000000013c684
 )
 
 // Arm64: Architectural Feature Access Control Register EL1.
@@ -50,9 +54,21 @@ const (
 
 // Arm64: System Control Register EL1.
 const (
-	_SCTLR_M = 1 << 0
-	_SCTLR_C = 1 << 2
-	_SCTLR_I = 1 << 12
+	_SCTLR_M           = 1 << 0
+	_SCTLR_C           = 1 << 2
+	_SCTLR_I           = 1 << 12
+	_SCTLR_DZE         = 1 << 14
+	_SCTLR_UCT         = 1 << 15
+	_SCTLR_UCI         = 1 << 26
+	_SCTLR_EL1_DEFAULT = _SCTLR_M | _SCTLR_C | _SCTLR_I | _SCTLR_UCT | _SCTLR_UCI | _SCTLR_DZE
+)
+
+// Arm64: Counter-timer Kernel Control Register el1.
+const (
+	_CNTKCTL_EL0PCTEN = 1 << 0
+	_CNTKCTL_EL0VCTEN = 1 << 1
+
+	_CNTKCTL_EL1_DEFAULT = _CNTKCTL_EL0PCTEN | _CNTKCTL_EL0VCTEN
 )
 
 // Arm64: Translation Control Register EL1.

pkg/sentry/platform/kvm/machine_arm64_unsafe.go

@@ -113,6 +113,34 @@ func (c *vCPU) initArchState() error {
 		return err
 	}
 
+	// cntkctl_el1
+	data = _CNTKCTL_EL1_DEFAULT
+	reg.id = _KVM_ARM64_REGS_CNTKCTL_EL1
+	if err := c.setOneRegister(&reg); err != nil {
+		return err
+	}
+
+	// cpacr_el1
+	data = 0
+	reg.id = _KVM_ARM64_REGS_CPACR_EL1
+	if err := c.setOneRegister(&reg); err != nil {
+		return err
+	}
+
+	// sctlr_el1
+	data = _SCTLR_EL1_DEFAULT
+	reg.id = _KVM_ARM64_REGS_SCTLR_EL1
+	if err := c.setOneRegister(&reg); err != nil {
+		return err
+	}
+
+	// tpidr_el1
+	reg.id = _KVM_ARM64_REGS_TPIDR_EL1
+	data = uint64(reflect.ValueOf(&c.CPU).Pointer() | ring0.KernelStartAddress)
+	if err := c.setOneRegister(&reg); err != nil {
+		return err
+	}
+
 	// sp_el1
 	data = c.CPU.StackTop()
 	reg.id = _KVM_ARM64_REGS_SP_EL1
@@ -127,13 +155,6 @@ func (c *vCPU) initArchState() error {
 		return err
 	}
 
-	// r8
-	reg.id = _KVM_ARM64_REGS_R8
-	data = uint64(reflect.ValueOf(&c.CPU).Pointer())
-	if err := c.setOneRegister(&reg); err != nil {
-		return err
-	}
-
 	// vbar_el1
 	reg.id = _KVM_ARM64_REGS_VBAR_EL1
 	vectorLocation := ring0.AddrOfVectors()
@@ -144,7 +165,8 @@ func (c *vCPU) initArchState() error {
 
 	// Use the address of the exception vector table as
 	// the MMIO address base.
-	arm64HypercallMMIOBase = vectorLocation
+	vectorLocationPhys, _, _ := translateToPhysical(vectorLocation)
+	arm64HypercallMMIOBase = vectorLocationPhys
 
 	// Initialize the PCID database.
 	if hasGuestPCID {