FROMGIT: arm64: Work around Cortex-A510 erratum 2454944

Cortex-A510 erratum 2454944 may cause clean cache lines to be
erroneously written back to memory, breaking the assumptions we rely on
for non-coherent DMA. Try to mitigate this by implementing special DMA
ops that do their best to avoid cacheable aliases via a combination of
bounce-buffering and manipulating the linear map directly, to minimise
the chance of DMA-mapped pages being speculated back into caches.

The other main concern is initial entry, where cache lines covering the
kernel image might potentially become affected between being cleaned by
the bootloader and the kernel being called, so perform some additional
maintenance to be safe in that regard too. Cortex-A510 supports S2FWB,
so KVM should be unaffected.

Bug: 223346425
(cherry picked from commit 5bb88dd8ed70973eeb15722710a46d60951c8255
 https://git.gitlab.arm.com/linux-arm/linux-rm.git arm64/2454944)
Change-Id: Iffd38bf97114f7151f01c70750b465fc991c89c8
Signed-off-by: Robin Murphy <robin.murphy@arm.com>
Signed-off-by: Beata Michalska <beata.michalska@arm.com>

Documentation/arm64/silicon-errata.rst

@@ -94,6 +94,8 @@ stable kernels.
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Cortex-A510     | #2051678        | ARM64_ERRATUM_2051678       |
 +----------------+-----------------+-----------------+-----------------------------+
+| ARM            | Cortex-A510     | #2454944        | ARM64_ERRATUM_2454944       |
++----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Cortex-A710     | #2054223        | ARM64_ERRATUM_2054223       |
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Neoverse-N1     | #1188873,1418040| ARM64_ERRATUM_1418040       |

arch/arm64/Kconfig

@@ -713,6 +713,36 @@ config ARM64_ERRATUM_2067961
 
 	  If unsure, say Y.
 
+config ARM64_ERRATUM_2454944
+	bool "Cortex-A510: 2454944: Unmodified cache line might be written back to memory"
+	select ARCH_HAS_TEARDOWN_DMA_OPS
+	default y
+	help
+	  This option adds the workaround for ARM Cortex-A510 erratum 2454944.
+
+	  Affected Cortex-A510 core might write unmodified cache lines back to
+	  memory, which breaks the assumptions upon which software coherency
+	  management for non-coherent DMA relies. If a cache line is
+	  speculatively fetched while a non-coherent device is writing directly
+	  to DRAM, and subsequently written back by natural eviction, data
+	  written by the device in the intervening period can be lost.
+
+	  The workaround is to enforce as far as reasonably possible that all
+	  non-coherent DMA transfers are bounced and/or remapped to minimise
+	  the chance that any Cacheable alias exists through which speculative
+	  cache fills could occur.
+
+	  This is quite involved and has unavoidable performance impact on
+	  affected systems.
+
+config ARM64_ERRATUM_2454944_DEBUG
+	bool "Extra debug checks for Cortex-A510 2454944"
+	depends on ARM64_ERRATUM_2454944
+	default y
+	help
+	  Enable additional checks and warnings to detect and mitigate driver
+	  bugs breaking the remapping workaround.
+
 config CAVIUM_ERRATUM_22375
 	bool "Cavium erratum 22375, 24313"
 	default y

arch/arm64/include/asm/cpucaps.h

@@ -72,7 +72,8 @@
 #define ARM64_WORKAROUND_TSB_FLUSH_FAILURE	61
 #define ARM64_SPECTRE_BHB			62
 
-/* kabi: reserve 63 - 76 for future cpu capabilities */
+/* kabi: reserve 63 - 74 for future cpu capabilities */
+#define ARM64_WORKAROUND_NO_DMA_ALIAS		75
 #define ARM64_NCAPS				76
 
 #endif /* __ASM_CPUCAPS_H */

arch/arm64/include/asm/page.h

@@ -43,6 +43,19 @@ typedef struct page *pgtable_t;
 
 extern int pfn_valid(unsigned long);
 
+#ifdef CONFIG_ARM64_ERRATUM_2454944_DEBUG
+#include <asm/cpufeature.h>
+
+void page_check_nc(struct page *page, int order);
+
+static inline void arch_free_page(struct page *page, int order)
+{
+	if (cpus_have_const_cap(ARM64_WORKAROUND_NO_DMA_ALIAS))
+		page_check_nc(page, order);
+}
+#define HAVE_ARCH_FREE_PAGE
+#endif
+
 #include <asm/memory.h>
 
 #endif /* !__ASSEMBLY__ */

arch/arm64/include/asm/vmalloc.h

@@ -1,4 +1,8 @@
 #ifndef _ASM_ARM64_VMALLOC_H
 #define _ASM_ARM64_VMALLOC_H
 
+#include <asm/cpufeature.h>
+
+#define arch_disable_lazy_vunmap cpus_have_const_cap(ARM64_WORKAROUND_NO_DMA_ALIAS)
+
 #endif /* _ASM_ARM64_VMALLOC_H */

arch/arm64/kernel/cpu_errata.c

@@ -555,6 +555,14 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
 		.capability = ARM64_WORKAROUND_TSB_FLUSH_FAILURE,
 		ERRATA_MIDR_RANGE_LIST(tsb_flush_fail_cpus),
 	},
+#endif
+#ifdef CONFIG_ARM64_ERRATUM_2454944
+	{
+		.desc = "ARM erratum 2454944",
+		.capability = ARM64_WORKAROUND_NO_DMA_ALIAS,
+		ERRATA_MIDR_RANGE(MIDR_CORTEX_A510, 0, 0, 1, 1),
+		MIDR_FIXED(MIDR_CPU_VAR_REV(1, 1), BIT(25)),
+	},
 #endif
 	{
 	}

arch/arm64/kernel/head.S

@@ -125,9 +125,13 @@ SYM_CODE_END(primary_entry)
 SYM_CODE_START_LOCAL(preserve_boot_args)
 	mov	x21, x0				// x21=FDT
 
-	adr_l	x0, boot_args			// record the contents of
-	stp	x21, x1, [x0]			// x0 .. x3 at kernel entry
-	stp	x2, x3, [x0, #16]
+	adr_l	x0, boot_args
+#ifdef CONFIG_ARM64_ERRATUM_2454944
+	dc	ivac, x0			// Cortex-A510 CWG is 64 bytes, so plenty
+	dsb	sy
+#endif
+	stp	x21, x1, [x0]			// record the contents of
+	stp	x2, x3, [x0, #16]		// x0 .. x3 at kernel entry
 
 	dmb	sy				// needed before dc ivac with
 						// MMU off
@@ -282,8 +286,17 @@ SYM_FUNC_START_LOCAL(__create_page_tables)
 	 * the kernel image, and thus are clean to the PoC per the boot
 	 * protocol.
 	 */
+#ifndef CONFIG_ARM64_ERRATUM_2454944
 	adrp	x0, init_pg_dir
 	adrp	x1, init_pg_end
+#else
+	/*
+	 * However if we can't even trust "clean" cache lines shadowing rodata,
+	 * then nuke the entire image. It's the only way to be sure.
+	 */
+	adrp	x0, _text
+	adrp	x1, _end
+#endif
 	sub	x1, x1, x0
 	bl	__inval_dcache_area
 
@@ -531,6 +544,10 @@ SYM_FUNC_END(init_kernel_el)
  */
 SYM_FUNC_START_LOCAL(set_cpu_boot_mode_flag)
 	adr_l	x1, __boot_cpu_mode
+#ifdef CONFIG_ARM64_ERRATUM_2454944
+	dc	ivac, x1
+	dsb	sy
+#endif
 	cmp	w0, #BOOT_CPU_MODE_EL2
 	b.ne	1f
 	add	x1, x1, #4

arch/arm64/mm/init.c

@@ -581,7 +581,8 @@ static void __init free_unused_memmap(void)
 void __init mem_init(void)
 {
 	if (swiotlb_force == SWIOTLB_FORCE ||
-	    max_pfn > PFN_DOWN(arm64_dma_phys_limit))
+	    max_pfn > PFN_DOWN(arm64_dma_phys_limit) ||
+	    cpus_have_cap(ARM64_WORKAROUND_NO_DMA_ALIAS))
 		swiotlb_init(1);
 	else
 		swiotlb_force = SWIOTLB_NO_FORCE;

arch/arm64/mm/mmu.c

@@ -502,7 +502,7 @@ static void __init map_mem(pgd_t *pgdp)
 	u64 i;
 
 	if (rodata_full || debug_pagealloc_enabled() ||
-	    IS_ENABLED(CONFIG_KFENCE))
+	    IS_ENABLED(CONFIG_KFENCE) || cpus_have_cap(ARM64_WORKAROUND_NO_DMA_ALIAS))
 		flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
 
 	/*