NVIDIA: VR: SAUCE: arm64: Add workaround to convert MT_NORMAL_NC to Device-nGnRE

Add CONFIG_ARM64_WORKAROUND_NC_TO_NGNRE configuration option that
enables conversion of MT_NORMAL_NC (Normal Non-Cacheable) memory
attribute to Device-nGnRE memory type in MAIR_EL1 for hardware that
requires stricter memory ordering or has issues with Non-Cacheable
memory mappings.

Key changes:

1. New memory type MT_NORMAL_NC_DMA (Attr5):
   - Introduced specifically for DMA coherent memory mappings
   - Configured with the same Normal Non-Cacheable attribute (0x44)
     as MT_NORMAL_NC (Attr2) by default
   - pgprot_dmacoherent uses MT_NORMAL_NC_DMA when workaround is
     enabled, MT_NORMAL_NC otherwise

2. MAIR_EL1 conversion via alternatives framework:
   - arch/arm64/mm/proc.S uses ARM64 alternatives to patch MAIR_EL1
     during early boot
   - Converts MT_NORMAL_NC (Attr2) from 0x44 to 0x04 (Device-nGnRE)
     using efficient bfi instruction
   - MT_NORMAL_NC_DMA (Attr5) keeps the same attribute value as
     MT_NORMAL_NC originally had
   - Zero performance overhead when workaround is disabled

3. Boot-time configuration:
   - Enabled via kernel command line: mair_el1_nc_to_ngnre=1
   - Boot CPU fixup in enable_nc_to_ngnre() applies conversion before
     alternatives are patched
   - Secondary CPUs automatically use patched alternatives in
     __cpu_setup
   - Runtime changes not supported as alternatives cannot be
     re-patched after boot

4. Errata framework integration:
   - Registered in arm64_errata[] array as ARM64_WORKAROUND_NC_TO_NGNRE
   - Capability type: ARM64_CPUCAP_BOOT_CPU_FEATURE
   - Uses cpucap_is_possible() for build-time capability checking

The workaround preserves pgprot_dmacoherent behavior while allowing
MT_NORMAL_NC to be converted to Device memory type for other mappings
that may be affected by hardware issues.

Signed-off-by: Shanker Donthineni <sdonthineni@nvidia.com>
Signed-off-by: Matthew R. Ochs <mochs@nvidia.com>
Acked-by: Nirmoy Das <nirmoyd@nvidia.com>
Acked-by: Carol L. Soto <csoto@nvidia.com>
Signed-off-by: Matthew R. Ochs <mochs@nvidia.com>

Author: shankerd04

arch/arm64/Kconfig

@@ -1695,6 +1695,36 @@ config ARM64_TAGGED_ADDR_ABI
 	  to system calls as pointer arguments. For details, see
 	  Documentation/arch/arm64/tagged-address-abi.rst.
 
+config ARM64_WORKAROUND_NC_TO_NGNRE
+	bool "Workaround: Convert MT_NORMAL_NC to Device-nGnRE"
+	default y
+	help
+	  This option enables a workaround that converts the MT_NORMAL_NC
+	  (Non-Cacheable) memory attribute to Device-nGnRE memory type in
+	  MAIR_EL1 (Memory Attribute Indirection Register).
+
+	  This workaround is useful for hardware that requires stricter
+	  memory ordering or has issues with Non-Cacheable memory mappings.
+
+	  A new memory type index MT_NORMAL_NC_DMA (Attr5) has been introduced
+	  specifically for DMA coherent memory mappings (pgprot_dmacoherent),
+	  configured with the same Normal Non-Cacheable attribute (0x44) as
+	  MT_NORMAL_NC (Attr2). When this workaround is enabled, it converts
+	  the NC attribute to Device-nGnRE (0x04), and pgprot_dmacoherent
+	  behavior remains the same as before.
+
+	  The workaround uses the ARM64 alternatives framework for efficient
+	  runtime patching with no performance overhead when disabled.
+
+	  This workaround can only be enabled at boot time via kernel command
+	  line parameter. Runtime changes are not supported because CPU
+	  alternatives cannot be re-patched after boot.
+
+	  Boot-time activation (kernel command line):
+	    mair_el1_nc_to_ngnre=1
+
+	  If unsure, say Y.
+
 menuconfig COMPAT
 	bool "Kernel support for 32-bit EL0"
 	depends on ARM64_4K_PAGES || EXPERT

arch/arm64/include/asm/cpucaps.h

@@ -73,6 +73,8 @@ cpucap_is_possible(const unsigned int cap)
 		return true;
 	case ARM64_HAS_PMUV3:
 		return IS_ENABLED(CONFIG_HW_PERF_EVENTS);
+	case ARM64_WORKAROUND_NC_TO_NGNRE:
+		return IS_ENABLED(CONFIG_ARM64_WORKAROUND_NC_TO_NGNRE);
 	}
 
 	return true;

arch/arm64/include/asm/memory.h

@@ -173,6 +173,7 @@
 #define MT_NORMAL_NC		2
 #define MT_DEVICE_nGnRnE	3
 #define MT_DEVICE_nGnRE		4
+#define MT_NORMAL_NC_DMA	5
 
 /*
  * Memory types for Stage-2 translation

arch/arm64/include/asm/pgtable.h

@@ -800,9 +800,15 @@ static inline void __set_puds(struct mm_struct *mm,
  * requires strict alignment and can also force write responses to come from the
  * endpoint.
  */
+#ifdef CONFIG_ARM64_WORKAROUND_NC_TO_NGNRE
+#define pgprot_dmacoherent(prot) \
+	__pgprot_modify(prot, PTE_ATTRINDX_MASK, \
+			PTE_ATTRINDX(MT_NORMAL_NC_DMA) | PTE_PXN | PTE_UXN)
+#else
 #define pgprot_dmacoherent(prot) \
 	__pgprot_modify(prot, PTE_ATTRINDX_MASK, \
 			PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN)
+#endif
 
 #define __HAVE_PHYS_MEM_ACCESS_PROT
 struct file;

arch/arm64/kernel/cpu_errata.c

@@ -8,11 +8,16 @@
 #include <linux/arm-smccc.h>
 #include <linux/types.h>
 #include <linux/cpu.h>
+#include <linux/arm-smccc.h>
 #include <asm/cpu.h>
 #include <asm/cputype.h>
 #include <asm/cpufeature.h>
 #include <asm/kvm_asm.h>
 #include <asm/smp_plat.h>
+#include <asm/memory.h>
+#include <asm/barrier.h>
+#include <asm/tlbflush.h>
+#include <asm/sysreg.h>
 
 static u64 target_impl_cpu_num;
 static struct target_impl_cpu *target_impl_cpus;
@@ -566,6 +571,97 @@ static const struct midr_range erratum_ac04_cpu_23_list[] = {
 };
 #endif
 
+#ifdef CONFIG_ARM64_WORKAROUND_NC_TO_NGNRE
+/*
+ * MAIR_EL1 MT_NORMAL_NC to Device-nGnRE Conversion
+ *
+ * Boot-time workaround that converts MT_NORMAL_NC attribute to Device-nGnRE
+ * (0x04) via kernel parameter for hardware-specific issues.
+ *
+ */
+
+/*
+ * Flag indicating if MT_NORMAL_NC to nGnRE conversion is enabled
+ */
+static int mair_el1_nc_to_ngnre __read_mostly = -1;
+
+/*
+ * Parse kernel command line parameter at boot:
+ *   mair_el1_nc_to_ngnre=1
+ * Enables MT_NORMAL_NC to Device-nGnRE conversion
+ */
+static int __init mair_el1_nc_setup(char *str)
+{
+	bool enable;
+	int ret;
+
+	ret = kstrtobool(str, &enable);
+	if (ret)
+		return ret;
+
+	mair_el1_nc_to_ngnre = enable ? 1 : 0;
+
+	pr_info("MAIR_EL1: MT_NORMAL_NC to Device-nGnRE conversion %s\n",
+		enable ? "enabled" : "disabled");
+
+	return 0;
+}
+early_param("mair_el1_nc_to_ngnre", mair_el1_nc_setup);
+
+/* Cpufeature capability check for MAIR NC to nGnRE workaround */
+static bool has_nc_ngnre_workaround(const struct arm64_cpu_capabilities *entry,
+				    int scope)
+{
+	/* Lazy initialization: check only once */
+	if (mair_el1_nc_to_ngnre == -1) {
+		if ((arm_smccc_get_soc_id_version() == 0x036b0410) &&
+		    (arm_smccc_get_soc_id_revision() < 5)) {
+			mair_el1_nc_to_ngnre = 1;
+			return true;
+		}
+		mair_el1_nc_to_ngnre = 0;
+	}
+
+	return mair_el1_nc_to_ngnre > 0;
+}
+
+/*
+ * Called by cpufeature framework when CPU comes online
+ * For boot CPU: alternatives not yet patched, so apply NC to nGnRE here
+ * For secondary CPUs: alternatives already patched in __cpu_setup
+ */
+static void enable_nc_to_ngnre(struct arm64_cpu_capabilities const *cap)
+{
+	u64 attr_mask, current_mair, new_mair;
+	u8 current_attr;
+
+	if (mair_el1_nc_to_ngnre <= 0)
+		return;
+
+	current_mair = read_sysreg(mair_el1);
+
+	/* Check if MT_NORMAL_NC is already Device-nGnRE */
+	attr_mask = GENMASK_ULL((MT_NORMAL_NC * 8) + 7, MT_NORMAL_NC * 8);
+	current_attr = FIELD_GET(attr_mask, current_mair);
+
+	/* Already set via alternatives (secondary CPU case) */
+	if (current_attr == MAIR_ATTR_DEVICE_nGnRE)
+		return;
+
+	/* Apply override for boot CPU */
+	new_mair = (current_mair & ~attr_mask) |
+		   ((u64)MAIR_ATTR_DEVICE_nGnRE << (MT_NORMAL_NC * 8));
+
+	write_sysreg(new_mair, mair_el1);
+	isb();
+	local_flush_tlb_all();
+
+	pr_info("CPU%d: MAIR_EL1 updated 0x%016llx -> 0x%016llx\n",
+		smp_processor_id(), current_mair, new_mair);
+}
+
+#endif /* CONFIG_ARM64_WORKAROUND_NC_TO_NGNRE */
+
 const struct arm64_cpu_capabilities arm64_errata[] = {
 #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE
 	{
@@ -907,6 +1003,15 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
 		.matches = has_impdef_pmuv3,
 		.cpu_enable = cpu_enable_impdef_pmuv3_traps,
 	},
+#ifdef CONFIG_ARM64_WORKAROUND_NC_TO_NGNRE
+	{
+		.desc = "MAIR_EL1 NC to nGnRE",
+		.capability = ARM64_WORKAROUND_NC_TO_NGNRE,
+		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
+		.matches = has_nc_ngnre_workaround,
+		.cpu_enable = enable_nc_to_ngnre,
+	},
+#endif
 	{
 	}
 };

arch/arm64/mm/proc.S

@@ -66,7 +66,8 @@
 	 MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRE, MT_DEVICE_nGnRE) |	\
 	 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_NC, MT_NORMAL_NC) |		\
 	 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL) |			\
-	 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL_TAGGED))
+	 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL_TAGGED) |		\
+	 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_NC, MT_NORMAL_NC_DMA))
 
 #ifdef CONFIG_CPU_PM
 /**
@@ -480,6 +481,14 @@ SYM_FUNC_START(__cpu_setup)
 	tcr	.req	x16
 	tcr2	.req	x15
 	mov_q	mair, MAIR_EL1_SET
+
+#ifdef CONFIG_ARM64_WORKAROUND_NC_TO_NGNRE
+alternative_if ARM64_WORKAROUND_NC_TO_NGNRE
+	mov	x9, #MAIR_ATTR_DEVICE_nGnRE
+	bfi	mair, x9, #(MT_NORMAL_NC * 8), #8
+alternative_else_nop_endif
+#endif
+
 	mov_q	tcr, TCR_T0SZ(IDMAP_VA_BITS) | TCR_T1SZ(VA_BITS_MIN) | TCR_CACHE_FLAGS | \
 		     TCR_SHARED | TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_ASID16 | \
 		     TCR_TBI0 | TCR_A1 | TCR_KASAN_SW_FLAGS | TCR_MTE_FLAGS

arch/arm64/tools/cpucaps

@@ -113,6 +113,7 @@ WORKAROUND_CAVIUM_TX2_219_PRFM
 WORKAROUND_CAVIUM_TX2_219_TVM
 WORKAROUND_CLEAN_CACHE
 WORKAROUND_DEVICE_LOAD_ACQUIRE
+WORKAROUND_NC_TO_NGNRE
 WORKAROUND_NVIDIA_CARMEL_CNP
 WORKAROUND_PMUV3_IMPDEF_TRAPS
 WORKAROUND_QCOM_FALKOR_E1003