NVIDIA: VR: SAUCE: arm64: Add workaround to convert MT_NORMAL_NC to Device-nGnRE

Add CONFIG_ARM64_WORKAROUND_NC_TO_NGNRE configuration option that
enables conversion of MT_NORMAL_NC (Normal Non-Cacheable) memory
attribute to Device-nGnRE memory type in MAIR_EL1 for hardware that
requires stricter memory ordering or has issues with Non-Cacheable
memory mappings.

Key changes:

1. New memory type MT_NORMAL_NC_DMA (Attr5):
   - Introduced specifically for DMA coherent memory mappings
   - Configured with the same Normal Non-Cacheable attribute (0x44)
     as MT_NORMAL_NC (Attr2) by default
   - pgprot_dmacoherent uses MT_NORMAL_NC_DMA when workaround is
     enabled, MT_NORMAL_NC otherwise

2. MAIR_EL1 conversion via alternatives framework:
   - arch/arm64/mm/proc.S uses ARM64 alternatives to patch MAIR_EL1
     during early boot
   - Converts MT_NORMAL_NC (Attr2) from 0x44 to 0x04 (Device-nGnRE)
     using efficient bfi instruction
   - MT_NORMAL_NC_DMA (Attr5) keeps the same attribute value as
     MT_NORMAL_NC originally had
   - Zero performance overhead when workaround is disabled

3. Boot-time configuration:
   - Enabled via kernel command line: mair_el1_nc_to_ngnre=1
   - Boot CPU fixup in enable_nc_to_ngnre() applies conversion before
     alternatives are patched
   - Secondary CPUs automatically use patched alternatives in
     __cpu_setup
   - Runtime changes not supported as alternatives cannot be
     re-patched after boot

4. Errata framework integration:
   - Registered in arm64_errata[] array as ARM64_WORKAROUND_NC_TO_NGNRE
   - Capability type: ARM64_CPUCAP_BOOT_CPU_FEATURE
   - Uses cpucap_is_possible() for build-time capability checking

The workaround preserves pgprot_dmacoherent behavior while allowing
MT_NORMAL_NC to be converted to Device memory type for other mappings
that may be affected by hardware issues. Ensure NC memory attribute
assignment is prevented for passthrough device MMIO regions.

Signed-off-by: Shanker Donthineni <sdonthineni@nvidia.com>
Signed-off-by: Matthew R. Ochs <mochs@nvidia.com>

Author: shankerd04

arch/arm64/Kconfig

@@ -1732,6 +1732,36 @@ config ARM64_TAGGED_ADDR_ABI
 	  to system calls as pointer arguments. For details, see
 	  Documentation/arch/arm64/tagged-address-abi.rst.
 
+config ARM64_WORKAROUND_NC_TO_NGNRE
+	bool "Workaround: Convert MT_NORMAL_NC to Device-nGnRE"
+	default y
+	help
+	  This option enables a workaround that converts the MT_NORMAL_NC
+	  (Non-Cacheable) memory attribute to Device-nGnRE memory type in
+	  MAIR_EL1 (Memory Attribute Indirection Register).
+
+	  This workaround is useful for hardware that requires stricter
+	  memory ordering or has issues with Non-Cacheable memory mappings.
+
+	  A new memory type index MT_NORMAL_NC_DMA (Attr5) has been introduced
+	  specifically for DMA coherent memory mappings (pgprot_dmacoherent),
+	  configured with the same Normal Non-Cacheable attribute (0x44) as
+	  MT_NORMAL_NC (Attr2). When this workaround is enabled, it converts
+	  the NC attribute to Device-nGnRE (0x04), and pgprot_dmacoherent
+	  behavior remains the same as before.
+
+	  The workaround uses the ARM64 alternatives framework for efficient
+	  runtime patching with no performance overhead when disabled.
+
+	  This workaround can only be enabled at boot time via kernel command
+	  line parameter. Runtime changes are not supported because CPU
+	  alternatives cannot be re-patched after boot.
+
+	  Boot-time activation (kernel command line):
+	    mair_el1_nc_to_ngnre=1
+
+	  If unsure, say Y.
+
 menuconfig COMPAT
 	bool "Kernel support for 32-bit EL0"
 	depends on ARM64_4K_PAGES || EXPERT

arch/arm64/include/asm/cpucaps.h

@@ -73,6 +73,8 @@ cpucap_is_possible(const unsigned int cap)
 		return true;
 	case ARM64_HAS_PMUV3:
 		return IS_ENABLED(CONFIG_HW_PERF_EVENTS);
+	case ARM64_WORKAROUND_NC_TO_NGNRE:
+		return IS_ENABLED(CONFIG_ARM64_WORKAROUND_NC_TO_NGNRE);
 	}
 
 	return true;

arch/arm64/include/asm/memory.h

@@ -173,6 +173,7 @@
 #define MT_NORMAL_NC		2
 #define MT_DEVICE_nGnRnE	3
 #define MT_DEVICE_nGnRE		4
+#define MT_NORMAL_NC_DMA	5
 
 /*
  * Memory types for Stage-2 translation

arch/arm64/include/asm/pgtable.h

@@ -795,9 +795,15 @@ static inline void __set_puds(struct mm_struct *mm,
  * requires strict alignment and can also force write responses to come from the
  * endpoint.
  */
+#ifdef CONFIG_ARM64_WORKAROUND_NC_TO_NGNRE
+#define pgprot_dmacoherent(prot) \
+	__pgprot_modify(prot, PTE_ATTRINDX_MASK, \
+			PTE_ATTRINDX(MT_NORMAL_NC_DMA) | PTE_PXN | PTE_UXN)
+#else
 #define pgprot_dmacoherent(prot) \
 	__pgprot_modify(prot, PTE_ATTRINDX_MASK, \
 			PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN)
+#endif
 
 #define __HAVE_PHYS_MEM_ACCESS_PROT
 struct file;

arch/arm64/kernel/cpu_errata.c

@@ -8,11 +8,16 @@
 #include <linux/arm-smccc.h>
 #include <linux/types.h>
 #include <linux/cpu.h>
+#include <linux/arm-smccc.h>
 #include <asm/cpu.h>
 #include <asm/cputype.h>
 #include <asm/cpufeature.h>
 #include <asm/kvm_asm.h>
 #include <asm/smp_plat.h>
+#include <asm/memory.h>
+#include <asm/barrier.h>
+#include <asm/tlbflush.h>
+#include <asm/sysreg.h>
 
 static u64 target_impl_cpu_num;
 static struct target_impl_cpu *target_impl_cpus;
@@ -565,6 +570,97 @@ static const struct midr_range erratum_ac04_cpu_23_list[] = {
 };
 #endif
 
+#ifdef CONFIG_ARM64_WORKAROUND_NC_TO_NGNRE
+/*
+ * MAIR_EL1 MT_NORMAL_NC to Device-nGnRE Conversion
+ *
+ * Boot-time workaround that converts MT_NORMAL_NC attribute to Device-nGnRE
+ * (0x04) via kernel parameter for hardware-specific issues.
+ *
+ */
+
+/*
+ * Flag indicating if MT_NORMAL_NC to nGnRE conversion is enabled
+ */
+static int mair_el1_nc_to_ngnre __read_mostly = -1;
+
+/*
+ * Parse kernel command line parameter at boot:
+ *   mair_el1_nc_to_ngnre=1
+ * Enables MT_NORMAL_NC to Device-nGnRE conversion
+ */
+static int __init mair_el1_nc_setup(char *str)
+{
+	bool enable;
+	int ret;
+
+	ret = kstrtobool(str, &enable);
+	if (ret)
+		return ret;
+
+	mair_el1_nc_to_ngnre = enable ? 1 : 0;
+
+	pr_info("MAIR_EL1: MT_NORMAL_NC to Device-nGnRE conversion %s\n",
+		enable ? "enabled" : "disabled");
+
+	return 0;
+}
+early_param("mair_el1_nc_to_ngnre", mair_el1_nc_setup);
+
+/* Cpufeature capability check for MAIR NC to nGnRE workaround */
+static bool has_nc_ngnre_workaround(const struct arm64_cpu_capabilities *entry,
+				    int scope)
+{
+	/* Lazy initialization: check only once */
+	if (mair_el1_nc_to_ngnre == -1) {
+		if ((arm_smccc_get_soc_id_version() == 0x036b0410) &&
+		    (arm_smccc_get_soc_id_revision() < 5)) {
+			mair_el1_nc_to_ngnre = 1;
+			return true;
+		}
+		mair_el1_nc_to_ngnre = 0;
+	}
+
+	return mair_el1_nc_to_ngnre > 0;
+}
+
+/*
+ * Called by cpufeature framework when CPU comes online
+ * For boot CPU: alternatives not yet patched, so apply NC to nGnRE here
+ * For secondary CPUs: alternatives already patched in __cpu_setup
+ */
+static void enable_nc_to_ngnre(struct arm64_cpu_capabilities const *cap)
+{
+	u64 attr_mask, current_mair, new_mair;
+	u8 current_attr;
+
+	if (mair_el1_nc_to_ngnre <= 0)
+		return;
+
+	current_mair = read_sysreg(mair_el1);
+
+	/* Check if MT_NORMAL_NC is already Device-nGnRE */
+	attr_mask = GENMASK_ULL((MT_NORMAL_NC * 8) + 7, MT_NORMAL_NC * 8);
+	current_attr = FIELD_GET(attr_mask, current_mair);
+
+	/* Already set via alternatives (secondary CPU case) */
+	if (current_attr == MAIR_ATTR_DEVICE_nGnRE)
+		return;
+
+	/* Apply override for boot CPU */
+	new_mair = (current_mair & ~attr_mask) |
+		   ((u64)MAIR_ATTR_DEVICE_nGnRE << (MT_NORMAL_NC * 8));
+
+	write_sysreg(new_mair, mair_el1);
+	isb();
+	local_flush_tlb_all();
+
+	pr_info("CPU%d: MAIR_EL1 updated 0x%016llx -> 0x%016llx\n",
+		smp_processor_id(), current_mair, new_mair);
+}
+
+#endif /* CONFIG_ARM64_WORKAROUND_NC_TO_NGNRE */
+
 const struct arm64_cpu_capabilities arm64_errata[] = {
 #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE
 	{
@@ -906,6 +1002,15 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
 		.matches = has_impdef_pmuv3,
 		.cpu_enable = cpu_enable_impdef_pmuv3_traps,
 	},
+#ifdef CONFIG_ARM64_WORKAROUND_NC_TO_NGNRE
+	{
+		.desc = "MAIR_EL1 NC to nGnRE",
+		.capability = ARM64_WORKAROUND_NC_TO_NGNRE,
+		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
+		.matches = has_nc_ngnre_workaround,
+		.cpu_enable = enable_nc_to_ngnre,
+	},
+#endif
 	{
 	}
 };

arch/arm64/kvm/mmu.c

@@ -1628,7 +1628,8 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
 	gfn = ipa >> PAGE_SHIFT;
 	mte_allowed = kvm_vma_mte_allowed(vma);
 
-	vfio_allow_any_uc = vma->vm_flags & VM_ALLOW_ANY_UNCACHED;
+	if (!cpus_have_cap(ARM64_WORKAROUND_NC_TO_NGNRE))
+		vfio_allow_any_uc = vma->vm_flags & VM_ALLOW_ANY_UNCACHED;
 
 	vm_flags = vma->vm_flags;
 

arch/arm64/mm/proc.S

@@ -66,7 +66,8 @@
 	 MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRE, MT_DEVICE_nGnRE) |	\
 	 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_NC, MT_NORMAL_NC) |		\
 	 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL) |			\
-	 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL_TAGGED))
+	 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL_TAGGED) |		\
+	 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_NC, MT_NORMAL_NC_DMA))
 
 #ifdef CONFIG_CPU_PM
 /**
@@ -467,6 +468,14 @@ SYM_FUNC_START(__cpu_setup)
 	tcr	.req	x16
 	tcr2	.req	x15
 	mov_q	mair, MAIR_EL1_SET
+
+#ifdef CONFIG_ARM64_WORKAROUND_NC_TO_NGNRE
+alternative_if ARM64_WORKAROUND_NC_TO_NGNRE
+	mov	x9, #MAIR_ATTR_DEVICE_nGnRE
+	bfi	mair, x9, #(MT_NORMAL_NC * 8), #8
+alternative_else_nop_endif
+#endif
+
 	mov_q	tcr, TCR_T0SZ(IDMAP_VA_BITS) | TCR_T1SZ(VA_BITS_MIN) | TCR_CACHE_FLAGS | \
 		     TCR_SHARED | TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_ASID16 | \
 		     TCR_TBI0 | TCR_A1 | TCR_KASAN_SW_FLAGS | TCR_MTE_FLAGS

arch/arm64/tools/cpucaps

@@ -112,6 +112,7 @@ WORKAROUND_CAVIUM_TX2_219_PRFM
 WORKAROUND_CAVIUM_TX2_219_TVM
 WORKAROUND_CLEAN_CACHE
 WORKAROUND_DEVICE_LOAD_ACQUIRE
+WORKAROUND_NC_TO_NGNRE
 WORKAROUND_NVIDIA_CARMEL_CNP
 WORKAROUND_PMUV3_IMPDEF_TRAPS
 WORKAROUND_QCOM_FALKOR_E1003