Arm64: Implement region write barriers

docs/design/coreclr/jit/GC-write-barriers.md

@@ -0,0 +1,96 @@
+# GC write barriers
+
+The GC write barrier function (JIT_WriteBarrier) is generally the hottest function in CoreCLR and is written in assembly. The full pseudo code for the function is as follows:
+
+
+````
+JIT_WriteBarrier(Object **dst, Object *ref)
+    Set *dst = ref
+
+    // Shadow Heap update
+    ifdef WRITE_BARRIER_CHECK:
+        if g_GCShadow != 0:
+            long *shadow_dst = g_GCShadow + (dst - g_lowest_address)
+            // Check shadow heap location is within shadow heap
+            if shadow_dst < g_GCShadowEnd:
+                *shadow_dst = ref
+                atomic: wait for stores to complete
+                if *dst != ref:
+                    *shadow_dst = INVALIDGCVALUE
+
+    // Update the write watch table, if it's in use
+    ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP:
+        if g_sw_ww_table != 0:
+            char *ww_table_dst = g_sw_ww_table + (dst>>11)
+            if *ww_table_dst != 0:
+                *ww_table_dst =  0xff
+
+    // Return if the reference is not in Gen 0
+    if ref < g_ephemeral_low || ref >= g_ephemeral_high:
+        return
+
+    // Region Checks
+    if g_region_to_generation_table != 0:
+
+        // Calculate region locations
+        char reg_loc_dst = *((dst >> g_region_shr) + g_region_to_generation_table)
+        char reg_loc_ref = *((ref >> g_region_shr) + g_region_to_generation_table)
+
+        // Return if the region we're storing into is Gen 0
+        if reg_loc_dst == 0:
+            return
+
+        // Check this is going from old to young
+        if reg_loc_dst >= reg_loc_ref:
+            return
+
+        // Bitwise write barriers only
+        if g_region_use_bitwise_write_barrier:
+
+            char *card_table_dst = (dst >> 11) + g_card_table
+            char dst_bit = 1 << (dst >> 8 && 7)
+
+            // Check if we need to update the card table
+            if *card_table_dst & dst_bit == 0:
+                return
+
+            // Atomically update the card table
+            lock: *card_table_dst |= dst_bit
+
+            goto CardBundle
+
+    // Check if we need to update the card table
+    char *card_table_dst = (dst >> 11) + g_card_table
+    if *card_table_dst == 0xff:
+        return
+
+    // Update the card table
+    *card_table_dst = 0xff
+
+CardBundle:
+
+    // Mark the card bundle table as dirty
+    Ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES:
+        char card_bundle_dst = (dst >> 21) + g_card_bundle_table
+        if *card_bundle_dst != 0xff:
+            *card_bundle_dst = 0xff
+
+````
+
+The Checked Write Barrier has additional checks:
+
+````
+JIT_CheckedWriteBarrier(Object **dst, Object *ref)
+
+    // Return if the destination is not on the heap
+    if ref < g_lowest_address || ref >= g_highest_address:
+        return
+
+    return JIT_WriteBarrier(dst, ref)
+````
+
+
+
+## WritebarrierManager
+
+On AMD64, there several different implementations of the write barrier function. Each implementation assumes different state and so can skip certain checks. The actual write barrier that is called is a copy of one of these implementations. The WritebarrierManager keeps track of which implementation is currently being used. As internal state changes, the WritebarrierManager updates the copy to the correct implementation. In practice, most of the internal state is fixed on startup, with only changes to/from use of write watch barriers changing during runtime.

src/coreclr/vm/CMakeLists.txt

@@ -745,11 +745,11 @@ if(CLR_CMAKE_TARGET_ARCH_AMD64)
     )
 
     set(VM_SOURCES_WKS_ARCH
-        ${ARCH_SOURCES_DIR}/jitinterfaceamd64.cpp
         ${ARCH_SOURCES_DIR}/profiler.cpp
         exceptionhandling.cpp
         gcinfodecoder.cpp
         jitinterfacegen.cpp
+        writebarriermanager.cpp
     )
 
     set(VM_HEADERS_WKS_ARCH
@@ -816,6 +816,7 @@ elseif(CLR_CMAKE_TARGET_ARCH_ARM64)
     set(VM_SOURCES_WKS_ARCH
         ${ARCH_SOURCES_DIR}/profiler.cpp
         gcinfodecoder.cpp
+        writebarriermanager.cpp
     )
 
     if(CLR_CMAKE_HOST_UNIX)

src/coreclr/vm/arm64/asmhelpers.S

@@ -194,8 +194,9 @@ LEAF_END ThePreStubPatch, _TEXT
 LEAF_ENTRY JIT_UpdateWriteBarrierState, _TEXT
     PROLOG_SAVE_REG_PAIR_INDEXED   fp, lr, -16
 
-    // x0-x7, x10 will contain intended new state
+    // x0-x7, x10-x11, x13-x14 will contain intended new state
     // x8 will preserve skipEphemeralCheck
+    // x9 will preserve writeableOffset
     // x12 will be used for pointers
 
     mov x8, x0
@@ -231,12 +232,21 @@ LOCAL_LABEL(EphemeralCheckEnabled):
     PREPARE_EXTERNAL_VAR g_highest_address, x12
     ldr  x6, [x12]
 
+    PREPARE_EXTERNAL_VAR g_region_to_generation_table, x12
+    ldr  x7, [x12]
+
+    PREPARE_EXTERNAL_VAR g_region_shr, x12
+    ldr  w10, [x12]
+
+    PREPARE_EXTERNAL_VAR g_region_use_bitwise_write_barrier, x12
+    ldr  w11, [x12]
+
 #ifdef WRITE_BARRIER_CHECK
     PREPARE_EXTERNAL_VAR g_GCShadow, x12
-    ldr  x7, [x12]
+    ldr  x13, [x12]
 
     PREPARE_EXTERNAL_VAR g_GCShadowEnd, x12
-    ldr  x10, [x12]
+    ldr  x14, [x12]
 #endif
 
     // Update wbs state
@@ -247,12 +257,12 @@ LOCAL_LABEL(EphemeralCheckEnabled):
     stp  x0, x1, [x12], 16
     stp  x2, x3, [x12], 16
     stp  x4, x5, [x12], 16
-    str  x6, [x12], 8
+    stp  x6, x7, [x12], 16
+    stp  w10, w11, [x12], 8
 #ifdef WRITE_BARRIER_CHECK
-    stp  x7, x10, [x12], 16
+    stp  x13, x14, [x12], 16
 #endif
 
-
     EPILOG_RESTORE_REG_PAIR_INDEXED fp, lr, 16
     EPILOG_RETURN
 LEAF_END JIT_UpdateWriteBarrierState