diff --git a/docs/design/coreclr/jit/GC-write-barriers.md b/docs/design/coreclr/jit/GC-write-barriers.md new file mode 100644 index 00000000000000..2fe5903817bf29 --- /dev/null +++ b/docs/design/coreclr/jit/GC-write-barriers.md @@ -0,0 +1,96 @@ +# GC write barriers + +The GC write barrier function (JIT_WriteBarrier) is generally the hottest function in CoreCLR and is written in assembly. The full pseudo code for the function is as follows: + + +```` +JIT_WriteBarrier(Object **dst, Object *ref) + Set *dst = ref + + // Shadow Heap update + ifdef WRITE_BARRIER_CHECK: + if g_GCShadow != 0: + long *shadow_dst = g_GCShadow + (dst - g_lowest_address) + // Check shadow heap location is within shadow heap + if shadow_dst < g_GCShadowEnd: + *shadow_dst = ref + atomic: wait for stores to complete + if *dst != ref: + *shadow_dst = INVALIDGCVALUE + + // Update the write watch table, if it's in use + ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP: + if g_sw_ww_table != 0: + char *ww_table_dst = g_sw_ww_table + (dst>>11) + if *ww_table_dst != 0: + *ww_table_dst = 0xff + + // Return if the reference is not in ephemeral generations + if ref < g_ephemeral_low || ref >= g_ephemeral_high: + return + + // Region Checks + if g_region_to_generation_table != 0: + + // Calculate region generations + char reg_loc_dst = *((dst >> g_region_shr) + g_region_to_generation_table) + char reg_loc_ref = *((ref >> g_region_shr) + g_region_to_generation_table) + + // Return if the region we're storing into is Gen 0 + if reg_loc_dst == 0: + return + + // Return if the new reference is not from old to young + if reg_loc_ref >= reg_loc_dst: + return + + // Bitwise write barriers only + if g_region_use_bitwise_write_barrier: + + char *card_table_dst = (dst >> 11) + g_card_table + char dst_bit = 1 << (dst >> 8 && 7) + + // Check if we need to update the card table + if *card_table_dst & dst_bit == 0: + return + + // Atomically update the card table + lock: *card_table_dst |= dst_bit + + goto CardBundle + + // Check if we need to update the card table + char *card_table_dst = (dst >> 11) + g_card_table + if *card_table_dst == 0xff: + return + + // Update the card table + *card_table_dst = 0xff + +CardBundle: + + // Mark the card bundle table as dirty + Ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES: + char card_bundle_dst = (dst >> 21) + g_card_bundle_table + if *card_bundle_dst != 0xff: + *card_bundle_dst = 0xff + +```` + +The Checked Write Barrier has additional checks: + +```` +JIT_CheckedWriteBarrier(Object **dst, Object *ref) + + // Return if the destination is not on the heap + if ref < g_lowest_address || ref >= g_highest_address: + return + + return JIT_WriteBarrier(dst, ref) +```` + + + +## WritebarrierManager + +On AMD64 and Arm64, there several different implementations of the write barrier function. Each implementation assumes different state and so can skip certain checks. The actual write barrier that is called is a copy of one of these implementations. The WritebarrierManager keeps track of which implementation is currently being used. As internal state changes, the WritebarrierManager updates the copy to the correct implementation. In practice, most of the internal state is fixed on startup, with only changes to/from use of write watch barriers changing during runtime. diff --git a/src/coreclr/pal/inc/unixasmmacrosarm64.inc b/src/coreclr/pal/inc/unixasmmacrosarm64.inc index 1e9a8a1e2bba7e..df8fa3ec0f8f19 100644 --- a/src/coreclr/pal/inc/unixasmmacrosarm64.inc +++ b/src/coreclr/pal/inc/unixasmmacrosarm64.inc @@ -37,6 +37,13 @@ C_FUNC(\Name): C_FUNC(\Name): .endm +// On MacOS, local labels cannot be used in arithmetic expressions. +#if defined(__APPLE__) +#define FIXUP_LABEL(name) name +#else +#define FIXUP_LABEL(name) .L##name +#endif + .macro LEAF_ENTRY Name, Section .global C_FUNC(\Name) #if defined(__APPLE__) diff --git a/src/coreclr/vm/CMakeLists.txt b/src/coreclr/vm/CMakeLists.txt index 2735b4b5a937a2..ddb0928c87b317 100644 --- a/src/coreclr/vm/CMakeLists.txt +++ b/src/coreclr/vm/CMakeLists.txt @@ -764,11 +764,11 @@ if(CLR_CMAKE_TARGET_ARCH_AMD64) ) set(VM_SOURCES_WKS_ARCH - ${ARCH_SOURCES_DIR}/jitinterfaceamd64.cpp ${ARCH_SOURCES_DIR}/profiler.cpp exceptionhandling.cpp gcinfodecoder.cpp jitinterfacegen.cpp + writebarriermanager.cpp ) set(VM_HEADERS_WKS_ARCH @@ -835,6 +835,7 @@ elseif(CLR_CMAKE_TARGET_ARCH_ARM64) set(VM_SOURCES_WKS_ARCH ${ARCH_SOURCES_DIR}/profiler.cpp gcinfodecoder.cpp + writebarriermanager.cpp ) if(CLR_CMAKE_HOST_UNIX) diff --git a/src/coreclr/vm/arm64/asmhelpers.S b/src/coreclr/vm/arm64/asmhelpers.S index 5fac7fae7aa838..bbfa2ad44b21ba 100644 --- a/src/coreclr/vm/arm64/asmhelpers.S +++ b/src/coreclr/vm/arm64/asmhelpers.S @@ -180,83 +180,6 @@ PATCH_LABEL ThePreStubPatchLabel ret lr LEAF_END ThePreStubPatch, _TEXT -// void JIT_UpdateWriteBarrierState(bool skipEphemeralCheck, size_t writeableOffset) -// -// Update shadow copies of the various state info required for barrier -// -// State info is contained in a literal pool at the end of the function -// Placed in text section so that it is close enough to use ldr literal and still -// be relocatable. Eliminates need for PREPARE_EXTERNAL_VAR in hot code. -// -// Align and group state info together so it fits in a single cache line -// and each entry can be written atomically -// -LEAF_ENTRY JIT_UpdateWriteBarrierState, _TEXT - PROLOG_SAVE_REG_PAIR_INDEXED fp, lr, -16 - - // x0-x7, x10 will contain intended new state - // x8 will preserve skipEphemeralCheck - // x12 will be used for pointers - - mov x8, x0 - mov x9, x1 - - PREPARE_EXTERNAL_VAR g_card_table, x12 - ldr x0, [x12] - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - PREPARE_EXTERNAL_VAR g_card_bundle_table, x12 - ldr x1, [x12] -#endif - -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - PREPARE_EXTERNAL_VAR g_sw_ww_table, x12 - ldr x2, [x12] -#endif - - PREPARE_EXTERNAL_VAR g_ephemeral_low, x12 - ldr x3, [x12] - - PREPARE_EXTERNAL_VAR g_ephemeral_high, x12 - ldr x4, [x12] - - cbz x8, LOCAL_LABEL(EphemeralCheckEnabled) - movz x3, #0 - movn x4, #0 -LOCAL_LABEL(EphemeralCheckEnabled): - - PREPARE_EXTERNAL_VAR g_lowest_address, x12 - ldr x5, [x12] - - PREPARE_EXTERNAL_VAR g_highest_address, x12 - ldr x6, [x12] - -#ifdef WRITE_BARRIER_CHECK - PREPARE_EXTERNAL_VAR g_GCShadow, x12 - ldr x7, [x12] - - PREPARE_EXTERNAL_VAR g_GCShadowEnd, x12 - ldr x10, [x12] -#endif - - // Update wbs state - PREPARE_EXTERNAL_VAR JIT_WriteBarrier_Table_Loc, x12 - ldr x12, [x12] - add x12, x12, x9 - - stp x0, x1, [x12], 16 - stp x2, x3, [x12], 16 - stp x4, x5, [x12], 16 - str x6, [x12], 8 -#ifdef WRITE_BARRIER_CHECK - stp x7, x10, [x12], 16 -#endif - - - EPILOG_RESTORE_REG_PAIR_INDEXED fp, lr, 16 - EPILOG_RETURN -LEAF_END JIT_UpdateWriteBarrierState - // ------------------------// ------------------------------------------------------------------ // __declspec(naked) void F_CALL_CONV JIT_WriteBarrier_Callable(Object **dst, Object* val) LEAF_ENTRY JIT_WriteBarrier_Callable, _TEXT diff --git a/src/coreclr/vm/arm64/asmhelpers.asm b/src/coreclr/vm/arm64/asmhelpers.asm index 3dd6edcb34ab2b..2e8b71ee3aa5e8 100644 --- a/src/coreclr/vm/arm64/asmhelpers.asm +++ b/src/coreclr/vm/arm64/asmhelpers.asm @@ -248,84 +248,6 @@ ThePreStubPatchLabel ret lr LEAF_END -;----------------------------------------------------------------------------- -; void JIT_UpdateWriteBarrierState(bool skipEphemeralCheck, size_t writeableOffset) -; -; Update shadow copies of the various state info required for barrier -; -; State info is contained in a literal pool at the end of the function -; Placed in text section so that it is close enough to use ldr literal and still -; be relocatable. Eliminates need for PREPARE_EXTERNAL_VAR in hot code. -; -; Align and group state info together so it fits in a single cache line -; and each entry can be written atomically -; - LEAF_ENTRY JIT_UpdateWriteBarrierState - PROLOG_SAVE_REG_PAIR fp, lr, #-16! - - ; x0-x7, x10 will contain intended new state - ; x8 will preserve skipEphemeralCheck - ; x12 will be used for pointers - - mov x8, x0 - mov x9, x1 - - adrp x12, g_card_table - ldr x0, [x12, g_card_table] - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - adrp x12, g_card_bundle_table - ldr x1, [x12, g_card_bundle_table] -#endif - -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - adrp x12, g_sw_ww_table - ldr x2, [x12, g_sw_ww_table] -#endif - - adrp x12, g_ephemeral_low - ldr x3, [x12, g_ephemeral_low] - - adrp x12, g_ephemeral_high - ldr x4, [x12, g_ephemeral_high] - - ; Check skipEphemeralCheck - cbz x8, EphemeralCheckEnabled - movz x3, #0 - movn x4, #0 - -EphemeralCheckEnabled - adrp x12, g_lowest_address - ldr x5, [x12, g_lowest_address] - - adrp x12, g_highest_address - ldr x6, [x12, g_highest_address] - -#ifdef WRITE_BARRIER_CHECK - adrp x12, $g_GCShadow - ldr x7, [x12, $g_GCShadow] - - adrp x12, $g_GCShadowEnd - ldr x10, [x12, $g_GCShadowEnd] -#endif - - ; Update wbs state - adrp x12, JIT_WriteBarrier_Table_Loc - ldr x12, [x12, JIT_WriteBarrier_Table_Loc] - add x12, x12, x9 - stp x0, x1, [x12], 16 - stp x2, x3, [x12], 16 - stp x4, x5, [x12], 16 - str x6, [x12], 8 -#ifdef WRITE_BARRIER_CHECK - stp x7, x10, [x12], 16 -#endif - - EPILOG_RESTORE_REG_PAIR fp, lr, #16! - EPILOG_RETURN - - LEAF_END JIT_UpdateWriteBarrierState - #ifdef FEATURE_COMINTEROP ; ------------------------------------------------------------------ diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index 0e223cbc1d33ab..2bea90942e66a4 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -3,6 +3,7 @@ #include "asmconstants.h" #include "unixasmmacros.inc" +#include "patchedcodeconstants.h" //----------------------------------------------------------------------------- // The following Macros help in WRITE_BARRIER Implementations @@ -85,6 +86,10 @@ WRITE_BARRIER_END JIT_CheckedWriteBarrier //----------------------------------------------------------------------------- // void JIT_WriteBarrier(Object** dst, Object* src) +// +// Empty function which at runtime is patched with one of the JIT_WriteBarrier_ +// functions below. +// // On entry: // x14 : the destination address (LHS of the assignment) // x15 : the object reference (RHS of the assignment) @@ -99,25 +104,82 @@ WRITE_BARRIER_END JIT_CheckedWriteBarrier // if you add more trashed registers. // WRITE_BARRIER_ENTRY JIT_WriteBarrier - stlr x15, [x14] + // This must be greater than the largest JIT_WriteBarrier_ function. + .space JIT_WriteBarrier_Size, 0 +WRITE_BARRIER_END JIT_WriteBarrier +//----------------------------------------------------------------------------- +// JIT_WriteBarrier_Table +// +// Patchable literal pool +// + .balign 64 // Align to power of two at least as big as patchable literal pool so that it fits optimally in cache line +WRITE_BARRIER_ENTRY JIT_WriteBarrier_Table +PATCH_LABEL JIT_WriteBarrier_Patch_Label_CardTable + .quad 0 +PATCH_LABEL JIT_WriteBarrier_Patch_Label_CardBundleTable + .quad 0 +PATCH_LABEL JIT_WriteBarrier_Patch_Label_WriteWatchTable + .quad 0 +PATCH_LABEL JIT_WriteBarrier_Patch_Label_Lower + .quad 0 +PATCH_LABEL JIT_WriteBarrier_Patch_Label_Upper + .quad 0 +LOCAL_LABEL(wbs_lowest_address): +PATCH_LABEL JIT_WriteBarrier_Patch_Label_LowestAddress + .quad 0 +LOCAL_LABEL(wbs_highest_address): +PATCH_LABEL JIT_WriteBarrier_Patch_Label_HighestAddress + .quad 0 +PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionToGeneration + .quad 0 +PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionShr + .quad 0 #ifdef WRITE_BARRIER_CHECK +PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadow + .quad 0 +PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadowEnd + .quad 0 +#endif +WRITE_BARRIER_END JIT_WriteBarrier_Table + +// ------------------------------------------------------------------ +// End of the writeable code region +LEAF_ENTRY JIT_PatchedCodeLast, _TEXT + ret lr +LEAF_END JIT_PatchedCodeLast, _TEXT + + + +//----------------------------------------------------------------------------- +// The following Macros are used by the different JIT_WriteBarrier_ functions. +// +// + +.macro WRITE_BARRIER_ENTRY_STUB start +FIXUP_LABEL(\start): + stlr x15, [x14] +.endm + + +.macro WRITE_BARRIER_SHADOW_UPDATE_STUB start + #ifdef WRITE_BARRIER_CHECK // Update GC Shadow Heap // Do not perform the work if g_GCShadow is 0 - ldr x12, LOCAL_LABEL(wbs_GCShadow) - cbz x12, LOCAL_LABEL(ShadowUpdateEnd) + ldr x12, JIT_WriteBarrier_Offset_GCShadow + FIXUP_LABEL(\start) + cbz x12, LOCAL_LABEL(ShadowUpdateEnd\@) // Compute address of shadow heap location: // pShadow = g_GCShadow + (x14 - g_lowest_address) - ldr x17, LOCAL_LABEL(wbs_lowest_address) + ldr x17, JIT_WriteBarrier_Offset_LowestAddress + FIXUP_LABEL(\start) sub x17, x14, x17 add x12, x17, x12 // if (pShadow >= g_GCShadowEnd) goto end - ldr x17, LOCAL_LABEL(wbs_GCShadowEnd) + ldr x17, JIT_WriteBarrier_Offset_GCShadowEnd + FIXUP_LABEL(\start) cmp x12, x17 - bhs LOCAL_LABEL(ShadowUpdateEnd) + bhs LOCAL_LABEL(ShadowUpdateEnd\@) // *pShadow = x15 str x15, [x12] @@ -129,96 +191,305 @@ WRITE_BARRIER_ENTRY JIT_WriteBarrier // if ([x14] == x15) goto end ldr x17, [x14] cmp x17, x15 - beq LOCAL_LABEL(ShadowUpdateEnd) + beq LOCAL_LABEL(ShadowUpdateEnd\@) // *pShadow = INVALIDGCVALUE (0xcccccccd) movz x17, #0xcccd movk x17, #0xcccc, LSL #16 str x17, [x12] - -LOCAL_LABEL(ShadowUpdateEnd): #endif +LOCAL_LABEL(ShadowUpdateEnd\@): +.endm + +.macro WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB start, exit #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP // Update the write watch table if necessary - ldr x12, LOCAL_LABEL(wbs_sw_ww_table) - cbz x12, LOCAL_LABEL(CheckCardTable) + ldr x12, JIT_WriteBarrier_Offset_WriteWatchTable + FIXUP_LABEL(\start) add x12, x12, x14, lsr #0xc // SoftwareWriteWatch::AddressToTableByteIndexShift ldrb w17, [x12] - cbnz x17, LOCAL_LABEL(CheckCardTable) + cbnz x17, LOCAL_LABEL(WriteWatchForGCHeapEnd\@) mov w17, #0xFF strb w17, [x12] +LOCAL_LABEL(WriteWatchForGCHeapEnd\@): #endif +.endm -LOCAL_LABEL(CheckCardTable): - // Branch to Exit if the reference is not in the Gen0 heap - ldr x12, LOCAL_LABEL(wbs_ephemeral_low) - ldr x17, LOCAL_LABEL(wbs_ephemeral_high) + +.macro WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB start, exit + // Branch to Exit if the reference is not in the ephemeral generations. + ldr x12, JIT_WriteBarrier_Offset_Lower + FIXUP_LABEL(\start) + cmp x15, x12 + blo LOCAL_LABEL(\exit) +.endm + + +.macro WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB start, exit + // Branch to Exit if the reference is not in the ephemeral generations. + ldr x12, JIT_WriteBarrier_Offset_Lower + FIXUP_LABEL(\start) + ldr x17, JIT_WriteBarrier_Offset_Upper + FIXUP_LABEL(\start) cmp x15, x12 ccmp x15, x17, #0x2, hs - bhs LOCAL_LABEL(Exit) + bhs LOCAL_LABEL(\exit) +.endm + + +.macro WRITE_BARRIER_REGION_CHECK_STUB start, exit + // Calculate region generations + ldr x17, JIT_WriteBarrier_Offset_RegionToGeneration + FIXUP_LABEL(\start) + ldr w12, JIT_WriteBarrier_Offset_RegionShr + FIXUP_LABEL(\start) + lsr x15, x15, x12 + add x15, x15, x17 // x15 = (RHS >> wbs_region_shr) + wbs_region_to_generation_table + lsr x12, x14, x12 + add x12, x12, x17 // x12 = (LHS >> wbs_region_shr) + wbs_region_to_generation_table + // Check whether the region we are storing into is gen 0 - nothing to do in this case + ldrb w12, [x12] + cbz w12, LOCAL_LABEL(\exit) + + // Return if the new reference is not from old to young + ldrb w15, [x15] + cmp w15, w12 + bhs LOCAL_LABEL(\exit) +.endm + + +.macro WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB start, exit // Check if we need to update the card table - ldr x12, LOCAL_LABEL(wbs_card_table) + lsr w17, w14, 8 + and w17, w17, 7 + movz w15, 1 + lsl w17, w15, w17 // w17 = 1 << (LHS >> 8 && 7) + ldr x12, JIT_WriteBarrier_Offset_CardTable + FIXUP_LABEL(\start) add x15, x12, x14, lsr #11 - ldrb w12, [x15] + ldrb w12, [x15] // w12 = [(LHS >> 11) + g_card_table] + tst w12, w17 + bne LOCAL_LABEL(\exit) + + // Atomically update the card table + // Requires LSE, but the code is only compiled for 8.0 + .word 0x383131FF // stsetb w17, [x15] +.endm + + +.macro WRITE_BARRIER_CHECK_CARD_TABLE_STUB start, exit + // Check if we need to update the card table + ldr x12, JIT_WriteBarrier_Offset_CardTable + FIXUP_LABEL(\start) + add x15, x12, x14, lsr #11 + ldrb w12, [x15] // w12 = [(RHS >> 11) + g_card_table] cmp x12, 0xFF - beq LOCAL_LABEL(Exit) + beq LOCAL_LABEL(\exit) // Update the card table mov x12, 0xFF strb w12, [x15] +.endm + +.macro WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB start, exit #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES // Check if we need to update the card bundle table - ldr x12, LOCAL_LABEL(wbs_card_bundle_table) + ldr x12, JIT_WriteBarrier_Offset_CardBundleTable + FIXUP_LABEL(\start) add x15, x12, x14, lsr #21 ldrb w12, [x15] cmp x12, 0xFF - beq LOCAL_LABEL(Exit) + beq LOCAL_LABEL(\exit) // Update the card bundle mov x12, 0xFF strb w12, [x15] #endif +.endm + -LOCAL_LABEL(Exit): +.macro WRITE_BARRIER_RETURN_STUB exit +LOCAL_LABEL(\exit): // Increment by 8 to implement JIT_ByRefWriteBarrier contract. // TODO: Consider duplicating the logic to get rid of this redundant 'add' // for JIT_WriteBarrier/JIT_CheckedWriteBarrier add x14, x14, 8 ret lr -WRITE_BARRIER_END JIT_WriteBarrier +.endm - // Begin patchable literal pool - .balign 64 // Align to power of two at least as big as patchable literal pool so that it fits optimally in cache line -WRITE_BARRIER_ENTRY JIT_WriteBarrier_Table -LOCAL_LABEL(wbs_begin): -LOCAL_LABEL(wbs_card_table): - .quad 0 -LOCAL_LABEL(wbs_card_bundle_table): - .quad 0 -LOCAL_LABEL(wbs_sw_ww_table): - .quad 0 -LOCAL_LABEL(wbs_ephemeral_low): - .quad 0 -LOCAL_LABEL(wbs_ephemeral_high): - .quad 0 -LOCAL_LABEL(wbs_lowest_address): - .quad 0 -LOCAL_LABEL(wbs_highest_address): - .quad 0 -#ifdef WRITE_BARRIER_CHECK -LOCAL_LABEL(wbs_GCShadow): - .quad 0 -LOCAL_LABEL(wbs_GCShadowEnd): - .quad 0 -#endif -WRITE_BARRIER_END JIT_WriteBarrier_Table +//----------------------------------------------------------------------------- +// void JIT_WriteBarrier_PreGrow64(Object** dst, Object* src) +// +// Skipped functionality: +// Does not update the write watch table +// Does not check wbs_ephemeral_high +// No region checks +// +LEAF_ENTRY JIT_WriteBarrier_PreGrow64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_PreGrow64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_PreGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB Start_PreGrow64, Exit_PreGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_PreGrow64, Exit_PreGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_PreGrow64, Exit_PreGrow64 + WRITE_BARRIER_RETURN_STUB Exit_PreGrow64 +LEAF_END_MARKED JIT_WriteBarrier_PreGrow64, _TEXT + + +//----------------------------------------------------------------------------- +// void JIT_WriteBarrier_PostGrow64(Object** dst, Object* src) +// +// Skipped functionality: +// Does not update the write watch table +// No region checks +// +LEAF_ENTRY JIT_WriteBarrier_PostGrow64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_PostGrow64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_PostGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_PostGrow64, Exit_PostGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_PostGrow64, Exit_PostGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_PostGrow64, Exit_PostGrow64 + WRITE_BARRIER_RETURN_STUB Exit_PostGrow64 +LEAF_END_MARKED JIT_WriteBarrier_PostGrow64, _TEXT + + +//----------------------------------------------------------------------------- +// void JIT_WriteBarrier_SVR64(Object** dst, Object* src) +// +// SVR GC has multiple heaps, so it cannot provide one single ephemeral region to bounds check +// against, so we just skip the bounds checking all together and do our card table update unconditionally. +// +// Skipped functionality: +// Does not update the write watch table +// Does not check wbs_ephemeral_high or wbs_ephemeral_low +// No region checks +// +LEAF_ENTRY JIT_WriteBarrier_SVR64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_SVR64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_SVR64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_SVR64, Exit_SVR64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_SVR64, Exit_SVR64 + WRITE_BARRIER_RETURN_STUB Exit_SVR64 +LEAF_END_MARKED JIT_WriteBarrier_SVR64, _TEXT + + +//----------------------------------------------------------------------------- +// void JIT_WriteBarrier_Byte_Region64(Object** dst, Object* src) +// +// Skipped functionality: +// Does not update the write watch table +// Bitwise updates for region checks +// +LEAF_ENTRY JIT_WriteBarrier_Byte_Region64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_Byte_Region64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_Byte_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Byte_Region64, Exit_Byte_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_Byte_Region64, Exit_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_Byte_Region64, Exit_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Byte_Region64, Exit_Byte_Region64 + WRITE_BARRIER_RETURN_STUB Exit_Byte_Region64 +LEAF_END_MARKED JIT_WriteBarrier_Byte_Region64, _TEXT + + +//----------------------------------------------------------------------------- +// void JIT_WriteBarrier_Bit_Region64(Object** dst, Object* src) +// +// Skipped functionality: +// Does not update the write watch table +// Does not call check card table stub +// +LEAF_ENTRY JIT_WriteBarrier_Bit_Region64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_Bit_Region64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_Bit_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Bit_Region64, Exit_Bit_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_Bit_Region64, Exit_Bit_Region64 + WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Start_Bit_Region64, Exit_Bit_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Bit_Region64, Exit_Bit_Region64 + WRITE_BARRIER_RETURN_STUB Exit_Bit_Region64 +LEAF_END_MARKED JIT_WriteBarrier_Bit_Region64, _TEXT + + +//----------------------------------------------------------------------------- +// void JIT_WriteBarrier_WriteWatch_PreGrow64(Object** dst, Object* src) +// +// Skipped functionality: +// Does not check wbs_ephemeral_high +// No region checks +// +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_PreGrow64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_PreGrow64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PreGrow64, Exit_WriteWatch_PreGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB Start_WriteWatch_PreGrow64, Exit_WriteWatch_PreGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_PreGrow64, Exit_WriteWatch_PreGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_PreGrow64, Exit_WriteWatch_PreGrow64 + WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_PreGrow64 +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT + + +//----------------------------------------------------------------------------- +// void JIT_WriteBarrier_WriteWatch_PostGrow64(Object** dst, Object* src) +// +// Skipped functionality: +// No region checks +// +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_PostGrow64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_PostGrow64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PostGrow64, Exit_WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_PostGrow64, Exit_WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_PostGrow64, Exit_WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_PostGrow64, Exit_WriteWatch_PostGrow64 + WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_PostGrow64 +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT + + +//----------------------------------------------------------------------------- +// void JIT_WriteBarrier_WriteWatch_SVR64(Object** dst, Object* src) +// +// SVR GC has multiple heaps, so it cannot provide one single ephemeral region to bounds check +// against, so we just skip the bounds checking all together and do our card table update unconditionally. +// +// Skipped functionality: +// Does not check wbs_ephemeral_high or wbs_ephemeral_low +// No region checks +// +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_SVR64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_SVR64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_SVR64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_SVR64, Exit_WriteWatch_SVR64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_SVR64, Exit_WriteWatch_SVR64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_SVR64, Exit_WriteWatch_SVR64 + WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_SVR64 +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_SVR64, _TEXT + + +//----------------------------------------------------------------------------- +// void JIT_WriteBarrier_WriteWatch_Byte_Region64(Object** dst, Object* src) +// +// Skipped functionality: +// Bitwise updates for region checks +// +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_Byte_Region64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_Byte_Region64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_Byte_Region64, Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_Byte_Region64, Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_WriteWatch_Byte_Region64, Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_Byte_Region64, Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_Byte_Region64, Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_Byte_Region64 +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT + + +//----------------------------------------------------------------------------- +// void JIT_WriteBarrier_WriteWatch_Bit_Region64(Object** dst, Object* src) +// +// Skipped functionality: +// Does not call check card table stub +// +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_Bit_Region64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_Bit_Region64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_Bit_Region64, Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_Bit_Region64, Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_WriteWatch_Bit_Region64, Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Start_WriteWatch_Bit_Region64, Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_Bit_Region64, Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_Bit_Region64 +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT -// ------------------------------------------------------------------ -// End of the writeable code region -LEAF_ENTRY JIT_PatchedCodeLast, _TEXT - ret lr -LEAF_END JIT_PatchedCodeLast, _TEXT diff --git a/src/coreclr/vm/arm64/patchedcode.asm b/src/coreclr/vm/arm64/patchedcode.asm index 454b8cac0c4abe..500f1044d488d7 100644 --- a/src/coreclr/vm/arm64/patchedcode.asm +++ b/src/coreclr/vm/arm64/patchedcode.asm @@ -4,6 +4,7 @@ #include "ksarm64.h" #include "asmconstants.h" #include "asmmacros.h" +#include "patchedcodeconstants.h" ;;like TEXTAREA, but with 64 byte alignment so that we can align the patchable pool below to 64 without warning AREA |.text|,ALIGN=6,CODE,READONLY @@ -38,32 +39,6 @@ ret lr LEAF_END - ; Begin patchable literal pool - ALIGN 64 ; Align to power of two at least as big as patchable literal pool so that it fits optimally in cache line - WRITE_BARRIER_ENTRY JIT_WriteBarrier_Table -wbs_begin -wbs_card_table - DCQ 0 -wbs_card_bundle_table - DCQ 0 -wbs_sw_ww_table - DCQ 0 -wbs_ephemeral_low - DCQ 0 -wbs_ephemeral_high - DCQ 0 -wbs_lowest_address - DCQ 0 -wbs_highest_address - DCQ 0 -#ifdef WRITE_BARRIER_CHECK -wbs_GCShadow - DCQ 0 -wbs_GCShadowEnd - DCQ 0 -#endif - WRITE_BARRIER_END JIT_WriteBarrier_Table - ;----------------------------------------------------------------------------- ; void JIT_ByRefWriteBarrier ; On entry: @@ -117,6 +92,9 @@ NotInHeap ;----------------------------------------------------------------------------- ; void JIT_WriteBarrier(Object** dst, Object* src) +; +; Empty function which at runtime is patched with one of the JIT_WriteBarrier_ +; functions below. ; On entry: ; x14 : the destination address (LHS of the assignment) ; x15 : the object reference (RHS of the assignment) @@ -131,107 +109,395 @@ NotInHeap ; if you add more trashed registers. ; WRITE_BARRIER_ENTRY JIT_WriteBarrier - stlr x15, [x14] +; This must be greater than the largest JIT_WriteBarrier_ function. + space (232*4), 0 + WRITE_BARRIER_END JIT_WriteBarrier + ; Begin patchable literal pool + ALIGN 64 ; Align to power of two at least as big as patchable literal pool so that it fits optimally in cache line + WRITE_BARRIER_ENTRY JIT_WriteBarrier_Table + PATCH_LABEL JIT_WriteBarrier_Patch_Label_CardTable + DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_CardBundleTable + DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_WriteWatchTable + DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_Lower + DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_Upper + DCQ 0 +wbs_lowest_address + PATCH_LABEL JIT_WriteBarrier_Patch_Label_LowestAddress + DCQ 0 +wbs_highest_address + PATCH_LABEL JIT_WriteBarrier_Patch_Label_HighestAddress + DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionToGeneration + DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionShr + DCQ 0 #ifdef WRITE_BARRIER_CHECK + PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadow + DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadowEnd + DCQ 0 +#endif + WRITE_BARRIER_END JIT_WriteBarrier_Table + +; ------------------------------------------------------------------ +; End of the writeable code region + LEAF_ENTRY JIT_PatchedCodeLast + ret lr + LEAF_END + + + +;----------------------------------------------------------------------------- +; The following Macros are used by the different JIT_WriteBarrier_ functions. +; +; + + MACRO + WRITE_BARRIER_ENTRY_STUB $name +start$name + stlr x15, [x14] + MEND + + + MACRO + WRITE_BARRIER_SHADOW_UPDATE_STUB $name + #ifdef WRITE_BARRIER_CHECK ; Update GC Shadow Heap ; Do not perform the work if g_GCShadow is 0 - ldr x12, wbs_GCShadow - cbz x12, ShadowUpdateEnd + ldr x12, JIT_WriteBarrier_Offset_GCShadow + start$name + cbz x12, ShadowUpdateEnd$name ; Compute address of shadow heap location: - ; pShadow = $g_GCShadow + (x14 - g_lowest_address) - ldr x17, wbs_lowest_address - sub x17, x14, x17 - add x12, x17, x12 + ; pShadow = g_GCShadow + (x14 - g_lowest_address) + ldr x17, JIT_WriteBarrier_Offset_LowestAddress + start$name + sub x17, x14, x17 + add x12, x17, x12 - ; if (pShadow >= $g_GCShadowEnd) goto end - ldr x17, wbs_GCShadowEnd - cmp x12, x17 - bhs ShadowUpdateEnd + ; if (pShadow >= g_GCShadowEnd) goto end + ldr x17, JIT_WriteBarrier_Offset_GCShadowEnd + start$name + cmp x12, x17 + bhs ShadowUpdateEnd$name ; *pShadow = x15 - str x15, [x12] + str x15, [x12] ; Ensure that the write to the shadow heap occurs before the read from the GC heap so that race ; conditions are caught by INVALIDGCVALUE. - dmb ish + dmb ish ; if ([x14] == x15) goto end - ldr x17, [x14] - cmp x17, x15 - beq ShadowUpdateEnd + ldr x17, [x14] + cmp x17, x15 + beq ShadowUpdateEnd$name ; *pShadow = INVALIDGCVALUE (0xcccccccd) - movz x17, #0xcccd - movk x17, #0xcccc, LSL #16 - str x17, [x12] - -ShadowUpdateEnd -#endif + movz x17, #0xcccd + movk x17, #0xcccc, LSL #16 + str x17, [x12] + #endif +ShadowUpdateEnd$name + MEND -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + MACRO + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB $name + #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP ; Update the write watch table if necessary - ldr x12, wbs_sw_ww_table - cbz x12, CheckCardTable - add x12, x12, x14, LSR #0xC // SoftwareWriteWatch::AddressToTableByteIndexShift - ldrb w17, [x12] - cbnz x17, CheckCardTable - mov w17, 0xFF - strb w17, [x12] -#endif + ldr x12, JIT_WriteBarrier_Offset_WriteWatchTable + start$name + ; SoftwareWriteWatch::AddressToTableByteIndexShift + add x12, x12, x14, lsr #0xc + ldrb w17, [x12] + cbnz x17, WriteWatchForGCHeapEnd$name + mov w17, #0xFF + strb w17, [x12] +WriteWatchForGCHeapEnd$name + #endif + MEND -CheckCardTable - ; Branch to Exit if the reference is not in the Gen0 heap - ldr x12, wbs_ephemeral_low - ldr x17, wbs_ephemeral_high - cmp x15, x12 - ccmp x15, x17, #0x2, hs - bhs Exit + MACRO + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB $name + ; Branch to Exit if the reference is not in the ephemeral generations. + ldr x12, JIT_WriteBarrier_Offset_Lower + start$name + cmp x15, x12 + blo exit$name + MEND - ; Check if we need to update the card table - ldr x12, wbs_card_table + MACRO + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB $name + ; Branch to Exit if the reference is not in the ephemeral generations. + ldr x12, JIT_WriteBarrier_Offset_Lower + start$name + ldr x17, JIT_WriteBarrier_Offset_Upper + start$name + cmp x15, x12 + ccmp x15, x17, #0x2, hs + bhs exit$name + MEND + + MACRO + WRITE_BARRIER_REGION_CHECK_STUB $name + ; Calculate region generations + ldr x17, JIT_WriteBarrier_Offset_RegionToGeneration + start$name + ldr w12, JIT_WriteBarrier_Offset_RegionShr + start$name + lsr x15, x15, x12 + add x15, x15, x17 ; x15 = (RHS >> wbs_region_shr) + wbs_region_to_generation_table + lsr x12, x14, x12 + add x12, x12, x17 ; x12 = (LHS >> wbs_region_shr) + wbs_region_to_generation_table + + ; Check whether the region we are storing into is gen 0 - nothing to do in this case + ldrb w12, [x12] + cbz w12, exit$name + + ; Return if the new reference is not from old to young + ldrb w15, [x15] + cmp w15, w12 + bhs exit$name + MEND - ; x15 := pointer into card table - add x15, x12, x14, lsr #11 + MACRO + WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB $name + ; Check if we need to update the card table + lsr w17, w14, 8 + and w17, w17, 7 + movz w15, 1 + lsl w17, w15, w17 ; w17 = 1 << (LHS >> 8 && 7) + ldr x12, JIT_WriteBarrier_Offset_CardTable + start$name + add x15, x12, x14, lsr #11 + ldrb w12, [x15] ; w12 = [(LHS >> 11) + g_card_table] + tst w12, w17 + bne exit$name + + ; Atomically update the card table + ; Requires LSE, but the code is only compiled for 8.0 + ; stsetb w17, [x15] + DCD 0x383131FF + MEND - ldrb w12, [x15] - cmp x12, 0xFF - beq Exit + MACRO + WRITE_BARRIER_CHECK_CARD_TABLE_STUB $name + ; Check if we need to update the card table + ldr x12, JIT_WriteBarrier_Offset_CardTable + start$name + add x15, x12, x14, lsr #11 + ; w12 = [(RHS >> 11) + g_card_table] + ldrb w12, [x15] + cmp x12, 0xFF + beq exit$name ; Update the card table - mov x12, 0xFF - strb w12, [x15] + mov x12, 0xFF + strb w12, [x15] + MEND -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + MACRO + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB $name + #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES ; Check if we need to update the card bundle table - ldr x12, wbs_card_bundle_table - - ; x15 := pointer into card bundle table - add x15, x12, x14, lsr #21 - - ldrb w12, [x15] - cmp x12, 0xFF - beq Exit - - mov x12, 0xFF - strb w12, [x15] -#endif + ldr x12, JIT_WriteBarrier_Offset_CardBundleTable + start$name + add x15, x12, x14, lsr #21 + ldrb w12, [x15] + cmp x12, 0xFF + beq exit$name + + ; Update the card bundle + mov x12, 0xFF + strb w12, [x15] + #endif + MEND -Exit + MACRO + WRITE_BARRIER_RETURN_STUB $name +exit$name ; Increment by 8 to implement JIT_ByRefWriteBarrier contract. ; TODO: Consider duplicating the logic to get rid of this redundant 'add' ; for JIT_WriteBarrier/JIT_CheckedWriteBarrier - add x14, x14, 8 - ret lr - WRITE_BARRIER_END JIT_WriteBarrier + add x14, x14, 8 + ret lr + MEND + + ;----------------------------------------------------------------------------- + ; void JIT_WriteBarrier_PreGrow64(Object** dst, Object* src) + ; + ; Skipped functionality: + ; Does not update the write watch table + ; Does not check wbs_ephemeral_high + ; No region checks + ; + WRITE_BARRIER_ENTRY JIT_WriteBarrier_PreGrow64 + WRITE_BARRIER_ENTRY_STUB PreGrow64 + WRITE_BARRIER_SHADOW_UPDATE_STUB PreGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB PreGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB PreGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB PreGrow64 + WRITE_BARRIER_RETURN_STUB PreGrow64 + WRITE_BARRIER_END JIT_WriteBarrier_PreGrow64 + + + ;----------------------------------------------------------------------------- + ; void JIT_WriteBarrier_PostGrow64(Object** dst, Object* src) + ; + ; Skipped functionality: + ; Does not update the write watch table + ; No region checks + ; + WRITE_BARRIER_ENTRY JIT_WriteBarrier_PostGrow64 + WRITE_BARRIER_ENTRY_STUB PostGrow64 + WRITE_BARRIER_SHADOW_UPDATE_STUB PostGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB PostGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB PostGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB PostGrow64 + WRITE_BARRIER_RETURN_STUB PostGrow64 + WRITE_BARRIER_END JIT_WriteBarrier_PostGrow64 + + + ;----------------------------------------------------------------------------- + ; void JIT_WriteBarrier_SVR64(Object** dst, Object* src) + ; + ; SVR GC has multiple heaps, so it cannot provide one single ephemeral region to bounds check + ; against, so we just skip the bounds checking all together and do our card table update unconditionally. + ; + ; Skipped functionality: + ; Does not update the write watch table + ; Does not check wbs_ephemeral_high or wbs_ephemeral_low + ; No region checks + ; + WRITE_BARRIER_ENTRY JIT_WriteBarrier_SVR64 + WRITE_BARRIER_ENTRY_STUB SVR64 + WRITE_BARRIER_SHADOW_UPDATE_STUB SVR64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB SVR64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB SVR64 + WRITE_BARRIER_RETURN_STUB SVR64 + WRITE_BARRIER_END JIT_WriteBarrier_SVR64 + + + ;----------------------------------------------------------------------------- + ; void JIT_WriteBarrier_Byte_Region64(Object** dst, Object* src) + ; + ; Skipped functionality: + ; Does not update the write watch table + ; Bitwise updates for region checks + ; + WRITE_BARRIER_ENTRY JIT_WriteBarrier_Byte_Region64 + WRITE_BARRIER_ENTRY_STUB Byte_Region64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Byte_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Byte_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Byte_Region64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Byte_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Byte_Region64 + WRITE_BARRIER_RETURN_STUB Byte_Region64 + WRITE_BARRIER_END JIT_WriteBarrier_Byte_Region64 + + + ;----------------------------------------------------------------------------- + ; void JIT_WriteBarrier_Bit_Region64(Object** dst, Object* src) + ; + ; Skipped functionality: + ; Does not update the write watch table + ; Does not call check card table stub + ; + WRITE_BARRIER_ENTRY JIT_WriteBarrier_Bit_Region64 + WRITE_BARRIER_ENTRY_STUB Bit_Region64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Bit_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Bit_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Bit_Region64 + WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Bit_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Bit_Region64 + WRITE_BARRIER_RETURN_STUB Bit_Region64 + WRITE_BARRIER_END JIT_WriteBarrier_Bit_Region64 + + + ;----------------------------------------------------------------------------- + ; void JIT_WriteBarrier_WriteWatch_PreGrow64(Object** dst, Object* src) + ; + ; Skipped functionality: + ; Does not check wbs_ephemeral_high + ; No region checks + ; + WRITE_BARRIER_ENTRY JIT_WriteBarrier_WriteWatch_PreGrow64 + WRITE_BARRIER_ENTRY_STUB WriteWatch_PreGrow64 + WRITE_BARRIER_SHADOW_UPDATE_STUB WriteWatch_PreGrow64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB WriteWatch_PreGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB WriteWatch_PreGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB WriteWatch_PreGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB WriteWatch_PreGrow64 + WRITE_BARRIER_RETURN_STUB WriteWatch_PreGrow64 + WRITE_BARRIER_END JIT_WriteBarrier_WriteWatch_PreGrow64 + + + ;----------------------------------------------------------------------------- + ; void JIT_WriteBarrier_WriteWatch_PostGrow64(Object** dst, Object* src) + ; + ; Skipped functionality: + ; No region checks + ; + WRITE_BARRIER_ENTRY JIT_WriteBarrier_WriteWatch_PostGrow64 + WRITE_BARRIER_ENTRY_STUB WriteWatch_PostGrow64 + WRITE_BARRIER_SHADOW_UPDATE_STUB WriteWatch_PostGrow64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB WriteWatch_PostGrow64 + WRITE_BARRIER_RETURN_STUB WriteWatch_PostGrow64 + WRITE_BARRIER_END JIT_WriteBarrier_WriteWatch_PostGrow64 + + + ;----------------------------------------------------------------------------- + ; void JIT_WriteBarrier_WriteWatch_SVR64(Object** dst, Object* src) + ; + ; SVR GC has multiple heaps, so it cannot provide one single ephemeral region to bounds check + ; against, so we just skip the bounds checking all together and do our card table update unconditionally. + ; + ; Skipped functionality: + ; Does not check wbs_ephemeral_high or wbs_ephemeral_low + ; No region checks + ; + WRITE_BARRIER_ENTRY JIT_WriteBarrier_WriteWatch_SVR64 + WRITE_BARRIER_ENTRY_STUB WriteWatch_SVR64 + WRITE_BARRIER_SHADOW_UPDATE_STUB WriteWatch_SVR64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB WriteWatch_SVR64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB WriteWatch_SVR64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB WriteWatch_SVR64 + WRITE_BARRIER_RETURN_STUB WriteWatch_SVR64 + WRITE_BARRIER_END JIT_WriteBarrier_WriteWatch_SVR64 + + + ;----------------------------------------------------------------------------- + ; void JIT_WriteBarrier_WriteWatch_Byte_Region64(Object** dst, Object* src) + ; + ; Skipped functionality: + ; Bitwise updates for region checks + ; + WRITE_BARRIER_ENTRY JIT_WriteBarrier_WriteWatch_Byte_Region64 + WRITE_BARRIER_ENTRY_STUB WriteWatch_Byte_Region64 + WRITE_BARRIER_SHADOW_UPDATE_STUB WriteWatch_Byte_Region64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB WriteWatch_Byte_Region64 + WRITE_BARRIER_REGION_CHECK_STUB WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB WriteWatch_Byte_Region64 + WRITE_BARRIER_RETURN_STUB WriteWatch_Byte_Region64 + WRITE_BARRIER_END JIT_WriteBarrier_WriteWatch_Byte_Region64 + + + ;----------------------------------------------------------------------------- + ; void JIT_WriteBarrier_WriteWatch_Bit_Region64(Object** dst, Object* src) + ; + ; Skipped functionality: + ; Does not call check card table stub + ; + WRITE_BARRIER_ENTRY JIT_WriteBarrier_WriteWatch_Bit_Region64 + WRITE_BARRIER_ENTRY_STUB WriteWatch_Bit_Region64 + WRITE_BARRIER_SHADOW_UPDATE_STUB WriteWatch_Bit_Region64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB WriteWatch_Bit_Region64 + WRITE_BARRIER_REGION_CHECK_STUB WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB WriteWatch_Bit_Region64 + WRITE_BARRIER_RETURN_STUB WriteWatch_Bit_Region64 + WRITE_BARRIER_END JIT_WriteBarrier_WriteWatch_Bit_Region64 -; ------------------------------------------------------------------ -; End of the writeable code region - LEAF_ENTRY JIT_PatchedCodeLast - ret lr - LEAF_END ; Must be at very end of file END diff --git a/src/coreclr/vm/arm64/patchedcodeconstants.h b/src/coreclr/vm/arm64/patchedcodeconstants.h new file mode 100644 index 00000000000000..1b4a32d210bd19 --- /dev/null +++ b/src/coreclr/vm/arm64/patchedcodeconstants.h @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// =========================================================================== +// File: patchedcodeconstants.h +// =========================================================================== + +#ifndef PATCHEDCODECONSTANTS_H +#define PATCHEDCODECONSTANTS_H + +// These are fixed constants becuase MacOS doesn't allow label arithmetic in +// LDR instructions. Asserts in writebarriermanager CALC_TABLE_LOCATION ensure +// the values are correct. + +#define JIT_WriteBarrier_Size 0x3a0 + +#ifdef TARGET_WINDOWS +#define JIT_WriteBarrier_Table_Offset (0x30 + JIT_WriteBarrier_Size) +#else +#define JIT_WriteBarrier_Table_Offset (0x2c + JIT_WriteBarrier_Size) +#endif + +#define JIT_WriteBarrier_Offset_CardTable (0x0 + JIT_WriteBarrier_Table_Offset) +#define JIT_WriteBarrier_Offset_CardBundleTable (0x8 + JIT_WriteBarrier_Table_Offset) +#define JIT_WriteBarrier_Offset_WriteWatchTable (0x10 + JIT_WriteBarrier_Table_Offset) +#define JIT_WriteBarrier_Offset_Lower (0x18 + JIT_WriteBarrier_Table_Offset) +#define JIT_WriteBarrier_Offset_Upper (0x20 + JIT_WriteBarrier_Table_Offset) +#define JIT_WriteBarrier_Offset_LowestAddress (0x28 + JIT_WriteBarrier_Table_Offset) +#define JIT_WriteBarrier_Offset_HighestAddress (0x30 + JIT_WriteBarrier_Table_Offset) +#define JIT_WriteBarrier_Offset_RegionToGeneration (0x38 + JIT_WriteBarrier_Table_Offset) +#define JIT_WriteBarrier_Offset_RegionShr (0x40 + JIT_WriteBarrier_Table_Offset) +#define JIT_WriteBarrier_Offset_GCShadow (0x48 + JIT_WriteBarrier_Table_Offset) +#define JIT_WriteBarrier_Offset_GCShadowEnd (0x50 + JIT_WriteBarrier_Table_Offset) + +#endif // PATCHEDCODECONSTANTS_H \ No newline at end of file diff --git a/src/coreclr/vm/arm64/stubs.cpp b/src/coreclr/vm/arm64/stubs.cpp index a02b97135f7466..0e20aed6751b38 100644 --- a/src/coreclr/vm/arm64/stubs.cpp +++ b/src/coreclr/vm/arm64/stubs.cpp @@ -13,7 +13,7 @@ #include "virtualcallstub.h" #include "jitinterface.h" #include "ecall.h" - +#include "writebarriermanager.h" #ifndef DACCESS_COMPILE //----------------------------------------------------------------------- @@ -835,23 +835,6 @@ void emitCOMStubCall (ComCallMethodDesc *pCOMMethodRX, ComCallMethodDesc *pCOMMe #endif // FEATURE_COMINTEROP #if !defined(DACCESS_COMPILE) -EXTERN_C void JIT_UpdateWriteBarrierState(bool skipEphemeralCheck, size_t writeableOffset); - -extern "C" void STDCALL JIT_PatchedCodeStart(); -extern "C" void STDCALL JIT_PatchedCodeLast(); - -static void UpdateWriteBarrierState(bool skipEphemeralCheck) -{ - BYTE *writeBarrierCodeStart = GetWriteBarrierCodeLocation((void*)JIT_PatchedCodeStart); - BYTE *writeBarrierCodeStartRW = writeBarrierCodeStart; - ExecutableWriterHolderNoLog writeBarrierWriterHolder; - if (IsWriteBarrierCopyEnabled()) - { - writeBarrierWriterHolder.AssignExecutableWriterHolder(writeBarrierCodeStart, (BYTE*)JIT_PatchedCodeLast - (BYTE*)JIT_PatchedCodeStart); - writeBarrierCodeStartRW = writeBarrierWriterHolder.GetRW(); - } - JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap(), writeBarrierCodeStartRW - writeBarrierCodeStart); -} void InitJITHelpers1() { @@ -859,6 +842,8 @@ void InitJITHelpers1() _ASSERTE(g_SystemInfo.dwNumberOfProcessors != 0); + g_WriteBarrierManager.Initialize(); + // Allocation helpers, faster but non-logging if (!((TrackAllocationsEnabled()) || (LoggingOn(LF_GCALLOC, LL_INFO10)) @@ -878,13 +863,8 @@ void InitJITHelpers1() ECall::DynamicallyAssignFCallImpl(GetEEFuncEntryPoint(AllocateString_MP_FastPortable), ECall::FastAllocateString); } } - - UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap()); } - -#else -void UpdateWriteBarrierState(bool) {} #endif // !defined(DACCESS_COMPILE) #ifdef TARGET_WINDOWS @@ -1005,37 +985,6 @@ LONG CLRNoCatchHandler(EXCEPTION_POINTERS* pExceptionInfo, PVOID pv) return EXCEPTION_CONTINUE_SEARCH; } -void FlushWriteBarrierInstructionCache() -{ - // this wouldn't be called in arm64, just to comply with gchelpers.h -} - -int StompWriteBarrierEphemeral(bool isRuntimeSuspended) -{ - UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap()); - return SWB_PASS; -} - -int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck) -{ - UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap()); - return SWB_PASS; -} - -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -int SwitchToWriteWatchBarrier(bool isRuntimeSuspended) -{ - UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap()); - return SWB_PASS; -} - -int SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended) -{ - UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap()); - return SWB_PASS; -} -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - #ifdef DACCESS_COMPILE BOOL GetAnyThunkTarget (T_CONTEXT *pctx, TADDR *pTarget, TADDR *pTargetMethodDesc) { diff --git a/src/coreclr/vm/gcenv.ee.cpp b/src/coreclr/vm/gcenv.ee.cpp index 4738b0d7aeb8fa..cfb265e18c5a92 100644 --- a/src/coreclr/vm/gcenv.ee.cpp +++ b/src/coreclr/vm/gcenv.ee.cpp @@ -1053,6 +1053,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) ThreadSuspend::RestartEE(FALSE, TRUE); } return; // unlike other branches we have already done cleanup so bailing out here + case WriteBarrierOp::StompEphemeral: assert(args->is_runtime_suspended && "the runtime must be suspended here!"); // StompEphemeral requires a new ephemeral low and a new ephemeral high @@ -1063,8 +1064,16 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) g_region_to_generation_table = args->region_to_generation_table; g_region_shr = args->region_shr; g_region_use_bitwise_write_barrier = args->region_use_bitwise_write_barrier; +#if defined(HOST_ARM64) + // Only allow bitwise write barriers if LSE atomics are present + if (!g_arm64_atomics_present) + { + g_region_use_bitwise_write_barrier = false; + } +#endif stompWBCompleteActions |= ::StompWriteBarrierEphemeral(args->is_runtime_suspended); break; + case WriteBarrierOp::Initialize: assert(args->is_runtime_suspended && "the runtime must be suspended here!"); // This operation should only be invoked once, upon initialization. @@ -1090,16 +1099,24 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) g_region_to_generation_table = args->region_to_generation_table; g_region_shr = args->region_shr; g_region_use_bitwise_write_barrier = args->region_use_bitwise_write_barrier; + g_ephemeral_low = args->ephemeral_low; + g_ephemeral_high = args->ephemeral_high; +#if defined(HOST_ARM64) + // Only allow bitwise write barriers if LSE atomics are present + if (!g_arm64_atomics_present) + { + g_region_use_bitwise_write_barrier = false; + } +#endif stompWBCompleteActions |= ::StompWriteBarrierResize(true, false); // StompWriteBarrierResize does not necessarily bash g_ephemeral_low - // usages, so we must do so here. This is particularly true on x86, + // usages, so we must do so here. This is particularly true on x86/Arm64, // where StompWriteBarrierResize will not bash g_ephemeral_low when // called with the parameters (true, false), as it is above. - g_ephemeral_low = args->ephemeral_low; - g_ephemeral_high = args->ephemeral_high; stompWBCompleteActions |= ::StompWriteBarrierEphemeral(true); break; + case WriteBarrierOp::SwitchToWriteWatch: #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP assert(args->is_runtime_suspended && "the runtime must be suspended here!"); @@ -1111,6 +1128,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) assert(!"should never be called without FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP"); #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP break; + case WriteBarrierOp::SwitchToNonWriteWatch: #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP assert(args->is_runtime_suspended && "the runtime must be suspended here!"); @@ -1121,6 +1139,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) assert(!"should never be called without FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP"); #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP break; + default: assert(!"unknown WriteBarrierOp enum"); } diff --git a/src/coreclr/vm/jitinterface.h b/src/coreclr/vm/jitinterface.h index 7f1835e458a53a..9506d2c45adc13 100644 --- a/src/coreclr/vm/jitinterface.h +++ b/src/coreclr/vm/jitinterface.h @@ -242,70 +242,6 @@ extern "C" FCDECL2(VOID, JIT_WriteBarrier_Callable, Object **dst, Object *ref); #define WriteBarrier_Helper JIT_WriteBarrier_Callable -#ifdef TARGET_AMD64 - - -class WriteBarrierManager -{ -public: - enum WriteBarrierType - { - WRITE_BARRIER_UNINITIALIZED, - WRITE_BARRIER_PREGROW64, - WRITE_BARRIER_POSTGROW64, -#ifdef FEATURE_SVR_GC - WRITE_BARRIER_SVR64, -#endif // FEATURE_SVR_GC - WRITE_BARRIER_BYTE_REGIONS64, - WRITE_BARRIER_BIT_REGIONS64, -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - WRITE_BARRIER_WRITE_WATCH_PREGROW64, - WRITE_BARRIER_WRITE_WATCH_POSTGROW64, -#ifdef FEATURE_SVR_GC - WRITE_BARRIER_WRITE_WATCH_SVR64, -#endif // FEATURE_SVR_GC - WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64, - WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64, -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - WRITE_BARRIER_BUFFER - }; - - WriteBarrierManager(); - void Initialize(); - - int UpdateEphemeralBounds(bool isRuntimeSuspended); - int UpdateWriteWatchAndCardTableLocations(bool isRuntimeSuspended, bool bReqUpperBoundsCheck); - -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - int SwitchToWriteWatchBarrier(bool isRuntimeSuspended); - int SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended); -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - size_t GetCurrentWriteBarrierSize(); - -protected: - size_t GetSpecificWriteBarrierSize(WriteBarrierType writeBarrier); - PBYTE CalculatePatchLocation(LPVOID base, LPVOID label, int offset); - PCODE GetCurrentWriteBarrierCode(); - int ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, bool isRuntimeSuspended); - bool NeedDifferentWriteBarrier(bool bReqUpperBoundsCheck, bool bUseBitwiseWriteBarrier, WriteBarrierType* pNewWriteBarrierType); - -private: - void Validate(); - - WriteBarrierType m_currentWriteBarrier; - - PBYTE m_pWriteWatchTableImmediate; // PREGROW | POSTGROW | SVR | WRITE_WATCH | REGION - PBYTE m_pLowerBoundImmediate; // PREGROW | POSTGROW | | WRITE_WATCH | REGION - PBYTE m_pCardTableImmediate; // PREGROW | POSTGROW | SVR | WRITE_WATCH | REGION - PBYTE m_pCardBundleTableImmediate; // PREGROW | POSTGROW | SVR | WRITE_WATCH | REGION - PBYTE m_pUpperBoundImmediate; // | POSTGROW | | WRITE_WATCH | REGION - PBYTE m_pRegionToGenTableImmediate; // | | | WRITE_WATCH | REGION - PBYTE m_pRegionShrDest; // | | | WRITE_WATCH | REGION - PBYTE m_pRegionShrSrc; // | | | WRITE_WATCH | RETION -}; - -#endif // TARGET_AMD64 - EXTERN_C FCDECL2_VV(INT64, JIT_LMul, INT64 val1, INT64 val2); #ifndef HOST_64BIT diff --git a/src/coreclr/vm/jitinterfacegen.cpp b/src/coreclr/vm/jitinterfacegen.cpp index 1d55e308402fe3..5dd08b3271d95d 100644 --- a/src/coreclr/vm/jitinterfacegen.cpp +++ b/src/coreclr/vm/jitinterfacegen.cpp @@ -17,6 +17,7 @@ #include "comdelegate.h" #include "field.h" #include "ecall.h" +#include "writebarriermanager.h" #ifdef HOST_64BIT @@ -32,10 +33,6 @@ EXTERN_C Object* AllocateStringFastUP (CLR_I4 cch); EXTERN_C Object* JIT_NewArr1OBJ_UP (CORINFO_CLASS_HANDLE arrayMT, INT_PTR size); EXTERN_C Object* JIT_NewArr1VC_UP (CORINFO_CLASS_HANDLE arrayMT, INT_PTR size); -#ifdef TARGET_AMD64 -extern WriteBarrierManager g_WriteBarrierManager; -#endif // TARGET_AMD64 - #endif // HOST_64BIT /*********************************************************************/ diff --git a/src/coreclr/vm/amd64/jitinterfaceamd64.cpp b/src/coreclr/vm/writebarriermanager.cpp similarity index 87% rename from src/coreclr/vm/amd64/jitinterfaceamd64.cpp rename to src/coreclr/vm/writebarriermanager.cpp index f9cd2a968a02a7..79e4b4bd0997d0 100644 --- a/src/coreclr/vm/amd64/jitinterfaceamd64.cpp +++ b/src/coreclr/vm/writebarriermanager.cpp @@ -2,19 +2,21 @@ // The .NET Foundation licenses this file to you under the MIT license. // =========================================================================== -// File: JITinterfaceCpu.CPP +// File: writebariermanager.cpp // =========================================================================== -// This contains JITinterface routines that are specific to the -// AMD64 platform. They are modeled after the X86 specific routines -// found in JITinterfaceX86.cpp or JIThelp.asm - +// This contains JITinterface routines for managing which write barrier function +// is currently in use, and patching all related constants. #include "common.h" #include "jitinterface.h" #include "eeconfig.h" #include "excep.h" #include "threadsuspend.h" +#include "writebarriermanager.h" +#if !defined(WRITE_BARRIER_VARS_INLINE) +#include "patchedcodeconstants.h" +#endif extern uint8_t* g_ephemeral_low; extern uint8_t* g_ephemeral_high; @@ -22,35 +24,58 @@ extern uint32_t* g_card_table; extern uint32_t* g_card_bundle_table; // Patch Labels for the various write barriers -EXTERN_C void JIT_WriteBarrier_End(); +EXTERN_C void JIT_WriteBarrier_End(); EXTERN_C void JIT_WriteBarrier_PreGrow64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_PreGrow64_End(); +EXTERN_C void JIT_WriteBarrier_PostGrow64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_PostGrow64_End(); +#ifdef FEATURE_SVR_GC +EXTERN_C void JIT_WriteBarrier_SVR64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_SVR64_End(); +#endif // FEATURE_SVR_GC +EXTERN_C void JIT_WriteBarrier_Byte_Region64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_Byte_Region64_End(); +EXTERN_C void JIT_WriteBarrier_Bit_Region64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_Bit_Region64_End(); +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP +EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_End(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_End(); +#ifdef FEATURE_SVR_GC +EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_End(); +#endif // FEATURE_SVR_GC +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_End(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_End(); +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + + +#if defined(WRITE_BARRIER_VARS_INLINE) + EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_Lower(); EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_CardTable(); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_CardBundleTable(); #endif -EXTERN_C void JIT_WriteBarrier_PreGrow64_End(); -EXTERN_C void JIT_WriteBarrier_PostGrow64(Object **dst, Object *ref); EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_Lower(); EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_Upper(); EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_CardTable(); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_CardBundleTable(); #endif -EXTERN_C void JIT_WriteBarrier_PostGrow64_End(); #ifdef FEATURE_SVR_GC -EXTERN_C void JIT_WriteBarrier_SVR64(Object **dst, Object *ref); EXTERN_C void JIT_WriteBarrier_SVR64_PatchLabel_CardTable(); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES EXTERN_C void JIT_WriteBarrier_SVR64_PatchLabel_CardBundleTable(); #endif -EXTERN_C void JIT_WriteBarrier_SVR64_End(); #endif // FEATURE_SVR_GC -EXTERN_C void JIT_WriteBarrier_Byte_Region64(Object **dst, Object *ref); EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_RegionToGeneration(); EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_RegionShrDest(); EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_Lower(); @@ -60,9 +85,7 @@ EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_CardTable(); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_CardBundleTable(); #endif -EXTERN_C void JIT_WriteBarrier_Byte_Region64_End(); -EXTERN_C void JIT_WriteBarrier_Bit_Region64(Object **dst, Object *ref); EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_RegionToGeneration(); EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_RegionShrDest(); EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_Lower(); @@ -72,20 +95,15 @@ EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_CardTable(); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_CardBundleTable(); #endif -EXTERN_C void JIT_WriteBarrier_Bit_Region64_End(); - #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64(Object **dst, Object *ref); EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_WriteWatchTable(); EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_Lower(); EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_CardTable(); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_CardBundleTable(); #endif -EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_End(); -EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64(Object **dst, Object *ref); EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_WriteWatchTable(); EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_Lower(); EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_Upper(); @@ -93,19 +111,15 @@ EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_CardTable(); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_CardBundleTable(); #endif -EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_End(); #ifdef FEATURE_SVR_GC -EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64(Object **dst, Object *ref); EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_WriteWatchTable(); EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_CardTable(); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_CardBundleTable(); #endif -EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_End(); #endif // FEATURE_SVR_GC -EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64(Object **dst, Object *ref); EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_WriteWatchTable(); EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_RegionToGeneration(); EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_RegionShrDest(); @@ -116,9 +130,7 @@ EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_CardTable(); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_CardBundleTable(); #endif -EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_End(); -EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64(Object **dst, Object *ref); EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_WriteWatchTable(); EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_RegionToGeneration(); EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_RegionShrDest(); @@ -129,16 +141,33 @@ EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_CardTable(); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_CardBundleTable(); #endif -EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_End(); #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -WriteBarrierManager g_WriteBarrierManager; +#else // WRITE_BARRIER_VARS_INLINE -// Use this somewhat hokey macro to concatenate the function start with the patch -// label. This allows the code below to look relatively nice, but relies on the -// naming convention which we have established for these helpers. -#define CALC_PATCH_LOCATION(func,label,offset) CalculatePatchLocation((PVOID)func, (PVOID)func##_##label, offset) +EXTERN_C void JIT_WriteBarrier_Table(); +EXTERN_C void JIT_WriteBarrier_Table_End(); +EXTERN_C void JIT_WriteBarrier_Patch_Label_WriteWatchTable(); +EXTERN_C void JIT_WriteBarrier_Patch_Label_RegionToGeneration(); +EXTERN_C void JIT_WriteBarrier_Patch_Label_RegionShr(); +EXTERN_C void JIT_WriteBarrier_Patch_Label_Lower(); +EXTERN_C void JIT_WriteBarrier_Patch_Label_Upper(); +EXTERN_C void JIT_WriteBarrier_Patch_Label_CardTable(); +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +EXTERN_C void JIT_WriteBarrier_Patch_Label_CardBundleTable(); +#endif +#if defined(TARGET_ARM64) +EXTERN_C void JIT_WriteBarrier_Patch_Label_LowestAddress(); +EXTERN_C void JIT_WriteBarrier_Patch_Label_HighestAddress(); +#if defined(WRITE_BARRIER_CHECK) +EXTERN_C void JIT_WriteBarrier_Patch_Label_GCShadow(); +EXTERN_C void JIT_WriteBarrier_Patch_Label_GCShadowEnd(); +#endif // WRITE_BARRIER_CHECK +#endif // TARGET_ARM64 +#endif // WRITE_BARRIER_VARS_INLINE + +WriteBarrierManager g_WriteBarrierManager; WriteBarrierManager::WriteBarrierManager() : m_currentWriteBarrier(WRITE_BARRIER_UNINITIALIZED) @@ -146,173 +175,6 @@ WriteBarrierManager::WriteBarrierManager() : LIMITED_METHOD_CONTRACT; } -#ifndef CODECOVERAGE // Deactivate alignment validation for code coverage builds - // because the instrumentation tool will not preserve alignment - // constraints and we will fail. - -void WriteBarrierManager::Validate() -{ - CONTRACTL - { - MODE_ANY; - GC_NOTRIGGER; - NOTHROW; - } - CONTRACTL_END; - - // we have an invariant that the addresses of all the values that we update in our write barrier - // helpers must be naturally aligned, this is so that the update can happen atomically since there - // are places where these values are updated while the EE is running - // NOTE: we can't call this from the ctor since our infrastructure isn't ready for assert dialogs - - PBYTE pLowerBoundImmediate, pUpperBoundImmediate, pCardTableImmediate; - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - PBYTE pCardBundleTableImmediate; -#endif - - pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_Lower, 2); - pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardTable, 2); - - _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); -#endif - - pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Lower, 2); - pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Upper, 2); - pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); -#endif - -#ifdef FEATURE_SVR_GC - pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); -#endif // FEATURE_MANUALLY_MANAGED_CARD_BUNDLES -#endif // FEATURE_SVR_GC - - PBYTE pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_RegionToGeneration, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pRegionToGenTableImmediate) & 0x7) == 0); - - pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_Lower, 2); - pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_Upper, 2); - pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_CardTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); -#endif - - pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_RegionToGeneration, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pRegionToGenTableImmediate) & 0x7) == 0); - - pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_Lower, 2); - pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_Upper, 2); - pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_CardTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); -#endif - -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - PBYTE pWriteWatchTableImmediate; - - pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_WriteWatchTable, 2); - pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_Lower, 2); - pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardTable, 2); - - _ASSERTE_ALL_BUILDS((reinterpret_cast(pWriteWatchTableImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); -#endif - - pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_WriteWatchTable, 2); - pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Lower, 2); - pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Upper, 2); - pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardTable, 2); - - _ASSERTE_ALL_BUILDS((reinterpret_cast(pWriteWatchTableImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); -#endif - -#ifdef FEATURE_SVR_GC - pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_WriteWatchTable, 2); - pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pWriteWatchTableImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); -#endif // FEATURE_MANUALLY_MANAGED_CARD_BUNDLES -#endif // FEATURE_SVR_GC - - pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_RegionToGeneration, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pRegionToGenTableImmediate) & 0x7) == 0); - - pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_Lower, 2); - pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_Upper, 2); - pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_CardTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); -#endif - - pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_RegionToGeneration, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pRegionToGenTableImmediate) & 0x7) == 0); - - pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_Lower, 2); - pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_Upper, 2); - pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_CardTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); -#endif - -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -} - -#endif // CODECOVERAGE - - PCODE WriteBarrierManager::GetCurrentWriteBarrierCode() { LIMITED_METHOD_CONTRACT; @@ -386,7 +248,11 @@ size_t WriteBarrierManager::GetSpecificWriteBarrierSize(WriteBarrierType writeBa return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_WriteWatch_Bit_Region64); #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP case WRITE_BARRIER_BUFFER: +#if defined(WRITE_BARRIER_VARS_INLINE) return MARKED_FUNCTION_SIZE(JIT_WriteBarrier); +#else + return (size_t)((LPBYTE)GetEEFuncEntryPoint(JIT_WriteBarrier_Table_End) - (LPBYTE)GetEEFuncEntryPoint(JIT_WriteBarrier)); +#endif default: UNREACHABLE_MSG("unexpected m_currentWriteBarrier!"); }; @@ -398,14 +264,6 @@ size_t WriteBarrierManager::GetCurrentWriteBarrierSize() return GetSpecificWriteBarrierSize(m_currentWriteBarrier); } -PBYTE WriteBarrierManager::CalculatePatchLocation(LPVOID base, LPVOID label, int offset) -{ - // the label should always come after the entrypoint for this funtion - _ASSERTE_ALL_BUILDS((LPBYTE)label > (LPBYTE)base); - - return (GetWriteBarrierCodeLocation((void*)JIT_WriteBarrier) + ((LPBYTE)GetEEFuncEntryPoint(label) - (LPBYTE)GetEEFuncEntryPoint(base) + offset)); -} - int WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, bool isRuntimeSuspended) { @@ -428,222 +286,16 @@ int WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, stompWBCompleteActions |= SWB_ICACHE_FLUSH; } - switch (newWriteBarrier) - { - case WRITE_BARRIER_PREGROW64: - { - m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_Lower, 2); - m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardTable, 2); - - // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#if defined(WRITE_BARRIER_VARS_INLINE) + UpdatePatchLocations(newWriteBarrier); #endif - break; - } - case WRITE_BARRIER_POSTGROW64: - { - m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Lower, 2); - m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Upper, 2); - m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardTable, 2); - - // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); -#endif - break; - } - -#ifdef FEATURE_SVR_GC - case WRITE_BARRIER_SVR64: - { - m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardTable, 2); - - // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); -#endif - break; - } -#endif // FEATURE_SVR_GC - - case WRITE_BARRIER_BYTE_REGIONS64: - m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_RegionToGeneration, 2); - m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_RegionShrDest, 3); - m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_RegionShrSrc, 3); - m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_Lower, 2); - m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_Upper, 2); - m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_CardTable, 2); - - // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pRegionToGenTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); - _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrDest); - _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrSrc); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); -#endif - break; - - case WRITE_BARRIER_BIT_REGIONS64: - m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_RegionToGeneration, 2); - m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_RegionShrDest, 3); - m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_RegionShrSrc, 3); - m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_Lower, 2); - m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_Upper, 2); - m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_CardTable, 2); - - // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pRegionToGenTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); - _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrDest); - _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrSrc); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); -#endif - break; - -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - case WRITE_BARRIER_WRITE_WATCH_PREGROW64: - { - m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_WriteWatchTable, 2); - m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_Lower, 2); - m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardTable, 2); - - // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); -#endif - break; - } - - case WRITE_BARRIER_WRITE_WATCH_POSTGROW64: - { - m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_WriteWatchTable, 2); - m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Lower, 2); - m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Upper, 2); - m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardTable, 2); - - // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); -#endif - break; - } - -#ifdef FEATURE_SVR_GC - case WRITE_BARRIER_WRITE_WATCH_SVR64: - { - m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_WriteWatchTable, 2); - m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardTable, 2); - - // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); -#endif - break; - } -#endif // FEATURE_SVR_GC - - case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: - m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_WriteWatchTable, 2); - m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_RegionToGeneration, 2); - m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_RegionShrDest, 3); - m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_RegionShrSrc, 3); - m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_Lower, 2); - m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_Upper, 2); - m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_CardTable, 2); - - // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pRegionToGenTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); - _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrDest); - _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrSrc); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); -#endif - break; - - case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: - m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_WriteWatchTable, 2); - m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_RegionToGeneration, 2); - m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_RegionShrDest, 3); - m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_RegionShrSrc, 3); - m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_Lower, 2); - m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_Upper, 2); - m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_CardTable, 2); - - // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pRegionToGenTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); - _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrDest); - _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrSrc); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); -#endif - break; - - -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - - default: - UNREACHABLE_MSG("unexpected write barrier type!"); - } - - stompWBCompleteActions |= UpdateEphemeralBounds(true); - stompWBCompleteActions |= UpdateWriteWatchAndCardTableLocations(true, false); + stompWBCompleteActions |= UpdateEphemeralBounds(true); + stompWBCompleteActions |= UpdateWriteWatchAndCardTableLocations(true, false); return stompWBCompleteActions; } -#undef CALC_PATCH_LOCATION - void WriteBarrierManager::Initialize() { CONTRACTL @@ -654,7 +306,6 @@ void WriteBarrierManager::Initialize() } CONTRACTL_END; - // Ensure that the generic JIT_WriteBarrier function buffer is large enough to hold any of the more specific // write barrier implementations. size_t cbWriteBarrierBuffer = GetSpecificWriteBarrierSize(WRITE_BARRIER_BUFFER); @@ -676,11 +327,50 @@ void WriteBarrierManager::Initialize() _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64)); #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -#if !defined(CODECOVERAGE) + +#if !defined(WRITE_BARRIER_VARS_INLINE) + + #define CALC_TABLE_LOCATION(var, offset) \ + assert(JIT_WriteBarrier_Offset_##offset == (PBYTE)JIT_WriteBarrier_Patch_Label_##offset - (PBYTE)JIT_WriteBarrier); \ + var = ((PBYTE)GetWriteBarrierCodeLocation((void*)JIT_WriteBarrier) + JIT_WriteBarrier_Offset_##offset); + + CALC_TABLE_LOCATION(m_pWriteWatchTableImmediate, WriteWatchTable); + CALC_TABLE_LOCATION(m_pRegionToGenTableImmediate, RegionToGeneration); + CALC_TABLE_LOCATION(m_pRegionShrDest, RegionShr); + CALC_TABLE_LOCATION(m_pLowerBoundImmediate, Lower); + CALC_TABLE_LOCATION(m_pUpperBoundImmediate, Upper); + CALC_TABLE_LOCATION(m_pCardTableImmediate, CardTable); +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + CALC_TABLE_LOCATION(m_pCardBundleTableImmediate, CardBundleTable); +#endif + +#if defined(TARGET_ARM64) + CALC_TABLE_LOCATION(m_lowestAddress, LowestAddress); + CALC_TABLE_LOCATION(m_highestAddress, HighestAddress); +#if defined(WRITE_BARRIER_CHECK) + CALC_TABLE_LOCATION(m_pGCShadow, GCShadow); + CALC_TABLE_LOCATION(m_pGCShadowEnd, GCShadowEnd); +#endif // WRITE_BARRIER_CHECK +#endif // TARGET_AMD64 + +#endif // !WRITE_BARRIER_VARS_INLINE + +#if !defined(CODECOVERAGE) && defined(WRITE_BARRIER_VARS_INLINE) Validate(); #endif } +template int updateVariable(PBYTE loc, T value) +{ + if (*(T*)loc != value) + { + ExecutableWriterHolder varWriterHolder((T*)loc, sizeof(T)); + *varWriterHolder.GetRW() = value; + return SWB_ICACHE_FLUSH; + } + return SWB_PASS; +} + bool WriteBarrierManager::NeedDifferentWriteBarrier(bool bReqUpperBoundsCheck, bool bUseBitwiseWriteBarrier, WriteBarrierType* pNewWriteBarrierType) { // Init code for the JIT_WriteBarrier assembly routine. Since it will be bashed everytime the GC Heap @@ -778,6 +468,8 @@ int WriteBarrierManager::UpdateEphemeralBounds(bool isRuntimeSuspended) return stompWBCompleteActions; #endif +#if defined(WRITE_BARRIER_VARS_INLINE) + switch (m_currentWriteBarrier) { case WRITE_BARRIER_POSTGROW64: @@ -788,45 +480,44 @@ int WriteBarrierManager::UpdateEphemeralBounds(bool isRuntimeSuspended) case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - { - // Change immediate if different from new g_ephermeral_high. - if (*(UINT64*)m_pUpperBoundImmediate != (size_t)g_ephemeral_high) - { - ExecutableWriterHolder upperBoundWriterHolder((UINT64*)m_pUpperBoundImmediate, sizeof(UINT64)); - *upperBoundWriterHolder.GetRW() = (size_t)g_ephemeral_high; - stompWBCompleteActions |= SWB_ICACHE_FLUSH; - } - } + stompWBCompleteActions |= updateVariable(m_pUpperBoundImmediate, (size_t)g_ephemeral_high); FALLTHROUGH; + case WRITE_BARRIER_PREGROW64: #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP case WRITE_BARRIER_WRITE_WATCH_PREGROW64: #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - { - // Change immediate if different from new g_ephermeral_low. - if (*(UINT64*)m_pLowerBoundImmediate != (size_t)g_ephemeral_low) - { - ExecutableWriterHolder lowerBoundImmediateWriterHolder((UINT64*)m_pLowerBoundImmediate, sizeof(UINT64)); - *lowerBoundImmediateWriterHolder.GetRW() = (size_t)g_ephemeral_low; - stompWBCompleteActions |= SWB_ICACHE_FLUSH; - } + stompWBCompleteActions |= updateVariable(m_pLowerBoundImmediate, (size_t)g_ephemeral_low); break; - } #ifdef FEATURE_SVR_GC case WRITE_BARRIER_SVR64: #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP case WRITE_BARRIER_WRITE_WATCH_SVR64: #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - { break; - } #endif // FEATURE_SVR_GC default: UNREACHABLE_MSG("unexpected m_currentWriteBarrier in UpdateEphemeralBounds"); } +#else + + stompWBCompleteActions |= updateVariable(m_pUpperBoundImmediate, (size_t)g_ephemeral_high); + stompWBCompleteActions |= updateVariable(m_pLowerBoundImmediate, (size_t)g_ephemeral_low); +#endif //WRITE_BARRIER_VARS_INLINE + + +#if defined(TARGET_ARM64) + stompWBCompleteActions |= updateVariable(m_lowestAddress, (size_t)g_lowest_address); + stompWBCompleteActions |= updateVariable(m_highestAddress, (size_t)g_highest_address); +#if defined(WRITE_BARRIER_CHECK) + stompWBCompleteActions |= updateVariable(m_pGCShadow, (size_t)g_GCShadow); + stompWBCompleteActions |= updateVariable(m_pGCShadowEnd, (size_t)g_GCShadowEnd); +#endif // WRITE_BARRIER_CHECK +#endif // TARGET_AMD64 + return stompWBCompleteActions; } @@ -850,6 +541,7 @@ int WriteBarrierManager::UpdateWriteWatchAndCardTableLocations(bool isRuntimeSus #endif #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP +#if defined(WRITE_BARRIER_VARS_INLINE) switch (m_currentWriteBarrier) { case WRITE_BARRIER_WRITE_WATCH_PREGROW64: @@ -859,65 +551,52 @@ int WriteBarrierManager::UpdateWriteWatchAndCardTableLocations(bool isRuntimeSus #endif // FEATURE_SVR_GC case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: - if (*(UINT64*)m_pWriteWatchTableImmediate != (size_t)g_sw_ww_table) - { - ExecutableWriterHolder writeWatchTableImmediateWriterHolder((UINT64*)m_pWriteWatchTableImmediate, sizeof(UINT64)); - *writeWatchTableImmediateWriterHolder.GetRW() = (size_t)g_sw_ww_table; - stompWBCompleteActions |= SWB_ICACHE_FLUSH; - } + stompWBCompleteActions |= updateVariable(m_pWriteWatchTableImmediate, (size_t)g_sw_ww_table); break; default: - break; // clang seems to require all enum values to be covered for some reason + break; } +#else + stompWBCompleteActions |= updateVariable(m_pWriteWatchTableImmediate, (size_t)g_sw_ww_table); +#endif #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + +#if defined(WRITE_BARRIER_VARS_INLINE) switch (m_currentWriteBarrier) { case WRITE_BARRIER_BYTE_REGIONS64: case WRITE_BARRIER_BIT_REGIONS64: case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: - if (*(UINT64*)m_pRegionToGenTableImmediate != (size_t)g_region_to_generation_table) - { - ExecutableWriterHolder writeWatchTableImmediateWriterHolder((UINT64*)m_pRegionToGenTableImmediate, sizeof(UINT64)); - *writeWatchTableImmediateWriterHolder.GetRW() = (size_t)g_region_to_generation_table; - stompWBCompleteActions |= SWB_ICACHE_FLUSH; - } - if (*m_pRegionShrDest != g_region_shr) - { - ExecutableWriterHolder writeWatchTableImmediateWriterHolder(m_pRegionShrDest, sizeof(UINT8)); - *writeWatchTableImmediateWriterHolder.GetRW() = g_region_shr; - stompWBCompleteActions |= SWB_ICACHE_FLUSH; - } - if (*m_pRegionShrSrc != g_region_shr) - { - ExecutableWriterHolder writeWatchTableImmediateWriterHolder(m_pRegionShrSrc, sizeof(UINT8)); - *writeWatchTableImmediateWriterHolder.GetRW() = g_region_shr; - stompWBCompleteActions |= SWB_ICACHE_FLUSH; - } + stompWBCompleteActions |= updateVariable(m_pRegionToGenTableImmediate, (size_t)g_region_to_generation_table); + stompWBCompleteActions |= updateVariable(m_pRegionShrDest, (size_t)g_region_shr); + stompWBCompleteActions |= updateVariable(m_pRegionShrSrc, (size_t)g_region_shr); break; default: - break; // clang seems to require all enum values to be covered for some reason - } - - if (*(UINT64*)m_pCardTableImmediate != (size_t)g_card_table) - { - ExecutableWriterHolder cardTableImmediateWriterHolder((UINT64*)m_pCardTableImmediate, sizeof(UINT64)); - *cardTableImmediateWriterHolder.GetRW() = (size_t)g_card_table; - stompWBCompleteActions |= SWB_ICACHE_FLUSH; + break; } +#else + stompWBCompleteActions |= updateVariable(m_pRegionToGenTableImmediate, (size_t)g_region_to_generation_table); + stompWBCompleteActions |= updateVariable(m_pRegionShrDest, g_region_shr); +#endif //WRITE_BARRIER_VARS_INLINE + stompWBCompleteActions |= updateVariable(m_pCardTableImmediate, (size_t)g_card_table); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - if (*(UINT64*)m_pCardBundleTableImmediate != (size_t)g_card_bundle_table) - { - ExecutableWriterHolder cardBundleTableImmediateWriterHolder((UINT64*)m_pCardBundleTableImmediate, sizeof(UINT64)); - *cardBundleTableImmediateWriterHolder.GetRW() = (size_t)g_card_bundle_table; - stompWBCompleteActions |= SWB_ICACHE_FLUSH; - } + stompWBCompleteActions |= updateVariable(m_pCardBundleTableImmediate, (size_t)g_card_bundle_table); #endif +#if defined(TARGET_ARM64) + stompWBCompleteActions |= updateVariable(m_lowestAddress, (size_t)g_lowest_address); + stompWBCompleteActions |= updateVariable(m_highestAddress, (size_t)g_highest_address); +#if defined(WRITE_BARRIER_CHECK) + stompWBCompleteActions |= updateVariable(m_pGCShadow, (size_t)g_GCShadow); + stompWBCompleteActions |= updateVariable(m_pGCShadowEnd, (size_t)g_GCShadowEnd); +#endif // WRITE_BARRIER_CHECK +#endif // TARGET_AMD64 + return stompWBCompleteActions; } @@ -999,7 +678,406 @@ int WriteBarrierManager::SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended) } #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -// This function bashes the super fast amd64 version of the JIT_WriteBarrier + +#if defined(WRITE_BARRIER_VARS_INLINE) + + +// Use this somewhat hokey macro to concatenate the function start with the patch +// label. This allows the code below to look relatively nice, but relies on the +// naming convention which we have established for these helpers. +#define CALC_PATCH_LOCATION(func,label,offset) CalculatePatchLocation((PVOID)func, (PVOID)func##_##label, offset) + +PBYTE WriteBarrierManager::CalculatePatchLocation(LPVOID base, LPVOID label, int inlineOffset) +{ + // the label should always come after or at the entrypoint for this funtion + _ASSERTE_ALL_BUILDS((LPBYTE)label >= (LPBYTE)base); + + BYTE* patchBase = GetWriteBarrierCodeLocation((void*)JIT_WriteBarrier); + return (patchBase + ((LPBYTE)GetEEFuncEntryPoint(label) - (LPBYTE)GetEEFuncEntryPoint(base))) + inlineOffset; +} + +// Deactivate alignment validation for code coverage builds +// because the instrumentation tool will not preserve alignment +// constraints and we will fail. +#if !defined(CODECOVERAGE) + +void WriteBarrierManager::Validate() +{ + CONTRACTL + { + MODE_ANY; + GC_NOTRIGGER; + NOTHROW; + } + CONTRACTL_END; + + // we have an invariant that the addresses of all the values that we update in our write barrier + // helpers must be naturally aligned, this is so that the update can happen atomically since there + // are places where these values are updated while the EE is running + // NOTE: we can't call this from the ctor since our infrastructure isn't ready for assert dialogs + + PBYTE pLowerBoundImmediate, pUpperBoundImmediate, pCardTableImmediate; + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + PBYTE pCardBundleTableImmediate; +#endif + + pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_Lower, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardTable, 2); + + _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif + + pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Lower, 2); + pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Upper, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif + +#ifdef FEATURE_SVR_GC + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif // FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +#endif // FEATURE_SVR_GC + + PBYTE pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_RegionToGeneration, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pRegionToGenTableImmediate) & 0x7) == 0); + + pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_Lower, 2); + pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_Upper, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_CardTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif + + pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_RegionToGeneration, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pRegionToGenTableImmediate) & 0x7) == 0); + + pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_Lower, 2); + pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_Upper, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_CardTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif + +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + PBYTE pWriteWatchTableImmediate; + + pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_WriteWatchTable, 2); + pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_Lower, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardTable, 2); + + _ASSERTE_ALL_BUILDS((reinterpret_cast(pWriteWatchTableImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif + + pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_WriteWatchTable, 2); + pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Lower, 2); + pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Upper, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardTable, 2); + + _ASSERTE_ALL_BUILDS((reinterpret_cast(pWriteWatchTableImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif + +#ifdef FEATURE_SVR_GC + pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_WriteWatchTable, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pWriteWatchTableImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif // FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +#endif // FEATURE_SVR_GC + + pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_RegionToGeneration, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pRegionToGenTableImmediate) & 0x7) == 0); + + pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_Lower, 2); + pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_Upper, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_CardTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif + + pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_RegionToGeneration, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pRegionToGenTableImmediate) & 0x7) == 0); + + pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_Lower, 2); + pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_Upper, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_CardTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif + +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP +} + +#endif // CODECOVERAGE + +void WriteBarrierManager::UpdatePatchLocations(WriteBarrierType newWriteBarrier) +{ + switch (newWriteBarrier) + { + case WRITE_BARRIER_PREGROW64: + { + m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_Lower, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + } + + case WRITE_BARRIER_POSTGROW64: + { + m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Lower, 2); + m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Upper, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + } + +#ifdef FEATURE_SVR_GC + case WRITE_BARRIER_SVR64: + { + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + } +#endif // FEATURE_SVR_GC + + case WRITE_BARRIER_BYTE_REGIONS64: + m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_RegionToGeneration, 2); + m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_RegionShrDest, 3); + m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_RegionShrSrc, 3); + m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_Lower, 2); + m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_Upper, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pRegionToGenTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrDest); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrSrc); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + + case WRITE_BARRIER_BIT_REGIONS64: + m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_RegionToGeneration, 2); + m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_RegionShrDest, 3); + m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_RegionShrSrc, 3); + m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_Lower, 2); + m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_Upper, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pRegionToGenTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrDest); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrSrc); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + case WRITE_BARRIER_WRITE_WATCH_PREGROW64: + { + m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_WriteWatchTable, 2); + m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_Lower, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + } + + case WRITE_BARRIER_WRITE_WATCH_POSTGROW64: + { + m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_WriteWatchTable, 2); + m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Lower, 2); + m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Upper, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + } + +#ifdef FEATURE_SVR_GC + case WRITE_BARRIER_WRITE_WATCH_SVR64: + { + m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_WriteWatchTable, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + } +#endif // FEATURE_SVR_GC + + case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: + m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_WriteWatchTable, 2); + m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_RegionToGeneration, 2); + m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_RegionShrDest, 3); + m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_RegionShrSrc, 3); + m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_Lower, 2); + m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_Upper, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pRegionToGenTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrDest); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrSrc); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + + case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: + m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_WriteWatchTable, 2); + m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_RegionToGeneration, 2); + m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_RegionShrDest, 3); + m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_RegionShrSrc, 3); + m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_Lower, 2); + m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_Upper, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pRegionToGenTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrDest); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrSrc); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + + +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + + default: + UNREACHABLE_MSG("unexpected write barrier type!"); + } +} + +#endif // WRITE_BARRIER_VARS_INLINE + + +// This function bashes the super fast version of the JIT_WriteBarrier // helper. It should be called by the GC whenever the ephermeral region // bounds get changed, but still remain on the top of the GC Heap. int StompWriteBarrierEphemeral(bool isRuntimeSuspended) @@ -1009,7 +1087,7 @@ int StompWriteBarrierEphemeral(bool isRuntimeSuspended) return g_WriteBarrierManager.UpdateEphemeralBounds(isRuntimeSuspended); } -// This function bashes the super fast amd64 versions of the JIT_WriteBarrier +// This function bashes the super fast versions of the JIT_WriteBarrier // helpers. It should be called by the GC whenever the ephermeral region gets moved // from being at the top of the GC Heap, and/or when the cards table gets moved. int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck) diff --git a/src/coreclr/vm/writebarriermanager.h b/src/coreclr/vm/writebarriermanager.h new file mode 100644 index 00000000000000..c8b8bc697b3f1e --- /dev/null +++ b/src/coreclr/vm/writebarriermanager.h @@ -0,0 +1,98 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// =========================================================================== +// File: writebarriermanager.h +// =========================================================================== + + +#ifndef WRITEBARRIERMANAGER_H +#define WRITEBARRIERMANAGER_H + +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + + +#if defined(TARGET_AMD64) +// Write barrier variables are inlined into the assembly code +#define WRITE_BARRIER_VARS_INLINE +// Else: Write barrier variables are in a table separate to the asm code +#endif + +class WriteBarrierManager +{ +public: + enum WriteBarrierType + { + WRITE_BARRIER_UNINITIALIZED, + WRITE_BARRIER_PREGROW64, + WRITE_BARRIER_POSTGROW64, +#ifdef FEATURE_SVR_GC + WRITE_BARRIER_SVR64, +#endif // FEATURE_SVR_GC + WRITE_BARRIER_BYTE_REGIONS64, + WRITE_BARRIER_BIT_REGIONS64, +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + WRITE_BARRIER_WRITE_WATCH_PREGROW64, + WRITE_BARRIER_WRITE_WATCH_POSTGROW64, +#ifdef FEATURE_SVR_GC + WRITE_BARRIER_WRITE_WATCH_SVR64, +#endif // FEATURE_SVR_GC + WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64, + WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64, +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + WRITE_BARRIER_BUFFER + }; + + WriteBarrierManager(); + void Initialize(); + + int UpdateEphemeralBounds(bool isRuntimeSuspended); + int UpdateWriteWatchAndCardTableLocations(bool isRuntimeSuspended, bool bReqUpperBoundsCheck); + +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + int SwitchToWriteWatchBarrier(bool isRuntimeSuspended); + int SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended); +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + size_t GetCurrentWriteBarrierSize(); + +private: + size_t GetSpecificWriteBarrierSize(WriteBarrierType writeBarrier); + PCODE GetCurrentWriteBarrierCode(); + int ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, bool isRuntimeSuspended); + bool NeedDifferentWriteBarrier(bool bReqUpperBoundsCheck, bool bUseBitwiseWriteBarrier, WriteBarrierType* pNewWriteBarrierType); + + +#if defined(WRITE_BARRIER_VARS_INLINE) + PBYTE CalculatePatchLocation(LPVOID base, LPVOID label, int offset); + void Validate(); + void UpdatePatchLocations(WriteBarrierType newWriteBarrier); +#endif // WRITE_BARRIER_VARS_INLINE + + + WriteBarrierType m_currentWriteBarrier; + + PBYTE m_pWriteWatchTableImmediate; // PREGROW | POSTGROW | SVR | WRITE_WATCH | REGION + PBYTE m_pLowerBoundImmediate; // PREGROW | POSTGROW | | WRITE_WATCH | REGION + PBYTE m_pCardTableImmediate; // PREGROW | POSTGROW | SVR | WRITE_WATCH | REGION + PBYTE m_pCardBundleTableImmediate; // PREGROW | POSTGROW | SVR | WRITE_WATCH | REGION + PBYTE m_pUpperBoundImmediate; // | POSTGROW | | WRITE_WATCH | REGION + PBYTE m_pRegionToGenTableImmediate; // | | | WRITE_WATCH | REGION + PBYTE m_pRegionShrDest; // | | | WRITE_WATCH | REGION + PBYTE m_pRegionShrSrc; // | | | WRITE_WATCH | RETION + +#if defined(TARGET_ARM64) + PBYTE m_lowestAddress; + PBYTE m_highestAddress; +#if defined(WRITE_BARRIER_CHECK) + PBYTE m_pGCShadow; + PBYTE m_pGCShadowEnd; +#endif // WRITE_BARRIER_CHECK +#endif // TARGET_AMD64 + +}; + +extern WriteBarrierManager g_WriteBarrierManager; + +#endif // TARGET_AMD64 || TARGET_ARM64 + +#endif // WRITEBARRIERMANAGER_H