From db6c2cf23906215c6075f23b11db97a12a5de43c Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Thu, 16 Jan 2025 11:41:21 +0000 Subject: [PATCH 01/42] Arm64: Implement region write barriers --- src/coreclr/vm/arm64/asmhelpers.S | 22 ++++++++--- src/coreclr/vm/arm64/patchedcode.S | 61 +++++++++++++++++++++++++++--- 2 files changed, 72 insertions(+), 11 deletions(-) diff --git a/src/coreclr/vm/arm64/asmhelpers.S b/src/coreclr/vm/arm64/asmhelpers.S index 65c1eaec4121ad..02a2372a48d38d 100644 --- a/src/coreclr/vm/arm64/asmhelpers.S +++ b/src/coreclr/vm/arm64/asmhelpers.S @@ -194,8 +194,9 @@ LEAF_END ThePreStubPatch, _TEXT LEAF_ENTRY JIT_UpdateWriteBarrierState, _TEXT PROLOG_SAVE_REG_PAIR_INDEXED fp, lr, -16 - // x0-x7, x10 will contain intended new state + // x0-x7, x10-x11, x13-x14 will contain intended new state // x8 will preserve skipEphemeralCheck + // x9 will preserve writeableOffset // x12 will be used for pointers mov x8, x0 @@ -231,12 +232,21 @@ LOCAL_LABEL(EphemeralCheckEnabled): PREPARE_EXTERNAL_VAR g_highest_address, x12 ldr x6, [x12] + PREPARE_EXTERNAL_VAR g_region_to_generation_table, x12 + ldr x7, [x12] + + PREPARE_EXTERNAL_VAR g_region_shr, x12 + ldr w10, [x12] + + PREPARE_EXTERNAL_VAR g_region_use_bitwise_write_barrier, x12 + ldr w11, [x12] + #ifdef WRITE_BARRIER_CHECK PREPARE_EXTERNAL_VAR g_GCShadow, x12 - ldr x7, [x12] + ldr x13, [x12] PREPARE_EXTERNAL_VAR g_GCShadowEnd, x12 - ldr x10, [x12] + ldr x14, [x12] #endif // Update wbs state @@ -247,12 +257,12 @@ LOCAL_LABEL(EphemeralCheckEnabled): stp x0, x1, [x12], 16 stp x2, x3, [x12], 16 stp x4, x5, [x12], 16 - str x6, [x12], 8 + stp x6, x7, [x12], 16 + stp w10, w11, [x12], 8 #ifdef WRITE_BARRIER_CHECK - stp x7, x10, [x12], 16 + stp x13, x14, [x12], 16 #endif - EPILOG_RESTORE_REG_PAIR_INDEXED fp, lr, 16 EPILOG_RETURN LEAF_END JIT_UpdateWriteBarrierState diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index 0e223cbc1d33ab..2238374954c05d 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -142,26 +142,70 @@ LOCAL_LABEL(ShadowUpdateEnd): #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP // Update the write watch table if necessary ldr x12, LOCAL_LABEL(wbs_sw_ww_table) - cbz x12, LOCAL_LABEL(CheckCardTable) + cbz x12, LOCAL_LABEL(CheckCardTableBounds) add x12, x12, x14, lsr #0xc // SoftwareWriteWatch::AddressToTableByteIndexShift ldrb w17, [x12] - cbnz x17, LOCAL_LABEL(CheckCardTable) + cbnz x17, LOCAL_LABEL(CheckCardTableBounds) mov w17, #0xFF strb w17, [x12] #endif -LOCAL_LABEL(CheckCardTable): - // Branch to Exit if the reference is not in the Gen0 heap +LOCAL_LABEL(CheckCardTableBounds): + // Branch to Exit if the reference is not in the heap ldr x12, LOCAL_LABEL(wbs_ephemeral_low) ldr x17, LOCAL_LABEL(wbs_ephemeral_high) cmp x15, x12 ccmp x15, x17, #0x2, hs bhs LOCAL_LABEL(Exit) + // Region Checks + + // Check if using regions + ldr x17, LOCAL_LABEL(wbs_region_to_generation_table) + cbz x17, LOCAL_LABEL(CheckCardTable) + + // Calculate region locations + ldr w12, LOCAL_LABEL(wbs_region_shr) + lsr x15, x15, x12 + add x15, x15, x17 // x15 = (RHS >> wbs_region_shr) + wbs_region_to_generation_table + lsr x12, x14, x12 + add x12, x12, x17 // x12 = (LHS >> wbs_region_shr) + wbs_region_to_generation_table + + // Check whether the region we're storing into is gen 0 - nothing to do in this case + ldrb w15, [x15] + cbz w15, LOCAL_LABEL(Exit) + + // Check this is going from old to young + ldrb w12, [x12] + cmp w12, w17 + bhs LOCAL_LABEL(Exit) + + // Bitwise write barriers only + ldr w17, LOCAL_LABEL(wbs_region_use_bitwise_write_barrier) + cbz w17, LOCAL_LABEL(CheckCardTable) + // Check if we need to update the card table + lsr x17, x14, 8 + and x17, x17, 7 + movz w15, 1 + lsl w17, w15, w17 // w17 = 1 << (LHS >> 8 && 7) ldr x12, LOCAL_LABEL(wbs_card_table) add x15, x12, x14, lsr #11 - ldrb w12, [x15] + ldrb w12, [x15] // w12 = [(RHS >> 11) + g_card_table] + cmp w12, w17 + bne LOCAL_LABEL(Exit) + + // Update the card table + strb w17, [x15] + b LOCAL_LABEL(CheckCardBundleTable) + + // End of Region Checks + +LOCAL_LABEL(CheckCardTable): + // Check if we need to update the card table + ldr x12, LOCAL_LABEL(wbs_card_table) + add x15, x12, x14, lsr #11 + ldrb w12, [x15] // w12 = [(RHS >> 11) + g_card_table] cmp x12, 0xFF beq LOCAL_LABEL(Exit) @@ -169,6 +213,7 @@ LOCAL_LABEL(CheckCardTable): mov x12, 0xFF strb w12, [x15] +LOCAL_LABEL(CheckCardBundleTable): #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES // Check if we need to update the card bundle table ldr x12, LOCAL_LABEL(wbs_card_bundle_table) @@ -208,6 +253,12 @@ LOCAL_LABEL(wbs_lowest_address): .quad 0 LOCAL_LABEL(wbs_highest_address): .quad 0 +LOCAL_LABEL(wbs_region_to_generation_table): + .quad 0 +LOCAL_LABEL(wbs_region_shr): + .word 0 +LOCAL_LABEL(wbs_region_use_bitwise_write_barrier): + .word 0 #ifdef WRITE_BARRIER_CHECK LOCAL_LABEL(wbs_GCShadow): .quad 0 From 5de7e0f4c9138e21aa5026a624431296f9334868 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Tue, 21 Jan 2025 11:31:56 +0000 Subject: [PATCH 02/42] Fix byte region barriers --- src/coreclr/vm/arm64/patchedcode.S | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index 2238374954c05d..b5212ad728b2f0 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -172,12 +172,12 @@ LOCAL_LABEL(CheckCardTableBounds): add x12, x12, x17 // x12 = (LHS >> wbs_region_shr) + wbs_region_to_generation_table // Check whether the region we're storing into is gen 0 - nothing to do in this case - ldrb w15, [x15] - cbz w15, LOCAL_LABEL(Exit) + ldrb w12, [x12] + cbz w12, LOCAL_LABEL(Exit) // Check this is going from old to young - ldrb w12, [x12] - cmp w12, w17 + ldrb w15, [x15] + cmp w15, w12 bhs LOCAL_LABEL(Exit) // Bitwise write barriers only From 9315aa1ff0264f7e17964ad78f33049fcbbcfb22 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Tue, 21 Jan 2025 12:59:45 +0000 Subject: [PATCH 03/42] Fix bit region barriers --- src/coreclr/vm/arm64/patchedcode.S | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index b5212ad728b2f0..667040ed406a11 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -185,18 +185,20 @@ LOCAL_LABEL(CheckCardTableBounds): cbz w17, LOCAL_LABEL(CheckCardTable) // Check if we need to update the card table - lsr x17, x14, 8 - and x17, x17, 7 + lsr w17, w14, 8 + and w17, w17, 7 movz w15, 1 - lsl w17, w15, w17 // w17 = 1 << (LHS >> 8 && 7) + lsl w17, w15, w17 // w17 = 1 << (RHS >> 8 && 7) ldr x12, LOCAL_LABEL(wbs_card_table) add x15, x12, x14, lsr #11 ldrb w12, [x15] // w12 = [(RHS >> 11) + g_card_table] cmp w12, w17 - bne LOCAL_LABEL(Exit) + beq LOCAL_LABEL(Exit) // Update the card table - strb w17, [x15] + // TODO: Is this correct? the AMD64 code is odd. + orr w12, w12, w17 + strb w12, [x15] b LOCAL_LABEL(CheckCardBundleTable) // End of Region Checks From d0e46f048f54615859cfdfb8a7b5047bf1cbe706 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Wed, 22 Jan 2025 10:29:32 +0000 Subject: [PATCH 04/42] test instead of cmp for bitwise write barriers --- src/coreclr/vm/arm64/patchedcode.S | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index 667040ed406a11..52126ca8cb9fd0 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -192,8 +192,8 @@ LOCAL_LABEL(CheckCardTableBounds): ldr x12, LOCAL_LABEL(wbs_card_table) add x15, x12, x14, lsr #11 ldrb w12, [x15] // w12 = [(RHS >> 11) + g_card_table] - cmp w12, w17 - beq LOCAL_LABEL(Exit) + tst w12, w17 + bne LOCAL_LABEL(Exit) // Update the card table // TODO: Is this correct? the AMD64 code is odd. From cb83f53eb7ad3b4a8dddd5bf1cb86467ce3e8b9a Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Wed, 22 Jan 2025 12:23:43 +0000 Subject: [PATCH 05/42] use LSE to atomically update bitwise write barriers --- src/coreclr/vm/arm64/asmhelpers.S | 7 +++++++ src/coreclr/vm/arm64/patchedcode.S | 7 +++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/coreclr/vm/arm64/asmhelpers.S b/src/coreclr/vm/arm64/asmhelpers.S index 02a2372a48d38d..ca0680771c3040 100644 --- a/src/coreclr/vm/arm64/asmhelpers.S +++ b/src/coreclr/vm/arm64/asmhelpers.S @@ -241,6 +241,13 @@ LOCAL_LABEL(EphemeralCheckEnabled): PREPARE_EXTERNAL_VAR g_region_use_bitwise_write_barrier, x12 ldr w11, [x12] + // Only allow bitwise write barriers if LSE atomics are present + PREPARE_EXTERNAL_VAR g_arm64_atomics_present, x12 + ldr w13, [x12] + cbnz w13, LOCAL_LABEL(AtomicsPresent) + movz w11, 0 +LOCAL_LABEL(AtomicsPresent): + #ifdef WRITE_BARRIER_CHECK PREPARE_EXTERNAL_VAR g_GCShadow, x12 ldr x13, [x12] diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index 52126ca8cb9fd0..75b049cd7e093e 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -195,10 +195,9 @@ LOCAL_LABEL(CheckCardTableBounds): tst w12, w17 bne LOCAL_LABEL(Exit) - // Update the card table - // TODO: Is this correct? the AMD64 code is odd. - orr w12, w12, w17 - strb w12, [x15] + // Atomically update the card table + // Requires LSE, but the code is only compiled for 8.0 + .word 0x383131FF // stsetb w17, [x15] b LOCAL_LABEL(CheckCardBundleTable) // End of Region Checks From c615772398585ceebd9df5ec1b49a8bfbf6504cc Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Thu, 23 Jan 2025 09:53:59 +0000 Subject: [PATCH 06/42] move atomics check into gcenv.ee.cpp --- src/coreclr/vm/arm64/asmhelpers.S | 7 ------- src/coreclr/vm/gcenv.ee.cpp | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/coreclr/vm/arm64/asmhelpers.S b/src/coreclr/vm/arm64/asmhelpers.S index ca0680771c3040..02a2372a48d38d 100644 --- a/src/coreclr/vm/arm64/asmhelpers.S +++ b/src/coreclr/vm/arm64/asmhelpers.S @@ -241,13 +241,6 @@ LOCAL_LABEL(EphemeralCheckEnabled): PREPARE_EXTERNAL_VAR g_region_use_bitwise_write_barrier, x12 ldr w11, [x12] - // Only allow bitwise write barriers if LSE atomics are present - PREPARE_EXTERNAL_VAR g_arm64_atomics_present, x12 - ldr w13, [x12] - cbnz w13, LOCAL_LABEL(AtomicsPresent) - movz w11, 0 -LOCAL_LABEL(AtomicsPresent): - #ifdef WRITE_BARRIER_CHECK PREPARE_EXTERNAL_VAR g_GCShadow, x12 ldr x13, [x12] diff --git a/src/coreclr/vm/gcenv.ee.cpp b/src/coreclr/vm/gcenv.ee.cpp index 55329274905a5d..bde9a19e58e84f 100644 --- a/src/coreclr/vm/gcenv.ee.cpp +++ b/src/coreclr/vm/gcenv.ee.cpp @@ -1063,6 +1063,13 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) g_region_to_generation_table = args->region_to_generation_table; g_region_shr = args->region_shr; g_region_use_bitwise_write_barrier = args->region_use_bitwise_write_barrier; +#if defined(HOST_ARM64) + // Only allow bitwise write barriers if LSE atomics are present + if (!g_arm64_atomics_present) + { + g_region_use_bitwise_write_barrier = false; + } +#endif stompWBCompleteActions |= ::StompWriteBarrierEphemeral(args->is_runtime_suspended); break; case WriteBarrierOp::Initialize: @@ -1090,6 +1097,13 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) g_region_to_generation_table = args->region_to_generation_table; g_region_shr = args->region_shr; g_region_use_bitwise_write_barrier = args->region_use_bitwise_write_barrier; +#if defined(HOST_ARM64) + // Only allow bitwise write barriers if LSE atomics are present + if (!g_arm64_atomics_present) + { + g_region_use_bitwise_write_barrier = false; + } +#endif stompWBCompleteActions |= ::StompWriteBarrierResize(true, false); // StompWriteBarrierResize does not necessarily bash g_ephemeral_low From 1c865f10ad3c4c88d81f85431b641e268e933185 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Thu, 23 Jan 2025 13:37:35 +0000 Subject: [PATCH 07/42] Skip ephemeral checks for regionless server GC, and refactor checks --- src/coreclr/vm/arm64/asmhelpers.S | 11 ++--------- src/coreclr/vm/arm64/stubs.cpp | 19 ++++++++++--------- src/coreclr/vm/gcenv.ee.cpp | 23 +++++++++++++++++++++-- 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/src/coreclr/vm/arm64/asmhelpers.S b/src/coreclr/vm/arm64/asmhelpers.S index 02a2372a48d38d..1f64ddf0026eba 100644 --- a/src/coreclr/vm/arm64/asmhelpers.S +++ b/src/coreclr/vm/arm64/asmhelpers.S @@ -180,7 +180,7 @@ PATCH_LABEL ThePreStubPatchLabel ret lr LEAF_END ThePreStubPatch, _TEXT -// void JIT_UpdateWriteBarrierState(bool skipEphemeralCheck, size_t writeableOffset) +// void JIT_UpdateWriteBarrierState(size_t writeableOffset) // // Update shadow copies of the various state info required for barrier // @@ -195,12 +195,10 @@ LEAF_ENTRY JIT_UpdateWriteBarrierState, _TEXT PROLOG_SAVE_REG_PAIR_INDEXED fp, lr, -16 // x0-x7, x10-x11, x13-x14 will contain intended new state - // x8 will preserve skipEphemeralCheck // x9 will preserve writeableOffset // x12 will be used for pointers - mov x8, x0 - mov x9, x1 + mov x9, x0 PREPARE_EXTERNAL_VAR g_card_table, x12 ldr x0, [x12] @@ -221,11 +219,6 @@ LEAF_ENTRY JIT_UpdateWriteBarrierState, _TEXT PREPARE_EXTERNAL_VAR g_ephemeral_high, x12 ldr x4, [x12] - cbz x8, LOCAL_LABEL(EphemeralCheckEnabled) - movz x3, #0 - movn x4, #0 -LOCAL_LABEL(EphemeralCheckEnabled): - PREPARE_EXTERNAL_VAR g_lowest_address, x12 ldr x5, [x12] diff --git a/src/coreclr/vm/arm64/stubs.cpp b/src/coreclr/vm/arm64/stubs.cpp index 825d2e0e4fd811..506583abd67304 100644 --- a/src/coreclr/vm/arm64/stubs.cpp +++ b/src/coreclr/vm/arm64/stubs.cpp @@ -865,12 +865,12 @@ void JIT_TailCall() } #if !defined(DACCESS_COMPILE) -EXTERN_C void JIT_UpdateWriteBarrierState(bool skipEphemeralCheck, size_t writeableOffset); +EXTERN_C void JIT_UpdateWriteBarrierState(size_t writeableOffset); extern "C" void STDCALL JIT_PatchedCodeStart(); extern "C" void STDCALL JIT_PatchedCodeLast(); -static void UpdateWriteBarrierState(bool skipEphemeralCheck) +static void UpdateWriteBarrierState() { BYTE *writeBarrierCodeStart = GetWriteBarrierCodeLocation((void*)JIT_PatchedCodeStart); BYTE *writeBarrierCodeStartRW = writeBarrierCodeStart; @@ -880,7 +880,8 @@ static void UpdateWriteBarrierState(bool skipEphemeralCheck) writeBarrierWriterHolder.AssignExecutableWriterHolder(writeBarrierCodeStart, (BYTE*)JIT_PatchedCodeLast - (BYTE*)JIT_PatchedCodeStart); writeBarrierCodeStartRW = writeBarrierWriterHolder.GetRW(); } - JIT_UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap(), writeBarrierCodeStartRW - writeBarrierCodeStart); + + JIT_UpdateWriteBarrierState(writeBarrierCodeStartRW - writeBarrierCodeStart); } void InitJITHelpers1() @@ -909,12 +910,12 @@ void InitJITHelpers1() } } - UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap()); + UpdateWriteBarrierState(); } #else -void UpdateWriteBarrierState(bool) {} +void UpdateWriteBarrierState() {} #endif // !defined(DACCESS_COMPILE) PTR_CONTEXT GetCONTEXTFromRedirectedStubStackFrame(T_DISPATCHER_CONTEXT * pDispatcherContext) @@ -1076,26 +1077,26 @@ void FlushWriteBarrierInstructionCache() int StompWriteBarrierEphemeral(bool isRuntimeSuspended) { - UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap()); + UpdateWriteBarrierState(); return SWB_PASS; } int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck) { - UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap()); + UpdateWriteBarrierState(); return SWB_PASS; } #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP int SwitchToWriteWatchBarrier(bool isRuntimeSuspended) { - UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap()); + UpdateWriteBarrierState(); return SWB_PASS; } int SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended) { - UpdateWriteBarrierState(GCHeapUtilities::IsServerHeap()); + UpdateWriteBarrierState(); return SWB_PASS; } #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP diff --git a/src/coreclr/vm/gcenv.ee.cpp b/src/coreclr/vm/gcenv.ee.cpp index bde9a19e58e84f..512131137711ac 100644 --- a/src/coreclr/vm/gcenv.ee.cpp +++ b/src/coreclr/vm/gcenv.ee.cpp @@ -1053,6 +1053,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) ThreadSuspend::RestartEE(FALSE, TRUE); } return; // unlike other branches we have already done cleanup so bailing out here + case WriteBarrierOp::StompEphemeral: assert(args->is_runtime_suspended && "the runtime must be suspended here!"); // StompEphemeral requires a new ephemeral low and a new ephemeral high @@ -1069,9 +1070,16 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) { g_region_use_bitwise_write_barrier = false; } + // Skip ephemeral checks for regionless server GC + if (GCHeapUtilities::IsServerHeap() && g_region_to_generation_table == nullptr) + { + g_ephemeral_low = 0; + g_ephemeral_high = 0; + } #endif stompWBCompleteActions |= ::StompWriteBarrierEphemeral(args->is_runtime_suspended); break; + case WriteBarrierOp::Initialize: assert(args->is_runtime_suspended && "the runtime must be suspended here!"); // This operation should only be invoked once, upon initialization. @@ -1097,23 +1105,32 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) g_region_to_generation_table = args->region_to_generation_table; g_region_shr = args->region_shr; g_region_use_bitwise_write_barrier = args->region_use_bitwise_write_barrier; + g_ephemeral_low = args->ephemeral_low; + g_ephemeral_high = args->ephemeral_high; #if defined(HOST_ARM64) // Only allow bitwise write barriers if LSE atomics are present if (!g_arm64_atomics_present) { g_region_use_bitwise_write_barrier = false; } + // Skip ephemeral checks for regionless server GC + if (GCHeapUtilities::IsServerHeap() && g_region_to_generation_table == nullptr) + { + g_ephemeral_low = 0; + g_ephemeral_high = 0; + } #endif stompWBCompleteActions |= ::StompWriteBarrierResize(true, false); +#if !defined(HOST_ARM64) // StompWriteBarrierResize does not necessarily bash g_ephemeral_low // usages, so we must do so here. This is particularly true on x86, // where StompWriteBarrierResize will not bash g_ephemeral_low when // called with the parameters (true, false), as it is above. - g_ephemeral_low = args->ephemeral_low; - g_ephemeral_high = args->ephemeral_high; stompWBCompleteActions |= ::StompWriteBarrierEphemeral(true); +#endif break; + case WriteBarrierOp::SwitchToWriteWatch: #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP assert(args->is_runtime_suspended && "the runtime must be suspended here!"); @@ -1125,6 +1142,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) assert(!"should never be called without FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP"); #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP break; + case WriteBarrierOp::SwitchToNonWriteWatch: #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP assert(args->is_runtime_suspended && "the runtime must be suspended here!"); @@ -1135,6 +1153,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) assert(!"should never be called without FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP"); #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP break; + default: assert(!"unknown WriteBarrierOp enum"); } From 15dde1b03b6f13aa0b1f44a8f9f7c6bc54c77982 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Fri, 24 Jan 2025 13:47:28 +0000 Subject: [PATCH 08/42] Move ephemeral checks back --- src/coreclr/vm/arm64/asmhelpers.S | 11 +++++++++-- src/coreclr/vm/arm64/stubs.cpp | 11 +++++++++-- src/coreclr/vm/gcenv.ee.cpp | 12 ------------ 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/coreclr/vm/arm64/asmhelpers.S b/src/coreclr/vm/arm64/asmhelpers.S index 1f64ddf0026eba..02a2372a48d38d 100644 --- a/src/coreclr/vm/arm64/asmhelpers.S +++ b/src/coreclr/vm/arm64/asmhelpers.S @@ -180,7 +180,7 @@ PATCH_LABEL ThePreStubPatchLabel ret lr LEAF_END ThePreStubPatch, _TEXT -// void JIT_UpdateWriteBarrierState(size_t writeableOffset) +// void JIT_UpdateWriteBarrierState(bool skipEphemeralCheck, size_t writeableOffset) // // Update shadow copies of the various state info required for barrier // @@ -195,10 +195,12 @@ LEAF_ENTRY JIT_UpdateWriteBarrierState, _TEXT PROLOG_SAVE_REG_PAIR_INDEXED fp, lr, -16 // x0-x7, x10-x11, x13-x14 will contain intended new state + // x8 will preserve skipEphemeralCheck // x9 will preserve writeableOffset // x12 will be used for pointers - mov x9, x0 + mov x8, x0 + mov x9, x1 PREPARE_EXTERNAL_VAR g_card_table, x12 ldr x0, [x12] @@ -219,6 +221,11 @@ LEAF_ENTRY JIT_UpdateWriteBarrierState, _TEXT PREPARE_EXTERNAL_VAR g_ephemeral_high, x12 ldr x4, [x12] + cbz x8, LOCAL_LABEL(EphemeralCheckEnabled) + movz x3, #0 + movn x4, #0 +LOCAL_LABEL(EphemeralCheckEnabled): + PREPARE_EXTERNAL_VAR g_lowest_address, x12 ldr x5, [x12] diff --git a/src/coreclr/vm/arm64/stubs.cpp b/src/coreclr/vm/arm64/stubs.cpp index 506583abd67304..92c5f0781274d5 100644 --- a/src/coreclr/vm/arm64/stubs.cpp +++ b/src/coreclr/vm/arm64/stubs.cpp @@ -865,7 +865,7 @@ void JIT_TailCall() } #if !defined(DACCESS_COMPILE) -EXTERN_C void JIT_UpdateWriteBarrierState(size_t writeableOffset); +EXTERN_C void JIT_UpdateWriteBarrierState(bool skipEphemeralCheck, size_t writeableOffset); extern "C" void STDCALL JIT_PatchedCodeStart(); extern "C" void STDCALL JIT_PatchedCodeLast(); @@ -881,7 +881,14 @@ static void UpdateWriteBarrierState() writeBarrierCodeStartRW = writeBarrierWriterHolder.GetRW(); } - JIT_UpdateWriteBarrierState(writeBarrierCodeStartRW - writeBarrierCodeStart); + // Skip ephemeral checks for regionless server GC + bool skipEphemeralCheck = false; + if (GCHeapUtilities::IsServerHeap() && g_region_to_generation_table == nullptr) + { + skipEphemeralCheck = true; + } + + JIT_UpdateWriteBarrierState(skipEphemeralCheck, writeBarrierCodeStartRW - writeBarrierCodeStart); } void InitJITHelpers1() diff --git a/src/coreclr/vm/gcenv.ee.cpp b/src/coreclr/vm/gcenv.ee.cpp index 512131137711ac..07d40f3a89f312 100644 --- a/src/coreclr/vm/gcenv.ee.cpp +++ b/src/coreclr/vm/gcenv.ee.cpp @@ -1070,12 +1070,6 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) { g_region_use_bitwise_write_barrier = false; } - // Skip ephemeral checks for regionless server GC - if (GCHeapUtilities::IsServerHeap() && g_region_to_generation_table == nullptr) - { - g_ephemeral_low = 0; - g_ephemeral_high = 0; - } #endif stompWBCompleteActions |= ::StompWriteBarrierEphemeral(args->is_runtime_suspended); break; @@ -1113,12 +1107,6 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) { g_region_use_bitwise_write_barrier = false; } - // Skip ephemeral checks for regionless server GC - if (GCHeapUtilities::IsServerHeap() && g_region_to_generation_table == nullptr) - { - g_ephemeral_low = 0; - g_ephemeral_high = 0; - } #endif stompWBCompleteActions |= ::StompWriteBarrierResize(true, false); From b5b28ce126e88500b4eb256970db8dbf01a23f17 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Mon, 27 Jan 2025 17:31:39 +0000 Subject: [PATCH 09/42] Add GC-write-barriers.md --- docs/design/coreclr/jit/GC-write-barriers.md | 79 ++++++++++++++++++++ src/coreclr/vm/arm64/patchedcode.S | 6 +- 2 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 docs/design/coreclr/jit/GC-write-barriers.md diff --git a/docs/design/coreclr/jit/GC-write-barriers.md b/docs/design/coreclr/jit/GC-write-barriers.md new file mode 100644 index 00000000000000..71551b442576ec --- /dev/null +++ b/docs/design/coreclr/jit/GC-write-barriers.md @@ -0,0 +1,79 @@ +# GC write barriers + +The GC write barrier function (JIT_WriteBarrier) is generally the hottest function in CoreCLR and is written in assembly. The full pseudo code for the function is as follows: + + +```` +JIT_WriteBarrier(Object **dst, Object *ref) + Set *dst = ref + + // Shadow Heap update: + ifdef TARGET_ARM64: + if *wbs_GCShadow != 0: + if g_GCShadow + (dst - g_lowest_address) < wbs_GCShadowEnd: + *(g_GCShadow + (dst - g_lowest_address) = ref + if *dst != ref: + *(g_GCShadow + (dst - g_lowest_address) = INVALIDGCVALUE + + // Write watch for GC Heap: + ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP: + if g_sw_ww_table != 0: + if *(g_sw_ww_table + (dst>>11)) != 0: + *(g_sw_ww_table + (dst>>11)) = 0xff + + + // Return if the reference is not in the heap + if ref < g_ephemeral_low || reg >= g_ephemeral_high: + return + + // Region Checks + if g_wbs_region_to_generation_table != 0: + + // Calculate region locations + char reg_loc_dst = *((dst >> g_region_shr) + g_region_to_generation_table) + char reg_loc_ref = *((ref >> g_region_shr) + g_region_to_generation_table) + + // Check whether the region we're storing into is gen 0 - nothing to do in this case + if reg_loc_dst == 0: + return + + // Check this is going from old to young + if reg_loc_dst >= reg_loc_ref: + return + + // Bitwise write barriers only + if g_region_use_bitwise_write_barrier: + + char *card_table_dst = (dst >> 11) + g_card_table + char dst_bit = 1 << (dst >> 8 && 7) + + // Check if we need to update the card table + if *card_table_dst & dst_bit == 0: + return + + // Atomically update the card table + lock: *card_table_dst |= dst_bit + + goto CardBundle + + // Check if we need to update the card table + char *card_table_dst = (dst >> 11) + g_card_table + if *card_table_dst == 0xff: + return + + // Update the card table + *card_table_dst = 0xff + +CardBundle: + + // Mark the card bundle table as dirty + Ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES: + char card_bundle_dst = (dst >> 21) + g_card_bundle_table + if *card_bundle_dst != 0xff: + *card_bundle_dst = 0xff + +```` + +## WritebarrierManager + +On AMD64, there several different implementations of the write barrier function. Each implementation assumes different state and so can skip certain checks. The actual write barrier that is called is a copy of one of these implementations. The WritebarrierManager keeps track of which implementation is currently being used. As internal state changes, the WritebarrierManager updates the copy to the correct implementation. In practice, most of the internal state is fixed on startup, with only changes to/from use of write watch barriers changing during runtime. diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index 75b049cd7e093e..98b9aedebde5bb 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -188,10 +188,10 @@ LOCAL_LABEL(CheckCardTableBounds): lsr w17, w14, 8 and w17, w17, 7 movz w15, 1 - lsl w17, w15, w17 // w17 = 1 << (RHS >> 8 && 7) + lsl w17, w15, w17 // w17 = 1 << (LHS >> 8 && 7) ldr x12, LOCAL_LABEL(wbs_card_table) add x15, x12, x14, lsr #11 - ldrb w12, [x15] // w12 = [(RHS >> 11) + g_card_table] + ldrb w12, [x15] // w12 = [(LHS >> 11) + g_card_table] tst w12, w17 bne LOCAL_LABEL(Exit) @@ -206,7 +206,7 @@ LOCAL_LABEL(CheckCardTable): // Check if we need to update the card table ldr x12, LOCAL_LABEL(wbs_card_table) add x15, x12, x14, lsr #11 - ldrb w12, [x15] // w12 = [(RHS >> 11) + g_card_table] + ldrb w12, [x15] // w12 = [(LHS >> 11) + g_card_table] cmp x12, 0xFF beq LOCAL_LABEL(Exit) From c27d51688eacd09f8bf07a22a77ca289d2457532 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Mon, 27 Jan 2025 18:08:18 +0000 Subject: [PATCH 10/42] More variables for the pseudo code --- docs/design/coreclr/jit/GC-write-barriers.md | 21 +++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/docs/design/coreclr/jit/GC-write-barriers.md b/docs/design/coreclr/jit/GC-write-barriers.md index 71551b442576ec..2c90ddf56f3d3e 100644 --- a/docs/design/coreclr/jit/GC-write-barriers.md +++ b/docs/design/coreclr/jit/GC-write-barriers.md @@ -7,20 +7,23 @@ The GC write barrier function (JIT_WriteBarrier) is generally the hottest functi JIT_WriteBarrier(Object **dst, Object *ref) Set *dst = ref - // Shadow Heap update: + // Shadow Heap update ifdef TARGET_ARM64: - if *wbs_GCShadow != 0: - if g_GCShadow + (dst - g_lowest_address) < wbs_GCShadowEnd: - *(g_GCShadow + (dst - g_lowest_address) = ref + if g_GCShadow != 0: + long *shadow_dst = g_GCShadow + (dst - g_lowest_address) + // Check shadow heap location is within shadow heap + if shadow_dst < g_GCShadowEnd: + *shadow_dst = ref + atomic: wait for stores to complete if *dst != ref: - *(g_GCShadow + (dst - g_lowest_address) = INVALIDGCVALUE + *shadow_dst = INVALIDGCVALUE - // Write watch for GC Heap: + // Update the write watch table, if it's in use ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP: if g_sw_ww_table != 0: - if *(g_sw_ww_table + (dst>>11)) != 0: - *(g_sw_ww_table + (dst>>11)) = 0xff - + char *ww_table_dst = g_sw_ww_table + (dst>>11) + if *ww_table_dst != 0: + *ww_table_dst = 0xff // Return if the reference is not in the heap if ref < g_ephemeral_low || reg >= g_ephemeral_high: From d014417877b6297fc72c0d496a24c2c225d1c8a2 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Mon, 27 Jan 2025 18:09:23 +0000 Subject: [PATCH 11/42] WRITE_BARRIER_CHECK instead of TARGET_ARM64 --- docs/design/coreclr/jit/GC-write-barriers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design/coreclr/jit/GC-write-barriers.md b/docs/design/coreclr/jit/GC-write-barriers.md index 2c90ddf56f3d3e..2e17656d697163 100644 --- a/docs/design/coreclr/jit/GC-write-barriers.md +++ b/docs/design/coreclr/jit/GC-write-barriers.md @@ -8,7 +8,7 @@ JIT_WriteBarrier(Object **dst, Object *ref) Set *dst = ref // Shadow Heap update - ifdef TARGET_ARM64: + ifdef WRITE_BARRIER_CHECK: if g_GCShadow != 0: long *shadow_dst = g_GCShadow + (dst - g_lowest_address) // Check shadow heap location is within shadow heap From 6068fb08af6e71e226150744335892648b5a0bdb Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Tue, 28 Jan 2025 12:30:45 +0000 Subject: [PATCH 12/42] Add JIT_CheckedWriteBarrier and fixups to doc --- docs/design/coreclr/jit/GC-write-barriers.md | 22 ++++++++++++++++---- src/coreclr/vm/arm64/patchedcode.S | 2 +- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/docs/design/coreclr/jit/GC-write-barriers.md b/docs/design/coreclr/jit/GC-write-barriers.md index 2e17656d697163..f9a8fdfff9fc2f 100644 --- a/docs/design/coreclr/jit/GC-write-barriers.md +++ b/docs/design/coreclr/jit/GC-write-barriers.md @@ -25,18 +25,18 @@ JIT_WriteBarrier(Object **dst, Object *ref) if *ww_table_dst != 0: *ww_table_dst = 0xff - // Return if the reference is not in the heap - if ref < g_ephemeral_low || reg >= g_ephemeral_high: + // Return if the reference is not in Gen 0 + if ref < g_ephemeral_low || ref >= g_ephemeral_high: return // Region Checks - if g_wbs_region_to_generation_table != 0: + if g_region_to_generation_table != 0: // Calculate region locations char reg_loc_dst = *((dst >> g_region_shr) + g_region_to_generation_table) char reg_loc_ref = *((ref >> g_region_shr) + g_region_to_generation_table) - // Check whether the region we're storing into is gen 0 - nothing to do in this case + // Return if the region we're storing into is Gen 0 if reg_loc_dst == 0: return @@ -77,6 +77,20 @@ CardBundle: ```` +The Checked Write Barrier has additional checks: + +```` +JIT_CheckedWriteBarrier(Object **dst, Object *ref) + + // Return if the destination is not on the heap + if ref < g_lowest_address || ref >= g_highest_address: + return + + return JIT_WriteBarrier(dst, ref) +```` + + + ## WritebarrierManager On AMD64, there several different implementations of the write barrier function. Each implementation assumes different state and so can skip certain checks. The actual write barrier that is called is a copy of one of these implementations. The WritebarrierManager keeps track of which implementation is currently being used. As internal state changes, the WritebarrierManager updates the copy to the correct implementation. In practice, most of the internal state is fixed on startup, with only changes to/from use of write watch barriers changing during runtime. diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index 98b9aedebde5bb..87c7507d9833ed 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -151,7 +151,7 @@ LOCAL_LABEL(ShadowUpdateEnd): #endif LOCAL_LABEL(CheckCardTableBounds): - // Branch to Exit if the reference is not in the heap + // Branch to Exit if the reference is not in the Gen0 heap ldr x12, LOCAL_LABEL(wbs_ephemeral_low) ldr x17, LOCAL_LABEL(wbs_ephemeral_high) cmp x15, x12 From a41643730f8647b0f3b499d7ff6285e43a994c59 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Mon, 13 Jan 2025 11:51:58 +0000 Subject: [PATCH 13/42] Use writebarriermanager for arm64. --- src/coreclr/vm/CMakeLists.txt | 4 + src/coreclr/vm/amd64/jitinterfaceamd64.cpp | 498 +------------- .../vm/arm64/JitHelpers_FastWriteBarriers.S | 55 ++ src/coreclr/vm/arm64/patchedcode.S | 12 + src/coreclr/vm/arm64/patchedcode.asm | 54 +- src/coreclr/vm/arm64/stubs.cpp | 74 +- src/coreclr/vm/gcenv.ee.cpp | 4 +- src/coreclr/vm/jitinterface.h | 18 +- src/coreclr/vm/writebarriermanager.cpp | 638 ++++++++++++++++++ 9 files changed, 811 insertions(+), 546 deletions(-) create mode 100644 src/coreclr/vm/arm64/JitHelpers_FastWriteBarriers.S create mode 100644 src/coreclr/vm/writebarriermanager.cpp diff --git a/src/coreclr/vm/CMakeLists.txt b/src/coreclr/vm/CMakeLists.txt index 88fa4ac9f88414..fc701ecbf3de84 100644 --- a/src/coreclr/vm/CMakeLists.txt +++ b/src/coreclr/vm/CMakeLists.txt @@ -651,6 +651,7 @@ elseif(CLR_CMAKE_TARGET_ARCH_ARM64) set(VM_SOURCES_WKS_ARCH_ASM ${ARCH_SOURCES_DIR}/AsmHelpers.asm ${ARCH_SOURCES_DIR}/CallDescrWorkerARM64.asm + ${ARCH_SOURCES_DIR}/JitHelpers_FastWriteBarriers.asm ${ARCH_SOURCES_DIR}/patchedcode.asm ${ARCH_SOURCES_DIR}/PInvokeStubs.asm ${ARCH_SOURCES_DIR}/thunktemplates.asm @@ -705,6 +706,7 @@ else(CLR_CMAKE_TARGET_WIN32) set(VM_SOURCES_WKS_ARCH_ASM ${ARCH_SOURCES_DIR}/asmhelpers.S ${ARCH_SOURCES_DIR}/calldescrworkerarm64.S + ${ARCH_SOURCES_DIR}/JitHelpers_FastWriteBarriers.S ${ARCH_SOURCES_DIR}/patchedcode.S ${ARCH_SOURCES_DIR}/pinvokestubs.S ${ARCH_SOURCES_DIR}/thunktemplates.S @@ -750,6 +752,7 @@ if(CLR_CMAKE_TARGET_ARCH_AMD64) exceptionhandling.cpp gcinfodecoder.cpp jitinterfacegen.cpp + writebarriermanager.cpp ) set(VM_HEADERS_WKS_ARCH @@ -816,6 +819,7 @@ elseif(CLR_CMAKE_TARGET_ARCH_ARM64) set(VM_SOURCES_WKS_ARCH ${ARCH_SOURCES_DIR}/profiler.cpp gcinfodecoder.cpp + writebarriermanager.cpp ) if(CLR_CMAKE_HOST_UNIX) diff --git a/src/coreclr/vm/amd64/jitinterfaceamd64.cpp b/src/coreclr/vm/amd64/jitinterfaceamd64.cpp index f9cd2a968a02a7..6296b788a855b0 100644 --- a/src/coreclr/vm/amd64/jitinterfaceamd64.cpp +++ b/src/coreclr/vm/amd64/jitinterfaceamd64.cpp @@ -16,13 +16,8 @@ #include "excep.h" #include "threadsuspend.h" -extern uint8_t* g_ephemeral_low; -extern uint8_t* g_ephemeral_high; -extern uint32_t* g_card_table; -extern uint32_t* g_card_bundle_table; // Patch Labels for the various write barriers -EXTERN_C void JIT_WriteBarrier_End(); EXTERN_C void JIT_WriteBarrier_PreGrow64(Object **dst, Object *ref); EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_Lower(); @@ -30,7 +25,6 @@ EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_CardTable(); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_CardBundleTable(); #endif -EXTERN_C void JIT_WriteBarrier_PreGrow64_End(); EXTERN_C void JIT_WriteBarrier_PostGrow64(Object **dst, Object *ref); EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_Lower(); @@ -39,7 +33,6 @@ EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_CardTable(); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_CardBundleTable(); #endif -EXTERN_C void JIT_WriteBarrier_PostGrow64_End(); #ifdef FEATURE_SVR_GC EXTERN_C void JIT_WriteBarrier_SVR64(Object **dst, Object *ref); @@ -47,7 +40,6 @@ EXTERN_C void JIT_WriteBarrier_SVR64_PatchLabel_CardTable(); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES EXTERN_C void JIT_WriteBarrier_SVR64_PatchLabel_CardBundleTable(); #endif -EXTERN_C void JIT_WriteBarrier_SVR64_End(); #endif // FEATURE_SVR_GC EXTERN_C void JIT_WriteBarrier_Byte_Region64(Object **dst, Object *ref); @@ -60,7 +52,6 @@ EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_CardTable(); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_CardBundleTable(); #endif -EXTERN_C void JIT_WriteBarrier_Byte_Region64_End(); EXTERN_C void JIT_WriteBarrier_Bit_Region64(Object **dst, Object *ref); EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_RegionToGeneration(); @@ -72,8 +63,6 @@ EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_CardTable(); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_CardBundleTable(); #endif -EXTERN_C void JIT_WriteBarrier_Bit_Region64_End(); - #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64(Object **dst, Object *ref); @@ -83,7 +72,6 @@ EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_CardTable(); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_CardBundleTable(); #endif -EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_End(); EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64(Object **dst, Object *ref); EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_WriteWatchTable(); @@ -93,7 +81,6 @@ EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_CardTable(); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_CardBundleTable(); #endif -EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_End(); #ifdef FEATURE_SVR_GC EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64(Object **dst, Object *ref); @@ -102,7 +89,6 @@ EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_CardTable(); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_CardBundleTable(); #endif -EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_End(); #endif // FEATURE_SVR_GC EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64(Object **dst, Object *ref); @@ -116,7 +102,6 @@ EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_CardTable(); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_CardBundleTable(); #endif -EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_End(); EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64(Object **dst, Object *ref); EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_WriteWatchTable(); @@ -129,22 +114,16 @@ EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_CardTable(); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_CardBundleTable(); #endif -EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_End(); #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -WriteBarrierManager g_WriteBarrierManager; +extern WriteBarrierManager g_WriteBarrierManager; // Use this somewhat hokey macro to concatenate the function start with the patch // label. This allows the code below to look relatively nice, but relies on the // naming convention which we have established for these helpers. #define CALC_PATCH_LOCATION(func,label,offset) CalculatePatchLocation((PVOID)func, (PVOID)func##_##label, offset) -WriteBarrierManager::WriteBarrierManager() : - m_currentWriteBarrier(WRITE_BARRIER_UNINITIALIZED) -{ - LIMITED_METHOD_CONTRACT; -} #ifndef CODECOVERAGE // Deactivate alignment validation for code coverage builds // because the instrumentation tool will not preserve alignment @@ -313,121 +292,8 @@ void WriteBarrierManager::Validate() #endif // CODECOVERAGE -PCODE WriteBarrierManager::GetCurrentWriteBarrierCode() -{ - LIMITED_METHOD_CONTRACT; - - switch (m_currentWriteBarrier) - { - case WRITE_BARRIER_PREGROW64: - return GetEEFuncEntryPoint(JIT_WriteBarrier_PreGrow64); - case WRITE_BARRIER_POSTGROW64: - return GetEEFuncEntryPoint(JIT_WriteBarrier_PostGrow64); -#ifdef FEATURE_SVR_GC - case WRITE_BARRIER_SVR64: - return GetEEFuncEntryPoint(JIT_WriteBarrier_SVR64); -#endif // FEATURE_SVR_GC - case WRITE_BARRIER_BYTE_REGIONS64: - return GetEEFuncEntryPoint(JIT_WriteBarrier_Byte_Region64); - case WRITE_BARRIER_BIT_REGIONS64: - return GetEEFuncEntryPoint(JIT_WriteBarrier_Bit_Region64); -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - case WRITE_BARRIER_WRITE_WATCH_PREGROW64: - return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_PreGrow64); - case WRITE_BARRIER_WRITE_WATCH_POSTGROW64: - return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_PostGrow64); -#ifdef FEATURE_SVR_GC - case WRITE_BARRIER_WRITE_WATCH_SVR64: - return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_SVR64); -#endif // FEATURE_SVR_GC - case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: - return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_Byte_Region64); - case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: - return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_Bit_Region64); -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - default: - UNREACHABLE_MSG("unexpected m_currentWriteBarrier!"); - }; -} - -size_t WriteBarrierManager::GetSpecificWriteBarrierSize(WriteBarrierType writeBarrier) +void WriteBarrierManager::UpdatePatchLocations() { -// marked asm functions are those which use the LEAF_END_MARKED macro to end them which -// creates a public Name_End label which can be used to figure out their size without -// having to create unwind info. -#define MARKED_FUNCTION_SIZE(pfn) (size_t)((LPBYTE)GetEEFuncEntryPoint(pfn##_End) - (LPBYTE)GetEEFuncEntryPoint(pfn)) - - switch (writeBarrier) - { - case WRITE_BARRIER_PREGROW64: - return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_PreGrow64); - case WRITE_BARRIER_POSTGROW64: - return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_PostGrow64); -#ifdef FEATURE_SVR_GC - case WRITE_BARRIER_SVR64: - return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_SVR64); -#endif // FEATURE_SVR_GC - case WRITE_BARRIER_BYTE_REGIONS64: - return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_Byte_Region64); - case WRITE_BARRIER_BIT_REGIONS64: - return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_Bit_Region64); -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - case WRITE_BARRIER_WRITE_WATCH_PREGROW64: - return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_WriteWatch_PreGrow64); - case WRITE_BARRIER_WRITE_WATCH_POSTGROW64: - return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_WriteWatch_PostGrow64); -#ifdef FEATURE_SVR_GC - case WRITE_BARRIER_WRITE_WATCH_SVR64: - return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_WriteWatch_SVR64); -#endif // FEATURE_SVR_GC - case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: - return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_WriteWatch_Byte_Region64); - case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: - return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_WriteWatch_Bit_Region64); -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - case WRITE_BARRIER_BUFFER: - return MARKED_FUNCTION_SIZE(JIT_WriteBarrier); - default: - UNREACHABLE_MSG("unexpected m_currentWriteBarrier!"); - }; -#undef MARKED_FUNCTION_SIZE -} - -size_t WriteBarrierManager::GetCurrentWriteBarrierSize() -{ - return GetSpecificWriteBarrierSize(m_currentWriteBarrier); -} - -PBYTE WriteBarrierManager::CalculatePatchLocation(LPVOID base, LPVOID label, int offset) -{ - // the label should always come after the entrypoint for this funtion - _ASSERTE_ALL_BUILDS((LPBYTE)label > (LPBYTE)base); - - return (GetWriteBarrierCodeLocation((void*)JIT_WriteBarrier) + ((LPBYTE)GetEEFuncEntryPoint(label) - (LPBYTE)GetEEFuncEntryPoint(base) + offset)); -} - - -int WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, bool isRuntimeSuspended) -{ - GCX_MAYBE_COOP_NO_THREAD_BROKEN((!isRuntimeSuspended && GetThreadNULLOk() != NULL)); - int stompWBCompleteActions = SWB_PASS; - if (!isRuntimeSuspended && m_currentWriteBarrier != WRITE_BARRIER_UNINITIALIZED) - { - ThreadSuspend::SuspendEE(ThreadSuspend::SUSPEND_FOR_GC_PREP); - stompWBCompleteActions |= SWB_EE_RESTART; - } - - _ASSERTE(m_currentWriteBarrier != newWriteBarrier); - m_currentWriteBarrier = newWriteBarrier; - - // the memcpy must come before the switch statement because the asserts inside the switch - // are actually looking into the JIT_WriteBarrier buffer - { - ExecutableWriterHolder writeBarrierWriterHolder(GetWriteBarrierCodeLocation((void*)JIT_WriteBarrier), GetCurrentWriteBarrierSize()); - memcpy(writeBarrierWriterHolder.GetRW(), (LPVOID)GetCurrentWriteBarrierCode(), GetCurrentWriteBarrierSize()); - stompWBCompleteActions |= SWB_ICACHE_FLUSH; - } - switch (newWriteBarrier) { case WRITE_BARRIER_PREGROW64: @@ -635,370 +501,10 @@ int WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, default: UNREACHABLE_MSG("unexpected write barrier type!"); } - - stompWBCompleteActions |= UpdateEphemeralBounds(true); - stompWBCompleteActions |= UpdateWriteWatchAndCardTableLocations(true, false); - - return stompWBCompleteActions; } #undef CALC_PATCH_LOCATION -void WriteBarrierManager::Initialize() -{ - CONTRACTL - { - MODE_ANY; - GC_NOTRIGGER; - NOTHROW; - } - CONTRACTL_END; - - - // Ensure that the generic JIT_WriteBarrier function buffer is large enough to hold any of the more specific - // write barrier implementations. - size_t cbWriteBarrierBuffer = GetSpecificWriteBarrierSize(WRITE_BARRIER_BUFFER); - - _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_PREGROW64)); - _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_POSTGROW64)); -#ifdef FEATURE_SVR_GC - _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_SVR64)); -#endif // FEATURE_SVR_GC - _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_BYTE_REGIONS64)); - _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_BIT_REGIONS64)); -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_WRITE_WATCH_PREGROW64)); - _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_WRITE_WATCH_POSTGROW64)); -#ifdef FEATURE_SVR_GC - _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_WRITE_WATCH_SVR64)); -#endif // FEATURE_SVR_GC - _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64)); - _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64)); -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - -#if !defined(CODECOVERAGE) - Validate(); -#endif -} - -bool WriteBarrierManager::NeedDifferentWriteBarrier(bool bReqUpperBoundsCheck, bool bUseBitwiseWriteBarrier, WriteBarrierType* pNewWriteBarrierType) -{ - // Init code for the JIT_WriteBarrier assembly routine. Since it will be bashed everytime the GC Heap - // changes size, we want to do most of the work just once. - // - // The actual JIT_WriteBarrier routine will only be called in free builds, but we keep this code (that - // modifies it) around in debug builds to check that it works (with assertions). - - - WriteBarrierType writeBarrierType = m_currentWriteBarrier; - - for(;;) - { - switch (writeBarrierType) - { - case WRITE_BARRIER_UNINITIALIZED: -#ifdef _DEBUG - // The default slow write barrier has some good asserts - if ((g_pConfig->GetHeapVerifyLevel() & EEConfig::HEAPVERIFY_BARRIERCHECK)) { - break; - } -#endif - if (g_region_shr != 0) - { - writeBarrierType = bUseBitwiseWriteBarrier ? WRITE_BARRIER_BIT_REGIONS64: WRITE_BARRIER_BYTE_REGIONS64; - } - else - { - writeBarrierType = GCHeapUtilities::IsServerHeap() ? WRITE_BARRIER_SVR64 : WRITE_BARRIER_PREGROW64; - } - continue; - - case WRITE_BARRIER_PREGROW64: - if (bReqUpperBoundsCheck) - { - writeBarrierType = WRITE_BARRIER_POSTGROW64; - } - break; - - case WRITE_BARRIER_POSTGROW64: - break; - -#ifdef FEATURE_SVR_GC - case WRITE_BARRIER_SVR64: - break; -#endif // FEATURE_SVR_GC - - case WRITE_BARRIER_BYTE_REGIONS64: - case WRITE_BARRIER_BIT_REGIONS64: - break; - -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - case WRITE_BARRIER_WRITE_WATCH_PREGROW64: - if (bReqUpperBoundsCheck) - { - writeBarrierType = WRITE_BARRIER_WRITE_WATCH_POSTGROW64; - } - break; - - case WRITE_BARRIER_WRITE_WATCH_POSTGROW64: - break; - -#ifdef FEATURE_SVR_GC - case WRITE_BARRIER_WRITE_WATCH_SVR64: - break; -#endif // FEATURE_SVR_GC - case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: - case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: - break; -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - - default: - UNREACHABLE_MSG("unexpected write barrier type!"); - } - break; - } - - *pNewWriteBarrierType = writeBarrierType; - return m_currentWriteBarrier != writeBarrierType; -} - -int WriteBarrierManager::UpdateEphemeralBounds(bool isRuntimeSuspended) -{ - WriteBarrierType newType; - if (NeedDifferentWriteBarrier(false, g_region_use_bitwise_write_barrier, &newType)) - { - return ChangeWriteBarrierTo(newType, isRuntimeSuspended); - } - - int stompWBCompleteActions = SWB_PASS; - -#ifdef _DEBUG - // Using debug-only write barrier? - if (m_currentWriteBarrier == WRITE_BARRIER_UNINITIALIZED) - return stompWBCompleteActions; -#endif - - switch (m_currentWriteBarrier) - { - case WRITE_BARRIER_POSTGROW64: - case WRITE_BARRIER_BYTE_REGIONS64: - case WRITE_BARRIER_BIT_REGIONS64: -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - case WRITE_BARRIER_WRITE_WATCH_POSTGROW64: - case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: - case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - { - // Change immediate if different from new g_ephermeral_high. - if (*(UINT64*)m_pUpperBoundImmediate != (size_t)g_ephemeral_high) - { - ExecutableWriterHolder upperBoundWriterHolder((UINT64*)m_pUpperBoundImmediate, sizeof(UINT64)); - *upperBoundWriterHolder.GetRW() = (size_t)g_ephemeral_high; - stompWBCompleteActions |= SWB_ICACHE_FLUSH; - } - } - FALLTHROUGH; - case WRITE_BARRIER_PREGROW64: -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - case WRITE_BARRIER_WRITE_WATCH_PREGROW64: -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - { - // Change immediate if different from new g_ephermeral_low. - if (*(UINT64*)m_pLowerBoundImmediate != (size_t)g_ephemeral_low) - { - ExecutableWriterHolder lowerBoundImmediateWriterHolder((UINT64*)m_pLowerBoundImmediate, sizeof(UINT64)); - *lowerBoundImmediateWriterHolder.GetRW() = (size_t)g_ephemeral_low; - stompWBCompleteActions |= SWB_ICACHE_FLUSH; - } - break; - } - -#ifdef FEATURE_SVR_GC - case WRITE_BARRIER_SVR64: -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - case WRITE_BARRIER_WRITE_WATCH_SVR64: -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - { - break; - } -#endif // FEATURE_SVR_GC - - default: - UNREACHABLE_MSG("unexpected m_currentWriteBarrier in UpdateEphemeralBounds"); - } - - return stompWBCompleteActions; -} - -int WriteBarrierManager::UpdateWriteWatchAndCardTableLocations(bool isRuntimeSuspended, bool bReqUpperBoundsCheck) -{ - // If we are told that we require an upper bounds check (GC did some heap reshuffling), - // we need to switch to the WriteBarrier_PostGrow function for good. - - WriteBarrierType newType; - if (NeedDifferentWriteBarrier(bReqUpperBoundsCheck, g_region_use_bitwise_write_barrier, &newType)) - { - return ChangeWriteBarrierTo(newType, isRuntimeSuspended); - } - - int stompWBCompleteActions = SWB_PASS; - -#ifdef _DEBUG - // Using debug-only write barrier? - if (m_currentWriteBarrier == WRITE_BARRIER_UNINITIALIZED) - return stompWBCompleteActions; -#endif - -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - switch (m_currentWriteBarrier) - { - case WRITE_BARRIER_WRITE_WATCH_PREGROW64: - case WRITE_BARRIER_WRITE_WATCH_POSTGROW64: -#ifdef FEATURE_SVR_GC - case WRITE_BARRIER_WRITE_WATCH_SVR64: -#endif // FEATURE_SVR_GC - case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: - case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: - if (*(UINT64*)m_pWriteWatchTableImmediate != (size_t)g_sw_ww_table) - { - ExecutableWriterHolder writeWatchTableImmediateWriterHolder((UINT64*)m_pWriteWatchTableImmediate, sizeof(UINT64)); - *writeWatchTableImmediateWriterHolder.GetRW() = (size_t)g_sw_ww_table; - stompWBCompleteActions |= SWB_ICACHE_FLUSH; - } - break; - - default: - break; // clang seems to require all enum values to be covered for some reason - } -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - - switch (m_currentWriteBarrier) - { - case WRITE_BARRIER_BYTE_REGIONS64: - case WRITE_BARRIER_BIT_REGIONS64: - case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: - case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: - if (*(UINT64*)m_pRegionToGenTableImmediate != (size_t)g_region_to_generation_table) - { - ExecutableWriterHolder writeWatchTableImmediateWriterHolder((UINT64*)m_pRegionToGenTableImmediate, sizeof(UINT64)); - *writeWatchTableImmediateWriterHolder.GetRW() = (size_t)g_region_to_generation_table; - stompWBCompleteActions |= SWB_ICACHE_FLUSH; - } - if (*m_pRegionShrDest != g_region_shr) - { - ExecutableWriterHolder writeWatchTableImmediateWriterHolder(m_pRegionShrDest, sizeof(UINT8)); - *writeWatchTableImmediateWriterHolder.GetRW() = g_region_shr; - stompWBCompleteActions |= SWB_ICACHE_FLUSH; - } - if (*m_pRegionShrSrc != g_region_shr) - { - ExecutableWriterHolder writeWatchTableImmediateWriterHolder(m_pRegionShrSrc, sizeof(UINT8)); - *writeWatchTableImmediateWriterHolder.GetRW() = g_region_shr; - stompWBCompleteActions |= SWB_ICACHE_FLUSH; - } - break; - - default: - break; // clang seems to require all enum values to be covered for some reason - } - - if (*(UINT64*)m_pCardTableImmediate != (size_t)g_card_table) - { - ExecutableWriterHolder cardTableImmediateWriterHolder((UINT64*)m_pCardTableImmediate, sizeof(UINT64)); - *cardTableImmediateWriterHolder.GetRW() = (size_t)g_card_table; - stompWBCompleteActions |= SWB_ICACHE_FLUSH; - } - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - if (*(UINT64*)m_pCardBundleTableImmediate != (size_t)g_card_bundle_table) - { - ExecutableWriterHolder cardBundleTableImmediateWriterHolder((UINT64*)m_pCardBundleTableImmediate, sizeof(UINT64)); - *cardBundleTableImmediateWriterHolder.GetRW() = (size_t)g_card_bundle_table; - stompWBCompleteActions |= SWB_ICACHE_FLUSH; - } -#endif - - return stompWBCompleteActions; -} - -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -int WriteBarrierManager::SwitchToWriteWatchBarrier(bool isRuntimeSuspended) -{ - WriteBarrierType newWriteBarrierType; - switch (m_currentWriteBarrier) - { - case WRITE_BARRIER_UNINITIALIZED: - // Using the debug-only write barrier - return SWB_PASS; - - case WRITE_BARRIER_PREGROW64: - newWriteBarrierType = WRITE_BARRIER_WRITE_WATCH_PREGROW64; - break; - - case WRITE_BARRIER_POSTGROW64: - newWriteBarrierType = WRITE_BARRIER_WRITE_WATCH_POSTGROW64; - break; - -#ifdef FEATURE_SVR_GC - case WRITE_BARRIER_SVR64: - newWriteBarrierType = WRITE_BARRIER_WRITE_WATCH_SVR64; - break; -#endif // FEATURE_SVR_GC - - case WRITE_BARRIER_BYTE_REGIONS64: - newWriteBarrierType = WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64; - break; - - case WRITE_BARRIER_BIT_REGIONS64: - newWriteBarrierType = WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64; - break; - - default: - UNREACHABLE(); - } - - return ChangeWriteBarrierTo(newWriteBarrierType, isRuntimeSuspended); -} - -int WriteBarrierManager::SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended) -{ - WriteBarrierType newWriteBarrierType; - switch (m_currentWriteBarrier) - { - case WRITE_BARRIER_UNINITIALIZED: - // Using the debug-only write barrier - return SWB_PASS; - - case WRITE_BARRIER_WRITE_WATCH_PREGROW64: - newWriteBarrierType = WRITE_BARRIER_PREGROW64; - break; - - case WRITE_BARRIER_WRITE_WATCH_POSTGROW64: - newWriteBarrierType = WRITE_BARRIER_POSTGROW64; - break; - -#ifdef FEATURE_SVR_GC - case WRITE_BARRIER_WRITE_WATCH_SVR64: - newWriteBarrierType = WRITE_BARRIER_SVR64; - break; -#endif // FEATURE_SVR_GC - - case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: - newWriteBarrierType = WRITE_BARRIER_BYTE_REGIONS64; - break; - - case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: - newWriteBarrierType = WRITE_BARRIER_BIT_REGIONS64; - break; - - default: - UNREACHABLE(); - } - - return ChangeWriteBarrierTo(newWriteBarrierType, isRuntimeSuspended); -} -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - // This function bashes the super fast amd64 version of the JIT_WriteBarrier // helper. It should be called by the GC whenever the ephermeral region // bounds get changed, but still remain on the top of the GC Heap. diff --git a/src/coreclr/vm/arm64/JitHelpers_FastWriteBarriers.S b/src/coreclr/vm/arm64/JitHelpers_FastWriteBarriers.S new file mode 100644 index 00000000000000..158022c1023857 --- /dev/null +++ b/src/coreclr/vm/arm64/JitHelpers_FastWriteBarriers.S @@ -0,0 +1,55 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#include "asmconstants.h" +#include "unixasmmacros.inc" + + +LEAF_ENTRY JIT_WriteBarrier_PostGrow64, _TEXT + nop +LEAF_END_MARKED JIT_WriteBarrier_PostGrow64, _TEXT + + +LEAF_ENTRY JIT_WriteBarrier_PreGrow64, _TEXT + nop +LEAF_END_MARKED JIT_WriteBarrier_PreGrow64, _TEXT + + +LEAF_ENTRY JIT_WriteBarrier_SVR64, _TEXT + nop +LEAF_END_MARKED JIT_WriteBarrier_SVR64, _TEXT + + +LEAF_ENTRY JIT_WriteBarrier_Byte_Region64, _TEXT + nop +LEAF_END_MARKED JIT_WriteBarrier_Byte_Region64, _TEXT + + +LEAF_ENTRY JIT_WriteBarrier_Bit_Region64, _TEXT + nop +LEAF_END_MARKED JIT_WriteBarrier_Bit_Region64, _TEXT + + +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT + nop +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT + + +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT + nop +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT + + +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_SVR64, _TEXT + nop +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_SVR64, _TEXT + + +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT + nop +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT + + +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT + nop +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index 87c7507d9833ed..1389919b38456f 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -241,29 +241,41 @@ WRITE_BARRIER_END JIT_WriteBarrier WRITE_BARRIER_ENTRY JIT_WriteBarrier_Table LOCAL_LABEL(wbs_begin): LOCAL_LABEL(wbs_card_table): +PATCH_LABEL JIT_WriteBarrier_Patch_Label_CardTable .quad 0 LOCAL_LABEL(wbs_card_bundle_table): +PATCH_LABEL JIT_WriteBarrier_Patch_Label_CardBundleTable .quad 0 LOCAL_LABEL(wbs_sw_ww_table): +PATCH_LABEL JIT_WriteBarrier_Patch_Label_WriteWatchTable .quad 0 LOCAL_LABEL(wbs_ephemeral_low): +PATCH_LABEL JIT_WriteBarrier_Patch_Label_Lower .quad 0 LOCAL_LABEL(wbs_ephemeral_high): +PATCH_LABEL JIT_WriteBarrier_Patch_Label_Upper .quad 0 LOCAL_LABEL(wbs_lowest_address): +PATCH_LABEL JIT_WriteBarrier_Patch_Label_LowestAddress .quad 0 LOCAL_LABEL(wbs_highest_address): +PATCH_LABEL JIT_WriteBarrier_Patch_Label_HighestAddress .quad 0 LOCAL_LABEL(wbs_region_to_generation_table): +PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionToGeneration .quad 0 LOCAL_LABEL(wbs_region_shr): +PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionShr .word 0 LOCAL_LABEL(wbs_region_use_bitwise_write_barrier): +PATCH_LABEL JIT_WriteBarrier_Patch_Label_UseBitwiseWriteBarrier .word 0 #ifdef WRITE_BARRIER_CHECK LOCAL_LABEL(wbs_GCShadow): +PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadow .quad 0 LOCAL_LABEL(wbs_GCShadowEnd): +PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadowEnd .quad 0 #endif WRITE_BARRIER_END JIT_WriteBarrier_Table diff --git a/src/coreclr/vm/arm64/patchedcode.asm b/src/coreclr/vm/arm64/patchedcode.asm index 454b8cac0c4abe..fbc623329e43f7 100644 --- a/src/coreclr/vm/arm64/patchedcode.asm +++ b/src/coreclr/vm/arm64/patchedcode.asm @@ -18,7 +18,7 @@ MACRO WRITE_BARRIER_ENTRY $name - LEAF_ENTRY $name + WRITE_BARRIER_ENTRY $name MEND ; WRITE_BARRIER_END @@ -28,13 +28,13 @@ MACRO WRITE_BARRIER_END $__write_barrier_name - LEAF_END_MARKED $__write_barrier_name + WRITE_BARRIER_END $__write_barrier_name MEND ; ------------------------------------------------------------------ ; Start of the writeable code region - LEAF_ENTRY JIT_PatchedCodeStart + WRITE_BARRIER_ENTRY JIT_PatchedCodeStart ret lr LEAF_END @@ -229,9 +229,55 @@ Exit ; ------------------------------------------------------------------ ; End of the writeable code region - LEAF_ENTRY JIT_PatchedCodeLast + WRITE_BARRIER_ENTRY JIT_PatchedCodeLast ret lr LEAF_END + + +WRITE_BARRIER_ENTRY JIT_WriteBarrier_PreGrow64 + nop +WRITE_BARRIER_END JIT_WriteBarrier_PreGrow64 + + +WRITE_BARRIER_ENTRY JIT_WriteBarrier_PreGrow64 + nop +WRITE_BARRIER_END JIT_WriteBarrier_PreGrow64 + + +WRITE_BARRIER_ENTRY JIT_WriteBarrier_SVR64 + nop +WRITE_BARRIER_END JIT_WriteBarrier_SVR64 + + +WRITE_BARRIER_ENTRY JIT_WriteBarrier_Byte_Region64 + nop +WRITE_BARRIER_END JIT_WriteBarrier_Byte_Region64 + + +WRITE_BARRIER_ENTRY JIT_WriteBarrier_Bit_Region64 + nop +WRITE_BARRIER_END JIT_WriteBarrier_Bit_Region64 + + +WRITE_BARRIER_ENTRY JIT_WriteBarrier_WriteWatch_PreGrow64 + nop +WRITE_BARRIER_END JIT_WriteBarrier_WriteWatch_PreGrow64 + + +WRITE_BARRIER_ENTRY JIT_WriteBarrier_WriteWatch_SVR64 + nop +WRITE_BARRIER_END JIT_WriteBarrier_WriteWatch_SVR64 + + +WRITE_BARRIER_ENTRY JIT_WriteBarrier_WriteWatch_Byte_Region64 + nop +WRITE_BARRIER_END JIT_WriteBarrier_WriteWatch_Byte_Region64 + + +WRITE_BARRIER_ENTRY JIT_WriteBarrier_WriteWatch_Bit_Region64 + nop +WRITE_BARRIER_END JIT_WriteBarrier_WriteWatch_Bit_Region64 + ; Must be at very end of file END diff --git a/src/coreclr/vm/arm64/stubs.cpp b/src/coreclr/vm/arm64/stubs.cpp index 71146e9de29422..208e2c435d835d 100644 --- a/src/coreclr/vm/arm64/stubs.cpp +++ b/src/coreclr/vm/arm64/stubs.cpp @@ -14,6 +14,7 @@ #include "jitinterface.h" #include "ecall.h" +extern WriteBarrierManager g_WriteBarrierManager; #ifndef DACCESS_COMPILE //----------------------------------------------------------------------- @@ -860,31 +861,6 @@ void emitCOMStubCall (ComCallMethodDesc *pCOMMethodRX, ComCallMethodDesc *pCOMMe #endif // FEATURE_COMINTEROP #if !defined(DACCESS_COMPILE) -EXTERN_C void JIT_UpdateWriteBarrierState(bool skipEphemeralCheck, size_t writeableOffset); - -extern "C" void STDCALL JIT_PatchedCodeStart(); -extern "C" void STDCALL JIT_PatchedCodeLast(); - -static void UpdateWriteBarrierState() -{ - BYTE *writeBarrierCodeStart = GetWriteBarrierCodeLocation((void*)JIT_PatchedCodeStart); - BYTE *writeBarrierCodeStartRW = writeBarrierCodeStart; - ExecutableWriterHolderNoLog writeBarrierWriterHolder; - if (IsWriteBarrierCopyEnabled()) - { - writeBarrierWriterHolder.AssignExecutableWriterHolder(writeBarrierCodeStart, (BYTE*)JIT_PatchedCodeLast - (BYTE*)JIT_PatchedCodeStart); - writeBarrierCodeStartRW = writeBarrierWriterHolder.GetRW(); - } - - // Skip ephemeral checks for regionless server GC - bool skipEphemeralCheck = false; - if (GCHeapUtilities::IsServerHeap() && g_region_to_generation_table == nullptr) - { - skipEphemeralCheck = true; - } - - JIT_UpdateWriteBarrierState(skipEphemeralCheck, writeBarrierCodeStartRW - writeBarrierCodeStart); -} void InitJITHelpers1() { @@ -892,6 +868,8 @@ void InitJITHelpers1() _ASSERTE(g_SystemInfo.dwNumberOfProcessors != 0); + g_WriteBarrierManager.Initialize(); + // Allocation helpers, faster but non-logging if (!((TrackAllocationsEnabled()) || (LoggingOn(LF_GCALLOC, LL_INFO10)) @@ -911,13 +889,8 @@ void InitJITHelpers1() ECall::DynamicallyAssignFCallImpl(GetEEFuncEntryPoint(AllocateString_MP_FastPortable), ECall::FastAllocateString); } } - - UpdateWriteBarrierState(); } - -#else -void UpdateWriteBarrierState() {} #endif // !defined(DACCESS_COMPILE) #ifdef TARGET_WINDOWS @@ -1075,37 +1048,58 @@ LONG CLRNoCatchHandler(EXCEPTION_POINTERS* pExceptionInfo, PVOID pv) return EXCEPTION_CONTINUE_SEARCH; } -void FlushWriteBarrierInstructionCache() +void WriteBarrierManager::Validate() { - // this wouldn't be called in arm64, just to comply with gchelpers.h + // Nothing to do } +void WriteBarrierManager::UpdatePatchLocations() +{ + // Nothing to do +} + +// This function bashes the super fast amd64 version of the JIT_WriteBarrier +// helper. It should be called by the GC whenever the ephermeral region +// bounds get changed, but still remain on the top of the GC Heap. int StompWriteBarrierEphemeral(bool isRuntimeSuspended) { - UpdateWriteBarrierState(); - return SWB_PASS; + WRAPPER_NO_CONTRACT; + + return g_WriteBarrierManager.UpdateEphemeralBounds(isRuntimeSuspended); } +// This function bashes the super fast amd64 versions of the JIT_WriteBarrier +// helpers. It should be called by the GC whenever the ephermeral region gets moved +// from being at the top of the GC Heap, and/or when the cards table gets moved. int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck) { - UpdateWriteBarrierState(); - return SWB_PASS; + WRAPPER_NO_CONTRACT; + + return g_WriteBarrierManager.UpdateWriteWatchAndCardTableLocations(isRuntimeSuspended, bReqUpperBoundsCheck); +} + +void FlushWriteBarrierInstructionCache() +{ + FlushInstructionCache(GetCurrentProcess(), GetWriteBarrierCodeLocation((PVOID)JIT_WriteBarrier), g_WriteBarrierManager.GetCurrentWriteBarrierSize()); } #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP int SwitchToWriteWatchBarrier(bool isRuntimeSuspended) { - UpdateWriteBarrierState(); - return SWB_PASS; + WRAPPER_NO_CONTRACT; + + return g_WriteBarrierManager.SwitchToWriteWatchBarrier(isRuntimeSuspended); } int SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended) { - UpdateWriteBarrierState(); - return SWB_PASS; + WRAPPER_NO_CONTRACT; + + return g_WriteBarrierManager.SwitchToNonWriteWatchBarrier(isRuntimeSuspended); } #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + #ifdef DACCESS_COMPILE BOOL GetAnyThunkTarget (T_CONTEXT *pctx, TADDR *pTarget, TADDR *pTargetMethodDesc) { diff --git a/src/coreclr/vm/gcenv.ee.cpp b/src/coreclr/vm/gcenv.ee.cpp index d065ef36a2ff5e..cfb265e18c5a92 100644 --- a/src/coreclr/vm/gcenv.ee.cpp +++ b/src/coreclr/vm/gcenv.ee.cpp @@ -1110,13 +1110,11 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) #endif stompWBCompleteActions |= ::StompWriteBarrierResize(true, false); -#if !defined(HOST_ARM64) // StompWriteBarrierResize does not necessarily bash g_ephemeral_low - // usages, so we must do so here. This is particularly true on x86, + // usages, so we must do so here. This is particularly true on x86/Arm64, // where StompWriteBarrierResize will not bash g_ephemeral_low when // called with the parameters (true, false), as it is above. stompWBCompleteActions |= ::StompWriteBarrierEphemeral(true); -#endif break; case WriteBarrierOp::SwitchToWriteWatch: diff --git a/src/coreclr/vm/jitinterface.h b/src/coreclr/vm/jitinterface.h index 7f1835e458a53a..23b96831b8d6df 100644 --- a/src/coreclr/vm/jitinterface.h +++ b/src/coreclr/vm/jitinterface.h @@ -242,7 +242,7 @@ extern "C" FCDECL2(VOID, JIT_WriteBarrier_Callable, Object **dst, Object *ref); #define WriteBarrier_Helper JIT_WriteBarrier_Callable -#ifdef TARGET_AMD64 +// #ifdef TARGET_AMD64 class WriteBarrierManager @@ -286,7 +286,8 @@ class WriteBarrierManager size_t GetSpecificWriteBarrierSize(WriteBarrierType writeBarrier); PBYTE CalculatePatchLocation(LPVOID base, LPVOID label, int offset); PCODE GetCurrentWriteBarrierCode(); - int ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, bool isRuntimeSuspended); + int ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, bool isRuntimeSuspended); + void UpdatePatchLocations(); bool NeedDifferentWriteBarrier(bool bReqUpperBoundsCheck, bool bUseBitwiseWriteBarrier, WriteBarrierType* pNewWriteBarrierType); private: @@ -302,10 +303,21 @@ class WriteBarrierManager PBYTE m_pRegionToGenTableImmediate; // | | | WRITE_WATCH | REGION PBYTE m_pRegionShrDest; // | | | WRITE_WATCH | REGION PBYTE m_pRegionShrSrc; // | | | WRITE_WATCH | RETION -}; +#if defined(TARGET_ARM64) + PBYTE m_pRegionUseBitwiseWriteBarrier; + PBYTE m_lowestAddress; + PBYTE m_highestAddress; +#if defined(WRITE_BARRIER_CHECK) + PBYTE m_pGCShadow; + PBYTE m_pGCShadowEnd; +#endif // WRITE_BARRIER_CHECK #endif // TARGET_AMD64 +}; + +// #endif // TARGET_AMD64 + EXTERN_C FCDECL2_VV(INT64, JIT_LMul, INT64 val1, INT64 val2); #ifndef HOST_64BIT diff --git a/src/coreclr/vm/writebarriermanager.cpp b/src/coreclr/vm/writebarriermanager.cpp new file mode 100644 index 00000000000000..bca55e9375db03 --- /dev/null +++ b/src/coreclr/vm/writebarriermanager.cpp @@ -0,0 +1,638 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// =========================================================================== +// File: writebariermanager.cpp +// =========================================================================== + +// This contains JITinterface routines for managing which write barrier function +// is currently in use, and patching all related constants. + +#include "common.h" +#include "jitinterface.h" +#include "eeconfig.h" +#include "excep.h" +#include "threadsuspend.h" + +extern uint8_t* g_ephemeral_low; +extern uint8_t* g_ephemeral_high; +extern uint32_t* g_card_table; +extern uint32_t* g_card_bundle_table; + +#if defined(TARGET_AMD64) +// Write barrier variables are inlined into the assembly code +#define WRITE_BARRIER_VARS_INLINE +// Else: Write barrier variables are in a table separate to the asm code +#endif + + +// Patch Labels for the various write barriers +EXTERN_C void JIT_WriteBarrier_End(); +EXTERN_C void JIT_WriteBarrier_PreGrow64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_PreGrow64_End(); +EXTERN_C void JIT_WriteBarrier_PostGrow64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_PostGrow64_End(); +#ifdef FEATURE_SVR_GC +EXTERN_C void JIT_WriteBarrier_SVR64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_SVR64_End(); +#endif // FEATURE_SVR_GC +EXTERN_C void JIT_WriteBarrier_Byte_Region64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_Byte_Region64_End(); +EXTERN_C void JIT_WriteBarrier_Bit_Region64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_Bit_Region64_End(); +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP +EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_End(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_End(); +#ifdef FEATURE_SVR_GC +EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_End(); +#endif // FEATURE_SVR_GC +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_End(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64(Object **dst, Object *ref); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_End(); +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + + +#if !defined(WRITE_BARRIER_VARS_INLINE) +EXTERN_C void JIT_WriteBarrier_Patch_Label_WriteWatchTable(); +EXTERN_C void JIT_WriteBarrier_Patch_Label_RegionToGeneration(); +EXTERN_C void JIT_WriteBarrier_Patch_Label_RegionShr(); +EXTERN_C void JIT_WriteBarrier_Patch_Label_Lower(); +EXTERN_C void JIT_WriteBarrier_Patch_Label_Upper(); +EXTERN_C void JIT_WriteBarrier_Patch_Label_CardTable(); +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +EXTERN_C void JIT_WriteBarrier_Patch_Label_CardBundleTable(); +#endif +#if defined(TARGET_ARM64) +EXTERN_C void JIT_WriteBarrier_Patch_Label_UseBitwiseWriteBarrier(); +EXTERN_C void JIT_WriteBarrier_Patch_Label_LowestAddress(); +EXTERN_C void JIT_WriteBarrier_Patch_Label_HighestAddress(); +#if defined(WRITE_BARRIER_CHECK) +EXTERN_C void JIT_WriteBarrier_Patch_Label_GCShadow(); +EXTERN_C void JIT_WriteBarrier_Patch_Label_GCShadowEnd(); +#endif // WRITE_BARRIER_CHECK +#endif // TARGET_ARM64 + +#endif // !WRITE_BARRIER_VARS_INLINE + + +WriteBarrierManager g_WriteBarrierManager; + +WriteBarrierManager::WriteBarrierManager() : + m_currentWriteBarrier(WRITE_BARRIER_UNINITIALIZED) +{ + LIMITED_METHOD_CONTRACT; +} + +PCODE WriteBarrierManager::GetCurrentWriteBarrierCode() +{ + LIMITED_METHOD_CONTRACT; + + return GetEEFuncEntryPoint(JIT_WriteBarrier); +// switch (m_currentWriteBarrier) +// { +// case WRITE_BARRIER_PREGROW64: +// return GetEEFuncEntryPoint(JIT_WriteBarrier_PreGrow64); +// case WRITE_BARRIER_POSTGROW64: +// return GetEEFuncEntryPoint(JIT_WriteBarrier_PostGrow64); +// #ifdef FEATURE_SVR_GC +// case WRITE_BARRIER_SVR64: +// return GetEEFuncEntryPoint(JIT_WriteBarrier_SVR64); +// #endif // FEATURE_SVR_GC +// case WRITE_BARRIER_BYTE_REGIONS64: +// return GetEEFuncEntryPoint(JIT_WriteBarrier_Byte_Region64); +// case WRITE_BARRIER_BIT_REGIONS64: +// return GetEEFuncEntryPoint(JIT_WriteBarrier_Bit_Region64); +// #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP +// case WRITE_BARRIER_WRITE_WATCH_PREGROW64: +// return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_PreGrow64); +// case WRITE_BARRIER_WRITE_WATCH_POSTGROW64: +// return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_PostGrow64); +// #ifdef FEATURE_SVR_GC +// case WRITE_BARRIER_WRITE_WATCH_SVR64: +// return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_SVR64); +// #endif // FEATURE_SVR_GC +// case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: +// return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_Byte_Region64); +// case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: +// return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_Bit_Region64); +// #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP +// default: +// UNREACHABLE_MSG("unexpected m_currentWriteBarrier!"); +// }; +} + +size_t WriteBarrierManager::GetSpecificWriteBarrierSize(WriteBarrierType writeBarrier) +{ +// marked asm functions are those which use the LEAF_END_MARKED macro to end them which +// creates a public Name_End label which can be used to figure out their size without +// having to create unwind info. +#define MARKED_FUNCTION_SIZE(pfn) (size_t)((LPBYTE)GetEEFuncEntryPoint(pfn##_End) - (LPBYTE)GetEEFuncEntryPoint(pfn)) + + switch (writeBarrier) + { + case WRITE_BARRIER_PREGROW64: + return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_PreGrow64); + case WRITE_BARRIER_POSTGROW64: + return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_PostGrow64); +#ifdef FEATURE_SVR_GC + case WRITE_BARRIER_SVR64: + return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_SVR64); +#endif // FEATURE_SVR_GC + case WRITE_BARRIER_BYTE_REGIONS64: + return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_Byte_Region64); + case WRITE_BARRIER_BIT_REGIONS64: + return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_Bit_Region64); +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + case WRITE_BARRIER_WRITE_WATCH_PREGROW64: + return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_WriteWatch_PreGrow64); + case WRITE_BARRIER_WRITE_WATCH_POSTGROW64: + return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_WriteWatch_PostGrow64); +#ifdef FEATURE_SVR_GC + case WRITE_BARRIER_WRITE_WATCH_SVR64: + return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_WriteWatch_SVR64); +#endif // FEATURE_SVR_GC + case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: + return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_WriteWatch_Byte_Region64); + case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: + return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_WriteWatch_Bit_Region64); +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + case WRITE_BARRIER_BUFFER: + return MARKED_FUNCTION_SIZE(JIT_WriteBarrier); + default: + UNREACHABLE_MSG("unexpected m_currentWriteBarrier!"); + }; +#undef MARKED_FUNCTION_SIZE +} + +size_t WriteBarrierManager::GetCurrentWriteBarrierSize() +{ + return GetSpecificWriteBarrierSize(m_currentWriteBarrier); +} + + +// +// m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_WriteWatchTable, 0); + +PBYTE WriteBarrierManager::CalculatePatchLocation(LPVOID base, LPVOID label, int offset) +{ + // the label should always come after or at the entrypoint for this funtion + _ASSERTE_ALL_BUILDS((LPBYTE)label >= (LPBYTE)base); + + BYTE* patchBase = GetWriteBarrierCodeLocation((void*)JIT_WriteBarrier); + return (patchBase + ((LPBYTE)GetEEFuncEntryPoint(label) - (LPBYTE)GetEEFuncEntryPoint(base) + offset)); +} + + +int WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, bool isRuntimeSuspended) +{ + GCX_MAYBE_COOP_NO_THREAD_BROKEN((!isRuntimeSuspended && GetThreadNULLOk() != NULL)); + int stompWBCompleteActions = SWB_PASS; + if (!isRuntimeSuspended && m_currentWriteBarrier != WRITE_BARRIER_UNINITIALIZED) + { + ThreadSuspend::SuspendEE(ThreadSuspend::SUSPEND_FOR_GC_PREP); + stompWBCompleteActions |= SWB_EE_RESTART; + } + + _ASSERTE(m_currentWriteBarrier != newWriteBarrier); + m_currentWriteBarrier = newWriteBarrier; + + // the memcpy must come before the switch statement because the asserts inside the switch + // are actually looking into the JIT_WriteBarrier buffer + // { + // ExecutableWriterHolder writeBarrierWriterHolder(GetWriteBarrierCodeLocation((void*)JIT_WriteBarrier), GetCurrentWriteBarrierSize()); + // memcpy(writeBarrierWriterHolder.GetRW(), (LPVOID)GetCurrentWriteBarrierCode(), GetCurrentWriteBarrierSize()); + // stompWBCompleteActions |= SWB_ICACHE_FLUSH; + // } + + UpdatePatchLocations(); + + stompWBCompleteActions |= UpdateEphemeralBounds(true); + stompWBCompleteActions |= UpdateWriteWatchAndCardTableLocations(true, false); + + return stompWBCompleteActions; +} + + +void WriteBarrierManager::Initialize() +{ + CONTRACTL + { + MODE_ANY; + GC_NOTRIGGER; + NOTHROW; + } + CONTRACTL_END; + + + // Ensure that the generic JIT_WriteBarrier function buffer is large enough to hold any of the more specific + // write barrier implementations. + size_t cbWriteBarrierBuffer = GetSpecificWriteBarrierSize(WRITE_BARRIER_BUFFER); + + _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_PREGROW64)); + _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_POSTGROW64)); +#ifdef FEATURE_SVR_GC + _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_SVR64)); +#endif // FEATURE_SVR_GC + _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_BYTE_REGIONS64)); + _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_BIT_REGIONS64)); +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_WRITE_WATCH_PREGROW64)); + _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_WRITE_WATCH_POSTGROW64)); +#ifdef FEATURE_SVR_GC + _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_WRITE_WATCH_SVR64)); +#endif // FEATURE_SVR_GC + _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64)); + _ASSERTE_ALL_BUILDS(cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64)); +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + + +#if !defined(WRITE_BARRIER_VARS_INLINE) + /// TODO: Want to move all the WriteBarrierManager code from AMD64 into here, but inside a define. + +#define CALC_PATCH_LOCATION(func,label,offset) CalculatePatchLocation((PVOID)func, (PVOID)func##_##label, offset) + + m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_WriteWatchTable, 0); + m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_RegionToGeneration, 0); + + // TODO: Only one of these! + m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_RegionShr, 0); + m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_RegionShr, 0); + + m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_Lower, 0); + m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_Upper, 0); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_CardTable, 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_CardBundleTable, 0); +#endif + +#if defined(TARGET_ARM64) + m_pRegionUseBitwiseWriteBarrier = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_UseBitwiseWriteBarrier, 0); +#if defined(WRITE_BARRIER_CHECK) + m_pGCShadow = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_GCShadow, 0); + m_pGCShadowEnd = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_GCShadowEnd, 0); + m_lowestAddress = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_LowestAddress, 0); + m_highestAddress = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_HighestAddress, 0); +#endif // WRITE_BARRIER_CHECK +#endif // TARGET_ARM64 + +#undef CALC_PATCH_LOCATION + +#endif // !WRITE_BARRIER_VARS_INLINE + +#if !defined(CODECOVERAGE) + Validate(); +#endif +} + +template int updateVariable(PBYTE loc, T value) +{ + if (*(T*)loc != value) + { +// #if defined(WRITE_BARRIER_VARS_INLINE) + ExecutableWriterHolder varWriterHolder((T*)loc, sizeof(T)); + *varWriterHolder.GetRW() = value; +// #else + +// BYTE *writeBarrierCodeStart = GetWriteBarrierCodeLocation((void*)JIT_PatchedCodeStart); +// BYTE *writeBarrierCodeStartRW = writeBarrierCodeStart; +// ExecutableWriterHolderNoLog writeBarrierWriterHolder; +// if (IsWriteBarrierCopyEnabled()) +// { +// writeBarrierWriterHolder.AssignExecutableWriterHolder(writeBarrierCodeStart, (BYTE*)JIT_PatchedCodeLast - (BYTE*)JIT_PatchedCodeStart); +// writeBarrierCodeStartRW = writeBarrierWriterHolder.GetRW(); +// } + +// *loc = value; +// #endif + return SWB_ICACHE_FLUSH; + } + return SWB_PASS; +} + +bool WriteBarrierManager::NeedDifferentWriteBarrier(bool bReqUpperBoundsCheck, bool bUseBitwiseWriteBarrier, WriteBarrierType* pNewWriteBarrierType) +{ + // Init code for the JIT_WriteBarrier assembly routine. Since it will be bashed everytime the GC Heap + // changes size, we want to do most of the work just once. + // + // The actual JIT_WriteBarrier routine will only be called in free builds, but we keep this code (that + // modifies it) around in debug builds to check that it works (with assertions). + + + WriteBarrierType writeBarrierType = m_currentWriteBarrier; + + for(;;) + { + switch (writeBarrierType) + { + case WRITE_BARRIER_UNINITIALIZED: +#ifdef _DEBUG + // The default slow write barrier has some good asserts + if ((g_pConfig->GetHeapVerifyLevel() & EEConfig::HEAPVERIFY_BARRIERCHECK)) { + break; + } +#endif + if (g_region_shr != 0) + { + writeBarrierType = bUseBitwiseWriteBarrier ? WRITE_BARRIER_BIT_REGIONS64: WRITE_BARRIER_BYTE_REGIONS64; + } + else + { + writeBarrierType = GCHeapUtilities::IsServerHeap() ? WRITE_BARRIER_SVR64 : WRITE_BARRIER_PREGROW64; + } + continue; + + case WRITE_BARRIER_PREGROW64: + if (bReqUpperBoundsCheck) + { + writeBarrierType = WRITE_BARRIER_POSTGROW64; + } + break; + + case WRITE_BARRIER_POSTGROW64: + break; + +#ifdef FEATURE_SVR_GC + case WRITE_BARRIER_SVR64: + break; +#endif // FEATURE_SVR_GC + + case WRITE_BARRIER_BYTE_REGIONS64: + case WRITE_BARRIER_BIT_REGIONS64: + break; + +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + case WRITE_BARRIER_WRITE_WATCH_PREGROW64: + if (bReqUpperBoundsCheck) + { + writeBarrierType = WRITE_BARRIER_WRITE_WATCH_POSTGROW64; + } + break; + + case WRITE_BARRIER_WRITE_WATCH_POSTGROW64: + break; + +#ifdef FEATURE_SVR_GC + case WRITE_BARRIER_WRITE_WATCH_SVR64: + break; +#endif // FEATURE_SVR_GC + case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: + case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: + break; +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + + default: + UNREACHABLE_MSG("unexpected write barrier type!"); + } + break; + } + + *pNewWriteBarrierType = writeBarrierType; + return m_currentWriteBarrier != writeBarrierType; +} + +int WriteBarrierManager::UpdateEphemeralBounds(bool isRuntimeSuspended) +{ + WriteBarrierType newType; + if (NeedDifferentWriteBarrier(false, g_region_use_bitwise_write_barrier, &newType)) + { + return ChangeWriteBarrierTo(newType, isRuntimeSuspended); + } + + int stompWBCompleteActions = SWB_PASS; + +#ifdef _DEBUG + // Using debug-only write barrier? + if (m_currentWriteBarrier == WRITE_BARRIER_UNINITIALIZED) + return stompWBCompleteActions; +#endif + +#if defined(WRITE_BARRIER_VARS_INLINE) + + switch (m_currentWriteBarrier) + { + case WRITE_BARRIER_POSTGROW64: + case WRITE_BARRIER_BYTE_REGIONS64: + case WRITE_BARRIER_BIT_REGIONS64: +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + case WRITE_BARRIER_WRITE_WATCH_POSTGROW64: + case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: + case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + stompWBCompleteActions |= updateVariable(m_pUpperBoundImmediate, (size_t)g_ephemeral_high); + FALLTHROUGH; + + case WRITE_BARRIER_PREGROW64: +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + case WRITE_BARRIER_WRITE_WATCH_PREGROW64: +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + stompWBCompleteActions |= updateVariable(m_pLowerBoundImmediate, (size_t)g_ephemeral_low); + break; + +#ifdef FEATURE_SVR_GC + case WRITE_BARRIER_SVR64: +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + case WRITE_BARRIER_WRITE_WATCH_SVR64: +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + break; +#endif // FEATURE_SVR_GC + + default: + UNREACHABLE_MSG("unexpected m_currentWriteBarrier in UpdateEphemeralBounds"); + } + +#else + + size_t e_high = (size_t)g_ephemeral_high; + size_t e_low = (size_t)g_ephemeral_low; + + // TODO: not required once we have separate functions +#if defined(TARGET_ARM64) + if (GCHeapUtilities::IsServerHeap() && g_region_to_generation_table == nullptr) + { + e_high = 0; + e_low = 0; + } +#endif // TARGET_ARM64 + + stompWBCompleteActions |= updateVariable(m_pUpperBoundImmediate, e_high); + stompWBCompleteActions |= updateVariable(m_pLowerBoundImmediate, e_low); +#endif //WRITE_BARRIER_VARS_INLINE + + +#if defined(TARGET_ARM64) + stompWBCompleteActions |= updateVariable(m_pRegionUseBitwiseWriteBarrier, g_region_use_bitwise_write_barrier); + stompWBCompleteActions |= updateVariable(m_lowestAddress, (size_t)g_lowest_address); + stompWBCompleteActions |= updateVariable(m_highestAddress, (size_t)g_highest_address); +#if defined(WRITE_BARRIER_CHECK) + stompWBCompleteActions |= updateVariable(m_pGCShadow, (size_t)g_GCShadow); + stompWBCompleteActions |= updateVariable(m_pGCShadowEnd, (size_t)g_GCShadowEnd); +#endif // WRITE_BARRIER_CHECK +#endif // TARGET_AMD64 + + return stompWBCompleteActions; +} + +int WriteBarrierManager::UpdateWriteWatchAndCardTableLocations(bool isRuntimeSuspended, bool bReqUpperBoundsCheck) +{ + // If we are told that we require an upper bounds check (GC did some heap reshuffling), + // we need to switch to the WriteBarrier_PostGrow function for good. + + WriteBarrierType newType; + if (NeedDifferentWriteBarrier(bReqUpperBoundsCheck, g_region_use_bitwise_write_barrier, &newType)) + { + return ChangeWriteBarrierTo(newType, isRuntimeSuspended); + } + + int stompWBCompleteActions = SWB_PASS; + +#ifdef _DEBUG + // Using debug-only write barrier? + if (m_currentWriteBarrier == WRITE_BARRIER_UNINITIALIZED) + return stompWBCompleteActions; +#endif + + +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP +#if defined(WRITE_BARRIER_VARS_INLINE) + switch (m_currentWriteBarrier) + { + case WRITE_BARRIER_WRITE_WATCH_PREGROW64: + case WRITE_BARRIER_WRITE_WATCH_POSTGROW64: +#ifdef FEATURE_SVR_GC + case WRITE_BARRIER_WRITE_WATCH_SVR64: +#endif // FEATURE_SVR_GC + case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: + case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: + stompWBCompleteActions |= updateVariable(m_pWriteWatchTableImmediate, (size_t)g_sw_ww_table); + break; + + default: + break; + } +#else + stompWBCompleteActions |= updateVariable(m_pWriteWatchTableImmediate, (size_t)g_sw_ww_table); +#endif +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + + +#if defined(WRITE_BARRIER_VARS_INLINE) + switch (m_currentWriteBarrier) + { + case WRITE_BARRIER_BYTE_REGIONS64: + case WRITE_BARRIER_BIT_REGIONS64: + case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: + case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: + stompWBCompleteActions |= updateVariable(m_pRegionToGenTableImmediate, (size_t)g_region_to_generation_table); + stompWBCompleteActions |= updateVariable(m_pRegionShrDest, (size_t)g_region_shr); + stompWBCompleteActions |= updateVariable(m_pRegionShrSrc, (size_t)g_region_shr); + break; + + default: + break; + } +#else + stompWBCompleteActions |= updateVariable(m_pRegionToGenTableImmediate, (size_t)g_region_to_generation_table); + stompWBCompleteActions |= updateVariable(m_pRegionShrDest, g_region_shr); + stompWBCompleteActions |= updateVariable(m_pRegionShrSrc, g_region_shr); +#endif //WRITE_BARRIER_VARS_INLINE + + stompWBCompleteActions |= updateVariable(m_pCardTableImmediate, (size_t)g_card_table); +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + stompWBCompleteActions |= updateVariable(m_pCardBundleTableImmediate, (size_t)g_card_bundle_table); +#endif + +#if defined(TARGET_ARM64) + stompWBCompleteActions |= updateVariable(m_pRegionUseBitwiseWriteBarrier, g_region_use_bitwise_write_barrier); + stompWBCompleteActions |= updateVariable(m_lowestAddress, (size_t)g_lowest_address); + stompWBCompleteActions |= updateVariable(m_highestAddress, (size_t)g_highest_address); +#if defined(WRITE_BARRIER_CHECK) + stompWBCompleteActions |= updateVariable(m_pGCShadow, (size_t)g_GCShadow); + stompWBCompleteActions |= updateVariable(m_pGCShadowEnd, (size_t)g_GCShadowEnd); +#endif // WRITE_BARRIER_CHECK +#endif // TARGET_AMD64 + + return stompWBCompleteActions; +} + +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP +int WriteBarrierManager::SwitchToWriteWatchBarrier(bool isRuntimeSuspended) +{ + WriteBarrierType newWriteBarrierType; + switch (m_currentWriteBarrier) + { + case WRITE_BARRIER_UNINITIALIZED: + // Using the debug-only write barrier + return SWB_PASS; + + case WRITE_BARRIER_PREGROW64: + newWriteBarrierType = WRITE_BARRIER_WRITE_WATCH_PREGROW64; + break; + + case WRITE_BARRIER_POSTGROW64: + newWriteBarrierType = WRITE_BARRIER_WRITE_WATCH_POSTGROW64; + break; + +#ifdef FEATURE_SVR_GC + case WRITE_BARRIER_SVR64: + newWriteBarrierType = WRITE_BARRIER_WRITE_WATCH_SVR64; + break; +#endif // FEATURE_SVR_GC + + case WRITE_BARRIER_BYTE_REGIONS64: + newWriteBarrierType = WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64; + break; + + case WRITE_BARRIER_BIT_REGIONS64: + newWriteBarrierType = WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64; + break; + + default: + UNREACHABLE(); + } + + return ChangeWriteBarrierTo(newWriteBarrierType, isRuntimeSuspended); +} + +int WriteBarrierManager::SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended) +{ + WriteBarrierType newWriteBarrierType; + switch (m_currentWriteBarrier) + { + case WRITE_BARRIER_UNINITIALIZED: + // Using the debug-only write barrier + return SWB_PASS; + + case WRITE_BARRIER_WRITE_WATCH_PREGROW64: + newWriteBarrierType = WRITE_BARRIER_PREGROW64; + break; + + case WRITE_BARRIER_WRITE_WATCH_POSTGROW64: + newWriteBarrierType = WRITE_BARRIER_POSTGROW64; + break; + +#ifdef FEATURE_SVR_GC + case WRITE_BARRIER_WRITE_WATCH_SVR64: + newWriteBarrierType = WRITE_BARRIER_SVR64; + break; +#endif // FEATURE_SVR_GC + + case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: + newWriteBarrierType = WRITE_BARRIER_BYTE_REGIONS64; + break; + + case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: + newWriteBarrierType = WRITE_BARRIER_BIT_REGIONS64; + break; + + default: + UNREACHABLE(); + } + + return ChangeWriteBarrierTo(newWriteBarrierType, isRuntimeSuspended); +} +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP From 86c2576d7f36c92fa46399b00c4822e5b1d2d5a7 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Tue, 25 Feb 2025 14:59:11 +0000 Subject: [PATCH 14/42] Add the different write barrier functions --- src/coreclr/vm/CMakeLists.txt | 2 - src/coreclr/vm/amd64/jitinterfaceamd64.cpp | 386 -------------- .../vm/arm64/JitHelpers_FastWriteBarriers.S | 55 -- src/coreclr/vm/arm64/patchedcode.S | 352 +++++++++++++ src/coreclr/vm/arm64/stubs.cpp | 10 - src/coreclr/vm/writebarriermanager.cpp | 472 ++++++++++++++++-- 6 files changed, 778 insertions(+), 499 deletions(-) delete mode 100644 src/coreclr/vm/arm64/JitHelpers_FastWriteBarriers.S diff --git a/src/coreclr/vm/CMakeLists.txt b/src/coreclr/vm/CMakeLists.txt index fc701ecbf3de84..d02f46b92fa42b 100644 --- a/src/coreclr/vm/CMakeLists.txt +++ b/src/coreclr/vm/CMakeLists.txt @@ -651,7 +651,6 @@ elseif(CLR_CMAKE_TARGET_ARCH_ARM64) set(VM_SOURCES_WKS_ARCH_ASM ${ARCH_SOURCES_DIR}/AsmHelpers.asm ${ARCH_SOURCES_DIR}/CallDescrWorkerARM64.asm - ${ARCH_SOURCES_DIR}/JitHelpers_FastWriteBarriers.asm ${ARCH_SOURCES_DIR}/patchedcode.asm ${ARCH_SOURCES_DIR}/PInvokeStubs.asm ${ARCH_SOURCES_DIR}/thunktemplates.asm @@ -706,7 +705,6 @@ else(CLR_CMAKE_TARGET_WIN32) set(VM_SOURCES_WKS_ARCH_ASM ${ARCH_SOURCES_DIR}/asmhelpers.S ${ARCH_SOURCES_DIR}/calldescrworkerarm64.S - ${ARCH_SOURCES_DIR}/JitHelpers_FastWriteBarriers.S ${ARCH_SOURCES_DIR}/patchedcode.S ${ARCH_SOURCES_DIR}/pinvokestubs.S ${ARCH_SOURCES_DIR}/thunktemplates.S diff --git a/src/coreclr/vm/amd64/jitinterfaceamd64.cpp b/src/coreclr/vm/amd64/jitinterfaceamd64.cpp index 6296b788a855b0..e5fd8fff60a250 100644 --- a/src/coreclr/vm/amd64/jitinterfaceamd64.cpp +++ b/src/coreclr/vm/amd64/jitinterfaceamd64.cpp @@ -119,392 +119,6 @@ EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_CardBundleTab extern WriteBarrierManager g_WriteBarrierManager; -// Use this somewhat hokey macro to concatenate the function start with the patch -// label. This allows the code below to look relatively nice, but relies on the -// naming convention which we have established for these helpers. -#define CALC_PATCH_LOCATION(func,label,offset) CalculatePatchLocation((PVOID)func, (PVOID)func##_##label, offset) - - -#ifndef CODECOVERAGE // Deactivate alignment validation for code coverage builds - // because the instrumentation tool will not preserve alignment - // constraints and we will fail. - -void WriteBarrierManager::Validate() -{ - CONTRACTL - { - MODE_ANY; - GC_NOTRIGGER; - NOTHROW; - } - CONTRACTL_END; - - // we have an invariant that the addresses of all the values that we update in our write barrier - // helpers must be naturally aligned, this is so that the update can happen atomically since there - // are places where these values are updated while the EE is running - // NOTE: we can't call this from the ctor since our infrastructure isn't ready for assert dialogs - - PBYTE pLowerBoundImmediate, pUpperBoundImmediate, pCardTableImmediate; - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - PBYTE pCardBundleTableImmediate; -#endif - - pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_Lower, 2); - pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardTable, 2); - - _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); -#endif - - pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Lower, 2); - pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Upper, 2); - pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); -#endif - -#ifdef FEATURE_SVR_GC - pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); -#endif // FEATURE_MANUALLY_MANAGED_CARD_BUNDLES -#endif // FEATURE_SVR_GC - - PBYTE pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_RegionToGeneration, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pRegionToGenTableImmediate) & 0x7) == 0); - - pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_Lower, 2); - pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_Upper, 2); - pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_CardTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); -#endif - - pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_RegionToGeneration, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pRegionToGenTableImmediate) & 0x7) == 0); - - pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_Lower, 2); - pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_Upper, 2); - pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_CardTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); -#endif - -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - PBYTE pWriteWatchTableImmediate; - - pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_WriteWatchTable, 2); - pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_Lower, 2); - pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardTable, 2); - - _ASSERTE_ALL_BUILDS((reinterpret_cast(pWriteWatchTableImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); -#endif - - pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_WriteWatchTable, 2); - pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Lower, 2); - pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Upper, 2); - pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardTable, 2); - - _ASSERTE_ALL_BUILDS((reinterpret_cast(pWriteWatchTableImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); -#endif - -#ifdef FEATURE_SVR_GC - pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_WriteWatchTable, 2); - pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pWriteWatchTableImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); -#endif // FEATURE_MANUALLY_MANAGED_CARD_BUNDLES -#endif // FEATURE_SVR_GC - - pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_RegionToGeneration, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pRegionToGenTableImmediate) & 0x7) == 0); - - pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_Lower, 2); - pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_Upper, 2); - pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_CardTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); -#endif - - pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_RegionToGeneration, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pRegionToGenTableImmediate) & 0x7) == 0); - - pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_Lower, 2); - pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_Upper, 2); - pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_CardTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); -#endif - -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -} - -#endif // CODECOVERAGE - - -void WriteBarrierManager::UpdatePatchLocations() -{ - switch (newWriteBarrier) - { - case WRITE_BARRIER_PREGROW64: - { - m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_Lower, 2); - m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardTable, 2); - - // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); -#endif - break; - } - - case WRITE_BARRIER_POSTGROW64: - { - m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Lower, 2); - m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Upper, 2); - m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardTable, 2); - - // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); -#endif - break; - } - -#ifdef FEATURE_SVR_GC - case WRITE_BARRIER_SVR64: - { - m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardTable, 2); - - // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); -#endif - break; - } -#endif // FEATURE_SVR_GC - - case WRITE_BARRIER_BYTE_REGIONS64: - m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_RegionToGeneration, 2); - m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_RegionShrDest, 3); - m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_RegionShrSrc, 3); - m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_Lower, 2); - m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_Upper, 2); - m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_CardTable, 2); - - // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pRegionToGenTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); - _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrDest); - _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrSrc); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); -#endif - break; - - case WRITE_BARRIER_BIT_REGIONS64: - m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_RegionToGeneration, 2); - m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_RegionShrDest, 3); - m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_RegionShrSrc, 3); - m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_Lower, 2); - m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_Upper, 2); - m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_CardTable, 2); - - // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pRegionToGenTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); - _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrDest); - _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrSrc); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); -#endif - break; - -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - case WRITE_BARRIER_WRITE_WATCH_PREGROW64: - { - m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_WriteWatchTable, 2); - m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_Lower, 2); - m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardTable, 2); - - // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); -#endif - break; - } - - case WRITE_BARRIER_WRITE_WATCH_POSTGROW64: - { - m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_WriteWatchTable, 2); - m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Lower, 2); - m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Upper, 2); - m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardTable, 2); - - // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); -#endif - break; - } - -#ifdef FEATURE_SVR_GC - case WRITE_BARRIER_WRITE_WATCH_SVR64: - { - m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_WriteWatchTable, 2); - m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardTable, 2); - - // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); -#endif - break; - } -#endif // FEATURE_SVR_GC - - case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: - m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_WriteWatchTable, 2); - m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_RegionToGeneration, 2); - m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_RegionShrDest, 3); - m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_RegionShrSrc, 3); - m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_Lower, 2); - m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_Upper, 2); - m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_CardTable, 2); - - // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pRegionToGenTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); - _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrDest); - _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrSrc); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); -#endif - break; - - case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: - m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_WriteWatchTable, 2); - m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_RegionToGeneration, 2); - m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_RegionShrDest, 3); - m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_RegionShrSrc, 3); - m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_Lower, 2); - m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_Upper, 2); - m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_CardTable, 2); - - // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pRegionToGenTableImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); - _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrDest); - _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrSrc); - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_CardBundleTable, 2); - _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); -#endif - break; - - -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - - default: - UNREACHABLE_MSG("unexpected write barrier type!"); - } -} - -#undef CALC_PATCH_LOCATION - // This function bashes the super fast amd64 version of the JIT_WriteBarrier // helper. It should be called by the GC whenever the ephermeral region // bounds get changed, but still remain on the top of the GC Heap. diff --git a/src/coreclr/vm/arm64/JitHelpers_FastWriteBarriers.S b/src/coreclr/vm/arm64/JitHelpers_FastWriteBarriers.S deleted file mode 100644 index 158022c1023857..00000000000000 --- a/src/coreclr/vm/arm64/JitHelpers_FastWriteBarriers.S +++ /dev/null @@ -1,55 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -#include "asmconstants.h" -#include "unixasmmacros.inc" - - -LEAF_ENTRY JIT_WriteBarrier_PostGrow64, _TEXT - nop -LEAF_END_MARKED JIT_WriteBarrier_PostGrow64, _TEXT - - -LEAF_ENTRY JIT_WriteBarrier_PreGrow64, _TEXT - nop -LEAF_END_MARKED JIT_WriteBarrier_PreGrow64, _TEXT - - -LEAF_ENTRY JIT_WriteBarrier_SVR64, _TEXT - nop -LEAF_END_MARKED JIT_WriteBarrier_SVR64, _TEXT - - -LEAF_ENTRY JIT_WriteBarrier_Byte_Region64, _TEXT - nop -LEAF_END_MARKED JIT_WriteBarrier_Byte_Region64, _TEXT - - -LEAF_ENTRY JIT_WriteBarrier_Bit_Region64, _TEXT - nop -LEAF_END_MARKED JIT_WriteBarrier_Bit_Region64, _TEXT - - -LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT - nop -LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT - - -LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT - nop -LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT - - -LEAF_ENTRY JIT_WriteBarrier_WriteWatch_SVR64, _TEXT - nop -LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_SVR64, _TEXT - - -LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT - nop -LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT - - -LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT - nop -LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index 1389919b38456f..3430de4e58f766 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -99,6 +99,7 @@ WRITE_BARRIER_END JIT_CheckedWriteBarrier // if you add more trashed registers. // WRITE_BARRIER_ENTRY JIT_WriteBarrier +PATCH_LABEL JIT_WriteBarrier_Label stlr x15, [x14] #ifdef WRITE_BARRIER_CHECK @@ -286,3 +287,354 @@ WRITE_BARRIER_END JIT_WriteBarrier_Table LEAF_ENTRY JIT_PatchedCodeLast, _TEXT ret lr LEAF_END JIT_PatchedCodeLast, _TEXT + + + +//----------------------------------------------------------------------------- +// The following Macros are used by the different writebarrier implementations +// +// + +#define WRITE_BARRIER_CONSTANT_OFFSET(start, constlabel) (constlabel - C_FUNC(JIT_WriteBarrier) + LOCAL_LABEL(start)) + + +.macro WRITE_BARRIER_ENTRY_STUB start +LOCAL_LABEL(\start): + stlr x15, [x14] +.endm + + +.macro WRITE_BARRIER_SHADOW_UPDATE_STUB start + #ifdef WRITE_BARRIER_CHECK + // Update GC Shadow Heap + + // Do not perform the work if g_GCShadow is 0 + ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_GCShadow) + cbz x12, LOCAL_LABEL(ShadowUpdateEnd\@) + + // Compute address of shadow heap location: + // pShadow = g_GCShadow + (x14 - g_lowest_address) + ldr x17, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_LowestAddress) + sub x17, x14, x17 + add x12, x17, x12 + + // if (pShadow >= g_GCShadowEnd) goto end + ldr x17, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_GCShadowEnd) + cmp x12, x17 + bhs LOCAL_LABEL(ShadowUpdateEnd\@) + + // *pShadow = x15 + str x15, [x12] + + // Ensure that the write to the shadow heap occurs before the read from the GC heap so that race + // conditions are caught by INVALIDGCVALUE. + dmb ish + + // if ([x14] == x15) goto end + ldr x17, [x14] + cmp x17, x15 + beq LOCAL_LABEL(ShadowUpdateEnd\@) + + // *pShadow = INVALIDGCVALUE (0xcccccccd) + movz x17, #0xcccd + movk x17, #0xcccc, LSL #16 + str x17, [x12] +#endif +LOCAL_LABEL(ShadowUpdateEnd\@): +.endm + + +.macro WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB start exit +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + // Update the write watch table if necessary + ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_WriteWatchTable) + add x12, x12, x14, lsr #0xc // SoftwareWriteWatch::AddressToTableByteIndexShift + ldrb w17, [x12] + cbnz x17, LOCAL_LABEL(WriteWatchForGCHeapEnd\@) + mov w17, #0xFF + strb w17, [x12] +LOCAL_LABEL(WriteWatchForGCHeapEnd\@): +#endif +.endm + + +.macro WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB start exit + // Branch to Exit if the reference is not in the heap + ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_Lower) + cmp x15, x12 + bhs LOCAL_LABEL(\exit) +.endm + + +.macro WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB start exit + // Branch to Exit if the reference is not in the Gen0 heap + ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_Lower) + ldr x17, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_Upper) + cmp x15, x12 + ccmp x15, x17, #0x2, hs + bhs LOCAL_LABEL(\exit) +.endm + + +.macro WRITE_BARRIER_REGION_CHECK_STUB start exit + // Calculate region locations + ldr x17, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_RegionToGeneration) + ldr w12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_RegionShr) + lsr x15, x15, x12 + add x15, x15, x17 // x15 = (RHS >> wbs_region_shr) + wbs_region_to_generation_table + lsr x12, x14, x12 + add x12, x12, x17 // x12 = (LHS >> wbs_region_shr) + wbs_region_to_generation_table + + // Check whether the region we're storing into is gen 0 - nothing to do in this case + ldrb w12, [x12] + cbz w12, LOCAL_LABEL(\exit) + + // Check this is going from old to young + ldrb w15, [x15] + cmp w15, w12 + bhs LOCAL_LABEL(\exit) +.endm + + +.macro WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB start exit + // Check if we need to update the card table + lsr w17, w14, 8 + and w17, w17, 7 + movz w15, 1 + lsl w17, w15, w17 // w17 = 1 << (LHS >> 8 && 7) + ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_CardTable) + add x15, x12, x14, lsr #11 + ldrb w12, [x15] // w12 = [(LHS >> 11) + g_card_table] + tst w12, w17 + bne LOCAL_LABEL(\exit) + + // Atomically update the card table + // Requires LSE, but the code is only compiled for 8.0 + .word 0x383131FF // stsetb w17, [x15] +.endm + + +.macro WRITE_BARRIER_CHECK_CARD_TABLE_STUB start exit +LOCAL_LABEL(CheckCardTable\@): + // Check if we need to update the card table + ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_CardTable) + add x15, x12, x14, lsr #11 + ldrb w12, [x15] // w12 = [(RHS >> 11) + g_card_table] + cmp x12, 0xFF + beq LOCAL_LABEL(\exit) + + // Update the card table + mov x12, 0xFF + strb w12, [x15] +.endm + + +.macro WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB start exit +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + // Check if we need to update the card bundle table + ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_CardBundleTable) + add x15, x12, x14, lsr #21 + ldrb w12, [x15] + cmp x12, 0xFF + beq LOCAL_LABEL(\exit) + + // Update the card bundle + mov x12, 0xFF + strb w12, [x15] +#endif +.endm + + +.macro WRITE_BARRIER_RETURN_STUB exit +LOCAL_LABEL(\exit): + // Increment by 8 to implement JIT_ByRefWriteBarrier contract. + // TODO: Consider duplicating the logic to get rid of this redundant 'add' + // for JIT_WriteBarrier/JIT_CheckedWriteBarrier + add x14, x14, 8 + ret lr +.endm + +//----------------------------------------------------------------------------- +// void JIT_WriteBarrier_PreGrow64(Object** dst, Object* src) +// +// Skipped functionality: +// Does not update the write watch table +// Does not check wbs_ephemeral_high +// No region checks +// +LEAF_ENTRY JIT_WriteBarrier_PreGrow64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_PreGrow64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_PreGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB Start_PreGrow64 Exit_PreGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_PreGrow64 Exit_PreGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_PreGrow64 Exit_PreGrow64 + WRITE_BARRIER_RETURN_STUB Exit_PreGrow64 +LEAF_END_MARKED JIT_WriteBarrier_PreGrow64, _TEXT + + +//----------------------------------------------------------------------------- +// void JIT_WriteBarrier_PostGrow64(Object** dst, Object* src) +// +// Skipped functionality: +// Does not update the write watch table +// No region checks +// +LEAF_ENTRY JIT_WriteBarrier_PostGrow64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_PostGrow64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_PostGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_PostGrow64 Exit_PostGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_PostGrow64 Exit_PostGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_PostGrow64 Exit_PostGrow64 + WRITE_BARRIER_RETURN_STUB Exit_PostGrow64 +LEAF_END_MARKED JIT_WriteBarrier_PostGrow64, _TEXT + + +//----------------------------------------------------------------------------- +// void JIT_WriteBarrier_SVR64(Object** dst, Object* src) +// +// SVR GC has multiple heaps, so it cannot provide one single ephemeral region to bounds check +// against, so we just skip the bounds checking all together and do our card table update unconditionally. +// +// Skipped functionality: +// Does not update the write watch table +// Does not check wbs_ephemeral_high or wbs_ephemeral_low +// No region checks +// +LEAF_ENTRY JIT_WriteBarrier_SVR64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_SVR64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_SVR64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_SVR64 Exit_SVR64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_SVR64 Exit_SVR64 + WRITE_BARRIER_RETURN_STUB Exit_SVR64 +LEAF_END_MARKED JIT_WriteBarrier_SVR64, _TEXT + + +//----------------------------------------------------------------------------- +// void JIT_WriteBarrier_Byte_Region64(Object** dst, Object* src) +// +// Skipped functionality: +// Does not update the write watch table +// Bitwise updates for region checks +// +LEAF_ENTRY JIT_WriteBarrier_Byte_Region64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_Byte_Region64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_Byte_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Byte_Region64 Exit_Byte_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_Byte_Region64 Exit_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_Byte_Region64 Exit_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Byte_Region64 Exit_Byte_Region64 + WRITE_BARRIER_RETURN_STUB Exit_Byte_Region64 +LEAF_END_MARKED JIT_WriteBarrier_Byte_Region64, _TEXT + + +//----------------------------------------------------------------------------- +// void JIT_WriteBarrier_Bit_Region64(Object** dst, Object* src) +// +// Skipped functionality: +// Does not update the write watch table +// Does not call check card table stub +// +LEAF_ENTRY JIT_WriteBarrier_Bit_Region64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_Bit_Region64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_Bit_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Bit_Region64 Exit_Bit_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_Bit_Region64 Exit_Bit_Region64 + WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Start_Bit_Region64 Exit_Bit_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Bit_Region64 Exit_Bit_Region64 + WRITE_BARRIER_RETURN_STUB Exit_Bit_Region64 +LEAF_END_MARKED JIT_WriteBarrier_Bit_Region64, _TEXT + + +//----------------------------------------------------------------------------- +// void JIT_WriteBarrier_WriteWatch_PreGrow64(Object** dst, Object* src) +// +// Skipped functionality: +// Does not check wbs_ephemeral_high +// No region checks +// +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_PreGrow64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_PreGrow64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PreGrow64 JIT_WriteBarrier_WriteWatch_PreGrow64Start + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 + WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_PreGrow64 +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT + + +//----------------------------------------------------------------------------- +// void JIT_WriteBarrier_WriteWatch_PostGrow64(Object** dst, Object* src) +// +// Skipped functionality: +// No region checks +// +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_PostGrow64Start + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_PostGrow64Start + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PostGrow64Start JIT_WriteBarrier_WriteWatch_PostGrow64Start + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_PostGrow64Start Exit_WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_PostGrow64Start Exit_WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_PostGrow64Start Exit_WriteWatch_PostGrow64 + WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_PostGrow64 +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT + + +//----------------------------------------------------------------------------- +// void JIT_WriteBarrier_WriteWatch_SVR64(Object** dst, Object* src) +// +// SVR GC has multiple heaps, so it cannot provide one single ephemeral region to bounds check +// against, so we just skip the bounds checking all together and do our card table update unconditionally. +// +// Skipped functionality: +// Does not check wbs_ephemeral_high or wbs_ephemeral_low +// No region checks +// +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_SVR64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_SVR64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_SVR64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_SVR64 JIT_WriteBarrier_WriteWatch_SVR64Start + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_SVR64 Exit_WriteWatch_SVR64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_SVR64 Exit_WriteWatch_SVR64 + WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_SVR64 +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_SVR64, _TEXT + + +//----------------------------------------------------------------------------- +// void JIT_WriteBarrier_WriteWatch_Byte_Region64(Object** dst, Object* src) +// +// Skipped functionality: +// Bitwise updates for region checks +// +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT +LOCAL_LABEL(JIT_WriteBarrier_WriteWatch_Byte_Region64Start): + WRITE_BARRIER_ENTRY_STUB Start_Byte_Region64Start + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_Byte_Region64Start + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_Byte_Region64Start JIT_WriteBarrier_WriteWatch_Byte_Region64Start + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Byte_Region64Start Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_Byte_Region64Start Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_Byte_Region64Start Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Byte_Region64Start Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_Byte_Region64 +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT + + +//----------------------------------------------------------------------------- +// void JIT_WriteBarrier_WriteWatch_Bit_Region64(Object** dst, Object* src) +// +// Skipped functionality: +// Does not call check card table stub +// +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT +LOCAL_LABEL(JIT_WriteBarrier_WriteWatch_Bit_Region64Start): + WRITE_BARRIER_ENTRY_STUB Start_Bit_Region64Start + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_Bit_Region64Start + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_Bit_Region64Start JIT_WriteBarrier_WriteWatch_Bit_Region64Start + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Bit_Region64Start Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_Bit_Region64Start Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Start_Bit_Region64Start Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Bit_Region64Start Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_Bit_Region64 +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT + + diff --git a/src/coreclr/vm/arm64/stubs.cpp b/src/coreclr/vm/arm64/stubs.cpp index 208e2c435d835d..745ab74dcbec01 100644 --- a/src/coreclr/vm/arm64/stubs.cpp +++ b/src/coreclr/vm/arm64/stubs.cpp @@ -1048,16 +1048,6 @@ LONG CLRNoCatchHandler(EXCEPTION_POINTERS* pExceptionInfo, PVOID pv) return EXCEPTION_CONTINUE_SEARCH; } -void WriteBarrierManager::Validate() -{ - // Nothing to do -} - -void WriteBarrierManager::UpdatePatchLocations() -{ - // Nothing to do -} - // This function bashes the super fast amd64 version of the JIT_WriteBarrier // helper. It should be called by the GC whenever the ephermeral region // bounds get changed, but still remain on the top of the GC Heap. diff --git a/src/coreclr/vm/writebarriermanager.cpp b/src/coreclr/vm/writebarriermanager.cpp index bca55e9375db03..72aa9203468595 100644 --- a/src/coreclr/vm/writebarriermanager.cpp +++ b/src/coreclr/vm/writebarriermanager.cpp @@ -78,6 +78,11 @@ EXTERN_C void JIT_WriteBarrier_Patch_Label_GCShadowEnd(); #endif // !WRITE_BARRIER_VARS_INLINE +// Use this somewhat hokey macro to concatenate the function start with the patch +// label. This allows the code below to look relatively nice, but relies on the +// naming convention which we have established for these helpers. +#define CALC_PATCH_LOCATION(func,label,offset) CalculatePatchLocation((PVOID)func, (PVOID)func##_##label, offset) + WriteBarrierManager g_WriteBarrierManager; @@ -91,38 +96,37 @@ PCODE WriteBarrierManager::GetCurrentWriteBarrierCode() { LIMITED_METHOD_CONTRACT; - return GetEEFuncEntryPoint(JIT_WriteBarrier); -// switch (m_currentWriteBarrier) -// { -// case WRITE_BARRIER_PREGROW64: -// return GetEEFuncEntryPoint(JIT_WriteBarrier_PreGrow64); -// case WRITE_BARRIER_POSTGROW64: -// return GetEEFuncEntryPoint(JIT_WriteBarrier_PostGrow64); -// #ifdef FEATURE_SVR_GC -// case WRITE_BARRIER_SVR64: -// return GetEEFuncEntryPoint(JIT_WriteBarrier_SVR64); -// #endif // FEATURE_SVR_GC -// case WRITE_BARRIER_BYTE_REGIONS64: -// return GetEEFuncEntryPoint(JIT_WriteBarrier_Byte_Region64); -// case WRITE_BARRIER_BIT_REGIONS64: -// return GetEEFuncEntryPoint(JIT_WriteBarrier_Bit_Region64); -// #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -// case WRITE_BARRIER_WRITE_WATCH_PREGROW64: -// return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_PreGrow64); -// case WRITE_BARRIER_WRITE_WATCH_POSTGROW64: -// return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_PostGrow64); -// #ifdef FEATURE_SVR_GC -// case WRITE_BARRIER_WRITE_WATCH_SVR64: -// return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_SVR64); -// #endif // FEATURE_SVR_GC -// case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: -// return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_Byte_Region64); -// case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: -// return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_Bit_Region64); -// #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -// default: -// UNREACHABLE_MSG("unexpected m_currentWriteBarrier!"); -// }; + switch (m_currentWriteBarrier) + { + case WRITE_BARRIER_PREGROW64: + return GetEEFuncEntryPoint(JIT_WriteBarrier_PreGrow64); + case WRITE_BARRIER_POSTGROW64: + return GetEEFuncEntryPoint(JIT_WriteBarrier_PostGrow64); +#ifdef FEATURE_SVR_GC + case WRITE_BARRIER_SVR64: + return GetEEFuncEntryPoint(JIT_WriteBarrier_SVR64); +#endif // FEATURE_SVR_GC + case WRITE_BARRIER_BYTE_REGIONS64: + return GetEEFuncEntryPoint(JIT_WriteBarrier_Byte_Region64); + case WRITE_BARRIER_BIT_REGIONS64: + return GetEEFuncEntryPoint(JIT_WriteBarrier_Bit_Region64); +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + case WRITE_BARRIER_WRITE_WATCH_PREGROW64: + return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_PreGrow64); + case WRITE_BARRIER_WRITE_WATCH_POSTGROW64: + return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_PostGrow64); +#ifdef FEATURE_SVR_GC + case WRITE_BARRIER_WRITE_WATCH_SVR64: + return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_SVR64); +#endif // FEATURE_SVR_GC + case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: + return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_Byte_Region64); + case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: + return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_Bit_Region64); +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + default: + UNREACHABLE_MSG("unexpected m_currentWriteBarrier!"); + }; } size_t WriteBarrierManager::GetSpecificWriteBarrierSize(WriteBarrierType writeBarrier) @@ -174,9 +178,6 @@ size_t WriteBarrierManager::GetCurrentWriteBarrierSize() } -// -// m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_WriteWatchTable, 0); - PBYTE WriteBarrierManager::CalculatePatchLocation(LPVOID base, LPVOID label, int offset) { // the label should always come after or at the entrypoint for this funtion @@ -202,11 +203,11 @@ int WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, // the memcpy must come before the switch statement because the asserts inside the switch // are actually looking into the JIT_WriteBarrier buffer - // { - // ExecutableWriterHolder writeBarrierWriterHolder(GetWriteBarrierCodeLocation((void*)JIT_WriteBarrier), GetCurrentWriteBarrierSize()); - // memcpy(writeBarrierWriterHolder.GetRW(), (LPVOID)GetCurrentWriteBarrierCode(), GetCurrentWriteBarrierSize()); - // stompWBCompleteActions |= SWB_ICACHE_FLUSH; - // } + { + ExecutableWriterHolder writeBarrierWriterHolder(GetWriteBarrierCodeLocation((void*)JIT_WriteBarrier), GetCurrentWriteBarrierSize()); + memcpy(writeBarrierWriterHolder.GetRW(), (LPVOID)GetCurrentWriteBarrierCode(), GetCurrentWriteBarrierSize()); + stompWBCompleteActions |= SWB_ICACHE_FLUSH; + } UpdatePatchLocations(); @@ -216,7 +217,6 @@ int WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, return stompWBCompleteActions; } - void WriteBarrierManager::Initialize() { CONTRACTL @@ -251,9 +251,6 @@ void WriteBarrierManager::Initialize() #if !defined(WRITE_BARRIER_VARS_INLINE) - /// TODO: Want to move all the WriteBarrierManager code from AMD64 into here, but inside a define. - -#define CALC_PATCH_LOCATION(func,label,offset) CalculatePatchLocation((PVOID)func, (PVOID)func##_##label, offset) m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_WriteWatchTable, 0); m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_RegionToGeneration, 0); @@ -280,8 +277,6 @@ void WriteBarrierManager::Initialize() #endif // WRITE_BARRIER_CHECK #endif // TARGET_ARM64 -#undef CALC_PATCH_LOCATION - #endif // !WRITE_BARRIER_VARS_INLINE #if !defined(CODECOVERAGE) @@ -636,3 +631,388 @@ int WriteBarrierManager::SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended) return ChangeWriteBarrierTo(newWriteBarrierType, isRuntimeSuspended); } #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + + +// Deactivate alignment validation for code coverage builds +// because the instrumentation tool will not preserve alignment +// constraints and we will fail. +#if !defined(CODECOVERAGE) + +void WriteBarrierManager::Validate() +{ +#if defined(WRITE_BARRIER_VARS_INLINE) + CONTRACTL + { + MODE_ANY; + GC_NOTRIGGER; + NOTHROW; + } + CONTRACTL_END; + + // we have an invariant that the addresses of all the values that we update in our write barrier + // helpers must be naturally aligned, this is so that the update can happen atomically since there + // are places where these values are updated while the EE is running + // NOTE: we can't call this from the ctor since our infrastructure isn't ready for assert dialogs + + PBYTE pLowerBoundImmediate, pUpperBoundImmediate, pCardTableImmediate; + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + PBYTE pCardBundleTableImmediate; +#endif + + pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_Lower, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardTable, 2); + + _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif + + pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Lower, 2); + pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Upper, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif + +#ifdef FEATURE_SVR_GC + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif // FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +#endif // FEATURE_SVR_GC + + PBYTE pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_RegionToGeneration, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pRegionToGenTableImmediate) & 0x7) == 0); + + pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_Lower, 2); + pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_Upper, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_CardTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif + + pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_RegionToGeneration, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pRegionToGenTableImmediate) & 0x7) == 0); + + pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_Lower, 2); + pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_Upper, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_CardTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif + +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + PBYTE pWriteWatchTableImmediate; + + pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_WriteWatchTable, 2); + pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_Lower, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardTable, 2); + + _ASSERTE_ALL_BUILDS((reinterpret_cast(pWriteWatchTableImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif + + pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_WriteWatchTable, 2); + pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Lower, 2); + pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Upper, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardTable, 2); + + _ASSERTE_ALL_BUILDS((reinterpret_cast(pWriteWatchTableImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif + +#ifdef FEATURE_SVR_GC + pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_WriteWatchTable, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pWriteWatchTableImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif // FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +#endif // FEATURE_SVR_GC + + pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_RegionToGeneration, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pRegionToGenTableImmediate) & 0x7) == 0); + + pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_Lower, 2); + pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_Upper, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_CardTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif + + pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_RegionToGeneration, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pRegionToGenTableImmediate) & 0x7) == 0); + + pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_Lower, 2); + pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_Upper, 2); + pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_CardTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pLowerBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pUpperBoundImmediate) & 0x7) == 0); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardTableImmediate) & 0x7) == 0); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS((reinterpret_cast(pCardBundleTableImmediate) & 0x7) == 0); +#endif + +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + +#endif // WRITE_BARRIER_VARS_INLINE +} + +#endif // CODECOVERAGE + +void WriteBarrierManager::UpdatePatchLocations() +{ +#if defined(WRITE_BARRIER_VARS_INLINE) + + switch (newWriteBarrier) + { + case WRITE_BARRIER_PREGROW64: + { + m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_Lower, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + } + + case WRITE_BARRIER_POSTGROW64: + { + m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Lower, 2); + m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Upper, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + } + +#ifdef FEATURE_SVR_GC + case WRITE_BARRIER_SVR64: + { + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + } +#endif // FEATURE_SVR_GC + + case WRITE_BARRIER_BYTE_REGIONS64: + m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_RegionToGeneration, 2); + m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_RegionShrDest, 3); + m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_RegionShrSrc, 3); + m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_Lower, 2); + m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_Upper, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pRegionToGenTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrDest); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrSrc); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Byte_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + + case WRITE_BARRIER_BIT_REGIONS64: + m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_RegionToGeneration, 2); + m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_RegionShrDest, 3); + m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_RegionShrSrc, 3); + m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_Lower, 2); + m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_Upper, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pRegionToGenTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrDest); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrSrc); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_Bit_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + case WRITE_BARRIER_WRITE_WATCH_PREGROW64: + { + m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_WriteWatchTable, 2); + m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_Lower, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + } + + case WRITE_BARRIER_WRITE_WATCH_POSTGROW64: + { + m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_WriteWatchTable, 2); + m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Lower, 2); + m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Upper, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + } + +#ifdef FEATURE_SVR_GC + case WRITE_BARRIER_WRITE_WATCH_SVR64: + { + m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_WriteWatchTable, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + } +#endif // FEATURE_SVR_GC + + case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64: + m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_WriteWatchTable, 2); + m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_RegionToGeneration, 2); + m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_RegionShrDest, 3); + m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_RegionShrSrc, 3); + m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_Lower, 2); + m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_Upper, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pRegionToGenTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrDest); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrSrc); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Byte_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + + case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64: + m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_WriteWatchTable, 2); + m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_RegionToGeneration, 2); + m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_RegionShrDest, 3); + m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_RegionShrSrc, 3); + m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_Lower, 2); + m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_Upper, 2); + m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_CardTable, 2); + + // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0). + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pRegionToGenTableImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrDest); + _ASSERTE_ALL_BUILDS( 0x16 == *(UINT8 *)m_pRegionShrSrc); + +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_Bit_Region64, Patch_Label_CardBundleTable, 2); + _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate); +#endif + break; + + +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + + default: + UNREACHABLE_MSG("unexpected write barrier type!"); + } +#endif // WRITE_BARRIER_VARS_INLINE +} From 38bc2a11989f3725a2489fb0086f803418aba2e0 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Tue, 4 Mar 2025 12:10:34 +0000 Subject: [PATCH 15/42] Don't clear ephemeral values --- src/coreclr/vm/writebarriermanager.cpp | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/coreclr/vm/writebarriermanager.cpp b/src/coreclr/vm/writebarriermanager.cpp index 72aa9203468595..3d379d20bdda94 100644 --- a/src/coreclr/vm/writebarriermanager.cpp +++ b/src/coreclr/vm/writebarriermanager.cpp @@ -442,20 +442,8 @@ int WriteBarrierManager::UpdateEphemeralBounds(bool isRuntimeSuspended) #else - size_t e_high = (size_t)g_ephemeral_high; - size_t e_low = (size_t)g_ephemeral_low; - - // TODO: not required once we have separate functions -#if defined(TARGET_ARM64) - if (GCHeapUtilities::IsServerHeap() && g_region_to_generation_table == nullptr) - { - e_high = 0; - e_low = 0; - } -#endif // TARGET_ARM64 - - stompWBCompleteActions |= updateVariable(m_pUpperBoundImmediate, e_high); - stompWBCompleteActions |= updateVariable(m_pLowerBoundImmediate, e_low); + stompWBCompleteActions |= updateVariable(m_pUpperBoundImmediate, (size_t)g_ephemeral_high); + stompWBCompleteActions |= updateVariable(m_pLowerBoundImmediate, (size_t)g_ephemeral_low); #endif //WRITE_BARRIER_VARS_INLINE From 5eed451433b255f8f192d8e26e683bda90981a01 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Tue, 4 Mar 2025 15:22:21 +0000 Subject: [PATCH 16/42] Only use offsets for inline variables --- src/coreclr/vm/writebarriermanager.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/coreclr/vm/writebarriermanager.cpp b/src/coreclr/vm/writebarriermanager.cpp index 3d379d20bdda94..777f9a4faae501 100644 --- a/src/coreclr/vm/writebarriermanager.cpp +++ b/src/coreclr/vm/writebarriermanager.cpp @@ -75,7 +75,6 @@ EXTERN_C void JIT_WriteBarrier_Patch_Label_GCShadow(); EXTERN_C void JIT_WriteBarrier_Patch_Label_GCShadowEnd(); #endif // WRITE_BARRIER_CHECK #endif // TARGET_ARM64 - #endif // !WRITE_BARRIER_VARS_INLINE // Use this somewhat hokey macro to concatenate the function start with the patch @@ -178,13 +177,18 @@ size_t WriteBarrierManager::GetCurrentWriteBarrierSize() } -PBYTE WriteBarrierManager::CalculatePatchLocation(LPVOID base, LPVOID label, int offset) +PBYTE WriteBarrierManager::CalculatePatchLocation(LPVOID base, LPVOID label, int inlineOffset) { // the label should always come after or at the entrypoint for this funtion _ASSERTE_ALL_BUILDS((LPBYTE)label >= (LPBYTE)base); BYTE* patchBase = GetWriteBarrierCodeLocation((void*)JIT_WriteBarrier); - return (patchBase + ((LPBYTE)GetEEFuncEntryPoint(label) - (LPBYTE)GetEEFuncEntryPoint(base) + offset)); + + PBYTE patchLocation = (patchBase + ((LPBYTE)GetEEFuncEntryPoint(label) - (LPBYTE)GetEEFuncEntryPoint(base))); +#if defined(WRITE_BARRIER_VARS_INLINE) + patchLocation+=offset; +#endif + return patchLocation; } From 6b29574789a8e622b74ffc1d66a4a284f4417a1d Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Tue, 4 Mar 2025 15:27:59 +0000 Subject: [PATCH 17/42] Remove jitinterfaceamd64.cpp --- src/coreclr/vm/CMakeLists.txt | 1 - src/coreclr/vm/amd64/jitinterfaceamd64.cpp | 161 --------------------- src/coreclr/vm/arm64/stubs.cpp | 42 ------ src/coreclr/vm/writebarriermanager.cpp | 140 +++++++++++++++++- 4 files changed, 138 insertions(+), 206 deletions(-) delete mode 100644 src/coreclr/vm/amd64/jitinterfaceamd64.cpp diff --git a/src/coreclr/vm/CMakeLists.txt b/src/coreclr/vm/CMakeLists.txt index d02f46b92fa42b..c23c2d8685316f 100644 --- a/src/coreclr/vm/CMakeLists.txt +++ b/src/coreclr/vm/CMakeLists.txt @@ -745,7 +745,6 @@ if(CLR_CMAKE_TARGET_ARCH_AMD64) ) set(VM_SOURCES_WKS_ARCH - ${ARCH_SOURCES_DIR}/jitinterfaceamd64.cpp ${ARCH_SOURCES_DIR}/profiler.cpp exceptionhandling.cpp gcinfodecoder.cpp diff --git a/src/coreclr/vm/amd64/jitinterfaceamd64.cpp b/src/coreclr/vm/amd64/jitinterfaceamd64.cpp deleted file mode 100644 index e5fd8fff60a250..00000000000000 --- a/src/coreclr/vm/amd64/jitinterfaceamd64.cpp +++ /dev/null @@ -1,161 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -// =========================================================================== -// File: JITinterfaceCpu.CPP -// =========================================================================== - -// This contains JITinterface routines that are specific to the -// AMD64 platform. They are modeled after the X86 specific routines -// found in JITinterfaceX86.cpp or JIThelp.asm - - -#include "common.h" -#include "jitinterface.h" -#include "eeconfig.h" -#include "excep.h" -#include "threadsuspend.h" - - -// Patch Labels for the various write barriers - -EXTERN_C void JIT_WriteBarrier_PreGrow64(Object **dst, Object *ref); -EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_Lower(); -EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_CardTable(); -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES -EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_CardBundleTable(); -#endif - -EXTERN_C void JIT_WriteBarrier_PostGrow64(Object **dst, Object *ref); -EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_Lower(); -EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_Upper(); -EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_CardTable(); -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES -EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_CardBundleTable(); -#endif - -#ifdef FEATURE_SVR_GC -EXTERN_C void JIT_WriteBarrier_SVR64(Object **dst, Object *ref); -EXTERN_C void JIT_WriteBarrier_SVR64_PatchLabel_CardTable(); -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES -EXTERN_C void JIT_WriteBarrier_SVR64_PatchLabel_CardBundleTable(); -#endif -#endif // FEATURE_SVR_GC - -EXTERN_C void JIT_WriteBarrier_Byte_Region64(Object **dst, Object *ref); -EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_RegionToGeneration(); -EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_RegionShrDest(); -EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_Lower(); -EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_Upper(); -EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_RegionShrSrc(); -EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_CardTable(); -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES -EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_CardBundleTable(); -#endif - -EXTERN_C void JIT_WriteBarrier_Bit_Region64(Object **dst, Object *ref); -EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_RegionToGeneration(); -EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_RegionShrDest(); -EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_Lower(); -EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_Upper(); -EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_RegionShrSrc(); -EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_CardTable(); -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES -EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_CardBundleTable(); -#endif - -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64(Object **dst, Object *ref); -EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_WriteWatchTable(); -EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_Lower(); -EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_CardTable(); -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES -EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_CardBundleTable(); -#endif - -EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64(Object **dst, Object *ref); -EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_WriteWatchTable(); -EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_Lower(); -EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_Upper(); -EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_CardTable(); -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES -EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_CardBundleTable(); -#endif - -#ifdef FEATURE_SVR_GC -EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64(Object **dst, Object *ref); -EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_WriteWatchTable(); -EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_CardTable(); -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES -EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_CardBundleTable(); -#endif -#endif // FEATURE_SVR_GC - -EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64(Object **dst, Object *ref); -EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_WriteWatchTable(); -EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_RegionToGeneration(); -EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_RegionShrDest(); -EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_Lower(); -EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_Upper(); -EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_RegionShrSrc(); -EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_CardTable(); -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES -EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_CardBundleTable(); -#endif - -EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64(Object **dst, Object *ref); -EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_WriteWatchTable(); -EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_RegionToGeneration(); -EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_RegionShrDest(); -EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_Lower(); -EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_Upper(); -EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_RegionShrSrc(); -EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_CardTable(); -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES -EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_CardBundleTable(); -#endif - -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - -extern WriteBarrierManager g_WriteBarrierManager; - -// This function bashes the super fast amd64 version of the JIT_WriteBarrier -// helper. It should be called by the GC whenever the ephermeral region -// bounds get changed, but still remain on the top of the GC Heap. -int StompWriteBarrierEphemeral(bool isRuntimeSuspended) -{ - WRAPPER_NO_CONTRACT; - - return g_WriteBarrierManager.UpdateEphemeralBounds(isRuntimeSuspended); -} - -// This function bashes the super fast amd64 versions of the JIT_WriteBarrier -// helpers. It should be called by the GC whenever the ephermeral region gets moved -// from being at the top of the GC Heap, and/or when the cards table gets moved. -int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck) -{ - WRAPPER_NO_CONTRACT; - - return g_WriteBarrierManager.UpdateWriteWatchAndCardTableLocations(isRuntimeSuspended, bReqUpperBoundsCheck); -} - -void FlushWriteBarrierInstructionCache() -{ - FlushInstructionCache(GetCurrentProcess(), GetWriteBarrierCodeLocation((PVOID)JIT_WriteBarrier), g_WriteBarrierManager.GetCurrentWriteBarrierSize()); -} - -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -int SwitchToWriteWatchBarrier(bool isRuntimeSuspended) -{ - WRAPPER_NO_CONTRACT; - - return g_WriteBarrierManager.SwitchToWriteWatchBarrier(isRuntimeSuspended); -} - -int SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended) -{ - WRAPPER_NO_CONTRACT; - - return g_WriteBarrierManager.SwitchToNonWriteWatchBarrier(isRuntimeSuspended); -} -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP diff --git a/src/coreclr/vm/arm64/stubs.cpp b/src/coreclr/vm/arm64/stubs.cpp index 745ab74dcbec01..eb777c1afe311d 100644 --- a/src/coreclr/vm/arm64/stubs.cpp +++ b/src/coreclr/vm/arm64/stubs.cpp @@ -1048,48 +1048,6 @@ LONG CLRNoCatchHandler(EXCEPTION_POINTERS* pExceptionInfo, PVOID pv) return EXCEPTION_CONTINUE_SEARCH; } -// This function bashes the super fast amd64 version of the JIT_WriteBarrier -// helper. It should be called by the GC whenever the ephermeral region -// bounds get changed, but still remain on the top of the GC Heap. -int StompWriteBarrierEphemeral(bool isRuntimeSuspended) -{ - WRAPPER_NO_CONTRACT; - - return g_WriteBarrierManager.UpdateEphemeralBounds(isRuntimeSuspended); -} - -// This function bashes the super fast amd64 versions of the JIT_WriteBarrier -// helpers. It should be called by the GC whenever the ephermeral region gets moved -// from being at the top of the GC Heap, and/or when the cards table gets moved. -int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck) -{ - WRAPPER_NO_CONTRACT; - - return g_WriteBarrierManager.UpdateWriteWatchAndCardTableLocations(isRuntimeSuspended, bReqUpperBoundsCheck); -} - -void FlushWriteBarrierInstructionCache() -{ - FlushInstructionCache(GetCurrentProcess(), GetWriteBarrierCodeLocation((PVOID)JIT_WriteBarrier), g_WriteBarrierManager.GetCurrentWriteBarrierSize()); -} - -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -int SwitchToWriteWatchBarrier(bool isRuntimeSuspended) -{ - WRAPPER_NO_CONTRACT; - - return g_WriteBarrierManager.SwitchToWriteWatchBarrier(isRuntimeSuspended); -} - -int SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended) -{ - WRAPPER_NO_CONTRACT; - - return g_WriteBarrierManager.SwitchToNonWriteWatchBarrier(isRuntimeSuspended); -} -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - - #ifdef DACCESS_COMPILE BOOL GetAnyThunkTarget (T_CONTEXT *pctx, TADDR *pTarget, TADDR *pTargetMethodDesc) { diff --git a/src/coreclr/vm/writebarriermanager.cpp b/src/coreclr/vm/writebarriermanager.cpp index 777f9a4faae501..65a990de085ef2 100644 --- a/src/coreclr/vm/writebarriermanager.cpp +++ b/src/coreclr/vm/writebarriermanager.cpp @@ -27,6 +27,7 @@ extern uint32_t* g_card_bundle_table; // Patch Labels for the various write barriers + EXTERN_C void JIT_WriteBarrier_End(); EXTERN_C void JIT_WriteBarrier_PreGrow64(Object **dst, Object *ref); EXTERN_C void JIT_WriteBarrier_PreGrow64_End(); @@ -56,7 +57,98 @@ EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_End(); #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -#if !defined(WRITE_BARRIER_VARS_INLINE) +#if defined(WRITE_BARRIER_VARS_INLINE) + +EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_Lower(); +EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_CardTable(); +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_CardBundleTable(); +#endif + +EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_Lower(); +EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_Upper(); +EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_CardTable(); +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_CardBundleTable(); +#endif + +#ifdef FEATURE_SVR_GC +EXTERN_C void JIT_WriteBarrier_SVR64_PatchLabel_CardTable(); +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +EXTERN_C void JIT_WriteBarrier_SVR64_PatchLabel_CardBundleTable(); +#endif +#endif // FEATURE_SVR_GC + +EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_RegionToGeneration(); +EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_RegionShrDest(); +EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_Lower(); +EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_Upper(); +EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_RegionShrSrc(); +EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_CardTable(); +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +EXTERN_C void JIT_WriteBarrier_Byte_Region64_Patch_Label_CardBundleTable(); +#endif + +EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_RegionToGeneration(); +EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_RegionShrDest(); +EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_Lower(); +EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_Upper(); +EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_RegionShrSrc(); +EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_CardTable(); +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +EXTERN_C void JIT_WriteBarrier_Bit_Region64_Patch_Label_CardBundleTable(); +#endif + +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP +EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_WriteWatchTable(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_Lower(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_CardTable(); +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_CardBundleTable(); +#endif + +EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_WriteWatchTable(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_Lower(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_Upper(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_CardTable(); +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_CardBundleTable(); +#endif + +#ifdef FEATURE_SVR_GC +EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_WriteWatchTable(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_CardTable(); +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_CardBundleTable(); +#endif +#endif // FEATURE_SVR_GC + +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_WriteWatchTable(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_RegionToGeneration(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_RegionShrDest(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_Lower(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_Upper(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_RegionShrSrc(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_CardTable(); +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +EXTERN_C void JIT_WriteBarrier_WriteWatch_Byte_Region64_Patch_Label_CardBundleTable(); +#endif + +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_WriteWatchTable(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_RegionToGeneration(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_RegionShrDest(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_Lower(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_Upper(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_RegionShrSrc(); +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_CardTable(); +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_CardBundleTable(); +#endif + +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + +#else // WRITE_BARRIER_VARS_INLINE + EXTERN_C void JIT_WriteBarrier_Patch_Label_WriteWatchTable(); EXTERN_C void JIT_WriteBarrier_Patch_Label_RegionToGeneration(); EXTERN_C void JIT_WriteBarrier_Patch_Label_RegionShr(); @@ -75,7 +167,7 @@ EXTERN_C void JIT_WriteBarrier_Patch_Label_GCShadow(); EXTERN_C void JIT_WriteBarrier_Patch_Label_GCShadowEnd(); #endif // WRITE_BARRIER_CHECK #endif // TARGET_ARM64 -#endif // !WRITE_BARRIER_VARS_INLINE +#endif // WRITE_BARRIER_VARS_INLINE // Use this somewhat hokey macro to concatenate the function start with the patch // label. This allows the code below to look relatively nice, but relies on the @@ -85,6 +177,50 @@ EXTERN_C void JIT_WriteBarrier_Patch_Label_GCShadowEnd(); WriteBarrierManager g_WriteBarrierManager; + +// This function bashes the super fast amd64 version of the JIT_WriteBarrier +// helper. It should be called by the GC whenever the ephermeral region +// bounds get changed, but still remain on the top of the GC Heap. +int StompWriteBarrierEphemeral(bool isRuntimeSuspended) +{ + WRAPPER_NO_CONTRACT; + + return g_WriteBarrierManager.UpdateEphemeralBounds(isRuntimeSuspended); +} + +// This function bashes the super fast amd64 versions of the JIT_WriteBarrier +// helpers. It should be called by the GC whenever the ephermeral region gets moved +// from being at the top of the GC Heap, and/or when the cards table gets moved. +int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck) +{ + WRAPPER_NO_CONTRACT; + + return g_WriteBarrierManager.UpdateWriteWatchAndCardTableLocations(isRuntimeSuspended, bReqUpperBoundsCheck); +} + +void FlushWriteBarrierInstructionCache() +{ + FlushInstructionCache(GetCurrentProcess(), GetWriteBarrierCodeLocation((PVOID)JIT_WriteBarrier), g_WriteBarrierManager.GetCurrentWriteBarrierSize()); +} + +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP +int SwitchToWriteWatchBarrier(bool isRuntimeSuspended) +{ + WRAPPER_NO_CONTRACT; + + return g_WriteBarrierManager.SwitchToWriteWatchBarrier(isRuntimeSuspended); +} + +int SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended) +{ + WRAPPER_NO_CONTRACT; + + return g_WriteBarrierManager.SwitchToNonWriteWatchBarrier(isRuntimeSuspended); +} +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + + + WriteBarrierManager::WriteBarrierManager() : m_currentWriteBarrier(WRITE_BARRIER_UNINITIALIZED) { From e3bbf96b3f9439a43a90cbbb551b63136ebadb8c Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Tue, 4 Mar 2025 15:51:03 +0000 Subject: [PATCH 18/42] Don't use m_pRegionShrSrc on Arm64 --- src/coreclr/vm/writebarriermanager.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/coreclr/vm/writebarriermanager.cpp b/src/coreclr/vm/writebarriermanager.cpp index 65a990de085ef2..4c03abe5494aeb 100644 --- a/src/coreclr/vm/writebarriermanager.cpp +++ b/src/coreclr/vm/writebarriermanager.cpp @@ -394,11 +394,7 @@ void WriteBarrierManager::Initialize() m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_WriteWatchTable, 0); m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_RegionToGeneration, 0); - - // TODO: Only one of these! m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_RegionShr, 0); - m_pRegionShrSrc = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_RegionShr, 0); - m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_Lower, 0); m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_Upper, 0); m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_CardTable, 0); @@ -661,7 +657,6 @@ int WriteBarrierManager::UpdateWriteWatchAndCardTableLocations(bool isRuntimeSus #else stompWBCompleteActions |= updateVariable(m_pRegionToGenTableImmediate, (size_t)g_region_to_generation_table); stompWBCompleteActions |= updateVariable(m_pRegionShrDest, g_region_shr); - stompWBCompleteActions |= updateVariable(m_pRegionShrSrc, g_region_shr); #endif //WRITE_BARRIER_VARS_INLINE stompWBCompleteActions |= updateVariable(m_pCardTableImmediate, (size_t)g_card_table); From 091e2654cda0a0ac7c5ce48725fcf581e6f6c9d2 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Tue, 4 Mar 2025 15:51:38 +0000 Subject: [PATCH 19/42] Remove commented code --- src/coreclr/vm/writebarriermanager.cpp | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/coreclr/vm/writebarriermanager.cpp b/src/coreclr/vm/writebarriermanager.cpp index 4c03abe5494aeb..4cebee79610a7e 100644 --- a/src/coreclr/vm/writebarriermanager.cpp +++ b/src/coreclr/vm/writebarriermanager.cpp @@ -424,22 +424,8 @@ template int updateVariable(PBYTE loc, T value) { if (*(T*)loc != value) { -// #if defined(WRITE_BARRIER_VARS_INLINE) ExecutableWriterHolder varWriterHolder((T*)loc, sizeof(T)); *varWriterHolder.GetRW() = value; -// #else - -// BYTE *writeBarrierCodeStart = GetWriteBarrierCodeLocation((void*)JIT_PatchedCodeStart); -// BYTE *writeBarrierCodeStartRW = writeBarrierCodeStart; -// ExecutableWriterHolderNoLog writeBarrierWriterHolder; -// if (IsWriteBarrierCopyEnabled()) -// { -// writeBarrierWriterHolder.AssignExecutableWriterHolder(writeBarrierCodeStart, (BYTE*)JIT_PatchedCodeLast - (BYTE*)JIT_PatchedCodeStart); -// writeBarrierCodeStartRW = writeBarrierWriterHolder.GetRW(); -// } - -// *loc = value; -// #endif return SWB_ICACHE_FLUSH; } return SWB_PASS; @@ -453,7 +439,6 @@ bool WriteBarrierManager::NeedDifferentWriteBarrier(bool bReqUpperBoundsCheck, b // The actual JIT_WriteBarrier routine will only be called in free builds, but we keep this code (that // modifies it) around in debug builds to check that it works (with assertions). - WriteBarrierType writeBarrierType = m_currentWriteBarrier; for(;;) From 91eb26f4f0b59359ce4a4d64f66818e7134074cd Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Tue, 4 Mar 2025 16:28:44 +0000 Subject: [PATCH 20/42] make writebarriermanager closer to original code --- src/coreclr/vm/writebarriermanager.cpp | 92 ++++++++++++-------------- 1 file changed, 44 insertions(+), 48 deletions(-) diff --git a/src/coreclr/vm/writebarriermanager.cpp b/src/coreclr/vm/writebarriermanager.cpp index 4cebee79610a7e..63d632c809e630 100644 --- a/src/coreclr/vm/writebarriermanager.cpp +++ b/src/coreclr/vm/writebarriermanager.cpp @@ -169,58 +169,13 @@ EXTERN_C void JIT_WriteBarrier_Patch_Label_GCShadowEnd(); #endif // TARGET_ARM64 #endif // WRITE_BARRIER_VARS_INLINE +WriteBarrierManager g_WriteBarrierManager; + // Use this somewhat hokey macro to concatenate the function start with the patch // label. This allows the code below to look relatively nice, but relies on the // naming convention which we have established for these helpers. #define CALC_PATCH_LOCATION(func,label,offset) CalculatePatchLocation((PVOID)func, (PVOID)func##_##label, offset) - -WriteBarrierManager g_WriteBarrierManager; - - -// This function bashes the super fast amd64 version of the JIT_WriteBarrier -// helper. It should be called by the GC whenever the ephermeral region -// bounds get changed, but still remain on the top of the GC Heap. -int StompWriteBarrierEphemeral(bool isRuntimeSuspended) -{ - WRAPPER_NO_CONTRACT; - - return g_WriteBarrierManager.UpdateEphemeralBounds(isRuntimeSuspended); -} - -// This function bashes the super fast amd64 versions of the JIT_WriteBarrier -// helpers. It should be called by the GC whenever the ephermeral region gets moved -// from being at the top of the GC Heap, and/or when the cards table gets moved. -int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck) -{ - WRAPPER_NO_CONTRACT; - - return g_WriteBarrierManager.UpdateWriteWatchAndCardTableLocations(isRuntimeSuspended, bReqUpperBoundsCheck); -} - -void FlushWriteBarrierInstructionCache() -{ - FlushInstructionCache(GetCurrentProcess(), GetWriteBarrierCodeLocation((PVOID)JIT_WriteBarrier), g_WriteBarrierManager.GetCurrentWriteBarrierSize()); -} - -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -int SwitchToWriteWatchBarrier(bool isRuntimeSuspended) -{ - WRAPPER_NO_CONTRACT; - - return g_WriteBarrierManager.SwitchToWriteWatchBarrier(isRuntimeSuspended); -} - -int SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended) -{ - WRAPPER_NO_CONTRACT; - - return g_WriteBarrierManager.SwitchToNonWriteWatchBarrier(isRuntimeSuspended); -} -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - - - WriteBarrierManager::WriteBarrierManager() : m_currentWriteBarrier(WRITE_BARRIER_UNINITIALIZED) { @@ -439,6 +394,7 @@ bool WriteBarrierManager::NeedDifferentWriteBarrier(bool bReqUpperBoundsCheck, b // The actual JIT_WriteBarrier routine will only be called in free builds, but we keep this code (that // modifies it) around in debug builds to check that it works (with assertions). + WriteBarrierType writeBarrierType = m_currentWriteBarrier; for(;;) @@ -600,7 +556,6 @@ int WriteBarrierManager::UpdateWriteWatchAndCardTableLocations(bool isRuntimeSus return stompWBCompleteActions; #endif - #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP #if defined(WRITE_BARRIER_VARS_INLINE) switch (m_currentWriteBarrier) @@ -1124,3 +1079,44 @@ void WriteBarrierManager::UpdatePatchLocations() } #endif // WRITE_BARRIER_VARS_INLINE } + +// This function bashes the super fast version of the JIT_WriteBarrier +// helper. It should be called by the GC whenever the ephermeral region +// bounds get changed, but still remain on the top of the GC Heap. +int StompWriteBarrierEphemeral(bool isRuntimeSuspended) +{ + WRAPPER_NO_CONTRACT; + + return g_WriteBarrierManager.UpdateEphemeralBounds(isRuntimeSuspended); +} + +// This function bashes the super fast versions of the JIT_WriteBarrier +// helpers. It should be called by the GC whenever the ephermeral region gets moved +// from being at the top of the GC Heap, and/or when the cards table gets moved. +int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck) +{ + WRAPPER_NO_CONTRACT; + + return g_WriteBarrierManager.UpdateWriteWatchAndCardTableLocations(isRuntimeSuspended, bReqUpperBoundsCheck); +} + +void FlushWriteBarrierInstructionCache() +{ + FlushInstructionCache(GetCurrentProcess(), GetWriteBarrierCodeLocation((PVOID)JIT_WriteBarrier), g_WriteBarrierManager.GetCurrentWriteBarrierSize()); +} + +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP +int SwitchToWriteWatchBarrier(bool isRuntimeSuspended) +{ + WRAPPER_NO_CONTRACT; + + return g_WriteBarrierManager.SwitchToWriteWatchBarrier(isRuntimeSuspended); +} + +int SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended) +{ + WRAPPER_NO_CONTRACT; + + return g_WriteBarrierManager.SwitchToNonWriteWatchBarrier(isRuntimeSuspended); +} +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP From b9471196de91c68e7bcf5f90a66579a7c31fe72a Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Tue, 4 Mar 2025 16:36:58 +0000 Subject: [PATCH 21/42] undo patchedcode.asm changes --- src/coreclr/vm/arm64/patchedcode.asm | 54 +++------------------------- 1 file changed, 4 insertions(+), 50 deletions(-) diff --git a/src/coreclr/vm/arm64/patchedcode.asm b/src/coreclr/vm/arm64/patchedcode.asm index fbc623329e43f7..454b8cac0c4abe 100644 --- a/src/coreclr/vm/arm64/patchedcode.asm +++ b/src/coreclr/vm/arm64/patchedcode.asm @@ -18,7 +18,7 @@ MACRO WRITE_BARRIER_ENTRY $name - WRITE_BARRIER_ENTRY $name + LEAF_ENTRY $name MEND ; WRITE_BARRIER_END @@ -28,13 +28,13 @@ MACRO WRITE_BARRIER_END $__write_barrier_name - WRITE_BARRIER_END $__write_barrier_name + LEAF_END_MARKED $__write_barrier_name MEND ; ------------------------------------------------------------------ ; Start of the writeable code region - WRITE_BARRIER_ENTRY JIT_PatchedCodeStart + LEAF_ENTRY JIT_PatchedCodeStart ret lr LEAF_END @@ -229,55 +229,9 @@ Exit ; ------------------------------------------------------------------ ; End of the writeable code region - WRITE_BARRIER_ENTRY JIT_PatchedCodeLast + LEAF_ENTRY JIT_PatchedCodeLast ret lr LEAF_END - - -WRITE_BARRIER_ENTRY JIT_WriteBarrier_PreGrow64 - nop -WRITE_BARRIER_END JIT_WriteBarrier_PreGrow64 - - -WRITE_BARRIER_ENTRY JIT_WriteBarrier_PreGrow64 - nop -WRITE_BARRIER_END JIT_WriteBarrier_PreGrow64 - - -WRITE_BARRIER_ENTRY JIT_WriteBarrier_SVR64 - nop -WRITE_BARRIER_END JIT_WriteBarrier_SVR64 - - -WRITE_BARRIER_ENTRY JIT_WriteBarrier_Byte_Region64 - nop -WRITE_BARRIER_END JIT_WriteBarrier_Byte_Region64 - - -WRITE_BARRIER_ENTRY JIT_WriteBarrier_Bit_Region64 - nop -WRITE_BARRIER_END JIT_WriteBarrier_Bit_Region64 - - -WRITE_BARRIER_ENTRY JIT_WriteBarrier_WriteWatch_PreGrow64 - nop -WRITE_BARRIER_END JIT_WriteBarrier_WriteWatch_PreGrow64 - - -WRITE_BARRIER_ENTRY JIT_WriteBarrier_WriteWatch_SVR64 - nop -WRITE_BARRIER_END JIT_WriteBarrier_WriteWatch_SVR64 - - -WRITE_BARRIER_ENTRY JIT_WriteBarrier_WriteWatch_Byte_Region64 - nop -WRITE_BARRIER_END JIT_WriteBarrier_WriteWatch_Byte_Region64 - - -WRITE_BARRIER_ENTRY JIT_WriteBarrier_WriteWatch_Bit_Region64 - nop -WRITE_BARRIER_END JIT_WriteBarrier_WriteWatch_Bit_Region64 - ; Must be at very end of file END From 580c16fbad96a877cb80978856ec4bd23db47bce Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Tue, 4 Mar 2025 16:52:26 +0000 Subject: [PATCH 22/42] Add writebarriermanager.h --- src/coreclr/vm/arm64/stubs.cpp | 3 +- src/coreclr/vm/jitinterface.h | 76 ------------------- src/coreclr/vm/jitinterfacegen.cpp | 5 +- src/coreclr/vm/writebarriermanager.cpp | 29 +++---- src/coreclr/vm/writebarriermanager.h | 101 +++++++++++++++++++++++++ 5 files changed, 115 insertions(+), 99 deletions(-) create mode 100644 src/coreclr/vm/writebarriermanager.h diff --git a/src/coreclr/vm/arm64/stubs.cpp b/src/coreclr/vm/arm64/stubs.cpp index eb777c1afe311d..a586686abd7fee 100644 --- a/src/coreclr/vm/arm64/stubs.cpp +++ b/src/coreclr/vm/arm64/stubs.cpp @@ -13,8 +13,7 @@ #include "virtualcallstub.h" #include "jitinterface.h" #include "ecall.h" - -extern WriteBarrierManager g_WriteBarrierManager; +#include "writebarriermanager.h" #ifndef DACCESS_COMPILE //----------------------------------------------------------------------- diff --git a/src/coreclr/vm/jitinterface.h b/src/coreclr/vm/jitinterface.h index 23b96831b8d6df..9506d2c45adc13 100644 --- a/src/coreclr/vm/jitinterface.h +++ b/src/coreclr/vm/jitinterface.h @@ -242,82 +242,6 @@ extern "C" FCDECL2(VOID, JIT_WriteBarrier_Callable, Object **dst, Object *ref); #define WriteBarrier_Helper JIT_WriteBarrier_Callable -// #ifdef TARGET_AMD64 - - -class WriteBarrierManager -{ -public: - enum WriteBarrierType - { - WRITE_BARRIER_UNINITIALIZED, - WRITE_BARRIER_PREGROW64, - WRITE_BARRIER_POSTGROW64, -#ifdef FEATURE_SVR_GC - WRITE_BARRIER_SVR64, -#endif // FEATURE_SVR_GC - WRITE_BARRIER_BYTE_REGIONS64, - WRITE_BARRIER_BIT_REGIONS64, -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - WRITE_BARRIER_WRITE_WATCH_PREGROW64, - WRITE_BARRIER_WRITE_WATCH_POSTGROW64, -#ifdef FEATURE_SVR_GC - WRITE_BARRIER_WRITE_WATCH_SVR64, -#endif // FEATURE_SVR_GC - WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64, - WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64, -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - WRITE_BARRIER_BUFFER - }; - - WriteBarrierManager(); - void Initialize(); - - int UpdateEphemeralBounds(bool isRuntimeSuspended); - int UpdateWriteWatchAndCardTableLocations(bool isRuntimeSuspended, bool bReqUpperBoundsCheck); - -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - int SwitchToWriteWatchBarrier(bool isRuntimeSuspended); - int SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended); -#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - size_t GetCurrentWriteBarrierSize(); - -protected: - size_t GetSpecificWriteBarrierSize(WriteBarrierType writeBarrier); - PBYTE CalculatePatchLocation(LPVOID base, LPVOID label, int offset); - PCODE GetCurrentWriteBarrierCode(); - int ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, bool isRuntimeSuspended); - void UpdatePatchLocations(); - bool NeedDifferentWriteBarrier(bool bReqUpperBoundsCheck, bool bUseBitwiseWriteBarrier, WriteBarrierType* pNewWriteBarrierType); - -private: - void Validate(); - - WriteBarrierType m_currentWriteBarrier; - - PBYTE m_pWriteWatchTableImmediate; // PREGROW | POSTGROW | SVR | WRITE_WATCH | REGION - PBYTE m_pLowerBoundImmediate; // PREGROW | POSTGROW | | WRITE_WATCH | REGION - PBYTE m_pCardTableImmediate; // PREGROW | POSTGROW | SVR | WRITE_WATCH | REGION - PBYTE m_pCardBundleTableImmediate; // PREGROW | POSTGROW | SVR | WRITE_WATCH | REGION - PBYTE m_pUpperBoundImmediate; // | POSTGROW | | WRITE_WATCH | REGION - PBYTE m_pRegionToGenTableImmediate; // | | | WRITE_WATCH | REGION - PBYTE m_pRegionShrDest; // | | | WRITE_WATCH | REGION - PBYTE m_pRegionShrSrc; // | | | WRITE_WATCH | RETION - -#if defined(TARGET_ARM64) - PBYTE m_pRegionUseBitwiseWriteBarrier; - PBYTE m_lowestAddress; - PBYTE m_highestAddress; -#if defined(WRITE_BARRIER_CHECK) - PBYTE m_pGCShadow; - PBYTE m_pGCShadowEnd; -#endif // WRITE_BARRIER_CHECK -#endif // TARGET_AMD64 - -}; - -// #endif // TARGET_AMD64 - EXTERN_C FCDECL2_VV(INT64, JIT_LMul, INT64 val1, INT64 val2); #ifndef HOST_64BIT diff --git a/src/coreclr/vm/jitinterfacegen.cpp b/src/coreclr/vm/jitinterfacegen.cpp index 1d55e308402fe3..5dd08b3271d95d 100644 --- a/src/coreclr/vm/jitinterfacegen.cpp +++ b/src/coreclr/vm/jitinterfacegen.cpp @@ -17,6 +17,7 @@ #include "comdelegate.h" #include "field.h" #include "ecall.h" +#include "writebarriermanager.h" #ifdef HOST_64BIT @@ -32,10 +33,6 @@ EXTERN_C Object* AllocateStringFastUP (CLR_I4 cch); EXTERN_C Object* JIT_NewArr1OBJ_UP (CORINFO_CLASS_HANDLE arrayMT, INT_PTR size); EXTERN_C Object* JIT_NewArr1VC_UP (CORINFO_CLASS_HANDLE arrayMT, INT_PTR size); -#ifdef TARGET_AMD64 -extern WriteBarrierManager g_WriteBarrierManager; -#endif // TARGET_AMD64 - #endif // HOST_64BIT /*********************************************************************/ diff --git a/src/coreclr/vm/writebarriermanager.cpp b/src/coreclr/vm/writebarriermanager.cpp index 63d632c809e630..8b106d6d181026 100644 --- a/src/coreclr/vm/writebarriermanager.cpp +++ b/src/coreclr/vm/writebarriermanager.cpp @@ -13,19 +13,13 @@ #include "eeconfig.h" #include "excep.h" #include "threadsuspend.h" +#include "writebarriermanager.h" extern uint8_t* g_ephemeral_low; extern uint8_t* g_ephemeral_high; extern uint32_t* g_card_table; extern uint32_t* g_card_bundle_table; -#if defined(TARGET_AMD64) -// Write barrier variables are inlined into the assembly code -#define WRITE_BARRIER_VARS_INLINE -// Else: Write barrier variables are in a table separate to the asm code -#endif - - // Patch Labels for the various write barriers EXTERN_C void JIT_WriteBarrier_End(); @@ -277,7 +271,7 @@ PBYTE WriteBarrierManager::CalculatePatchLocation(LPVOID base, LPVOID label, int PBYTE patchLocation = (patchBase + ((LPBYTE)GetEEFuncEntryPoint(label) - (LPBYTE)GetEEFuncEntryPoint(base))); #if defined(WRITE_BARRIER_VARS_INLINE) - patchLocation+=offset; + patchLocation += inlineOffset; #endif return patchLocation; } @@ -304,7 +298,9 @@ int WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, stompWBCompleteActions |= SWB_ICACHE_FLUSH; } - UpdatePatchLocations(); +#if defined(WRITE_BARRIER_VARS_INLINE) + UpdatePatchLocations(newWriteBarrier); +#endif stompWBCompleteActions |= UpdateEphemeralBounds(true); stompWBCompleteActions |= UpdateWriteWatchAndCardTableLocations(true, false); @@ -370,7 +366,7 @@ void WriteBarrierManager::Initialize() #endif // !WRITE_BARRIER_VARS_INLINE -#if !defined(CODECOVERAGE) +#if !defined(CODECOVERAGE) && defined(WRITE_BARRIER_VARS_INLINE) Validate(); #endif } @@ -696,6 +692,8 @@ int WriteBarrierManager::SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended) #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP +#if defined(WRITE_BARRIER_VARS_INLINE) + // Deactivate alignment validation for code coverage builds // because the instrumentation tool will not preserve alignment // constraints and we will fail. @@ -703,7 +701,6 @@ int WriteBarrierManager::SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended) void WriteBarrierManager::Validate() { -#if defined(WRITE_BARRIER_VARS_INLINE) CONTRACTL { MODE_ANY; @@ -860,16 +857,12 @@ void WriteBarrierManager::Validate() #endif #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - -#endif // WRITE_BARRIER_VARS_INLINE } #endif // CODECOVERAGE -void WriteBarrierManager::UpdatePatchLocations() +void WriteBarrierManager::UpdatePatchLocations(WriteBarrierType newWriteBarrier) { -#if defined(WRITE_BARRIER_VARS_INLINE) - switch (newWriteBarrier) { case WRITE_BARRIER_PREGROW64: @@ -1077,9 +1070,11 @@ void WriteBarrierManager::UpdatePatchLocations() default: UNREACHABLE_MSG("unexpected write barrier type!"); } -#endif // WRITE_BARRIER_VARS_INLINE } +#endif // WRITE_BARRIER_VARS_INLINE + + // This function bashes the super fast version of the JIT_WriteBarrier // helper. It should be called by the GC whenever the ephermeral region // bounds get changed, but still remain on the top of the GC Heap. diff --git a/src/coreclr/vm/writebarriermanager.h b/src/coreclr/vm/writebarriermanager.h new file mode 100644 index 00000000000000..8a140179db971c --- /dev/null +++ b/src/coreclr/vm/writebarriermanager.h @@ -0,0 +1,101 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// =========================================================================== +// File: writebarriermanager.h +// =========================================================================== + + +#ifndef WRITEBARRIERMANAGER_H +#define WRITEBARRIERMANAGER_H + +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + + +#if defined(TARGET_AMD64) +// Write barrier variables are inlined into the assembly code +#define WRITE_BARRIER_VARS_INLINE +// Else: Write barrier variables are in a table separate to the asm code +#endif + +class WriteBarrierManager +{ +public: + enum WriteBarrierType + { + WRITE_BARRIER_UNINITIALIZED, + WRITE_BARRIER_PREGROW64, + WRITE_BARRIER_POSTGROW64, +#ifdef FEATURE_SVR_GC + WRITE_BARRIER_SVR64, +#endif // FEATURE_SVR_GC + WRITE_BARRIER_BYTE_REGIONS64, + WRITE_BARRIER_BIT_REGIONS64, +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + WRITE_BARRIER_WRITE_WATCH_PREGROW64, + WRITE_BARRIER_WRITE_WATCH_POSTGROW64, +#ifdef FEATURE_SVR_GC + WRITE_BARRIER_WRITE_WATCH_SVR64, +#endif // FEATURE_SVR_GC + WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64, + WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64, +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + WRITE_BARRIER_BUFFER + }; + + WriteBarrierManager(); + void Initialize(); + + int UpdateEphemeralBounds(bool isRuntimeSuspended); + int UpdateWriteWatchAndCardTableLocations(bool isRuntimeSuspended, bool bReqUpperBoundsCheck); + +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + int SwitchToWriteWatchBarrier(bool isRuntimeSuspended); + int SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended); +#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + size_t GetCurrentWriteBarrierSize(); + +protected: + size_t GetSpecificWriteBarrierSize(WriteBarrierType writeBarrier); + PBYTE CalculatePatchLocation(LPVOID base, LPVOID label, int offset); + PCODE GetCurrentWriteBarrierCode(); + int ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, bool isRuntimeSuspended); + bool NeedDifferentWriteBarrier(bool bReqUpperBoundsCheck, bool bUseBitwiseWriteBarrier, WriteBarrierType* pNewWriteBarrierType); + + +#if defined(WRITE_BARRIER_VARS_INLINE) +private: + void Validate(); + void UpdatePatchLocations(WriteBarrierType newWriteBarrier); +#endif // WRITE_BARRIER_VARS_INLINE + +private: + + WriteBarrierType m_currentWriteBarrier; + + PBYTE m_pWriteWatchTableImmediate; // PREGROW | POSTGROW | SVR | WRITE_WATCH | REGION + PBYTE m_pLowerBoundImmediate; // PREGROW | POSTGROW | | WRITE_WATCH | REGION + PBYTE m_pCardTableImmediate; // PREGROW | POSTGROW | SVR | WRITE_WATCH | REGION + PBYTE m_pCardBundleTableImmediate; // PREGROW | POSTGROW | SVR | WRITE_WATCH | REGION + PBYTE m_pUpperBoundImmediate; // | POSTGROW | | WRITE_WATCH | REGION + PBYTE m_pRegionToGenTableImmediate; // | | | WRITE_WATCH | REGION + PBYTE m_pRegionShrDest; // | | | WRITE_WATCH | REGION + PBYTE m_pRegionShrSrc; // | | | WRITE_WATCH | RETION + +#if defined(TARGET_ARM64) + PBYTE m_pRegionUseBitwiseWriteBarrier; + PBYTE m_lowestAddress; + PBYTE m_highestAddress; +#if defined(WRITE_BARRIER_CHECK) + PBYTE m_pGCShadow; + PBYTE m_pGCShadowEnd; +#endif // WRITE_BARRIER_CHECK +#endif // TARGET_AMD64 + +}; + +extern WriteBarrierManager g_WriteBarrierManager; + +#endif // TARGET_AMD64 || TARGET_ARM64 + +#endif // WRITEBARRIERMANAGER_H From 7993687ce80bb393a671d9524987c3e220baf66c Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Wed, 5 Mar 2025 10:44:54 +0000 Subject: [PATCH 23/42] Update doc --- docs/design/coreclr/jit/GC-write-barriers.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/design/coreclr/jit/GC-write-barriers.md b/docs/design/coreclr/jit/GC-write-barriers.md index f9a8fdfff9fc2f..8a9c987b4a41e0 100644 --- a/docs/design/coreclr/jit/GC-write-barriers.md +++ b/docs/design/coreclr/jit/GC-write-barriers.md @@ -25,7 +25,7 @@ JIT_WriteBarrier(Object **dst, Object *ref) if *ww_table_dst != 0: *ww_table_dst = 0xff - // Return if the reference is not in Gen 0 + // Return if the reference is not in ephemeral generations. if ref < g_ephemeral_low || ref >= g_ephemeral_high: return @@ -93,4 +93,4 @@ JIT_CheckedWriteBarrier(Object **dst, Object *ref) ## WritebarrierManager -On AMD64, there several different implementations of the write barrier function. Each implementation assumes different state and so can skip certain checks. The actual write barrier that is called is a copy of one of these implementations. The WritebarrierManager keeps track of which implementation is currently being used. As internal state changes, the WritebarrierManager updates the copy to the correct implementation. In practice, most of the internal state is fixed on startup, with only changes to/from use of write watch barriers changing during runtime. +On AMD64 and Arm64, there several different implementations of the write barrier function. Each implementation assumes different state and so can skip certain checks. The actual write barrier that is called is a copy of one of these implementations. The WritebarrierManager keeps track of which implementation is currently being used. As internal state changes, the WritebarrierManager updates the copy to the correct implementation. In practice, most of the internal state is fixed on startup, with only changes to/from use of write watch barriers changing during runtime. From 90d889bbcb52375434f5d99356097eefa9e780d6 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Wed, 5 Mar 2025 11:59:50 +0000 Subject: [PATCH 24/42] Replace JIT_WriteBarrier with zeroed memory --- src/coreclr/vm/arm64/patchedcode.S | 160 +++-------------------------- 1 file changed, 12 insertions(+), 148 deletions(-) diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index 3430de4e58f766..ba646cb023df3a 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -85,6 +85,10 @@ WRITE_BARRIER_END JIT_CheckedWriteBarrier //----------------------------------------------------------------------------- // void JIT_WriteBarrier(Object** dst, Object* src) +// +// Empty function which at runtime is patched with one JIT_WriteBarrier_ +// functions below. +// // On entry: // x14 : the destination address (LHS of the assignment) // x15 : the object reference (RHS of the assignment) @@ -99,161 +103,26 @@ WRITE_BARRIER_END JIT_CheckedWriteBarrier // if you add more trashed registers. // WRITE_BARRIER_ENTRY JIT_WriteBarrier -PATCH_LABEL JIT_WriteBarrier_Label - stlr x15, [x14] - -#ifdef WRITE_BARRIER_CHECK - // Update GC Shadow Heap - - // Do not perform the work if g_GCShadow is 0 - ldr x12, LOCAL_LABEL(wbs_GCShadow) - cbz x12, LOCAL_LABEL(ShadowUpdateEnd) - - // Compute address of shadow heap location: - // pShadow = g_GCShadow + (x14 - g_lowest_address) - ldr x17, LOCAL_LABEL(wbs_lowest_address) - sub x17, x14, x17 - add x12, x17, x12 - - // if (pShadow >= g_GCShadowEnd) goto end - ldr x17, LOCAL_LABEL(wbs_GCShadowEnd) - cmp x12, x17 - bhs LOCAL_LABEL(ShadowUpdateEnd) - - // *pShadow = x15 - str x15, [x12] - - // Ensure that the write to the shadow heap occurs before the read from the GC heap so that race - // conditions are caught by INVALIDGCVALUE. - dmb ish - - // if ([x14] == x15) goto end - ldr x17, [x14] - cmp x17, x15 - beq LOCAL_LABEL(ShadowUpdateEnd) - - // *pShadow = INVALIDGCVALUE (0xcccccccd) - movz x17, #0xcccd - movk x17, #0xcccc, LSL #16 - str x17, [x12] - -LOCAL_LABEL(ShadowUpdateEnd): -#endif - -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - // Update the write watch table if necessary - ldr x12, LOCAL_LABEL(wbs_sw_ww_table) - cbz x12, LOCAL_LABEL(CheckCardTableBounds) - add x12, x12, x14, lsr #0xc // SoftwareWriteWatch::AddressToTableByteIndexShift - ldrb w17, [x12] - cbnz x17, LOCAL_LABEL(CheckCardTableBounds) - mov w17, #0xFF - strb w17, [x12] -#endif - -LOCAL_LABEL(CheckCardTableBounds): - // Branch to Exit if the reference is not in the Gen0 heap - ldr x12, LOCAL_LABEL(wbs_ephemeral_low) - ldr x17, LOCAL_LABEL(wbs_ephemeral_high) - cmp x15, x12 - ccmp x15, x17, #0x2, hs - bhs LOCAL_LABEL(Exit) - - // Region Checks - - // Check if using regions - ldr x17, LOCAL_LABEL(wbs_region_to_generation_table) - cbz x17, LOCAL_LABEL(CheckCardTable) - - // Calculate region locations - ldr w12, LOCAL_LABEL(wbs_region_shr) - lsr x15, x15, x12 - add x15, x15, x17 // x15 = (RHS >> wbs_region_shr) + wbs_region_to_generation_table - lsr x12, x14, x12 - add x12, x12, x17 // x12 = (LHS >> wbs_region_shr) + wbs_region_to_generation_table - - // Check whether the region we're storing into is gen 0 - nothing to do in this case - ldrb w12, [x12] - cbz w12, LOCAL_LABEL(Exit) - - // Check this is going from old to young - ldrb w15, [x15] - cmp w15, w12 - bhs LOCAL_LABEL(Exit) - - // Bitwise write barriers only - ldr w17, LOCAL_LABEL(wbs_region_use_bitwise_write_barrier) - cbz w17, LOCAL_LABEL(CheckCardTable) - - // Check if we need to update the card table - lsr w17, w14, 8 - and w17, w17, 7 - movz w15, 1 - lsl w17, w15, w17 // w17 = 1 << (LHS >> 8 && 7) - ldr x12, LOCAL_LABEL(wbs_card_table) - add x15, x12, x14, lsr #11 - ldrb w12, [x15] // w12 = [(LHS >> 11) + g_card_table] - tst w12, w17 - bne LOCAL_LABEL(Exit) - - // Atomically update the card table - // Requires LSE, but the code is only compiled for 8.0 - .word 0x383131FF // stsetb w17, [x15] - b LOCAL_LABEL(CheckCardBundleTable) - - // End of Region Checks - -LOCAL_LABEL(CheckCardTable): - // Check if we need to update the card table - ldr x12, LOCAL_LABEL(wbs_card_table) - add x15, x12, x14, lsr #11 - ldrb w12, [x15] // w12 = [(LHS >> 11) + g_card_table] - cmp x12, 0xFF - beq LOCAL_LABEL(Exit) - - // Update the card table - mov x12, 0xFF - strb w12, [x15] - -LOCAL_LABEL(CheckCardBundleTable): -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - // Check if we need to update the card bundle table - ldr x12, LOCAL_LABEL(wbs_card_bundle_table) - add x15, x12, x14, lsr #21 - ldrb w12, [x15] - cmp x12, 0xFF - beq LOCAL_LABEL(Exit) - - // Update the card bundle - mov x12, 0xFF - strb w12, [x15] -#endif - -LOCAL_LABEL(Exit): - // Increment by 8 to implement JIT_ByRefWriteBarrier contract. - // TODO: Consider duplicating the logic to get rid of this redundant 'add' - // for JIT_WriteBarrier/JIT_CheckedWriteBarrier - add x14, x14, 8 - ret lr + // This must be greater than the largest JIT_WriteBarrier_ function. + .space (232*4), 0 WRITE_BARRIER_END JIT_WriteBarrier - // Begin patchable literal pool +//----------------------------------------------------------------------------- +// JIT_WriteBarrier_Table +// +// Patchable literal pool +// .balign 64 // Align to power of two at least as big as patchable literal pool so that it fits optimally in cache line WRITE_BARRIER_ENTRY JIT_WriteBarrier_Table LOCAL_LABEL(wbs_begin): -LOCAL_LABEL(wbs_card_table): PATCH_LABEL JIT_WriteBarrier_Patch_Label_CardTable .quad 0 -LOCAL_LABEL(wbs_card_bundle_table): PATCH_LABEL JIT_WriteBarrier_Patch_Label_CardBundleTable .quad 0 -LOCAL_LABEL(wbs_sw_ww_table): PATCH_LABEL JIT_WriteBarrier_Patch_Label_WriteWatchTable .quad 0 -LOCAL_LABEL(wbs_ephemeral_low): PATCH_LABEL JIT_WriteBarrier_Patch_Label_Lower .quad 0 -LOCAL_LABEL(wbs_ephemeral_high): PATCH_LABEL JIT_WriteBarrier_Patch_Label_Upper .quad 0 LOCAL_LABEL(wbs_lowest_address): @@ -262,20 +131,15 @@ PATCH_LABEL JIT_WriteBarrier_Patch_Label_LowestAddress LOCAL_LABEL(wbs_highest_address): PATCH_LABEL JIT_WriteBarrier_Patch_Label_HighestAddress .quad 0 -LOCAL_LABEL(wbs_region_to_generation_table): PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionToGeneration .quad 0 -LOCAL_LABEL(wbs_region_shr): PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionShr .word 0 -LOCAL_LABEL(wbs_region_use_bitwise_write_barrier): PATCH_LABEL JIT_WriteBarrier_Patch_Label_UseBitwiseWriteBarrier .word 0 #ifdef WRITE_BARRIER_CHECK -LOCAL_LABEL(wbs_GCShadow): PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadow .quad 0 -LOCAL_LABEL(wbs_GCShadowEnd): PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadowEnd .quad 0 #endif @@ -291,7 +155,7 @@ LEAF_END JIT_PatchedCodeLast, _TEXT //----------------------------------------------------------------------------- -// The following Macros are used by the different writebarrier implementations +// The following Macros are used by the different JIT_WriteBarrier_ functions. // // From da7b5c0cec1a3a3d9d39f01b9433c37c78622e50 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Wed, 5 Mar 2025 12:27:56 +0000 Subject: [PATCH 25/42] Fix writebarrier macro labels --- src/coreclr/vm/arm64/patchedcode.S | 44 +++++++++++++++--------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index ba646cb023df3a..758d792f4571a6 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -419,7 +419,7 @@ LEAF_END_MARKED JIT_WriteBarrier_Bit_Region64, _TEXT LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_PreGrow64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_PreGrow64 - WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PreGrow64 JIT_WriteBarrier_WriteWatch_PreGrow64Start + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 @@ -434,12 +434,12 @@ LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT // No region checks // LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT - WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_PostGrow64Start - WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_PostGrow64Start - WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PostGrow64Start JIT_WriteBarrier_WriteWatch_PostGrow64Start - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_PostGrow64Start Exit_WriteWatch_PostGrow64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_PostGrow64Start Exit_WriteWatch_PostGrow64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_PostGrow64Start Exit_WriteWatch_PostGrow64 + WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_PostGrow64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_PostGrow64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PostGrow64 Exit_WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_PostGrow64 Exit_WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_PostGrow64 Exit_WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_PostGrow64 Exit_WriteWatch_PostGrow64 WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_PostGrow64 LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT @@ -457,7 +457,7 @@ LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT LEAF_ENTRY JIT_WriteBarrier_WriteWatch_SVR64, _TEXT WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_SVR64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_SVR64 - WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_SVR64 JIT_WriteBarrier_WriteWatch_SVR64Start + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_SVR64 Exit_WriteWatch_SVR64 WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_SVR64 Exit_WriteWatch_SVR64 WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_SVR64 Exit_WriteWatch_SVR64 WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_SVR64 @@ -472,13 +472,13 @@ LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_SVR64, _TEXT // LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT LOCAL_LABEL(JIT_WriteBarrier_WriteWatch_Byte_Region64Start): - WRITE_BARRIER_ENTRY_STUB Start_Byte_Region64Start - WRITE_BARRIER_SHADOW_UPDATE_STUB Start_Byte_Region64Start - WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_Byte_Region64Start JIT_WriteBarrier_WriteWatch_Byte_Region64Start - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Byte_Region64Start Exit_WriteWatch_Byte_Region64 - WRITE_BARRIER_REGION_CHECK_STUB Start_Byte_Region64Start Exit_WriteWatch_Byte_Region64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_Byte_Region64Start Exit_WriteWatch_Byte_Region64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Byte_Region64Start Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_Byte_Region64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_Byte_Region64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_Byte_Region64 LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT @@ -491,13 +491,13 @@ LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT // LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT LOCAL_LABEL(JIT_WriteBarrier_WriteWatch_Bit_Region64Start): - WRITE_BARRIER_ENTRY_STUB Start_Bit_Region64Start - WRITE_BARRIER_SHADOW_UPDATE_STUB Start_Bit_Region64Start - WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_Bit_Region64Start JIT_WriteBarrier_WriteWatch_Bit_Region64Start - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Bit_Region64Start Exit_WriteWatch_Bit_Region64 - WRITE_BARRIER_REGION_CHECK_STUB Start_Bit_Region64Start Exit_WriteWatch_Bit_Region64 - WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Start_Bit_Region64Start Exit_WriteWatch_Bit_Region64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Bit_Region64Start Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_Bit_Region64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_Bit_Region64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_Bit_Region64 LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT From dc6d0a7df81871cd7940fefdb8811bdac1548ab5 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Wed, 5 Mar 2025 12:40:15 +0000 Subject: [PATCH 26/42] Fix JIT_WriteBarrier_PreGrow64 checks --- src/coreclr/vm/arm64/patchedcode.S | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index 758d792f4571a6..3ffaa317b12489 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -223,15 +223,15 @@ LOCAL_LABEL(WriteWatchForGCHeapEnd\@): .macro WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB start exit - // Branch to Exit if the reference is not in the heap + // Branch to Exit if the reference is not in the ephemeral generations. ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_Lower) cmp x15, x12 - bhs LOCAL_LABEL(\exit) + blo LOCAL_LABEL(\exit) .endm .macro WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB start exit - // Branch to Exit if the reference is not in the Gen0 heap + // Branch to Exit if the reference is not in the ephemeral generations. ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_Lower) ldr x17, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_Upper) cmp x15, x12 From 6e6684282537eb80f8c595abee3264e851219452 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Wed, 5 Mar 2025 13:47:13 +0000 Subject: [PATCH 27/42] remove unused label --- src/coreclr/vm/arm64/patchedcode.S | 1 - 1 file changed, 1 deletion(-) diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index 3ffaa317b12489..40a8fc7ef5d6ce 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -279,7 +279,6 @@ LOCAL_LABEL(WriteWatchForGCHeapEnd\@): .macro WRITE_BARRIER_CHECK_CARD_TABLE_STUB start exit -LOCAL_LABEL(CheckCardTable\@): // Check if we need to update the card table ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_CardTable) add x15, x12, x14, lsr #11 From 729cb776c59f0ecd6222a070e246d91f0407d511 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Wed, 5 Mar 2025 14:20:54 +0000 Subject: [PATCH 28/42] Remove JIT_UpdateWriteBarrierState --- src/coreclr/vm/arm64/asmhelpers.S | 87 -------------------------- src/coreclr/vm/arm64/asmhelpers.asm | 78 ----------------------- src/coreclr/vm/arm64/patchedcode.S | 2 - src/coreclr/vm/writebarriermanager.cpp | 10 +-- src/coreclr/vm/writebarriermanager.h | 1 - 5 files changed, 2 insertions(+), 176 deletions(-) diff --git a/src/coreclr/vm/arm64/asmhelpers.S b/src/coreclr/vm/arm64/asmhelpers.S index 959999428d9c6a..4b3c6a63c89cd5 100644 --- a/src/coreclr/vm/arm64/asmhelpers.S +++ b/src/coreclr/vm/arm64/asmhelpers.S @@ -180,93 +180,6 @@ PATCH_LABEL ThePreStubPatchLabel ret lr LEAF_END ThePreStubPatch, _TEXT -// void JIT_UpdateWriteBarrierState(bool skipEphemeralCheck, size_t writeableOffset) -// -// Update shadow copies of the various state info required for barrier -// -// State info is contained in a literal pool at the end of the function -// Placed in text section so that it is close enough to use ldr literal and still -// be relocatable. Eliminates need for PREPARE_EXTERNAL_VAR in hot code. -// -// Align and group state info together so it fits in a single cache line -// and each entry can be written atomically -// -LEAF_ENTRY JIT_UpdateWriteBarrierState, _TEXT - PROLOG_SAVE_REG_PAIR_INDEXED fp, lr, -16 - - // x0-x7, x10-x11, x13-x14 will contain intended new state - // x8 will preserve skipEphemeralCheck - // x9 will preserve writeableOffset - // x12 will be used for pointers - - mov x8, x0 - mov x9, x1 - - PREPARE_EXTERNAL_VAR g_card_table, x12 - ldr x0, [x12] - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - PREPARE_EXTERNAL_VAR g_card_bundle_table, x12 - ldr x1, [x12] -#endif - -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - PREPARE_EXTERNAL_VAR g_sw_ww_table, x12 - ldr x2, [x12] -#endif - - PREPARE_EXTERNAL_VAR g_ephemeral_low, x12 - ldr x3, [x12] - - PREPARE_EXTERNAL_VAR g_ephemeral_high, x12 - ldr x4, [x12] - - cbz x8, LOCAL_LABEL(EphemeralCheckEnabled) - movz x3, #0 - movn x4, #0 -LOCAL_LABEL(EphemeralCheckEnabled): - - PREPARE_EXTERNAL_VAR g_lowest_address, x12 - ldr x5, [x12] - - PREPARE_EXTERNAL_VAR g_highest_address, x12 - ldr x6, [x12] - - PREPARE_EXTERNAL_VAR g_region_to_generation_table, x12 - ldr x7, [x12] - - PREPARE_EXTERNAL_VAR g_region_shr, x12 - ldr w10, [x12] - - PREPARE_EXTERNAL_VAR g_region_use_bitwise_write_barrier, x12 - ldr w11, [x12] - -#ifdef WRITE_BARRIER_CHECK - PREPARE_EXTERNAL_VAR g_GCShadow, x12 - ldr x13, [x12] - - PREPARE_EXTERNAL_VAR g_GCShadowEnd, x12 - ldr x14, [x12] -#endif - - // Update wbs state - PREPARE_EXTERNAL_VAR JIT_WriteBarrier_Table_Loc, x12 - ldr x12, [x12] - add x12, x12, x9 - - stp x0, x1, [x12], 16 - stp x2, x3, [x12], 16 - stp x4, x5, [x12], 16 - stp x6, x7, [x12], 16 - stp w10, w11, [x12], 8 -#ifdef WRITE_BARRIER_CHECK - stp x13, x14, [x12], 16 -#endif - - EPILOG_RESTORE_REG_PAIR_INDEXED fp, lr, 16 - EPILOG_RETURN -LEAF_END JIT_UpdateWriteBarrierState - // ------------------------// ------------------------------------------------------------------ // __declspec(naked) void F_CALL_CONV JIT_WriteBarrier_Callable(Object **dst, Object* val) LEAF_ENTRY JIT_WriteBarrier_Callable, _TEXT diff --git a/src/coreclr/vm/arm64/asmhelpers.asm b/src/coreclr/vm/arm64/asmhelpers.asm index 081ee30a97c431..88b46be87118c6 100644 --- a/src/coreclr/vm/arm64/asmhelpers.asm +++ b/src/coreclr/vm/arm64/asmhelpers.asm @@ -244,84 +244,6 @@ ThePreStubPatchLabel ret lr LEAF_END -;----------------------------------------------------------------------------- -; void JIT_UpdateWriteBarrierState(bool skipEphemeralCheck, size_t writeableOffset) -; -; Update shadow copies of the various state info required for barrier -; -; State info is contained in a literal pool at the end of the function -; Placed in text section so that it is close enough to use ldr literal and still -; be relocatable. Eliminates need for PREPARE_EXTERNAL_VAR in hot code. -; -; Align and group state info together so it fits in a single cache line -; and each entry can be written atomically -; - LEAF_ENTRY JIT_UpdateWriteBarrierState - PROLOG_SAVE_REG_PAIR fp, lr, #-16! - - ; x0-x7, x10 will contain intended new state - ; x8 will preserve skipEphemeralCheck - ; x12 will be used for pointers - - mov x8, x0 - mov x9, x1 - - adrp x12, g_card_table - ldr x0, [x12, g_card_table] - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - adrp x12, g_card_bundle_table - ldr x1, [x12, g_card_bundle_table] -#endif - -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - adrp x12, g_sw_ww_table - ldr x2, [x12, g_sw_ww_table] -#endif - - adrp x12, g_ephemeral_low - ldr x3, [x12, g_ephemeral_low] - - adrp x12, g_ephemeral_high - ldr x4, [x12, g_ephemeral_high] - - ; Check skipEphemeralCheck - cbz x8, EphemeralCheckEnabled - movz x3, #0 - movn x4, #0 - -EphemeralCheckEnabled - adrp x12, g_lowest_address - ldr x5, [x12, g_lowest_address] - - adrp x12, g_highest_address - ldr x6, [x12, g_highest_address] - -#ifdef WRITE_BARRIER_CHECK - adrp x12, $g_GCShadow - ldr x7, [x12, $g_GCShadow] - - adrp x12, $g_GCShadowEnd - ldr x10, [x12, $g_GCShadowEnd] -#endif - - ; Update wbs state - adrp x12, JIT_WriteBarrier_Table_Loc - ldr x12, [x12, JIT_WriteBarrier_Table_Loc] - add x12, x12, x9 - stp x0, x1, [x12], 16 - stp x2, x3, [x12], 16 - stp x4, x5, [x12], 16 - str x6, [x12], 8 -#ifdef WRITE_BARRIER_CHECK - stp x7, x10, [x12], 16 -#endif - - EPILOG_RESTORE_REG_PAIR fp, lr, #16! - EPILOG_RETURN - - LEAF_END JIT_UpdateWriteBarrierState - #ifdef FEATURE_COMINTEROP ; ------------------------------------------------------------------ diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index 40a8fc7ef5d6ce..08c2d7b4c266b9 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -135,8 +135,6 @@ PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionToGeneration .quad 0 PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionShr .word 0 -PATCH_LABEL JIT_WriteBarrier_Patch_Label_UseBitwiseWriteBarrier - .word 0 #ifdef WRITE_BARRIER_CHECK PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadow .quad 0 diff --git a/src/coreclr/vm/writebarriermanager.cpp b/src/coreclr/vm/writebarriermanager.cpp index 8b106d6d181026..14e567e7176ca5 100644 --- a/src/coreclr/vm/writebarriermanager.cpp +++ b/src/coreclr/vm/writebarriermanager.cpp @@ -153,7 +153,6 @@ EXTERN_C void JIT_WriteBarrier_Patch_Label_CardTable(); EXTERN_C void JIT_WriteBarrier_Patch_Label_CardBundleTable(); #endif #if defined(TARGET_ARM64) -EXTERN_C void JIT_WriteBarrier_Patch_Label_UseBitwiseWriteBarrier(); EXTERN_C void JIT_WriteBarrier_Patch_Label_LowestAddress(); EXTERN_C void JIT_WriteBarrier_Patch_Label_HighestAddress(); #if defined(WRITE_BARRIER_CHECK) @@ -354,15 +353,12 @@ void WriteBarrierManager::Initialize() m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_CardBundleTable, 0); #endif -#if defined(TARGET_ARM64) - m_pRegionUseBitwiseWriteBarrier = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_UseBitwiseWriteBarrier, 0); -#if defined(WRITE_BARRIER_CHECK) +#if defined(TARGET_ARM64) && defined(WRITE_BARRIER_CHECK) m_pGCShadow = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_GCShadow, 0); m_pGCShadowEnd = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_GCShadowEnd, 0); m_lowestAddress = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_LowestAddress, 0); m_highestAddress = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_HighestAddress, 0); -#endif // WRITE_BARRIER_CHECK -#endif // TARGET_ARM64 +#endif // TARGET_ARM64 && WRITE_BARRIER_CHECK #endif // !WRITE_BARRIER_VARS_INLINE @@ -521,7 +517,6 @@ int WriteBarrierManager::UpdateEphemeralBounds(bool isRuntimeSuspended) #if defined(TARGET_ARM64) - stompWBCompleteActions |= updateVariable(m_pRegionUseBitwiseWriteBarrier, g_region_use_bitwise_write_barrier); stompWBCompleteActions |= updateVariable(m_lowestAddress, (size_t)g_lowest_address); stompWBCompleteActions |= updateVariable(m_highestAddress, (size_t)g_highest_address); #if defined(WRITE_BARRIER_CHECK) @@ -601,7 +596,6 @@ int WriteBarrierManager::UpdateWriteWatchAndCardTableLocations(bool isRuntimeSus #endif #if defined(TARGET_ARM64) - stompWBCompleteActions |= updateVariable(m_pRegionUseBitwiseWriteBarrier, g_region_use_bitwise_write_barrier); stompWBCompleteActions |= updateVariable(m_lowestAddress, (size_t)g_lowest_address); stompWBCompleteActions |= updateVariable(m_highestAddress, (size_t)g_highest_address); #if defined(WRITE_BARRIER_CHECK) diff --git a/src/coreclr/vm/writebarriermanager.h b/src/coreclr/vm/writebarriermanager.h index 8a140179db971c..bf85978b14f33c 100644 --- a/src/coreclr/vm/writebarriermanager.h +++ b/src/coreclr/vm/writebarriermanager.h @@ -83,7 +83,6 @@ class WriteBarrierManager PBYTE m_pRegionShrSrc; // | | | WRITE_WATCH | RETION #if defined(TARGET_ARM64) - PBYTE m_pRegionUseBitwiseWriteBarrier; PBYTE m_lowestAddress; PBYTE m_highestAddress; #if defined(WRITE_BARRIER_CHECK) From e0759b87437699501c67ec6379e871fd64593ac4 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Wed, 5 Mar 2025 14:58:09 +0000 Subject: [PATCH 29/42] first attempt at windows .asm --- src/coreclr/vm/arm64/patchedcode.S | 2 +- src/coreclr/vm/arm64/patchedcode.asm | 471 +++++++++++++++++++++------ 2 files changed, 371 insertions(+), 102 deletions(-) diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index 08c2d7b4c266b9..18bfa2cdda5876 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -86,7 +86,7 @@ WRITE_BARRIER_END JIT_CheckedWriteBarrier //----------------------------------------------------------------------------- // void JIT_WriteBarrier(Object** dst, Object* src) // -// Empty function which at runtime is patched with one JIT_WriteBarrier_ +// Empty function which at runtime is patched with one of the JIT_WriteBarrier_ // functions below. // // On entry: diff --git a/src/coreclr/vm/arm64/patchedcode.asm b/src/coreclr/vm/arm64/patchedcode.asm index 454b8cac0c4abe..6486eef99b336f 100644 --- a/src/coreclr/vm/arm64/patchedcode.asm +++ b/src/coreclr/vm/arm64/patchedcode.asm @@ -42,26 +42,32 @@ ALIGN 64 ; Align to power of two at least as big as patchable literal pool so that it fits optimally in cache line WRITE_BARRIER_ENTRY JIT_WriteBarrier_Table wbs_begin -wbs_card_table +PATCH_LABEL JIT_WriteBarrier_Patch_Label_CardTable DCQ 0 -wbs_card_bundle_table +PATCH_LABEL JIT_WriteBarrier_Patch_Label_CardBundleTable DCQ 0 -wbs_sw_ww_table +PATCH_LABEL JIT_WriteBarrier_Patch_Label_WriteWatchTable DCQ 0 -wbs_ephemeral_low +PATCH_LABEL JIT_WriteBarrier_Patch_Label_Lower DCQ 0 -wbs_ephemeral_high +PATCH_LABEL JIT_WriteBarrier_Patch_Label_Upper DCQ 0 wbs_lowest_address +PATCH_LABEL JIT_WriteBarrier_Patch_Label_LowestAddress DCQ 0 wbs_highest_address +PATCH_LABEL JIT_WriteBarrier_Patch_Label_HighestAddress DCQ 0 -#ifdef WRITE_BARRIER_CHECK -wbs_GCShadow +PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionToGeneration DCQ 0 -wbs_GCShadowEnd +PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionShr + DCW 0 +ifdef WRITE_BARRIER_CHECK +PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadow DCQ 0 -#endif +PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadowEnd + DCQ 0 +endif WRITE_BARRIER_END JIT_WriteBarrier_Table ;----------------------------------------------------------------------------- @@ -117,6 +123,9 @@ NotInHeap ;----------------------------------------------------------------------------- ; void JIT_WriteBarrier(Object** dst, Object* src) +; +; Empty function which at runtime is patched with one of the JIT_WriteBarrier_ +; functions below. ; On entry: ; x14 : the destination address (LHS of the assignment) ; x15 : the object reference (RHS of the assignment) @@ -131,107 +140,367 @@ NotInHeap ; if you add more trashed registers. ; WRITE_BARRIER_ENTRY JIT_WriteBarrier - stlr x15, [x14] +; This must be greater than the largest JIT_WriteBarrier_ function. + space (232*4), 0 + WRITE_BARRIER_END JIT_WriteBarrier + +; ------------------------------------------------------------------ +; End of the writeable code region + LEAF_ENTRY JIT_PatchedCodeLast + ret lr + LEAF_END + +; Must be at very end of file + END + + + +;----------------------------------------------------------------------------- +; The following Macros are used by the different JIT_WriteBarrier_ functions. +; +; + +#define WRITE_BARRIER_CONSTANT_OFFSET(start, constlabel) (constlabel - C_FUNC(JIT_WriteBarrier) + start) + -#ifdef WRITE_BARRIER_CHECK - ; Update GC Shadow Heap +WRITE_BARRIER_ENTRY_STUB macro start +start + stlr x15, [x14] +endm - ; Do not perform the work if g_GCShadow is 0 - ldr x12, wbs_GCShadow - cbz x12, ShadowUpdateEnd - ; Compute address of shadow heap location: - ; pShadow = $g_GCShadow + (x14 - g_lowest_address) - ldr x17, wbs_lowest_address - sub x17, x14, x17 - add x12, x17, x12 +WRITE_BARRIER_SHADOW_UPDATE_STUB macro start +ifdef WRITE_BARRIER_CHECK +; Update GC Shadow Heap - ; if (pShadow >= $g_GCShadowEnd) goto end - ldr x17, wbs_GCShadowEnd - cmp x12, x17 - bhs ShadowUpdateEnd +; Do not perform the work if g_GCShadow is 0 + ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_GCShadow) + cbz x12, ShadowUpdateEnd - ; *pShadow = x15 - str x15, [x12] +; Compute address of shadow heap location: +; pShadow = g_GCShadow + (x14 - g_lowest_address) + ldr x17, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_LowestAddress) + sub x17, x14, x17 + add x12, x17, x12 - ; Ensure that the write to the shadow heap occurs before the read from the GC heap so that race - ; conditions are caught by INVALIDGCVALUE. - dmb ish +; if (pShadow >= g_GCShadowEnd) goto end + ldr x17, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_GCShadowEnd) + cmp x12, x17 + bhs ShadowUpdateEnd - ; if ([x14] == x15) goto end - ldr x17, [x14] - cmp x17, x15 - beq ShadowUpdateEnd +; *pShadow = x15 + str x15, [x12] - ; *pShadow = INVALIDGCVALUE (0xcccccccd) - movz x17, #0xcccd - movk x17, #0xcccc, LSL #16 - str x17, [x12] +; Ensure that the write to the shadow heap occurs before the read from the GC heap so that race +; conditions are caught by INVALIDGCVALUE. + dmb ish +; if ([x14] == x15) goto end + ldr x17, [x14] + cmp x17, x15 + beq ShadowUpdateEnd + +; *pShadow = INVALIDGCVALUE (0xcccccccd) + movz x17, #0xcccd + movk x17, #0xcccc, LSL #16 + str x17, [x12] +endif ShadowUpdateEnd -#endif - -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - ; Update the write watch table if necessary - ldr x12, wbs_sw_ww_table - cbz x12, CheckCardTable - add x12, x12, x14, LSR #0xC // SoftwareWriteWatch::AddressToTableByteIndexShift - ldrb w17, [x12] - cbnz x17, CheckCardTable - mov w17, 0xFF - strb w17, [x12] -#endif - -CheckCardTable - ; Branch to Exit if the reference is not in the Gen0 heap - ldr x12, wbs_ephemeral_low - ldr x17, wbs_ephemeral_high - cmp x15, x12 - ccmp x15, x17, #0x2, hs - bhs Exit - - ; Check if we need to update the card table - ldr x12, wbs_card_table - - ; x15 := pointer into card table - add x15, x12, x14, lsr #11 - - ldrb w12, [x15] - cmp x12, 0xFF - beq Exit - - ; Update the card table - mov x12, 0xFF - strb w12, [x15] - -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - ; Check if we need to update the card bundle table - ldr x12, wbs_card_bundle_table - - ; x15 := pointer into card bundle table - add x15, x12, x14, lsr #21 - - ldrb w12, [x15] - cmp x12, 0xFF - beq Exit - - mov x12, 0xFF - strb w12, [x15] -#endif - -Exit - ; Increment by 8 to implement JIT_ByRefWriteBarrier contract. - ; TODO: Consider duplicating the logic to get rid of this redundant 'add' - ; for JIT_WriteBarrier/JIT_CheckedWriteBarrier - add x14, x14, 8 - ret lr - WRITE_BARRIER_END JIT_WriteBarrier +endm + + + +WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB macro start exit +ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP +; Update the write watch table if necessary + ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_WriteWatchTable) +; SoftwareWriteWatch::AddressToTableByteIndexShift + add x12, x12, x14, lsr #0xc + ldrb w17, [x12] + cbnz x17, WriteWatchForGCHeapEnd + mov w17, #0xFF + strb w17, [x12] +WriteWatchForGCHeapEnd +endif +endm + + +WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB macro start exit +; Branch to Exit if the reference is not in the ephemeral generations. + ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_Lower) + cmp x15, x12 + blo exit +endm + + +WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB macro start exit +; Branch to Exit if the reference is not in the ephemeral generations. + ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_Lower) + ldr x17, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_Upper) + cmp x15, x12 + ccmp x15, x17, #0x2, hs + bhs exit +endm + + +WRITE_BARRIER_REGION_CHECK_STUB macro start exit +; Calculate region locations + ldr x17, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_RegionToGeneration) + ldr w12, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_RegionShr) + lsr x15, x15, x12 + add x15, x15, x17 ; x15 = (RHS >> wbs_region_shr) + wbs_region_to_generation_table + lsr x12, x14, x12 + add x12, x12, x17 ; x12 = (LHS >> wbs_region_shr) + wbs_region_to_generation_table + +; Check whether the region we're storing into is gen 0 - nothing to do in this case + ldrb w12, [x12] + cbz w12, exit + +; Check this is going from old to young + ldrb w15, [x15] + cmp w15, w12 + bhs exit +endm + + +WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB macro start exit +; Check if we need to update the card table + lsr w17, w14, 8 + and w17, w17, 7 + movz w15, 1 + lsl w17, w15, w17 ; w17 = 1 << (LHS >> 8 && 7) + ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_CardTable) + add x15, x12, x14, lsr #11 + ldrb w12, [x15] ; w12 = [(LHS >> 11) + g_card_table] + tst w12, w17 + bne exit + +; Atomically update the card table +; Requires LSE, but the code is only compiled for 8.0 +; stsetb w17, [x15] + word 0x383131FF +endm + + +WRITE_BARRIER_CHECK_CARD_TABLE_STUB macro start exit +; Check if we need to update the card table + ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_CardTable) + add x15, x12, x14, lsr #11 +; w12 = [(RHS >> 11) + g_card_table] + ldrb w12, [x15] + cmp x12, 0xFF + beq exit + +; Update the card table + mov x12, 0xFF + strb w12, [x15] +endm + + +WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB macro start exit +ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +; Check if we need to update the card bundle table + ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_CardBundleTable) + add x15, x12, x14, lsr #21 + ldrb w12, [x15] + cmp x12, 0xFF + beq exit + +; Update the card bundle + mov x12, 0xFF + strb w12, [x15] +endif +endm + + +WRITE_BARRIER_RETURN_STUB macro exit +exit +; Increment by 8 to implement JIT_ByRefWriteBarrier contract. +; TODO: Consider duplicating the logic to get rid of this redundant 'add' +; for JIT_WriteBarrier/JIT_CheckedWriteBarrier + add x14, x14, 8 + ret lr +endm -; ------------------------------------------------------------------ -; End of the writeable code region - LEAF_ENTRY JIT_PatchedCodeLast - ret lr - LEAF_END +;----------------------------------------------------------------------------- +; void JIT_WriteBarrier_PreGrow64(Object** dst, Object* src) +; +; Skipped functionality: +; Does not update the write watch table +; Does not check wbs_ephemeral_high +; No region checks +; +LEAF_ENTRY JIT_WriteBarrier_PreGrow64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_PreGrow64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_PreGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB Start_PreGrow64 Exit_PreGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_PreGrow64 Exit_PreGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_PreGrow64 Exit_PreGrow64 + WRITE_BARRIER_RETURN_STUB Exit_PreGrow64 +LEAF_END_MARKED JIT_WriteBarrier_PreGrow64, _TEXT -; Must be at very end of file - END + +;----------------------------------------------------------------------------- +; void JIT_WriteBarrier_PostGrow64(Object** dst, Object* src) +; +; Skipped functionality: +; Does not update the write watch table +; No region checks +; +LEAF_ENTRY JIT_WriteBarrier_PostGrow64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_PostGrow64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_PostGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_PostGrow64 Exit_PostGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_PostGrow64 Exit_PostGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_PostGrow64 Exit_PostGrow64 + WRITE_BARRIER_RETURN_STUB Exit_PostGrow64 +LEAF_END_MARKED JIT_WriteBarrier_PostGrow64, _TEXT + + +;----------------------------------------------------------------------------- +; void JIT_WriteBarrier_SVR64(Object** dst, Object* src) +; +; SVR GC has multiple heaps, so it cannot provide one single ephemeral region to bounds check +; against, so we just skip the bounds checking all together and do our card table update unconditionally. +; +; Skipped functionality: +; Does not update the write watch table +; Does not check wbs_ephemeral_high or wbs_ephemeral_low +; No region checks +; +LEAF_ENTRY JIT_WriteBarrier_SVR64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_SVR64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_SVR64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_SVR64 Exit_SVR64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_SVR64 Exit_SVR64 + WRITE_BARRIER_RETURN_STUB Exit_SVR64 +LEAF_END_MARKED JIT_WriteBarrier_SVR64, _TEXT + + +;----------------------------------------------------------------------------- +; void JIT_WriteBarrier_Byte_Region64(Object** dst, Object* src) +; +; Skipped functionality: +; Does not update the write watch table +; Bitwise updates for region checks +; +LEAF_ENTRY JIT_WriteBarrier_Byte_Region64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_Byte_Region64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_Byte_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Byte_Region64 Exit_Byte_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_Byte_Region64 Exit_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_Byte_Region64 Exit_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Byte_Region64 Exit_Byte_Region64 + WRITE_BARRIER_RETURN_STUB Exit_Byte_Region64 +LEAF_END_MARKED JIT_WriteBarrier_Byte_Region64, _TEXT + + +;----------------------------------------------------------------------------- +; void JIT_WriteBarrier_Bit_Region64(Object** dst, Object* src) +; +; Skipped functionality: +; Does not update the write watch table +; Does not call check card table stub +; +LEAF_ENTRY JIT_WriteBarrier_Bit_Region64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_Bit_Region64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_Bit_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Bit_Region64 Exit_Bit_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_Bit_Region64 Exit_Bit_Region64 + WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Start_Bit_Region64 Exit_Bit_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Bit_Region64 Exit_Bit_Region64 + WRITE_BARRIER_RETURN_STUB Exit_Bit_Region64 +LEAF_END_MARKED JIT_WriteBarrier_Bit_Region64, _TEXT + + +;----------------------------------------------------------------------------- +; void JIT_WriteBarrier_WriteWatch_PreGrow64(Object** dst, Object* src) +; +; Skipped functionality: +; Does not check wbs_ephemeral_high +; No region checks +; +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_PreGrow64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_PreGrow64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 + WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_PreGrow64 +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT + + +;----------------------------------------------------------------------------- +; void JIT_WriteBarrier_WriteWatch_PostGrow64(Object** dst, Object* src) +; +; Skipped functionality: +; No region checks +; +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_PostGrow64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_PostGrow64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PostGrow64 Exit_WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_PostGrow64 Exit_WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_PostGrow64 Exit_WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_PostGrow64 Exit_WriteWatch_PostGrow64 + WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_PostGrow64 +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT + + +;----------------------------------------------------------------------------- +; void JIT_WriteBarrier_WriteWatch_SVR64(Object** dst, Object* src) +; +; SVR GC has multiple heaps, so it cannot provide one single ephemeral region to bounds check +; against, so we just skip the bounds checking all together and do our card table update unconditionally. +; +; Skipped functionality: +; Does not check wbs_ephemeral_high or wbs_ephemeral_low +; No region checks +; +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_SVR64, _TEXT + WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_SVR64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_SVR64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_SVR64 Exit_WriteWatch_SVR64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_SVR64 Exit_WriteWatch_SVR64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_SVR64 Exit_WriteWatch_SVR64 + WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_SVR64 +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_SVR64, _TEXT + + +;----------------------------------------------------------------------------- +; void JIT_WriteBarrier_WriteWatch_Byte_Region64(Object** dst, Object* src) +; +; Skipped functionality: +; Bitwise updates for region checks +; +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT +JIT_WriteBarrier_WriteWatch_Byte_Region64Start + WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_Byte_Region64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_Byte_Region64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_Byte_Region64 +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT + + +;----------------------------------------------------------------------------- +; void JIT_WriteBarrier_WriteWatch_Bit_Region64(Object** dst, Object* src) +; +; Skipped functionality: +; Does not call check card table stub +; +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT +JIT_WriteBarrier_WriteWatch_Bit_Region64Start + WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_Bit_Region64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_Bit_Region64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_Bit_Region64 +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT From e026a06a230bd8838e023fc513c16a7a7aac1bc5 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Wed, 5 Mar 2025 18:07:56 +0000 Subject: [PATCH 30/42] remove wbs_begin --- src/coreclr/vm/arm64/patchedcode.S | 1 - src/coreclr/vm/arm64/patchedcode.asm | 43 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index 18bfa2cdda5876..6636c781f0679a 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -114,7 +114,6 @@ WRITE_BARRIER_END JIT_WriteBarrier // .balign 64 // Align to power of two at least as big as patchable literal pool so that it fits optimally in cache line WRITE_BARRIER_ENTRY JIT_WriteBarrier_Table -LOCAL_LABEL(wbs_begin): PATCH_LABEL JIT_WriteBarrier_Patch_Label_CardTable .quad 0 PATCH_LABEL JIT_WriteBarrier_Patch_Label_CardBundleTable diff --git a/src/coreclr/vm/arm64/patchedcode.asm b/src/coreclr/vm/arm64/patchedcode.asm index 6486eef99b336f..9b551d4ba8e8d7 100644 --- a/src/coreclr/vm/arm64/patchedcode.asm +++ b/src/coreclr/vm/arm64/patchedcode.asm @@ -41,31 +41,30 @@ ; Begin patchable literal pool ALIGN 64 ; Align to power of two at least as big as patchable literal pool so that it fits optimally in cache line WRITE_BARRIER_ENTRY JIT_WriteBarrier_Table -wbs_begin -PATCH_LABEL JIT_WriteBarrier_Patch_Label_CardTable - DCQ 0 -PATCH_LABEL JIT_WriteBarrier_Patch_Label_CardBundleTable - DCQ 0 -PATCH_LABEL JIT_WriteBarrier_Patch_Label_WriteWatchTable - DCQ 0 -PATCH_LABEL JIT_WriteBarrier_Patch_Label_Lower - DCQ 0 -PATCH_LABEL JIT_WriteBarrier_Patch_Label_Upper - DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_CardTable + DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_CardBundleTable + DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_WriteWatchTable + DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_Lower + DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_Upper + DCQ 0 wbs_lowest_address -PATCH_LABEL JIT_WriteBarrier_Patch_Label_LowestAddress - DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_LowestAddress + DCQ 0 wbs_highest_address -PATCH_LABEL JIT_WriteBarrier_Patch_Label_HighestAddress - DCQ 0 -PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionToGeneration - DCQ 0 -PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionShr - DCW 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_HighestAddress + DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionToGeneration + DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionShr + DCW 0 ifdef WRITE_BARRIER_CHECK -PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadow - DCQ 0 -PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadowEnd + PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadow + DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadowEnd DCQ 0 endif WRITE_BARRIER_END JIT_WriteBarrier_Table From 678f8e162f7dff6a03cbd5da05a8ce370d005331 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Fri, 7 Mar 2025 10:18:22 +0000 Subject: [PATCH 31/42] fix ifdefs for windows --- src/coreclr/vm/arm64/patchedcode.asm | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/coreclr/vm/arm64/patchedcode.asm b/src/coreclr/vm/arm64/patchedcode.asm index 9b551d4ba8e8d7..6690b2b2b378ca 100644 --- a/src/coreclr/vm/arm64/patchedcode.asm +++ b/src/coreclr/vm/arm64/patchedcode.asm @@ -61,12 +61,12 @@ wbs_highest_address DCQ 0 PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionShr DCW 0 -ifdef WRITE_BARRIER_CHECK +#ifdef WRITE_BARRIER_CHECK PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadow DCQ 0 PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadowEnd DCQ 0 -endif +#endif WRITE_BARRIER_END JIT_WriteBarrier_Table ;----------------------------------------------------------------------------- @@ -169,7 +169,7 @@ endm WRITE_BARRIER_SHADOW_UPDATE_STUB macro start -ifdef WRITE_BARRIER_CHECK +#ifdef WRITE_BARRIER_CHECK ; Update GC Shadow Heap ; Do not perform the work if g_GCShadow is 0 @@ -203,14 +203,14 @@ ifdef WRITE_BARRIER_CHECK movz x17, #0xcccd movk x17, #0xcccc, LSL #16 str x17, [x12] -endif +#endif ShadowUpdateEnd endm WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB macro start exit -ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP +#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP ; Update the write watch table if necessary ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_WriteWatchTable) ; SoftwareWriteWatch::AddressToTableByteIndexShift @@ -220,7 +220,7 @@ ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP mov w17, #0xFF strb w17, [x12] WriteWatchForGCHeapEnd -endif +#endif endm @@ -297,7 +297,7 @@ endm WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB macro start exit -ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES +#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES ; Check if we need to update the card bundle table ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_CardBundleTable) add x15, x12, x14, lsr #21 @@ -308,7 +308,7 @@ ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES ; Update the card bundle mov x12, 0xFF strb w12, [x15] -endif +#endif endm From 5a156d34a505bf4a9520989a53401796f77fc136 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Fri, 7 Mar 2025 11:45:35 +0000 Subject: [PATCH 32/42] Remove _TEXT --- src/coreclr/vm/arm64/patchedcode.asm | 40 ++++++++++++++-------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/coreclr/vm/arm64/patchedcode.asm b/src/coreclr/vm/arm64/patchedcode.asm index 6690b2b2b378ca..84c76f90d426e7 100644 --- a/src/coreclr/vm/arm64/patchedcode.asm +++ b/src/coreclr/vm/arm64/patchedcode.asm @@ -329,14 +329,14 @@ endm ; Does not check wbs_ephemeral_high ; No region checks ; -LEAF_ENTRY JIT_WriteBarrier_PreGrow64, _TEXT +LEAF_ENTRY JIT_WriteBarrier_PreGrow64 WRITE_BARRIER_ENTRY_STUB Start_PreGrow64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_PreGrow64 WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB Start_PreGrow64 Exit_PreGrow64 WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_PreGrow64 Exit_PreGrow64 WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_PreGrow64 Exit_PreGrow64 WRITE_BARRIER_RETURN_STUB Exit_PreGrow64 -LEAF_END_MARKED JIT_WriteBarrier_PreGrow64, _TEXT +LEAF_END_MARKED JIT_WriteBarrier_PreGrow64 ;----------------------------------------------------------------------------- @@ -346,14 +346,14 @@ LEAF_END_MARKED JIT_WriteBarrier_PreGrow64, _TEXT ; Does not update the write watch table ; No region checks ; -LEAF_ENTRY JIT_WriteBarrier_PostGrow64, _TEXT +LEAF_ENTRY JIT_WriteBarrier_PostGrow64 WRITE_BARRIER_ENTRY_STUB Start_PostGrow64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_PostGrow64 WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_PostGrow64 Exit_PostGrow64 WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_PostGrow64 Exit_PostGrow64 WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_PostGrow64 Exit_PostGrow64 WRITE_BARRIER_RETURN_STUB Exit_PostGrow64 -LEAF_END_MARKED JIT_WriteBarrier_PostGrow64, _TEXT +LEAF_END_MARKED JIT_WriteBarrier_PostGrow64 ;----------------------------------------------------------------------------- @@ -367,13 +367,13 @@ LEAF_END_MARKED JIT_WriteBarrier_PostGrow64, _TEXT ; Does not check wbs_ephemeral_high or wbs_ephemeral_low ; No region checks ; -LEAF_ENTRY JIT_WriteBarrier_SVR64, _TEXT +LEAF_ENTRY JIT_WriteBarrier_SVR64 WRITE_BARRIER_ENTRY_STUB Start_SVR64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_SVR64 WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_SVR64 Exit_SVR64 WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_SVR64 Exit_SVR64 WRITE_BARRIER_RETURN_STUB Exit_SVR64 -LEAF_END_MARKED JIT_WriteBarrier_SVR64, _TEXT +LEAF_END_MARKED JIT_WriteBarrier_SVR64 ;----------------------------------------------------------------------------- @@ -383,7 +383,7 @@ LEAF_END_MARKED JIT_WriteBarrier_SVR64, _TEXT ; Does not update the write watch table ; Bitwise updates for region checks ; -LEAF_ENTRY JIT_WriteBarrier_Byte_Region64, _TEXT +LEAF_ENTRY JIT_WriteBarrier_Byte_Region64 WRITE_BARRIER_ENTRY_STUB Start_Byte_Region64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_Byte_Region64 WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Byte_Region64 Exit_Byte_Region64 @@ -391,7 +391,7 @@ LEAF_ENTRY JIT_WriteBarrier_Byte_Region64, _TEXT WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_Byte_Region64 Exit_Byte_Region64 WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Byte_Region64 Exit_Byte_Region64 WRITE_BARRIER_RETURN_STUB Exit_Byte_Region64 -LEAF_END_MARKED JIT_WriteBarrier_Byte_Region64, _TEXT +LEAF_END_MARKED JIT_WriteBarrier_Byte_Region64 ;----------------------------------------------------------------------------- @@ -401,7 +401,7 @@ LEAF_END_MARKED JIT_WriteBarrier_Byte_Region64, _TEXT ; Does not update the write watch table ; Does not call check card table stub ; -LEAF_ENTRY JIT_WriteBarrier_Bit_Region64, _TEXT +LEAF_ENTRY JIT_WriteBarrier_Bit_Region64 WRITE_BARRIER_ENTRY_STUB Start_Bit_Region64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_Bit_Region64 WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Bit_Region64 Exit_Bit_Region64 @@ -409,7 +409,7 @@ LEAF_ENTRY JIT_WriteBarrier_Bit_Region64, _TEXT WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Start_Bit_Region64 Exit_Bit_Region64 WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Bit_Region64 Exit_Bit_Region64 WRITE_BARRIER_RETURN_STUB Exit_Bit_Region64 -LEAF_END_MARKED JIT_WriteBarrier_Bit_Region64, _TEXT +LEAF_END_MARKED JIT_WriteBarrier_Bit_Region64 ;----------------------------------------------------------------------------- @@ -419,7 +419,7 @@ LEAF_END_MARKED JIT_WriteBarrier_Bit_Region64, _TEXT ; Does not check wbs_ephemeral_high ; No region checks ; -LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PreGrow64 WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_PreGrow64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_PreGrow64 WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 @@ -427,7 +427,7 @@ LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_PreGrow64 -LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PreGrow64 ;----------------------------------------------------------------------------- @@ -436,7 +436,7 @@ LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT ; Skipped functionality: ; No region checks ; -LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PostGrow64 WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_PostGrow64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_PostGrow64 WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PostGrow64 Exit_WriteWatch_PostGrow64 @@ -444,7 +444,7 @@ LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_PostGrow64 Exit_WriteWatch_PostGrow64 WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_PostGrow64 Exit_WriteWatch_PostGrow64 WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_PostGrow64 -LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PostGrow64 ;----------------------------------------------------------------------------- @@ -457,14 +457,14 @@ LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT ; Does not check wbs_ephemeral_high or wbs_ephemeral_low ; No region checks ; -LEAF_ENTRY JIT_WriteBarrier_WriteWatch_SVR64, _TEXT +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_SVR64 WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_SVR64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_SVR64 WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_SVR64 Exit_WriteWatch_SVR64 WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_SVR64 Exit_WriteWatch_SVR64 WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_SVR64 Exit_WriteWatch_SVR64 WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_SVR64 -LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_SVR64, _TEXT +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_SVR64 ;----------------------------------------------------------------------------- @@ -473,7 +473,7 @@ LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_SVR64, _TEXT ; Skipped functionality: ; Bitwise updates for region checks ; -LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Byte_Region64 JIT_WriteBarrier_WriteWatch_Byte_Region64Start WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_Byte_Region64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_Byte_Region64 @@ -483,7 +483,7 @@ JIT_WriteBarrier_WriteWatch_Byte_Region64Start WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_Byte_Region64 -LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Byte_Region64 ;----------------------------------------------------------------------------- @@ -492,7 +492,7 @@ LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT ; Skipped functionality: ; Does not call check card table stub ; -LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT +LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Bit_Region64 JIT_WriteBarrier_WriteWatch_Bit_Region64Start WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_Bit_Region64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_Bit_Region64 @@ -502,4 +502,4 @@ JIT_WriteBarrier_WriteWatch_Bit_Region64Start WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_Bit_Region64 -LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT +LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Bit_Region64 From b318113810f60e7561145bfb2407afb71a2dbcbc Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Fri, 7 Mar 2025 17:45:27 +0000 Subject: [PATCH 33/42] Fix macro arg passing on OSX --- src/coreclr/vm/arm64/patchedcode.S | 90 +++++++++++++++--------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index 6636c781f0679a..c4d36ab160eab5 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -156,7 +156,7 @@ LEAF_END JIT_PatchedCodeLast, _TEXT // // -#define WRITE_BARRIER_CONSTANT_OFFSET(start, constlabel) (constlabel - C_FUNC(JIT_WriteBarrier) + LOCAL_LABEL(start)) +#define WRITE_BARRIER_CONSTANT_OFFSET(start, constlabel) (C_FUNC(constlabel) - C_FUNC(JIT_WriteBarrier) + LOCAL_LABEL(start)) .macro WRITE_BARRIER_ENTRY_STUB start @@ -205,7 +205,7 @@ LOCAL_LABEL(ShadowUpdateEnd\@): .endm -.macro WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB start exit +.macro WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB start, exit #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP // Update the write watch table if necessary ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_WriteWatchTable) @@ -219,7 +219,7 @@ LOCAL_LABEL(WriteWatchForGCHeapEnd\@): .endm -.macro WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB start exit +.macro WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB start, exit // Branch to Exit if the reference is not in the ephemeral generations. ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_Lower) cmp x15, x12 @@ -227,7 +227,7 @@ LOCAL_LABEL(WriteWatchForGCHeapEnd\@): .endm -.macro WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB start exit +.macro WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB start, exit // Branch to Exit if the reference is not in the ephemeral generations. ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_Lower) ldr x17, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_Upper) @@ -237,7 +237,7 @@ LOCAL_LABEL(WriteWatchForGCHeapEnd\@): .endm -.macro WRITE_BARRIER_REGION_CHECK_STUB start exit +.macro WRITE_BARRIER_REGION_CHECK_STUB start, exit // Calculate region locations ldr x17, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_RegionToGeneration) ldr w12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_RegionShr) @@ -257,7 +257,7 @@ LOCAL_LABEL(WriteWatchForGCHeapEnd\@): .endm -.macro WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB start exit +.macro WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB start, exit // Check if we need to update the card table lsr w17, w14, 8 and w17, w17, 7 @@ -275,7 +275,7 @@ LOCAL_LABEL(WriteWatchForGCHeapEnd\@): .endm -.macro WRITE_BARRIER_CHECK_CARD_TABLE_STUB start exit +.macro WRITE_BARRIER_CHECK_CARD_TABLE_STUB start, exit // Check if we need to update the card table ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_CardTable) add x15, x12, x14, lsr #11 @@ -289,7 +289,7 @@ LOCAL_LABEL(WriteWatchForGCHeapEnd\@): .endm -.macro WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB start exit +.macro WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB start, exit #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES // Check if we need to update the card bundle table ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_CardBundleTable) @@ -325,9 +325,9 @@ LOCAL_LABEL(\exit): LEAF_ENTRY JIT_WriteBarrier_PreGrow64, _TEXT WRITE_BARRIER_ENTRY_STUB Start_PreGrow64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_PreGrow64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB Start_PreGrow64 Exit_PreGrow64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_PreGrow64 Exit_PreGrow64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_PreGrow64 Exit_PreGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB Start_PreGrow64, Exit_PreGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_PreGrow64, Exit_PreGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_PreGrow64, Exit_PreGrow64 WRITE_BARRIER_RETURN_STUB Exit_PreGrow64 LEAF_END_MARKED JIT_WriteBarrier_PreGrow64, _TEXT @@ -342,9 +342,9 @@ LEAF_END_MARKED JIT_WriteBarrier_PreGrow64, _TEXT LEAF_ENTRY JIT_WriteBarrier_PostGrow64, _TEXT WRITE_BARRIER_ENTRY_STUB Start_PostGrow64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_PostGrow64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_PostGrow64 Exit_PostGrow64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_PostGrow64 Exit_PostGrow64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_PostGrow64 Exit_PostGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_PostGrow64, Exit__PostGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_PostGrow64, Exit__PostGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_PostGrow64, Exit__PostGrow64 WRITE_BARRIER_RETURN_STUB Exit_PostGrow64 LEAF_END_MARKED JIT_WriteBarrier_PostGrow64, _TEXT @@ -363,8 +363,8 @@ LEAF_END_MARKED JIT_WriteBarrier_PostGrow64, _TEXT LEAF_ENTRY JIT_WriteBarrier_SVR64, _TEXT WRITE_BARRIER_ENTRY_STUB Start_SVR64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_SVR64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_SVR64 Exit_SVR64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_SVR64 Exit_SVR64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_SVR64, Exit__SVR64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_SVR64, Exit__SVR64 WRITE_BARRIER_RETURN_STUB Exit_SVR64 LEAF_END_MARKED JIT_WriteBarrier_SVR64, _TEXT @@ -379,10 +379,10 @@ LEAF_END_MARKED JIT_WriteBarrier_SVR64, _TEXT LEAF_ENTRY JIT_WriteBarrier_Byte_Region64, _TEXT WRITE_BARRIER_ENTRY_STUB Start_Byte_Region64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_Byte_Region64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Byte_Region64 Exit_Byte_Region64 - WRITE_BARRIER_REGION_CHECK_STUB Start_Byte_Region64 Exit_Byte_Region64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_Byte_Region64 Exit_Byte_Region64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Byte_Region64 Exit_Byte_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Byte_Region64, Exit__Byte_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_Byte_Region64, Exit__Byte_Region64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_Byte_Region64, Exit__Byte_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Byte_Region64, Exit__Byte_Region64 WRITE_BARRIER_RETURN_STUB Exit_Byte_Region64 LEAF_END_MARKED JIT_WriteBarrier_Byte_Region64, _TEXT @@ -397,10 +397,10 @@ LEAF_END_MARKED JIT_WriteBarrier_Byte_Region64, _TEXT LEAF_ENTRY JIT_WriteBarrier_Bit_Region64, _TEXT WRITE_BARRIER_ENTRY_STUB Start_Bit_Region64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_Bit_Region64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Bit_Region64 Exit_Bit_Region64 - WRITE_BARRIER_REGION_CHECK_STUB Start_Bit_Region64 Exit_Bit_Region64 - WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Start_Bit_Region64 Exit_Bit_Region64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Bit_Region64 Exit_Bit_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Bit_Region64, Exit__Bit_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_Bit_Region64, Exit__Bit_Region64 + WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Start_Bit_Region64, Exit__Bit_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Bit_Region64, Exit__Bit_Region64 WRITE_BARRIER_RETURN_STUB Exit_Bit_Region64 LEAF_END_MARKED JIT_WriteBarrier_Bit_Region64, _TEXT @@ -415,10 +415,10 @@ LEAF_END_MARKED JIT_WriteBarrier_Bit_Region64, _TEXT LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_PreGrow64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_PreGrow64 - WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PreGrow64, Exit__WriteWatch_PreGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB Start_WriteWatch_PreGrow64, Exit__WriteWatch_PreGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_PreGrow64, Exit__WriteWatch_PreGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_PreGrow64, Exit__WriteWatch_PreGrow64 WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_PreGrow64 LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT @@ -432,10 +432,10 @@ LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_PostGrow64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_PostGrow64 - WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PostGrow64 Exit_WriteWatch_PostGrow64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_PostGrow64 Exit_WriteWatch_PostGrow64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_PostGrow64 Exit_WriteWatch_PostGrow64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_PostGrow64 Exit_WriteWatch_PostGrow64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PostGrow64, Exit__WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_PostGrow64, Exit__WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_PostGrow64, Exit__WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_PostGrow64, Exit__WriteWatch_PostGrow64 WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_PostGrow64 LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT @@ -453,9 +453,9 @@ LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT LEAF_ENTRY JIT_WriteBarrier_WriteWatch_SVR64, _TEXT WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_SVR64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_SVR64 - WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_SVR64 Exit_WriteWatch_SVR64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_SVR64 Exit_WriteWatch_SVR64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_SVR64 Exit_WriteWatch_SVR64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_SVR64, Exit__WriteWatch_SVR64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_SVR64, Exit__WriteWatch_SVR64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_SVR64, Exit__WriteWatch_SVR64 WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_SVR64 LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_SVR64, _TEXT @@ -470,11 +470,11 @@ LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT LOCAL_LABEL(JIT_WriteBarrier_WriteWatch_Byte_Region64Start): WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_Byte_Region64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_Byte_Region64 - WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 - WRITE_BARRIER_REGION_CHECK_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_Byte_Region64, Exit__WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_Byte_Region64, Exit__WriteWatch_Byte_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_WriteWatch_Byte_Region64, Exit__WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_Byte_Region64, Exit__WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_Byte_Region64, Exit__WriteWatch_Byte_Region64 WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_Byte_Region64 LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT @@ -489,11 +489,11 @@ LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT LOCAL_LABEL(JIT_WriteBarrier_WriteWatch_Bit_Region64Start): WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_Bit_Region64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_Bit_Region64 - WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 - WRITE_BARRIER_REGION_CHECK_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 - WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_Bit_Region64, Exit__WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_Bit_Region64, Exit__WriteWatch_Bit_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_WriteWatch_Bit_Region64, Exit__WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Start_WriteWatch_Bit_Region64, Exit__WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_Bit_Region64, Exit__WriteWatch_Bit_Region64 WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_Bit_Region64 LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT From 9312f3c4ab3d237018bd208b6ae52e320e666b90 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Wed, 12 Mar 2025 12:11:05 +0000 Subject: [PATCH 34/42] Fix patching arithmetic to work on MacOS --- src/coreclr/pal/inc/unixasmmacrosarm64.inc | 7 ++ src/coreclr/vm/arm64/patchedcode.S | 103 ++++++++++---------- src/coreclr/vm/arm64/patchedcodeconstants.h | 29 ++++++ src/coreclr/vm/writebarriermanager.cpp | 76 ++++++++------- src/coreclr/vm/writebarriermanager.h | 6 +- 5 files changed, 130 insertions(+), 91 deletions(-) create mode 100644 src/coreclr/vm/arm64/patchedcodeconstants.h diff --git a/src/coreclr/pal/inc/unixasmmacrosarm64.inc b/src/coreclr/pal/inc/unixasmmacrosarm64.inc index 9e86779d4511bc..69fe4e8164717d 100644 --- a/src/coreclr/pal/inc/unixasmmacrosarm64.inc +++ b/src/coreclr/pal/inc/unixasmmacrosarm64.inc @@ -26,6 +26,13 @@ C_FUNC(\Name): .endm +// On MacOS, local labels cannot be used in arithmetic expressions. +#if defined(__APPLE__) +#define FIXUP_LABEL(name) name +#else +#define FIXUP_LABEL(name) .L##name +#endif + .macro LEAF_ENTRY Name, Section .global C_FUNC(\Name) #if defined(__APPLE__) diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index c4d36ab160eab5..71e4459b6bbd0e 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -3,6 +3,7 @@ #include "asmconstants.h" #include "unixasmmacros.inc" +#include "patchedcodeconstants.h" //----------------------------------------------------------------------------- // The following Macros help in WRITE_BARRIER Implementations @@ -104,7 +105,7 @@ WRITE_BARRIER_END JIT_CheckedWriteBarrier // WRITE_BARRIER_ENTRY JIT_WriteBarrier // This must be greater than the largest JIT_WriteBarrier_ function. - .space (232*4), 0 + .space JIT_WriteBarrier_Size, 0 WRITE_BARRIER_END JIT_WriteBarrier //----------------------------------------------------------------------------- @@ -133,7 +134,7 @@ PATCH_LABEL JIT_WriteBarrier_Patch_Label_HighestAddress PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionToGeneration .quad 0 PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionShr - .word 0 + .quad 0 #ifdef WRITE_BARRIER_CHECK PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadow .quad 0 @@ -142,7 +143,6 @@ PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadowEnd #endif WRITE_BARRIER_END JIT_WriteBarrier_Table - // ------------------------------------------------------------------ // End of the writeable code region LEAF_ENTRY JIT_PatchedCodeLast, _TEXT @@ -156,11 +156,8 @@ LEAF_END JIT_PatchedCodeLast, _TEXT // // -#define WRITE_BARRIER_CONSTANT_OFFSET(start, constlabel) (C_FUNC(constlabel) - C_FUNC(JIT_WriteBarrier) + LOCAL_LABEL(start)) - - .macro WRITE_BARRIER_ENTRY_STUB start -LOCAL_LABEL(\start): +FIXUP_LABEL(\start): stlr x15, [x14] .endm @@ -170,17 +167,17 @@ LOCAL_LABEL(\start): // Update GC Shadow Heap // Do not perform the work if g_GCShadow is 0 - ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_GCShadow) + ldr x12, JIT_WriteBarrier_Offset_GCShadow + FIXUP_LABEL(\start) cbz x12, LOCAL_LABEL(ShadowUpdateEnd\@) // Compute address of shadow heap location: // pShadow = g_GCShadow + (x14 - g_lowest_address) - ldr x17, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_LowestAddress) + ldr x17, JIT_WriteBarrier_Offset_LowestAddress + FIXUP_LABEL(\start) sub x17, x14, x17 add x12, x17, x12 // if (pShadow >= g_GCShadowEnd) goto end - ldr x17, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_GCShadowEnd) + ldr x17, JIT_WriteBarrier_Offset_GCShadowEnd + FIXUP_LABEL(\start) cmp x12, x17 bhs LOCAL_LABEL(ShadowUpdateEnd\@) @@ -208,7 +205,7 @@ LOCAL_LABEL(ShadowUpdateEnd\@): .macro WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB start, exit #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP // Update the write watch table if necessary - ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_WriteWatchTable) + ldr x12, JIT_WriteBarrier_Offset_WriteWatchTable + FIXUP_LABEL(\start) add x12, x12, x14, lsr #0xc // SoftwareWriteWatch::AddressToTableByteIndexShift ldrb w17, [x12] cbnz x17, LOCAL_LABEL(WriteWatchForGCHeapEnd\@) @@ -221,7 +218,7 @@ LOCAL_LABEL(WriteWatchForGCHeapEnd\@): .macro WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB start, exit // Branch to Exit if the reference is not in the ephemeral generations. - ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_Lower) + ldr x12, JIT_WriteBarrier_Offset_Lower + FIXUP_LABEL(\start) cmp x15, x12 blo LOCAL_LABEL(\exit) .endm @@ -229,8 +226,8 @@ LOCAL_LABEL(WriteWatchForGCHeapEnd\@): .macro WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB start, exit // Branch to Exit if the reference is not in the ephemeral generations. - ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_Lower) - ldr x17, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_Upper) + ldr x12, JIT_WriteBarrier_Offset_Lower + FIXUP_LABEL(\start) + ldr x17, JIT_WriteBarrier_Offset_Upper + FIXUP_LABEL(\start) cmp x15, x12 ccmp x15, x17, #0x2, hs bhs LOCAL_LABEL(\exit) @@ -239,8 +236,8 @@ LOCAL_LABEL(WriteWatchForGCHeapEnd\@): .macro WRITE_BARRIER_REGION_CHECK_STUB start, exit // Calculate region locations - ldr x17, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_RegionToGeneration) - ldr w12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_RegionShr) + ldr x17, JIT_WriteBarrier_Offset_RegionToGeneration + FIXUP_LABEL(\start) + ldr w12, JIT_WriteBarrier_Offset_RegionShr + FIXUP_LABEL(\start) lsr x15, x15, x12 add x15, x15, x17 // x15 = (RHS >> wbs_region_shr) + wbs_region_to_generation_table lsr x12, x14, x12 @@ -263,7 +260,7 @@ LOCAL_LABEL(WriteWatchForGCHeapEnd\@): and w17, w17, 7 movz w15, 1 lsl w17, w15, w17 // w17 = 1 << (LHS >> 8 && 7) - ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_CardTable) + ldr x12, JIT_WriteBarrier_Offset_CardTable + FIXUP_LABEL(\start) add x15, x12, x14, lsr #11 ldrb w12, [x15] // w12 = [(LHS >> 11) + g_card_table] tst w12, w17 @@ -277,7 +274,7 @@ LOCAL_LABEL(WriteWatchForGCHeapEnd\@): .macro WRITE_BARRIER_CHECK_CARD_TABLE_STUB start, exit // Check if we need to update the card table - ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_CardTable) + ldr x12, JIT_WriteBarrier_Offset_CardTable + FIXUP_LABEL(\start) add x15, x12, x14, lsr #11 ldrb w12, [x15] // w12 = [(RHS >> 11) + g_card_table] cmp x12, 0xFF @@ -292,7 +289,7 @@ LOCAL_LABEL(WriteWatchForGCHeapEnd\@): .macro WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB start, exit #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES // Check if we need to update the card bundle table - ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(\start, JIT_WriteBarrier_Patch_Label_CardBundleTable) + ldr x12, JIT_WriteBarrier_Offset_CardBundleTable + FIXUP_LABEL(\start) add x15, x12, x14, lsr #21 ldrb w12, [x15] cmp x12, 0xFF @@ -342,9 +339,9 @@ LEAF_END_MARKED JIT_WriteBarrier_PreGrow64, _TEXT LEAF_ENTRY JIT_WriteBarrier_PostGrow64, _TEXT WRITE_BARRIER_ENTRY_STUB Start_PostGrow64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_PostGrow64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_PostGrow64, Exit__PostGrow64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_PostGrow64, Exit__PostGrow64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_PostGrow64, Exit__PostGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_PostGrow64, Exit_PostGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_PostGrow64, Exit_PostGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_PostGrow64, Exit_PostGrow64 WRITE_BARRIER_RETURN_STUB Exit_PostGrow64 LEAF_END_MARKED JIT_WriteBarrier_PostGrow64, _TEXT @@ -363,8 +360,8 @@ LEAF_END_MARKED JIT_WriteBarrier_PostGrow64, _TEXT LEAF_ENTRY JIT_WriteBarrier_SVR64, _TEXT WRITE_BARRIER_ENTRY_STUB Start_SVR64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_SVR64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_SVR64, Exit__SVR64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_SVR64, Exit__SVR64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_SVR64, Exit_SVR64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_SVR64, Exit_SVR64 WRITE_BARRIER_RETURN_STUB Exit_SVR64 LEAF_END_MARKED JIT_WriteBarrier_SVR64, _TEXT @@ -379,10 +376,10 @@ LEAF_END_MARKED JIT_WriteBarrier_SVR64, _TEXT LEAF_ENTRY JIT_WriteBarrier_Byte_Region64, _TEXT WRITE_BARRIER_ENTRY_STUB Start_Byte_Region64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_Byte_Region64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Byte_Region64, Exit__Byte_Region64 - WRITE_BARRIER_REGION_CHECK_STUB Start_Byte_Region64, Exit__Byte_Region64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_Byte_Region64, Exit__Byte_Region64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Byte_Region64, Exit__Byte_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Byte_Region64, Exit_Byte_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_Byte_Region64, Exit_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_Byte_Region64, Exit_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Byte_Region64, Exit_Byte_Region64 WRITE_BARRIER_RETURN_STUB Exit_Byte_Region64 LEAF_END_MARKED JIT_WriteBarrier_Byte_Region64, _TEXT @@ -397,10 +394,10 @@ LEAF_END_MARKED JIT_WriteBarrier_Byte_Region64, _TEXT LEAF_ENTRY JIT_WriteBarrier_Bit_Region64, _TEXT WRITE_BARRIER_ENTRY_STUB Start_Bit_Region64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_Bit_Region64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Bit_Region64, Exit__Bit_Region64 - WRITE_BARRIER_REGION_CHECK_STUB Start_Bit_Region64, Exit__Bit_Region64 - WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Start_Bit_Region64, Exit__Bit_Region64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Bit_Region64, Exit__Bit_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Bit_Region64, Exit_Bit_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_Bit_Region64, Exit_Bit_Region64 + WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Start_Bit_Region64, Exit_Bit_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Bit_Region64, Exit_Bit_Region64 WRITE_BARRIER_RETURN_STUB Exit_Bit_Region64 LEAF_END_MARKED JIT_WriteBarrier_Bit_Region64, _TEXT @@ -415,10 +412,10 @@ LEAF_END_MARKED JIT_WriteBarrier_Bit_Region64, _TEXT LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_PreGrow64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_PreGrow64 - WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PreGrow64, Exit__WriteWatch_PreGrow64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB Start_WriteWatch_PreGrow64, Exit__WriteWatch_PreGrow64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_PreGrow64, Exit__WriteWatch_PreGrow64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_PreGrow64, Exit__WriteWatch_PreGrow64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PreGrow64, Exit_WriteWatch_PreGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB Start_WriteWatch_PreGrow64, Exit_WriteWatch_PreGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_PreGrow64, Exit_WriteWatch_PreGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_PreGrow64, Exit_WriteWatch_PreGrow64 WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_PreGrow64 LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT @@ -432,10 +429,10 @@ LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PreGrow64, _TEXT LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_PostGrow64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_PostGrow64 - WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PostGrow64, Exit__WriteWatch_PostGrow64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_PostGrow64, Exit__WriteWatch_PostGrow64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_PostGrow64, Exit__WriteWatch_PostGrow64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_PostGrow64, Exit__WriteWatch_PostGrow64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PostGrow64, Exit_WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_PostGrow64, Exit_WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_PostGrow64, Exit_WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_PostGrow64, Exit_WriteWatch_PostGrow64 WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_PostGrow64 LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT @@ -453,9 +450,9 @@ LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PostGrow64, _TEXT LEAF_ENTRY JIT_WriteBarrier_WriteWatch_SVR64, _TEXT WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_SVR64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_SVR64 - WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_SVR64, Exit__WriteWatch_SVR64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_SVR64, Exit__WriteWatch_SVR64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_SVR64, Exit__WriteWatch_SVR64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_SVR64, Exit_WriteWatch_SVR64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_SVR64, Exit_WriteWatch_SVR64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_SVR64, Exit_WriteWatch_SVR64 WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_SVR64 LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_SVR64, _TEXT @@ -470,11 +467,11 @@ LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT LOCAL_LABEL(JIT_WriteBarrier_WriteWatch_Byte_Region64Start): WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_Byte_Region64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_Byte_Region64 - WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_Byte_Region64, Exit__WriteWatch_Byte_Region64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_Byte_Region64, Exit__WriteWatch_Byte_Region64 - WRITE_BARRIER_REGION_CHECK_STUB Start_WriteWatch_Byte_Region64, Exit__WriteWatch_Byte_Region64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_Byte_Region64, Exit__WriteWatch_Byte_Region64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_Byte_Region64, Exit__WriteWatch_Byte_Region64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_Byte_Region64, Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_Byte_Region64, Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_WriteWatch_Byte_Region64, Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_Byte_Region64, Exit_WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_Byte_Region64, Exit_WriteWatch_Byte_Region64 WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_Byte_Region64 LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT @@ -489,11 +486,11 @@ LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT LOCAL_LABEL(JIT_WriteBarrier_WriteWatch_Bit_Region64Start): WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_Bit_Region64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_Bit_Region64 - WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_Bit_Region64, Exit__WriteWatch_Bit_Region64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_Bit_Region64, Exit__WriteWatch_Bit_Region64 - WRITE_BARRIER_REGION_CHECK_STUB Start_WriteWatch_Bit_Region64, Exit__WriteWatch_Bit_Region64 - WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Start_WriteWatch_Bit_Region64, Exit__WriteWatch_Bit_Region64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_Bit_Region64, Exit__WriteWatch_Bit_Region64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_Bit_Region64, Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_Bit_Region64, Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Start_WriteWatch_Bit_Region64, Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Start_WriteWatch_Bit_Region64, Exit_WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_Bit_Region64, Exit_WriteWatch_Bit_Region64 WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_Bit_Region64 LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT diff --git a/src/coreclr/vm/arm64/patchedcodeconstants.h b/src/coreclr/vm/arm64/patchedcodeconstants.h new file mode 100644 index 00000000000000..5ad9dcf1ee4113 --- /dev/null +++ b/src/coreclr/vm/arm64/patchedcodeconstants.h @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// =========================================================================== +// File: patchedcodeconstants.h +// =========================================================================== + +#ifndef PATCHEDCODECONSTANTS_H +#define PATCHEDCODECONSTANTS_H + +// These are fixed constants becuase MacOS doesn't allow label arithmetic in +// LDR instructions. Asserts in writebarriermanager CALC_TABLE_LOCATION ensure +// the values are correct. + +#define JIT_WriteBarrier_Size 0x3a0 +#define JIT_WriteBarrier_Table_Offset (0x2c + JIT_WriteBarrier_Size) +#define JIT_WriteBarrier_Offset_CardTable (0x0 + JIT_WriteBarrier_Table_Offset) +#define JIT_WriteBarrier_Offset_CardBundleTable (0x8 + JIT_WriteBarrier_Table_Offset) +#define JIT_WriteBarrier_Offset_WriteWatchTable (0x10 + JIT_WriteBarrier_Table_Offset) +#define JIT_WriteBarrier_Offset_Lower (0x18 + JIT_WriteBarrier_Table_Offset) +#define JIT_WriteBarrier_Offset_Upper (0x20 + JIT_WriteBarrier_Table_Offset) +#define JIT_WriteBarrier_Offset_LowestAddress (0x28 + JIT_WriteBarrier_Table_Offset) +#define JIT_WriteBarrier_Offset_HighestAddress (0x30 + JIT_WriteBarrier_Table_Offset) +#define JIT_WriteBarrier_Offset_RegionToGeneration (0x38 + JIT_WriteBarrier_Table_Offset) +#define JIT_WriteBarrier_Offset_RegionShr (0x40 + JIT_WriteBarrier_Table_Offset) +#define JIT_WriteBarrier_Offset_GCShadow (0x48 + JIT_WriteBarrier_Table_Offset) +#define JIT_WriteBarrier_Offset_GCShadowEnd (0x50 + JIT_WriteBarrier_Table_Offset) + +#endif // PATCHEDCODECONSTANTS_H \ No newline at end of file diff --git a/src/coreclr/vm/writebarriermanager.cpp b/src/coreclr/vm/writebarriermanager.cpp index 14e567e7176ca5..6c037b9a88d05e 100644 --- a/src/coreclr/vm/writebarriermanager.cpp +++ b/src/coreclr/vm/writebarriermanager.cpp @@ -14,6 +14,9 @@ #include "excep.h" #include "threadsuspend.h" #include "writebarriermanager.h" +#if !defined(WRITE_BARRIER_VARS_INLINE) +#include "patchedcodeconstants.h" +#endif extern uint8_t* g_ephemeral_low; extern uint8_t* g_ephemeral_high; @@ -143,6 +146,8 @@ EXTERN_C void JIT_WriteBarrier_WriteWatch_Bit_Region64_Patch_Label_CardBundleTab #else // WRITE_BARRIER_VARS_INLINE +EXTERN_C void JIT_WriteBarrier_Table(); +EXTERN_C void JIT_WriteBarrier_Table_End(); EXTERN_C void JIT_WriteBarrier_Patch_Label_WriteWatchTable(); EXTERN_C void JIT_WriteBarrier_Patch_Label_RegionToGeneration(); EXTERN_C void JIT_WriteBarrier_Patch_Label_RegionShr(); @@ -164,11 +169,6 @@ EXTERN_C void JIT_WriteBarrier_Patch_Label_GCShadowEnd(); WriteBarrierManager g_WriteBarrierManager; -// Use this somewhat hokey macro to concatenate the function start with the patch -// label. This allows the code below to look relatively nice, but relies on the -// naming convention which we have established for these helpers. -#define CALC_PATCH_LOCATION(func,label,offset) CalculatePatchLocation((PVOID)func, (PVOID)func##_##label, offset) - WriteBarrierManager::WriteBarrierManager() : m_currentWriteBarrier(WRITE_BARRIER_UNINITIALIZED) { @@ -248,7 +248,11 @@ size_t WriteBarrierManager::GetSpecificWriteBarrierSize(WriteBarrierType writeBa return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_WriteWatch_Bit_Region64); #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP case WRITE_BARRIER_BUFFER: +#if defined(WRITE_BARRIER_VARS_INLINE) return MARKED_FUNCTION_SIZE(JIT_WriteBarrier); +#else + return (size_t)((LPBYTE)GetEEFuncEntryPoint(JIT_WriteBarrier_Table_End) - (LPBYTE)GetEEFuncEntryPoint(JIT_WriteBarrier)); +#endif default: UNREACHABLE_MSG("unexpected m_currentWriteBarrier!"); }; @@ -261,21 +265,6 @@ size_t WriteBarrierManager::GetCurrentWriteBarrierSize() } -PBYTE WriteBarrierManager::CalculatePatchLocation(LPVOID base, LPVOID label, int inlineOffset) -{ - // the label should always come after or at the entrypoint for this funtion - _ASSERTE_ALL_BUILDS((LPBYTE)label >= (LPBYTE)base); - - BYTE* patchBase = GetWriteBarrierCodeLocation((void*)JIT_WriteBarrier); - - PBYTE patchLocation = (patchBase + ((LPBYTE)GetEEFuncEntryPoint(label) - (LPBYTE)GetEEFuncEntryPoint(base))); -#if defined(WRITE_BARRIER_VARS_INLINE) - patchLocation += inlineOffset; -#endif - return patchLocation; -} - - int WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, bool isRuntimeSuspended) { GCX_MAYBE_COOP_NO_THREAD_BROKEN((!isRuntimeSuspended && GetThreadNULLOk() != NULL)); @@ -317,7 +306,6 @@ void WriteBarrierManager::Initialize() } CONTRACTL_END; - // Ensure that the generic JIT_WriteBarrier function buffer is large enough to hold any of the more specific // write barrier implementations. size_t cbWriteBarrierBuffer = GetSpecificWriteBarrierSize(WRITE_BARRIER_BUFFER); @@ -342,23 +330,28 @@ void WriteBarrierManager::Initialize() #if !defined(WRITE_BARRIER_VARS_INLINE) - m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_WriteWatchTable, 0); - m_pRegionToGenTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_RegionToGeneration, 0); - m_pRegionShrDest = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_RegionShr, 0); - m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_Lower, 0); - m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_Upper, 0); - m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_CardTable, 0); + #define CALC_TABLE_LOCATION(var, offset) \ + assert(JIT_WriteBarrier_Offset_##offset == (PBYTE)JIT_WriteBarrier_Patch_Label_##offset - (PBYTE)JIT_WriteBarrier); \ + var = (PBYTE)((long)GetWriteBarrierCodeLocation((void*)JIT_WriteBarrier) + (long)JIT_WriteBarrier_Offset_##offset); + CALC_TABLE_LOCATION(m_pWriteWatchTableImmediate, WriteWatchTable); + CALC_TABLE_LOCATION(m_pRegionToGenTableImmediate, RegionToGeneration); + CALC_TABLE_LOCATION(m_pRegionShrDest, RegionShr); + CALC_TABLE_LOCATION(m_pLowerBoundImmediate, Lower); + CALC_TABLE_LOCATION(m_pUpperBoundImmediate, Upper); + CALC_TABLE_LOCATION(m_pCardTableImmediate, CardTable); #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_CardBundleTable, 0); + CALC_TABLE_LOCATION(m_pCardBundleTableImmediate, CardBundleTable); #endif -#if defined(TARGET_ARM64) && defined(WRITE_BARRIER_CHECK) - m_pGCShadow = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_GCShadow, 0); - m_pGCShadowEnd = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_GCShadowEnd, 0); - m_lowestAddress = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_LowestAddress, 0); - m_highestAddress = CALC_PATCH_LOCATION(JIT_WriteBarrier, Patch_Label_HighestAddress, 0); -#endif // TARGET_ARM64 && WRITE_BARRIER_CHECK +#if defined(TARGET_ARM64) + CALC_TABLE_LOCATION(m_lowestAddress, LowestAddress); + CALC_TABLE_LOCATION(m_highestAddress, HighestAddress); +#if defined(WRITE_BARRIER_CHECK) + CALC_TABLE_LOCATION(m_pGCShadow, GCShadow); + CALC_TABLE_LOCATION(m_pGCShadowEnd, GCShadowEnd); +#endif // WRITE_BARRIER_CHECK +#endif // TARGET_AMD64 #endif // !WRITE_BARRIER_VARS_INLINE @@ -688,6 +681,21 @@ int WriteBarrierManager::SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended) #if defined(WRITE_BARRIER_VARS_INLINE) + +// Use this somewhat hokey macro to concatenate the function start with the patch +// label. This allows the code below to look relatively nice, but relies on the +// naming convention which we have established for these helpers. +#define CALC_PATCH_LOCATION(func,label,offset) CalculatePatchLocation((PVOID)func, (PVOID)func##_##label, offset) + +PBYTE WriteBarrierManager::CalculatePatchLocation(LPVOID base, LPVOID label, int inlineOffset) +{ + // the label should always come after or at the entrypoint for this funtion + _ASSERTE_ALL_BUILDS((LPBYTE)label >= (LPBYTE)base); + + BYTE* patchBase = GetWriteBarrierCodeLocation((void*)JIT_WriteBarrier); + return (patchBase + ((LPBYTE)GetEEFuncEntryPoint(label) - (LPBYTE)GetEEFuncEntryPoint(base))) + inlineOffset; +} + // Deactivate alignment validation for code coverage builds // because the instrumentation tool will not preserve alignment // constraints and we will fail. diff --git a/src/coreclr/vm/writebarriermanager.h b/src/coreclr/vm/writebarriermanager.h index bf85978b14f33c..c8b8bc697b3f1e 100644 --- a/src/coreclr/vm/writebarriermanager.h +++ b/src/coreclr/vm/writebarriermanager.h @@ -55,21 +55,19 @@ class WriteBarrierManager #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP size_t GetCurrentWriteBarrierSize(); -protected: +private: size_t GetSpecificWriteBarrierSize(WriteBarrierType writeBarrier); - PBYTE CalculatePatchLocation(LPVOID base, LPVOID label, int offset); PCODE GetCurrentWriteBarrierCode(); int ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, bool isRuntimeSuspended); bool NeedDifferentWriteBarrier(bool bReqUpperBoundsCheck, bool bUseBitwiseWriteBarrier, WriteBarrierType* pNewWriteBarrierType); #if defined(WRITE_BARRIER_VARS_INLINE) -private: + PBYTE CalculatePatchLocation(LPVOID base, LPVOID label, int offset); void Validate(); void UpdatePatchLocations(WriteBarrierType newWriteBarrier); #endif // WRITE_BARRIER_VARS_INLINE -private: WriteBarrierType m_currentWriteBarrier; From 0f216735dbaaf6eabe6dc19ff3e991ebcbd07021 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Thu, 13 Mar 2025 09:55:44 +0000 Subject: [PATCH 35/42] Fixup comments in doc and assembly --- docs/design/coreclr/jit/GC-write-barriers.md | 8 ++++---- src/coreclr/vm/arm64/patchedcode.S | 4 ++-- src/coreclr/vm/arm64/patchedcode.asm | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/design/coreclr/jit/GC-write-barriers.md b/docs/design/coreclr/jit/GC-write-barriers.md index 8a9c987b4a41e0..2fe5903817bf29 100644 --- a/docs/design/coreclr/jit/GC-write-barriers.md +++ b/docs/design/coreclr/jit/GC-write-barriers.md @@ -25,14 +25,14 @@ JIT_WriteBarrier(Object **dst, Object *ref) if *ww_table_dst != 0: *ww_table_dst = 0xff - // Return if the reference is not in ephemeral generations. + // Return if the reference is not in ephemeral generations if ref < g_ephemeral_low || ref >= g_ephemeral_high: return // Region Checks if g_region_to_generation_table != 0: - // Calculate region locations + // Calculate region generations char reg_loc_dst = *((dst >> g_region_shr) + g_region_to_generation_table) char reg_loc_ref = *((ref >> g_region_shr) + g_region_to_generation_table) @@ -40,8 +40,8 @@ JIT_WriteBarrier(Object **dst, Object *ref) if reg_loc_dst == 0: return - // Check this is going from old to young - if reg_loc_dst >= reg_loc_ref: + // Return if the new reference is not from old to young + if reg_loc_ref >= reg_loc_dst: return // Bitwise write barriers only diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index 71e4459b6bbd0e..4b567a8d864cbc 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -235,7 +235,7 @@ LOCAL_LABEL(WriteWatchForGCHeapEnd\@): .macro WRITE_BARRIER_REGION_CHECK_STUB start, exit - // Calculate region locations + // Calculate region generations ldr x17, JIT_WriteBarrier_Offset_RegionToGeneration + FIXUP_LABEL(\start) ldr w12, JIT_WriteBarrier_Offset_RegionShr + FIXUP_LABEL(\start) lsr x15, x15, x12 @@ -247,7 +247,7 @@ LOCAL_LABEL(WriteWatchForGCHeapEnd\@): ldrb w12, [x12] cbz w12, LOCAL_LABEL(\exit) - // Check this is going from old to young + // Return if the new reference is not from old to young ldrb w15, [x15] cmp w15, w12 bhs LOCAL_LABEL(\exit) diff --git a/src/coreclr/vm/arm64/patchedcode.asm b/src/coreclr/vm/arm64/patchedcode.asm index 84c76f90d426e7..feda1b076a78c3 100644 --- a/src/coreclr/vm/arm64/patchedcode.asm +++ b/src/coreclr/vm/arm64/patchedcode.asm @@ -243,7 +243,7 @@ endm WRITE_BARRIER_REGION_CHECK_STUB macro start exit -; Calculate region locations +; Calculate region generations ldr x17, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_RegionToGeneration) ldr w12, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_RegionShr) lsr x15, x15, x12 @@ -255,7 +255,7 @@ WRITE_BARRIER_REGION_CHECK_STUB macro start exit ldrb w12, [x12] cbz w12, exit -; Check this is going from old to young +; Return if the new reference is not from old to young ldrb w15, [x15] cmp w15, w12 bhs exit From 48620bb888b3710eaf31afb085556b1079690b62 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Thu, 13 Mar 2025 11:01:53 +0000 Subject: [PATCH 36/42] fix location casts for windows --- src/coreclr/vm/writebarriermanager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/vm/writebarriermanager.cpp b/src/coreclr/vm/writebarriermanager.cpp index 6c037b9a88d05e..5c8f043fc06f52 100644 --- a/src/coreclr/vm/writebarriermanager.cpp +++ b/src/coreclr/vm/writebarriermanager.cpp @@ -332,7 +332,7 @@ void WriteBarrierManager::Initialize() #define CALC_TABLE_LOCATION(var, offset) \ assert(JIT_WriteBarrier_Offset_##offset == (PBYTE)JIT_WriteBarrier_Patch_Label_##offset - (PBYTE)JIT_WriteBarrier); \ - var = (PBYTE)((long)GetWriteBarrierCodeLocation((void*)JIT_WriteBarrier) + (long)JIT_WriteBarrier_Offset_##offset); + var = (PBYTE)((BYTE*)GetWriteBarrierCodeLocation((void*)JIT_WriteBarrier) + (PBYTE)JIT_WriteBarrier_Offset_##offset); CALC_TABLE_LOCATION(m_pWriteWatchTableImmediate, WriteWatchTable); CALC_TABLE_LOCATION(m_pRegionToGenTableImmediate, RegionToGeneration); From d97e4c1df3127b34999e33e54d5ba3b3ba960534 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Thu, 13 Mar 2025 11:19:33 +0000 Subject: [PATCH 37/42] copy patching changes to windows --- src/coreclr/vm/arm64/patchedcode.S | 2 +- src/coreclr/vm/arm64/patchedcode.asm | 30 +++++++++++++--------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index 4b567a8d864cbc..b3d5bc6566f2ff 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -243,7 +243,7 @@ LOCAL_LABEL(WriteWatchForGCHeapEnd\@): lsr x12, x14, x12 add x12, x12, x17 // x12 = (LHS >> wbs_region_shr) + wbs_region_to_generation_table - // Check whether the region we're storing into is gen 0 - nothing to do in this case + // Check whether the region we are storing into is gen 0 - nothing to do in this case ldrb w12, [x12] cbz w12, LOCAL_LABEL(\exit) diff --git a/src/coreclr/vm/arm64/patchedcode.asm b/src/coreclr/vm/arm64/patchedcode.asm index feda1b076a78c3..4a760bbb0cdd47 100644 --- a/src/coreclr/vm/arm64/patchedcode.asm +++ b/src/coreclr/vm/arm64/patchedcode.asm @@ -4,6 +4,7 @@ #include "ksarm64.h" #include "asmconstants.h" #include "asmmacros.h" +#include "patchedcodeconstants.h" ;;like TEXTAREA, but with 64 byte alignment so that we can align the patchable pool below to 64 without warning AREA |.text|,ALIGN=6,CODE,READONLY @@ -159,9 +160,6 @@ NotInHeap ; ; -#define WRITE_BARRIER_CONSTANT_OFFSET(start, constlabel) (constlabel - C_FUNC(JIT_WriteBarrier) + start) - - WRITE_BARRIER_ENTRY_STUB macro start start stlr x15, [x14] @@ -173,17 +171,17 @@ WRITE_BARRIER_SHADOW_UPDATE_STUB macro start ; Update GC Shadow Heap ; Do not perform the work if g_GCShadow is 0 - ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_GCShadow) + ldr x12, JIT_WriteBarrier_Offset_GCShadow + start cbz x12, ShadowUpdateEnd ; Compute address of shadow heap location: ; pShadow = g_GCShadow + (x14 - g_lowest_address) - ldr x17, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_LowestAddress) + ldr x17, JIT_WriteBarrier_Offset_LowestAddress + start sub x17, x14, x17 add x12, x17, x12 ; if (pShadow >= g_GCShadowEnd) goto end - ldr x17, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_GCShadowEnd) + ldr x17, JIT_WriteBarrier_Offset_GCShadowEnd + start cmp x12, x17 bhs ShadowUpdateEnd @@ -212,7 +210,7 @@ endm WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB macro start exit #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP ; Update the write watch table if necessary - ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_WriteWatchTable) + ldr x12, JIT_WriteBarrier_Offset_WriteWatchTable + start ; SoftwareWriteWatch::AddressToTableByteIndexShift add x12, x12, x14, lsr #0xc ldrb w17, [x12] @@ -226,7 +224,7 @@ endm WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB macro start exit ; Branch to Exit if the reference is not in the ephemeral generations. - ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_Lower) + ldr x12, JIT_WriteBarrier_Offset_Lower + start cmp x15, x12 blo exit endm @@ -234,8 +232,8 @@ endm WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB macro start exit ; Branch to Exit if the reference is not in the ephemeral generations. - ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_Lower) - ldr x17, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_Upper) + ldr x12, JIT_WriteBarrier_Offset_Lower + start + ldr x17, JIT_WriteBarrier_Offset_Upper + start cmp x15, x12 ccmp x15, x17, #0x2, hs bhs exit @@ -244,14 +242,14 @@ endm WRITE_BARRIER_REGION_CHECK_STUB macro start exit ; Calculate region generations - ldr x17, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_RegionToGeneration) - ldr w12, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_RegionShr) + ldr x17, JIT_WriteBarrier_Offset_RegionToGeneration + start + ldr w12, JIT_WriteBarrier_Offset_RegionShr + start lsr x15, x15, x12 add x15, x15, x17 ; x15 = (RHS >> wbs_region_shr) + wbs_region_to_generation_table lsr x12, x14, x12 add x12, x12, x17 ; x12 = (LHS >> wbs_region_shr) + wbs_region_to_generation_table -; Check whether the region we're storing into is gen 0 - nothing to do in this case +; Check whether the region we are storing into is gen 0 - nothing to do in this case ldrb w12, [x12] cbz w12, exit @@ -268,7 +266,7 @@ WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB macro start exit and w17, w17, 7 movz w15, 1 lsl w17, w15, w17 ; w17 = 1 << (LHS >> 8 && 7) - ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_CardTable) + ldr x12, JIT_WriteBarrier_Offset_CardTable + start add x15, x12, x14, lsr #11 ldrb w12, [x15] ; w12 = [(LHS >> 11) + g_card_table] tst w12, w17 @@ -283,7 +281,7 @@ endm WRITE_BARRIER_CHECK_CARD_TABLE_STUB macro start exit ; Check if we need to update the card table - ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_CardTable) + ldr x12, JIT_WriteBarrier_Offset_CardTable + start add x15, x12, x14, lsr #11 ; w12 = [(RHS >> 11) + g_card_table] ldrb w12, [x15] @@ -299,7 +297,7 @@ endm WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB macro start exit #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES ; Check if we need to update the card bundle table - ldr x12, WRITE_BARRIER_CONSTANT_OFFSET(start, JIT_WriteBarrier_Patch_Label_CardBundleTable) + ldr x12, JIT_WriteBarrier_Offset_CardBundleTable + start add x15, x12, x14, lsr #21 ldrb w12, [x15] cmp x12, 0xFF From 8f734688d3c61c1790729bb1cbd0c67282093ae8 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Thu, 13 Mar 2025 16:40:29 +0000 Subject: [PATCH 38/42] fix location casts again --- src/coreclr/vm/writebarriermanager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/vm/writebarriermanager.cpp b/src/coreclr/vm/writebarriermanager.cpp index 5c8f043fc06f52..79e4b4bd0997d0 100644 --- a/src/coreclr/vm/writebarriermanager.cpp +++ b/src/coreclr/vm/writebarriermanager.cpp @@ -332,7 +332,7 @@ void WriteBarrierManager::Initialize() #define CALC_TABLE_LOCATION(var, offset) \ assert(JIT_WriteBarrier_Offset_##offset == (PBYTE)JIT_WriteBarrier_Patch_Label_##offset - (PBYTE)JIT_WriteBarrier); \ - var = (PBYTE)((BYTE*)GetWriteBarrierCodeLocation((void*)JIT_WriteBarrier) + (PBYTE)JIT_WriteBarrier_Offset_##offset); + var = ((PBYTE)GetWriteBarrierCodeLocation((void*)JIT_WriteBarrier) + JIT_WriteBarrier_Offset_##offset); CALC_TABLE_LOCATION(m_pWriteWatchTableImmediate, WriteWatchTable); CALC_TABLE_LOCATION(m_pRegionToGenTableImmediate, RegionToGeneration); From b5cbb9ef1cae6311dc57023ec01e1458b76ac156 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Fri, 14 Mar 2025 10:14:36 +0000 Subject: [PATCH 39/42] remove extra labels --- src/coreclr/vm/arm64/patchedcode.S | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index b3d5bc6566f2ff..2bea90942e66a4 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -464,7 +464,6 @@ LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_SVR64, _TEXT // Bitwise updates for region checks // LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT -LOCAL_LABEL(JIT_WriteBarrier_WriteWatch_Byte_Region64Start): WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_Byte_Region64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_Byte_Region64 WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_Byte_Region64, Exit_WriteWatch_Byte_Region64 @@ -483,7 +482,6 @@ LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Byte_Region64, _TEXT // Does not call check card table stub // LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Bit_Region64, _TEXT -LOCAL_LABEL(JIT_WriteBarrier_WriteWatch_Bit_Region64Start): WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_Bit_Region64 WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_Bit_Region64 WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_Bit_Region64, Exit_WriteWatch_Bit_Region64 From 4dc3f636b3d6008d4ae55253900b32cee0df4e4e Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Fri, 14 Mar 2025 10:34:10 +0000 Subject: [PATCH 40/42] fix windows build --- src/coreclr/vm/arm64/patchedcode.asm | 720 +++++++++++++-------------- 1 file changed, 360 insertions(+), 360 deletions(-) diff --git a/src/coreclr/vm/arm64/patchedcode.asm b/src/coreclr/vm/arm64/patchedcode.asm index 4a760bbb0cdd47..fcfd76c286ebb3 100644 --- a/src/coreclr/vm/arm64/patchedcode.asm +++ b/src/coreclr/vm/arm64/patchedcode.asm @@ -39,37 +39,6 @@ ret lr LEAF_END - ; Begin patchable literal pool - ALIGN 64 ; Align to power of two at least as big as patchable literal pool so that it fits optimally in cache line - WRITE_BARRIER_ENTRY JIT_WriteBarrier_Table - PATCH_LABEL JIT_WriteBarrier_Patch_Label_CardTable - DCQ 0 - PATCH_LABEL JIT_WriteBarrier_Patch_Label_CardBundleTable - DCQ 0 - PATCH_LABEL JIT_WriteBarrier_Patch_Label_WriteWatchTable - DCQ 0 - PATCH_LABEL JIT_WriteBarrier_Patch_Label_Lower - DCQ 0 - PATCH_LABEL JIT_WriteBarrier_Patch_Label_Upper - DCQ 0 -wbs_lowest_address - PATCH_LABEL JIT_WriteBarrier_Patch_Label_LowestAddress - DCQ 0 -wbs_highest_address - PATCH_LABEL JIT_WriteBarrier_Patch_Label_HighestAddress - DCQ 0 - PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionToGeneration - DCQ 0 - PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionShr - DCW 0 -#ifdef WRITE_BARRIER_CHECK - PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadow - DCQ 0 - PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadowEnd - DCQ 0 -#endif - WRITE_BARRIER_END JIT_WriteBarrier_Table - ;----------------------------------------------------------------------------- ; void JIT_ByRefWriteBarrier ; On entry: @@ -144,15 +113,43 @@ NotInHeap space (232*4), 0 WRITE_BARRIER_END JIT_WriteBarrier + ; Begin patchable literal pool + ALIGN 64 ; Align to power of two at least as big as patchable literal pool so that it fits optimally in cache line + WRITE_BARRIER_ENTRY JIT_WriteBarrier_Table + PATCH_LABEL JIT_WriteBarrier_Patch_Label_CardTable + DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_CardBundleTable + DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_WriteWatchTable + DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_Lower + DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_Upper + DCQ 0 +wbs_lowest_address + PATCH_LABEL JIT_WriteBarrier_Patch_Label_LowestAddress + DCQ 0 +wbs_highest_address + PATCH_LABEL JIT_WriteBarrier_Patch_Label_HighestAddress + DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionToGeneration + DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionShr + DCW 0 +#ifdef WRITE_BARRIER_CHECK + PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadow + DCQ 0 + PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadowEnd + DCQ 0 +#endif + WRITE_BARRIER_END JIT_WriteBarrier_Table + ; ------------------------------------------------------------------ ; End of the writeable code region LEAF_ENTRY JIT_PatchedCodeLast ret lr LEAF_END -; Must be at very end of file - END - ;----------------------------------------------------------------------------- @@ -160,344 +157,347 @@ NotInHeap ; ; -WRITE_BARRIER_ENTRY_STUB macro start -start - stlr x15, [x14] -endm - - -WRITE_BARRIER_SHADOW_UPDATE_STUB macro start -#ifdef WRITE_BARRIER_CHECK -; Update GC Shadow Heap - -; Do not perform the work if g_GCShadow is 0 - ldr x12, JIT_WriteBarrier_Offset_GCShadow + start - cbz x12, ShadowUpdateEnd - -; Compute address of shadow heap location: -; pShadow = g_GCShadow + (x14 - g_lowest_address) - ldr x17, JIT_WriteBarrier_Offset_LowestAddress + start - sub x17, x14, x17 - add x12, x17, x12 - -; if (pShadow >= g_GCShadowEnd) goto end - ldr x17, JIT_WriteBarrier_Offset_GCShadowEnd + start - cmp x12, x17 - bhs ShadowUpdateEnd - -; *pShadow = x15 - str x15, [x12] - -; Ensure that the write to the shadow heap occurs before the read from the GC heap so that race -; conditions are caught by INVALIDGCVALUE. - dmb ish - -; if ([x14] == x15) goto end - ldr x17, [x14] - cmp x17, x15 - beq ShadowUpdateEnd - -; *pShadow = INVALIDGCVALUE (0xcccccccd) - movz x17, #0xcccd - movk x17, #0xcccc, LSL #16 - str x17, [x12] -#endif -ShadowUpdateEnd -endm - - - -WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB macro start exit -#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -; Update the write watch table if necessary - ldr x12, JIT_WriteBarrier_Offset_WriteWatchTable + start -; SoftwareWriteWatch::AddressToTableByteIndexShift - add x12, x12, x14, lsr #0xc - ldrb w17, [x12] - cbnz x17, WriteWatchForGCHeapEnd - mov w17, #0xFF - strb w17, [x12] -WriteWatchForGCHeapEnd -#endif -endm - - -WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB macro start exit -; Branch to Exit if the reference is not in the ephemeral generations. - ldr x12, JIT_WriteBarrier_Offset_Lower + start - cmp x15, x12 - blo exit -endm - - -WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB macro start exit -; Branch to Exit if the reference is not in the ephemeral generations. - ldr x12, JIT_WriteBarrier_Offset_Lower + start - ldr x17, JIT_WriteBarrier_Offset_Upper + start - cmp x15, x12 - ccmp x15, x17, #0x2, hs - bhs exit -endm - - -WRITE_BARRIER_REGION_CHECK_STUB macro start exit -; Calculate region generations - ldr x17, JIT_WriteBarrier_Offset_RegionToGeneration + start - ldr w12, JIT_WriteBarrier_Offset_RegionShr + start - lsr x15, x15, x12 - add x15, x15, x17 ; x15 = (RHS >> wbs_region_shr) + wbs_region_to_generation_table - lsr x12, x14, x12 - add x12, x12, x17 ; x12 = (LHS >> wbs_region_shr) + wbs_region_to_generation_table - -; Check whether the region we are storing into is gen 0 - nothing to do in this case - ldrb w12, [x12] - cbz w12, exit - -; Return if the new reference is not from old to young - ldrb w15, [x15] - cmp w15, w12 - bhs exit -endm - - -WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB macro start exit -; Check if we need to update the card table - lsr w17, w14, 8 - and w17, w17, 7 - movz w15, 1 - lsl w17, w15, w17 ; w17 = 1 << (LHS >> 8 && 7) - ldr x12, JIT_WriteBarrier_Offset_CardTable + start - add x15, x12, x14, lsr #11 - ldrb w12, [x15] ; w12 = [(LHS >> 11) + g_card_table] - tst w12, w17 - bne exit - -; Atomically update the card table -; Requires LSE, but the code is only compiled for 8.0 -; stsetb w17, [x15] - word 0x383131FF -endm - - -WRITE_BARRIER_CHECK_CARD_TABLE_STUB macro start exit -; Check if we need to update the card table - ldr x12, JIT_WriteBarrier_Offset_CardTable + start - add x15, x12, x14, lsr #11 -; w12 = [(RHS >> 11) + g_card_table] - ldrb w12, [x15] - cmp x12, 0xFF - beq exit - -; Update the card table - mov x12, 0xFF - strb w12, [x15] -endm - - -WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB macro start exit -#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES -; Check if we need to update the card bundle table - ldr x12, JIT_WriteBarrier_Offset_CardBundleTable + start - add x15, x12, x14, lsr #21 - ldrb w12, [x15] - cmp x12, 0xFF - beq exit - -; Update the card bundle - mov x12, 0xFF - strb w12, [x15] -#endif -endm - - -WRITE_BARRIER_RETURN_STUB macro exit -exit -; Increment by 8 to implement JIT_ByRefWriteBarrier contract. -; TODO: Consider duplicating the logic to get rid of this redundant 'add' -; for JIT_WriteBarrier/JIT_CheckedWriteBarrier - add x14, x14, 8 - ret lr -endm - -;----------------------------------------------------------------------------- -; void JIT_WriteBarrier_PreGrow64(Object** dst, Object* src) -; -; Skipped functionality: -; Does not update the write watch table -; Does not check wbs_ephemeral_high -; No region checks -; -LEAF_ENTRY JIT_WriteBarrier_PreGrow64 - WRITE_BARRIER_ENTRY_STUB Start_PreGrow64 - WRITE_BARRIER_SHADOW_UPDATE_STUB Start_PreGrow64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB Start_PreGrow64 Exit_PreGrow64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_PreGrow64 Exit_PreGrow64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_PreGrow64 Exit_PreGrow64 - WRITE_BARRIER_RETURN_STUB Exit_PreGrow64 -LEAF_END_MARKED JIT_WriteBarrier_PreGrow64 - - -;----------------------------------------------------------------------------- -; void JIT_WriteBarrier_PostGrow64(Object** dst, Object* src) -; -; Skipped functionality: -; Does not update the write watch table -; No region checks -; -LEAF_ENTRY JIT_WriteBarrier_PostGrow64 - WRITE_BARRIER_ENTRY_STUB Start_PostGrow64 - WRITE_BARRIER_SHADOW_UPDATE_STUB Start_PostGrow64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_PostGrow64 Exit_PostGrow64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_PostGrow64 Exit_PostGrow64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_PostGrow64 Exit_PostGrow64 - WRITE_BARRIER_RETURN_STUB Exit_PostGrow64 -LEAF_END_MARKED JIT_WriteBarrier_PostGrow64 + MACRO + WRITE_BARRIER_ENTRY_STUB $name +start$name + stlr x15, [x14] + MEND -;----------------------------------------------------------------------------- -; void JIT_WriteBarrier_SVR64(Object** dst, Object* src) -; -; SVR GC has multiple heaps, so it cannot provide one single ephemeral region to bounds check -; against, so we just skip the bounds checking all together and do our card table update unconditionally. -; -; Skipped functionality: -; Does not update the write watch table -; Does not check wbs_ephemeral_high or wbs_ephemeral_low -; No region checks -; -LEAF_ENTRY JIT_WriteBarrier_SVR64 - WRITE_BARRIER_ENTRY_STUB Start_SVR64 - WRITE_BARRIER_SHADOW_UPDATE_STUB Start_SVR64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_SVR64 Exit_SVR64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_SVR64 Exit_SVR64 - WRITE_BARRIER_RETURN_STUB Exit_SVR64 -LEAF_END_MARKED JIT_WriteBarrier_SVR64 - + MACRO + WRITE_BARRIER_SHADOW_UPDATE_STUB $name + #ifdef WRITE_BARRIER_CHECK + ; Update GC Shadow Heap + + ; Do not perform the work if g_GCShadow is 0 + ldr x12, JIT_WriteBarrier_Offset_GCShadow + start$name + cbz x12, ShadowUpdateEnd$name + + ; Compute address of shadow heap location: + ; pShadow = g_GCShadow + (x14 - g_lowest_address) + ldr x17, JIT_WriteBarrier_Offset_LowestAddress + start$name + sub x17, x14, x17 + add x12, x17, x12 + + ; if (pShadow >= g_GCShadowEnd) goto end + ldr x17, JIT_WriteBarrier_Offset_GCShadowEnd + start$name + cmp x12, x17 + bhs ShadowUpdateEnd$name + + ; *pShadow = x15 + str x15, [x12] + + ; Ensure that the write to the shadow heap occurs before the read from the GC heap so that race + ; conditions are caught by INVALIDGCVALUE. + dmb ish + + ; if ([x14] == x15) goto end + ldr x17, [x14] + cmp x17, x15 + beq ShadowUpdateEnd$name + + ; *pShadow = INVALIDGCVALUE (0xcccccccd) + movz x17, #0xcccd + movk x17, #0xcccc, LSL #16 + str x17, [x12] + #endif +ShadowUpdateEnd$name + MEND -;----------------------------------------------------------------------------- -; void JIT_WriteBarrier_Byte_Region64(Object** dst, Object* src) -; -; Skipped functionality: -; Does not update the write watch table -; Bitwise updates for region checks -; -LEAF_ENTRY JIT_WriteBarrier_Byte_Region64 - WRITE_BARRIER_ENTRY_STUB Start_Byte_Region64 - WRITE_BARRIER_SHADOW_UPDATE_STUB Start_Byte_Region64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Byte_Region64 Exit_Byte_Region64 - WRITE_BARRIER_REGION_CHECK_STUB Start_Byte_Region64 Exit_Byte_Region64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_Byte_Region64 Exit_Byte_Region64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Byte_Region64 Exit_Byte_Region64 - WRITE_BARRIER_RETURN_STUB Exit_Byte_Region64 -LEAF_END_MARKED JIT_WriteBarrier_Byte_Region64 + MACRO + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB $name + #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP + ; Update the write watch table if necessary + ldr x12, JIT_WriteBarrier_Offset_WriteWatchTable + start$name + ; SoftwareWriteWatch::AddressToTableByteIndexShift + add x12, x12, x14, lsr #0xc + ldrb w17, [x12] + cbnz x17, WriteWatchForGCHeapEnd$name + mov w17, #0xFF + strb w17, [x12] +WriteWatchForGCHeapEnd$name + #endif + MEND + MACRO + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB $name + ; Branch to Exit if the reference is not in the ephemeral generations. + ldr x12, JIT_WriteBarrier_Offset_Lower + start$name + cmp x15, x12 + blo exit$name + MEND -;----------------------------------------------------------------------------- -; void JIT_WriteBarrier_Bit_Region64(Object** dst, Object* src) -; -; Skipped functionality: -; Does not update the write watch table -; Does not call check card table stub -; -LEAF_ENTRY JIT_WriteBarrier_Bit_Region64 - WRITE_BARRIER_ENTRY_STUB Start_Bit_Region64 - WRITE_BARRIER_SHADOW_UPDATE_STUB Start_Bit_Region64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_Bit_Region64 Exit_Bit_Region64 - WRITE_BARRIER_REGION_CHECK_STUB Start_Bit_Region64 Exit_Bit_Region64 - WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Start_Bit_Region64 Exit_Bit_Region64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_Bit_Region64 Exit_Bit_Region64 - WRITE_BARRIER_RETURN_STUB Exit_Bit_Region64 -LEAF_END_MARKED JIT_WriteBarrier_Bit_Region64 + MACRO + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB $name + ; Branch to Exit if the reference is not in the ephemeral generations. + ldr x12, JIT_WriteBarrier_Offset_Lower + start$name + ldr x17, JIT_WriteBarrier_Offset_Upper + start$name + cmp x15, x12 + ccmp x15, x17, #0x2, hs + bhs exit$name + MEND + MACRO + WRITE_BARRIER_REGION_CHECK_STUB $name + ; Calculate region generations + ldr x17, JIT_WriteBarrier_Offset_RegionToGeneration + start$name + ldr w12, JIT_WriteBarrier_Offset_RegionShr + start$name + lsr x15, x15, x12 + add x15, x15, x17 ; x15 = (RHS >> wbs_region_shr) + wbs_region_to_generation_table + lsr x12, x14, x12 + add x12, x12, x17 ; x12 = (LHS >> wbs_region_shr) + wbs_region_to_generation_table + + ; Check whether the region we are storing into is gen 0 - nothing to do in this case + ldrb w12, [x12] + cbz w12, exit$name + + ; Return if the new reference is not from old to young + ldrb w15, [x15] + cmp w15, w12 + bhs exit$name + MEND -;----------------------------------------------------------------------------- -; void JIT_WriteBarrier_WriteWatch_PreGrow64(Object** dst, Object* src) -; -; Skipped functionality: -; Does not check wbs_ephemeral_high -; No region checks -; -LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PreGrow64 - WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_PreGrow64 - WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_PreGrow64 - WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_PreGrow64 Exit_WriteWatch_PreGrow64 - WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_PreGrow64 -LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PreGrow64 + MACRO + WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB $name + ; Check if we need to update the card table + lsr w17, w14, 8 + and w17, w17, 7 + movz w15, 1 + lsl w17, w15, w17 ; w17 = 1 << (LHS >> 8 && 7) + ldr x12, JIT_WriteBarrier_Offset_CardTable + start$name + add x15, x12, x14, lsr #11 + ldrb w12, [x15] ; w12 = [(LHS >> 11) + g_card_table] + tst w12, w17 + bne exit$name + + ; Atomically update the card table + ; Requires LSE, but the code is only compiled for 8.0 + ; stsetb w17, [x15] + DCD 0x383131FF + MEND + MACRO + WRITE_BARRIER_CHECK_CARD_TABLE_STUB $name + ; Check if we need to update the card table + ldr x12, JIT_WriteBarrier_Offset_CardTable + start$name + add x15, x12, x14, lsr #11 + ; w12 = [(RHS >> 11) + g_card_table] + ldrb w12, [x15] + cmp x12, 0xFF + beq exit$name + + ; Update the card table + mov x12, 0xFF + strb w12, [x15] + MEND -;----------------------------------------------------------------------------- -; void JIT_WriteBarrier_WriteWatch_PostGrow64(Object** dst, Object* src) -; -; Skipped functionality: -; No region checks -; -LEAF_ENTRY JIT_WriteBarrier_WriteWatch_PostGrow64 - WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_PostGrow64 - WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_PostGrow64 - WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_PostGrow64 Exit_WriteWatch_PostGrow64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_PostGrow64 Exit_WriteWatch_PostGrow64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_PostGrow64 Exit_WriteWatch_PostGrow64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_PostGrow64 Exit_WriteWatch_PostGrow64 - WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_PostGrow64 -LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_PostGrow64 + MACRO + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB $name + #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES + ; Check if we need to update the card bundle table + ldr x12, JIT_WriteBarrier_Offset_CardBundleTable + start$name + add x15, x12, x14, lsr #21 + ldrb w12, [x15] + cmp x12, 0xFF + beq exit$name + + ; Update the card bundle + mov x12, 0xFF + strb w12, [x15] + #endif + MEND + MACRO + WRITE_BARRIER_RETURN_STUB $name +exit$name + ; Increment by 8 to implement JIT_ByRefWriteBarrier contract. + ; TODO: Consider duplicating the logic to get rid of this redundant 'add' + ; for JIT_WriteBarrier/JIT_CheckedWriteBarrier + add x14, x14, 8 + ret lr + MEND -;----------------------------------------------------------------------------- -; void JIT_WriteBarrier_WriteWatch_SVR64(Object** dst, Object* src) -; -; SVR GC has multiple heaps, so it cannot provide one single ephemeral region to bounds check -; against, so we just skip the bounds checking all together and do our card table update unconditionally. -; -; Skipped functionality: -; Does not check wbs_ephemeral_high or wbs_ephemeral_low -; No region checks -; -LEAF_ENTRY JIT_WriteBarrier_WriteWatch_SVR64 - WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_SVR64 - WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_SVR64 - WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_SVR64 Exit_WriteWatch_SVR64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_SVR64 Exit_WriteWatch_SVR64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_SVR64 Exit_WriteWatch_SVR64 - WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_SVR64 -LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_SVR64 + ;----------------------------------------------------------------------------- + ; void JIT_WriteBarrier_PreGrow64(Object** dst, Object* src) + ; + ; Skipped functionality: + ; Does not update the write watch table + ; Does not check wbs_ephemeral_high + ; No region checks + ; + WRITE_BARRIER_ENTRY JIT_WriteBarrier_PreGrow64 + WRITE_BARRIER_ENTRY_STUB PreGrow64 + WRITE_BARRIER_SHADOW_UPDATE_STUB PreGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB PreGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB PreGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB PreGrow64 + WRITE_BARRIER_RETURN_STUB PreGrow64 + WRITE_BARRIER_END JIT_WriteBarrier_PreGrow64 + + + ;----------------------------------------------------------------------------- + ; void JIT_WriteBarrier_PostGrow64(Object** dst, Object* src) + ; + ; Skipped functionality: + ; Does not update the write watch table + ; No region checks + ; + WRITE_BARRIER_ENTRY JIT_WriteBarrier_PostGrow64 + WRITE_BARRIER_ENTRY_STUB PostGrow64 + WRITE_BARRIER_SHADOW_UPDATE_STUB PostGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB PostGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB PostGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB PostGrow64 + WRITE_BARRIER_RETURN_STUB PostGrow64 + WRITE_BARRIER_END JIT_WriteBarrier_PostGrow64 + + + ;----------------------------------------------------------------------------- + ; void JIT_WriteBarrier_SVR64(Object** dst, Object* src) + ; + ; SVR GC has multiple heaps, so it cannot provide one single ephemeral region to bounds check + ; against, so we just skip the bounds checking all together and do our card table update unconditionally. + ; + ; Skipped functionality: + ; Does not update the write watch table + ; Does not check wbs_ephemeral_high or wbs_ephemeral_low + ; No region checks + ; + WRITE_BARRIER_ENTRY JIT_WriteBarrier_SVR64 + WRITE_BARRIER_ENTRY_STUB SVR64 + WRITE_BARRIER_SHADOW_UPDATE_STUB SVR64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB SVR64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB SVR64 + WRITE_BARRIER_RETURN_STUB SVR64 + WRITE_BARRIER_END JIT_WriteBarrier_SVR64 -;----------------------------------------------------------------------------- -; void JIT_WriteBarrier_WriteWatch_Byte_Region64(Object** dst, Object* src) -; -; Skipped functionality: -; Bitwise updates for region checks -; -LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Byte_Region64 -JIT_WriteBarrier_WriteWatch_Byte_Region64Start - WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_Byte_Region64 - WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_Byte_Region64 - WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 - WRITE_BARRIER_REGION_CHECK_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 - WRITE_BARRIER_CHECK_CARD_TABLE_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_Byte_Region64 Exit_WriteWatch_Byte_Region64 - WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_Byte_Region64 -LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Byte_Region64 + ;----------------------------------------------------------------------------- + ; void JIT_WriteBarrier_Byte_Region64(Object** dst, Object* src) + ; + ; Skipped functionality: + ; Does not update the write watch table + ; Bitwise updates for region checks + ; + WRITE_BARRIER_ENTRY JIT_WriteBarrier_Byte_Region64 + WRITE_BARRIER_ENTRY_STUB Byte_Region64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Byte_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Byte_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Byte_Region64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB Byte_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Byte_Region64 + WRITE_BARRIER_RETURN_STUB Byte_Region64 + WRITE_BARRIER_END JIT_WriteBarrier_Byte_Region64 + + + ;----------------------------------------------------------------------------- + ; void JIT_WriteBarrier_Bit_Region64(Object** dst, Object* src) + ; + ; Skipped functionality: + ; Does not update the write watch table + ; Does not call check card table stub + ; + WRITE_BARRIER_ENTRY JIT_WriteBarrier_Bit_Region64 + WRITE_BARRIER_ENTRY_STUB Bit_Region64 + WRITE_BARRIER_SHADOW_UPDATE_STUB Bit_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Bit_Region64 + WRITE_BARRIER_REGION_CHECK_STUB Bit_Region64 + WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Bit_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Bit_Region64 + WRITE_BARRIER_RETURN_STUB Bit_Region64 + WRITE_BARRIER_END JIT_WriteBarrier_Bit_Region64 + + + ;----------------------------------------------------------------------------- + ; void JIT_WriteBarrier_WriteWatch_PreGrow64(Object** dst, Object* src) + ; + ; Skipped functionality: + ; Does not check wbs_ephemeral_high + ; No region checks + ; + WRITE_BARRIER_ENTRY JIT_WriteBarrier_WriteWatch_PreGrow64 + WRITE_BARRIER_ENTRY_STUB WriteWatch_PreGrow64 + WRITE_BARRIER_SHADOW_UPDATE_STUB WriteWatch_PreGrow64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB WriteWatch_PreGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_STUB WriteWatch_PreGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB WriteWatch_PreGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB WriteWatch_PreGrow64 + WRITE_BARRIER_RETURN_STUB WriteWatch_PreGrow64 + WRITE_BARRIER_END JIT_WriteBarrier_WriteWatch_PreGrow64 + + + ;----------------------------------------------------------------------------- + ; void JIT_WriteBarrier_WriteWatch_PostGrow64(Object** dst, Object* src) + ; + ; Skipped functionality: + ; No region checks + ; + WRITE_BARRIER_ENTRY JIT_WriteBarrier_WriteWatch_PostGrow64 + WRITE_BARRIER_ENTRY_STUB WriteWatch_PostGrow64 + WRITE_BARRIER_SHADOW_UPDATE_STUB WriteWatch_PostGrow64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB WriteWatch_PostGrow64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB WriteWatch_PostGrow64 + WRITE_BARRIER_RETURN_STUB WriteWatch_PostGrow64 + WRITE_BARRIER_END JIT_WriteBarrier_WriteWatch_PostGrow64 + + + ;----------------------------------------------------------------------------- + ; void JIT_WriteBarrier_WriteWatch_SVR64(Object** dst, Object* src) + ; + ; SVR GC has multiple heaps, so it cannot provide one single ephemeral region to bounds check + ; against, so we just skip the bounds checking all together and do our card table update unconditionally. + ; + ; Skipped functionality: + ; Does not check wbs_ephemeral_high or wbs_ephemeral_low + ; No region checks + ; + WRITE_BARRIER_ENTRY JIT_WriteBarrier_WriteWatch_SVR64 + WRITE_BARRIER_ENTRY_STUB WriteWatch_SVR64 + WRITE_BARRIER_SHADOW_UPDATE_STUB WriteWatch_SVR64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB WriteWatch_SVR64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB WriteWatch_SVR64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB WriteWatch_SVR64 + WRITE_BARRIER_RETURN_STUB WriteWatch_SVR64 + WRITE_BARRIER_END JIT_WriteBarrier_WriteWatch_SVR64 + + + ;----------------------------------------------------------------------------- + ; void JIT_WriteBarrier_WriteWatch_Byte_Region64(Object** dst, Object* src) + ; + ; Skipped functionality: + ; Bitwise updates for region checks + ; + WRITE_BARRIER_ENTRY JIT_WriteBarrier_WriteWatch_Byte_Region64 + WRITE_BARRIER_ENTRY_STUB WriteWatch_Byte_Region64 + WRITE_BARRIER_SHADOW_UPDATE_STUB WriteWatch_Byte_Region64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB WriteWatch_Byte_Region64 + WRITE_BARRIER_REGION_CHECK_STUB WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_TABLE_STUB WriteWatch_Byte_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB WriteWatch_Byte_Region64 + WRITE_BARRIER_RETURN_STUB WriteWatch_Byte_Region64 + WRITE_BARRIER_END JIT_WriteBarrier_WriteWatch_Byte_Region64 + + + ;----------------------------------------------------------------------------- + ; void JIT_WriteBarrier_WriteWatch_Bit_Region64(Object** dst, Object* src) + ; + ; Skipped functionality: + ; Does not call check card table stub + ; + WRITE_BARRIER_ENTRY JIT_WriteBarrier_WriteWatch_Bit_Region64 + WRITE_BARRIER_ENTRY_STUB WriteWatch_Bit_Region64 + WRITE_BARRIER_SHADOW_UPDATE_STUB WriteWatch_Bit_Region64 + WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB WriteWatch_Bit_Region64 + WRITE_BARRIER_REGION_CHECK_STUB WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB WriteWatch_Bit_Region64 + WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB WriteWatch_Bit_Region64 + WRITE_BARRIER_RETURN_STUB WriteWatch_Bit_Region64 + WRITE_BARRIER_END JIT_WriteBarrier_WriteWatch_Bit_Region64 -;----------------------------------------------------------------------------- -; void JIT_WriteBarrier_WriteWatch_Bit_Region64(Object** dst, Object* src) -; -; Skipped functionality: -; Does not call check card table stub -; -LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Bit_Region64 -JIT_WriteBarrier_WriteWatch_Bit_Region64Start - WRITE_BARRIER_ENTRY_STUB Start_WriteWatch_Bit_Region64 - WRITE_BARRIER_SHADOW_UPDATE_STUB Start_WriteWatch_Bit_Region64 - WRITE_BARRIER_WRITE_WATCH_FOR_GC_HEAP_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 - WRITE_BARRIER_CHECK_EPHEMERAL_LOW_AND_HIGH_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 - WRITE_BARRIER_REGION_CHECK_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 - WRITE_BARRIER_CHECK_BIT_REGIONS_CARD_TABLE_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 - WRITE_BARRIER_CHECK_CARD_BUNDLE_TABLE_STUB Start_WriteWatch_Bit_Region64 Exit_WriteWatch_Bit_Region64 - WRITE_BARRIER_RETURN_STUB Exit_WriteWatch_Bit_Region64 -LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Bit_Region64 +; Must be at very end of file + END From 6ea99c0ea088ad3a1611700f17bcc1ad751af181 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Fri, 14 Mar 2025 17:39:02 +0000 Subject: [PATCH 41/42] fix patched constants on windows --- src/coreclr/vm/arm64/patchedcodeconstants.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/coreclr/vm/arm64/patchedcodeconstants.h b/src/coreclr/vm/arm64/patchedcodeconstants.h index 5ad9dcf1ee4113..1b4a32d210bd19 100644 --- a/src/coreclr/vm/arm64/patchedcodeconstants.h +++ b/src/coreclr/vm/arm64/patchedcodeconstants.h @@ -13,7 +13,13 @@ // the values are correct. #define JIT_WriteBarrier_Size 0x3a0 + +#ifdef TARGET_WINDOWS +#define JIT_WriteBarrier_Table_Offset (0x30 + JIT_WriteBarrier_Size) +#else #define JIT_WriteBarrier_Table_Offset (0x2c + JIT_WriteBarrier_Size) +#endif + #define JIT_WriteBarrier_Offset_CardTable (0x0 + JIT_WriteBarrier_Table_Offset) #define JIT_WriteBarrier_Offset_CardBundleTable (0x8 + JIT_WriteBarrier_Table_Offset) #define JIT_WriteBarrier_Offset_WriteWatchTable (0x10 + JIT_WriteBarrier_Table_Offset) From 75d2443ff29c8b728598a2a064f9525f99f36a30 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Wed, 19 Mar 2025 11:57:05 +0000 Subject: [PATCH 42/42] Fix RegionShr size on windows --- src/coreclr/vm/arm64/patchedcode.asm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/vm/arm64/patchedcode.asm b/src/coreclr/vm/arm64/patchedcode.asm index fcfd76c286ebb3..500f1044d488d7 100644 --- a/src/coreclr/vm/arm64/patchedcode.asm +++ b/src/coreclr/vm/arm64/patchedcode.asm @@ -135,12 +135,12 @@ wbs_highest_address PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionToGeneration DCQ 0 PATCH_LABEL JIT_WriteBarrier_Patch_Label_RegionShr - DCW 0 + DCQ 0 #ifdef WRITE_BARRIER_CHECK PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadow DCQ 0 PATCH_LABEL JIT_WriteBarrier_Patch_Label_GCShadowEnd - DCQ 0 + DCQ 0 #endif WRITE_BARRIER_END JIT_WriteBarrier_Table