From f702440fcd8c14c8ca0fc07002beb90140ff3286 Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Thu, 23 Jul 2026 21:11:32 +0200 Subject: [PATCH 1/3] Fix --- .../share/gc/shenandoah/shenandoahHeap.cpp | 8 +- .../gc/shenandoah/shenandoahHeapRegion.cpp | 47 ++++++------ .../gc/shenandoah/shenandoahHeapRegion.hpp | 74 +++++++++++-------- .../shenandoahHeapRegion.inline.hpp | 4 +- .../gc/shenandoah/vmStructs_shenandoah.hpp | 2 +- 5 files changed, 77 insertions(+), 58 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 84add03f1bac0..d5677c5a14ac8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -355,8 +355,10 @@ jint ShenandoahHeap::initialize() { // // Create regions and region sets // - size_t region_align = align_up(sizeof(ShenandoahHeapRegion), SHENANDOAH_CACHE_LINE_SIZE); - size_t region_storage_size_orig = region_align * _num_regions; + size_t region_granule = align_up(sizeof(ShenandoahHeapRegion), SHENANDOAH_CACHE_LINE_SIZE); + assert(region_granule <= SHENANDOAH_CACHE_LINE_SIZE*2, + "Performance: Should take only a few cache lines: %zu", sizeof(ShenandoahHeapRegion)); + size_t region_storage_size_orig = region_granule * _num_regions; size_t region_storage_size = align_up(region_storage_size_orig, MAX2(region_page_size, os::vm_allocation_granularity())); @@ -416,7 +418,7 @@ jint ShenandoahHeap::initialize() { for (size_t i = 0; i < _num_regions; i++) { HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i; bool is_committed = i < num_committed_regions; - void* loc = region_storage.base() + i * region_align; + void* loc = region_storage.base() + i * region_granule; ShenandoahHeapRegion* r = new (loc) ShenandoahHeapRegion(start, i, is_committed); assert(is_aligned(r, SHENANDOAH_CACHE_LINE_SIZE), "Sanity"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 66eaed2c22218..02cfb21b84b5c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -57,36 +57,37 @@ size_t ShenandoahHeapRegion::MaxTLABSizeBytes = 0; size_t ShenandoahHeapRegion::MaxTLABSizeWords = 0; ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed) : - _index(index), _bottom(start), _end(start + RegionSizeWords), - _new_top(nullptr), + _index(index), _empty_time(os::elapsedTime()), + _new_top(nullptr), _top_before_promoted(nullptr), _top_at_evac_start(start), + _coalesce_and_fill_boundary(start), + _mixed_candidate_garbage_words(0), _state(committed ? _empty_committed : _empty_uncommitted), + _age(0), + #ifdef SHENANDOAH_CENSUS_NOISE + _youth(0), + #endif + _needs_bitmap_reset(false), + _promoted_in_place(false), _top(start), + _update_watermark(start), _tlab_allocs(0), _gclab_allocs(0), _plab_allocs(0), _live_data(0), - _critical_pins(0), - _mixed_candidate_garbage_words(0), - _update_watermark(start), - _age(0), -#ifdef SHENANDOAH_CENSUS_NOISE - _youth(0), -#endif // SHENANDOAH_CENSUS_NOISE - _needs_bitmap_reset(false) - { - + _critical_pins(0) +{ assert(Universe::on_page_boundary(_bottom) && Universe::on_page_boundary(_end), "invalid space boundaries"); if (ZapUnusedHeapArea && committed) { SpaceMangler::mangle_region(MemRegion(_bottom, _end)); } - _recycling.unset(); - _has_self_forwards.unset(); + _recycling.release_store(false); + _has_self_forwards.release_store(false); } void ShenandoahHeapRegion::report_illegal_transition(const char *method) { @@ -386,7 +387,7 @@ void ShenandoahHeapRegion::set_live_data(size_t s) { void ShenandoahHeapRegion::print_on(outputStream* st) const { st->print("|"); - st->print("%5zu", this->_index); + st->print(UINT32_FORMAT_W(5), this->_index); switch (state()) { case _empty_uncommitted: @@ -569,7 +570,7 @@ ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const { void ShenandoahHeapRegion::recycle_internal() { - assert(_recycling.is_set() && is_trash(), "Wrong state"); + assert(_recycling.load_acquire() && is_trash(), "Wrong state"); ShenandoahHeap* heap = ShenandoahHeap::heap(); _top_at_evac_start = _bottom; @@ -600,7 +601,7 @@ void ShenandoahHeapRegion::try_recycle_under_lock() { if (!is_trash()) { return; } - if (_recycling.try_set()) { + if (_recycling.compare_set(false, true)) { if (is_trash()) { // At freeset rebuild time, which precedes recycling of collection set, we treat all cset regions as // part of capacity, as empty, as fully available, and as unaffiliated. This provides short-lived optimism @@ -608,12 +609,12 @@ void ShenandoahHeapRegion::try_recycle_under_lock() { // by more time-precise accounting of these details. recycle_internal(); } - _recycling.unset(); + _recycling.release_store(false); } else { // Ensure recycling is unset before returning to mutator to continue memory allocation. // Otherwise, the mutator might see region as fully recycled and might change its affiliation only to have // the racing GC worker thread overwrite its affiliation to FREE. - while (_recycling.is_set()) { + while (_recycling.load_acquire()) { if (os::is_MP()) { SpinPause(); } else { @@ -631,7 +632,7 @@ void ShenandoahHeapRegion::try_recycle() { if (!is_trash()) { return; } - if (_recycling.try_set()) { + if (_recycling.compare_set(false, true)) { // Double check region state after win the race to set recycling flag if (is_trash()) { // At freeset rebuild time, which precedes recycling of collection set, we treat all cset regions as @@ -640,7 +641,7 @@ void ShenandoahHeapRegion::try_recycle() { // by more time-precise accounting of these details. recycle_internal(); } - _recycling.unset(); + _recycling.release_store(false); } } @@ -818,12 +819,12 @@ void ShenandoahHeapRegion::set_state(RegionState to) { } void ShenandoahHeapRegion::record_pin() { - _critical_pins.add_then_fetch((size_t)1); + _critical_pins.add_then_fetch((uint32_t)1); } void ShenandoahHeapRegion::record_unpin() { assert(pin_count() > 0, "Region %zu should have non-zero pins", index()); - _critical_pins.sub_then_fetch((size_t)1); + _critical_pins.sub_then_fetch((uint32_t)1); } size_t ShenandoahHeapRegion::pin_count() const { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index cf60f18e48f3e..74d82ecc2da4f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -239,49 +239,65 @@ class ShenandoahHeapRegion { static size_t MaxTLABSizeBytes; static size_t MaxTLABSizeWords; - // Never updated fields - size_t const _index; + // NOTE: This class is allocated specially by ShenandoahHeap, starting on cache line + // boundary. We have to carefully manage the order of these fields and the total density + // of the instance. Sometimes we arrange the fields in "unnatural" order to get more + // compact memory representation. If you change/rearrange the fields here, check the new + // layout with pahole. + + // ------------- Never updated fields + // These are the fastest to access, without contention with other updated fields. + HeapWord* const _bottom; HeapWord* const _end; + uint32_t const _index; - // Rarely updated fields - HeapWord* _new_top; - double _empty_time; + // ------------- Seldom updated fields + // These fields are updated rarely, and can afford introducing some contention. + float _empty_time; + HeapWord* _new_top; HeapWord* _top_before_promoted; HeapWord* _top_at_evac_start; - // Seldom updated fields - Atomic _state; - HeapWord* _coalesce_and_fill_boundary; // for old regions not selected as collection set candidates. + // For old regions not selected as collection set candidates + HeapWord* _coalesce_and_fill_boundary; - // Frequently updated fields - HeapWord* _top; + uint32_t _mixed_candidate_garbage_words; - size_t _tlab_allocs; - size_t _gclab_allocs; - size_t _plab_allocs; + Atomic _state; - Atomic _live_data; - Atomic _critical_pins; + // Set when an evacuation failure self-forwarded at least one object in this + // region. The drain at degen/full GC entry scans flagged regions and CAS- + // clears the self_fwd bits. Safety-net reset on region recycle. + Atomic _has_self_forwards; - size_t _mixed_candidate_garbage_words; + // Used to indicate that the region is being recycled; see try_recycle*() + Atomic _recycling; - Atomic _update_watermark; + // Aging and census data + uint8_t _age; + CENSUS_NOISE(uint8_t _youth;) + + // This is only read/written by a gc worker to avoid unnecessary bitmap resets + bool _needs_bitmap_reset; - uint _age; bool _promoted_in_place; - CENSUS_NOISE(uint _youth;) // tracks epochs of retrograde ageing (rejuvenation) - ShenandoahSharedFlag _recycling; // Used to indicate that the region is being recycled; see try_recycle*(). + // ------------- Frequently updated fields + // We expect lots of writes to these fields, so they must be farther away + // from the hot fields above. These fields are used in hot GC code, + // e.g. allocation, pinning, marking. - // Set when an evacuation failure self-forwarded at least one object in this - // region. The drain at degen/full GC entry scans flagged regions and CAS- - // clears the self_fwd bits. Safety-net reset on region recycle. - ShenandoahSharedFlag _has_self_forwards; + HeapWord* _top; + Atomic _update_watermark; - // This is only read/written by a gc worker to avoid unnecessary bitmap resets - bool _needs_bitmap_reset; + uint32_t _tlab_allocs; + uint32_t _gclab_allocs; + uint32_t _plab_allocs; + + Atomic _live_data; + Atomic _critical_pins; public: ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed); @@ -535,9 +551,9 @@ class ShenandoahHeapRegion { // Self-forward accounting: set by an evacuating thread after it successfully // installs a self-forward mark on an object in this region. Tested and cleared // at the drain phase (degen/full GC entry) and again on region recycle. - bool has_self_forwards() const { return _has_self_forwards.is_set(); } - void set_has_self_forwards() { _has_self_forwards.set(); } - void clear_has_self_forwards() { _has_self_forwards.unset(); } + bool has_self_forwards() const { return _has_self_forwards.load_acquire(); } + void set_has_self_forwards() { _has_self_forwards.release_store(true); } + void clear_has_self_forwards() { _has_self_forwards.release_store(false); } private: void decrement_humongous_waste(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index f004fdf0ea2be..6df1a7732046e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -93,11 +93,11 @@ inline void ShenandoahHeapRegion::increase_live_data_gc_words(size_t s) { } inline void ShenandoahHeapRegion::internal_increase_live_data(size_t s) { - _live_data.add_then_fetch(s, memory_order_relaxed); + _live_data.add_then_fetch(checked_cast(s), memory_order_relaxed); } inline void ShenandoahHeapRegion::clear_live_data() { - _live_data.store_relaxed((size_t)0); + _live_data.store_relaxed(0); _promoted_in_place = false; } diff --git a/src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp b/src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp index 4299bdb8126fc..cab1abad10ffa 100644 --- a/src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp +++ b/src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp @@ -40,7 +40,7 @@ static_field(ShenandoahHeapRegion, RegionSizeBytes, size_t) \ static_field(ShenandoahHeapRegion, RegionSizeBytesShift, size_t) \ nonstatic_field(ShenandoahHeapRegion, _state, Atomic) \ - nonstatic_field(ShenandoahHeapRegion, _index, size_t const) \ + nonstatic_field(ShenandoahHeapRegion, _index, uint32_t const) \ nonstatic_field(ShenandoahHeapRegion, _bottom, HeapWord* const) \ nonstatic_field(ShenandoahHeapRegion, _top, HeapWord*) \ nonstatic_field(ShenandoahHeapRegion, _end, HeapWord* const) \ From 4bec427419dcb31e8b62a8952591f6d2594968d6 Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Fri, 24 Jul 2026 17:16:00 +0200 Subject: [PATCH 2/3] Rewind some fields back to size_t --- src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp | 2 +- src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp | 7 +++++-- .../share/gc/shenandoah/shenandoahHeapRegion.inline.hpp | 8 ++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 02cfb21b84b5c..536b5eca16ebb 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -59,7 +59,7 @@ size_t ShenandoahHeapRegion::MaxTLABSizeWords = 0; ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed) : _bottom(start), _end(start + RegionSizeWords), - _index(index), + _index(checked_cast(index)), _empty_time(os::elapsedTime()), _new_top(nullptr), _top_before_promoted(nullptr), diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 74d82ecc2da4f..4f46ca9d96e74 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -263,7 +263,7 @@ class ShenandoahHeapRegion { // For old regions not selected as collection set candidates HeapWord* _coalesce_and_fill_boundary; - uint32_t _mixed_candidate_garbage_words; + size_t _mixed_candidate_garbage_words; Atomic _state; @@ -292,11 +292,14 @@ class ShenandoahHeapRegion { HeapWord* _top; Atomic _update_watermark; + // These are only for regular allocs, so they cannot be larger than a single region. uint32_t _tlab_allocs; uint32_t _gclab_allocs; uint32_t _plab_allocs; - Atomic _live_data; + // Live data can include humongous region size. + Atomic _live_data; + Atomic _critical_pins; public: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index 6df1a7732046e..d79d2587953f2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -75,11 +75,11 @@ inline void ShenandoahHeapRegion::adjust_alloc_metadata(const ShenandoahAllocReq // Only need to update alloc metadata for lab alloc, shared alloc is counted implicitly by tlab/gclab allocs if (req.is_lab_alloc()) { if (req.is_mutator_alloc()) { - _tlab_allocs += size; + _tlab_allocs = checked_cast(_tlab_allocs + size); } else if (req.is_old()) { - _plab_allocs += size; + _plab_allocs = checked_cast(_plab_allocs + size); } else { - _gclab_allocs += size; + _gclab_allocs = checked_cast(_gclab_allocs + size); } } } @@ -93,7 +93,7 @@ inline void ShenandoahHeapRegion::increase_live_data_gc_words(size_t s) { } inline void ShenandoahHeapRegion::internal_increase_live_data(size_t s) { - _live_data.add_then_fetch(checked_cast(s), memory_order_relaxed); + _live_data.add_then_fetch(s, memory_order_relaxed); } inline void ShenandoahHeapRegion::clear_live_data() { From 9c1d0ea183b8a6a333e9540867a50bb3b447e54e Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Fri, 24 Jul 2026 18:04:00 +0200 Subject: [PATCH 3/3] More polish --- .../gc/shenandoah/shenandoahHeapRegion.cpp | 24 +++++----- .../gc/shenandoah/shenandoahHeapRegion.hpp | 47 +++++++++---------- .../shenandoahHeapRegion.inline.hpp | 2 +- 3 files changed, 35 insertions(+), 38 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 536b5eca16ebb..488b48d9ad0f1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -60,26 +60,26 @@ ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool c _bottom(start), _end(start + RegionSizeWords), _index(checked_cast(index)), - _empty_time(os::elapsedTime()), + _state(committed ? _empty_committed : _empty_uncommitted), _new_top(nullptr), _top_before_promoted(nullptr), _top_at_evac_start(start), _coalesce_and_fill_boundary(start), _mixed_candidate_garbage_words(0), - _state(committed ? _empty_committed : _empty_uncommitted), + _live_data(0), + _critical_pins(0), + _update_watermark(start), + _top(start), + _tlab_allocs(0), + _gclab_allocs(0), + _plab_allocs(0), + _empty_time(os::elapsedTime()), _age(0), #ifdef SHENANDOAH_CENSUS_NOISE _youth(0), #endif _needs_bitmap_reset(false), - _promoted_in_place(false), - _top(start), - _update_watermark(start), - _tlab_allocs(0), - _gclab_allocs(0), - _plab_allocs(0), - _live_data(0), - _critical_pins(0) + _promoted_in_place(false) { assert(Universe::on_page_boundary(_bottom) && Universe::on_page_boundary(_end), "invalid space boundaries"); @@ -819,12 +819,12 @@ void ShenandoahHeapRegion::set_state(RegionState to) { } void ShenandoahHeapRegion::record_pin() { - _critical_pins.add_then_fetch((uint32_t)1); + _critical_pins.add_then_fetch((size_t)1); } void ShenandoahHeapRegion::record_unpin() { assert(pin_count() > 0, "Region %zu should have non-zero pins", index()); - _critical_pins.sub_then_fetch((uint32_t)1); + _critical_pins.sub_then_fetch((size_t)1); } size_t ShenandoahHeapRegion::pin_count() const { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 4f46ca9d96e74..fb2b40b6cb6eb 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -245,17 +245,16 @@ class ShenandoahHeapRegion { // compact memory representation. If you change/rearrange the fields here, check the new // layout with pahole. - // ------------- Never updated fields - // These are the fastest to access, without contention with other updated fields. + // We split fields in two groups to hit two different cache lines. The first cache line + // contains never/seldom updated fields, and also the fields we consult very often, like + // region state. The second cache line contains heavily updated fields, where contention + // is unfortunate but at least contained. HeapWord* const _bottom; HeapWord* const _end; uint32_t const _index; - // ------------- Seldom updated fields - // These fields are updated rarely, and can afford introducing some contention. - - float _empty_time; + Atomic _state; HeapWord* _new_top; HeapWord* _top_before_promoted; HeapWord* _top_at_evac_start; @@ -265,7 +264,23 @@ class ShenandoahHeapRegion { size_t _mixed_candidate_garbage_words; - Atomic _state; + // --- cache line boundary here -- + + // Live data can include humongous region size. + Atomic _live_data; + + Atomic _critical_pins; + + Atomic _update_watermark; + + HeapWord* _top; + + // These are only for regular allocs, so they cannot be larger than a single region. + uint32_t _tlab_allocs; + uint32_t _gclab_allocs; + uint32_t _plab_allocs; + + float _empty_time; // Set when an evacuation failure self-forwarded at least one object in this // region. The drain at degen/full GC entry scans flagged regions and CAS- @@ -284,24 +299,6 @@ class ShenandoahHeapRegion { bool _promoted_in_place; - // ------------- Frequently updated fields - // We expect lots of writes to these fields, so they must be farther away - // from the hot fields above. These fields are used in hot GC code, - // e.g. allocation, pinning, marking. - - HeapWord* _top; - Atomic _update_watermark; - - // These are only for regular allocs, so they cannot be larger than a single region. - uint32_t _tlab_allocs; - uint32_t _gclab_allocs; - uint32_t _plab_allocs; - - // Live data can include humongous region size. - Atomic _live_data; - - Atomic _critical_pins; - public: ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index d79d2587953f2..933b7e286fae3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -97,7 +97,7 @@ inline void ShenandoahHeapRegion::internal_increase_live_data(size_t s) { } inline void ShenandoahHeapRegion::clear_live_data() { - _live_data.store_relaxed(0); + _live_data.store_relaxed((size_t)0); _promoted_in_place = false; }