8382277: Shenandoah: Improve mutator runtime entry points

Reviewed-by: kdnilsen, xpeng, rkennke

Author: shipilev

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp

@@ -96,7 +96,6 @@ class ShenandoahBarrierSet: public BarrierSet {
   void on_thread_detach(Thread* thread) override;
 
   static inline oop resolve_forwarded_not_null(oop p);
-  static inline oop resolve_forwarded_not_null_mutator(oop p);
   static inline oop resolve_forwarded(oop p);
 
   template <DecoratorSet decorators, typename T>
@@ -109,7 +108,7 @@ class ShenandoahBarrierSet: public BarrierSet {
 
   inline oop load_reference_barrier(oop obj);
 
-  template <class T>
+  template <DecoratorSet decorators, class T>
   inline oop load_reference_barrier_mutator(oop obj, T* load_addr);
 
   template <class T>

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp

@@ -56,16 +56,49 @@ inline oop ShenandoahBarrierSet::resolve_forwarded(oop p) {
   }
 }
 
-inline oop ShenandoahBarrierSet::resolve_forwarded_not_null_mutator(oop p) {
-  return ShenandoahForwarding::get_forwardee_mutator(p);
-}
-
-template <class T>
+template <DecoratorSet decorators, class T>
 inline oop ShenandoahBarrierSet::load_reference_barrier_mutator(oop obj, T* load_addr) {
-  assert(ShenandoahLoadRefBarrier, "should be enabled");
-  shenandoah_assert_in_cset(load_addr, obj);
+  assert(ShenandoahLoadRefBarrier, "Should be enabled");
+
+  constexpr bool on_weak    = HasDecorator<decorators, ON_WEAK_OOP_REF>::value;
+  constexpr bool on_phantom = HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value;
+
+  // Handle nulls. Strong loads filtered nulls with cset checks.
+  // Weak/phantom loads need to check for nulls here.
+  if (on_weak || on_phantom) {
+    if (obj == nullptr) {
+      return nullptr;
+    }
+  } else {
+    assert(obj != nullptr, "Should have been filtered before");
+  }
+
+  // Prevent resurrection of unreachable phantom (i.e. weak-native) references.
+  if (on_phantom &&
+      _heap->is_concurrent_weak_root_in_progress() &&
+      _heap->is_in_active_generation(obj) &&
+      !_heap->marking_context()->is_marked(obj)) {
+    return nullptr;
+  }
+
+  // Prevent resurrection of unreachable weak references.
+  if (on_weak &&
+      _heap->is_concurrent_weak_root_in_progress() &&
+      _heap->is_in_active_generation(obj) &&
+      !_heap->marking_context()->is_marked_strong(obj)) {
+    return nullptr;
+  }
+
+  // Weak/phantom loads need additional cset check.
+  if (on_phantom || on_weak) {
+    if (!_heap->has_forwarded_objects() || !_heap->in_collection_set(obj)) {
+      return obj;
+    }
+  } else {
+    shenandoah_assert_in_cset(load_addr, obj);
+  }
 
-  oop fwd = resolve_forwarded_not_null_mutator(obj);
+  oop fwd = ShenandoahForwarding::get_forwardee_mutator(obj);
   if (obj == fwd) {
     assert(_heap->is_evacuation_in_progress(), "evac should be in progress");
     Thread* const t = Thread::current();

src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp

@@ -49,27 +49,27 @@ JRT_LEAF(void, ShenandoahRuntime::write_barrier_pre(oopDesc* orig))
 JRT_END
 
 JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_strong(oopDesc* src, oop* load_addr))
-  return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src, load_addr);
+  return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator<ON_STRONG_OOP_REF, oop>(src, load_addr);
 JRT_END
 
 JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_strong_narrow(oopDesc* src, narrowOop* load_addr))
-  return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src, load_addr);
+  return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator<ON_STRONG_OOP_REF, narrowOop>(src, load_addr);
 JRT_END
 
 JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_weak(oopDesc* src, oop* load_addr))
-  return (oopDesc*) ShenandoahBarrierSet::barrier_set()->load_reference_barrier<oop>(ON_WEAK_OOP_REF, oop(src), load_addr);
+  return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator<ON_WEAK_OOP_REF, oop>(src, load_addr);
 JRT_END
 
 JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_weak_narrow(oopDesc* src, narrowOop* load_addr))
-  return (oopDesc*) ShenandoahBarrierSet::barrier_set()->load_reference_barrier<narrowOop>(ON_WEAK_OOP_REF, oop(src), load_addr);
+  return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator<ON_WEAK_OOP_REF, narrowOop>(src, load_addr);
 JRT_END
 
 JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_phantom(oopDesc* src, oop* load_addr))
-  return (oopDesc*) ShenandoahBarrierSet::barrier_set()->load_reference_barrier<oop>(ON_PHANTOM_OOP_REF, oop(src), load_addr);
+  return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator<ON_PHANTOM_OOP_REF, oop>(src, load_addr);
 JRT_END
 
 JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_phantom_narrow(oopDesc* src, narrowOop* load_addr))
-  return (oopDesc*) ShenandoahBarrierSet::barrier_set()->load_reference_barrier<narrowOop>(ON_PHANTOM_OOP_REF, oop(src), load_addr);
+  return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator<ON_PHANTOM_OOP_REF, narrowOop>(src, load_addr);
 JRT_END
 
 JRT_LEAF(void, ShenandoahRuntime::clone_barrier(oopDesc* src))