8376135: [lworld] Add JIT support for NULLABLE_NON_ATOMIC_FLAT layout

Reviewed-by: thartmann, fparain

Author: merykitty

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -7261,17 +7261,17 @@ bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, in
       b(L_notNull);
       bind(L_null);
       // Set null marker to zero to signal that the argument is null.
-      // Also set all oop fields to zero to make the GC happy.
+      // Also set all fields to zero since the runtime requires a canonical
+      // representation of a flat null.
       stream.reset(sig_index, to_index);
       while (stream.next(toReg, bt)) {
-        if (sig->at(stream.sig_index())._offset == -1 ||
-            bt == T_OBJECT || bt == T_ARRAY) {
-          if (toReg->is_stack()) {
-            int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size;
-            str(zr, Address(sp, st_off));
-          } else {
-            mov(toReg->as_Register(), zr);
-          }
+        if (toReg->is_stack()) {
+          int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size;
+          str(zr, Address(sp, st_off));
+        } else if (toReg->is_FloatRegister()) {
+          mov(toReg->as_FloatRegister(), T2S, 0);
+        } else {
+          mov(toReg->as_Register(), zr);
         }
       }
       bind(L_notNull);

src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp

@@ -3055,7 +3055,7 @@ BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(con
     j++;
   }
   assert(j == regs->length(), "missed a field?");
-  if (vk->has_nullable_atomic_layout()) {
+  if (vk->supports_nullable_layouts()) {
     // Zero the null marker (setting it to 1 would be better but would require an additional register)
     __ strb(zr, Address(r0, vk->null_marker_offset()));
   }
@@ -3067,7 +3067,8 @@ BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(con
   Label not_null;
   __ cbnz(r0, not_null);
 
-  // Return value is null. Zero oop registers to make the GC happy.
+  // Return value is null. Zero all registers because the runtime requires a canonical
+  // representation of a flat null.
   j = 1;
   for (int i = 0; i < sig_vk->length(); i++) {
     BasicType bt = sig_vk->at(i)._bt;
@@ -3081,9 +3082,12 @@ BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(con
       }
       continue;
     }
-    if (bt == T_OBJECT || bt == T_ARRAY) {
-      VMRegPair pair = regs->at(j);
-      VMReg r_1 = pair.first();
+
+    VMRegPair pair = regs->at(j);
+    VMReg r_1 = pair.first();
+    if (r_1->is_FloatRegister()) {
+      __ mov(r_1->as_FloatRegister(), Assembler::T2S, 0);
+    } else {
       __ mov(r_1->as_Register(), zr);
     }
     j++;

src/hotspot/cpu/x86/macroAssembler_x86.cpp

@@ -6380,17 +6380,17 @@ bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, in
       jmp(L_notNull);
       bind(L_null);
       // Set null marker to zero to signal that the argument is null.
-      // Also set all oop fields to zero to make the GC happy.
+      // Also set all fields to zero since the runtime requires a canonical
+      // representation of a flat null.
       stream.reset(sig_index, to_index);
       while (stream.next(toReg, bt)) {
-        if (sig->at(stream.sig_index())._offset == -1 ||
-            bt == T_OBJECT || bt == T_ARRAY) {
-          if (toReg->is_stack()) {
-            int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
-            movq(Address(rsp, st_off), 0);
-          } else {
-            xorq(toReg->as_Register(), toReg->as_Register());
-          }
+        if (toReg->is_stack()) {
+          int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
+          movq(Address(rsp, st_off), 0);
+        } else if (toReg->is_XMMRegister()) {
+          xorps(toReg->as_XMMRegister(), toReg->as_XMMRegister());
+        } else {
+          xorl(toReg->as_Register(), toReg->as_Register());
         }
       }
       bind(L_notNull);

src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp

@@ -3848,7 +3848,7 @@ BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(con
     j++;
   }
   assert(j == regs->length(), "missed a field?");
-  if (vk->has_nullable_atomic_layout()) {
+  if (vk->supports_nullable_layouts()) {
     // Set the null marker
     __ movb(Address(rax, vk->null_marker_offset()), 1);
   }
@@ -3861,7 +3861,8 @@ BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(con
   __ testptr(rax, rax);
   __ jcc(Assembler::notZero, not_null);
 
-  // Return value is null. Zero oop registers to make the GC happy.
+  // Return value is null. Zero all registers because the runtime requires a canonical
+  // representation of a flat null.
   j = 1;
   for (int i = 0; i < sig_vk->length(); i++) {
     BasicType bt = sig_vk->at(i)._bt;
@@ -3875,10 +3876,13 @@ BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(con
       }
       continue;
     }
-    if (bt == T_OBJECT || bt == T_ARRAY) {
-      VMRegPair pair = regs->at(j);
-      VMReg r_1 = pair.first();
-      __ xorq(r_1->as_Register(), r_1->as_Register());
+
+    VMRegPair pair = regs->at(j);
+    VMReg r_1 = pair.first();
+    if (r_1->is_XMMRegister()) {
+      __ xorps(r_1->as_XMMRegister(), r_1->as_XMMRegister());
+    } else {
+      __ xorl(r_1->as_Register(), r_1->as_Register());
     }
     j++;
   }

src/hotspot/share/c1/c1_Canonicalizer.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -468,11 +468,8 @@ void Canonicalizer::do_CompareOp      (CompareOp*       x) {
 
 
 void Canonicalizer::do_IfOp(IfOp* x) {
-  // Currently, Canonicalizer is only used by GraphBuilder,
-  // and IfOp is not created by GraphBuilder but only later
-  // when eliminating conditional expressions with CE_Eliminator,
-  // so this method will not be called.
-  ShouldNotReachHere();
+  // Currently, Canonicalizer is only used by GraphBuilder, and IfOp is only created by
+  // GraphBuilder when loading/storing flat fields, do nothing for now.
 }
 
 

src/hotspot/share/c1/c1_GraphBuilder.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,7 +26,9 @@
 #include "c1/c1_CFGPrinter.hpp"
 #include "c1/c1_Compilation.hpp"
 #include "c1/c1_GraphBuilder.hpp"
+#include "c1/c1_Instruction.hpp"
 #include "c1/c1_InstructionPrinter.hpp"
+#include "c1/c1_ValueType.hpp"
 #include "ci/ciCallSite.hpp"
 #include "ci/ciField.hpp"
 #include "ci/ciFlatArrayKlass.hpp"
@@ -1983,28 +1985,28 @@ void GraphBuilder::access_field(Bytecodes::Code code) {
           // Flat field
           assert(!needs_patching, "Can't patch flat inline type field access");
           ciInlineKlass* inline_klass = field->type()->as_inline_klass();
-          bool is_naturally_atomic = inline_klass->nof_declared_nonstatic_fields() <= 1;
-          bool needs_atomic_access = !field->is_null_free() || (field->is_volatile() && !is_naturally_atomic);
-          if (needs_atomic_access) {
+          if (field->is_atomic()) {
             assert(!has_pending_field_access(), "Pending field accesses are not supported");
             LoadField* load = new LoadField(obj, offset, field, false, state_before, needs_patching);
             push(type, append(load));
           } else {
-            assert(field->is_null_free(), "must be null-free");
             // Look at the next bytecode to check if we can delay the field access
             bool can_delay_access = false;
-            ciBytecodeStream s(method());
-            s.force_bci(bci());
-            s.next();
-            if (s.cur_bc() == Bytecodes::_getfield && !needs_patching) {
-              ciField* next_field = s.get_field(will_link);
-              bool next_needs_patching = !next_field->holder()->is_loaded() ||
-                                         !next_field->will_link(method(), Bytecodes::_getfield) ||
-                                         PatchALot;
-              // We can't update the offset for atomic accesses
-              bool next_needs_atomic_access = !next_field->is_null_free() || next_field->is_volatile();
-              can_delay_access = C1UseDelayedFlattenedFieldReads && !next_needs_patching && !next_needs_atomic_access;
+            if (field->is_null_free()) {
+              ciBytecodeStream s(method());
+              s.force_bci(bci());
+              s.next();
+              if (s.cur_bc() == Bytecodes::_getfield && !needs_patching) {
+                ciField* next_field = s.get_field(will_link);
+                bool next_needs_patching = !next_field->holder()->is_loaded() ||
+                                          !next_field->will_link(method(), Bytecodes::_getfield) ||
+                                          PatchALot;
+                // We can't update the offset for atomic accesses
+                bool next_needs_atomic_access = next_field->is_flat() && next_field->is_atomic();
+                can_delay_access = C1UseDelayedFlattenedFieldReads && !next_needs_patching && !next_needs_atomic_access && next_field->is_null_free();
+              }
             }
+
             if (can_delay_access) {
               if (has_pending_load_indexed()) {
                 pending_load_indexed()->update(field, offset - field->holder()->as_inline_klass()->payload_offset());
@@ -2018,8 +2020,8 @@ void GraphBuilder::access_field(Bytecodes::Code code) {
             } else {
               scope()->set_wrote_final();
               scope()->set_wrote_fields();
-              bool need_membar = false;
               if (has_pending_load_indexed()) {
+                assert(field->is_null_free(), "nullable fields do not support delayed accesses yet");
                 assert(!needs_patching, "Can't patch delayed field access");
                 pending_load_indexed()->update(field, offset - field->holder()->as_inline_klass()->payload_offset());
                 NewInstance* vt = new NewInstance(inline_klass, pending_load_indexed()->state_before(), false, true);
@@ -2028,34 +2030,49 @@ void GraphBuilder::access_field(Bytecodes::Code code) {
                 apush(append_split(vt));
                 append(pending_load_indexed()->load_instr());
                 set_pending_load_indexed(nullptr);
-                need_membar = true;
+              } else if (has_pending_field_access()) {
+                assert(field->is_null_free(), "nullable fields do not support delayed accesses yet");
+                state_before = pending_field_access()->state_before();
+                NewInstance* new_instance = new NewInstance(inline_klass, state_before, false, true);
+                _memory->new_instance(new_instance);
+                apush(append_split(new_instance));
+                copy_inline_content(inline_klass, pending_field_access()->obj(),
+                                    pending_field_access()->offset() + field->offset_in_bytes() - field->holder()->as_inline_klass()->payload_offset(),
+                                    new_instance, inline_klass->payload_offset(), state_before);
+                set_pending_field_access(nullptr);
               } else {
-                if (has_pending_field_access()) {
-                  state_before = pending_field_access()->state_before();
+                if (!field->is_null_free() && !inline_klass->is_initialized()) {
+                  // Cannot allocate an instance of inline_klass because it may have not been
+                  // initialized, bailout for now
+                  bailout("load from an uninitialized nullable non-atomic flat field");
+                  return;
                 }
+
                 NewInstance* new_instance = new NewInstance(inline_klass, state_before, false, true);
                 _memory->new_instance(new_instance);
-                apush(append_split(new_instance));
-                if (has_pending_field_access()) {
-                  copy_inline_content(inline_klass, pending_field_access()->obj(),
-                                      pending_field_access()->offset() + field->offset_in_bytes() - field->holder()->as_inline_klass()->payload_offset(),
-                                      new_instance, inline_klass->payload_offset(), state_before);
-                  set_pending_field_access(nullptr);
-                } else {
-                  if (field->type()->as_instance_klass()->is_initialized() && field->type()->as_inline_klass()->is_empty()) {
-                    // Needs an explicit null check because below code does not perform any actual load if there are no fields
-                    null_check(obj);
-                  }
-                  copy_inline_content(inline_klass, obj, field->offset_in_bytes(), new_instance, inline_klass->payload_offset(), state_before);
+                append_split(new_instance);
+
+                if (inline_klass->is_initialized() && inline_klass->is_empty()) {
+                  // Needs an explicit null check because below code does not perform any actual load if there are no fields
+                  null_check(obj);
                 }
-                need_membar = true;
-              }
-              if (need_membar) {
-                // If we allocated a new instance ensure the stores to copy the
-                // field contents are visible before any subsequent store that
-                // publishes this reference.
-                append(new MemBar(lir_membar_storestore));
+                copy_inline_content(inline_klass, obj, field->offset_in_bytes(), new_instance, inline_klass->payload_offset(), state_before);
+
+                Instruction* result = new_instance;
+                if (!field->is_null_free()) {
+                  Value int_zero = append(new Constant(intZero));
+                  Value object_null = append(new Constant(objectNull));
+                  Value nm_offset = append(new Constant(new LongConstant(offset + inline_klass->null_marker_offset_in_payload())));
+                  Value nm = append(new UnsafeGet(T_BOOLEAN, obj, nm_offset, false));
+                  result = append(new IfOp(nm, Instruction::neq, int_zero, new_instance, object_null, state_before, false));
+                }
+                apush(result);
               }
+
+              // If we allocated a new instance ensure the stores to copy the
+              // field contents are visible before any subsequent store that
+              // publishes this reference.
+              append(new MemBar(lir_membar_storestore));
             }
           }
         }
@@ -2090,16 +2107,33 @@ void GraphBuilder::access_field(Bytecodes::Code code) {
         // Flat field
         assert(!needs_patching, "Can't patch flat inline type field access");
         ciInlineKlass* inline_klass = field->type()->as_inline_klass();
-        bool is_naturally_atomic = inline_klass->nof_declared_nonstatic_fields() <= 1;
-        bool needs_atomic_access = !field->is_null_free() || (field->is_volatile() && !is_naturally_atomic);
-        if (needs_atomic_access) {
+        if (field->is_atomic()) {
           if (field->is_null_free()) {
             null_check(val);
           }
           append(new StoreField(obj, offset, field, val, false, state_before, needs_patching));
-        } else {
-          assert(field->is_null_free(), "must be null-free");
+        } else if (field->is_null_free()) {
+          assert(!inline_klass->is_empty(), "should have been handled");
           copy_inline_content(inline_klass, val, inline_klass->payload_offset(), obj, offset, state_before, field);
+        } else {
+          if (!inline_klass->is_initialized()) {
+            // null_reset_value is not available, bailout for now
+            bailout("store to an uninitialized nullable non-atomic flat field");
+            return;
+          }
+
+          // Store the subfields when field is a nullable non-atomic field
+          Value object_null = append(new Constant(objectNull));
+          Value null_reset_value = append(new Constant(new ObjectConstant(inline_klass->get_null_reset_value().as_object())));
+          Value src = append(new IfOp(val, Instruction::neq, object_null, val, null_reset_value, state_before, false));
+          copy_inline_content(inline_klass, src, inline_klass->payload_offset(), obj, offset, state_before);
+
+          // Store the null marker
+          Value int_one = append(new Constant(new IntConstant(1)));
+          Value int_zero = append(new Constant(intZero));
+          Value nm = append(new IfOp(val, Instruction::neq, object_null, int_one, int_zero, state_before, false));
+          Value nm_offset = append(new Constant(new LongConstant(offset + inline_klass->null_marker_offset_in_payload())));
+          append(new UnsafePut(T_BOOLEAN, obj, nm_offset, nm, false));
         }
       }
       break;

src/hotspot/share/c1/c1_LIRGenerator.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1650,9 +1650,7 @@ void LIRGenerator::do_StoreField(StoreField* x) {
     ciInlineKlass* vk = field->type()->as_inline_klass();
 
 #ifdef ASSERT
-    bool is_naturally_atomic = vk->nof_declared_nonstatic_fields() <= 1;
-    bool needs_atomic_access = !field->is_null_free() || (field->is_volatile() && !is_naturally_atomic);
-    assert(needs_atomic_access, "No atomic access required");
+    assert(field->is_atomic(), "No atomic access required %s.%s", field->holder()->name()->as_utf8(), field->name()->as_utf8());
     // ZGC does not support compressed oops, so only one oop can be in the payload which is written by a "normal" oop store.
     assert(!vk->contains_oops() || !UseZGC, "ZGC does not support embedded oops in flat fields");
 #endif
@@ -2098,9 +2096,7 @@ void LIRGenerator::do_LoadField(LoadField* x) {
   if (field->is_flat()) {
     ciInlineKlass* vk = field->type()->as_inline_klass();
 #ifdef ASSERT
-    bool is_naturally_atomic = vk->nof_declared_nonstatic_fields() <= 1;
-    bool needs_atomic_access = !field->is_null_free() || (field->is_volatile() && !is_naturally_atomic);
-    assert(needs_atomic_access, "No atomic access required");
+    assert(field->is_atomic(), "No atomic access required");
     assert(x->state_before() != nullptr, "Needs state before");
 #endif