8370519: C2: Hit MemLimit when running with +VerifyLoopOptimizations

rwestrel · benoitmaillard · rwestrel · commit 176422b885d2 · 2026-02-02T11:43:30.000Z
Co-authored-by: Benoît Maillard &lt;bmaillard@openjdk.org&gt;
Reviewed-by: mhaessig, bmaillard, epeter
diff --git a/src/hotspot/share/memory/arena.hpp b/src/hotspot/share/memory/arena.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -101,6 +101,7 @@ class Chunk {
   FN(ra,          Resource areas) \
   FN(node,        C2 Node arena) \
   FN(comp,        C2 Compile arena) \
+  FN(idealloop,   C2 Ideal Loop arena) \
   FN(type,        C2 Type arena) \
   FN(states,      C2 Matcher States Arena) \
   FN(reglive,     C2 Register Allocation Live Arenas) \
diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -3752,6 +3752,20 @@ void CountedLoopEndNode::dump_spec(outputStream *st) const {
 }
 #endif
 
+IdealLoopTree::IdealLoopTree(PhaseIdealLoop* phase, Node* head, Node* tail): _parent(nullptr), _next(nullptr), _child(nullptr),
+                                                                             _head(head), _tail(tail),
+                                                                             _phase(phase),
+                                                                             _local_loop_unroll_limit(0), _local_loop_unroll_factor(0),
+                                                                             _body(phase->arena()),
+                                                                             _nest(0), _irreducible(0), _has_call(0), _has_sfpt(0), _rce_candidate(0),
+                                                                             _has_range_checks(0), _has_range_checks_computed(0),
+                                                                             _safepts(nullptr),
+                                                                             _required_safept(nullptr),
+                                                                             _allow_optimizations(true) {
+  precond(_head != nullptr);
+  precond(_tail != nullptr);
+}
+
 //=============================================================================
 //------------------------------is_member--------------------------------------
 // Is 'l' a member of 'this'?
@@ -5089,8 +5103,8 @@ void PhaseIdealLoop::build_and_optimize() {
   // Since nodes do not have a slot for immediate dominator, make
   // a persistent side array for that info indexed on node->_idx.
   _idom_size = C->unique();
-  _idom      = NEW_RESOURCE_ARRAY( Node*, _idom_size );
-  _dom_depth = NEW_RESOURCE_ARRAY( uint,  _idom_size );
+  _idom      = NEW_ARENA_ARRAY(&_arena, Node*, _idom_size);
+  _dom_depth = NEW_ARENA_ARRAY(&_arena, uint,  _idom_size);
   _dom_stk   = nullptr; // Allocated on demand in recompute_dom_depth
   memset( _dom_depth, 0, _idom_size * sizeof(uint) );
 
@@ -5691,8 +5705,8 @@ void PhaseIdealLoop::set_idom(Node* d, Node* n, uint dom_depth) {
   uint idx = d->_idx;
   if (idx >= _idom_size) {
     uint newsize = next_power_of_2(idx);
-    _idom      = REALLOC_RESOURCE_ARRAY( Node*,     _idom,_idom_size,newsize);
-    _dom_depth = REALLOC_RESOURCE_ARRAY( uint, _dom_depth,_idom_size,newsize);
+    _idom      = REALLOC_ARENA_ARRAY(&_arena, Node*,     _idom,_idom_size,newsize);
+    _dom_depth = REALLOC_ARENA_ARRAY(&_arena,  uint, _dom_depth,_idom_size,newsize);
     memset( _dom_depth + _idom_size, 0, (newsize - _idom_size) * sizeof(uint) );
     _idom_size = newsize;
   }
diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -669,21 +669,7 @@ class IdealLoopTree : public ResourceObj {
   Node_List* _required_safept;  // A inner loop cannot delete these safepts;
   bool  _allow_optimizations;   // Allow loop optimizations
 
-  IdealLoopTree( PhaseIdealLoop* phase, Node *head, Node *tail )
-    : _parent(nullptr), _next(nullptr), _child(nullptr),
-      _head(head), _tail(tail),
-      _phase(phase),
-      _local_loop_unroll_limit(0), _local_loop_unroll_factor(0),
-      _body(Compile::current()->comp_arena()),
-      _nest(0), _irreducible(0), _has_call(0), _has_sfpt(0), _rce_candidate(0),
-      _has_range_checks(0), _has_range_checks_computed(0),
-      _safepts(nullptr),
-      _required_safept(nullptr),
-      _allow_optimizations(true)
-  {
-    precond(_head != nullptr);
-    precond(_tail != nullptr);
-  }
+  IdealLoopTree(PhaseIdealLoop* phase, Node* head, Node* tail);
 
   // Is 'l' a member of 'this'?
   bool is_member(const IdealLoopTree *l) const; // Test for nested membership
@@ -889,6 +875,8 @@ class PhaseIdealLoop : public PhaseTransform {
   friend class ShenandoahBarrierC2Support;
   friend class AutoNodeBudget;
 
+  Arena _arena; // For data whose lifetime is a single pass of loop optimizations
+
   // Map loop membership for CFG nodes, and ctrl for non-CFG nodes.
   //
   // Exception: dead CFG nodes may instead have a ctrl/idom forwarding
@@ -1049,6 +1037,8 @@ class PhaseIdealLoop : public PhaseTransform {
 
   PhaseIterGVN &igvn() const { return _igvn; }
 
+  Arena* arena() { return &_arena; };
+
   bool has_node(const Node* n) const {
     guarantee(n != nullptr, "No Node.");
     return _loop_or_ctrl[n->_idx] != nullptr;
@@ -1223,7 +1213,8 @@ class PhaseIdealLoop : public PhaseTransform {
   // Compute the Ideal Node to Loop mapping
   PhaseIdealLoop(PhaseIterGVN& igvn, LoopOptsMode mode) :
     PhaseTransform(Ideal_Loop),
-    _loop_or_ctrl(igvn.C->comp_arena()),
+    _arena(mtCompiler, Arena::Tag::tag_idealloop),
+    _loop_or_ctrl(&_arena),
     _igvn(igvn),
     _verify_me(nullptr),
     _verify_only(false),
@@ -1238,7 +1229,8 @@ class PhaseIdealLoop : public PhaseTransform {
   // or only verify that the graph is valid if verify_me is null.
   PhaseIdealLoop(PhaseIterGVN& igvn, const PhaseIdealLoop* verify_me = nullptr) :
     PhaseTransform(Ideal_Loop),
-    _loop_or_ctrl(igvn.C->comp_arena()),
+    _arena(mtCompiler, Arena::Tag::tag_idealloop),
+    _loop_or_ctrl(&_arena),
     _igvn(igvn),
     _verify_me(verify_me),
     _verify_only(verify_me == nullptr),
diff --git a/test/hotspot/jtreg/compiler/c2/TestVerifyLoopOptimizationsHighMemUsage.java b/test/hotspot/jtreg/compiler/c2/TestVerifyLoopOptimizationsHighMemUsage.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @key stress randomness
+ * @bug 8370519
+ * @summary C2: Hit MemLimit when running with +VerifyLoopOptimizations
+ * @run main/othervm -XX:CompileCommand=compileonly,${test.main.class}::* -XX:-TieredCompilation -Xbatch
+ *                   -XX:+UnlockDiagnosticVMOptions -XX:+IgnoreUnrecognizedVMOptions
+ *                   -XX:+StressLoopPeeling -XX:+VerifyLoopOptimizations
+ *                   -XX:CompileCommand=memlimit,${test.main.class}::*,600M~crash
+ *                   -XX:StressSeed=3106998670 ${test.main.class}
+ * @run main ${test.main.class}
+ */
+
+package compiler.c2;
+
+public class TestVerifyLoopOptimizationsHighMemUsage {
+    public static final int N = 400;
+    public static long instanceCount = -13L;
+    public static volatile short sFld = -16143;
+    public static int iFld = -159;
+    public static float fArrFld[] = new float[N];
+
+    public static long lMeth(int i1) {
+        int i2 = 11, i3 = 37085, i4 = 177, i5 = 190, i6 = -234, i7 = 13060,
+                iArr[] = new int[N];
+        float f = 1.179F;
+        double d = 2.9685;
+        long lArr[] = new long[N];
+        for (i2 = 15; i2 < 330; ++i2)
+            for (i4 = 1; i4 < 5; ++i4) {
+                fArrFld[i4 + 1] = (++i1);
+                for (i6 = 2; i6 > 1; i6 -= 3)
+                    switch ((i2 * 5) + 54) {
+                        case 156:
+                            if (i4 != 0)
+                                ;
+                        case 168:
+                        case 342:
+                        case 283:
+                        case 281:
+                        case 328:
+                        case 322:
+                        case 228:
+                        case 114:
+                        case 207:
+                        case 209:
+                        case 354:
+                        case 108:
+                            i1 <<= i1;
+                        case 398:
+                        case 144:
+                        case 218:
+                        case 116:
+                        case 296:
+                        case 198:
+                        case 173:
+                        case 105:
+                        case 120:
+                        case 248:
+                        case 140:
+                        case 352:
+                            try {
+                            } catch (ArithmeticException a_e) {
+                            }
+                        case 404:
+                            i5 += (i6 ^ instanceCount);
+                        case 370:
+                        case 211:
+                        case 231:
+                            try {
+                            } catch (ArithmeticException a_e) {
+                            }
+                        case 251:
+                        case 179:
+                            f += (((i6 * sFld) + i4) -
+                                    iFld);
+                    }
+            }
+        long meth_res = i1 + i2 + i3 + i4 + i5 + i6 + i7 + Float.floatToIntBits(f) +
+                Double.doubleToLongBits(d) + +checkSum(iArr) +
+                checkSum(lArr);
+        return meth_res;
+    }
+
+    public static long checkSum(int[] a) {
+        long sum = 0;
+        for (int j = 0; j < a.length; j++)
+            sum += (a[j] / (j + 1) + a[j] % (j + 1));
+        return sum;
+    }
+
+    public static long checkSum(long[] a) {
+        long sum = 0;
+        for (int j = 0; j < a.length; j++)
+            sum += (a[j] / (j + 1) + a[j] % (j + 1));
+        return sum;
+    }
+
+  public static void main(String[] strArr) {
+    for (int i = 0; i < 10; i++)
+      lMeth(-159);
+  }
+}
