8377163: C2: Iteration split must take into consideration sunk stores

Reviewed-by: chagedorn, dfenacci

Author: merykitty

src/hotspot/share/opto/loopTransform.cpp

@@ -35,7 +35,9 @@
 #include "opto/loopnode.hpp"
 #include "opto/movenode.hpp"
 #include "opto/mulnode.hpp"
+#include "opto/node.hpp"
 #include "opto/opaquenode.hpp"
+#include "opto/opcodes.hpp"
 #include "opto/phase.hpp"
 #include "opto/predicates.hpp"
 #include "opto/rootnode.hpp"
@@ -1774,13 +1776,39 @@ Node *PhaseIdealLoop::insert_post_loop(IdealLoopTree* loop, Node_List& old_new,
   for (DUIterator i = main_head->outs(); main_head->has_out(i); i++) {
     Node* main_phi = main_head->out(i);
     if (main_phi->is_Phi() && main_phi->in(0) == main_head && main_phi->outcnt() > 0) {
-      Node* cur_phi = old_new[main_phi->_idx];
+      Node* post_phi = old_new[main_phi->_idx];
+      Node* loopback_input = main_phi->in(LoopNode::LoopBackControl);
       Node* fallnew = clone_up_backedge_goo(main_head->back_control(),
                                             post_head->init_control(),
-                                            main_phi->in(LoopNode::LoopBackControl),
+                                            loopback_input,
                                             visited, clones);
-      _igvn.hash_delete(cur_phi);
-      cur_phi->set_req(LoopNode::EntryControl, fallnew);
+      // Technically, the entry value of post_phi must be the loop back input of the corresponding
+      // Phi of the outer loop, not the Phi of the inner loop (i.e. main_phi). However, we have not
+      // constructed the Phis for the OuterStripMinedLoop yet, so the input must be inferred from
+      // the loop back input of main_phi.
+      // - If post_phi is a data Phi, then we can use the loop back input of main_phi.
+      // - If post_phi is a memory Phi, since Stores can be sunk below the inner loop, but still
+      //   inside the outer loop, we have 2 cases:
+      //   + If the loop back input of main_phi is on the backedge, then the entry input of
+      //     post_phi is the clone of the node on the entry of post_head, similar to when post_phi
+      //     is a data Phi.
+      //   + If the loop back input of main_phi is not on the backedge, we need to find whether
+      //     there is a sunk Store corresponding to post_phi, if there is any, the latest such
+      //     store will be the entry input of post_phi. Fortunately, the safepoint at the exit of
+      //     the outer loop captures all memory states, so we can use it as the entry input of
+      //     post_phi.
+      //   Another way to see it is that, the memory phi should capture the latest state at the
+      //   post-loop entry. If loopback_input is cloned by clone_up_backedge_goo, it is pinned at
+      //   the post-loop entry, and is surely the latest state. Otherwise, the latest memory state
+      //   corresponding to post_phi is the memory state at the exit of the outer main-loop, which
+      //   is captured by the safepoint there.
+      if (main_head->is_strip_mined() && fallnew == loopback_input && post_phi->is_memory_phi()) {
+        SafePointNode* main_safepoint = main_head->outer_safepoint();
+        assert(main_safepoint != nullptr, "outer loop must have a safepoint");
+        fallnew = main_safepoint->memory();
+      }
+      _igvn.hash_delete(post_phi);
+      post_phi->set_req(LoopNode::EntryControl, fallnew);
     }
   }
   // Store nodes that were moved to the outer loop by PhaseIdealLoop::try_move_store_after_loop

test/hotspot/jtreg/compiler/loopopts/TestIterationSplitWithSunkStores.java

@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package compiler.loopopts;
+
+import java.util.Objects;
+import jdk.internal.misc.Unsafe;
+
+/*
+ * @test
+ * @bug 8377163
+ * @summary Iteration splitting a counted loop with sunk stores should connect the memory phi of
+ *          the post loop to the sunk store in the main loop, not the store at the loop back input
+ *          of the corresponding phi of the main loop.
+ * @modules java.base/jdk.internal.misc
+ * @run main ${test.main.class}
+ * @run main/othervm -Xbatch -XX:-TieredCompilation ${test.main.class}
+ */
+public class TestIterationSplitWithSunkStores {
+    private static final Unsafe U = Unsafe.getUnsafe();
+
+    public static void main(String[] args) {
+        test1();
+
+        int[] array = new int[1000];
+        MyInteger v = new MyInteger(0);
+        for (int i = 0; i < 100; i++) {
+            test2(array, v, v, v, v);
+        }
+    }
+
+    private static void test1() {
+        int[] dst = new int[5];
+        for (long i = 0L; i < 20_000; i++) {
+            test1(dst, 1);
+            for (int j = 1; j < 5; j++) {
+                if (dst[j] != j) {
+                    throw new RuntimeException("Bad copy");
+                }
+            }
+        }
+    }
+
+    private static void test1(int[] dst, int dstPos) {
+        int[] src = new int[4];
+        src[0] = new MyInteger(1).v();
+        src[1] = 2;
+        src[2] = 3;
+        src[3] = 4;
+        System.arraycopy(src, 0, dst, dstPos, 4);
+    }
+
+    private static void test2(int[] array, MyInteger v1, MyInteger v2, MyInteger v3, MyInteger v4) {
+        Objects.requireNonNull(array);
+        Objects.requireNonNull(v1);
+        Objects.requireNonNull(v2);
+        Objects.requireNonNull(v3);
+        Objects.requireNonNull(v4);
+
+        // Using Unsafe to access the array so that the stores can be sunk without loop
+        // predication. This is because store sinking is only attempted during the first and the
+        // last loop opt passes, and we need it to occur before iteration splitting.
+        for (int i = 0; i < array.length; i++) {
+            long elemOffset = Unsafe.ARRAY_INT_BASE_OFFSET + (long) i * Unsafe.ARRAY_INT_INDEX_SCALE;
+            int e = U.getInt(array, elemOffset);
+            U.putInt(array, elemOffset, e + 1);
+
+            // These 4 stores can all be sunk, but depending on the order in which they are
+            // visited, it is most likely that only some of them are actually sunk
+            v1.v = e + 1;
+            v2.v = e + 2;
+            v3.v = e + 3;
+            v4.v = e + 4;
+        }
+    }
+
+    static class MyInteger {
+        public int v;
+
+        public MyInteger(int v) {
+            for (int i = 0; i < 32; i++) {
+                if (i < 10) {
+                    this.v = v;
+                }
+            }
+            this.v = v;
+        }
+
+        public int v() {
+            return v;
+        }
+    }
+}