[AIEX] OuterLoopPipeliner Bug fix 1

llvm/lib/Target/AIE/Utils/AIEIRUtils.cpp

@@ -4,21 +4,40 @@
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// (c) Copyright 2025 Advanced Micro Devices, Inc. or its affiliates
+// (c) Copyright 2025-2026 Advanced Micro Devices, Inc. or its affiliates
 //
 //===----------------------------------------------------------------------===//
 
 #include "AIEIRUtils.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/Instructions.h"
-#include "llvm/IR/PatternMatch.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
 #include "llvm/Support/KnownBits.h"
 #include "llvm/Transforms/InstCombine/InstCombiner.h"
 #include "llvm/Transforms/Utils/Local.h"
 
 namespace llvm::AIEIRUtils {
 
+static Intrinsic::ID getIntrinsicID(const Instruction *I) {
+  const auto *Call = dyn_cast<CallInst>(I);
+  if (!Call)
+    return Intrinsic::not_intrinsic;
+  const auto *Fn = Call->getCalledFunction();
+  return Fn ? Fn->getIntrinsicID() : Intrinsic::not_intrinsic;
+}
+
+bool isHardwareLoopSetup(const Instruction *I) {
+  Intrinsic::ID IID = getIntrinsicID(I);
+  return IID == Intrinsic::set_loop_iterations ||
+         IID == Intrinsic::start_loop_iterations;
+}
+
+bool isHardwareLoopDecrement(const Instruction *I) {
+  return getIntrinsicID(I) == Intrinsic::loop_decrement;
+}
+
 std::optional<Instruction *> instCombineDemandedBits(InstCombiner &IC,
                                                      IntrinsicInst &II,
                                                      unsigned NumBits,

llvm/lib/Target/AIE/Utils/AIEIRUtils.h

@@ -4,7 +4,7 @@
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// (c) Copyright 2025 Advanced Micro Devices, Inc. or its affiliates
+// (c) Copyright 2025-2026 Advanced Micro Devices, Inc. or its affiliates
 //
 //===----------------------------------------------------------------------===//
 
@@ -22,6 +22,15 @@ class InstCombiner;
 
 namespace llvm::AIEIRUtils {
 
+/// True if I is a call to @llvm.set.loop.iterations or
+/// @llvm.start.loop.iterations, the intrinsics that establish a hardware-loop
+/// trip count.
+bool isHardwareLoopSetup(const Instruction *I);
+
+/// True if I is a call to @llvm.loop.decrement, the intrinsic that controls a
+/// hardware-loop latch branch.
+bool isHardwareLoopDecrement(const Instruction *I);
+
 /// Helper function to recursively check if a user (and all its users if it's a
 /// bitcast) access lanes higher than HighestLane.
 bool checkIfUsersDontAccessLanesHigherThan(Instruction *User, Type *CurrentType,

llvm/test/CodeGen/AIE/aie2p/schedule/outer-loop-pipeliner/outer-loop-pipelining-exit-no-phi.ll

@@ -0,0 +1,146 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+;
+; (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates
+;
+; RUN: llc -mtriple=aie2p -O2 -aie-enable-outer-loop-pipelining \
+; RUN:     -stop-after=irtranslator \
+; RUN:     -o - %s 2>&1 | FileCheck %s
+;
+; section); regenerate them whenever the pass output changes.
+
+; A latch-accumulated value is read after the loop, but the exit block has NO
+; phi: the latch dominates the exit (no preheader-bypass edge), so LCSSA
+; rematerializes the live-out as a plain instruction whose operands are the
+; loop-carried %sum and %acc.next. The pipeliner peels the last iteration into
+; lastiter.stage1.bottom, so the exit's operands must be remapped to the
+; last-iteration values — otherwise they dangle to poison and the function
+; returns garbage. Guards the non-phi exit live-out remap path.
+
+;
+;
+;
+;
+;
+;
+;
+;
+
+define i32 @exit_uses_latch_def(ptr noalias %a, ptr noalias %c, i32 %N, i32 %M) {
+  ; CHECK-LABEL: name: exit_uses_latch_def
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT:   liveins: $p0, $p1, $r1, $r2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(p0) = COPY $p0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(p0) = COPY $p1
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $r1
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $r2
+  ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2.steady.preheader:
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32) from %ir.a)
+  ; CHECK-NEXT:   [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY2]], [[C]]
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3.steady.header:
+  ; CHECK-NEXT:   successors: %bb.4(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.2, %20(s32), %bb.5
+  ; CHECK-NEXT:   [[PHI1:%[0-9]+]]:_(p0) = G_PHI [[COPY]](p0), %bb.2, %14(p0), %bb.5
+  ; CHECK-NEXT:   [[PHI2:%[0-9]+]]:_(p0) = G_PHI [[COPY1]](p0), %bb.2, %15(p0), %bb.5
+  ; CHECK-NEXT:   [[PHI3:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.2, %19(s32), %bb.5
+  ; CHECK-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[LOAD]](s32), %bb.2, %22(s32), %bb.5
+  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.set.loop.iterations), [[COPY3]](s32)
+  ; CHECK-NEXT:   [[C2:%[0-9]+]]:_(s20) = G_CONSTANT i20 4
+  ; CHECK-NEXT:   %14:_(p0) = nuw nusw G_PTR_ADD [[PHI1]], [[C2]](s20)
+  ; CHECK-NEXT:   %15:_(p0) = nuw nusw G_PTR_ADD [[PHI2]], [[C2]](s20)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.4.steady.inner.header:
+  ; CHECK-NEXT:   successors: %bb.4(0x7c000000), %bb.5(0x04000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI5:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.3, %17(s32), %bb.4
+  ; CHECK-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI5]], [[PHI4]]
+  ; CHECK-NEXT:   [[INT:%[0-9]+]]:_(s1) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.loop.decrement), [[C]](s32)
+  ; CHECK-NEXT:   G_BRCOND [[INT]](s1), %bb.4
+  ; CHECK-NEXT:   G_BR %bb.5
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.5.steady.latch:
+  ; CHECK-NEXT:   successors: %bb.3(0x7c000000), %bb.6(0x04000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   G_STORE [[ADD]](s32), [[PHI2]](p0) :: (store (s32) into %ir.c.ptr.steady)
+  ; CHECK-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI3]], [[ADD]]
+  ; CHECK-NEXT:   [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[PHI]], [[C]]
+  ; CHECK-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[ADD2]](s32), [[SUB]]
+  ; CHECK-NEXT:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD %14(p0) :: (load (s32) from %ir.a.ptr.next.steady)
+  ; CHECK-NEXT:   G_BRCOND [[ICMP]](s1), %bb.3
+  ; CHECK-NEXT:   G_BR %bb.6
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.6.lastiter.prologue:
+  ; CHECK-NEXT:   successors: %bb.7(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.set.loop.iterations), [[COPY3]](s32)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.7.steady.inner.header.lastiter:
+  ; CHECK-NEXT:   successors: %bb.7(0x7c000000), %bb.8(0x04000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI6:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.6, %24(s32), %bb.7
+  ; CHECK-NEXT:   [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[PHI6]], [[LOAD1]]
+  ; CHECK-NEXT:   [[INT1:%[0-9]+]]:_(s1) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.loop.decrement), [[C]](s32)
+  ; CHECK-NEXT:   G_BRCOND [[INT1]](s1), %bb.7
+  ; CHECK-NEXT:   G_BR %bb.8
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.8.lastiter.epilogue:
+  ; CHECK-NEXT:   successors: %bb.9(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   G_STORE [[ADD3]](s32), %15(p0) :: (store (s32) into %ir.c.ptr.next.steady)
+  ; CHECK-NEXT:   [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[ADD3]]
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.9.exit:
+  ; CHECK-NEXT:   [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[ADD3]]
+  ; CHECK-NEXT:   [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[ADD5]], [[C]]
+  ; CHECK-NEXT:   $r0 = COPY [[ADD6]](s32)
+  ; CHECK-NEXT:   PseudoRET implicit $lr, implicit $r0
+entry:
+  br label %outer.header
+
+outer.header:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %outer.latch ]
+  %a.ptr = phi ptr [ %a, %entry ], [ %a.ptr.next, %outer.latch ]
+  %c.ptr = phi ptr [ %c, %entry ], [ %c.ptr.next, %outer.latch ]
+  %sum = phi i32 [ 0, %entry ], [ %sum.next, %outer.latch ]
+  %v0 = load i32, ptr %a.ptr, align 4
+  call void @llvm.set.loop.iterations.i32(i32 %M)
+  br label %inner.header
+
+inner.header:
+  %acc = phi i32 [ 0, %outer.header ], [ %acc.next, %inner.header ]
+  %acc.next = add i32 %acc, %v0
+  %inner.cond = call i1 @llvm.loop.decrement.i32(i32 1)
+  br i1 %inner.cond, label %inner.header, label %outer.latch, !llvm.loop !1
+
+outer.latch:
+  store i32 %acc.next, ptr %c.ptr, align 4
+  %sum.next = add i32 %sum, %acc.next
+  %a.ptr.next = getelementptr inbounds i32, ptr %a.ptr, i32 1
+  %c.ptr.next = getelementptr inbounds i32, ptr %c.ptr, i32 1
+  %i.next = add i32 %i, 1
+  %outer.cond = icmp slt i32 %i.next, %N
+  br i1 %outer.cond, label %outer.header, label %exit, !llvm.loop !0
+
+exit:
+  %result = add i32 %sum.next, 1
+  ret i32 %result
+}
+
+declare void @llvm.set.loop.iterations.i32(i32)
+declare i1 @llvm.loop.decrement.i32(i32)
+
+!0 = distinct !{!0, !2, !3}
+!1 = distinct !{!1, !2}
+!2 = !{!"llvm.loop.mustprogress"}
+!3 = !{!"llvm.loop.itercount.range", i32 2}

llvm/test/CodeGen/AIE/aie2p/outer-loop-pipelining-gep.ll → llvm/test/CodeGen/AIE/aie2p/schedule/outer-loop-pipeliner/outer-loop-pipelining-gep.ll

@@ -17,15 +17,15 @@
 ;
 ; This tests that address computation instructions (GEPs) between PHI pointers
 ; and loads are correctly identified as part of the data-load chain and are
-; included in the warm-up and epilogue clones.
+; included in the peel and epilogue clones.
 ;
 ; Input structure:
 ;   %a.ptr = phi ptr [...]
 ;   %a.gep = getelementptr i32, ptr %a.ptr, i32 %offset   ; GEP between PHI and load
 ;   %v0 = load i32, ptr %a.gep
 ;
 ; Expected after transformation:
-;   - Warm-up: GEP cloned with initial PHI values, load uses cloned GEP
+;   - Peel: GEP cloned with initial PHI values, load uses cloned GEP
 ;   - Outer header: pipelined PHIs for both GEPs and loads
 ;   - Epilogue: GEP cloned with next-iteration pointers, load uses cloned GEP
 ;
@@ -34,40 +34,41 @@
 
 ; CHECK-LABEL: define void @nested_loop_with_gep
 
-; Warm-up block: GEPs + loads cloned with initial values (using %a, %b)
-; CHECK: outer.header.peel.pro:
-; CHECK:   %a.gep.peel = getelementptr inbounds i32, ptr %a, i32 %offset
-; CHECK:   %b.gep.peel = getelementptr inbounds i32, ptr %b, i32 %offset
-; CHECK:   %v0.peel = load i32, ptr %a.gep.peel, align 4
-; CHECK:   %v1.peel = load i32, ptr %b.gep.peel, align 4
+; Peel block: GEPs + loads cloned with initial values (using %a, %b)
+; CHECK: steady.preheader:
+; CHECK:   %a.gep.steady.peel = getelementptr inbounds i32, ptr %a, i32 %offset
+; CHECK:   %b.gep.steady.peel = getelementptr inbounds i32, ptr %b, i32 %offset
+; CHECK:   %v0.steady.peel = load i32, ptr %a.gep.steady.peel, align 4
+; CHECK:   %v1.steady.peel = load i32, ptr %b.gep.steady.peel, align 4
 ; CHECK-NOT:  call void @llvm.set.loop.iterations
-; CHECK:      br label %outer.header
+; CHECK:      br label %steady.header
 
 ; Outer header: pipelined PHIs for GEPs and loads
-; CHECK: outer.header:
-; CHECK-DAG:   %a.gep.phi = phi ptr [ %a.gep.peel, %outer.header.peel.pro ], [ %a.gep.epi, %outer.latch ]
-; CHECK-DAG:   %b.gep.phi = phi ptr [ %b.gep.peel, %outer.header.peel.pro ], [ %b.gep.epi, %outer.latch ]
-; CHECK-DAG:   %v0.phi = phi i32 [ %v0.peel, %outer.header.peel.pro ], [ %v0.epi, %outer.latch ]
-; CHECK-DAG:   %v1.phi = phi i32 [ %v1.peel, %outer.header.peel.pro ], [ %v1.epi, %outer.latch ]
+; CHECK: steady.header:
+; CHECK-DAG:   %a.gep.steady.phi = phi ptr [ %a.gep.steady.peel, %steady.preheader ], [ %a.gep.steady.epi, %steady.latch ]
+; CHECK-DAG:   %b.gep.steady.phi = phi ptr [ %b.gep.steady.peel, %steady.preheader ], [ %b.gep.steady.epi, %steady.latch ]
+; CHECK-DAG:   %v0.steady.phi = phi i32 [ %v0.steady.peel, %steady.preheader ], [ %v0.steady.epi, %steady.latch ]
+; CHECK-DAG:   %v1.steady.phi = phi i32 [ %v1.steady.peel, %steady.preheader ], [ %v1.steady.epi, %steady.latch ]
 ; CHECK:   call void @llvm.set.loop.iterations.i32(i32 %M)
-; CHECK:   br label %inner.header
+; CHECK:   br label %steady.inner.header
 
 ; Outer latch: stores + GEPs + loads for NEXT iteration (uses a.ptr.next, b.ptr.next)
-; CHECK: outer.latch:
+; CHECK: steady.latch:
 ; CHECK:   store i32
-; CHECK:   %a.gep.epi = getelementptr inbounds i32, ptr %a.ptr.next, i32 %offset
-; CHECK:   %b.gep.epi = getelementptr inbounds i32, ptr %b.ptr.next, i32 %offset
-; CHECK:   %v0.epi = load i32, ptr %a.gep.epi, align 4
-; CHECK:   %v1.epi = load i32, ptr %b.gep.epi, align 4
-; CHECK:   br i1 %outer.cond, label %outer.header, label %cooldown.entry
-
-; Cool-down entry: set.loop.iterations for last iteration
-; CHECK: cooldown.entry:
+; CHECK:   %a.gep.steady.epi = getelementptr inbounds i32, ptr %a.ptr.next.steady, i32 %offset
+; CHECK:   %b.gep.steady.epi = getelementptr inbounds i32, ptr %b.ptr.next.steady, i32 %offset
+; CHECK:   %v0.steady.epi = load i32, ptr %a.gep.steady.epi, align 4
+; CHECK:   %v1.steady.epi = load i32, ptr %b.gep.steady.epi, align 4
+; CHECK:   br i1 %outer.cond.steady, label %steady.header, label %lastiter.prologue
+
+; Cool-down (last iteration) blocks follow the steady loop.
+; Last-iteration entry: set.loop.iterations for last iteration
+; CHECK: lastiter.prologue:
 ; CHECK:   call void @llvm.set.loop.iterations.i32(i32 %M)
-; CHECK:   br label %inner.header.cd
+; CHECK:   br label %steady.inner.header.lastiter
 
-; Cool-down exit: stores only (no prologue GEPs, no prologue loads)
-; CHECK: cooldown.exit:
+; Last-iteration exit: stores only (no prologue GEPs, no prologue loads)
+; CHECK: lastiter.epilogue:
 ; CHECK:   store i32
 ; CHECK:   br label %exit