[VPlan] Restore all-operands-inv WidenGEP logic

[artagnon]

Restore the all-operands-invariant handling in WidenGEP::execute prior to 37f7b31 (Reland [VPlan] Handle WidenGEP in narrowToSingleScalars), as crashes have been reported.

Fixes #173761.

[llvmbot]

@llvm/pr-subscribers-vectorizers

@llvm/pr-subscribers-llvm-transforms

Author: Ramkumar Ramachandra (artagnon)

Changes

Restore the all-operands-invariant handling in WidenGEP::execute prior to 37f7b31 (Reland [VPlan] Handle WidenGEP in narrowToSingleScalars), as crashes have been reported.

Fixes #173761.

---

Full diff: https://github.com/llvm/llvm-project/pull/174416.diff

2 Files Affected:

• (modified) llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp (+27-4)
• (modified) llvm/test/Transforms/LoopVectorize/widen-gep-all-indices-invariant.ll (+51)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 8044d4d955bc5..34528dc21d3f4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2503,10 +2503,33 @@ void VPWidenGEPRecipe::execute(VPTransformState &State) {
   // is vector-typed. Thus, to keep the representation compact, we only use
   // vector-typed operands for loop-varying values.
 
-  assert(
-      any_of(operands(),
-             [](VPValue *Op) { return !Op->isDefinedOutsideLoopRegions(); }) &&
-      "Expected at least one loop-variant operand");
+  bool AllOperandsAreInvariant = all_of(operands(), [](VPValue *Op) {
+    return Op->isDefinedOutsideLoopRegions();
+  });
+  if (AllOperandsAreInvariant) {
+    // If we are vectorizing, but the GEP has only loop-invariant operands,
+    // the GEP we build (by only using vector-typed operands for
+    // loop-varying values) would be a scalar pointer. Thus, to ensure we
+    // produce a vector of pointers, we need to either arbitrarily pick an
+    // operand to broadcast, or broadcast a clone of the original GEP.
+    // Here, we broadcast a clone of the original.
+    //
+    // TODO: If at some point we decide to scalarize instructions having
+    //       loop-invariant operands, this special case will no longer be
+    //       required. We would add the scalarization decision to
+    //       collectLoopScalars() and teach getVectorValue() to broadcast
+    //       the lane-zero scalar value.
+    SmallVector<Value *> Ops;
+    for (unsigned I = 0, E = getNumOperands(); I != E; I++)
+      Ops.push_back(State.get(getOperand(I), VPLane(0)));
+
+    auto *NewGEP =
+        State.Builder.CreateGEP(getSourceElementType(), Ops[0], drop_begin(Ops),
+                                "", getGEPNoWrapFlags());
+    Value *Splat = State.Builder.CreateVectorSplat(State.VF, NewGEP);
+    State.set(this, Splat);
+    return;
+  }
 
   // If the GEP has at least one loop-varying operand, we are sure to
   // produce a vector of pointers unless VF is scalar.
diff --git a/llvm/test/Transforms/LoopVectorize/widen-gep-all-indices-invariant.ll b/llvm/test/Transforms/LoopVectorize/widen-gep-all-indices-invariant.ll
index 90ef97609e096..ee9cf443a1f77 100644
--- a/llvm/test/Transforms/LoopVectorize/widen-gep-all-indices-invariant.ll
+++ b/llvm/test/Transforms/LoopVectorize/widen-gep-all-indices-invariant.ll
@@ -126,3 +126,54 @@ loop:
 exit:
   ret void
 }
+
+define void @pr173761(i8 %c, ptr %p, ptr noalias %q, ptr noalias %r) {
+; CHECK-LABEL: define void @pr173761
+; CHECK-SAME: (i8 [[C:%.*]], ptr [[P:%.*]], ptr noalias [[Q:%.*]], ptr noalias [[R:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[C]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x ptr> poison, ptr [[P]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x ptr> [[BROADCAST_SPLATINSERT1]], <4 x ptr> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 16
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP0]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x ptr> [[BROADCAST_SPLATINSERT3]], <4 x ptr> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <4 x i8> [[BROADCAST_SPLAT]] to <4 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x ptr> [[BROADCAST_SPLAT4]], <4 x ptr> [[BROADCAST_SPLAT2]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT:    store i8 [[TMP3]], ptr [[R]], align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x ptr> [[TMP2]], i32 3
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
+; CHECK-NEXT:    store i8 [[TMP5]], ptr [[Q]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i16 [ 0, %entry ], [ %iv.next, %loop ]
+  %ld.p = load i8, ptr %p
+  store i8 %ld.p, ptr %r
+  %gep.inv = getelementptr inbounds nuw i8, ptr %p, i64 16
+  %c.trunc = trunc i8 %c to i1
+  %sel.inv = select i1 %c.trunc, ptr %gep.inv, ptr %p
+  %ld.sel.inv = load i8, ptr %sel.inv
+  store i8 %ld.sel.inv, ptr %q
+  %iv.next = add i16 %iv, 1
+  %ec = icmp ne i16 %iv.next, 1024
+  br i1 %ec, label %loop, label %exit
+
+exit:
+  ret void
+}

[ElvisWang123]

This probably also fixes #173755 and #173753.

[fhahn]

LGTM, thanks

[github-actions]

🪟 Windows x64 Test Results

• 129159 tests passed
• 2842 tests skipped
• 1 test failed

Failed Tests

(click on a test name to see its output)

LLVM

LLVM.CodeGen/RISCV/riscv-macho.ll

Exit Code: 3221225477

Command Output (stdout):
--
# RUN: at line 1
c:\_work\llvm-project\llvm-project\build\bin\llc.exe -mtriple=riscv32-apple-macho C:\_work\llvm-project\llvm-project\llvm\test\CodeGen\RISCV\riscv-macho.ll -o - | c:\_work\llvm-project\llvm-project\build\bin\filecheck.exe C:\_work\llvm-project\llvm-project\llvm\test\CodeGen\RISCV\riscv-macho.ll
# executed command: 'c:\_work\llvm-project\llvm-project\build\bin\llc.exe' -mtriple=riscv32-apple-macho 'C:\_work\llvm-project\llvm-project\llvm\test\CodeGen\RISCV\riscv-macho.ll' -o -
# note: command had no output on stdout or stderr
# executed command: 'c:\_work\llvm-project\llvm-project\build\bin\filecheck.exe' 'C:\_work\llvm-project\llvm-project\llvm\test\CodeGen\RISCV\riscv-macho.ll'
# note: command had no output on stdout or stderr
# RUN: at line 2
c:\_work\llvm-project\llvm-project\build\bin\llc.exe -mtriple=riscv32-apple-macho -filetype=obj C:\_work\llvm-project\llvm-project\llvm\test\CodeGen\RISCV\riscv-macho.ll -o C:\_work\llvm-project\llvm-project\build\test\CodeGen\RISCV\Output\riscv-macho.ll.tmp.o
# executed command: 'c:\_work\llvm-project\llvm-project\build\bin\llc.exe' -mtriple=riscv32-apple-macho -filetype=obj 'C:\_work\llvm-project\llvm-project\llvm\test\CodeGen\RISCV\riscv-macho.ll' -o 'C:\_work\llvm-project\llvm-project\build\test\CodeGen\RISCV\Output\riscv-macho.ll.tmp.o'
# .---command stderr------------
# | PLEASE submit a bug report to https://github.com/llvm/llvm-project/issues/ and include the crash backtrace and instructions to reproduce the bug.
# | Stack dump:
# | 0.	Program arguments: c:\\_work\\llvm-project\\llvm-project\\build\\bin\\llc.exe -mtriple=riscv32-apple-macho -filetype=obj C:\\_work\\llvm-project\\llvm-project\\llvm\\test\\CodeGen\\RISCV\\riscv-macho.ll -o C:\\_work\\llvm-project\\llvm-project\\build\\test\\CodeGen\\RISCV\\Output\\riscv-macho.ll.tmp.o
# | Exception Code: 0xC0000005
# |  #0 0x00007ff81dc92004 (C:\Windows\SYSTEM32\ntdll.dll+0x12004)
# |  #1 0x00007ff81dc91dda (C:\Windows\SYSTEM32\ntdll.dll+0x11dda)
# |  #2 0x00007ff80b26ff29 (C:\Windows\System32\ucrtbase.dll+0xff29)
# |  #3 0x00007ff76c1f1429 (c:\_work\llvm-project\llvm-project\build\bin\llc.exe+0x6f1429)
# |  #4 0x00007ff76e9500f3 (c:\_work\llvm-project\llvm-project\build\bin\llc.exe+0x2e500f3)
# |  #5 0x00007ff76d98a4d8 (c:\_work\llvm-project\llvm-project\build\bin\llc.exe+0x1e8a4d8)
# |  #6 0x00007ff76c36f105 (c:\_work\llvm-project\llvm-project\build\bin\llc.exe+0x86f105)
# |  #7 0x00007ff76bfc19a1 (c:\_work\llvm-project\llvm-project\build\bin\llc.exe+0x4c19a1)
# |  #8 0x00007ff76bfb9fb1 (c:\_work\llvm-project\llvm-project\build\bin\llc.exe+0x4b9fb1)
# |  #9 0x00007ff76bb07c0c (c:\_work\llvm-project\llvm-project\build\bin\llc.exe+0x7c0c)
# | #10 0x00007ff76bb04e86 (c:\_work\llvm-project\llvm-project\build\bin\llc.exe+0x4e86)
# | #11 0x00007ff77011a194 (c:\_work\llvm-project\llvm-project\build\bin\llc.exe+0x461a194)
# | #12 0x00007ff818964cb0 (C:\Windows\System32\KERNEL32.DLL+0x14cb0)
# | #13 0x00007ff81dcfedcb (C:\Windows\SYSTEM32\ntdll.dll+0x7edcb)
# `-----------------------------
# error: command failed with exit status: 0xc0000005

--

If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the infrastructure label.