Fix and test infinite recursion in LLVM debug data emission (#11108)

There's a bug with the debug info in the LLVM emitter, where a struct
that references itself (even through a pointer) causes infinite
recursion and the compiler crashing.

For example, the following would cause the crash when `-g3` is
specified:
```slang
struct SelfReferential
{
    SelfReferential* self;
}
```

This is kind-of a hotfix, because it is blocking me in my own project
(but I'm already running my own branch with all of my PRs merged on that
one, so no rush to actually merge this.) This simple approach can result
in the loss of some type info in debug data when these problematic
cycles are formed; there may be a better way that would retain this
info, but that would likely require more thorough refactoring.

Author: juliusikkala

source/slang-llvm/slang-llvm-builder.cpp

@@ -365,6 +365,10 @@ class LLVMBuilder : public ComBaseObject, public ILLVMBuilder
         LLVMDebugNode* file,
         int line) override;
     SLANG_NO_THROW LLVMDebugNode* SLANG_MCALL
+    getDebugForwardDeclareType(CharSlice name, LLVMDebugNode* file, int line) override;
+    SLANG_NO_THROW void SLANG_MCALL
+    replaceDebugForwardDeclareType(LLVMDebugNode* replace, LLVMDebugNode* with) override;
+    SLANG_NO_THROW LLVMDebugNode* SLANG_MCALL
     getDebugFunctionType(LLVMDebugNode* returnType, Slice<LLVMDebugNode*> paramTypes) override;
     SLANG_NO_THROW LLVMDebugNode* SLANG_MCALL getDebugFunction(
         LLVMDebugNode* funcType,
@@ -719,16 +723,16 @@ void LLVMBuilder::finalize()
     // llvmModule->print(rso, nullptr);
     // printf("%s\n", out.c_str());
 
+    // Debug info must be finalized before verifyModule, as otherwise the
+    // unresolved debug nodes can cause verification errors.
+    if (options.debugLevel != SLANG_DEBUG_INFO_LEVEL_NONE)
+        llvmDebugBuilder->finalize();
+
     llvm::verifyModule(*llvmModule, &llvm::errs());
 
     // O0 is separately handled inside `optimize()`; we need to call it in
     // any case to make sure that `ForceInline` functions get inlined.
     optimize();
-
-    if (options.debugLevel != SLANG_DEBUG_INFO_LEVEL_NONE)
-    {
-        llvmDebugBuilder->finalize();
-    }
 }
 
 void LLVMBuilder::emitGlobalLLVMIR(const std::string& textIR)
@@ -1800,6 +1804,29 @@ LLVMDebugNode* LLVMBuilder::getDebugStructType(
         fieldTypes);
 }
 
+LLVMDebugNode* LLVMBuilder::getDebugForwardDeclareType(
+    CharSlice name,
+    LLVMDebugNode* file,
+    int line)
+{
+    if (!file)
+        file = compileUnit->getFile();
+    llvm::DIFile* llvmFile = llvm::cast<llvm::DIFile>(file);
+
+    return llvmDebugBuilder->createReplaceableCompositeType(
+        llvm::dwarf::DW_TAG_structure_type,
+        charSliceToLLVM(name),
+        llvmFile,
+        llvmFile,
+        line);
+}
+
+void LLVMBuilder::replaceDebugForwardDeclareType(LLVMDebugNode* replace, LLVMDebugNode* with)
+{
+    llvm::TempMDNode fwdDecl(replace);
+    llvmDebugBuilder->replaceTemporary(std::move(fwdDecl), with);
+}
+
 LLVMDebugNode* LLVMBuilder::getDebugFunctionType(
     LLVMDebugNode* returnType,
     Slice<LLVMDebugNode*> paramTypes)
@@ -2349,7 +2376,7 @@ SlangResult LLVMBuilder::generateJITLibrary(IArtifact** outArtifact)
 
 } // namespace slang_llvm
 
-extern "C" SLANG_DLL_EXPORT SlangResult createLLVMBuilder_V2(
+extern "C" SLANG_DLL_EXPORT SlangResult createLLVMBuilder_V3(
     const SlangUUID& intfGuid,
     Slang::ILLVMBuilder** out,
     Slang::LLVMBuilderOptions options,

source/slang-llvm/slang-llvm-builder.h

@@ -326,6 +326,12 @@ class ILLVMBuilder : public ICastable
         int64_t alignment,
         LLVMDebugNode* file,
         int line) = 0;
+
+    virtual SLANG_NO_THROW LLVMDebugNode* SLANG_MCALL
+    getDebugForwardDeclareType(CharSlice name, LLVMDebugNode* file, int line) = 0;
+    virtual SLANG_NO_THROW void SLANG_MCALL
+    replaceDebugForwardDeclareType(LLVMDebugNode* replace, LLVMDebugNode* with) = 0;
+
     virtual SLANG_NO_THROW LLVMDebugNode* SLANG_MCALL
     getDebugFunctionType(LLVMDebugNode* returnType, Slice<LLVMDebugNode*> paramTypes) = 0;
     virtual SLANG_NO_THROW LLVMDebugNode* SLANG_MCALL getDebugFunction(

source/slang/slang-emit-llvm.cpp

@@ -168,6 +168,8 @@ class LLVMTypeTranslator
     Dictionary<IRTypeLayoutRules*, Dictionary<IRType*, LLVMDebugNode*>> debugTypeMap;
 
 public:
+    Dictionary<LLVMDebugNode*, LLVMDebugNode*> forwardDeclaredDebugTypes;
+
     LLVMTypeTranslator(
         ILLVMBuilder* builder,
         TargetRequest* targetReq,
@@ -453,6 +455,20 @@ class LLVMTypeTranslator
                 int line;
                 findDebugLocation(*instToDebugLLVM, structType, file, line);
 
+                CharSlice linkageName, prettyName;
+                maybeGetName(&linkageName, &prettyName, type);
+
+                LLVMDebugNode* forwardDeclaredType = nullptr;
+                {
+                    // We forward-declare the type itself initially, so that
+                    // self-referential structs get some useful debug data as
+                    // well. These get replaced later with the final type.
+                    auto& types = debugTypeMap[rules];
+                    forwardDeclaredType =
+                        builder->getDebugForwardDeclareType(prettyName, file, line);
+                    types[type] = forwardDeclaredType;
+                }
+
                 List<LLVMDebugNode*> types;
                 for (auto field : structType->getFields())
                 {
@@ -466,20 +482,18 @@ class LLVMTypeTranslator
                     IRIntegerValue offset = getOffset(field, rules);
 
                     IRStructKey* key = field->getKey();
-                    CharSlice linkageName, prettyName;
-                    maybeGetName(&linkageName, &prettyName, key);
+                    CharSlice fieldLinkageName, fieldPrettyName;
+                    maybeGetName(&fieldLinkageName, &fieldPrettyName, key);
 
                     types.add(builder->getDebugStructField(
                         debugType,
-                        prettyName,
+                        fieldPrettyName,
                         offset,
                         fieldSizeAndAlignment.size,
                         fieldSizeAndAlignment.alignment,
                         file,
                         line));
                 }
-                CharSlice linkageName, prettyName;
-                maybeGetName(&linkageName, &prettyName, type);
                 sizeAndAlignment.size = align(sizeAndAlignment.size, sizeAndAlignment.alignment);
 
                 llvmType = builder->getDebugStructType(
@@ -489,6 +503,7 @@ class LLVMTypeTranslator
                     sizeAndAlignment.alignment,
                     file,
                     line);
+                forwardDeclaredDebugTypes[forwardDeclaredType] = llvmType;
             }
             break;
 
@@ -688,13 +703,13 @@ struct LLVMEmitter
             return SLANG_FAIL;
         }
 
-        using BuilderFuncV2 = SlangResult (*)(
+        using BuilderFuncV3 = SlangResult (*)(
             const SlangUUID& intfGuid,
             Slang::ILLVMBuilder** out,
             Slang::LLVMBuilderOptions options,
             Slang::IArtifact** outErrorArtifact);
 
-        auto builderFunc = (BuilderFuncV2)library->findFuncByName("createLLVMBuilder_V2");
+        auto builderFunc = (BuilderFuncV3)library->findFuncByName("createLLVMBuilder_V3");
         if (!builderFunc)
             return SLANG_FAIL;
 
@@ -2862,6 +2877,12 @@ struct LLVMEmitter
         emitGlobalDeclarations(irModule);
         emitGlobalFunctions(irModule);
         emitGlobalInstructionCtor();
+
+        // Some debug types may have been left as forward declared if they
+        // were self-referential (e.g., struct containing a pointer to itself).
+        // We have to resolve these last.
+        for (auto [fwdDecl, concrete] : types->forwardDeclaredDebugTypes)
+            builder->replaceDebugForwardDeclareType(fwdDecl, concrete);
     }
 };
 

tests/bugs/llvm-debug-data-recursion.slang

@@ -0,0 +1,20 @@
+// This test just needs to run without crashing.
+//TEST(compute):COMPARE_COMPUTE_EX(filecheck-buffer=CHECK): -llvm -compute -Xslang -O0 -Xslang -g3
+//TEST_INPUT:set outputBuffer = out ubuffer(data=[0 0 0 0], stride=4)
+RWStructuredBuffer<int> outputBuffer;
+
+struct SelfReferential
+{
+    // There used to be a bug in the debug data emission here, where this would
+    // cause infinite recursion inside the LLVM emitter.
+    SelfReferential* self;
+}
+
+[numthreads(1,1,1)]
+void computeMain(uint2 tid : SV_DispatchThreadID)
+{
+    SelfReferential self;
+    self.self = &self;
+    // CHECK: 0
+    outputBuffer[0] = 0;
+}