Fix crash when HitObject::TraceRay payload is an empty struct (#10634)

Extend legalizeEmptyRayPayloadsForHLSL to also handle global variables
with IRVulkanRayPayloadDecoration whose element struct type is empty.
Previously only struct types with the decoration were handled; the
[__vulkanRayPayload] static variable in __spirvTraceRayHitObjectEXT
places the decoration on the variable, not the struct type, causing the
type legalization pass to crash with "non-simple operand(s)\!".

Fixes #9757

---------

Co-authored-by: Ellie Hermaszewska <ellieh@nvidia.com>

Author: zangold-nv

source/slang/slang-ir-hlsl-legalize.cpp

@@ -126,13 +126,34 @@ void legalizeEmptyRayPayloadsForHLSL(IRModule* module)
     // First, collect all empty ray payload structs to process.
     // We must collect first because the processing phase inserts new global
     // instructions (struct keys, string values) which would invalidate the iterator.
-    List<IRStructType*> emptyRayPayloadStructs;
+    HashSet<IRStructType*> emptyRayPayloadStructs;
 
     for (auto globalInst : module->getGlobalInsts())
     {
         auto structType = as<IRStructType>(globalInst);
         if (!structType)
+        {
+            // Also check global variables with IRVulkanRayPayloadDecoration.
+            // These arise from [__vulkanRayPayload] parameters in built-in functions
+            // (e.g. __spirvTraceRayHitObjectEXT) where the decoration is on the
+            // variable rather than the struct type itself.
+            auto globalVar = as<IRGlobalVar>(globalInst);
+            if (!globalVar)
+                continue;
+            if (!globalVar->findDecoration<IRVulkanRayPayloadDecoration>())
+                continue;
+            auto ptrType = as<IRPtrTypeBase>(globalVar->getDataType());
+            if (!ptrType)
+                continue;
+            structType = as<IRStructType>(ptrType->getValueType());
+            if (!structType)
+                continue;
+            // Check if the struct is empty
+            if (structType->getFields().begin() != structType->getFields().end())
+                continue;
+            emptyRayPayloadStructs.add(structType);
             continue;
+        }
 
         // Check if this struct has ray payload decoration
         auto rayPayloadDec = structType->findDecoration<IRRayPayloadDecoration>();

tests/bugs/gh-9757.slang

@@ -0,0 +1,29 @@
+// Test for issue #9757: InternalError when HitObject::TraceRay payload is an empty struct.
+// Empty ray payload structs should compile without crashing.
+
+//TEST:SIMPLE(filecheck=CHECK): -target spirv-asm -emit-spirv-directly -capability spvShaderInvocationReorderNV -entry main -stage raygeneration
+
+struct Payload{};
+
+RaytracingAccelerationStructure tlas;
+RWTexture2D<float> outImage;
+
+[shader("raygeneration")]
+void main()
+{
+    RayDesc ray;
+    ray.Origin = float3(0.0);
+    ray.Direction = float3(0.0, 0.0, 1.0);
+    ray.TMin = 0.0;
+    ray.TMax = 1e30;
+
+    Payload payload;
+    HitObject hitObj = HitObject::TraceRay(tlas, 0, 0xFF, 0, 0, 0, ray, payload);
+    if (hitObj.IsHit())
+    {
+        outImage[DispatchRaysIndex().xy] = 1.0f;
+    }
+
+    // CHECK: OpMemberName %Payload 0 "_slang_dummy"
+    // CHECK: OpHitObjectTraceRayNV
+}