Handle circular references in OpenAPI schema generation

Enhance SignalROpenApiDocumentGenerator to prevent stack overflows by tracking processed types and returning schema references for self-referencing types. Add unit test and new hub/type for circular reference scenarios. Update GitHub Actions to publish non-draft releases.

Author: icnocop

.github/workflows/build.yml

@@ -96,6 +96,6 @@ jobs:
         name: v${{ env.PACKAGE_VERSION }}
         tag_name: v${{ env.PACKAGE_VERSION }}
         generate_release_notes: true
-        draft: true
+        draft: false
         prerelease: ${{ env.IS_RELEASE != 'True' }}
         files: ./artifacts/*.nupkg

src/SignalR.OpenApi/Generation/SignalROpenApiDocumentGenerator.cs

@@ -20,6 +20,7 @@ public sealed class SignalROpenApiDocumentGenerator : ISignalROpenApiDocumentGen
     private readonly SignalROpenApiOptions options;
     private readonly IServiceProvider serviceProvider;
     private readonly IReadOnlyList<ISignalROpenApiSchemaProcessor> schemaProcessors;
+    private readonly Dictionary<Type, string> schemaRegistry = [];
     private OpenApiDocument? currentDocument;
 
     /// <summary>
@@ -57,6 +58,7 @@ public OpenApiDocument GenerateDocument(IReadOnlyList<SignalRHubInfo> hubs)
         };
 
         this.currentDocument = document;
+        this.schemaRegistry.Clear();
 
         var requiresAuth = false;
 
@@ -510,12 +512,27 @@ private OpenApiSchema CreateSchemaForType(Type type)
             };
         }
 
-        // Complex object
-        return CreateObjectSchema(type);
+        // Complex object — check for circular references
+        if (this.schemaRegistry.TryGetValue(type, out var schemaName))
+        {
+            return new OpenApiSchema
+            {
+                Reference = new OpenApiReference
+                {
+                    Type = ReferenceType.Schema,
+                    Id = schemaName,
+                },
+            };
+        }
+
+        return this.CreateObjectSchema(type);
     }
 
     private OpenApiSchema CreateObjectSchema(Type type)
     {
+        var typeName = type.Name;
+        this.schemaRegistry[type] = typeName;
+
         var schema = new OpenApiSchema
         {
             Type = "object",
@@ -551,6 +568,11 @@ private OpenApiSchema CreateObjectSchema(Type type)
 
         this.ApplySchemaProcessors(schema, type);
 
+        if (this.currentDocument is not null)
+        {
+            this.currentDocument.Components.Schemas[typeName] = schema;
+        }
+
         return schema;
     }
 

test/SignalR.OpenApi.Tests/SignalROpenApiDocumentGeneratorTests.cs

@@ -654,6 +654,20 @@ public void GenerateDocument_FlattenedBody_FalseForPrimitiveParameters()
         Assert.IsFalse(flattenedBody, "flattenedBody should be false for primitive parameters.");
     }
 
+    /// <summary>
+    /// Verifies that self-referencing types do not cause a StackOverflowException.
+    /// </summary>
+    [TestMethod]
+    public void GenerateDocument_SelfReferencingType_DoesNotStackOverflow()
+    {
+        var (discoverer, generator) = CreateServices();
+        var hubs = discoverer.DiscoverHubs();
+        var doc = generator.GenerateDocument(hubs);
+
+        Assert.IsTrue(doc.Paths.ContainsKey("/hubs/CircularRef/ProcessNode"));
+        Assert.IsTrue(doc.Components.Schemas.ContainsKey("TreeNode"));
+    }
+
     private static (ReflectionHubDiscoverer Discoverer, SignalROpenApiDocumentGenerator Generator) CreateServices(
         Action<SignalROpenApiOptions>? configure = null)
     {

test/SignalR.OpenApi.Tests/TestHubs/CircularRefHub.cs

@@ -0,0 +1,21 @@
+// Copyright (c) SignalR.OpenApi Contributors. Licensed under the MIT License.
+
+using Microsoft.AspNetCore.SignalR;
+
+namespace SignalR.OpenApi.Tests.TestHubs;
+
+/// <summary>
+/// A hub for testing circular reference handling.
+/// </summary>
+public class CircularRefHub : Hub
+{
+    /// <summary>
+    /// Processes a tree node with self-referencing children.
+    /// </summary>
+    /// <param name="node">The tree node.</param>
+    /// <returns>The processed node.</returns>
+    public Task<TreeNode> ProcessNode(TreeNode node)
+    {
+        return Task.FromResult(node);
+    }
+}

test/SignalR.OpenApi.Tests/TestHubs/TreeNode.cs

@@ -0,0 +1,24 @@
+// Copyright (c) SignalR.OpenApi Contributors. Licensed under the MIT License.
+
+namespace SignalR.OpenApi.Tests.TestHubs;
+
+/// <summary>
+/// A tree node with self-referencing children.
+/// </summary>
+public class TreeNode
+{
+    /// <summary>
+    /// Gets or sets the name.
+    /// </summary>
+    public string Name { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Gets or sets the parent node (self-reference).
+    /// </summary>
+    public TreeNode? Parent { get; set; }
+
+    /// <summary>
+    /// Gets or sets the child nodes (self-referencing collection).
+    /// </summary>
+    public List<TreeNode>? Children { get; set; }
+}