fix(openapi): handle recursive PEP 695 type aliases in schema generation

`SchemaCreator.for_type_alias_type` unwrapped a `type` alias's `__value__`
and recursed with no cycle guard, so a self-referential alias such as
`type JSON = None | bool | str | list[JSON] | dict[str, JSON]` blew the
stack with a RecursionError when building the OpenAPI document. Litestar
already guards self-referential *models* via a registered `$ref`, but that
guard was never extended to the type-alias path added in #3715.

A self-referential alias is now registered as a named component and its
recursive occurrences resolve to a `$ref`, mirroring how recursive models
are handled; non-recursive aliases are still inlined. Mutually recursive
aliases terminate as well: the cycle entry point becomes the component, and
aliases reached while expanding it are inlined. When an alias's value is
itself a single component (e.g. `type Node = Container` where `Container`
refers back to `Node`), the component defers to it via `allOf`.

`_get_normalized_schema_key` now keys `TypeAliasType`, which exposes
`__name__` but no `__qualname__`.

Because schemas for recursive aliases now generate, example generation
becomes reachable for them and would itself recurse without bound (the
example factory guards recursive models but not aliases). Example
generation is now skipped for any annotation that is, or contains, a
self-referential alias; non-recursive aliases reused within a single
annotation are unaffected.

Fixes #4843

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: declaresub

litestar/_openapi/datastructures.py

@@ -64,8 +64,12 @@ def _get_normalized_schema_key(field_definition: FieldDefinition) -> tuple[str,
         return (override,)
 
     annotation = field_definition.annotation
-    module = getattr(annotation, "__module__", "")
-    name = str(annotation)[len(module) + 1 :] if isinstance(annotation, _GenericAlias) else annotation.__qualname__
+    module = getattr(annotation, "__module__", None) or ""
+    if isinstance(annotation, _GenericAlias):
+        name = str(annotation)[len(module) + 1 :]
+    else:
+        # ``TypeAliasType`` (PEP 695 / ``typing_extensions``) exposes ``__name__`` but no ``__qualname__``.
+        name = getattr(annotation, "__qualname__", None) or annotation.__name__
     name = name.replace(".<locals>.", ".")
     return *module.split("."), re.sub(INVALID_KEY_CHARACTER_PATTERN, "_", name)
 

litestar/_openapi/schema_generation/examples.py

@@ -16,6 +16,7 @@
 from litestar.openapi.spec import Example
 from litestar.plugins.pydantic.utils import is_pydantic_model_instance
 from litestar.types import Empty
+from litestar.typing import TypeAliasTypes
 
 if TYPE_CHECKING:
     from litestar.typing import FieldDefinition
@@ -58,6 +59,51 @@ def _normalize_example_value(value: Any) -> Any:
     return value
 
 
+def _type_alias_is_recursive(alias: Any) -> bool:
+    """Return whether expanding ``alias`` would recurse, i.e. the alias is reachable from its own value.
+
+    This covers both directly self-referential aliases (``type JSON = ... | list[JSON]``) and mutually
+    recursive ones (``type A = B`` / ``type B = A``), where ``alias`` is reached again via another alias.
+    """
+    target = id(alias)
+    seen: set[int] = set()
+    stack = [alias.__value__]
+    while stack:
+        obj = stack.pop()
+        if id(obj) in seen:
+            continue
+        seen.add(id(obj))
+        if isinstance(obj, TypeAliasTypes):
+            if id(obj) == target:
+                return True
+            stack.append(obj.__value__)
+        stack.extend(get_args(obj))
+    return False
+
+
+def _contains_recursive_type_alias(annotation: Any) -> bool:
+    """Return whether ``annotation`` is, or contains, a self-referential PEP 695 ``type`` alias.
+
+    The example factory does not guard against recursive ``type`` aliases (it does for recursive models) and
+    would recurse without bound while building an example value for one, e.g. ``dict[str, JSON]`` where
+    ``type JSON = ... | dict[str, JSON]``. This detects such aliases so example generation can be skipped.
+    A non-recursive alias reused several times in one annotation (``tuple[Name, Name]``) is *not* flagged.
+    """
+    seen: set[int] = set()
+    stack = [annotation]
+    while stack:
+        obj = stack.pop()
+        if id(obj) in seen:
+            continue
+        seen.add(id(obj))
+        if isinstance(obj, TypeAliasTypes):
+            if _type_alias_is_recursive(obj):
+                return True
+            stack.append(obj.__value__)
+        stack.extend(get_args(obj))
+    return False
+
+
 def _create_field_meta(field: FieldDefinition) -> FieldMeta:
     return FieldMeta.from_type(
         annotation=field.annotation,
@@ -75,6 +121,9 @@ def create_examples_for_field(field: FieldDefinition) -> list[Example]:
     Returns:
         A list including a single example.
     """
+    if _contains_recursive_type_alias(field.annotation):
+        # the example factory would recurse without bound on a self-referential ``type`` alias
+        return []
     try:
         field_meta = _create_field_meta(replace(field, annotation=_normalize_example_value(field.annotation)))
         value = ExampleFactory.get_field_value(field_meta)

litestar/_openapi/schema_generation/schema.py

@@ -5,6 +5,7 @@
 from collections import OrderedDict, defaultdict, deque
 from collections.abc import Hashable, Iterable, Mapping, MutableMapping, MutableSequence, Sequence
 from copy import copy
+from dataclasses import fields
 from datetime import date, datetime, time, timedelta
 from decimal import Decimal
 from enum import Enum
@@ -219,7 +220,14 @@ def create_schema_for_annotation(annotation: Any) -> Schema:
 
 
 class SchemaCreator:
-    __slots__ = ("generate_examples", "plugins", "prefer_alias", "schema_registry")
+    __slots__ = (
+        "_in_progress_alias_ids",
+        "_recursive_alias_ids",
+        "generate_examples",
+        "plugins",
+        "prefer_alias",
+        "schema_registry",
+    )
 
     def __init__(
         self,
@@ -240,6 +248,10 @@ def __init__(
         self.plugins = plugins if plugins is not None else []
         self.prefer_alias = prefer_alias
         self.schema_registry = schema_registry or SchemaRegistry()
+        # ids of ``TypeAliasType`` annotations currently being expanded, used to detect self-references
+        self._in_progress_alias_ids: set[int] = set()
+        # ids of ``TypeAliasType`` annotations found to be self-referential during expansion
+        self._recursive_alias_ids: set[int] = set()
 
     @classmethod
     def from_openapi_context(cls, context: OpenAPIContext, prefer_alias: bool = True, **kwargs: Any) -> Self:
@@ -343,14 +355,58 @@ def for_new_type(self, field_definition: FieldDefinition) -> Schema | Reference:
         )
 
     def for_type_alias_type(self, field_definition: FieldDefinition) -> Schema | Reference:
-        return self.for_field_definition(
-            FieldDefinition.from_kwarg(
-                annotation=field_definition.annotation.__value__,
-                name=field_definition.name,
-                default=field_definition.default,
-                kwarg_definition=field_definition.kwarg_definition,
+        """Create a schema for a PEP 695 ``type`` alias (``TypeAliasType``).
+
+        The alias's ``__value__`` is unwrapped and a schema is created for it. Self-referential aliases,
+        e.g. ``type JSON = str | int | list[JSON] | dict[str, JSON]``, are registered as named components so
+        that recursive occurrences resolve to a ``$ref`` -- mirroring how self-referential models are handled
+        (see :meth:`create_component_schema`) and breaking what would otherwise be unbounded recursion.
+        Non-recursive aliases are inlined as before.
+
+        Note:
+            Only the alias that is the entry point of a cycle becomes a component; any *mutually* recursive
+            aliases reached while expanding it are inlined. The result is always valid and finite, but for
+            mutually recursive aliases used at several independent entry points the emitted components are not
+            minimal (each inlines a copy of the other's value).
+        """
+        alias_id = id(field_definition.annotation)
+
+        if alias_id in self._in_progress_alias_ids:
+            # We are already expanding this alias higher up the call stack: this is a recursive reference.
+            # Register it as a component (if not already) and return a reference to break the cycle.
+            self._recursive_alias_ids.add(alias_id)
+            self.schema_registry.get_schema_for_field_definition(field_definition)
+            return self.schema_registry.get_reference_for_field_definition(field_definition)  # type: ignore[return-value]
+
+        self._in_progress_alias_ids.add(alias_id)
+        try:
+            schema = self.for_field_definition(
+                FieldDefinition.from_kwarg(
+                    annotation=field_definition.annotation.__value__,
+                    name=field_definition.name,
+                    default=field_definition.default,
+                    kwarg_definition=field_definition.kwarg_definition,
+                )
             )
-        )
+        finally:
+            self._in_progress_alias_ids.discard(alias_id)
+
+        if alias_id not in self._recursive_alias_ids:
+            # No self-reference was encountered while expanding the value: inline the schema as before.
+            return schema
+
+        self._recursive_alias_ids.discard(alias_id)
+        # The alias was registered as a component while expanding its value (the recursive references above
+        # already point at it). Populate that component with the resolved schema and return a reference to it.
+        component = self.schema_registry.get_schema_for_field_definition(field_definition)
+        if isinstance(schema, Schema):
+            for schema_field in fields(Schema):
+                setattr(component, schema_field.name, getattr(schema, schema_field.name))
+        else:
+            # the value resolved to a reference (e.g. ``type Alias = SomeModel``); defer to it
+            component.all_of = [schema]
+        component.title = field_definition.annotation.__name__
+        return self.schema_registry.get_reference_for_field_definition(field_definition) or component
 
     @staticmethod
     def for_upload_file(field_definition: FieldDefinition) -> Schema:

tests/unit/test_openapi/test_schema.py

@@ -34,6 +34,7 @@
 from litestar.di import NamedDependency, Provide
 from litestar.enums import ParamType
 from litestar.exceptions import ImproperlyConfiguredException
+from litestar.openapi import OpenAPIConfig
 from litestar.openapi.spec import ExternalDocumentation, OpenAPIType, Reference
 from litestar.openapi.spec.example import Example
 from litestar.openapi.spec.parameter import Parameter as OpenAPIParameter
@@ -837,6 +838,136 @@ def handler(query_param: Annotated[annotation, QueryParameter(description="foo")
     assert param.description == "foo"
 
 
+@pytest.mark.skipif(sys.version_info < (3, 12), reason="type keyword not available before 3.12")
+def test_recursive_type_alias_type() -> None:
+    # https://github.com/litestar-org/litestar/issues/4843
+    # A self-referential PEP 695 ``type`` alias must not blow the stack when generating the OpenAPI schema.
+    ctx: dict[str, Any] = {"__name__": __name__}
+    exec("type JSON = None | bool | str | float | int | list[JSON] | dict[str, JSON]", ctx, None)
+    annotation = ctx["JSON"]
+
+    @get("/")
+    def handler() -> annotation:  # type: ignore[valid-type]
+        return {}
+
+    app = Litestar([handler])
+    schema = app.openapi_schema.to_schema()
+
+    # the alias is registered as a named component and the response references it
+    assert "JSON" in schema["components"]["schemas"]
+    response_schema = schema["paths"]["/"]["get"]["responses"]["200"]["content"]["application/json"]["schema"]
+    assert response_schema == {"$ref": "#/components/schemas/JSON"}
+
+    # the recursive occurrences inside the component resolve to a ``$ref`` back to the alias, breaking the cycle
+    component = schema["components"]["schemas"]["JSON"]
+    one_of = component["oneOf"]
+    array_schema = next(s for s in one_of if s.get("type") == "array")
+    object_schema = next(s for s in one_of if s.get("type") == "object")
+    assert array_schema["items"] == {"$ref": "#/components/schemas/JSON"}
+    assert object_schema["additionalProperties"] == {"$ref": "#/components/schemas/JSON"}
+
+
+@pytest.mark.skipif(sys.version_info < (3, 12), reason="type keyword not available before 3.12")
+def test_mutually_recursive_type_alias_types() -> None:
+    # Mutually recursive aliases must terminate as well, not just directly self-referential ones.
+    ctx: dict[str, Any] = {"__name__": __name__}
+    exec("type A = str | list[B]\ntype B = int | dict[str, A]", ctx, None)
+    annotation = ctx["A"]
+
+    @get("/")
+    def handler() -> annotation:  # type: ignore[valid-type]
+        return ""
+
+    app = Litestar([handler])
+    # smoke test: this must not raise ``RecursionError``
+    schema = app.openapi_schema.to_schema()
+    response_schema = schema["paths"]["/"]["get"]["responses"]["200"]["content"]["application/json"]["schema"]
+    assert response_schema == {"$ref": "#/components/schemas/A"}
+
+
+@pytest.mark.skipif(sys.version_info < (3, 12), reason="type keyword not available before 3.12")
+def test_type_alias_type_to_recursive_model() -> None:
+    # An alias whose value is a single component (here a dataclass) that recurses back into the alias: the alias's
+    # value resolves to a ``Reference`` rather than a ``Schema``, so the component defers to it via ``allOf``.
+    # ``exec`` into the module globals so the dataclass's forward reference to the alias can be resolved.
+    exec(
+        "@dataclass\nclass _AliasContainer:\n    children: 'list[_AliasNode]'\ntype _AliasNode = _AliasContainer",
+        globals(),
+    )
+    annotation = globals()["_AliasNode"]
+
+    @get("/")
+    def handler() -> annotation:  # type: ignore[valid-type]
+        ...
+
+    app = Litestar([handler])
+    schema = app.openapi_schema.to_schema()
+    schemas = schema["components"]["schemas"]
+    assert schema["paths"]["/"]["get"]["responses"]["200"]["content"]["application/json"]["schema"] == {
+        "$ref": "#/components/schemas/_AliasNode"
+    }
+    # the ``_AliasNode`` component defers to the ``_AliasContainer`` component
+    assert schemas["_AliasNode"]["allOf"] == [{"$ref": "#/components/schemas/_AliasContainer"}]
+    # and ``_AliasContainer`` references ``_AliasNode`` back, completing the cycle without recursing infinitely
+    assert schemas["_AliasContainer"]["properties"]["children"]["items"] == {"$ref": "#/components/schemas/_AliasNode"}
+
+
+@pytest.mark.skipif(sys.version_info < (3, 12), reason="type keyword not available before 3.12")
+def test_recursive_type_alias_type_with_example_generation() -> None:
+    # https://github.com/litestar-org/litestar/issues/4843
+    # The example factory does not guard against recursive ``type`` aliases, so example generation must be
+    # skipped for them -- otherwise it recurses without bound (here ``dict[str, JSON]`` is the dangerous shape,
+    # processed after the alias has finished expanding).
+    ctx: dict[str, Any] = {"__name__": __name__}
+    exec("type JSON = None | bool | str | float | int | list[JSON] | dict[str, JSON]", ctx, None)
+    annotation = ctx["JSON"]
+
+    @get("/")
+    def handler() -> dict[str, annotation]:  # type: ignore[valid-type]
+        return {}
+
+    app = Litestar([handler], openapi_config=OpenAPIConfig(title="t", version="1", create_examples=True))
+    # must terminate (no ``RecursionError`` / hang) -- a non-recursive annotation containing the alias
+    schema = app.openapi_schema.to_schema()
+    component = schema["components"]["schemas"]["JSON"]
+    # example generation was skipped for the self-referential shapes, so no value was fabricated for them
+    assert component.get("examples", []) == []
+    array_member = next(s for s in component["oneOf"] if s.get("type") == "array")
+    assert array_member.get("examples", []) == []
+
+
+@pytest.mark.skipif(sys.version_info < (3, 12), reason="type keyword not available before 3.12")
+def test_contains_recursive_type_alias_detection() -> None:
+    from litestar._openapi.schema_generation.examples import _contains_recursive_type_alias
+
+    # the composite annotations are built inside ``exec`` so they are not analysed as type expressions
+    ctx: dict[str, Any] = {"__name__": __name__}
+    exec(
+        "type Name = str\n"
+        "type JSON = None | str | list[JSON]\n"
+        "type Wrapper = list[JSON]\n"
+        "type A = str | list[B]\n"
+        "type B = int | dict[str, A]\n"
+        "dict_str_json = dict[str, JSON]\n"
+        "tuple_name_name = tuple[Name, Name]\n"
+        "dict_name_name = dict[Name, Name]",
+        ctx,
+        None,
+    )
+
+    # directly recursive, mutually recursive, and aliases that *contain* a recursive one
+    assert _contains_recursive_type_alias(ctx["JSON"]) is True
+    assert _contains_recursive_type_alias(ctx["dict_str_json"]) is True
+    assert _contains_recursive_type_alias(ctx["Wrapper"]) is True
+    assert _contains_recursive_type_alias(ctx["A"]) is True
+    assert _contains_recursive_type_alias(ctx["B"]) is True
+    # a non-recursive alias is not flagged -- even when reused several times in one annotation
+    assert _contains_recursive_type_alias(ctx["Name"]) is False
+    assert _contains_recursive_type_alias(ctx["tuple_name_name"]) is False
+    assert _contains_recursive_type_alias(ctx["dict_name_name"]) is False
+    assert _contains_recursive_type_alias(int) is False
+
+
 def test_decimal_schema_type() -> None:
     from litestar._openapi.schema_generation.schema import create_schema_for_annotation