Fix circular-ref when navigating paging properties (#7480)

issue: #7436

still validating if this will solve the issue

---------

Co-authored-by: Alitzel Mendez <[email protected]>
Co-authored-by: Timothee Guerin <[email protected]>

Author: 3 people

.chronus/changes/inifite-loop-2025-4-27-14-20-40.md

@@ -0,0 +1,7 @@
+---
+changeKind: fix
+packages:
+  - "@typespec/compiler"
+---
+
+Fix infinite recursion when navigating paging properties by detecting and handling circular model references.

packages/compiler/src/lib/paging.ts

@@ -298,21 +298,24 @@ function navigateProperties(
   type: Type,
   callback: (prop: ModelProperty, path: ModelProperty[]) => void,
   path: ModelProperty[] = [],
+  visited: Set<Type> = new Set(),
 ): void {
+  if (visited.has(type)) return;
+  visited.add(type);
   switch (type.kind) {
     case "Model":
       for (const prop of type.properties.values()) {
         callback(prop, [...path, prop]);
-        navigateProperties(prop.type, callback, [...path, prop]);
+        navigateProperties(prop.type, callback, [...path, prop], visited);
       }
       break;
     case "Union":
       for (const member of type.variants.values()) {
-        navigateProperties(member, callback, path);
+        navigateProperties(member, callback, path, visited);
       }
       break;
     case "UnionVariant":
-      navigateProperties(type.type, callback, path);
+      navigateProperties(type.type, callback, path, visited);
       break;
   }
 }

packages/compiler/test/decorators/paging.test.ts

@@ -1,7 +1,7 @@
 import { beforeEach, describe, expect, it } from "vitest";
 import { ignoreDiagnostics, ModelProperty, Operation } from "../../src/index.js";
 import { getPagingOperation, PagingOperation } from "../../src/lib/paging.js";
-import { expectDiagnostics } from "../../src/testing/expect.js";
+import { expectDiagnosticEmpty, expectDiagnostics } from "../../src/testing/expect.js";
 import { createTestRunner } from "../../src/testing/test-host.js";
 import { BasicTestRunner } from "../../src/testing/types.js";
 
@@ -39,7 +39,20 @@ it("emit error if missing pageItems property", async () => {
   });
 });
 
-describe("emit conflict diagnostic if multiple properties are annotated with teh same property marker", () => {
+it("@list decorator handle recursive models without infinite loop", async () => {
+  const diagnostics = await runner.diagnose(`
+      model MyPage {
+        selfRef?: MyPage;
+        @pageItems items: string[];
+        @nextLink next: string;
+      }
+
+      @list op foo(): MyPage;
+    `);
+  expectDiagnosticEmpty(diagnostics);
+});
+
+describe("emit conflict diagnostic if multiple properties are annotated with the same property marker", () => {
   it.each([
     ["offset", "int32"],
     ["pageSize", "int32"],