fix(ci): bound integration network probes

Author: terion-labs

tests/integration-scenario-timeouts.test.ts

@@ -0,0 +1,14 @@
+import { readFileSync } from "node:fs";
+import { describe, expect, test } from "bun:test";
+
+describe("integration scenario timeout guardrails", () => {
+  const commonSource = readFileSync(new URL("./integration/scenarios/common.ts", import.meta.url), "utf8");
+
+  test("logs and bounds LXD API verification after identity-provider switches", () => {
+    expect(commonSource).toContain("verify ${host.label} LXD API");
+    expect(commonSource).toContain("verified ${host.label} LXD API");
+    expect(commonSource).toContain("external OIDC LXD API for ${host.label}");
+    expect(commonSource).toContain("local ZITADEL LXD API for ${host.label}");
+    expect(commonSource.match(/verifyLxdApi\(host, context\)/g)?.length).toBe(2);
+  });
+});

tests/integration/assertions/http.test.ts

@@ -1,5 +1,6 @@
+import { readFileSync } from "node:fs";
 import { describe, expect, test } from "bun:test";
-import { parseCurlHttpBodyResult } from "./http";
+import { CURL_PROCESS_TIMEOUT_MS, HTTP_FETCH_TIMEOUT_MS, parseCurlHttpBodyResult } from "./http";
 
 describe("HTTP assertion helpers", () => {
   test("uses curl's final status output instead of header-looking body content", () => {
@@ -22,4 +23,14 @@ describe("HTTP assertion helpers", () => {
     expect(() => parseCurlHttpBodyResult("", "body")).toThrow("valid final HTTP status");
     expect(() => parseCurlHttpBodyResult("HTTP/2 302", "body")).toThrow("valid final HTTP status");
   });
+
+  test("bounds every network attempt below the outer poll deadline", () => {
+    const source = readFileSync(new URL("./http.ts", import.meta.url), "utf8");
+
+    expect(HTTP_FETCH_TIMEOUT_MS).toBeLessThan(30000);
+    expect(CURL_PROCESS_TIMEOUT_MS).toBeGreaterThan(HTTP_FETCH_TIMEOUT_MS);
+    expect(source.match(/timeoutMs: CURL_PROCESS_TIMEOUT_MS/g)?.length).toBe(2);
+    expect(source.match(/await fetchWithTimeout\(url/g)?.length).toBe(1);
+    expect(source.match(/await fetchTextWithTimeout\(url/g)?.length).toBe(1);
+  });
 });

tests/integration/assertions/http.ts

@@ -11,6 +11,10 @@ type HttpAssertionOptions = {
 
 type JsonValidator = (value: unknown) => void;
 
+export const HTTP_FETCH_TIMEOUT_MS = 20000;
+export const CURL_PROCESS_TIMEOUT_MS = 30000;
+const HTTP_POLL_INTERVAL_MS = 5000;
+
 function curlResolveArgs(url: string, resolveIp?: string): string[] {
   if (!resolveIp) {
     return [];
@@ -29,21 +33,43 @@ export function parseCurlHttpBodyResult(stdout: string, body: string): { status:
   return { status, body };
 }
 
+async function withFetchTimeout<T>(task: (signal: AbortSignal) => Promise<T>): Promise<T> {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), HTTP_FETCH_TIMEOUT_MS);
+  try {
+    return await task(controller.signal);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+async function fetchWithTimeout(url: string, init: RequestInit): Promise<Response> {
+  return await withFetchTimeout((signal) => fetch(url, { ...init, signal }));
+}
+
+async function fetchTextWithTimeout(url: string, init: RequestInit): Promise<{ response: Response; body: string }> {
+  return await withFetchTimeout(async (signal) => {
+    const response = await fetch(url, { ...init, signal });
+    const body = await response.text();
+    return { response, body };
+  });
+}
+
 /** Polls an HTTP endpoint until it returns one of the expected status codes. */
 export async function waitForHttpStatus(url: string, expectedStatuses: number[], timeoutMs = 180000): Promise<Response> {
   const deadline = Date.now() + timeoutMs;
   let lastResponse: Response | null = null;
   while (Date.now() < deadline) {
     try {
-      const response = await fetch(url, { redirect: "manual" });
+      const response = await fetchWithTimeout(url, { redirect: "manual" });
       lastResponse = response;
       if (expectedStatuses.includes(response.status)) {
         return response;
       }
     } catch {
       // Ignore transient DNS/TLS startup errors while services converge.
     }
-    await Bun.sleep(5000);
+    await Bun.sleep(HTTP_POLL_INTERVAL_MS);
   }
   throw new Error(`timed out waiting for ${url} to return one of [${expectedStatuses.join(", ")}], last status: ${lastResponse?.status ?? "none"}`);
 }
@@ -70,7 +96,7 @@ async function readHttpsBody(url: string, options: HttpAssertionOptions = {}): P
       ...(options.insecure ? ["-k"] : []),
       ...curlResolveArgs(url, options.resolveIp),
       url
-    ]);
+    ], { timeoutMs: CURL_PROCESS_TIMEOUT_MS });
     if (result.exitCode !== 0) {
       throw new Error(result.stderr.trim() || result.stdout.trim() || `failed to fetch ${url}`);
     }
@@ -123,7 +149,7 @@ export async function waitForHttpStatusResolved(
       ...(insecure ? ["-k"] : []),
       ...curlResolveArgs(url, resolveIp),
       url
-    ]);
+    ], { timeoutMs: CURL_PROCESS_TIMEOUT_MS });
     const status = (result.stdout || "").trim();
     lastStatus = status || lastStatus;
     lastError = result.stderr.trim() || result.stdout.trim() || lastError;
@@ -135,7 +161,7 @@ export async function waitForHttpStatusResolved(
       }
     }
 
-    await Bun.sleep(5000);
+    await Bun.sleep(HTTP_POLL_INTERVAL_MS);
   }
 
   throw new Error(
@@ -168,8 +194,7 @@ export async function expectHttpBodyContains(
           return;
         }
       } else {
-        const response = await fetch(url, { redirect: "follow" });
-        const body = await response.text();
+        const { response, body } = await fetchTextWithTimeout(url, { redirect: "follow" });
         lastStatus = String(response.status);
         lastBody = body;
         if (body.includes(needle)) {
@@ -180,7 +205,7 @@ export async function expectHttpBodyContains(
       lastBody = String(error);
     }
 
-    await Bun.sleep(5000);
+    await Bun.sleep(HTTP_POLL_INTERVAL_MS);
   }
 
   const bodySnippet = lastBody.replace(/\s+/g, " ").trim().slice(0, 400);
@@ -217,7 +242,7 @@ export async function expectHttpsJson(
       lastError = error instanceof Error ? error.message : String(error);
     }
 
-    await Bun.sleep(5000);
+    await Bun.sleep(HTTP_POLL_INTERVAL_MS);
   }
 
   const bodySnippet = lastBody.replace(/\s+/g, " ").trim().slice(0, 400);

tests/integration/lib/process.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "bun:test";
-import { DEFAULT_LOCAL_PROCESS_ATTEMPTS, isRetryableLocalProcessFailure, run } from "./process";
+import { DEFAULT_LOCAL_PROCESS_ATTEMPTS, isRetryableLocalProcessFailure, run, runAllowFailure } from "./process";
 
 describe("integration process helpers", () => {
   test("renders stdout and stderr for failed commands", async () => {
@@ -14,4 +14,13 @@ describe("integration process helpers", () => {
     expect(isRetryableLocalProcessFailure({ exitCode: 127, stdout: "", stderr: "bash: missing-command: command not found" })).toBeFalse();
     expect(isRetryableLocalProcessFailure({ exitCode: 124, stdout: "", stderr: "EBADF: bad file descriptor, epoll_ctl" })).toBeFalse();
   });
+
+  test("terminates timed-out commands before resolving", async () => {
+    const startedAt = Date.now();
+    const result = await runAllowFailure(["bash", "-lc", "trap '' TERM; sleep 10"], { timeoutMs: 100 });
+
+    expect(result.exitCode).toBe(124);
+    expect(result.stderr).toContain("command timed out after 100ms");
+    expect(Date.now() - startedAt).toBeLessThan(5000);
+  });
 });

tests/integration/lib/process.ts

@@ -55,8 +55,10 @@ async function runOnceAllowFailure(cmd: string[], options: CommandOptions = {}):
   return await new Promise<CommandResult>((resolve) => {
     const stdout: Buffer[] = [];
     const stderr: Buffer[] = [];
+    const useProcessGroup = options.timeoutMs !== undefined && process.platform !== "win32";
     const proc = spawn(cmd[0] ?? "", cmd.slice(1), {
       cwd: options.cwd,
+      detached: useProcessGroup,
       env: options.env ? { ...process.env, ...options.env } : process.env,
       stdio: [options.stdin !== undefined ? "pipe" : "ignore", "pipe", "pipe"]
     });
@@ -80,6 +82,18 @@ async function runOnceAllowFailure(cmd: string[], options: CommandOptions = {}):
       resolve(result);
     };
 
+    const killProcess = (signal: NodeJS.Signals): void => {
+      if (useProcessGroup && proc.pid) {
+        try {
+          process.kill(-proc.pid, signal);
+          return;
+        } catch {
+          // Fall back to killing the direct child if process-group signaling is unavailable.
+        }
+      }
+      proc.kill(signal);
+    };
+
     proc.stdout?.on("data", (chunk: Buffer | string) => {
       stdout.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
     });
@@ -96,22 +110,24 @@ async function runOnceAllowFailure(cmd: string[], options: CommandOptions = {}):
     proc.on("close", (code, signal) => {
       const stderrText = Buffer.concat(stderr).toString("utf8");
       finish({
-        exitCode: code ?? (signal ? 128 + signalNumber(signal) : 1),
+        exitCode: timedOut ? 124 : code ?? (signal ? 128 + signalNumber(signal) : 1),
         stdout: Buffer.concat(stdout).toString("utf8"),
         stderr: timedOut ? `${stderrText}${stderrText ? "\n" : ""}command timed out after ${options.timeoutMs}ms` : stderrText
       });
     });
 
     if (options.timeoutMs !== undefined) {
       timeout = setTimeout(() => {
+        if (settled) {
+          return;
+        }
         timedOut = true;
-        proc.kill("SIGTERM");
-        killTimer = setTimeout(() => proc.kill("SIGKILL"), 2000);
-        finish({
-          exitCode: 124,
-          stdout: Buffer.concat(stdout).toString("utf8"),
-          stderr: `${Buffer.concat(stderr).toString("utf8")}${stderr.length > 0 ? "\n" : ""}command timed out after ${options.timeoutMs}ms`
-        });
+        killProcess("SIGTERM");
+        killTimer = setTimeout(() => {
+          if (!settled) {
+            killProcess("SIGKILL");
+          }
+        }, 2000);
       }, options.timeoutMs);
     }
 

tests/integration/scenarios/common.ts

@@ -324,7 +324,8 @@ export async function verifyManagementSurfaces(
 }
 
 /** Verifies the public LXD endpoint serves the real API over trusted TLS and does not expose trusted anonymous access. */
-export async function verifyLxdApi(host: ManagedHost): Promise<void> {
+export async function verifyLxdApi(host: ManagedHost, context?: IntegrationContext): Promise<void> {
+  context?.logger.info(`verify ${host.label} LXD API`);
   await expectHttpsJson(
     `https://${host.domains.lxd}/1.0`,
     (body) => {
@@ -351,6 +352,7 @@ export async function verifyLxdApi(host: ManagedHost): Promise<void> {
     },
     { timeoutMs: 300000, resolveIp: host.server.ipv4 }
   );
+  context?.logger.info(`verified ${host.label} LXD API`);
 }
 
 /** Verifies a real browser login through LXD's public OIDC flow. */
@@ -480,17 +482,19 @@ ${remoteCtl("set idp oidc")} \\
   --lxd-oidc-client ${shellArg(fixture.lxdClientId)} \\
   --admin-group ${shellArg(fixture.adminGroup)}`
   );
-  await verifyManagementSurfaces(context, host, fixture.adminUser);
-  await verifyLxdApi(host);
+  await withStepTimeout(`external OIDC management surfaces for ${host.label}`, 15 * 60 * 1000, () =>
+    verifyManagementSurfaces(context, host, fixture.adminUser)
+  );
+  await withStepTimeout(`external OIDC LXD API for ${host.label}`, 6 * 60 * 1000, () => verifyLxdApi(host, context));
 }
 
 /** Reconfigures the primary host back to local ZITADEL and validates its management UIs. */
 export async function switchBackToLocalIdp(context: IntegrationContext, host: ManagedHost): Promise<void> {
   const ssh = context.ssh(host);
   await runDetachedRemoteCommand(ssh, "switch-local-idp", remoteCtl("set idp local"));
   const admin = await readLocalZitadelAdmin(ssh);
-  await verifyManagementSurfaces(context, host, admin);
-  await verifyLxdApi(host);
+  await withStepTimeout(`local ZITADEL management surfaces for ${host.label}`, 15 * 60 * 1000, () => verifyManagementSurfaces(context, host, admin));
+  await withStepTimeout(`local ZITADEL LXD API for ${host.label}`, 6 * 60 * 1000, () => verifyLxdApi(host, context));
 }
 
 /** Runs a small route-auth matrix against the currently configured OIDC provider. */

tests/integration/scenarios/smoke.ts

@@ -71,7 +71,7 @@ export async function runSmokeSuite(context: IntegrationContext): Promise<void>
 
     const localAdmin = await readLocalZitadelAdmin(primarySsh);
     await verifyManagementSurfaces(context, primary, localAdmin);
-    await verifyLxdApi(primary);
+    await verifyLxdApi(primary, context);
 
     const plainRoute = `https://plain-${context.config.slug}.${rootDomain}:8080`;
     const authRoute = `https://auth-${context.config.slug}.${rootDomain}:8080@auth`;