fix(ci): hard-bound LXD API probes

Author: terion-labs

tests/integration-scenario-timeouts.test.ts

@@ -7,6 +7,8 @@ describe("integration scenario timeout guardrails", () => {
   test("logs and bounds LXD API verification after identity-provider switches", () => {
     expect(commonSource).toContain("verify ${host.label} LXD API");
     expect(commonSource).toContain("verified ${host.label} LXD API");
+    expect(commonSource).toContain("LXD API verification for ${host.label}");
+    expect(commonSource).toContain("timeoutMs: LXD_API_POLL_TIMEOUT_MS");
     expect(commonSource).toContain("external OIDC LXD API for ${host.label}");
     expect(commonSource).toContain("local ZITADEL LXD API for ${host.label}");
     expect(commonSource.match(/verifyLxdApi\(host, context\)/g)?.length).toBe(2);

tests/integration/lib/process.test.ts

@@ -23,4 +23,13 @@ describe("integration process helpers", () => {
     expect(result.stderr).toContain("command timed out after 100ms");
     expect(Date.now() - startedAt).toBeLessThan(5000);
   });
+
+  test("terminates timed-out process groups with inherited pipes", async () => {
+    const startedAt = Date.now();
+    const result = await runAllowFailure(["bash", "-lc", "sleep 10 & exit 0"], { timeoutMs: 100 });
+
+    expect(result.exitCode).toBe(124);
+    expect(result.stderr).toContain("command timed out after 100ms");
+    expect(Date.now() - startedAt).toBeLessThan(5000);
+  });
 });

tests/integration/lib/process.ts

@@ -67,6 +67,16 @@ async function runOnceAllowFailure(cmd: string[], options: CommandOptions = {}):
     let timedOut = false;
     let timeout: Timer | undefined;
     let killTimer: Timer | undefined;
+    let forceFinishTimer: Timer | undefined;
+
+    const timeoutResult = (): CommandResult => {
+      const stderrText = Buffer.concat(stderr).toString("utf8");
+      return {
+        exitCode: 124,
+        stdout: Buffer.concat(stdout).toString("utf8"),
+        stderr: `${stderrText}${stderrText ? "\n" : ""}command timed out after ${options.timeoutMs}ms`
+      };
+    };
 
     const finish = (result: CommandResult): void => {
       if (settled) {
@@ -79,6 +89,9 @@ async function runOnceAllowFailure(cmd: string[], options: CommandOptions = {}):
       if (killTimer) {
         clearTimeout(killTimer);
       }
+      if (forceFinishTimer) {
+        clearTimeout(forceFinishTimer);
+      }
       resolve(result);
     };
 
@@ -108,11 +121,10 @@ async function runOnceAllowFailure(cmd: string[], options: CommandOptions = {}):
       });
     });
     proc.on("close", (code, signal) => {
-      const stderrText = Buffer.concat(stderr).toString("utf8");
       finish({
         exitCode: timedOut ? 124 : code ?? (signal ? 128 + signalNumber(signal) : 1),
         stdout: Buffer.concat(stdout).toString("utf8"),
-        stderr: timedOut ? `${stderrText}${stderrText ? "\n" : ""}command timed out after ${options.timeoutMs}ms` : stderrText
+        stderr: timedOut ? timeoutResult().stderr : Buffer.concat(stderr).toString("utf8")
       });
     });
 
@@ -126,6 +138,16 @@ async function runOnceAllowFailure(cmd: string[], options: CommandOptions = {}):
         killTimer = setTimeout(() => {
           if (!settled) {
             killProcess("SIGKILL");
+            forceFinishTimer = setTimeout(() => {
+              if (settled) {
+                return;
+              }
+              proc.stdout?.destroy();
+              proc.stderr?.destroy();
+              proc.stdin?.destroy();
+              proc.unref();
+              finish(timeoutResult());
+            }, 1000);
           }
         }, 2000);
       }, options.timeoutMs);

tests/integration/scenarios/common.ts

@@ -40,6 +40,9 @@ type InstallOptions = {
   zitadelAdminEmail?: string;
 };
 
+const LXD_API_POLL_TIMEOUT_MS = 90 * 1000;
+const LXD_API_VERIFY_TIMEOUT_MS = 2 * 60 * 1000;
+
 function baseEmail(ctx: IntegrationContext): string {
   return `terrarium+${ctx.config.slug}@${ctx.config.ipDnsDomain}`;
 }
@@ -325,34 +328,36 @@ export async function verifyManagementSurfaces(
 
 /** Verifies the public LXD endpoint serves the real API over trusted TLS and does not expose trusted anonymous access. */
 export async function verifyLxdApi(host: ManagedHost, context?: IntegrationContext): Promise<void> {
-  context?.logger.info(`verify ${host.label} LXD API`);
-  await expectHttpsJson(
-    `https://${host.domains.lxd}/1.0`,
-    (body) => {
-      if (!isObject(body)) {
-        throw new Error("LXD API root did not return an object");
-      }
-
-      const metadata = body.metadata;
-      if (!isObject(metadata)) {
-        throw new Error("LXD API root did not include metadata");
-      }
-
-      if (!Array.isArray(metadata.api_extensions)) {
-        throw new Error("LXD API root did not include api_extensions");
-      }
-
-      const auth = typeof metadata.auth === "string" ? metadata.auth.toLowerCase() : "";
-      if (!auth) {
-        throw new Error("LXD API root did not include auth state");
-      }
-      if (auth === "trusted") {
-        throw new Error("LXD API root allowed trusted anonymous access");
-      }
-    },
-    { timeoutMs: 300000, resolveIp: host.server.ipv4 }
-  );
-  context?.logger.info(`verified ${host.label} LXD API`);
+  await withStepTimeout(`LXD API verification for ${host.label}`, LXD_API_VERIFY_TIMEOUT_MS, async () => {
+    context?.logger.info(`verify ${host.label} LXD API`);
+    await expectHttpsJson(
+      `https://${host.domains.lxd}/1.0`,
+      (body) => {
+        if (!isObject(body)) {
+          throw new Error("LXD API root did not return an object");
+        }
+
+        const metadata = body.metadata;
+        if (!isObject(metadata)) {
+          throw new Error("LXD API root did not include metadata");
+        }
+
+        if (!Array.isArray(metadata.api_extensions)) {
+          throw new Error("LXD API root did not include api_extensions");
+        }
+
+        const auth = typeof metadata.auth === "string" ? metadata.auth.toLowerCase() : "";
+        if (!auth) {
+          throw new Error("LXD API root did not include auth state");
+        }
+        if (auth === "trusted") {
+          throw new Error("LXD API root allowed trusted anonymous access");
+        }
+      },
+      { timeoutMs: LXD_API_POLL_TIMEOUT_MS, resolveIp: host.server.ipv4 }
+    );
+    context?.logger.info(`verified ${host.label} LXD API`);
+  });
 }
 
 /** Verifies a real browser login through LXD's public OIDC flow. */