feat(destroy): graceful SIGINT handling — release lock + preserve state on interrupt (#826)

Author: go-to-k

docs/_generated/integ-last-run.tsv

@@ -119,7 +119,7 @@ kms-encryption	2026-06-10T10:20:55Z	PASS		standard	coverage spread; 3 deleted 0
 sns-sqs-event	2026-06-10T10:20:55Z	PASS		verify.sh	coverage spread; 19 deleted 0 errors
 vpc-lambda	2026-06-10T10:20:55Z	PASS		standard	coverage spread; 16 deleted 0 errors, VPC/ENI clean
 drift-revert-vpc	2026-06-11T06:05:51Z	PASS		verify.sh	order-normalize backport; 21 deleted 0 err 0 orphan
-microservices	2026-06-13T04:12:49Z	PASS	39	standard	#804 incremental destroy persistence; 19 deleted 0 err 0 orphan
+microservices	2026-06-13T06:25:57Z	PASS	34	standard	#816 SIGINT handler; normal destroy unaffected; 19 deleted 0 err 0 orphan
 lambda	2026-06-13T04:41:51Z	PASS	76	verify.sh	#808 broad integ + cdkd events live-tested (deploy+destroy runs persisted); 9 deleted 0 err
 vpc-nat-gateway	2026-06-13T05:15:45Z	PASS	328	standard	#817 IGW/NAT delete-order; 21 deleted 0 err 0 orphan (NAT before IGW/EIP)
 cross-region-state-bucket	2026-06-13T06:16:49Z	PASS	34	verify.sh	#819 exports-index region-corrected client; no 301; 1 deleted 0 err, temp bucket cleaned

docs/changelog-cdkd.md

@@ -592,6 +592,28 @@ Default: **15 minutes**
 
 Even if a process crashes, after 15 minutes the old lock is considered stale and can be force released.
 
+### Destroy interruption (Ctrl-C)
+
+`cdkd destroy` and `cdkd state destroy` handle the first `Ctrl-C` (SIGINT)
+gracefully (issue [#816](https://github.com/go-to-k/cdkd/issues/816)),
+mirroring Terraform:
+
+- **First Ctrl-C** stops scheduling new deletes. Any provider delete already
+  in flight is allowed to finish (it is not cancelled). The runner then flushes
+  the incremental destroy state (the same per-resource save-chain that powers
+  the partial-failure path — see "Incremental destroy persistence" below), so
+  the preserved `state.json` lists only the resources that still exist.
+  Finally it **releases the stack lock** and the command exits non-zero. A
+  re-run of `cdkd destroy` resumes cleanly with no replay and no wait for the
+  lock TTL.
+- **Second Ctrl-C** force-quits immediately (`process.exit(130)`) without
+  waiting for the in-flight delete. In that case the lock may be left behind
+  and is reclaimed after the TTL above (or cleared with `cdkd force-unlock`).
+
+This is why an interrupted destroy no longer strands the lock for its full
+TTL: only an ungraceful kill (`SIGKILL`, a second Ctrl-C, or a crash) leaves a
+stale lock.
+
 ## State Saving and Updating
 
 ### Initial Save (New Stack)

docs/state-management.md

@@ -31,6 +31,28 @@ Locked by: user@hostname:12345, operation: deploy
 - Another process is deploying the same stack
 - Previous process crashed and lock remains
 
+> **Note:** A first `Ctrl-C` during `cdkd destroy` / `cdkd state destroy` no
+> longer strands the lock — the graceful-SIGINT handler (issue
+> [#816](https://github.com/go-to-k/cdkd/issues/816)) finishes any in-flight
+> delete, flushes the incremental state, and **releases the lock** before
+> exiting non-zero. A re-run resumes immediately without waiting out the lock
+> TTL.
+>
+> A **second** `Ctrl-C` force-quits immediately (`exit 130`) without waiting
+> for the in-flight delete. Because the force-quit path cannot run the normal
+> lock-release cleanup, it fires a **best-effort** (un-awaited) lock release
+> AND prints the exact recovery command to stderr:
+>
+> ```text
+> Force-quit: stack lock may not be released. If the next run reports a lock, run: cdkd force-unlock MyStack
+> ```
+>
+> The best-effort release usually lands before the process dies, so most
+> force-quits leave no lock; if a subsequent run reports a lock, run the
+> printed `cdkd force-unlock <stackName>` (or the steps below) to clear it. A
+> leftover lock therefore means an ungraceful kill (`SIGKILL`, a force-quit
+> whose best-effort release did not complete, or a crash).
+
 #### Solutions
 
 **1. Check if another process is running**

docs/troubleshooting.md

@@ -175,6 +175,14 @@ export interface DestroyRunnerResult {
   retainedCount: number;
   /** Number of resources that failed to delete. State is preserved on >0 errors. */
   errorCount: number;
+  /**
+   * True when a graceful SIGINT (issue #816) stopped the destroy early. The
+   * in-flight deletes finished, the (trimmed) state was preserved, and the
+   * lock was released — but resources may remain, so the caller surfaces a
+   * non-zero exit. Distinct from `errorCount > 0` (a resource actually failed
+   * to delete): an interrupt is a user-requested stop, not a failure.
+   */
+  interrupted: boolean;
 }
 
 /**
@@ -303,6 +311,7 @@ export async function runDestroyForStack(
     deletedCount: 0,
     retainedCount: 0,
     errorCount: 0,
+    interrupted: false,
   };
 
   const resourceCount = Object.keys(state.resources).length;
@@ -523,6 +532,66 @@ export async function runDestroyForStack(
   const renderer = getLiveRenderer();
   renderer.start();
 
+  // Graceful SIGINT handling (issue #816, Terraform parity). The first
+  // Ctrl-C flips `draining` true: the reverse-DAG delete loop below stops
+  // SCHEDULING new deletes (it checks the flag before each level and before
+  // dispatching each resource), but the already-dispatched in-flight
+  // `provider.delete` calls in the current level are awaited to completion.
+  // Control then falls through to the `finally` block, which flushes the
+  // incremental save-chain (issue #804) — leaving a clean, minimal preserved
+  // state — and releases the stack lock. Without this the process would die
+  // mid-destroy, skip the `finally`, and strand the lock for its 30m TTL.
+  //
+  // A SECOND Ctrl-C bypasses graceful shutdown entirely (`process.exit(130)`)
+  // — the user has decided not to wait for the in-flight call.
+  //
+  // The handler reads/writes ONLY this call's closure state, and is removed in
+  // the `finally` below, so no listener leaks across stacks. Nested-stack
+  // destroys recurse into `runDestroyForStack`, registering one handler per
+  // level — Node delivers SIGINT to every listener, so the first Ctrl-C drains
+  // the parent AND every in-flight child, which is the intended behavior.
+  let draining = false;
+  const sigintHandler = (): void => {
+    if (draining) {
+      // Second Ctrl-C: force-quit without waiting for the in-flight delete.
+      // The synchronous `process.exit(130)` bypasses the `finally` below,
+      // so the stack lock is NOT released through the normal path (issue
+      // #816). Fire a best-effort, un-awaited release first — it MAY land
+      // before the process dies on a fast network — but always print the
+      // exact recovery command so the user can recover deterministically if
+      // it does not (a force-quit leaving a stranded lock would otherwise
+      // re-introduce the 30m-TTL wait this issue fixes, just on this path).
+      void ctx.lockManager.releaseLock(stackName, regionForState).catch(() => {
+        /* best-effort: the recovery line below is the real guarantee */
+      });
+      process.stderr.write(
+        `\nForce-quit: stack lock may not be released. If the next run reports a lock, run: ` +
+          `cdkd force-unlock ${stackName}\n`
+      );
+      process.exit(130);
+    }
+    draining = true;
+    // Route the notice through the live renderer so it doesn't collide with
+    // the in-flight task display.
+    renderer.printAbove(() => {
+      process.stderr.write(
+        '\nInterrupted — finishing in-flight deletes, then flushing state and releasing the lock ' +
+          '(press Ctrl-C again to force-quit)...\n'
+      );
+    });
+  };
+  // Each nested-stack level recurses into `runDestroyForStack` and registers
+  // its own SIGINT listener, and each in-flight provider that installs its own
+  // SIGINT handler (CustomResource / CloudFront / ACM / Route53) adds one more.
+  // Deep nesting + high `--concurrency` can legitimately exceed Node's default
+  // 10-listener cap and emit a scary MaxListenersExceededWarning that is NOT a
+  // leak (every listener is removed in its own `finally`). Raise the ceiling
+  // with generous headroom for real fan-out while still leaving the warning
+  // active above it so an ACTUAL listener leak is not masked. `Math.max` keeps
+  // this safe under recursion (never lowers an already-raised limit).
+  process.setMaxListeners(Math.max(process.getMaxListeners(), 100));
+  process.on('SIGINT', sigintHandler);
+
   try {
     logger.info('Building dependency graph...');
 
@@ -584,6 +653,15 @@ export async function runDestroyForStack(
 
     // Process levels in reverse order for deletion.
     for (let levelIndex = executionLevels.length - 1; levelIndex >= 0; levelIndex--) {
+      // Graceful SIGINT (issue #816): once draining, do not start a new
+      // deletion level. Any level already in flight finished via its own
+      // `Promise.all` below; remaining levels are left untouched and their
+      // resources stay in the preserved state for a clean re-run.
+      if (draining) {
+        logger.debug('Interrupted (draining) — not scheduling further deletion levels');
+        break;
+      }
+
       const level = executionLevels[levelIndex];
       if (!level) continue;
 
@@ -594,6 +672,14 @@ export async function runDestroyForStack(
       const stackRegion = state.region ?? ctx.baseRegion;
 
       const deletePromises = level.map(async (logicalId) => {
+        // Graceful SIGINT (issue #816): if the interrupt landed after this
+        // level's promises were created but before this resource's delete was
+        // dispatched, skip it. It stays in the preserved state for re-run.
+        // (Deletes already in flight when the interrupt arrives are NOT
+        // cancelled — they run to completion; only not-yet-dispatched ones
+        // bail here.)
+        if (draining) return;
+
         const resource = state.resources[logicalId];
         if (!resource) {
           logger.warn(`Resource ${logicalId} not found in state, skipping`);
@@ -826,12 +912,21 @@ export async function runDestroyForStack(
       await Promise.all(deletePromises);
     }
 
+    // Carry the graceful-interrupt outcome (issue #816) into the result so the
+    // CLI surfaces a non-zero exit. Read AFTER the level loop so a SIGINT that
+    // arrived while the final level was draining is still observed.
+    result.interrupted = draining;
+
     // Flush pending incremental persists BEFORE the final state decision so
     // a chained write can never land after deleteState and re-create the
     // state file. The chain never rejects (each link catches internally).
     await saveChain;
 
-    if (result.errorCount === 0) {
+    // Preserve state (rather than delete it) when there were delete errors OR
+    // the destroy was gracefully interrupted (issue #816). An interrupt leaves
+    // not-yet-deleted resources, so deleting the state file would orphan them.
+    const preserveState = result.errorCount > 0 || result.interrupted;
+    if (!preserveState) {
       await ctx.stateBackend.deleteState(stackName, regionForState);
       logger.debug('State deleted');
       // Drop this stack's entries from the exports index so the next
@@ -856,26 +951,42 @@ export async function runDestroyForStack(
             `The state file may still list already-deleted resources; a re-run resolves them idempotently.`
         );
       }
-      logger.warn(`${result.errorCount} resource(s) failed to delete. State preserved.`);
+      if (result.interrupted) {
+        logger.warn(
+          `Destroy interrupted — ${Object.keys(remainingResources).length} resource(s) not deleted. State preserved.`
+        );
+      } else {
+        logger.warn(`${result.errorCount} resource(s) failed to delete. State preserved.`);
+      }
     }
 
-    // Summary glyph distinguishes clean destroy (✓) from partial failure
-    // (⚠). The CLI's exit code reflects the same split (0 vs 2) — see
-    // PartialFailureError in src/utils/error-handler.ts. Without the
+    // Summary glyph distinguishes clean destroy (✓) from partial failure /
+    // interrupt (⚠). The CLI's exit code reflects the same split (0 vs 2) —
+    // see PartialFailureError in src/utils/error-handler.ts. Without the
     // visual marker, a partial failure scrolls past in the same shape
     // as a successful destroy and gets missed in CI / bench output.
     const retainedSuffix = result.retainedCount > 0 ? `, ${result.retainedCount} retained` : '';
-    if (result.errorCount === 0) {
+    if (!preserveState) {
       logger.info(
         `\n${green('✓')} ${bold(`Stack ${stackName} destroyed`)} (${green(result.deletedCount)} deleted${retainedSuffix}, ${result.errorCount} errors)`
       );
+    } else if (result.interrupted && result.errorCount === 0) {
+      logger.warn(
+        `\n${yellow('⚠')} ${bold(`Stack ${stackName} destroy interrupted`)} (${green(result.deletedCount)} deleted${retainedSuffix}, ${result.errorCount} errors). ` +
+          `State preserved — re-run 'cdkd destroy' / 'cdkd state destroy' to finish.`
+      );
     } else {
       logger.warn(
         `\n${yellow('⚠')} ${bold(`Stack ${stackName} partially destroyed`)} (${green(result.deletedCount)} deleted${retainedSuffix}, ${red(result.errorCount)} errors). ` +
           `State preserved — re-run 'cdkd destroy' / 'cdkd state destroy' to clean up.`
       );
     }
   } finally {
+    // Remove our SIGINT listener so it never leaks past this call (each
+    // call registers and removes its own function reference — important for
+    // nested-stack recursion, where one handler is registered per level).
+    process.removeListener('SIGINT', sigintHandler);
+
     // Stop live renderer before releasing the lock so any pending in-flight
     // task lines are cleared cleanly.
     renderer.stop();

src/cli/commands/destroy-runner.ts

@@ -286,6 +286,9 @@ async function destroyCommand(
     // Hoisted out of the per-stack loop so the upfront nested-child-by-name
     // refusal (after the empty-match gate below) can use the same accumulator.
     let totalErrors = 0;
+    // Set true when a per-stack destroy was gracefully interrupted (issue
+    // #816). Stops the multi-stack loop and surfaces a non-zero exit below.
+    let interrupted = false;
 
     let stackNames: string[];
     if (options.all) {
@@ -560,6 +563,7 @@ async function destroyCommand(
             })
         );
         totalErrors += result.errorCount;
+        if (result.interrupted) interrupted = true;
 
         // Map the per-stack runner outcome to a run-level result. A
         // partial-failure (errorCount > 0) is a FAILED run; cancelled /
@@ -584,6 +588,11 @@ async function destroyCommand(
       } finally {
         await eventRecorder.finalize(destroyRunResult);
       }
+
+      // Graceful SIGINT (issue #816): do not start destroying further stacks
+      // once the user has asked to stop. The interrupted stack already
+      // finished its in-flight deletes, preserved state, and released its lock.
+      if (interrupted) break;
     }
 
     if (totalErrors > 0) {
@@ -597,6 +606,14 @@ async function destroyCommand(
           `inspect 'cdkd state show <stack>' and re-run 'cdkd destroy' to retry.`
       );
     }
+    if (interrupted) {
+      // Graceful SIGINT (issue #816): in-flight deletes finished, state was
+      // preserved (trimmed), and the lock was released. Surface a non-zero
+      // exit so scripts / CI see the destroy did not complete.
+      throw new PartialFailureError(
+        `Destroy interrupted by Ctrl-C. State preserved — re-run 'cdkd destroy' to finish.`
+      );
+    }
   } finally {
     // Cleanup AWS clients
     awsClients.destroy();

src/cli/commands/destroy.ts

@@ -1184,6 +1184,9 @@ async function stateDestroyCommand(
     logger.info(`Found ${stackNames.length} stack(s) to destroy: ${stackNames.join(', ')}`);
 
     let totalErrors = 0;
+    // Set true when a per-stack destroy was gracefully interrupted (issue
+    // #816). Stops the multi-stack loop and surfaces a non-zero exit below.
+    let interrupted = false;
     for (const stackName of stackNames) {
       // After PR 1, the same stackName can have state in multiple regions.
       // Pick the right ref(s):
@@ -1293,7 +1296,16 @@ async function stateDestroyCommand(
             })
         );
         totalErrors += result.errorCount;
+        if (result.interrupted) interrupted = true;
+        // Graceful interrupt (issue #816): stop iterating this stack's regions.
+        if (interrupted) break;
       }
+
+      // Graceful interrupt (issue #816): stop the outer multi-stack loop too —
+      // do not start destroying further stacks once the user has asked to
+      // stop. Explicit guard mirroring destroy.ts's stack-loop break (the
+      // inner `break` above only exits the per-region loop).
+      if (interrupted) break;
     }
 
     if (totalErrors > 0) {
@@ -1305,6 +1317,14 @@ async function stateDestroyCommand(
           `inspect 'cdkd state show <stack>' and re-run 'cdkd state destroy' to retry.`
       );
     }
+    if (interrupted) {
+      // Graceful SIGINT (issue #816): in-flight deletes finished, state was
+      // preserved (trimmed), and the lock was released. Surface a non-zero
+      // exit so scripts / CI see the destroy did not complete.
+      throw new PartialFailureError(
+        `Destroy interrupted by Ctrl-C. State preserved — re-run 'cdkd state destroy' to finish.`
+      );
+    }
   } finally {
     setup.dispose();
   }