cleanup

Author: grutt

frontend/docs/pages/home/cancellation.mdx

@@ -33,6 +33,26 @@ When a task is canceled, Hatchet sends a cancellation signal to the task. The ta
 
     />
 
+### AbortError behavior
+
+Hatchet cancellation in TypeScript is driven by an internal `ctx.abortController.signal` (an `AbortSignal`). When a task is cancelled, Hatchet aborts that signal and cancellation-aware operations (like waiting on a child run result) will reject with an **`AbortError`**.
+
+<Callout type="warning">
+  **Important:** JavaScript/TypeScript cannot make cancellation “uncatchable”. A broad `catch (e) { ... }` can swallow cancellation. Hatchet’s SDK will avoid enqueueing new child runs once the parent task is cancelled, and it will not report a cancelled task as “completed” even if user code catches the abort — but your code should still exit quickly to avoid wasted work.
+</Callout>
+
+If you must catch errors, re-throw abort/cancellation errors:
+
+```ts
+try {
+  // ... work ...
+  await simple.run({});
+} catch (e) {
+  ctx.rethrowIfCancelled(e);
+  // ... other error handling ...
+}
+```
+
   </Tabs.Tab>
   <Tabs.Tab title="Go">
     <Snippet src={snippets.go.cancellations.main.cancelled_task}/>

sdks/typescript/src/clients/worker/worker.ts

@@ -31,6 +31,7 @@ import { WebhookWorkerCreateRequest } from '@clients/rest/generated/data-contrac
 import { WorkflowDefinition } from '@hatchet/v1';
 import { NonRetryableError } from '@hatchet/v1/task';
 import { applyNamespace } from '@hatchet/util/apply-namespace';
+import { throwIfAborted } from '@hatchet/util/abort-error';
 import { V0Context, CreateStep, V0DurableContext, mapRateLimit, StepRunFunction } from '../../step';
 import { WorkerLabels } from '../dispatcher/dispatcher-client';
 
@@ -253,10 +254,16 @@ export class V0Worker {
       }
 
       const run = async () => {
+        // Precheck: if cancellation already happened, don't execute user code.
+        throwIfAborted(context.controller?.signal);
         return step(context);
       };
 
       const success = async (result: any) => {
+        // If cancellation happened, do not report completion.
+        if (context.controller?.signal?.aborted) {
+          return;
+        }
         this.logger.info(taskRunLog(taskName, taskRunExternalId, 'completed'));
 
         try {
@@ -302,6 +309,10 @@ export class V0Worker {
       };
 
       const failure = async (error: any) => {
+        // If cancellation happened, do not report failure.
+        if (context.controller?.signal?.aborted) {
+          return;
+        }
         this.logger.error(taskRunLog(taskName, taskRunExternalId, `failed: ${error.message}`));
 
         if (error.stack) {
@@ -341,6 +352,19 @@ export class V0Worker {
             await failure(e);
             return;
           }
+
+          // Postcheck: user code may swallow AbortError; don't report completion after cancellation.
+          // If we reached this point and the signal is aborted, the task likely caught/ignored cancellation.
+          if (context.controller?.signal?.aborted) {
+            this.logger.warn(
+              `Cancellation: task run ${taskRunExternalId} returned after cancellation was signaled. ` +
+                `This usually means an AbortError was caught and not propagated. ` +
+                `See https://docs.hatchet.run/home/cancellation`
+            );
+            return;
+          }
+          throwIfAborted(context.controller?.signal);
+
           await success(result);
         })()
       );

sdks/typescript/src/util/abort-error.ts

@@ -4,3 +4,89 @@ export function createAbortError(message = 'Operation aborted'): Error {
   err.code = 'ABORT_ERR';
   return err as Error;
 }
+
+export function isAbortError(err: unknown): err is Error {
+  return (
+    err instanceof Error &&
+    (err.name === 'AbortError' || (err as any).code === 'ABORT_ERR')
+  );
+}
+
+/**
+ * Helper to be used inside broad `catch` blocks so cancellation isn't accidentally swallowed.
+ *
+ * Example:
+ * ```ts
+ * try { ... } catch (e) { rethrowIfAborted(e); ... }
+ * ```
+ */
+export function rethrowIfAborted(err: unknown): void {
+  if (isAbortError(err)) {
+    throw err;
+  }
+}
+
+export type ThrowIfAbortedOpts = {
+  /**
+   * Optional: called before throwing when the signal is aborted.
+   * This lets callsites attach logging without coupling this util to a logger implementation.
+   */
+  warn?: (message: string) => void;
+
+  /**
+   * If true, emits a generic warning intended for "trigger/enqueue" paths.
+   */
+  isTrigger?: boolean;
+
+  /**
+   * Optional context used to make warnings consistent, e.g. "task run <id>".
+   */
+  context?: string;
+
+  /**
+   * Message used when the AbortSignal doesn't provide a reason.
+   */
+  defaultMessage?: string;
+};
+
+/**
+ * Throws an AbortError if the provided signal is aborted.
+ *
+ * Notes:
+ * - In JS/TS, `catch` can swallow any thrown value, so this is best-effort.
+ * - We prefer throwing the signal's `reason` when it is already an Error.
+ */
+export function throwIfAborted(
+  signal: AbortSignal | undefined,
+  optsOrDefaultMessage: ThrowIfAbortedOpts | string = 'Operation cancelled by AbortSignal'
+): void {
+  if (!signal?.aborted) {
+    return;
+  }
+
+  const opts: ThrowIfAbortedOpts =
+    typeof optsOrDefaultMessage === 'string'
+      ? { defaultMessage: optsOrDefaultMessage }
+      : (optsOrDefaultMessage ?? {});
+
+  if (opts.isTrigger) {
+    const ctx = opts.context ? `${opts.context} ` : '';
+    opts.warn?.(
+      `Cancellation: ${ctx}attempted to enqueue/trigger work after cancellation was signaled. ` +
+        `This usually means an AbortError was caught and not propagated. ` +
+        `See https://docs.hatchet.run/home/cancellation`
+    );
+  }
+
+  const reason = (signal as any).reason;
+
+  if (reason instanceof Error) {
+    throw reason;
+  }
+
+  if (typeof reason === 'string' && reason.length > 0) {
+    throw createAbortError(reason);
+  }
+
+  throw createAbortError(opts.defaultMessage ?? 'Operation cancelled by AbortSignal');
+}

sdks/typescript/src/v1/client/worker/context.ts

@@ -21,7 +21,7 @@ import { Action as ConditionAction } from '@hatchet/protoc/v1/shared/condition';
 import { HatchetClient } from '@hatchet/v1';
 import { ContextWorker, NextStep } from '@hatchet/step';
 import { applyNamespace } from '@hatchet/util/apply-namespace';
-import { createAbortError } from '@hatchet/util/abort-error';
+import { createAbortError, rethrowIfAborted } from '@hatchet/util/abort-error';
 import { V1Worker } from './worker-internal';
 import { Duration } from '../duration';
 
@@ -97,6 +97,18 @@ export class Context<T, K = {}> {
     }
   }
 
+  /**
+   * Helper for broad `catch` blocks so cancellation isn't accidentally swallowed.
+   *
+   * Example:
+   * ```ts
+   * try { ... } catch (e) { ctx.rethrowIfCancelled(e); ... }
+   * ```
+   */
+  rethrowIfCancelled(err: unknown): void {
+    rethrowIfAborted(err);
+  }
+
   async cancel() {
     await this.v1.runs.cancel({
       ids: [this.action.taskRunExternalId],

sdks/typescript/src/v1/client/worker/worker-internal.ts

@@ -37,6 +37,7 @@ import { WorkerLabels } from '@hatchet/clients/dispatcher/dispatcher-client';
 import { CreateStep, mapRateLimit, StepRunFunction } from '@hatchet/step';
 import { applyNamespace } from '@hatchet/util/apply-namespace';
 import sleep from '@hatchet/util/sleep';
+import { throwIfAborted } from '@hatchet/util/abort-error';
 import { Context, DurableContext } from './context';
 import { parentRunContextManager } from '../../parent-run-context-vars';
 import { HealthServer, workerStatus, type WorkerStatus } from './health-server';
@@ -544,7 +545,11 @@ export class V1Worker {
             desiredWorkerId: this.workerId || '',
             signal: context.abortController.signal,
           },
-          () => step(context)
+          () => {
+            // Precheck: if cancellation already happened, don't execute user code.
+            throwIfAborted(context.abortController.signal);
+            return step(context);
+          }
         );
       };
 
@@ -641,6 +646,19 @@ export class V1Worker {
             await failure(e);
             return;
           }
+
+          // Postcheck: user code may swallow AbortError; don't report completion after cancellation.
+          // If we reached this point and the signal is aborted, the task likely caught/ignored cancellation.
+          if (context.abortController.signal.aborted) {
+            this.logger.warn(
+              `Cancellation: task run ${taskRunExternalId} returned after cancellation was signaled. ` +
+                `This usually means an AbortError was caught and not propagated. ` +
+                `See https://docs.hatchet.run/home/cancellation`
+            );
+            return;
+          }
+          throwIfAborted(context.abortController.signal);
+
           await success(result);
         })()
       );
@@ -848,11 +866,9 @@ export class V1Worker {
           ]);
 
           if (winner === 'warn') {
-            const elapsedSeconds = (Date.now() - start) / 1000;
+            const milliseconds = Date.now() - start;
             this.logger.warn(
-              `Cancellation: task run ${taskRunExternalId} has not cancelled after ${elapsedSeconds.toFixed(
-                1
-              )}s. Consider checking for blocking operations. ` +
+              `Cancellation: task run ${taskRunExternalId} has not cancelled after ${milliseconds}ms. Consider checking for blocking operations. ` +
                 `See https://docs.hatchet.run/home/cancellation`
             );
           }
@@ -870,10 +886,10 @@ export class V1Worker {
           this.logger.info(taskRunLog(taskName, taskRunExternalId, 'cancelled'));
         } else {
           const totalElapsedMs = Date.now() - start;
-          this.logger.warn(
-            `Cancellation: task run ${taskRunExternalId} still running after grace period ` +
-              `${gracePeriodMs}ms (elapsed ${totalElapsedMs.toFixed(1)}ms). ` +
-              `JavaScript cannot force-kill user code; ensure your tasks honor ctx.abortController.signal.`
+          this.logger.error(
+            `Cancellation: task run ${taskRunExternalId} still running after cancellation grace period ` +
+              `${totalElapsedMs}ms.\n` +
+              `JavaScript cannot force-kill user code; see: https://docs.hatchet.run/home/cancellation`
           );
         }
       }

sdks/typescript/src/v1/declaration.ts

@@ -25,26 +25,10 @@ import { MetricsClient } from './client/features/metrics';
 import { InputType, OutputType, UnknownInputType, JsonObject } from './types';
 import { Context, DurableContext } from './client/worker/context';
 import { parentRunContextManager } from './parent-run-context-vars';
-import { createAbortError } from '@hatchet/util/abort-error';
+import { throwIfAborted } from '@hatchet/util/abort-error';
 
 const UNBOUND_ERR = new Error('workflow unbound to hatchet client, hint: use client.run instead');
 
-function throwIfAborted(signal: AbortSignal | undefined): void {
-  if (!signal?.aborted) {
-    return;
-  }
-
-  const reason = (signal as any).reason;
-
-  if (reason instanceof Error) {
-    throw reason;
-  }
-
-  throw createAbortError(
-    typeof reason === 'string' && reason.length > 0 ? reason : 'Operation cancelled by AbortSignal'
-  );
-}
-
 // eslint-disable-next-line no-shadow
 export enum Priority {
   LOW = 1,
@@ -340,7 +324,13 @@ export class BaseWorkflowDeclaration<
 
     // Precheck: if we're being called from a cancelled parent task, do not enqueue more work.
     // The signal is inherited from the parent task's `ctx.abortController.signal`.
-    throwIfAborted(inheritedSignal);
+    throwIfAborted(inheritedSignal, {
+      isTrigger: true,
+      context: parentRunContext?.parentTaskRunExternalId
+        ? `task run ${parentRunContext.parentTaskRunExternalId}`
+        : undefined,
+      warn: (message) => this.client!.admin.logger.warn(message),
+    });
 
     parentRunContextManager.incrementChildIndex(Array.isArray(input) ? input.length : 1);
 
@@ -461,7 +451,10 @@ export class BaseWorkflowDeclaration<
     }
 
     // If called from within a cancelled parent task, do not enqueue scheduled work.
-    throwIfAborted(parentRunContextManager.getContext()?.signal);
+    throwIfAborted(parentRunContextManager.getContext()?.signal, {
+      isTrigger: true,
+      warn: (message) => this.client!.admin.logger.warn(message),
+    });
 
     const scheduled = this.client.scheduled.create(this.definition.name, {
       triggerAt: enqueueAt,
@@ -506,7 +499,10 @@ export class BaseWorkflowDeclaration<
     }
 
     // If called from within a cancelled parent task, do not enqueue cron work.
-    throwIfAborted(parentRunContextManager.getContext()?.signal);
+    throwIfAborted(parentRunContextManager.getContext()?.signal, {
+      isTrigger: true,
+      warn: (message) => this.client!.admin.logger.warn(message),
+    });
 
     const cronDef = this.client.crons.create(this.definition.name, {
       expression,