Merge pull request coleam00#962 from dynamous-community/archon/task-fix-issue-961

fix: prevent SIGTERM from orphaning CLI workflow runs

Author: Wirasm

packages/cli/src/cli.ts

@@ -65,6 +65,7 @@ import { chatCommand } from './commands/chat';
 import { setupCommand } from './commands/setup';
 import { validateWorkflowsCommand, validateCommandsCommand } from './commands/validate';
 import { closeDatabase } from '@archon/core';
+import { createWorkflowStore } from '@archon/core/workflows/store-adapter';
 import { setLogLevel, createLogger } from '@archon/paths';
 import * as git from '@archon/git';
 
@@ -214,6 +215,14 @@ async function main(): Promise<number> {
       setLogLevel('debug');
     }
 
+    // Mark workflow runs orphaned by previous process termination as failed
+    void createWorkflowStore()
+      .failOrphanedRuns()
+      .catch((err: unknown) => {
+        const e = err as Error;
+        getLog().warn({ err: e, errorType: e.constructor.name }, 'cli.fail_orphans_failed');
+      });
+
     // Validate working directory exists
     let effectiveCwd = cwd;
     if (requiresGitRepo) {

packages/cli/src/commands/workflow.ts

@@ -518,20 +518,36 @@ export async function workflowRunCommand(
   );
 
   // Register cleanup handlers for graceful termination
-  const cleanup = async (signal: string): Promise<void> => {
+  let terminating = false;
+  const cleanup = (signal: string): void => {
+    if (terminating) return;
+    terminating = true;
     getLog().info({ conversationId: conversation.id, signal }, 'workflow.process_terminating');
-    try {
-      const activeRun = await workflowDb.getActiveWorkflowRun(conversation.id);
-      if (activeRun) {
-        await workflowDb.failWorkflowRun(activeRun.id, `Process terminated (${signal})`);
-      }
-    } catch (err) {
-      getLog().error({ err: err as Error }, 'workflow.termination_cleanup_failed');
-    }
-    process.exit(1);
+    workflowDb
+      .getActiveWorkflowRun(conversation.id)
+      .then(activeRun => {
+        if (activeRun) {
+          return workflowDb.failWorkflowRun(activeRun.id, `Process terminated (${signal})`);
+        }
+        return undefined;
+      })
+      .catch((err: unknown) => {
+        const e = err as Error;
+        getLog().error(
+          { err: e, errorType: e.constructor.name },
+          'workflow.termination_cleanup_failed'
+        );
+      })
+      .finally(() => {
+        process.exit(1);
+      });
   };
-  process.on('SIGTERM', () => void cleanup('SIGTERM'));
-  process.on('SIGINT', () => void cleanup('SIGINT'));
+  process.once('SIGTERM', () => {
+    cleanup('SIGTERM');
+  });
+  process.once('SIGINT', () => {
+    cleanup('SIGINT');
+  });
 
   // Execute workflow with workingCwd (may be worktree path)
   const result = await executeWorkflow(
@@ -730,14 +746,13 @@ export async function workflowStatusCommand(json?: boolean, verbose?: boolean):
       if (nodes.length > 0) {
         console.log('  Nodes:');
         for (const node of nodes) {
-          const icon =
-            node.state === 'completed'
-              ? '✓'
-              : node.state === 'failed'
-                ? '✗'
-                : node.state === 'skipped'
-                  ? '-'
-                  : '◌';
+          const iconMap: Record<string, string> = {
+            completed: '✓',
+            failed: '✗',
+            skipped: '-',
+            running: '◌',
+          };
+          const icon = iconMap[node.state] ?? '◌';
           const duration =
             node.durationMs !== undefined ? ` (${formatDuration(node.durationMs)})` : '';
           const stateLabel = node.state === 'running' ? ' (running)' : '';

packages/core/src/db/workflows.test.ts

@@ -489,7 +489,42 @@ describe('workflows database', () => {
       expect(query).toContain('working_path = $2');
       expect(query).not.toContain('conversation_id');
       expect(query).toContain('ORDER BY started_at DESC');
-      expect(params).toEqual(['feature-development', '/repo/path']);
+      expect(params).toEqual(['feature-development', '/repo/path', 1]);
+    });
+
+    test('returns a stale running run (no activity for >1 day)', async () => {
+      const twoDaysAgo = new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString();
+      const staleRun = {
+        ...mockWorkflowRun,
+        status: 'running' as const,
+        working_path: '/repo/path',
+        last_activity_at: twoDaysAgo,
+      };
+      mockQuery.mockResolvedValueOnce(createQueryResult([staleRun]));
+
+      const result = await findResumableRun('feature-development', '/repo/path');
+
+      expect(result).toEqual(staleRun);
+      const [query, params] = mockQuery.mock.calls[0] as [string, unknown[]];
+      expect(query).toContain("status = 'running'");
+      expect(query).toContain('last_activity_at');
+      expect(params).toEqual(['feature-development', '/repo/path', 1]);
+    });
+
+    test('returns a running run with null last_activity_at (never recorded activity)', async () => {
+      const staleRun = {
+        ...mockWorkflowRun,
+        status: 'running' as const,
+        working_path: '/repo/path',
+        last_activity_at: null,
+      };
+      mockQuery.mockResolvedValueOnce(createQueryResult([staleRun]));
+
+      const result = await findResumableRun('feature-development', '/repo/path');
+
+      expect(result).toEqual(staleRun);
+      const [query] = mockQuery.mock.calls[0] as [string, unknown[]];
+      expect(query).toContain('last_activity_at IS NULL');
     });
 
     test('returns null when no resumable run exists', async () => {

packages/core/src/db/workflows.ts

@@ -226,21 +226,28 @@ export async function findResumableRun(
   workflowName: string,
   workingPath: string
 ): Promise<WorkflowRun | null> {
+  const dialect = getDialect();
   try {
     const result = await pool.query<WorkflowRun>(
       `SELECT * FROM remote_agent_workflow_runs
        WHERE workflow_name = $1
          AND working_path = $2
-         AND status IN ('failed', 'paused')
+         AND (
+           status IN ('failed', 'paused')
+           OR (status = 'running' AND (last_activity_at IS NULL OR last_activity_at < ${dialect.nowMinusDays(3)})) -- nowMinusDays(3): param index 3, value bound as $3 = 1 day
+         )
        ORDER BY started_at DESC
        LIMIT 1`,
-      [workflowName, workingPath]
+      [workflowName, workingPath, 1]
     );
     const row = result.rows[0];
     return row ? normalizeWorkflowRun(row) : null;
   } catch (error) {
     const err = error as Error;
-    getLog().warn({ err, workflowName, workingPath }, 'db.workflow_run_find_resumable_failed');
+    getLog().error(
+      { err, errorType: err.constructor.name, workflowName, workingPath },
+      'db.workflow_run_find_resumable_failed'
+    );
     throw new Error(`Failed to find resumable run: ${err.message}`);
   }
 }