Merge pull request #3 from b1rdmania/feat/v0.6.1-dual-timeout

v0.6.1 — Dual timeout for stuck agents

Author: b1rdmania

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## v0.6.1 (2026-03-19) — Dual timeout for stuck agents
+
+### Fixes
+- **Dual agent timeout** replaces the old single 30-min timer:
+  - **Idle timeout** (10 min default, `AGENT_IDLE_TIMEOUT` env var): reset on any stdout activity. An agent that produces no output for 10 minutes is considered stuck and killed. This catches the "hung waiting on API" case that was causing 30-40 min hangs.
+  - **Absolute ceiling** (45 min default, `AGENT_ABSOLUTE_TIMEOUT` env var): never resets, regardless of any activity. Hard cap for runaway agents producing garbage output.
+- Both timeouts log their reason (`idle` vs `absolute`) to make post-mortems easier.
+
 ## v0.6.0 (2026-03-19) — Reliability + remote control
 
 ### Fixes

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ghostclaw",
-  "version": "0.6.0",
+  "version": "0.6.1",
   "description": "Personal AI assistant. Bare metal, Telegram-first, no containers.",
   "type": "module",
   "main": "dist/index.js",

src/config.ts

@@ -39,6 +39,18 @@ export const CONTAINER_MAX_OUTPUT_SIZE = parseInt(
 ); // 10MB default
 export const IPC_POLL_INTERVAL = 1000;
 export const IDLE_TIMEOUT = parseInt(process.env.IDLE_TIMEOUT || '1800000', 10); // 30min default — how long to keep container alive after last result
+
+// Agent process timeouts (two independent timers in container-runner)
+// Idle: reset on any stdout activity. Agent with no stdout for this long is stuck → kill.
+export const AGENT_IDLE_TIMEOUT = parseInt(
+  process.env.AGENT_IDLE_TIMEOUT || '600000',
+  10,
+); // 10 min default
+// Absolute: never resets. Hard ceiling regardless of any activity.
+export const AGENT_ABSOLUTE_TIMEOUT = parseInt(
+  process.env.AGENT_ABSOLUTE_TIMEOUT || '2700000',
+  10,
+); // 45 min default
 export const MAX_CONCURRENT_CONTAINERS = Math.max(
   1,
   parseInt(process.env.MAX_CONCURRENT_CONTAINERS || '5', 10) || 5,

src/container-runner.test.ts

@@ -14,6 +14,8 @@ vi.mock('./config.js', () => ({
   DATA_DIR: '/tmp/ghostclaw-test-data',
   GROUPS_DIR: '/tmp/ghostclaw-test-groups',
   IDLE_TIMEOUT: 1800000, // 30min
+  AGENT_IDLE_TIMEOUT: 5000, // 5s in tests
+  AGENT_ABSOLUTE_TIMEOUT: 10000, // 10s in tests
   TIMEZONE: 'America/Los_Angeles',
 }));
 
@@ -116,7 +118,7 @@ describe('container-runner timeout behavior', () => {
     vi.useRealTimers();
   });
 
-  it('timeout after output resolves as success', async () => {
+  it('absolute timeout after output resolves as error', async () => {
     const onOutput = vi.fn(async () => {});
     const resultPromise = runContainerAgent(
       testGroup,
@@ -125,7 +127,7 @@ describe('container-runner timeout behavior', () => {
       onOutput,
     );
 
-    // Emit output with a result
+    // Emit output with a result (also resets idle timer via stdout data)
     emitOutputMarker(fakeProc, {
       status: 'success',
       result: 'Here is my response',
@@ -135,24 +137,24 @@ describe('container-runner timeout behavior', () => {
     // Let output processing settle
     await vi.advanceTimersByTimeAsync(10);
 
-    // Fire the hard timeout (IDLE_TIMEOUT + 30s = 1830000ms)
-    await vi.advanceTimersByTimeAsync(1830000);
+    // Fire the absolute ceiling (AGENT_ABSOLUTE_TIMEOUT = 10000ms in tests)
+    await vi.advanceTimersByTimeAsync(10000);
 
-    // Emit close event (as if container was stopped by the timeout)
+    // Emit close event (as if process was stopped by the timeout)
     fakeProc.emit('close', 137);
 
-    // Let the promise resolve
     await vi.advanceTimersByTimeAsync(10);
 
     const result = await resultPromise;
-    expect(result.status).toBe('success');
-    expect(result.newSessionId).toBe('session-123');
+    // Timeout always resolves as error — index.ts outputSentToUser guard prevents cursor rollback
+    expect(result.status).toBe('error');
+    expect(result.error).toContain('timed out');
     expect(onOutput).toHaveBeenCalledWith(
       expect.objectContaining({ result: 'Here is my response' }),
     );
   });
 
-  it('timeout with no output resolves as error', async () => {
+  it('idle timeout with no stdout resolves as error', async () => {
     const onOutput = vi.fn(async () => {});
     const resultPromise = runContainerAgent(
       testGroup,
@@ -161,12 +163,10 @@ describe('container-runner timeout behavior', () => {
       onOutput,
     );
 
-    // No output emitted — fire the hard timeout
-    await vi.advanceTimersByTimeAsync(1830000);
+    // No stdout at all — fire the idle timeout (AGENT_IDLE_TIMEOUT = 5000ms in tests)
+    await vi.advanceTimersByTimeAsync(5000);
 
-    // Emit close event
     fakeProc.emit('close', 137);
-
     await vi.advanceTimersByTimeAsync(10);
 
     const result = await resultPromise;
@@ -175,6 +175,32 @@ describe('container-runner timeout behavior', () => {
     expect(onOutput).not.toHaveBeenCalled();
   });
 
+  it('stdout activity resets idle timer', async () => {
+    const onOutput = vi.fn(async () => {});
+    const resultPromise = runContainerAgent(
+      testGroup,
+      testInput,
+      () => {},
+      onOutput,
+    );
+
+    // Emit raw stdout at 4s — just before idle timeout (5s in tests)
+    await vi.advanceTimersByTimeAsync(4000);
+    fakeProc.stdout.push('thinking...\n');
+
+    // Idle timer should have reset — advance another 4s (total 8s, but idle only 4s since last stdout)
+    await vi.advanceTimersByTimeAsync(4000);
+
+    // Agent hasn't timed out yet — now emit proper output and close normally
+    emitOutputMarker(fakeProc, { status: 'success', result: 'Done' });
+    await vi.advanceTimersByTimeAsync(10);
+    fakeProc.emit('close', 0);
+    await vi.advanceTimersByTimeAsync(10);
+
+    const result = await resultPromise;
+    expect(result.status).toBe('success');
+  });
+
   it('normal exit after output resolves as success', async () => {
     const onOutput = vi.fn(async () => {});
     const resultPromise = runContainerAgent(

src/container-runner.ts

@@ -7,11 +7,12 @@ import fs from 'fs';
 import path from 'path';
 
 import {
+  AGENT_ABSOLUTE_TIMEOUT,
+  AGENT_IDLE_TIMEOUT,
   CONTAINER_MAX_OUTPUT_SIZE,
   CONTAINER_TIMEOUT,
   DATA_DIR,
   GROUPS_DIR,
-  IDLE_TIMEOUT,
 } from './config.js';
 import { readEnvFile } from './env.js';
 import { resolveGroupFolderPath, resolveGroupIpcPath } from './group-folder.js';
@@ -291,6 +292,7 @@ export async function runContainerAgent(
 
     agentProcess.stdout.on('data', (data) => {
       const chunk = data.toString();
+      resetIdleTimer();
 
       if (!stdoutTruncated) {
         const remaining = CONTAINER_MAX_OUTPUT_SIZE - stdout.length;
@@ -324,7 +326,6 @@ export async function runContainerAgent(
               newSessionId = parsed.newSessionId;
             }
             hadStreamingOutput = true;
-            resetTimeout();
             outputChain = outputChain.then(() => onOutput(parsed));
           } catch (err) {
             logger.warn(
@@ -357,33 +358,53 @@ export async function runContainerAgent(
     });
 
     let timedOut = false;
+    let timeoutReason: 'idle' | 'absolute' | null = null;
     let hadStreamingOutput = false;
     const configTimeout = group.containerConfig?.timeout || CONTAINER_TIMEOUT;
-    const timeoutMs = Math.max(configTimeout, IDLE_TIMEOUT + 30_000);
+    const absoluteTimeoutMs = Math.min(configTimeout, AGENT_ABSOLUTE_TIMEOUT);
+    const idleTimeoutMs = Math.min(AGENT_IDLE_TIMEOUT, absoluteTimeoutMs);
 
-    const killOnTimeout = () => {
+    let timeoutHandled = false;
+    const killOnTimeout = (reason: 'idle' | 'absolute') => {
+      if (timeoutHandled) return;
+      timeoutHandled = true;
       timedOut = true;
+      timeoutReason = reason;
       logger.error(
-        { group: group.name, processName },
-        'Agent timeout, killing process',
+        { group: group.name, processName, reason },
+        reason === 'idle'
+          ? 'Agent idle timeout — no stdout for too long, killing process'
+          : 'Agent absolute timeout — hard ceiling reached, killing process',
       );
       agentProcess.kill('SIGTERM');
+      agentProcess.once('close', () => {
+        // Process already dead, nothing more to do
+      });
       setTimeout(() => {
-        if (!agentProcess.killed) {
+        try {
           agentProcess.kill('SIGKILL');
+        } catch {
+          // already dead
         }
       }, 15000);
     };
 
-    let timeout = setTimeout(killOnTimeout, timeoutMs);
-
-    const resetTimeout = () => {
-      clearTimeout(timeout);
-      timeout = setTimeout(killOnTimeout, timeoutMs);
+    // Idle timer — reset on any stdout activity
+    let idleTimer = setTimeout(() => killOnTimeout('idle'), idleTimeoutMs);
+    const resetIdleTimer = () => {
+      clearTimeout(idleTimer);
+      idleTimer = setTimeout(() => killOnTimeout('idle'), idleTimeoutMs);
     };
 
+    // Absolute ceiling — never resets
+    const absoluteTimer = setTimeout(
+      () => killOnTimeout('absolute'),
+      absoluteTimeoutMs,
+    );
+
     agentProcess.on('close', (code) => {
-      clearTimeout(timeout);
+      clearTimeout(idleTimer);
+      clearTimeout(absoluteTimer);
       const duration = Date.now() - startTime;
 
       if (timedOut) {
@@ -402,30 +423,32 @@ export async function runContainerAgent(
           ].join('\n'),
         );
 
-        if (hadStreamingOutput) {
-          logger.info(
-            { group: group.name, processName, duration, code },
-            'Agent timed out after output (idle cleanup)',
-          );
-          outputChain.then(() => {
-            resolve({
-              status: 'success',
-              result: null,
-              newSessionId,
-            });
-          });
-          return;
-        }
+        const timeoutMs =
+          timeoutReason === 'idle' ? idleTimeoutMs : absoluteTimeoutMs;
+        const timeoutLabel =
+          timeoutReason === 'idle'
+            ? `idle timeout (${idleTimeoutMs}ms no stdout)`
+            : `absolute timeout (${absoluteTimeoutMs}ms ceiling)`;
 
         logger.error(
-          { group: group.name, processName, duration, code },
-          'Agent timed out with no output',
+          {
+            group: group.name,
+            processName,
+            duration,
+            code,
+            timeoutReason,
+            hadStreamingOutput,
+          },
+          `Agent timed out (${timeoutLabel})`,
         );
 
-        resolve({
-          status: 'error',
-          result: null,
-          error: `Agent timed out after ${configTimeout}ms`,
+        outputChain.then(() => {
+          resolve({
+            status: 'error',
+            result: null,
+            error: `Agent timed out after ${timeoutMs}ms (${timeoutLabel})`,
+            newSessionId,
+          });
         });
         return;
       }
@@ -557,7 +580,8 @@ export async function runContainerAgent(
     });
 
     agentProcess.on('error', (err) => {
-      clearTimeout(timeout);
+      clearTimeout(idleTimer);
+      clearTimeout(absoluteTimer);
       logger.error(
         { group: group.name, processName, error: err },
         'Agent spawn error',