v0.6.1 — Dual timeout for stuck agents

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## v0.6.1 (2026-03-19) — Dual timeout for stuck agents
+
+### Fixes
+- **Dual agent timeout** replaces the old single 30-min timer:
+  - **Idle timeout** (10 min default, `AGENT_IDLE_TIMEOUT` env var): reset on any stdout activity. An agent that produces no output for 10 minutes is considered stuck and killed. This catches the "hung waiting on API" case that was causing 30-40 min hangs.
+  - **Absolute ceiling** (45 min default, `AGENT_ABSOLUTE_TIMEOUT` env var): never resets, regardless of any activity. Hard cap for runaway agents producing garbage output.
+- Both timeouts log their reason (`idle` vs `absolute`) to make post-mortems easier.
+
 ## v0.6.0 (2026-03-19) — Reliability + remote control
 
 ### Fixes

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ghostclaw",
-  "version": "0.6.0",
+  "version": "0.6.1",
   "description": "Personal AI assistant. Bare metal, Telegram-first, no containers.",
   "type": "module",
   "main": "dist/index.js",

src/config.ts

@@ -39,6 +39,18 @@ export const CONTAINER_MAX_OUTPUT_SIZE = parseInt(
 ); // 10MB default
 export const IPC_POLL_INTERVAL = 1000;
 export const IDLE_TIMEOUT = parseInt(process.env.IDLE_TIMEOUT || '1800000', 10); // 30min default — how long to keep container alive after last result
+
+// Agent process timeouts (two independent timers in container-runner)
+// Idle: reset on any stdout activity. Agent with no stdout for this long is stuck → kill.
+export const AGENT_IDLE_TIMEOUT = parseInt(
+  process.env.AGENT_IDLE_TIMEOUT || '600000',
+  10,
+); // 10 min default
+// Absolute: never resets. Hard ceiling regardless of any activity.
+export const AGENT_ABSOLUTE_TIMEOUT = parseInt(
+  process.env.AGENT_ABSOLUTE_TIMEOUT || '2700000',
+  10,
+); // 45 min default
 export const MAX_CONCURRENT_CONTAINERS = Math.max(
   1,
   parseInt(process.env.MAX_CONCURRENT_CONTAINERS || '5', 10) || 5,

src/container-runner.test.ts

@@ -14,6 +14,8 @@ vi.mock('./config.js', () => ({
   DATA_DIR: '/tmp/ghostclaw-test-data',
   GROUPS_DIR: '/tmp/ghostclaw-test-groups',
   IDLE_TIMEOUT: 1800000, // 30min
+  AGENT_IDLE_TIMEOUT: 5000, // 5s in tests
+  AGENT_ABSOLUTE_TIMEOUT: 10000, // 10s in tests
   TIMEZONE: 'America/Los_Angeles',
 }));
 
@@ -116,7 +118,7 @@ describe('container-runner timeout behavior', () => {
     vi.useRealTimers();
   });
 
-  it('timeout after output resolves as success', async () => {
+  it('absolute timeout after output resolves as success', async () => {
     const onOutput = vi.fn(async () => {});
     const resultPromise = runContainerAgent(
       testGroup,
@@ -125,7 +127,7 @@ describe('container-runner timeout behavior', () => {
       onOutput,
     );
 
-    // Emit output with a result
+    // Emit output with a result (also resets idle timer via stdout data)
     emitOutputMarker(fakeProc, {
       status: 'success',
       result: 'Here is my response',
@@ -135,13 +137,12 @@ describe('container-runner timeout behavior', () => {
     // Let output processing settle
     await vi.advanceTimersByTimeAsync(10);
 
-    // Fire the hard timeout (IDLE_TIMEOUT + 30s = 1830000ms)
-    await vi.advanceTimersByTimeAsync(1830000);
+    // Fire the absolute ceiling (AGENT_ABSOLUTE_TIMEOUT = 10000ms in tests)
+    await vi.advanceTimersByTimeAsync(10000);
 
-    // Emit close event (as if container was stopped by the timeout)
+    // Emit close event (as if process was stopped by the timeout)
     fakeProc.emit('close', 137);
 
-    // Let the promise resolve
     await vi.advanceTimersByTimeAsync(10);
 
     const result = await resultPromise;
@@ -152,7 +153,7 @@ describe('container-runner timeout behavior', () => {
     );
   });
 
-  it('timeout with no output resolves as error', async () => {
+  it('idle timeout with no stdout resolves as error', async () => {
     const onOutput = vi.fn(async () => {});
     const resultPromise = runContainerAgent(
       testGroup,
@@ -161,12 +162,10 @@ describe('container-runner timeout behavior', () => {
       onOutput,
     );
 
-    // No output emitted — fire the hard timeout
-    await vi.advanceTimersByTimeAsync(1830000);
+    // No stdout at all — fire the idle timeout (AGENT_IDLE_TIMEOUT = 5000ms in tests)
+    await vi.advanceTimersByTimeAsync(5000);
 
-    // Emit close event
     fakeProc.emit('close', 137);
-
     await vi.advanceTimersByTimeAsync(10);
 
     const result = await resultPromise;
@@ -175,6 +174,32 @@ describe('container-runner timeout behavior', () => {
     expect(onOutput).not.toHaveBeenCalled();
   });
 
+  it('stdout activity resets idle timer', async () => {
+    const onOutput = vi.fn(async () => {});
+    const resultPromise = runContainerAgent(
+      testGroup,
+      testInput,
+      () => {},
+      onOutput,
+    );
+
+    // Emit raw stdout at 4s — just before idle timeout (5s in tests)
+    await vi.advanceTimersByTimeAsync(4000);
+    fakeProc.stdout.push('thinking...\n');
+
+    // Idle timer should have reset — advance another 4s (total 8s, but idle only 4s since last stdout)
+    await vi.advanceTimersByTimeAsync(4000);
+
+    // Agent hasn't timed out yet — now emit proper output and close normally
+    emitOutputMarker(fakeProc, { status: 'success', result: 'Done' });
+    await vi.advanceTimersByTimeAsync(10);
+    fakeProc.emit('close', 0);
+    await vi.advanceTimersByTimeAsync(10);
+
+    const result = await resultPromise;
+    expect(result.status).toBe('success');
+  });
+
   it('normal exit after output resolves as success', async () => {
     const onOutput = vi.fn(async () => {});
     const resultPromise = runContainerAgent(

src/container-runner.ts

@@ -7,11 +7,12 @@ import fs from 'fs';
 import path from 'path';
 
 import {
+  AGENT_ABSOLUTE_TIMEOUT,
+  AGENT_IDLE_TIMEOUT,
   CONTAINER_MAX_OUTPUT_SIZE,
   CONTAINER_TIMEOUT,
   DATA_DIR,
   GROUPS_DIR,
-  IDLE_TIMEOUT,
 } from './config.js';
 import { readEnvFile } from './env.js';
 import { resolveGroupFolderPath, resolveGroupIpcPath } from './group-folder.js';
@@ -291,6 +292,7 @@ export async function runContainerAgent(
 
     agentProcess.stdout.on('data', (data) => {
       const chunk = data.toString();
+      resetIdleTimer();
 
       if (!stdoutTruncated) {
         const remaining = CONTAINER_MAX_OUTPUT_SIZE - stdout.length;
@@ -324,7 +326,6 @@ export async function runContainerAgent(
               newSessionId = parsed.newSessionId;
             }
             hadStreamingOutput = true;
-            resetTimeout();
             outputChain = outputChain.then(() => onOutput(parsed));
           } catch (err) {
             logger.warn(
@@ -359,31 +360,42 @@ export async function runContainerAgent(
     let timedOut = false;
     let hadStreamingOutput = false;
     const configTimeout = group.containerConfig?.timeout || CONTAINER_TIMEOUT;
-    const timeoutMs = Math.max(configTimeout, IDLE_TIMEOUT + 30_000);
+    const idleTimeoutMs = Math.min(
+      AGENT_IDLE_TIMEOUT,
+      Math.max(configTimeout, AGENT_ABSOLUTE_TIMEOUT),
+    );
+    const absoluteTimeoutMs = Math.max(configTimeout, AGENT_ABSOLUTE_TIMEOUT);
 
-    const killOnTimeout = () => {
+    const killOnTimeout = (reason: 'idle' | 'absolute') => {
       timedOut = true;
       logger.error(
-        { group: group.name, processName },
-        'Agent timeout, killing process',
+        { group: group.name, processName, reason },
+        reason === 'idle'
+          ? 'Agent idle timeout — no stdout for too long, killing process'
+          : 'Agent absolute timeout — hard ceiling reached, killing process',
       );
       agentProcess.kill('SIGTERM');
       setTimeout(() => {
-        if (!agentProcess.killed) {
-          agentProcess.kill('SIGKILL');
-        }
+        if (!agentProcess.killed) agentProcess.kill('SIGKILL');
       }, 15000);
     };
 
-    let timeout = setTimeout(killOnTimeout, timeoutMs);
-
-    const resetTimeout = () => {
-      clearTimeout(timeout);
-      timeout = setTimeout(killOnTimeout, timeoutMs);
+    // Idle timer — reset on any stdout activity
+    let idleTimer = setTimeout(() => killOnTimeout('idle'), idleTimeoutMs);
+    const resetIdleTimer = () => {
+      clearTimeout(idleTimer);
+      idleTimer = setTimeout(() => killOnTimeout('idle'), idleTimeoutMs);
     };
 
+    // Absolute ceiling — never resets
+    const absoluteTimer = setTimeout(
+      () => killOnTimeout('absolute'),
+      absoluteTimeoutMs,
+    );
+
     agentProcess.on('close', (code) => {
-      clearTimeout(timeout);
+      clearTimeout(idleTimer);
+      clearTimeout(absoluteTimer);
       const duration = Date.now() - startTime;
 
       if (timedOut) {
@@ -557,7 +569,8 @@ export async function runContainerAgent(
     });
 
     agentProcess.on('error', (err) => {
-      clearTimeout(timeout);
+      clearTimeout(idleTimer);
+      clearTimeout(absoluteTimer);
       logger.error(
         { group: group.name, processName, error: err },
         'Agent spawn error',