Skip to content

Commit 8f9fa2b

Browse files
EchoOfDawnInstar Agent (echo)claude
authored
fix(reaper): detect multi-commit squash-merges via GitHub merged-PR state (disk-accumulation root cause) (#1290)
* fix(reaper): detect multi-commit squash-merges via GitHub merged-PR state (disk-accumulation root cause) The AgentWorktreeReaper's merged-check used `git cherry` (patch-id) ONLY, which cannot detect a MULTI-commit branch that was SQUASH-merged. Since this project squash-merges every PR, merged worktrees were kept forever and accumulated (~118GB/290 observed), contributing to the 2026-06-26 kernel panic. Diagnosed in the stalled 28730 session (CMT-1812), picked up here. - New `fetchMergedPrHeadOids()` — ONE `gh pr list --state merged` call per sweep (cached 60s, funneled through withSyncOp). Fail-safe to empty map on any error. - `isMerged` falls back to the PR map ONLY when `git cherry` says unmerged, and treats a worktree as merged only when its branch's merged-PR head OID EXACTLY matches the worktree HEAD (post-merge commits are still KEPT). isClean still independently blocks any dirty worktree. - New config `monitoring.agentWorktreeReaper.githubMergeCheck` (default true). Reaper still ships OFF + dry-run. - Tests: 9 new; CLAUDE.md awareness updated. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * docs(release): add Evidence section to reaper squash-merge fragment Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --------- Co-authored-by: Instar Agent (echo) <echo@instar.local> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
1 parent acf0e82 commit 8f9fa2b

12 files changed

Lines changed: 426 additions & 4 deletions
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"ts": "2026-06-26T23:07:20.584Z",
3+
"slug": "reaper-squash-merge-aware",
4+
"suggestedTier": 2,
5+
"declaredTier": 1,
6+
"riskFloor": 2,
7+
"riskFloorReasons": [
8+
"safety-invariant proximity: src/monitoring/AgentWorktreeReaper.ts matches session reaper",
9+
"irreversibility: src/core/PostUpdateMigrator.ts touches PostUpdateMigrator",
10+
"migration / fleet-rollout surface: src/core/PostUpdateMigrator.ts touches PostUpdateMigrator (fleet migration machinery)"
11+
],
12+
"belowFloor": true,
13+
"files": 5,
14+
"loc": 106,
15+
"causalAutopsy": null,
16+
"verdict": "blocked"
17+
}
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"ts": "2026-06-26T23:08:05.065Z",
3+
"slug": "reaper-squash-merge-aware",
4+
"suggestedTier": 2,
5+
"declaredTier": 1,
6+
"riskFloor": 2,
7+
"riskFloorReasons": [
8+
"safety-invariant proximity: src/monitoring/AgentWorktreeReaper.ts matches session reaper",
9+
"irreversibility: src/core/PostUpdateMigrator.ts touches PostUpdateMigrator",
10+
"migration / fleet-rollout surface: src/core/PostUpdateMigrator.ts touches PostUpdateMigrator (fleet migration machinery)"
11+
],
12+
"belowFloor": true,
13+
"files": 5,
14+
"loc": 106,
15+
"causalAutopsy": null,
16+
"verdict": "blocked"
17+
}
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"ts": "2026-06-26T23:48:08.434Z",
3+
"slug": "reaper-squash-merge-aware",
4+
"suggestedTier": 2,
5+
"declaredTier": 1,
6+
"riskFloor": 2,
7+
"riskFloorReasons": [
8+
"safety-invariant proximity: src/monitoring/AgentWorktreeReaper.ts matches session reaper",
9+
"irreversibility: src/core/PostUpdateMigrator.ts touches PostUpdateMigrator",
10+
"migration / fleet-rollout surface: src/core/PostUpdateMigrator.ts touches PostUpdateMigrator (fleet migration machinery)"
11+
],
12+
"belowFloor": true,
13+
"files": 5,
14+
"loc": 106,
15+
"causalAutopsy": {
16+
"origin": "latent",
17+
"notes": "Latent blind spot in the reaper's git-cherry merged-check: it documents that multi-commit squash-merges are not detected (their per-commit patch-ids differ from the squashed commit) and conservatively KEEPS them. Because this project squash-merges every PR, merged worktrees accumulated forever (~118GB/290 observed) — a contributor to the 2026-06-26 resource-exhaustion kernel panic. Diagnosed in the stalled 28730 session (CMT-1812), picked up here."
18+
},
19+
"verdict": "pass"
20+
}

src/commands/server.ts

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15717,7 +15717,13 @@ export async function startServer(options: StartOptions): Promise<void> {
1571715717
const { makeAgentWorktreeReaperDeps } = await import('../monitoring/agentWorktreeGit.js');
1571815718
const _agentWorktreesDir = path.join(path.dirname(config.stateDir), '.worktrees');
1571915719
const agentWorktreeReaper = new AgentWorktreeReaper(
15720-
makeAgentWorktreeReaperDeps({ instarRepo: config.projectDir, worktreesDir: _agentWorktreesDir }),
15720+
makeAgentWorktreeReaperDeps({
15721+
instarRepo: config.projectDir,
15722+
worktreesDir: _agentWorktreesDir,
15723+
// Multi-commit squash-merge detection via GitHub merged-PR state (default
15724+
// on; fail-safe to cherry-only). Off only if explicitly disabled in config.
15725+
githubMergeCheck: config.monitoring?.agentWorktreeReaper?.githubMergeCheck ?? true,
15726+
}),
1572115727
config.monitoring?.agentWorktreeReaper,
1572215728
);
1572315729
agentWorktreeReaper.start();

src/core/PostUpdateMigrator.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5856,8 +5856,9 @@ Every guard (monitoring sentinels, reapers, the scheduler, …) is graded by wha
58565856
CLI-created worktrees under \`~/.instar/agents/<agent>/.worktrees/\` accumulate (each is a full source tree). The AgentWorktreeReaper reclaims ones that are **merged + clean + not-in-use** — for a merged branch the work is in main, so removing the checkout loses nothing (the branch + commits remain). It NEVER touches a worktree with uncommitted changes, an unmerged branch, a live lock, or a running process whose cwd is inside it. Ships **OFF + dry-run** (it deletes on a heuristic).
58575857

58585858
- See what's reclaimable (and why each is kept): \`curl -H "Authorization: Bearer $AUTH" http://localhost:4040/worktrees/agent-reaper\` → per-worktree verdict (in-use / uncommitted-changes / unmerged / reap-eligible) + the reclaimable count.
5859+
- **Squash-merge detection (the accumulation fix):** the merged-check is patch-id (\`git cherry\`) FIRST — which cannot see a MULTI-commit branch that was SQUASH-merged (its commits' SHAs/patch-ids differ from the single squashed commit), so those worktrees used to pile up forever. The reaper now ALSO consults GitHub merged-PR state (ONE \`gh\` call per sweep) and treats a worktree as merged when its branch has a merged PR whose head commit EXACTLY matches the worktree's HEAD (so a branch with commits added AFTER the merge is still kept). Fail-safe: any \`gh\` error degrades to cherry-only (KEEP). Off-switch: \`{"monitoring": {"agentWorktreeReaper": {"githubMergeCheck": false}}}\`.
58595860
- Review the dry-run report FIRST, then enable in \`.instar/config.json\`: \`{"monitoring": {"agentWorktreeReaper": {"enabled": true, "dryRun": false}}}\`. Tune \`maxReapsPerPass\` (default 20).
5860-
- Pairs with the Spotlight-exclusion marker (fewer worktrees = less disk AND less macOS indexing). Proactive: user asks "why is my disk full of worktrees?" / "clean up old worktrees?" → GET /worktrees/agent-reaper.
5861+
- Pairs with the Spotlight-exclusion marker (fewer worktrees = less disk AND less macOS indexing). Proactive: user asks "why is my disk full of worktrees?" / "clean up old worktrees?" / "why is the reaper calling GitHub?" → GET /worktrees/agent-reaper; the gh call is the squash-merge detection above.
58615862
`;
58625863
content += '\n' + section;
58635864
patched = true;

src/core/types.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4947,6 +4947,9 @@ export interface MonitoringConfig {
49474947
dryRun?: boolean;
49484948
reapIntervalMs?: number;
49494949
maxReapsPerPass?: number;
4950+
/** Catch multi-commit squash-merges via GitHub merged-PR state (default true;
4951+
* fail-safe to git-cherry-only). Set false to disable the network call. */
4952+
githubMergeCheck?: boolean;
49504953
};
49514954
/**
49524955
* OrphanedWorkSentinel — the silent-uncommitted-death backstop (2026-06-12,

src/monitoring/AgentWorktreeReaper.ts

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,14 @@ export interface AgentWorktreeReaperConfig {
4141
* permanently-unremovable worktree can't be retried forever. 0 disables the brake.
4242
*/
4343
maxReclaimFailuresPerPath: number;
44+
/**
45+
* When true (default), merged-detection falls back to GitHub merged-PR state to
46+
* catch MULTI-COMMIT squash-merges that `git cherry` (patch-id) cannot — the
47+
* disk-accumulation root cause where squash-merged worktrees are kept forever.
48+
* One `gh` call per sweep, fail-safe to cherry-only (KEEP) on any error. Set
49+
* false to disable the network call and restore the legacy cherry-only behavior.
50+
*/
51+
githubMergeCheck: boolean;
4452
}
4553

4654
export const DEFAULT_AGENT_WORKTREE_REAPER_CONFIG: AgentWorktreeReaperConfig = {
@@ -49,6 +57,7 @@ export const DEFAULT_AGENT_WORKTREE_REAPER_CONFIG: AgentWorktreeReaperConfig = {
4957
reapIntervalMs: 24 * 3600 * 1000,
5058
maxReapsPerPass: 20,
5159
maxReclaimFailuresPerPath: 3,
60+
githubMergeCheck: true,
5261
};
5362

5463
export interface WorktreeInfo {

src/monitoring/agentWorktreeGit.ts

Lines changed: 82 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import fs from 'node:fs';
1111
import path from 'node:path';
1212
import { execFileSync } from 'node:child_process';
1313
import { SafeGitExecutor } from '../core/SafeGitExecutor.js';
14+
import { withSyncOp } from '../core/InFlightSyncOpMarker.js';
1415
import { classifyPorcelain } from '../core/worktreeDirtyCheck.js';
1516
import type { AgentWorktreeReaperDeps, WorktreeInfo } from './AgentWorktreeReaper.js';
1617

@@ -88,6 +89,54 @@ export function isBranchMerged(readGit: ReadGit, repo: string, baseRef: string,
8889
return lines.every((l) => l.startsWith('-'));
8990
}
9091

92+
/** Run `gh` and return stdout, or null on ANY failure (gh missing, not authed,
93+
* not a GitHub repo, timeout). Null is the conservative signal → caller keeps. */
94+
export type RunGh = (args: string[], cwd: string) => string | null;
95+
96+
const defaultRunGh: RunGh = (args, cwd) => {
97+
try {
98+
// Funnel through withSyncOp so the in-flight marker sees this blocking spawn
99+
// (event-loop-resilience spec): the reaper runs in-process on a timer, and a
100+
// bounded gh call must not read as a "stuck" event loop to the watchdogs.
101+
return withSyncOp(() => execFileSync('gh', args, { cwd, encoding: 'utf-8', timeout: 30_000, maxBuffer: 16 * 1024 * 1024 }));
102+
} catch {
103+
return null; // @silent-fallback-ok — gh unavailable ⇒ no PR signal ⇒ KEEP (conservative)
104+
}
105+
};
106+
107+
/**
108+
* Fetch a map of `headRefName → headRefOid` for MERGED PRs, to detect
109+
* MULTI-COMMIT squash-merges that `git cherry` (patch-id) cannot — the
110+
* disk-accumulation root cause: a multi-commit branch squash-merged into one
111+
* commit on main has different commit SHAs/patch-ids, so cherry reports it
112+
* UNMERGED and the worktree is kept forever. A merged PR is the authoritative
113+
* "the content is in main" signal; pairing it with an EXACT head-OID match (in
114+
* the caller) ensures a branch with commits ADDED AFTER the merge is still kept.
115+
*
116+
* ONE `gh` call per sweep (bounded `--limit`); fail-safe to an EMPTY map on any
117+
* error so the reaper degrades to exactly today's cherry-only behavior (KEEP).
118+
*/
119+
export function fetchMergedPrHeadOids(repo: string, opts?: { runGh?: RunGh; limit?: number }): Map<string, string> {
120+
const runGh = opts?.runGh ?? defaultRunGh;
121+
const limit = opts?.limit ?? 500;
122+
const map = new Map<string, string>();
123+
const out = runGh(['pr', 'list', '--state', 'merged', '--json', 'headRefName,headRefOid', '--limit', String(limit)], repo);
124+
if (!out) return map; // conservative: no signal
125+
let arr: unknown;
126+
try { arr = JSON.parse(out); } catch { return map; }
127+
if (!Array.isArray(arr)) return map;
128+
for (const row of arr) {
129+
if (!row || typeof row !== 'object') continue;
130+
const name = (row as { headRefName?: unknown }).headRefName;
131+
const oid = (row as { headRefOid?: unknown }).headRefOid;
132+
if (typeof name === 'string' && typeof oid === 'string' && name && oid) {
133+
// Latest merged PR for a (reused) branch name wins — gh returns newest first.
134+
if (!map.has(name)) map.set(name, oid);
135+
}
136+
}
137+
return map;
138+
}
139+
91140
/**
92141
* Build the production git/fs-backed deps for the AgentWorktreeReaper.
93142
* `worktreesDir` bounds which `git worktree list` entries are considered (the
@@ -101,10 +150,18 @@ export function makeAgentWorktreeReaperDeps(opts: {
101150
/** Override the process-cwd scanner (testing). Returns the set of worktree
102151
* ROOT paths that currently have a live process cwd inside them. */
103152
cwdRoots?: () => Set<string>;
153+
/** When true (default), `isMerged` falls back to GitHub merged-PR state to
154+
* detect multi-commit squash-merges that `git cherry` cannot. Fail-safe:
155+
* any gh error degrades to cherry-only (KEEP). Set false to disable the
156+
* network call entirely (the legacy cherry-only behavior). */
157+
githubMergeCheck?: boolean;
158+
/** Override the merged-PR map source (testing). Returns headRefName→headRefOid. */
159+
mergedPrMap?: () => Map<string, string>;
104160
now?: () => number;
105161
}): AgentWorktreeReaperDeps {
106162
const readGit = opts.readGit ?? defaultReadGit;
107163
const repo = opts.instarRepo;
164+
const githubMergeCheck = opts.githubMergeCheck ?? true;
108165
const worktreesReal = (() => { try { return fs.realpathSync(opts.worktreesDir); } catch { return opts.worktreesDir; } })();
109166
const within = (p: string): boolean => {
110167
const real = (() => { try { return fs.realpathSync(p); } catch { return p; } })();
@@ -144,6 +201,19 @@ export function makeAgentWorktreeReaperDeps(opts: {
144201
return cwdCache;
145202
};
146203

204+
// Merged-PR map (headRefName→headRefOid), cached for a short TTL so a single
205+
// reap pass makes ONE `gh` call, not one per worktree. The map only fixes the
206+
// multi-commit-squash blind spot in `git cherry`; it is consulted lazily (only
207+
// when cherry says unmerged) so a fully cherry-detectable repo never calls gh.
208+
const mergedPrMapFn = opts.mergedPrMap ?? (() => fetchMergedPrHeadOids(repo));
209+
let prMapCache: Map<string, string> | null = null;
210+
let prMapCacheAt = 0;
211+
const mergedPrMapCached = (): Map<string, string> => {
212+
const t = Date.now();
213+
if (!prMapCache || t - prMapCacheAt > 60_000) { prMapCache = mergedPrMapFn(); prMapCacheAt = t; }
214+
return prMapCache;
215+
};
216+
147217
return {
148218
listWorktrees: (): WorktreeInfo[] => {
149219
let porcelain: string;
@@ -189,7 +259,18 @@ export function makeAgentWorktreeReaperDeps(opts: {
189259
isMerged: (info: WorktreeInfo): boolean => {
190260
const base = resolveBaseRef(readGit, repo);
191261
if (!base || !info.headSha) return false;
192-
return isBranchMerged(readGit, repo, base, info.headSha);
262+
// 1) Patch-id equivalence (fast, offline): fast-forward / merge / rebase /
263+
// single-commit-squash. Never false-positives "merged".
264+
if (isBranchMerged(readGit, repo, base, info.headSha)) return true;
265+
// 2) Multi-commit squash-merge: `git cherry` cannot see it (SHAs differ).
266+
// Consult GitHub merged-PR state — but require an EXACT head-OID match so
267+
// a branch with commits ADDED AFTER the merge is still KEPT (those would
268+
// be unmerged work). Fail-safe: an empty/missing map ⇒ KEEP.
269+
if (githubMergeCheck && info.branch) {
270+
const oid = mergedPrMapCached().get(info.branch);
271+
if (oid && oid === info.headSha) return true;
272+
}
273+
return false;
193274
},
194275

195276
isInUse: (p: string): boolean => {

tests/unit/agent-worktree-reaper.test.ts

Lines changed: 94 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import {
1212
type AgentWorktreeReaperDeps,
1313
type WorktreeInfo,
1414
} from '../../src/monitoring/AgentWorktreeReaper.js';
15-
import { isBranchMerged, resolveBaseRef, makeAgentWorktreeReaperDeps, REAPER_RESIDUE_DENYLIST, type ReadGit } from '../../src/monitoring/agentWorktreeGit.js';
15+
import { isBranchMerged, resolveBaseRef, makeAgentWorktreeReaperDeps, fetchMergedPrHeadOids, REAPER_RESIDUE_DENYLIST, type ReadGit, type RunGh } from '../../src/monitoring/agentWorktreeGit.js';
1616
import fs from 'node:fs';
1717
import os from 'node:os';
1818
import path from 'node:path';
@@ -131,6 +131,99 @@ describe('isBranchMerged (git cherry) — conservative, never false-positive', (
131131
});
132132
});
133133

134+
describe('fetchMergedPrHeadOids — gh-backed multi-commit-squash detection', () => {
135+
const ghOut = (rows: Array<{ headRefName: string; headRefOid: string }>): RunGh => () => JSON.stringify(rows);
136+
137+
it('parses gh merged-PR JSON into a headRefName→headRefOid map', () => {
138+
const m = fetchMergedPrHeadOids('/repo', { runGh: ghOut([
139+
{ headRefName: 'echo/feat-a', headRefOid: 'oidA' },
140+
{ headRefName: 'echo/feat-b', headRefOid: 'oidB' },
141+
]) });
142+
expect(m.get('echo/feat-a')).toBe('oidA');
143+
expect(m.get('echo/feat-b')).toBe('oidB');
144+
expect(m.size).toBe(2);
145+
});
146+
147+
it('keeps the FIRST (newest) entry when a branch name is reused', () => {
148+
const m = fetchMergedPrHeadOids('/repo', { runGh: ghOut([
149+
{ headRefName: 'echo/reused', headRefOid: 'newest' },
150+
{ headRefName: 'echo/reused', headRefOid: 'older' },
151+
]) });
152+
expect(m.get('echo/reused')).toBe('newest');
153+
});
154+
155+
it('fail-safe to EMPTY map when gh is unavailable (null) — conservative KEEP', () => {
156+
const m = fetchMergedPrHeadOids('/repo', { runGh: () => null });
157+
expect(m.size).toBe(0);
158+
});
159+
160+
it('fail-safe to EMPTY map on malformed JSON', () => {
161+
const m = fetchMergedPrHeadOids('/repo', { runGh: () => 'not json' });
162+
expect(m.size).toBe(0);
163+
});
164+
165+
it('ignores rows missing name or oid', () => {
166+
const m = fetchMergedPrHeadOids('/repo', { runGh: () => JSON.stringify([
167+
{ headRefName: 'ok', headRefOid: 'oidOk' },
168+
{ headRefName: '', headRefOid: 'x' },
169+
{ headRefOid: 'noName' },
170+
{ headRefName: 'noOid' },
171+
]) });
172+
expect(m.get('ok')).toBe('oidOk');
173+
expect(m.size).toBe(1);
174+
});
175+
});
176+
177+
describe('makeAgentWorktreeReaperDeps.isMerged — multi-commit squash via PR map', () => {
178+
// readGit: rev-parse resolves a base; cherry reports UNMERGED (a "+" commit),
179+
// which is exactly the multi-commit-squash blind spot.
180+
const cherryUnmergedGit: ReadGit = (args) => {
181+
if (args.includes('rev-parse')) return ''; // base resolves
182+
if (args.includes('cherry')) return '+ aaa\n+ bbb'; // looks unmerged
183+
throw new Error('unexpected git call: ' + args.join(' '));
184+
};
185+
const wt = (branch: string, headSha: string) => ({ path: '/x', branch, headSha });
186+
187+
it('MERGED when cherry says unmerged but a merged PR head-OID matches exactly', () => {
188+
const deps = makeAgentWorktreeReaperDeps({
189+
instarRepo: '/repo', worktreesDir: '/repo/.worktrees', readGit: cherryUnmergedGit,
190+
githubMergeCheck: true,
191+
mergedPrMap: () => new Map([['echo/feat', 'SQUASHED_OID']]),
192+
});
193+
expect(deps.isMerged(wt('echo/feat', 'SQUASHED_OID'))).toBe(true);
194+
});
195+
196+
it('KEEP when the branch advanced past the merged PR (head-OID mismatch = unmerged work)', () => {
197+
const deps = makeAgentWorktreeReaperDeps({
198+
instarRepo: '/repo', worktreesDir: '/repo/.worktrees', readGit: cherryUnmergedGit,
199+
githubMergeCheck: true,
200+
mergedPrMap: () => new Map([['echo/feat', 'MERGED_OID']]),
201+
});
202+
// worktree HEAD is a NEW commit added after the merge → must be KEPT
203+
expect(deps.isMerged(wt('echo/feat', 'NEWER_OID_WITH_UNMERGED_WORK'))).toBe(false);
204+
});
205+
206+
it('KEEP when no merged PR exists for the branch', () => {
207+
const deps = makeAgentWorktreeReaperDeps({
208+
instarRepo: '/repo', worktreesDir: '/repo/.worktrees', readGit: cherryUnmergedGit,
209+
githubMergeCheck: true,
210+
mergedPrMap: () => new Map(),
211+
});
212+
expect(deps.isMerged(wt('echo/feat', 'anySha'))).toBe(false);
213+
});
214+
215+
it('KEEP (legacy cherry-only) when githubMergeCheck is disabled — never calls the PR map', () => {
216+
const prMap = vi.fn(() => new Map([['echo/feat', 'SQUASHED_OID']]));
217+
const deps = makeAgentWorktreeReaperDeps({
218+
instarRepo: '/repo', worktreesDir: '/repo/.worktrees', readGit: cherryUnmergedGit,
219+
githubMergeCheck: false,
220+
mergedPrMap: prMap,
221+
});
222+
expect(deps.isMerged(wt('echo/feat', 'SQUASHED_OID'))).toBe(false);
223+
expect(prMap).not.toHaveBeenCalled();
224+
});
225+
});
226+
134227
describe('makeAgentWorktreeReaperDeps.isInUse — lock OR live process cwd', () => {
135228
let tmp: string;
136229
beforeEach(() => { tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'awr-inuse-')); });

0 commit comments

Comments
 (0)