-
Notifications
You must be signed in to change notification settings - Fork 10
Expand file tree
/
Copy pathquality-check.sh
More file actions
executable file
·692 lines (635 loc) · 30.4 KB
/
Copy pathquality-check.sh
File metadata and controls
executable file
·692 lines (635 loc) · 30.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
#!/bin/bash
# Quality Check Script for BESS Manager
# Run this script before committing to ensure all files meet quality standards
set -e
echo "🔍 Running BESS Manager Quality Checks..."
echo "========================================"
# Check if we're in the right directory
if [ ! -f "CLAUDE.md" ]; then
echo "❌ Error: Run this script from the project root directory"
exit 1
fi
# Initialize counters
ERRORS=0
WARNINGS=0
# Resolve a Python tool: the project venv first, then PATH. Prints nothing and
# returns 1 when the tool is available in neither.
#
# The venv lookup matters because the documented way to run anything here is
# `.venv/bin/<tool>` (see CLAUDE.md) — a fresh git worktree has a .venv but
# usually no activated shell, so a bare `command -v pytest` finds nothing.
#
# A missing tool is an ERROR, not a warning: this script is the pre-commit
# gate, and skipping its three most important checks while printing
# "Errors: 0" reports success for a run that verified nothing. That is exactly
# how Black violations reached CI from a fresh worktree.
py_tool() {
if [ -x ".venv/bin/$1" ]; then
echo ".venv/bin/$1"
elif command -v "$1" >/dev/null 2>&1; then
echo "$1"
else
return 1
fi
}
echo ""
echo "📋 Running Python tests..."
echo "---------------------------"
if PYTEST=$(py_tool pytest); then
echo "🔸 Running fast tests (use '$PYTEST' directly to include slow algorithm tests)..."
if ! "$PYTEST" -m "not slow" --tb=short -q; then
echo "❌ Tests failed"
ERRORS=$((ERRORS + 1))
else
echo "✅ Fast tests passed"
fi
else
echo "❌ pytest not found in .venv/bin or on PATH — cannot verify tests."
echo " Install with: python3 -m venv .venv && .venv/bin/pip install -r requirements-dev.txt"
ERRORS=$((ERRORS + 1))
fi
echo ""
echo "📋 Checking Python code quality..."
echo "-----------------------------------"
# Black and Ruff violations are ERRORs, not warnings: both are hard CI
# failures, so a gate that reports them as warnings and still exits 0 sends
# code to CI that is already known to fail.
#
# Check if Python files exist
if find . -name "*.py" -not -path "./build/*" -not -path "./.venv/*" -not -path "./frontend/node_modules/*" | grep -q .; then
# Run Black formatting check
if BLACK=$(py_tool black); then
echo "🔸 Checking Black formatting..."
if ! "$BLACK" --check . --exclude="/(build|\.venv|node_modules)/" >/dev/null 2>&1; then
echo "❌ Black formatting issues found. Run: $BLACK ."
ERRORS=$((ERRORS + 1))
else
echo "✅ Black formatting OK"
fi
else
echo "❌ Black not found in .venv/bin or on PATH — cannot verify formatting."
echo " Install with: python3 -m venv .venv && .venv/bin/pip install -r requirements-dev.txt"
ERRORS=$((ERRORS + 1))
fi
# Run Ruff linting check
if RUFF=$(py_tool ruff); then
echo "🔸 Checking Ruff linting..."
if ! "$RUFF" check . --exclude="build,.venv,node_modules" >/dev/null 2>&1; then
echo "❌ Ruff linting issues found. Run: $RUFF check --fix ."
ERRORS=$((ERRORS + 1))
else
echo "✅ Ruff linting OK"
fi
else
echo "❌ Ruff not found in .venv/bin or on PATH — cannot verify linting."
echo " Install with: python3 -m venv .venv && .venv/bin/pip install -r requirements-dev.txt"
ERRORS=$((ERRORS + 1))
fi
# mypy, scoped to files this branch actually changed.
#
# `docs/agents/rules.md` requires mypy to pass, but nothing ran it: not
# this gate, not CI, and it was not even in requirements-dev.txt. Turning
# it on repo-wide is not an option — a measured 2914 errors across 191
# files, 417 of them outside tests. So the rule is enforced going forward
# instead of retroactively: code you touch must type-check, and the legacy
# backlog burns down as files get edited.
#
# Three sources, because each misses something the others catch:
# committed-on-this-branch, uncommitted-but-tracked, and untracked. The
# last one matters most — a brand-new file is untracked until its first
# commit, and this gate is meant to run BEFORE that commit. Leaving it out
# let a probe file with a known error pass the gate silently.
#
# A file deleted on this branch still appears in the diff, hence the -e
# test. No changed Python files is a pass, not a skip-with-warning.
#
# An unresolvable origin/main is an ERROR, not a warning: warnings exit
# 0, so a run that type-checked nothing would print "Errors: 0" and
# report success for a check that never happened.
if MYPY=$(py_tool mypy); then
echo "🔸 Checking mypy on changed files..."
base=$(git merge-base origin/main HEAD 2>/dev/null || echo "")
if [ -z "$base" ]; then
echo "❌ Cannot resolve origin/main — mypy checked nothing."
echo " Run: git fetch origin main"
ERRORS=$((ERRORS + 1))
else
# sort -u is load-bearing: a file changed on the branch AND dirty
# in the working tree appears in both diffs, and mypy fails with
# "Duplicate module named ..." when handed the same path twice.
changed=""
while IFS= read -r f; do
case "$f" in *.py) [ -e "$f" ] && changed="$changed $f" ;; esac
done <<EOF
$( { git diff --name-only "$base" HEAD; git diff --name-only HEAD; git ls-files --others --exclude-standard; } | sort -u)
EOF
if [ -z "$changed" ]; then
echo "✅ mypy OK (no changed Python files)"
elif ! "$MYPY" --explicit-package-bases --ignore-missing-imports $changed >/dev/null 2>&1; then
echo "❌ mypy errors in changed files. Run:"
echo " $MYPY --explicit-package-bases --ignore-missing-imports$changed"
ERRORS=$((ERRORS + 1))
else
echo "✅ mypy OK (changed files)"
fi
fi
else
echo "❌ mypy not found in .venv/bin or on PATH — cannot verify types."
echo " Install with: .venv/bin/pip install -r requirements-dev.txt"
ERRORS=$((ERRORS + 1))
fi
else
echo "ℹ️ No Python files found to check"
fi
echo ""
echo "📋 Checking TypeScript code quality..."
echo "--------------------------------------"
# Check if TypeScript files exist in frontend
if [ -d "frontend" ] && find frontend/src -name "*.ts" -o -name "*.tsx" 2>/dev/null | grep -q .; then
cd frontend
# Check if package.json exists
if [ -f "package.json" ]; then
# Run frontend tests
if command -v npm >/dev/null 2>&1; then
echo "🔸 Running frontend tests..."
if npm test 2>/dev/null; then
echo "✅ Frontend tests passed"
else
echo "❌ Frontend tests failed"
ERRORS=$((ERRORS + 1))
fi
echo "🔸 Checking TypeScript compilation..."
if npm run type-check >/dev/null 2>&1; then
echo "✅ TypeScript compilation OK"
else
echo "⚠️ TypeScript compilation issues found. Run: npm run type-check"
WARNINGS=$((WARNINGS + 1))
fi
# Check ESLint
echo "🔸 Checking ESLint..."
if npm run lint >/dev/null 2>&1; then
echo "✅ ESLint OK"
else
echo "⚠️ ESLint issues found. Run: npm run lint:fix"
WARNINGS=$((WARNINGS + 1))
fi
else
echo "⚠️ npm not installed. Install Node.js and npm"
WARNINGS=$((WARNINGS + 1))
fi
else
echo "⚠️ No package.json found in frontend directory"
WARNINGS=$((WARNINGS + 1))
fi
cd ..
else
echo "ℹ️ No TypeScript files found to check"
fi
echo ""
echo "📋 Checking permission surface..."
echo "-------------------------------------------"
# Replaces the hook-matrix gate deleted with the hooks (#588). verify-sandbox.sh
# cannot fill that role -- it exits 2 unless the Bash tool runs it in a
# sandboxed session, so it can never be a CI or pre-commit check. What IS
# statically checkable is that the rules which stand in for the deleted hooks
# are still present. Every entry below was a real regression at some point:
# option-first `git stash` forms fell through to `auto` because the deny list
# enumerated literal subcommands, and the GitHub-publishing guards were dropped
# entirely -- effects the sandbox cannot contain, since it bounds the
# filesystem, not the network.
# `if ! ...` is load-bearing: `set -e` (line 6) aborts the whole script on a
# bare failing statement, so a plain heredoc here would skip the ERRORS
# increment, the checks below it, AND the final summary -- a missing rule would
# stop the run mid-file with no verdict, which is the opposite of a gate.
if ! python3 - <<'PY'
import json, re, sys
# Patterns match the command AS WRITTEN -- prefix globbing, no normalisation.
# `gh api` is guarded by a BLANKET rule on purpose: the dangerous shapes put
# their marker at an arbitrary argument position (`gh api <path> -X PUT`,
# `gh api <path> -f k=v`), which a prefix glob cannot reach. Enumerating them
# left real holes twice. Narrowing it back to specific forms re-opens the
# holes, so the check requires the blanket spelling rather than merely "some
# rule exists".
#
# `git push` USED to be in that same sentence and no longer is. It was never
# the glob that made it safe -- the glob was a blunt instrument compensating
# for having no guard at the only layer that can actually see a ref update.
# The four leaks it was blanket-guarding against are now refused SERVER-SIDE
# by GitHub rulesets, which evaluate the resulting ref rather than the command
# string, so argument order cannot evade them:
#
# git push origin main --force -> non_fast_forward on ~DEFAULT_BRANCH
# git push origin +beta-release-9.9 -> non_fast_forward on beta-release-*
# git push origin +release-9.9 -> non_fast_forward on release-*
# git push origin --delete <ref> -> deletion on both of the above
# git push origin v9.9.0 (force/move) -> non_fast_forward on ~ALL tags
#
# All rulesets are enforcement=active with an EMPTY bypass_actors list, so
# they bind the repo owner too -- which matters, because local pushes
# authenticate as the owner (osxkeychain), never as bess-agent.
#
# Do NOT re-add a `Bash(git push*)` ask on the grounds that "a guard is
# missing". Check the rulesets first:
#
# gh api repos/johanzander/bess-manager/rulesets
# gh api repos/johanzander/bess-manager-beta/rulesets
#
# What this deliberately does NOT cover: force-pushing or deleting a FEATURE
# branch (fix/**, feat/**) on origin. An accepted residual -- but NOT because
# "the damage is bounded to your own unmerged branch". ~20 worktrees push in
# parallel as the same identity, so a misaimed --force destroys another agent's
# commits and closes its PR, with the recovering reflog sitting in a different
# worktree.
#
# Every entry below is a rule whose deletion is the exact regression this gate
# was written for -- the GitHub-reaching and history-destroying guards. Keep
# this list in sync with the ask/deny lists; a rule absent from here is a rule
# that can be silently removed.
REQUIRED = {
"deny": [
"Bash(git stash -*)", "Bash(git stash --*)", "Bash(git stash)",
"Bash(git -* stash)", "Bash(git -* stash pop*)",
"Bash(git -* stash drop*)", "Bash(git -* stash clear*)",
"Bash(podman machine rm)", "Bash(podman system reset)",
],
"ask": [
"Bash(gh api)", "Bash(gh api *)",
"Bash(gh pr merge*)", "Bash(gh repo edit*)",
"Bash(gh release create*)", "Bash(gh release edit*)",
"Bash(gh release delete*)", "Bash(gh release delete-asset*)",
"Bash(gh release upload*)",
"Bash(gh repo delete*)", "Bash(gh secret*)", "Bash(gh workflow run*)",
"Bash(git gc*)", "Bash(git prune*)", "Bash(git repack*)",
"Bash(git maintenance*)",
"Bash(git reflog expire*)", "Bash(git reflog delete*)",
"Bash(git update-ref*)",
"Bash(git tag -d*)", "Bash(git tag --delete*)", "Bash(git tag -f*)",
"Bash(sudo *)",
],
}
# Presence checks alone kept passing while real command spellings slipped
# through -- four review rounds of the same class of bug. So also assert, per
# COMMAND STRING, that something actually matches it. Patterns are prefix
# globs over the command as written, which is the semantics that keeps
# surprising people: `git stash pop` is covered while `git -C x stash pop` is
# not, because the rule anchors on the literal `git stash`.
#
# Add a line here whenever a new spelling is found in the wild. A rule that
# looks right and matches nothing is the failure mode this exists to catch.
# MUST_BE_DENIED is checked against `deny` ONLY. Checking these against
# deny+ask would certify a one-keystroke `ask` for commands policy says are
# unapprovable -- and that is not hypothetical: an earlier `Bash(git -*)` ask
# matched `git -C x stash pop` while no deny did, silently downgrading the
# stash prohibition to a prompt, and a deny+ask gate reported it green.
MUST_BE_DENIED = [
# Every mutating stash form, in both plain and global-option spellings.
# `clear` and `drop` destroy other agents' entries irreversibly.
"git stash", "git stash push", "git stash push -u", "git stash save wip",
"git stash pop", "git stash apply", "git stash apply stash@{0}",
"git stash drop", "git stash drop stash@{1}", "git stash clear",
"git stash branch tmp", "git stash store abc123", "git stash create",
"git stash -u", "git stash --include-untracked",
"git -C ../bess-manager-feature stash pop",
"git -C .claude/worktrees/x stash drop",
"git --git-dir=/tmp/r/.git stash drop",
# The shared podman VM: destruction is unrecoverable and outside the sandbox.
"podman machine rm", "podman system reset",
]
# Checked against deny + ask: a prompt is an acceptable outcome for these.
MUST_BE_GUARDED = [
# git's global options may precede the subcommand -- the hook this
# replaced normalised for exactly this, and local-agent-environment.md
# teaches `git -C` as
# the cross-checkout idiom, so it is the spelling most likely to be used.
"git --no-pager gc --prune=now",
"git -C sub tag -d v9.9.0",
"git -C sub update-ref -d refs/heads/x",
# history destruction, incl. the spellings that are NOT `gc`/`reflog expire`
"git tag --delete v9.9.0", "git tag -d v9.9.0",
"git tag -f v9.9.0 abc123",
"git reflog expire --expire=now --all", "git reflog delete HEAD@{0}",
"git gc --prune=now", "git prune", "git repack -d",
"git maintenance run --task=gc",
"git update-ref -d refs/tags/v9.9.0",
# gh reaching GitHub, including the raw API path
"gh api repos/o/r/pulls/1/merge -X PUT",
"gh api repos/o/r/releases -f tag_name=v1",
"gh pr merge 588 --squash",
# Publishing a release escapes to GitHub irreversibly. The read-only verbs
# are exempt (see MUST_NOT_BE_GUARDED) -- the gate pins both directions so
# the split cannot silently collapse back to a blanket rule or a hole.
"gh release create v9.9.0", "gh release create v9.9.0 -R owner/repo",
"gh release edit v9.9.0 --draft=false",
"gh release delete v9.9.0 --yes",
"gh release delete-asset v9.9.0 addon.zip",
"gh release upload v9.9.0 addon.zip",
"gh secret set FOO", "gh workflow run ci.yml", "gh repo edit --visibility private",
# Unrecoverable gh mutations. `gh repo delete` was unguarded while the far
# milder `gh repo edit` asked -- it escapes to GitHub and git cannot undo
# it, which is this file's stated standard for the ask list.
"gh repo delete owner/repo --yes", "gh pr close 1",
"gh issue delete 1", "gh cache delete --all",
"sudo rm -rf /",
]
# NOT modelled here, deliberately:
#
# * COMPOUND COMMANDS. `cd frontend && git push origin main` matches nothing in
# this matcher, because matches() emulates a single command string. The
# harness decomposes on &&/;/| before applying rules, so the real decision is
# made on `git push origin main` -- do not "fix" this by adding compound
# entries here; they would fail against a matcher that is correct for what it
# models. Verifying the decomposition claim needs a live permission test, not
# this gate.
# * GREEDY GLOBS. `*` compiles to `.*`, which spans spaces, so `git -* push*`
# also matches e.g. a commit whose MESSAGE contains " push". That is a false
# PROMPT, not a hole, and it is accepted: the alternative is dropping the
# global-option guard on push, which is a real bypass. Precision here is
# bounded by prefix globbing -- when the choice is between an extra prompt
# and a gap, take the prompt.
# Read-only git must NOT be caught: an autonomous run executing rules.md's
# cross-checkout procedure (`git diff -- f | git -C <wt> apply`, then
# `git -C <wt> diff -- f` to verify) would otherwise stall on inspection
# commands. A blanket `Bash(git -*)` did exactly that, which is why the
# global-option rules name a verb.
MUST_NOT_BE_GUARDED = [
"git -C sub status --short",
"git --no-pager log --oneline -5",
"git -C .claude/worktrees/x diff -- file.py",
"git -C .claude/worktrees/x apply",
"git status", "git diff", "git log --oneline",
# Pushing. Enforcement moved to GitHub rulesets (see the REQUIRED comment
# above), so these must run UNATTENDED -- implement-issue Step 9 and every
# PR-refresh push. Pinned in this direction on purpose: the failure mode
# being guarded against is someone re-adding a `Bash(git push*)` ask
# because it "looks unguarded", which would silently restore the stall the
# rulesets were created to remove. If you believe a guard is missing, read
# the rulesets before touching this list.
#
# These entries assert only that the commands are LOCALLY unguarded. Which
# of them GitHub also refuses is a separate question, and the answer is not
# "all of them" -- do not read this list as a protection matrix:
#
# main --force, +beta-release-*, v9.9.0 (tag) -> refused by a ruleset
# +release-X.Y (rewrite) -> refused by a ruleset
# --delete release-X.Y -> ALLOWED, deliberately
#
# That last line is a deliberate asymmetry, not a residual and not an
# oversight to "fix" by putting the prompt back. `release-X.Y` is the
# short-lived hotfix branch created by the release skill (steps 2-6);
# rewriting it is refused by the `release-*` ruleset, while deleting it
# once the release is out is ordinary cleanup. The protected TAG pins the
# released commit, so a spent branch costs nothing to lose.
"git push", "git push -u origin main",
"git push origin main --force",
"git push origin +beta-release-9.9",
"git push origin --delete release-9.9",
"git push origin v9.9.0",
"git -C .claude/worktrees/x push origin main",
"git -c push.default=current push beta main",
# Read-only stash inspection, which local-agent-environment.md and
# rules.md both promise
# keeps working. A blanket `git -* stash *` DENY caught these, and deny has
# no override -- so the cross-checkout recipe was hard-blocked, not merely
# prompted. That is why the stash twins name a verb.
"git stash list", "git stash show",
"git -C sub stash list",
"git -C .claude/worktrees/x stash show",
# implement-issue Step 4 prunes worktrees in a loop;
# local-agent-environment.md argues that
# must stay unattended. A `git -* prune*` twin caught `worktree prune` and
# `remote prune` through the same greedy glob, so the twin was dropped.
"git -C /main worktree prune", "git worktree prune",
"git -C sub remote prune origin",
# Reading releases changes nothing on GitHub, and the release/beta flows
# check the published version constantly ("always check the current
# published version before tagging"). A blanket `gh release*` ask made
# every one of those a prompt, which is why these verbs are named.
"gh release list -L 5 -R johanzander/bess-manager-beta",
"gh release list", "gh release view v9.9.0",
"gh release view --json tagName",
]
def matches(pattern: str, command: str) -> bool:
"""Prefix-glob match over the raw command, mirroring the documented rules."""
inner = pattern[len("Bash(") : -1] if pattern.startswith("Bash(") else pattern
return re.fullmatch(re.escape(inner).replace(r"\*", ".*"), command) is not None
perms = json.load(open(".claude/settings.json"))["permissions"]
bad = [(k, p) for k, ps in REQUIRED.items() for p in ps if p not in perms.get(k, [])]
for k, p in bad:
print(f"❌ permissions.{k} is missing {p}")
denies = perms.get("deny", [])
guards = denies + perms.get("ask", [])
for cmd in MUST_BE_DENIED:
if not any(matches(p, cmd) for p in denies):
why = " (only an ask matches -- deny > ask, so this is a prompt, not a block)" \
if any(matches(p, cmd) for p in guards) else ""
print(f"❌ no DENY rule matches: {cmd}{why}")
bad.append(("deny", cmd))
for cmd in MUST_BE_GUARDED:
if not any(matches(p, cmd) for p in guards):
print(f"❌ no deny/ask rule matches: {cmd}")
bad.append(("ask", cmd))
for cmd in MUST_NOT_BE_GUARDED:
hit = [p for p in guards if matches(p, cmd)]
if hit:
print(f"❌ read-only command is gated by {hit[0]}: {cmd}")
bad.append(("ask", cmd))
if bad:
sys.exit(1)
total = len(MUST_BE_DENIED) + len(MUST_BE_GUARDED) + len(MUST_NOT_BE_GUARDED)
print(f"✅ Permission surface intact ({total} command shapes checked, "
f"{len(MUST_BE_DENIED)} require deny, {len(MUST_NOT_BE_GUARDED)} must stay unattended)")
PY
then
ERRORS=$((ERRORS + 1))
fi
echo ""
echo "📋 Checking bot workflow publish contract..."
echo "-------------------------------------------"
# Stage 2 (issue-analyze.yml) spent three days exiting `success` while posting
# nothing -- five of nine non-skipped runs produced no comment and no label,
# burning $0.33-1.58 each (#646). Every layer reported healthy: the run was
# green, is_error false, permission_denials_count 0.
#
# Two independent things have to hold, and this gate checks BOTH because either
# one alone leaves the failure silent:
#
# 1. The prompt must tell the agent to RUN a command to publish. Stage 2 said
# "Post ONE comment on the issue with this structure:" followed by a
# markdown template -- a description of a document, not an instruction to
# execute anything. These workflows run in AGENT mode (use_sticky_comment
# false, track_progress false), so the action posts nothing on the agent's
# behalf: an agent that renders the template as its final assistant message
# has, by its own lights, finished. It then never reaches the labelling
# step, which is why the label was untouched too.
#
# The contrast is what makes this more than a story. On the SAME action
# version (459ad358 / CLI 2.1.234), Stage 4's review -- whose prompt names
# `gh pr review` explicitly -- landed an APPROVED verdict, while Stage 2
# went silent three times in three seconds. Stage 2 was the only bot
# workflow whose publish step was prose, and the only one that failed to
# publish.
#
# 2. The workflow must VERIFY it afterwards. The prompt fix is a behavioural
# argument about what a model will infer, so it cannot be trusted on its
# own -- the next upstream version may infer differently, exactly as this
# one did. Only a post-condition on the job makes that loud instead of
# green.
#
# Deliberately NOT checked for, and deliberately not built: a step that posts
# the comment on the agent's behalf when it didn't. That is a second publishing
# path whose only job is to route around the first one failing, and it would
# mask the regression rather than surface it (docs/agents/rules.md, Debugging
# Protocol step 8). The guard asserts; it does not repair.
if ! python3 - <<'PY'
import pathlib, re, sys
WORKFLOWS = pathlib.Path(".github/workflows")
# Workflows whose whole product is a comment on the issue. Stage 3 is excluded
# on purpose: its product is a PR, which is observable without this check.
PUBLISHERS = {
"issue-triage.yml": "gh issue comment",
"issue-analyze.yml": "gh issue comment",
}
bad = []
for name, command in PUBLISHERS.items():
path = WORKFLOWS / name
if not path.is_file():
print(f"❌ {name} is missing -- the publish contract cannot be checked")
bad.append(name)
continue
if command not in path.read_text():
print(
f"❌ {name} never names `{command}`. Its publish step is prose, so "
"the agent can render it as a final message and exit success "
"having written nothing to GitHub (#646)."
)
bad.append(name)
# The post-agent guard: a run that published nothing must fail the job. Checked
# by the shape that actually does the work -- a step keyed on `if: always()`
# that re-reads the issue -- rather than by step name, which renames freely.
analyze = WORKFLOWS / "issue-analyze.yml"
if analyze.is_file():
text = analyze.read_text()
has_always = re.search(r"^\s*if:\s*always\(\)\s*$", text, re.MULTILINE)
has_readback = "--json comments" in text and "--json labels" in text
if not (has_always and has_readback):
print(
"❌ issue-analyze.yml has no post-agent verification step. A run "
"that posts no comment and applies no label must FAIL the job, "
"not exit success (#646 acceptance criterion)."
)
bad.append("issue-analyze.yml:guard")
if bad:
sys.exit(1)
print(f"✅ Bot workflow publish contract intact ({len(PUBLISHERS)} publishers "
"name their command, Stage 2 verifies it posted)")
PY
then
ERRORS=$((ERRORS + 1))
fi
echo ""
echo "📋 Checking agent context budget..."
echo "-------------------------------------------"
# Every bot stage loads CLAUDE.md before it does anything, on the main agent
# and again on its sub-agent, and re-sends it on every turn. It is the largest
# single item in the fixed context floor, so its size is a per-run cost on
# Stages 1-5 and on every local session.
#
# It reached 45,488 B once, of which 33,743 B (74%) was one section describing
# this machine -- the macOS sandbox, podman, worktrees, Playwright, the
# permission rules. None of it can apply on a fresh ubuntu runner under
# `--permission-mode bypassPermissions`, and it was billed on every turn of
# every stage anyway (#650).
#
# The fix was to RELOCATE that content, not delete it: it lives in
# docs/agents/local-agent-environment.md and is reached through the Agent
# Documentation Index like every other doc. This gate keeps it from creeping
# back inline. If you need to raise the cap, move content into docs/agents/
# and link it instead -- that is the mechanism CLAUDE.md already uses.
CLAUDE_MD_MAX_BYTES=16000
CLAUDE_MD_BYTES=$(wc -c < CLAUDE.md | tr -d ' ')
if [ "$CLAUDE_MD_BYTES" -gt "$CLAUDE_MD_MAX_BYTES" ]; then
echo "❌ CLAUDE.md is ${CLAUDE_MD_BYTES} B, over the ${CLAUDE_MD_MAX_BYTES} B budget"
echo " Every bot stage loads this file on every turn, on two agents."
echo " Move the new material into docs/agents/ and link it from the"
echo " Agent Documentation Index rather than raising the cap (#650)."
ERRORS=$((ERRORS + 1))
else
echo "✅ CLAUDE.md within context budget (${CLAUDE_MD_BYTES} B / ${CLAUDE_MD_MAX_BYTES} B)"
fi
echo ""
echo "📋 Checking scenario discovery coverage..."
echo "-------------------------------------------"
SCENARIO_DIR="scripts/mock_ha/scenarios"
MISSING_DISCOVERY=0
if [ -d "$SCENARIO_DIR" ]; then
for f in "$SCENARIO_DIR"/ci-wizard-*.json; do
name=$(basename "$f")
if ! python3 -c "import json,sys; sys.exit(0 if 'expected_discovery' in json.load(open('$f')) else 1)" 2>/dev/null; then
echo "❌ $name is missing expected_discovery section"
MISSING_DISCOVERY=$((MISSING_DISCOVERY + 1))
fi
done
if [ $MISSING_DISCOVERY -eq 0 ]; then
echo "✅ All ci-wizard-* scenarios have expected_discovery"
else
echo "❌ $MISSING_DISCOVERY scenario(s) missing expected_discovery — add assertions before releasing"
ERRORS=$((ERRORS + 1))
fi
else
echo "ℹ️ No scenario directory found"
fi
echo ""
echo "📋 Checking Markdown files..."
echo "------------------------------"
# Find project markdown files (exclude node_modules, build, .venv)
MD_FILES=$(find . -name "*.md" -not -path "./node_modules/*" -not -path "./build/*" -not -path "./.venv/*" -not -path "./frontend/node_modules/*" -not -path "./.git/*" -not -path "./.pytest_cache/*" 2>/dev/null | head -20)
if [ -n "$MD_FILES" ]; then
echo "🔸 Found markdown files:"
echo "$MD_FILES" | sed 's/^/ /'
# Check for common markdown issues
echo "🔸 Checking for common markdown issues..."
# Check for trailing spaces
if echo "$MD_FILES" | xargs grep -l " $" 2>/dev/null | grep -q .; then
echo "⚠️ Files with trailing spaces found:"
echo "$MD_FILES" | xargs grep -l " $" 2>/dev/null | sed 's/^/ /'
WARNINGS=$((WARNINGS + 1))
fi
# Check for multiple consecutive blank lines
if echo "$MD_FILES" | xargs grep -l "^$" 2>/dev/null | xargs grep -Pzo "\n\n\n" 2>/dev/null | grep -q .; then
echo "⚠️ Files with multiple consecutive blank lines found"
WARNINGS=$((WARNINGS + 1))
fi
# Check for missing blank lines before headers
HEADER_ISSUES=0
for file in $MD_FILES; do
if grep -Pzl ".*[^\n]\n#" "$file" 2>/dev/null; then
HEADER_ISSUES=$((HEADER_ISSUES + 1))
fi
done
if [ $HEADER_ISSUES -gt 0 ]; then
echo "⚠️ $HEADER_ISSUES files with headers missing blank lines"
WARNINGS=$((WARNINGS + 1))
fi
if [ $WARNINGS -eq 0 ] || [ $HEADER_ISSUES -eq 0 ]; then
echo "✅ Basic markdown formatting OK"
fi
else
echo "ℹ️ No markdown files found to check"
fi
echo ""
echo "📋 Summary"
echo "----------"
echo "Errors: $ERRORS"
echo "Warnings: $WARNINGS"
if [ $ERRORS -eq 0 ] && [ $WARNINGS -eq 0 ]; then
echo "🎉 All quality checks passed!"
exit 0
elif [ $ERRORS -eq 0 ]; then
echo "⚠️ Quality checks completed with $WARNINGS warnings"
echo "💡 Consider fixing warnings before committing"
exit 0
else
echo "❌ Quality checks failed with $ERRORS errors and $WARNINGS warnings"
echo "🔧 Please fix all errors before committing"
exit 1
fi