Merge pull request #52 from alvinhui/main

Fix harness-creator bugs and sharpen skill triggering

Author: sanbuphy

skills/harness-creator/SKILL.md

@@ -1,9 +1,13 @@
 ---
 name: harness-creator
 description: >-
-  Build, audit, and improve lightweight harnesses for AI coding agents: AGENTS.md/CLAUDE.md,
-  feature state, verification workflows, scope boundaries, lifecycle handoff,
-  memory persistence, context control, tool safety, and multi-agent coordination.
+  Build, audit, and improve harnesses that make AI coding agents reliable: AGENTS.md/CLAUDE.md
+  instruction files, feature/state tracking, verification gates, scope boundaries, session
+  handoff, memory persistence, context budgets, tool-permission safety, and multi-agent
+  coordination. Use this whenever a coding agent is unreliable across sessions — forgets context,
+  drifts out of scope, claims "done" before tests pass, or starts each session inconsistently —
+  or when creating or assessing AGENTS.md, CLAUDE.md, feature_list.json, init.sh, progress.md, or
+  session-handoff files. Reach for it even if the user never says the word "harness."
 license: MIT
 ---
 
@@ -69,7 +73,7 @@ node skills/harness-creator/scripts/render-assessment-html.mjs --target /path/to
 node skills/harness-creator/scripts/run-benchmark.mjs --target /path/to/project --html /path/to/report.html
 ```
 
-Be clear that this is a structural benchmark. Real effectiveness still needs before/after agent sessions on representative tasks.
+Be clear that this is a structural benchmark. The benchmark first runs a self-check — it scaffolds a throwaway harness and validates it, proving the bundled scripts work end-to-end — then scores the target and eval coverage. Real effectiveness still needs before/after agent sessions on representative tasks.
 
 ## When to Read References
 

skills/harness-creator/SKILL.md.en

@@ -1,9 +1,13 @@
 ---
 name: harness-creator
 description: >-
-  Build, audit, and improve lightweight harnesses for AI coding agents: AGENTS.md/CLAUDE.md,
-  feature state, verification workflows, scope boundaries, lifecycle handoff,
-  memory persistence, context control, tool safety, and multi-agent coordination.
+  Build, audit, and improve harnesses that make AI coding agents reliable: AGENTS.md/CLAUDE.md
+  instruction files, feature/state tracking, verification gates, scope boundaries, session
+  handoff, memory persistence, context budgets, tool-permission safety, and multi-agent
+  coordination. Use this whenever a coding agent is unreliable across sessions — forgets context,
+  drifts out of scope, claims "done" before tests pass, or starts each session inconsistently —
+  or when creating or assessing AGENTS.md, CLAUDE.md, feature_list.json, init.sh, progress.md, or
+  session-handoff files. Reach for it even if the user never says the word "harness."
 license: MIT
 ---
 
@@ -69,7 +73,7 @@ node skills/harness-creator/scripts/render-assessment-html.mjs --target /path/to
 node skills/harness-creator/scripts/run-benchmark.mjs --target /path/to/project --html /path/to/report.html
 ```
 
-Be clear that this is a structural benchmark. Real effectiveness still needs before/after agent sessions on representative tasks.
+Be clear that this is a structural benchmark. The benchmark first runs a self-check — it scaffolds a throwaway harness and validates it, proving the bundled scripts work end-to-end — then scores the target and eval coverage. Real effectiveness still needs before/after agent sessions on representative tasks.
 
 ## When to Read References
 

skills/harness-creator/references/tool-registry-pattern.md

@@ -161,8 +161,7 @@ protected_commands:
 
 ## Related Patterns
 
-- [Lifecycle & Bootstrap](lifecycle-bootstrap-pattern.md) — How tools are registered at init
-- Hook Lifecycle (`hook-lifecycle-pattern.md`) — Pre/post tool execution hooks
+- [Lifecycle & Bootstrap](lifecycle-bootstrap-pattern.md) — How tools are registered at init, and pre/post-tool hook dispatch
 
 ## Template: Tool Safety Checklist
 

skills/harness-creator/scripts/lib/harness-utils.mjs

@@ -150,9 +150,14 @@ export function verificationCommands(project, explicitPackageManager) {
   };
 
   if (project.stack === 'python') {
+    // python3 is the portable name (many systems no longer ship a bare `python`).
+    // pytest exits 5 when it collects zero tests — harmless here, so don't let `set -e`
+    // treat "no tests yet" as a failure. compileall's -x skips virtualenvs and build
+    // artifacts so a syntax check doesn't choke on dependencies it shouldn't compile.
+    const py = 'python3';
     return [
-      'python -m pytest',
-      'python -m compileall .'
+      `${py} -m pytest || [ $? -eq 5 ]`,
+      `${py} -m compileall -q -x '(^|/)(\\.?venv|env|node_modules|build|dist|__pycache__)(/|$)' .`
     ];
   }
 
@@ -224,17 +229,17 @@ export function scoreHarness(files) {
   const checks = {
     instructions: [
       hasFile(byPath, ['AGENTS.md', 'CLAUDE.md'], 'Agent instruction file exists'),
-      textHas(agents, ['Startup Workflow', 'Before writing code'], 'Startup workflow documented'),
-      textHas(agents, ['Definition of Done', 'done only when'], 'Definition of done documented'),
-      textHas(agents, ['Verification Commands', './init.sh', 'test', 'verify'], 'Verification commands discoverable'),
-      textHas(agents, ['feature_list.json', 'progress.md'], 'State artifacts routed from instructions')
+      structuredHas(agents, ['Startup Workflow', 'Before writing code'], 'Startup workflow documented'),
+      structuredHas(agents, ['Definition of Done', 'done only when'], 'Definition of done documented'),
+      structuredHas(agents, ['Verification Commands', './init.sh', 'test', 'verify'], 'Verification commands discoverable'),
+      structuredHas(agents, ['feature_list.json', 'progress.md'], 'State artifacts routed from instructions')
     ],
     state: [
       hasFile(byPath, ['feature_list.json', 'feature-list.json'], 'Feature tracker exists'),
       jsonFeatureList(featureList, 'Feature tracker is valid and has feature fields'),
       hasFile(byPath, ['progress.md'], 'Progress log exists'),
-      textHas(progress, ['Current State', 'What', 'Next'], 'Progress log supports restart'),
-      textHas(handoff || progress, ['Blockers', 'Files', 'Next Session'], 'Handoff captures blockers/files/next step')
+      structuredHas(progress, ['Current State', 'What', 'Next'], 'Progress log supports restart'),
+      structuredHas(handoff || progress, ['Blockers', 'Files', 'Next Session'], 'Handoff captures blockers/files/next step')
     ],
     verification: [
       hasFile(byPath, ['init.sh'], 'Verification entrypoint exists'),
@@ -244,17 +249,17 @@ export function scoreHarness(files) {
       textHas(allText, ['Evidence', 'Verification Evidence', 'command and output'], 'Verification evidence is recorded')
     ],
     scope: [
-      textHas(agents, ['One feature at a time', 'one-feature-at-a-time'], 'One-feature-at-a-time rule exists'),
+      structuredHas(agents, ['One feature at a time', 'one-feature-at-a-time'], 'One-feature-at-a-time rule exists'),
       textHas(featureList, ['dependencies'], 'Feature dependencies are tracked'),
       textHas(agents + featureList, ['status'], 'Feature status is explicit'),
-      textHas(agents, ['Stay in scope', 'scope'], 'Scope boundary documented'),
-      textHas(agents, ['Definition of Done'], 'Completion gate limits scope closure')
+      structuredHas(agents, ['Stay in scope', 'scope'], 'Scope boundary documented'),
+      structuredHas(agents, ['Definition of Done'], 'Completion gate limits scope closure')
     ],
     lifecycle: [
       hasFile(byPath, ['init.sh'], 'Startup script exists'),
-      textHas(agents, ['End of Session', 'Before ending'], 'End-of-session procedure exists'),
+      structuredHas(agents, ['End of Session', 'Before ending'], 'End-of-session procedure exists'),
       hasFile(byPath, ['session-handoff.md'], 'Session handoff template exists'),
-      textHas(progress + handoff, ['Last Updated', 'Current Objective', 'Recommended Next Step'], 'Session restart markers exist'),
+      structuredHas(progress + '\n' + handoff, ['Last Updated', 'Current Objective', 'Recommended Next Step'], 'Session restart markers exist'),
       textHas(agents + init, ['restartable', 'clean', 'Next steps'], 'Clean restart path documented')
     ]
   };
@@ -272,7 +277,10 @@ export function scoreHarness(files) {
 
   const total = Object.values(subsystems).reduce((sum, item) => sum + item.score, 0);
   const overall = Math.round((total / (SUBSYSTEMS.length * 5)) * 100);
-  const bottleneck = Object.entries(subsystems).sort((a, b) => a[1].score - b[1].score)[0][0];
+  const ranked = Object.entries(subsystems).sort((a, b) => a[1].score - b[1].score);
+  // A bottleneck only means something when a subsystem is weaker than the rest.
+  // When every subsystem already maxes out, reporting one is misleading.
+  const bottleneck = ranked[0][1].score === 5 ? null : ranked[0][0];
   return { overall, bottleneck, subsystems };
 }
 
@@ -285,6 +293,31 @@ function textHas(text, needles, message) {
   return { pass: needles.some((needle) => lower.includes(needle.toLowerCase())), message };
 }
 
+// A real instruction doc carries its load-bearing phrases in structure — headings,
+// list items, tables, fenced code, or bold lead-ins — not in free prose. Scoring only
+// the structured lines means a genuine harness still passes, while a file that just
+// sprinkles the right keywords across a paragraph to game the score does not.
+function structuredText(markdown) {
+  const kept = [];
+  let inFence = false;
+  for (const raw of markdown.split(/\r?\n/)) {
+    const line = raw.trim();
+    if (/^(```|~~~)/.test(line)) { inFence = !inFence; continue; }
+    if (inFence) { kept.push(line); continue; }
+    if (!line) continue;
+    const isHeading = /^#{1,6}\s/.test(line);
+    const isList = /^([-*+]|\d+\.)\s/.test(line);
+    const isTable = line.startsWith('|');
+    const isBoldLead = /^\*\*[^*]+\*\*/.test(line);
+    if (isHeading || isList || isTable || isBoldLead) kept.push(line);
+  }
+  return kept.join('\n');
+}
+
+function structuredHas(markdown, needles, message) {
+  return textHas(structuredText(markdown), needles, message);
+}
+
 function jsonFeatureList(text, message) {
   try {
     const parsed = JSON.parse(text);
@@ -324,7 +357,7 @@ export function formatScoreReport(result, root = '.') {
   const lines = [
     `Harness validation for ${root}`,
     `Overall: ${result.overall}/100`,
-    `Bottleneck: ${result.bottleneck}`,
+    `Bottleneck: ${result.bottleneck ?? 'none — all subsystems at full score'}`,
     ''
   ];
 
@@ -378,7 +411,7 @@ export function htmlReport(result, title = 'Harness Assessment') {
       <p>Five-subsystem harness validation report.</p>
       <div class="summary">
         <div class="metric">Overall<strong>${result.overall}/100</strong></div>
-        <div class="metric">Bottleneck<strong>${escapeHtml(result.bottleneck)}</strong></div>
+        <div class="metric">Bottleneck<strong>${escapeHtml(result.bottleneck ?? 'none')}</strong></div>
       </div>
     </header>
     ${rows}

skills/harness-creator/scripts/run-benchmark.mjs

@@ -1,5 +1,9 @@
 #!/usr/bin/env node
+import { execFile } from 'node:child_process';
+import { mkdtemp, rm, writeFile } from 'node:fs/promises';
+import os from 'node:os';
 import path from 'node:path';
+import { promisify } from 'node:util';
 import { fileURLToPath } from 'node:url';
 import {
   formatScoreReport,
@@ -11,17 +15,20 @@ import {
   writeText
 } from './lib/harness-utils.mjs';
 
+const execFileAsync = promisify(execFile);
+
 const args = parseArgs(process.argv.slice(2));
 const scriptDir = path.dirname(fileURLToPath(import.meta.url));
 const skillRoot = path.resolve(scriptDir, '..');
 
 if (args.help) {
-  console.log(`Usage: node scripts/run-benchmark.mjs [--target DIR] [--output FILE] [--html FILE]
+  console.log(`Usage: node scripts/run-benchmark.mjs [--target DIR] [--output FILE] [--html FILE] [--no-self-check]
 
 Runs a lightweight harness benchmark:
-  1. Scores the current target harness.
-  2. Checks eval coverage in evals/evals.json.
-  3. Produces a JSON report and optional HTML report.
+  1. Self-check: scaffold a throwaway harness and confirm it validates (proves the scripts work).
+  2. Scores the current target harness.
+  3. Checks eval coverage in evals/evals.json.
+  4. Produces a JSON report and optional HTML report.
 
 This is a structural benchmark, not an LLM judge. Use it before/after real agent sessions.`);
   process.exit(0);
@@ -34,9 +41,11 @@ const evalPath = path.resolve(args.evals || path.join(skillRoot, 'evals', 'evals
 const harnessResult = scoreHarness(await loadHarnessFiles(target));
 const evals = await readJson(evalPath);
 const evalResult = scoreEvals(evals);
+const selfCheck = args.noSelfCheck ? { skipped: true } : await runSelfCheck();
 const report = {
   generatedAt: new Date().toISOString(),
   target,
+  selfCheck,
   harness: harnessResult,
   evals: evalResult,
   recommendation: recommend(harnessResult, evalResult)
@@ -45,6 +54,10 @@ const report = {
 await writeText(output, `${JSON.stringify(report, null, 2)}\n`);
 console.log(`Benchmark report written to ${output}`);
 console.log('');
+if (!selfCheck.skipped) {
+  console.log(`Self-check: ${selfCheck.pass ? 'PASS' : 'FAIL'} — scaffolded harness scored ${selfCheck.score}/100`);
+  if (!selfCheck.pass && selfCheck.error) console.log(`  ${selfCheck.error}`);
+}
 console.log(formatScoreReport(harnessResult, target));
 console.log(`Eval coverage: ${evalResult.score}/100 (${evalResult.passed}/${evalResult.total})`);
 console.log(`Recommendation: ${report.recommendation}`);
@@ -55,10 +68,39 @@ if (args.html) {
   console.log(`HTML benchmark report written to ${htmlPath}`);
 }
 
-if (harnessResult.overall < Number(args.minScore || 70) || evalResult.score < Number(args.minEvalScore || 80)) {
+if (
+  harnessResult.overall < Number(args.minScore || 70) ||
+  evalResult.score < Number(args.minEvalScore || 80) ||
+  selfCheck.pass === false
+) {
   process.exitCode = 1;
 }
 
+// Prove the bundled scripts actually work end-to-end: scaffold a harness into a throwaway
+// directory, then score it. A structural eval-coverage check can't catch a broken
+// create-harness.mjs — this can. Failure here means the skill ships broken, not just thin.
+async function runSelfCheck() {
+  let dir;
+  try {
+    dir = await mkdtemp(path.join(os.tmpdir(), 'harness-selfcheck-'));
+    await writeFile(
+      path.join(dir, 'package.json'),
+      JSON.stringify({ name: 'selfcheck', scripts: { check: 'tsc', test: 'vitest run', build: 'vite build' } })
+    );
+    await execFileAsync('node', [path.join(scriptDir, 'create-harness.mjs'), '--target', dir]);
+    const scored = scoreHarness(await loadHarnessFiles(dir));
+    return {
+      pass: scored.overall >= Number(args.minSelfCheckScore || 90),
+      score: scored.overall,
+      bottleneck: scored.bottleneck
+    };
+  } catch (error) {
+    return { pass: false, score: 0, error: error.message };
+  } finally {
+    if (dir) await rm(dir, { recursive: true, force: true });
+  }
+}
+
 function scoreEvals(evalsJson) {
   const cases = Array.isArray(evalsJson.evals) ? evalsJson.evals : [];
   const checks = [];
@@ -97,8 +139,14 @@ function recommend(harnessResult, evalResult) {
 }
 
 function renderBenchmarkHtml(report) {
+  const selfCheckSection = report.selfCheck?.skipped
+    ? ''
+    : `<section>
+      <h2>Script Self-Check <span>${report.selfCheck.pass ? 'PASS' : 'FAIL'}</span></h2>
+      <p>Scaffolded a throwaway harness and scored it ${report.selfCheck.score}/100 — confirms the bundled scripts run end-to-end.${report.selfCheck.error ? ` Error: ${escapeHtml(report.selfCheck.error)}` : ''}</p>
+    </section>`;
   const evalHtml = htmlReport(report.harness, `Harness Benchmark: ${path.basename(report.target)}`)
-    .replace('</main>', `<section>
+    .replace('</main>', `${selfCheckSection}<section>
       <h2>Eval Coverage <span>${report.evals.score}/100</span></h2>
       <p>${report.evals.passed}/${report.evals.total} benchmark checks passed across ${report.evals.cases} eval cases.</p>
       <ul>${report.evals.checks.map((check) => `<li class="${check.pass ? 'pass' : 'fail'}">${check.pass ? 'PASS' : 'FAIL'} ${escapeHtml(check.message)}</li>`).join('')}</ul>

skills/harness-creator/templates/init.sh

@@ -44,8 +44,11 @@ if [ -f package.json ]; then
   }
 elif [ -f pyproject.toml ] || [ -f requirements.txt ]; then
   echo "=== Running Python verification ==="
-  python -m pytest
-  python -m compileall .
+  PY="$(command -v python3 || command -v python)"
+  # pytest exits 5 when no tests are collected — not a failure for a fresh project.
+  "$PY" -m pytest || [ $? -eq 5 ]
+  # -x skips virtualenvs/build dirs so the syntax check doesn't compile dependencies.
+  "$PY" -m compileall -q -x '(^|/)(\.?venv|env|node_modules|build|dist|__pycache__)(/|$)' .
 elif [ -f go.mod ]; then
   echo "=== Running Go verification ==="
   go test ./...