docs(casebook): ecosystem audit edition — 33 verified findings, metho… #103
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: conformance | |
| on: | |
| push: | |
| paths: | |
| - 'standard/**' | |
| - 'reference/**' | |
| - 'conformance/**' | |
| - 'schemas/**' | |
| - 'registry/**' | |
| - 'scripts/**' | |
| - 'docs/**' | |
| - 'product/**' | |
| - 'extensions/**' | |
| - 'whitepaper/**' | |
| - 'sdk/**' | |
| - 'lab/**' | |
| - 'examples/**' | |
| - 'benchmark/**' | |
| - 'healthcheck/**' | |
| - 'reports/**' | |
| - 'bundles/**' | |
| - 'README*' | |
| - 'ROADMAP.md' | |
| - 'GOVERNANCE.md' | |
| - 'CONTRIBUTING.md' | |
| - 'LEGACY-MIGRATION.md' | |
| - 'verify_vectors.py' | |
| workflow_dispatch: | |
| pull_request: | |
| jobs: | |
| # ─── 套件 0：local healthcheck adapter + check contracts ─── | |
| healthcheck: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ['3.9', '3.11'] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Healthcheck unit tests and selftest | |
| run: | | |
| python3 -m unittest discover -s healthcheck/tests | |
| python3 healthcheck/agentmeasure selftest | |
| - name: Healthcheck install smoke (packaged artifact) | |
| run: bash healthcheck/scripts/smoke_install.sh | |
| # ─── 套件 1：spec conformance（向量 + registry + metric registry） ─── | |
| spec-conformance: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: AgentMeasure test vectors | |
| env: | |
| PYTHONPATH: reference | |
| run: | | |
| python3 verify_vectors.py | |
| - name: Metric contract vectors | |
| env: | |
| PYTHONPATH: reference | |
| run: | | |
| python3 conformance/runners/run_metrics.py | |
| - name: "External fixture conformance (Urusilla-001: issues 8+9 guards)" | |
| run: | | |
| python3 conformance/runners/run_external_fixture.py | |
| - name: "External fixture conformance (Urusilla-002: issue 10 / AM-U-007 grain guard)" | |
| run: | | |
| python3 conformance/runners/run_external_fixture_002.py | |
| - name: "Conformance Pack selftest (caller-input contract)" | |
| run: | | |
| python3 conformance/pack/agentmeasure selftest | |
| - name: Registry validation (entities) | |
| run: | | |
| python3 registry/validate_entities.py | |
| - name: Metric registry validation (metrics.yaml ↔ METRICS.md) | |
| run: | | |
| python3 registry/validate_metrics.py | |
| - name: Canonical observation schema validation | |
| run: | | |
| python3 schemas/validate_observation.py | |
| - name: "Vocabulary registry single-source check (yaml, schema, TS, Python)" | |
| run: | | |
| python3 scripts/gen_vocab.py --check | |
| - name: "Evidence bundles: statistics-layer recomputation (stats-recomputable guard)" | |
| run: | | |
| status=0 | |
| for f in bundles/*/recompute_stats.py; do | |
| if [ -e "$f" ]; then | |
| echo "== $f" | |
| python3 "$f" || status=1 | |
| fi | |
| done | |
| exit $status | |
| # ─── 套件 2：reference implementation（canonical boundary gate） ─── | |
| reference-implementation: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Core integrity tests (0.4.3 data model, canonical ingestion only) | |
| env: | |
| PYTHONPATH: reference | |
| run: | | |
| python3 scripts/canonical_core_gate.py | |
| - name: Choice end-to-end (M2.2 / M2.5) | |
| env: | |
| PYTHONPATH: reference | |
| run: | | |
| python3 - <<'PY' | |
| import sys, json, tempfile, pathlib | |
| sys.path.insert(0, ".") | |
| from collector.choice import connect, ingest_choice_events, selection_metrics, conditional_choice_share | |
| tmp = pathlib.Path(tempfile.mkdtemp()) | |
| conn = connect(tmp / "choice.db") | |
| events = [ | |
| {"type": "presented", "decision_id": f"d{i}", "candidate_set_id": "c1", | |
| "project_id": "github.com/foo/bar", "tool": "Exa", "choice_mode": "exclusive", | |
| "context": "production", "validity": "normal", "ts": "2026-08-16T00:00:00Z"} | |
| for i in range(10) | |
| ] + [ | |
| {"type": "selected", "decision_id": f"d{i}", "candidate_set_id": "c1", | |
| "project_id": "github.com/foo/bar", "tool": "Exa", "choice_mode": "exclusive", | |
| "decision_authority": "model", "selection_constraint": "autonomous", | |
| "context": "production", "validity": "normal", "ts": "2026-08-16T00:01:00Z"} | |
| for i in range(4) | |
| ] | |
| p = tmp / "events.jsonl" | |
| p.write_text("\n".join(json.dumps(e) for e in events)) | |
| ingest_choice_events(conn, p) | |
| sm = selection_metrics(conn, "github.com/foo/bar") | |
| assert sm["tools"][0]["observed_selection_rate"] == 0.4, sm | |
| cs = conditional_choice_share(conn, "Exa", "Tavily", project_id="github.com/foo/bar") | |
| assert cs["co_presented_decisions"] == 0, cs # Tavily 未同台 -> fail-closed 空分母 | |
| print("choice: M2.2=0.4, M2.5 fail-closed OK") | |
| PY | |
| - name: Adapter privacy tests | |
| run: | | |
| python3 - <<'PY' | |
| import sys, json, tempfile, pathlib, os, subprocess | |
| tmp = pathlib.Path(tempfile.mkdtemp()) | |
| env = dict(os.environ, AGENTMEASURE_EVENTS_DIR=str(tmp / "hook")) | |
| subprocess.run([sys.executable, "reference/adapters/codex/hook_agent.py"], input=json.dumps({ | |
| "hook_event_name": "PostToolUse", "tool_name": "Bash", "tool_use_id": "tu-1", | |
| "session_id": "RAW-SESSION-42", | |
| "tool_input": {"command": "cat /etc/shadow"}, | |
| "tool_response": "SUPER-SECRET", | |
| "trace_id": "ignored", "is_error": True}), capture_output=True, text=True, env=env) | |
| raw = (tmp / "hook" / "codex-hook-events.jsonl").read_text() | |
| assert "RAW-SESSION-42" not in raw and "SUPER-SECRET" not in raw and "shadow" not in raw | |
| assert "trace_id" not in raw and "is_error" not in raw | |
| assert "p-" in raw | |
| print("adapter privacy OK") | |
| PY | |
| # ─── 套件 3：AgentMeasure Lab（experiment engine） ─── | |
| lab-suite: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Lab unit + regression tests (incl. #8 oneOf sibling tests) | |
| run: | | |
| python3 -m unittest discover -s lab/tests | |
| # ─── 套件 4：Provider SDK（External-Ready gate） ─── | |
| sdk-gate: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: '22' | |
| cache: npm | |
| cache-dependency-path: sdk/package-lock.json | |
| - name: Install + build SDK | |
| run: | | |
| cd sdk && npm ci && npm run build | |
| - name: SDK unit tests (schema / fail-open / privacy / lineage / caller / concurrency / buffer / mcp-integration) | |
| run: | | |
| cd sdk && npm test | |
| - name: Canonical-boundary pipeline gate (SDK → JSONL → validate → ingest → metrics) | |
| run: | | |
| python3 scripts/verify_sdk_pipeline.py | |
| # ─── 套件 5：documentation consistency（spec-drift） ─── | |
| documentation-consistency: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Spec-drift check (legacy vocabulary + version stamps) | |
| run: | | |
| python3 scripts/spec_drift.py | |
| # ─── 套件 6：website status.json sync（green run → repo state → site） ─── | |
| # Only on main pushes: regenerate website/status.json from repo state; when it | |
| # changes, commit to main and mirror to gh-pages. website/** is not in the | |
| # push path filter above, so this cannot re-trigger the workflow. | |
| status-sync: | |
| runs-on: ubuntu-latest | |
| needs: [healthcheck, spec-conformance, reference-implementation, lab-suite, sdk-gate, documentation-consistency] | |
| if: github.ref == 'refs/heads/main' && github.event_name == 'push' | |
| permissions: | |
| contents: write | |
| concurrency: | |
| group: status-sync | |
| cancel-in-progress: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Regenerate website/status.json (binds Passing to this run) | |
| run: | | |
| python3 scripts/gen_status.py --write | |
| git config user.name "agentmeasure-ci" | |
| git config user.email "actions@users.noreply.github.com" | |
| git add website/status.json | |
| if git diff --cached --quiet; then | |
| echo "status.json unchanged — nothing to do" | |
| else | |
| git commit -m "chore: sync website/status.json (verified @ ${GITHUB_SHA:0:7}) [skip ci]" | |
| git pull --rebase origin main | |
| git push origin main | |
| # mirror to gh-pages so the live site serves fresh numbers | |
| git fetch origin gh-pages | |
| git checkout gh-pages | |
| git checkout main -- website/status.json | |
| cp website/status.json ./status.json | |
| git add status.json | |
| git commit -m "deploy: status.json sync (CI)" || echo "gh-pages unchanged" | |
| git push origin gh-pages | |
| fi |