conformance.yml

name: conformance

on:
  push:
    paths:
      - 'standard/**'
      - 'reference/**'
      - 'conformance/**'
      - 'schemas/**'
      - 'registry/**'
      - 'scripts/**'
      - 'docs/**'
      - 'product/**'
      - 'extensions/**'
      - 'whitepaper/**'
      - 'sdk/**'
      - 'lab/**'
      - 'examples/**'
      - 'benchmark/**'
      - 'healthcheck/**'
      - 'reports/**'
      - 'bundles/**'
      - 'README*'
      - 'ROADMAP.md'
      - 'GOVERNANCE.md'
      - 'CONTRIBUTING.md'
      - 'LEGACY-MIGRATION.md'
      - 'verify_vectors.py'
  workflow_dispatch:
  pull_request:

jobs:
  # ─── 套件 0：local healthcheck adapter + check contracts ───
  healthcheck:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ['3.9', '3.11']
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Healthcheck unit tests and selftest
        run: |
          python3 -m unittest discover -s healthcheck/tests
          python3 healthcheck/agentmeasure selftest
      - name: Healthcheck install smoke (packaged artifact)
        run: bash healthcheck/scripts/smoke_install.sh

  # ─── 套件 1：spec conformance（向量 + registry + metric registry） ───
  spec-conformance:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: AgentMeasure test vectors
        env:
          PYTHONPATH: reference
        run: |
          python3 verify_vectors.py
      - name: Metric contract vectors
        env:
          PYTHONPATH: reference
        run: |
          python3 conformance/runners/run_metrics.py
      - name: "External fixture conformance (Urusilla-001: issues 8+9 guards)"
        run: |
          python3 conformance/runners/run_external_fixture.py
      - name: "External fixture conformance (Urusilla-002: issue 10 / AM-U-007 grain guard)"
        run: |
          python3 conformance/runners/run_external_fixture_002.py
      - name: "Conformance Pack selftest (caller-input contract)"
        run: |
          python3 conformance/pack/agentmeasure selftest
      - name: Registry validation (entities)
        run: |
          python3 registry/validate_entities.py
      - name: Metric registry validation (metrics.yaml ↔ METRICS.md)
        run: |
          python3 registry/validate_metrics.py
      - name: Canonical observation schema validation
        run: |
          python3 schemas/validate_observation.py
      - name: "Vocabulary registry single-source check (yaml, schema, TS, Python)"
        run: |
          python3 scripts/gen_vocab.py --check
      - name: "Evidence bundles: statistics-layer recomputation (stats-recomputable guard)"
        run: |
          status=0
          for f in bundles/*/recompute_stats.py; do
            if [ -e "$f" ]; then
              echo "== $f"
              python3 "$f" || status=1
            fi
          done
          exit $status

  # ─── 套件 2：reference implementation（canonical boundary gate） ───
  reference-implementation:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: Core integrity tests (0.4.3 data model, canonical ingestion only)
        env:
          PYTHONPATH: reference
        run: |
          python3 scripts/canonical_core_gate.py
      - name: Choice end-to-end (M2.2 / M2.5)
        env:
          PYTHONPATH: reference
        run: |
          python3 - <<'PY'
          import sys, json, tempfile, pathlib
          sys.path.insert(0, ".")
          from collector.choice import connect, ingest_choice_events, selection_metrics, conditional_choice_share
          tmp = pathlib.Path(tempfile.mkdtemp())
          conn = connect(tmp / "choice.db")
          events = [
              {"type": "presented", "decision_id": f"d{i}", "candidate_set_id": "c1",
               "project_id": "github.com/foo/bar", "tool": "Exa", "choice_mode": "exclusive",
               "context": "production", "validity": "normal", "ts": "2026-08-16T00:00:00Z"}
              for i in range(10)
          ] + [
              {"type": "selected", "decision_id": f"d{i}", "candidate_set_id": "c1",
               "project_id": "github.com/foo/bar", "tool": "Exa", "choice_mode": "exclusive",
               "decision_authority": "model", "selection_constraint": "autonomous",
               "context": "production", "validity": "normal", "ts": "2026-08-16T00:01:00Z"}
              for i in range(4)
          ]
          p = tmp / "events.jsonl"
          p.write_text("\n".join(json.dumps(e) for e in events))
          ingest_choice_events(conn, p)
          sm = selection_metrics(conn, "github.com/foo/bar")
          assert sm["tools"][0]["observed_selection_rate"] == 0.4, sm
          cs = conditional_choice_share(conn, "Exa", "Tavily", project_id="github.com/foo/bar")
          assert cs["co_presented_decisions"] == 0, cs  # Tavily 未同台 -> fail-closed 空分母
          print("choice: M2.2=0.4, M2.5 fail-closed OK")
          PY
      - name: Adapter privacy tests
        run: |
          python3 - <<'PY'
          import sys, json, tempfile, pathlib, os, subprocess
          tmp = pathlib.Path(tempfile.mkdtemp())
          env = dict(os.environ, AGENTMEASURE_EVENTS_DIR=str(tmp / "hook"))
          subprocess.run([sys.executable, "reference/adapters/codex/hook_agent.py"], input=json.dumps({
              "hook_event_name": "PostToolUse", "tool_name": "Bash", "tool_use_id": "tu-1",
              "session_id": "RAW-SESSION-42",
              "tool_input": {"command": "cat /etc/shadow"},
              "tool_response": "SUPER-SECRET",
              "trace_id": "ignored", "is_error": True}), capture_output=True, text=True, env=env)
          raw = (tmp / "hook" / "codex-hook-events.jsonl").read_text()
          assert "RAW-SESSION-42" not in raw and "SUPER-SECRET" not in raw and "shadow" not in raw
          assert "trace_id" not in raw and "is_error" not in raw
          assert "p-" in raw
          print("adapter privacy OK")
          PY

  # ─── 套件 3：AgentMeasure Lab（experiment engine） ───
  lab-suite:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: Lab unit + regression tests (incl. #8 oneOf sibling tests)
        run: |
          python3 -m unittest discover -s lab/tests

  # ─── 套件 4：Provider SDK（External-Ready gate） ───
  sdk-gate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - uses: actions/setup-node@v4
        with:
          node-version: '22'
          cache: npm
          cache-dependency-path: sdk/package-lock.json
      - name: Install + build SDK
        run: |
          cd sdk && npm ci && npm run build
      - name: SDK unit tests (schema / fail-open / privacy / lineage / caller / concurrency / buffer / mcp-integration)
        run: |
          cd sdk && npm test
      - name: Canonical-boundary pipeline gate (SDK → JSONL → validate → ingest → metrics)
        run: |
          python3 scripts/verify_sdk_pipeline.py

  # ─── 套件 5：documentation consistency（spec-drift） ───
  documentation-consistency:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: Spec-drift check (legacy vocabulary + version stamps)
        run: |
          python3 scripts/spec_drift.py

  # ─── 套件 6：website status.json sync（green run → repo state → site） ───
  # Only on main pushes: regenerate website/status.json from repo state; when it
  # changes, commit to main and mirror to gh-pages. website/** is not in the
  # push path filter above, so this cannot re-trigger the workflow.
  status-sync:
    runs-on: ubuntu-latest
    needs: [healthcheck, spec-conformance, reference-implementation, lab-suite, sdk-gate, documentation-consistency]
    if: github.ref == 'refs/heads/main' && github.event_name == 'push'
    permissions:
      contents: write
    concurrency:
      group: status-sync
      cancel-in-progress: false
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          token: ${{ secrets.GITHUB_TOKEN }}
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: Regenerate website/status.json (binds Passing to this run)
        run: |
          python3 scripts/gen_status.py --write
          git config user.name "agentmeasure-ci"
          git config user.email "actions@users.noreply.github.com"
          git add website/status.json
          if git diff --cached --quiet; then
            echo "status.json unchanged — nothing to do"
          else
            git commit -m "chore: sync website/status.json (verified @ ${GITHUB_SHA:0:7}) [skip ci]"
            git pull --rebase origin main
            git push origin main
            # mirror to gh-pages so the live site serves fresh numbers
            git fetch origin gh-pages
            git checkout gh-pages
            git checkout main -- website/status.json
            cp website/status.json ./status.json
            git add status.json
            git commit -m "deploy: status.json sync (CI)" || echo "gh-pages unchanged"
            git push origin gh-pages
          fi