Run agent-semantic eval policies in parallel and surface results in Job Summary.

FAUST-BENCHOU · cursoragent · FAUST-BENCHOU · commit f9c1d2b6abfd · 2026-07-23T14:45:46.000+08:00
Each matrix shard boots Kind with the target POLICY/PRIOR_MIX, then merge
writes the compare table to the Actions Summary tab.

Co-authored-by: Cursor &lt;cursoragent@cursor.com&gt;
diff --git a/.github/workflows/e2e-kind.yaml b/.github/workflows/e2e-kind.yaml
@@ -67,10 +67,31 @@ jobs:
           KIND_CLUSTER_NAME: actordock-functional
         run: ./hack/kind-down.sh || true
 
-  # agent-semantic@v2 ablation: random | resource-evict | semantic L1 | semantic L1+L3
+  # Parallel ablation: one Kind cluster per policy variant (no mid-run POLICY switch).
   e2e-eval:
     runs-on: ubuntu-latest
-    timeout-minutes: 120
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - label: random
+            policy: random
+            prior_mix: ""
+            cluster: actordock-eval-random
+          - label: resource-evict
+            policy: resource-evict
+            prior_mix: ""
+            cluster: actordock-eval-re
+          - label: semantic-score-l1
+            policy: semantic-score
+            prior_mix: "0"
+            cluster: actordock-eval-l1
+          - label: semantic-score
+            policy: semantic-score
+            prior_mix: "0.3"
+            cluster: actordock-eval-full
+    name: e2e-eval (${{ matrix.label }})
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
@@ -88,15 +109,16 @@ jobs:
 
       - name: Create cluster and deploy
         env:
-          KIND_CLUSTER_NAME: actordock-eval
-          POLICY: fifo
+          KIND_CLUSTER_NAME: ${{ matrix.cluster }}
+          POLICY: ${{ matrix.policy }}
+          SEMANTIC_PRIOR_MIX: ${{ matrix.prior_mix }}
         run: |
           chmod +x hack/*.sh hack/replay-agent-semantic.py
           ./hack/kind-up.sh
 
-      - name: Agent-semantic ablation (8 agents / 2 workers)
+      - name: Agent-semantic replay (8 agents / 2 workers)
         env:
-          KIND_CLUSTER_NAME: actordock-eval
+          KIND_CLUSTER_NAME: ${{ matrix.cluster }}
           E2E_SUITE: agent-semantic
           ACTORDOCK_API: http://127.0.0.1:18080
           EVAL_OUT_DIR: docs/eval/results
@@ -105,29 +127,140 @@ jobs:
           AGENT_SEMANTIC_MIN_WORKERS: "2"
           AGENT_SEMANTIC_SPEED: "60"
           AGENT_SEMANTIC_MIN_LOCK: "0.25"
-          AGENT_SEMANTIC_POLICIES: "random,resource-evict,semantic-score-l1,semantic-score"
+          AGENT_SEMANTIC_POLICIES: ${{ matrix.label }}
+          AGENT_SEMANTIC_SWITCH_POLICY: "0"
         run: ./hack/verify-local.sh
 
-      - name: Upload agent-semantic results
+      - name: Publish policy summary on Actions page
+        if: success() || failure()
+        env:
+          LABEL: ${{ matrix.label }}
+        run: |
+          python3 - <<'PY'
+          import json, os
+          from pathlib import Path
+          label = os.environ["LABEL"]
+          out = Path(os.environ.get("GITHUB_STEP_SUMMARY", "/dev/null"))
+          p = Path(f"docs/eval/results/agent_semantic_v2__{label}.json")
+          lines = [f"## agent-semantic `{label}`", ""]
+          if not p.exists():
+              lines += [f"_missing {p}_", ""]
+          else:
+              r = json.loads(p.read_text())
+              lines += [
+                  "| metric | value |",
+                  "|--------|------:|",
+                  f"| sessions ok/fail | {r.get('sessions_ok')}/{r.get('sessions_failed')} |",
+                  f"| mid_tool | {r.get('mid_tool_suspend', 0)} |",
+                  f"| mid_tool_rate | {float(r.get('mid_tool_rate') or 0):.2f} |",
+                  f"| suspend | {r.get('suspend_total', 0)} |",
+                  f"| hard_rate | {float(r.get('victim_l3_hard_rate') or 0):.2f} |",
+                  f"| vic_hard / vic_easy | {r.get('victim_l3_hard', 0)} / {r.get('victim_l3_easy', 0)} |",
+                  f"| resume_hard_s / resume_easy_s | {float(r.get('resume_sec_l3_hard') or 0):.3f} / {float(r.get('resume_sec_l3_easy') or 0):.3f} |",
+                  f"| victim_cx | {float(r.get('victim_complexity_mean') or 0):.3f} |",
+                  f"| resume_wait_s | {float(r.get('resume_wait_mean_s') or 0):.3f} |",
+                  f"| wall_s | {float(r.get('wall_sec') or 0):.1f} |",
+                  "",
+                  f"victims=`{r.get('victim_by_cohort')}`",
+                  "",
+              ]
+          out.write_text("\n".join(lines) + "\n")
+          print("".join(line + "\n" for line in lines))
+          PY
+
+      - name: Upload policy result
         if: success() || failure()
         uses: actions/upload-artifact@v4
         with:
-          name: agent-semantic-policy-compare
-          path: |
-            docs/eval/results/policy_compare_agent_semantic_v2.md
-            docs/eval/results/agent_semantic_v2__*.json
+          name: agent-semantic-${{ matrix.label }}
+          path: docs/eval/results/agent_semantic_v2__${{ matrix.label }}.json
           if-no-files-found: warn
 
       - name: Dump on failure
         if: failure()
         run: |
-          kubectl --context kind-actordock-eval -n actordock get pods -o wide || true
-          kubectl --context kind-actordock-eval -n actordock describe pods || true
-          kubectl --context kind-actordock-eval -n actordock logs deploy/controlplane --all-containers=true || true
-          kubectl --context kind-actordock-eval -n actordock logs statefulset/worker --all-containers=true || true
+          kubectl --context kind-${{ matrix.cluster }} -n actordock get pods -o wide || true
+          kubectl --context kind-${{ matrix.cluster }} -n actordock describe pods || true
+          kubectl --context kind-${{ matrix.cluster }} -n actordock logs deploy/controlplane --all-containers=true || true
+          kubectl --context kind-${{ matrix.cluster }} -n actordock logs statefulset/worker --all-containers=true || true
 
       - name: Teardown
         if: always()
         env:
-          KIND_CLUSTER_NAME: actordock-eval
+          KIND_CLUSTER_NAME: ${{ matrix.cluster }}
         run: ./hack/kind-down.sh || true
+
+  e2e-eval-merge:
+    runs-on: ubuntu-latest
+    needs: e2e-eval
+    if: always() && !cancelled()
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Download policy artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: agent-semantic-*
+          path: docs/eval/results/parts
+          merge-multiple: true
+
+      - name: Merge compare table + gate
+        env:
+          EVAL_OUT_DIR: docs/eval/results
+        run: |
+          chmod +x hack/replay-agent-semantic.py
+          mkdir -p docs/eval/results
+          # Flatten downloaded JSONs into results/
+          find docs/eval/results/parts -name 'agent_semantic_v2__*.json' -exec cp -f {} docs/eval/results/ \;
+          ls -la docs/eval/results/agent_semantic_v2__*.json
+          python3 hack/replay-agent-semantic.py --merge-only --out docs/eval/results
+          python3 - <<'PY'
+          import json, os, sys
+          from pathlib import Path
+          out = Path("docs/eval/results")
+          want = ["random", "resource-evict", "semantic-score-l1", "semantic-score"]
+          failed = []
+          missing = []
+          for label in want:
+              p = out / f"agent_semantic_v2__{label}.json"
+              if not p.exists():
+                  missing.append(label)
+                  continue
+              r = json.loads(p.read_text())
+              if int(r.get("sessions_failed") or 0) != 0 or int(r.get("sessions_ok") or 0) <= 0:
+                  failed.append(f"{p.name}: ok={r.get('sessions_ok')} fail={r.get('sessions_failed')}")
+          compare = out / "policy_compare_agent_semantic_v2.md"
+          if not compare.exists():
+              sys.exit("missing policy_compare_agent_semantic_v2.md")
+          body = compare.read_text()
+          print(body)
+          # Show full compare on the Actions run Summary tab (no artifact download needed).
+          summary = Path(os.environ.get("GITHUB_STEP_SUMMARY", ""))
+          if str(summary):
+              with summary.open("a", encoding="utf-8") as f:
+                  f.write(body)
+                  if missing:
+                      f.write("\n\n**MISSING:** " + ", ".join(missing) + "\n")
+                  if failed:
+                      f.write("\n\n**FAILED:**\n\n- " + "\n- ".join(failed) + "\n")
+          if missing:
+              print("MISSING:", ", ".join(missing))
+              sys.exit(1)
+          if failed:
+              print("FAILED:", *failed, sep="\n  ")
+              sys.exit(1)
+          print(f"ok: {len(want)} policy report(s)")
+          PY
+
+      - name: Upload merged compare (optional archive)
+        if: success() || failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: agent-semantic-policy-compare
+          path: |
+            docs/eval/results/policy_compare_agent_semantic_v2.md
+            docs/eval/results/agent_semantic_v2__*.json
+          if-no-files-found: warn
diff --git a/docs/eval/README.md b/docs/eval/README.md
@@ -32,9 +32,9 @@ Each dataset version directory should include:
 - Metrics table aligned with `docs/research/metrics.md`
 - Optional: raw decision logs for debugging
 
-CI job `e2e-eval` runs `E2E_SUITE=agent-semantic`: 8 agents / 2 workers on
-`datasets/agent-semantic@v2`; policies
-`random,resource-evict,semantic-score-l1,semantic-score` (L3 ablation);
-artifact `agent-semantic-policy-compare`.
+CI job `e2e-eval` runs `E2E_SUITE=agent-semantic` in a **matrix** (one Kind cluster per
+policy: `random`, `resource-evict`, `semantic-score-l1`, `semantic-score`);
+Primary view: GitHub Actions **Summary** tab (full compare table + per-policy metrics).
+Artifacts remain as optional archive.
 
 Do not commit huge binary checkpoints here; link or use Git LFS if needed.
diff --git a/docs/eval/agent-semantic-workload.md b/docs/eval/agent-semantic-workload.md
@@ -198,7 +198,7 @@ Refuse to publish the package if:
 |------|------|-----|
 | **A. Live Kind replay** | Primary claim for Actordock | Port-forward CP; for each policy, Create/Resume sandboxes on `arrival_ts` schedule; drive agent (or phase-faithful stub) from `tool_trace`; POST L1/L3 signals; collect `/metrics` + victim logs |
 | **B. Offline decision replay** | Fast ablation of keepScore | Feed cached signals + arrivals into a Place/Evict simulator that embeds `semantic-score` / baselines; no gVisor | 
-| **C. CI agent-semantic** | PR / main Kind job | `E2E_SUITE=agent-semantic`: 8 agents / 2 workers; ablation matrix (random, resource-evict, semantic-score-l1, semantic-score) |
+| **C. CI agent-semantic** | PR / main Kind job | Matrix: one cluster per policy (`random`, `resource-evict`, `semantic-score-l1`, `semantic-score`); merge job builds compare table |
 
 Primary paper/demo numbers should come from **Mode A** (or A+B agreement).
 
diff --git a/docs/eval/results/README.md b/docs/eval/results/README.md
@@ -1,7 +1,7 @@
 # Eval results
 
-Generated by `hack/replay-agent-semantic.py` (CI job `e2e-eval` uploads artifact
-`agent-semantic-policy-compare`). Artifacts are gitignored except this README.
+Generated by `hack/replay-agent-semantic.py`. In CI, the compare table is written to the
+Actions **Summary** tab (Job Summary); artifacts are optional.
 
 - `policy_compare_agent_semantic_v2.md` — cross-policy table
 - `agent_semantic_v2__<policy>.json` — per-policy metrics
diff --git a/e2e/README.md b/e2e/README.md
@@ -23,7 +23,7 @@ E2E_SUITE=agent-semantic ./hack/verify-local.sh  # dataset policy compare
 E2E_SUITE=all ./hack/verify-local.sh             # both
 ```
 
-CI: **functional** + **e2e-eval** (`random` / `resource-evict` / `semantic-score-l1` / `semantic-score`).
+CI: **functional** + **e2e-eval** matrix (`random` / `resource-evict` / `semantic-score-l1` / `semantic-score` in parallel) + merge compare.
 
 ## Functional suite
 
@@ -54,5 +54,5 @@ CI: **functional** + **e2e-eval** (`random` / `resource-evict` / `semantic-score
 
 `semantic-score-l1` = L1 lock only (`SEMANTIC_PRIOR_MIX=0`); `semantic-score` = L1+L3 (`0.3`).
 
-Outputs `docs/eval/results/agent_semantic_v2__*.json` + `policy_compare_agent_semantic_v2.md`.
+Outputs also appear on the Actions **Summary** tab (Job Summary); artifacts are optional archive only.
 Gate: every policy `sessions_failed=0`.
diff --git a/hack/kind-up.sh b/hack/kind-up.sh
@@ -33,7 +33,7 @@ echo "==> load into kind"
 kind load docker-image actordock/controlplane:dev --name "${CLUSTER_NAME}"
 kind load docker-image actordock/worker:dev --name "${CLUSTER_NAME}"
 
-echo "==> deploy rustfs + actordock (POLICY=${POLICY})"
+echo "==> deploy rustfs + actordock (POLICY=${POLICY}${SEMANTIC_PRIOR_MIX:+ PRIOR_MIX=${SEMANTIC_PRIOR_MIX}})"
 kubectl apply -f "${ROOT}/manifests/kind/rustfs.yaml"
 kubectl -n actordock rollout status deploy/rustfs --timeout=180s
 # Recreate bucket-init job so re-runs are idempotent.
@@ -42,7 +42,11 @@ kubectl apply -f "${ROOT}/manifests/kind/rustfs.yaml"
 kubectl -n actordock wait --for=condition=complete job/rustfs-bucket-init --timeout=180s
 
 kubectl apply -f "${ROOT}/manifests/kind/actordock.yaml"
-kubectl -n actordock set env deployment/controlplane POLICY="${POLICY}"
+CP_ENV=(POLICY="${POLICY}")
+if [[ -n "${SEMANTIC_PRIOR_MIX:-}" ]]; then
+  CP_ENV+=("SEMANTIC_PRIOR_MIX=${SEMANTIC_PRIOR_MIX}")
+fi
+kubectl -n actordock set env deployment/controlplane "${CP_ENV[@]}"
 # Ensure pods pick up freshly loaded images.
 kubectl -n actordock rollout restart deployment/controlplane deployment/redis
 kubectl -n actordock rollout restart statefulset/worker
diff --git a/hack/replay-agent-semantic.py b/hack/replay-agent-semantic.py
@@ -37,7 +37,7 @@
 import urllib.request
 from collections import defaultdict
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, fields as dc_fields
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any
@@ -923,6 +923,42 @@ def run_policy(
     return result
 
 
+def load_policy_result(path: Path) -> PolicyResult:
+    raw = json.loads(path.read_text())
+    allowed = {f.name for f in dc_fields(PolicyResult)}
+    return PolicyResult(**{k: v for k, v in raw.items() if k in allowed})
+
+
+COMPARE_ORDER = (
+    "random",
+    "resource-evict",
+    "semantic-score-l1",
+    "semantic-score",
+    "fifo",
+    "lru-idle",
+)
+
+
+def merge_results_dir(out: Path) -> Path:
+    """Load agent_semantic_v2__*.json under out and write policy_compare markdown."""
+    files = sorted(out.glob("agent_semantic_v2__*.json"))
+    if not files:
+        raise SystemExit(f"no agent_semantic_v2__*.json under {out}")
+    by_label = {p.stem.replace("agent_semantic_v2__", "", 1): load_policy_result(p) for p in files}
+    ordered: list[PolicyResult] = []
+    seen: set[str] = set()
+    for label in COMPARE_ORDER:
+        if label in by_label:
+            ordered.append(by_label[label])
+            seen.add(label)
+    for label, res in sorted(by_label.items()):
+        if label not in seen:
+            ordered.append(res)
+    compare = out / "policy_compare_agent_semantic_v2.md"
+    write_compare(ordered, compare)
+    return compare
+
+
 def write_compare(results: list[PolicyResult], path: Path) -> None:
     lines = [
         "# agent-semantic policy compare",
@@ -1018,8 +1054,20 @@ def main() -> None:
     ap.add_argument("--resume-timeout", type=float, default=300.0)
     ap.add_argument("--out", type=Path, default=DEFAULT_OUT)
     ap.add_argument("--dry-run", action="store_true", help="Load dataset only")
+    ap.add_argument(
+        "--merge-only",
+        action="store_true",
+        help="Only merge agent_semantic_v2__*.json under --out into policy_compare markdown",
+    )
     args = ap.parse_args()
 
+    if args.merge_only:
+        args.out.mkdir(parents=True, exist_ok=True)
+        compare = merge_results_dir(args.out)
+        log(f"[merge] {compare}")
+        sys.stdout.write(compare.read_text())
+        return
+
     sessions_path = args.dataset / "sessions.jsonl"
     if not sessions_path.exists():
         raise SystemExit(f"missing {sessions_path}")
diff --git a/hack/verify-local.sh b/hack/verify-local.sh
@@ -58,12 +58,26 @@ run_agent_semantic() {
   ensure_api_pf
   mkdir -p "${EVAL_OUT_DIR}"
   chmod +x "${ROOT}/hack/replay-agent-semantic.py"
-  echo "==> agent-semantic replay limit=${AGENT_SEMANTIC_LIMIT} inflight=${AGENT_SEMANTIC_INFLIGHT} policies=${AGENT_SEMANTIC_POLICIES}"
+  SWITCH_ARGS=()
+  # Default: switch when multiple policies; CI matrix sets AGENT_SEMANTIC_SWITCH_POLICY=0.
+  switch="${AGENT_SEMANTIC_SWITCH_POLICY:-}"
+  if [[ -z "${switch}" ]]; then
+    IFS=',' read -r -a _pols <<< "${AGENT_SEMANTIC_POLICIES}"
+    if ((${#_pols[@]} > 1)); then
+      switch=1
+    else
+      switch=0
+    fi
+  fi
+  if [[ "${switch}" == "1" || "${switch}" == "true" ]]; then
+    SWITCH_ARGS=(--switch-policy)
+  fi
+  echo "==> agent-semantic replay limit=${AGENT_SEMANTIC_LIMIT} inflight=${AGENT_SEMANTIC_INFLIGHT} policies=${AGENT_SEMANTIC_POLICIES} switch=${switch}"
   python3 "${ROOT}/hack/replay-agent-semantic.py" \
     --api "${ACTORDOCK_API}" \
     --dataset "${AGENT_SEMANTIC_DATASET}" \
     --policies "${AGENT_SEMANTIC_POLICIES}" \
-    --switch-policy \
+    "${SWITCH_ARGS[@]}" \
     --namespace "${ACTORDOCK_NAMESPACE}" \
     --min-workers "${AGENT_SEMANTIC_MIN_WORKERS}" \
     --max-inflight "${AGENT_SEMANTIC_INFLIGHT}" \
