diff --git a/CATALOG.md b/CATALOG.md index 58cd2eb1..c1ed86dc 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -1,7 +1,7 @@ # Current eval catalog and evidence -This is the sole current corpus surface: **45 included cells** (14 model-free, +This is the sole current corpus surface: **46 included cells** (15 model-free, 31 model-backed) and **4 retired exclusions**. Inventory fields are derived from the active KDL and executable judge scripts. Latest accepted PASS evidence remains distinct from the append-only run history: a failed last run is visible without being advertised as accepted evidence, and cells with no @@ -43,6 +43,7 @@ harness-native loader plus canonical hook file. | `render-target-safety` | model-free | — | 0 | none | `90s` | 5 | — | **NO STRUCTURED RUN** | | `restart-continuity` | Claude | `claude-sonnet-5` / medium | 2 | medium | `1200s` | 5 | — | **NO STRUCTURED RUN** | | `security-audit` | Claude | `claude-sonnet-5` / medium | 2 | medium | `1200s` | 4 | — | **NO STRUCTURED RUN** | +| `shared-workspace-render-ownership` | model-free | — | 0 | none | `60s` | 5 | — | **NO STRUCTURED RUN** | | `signal-rename` | Claude | `claude-sonnet-5` / medium | 4 | high | `1800s` | 5 | — | **NO STRUCTURED RUN** | | `signal-rename-codex` | Codex | `gpt-5.6-sol` / medium | 4 | high | `1800s` | 5 | **PASS** 2026-07-26, st2 `9d26245`, 8m07s, [`67b45d2`](https://github.com/compoundingtech/evals/commit/67b45d2694ac40762b09f51bf625d092ab68de74) | **NO STRUCTURED RUN** | | `skill-inheritance` | Claude | `claude-sonnet-5` / medium | 1 | low | `420s` | 3 | — | **NO STRUCTURED RUN** | @@ -91,13 +92,14 @@ while the last-run column makes a recorded failure distinct from a cell with no - `presence-ding-matrix` / `cell` — deterministic native presence and DING probe; no harness seat - `reconcile-retire-keep` / `cell` — deterministic native reconcile lifecycle probe; no harness seat - `render-target-safety` / `cell` — deterministic native render-policy probe; no harness seat +- `shared-workspace-render-ownership` / `cell` — deterministic native shared-render admission probe; no harness seat - `strict-validation-json` / `cell` — deterministic native validation-diagnostics probe; no harness seat - `st2-doctor-structure` / `cell` — deterministic catalog-health probe; no harness seat - `st2-network` / `cell` — deterministic catalog/message round-trip probe; no harness seat - `targeted-reconcile-isolation` / `cell` — deterministic selected reconcile isolation probe; no bus-connected agent - `two-networks-coexist` / `cell` — deterministic catalog-isolation probe; no harness seat -These **15** rows are generated from structured exclusions. A cell-level row is required for +These **16** rows are generated from structured exclusions. A cell-level row is required for every derived model-free cell. The one-shot docs cold reader is a model-backed grader, but not a bus-connected agent: it has no st2 identity, DING sidecar, or lifecycle hooks. diff --git a/cells/shared-workspace-render-ownership/README.md b/cells/shared-workspace-render-ownership/README.md new file mode 100644 index 00000000..ceb099e3 --- /dev/null +++ b/cells/shared-workspace-render-ownership/README.md @@ -0,0 +1,9 @@ +# shared-workspace-render-ownership + +Model-free admission coverage for multiple declarations that render into one +workspace. Incompatible persona bytes must fail validation and both full and +targeted materialization before any write. A separate control proves that +byte-identical shared claims remain valid and idempotent. + +This is a workspace-ownership discriminator, not a provider test: it starts no +PTY, Claude, or Codex session. diff --git a/cells/shared-workspace-render-ownership/fixture/conflict/_templates/orchestrator.md b/cells/shared-workspace-render-ownership/fixture/conflict/_templates/orchestrator.md new file mode 100644 index 00000000..849f3c8b --- /dev/null +++ b/cells/shared-workspace-render-ownership/fixture/conflict/_templates/orchestrator.md @@ -0,0 +1 @@ +orchestrator diff --git a/cells/shared-workspace-render-ownership/fixture/conflict/_templates/worker.md b/cells/shared-workspace-render-ownership/fixture/conflict/_templates/worker.md new file mode 100644 index 00000000..ff0d0382 --- /dev/null +++ b/cells/shared-workspace-render-ownership/fixture/conflict/_templates/worker.md @@ -0,0 +1 @@ +worker diff --git a/cells/shared-workspace-render-ownership/fixture/conflict/agents/evalhost/orchestrator/agent.kdl b/cells/shared-workspace-render-ownership/fixture/conflict/agents/evalhost/orchestrator/agent.kdl new file mode 100644 index 00000000..7ee57dc9 --- /dev/null +++ b/cells/shared-workspace-render-ownership/fixture/conflict/agents/evalhost/orchestrator/agent.kdl @@ -0,0 +1,9 @@ +agent "orchestrator" { + host "evalhost" + workspace "$CATALOG/shared" + command "true" + + render { + copy "_templates/orchestrator.md" ".st2/PERSONA.md" + } +} diff --git a/cells/shared-workspace-render-ownership/fixture/conflict/agents/evalhost/worker/agent.kdl b/cells/shared-workspace-render-ownership/fixture/conflict/agents/evalhost/worker/agent.kdl new file mode 100644 index 00000000..7957b2c8 --- /dev/null +++ b/cells/shared-workspace-render-ownership/fixture/conflict/agents/evalhost/worker/agent.kdl @@ -0,0 +1,9 @@ +agent "worker" { + host "evalhost" + workspace "$CATALOG/shared" + command "true" + + render { + copy "_templates/worker.md" ".st2/PERSONA.md" + } +} diff --git a/cells/shared-workspace-render-ownership/fixture/conflict/shared/.keep b/cells/shared-workspace-render-ownership/fixture/conflict/shared/.keep new file mode 100644 index 00000000..ee8c1ee4 --- /dev/null +++ b/cells/shared-workspace-render-ownership/fixture/conflict/shared/.keep @@ -0,0 +1 @@ +fixture diff --git a/cells/shared-workspace-render-ownership/fixture/equivalent/_templates/shared.md b/cells/shared-workspace-render-ownership/fixture/equivalent/_templates/shared.md new file mode 100644 index 00000000..3a36983c --- /dev/null +++ b/cells/shared-workspace-render-ownership/fixture/equivalent/_templates/shared.md @@ -0,0 +1 @@ +shared-bus diff --git a/cells/shared-workspace-render-ownership/fixture/equivalent/agents/evalhost/a/agent.kdl b/cells/shared-workspace-render-ownership/fixture/equivalent/agents/evalhost/a/agent.kdl new file mode 100644 index 00000000..d1039ec8 --- /dev/null +++ b/cells/shared-workspace-render-ownership/fixture/equivalent/agents/evalhost/a/agent.kdl @@ -0,0 +1,9 @@ +agent "a" { + host "evalhost" + workspace "$CATALOG/shared" + command "true" + + render { + copy "_templates/shared.md" ".st2/bus.md" + } +} diff --git a/cells/shared-workspace-render-ownership/fixture/equivalent/agents/evalhost/b/agent.kdl b/cells/shared-workspace-render-ownership/fixture/equivalent/agents/evalhost/b/agent.kdl new file mode 100644 index 00000000..f9290870 --- /dev/null +++ b/cells/shared-workspace-render-ownership/fixture/equivalent/agents/evalhost/b/agent.kdl @@ -0,0 +1,9 @@ +agent "b" { + host "evalhost" + workspace "$CATALOG/shared" + command "true" + + render { + copy "_templates/shared.md" ".st2/bus.md" + } +} diff --git a/cells/shared-workspace-render-ownership/fixture/equivalent/shared/.keep b/cells/shared-workspace-render-ownership/fixture/equivalent/shared/.keep new file mode 100644 index 00000000..ee8c1ee4 --- /dev/null +++ b/cells/shared-workspace-render-ownership/fixture/equivalent/shared/.keep @@ -0,0 +1 @@ +fixture diff --git a/cells/shared-workspace-render-ownership/fixture/inspect.sh b/cells/shared-workspace-render-ownership/fixture/inspect.sh new file mode 100755 index 00000000..7327eb89 --- /dev/null +++ b/cells/shared-workspace-render-ownership/fixture/inspect.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +conflict="${CATALOG:?CATALOG must be set}/conflict/shared" +equivalent="$CATALOG/equivalent/shared" + +test ! -e "$conflict/.st2/PERSONA.md" +echo "FULL-CONFLICT-PREWRITE-GREEN-95" + +test ! -e "$conflict/.st2/PERSONA.md" +echo "TARGETED-CONFLICT-PREWRITE-GREEN-95" + +test "$(cat "$equivalent/.st2/bus.md")" = "shared-bus" +echo "EQUIVALENT-SHARED-GREEN-95" diff --git a/cells/shared-workspace-render-ownership/shared-workspace-render-ownership.kdl b/cells/shared-workspace-render-ownership/shared-workspace-render-ownership.kdl new file mode 100644 index 00000000..fc7ee34f --- /dev/null +++ b/cells/shared-workspace-render-ownership/shared-workspace-render-ownership.kdl @@ -0,0 +1,42 @@ +// A shared workspace target has one coherent desired state across its active owners. +eval { + copy "./fixture" + max-timeout "60s" + + run "validate-conflict" { + allow-nonzero + command #"st2 validate --json --strict --catalog "$CATALOG/conflict" --host evalhost"# + } + run "materialize-conflict" { + allow-nonzero + command #"st2 up --catalog "$CATALOG/conflict" --host evalhost --materialize-only"# + } + run "materialize-targeted-conflict" { + allow-nonzero + command #"st2 up --catalog "$CATALOG/conflict" --host evalhost --materialize-only --task evalhost.worker.agent"# + } + run "materialize-equivalent" { + command #"st2 up --catalog "$CATALOG/equivalent" --host evalhost --materialize-only"# + } + run "inspect" { + command "bash ./inspect.sh" + } + + judges { + judge "VALIDATION — incompatible shared target ownership is a stable error" { + exec #"test "$RUN_validate_conflict_EXIT" -ne 0 && jq -e '.issues[] | select(.severity == "error" and .code == "render-owner-conflict")' $RUNS_DIR/validate-conflict.out >/dev/null"# + } + judge "PREWRITE — full materialization refuses before either owner writes" { + exec #"test "$RUN_materialize_conflict_EXIT" -ne 0 && grep -Fqx FULL-CONFLICT-PREWRITE-GREEN-95 $RUNS_DIR/inspect.out"# + } + judge "TARGETED — selecting one owner cannot bypass its sibling's claim" { + exec #"test "$RUN_materialize_targeted_conflict_EXIT" -ne 0 && grep -Fqx TARGETED-CONFLICT-PREWRITE-GREEN-95 $RUNS_DIR/inspect.out"# + } + judge "EQUIVALENT — byte-identical shared claims remain valid and idempotent" { + exec #"grep -Fqx EQUIVALENT-SHARED-GREEN-95 $RUNS_DIR/inspect.out"# + } + judge "DIAGNOSTIC — the conflict names the target and both owners" { + exec #"grep -F '.st2/PERSONA.md' $RUNS_DIR/materialize-conflict.err >/dev/null && grep -F 'evalhost.worker' $RUNS_DIR/materialize-conflict.err >/dev/null && grep -F 'evalhost.orchestrator' $RUNS_DIR/materialize-conflict.err >/dev/null"# + } + } +} diff --git a/docs/vrs/spec.md b/docs/vrs/spec.md index f7325d17..d3a66d41 100644 --- a/docs/vrs/spec.md +++ b/docs/vrs/spec.md @@ -33,6 +33,9 @@ tracked cell - Hooks and personas must be baked into the fixture for every launched agent so the eval tests a controlled, reproducible real-work environment rather than mutable external defaults. +- `shared-workspace-render-ownership` proves that incompatible declarations + cannot race on one workspace target, including through targeted + materialization, while byte-equivalent shared claims remain valid. - An LLM judge is not a defect. Its provider, model, prompt, inputs, and failure behavior must be explicit, and mutation checks must demonstrate useful discrimination. diff --git a/evidence/harness-exclusions.tsv b/evidence/harness-exclusions.tsv index fb928d87..6de1f5e8 100644 --- a/evidence/harness-exclusions.tsv +++ b/evidence/harness-exclusions.tsv @@ -9,6 +9,7 @@ pty-send-peek cell deterministic PTY transport probe; no harness seat presence-ding-matrix cell deterministic native presence and DING probe; no harness seat reconcile-retire-keep cell deterministic native reconcile lifecycle probe; no harness seat render-target-safety cell deterministic native render-policy probe; no harness seat +shared-workspace-render-ownership cell deterministic native shared-render admission probe; no harness seat strict-validation-json cell deterministic native validation-diagnostics probe; no harness seat st2-doctor-structure cell deterministic catalog-health probe; no harness seat st2-network cell deterministic catalog/message round-trip probe; no harness seat