feat(data): add nine dream-system validation scenarios

ntindle · ntindle · commit 409e39d38a1e · 2026-05-13T14:06:12.000-05:00
Add `data/dream-validation.yaml` covering the four roadmap items the
new scorer kinds were built to grade:

P-1.3 retract-vs-soft-delete (Snodgrass bi-temporal):
  - dream-demotion-retract-discipline (passes against compliant fixture)
  - dream-demotion-snodgrass-violation (negative: fixture sets both
    timestamps; scenario MUST fail on timestamp_discipline)

P0.3a stale-fact deprecation:
  - dream-demotion-stale-fact (pricing edge correctly demoted to
    superseded)

P0.3b scoped cascading expiry:
  - dream-demotion-cascade-bounded (single-hop discipline held)
  - dream-demotion-cascade-runaway (negative: fixture touches a 2+ hop
    edge; scenario MUST fail on cascade_bounded)

P1 procedure synthesis (per `dreaming-procedures.md`):
  - dream-procedure-weekly-report (4 ordered steps + 2 parameters)
  - dream-procedure-client-onboarding (different workflow, exercises
    the scorer on a non-degenerate case)

P2 memory dedup:
  - dream-dedup-near-duplicates (clean merge of Sarah-billing facts;
    HubSpot + fiscal-year stay singletons)
  - dream-dedup-false-positive (negative: over-merges Sarah-billing
    with Sarah-manager; scenario MUST fail on ARI + pairwise precision)

Eight JSON fixtures under `data/fixtures/dream/` back the offline path;
each scenario also accepts a `source.raw_exchange_key` swap when
AutoGPT's dream pass starts emitting structured payloads inline.

Pack-level pinning in `tests/unit/dream-validation.test.ts` asserts:
- ship-shape counts (&gt;=4 demotion, &gt;=2 procedure, &gt;=2 dedup)
- every fixture exists on disk relative to the YAML
- happy-path scenarios pass against their golden
- negative scenarios fail in the expected way (timestamp_discipline,
  cascade_bounded, over-merge)

Docs updated: `platform.md` gets a new spec scenario, `current-state.md`
marks it implemented, `e2e-checklist.md` references the test files,
generated workspace inventory + quality score refreshed.
diff --git a/data/dream-validation.yaml b/data/dream-validation.yaml
diff --git a/data/fixtures/dream/dedup-false-positive.json b/data/fixtures/dream/dedup-false-positive.json
@@ -0,0 +1,10 @@
+{
+  "clusters": [
+    [
+      "fact-sarah-billing-1",
+      "fact-sarah-billing-2",
+      "fact-sarah-manager-1"
+    ],
+    ["fact-hubspot-crm-1"]
+  ]
+}
diff --git a/data/fixtures/dream/dedup-near-duplicates.json b/data/fixtures/dream/dedup-near-duplicates.json
@@ -0,0 +1,7 @@
+{
+  "clusters": [
+    ["fact-sarah-billing-1", "fact-sarah-billing-2"],
+    ["fact-hubspot-crm-1"],
+    ["fact-fiscal-year-1"]
+  ]
+}
diff --git a/data/fixtures/dream/demotion-cascade-bounded.json b/data/fixtures/dream/demotion-cascade-bounded.json
@@ -0,0 +1,8 @@
+{
+  "observed": ["edge-northstar-marcus", "edge-northstar-cto"],
+  "cascade_touched": ["edge-northstar-marcus", "edge-northstar-cto"],
+  "retract_actions": [
+    { "uuid": "edge-northstar-marcus", "expired_at_set": true, "invalid_at_set": false, "status": "superseded" },
+    { "uuid": "edge-northstar-cto", "expired_at_set": true, "invalid_at_set": false, "status": "superseded" }
+  ]
+}
diff --git a/data/fixtures/dream/demotion-cascade-runaway.json b/data/fixtures/dream/demotion-cascade-runaway.json
@@ -0,0 +1,13 @@
+{
+  "observed": ["edge-northstar-marcus", "edge-northstar-cto", "edge-hubspot-crm"],
+  "cascade_touched": [
+    "edge-northstar-marcus",
+    "edge-northstar-cto",
+    "edge-hubspot-crm"
+  ],
+  "retract_actions": [
+    { "uuid": "edge-northstar-marcus", "expired_at_set": true, "invalid_at_set": false, "status": "superseded" },
+    { "uuid": "edge-northstar-cto", "expired_at_set": true, "invalid_at_set": false, "status": "superseded" },
+    { "uuid": "edge-hubspot-crm", "expired_at_set": true, "invalid_at_set": false, "status": "superseded" }
+  ]
+}
diff --git a/data/fixtures/dream/demotion-snodgrass-retract.json b/data/fixtures/dream/demotion-snodgrass-retract.json
@@ -0,0 +1,6 @@
+{
+  "observed": ["edge-budget-q2"],
+  "retract_actions": [
+    { "uuid": "edge-budget-q2", "expired_at_set": true, "invalid_at_set": false, "status": "superseded" }
+  ]
+}
diff --git a/data/fixtures/dream/demotion-snodgrass-violation.json b/data/fixtures/dream/demotion-snodgrass-violation.json
@@ -0,0 +1,11 @@
+{
+  "observed": ["edge-budget-q2"],
+  "retract_actions": [
+    {
+      "uuid": "edge-budget-q2",
+      "expired_at_set": true,
+      "invalid_at_set": true,
+      "status": "superseded"
+    }
+  ]
+}
diff --git a/data/fixtures/dream/demotion-stale-fact.json b/data/fixtures/dream/demotion-stale-fact.json
@@ -0,0 +1,11 @@
+{
+  "observed": ["edge-pricing-old"],
+  "retract_actions": [
+    {
+      "uuid": "edge-pricing-old",
+      "expired_at_set": true,
+      "invalid_at_set": false,
+      "status": "superseded"
+    }
+  ]
+}
diff --git a/data/fixtures/dream/procedure-onboarding.json b/data/fixtures/dream/procedure-onboarding.json
@@ -0,0 +1,9 @@
+{
+  "steps": [
+    "create folder in workspace",
+    "draft welcome email",
+    "schedule kickoff call",
+    "share onboarding doc"
+  ],
+  "parameters": ["client_name", "kickoff_date"]
+}
diff --git a/data/fixtures/dream/procedure-weekly-report.json b/data/fixtures/dream/procedure-weekly-report.json
@@ -0,0 +1,9 @@
+{
+  "steps": [
+    "pull metrics from analytics",
+    "summarize per channel",
+    "draft email to stakeholders",
+    "send"
+  ],
+  "parameters": ["recipient_list", "week_window"]
+}
diff --git a/docs/QUALITY_SCORE.md b/docs/QUALITY_SCORE.md
@@ -1,6 +1,6 @@
 # Quality Score
 
-Last updated: 2026-05-12
+Last updated: 2026-05-13
 
 ## Health summary
 
diff --git a/docs/generated/workspace-inventory.md b/docs/generated/workspace-inventory.md
@@ -1,6 +1,6 @@
 # Workspace Inventory
 
-Generated: 2026-05-12T19:50:46.522Z
+Generated: 2026-05-13T19:05:50.184Z
 
 ```text
   AGENTS.md
@@ -402,8 +402,20 @@ src/cli/
   src/cli/main.ts
 src/domains/
 src/domains/evaluation/
+  src/domains/evaluation/clustering.test.ts
+  src/domains/evaluation/clustering.ts
+  src/domains/evaluation/dedup-scorer.test.ts
+  src/domains/evaluation/dedup-scorer.ts
+  src/domains/evaluation/demotion-match.test.ts
+  src/domains/evaluation/demotion-match.ts
+  src/domains/evaluation/demotion-scorer.test.ts
+  src/domains/evaluation/demotion-scorer.ts
   src/domains/evaluation/judge.ts
   src/domains/evaluation/ports.ts
+  src/domains/evaluation/procedure-match.test.ts
+  src/domains/evaluation/procedure-match.ts
+  src/domains/evaluation/procedure-scorer.test.ts
+  src/domains/evaluation/procedure-scorer.ts
   src/domains/evaluation/ranking.test.ts
   src/domains/evaluation/ranking.ts
   src/domains/evaluation/retrieval-scorer.test.ts
diff --git a/docs/product-specs/current-state.md b/docs/product-specs/current-state.md
@@ -36,6 +36,7 @@ Last validated against `platform.md`: 2026-04-17
 - [x] Docker Compose readiness waits for server readiness
 - [x] Human scoring drains an unscored backlog one chat at a time
 - [x] Ranking-scored scenarios grade retrieval relevance against a curated golden set
+- [x] Dream-system scenarios validate demotion, procedure, and dedup behavior
 
 ## Notes
 
diff --git a/docs/product-specs/e2e-checklist.md b/docs/product-specs/e2e-checklist.md
@@ -33,3 +33,4 @@ Derived from `platform.md`. Every scenario should have a coverage owner.
 | Docker Compose readiness waits for server readiness | `docker-compose.yml` + `docs/playbooks/agent-probe-server.md` + `docker compose config` | ✅ covered |
 | Human scoring drains an unscored backlog one chat at a time | `tests/integration/server/human-scoring.test.ts` + `tests/unit/persistence/human-scoring.test.ts` | ✅ covered |
 | Ranking-scored scenarios grade retrieval relevance against a curated golden set | `src/domains/evaluation/ranking.test.ts` + `src/domains/evaluation/retrieval-scorer.test.ts` + `tests/unit/retrieval-memory.test.ts` + `tests/unit/runner.test.ts` | ✅ covered |
+| Dream-system scenarios validate demotion, procedure, and dedup behavior | `src/domains/evaluation/clustering.test.ts` + `src/domains/evaluation/demotion-match.test.ts` + `src/domains/evaluation/procedure-match.test.ts` + `src/domains/evaluation/{demotion,procedure,dedup}-scorer.test.ts` + `tests/unit/dream-validation.test.ts` | ✅ covered |
diff --git a/docs/product-specs/platform.md b/docs/product-specs/platform.md
@@ -313,6 +313,26 @@ average with `pass_threshold`, and forces a scenario fail when any
 persisted to `retrieval_scores` keyed by `scenario_run_id` for replay, and
 the rendered run report surfaces them alongside the LLM-judge dimensions.
 
+### Dream-system scenarios validate demotion, procedure, and dedup behavior
+
+**Given** a scenario YAML that declares one of `demotion:`, `procedure:`, or
+`dedup:` (each a sibling of the existing `retrieval:` block) with its own
+golden expectation, weight, threshold, and `source` (fixture or
+`raw_exchange_key`)
+**When** AgentProbe runs the scenario and the adapter returns the
+corresponding payload (observed demotions / extracted procedure / predicted
+clusters) inline on the last reply's `rawExchange` or via a JSON fixture
+**Then** the runner computes the appropriate metric set — set
+precision/recall/F1 + Snodgrass timestamp discipline + single-hop cascade
+bound for demotion; step-coverage F1 + LCS-normalized order similarity +
+parameter Jaccard for procedure; pairwise P/R/F1 + Adjusted Rand Index for
+dedup — aggregates under per-metric weights, and forces a fail on hard
+violations (Snodgrass conflict, runaway cascade, over- or under-merge below
+threshold). Per-metric and aggregate scores are persisted to
+`demotion_scores`, `procedure_scores`, and `dedup_scores` tables keyed by
+`scenario_run_id`, and the rendered run report surfaces them alongside the
+LLM-judge and retrieval dimensions.
+
 ### Database URL credentials stay redacted in operator-visible output
 
 **Given** an operator configures persistence with a database URL that contains
diff --git a/tests/unit/dream-validation.test.ts b/tests/unit/dream-validation.test.ts
@@ -0,0 +1,120 @@
+import { describe, expect, test } from "bun:test";
+import { existsSync } from "node:fs";
+import { resolve } from "node:path";
+
+import { scoreScenarioDedup } from "../../src/domains/evaluation/dedup-scorer.ts";
+import { scoreScenarioDemotion } from "../../src/domains/evaluation/demotion-scorer.ts";
+import { scoreScenarioProcedure } from "../../src/domains/evaluation/procedure-scorer.ts";
+import { parseScenarioYaml } from "../../src/domains/validation/load-suite.ts";
+
+const SCENARIOS_PATH = resolve(
+  import.meta.dir,
+  "..",
+  "..",
+  "data",
+  "dream-validation.yaml",
+);
+
+describe("dream-validation pack", () => {
+  const parsed = parseScenarioYaml(SCENARIOS_PATH);
+  const scenarios = parsed.scenarios;
+
+  function requireScenario(id: string) {
+    const scenario = scenarios.find((s) => s.id === id);
+    if (!scenario) {
+      throw new Error(`Missing scenario: ${id}`);
+    }
+    return scenario;
+  }
+
+  test("ships at least four demotion, two procedure, and two dedup scenarios", () => {
+    const demotion = scenarios.filter((s) => s.demotion !== undefined);
+    const procedure = scenarios.filter((s) => s.procedure !== undefined);
+    const dedup = scenarios.filter((s) => s.dedup !== undefined);
+    expect(demotion.length).toBeGreaterThanOrEqual(4);
+    expect(procedure.length).toBeGreaterThanOrEqual(2);
+    expect(dedup.length).toBeGreaterThanOrEqual(2);
+  });
+
+  test("every scenario references a fixture that exists on disk", () => {
+    for (const scenario of scenarios) {
+      const fixture =
+        scenario.demotion?.source?.fixture ??
+        scenario.procedure?.source?.fixture ??
+        scenario.dedup?.source?.fixture;
+      expect(fixture).toBeDefined();
+      const resolved = resolve(SCENARIOS_PATH, "..", fixture ?? "");
+      expect(existsSync(resolved)).toBe(true);
+    }
+  });
+
+  test("Snodgrass-respecting retract scenario passes", () => {
+    const scenario = requireScenario("dream-demotion-retract-discipline");
+    const score = scoreScenarioDemotion(scenario, {
+      scenariosPath: SCENARIOS_PATH,
+    });
+    expect(score?.source).toBe("fixture");
+    expect(score?.timestampViolationCount).toBe(0);
+    expect(score?.passed).toBe(true);
+  });
+
+  test("Snodgrass-violating retract scenario fails on timestamp discipline", () => {
+    const scenario = requireScenario("dream-demotion-snodgrass-violation");
+    const score = scoreScenarioDemotion(scenario, {
+      scenariosPath: SCENARIOS_PATH,
+    });
+    expect(score?.timestampViolationCount).toBeGreaterThan(0);
+    expect(score?.passed).toBe(false);
+  });
+
+  test("bounded cascade scenario passes (single-hop discipline held)", () => {
+    const scenario = requireScenario("dream-demotion-cascade-bounded");
+    const score = scoreScenarioDemotion(scenario, {
+      scenariosPath: SCENARIOS_PATH,
+    });
+    expect(score?.cascadeBounded).toBe(true);
+    expect(score?.passed).toBe(true);
+  });
+
+  test("runaway cascade scenario fails on cascade_bounded", () => {
+    const scenario = requireScenario("dream-demotion-cascade-runaway");
+    const score = scoreScenarioDemotion(scenario, {
+      scenariosPath: SCENARIOS_PATH,
+    });
+    expect(score?.cascadeBounded).toBe(false);
+    expect(score?.passed).toBe(false);
+  });
+
+  test("weekly-report procedure scenario passes against its golden", () => {
+    const scenario = requireScenario("dream-procedure-weekly-report");
+    const score = scoreScenarioProcedure(scenario, {
+      scenariosPath: SCENARIOS_PATH,
+    });
+    expect(score?.source).toBe("fixture");
+    expect(score?.passed).toBe(true);
+  });
+
+  test("client-onboarding procedure scenario passes against its golden", () => {
+    const scenario = requireScenario("dream-procedure-client-onboarding");
+    const score = scoreScenarioProcedure(scenario, {
+      scenariosPath: SCENARIOS_PATH,
+    });
+    expect(score?.passed).toBe(true);
+  });
+
+  test("clean dedup scenario passes (no over-merge, no under-merge)", () => {
+    const scenario = requireScenario("dream-dedup-near-duplicates");
+    const score = scoreScenarioDedup(scenario, {
+      scenariosPath: SCENARIOS_PATH,
+    });
+    expect(score?.passed).toBe(true);
+  });
+
+  test("over-merge dedup scenario fails on pairwise precision + ARI", () => {
+    const scenario = requireScenario("dream-dedup-false-positive");
+    const score = scoreScenarioDedup(scenario, {
+      scenariosPath: SCENARIOS_PATH,
+    });
+    expect(score?.passed).toBe(false);
+  });
+});
