fix: guard --recreate against deleting non-benchmark indexes; fail on zero latency baseline

Addresses two review findings:

- BLOCKING: --recreate deleted whatever MOSS_INDEX_NAME pointed at, so a
  developer with a shared or production Moss project in their env could
  destroy a non-benchmark index. Deletion is now guarded three ways: the
  derived benchmark-ci-<signature> name deletes without confirmation, an
  overridden name inside the benchmark-ci-* namespace additionally
  requires the new --force flag, and names outside that namespace are
  refused even with --force.

- CONSIDER: the checked-in zero latency baseline made
  test_no_latency_regression skip on every run, leaving the latency
  guard silently inactive until someone manually armed it. A zero
  baseline now FAILS comparison runs with the arming procedure in the
  message. update_baseline dispatch runs are unaffected (they do not
  pass --baseline-file), and fork PRs skip earlier at the credentials
  gate, so the failure lands exactly on trusted runs that should be
  enforcing the guard. baseline.json note and README updated; the first
  trusted CI run's artifact is the natural baseline to commit.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: Sravan1011

benchmarks/ci/README.md

@@ -50,6 +50,12 @@ expected document IDs to `ground_truth.json`. Commit the updated file.
 After a corpus or model change, pass `--recreate` so the index is rebuilt
 from the current corpus before the ground truth is captured.
 
+`--recreate` deletion is guarded: only the derived `benchmark-ci-<hash>`
+index is deleted without confirmation. If `MOSS_INDEX_NAME` overrides the
+name, deletion additionally requires `--force`, and indexes outside the
+`benchmark-ci-*` namespace are never deleted — so a stray environment
+variable pointing at a shared or production index cannot be destroyed.
+
 > **Note**: the ground truth is a *ranking-stability reference* generated by
 > Moss itself at a known-good commit — not an independent relevance judgment.
 > The recall gate detects changes in retrieval behavior; an intentional
@@ -63,6 +69,10 @@ The harness compares the current run's metrics against `baseline.json`:
 - **Latency**: Fails if P95 increases by more than the threshold (default 20%)
 - **Recall**: Fails if Recall@5 drops by more than the threshold (default 5pp)
 
+A zero latency baseline **fails** comparison runs rather than skipping — the
+guard cannot stay silently inactive. To arm it, commit a CI-captured
+baseline (see "Updating the baseline" below).
+
 Thresholds are configurable via CLI flags:
 
 ```bash

benchmarks/ci/baseline.json

@@ -1,7 +1,7 @@
 {
   "commit": "f4732a1",
   "timestamp": "2026-07-20T05:35:00+00:00",
-  "_note": "Recall values are measured (hardware-independent) so the recall guard is active. Latency values are intentionally zero \u2014 latency is hardware-dependent, so the guard skips until a baseline captured on CI runners replaces this file (run the Benchmark workflow, download the benchmark-results-<sha> artifact, commit it here).",
+  "_note": "Recall values are measured (hardware-independent) so the recall guard is active. Latency values are intentionally zero and the latency guard FAILS on zero baselines in comparison runs \u2014 the first trusted CI run will be red until a CI-captured baseline is committed: run the Benchmark workflow (update_baseline=true also works), download the benchmark-results-<sha> artifact, and commit it here. Latency baselines must come from CI runners; numbers from other hardware are not comparable.",
   "latency_ms": {
     "p50": 0,
     "p95": 0,

benchmarks/ci/generate_ground_truth.py

@@ -36,6 +36,7 @@
 
 from bench_queries import (
     DOC_COUNT,
+    INDEX_NAME_PREFIX,
     MODEL_ID,
     QUERIES,
     corpus_signature,
@@ -70,7 +71,32 @@ async def _create_index(client, index_name: str, corpus_slice: list[dict]) -> No
     print(f"Created index '{index_name}' with {result.doc_count} docs")
 
 
-async def main(recreate: bool) -> None:
+def _guard_deletion(index_name: str, derived_name: str, force: bool) -> None:
+    """Refuse to delete indexes that are not clearly benchmark-owned.
+
+    MOSS_INDEX_NAME is a documented override, so a developer whose
+    environment points at a shared or production Moss project could
+    otherwise aim --recreate at a non-benchmark index and destroy it.
+    """
+    if index_name == derived_name:
+        return  # the derived benchmark index — always safe to recreate
+    if not index_name.startswith(f"{INDEX_NAME_PREFIX}-"):
+        print(
+            f"Error: refusing to delete index '{index_name}' — it is outside "
+            f"the benchmark namespace ('{INDEX_NAME_PREFIX}-*'). Unset "
+            "MOSS_INDEX_NAME or point it at a benchmark index."
+        )
+        sys.exit(1)
+    if not force:
+        print(
+            f"Error: MOSS_INDEX_NAME overrides the derived name "
+            f"('{index_name}' != '{derived_name}'). Pass --force to confirm "
+            "deleting the overridden benchmark index."
+        )
+        sys.exit(1)
+
+
+async def main(recreate: bool, force: bool) -> None:
     from moss import MossClient, QueryOptions
 
     project_id = os.getenv("MOSS_PROJECT_ID")
@@ -94,6 +120,7 @@ async def main(recreate: bool) -> None:
     existing = {idx.name for idx in await client.list_indexes()}
 
     if index_name in existing and recreate:
+        _guard_deletion(index_name, index_name_for(signature), force)
         print(f"--recreate: deleting existing index '{index_name}'")
         await client.delete_index(index_name)
         existing.discard(index_name)
@@ -147,5 +174,12 @@ async def main(recreate: bool) -> None:
         "querying. Required after a corpus or embedding-model change so the "
         "ground truth reflects the current data.",
     )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Confirm --recreate deletion when MOSS_INDEX_NAME overrides the "
+        "derived index name. Only benchmark-namespace indexes "
+        "(benchmark-ci-*) can be deleted even with this flag.",
+    )
     args = parser.parse_args()
-    asyncio.run(main(recreate=args.recreate))
+    asyncio.run(main(recreate=args.recreate, force=args.force))

benchmarks/ci/test_bench_ci_moss.py

@@ -421,7 +421,16 @@ def test_no_latency_regression(self, request, benchmark_results):
         _assert_baseline_compatible(baseline, benchmark_results)
 
         if baseline_p95 == 0:
-            pytest.skip("Baseline p95 is zero — cannot compute regression ratio")
+            # A zero baseline means the latency guard has never been armed.
+            # Skipping here would let every run pass with the guard silently
+            # inactive — fail instead, with the arming procedure.
+            pytest.fail(
+                "Baseline p95 is zero — the latency guard is not armed. Run the "
+                "Benchmark workflow with update_baseline=true, download the "
+                "benchmark-results-<sha> artifact, and commit it as "
+                "benchmarks/ci/baseline.json (values must come from CI runners; "
+                "this run's artifact works too)."
+            )
 
         regression = (current_p95 - baseline_p95) / baseline_p95