test(benchmark): fail loud when a committed report file is missing

CHANGELOG.md

@@ -146,6 +146,12 @@ The format is adapted from [Keep a Changelog](https://keepachangelog.com/); vers
   (pre-existing, introduced with the records in v0.8.0-era curation). Metadata only: `provenance`
   content is not report-counted, so rankings/counts/reports are unchanged; the curation test now
   pins `grc` per record. Both `presentation.json` copies stay byte-identical.
+- **Committed-report reproducibility tests now fail loud on a missing result file.** The
+  threshold-t2 / bm25 / bm25-threshold reproducibility tests silently `skipTest`ed when the
+  committed JSON was absent — so a deleted or renamed report read as "reproducibility verified"
+  when nothing ran (cross-model review finding on #48). A missing committed report is a broken
+  contract pin and now fails with the exact regenerate command, matching the baseline test's
+  behavior. No scoring, metric, or report change.
 
 ### Changed
 - Deferred follow-up: rename the older controller `renderer-bypass` boundary reason to

tests/test_retrieval_benchmark.py

@@ -280,7 +280,14 @@ def test_candidate_cli_validation(self):
     def test_committed_threshold_t2_is_reproducible(self):
         t2_path = ROOT / "docs" / "benchmarks" / "results" / "retrieval-v1-lexical-threshold-t2.json"
         if not t2_path.exists():
-            self.skipTest("threshold t=2 result not yet committed")
+            # Committed reports are a contract — a missing file is a broken pin, not a skip
+            # (a silent skip here once read as "reproducibility verified" when nothing ran).
+            self.fail(
+                "committed threshold t=2 result is missing — regenerate with:\n"
+                "  python scripts/run_retrieval_benchmark.py --candidate lexical-threshold"
+                " --threshold 2 \\\n"
+                "    --json-out docs/benchmarks/results/retrieval-v1-lexical-threshold-t2.json"
+            )
         committed = json.loads(t2_path.read_text(encoding="utf-8"))
         fresh = self.bm.reproducible_view(self.bm.run_benchmark(
             "project", self.bm.DEFAULT_KS,
@@ -394,7 +401,12 @@ def test_bm25_cli_validation(self):
     def test_committed_bm25_is_reproducible(self):
         path = ROOT / "docs" / "benchmarks" / "results" / "retrieval-v1-lexical-bm25.json"
         if not path.exists():
-            self.skipTest("bm25 result not yet committed")
+            # Committed reports are a contract — a missing file is a broken pin, not a skip.
+            self.fail(
+                "committed bm25 result is missing — regenerate with:\n"
+                "  python scripts/run_retrieval_benchmark.py --candidate lexical-bm25 \\\n"
+                "    --json-out docs/benchmarks/results/retrieval-v1-lexical-bm25.json"
+            )
         committed = json.loads(path.read_text(encoding="utf-8"))
         fresh = self.bm.reproducible_view(self.bm.run_benchmark(
             "project", self.bm.DEFAULT_KS, candidate=dict(self.BM25)))
@@ -470,7 +482,16 @@ def test_committed_bm25_threshold_reports_are_reproducible(self):
             path = ROOT / "docs" / "benchmarks" / "results" / (
                 "retrieval-v1-lexical-bm25-threshold-t{}.json".format(threshold))
             if not path.exists():
-                self.skipTest("bm25 threshold t={} result not yet committed".format(threshold))
+                # Committed reports are a contract — a missing file is a broken pin, not a skip.
+                self.fail(
+                    "committed bm25 threshold t={} result is missing — regenerate with:\n"
+                    "  python scripts/run_retrieval_benchmark.py --candidate"
+                    " lexical-bm25-threshold --threshold {} \\\n"
+                    "    --json-out docs/benchmarks/results/"
+                    "retrieval-v1-lexical-bm25-threshold-t{}.json".format(
+                        threshold, threshold, threshold
+                    )
+                )
             committed = json.loads(path.read_text(encoding="utf-8"))
             fresh = self.bm.reproducible_view(self.bm.run_benchmark(
                 "project", self.bm.DEFAULT_KS, candidate=dict(candidate)))