diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bc9907..8871ece 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -146,6 +146,12 @@ The format is adapted from [Keep a Changelog](https://keepachangelog.com/); vers (pre-existing, introduced with the records in v0.8.0-era curation). Metadata only: `provenance` content is not report-counted, so rankings/counts/reports are unchanged; the curation test now pins `grc` per record. Both `presentation.json` copies stay byte-identical. +- **Committed-report reproducibility tests now fail loud on a missing result file.** The + threshold-t2 / bm25 / bm25-threshold reproducibility tests silently `skipTest`ed when the + committed JSON was absent — so a deleted or renamed report read as "reproducibility verified" + when nothing ran (cross-model review finding on #48). A missing committed report is a broken + contract pin and now fails with the exact regenerate command, matching the baseline test's + behavior. No scoring, metric, or report change. ### Changed - Deferred follow-up: rename the older controller `renderer-bypass` boundary reason to diff --git a/tests/test_retrieval_benchmark.py b/tests/test_retrieval_benchmark.py index dd293f0..8183d15 100644 --- a/tests/test_retrieval_benchmark.py +++ b/tests/test_retrieval_benchmark.py @@ -280,7 +280,14 @@ def test_candidate_cli_validation(self): def test_committed_threshold_t2_is_reproducible(self): t2_path = ROOT / "docs" / "benchmarks" / "results" / "retrieval-v1-lexical-threshold-t2.json" if not t2_path.exists(): - self.skipTest("threshold t=2 result not yet committed") + # Committed reports are a contract — a missing file is a broken pin, not a skip + # (a silent skip here once read as "reproducibility verified" when nothing ran). + self.fail( + "committed threshold t=2 result is missing — regenerate with:\n" + " python scripts/run_retrieval_benchmark.py --candidate lexical-threshold" + " --threshold 2 \\\n" + " --json-out docs/benchmarks/results/retrieval-v1-lexical-threshold-t2.json" + ) committed = json.loads(t2_path.read_text(encoding="utf-8")) fresh = self.bm.reproducible_view(self.bm.run_benchmark( "project", self.bm.DEFAULT_KS, @@ -394,7 +401,12 @@ def test_bm25_cli_validation(self): def test_committed_bm25_is_reproducible(self): path = ROOT / "docs" / "benchmarks" / "results" / "retrieval-v1-lexical-bm25.json" if not path.exists(): - self.skipTest("bm25 result not yet committed") + # Committed reports are a contract — a missing file is a broken pin, not a skip. + self.fail( + "committed bm25 result is missing — regenerate with:\n" + " python scripts/run_retrieval_benchmark.py --candidate lexical-bm25 \\\n" + " --json-out docs/benchmarks/results/retrieval-v1-lexical-bm25.json" + ) committed = json.loads(path.read_text(encoding="utf-8")) fresh = self.bm.reproducible_view(self.bm.run_benchmark( "project", self.bm.DEFAULT_KS, candidate=dict(self.BM25))) @@ -470,7 +482,16 @@ def test_committed_bm25_threshold_reports_are_reproducible(self): path = ROOT / "docs" / "benchmarks" / "results" / ( "retrieval-v1-lexical-bm25-threshold-t{}.json".format(threshold)) if not path.exists(): - self.skipTest("bm25 threshold t={} result not yet committed".format(threshold)) + # Committed reports are a contract — a missing file is a broken pin, not a skip. + self.fail( + "committed bm25 threshold t={} result is missing — regenerate with:\n" + " python scripts/run_retrieval_benchmark.py --candidate" + " lexical-bm25-threshold --threshold {} \\\n" + " --json-out docs/benchmarks/results/" + "retrieval-v1-lexical-bm25-threshold-t{}.json".format( + threshold, threshold, threshold + ) + ) committed = json.loads(path.read_text(encoding="utf-8")) fresh = self.bm.reproducible_view(self.bm.run_benchmark( "project", self.bm.DEFAULT_KS, candidate=dict(candidate)))