diff --git a/benchmarks/scripts/build_vended_models.py b/benchmarks/scripts/build_vended_models.py index 167bb69f..92a05d3c 100644 --- a/benchmarks/scripts/build_vended_models.py +++ b/benchmarks/scripts/build_vended_models.py @@ -31,8 +31,7 @@ import argparse import json import logging -import subprocess # nosec B404 - used with list args, no shell, hardcoded 'git' -from datetime import date +import hashlib from pathlib import Path from typing import Any @@ -53,8 +52,12 @@ DEFAULT_OUT = _REPO_ROOT / "vend" / "swe-router" / "models.json" # Bumped when the shape of models.json changes in a way a consumer would notice. -# Consumers pin this; the skill refuses a major it does not know. -SCHEMA_VERSION = "1.0" +# Consumers pin this; the skill refuses a major it does not know. 1.1 replaced +# provenance.source_commit (a git sha) with source_sha256 (a content hash) and +# took measured_on from the run summaries instead of a commit date. +SCHEMA_VERSION = "1.1" +# provenance.source_sha256 length: enough to tell versions apart, short to read. +SOURCE_HASH_CHARS = 12 # The judge is not recorded in the frontier JSON, so it is stated here. Keep in # step with codex_judge.py's default (JUDGE_MODEL overrides it per run). @@ -77,78 +80,59 @@ } -def _git_commit(source: Path) -> str | None: - """Return the short sha of the commit that last changed ``source``. +def _source_sha256(source: Path) -> str: + """Return the first 12 hex digits of the SHA-256 of ``source``'s bytes. - Deliberately not HEAD. HEAD moves with every commit to the repository, so - stamping it would make the generated file differ after any unrelated change - and turn the --check guard into noise. The commit that last touched the - source is what actually identifies this version of the data. + Identifies the exact version of the data from the file itself. An earlier + version stamped the commit that last touched the file, which a squash merge + replaces: the stamp made on the branch then no longer matched ``main``, and + the committed-artifact check failed with every score still correct (after + #184, #210 and #219). A content hash is the same on the branch and after the + merge, and anyone can check it with ``sha256sum | cut -c1-12``. Args: source: The frontier JSON. Returns: - The abbreviated commit, or None outside a git checkout or for a file - that has never been committed. + A 12-character lowercase hex string. """ - try: - out = subprocess.run( # nosec B603 B607 - hardcoded 'git', list args, no shell - [ - "git", - "-C", - str(source.parent), - "log", - "-1", - "--format=%h", - "--", - source.name, - ], - capture_output=True, - text=True, - timeout=10, - check=False, - ) - except (OSError, subprocess.SubprocessError): - return None - return out.stdout.strip() or None + return hashlib.sha256(source.read_bytes()).hexdigest()[:SOURCE_HASH_CHARS] -def _measured_on(source: Path) -> str: - """Return the date the source frontier was last modified, ISO format. +def _latest_run_date( + runs_dir: Path, harness: str, skill: str, dataset: str, models: set[str] +) -> str | None: + """Return the latest ``run_date`` among the run summaries behind the frontier. - Uses the file's last git commit date when available, so regenerating without - changing the data does not advance the measurement date. Falls back to the - filesystem mtime. + That is when the newest measurement in the file was taken, which is what a + consumer means by "measured on". It comes from committed data, so it does + not move when a commit is rewritten, and it does not depend on anyone's + timezone. Args: - source: The frontier JSON. + runs_dir: The swe-benchmark-data root. + harness: Harness folder, e.g. "omp". + skill: Skill folder, e.g. "swe3". + dataset: Dataset scope folder. + models: The model slugs the frontier JSON lists; other runs are ignored. Returns: - An ISO date string. + An ISO date, or None when no matching summary records a ``run_date``. """ - try: - out = subprocess.run( # nosec B603 B607 - hardcoded 'git', list args, no shell - [ - "git", - "-C", - str(source.parent), - "log", - "-1", - "--format=%cs", - "--", - source.name, - ], - capture_output=True, - text=True, - timeout=10, - check=False, + dates = [] + pattern = f"*/{harness}/{skill}/{dataset}/run-summary.json" + for path in runs_dir.glob(pattern): + summary = json.loads(path.read_text(encoding="utf-8")) + if summary.get("model_slug") in models and summary.get("run_date"): + dates.append(summary["run_date"]) + if not dates: + logger.warning( + "no run_date in any %s summary for the frontier's models; " + "measured_on will be null", + pattern, ) - if out.stdout.strip(): - return out.stdout.strip() - except (OSError, subprocess.SubprocessError): - pass - return date.fromtimestamp(source.stat().st_mtime).isoformat() + return None + return max(dates) def _relative_or_name(source: Path, repo_root: Path) -> str: @@ -243,7 +227,8 @@ def build( Args: source: Path to a ``pareto-frontier-*.json``. - repo_root: Repository root, used for the provenance commit. + repo_root: Repository root, used to record the source path. + runs_dir: The swe-benchmark-data root; defaults to the committed one. Returns: The payload to write as models.json. @@ -264,11 +249,19 @@ def build( bedrock = {m["model"] for m in data.get("bedrock_frontier", [])} self_hosted = {m["model"] for m in data.get("self_hosted_frontier", [])} + runs_dir = runs_dir if runs_dir is not None else DEFAULT_RUNS_DIR by_tier = per_tier_stats( - runs_dir if runs_dir is not None else DEFAULT_RUNS_DIR, + runs_dir, + data.get("harness", ""), + data.get("skill", ""), + data.get("repo", ""), + ) + measured_on = _latest_run_date( + runs_dir, data.get("harness", ""), data.get("skill", ""), data.get("repo", ""), + {m["model"] for m in models}, ) out_models = [] for m in sorted(models, key=lambda x: -x["mean_score"]): @@ -305,9 +298,9 @@ def build( "schema_version": SCHEMA_VERSION, "generated_by": "benchmarks/scripts/build_vended_models.py", "provenance": { - "measured_on": _measured_on(source), + "measured_on": measured_on, "source_file": _relative_or_name(source, repo_root), - "source_commit": _git_commit(source), + "source_sha256": _source_sha256(source), "harness": data.get("harness"), "skill": data.get("skill"), "dataset": data.get("repo"), diff --git a/benchmarks/tests/test_build_vended_models.py b/benchmarks/tests/test_build_vended_models.py index 69772450..05504792 100644 --- a/benchmarks/tests/test_build_vended_models.py +++ b/benchmarks/tests/test_build_vended_models.py @@ -2,9 +2,9 @@ from __future__ import annotations +import hashlib import importlib.util import json -import subprocess import sys import tempfile import unittest @@ -67,6 +67,19 @@ def _write(data: dict) -> Path: return Path(tmp.name) +def _runs(dates: dict[str, str | None]) -> Path: + """Write one omp/swe3/v2 run summary per model, with the given run_date.""" + root = Path(tempfile.mkdtemp()) + for model, run_date in dates.items(): + folder = root / model / "omp" / "swe3" / "mcp-gateway-registry-v2" + folder.mkdir(parents=True) + summary: dict = {"model_slug": model, "tasks": []} + if run_date: + summary["run_date"] = run_date + (folder / "run-summary.json").write_text(json.dumps(summary), "utf-8") + return root + + class BuildTest(unittest.TestCase): def test_includes_every_model_not_only_the_frontier(self) -> None: # The skill filters to what the user's assistant offers before ranking, @@ -119,7 +132,8 @@ def test_incomplete_runs_stay_visible(self) -> None: def test_provenance_carries_the_measurement_context(self) -> None: # A vended file has no reader who knows this repo, so the context # travels with the data. - out = bvm.build(_write(_frontier()), _REPO_ROOT) + runs = _runs({"big": "2026-10-01", "small": "2026-09-01"}) + out = bvm.build(_write(_frontier()), _REPO_ROOT, runs_dir=runs) p = out["provenance"] self.assertEqual(p["harness"], "omp") self.assertEqual(p["skill"], "swe3") @@ -141,6 +155,40 @@ def test_empty_source_is_a_clear_error(self) -> None: with self.assertRaisesRegex(SystemExit, "no all_models"): bvm.build(_write(_frontier(all_models=[])), _REPO_ROOT) + def test_provenance_ignores_where_the_file_lives(self) -> None: + # The squash-merge case: the same bytes must stamp the same provenance, + # whatever commit or path they arrived under. + runs = _runs({"big": "2026-10-01"}) + first = bvm.build(_write(_frontier()), _REPO_ROOT, runs_dir=runs) + second = bvm.build(_write(_frontier()), _REPO_ROOT, runs_dir=runs) + keys = ("measured_on", "source_sha256") + self.assertEqual( + [first["provenance"][k] for k in keys], + [second["provenance"][k] for k in keys], + ) + + def test_source_sha256_changes_with_the_data(self) -> None: + runs = _runs({}) + before = bvm.build(_write(_frontier()), _REPO_ROOT, runs_dir=runs) + after = bvm.build(_write(_frontier(note="new")), _REPO_ROOT, runs_dir=runs) + self.assertNotEqual( + before["provenance"]["source_sha256"], + after["provenance"]["source_sha256"], + ) + + def test_measured_on_is_the_latest_run_of_a_frontier_model(self) -> None: + # "other" is not in the frontier JSON, so its later date is ignored. + runs = _runs( + {"big": "2026-10-01", "small": "2026-09-15", "other": "2026-12-31"} + ) + out = bvm.build(_write(_frontier()), _REPO_ROOT, runs_dir=runs) + self.assertEqual(out["provenance"]["measured_on"], "2026-10-01") + + def test_measured_on_is_null_without_any_run_date(self) -> None: + runs = _runs({"big": None, "small": None}) + out = bvm.build(_write(_frontier()), _REPO_ROOT, runs_dir=runs) + self.assertIsNone(out["provenance"]["measured_on"]) + class CommittedArtifactTest(unittest.TestCase): """The vended files are committed, so they are checked like any other input.""" @@ -194,35 +242,11 @@ def test_tier_scores_bracket_the_overall_mean(self) -> None: self.assertLessEqual(min(tiers), m["score"] + 0.01, m["model"]) self.assertGreaterEqual(max(tiers), m["score"] - 0.01, m["model"]) - def test_source_commit_tracks_the_frontier_not_head(self) -> None: - # Stamping HEAD would make the file differ after every unrelated commit - # and turn --check into noise. It must identify the data's version. + def test_source_sha256_matches_the_committed_frontier_bytes(self) -> None: + # What `sha256sum | cut -c1-12` prints, so anyone can check it. payload = bvm.build(bvm.DEFAULT_SOURCE, _REPO_ROOT) - stamped = payload["provenance"]["source_commit"] - head = subprocess.run( - ["git", "-C", str(_REPO_ROOT), "rev-parse", "--short", "HEAD"], - capture_output=True, - text=True, - check=False, - ).stdout.strip() - last_touch = subprocess.run( - [ - "git", - "-C", - str(_REPO_ROOT), - "log", - "-1", - "--format=%h", - "--", - str(bvm.DEFAULT_SOURCE.relative_to(_REPO_ROOT)), - ], - capture_output=True, - text=True, - check=False, - ).stdout.strip() - self.assertEqual(stamped, last_touch) - if head != last_touch: - self.assertNotEqual(stamped, head) + digest = hashlib.sha256(bvm.DEFAULT_SOURCE.read_bytes()).hexdigest()[:12] + self.assertEqual(payload["provenance"]["source_sha256"], digest) def test_vend_dir_holds_exactly_the_portable_files(self) -> None: # The portability contract: the skill must work in a directory holding diff --git a/vend/swe-router/README.md b/vend/swe-router/README.md index 901a01c4..eec1ab26 100644 --- a/vend/swe-router/README.md +++ b/vend/swe-router/README.md @@ -82,7 +82,7 @@ If it is firing when you do not want it, narrow the `Run it BEFORE…` sentence **Self-hosted costs are measured under load.** Hosted figures come from a metered bill. Self-hosted figures come from the server's hourly price divided by throughput measured at a stated concurrency — a platform team serving a group of developers, not one person with an idle GPU. Both are a cost per task, and the skill ranks them together. -**These numbers move.** A fix to token accounting once changed `claude-opus-5` from $7.63 to $11.95 per task while every score stayed identical. Check `provenance.measured_on` and refetch if it looks old. +**These numbers move.** A fix to token accounting once changed `claude-opus-5` from $7.63 to $11.95 per task while every score stayed identical. Check `provenance.measured_on`, the date of the newest run in the file, and refetch if it looks old. `provenance.source_sha256` identifies the exact data: it is the first 12 hex digits of the SHA-256 of the frontier file it was built from, so two copies with the same value hold the same numbers. ## How it decides diff --git a/vend/swe-router/models.json b/vend/swe-router/models.json index 9c85849c..e91df795 100644 --- a/vend/swe-router/models.json +++ b/vend/swe-router/models.json @@ -1,10 +1,10 @@ { - "schema_version": "1.0", + "schema_version": "1.1", "generated_by": "benchmarks/scripts/build_vended_models.py", "provenance": { - "measured_on": "2026-10-10", + "measured_on": "2026-10-11", "source_file": "docs/metrics/pareto-frontier-omp-swe3.json", - "source_commit": "1c86108", + "source_sha256": "72b1ed9ee222", "harness": "omp", "skill": "swe3", "dataset": "mcp-gateway-registry-v2",