From 24e15fab367c732b8ae94d573571b763b8476f5d Mon Sep 17 00:00:00 2001 From: Amit Arora Date: Tue, 13 Oct 2026 17:00:08 +0000 Subject: [PATCH] Stamp models.json provenance from the data, not from git build_vended_models.py stamped models.json with the commit that last touched the omp frontier JSON and that commit's date. A squash merge replaces the branch commit, so the stamp made on a PR no longer matched main and test_committed_models_json_matches_the_generator failed with every score still correct. It happened after #184, #210 and #219. Both fields now come from committed data, which a merge does not rewrite: - source_commit becomes source_sha256: the first 12 hex digits of the frontier JSON's SHA-256, the value `sha256sum | cut -c1-12` prints. It changes only when the data does. - measured_on is the latest run_date among the run summaries of the models in the frontier, rather than a commit date in the merger's timezone. For the current data that is 2026-10-11, Sonnet 5.5's run. schema_version goes to 1.1 for the renamed field. route.py and the skill check only the major version, so both keep working. The generator no longer runs git, which removes its two subprocess calls. Tests: the git-based provenance test is replaced by checks that the hash matches the file bytes, changes with the data and ignores where the file lives, and that measured_on takes the latest frontier model's run date and is null when no summary records one. A simulated squash merge onto main, with a different commit, date and timezone, passes --check. --- benchmarks/scripts/build_vended_models.py | 121 +++++++++---------- benchmarks/tests/test_build_vended_models.py | 84 ++++++++----- vend/swe-router/README.md | 2 +- vend/swe-router/models.json | 6 +- 4 files changed, 115 insertions(+), 98 deletions(-) diff --git a/benchmarks/scripts/build_vended_models.py b/benchmarks/scripts/build_vended_models.py index 167bb69f..92a05d3c 100644 --- a/benchmarks/scripts/build_vended_models.py +++ b/benchmarks/scripts/build_vended_models.py @@ -31,8 +31,7 @@ import argparse import json import logging -import subprocess # nosec B404 - used with list args, no shell, hardcoded 'git' -from datetime import date +import hashlib from pathlib import Path from typing import Any @@ -53,8 +52,12 @@ DEFAULT_OUT = _REPO_ROOT / "vend" / "swe-router" / "models.json" # Bumped when the shape of models.json changes in a way a consumer would notice. -# Consumers pin this; the skill refuses a major it does not know. -SCHEMA_VERSION = "1.0" +# Consumers pin this; the skill refuses a major it does not know. 1.1 replaced +# provenance.source_commit (a git sha) with source_sha256 (a content hash) and +# took measured_on from the run summaries instead of a commit date. +SCHEMA_VERSION = "1.1" +# provenance.source_sha256 length: enough to tell versions apart, short to read. +SOURCE_HASH_CHARS = 12 # The judge is not recorded in the frontier JSON, so it is stated here. Keep in # step with codex_judge.py's default (JUDGE_MODEL overrides it per run). @@ -77,78 +80,59 @@ } -def _git_commit(source: Path) -> str | None: - """Return the short sha of the commit that last changed ``source``. +def _source_sha256(source: Path) -> str: + """Return the first 12 hex digits of the SHA-256 of ``source``'s bytes. - Deliberately not HEAD. HEAD moves with every commit to the repository, so - stamping it would make the generated file differ after any unrelated change - and turn the --check guard into noise. The commit that last touched the - source is what actually identifies this version of the data. + Identifies the exact version of the data from the file itself. An earlier + version stamped the commit that last touched the file, which a squash merge + replaces: the stamp made on the branch then no longer matched ``main``, and + the committed-artifact check failed with every score still correct (after + #184, #210 and #219). A content hash is the same on the branch and after the + merge, and anyone can check it with ``sha256sum | cut -c1-12``. Args: source: The frontier JSON. Returns: - The abbreviated commit, or None outside a git checkout or for a file - that has never been committed. + A 12-character lowercase hex string. """ - try: - out = subprocess.run( # nosec B603 B607 - hardcoded 'git', list args, no shell - [ - "git", - "-C", - str(source.parent), - "log", - "-1", - "--format=%h", - "--", - source.name, - ], - capture_output=True, - text=True, - timeout=10, - check=False, - ) - except (OSError, subprocess.SubprocessError): - return None - return out.stdout.strip() or None + return hashlib.sha256(source.read_bytes()).hexdigest()[:SOURCE_HASH_CHARS] -def _measured_on(source: Path) -> str: - """Return the date the source frontier was last modified, ISO format. +def _latest_run_date( + runs_dir: Path, harness: str, skill: str, dataset: str, models: set[str] +) -> str | None: + """Return the latest ``run_date`` among the run summaries behind the frontier. - Uses the file's last git commit date when available, so regenerating without - changing the data does not advance the measurement date. Falls back to the - filesystem mtime. + That is when the newest measurement in the file was taken, which is what a + consumer means by "measured on". It comes from committed data, so it does + not move when a commit is rewritten, and it does not depend on anyone's + timezone. Args: - source: The frontier JSON. + runs_dir: The swe-benchmark-data root. + harness: Harness folder, e.g. "omp". + skill: Skill folder, e.g. "swe3". + dataset: Dataset scope folder. + models: The model slugs the frontier JSON lists; other runs are ignored. Returns: - An ISO date string. + An ISO date, or None when no matching summary records a ``run_date``. """ - try: - out = subprocess.run( # nosec B603 B607 - hardcoded 'git', list args, no shell - [ - "git", - "-C", - str(source.parent), - "log", - "-1", - "--format=%cs", - "--", - source.name, - ], - capture_output=True, - text=True, - timeout=10, - check=False, + dates = [] + pattern = f"*/{harness}/{skill}/{dataset}/run-summary.json" + for path in runs_dir.glob(pattern): + summary = json.loads(path.read_text(encoding="utf-8")) + if summary.get("model_slug") in models and summary.get("run_date"): + dates.append(summary["run_date"]) + if not dates: + logger.warning( + "no run_date in any %s summary for the frontier's models; " + "measured_on will be null", + pattern, ) - if out.stdout.strip(): - return out.stdout.strip() - except (OSError, subprocess.SubprocessError): - pass - return date.fromtimestamp(source.stat().st_mtime).isoformat() + return None + return max(dates) def _relative_or_name(source: Path, repo_root: Path) -> str: @@ -243,7 +227,8 @@ def build( Args: source: Path to a ``pareto-frontier-*.json``. - repo_root: Repository root, used for the provenance commit. + repo_root: Repository root, used to record the source path. + runs_dir: The swe-benchmark-data root; defaults to the committed one. Returns: The payload to write as models.json. @@ -264,11 +249,19 @@ def build( bedrock = {m["model"] for m in data.get("bedrock_frontier", [])} self_hosted = {m["model"] for m in data.get("self_hosted_frontier", [])} + runs_dir = runs_dir if runs_dir is not None else DEFAULT_RUNS_DIR by_tier = per_tier_stats( - runs_dir if runs_dir is not None else DEFAULT_RUNS_DIR, + runs_dir, + data.get("harness", ""), + data.get("skill", ""), + data.get("repo", ""), + ) + measured_on = _latest_run_date( + runs_dir, data.get("harness", ""), data.get("skill", ""), data.get("repo", ""), + {m["model"] for m in models}, ) out_models = [] for m in sorted(models, key=lambda x: -x["mean_score"]): @@ -305,9 +298,9 @@ def build( "schema_version": SCHEMA_VERSION, "generated_by": "benchmarks/scripts/build_vended_models.py", "provenance": { - "measured_on": _measured_on(source), + "measured_on": measured_on, "source_file": _relative_or_name(source, repo_root), - "source_commit": _git_commit(source), + "source_sha256": _source_sha256(source), "harness": data.get("harness"), "skill": data.get("skill"), "dataset": data.get("repo"), diff --git a/benchmarks/tests/test_build_vended_models.py b/benchmarks/tests/test_build_vended_models.py index 69772450..05504792 100644 --- a/benchmarks/tests/test_build_vended_models.py +++ b/benchmarks/tests/test_build_vended_models.py @@ -2,9 +2,9 @@ from __future__ import annotations +import hashlib import importlib.util import json -import subprocess import sys import tempfile import unittest @@ -67,6 +67,19 @@ def _write(data: dict) -> Path: return Path(tmp.name) +def _runs(dates: dict[str, str | None]) -> Path: + """Write one omp/swe3/v2 run summary per model, with the given run_date.""" + root = Path(tempfile.mkdtemp()) + for model, run_date in dates.items(): + folder = root / model / "omp" / "swe3" / "mcp-gateway-registry-v2" + folder.mkdir(parents=True) + summary: dict = {"model_slug": model, "tasks": []} + if run_date: + summary["run_date"] = run_date + (folder / "run-summary.json").write_text(json.dumps(summary), "utf-8") + return root + + class BuildTest(unittest.TestCase): def test_includes_every_model_not_only_the_frontier(self) -> None: # The skill filters to what the user's assistant offers before ranking, @@ -119,7 +132,8 @@ def test_incomplete_runs_stay_visible(self) -> None: def test_provenance_carries_the_measurement_context(self) -> None: # A vended file has no reader who knows this repo, so the context # travels with the data. - out = bvm.build(_write(_frontier()), _REPO_ROOT) + runs = _runs({"big": "2026-10-01", "small": "2026-09-01"}) + out = bvm.build(_write(_frontier()), _REPO_ROOT, runs_dir=runs) p = out["provenance"] self.assertEqual(p["harness"], "omp") self.assertEqual(p["skill"], "swe3") @@ -141,6 +155,40 @@ def test_empty_source_is_a_clear_error(self) -> None: with self.assertRaisesRegex(SystemExit, "no all_models"): bvm.build(_write(_frontier(all_models=[])), _REPO_ROOT) + def test_provenance_ignores_where_the_file_lives(self) -> None: + # The squash-merge case: the same bytes must stamp the same provenance, + # whatever commit or path they arrived under. + runs = _runs({"big": "2026-10-01"}) + first = bvm.build(_write(_frontier()), _REPO_ROOT, runs_dir=runs) + second = bvm.build(_write(_frontier()), _REPO_ROOT, runs_dir=runs) + keys = ("measured_on", "source_sha256") + self.assertEqual( + [first["provenance"][k] for k in keys], + [second["provenance"][k] for k in keys], + ) + + def test_source_sha256_changes_with_the_data(self) -> None: + runs = _runs({}) + before = bvm.build(_write(_frontier()), _REPO_ROOT, runs_dir=runs) + after = bvm.build(_write(_frontier(note="new")), _REPO_ROOT, runs_dir=runs) + self.assertNotEqual( + before["provenance"]["source_sha256"], + after["provenance"]["source_sha256"], + ) + + def test_measured_on_is_the_latest_run_of_a_frontier_model(self) -> None: + # "other" is not in the frontier JSON, so its later date is ignored. + runs = _runs( + {"big": "2026-10-01", "small": "2026-09-15", "other": "2026-12-31"} + ) + out = bvm.build(_write(_frontier()), _REPO_ROOT, runs_dir=runs) + self.assertEqual(out["provenance"]["measured_on"], "2026-10-01") + + def test_measured_on_is_null_without_any_run_date(self) -> None: + runs = _runs({"big": None, "small": None}) + out = bvm.build(_write(_frontier()), _REPO_ROOT, runs_dir=runs) + self.assertIsNone(out["provenance"]["measured_on"]) + class CommittedArtifactTest(unittest.TestCase): """The vended files are committed, so they are checked like any other input.""" @@ -194,35 +242,11 @@ def test_tier_scores_bracket_the_overall_mean(self) -> None: self.assertLessEqual(min(tiers), m["score"] + 0.01, m["model"]) self.assertGreaterEqual(max(tiers), m["score"] - 0.01, m["model"]) - def test_source_commit_tracks_the_frontier_not_head(self) -> None: - # Stamping HEAD would make the file differ after every unrelated commit - # and turn --check into noise. It must identify the data's version. + def test_source_sha256_matches_the_committed_frontier_bytes(self) -> None: + # What `sha256sum | cut -c1-12` prints, so anyone can check it. payload = bvm.build(bvm.DEFAULT_SOURCE, _REPO_ROOT) - stamped = payload["provenance"]["source_commit"] - head = subprocess.run( - ["git", "-C", str(_REPO_ROOT), "rev-parse", "--short", "HEAD"], - capture_output=True, - text=True, - check=False, - ).stdout.strip() - last_touch = subprocess.run( - [ - "git", - "-C", - str(_REPO_ROOT), - "log", - "-1", - "--format=%h", - "--", - str(bvm.DEFAULT_SOURCE.relative_to(_REPO_ROOT)), - ], - capture_output=True, - text=True, - check=False, - ).stdout.strip() - self.assertEqual(stamped, last_touch) - if head != last_touch: - self.assertNotEqual(stamped, head) + digest = hashlib.sha256(bvm.DEFAULT_SOURCE.read_bytes()).hexdigest()[:12] + self.assertEqual(payload["provenance"]["source_sha256"], digest) def test_vend_dir_holds_exactly_the_portable_files(self) -> None: # The portability contract: the skill must work in a directory holding diff --git a/vend/swe-router/README.md b/vend/swe-router/README.md index 901a01c4..eec1ab26 100644 --- a/vend/swe-router/README.md +++ b/vend/swe-router/README.md @@ -82,7 +82,7 @@ If it is firing when you do not want it, narrow the `Run it BEFORE…` sentence **Self-hosted costs are measured under load.** Hosted figures come from a metered bill. Self-hosted figures come from the server's hourly price divided by throughput measured at a stated concurrency — a platform team serving a group of developers, not one person with an idle GPU. Both are a cost per task, and the skill ranks them together. -**These numbers move.** A fix to token accounting once changed `claude-opus-5` from $7.63 to $11.95 per task while every score stayed identical. Check `provenance.measured_on` and refetch if it looks old. +**These numbers move.** A fix to token accounting once changed `claude-opus-5` from $7.63 to $11.95 per task while every score stayed identical. Check `provenance.measured_on`, the date of the newest run in the file, and refetch if it looks old. `provenance.source_sha256` identifies the exact data: it is the first 12 hex digits of the SHA-256 of the frontier file it was built from, so two copies with the same value hold the same numbers. ## How it decides diff --git a/vend/swe-router/models.json b/vend/swe-router/models.json index 9c85849c..e91df795 100644 --- a/vend/swe-router/models.json +++ b/vend/swe-router/models.json @@ -1,10 +1,10 @@ { - "schema_version": "1.0", + "schema_version": "1.1", "generated_by": "benchmarks/scripts/build_vended_models.py", "provenance": { - "measured_on": "2026-10-10", + "measured_on": "2026-10-11", "source_file": "docs/metrics/pareto-frontier-omp-swe3.json", - "source_commit": "1c86108", + "source_sha256": "72b1ed9ee222", "harness": "omp", "skill": "swe3", "dataset": "mcp-gateway-registry-v2",