Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 57 additions & 64 deletions benchmarks/scripts/build_vended_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,7 @@
import argparse
import json
import logging
import subprocess # nosec B404 - used with list args, no shell, hardcoded 'git'
from datetime import date
import hashlib
from pathlib import Path
from typing import Any

Expand All @@ -53,8 +52,12 @@
DEFAULT_OUT = _REPO_ROOT / "vend" / "swe-router" / "models.json"

# Bumped when the shape of models.json changes in a way a consumer would notice.
# Consumers pin this; the skill refuses a major it does not know.
SCHEMA_VERSION = "1.0"
# Consumers pin this; the skill refuses a major it does not know. 1.1 replaced
# provenance.source_commit (a git sha) with source_sha256 (a content hash) and
# took measured_on from the run summaries instead of a commit date.
SCHEMA_VERSION = "1.1"
# provenance.source_sha256 length: enough to tell versions apart, short to read.
SOURCE_HASH_CHARS = 12

# The judge is not recorded in the frontier JSON, so it is stated here. Keep in
# step with codex_judge.py's default (JUDGE_MODEL overrides it per run).
Expand All @@ -77,78 +80,59 @@
}


def _git_commit(source: Path) -> str | None:
"""Return the short sha of the commit that last changed ``source``.
def _source_sha256(source: Path) -> str:
"""Return the first 12 hex digits of the SHA-256 of ``source``'s bytes.

Deliberately not HEAD. HEAD moves with every commit to the repository, so
stamping it would make the generated file differ after any unrelated change
and turn the --check guard into noise. The commit that last touched the
source is what actually identifies this version of the data.
Identifies the exact version of the data from the file itself. An earlier
version stamped the commit that last touched the file, which a squash merge
replaces: the stamp made on the branch then no longer matched ``main``, and
the committed-artifact check failed with every score still correct (after
#184, #210 and #219). A content hash is the same on the branch and after the
merge, and anyone can check it with ``sha256sum <file> | cut -c1-12``.

Args:
source: The frontier JSON.

Returns:
The abbreviated commit, or None outside a git checkout or for a file
that has never been committed.
A 12-character lowercase hex string.
"""
try:
out = subprocess.run( # nosec B603 B607 - hardcoded 'git', list args, no shell
[
"git",
"-C",
str(source.parent),
"log",
"-1",
"--format=%h",
"--",
source.name,
],
capture_output=True,
text=True,
timeout=10,
check=False,
)
except (OSError, subprocess.SubprocessError):
return None
return out.stdout.strip() or None
return hashlib.sha256(source.read_bytes()).hexdigest()[:SOURCE_HASH_CHARS]


def _measured_on(source: Path) -> str:
"""Return the date the source frontier was last modified, ISO format.
def _latest_run_date(
runs_dir: Path, harness: str, skill: str, dataset: str, models: set[str]
) -> str | None:
"""Return the latest ``run_date`` among the run summaries behind the frontier.

Uses the file's last git commit date when available, so regenerating without
changing the data does not advance the measurement date. Falls back to the
filesystem mtime.
That is when the newest measurement in the file was taken, which is what a
consumer means by "measured on". It comes from committed data, so it does
not move when a commit is rewritten, and it does not depend on anyone's
timezone.

Args:
source: The frontier JSON.
runs_dir: The swe-benchmark-data root.
harness: Harness folder, e.g. "omp".
skill: Skill folder, e.g. "swe3".
dataset: Dataset scope folder.
models: The model slugs the frontier JSON lists; other runs are ignored.

Returns:
An ISO date string.
An ISO date, or None when no matching summary records a ``run_date``.
"""
try:
out = subprocess.run( # nosec B603 B607 - hardcoded 'git', list args, no shell
[
"git",
"-C",
str(source.parent),
"log",
"-1",
"--format=%cs",
"--",
source.name,
],
capture_output=True,
text=True,
timeout=10,
check=False,
dates = []
pattern = f"*/{harness}/{skill}/{dataset}/run-summary.json"
for path in runs_dir.glob(pattern):
summary = json.loads(path.read_text(encoding="utf-8"))
if summary.get("model_slug") in models and summary.get("run_date"):
dates.append(summary["run_date"])
if not dates:
logger.warning(
"no run_date in any %s summary for the frontier's models; "
"measured_on will be null",
pattern,
)
if out.stdout.strip():
return out.stdout.strip()
except (OSError, subprocess.SubprocessError):
pass
return date.fromtimestamp(source.stat().st_mtime).isoformat()
return None
return max(dates)


def _relative_or_name(source: Path, repo_root: Path) -> str:
Expand Down Expand Up @@ -243,7 +227,8 @@ def build(

Args:
source: Path to a ``pareto-frontier-*.json``.
repo_root: Repository root, used for the provenance commit.
repo_root: Repository root, used to record the source path.
runs_dir: The swe-benchmark-data root; defaults to the committed one.

Returns:
The payload to write as models.json.
Expand All @@ -264,11 +249,19 @@ def build(
bedrock = {m["model"] for m in data.get("bedrock_frontier", [])}
self_hosted = {m["model"] for m in data.get("self_hosted_frontier", [])}

runs_dir = runs_dir if runs_dir is not None else DEFAULT_RUNS_DIR
by_tier = per_tier_stats(
runs_dir if runs_dir is not None else DEFAULT_RUNS_DIR,
runs_dir,
data.get("harness", ""),
data.get("skill", ""),
data.get("repo", ""),
)
measured_on = _latest_run_date(
runs_dir,
data.get("harness", ""),
data.get("skill", ""),
data.get("repo", ""),
{m["model"] for m in models},
)
out_models = []
for m in sorted(models, key=lambda x: -x["mean_score"]):
Expand Down Expand Up @@ -305,9 +298,9 @@ def build(
"schema_version": SCHEMA_VERSION,
"generated_by": "benchmarks/scripts/build_vended_models.py",
"provenance": {
"measured_on": _measured_on(source),
"measured_on": measured_on,
"source_file": _relative_or_name(source, repo_root),
"source_commit": _git_commit(source),
"source_sha256": _source_sha256(source),
"harness": data.get("harness"),
"skill": data.get("skill"),
"dataset": data.get("repo"),
Expand Down
84 changes: 54 additions & 30 deletions benchmarks/tests/test_build_vended_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@

from __future__ import annotations

import hashlib
import importlib.util
import json
import subprocess
import sys
import tempfile
import unittest
Expand Down Expand Up @@ -67,6 +67,19 @@ def _write(data: dict) -> Path:
return Path(tmp.name)


def _runs(dates: dict[str, str | None]) -> Path:
"""Write one omp/swe3/v2 run summary per model, with the given run_date."""
root = Path(tempfile.mkdtemp())
for model, run_date in dates.items():
folder = root / model / "omp" / "swe3" / "mcp-gateway-registry-v2"
folder.mkdir(parents=True)
summary: dict = {"model_slug": model, "tasks": []}
if run_date:
summary["run_date"] = run_date
(folder / "run-summary.json").write_text(json.dumps(summary), "utf-8")
return root


class BuildTest(unittest.TestCase):
def test_includes_every_model_not_only_the_frontier(self) -> None:
# The skill filters to what the user's assistant offers before ranking,
Expand Down Expand Up @@ -119,7 +132,8 @@ def test_incomplete_runs_stay_visible(self) -> None:
def test_provenance_carries_the_measurement_context(self) -> None:
# A vended file has no reader who knows this repo, so the context
# travels with the data.
out = bvm.build(_write(_frontier()), _REPO_ROOT)
runs = _runs({"big": "2026-10-01", "small": "2026-09-01"})
out = bvm.build(_write(_frontier()), _REPO_ROOT, runs_dir=runs)
p = out["provenance"]
self.assertEqual(p["harness"], "omp")
self.assertEqual(p["skill"], "swe3")
Expand All @@ -141,6 +155,40 @@ def test_empty_source_is_a_clear_error(self) -> None:
with self.assertRaisesRegex(SystemExit, "no all_models"):
bvm.build(_write(_frontier(all_models=[])), _REPO_ROOT)

def test_provenance_ignores_where_the_file_lives(self) -> None:
# The squash-merge case: the same bytes must stamp the same provenance,
# whatever commit or path they arrived under.
runs = _runs({"big": "2026-10-01"})
first = bvm.build(_write(_frontier()), _REPO_ROOT, runs_dir=runs)
second = bvm.build(_write(_frontier()), _REPO_ROOT, runs_dir=runs)
keys = ("measured_on", "source_sha256")
self.assertEqual(
[first["provenance"][k] for k in keys],
[second["provenance"][k] for k in keys],
)

def test_source_sha256_changes_with_the_data(self) -> None:
runs = _runs({})
before = bvm.build(_write(_frontier()), _REPO_ROOT, runs_dir=runs)
after = bvm.build(_write(_frontier(note="new")), _REPO_ROOT, runs_dir=runs)
self.assertNotEqual(
before["provenance"]["source_sha256"],
after["provenance"]["source_sha256"],
)

def test_measured_on_is_the_latest_run_of_a_frontier_model(self) -> None:
# "other" is not in the frontier JSON, so its later date is ignored.
runs = _runs(
{"big": "2026-10-01", "small": "2026-09-15", "other": "2026-12-31"}
)
out = bvm.build(_write(_frontier()), _REPO_ROOT, runs_dir=runs)
self.assertEqual(out["provenance"]["measured_on"], "2026-10-01")

def test_measured_on_is_null_without_any_run_date(self) -> None:
runs = _runs({"big": None, "small": None})
out = bvm.build(_write(_frontier()), _REPO_ROOT, runs_dir=runs)
self.assertIsNone(out["provenance"]["measured_on"])


class CommittedArtifactTest(unittest.TestCase):
"""The vended files are committed, so they are checked like any other input."""
Expand Down Expand Up @@ -194,35 +242,11 @@ def test_tier_scores_bracket_the_overall_mean(self) -> None:
self.assertLessEqual(min(tiers), m["score"] + 0.01, m["model"])
self.assertGreaterEqual(max(tiers), m["score"] - 0.01, m["model"])

def test_source_commit_tracks_the_frontier_not_head(self) -> None:
# Stamping HEAD would make the file differ after every unrelated commit
# and turn --check into noise. It must identify the data's version.
def test_source_sha256_matches_the_committed_frontier_bytes(self) -> None:
# What `sha256sum <file> | cut -c1-12` prints, so anyone can check it.
payload = bvm.build(bvm.DEFAULT_SOURCE, _REPO_ROOT)
stamped = payload["provenance"]["source_commit"]
head = subprocess.run(
["git", "-C", str(_REPO_ROOT), "rev-parse", "--short", "HEAD"],
capture_output=True,
text=True,
check=False,
).stdout.strip()
last_touch = subprocess.run(
[
"git",
"-C",
str(_REPO_ROOT),
"log",
"-1",
"--format=%h",
"--",
str(bvm.DEFAULT_SOURCE.relative_to(_REPO_ROOT)),
],
capture_output=True,
text=True,
check=False,
).stdout.strip()
self.assertEqual(stamped, last_touch)
if head != last_touch:
self.assertNotEqual(stamped, head)
digest = hashlib.sha256(bvm.DEFAULT_SOURCE.read_bytes()).hexdigest()[:12]
self.assertEqual(payload["provenance"]["source_sha256"], digest)

def test_vend_dir_holds_exactly_the_portable_files(self) -> None:
# The portability contract: the skill must work in a directory holding
Expand Down
2 changes: 1 addition & 1 deletion vend/swe-router/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ If it is firing when you do not want it, narrow the `Run it BEFORE…` sentence

**Self-hosted costs are measured under load.** Hosted figures come from a metered bill. Self-hosted figures come from the server's hourly price divided by throughput measured at a stated concurrency — a platform team serving a group of developers, not one person with an idle GPU. Both are a cost per task, and the skill ranks them together.

**These numbers move.** A fix to token accounting once changed `claude-opus-5` from $7.63 to $11.95 per task while every score stayed identical. Check `provenance.measured_on` and refetch if it looks old.
**These numbers move.** A fix to token accounting once changed `claude-opus-5` from $7.63 to $11.95 per task while every score stayed identical. Check `provenance.measured_on`, the date of the newest run in the file, and refetch if it looks old. `provenance.source_sha256` identifies the exact data: it is the first 12 hex digits of the SHA-256 of the frontier file it was built from, so two copies with the same value hold the same numbers.

## How it decides

Expand Down
6 changes: 3 additions & 3 deletions vend/swe-router/models.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"schema_version": "1.0",
"schema_version": "1.1",
"generated_by": "benchmarks/scripts/build_vended_models.py",
"provenance": {
"measured_on": "2026-10-10",
"measured_on": "2026-10-11",
"source_file": "docs/metrics/pareto-frontier-omp-swe3.json",
"source_commit": "1c86108",
"source_sha256": "72b1ed9ee222",
"harness": "omp",
"skill": "swe3",
"dataset": "mcp-gateway-registry-v2",
Expand Down
Loading