Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -550,6 +550,7 @@ Read the doc that covers what you are about to do rather than rediscovering it.
- [docs/best-harness-selection.md](docs/best-harness-selection.md) -- how the combined cost/quality chart picks one harness per model.
- [docs/results-swe3.md](docs/results-swe3.md), [docs/results-swe2.md](docs/results-swe2.md) -- published results and their footnotes.
- [docs/results-swe3-v2.md](docs/results-swe3-v2.md) -- results on the v2 dataset (15 release-sourced tasks, complexity-balanced). **Never merge v2 scores into a v1 table**; the task sets, refs and difficulty mixes differ.
- [docs/results-cli-cli-v1.md](docs/results-cli-cli-v1.md) -- the cross-repo check (#221): Claude 5.5 models on 21 cli/cli (Go) tasks, and whether v2's rankings, score levels and swe-router columns carry over. Keep its tables apart from v1 and v2.
- [docs/model-selection-by-complexity.md](docs/model-selection-by-complexity.md) -- which model to use for which class of work, derived from the v2 complexity tiers.

**Questions rather than reference pages:**
Expand Down
94 changes: 90 additions & 4 deletions benchmarks/scripts/eval_swe_router.py
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,7 @@ def _evaluate(
judged_tiers: dict[str, str] | None = None,
column_for: Callable[[str], dict[str, str]] | None = None,
stats_for: Callable[[str], dict[str, dict[str, Any]]] | None = None,
fixed_stats: dict[str, dict[str, Any]] | None = None,
) -> dict[str, Any]:
"""Replay the router over every task and compare it to the baseline.

Expand All @@ -480,6 +481,9 @@ def _evaluate(
rubric buckets do. Overrides ``judged_tiers`` and the dataset label.
stats_for: Given the task being routed, every model's stats with one
predicted score under ``PREDICTED_TIER``. Overrides ``column_for``.
fixed_stats: Every model's stats from ANOTHER dataset, routed against
for every task. ``results`` then only supplies each routed task's
outcome. ``column_for`` still names the routed task's column.

Returns:
A result mapping with ``rows`` (one per task) and ``totals``.
Expand All @@ -497,7 +501,8 @@ def _evaluate(
tmp_dir = Path(tempfile.mkdtemp(prefix="router-eval-"))
in_sample_path = tmp_dir / "models-in-sample.json"
in_sample_path.write_text(
json.dumps(_models_json(_tier_stats(results), provenance)), encoding="utf-8"
json.dumps(_models_json(fixed_stats or _tier_stats(results), provenance)),
encoding="utf-8",
)
for task in tasks:
task_id = task["id"]
Expand All @@ -511,7 +516,9 @@ def _evaluate(
columns = column_for(task_id)
tier = columns[task_id]
grouped = _regrouped(results, columns)
if stats_for is not None:
if fixed_stats is not None:
models_path = in_sample_path
elif stats_for is not None:
tier = PREDICTED_TIER
models_path = tmp_dir / f"models-{task_id}.json"
models_path.write_text(
Expand Down Expand Up @@ -895,7 +902,10 @@ def _document(report: dict[str, Any]) -> str:
cfg = report["config"]
prov = report["provenance"]
sampling = (
"Leave-one-out: each task routes from tier means recomputed with that "
f"Zero-shot: every task routes from columns built on `{cfg['columns_from']}` "
"alone, a different repository, so no pick saw any task of this one."
if cfg.get("columns_from")
else "Leave-one-out: each task routes from tier means recomputed with that "
"task excluded, so no pick knows the run it is scored against."
if cfg["holdout"]
else "In-sample: the router read tier means computed from these same "
Expand Down Expand Up @@ -1030,6 +1040,44 @@ def column_for(routed: str) -> dict[str, str]:
return floors, column_for, meta


def _columns_from_other_dataset(
dataset_path: Path,
rubric_inputs: Path,
harness: str,
skill: str,
) -> tuple[dict[str, dict[str, Any]], str]:
"""Every model's bucket columns, built from a different dataset.

This is the zero-shot case: the router has only another repository's
benchmark to go on. Each of that dataset's tasks is filed under the bucket
its own rubric pass gave it, and the columns are its models' means there.
Buckets are absolute, so the routed tasks are placed by the same fixed
ranges.

Args:
dataset_path: The dataset whose results build the columns.
rubric_inputs: That dataset's rubric pass.
harness: Harness folder.
skill: Skill folder.

Returns:
``_tier_stats`` output for that dataset, and its scope.

Raises:
SystemExit: If the dataset has no results or the rubric pass misses a task.
"""
tasks, scope = _load_tasks(dataset_path)
results = _load_results(harness, skill, scope)
if not results:
raise SystemExit(
f"no results for --columns-from under {harness}/{skill}/{scope}"
)
_, column_for, _ = _apply_rubric_inputs(rubric_inputs, tasks, "absolute", False)
known = {t["id"] for t in tasks}
columns = {tid: c for tid, c in column_for("").items() if tid in known}
return _tier_stats(_regrouped(results, columns)), scope


def _parse_args() -> argparse.Namespace:
"""Parse command-line arguments."""
parser = argparse.ArgumentParser(
Expand Down Expand Up @@ -1097,6 +1145,20 @@ def _parse_args() -> argparse.Namespace:
"column means, or against each model's score predicted from the task's "
"difficulty sum by a line fitted over the other tasks. Default: %(default)s.",
)
parser.add_argument(
"--columns-from",
type=Path,
default=None,
help="Zero-shot replay: build the bucket columns from THIS other dataset's "
"results instead of --dataset's, and route --dataset's tasks against them. "
"Needs --rubric-inputs and --columns-rubric-inputs; absolute buckets only.",
)
parser.add_argument(
"--columns-rubric-inputs",
type=Path,
default=None,
help="The rubric pass for --columns-from's dataset.",
)
parser.add_argument(
"--floors-file",
type=Path,
Expand Down Expand Up @@ -1193,6 +1255,24 @@ def main() -> None:

stats_for = partial(_predicted_stats, results, sums, holdout=args.holdout)
judged_meta["column_model"] = "regression"
fixed_stats = None
if args.columns_from:
if not (args.rubric_inputs and args.columns_rubric_inputs):
raise SystemExit(
"--columns-from needs --rubric-inputs and --columns-rubric-inputs"
)
if bucketing != "absolute" or args.column_model != "buckets" or args.holdout:
raise SystemExit(
"--columns-from supports absolute buckets only, without --holdout "
"(the routed tasks are never in the columns)"
)
columns_path = args.columns_from
if not columns_path.is_absolute():
columns_path = BENCHMARKS_DIR / columns_path
fixed_stats, columns_scope = _columns_from_other_dataset(
columns_path, args.columns_rubric_inputs, args.harness, args.skill
)
judged_meta["columns_from"] = columns_scope
if args.judged_inputs:
judged = json.loads(args.judged_inputs.read_text(encoding="utf-8"))
judged_meta = judged.get("judged_by") or {}
Expand Down Expand Up @@ -1236,6 +1316,7 @@ def main() -> None:
judged_tiers=judged_tiers,
column_for=column_for,
stats_for=stats_for,
fixed_stats=fixed_stats,
)
except RouteError as exc:
raise SystemExit(f"routing failed at floor {floor}: {exc}") from exc
Expand All @@ -1245,7 +1326,11 @@ def main() -> None:
"mean score %.2f vs %.2f (%+.2f; %+.2f with failures as 0), "
"%d task(s) under floor",
floor,
"leave-one-out" if args.holdout else "in-sample",
"zero-shot"
if fixed_stats
else "leave-one-out"
if args.holdout
else "in-sample",
totals["routed_total_cost_usd"],
totals["baseline_total_cost_usd"],
totals["cost_saving_pct"],
Expand All @@ -1270,6 +1355,7 @@ def main() -> None:
"no_allow_list": args.no_allow_list,
"tie_band": args.tie_band,
"holdout": args.holdout,
"columns_from": judged_meta.get("columns_from"),
"judged_by": judged_meta,
"floor_note": (
"Required quality per task, from the rubric pass by "
Expand Down
32 changes: 25 additions & 7 deletions benchmarks/scripts/run-swe-router-headless.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,10 +100,10 @@
MODE_FLOOR = "floor"
MODE_RUBRIC = "rubric"
MODES = (MODE_FLOOR, MODE_RUBRIC)
DEFAULT_OUT = {
MODE_FLOOR: REPO_ROOT / "docs" / "metrics" / "swe-router-judged-inputs.json",
MODE_RUBRIC: REPO_ROOT / "docs" / "metrics" / "swe-router-rubric-inputs.json",
}
DEFAULT_FLOOR_OUT = REPO_ROOT / "docs" / "metrics" / "swe-router-judged-inputs.json"
# A rubric pass describes one dataset's tasks, so its default file name carries
# the dataset scope and the judging model: one file per dataset and judge.
RUBRIC_OUT_TEMPLATE = "swe-router-rubric-{scope}-{model}.json"

DEFAULT_DATASET = "dataset/mcp-gateway-registry-v2.yaml"
DEFAULT_TIMEOUT_SECONDS = 900
Expand Down Expand Up @@ -150,6 +150,24 @@ def _utc_now_iso() -> str:
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")


def _default_out(
mode: str,
dataset: Dataset,
dataset_path: Path,
config: RunnerConfig,
) -> Path:
"""Where to write the judged inputs when --out is not given.

A rubric pass is named after its dataset and judge, so passes over two
datasets, or by two judges, never overwrite each other.
"""
if mode != MODE_RUBRIC:
return DEFAULT_FLOOR_OUT
scope = dataset.output_scope or dataset_path.stem
name = RUBRIC_OUT_TEMPLATE.format(scope=scope, model=config.model_slug)
return REPO_ROOT / "docs" / "metrics" / name


def _build_prompt(task: Task, clone_path: Path) -> str:
"""Build the step-1 prompt: the skill, the task, and the output contract.

Expand Down Expand Up @@ -907,8 +925,8 @@ def _parse_args() -> argparse.Namespace:
type=Path,
default=None,
help="Where to write the judged inputs. Default: "
"docs/metrics/swe-router-judged-inputs.json, or "
"docs/metrics/swe-router-rubric-inputs.json in rubric mode.",
"docs/metrics/swe-router-judged-inputs.json, or in rubric mode "
"docs/metrics/swe-router-rubric-<dataset scope>-<model slug>.json.",
)
parser.add_argument(
"--out-md",
Expand Down Expand Up @@ -972,7 +990,7 @@ def main() -> None:
except DatasetError as exc:
raise SystemExit(f"dataset error: {exc}") from exc
tasks = _select_tasks(dataset, args.tasks.split(",") if args.tasks else [])
args.out = args.out or DEFAULT_OUT[args.mode]
args.out = args.out or _default_out(args.mode, dataset, dataset_path, config)
rubric_text, policy = "", None
if args.mode == MODE_RUBRIC:
try:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
{
"task": "auth-clipboard-config-default",
"model": "us.anthropic.claude-haiku-5-5",
"scores": {
"github_issue": {
"completeness": 0,
"correctness": 0,
"specificity": 0,
"risk_awareness": 0,
"total": 0,
"notes": "Artifact not produced by the candidate run."
},
"lld": {
"completeness": 0,
"correctness": 0,
"specificity": 0,
"risk_awareness": 0,
"total": 0,
"notes": "Artifact not produced by the candidate run."
},
"review": {
"completeness": 0,
"correctness": 0,
"specificity": 0,
"risk_awareness": 0,
"total": 0,
"notes": "Artifact not produced by the candidate run."
},
"testing": {
"completeness": 0,
"correctness": 0,
"specificity": 0,
"risk_awareness": 0,
"total": 0,
"notes": "Artifact not produced by the candidate run."
},
"implementation": {
"completeness": 0,
"correctness": 0,
"specificity": 0,
"risk_awareness": 0,
"total": 0,
"notes": "Artifact not produced by the candidate run."
}
},
"task_score": 0.0,
"verdict": "MODEL FAILURE: the candidate run did not produce the required artifact(s): github-issue.md, lld.md, review.md, testing.md. Scored 0.",
"judge": {
"model": "openai.gpt-5.6-sol",
"provider": "codex-exec",
"repo_grounded": false,
"scored_zero_missing_artifacts": [
"github-issue.md",
"lld.md",
"review.md",
"testing.md"
],
"evaluated_at": "2026-10-14T13:47:44.669482Z"
}
}
Loading
Loading