Skip to content

Commit c8be585

Browse files
authored
feat(crosscheck): add R6 prompt addendum and status (#287)
1 parent 979f07e commit c8be585

4 files changed

Lines changed: 380 additions & 22 deletions

File tree

bin/fm-crosscheck.py

Lines changed: 138 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3291,6 +3291,32 @@ def reviewer_candidates(
32913291
"""
32923292

32933293
allow_same_model = same_model_review_enabled(home)
3294+
validated = reviewer_roster(home)
3295+
# Independence is compared on the model FAMILY, not the exact id: a
3296+
# `gpt-5.5` author admitting a `gpt-5.6-sol` reviewer is the same-family
3297+
# review this requirement exists to prevent (cc-4dcd7873f71a). The durable
3298+
# ledger marker keeps its `same-model` spelling, which older records
3299+
# already carry; it now means "shares the author's model family".
3300+
author_family = model_family(meta["model"])
3301+
eligible: list[dict[str, str]] = []
3302+
for roster_entry in validated:
3303+
reviewer = dict(roster_entry)
3304+
model_is_separate = model_family(reviewer["model"]) != author_family
3305+
if model_is_separate or allow_same_model:
3306+
if not model_is_separate:
3307+
reviewer["model_independence"] = "same-model"
3308+
eligible.append(reviewer)
3309+
if eligible:
3310+
return eligible
3311+
fail(
3312+
"reviewer model policy found no configured reviewer outside the model "
3313+
f"family of {meta['model']!r}"
3314+
)
3315+
3316+
3317+
def reviewer_roster(home: Path) -> list[dict[str, str]]:
3318+
"""Return the validated reviewer roster in configured serving order."""
3319+
32943320
config_path = Path(
32953321
environment_value(
32963322
"FM_CROSSCHECK_REVIEWER_CONFIG",
@@ -3362,25 +3388,7 @@ def reviewer_candidates(
33623388
),
33633389
}
33643390
)
3365-
# Independence is compared on the model FAMILY, not the exact id: a
3366-
# `gpt-5.5` author admitting a `gpt-5.6-sol` reviewer is the same-family
3367-
# review this requirement exists to prevent (cc-4dcd7873f71a). The durable
3368-
# ledger marker keeps its `same-model` spelling, which older records
3369-
# already carry; it now means "shares the author's model family".
3370-
author_family = model_family(meta["model"])
3371-
eligible: list[dict[str, str]] = []
3372-
for reviewer in validated:
3373-
model_is_separate = model_family(reviewer["model"]) != author_family
3374-
if model_is_separate or allow_same_model:
3375-
if not model_is_separate:
3376-
reviewer["model_independence"] = "same-model"
3377-
eligible.append(reviewer)
3378-
if eligible:
3379-
return eligible
3380-
fail(
3381-
"reviewer model policy found no configured reviewer outside the model "
3382-
f"family of {meta['model']!r}"
3383-
)
3391+
return validated
33843392

33853393

33863394
def review_output_schema(
@@ -3594,7 +3602,7 @@ def make_prompt(
35943602
You are using the same model as the author and may share the author's blind spots and priors.
35953603
Compensate explicitly: attack the change adversarially, try to falsify the author's claims rather than confirm them, and default to reporting a finding when uncertain.
35963604
"""
3597-
return f"""You are the independent merge-gate reviewer for a pull request.
3605+
prompt = f"""You are the independent merge-gate reviewer for a pull request.
35983606
{same_model_warning}Review exact head {snapshot_value['head_sha']} against exact base {snapshot_value['base_sha']}.
35993607
Perform a rigorous release-readiness review of the full diff and the PR's own claims.
36003608
Do not trust the PR description or a previous clean run.
@@ -3645,6 +3653,20 @@ def make_prompt(
36453653
Bounded durable-finding lifecycle metadata and proof digests:
36463654
{json.dumps(projection, indent=2, sort_keys=True)}
36473655
"""
3656+
# The shared prompt is intentionally byte-identical for every Codex-family
3657+
# reviewer. Cross-family models receive only this appended clarification:
3658+
# the exact-SHA verdict check below remains the authority and is not
3659+
# weakened to accommodate a reviewer that omitted its required literals.
3660+
if model_family(config["model"]) != "openai":
3661+
prompt += f"""
3662+
REPRODUCTION COMMAND FORMAT - EXACT REQUIREMENT:
3663+
The literal string you place in `executed_reproduction.command` MUST contain, verbatim, both
3664+
full 40-character SHAs: exact base {snapshot_value['base_sha']} and exact head {snapshot_value['head_sha']}.
3665+
Example: bash .crosscheck/reproductions/repro.sh {snapshot_value['base_sha']} {snapshot_value['head_sha']}
3666+
A command that omits either SHA, abbreviates it, or references it through a shell variable is
3667+
refused and the entire review is discarded as UNREVIEWED.
3668+
"""
3669+
return prompt
36483670

36493671

36503672
def reviewer_timeout() -> int:
@@ -5298,6 +5320,94 @@ def timings_crosscheck(home: Path, task_id: str) -> int:
52985320
return 0
52995321

53005322

5323+
def latest_review_family(
5324+
data: Path,
5325+
) -> tuple[str, str, str] | None:
5326+
"""Return the latest recorded run's time, task id, and review family."""
5327+
5328+
if not data.exists() and not data.is_symlink():
5329+
return None
5330+
require(
5331+
data.is_dir() and not data.is_symlink(),
5332+
f"crosscheck data root is not a directory: {data}",
5333+
)
5334+
try:
5335+
task_dirs = sorted(data.iterdir(), key=lambda path: path.name)
5336+
except OSError as exc:
5337+
fail(f"crosscheck data root inspection failed at {data}: {exc}")
5338+
latest: tuple[str, str, str] | None = None
5339+
for task_dir in task_dirs:
5340+
try:
5341+
is_task_dir = task_dir.is_dir() and not task_dir.is_symlink()
5342+
except OSError as exc:
5343+
fail(f"crosscheck task data inspection failed at {task_dir}: {exc}")
5344+
if not is_task_dir:
5345+
continue
5346+
ledger_path = task_dir / "crosscheck-ledger.json"
5347+
if not ledger_path.exists() and not ledger_path.is_symlink():
5348+
continue
5349+
raw = read_json(
5350+
ledger_path,
5351+
"findings ledger",
5352+
maximum_bytes=MAX_LEDGER_BYTES,
5353+
maximum_items=262_144,
5354+
)
5355+
require(isinstance(raw, dict), "existing findings ledger must be an object")
5356+
task_id = require_string(raw.get("task_id"), "ledger.task_id")
5357+
require(
5358+
ID_RE.fullmatch(task_id) is not None,
5359+
f"ledger task_id is invalid at {ledger_path}",
5360+
)
5361+
url = require_string(raw.get("pull_request"), "ledger.pull_request")
5362+
ledger = validate_ledger(raw, task_id, url)
5363+
if not ledger["runs"]:
5364+
continue
5365+
run = ledger["runs"][-1]
5366+
reviewer = run.get("reviewer")
5367+
family = (
5368+
reviewer.get("review_family_mode") or "none"
5369+
if isinstance(reviewer, dict)
5370+
else "none"
5371+
)
5372+
candidate = (run["at"], task_id, family)
5373+
if latest is None or candidate[:2] > latest[:2]:
5374+
latest = candidate
5375+
return latest
5376+
5377+
5378+
def status_crosscheck(home: Path) -> int:
5379+
"""Print the configured serving family and latest durable run family.
5380+
5381+
This is a read-only operator view. It deliberately takes no task lock and
5382+
creates no state, so checking whether the primary lane or fallback is at
5383+
the front of the roster cannot interfere with an in-flight review.
5384+
"""
5385+
5386+
try:
5387+
roster = reviewer_roster(home)
5388+
relaxation = "on" if same_model_review_enabled(home) else "off"
5389+
data = Path(environment_value("FM_DATA_OVERRIDE", str(home / "data")))
5390+
latest = latest_review_family(data)
5391+
except CrosscheckError as exc:
5392+
tool_fail(f"status preflight failed: {exc}")
5393+
serving = roster[0]
5394+
if cross_family_lane_for_model(serving["model"]) is not None:
5395+
lane = "cross-family serving"
5396+
else:
5397+
lane = "codex fallback active"
5398+
print(
5399+
f"crosscheck lane: {lane} "
5400+
f"({serving['harness']} {serving['model']}, roster entry 1)"
5401+
)
5402+
print(f"crosscheck same-model relaxation: {relaxation}")
5403+
if latest is None:
5404+
print("crosscheck last review family: none")
5405+
else:
5406+
at, task_id, family = latest
5407+
print(f"crosscheck last review family: {family} ({task_id} at {at})")
5408+
return 0
5409+
5410+
53015411
def load_azure_crosscheck_adapter(root: Path) -> Any:
53025412
"""Load the dedicated Azure review/ledger adapter without weakening local review."""
53035413

@@ -5379,6 +5489,7 @@ def build_parser() -> argparse.ArgumentParser:
53795489
command.add_argument("pr_url")
53805490
timings = subparsers.add_parser("timings")
53815491
timings.add_argument("task_id")
5492+
subparsers.add_parser("status")
53825493
merge = subparsers.add_parser("merge")
53835494
merge.add_argument("task_id")
53845495
merge.add_argument("pr_url")
@@ -5414,9 +5525,10 @@ def assert_supported_interpreter() -> None:
54145525

54155526
def main() -> int:
54165527
args = build_parser().parse_args()
5417-
if ID_RE.fullmatch(args.task_id) is None:
5528+
task_id = getattr(args, "task_id", None)
5529+
if task_id is not None and ID_RE.fullmatch(task_id) is None:
54185530
print(
5419-
f"CROSSCHECK TOOL-FAILURE: task id validation rejected {args.task_id!r}",
5531+
f"CROSSCHECK TOOL-FAILURE: task id validation rejected {task_id!r}",
54205532
file=sys.stderr,
54215533
)
54225534
return 1
@@ -5433,6 +5545,10 @@ def main() -> int:
54335545
home = Path(environment_value("FM_HOME", str(root))).resolve()
54345546
state = Path(environment_value("FM_STATE_OVERRIDE", str(home / "state")))
54355547
try:
5548+
if args.command == "status":
5549+
# Read-only, so it takes no run lock and creates no state: asking
5550+
# which review family is serving must never interfere with a run.
5551+
return status_crosscheck(home)
54365552
if args.command == "timings":
54375553
# Read-only, so it takes no run lock and creates no state: asking
54385554
# where the time went must never block or be blocked by a review.

bin/fm-crosscheck.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
# Usage:
55
# fm-crosscheck.sh run <task-id> <full GitHub PR URL>
66
# fm-crosscheck.sh verify <task-id> <full GitHub PR URL>
7+
# fm-crosscheck.sh status
78
# fm-crosscheck.sh timings <task-id>
89
# fm-crosscheck.sh merge <task-id> <full GitHub PR URL> <reviewed SHA> <method> [--allow-queue]
910
#
@@ -13,6 +14,8 @@
1314
# and claims document to be clear, and prints only the reviewed SHA.
1415
# `timings` is the read-only C1 breakdown: it prints the per-phase duration
1516
# table every recorded run carries, takes no lock, and changes nothing.
17+
# `status` is the read-only R6 family view: it prints the serving roster family,
18+
# same-model relaxation, and latest durable review family without taking a lock.
1619
# `merge` repeats that verification and is the sole entrypoint to the private
1720
# exact-SHA GitHub merge or merge-queue primitive.
1821
#

docs/crosscheck.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,12 @@ The accepted profiles are Pi at xhigh on every registered cross-family model (to
8484
Reviewer independence is compared on the model FAMILY, not the exact id, so a `gpt-5.5` author is not admitted a `gpt-5.6-sol` reviewer (finding cc-4dcd7873f71a); an unrecognized model remains its own family.
8585
Absent reviewer configuration, unavailable reviewer credentials, or model-policy mismatch produces `CROSSCHECK TOOL-FAILURE` and a nonzero exit before reviewer launch.
8686

87+
The lock-free status read reports whether the roster's first serving family is the cross-family primary or the Codex fallback, the current `crosscheck-same-model` setting, and the latest durable run's `review_family_mode` when a run exists.
88+
89+
```sh
90+
bin/fm-crosscheck.sh status
91+
```
92+
8793
Crosscheck requires Python 3.11 or newer and refuses to run on anything older.
8894
This is a safety floor rather than a style preference: the bounded-read layer rejects hostile JSON integers by relying on CPython's integer/string conversion limit, which first exists in 3.11, and on an older interpreter that rejection silently stops happening while every banner the gate prints reads exactly the same.
8995
Stock macOS `python3` is 3.9, so `bin/fm-crosscheck.sh` resolves a supported sibling interpreter instead of assuming `python3` qualifies, and `bin/fm-crosscheck.py` enforces the same minimum itself so a direct invocation cannot bypass it.

0 commit comments

Comments
 (0)