diff --git a/bin/fm-crosscheck.py b/bin/fm-crosscheck.py index 3e74afe64b4..4dd87b2f8ca 100755 --- a/bin/fm-crosscheck.py +++ b/bin/fm-crosscheck.py @@ -3291,6 +3291,32 @@ def reviewer_candidates( """ allow_same_model = same_model_review_enabled(home) + validated = reviewer_roster(home) + # Independence is compared on the model FAMILY, not the exact id: a + # `gpt-5.5` author admitting a `gpt-5.6-sol` reviewer is the same-family + # review this requirement exists to prevent (cc-4dcd7873f71a). The durable + # ledger marker keeps its `same-model` spelling, which older records + # already carry; it now means "shares the author's model family". + author_family = model_family(meta["model"]) + eligible: list[dict[str, str]] = [] + for roster_entry in validated: + reviewer = dict(roster_entry) + model_is_separate = model_family(reviewer["model"]) != author_family + if model_is_separate or allow_same_model: + if not model_is_separate: + reviewer["model_independence"] = "same-model" + eligible.append(reviewer) + if eligible: + return eligible + fail( + "reviewer model policy found no configured reviewer outside the model " + f"family of {meta['model']!r}" + ) + + +def reviewer_roster(home: Path) -> list[dict[str, str]]: + """Return the validated reviewer roster in configured serving order.""" + config_path = Path( environment_value( "FM_CROSSCHECK_REVIEWER_CONFIG", @@ -3362,25 +3388,7 @@ def reviewer_candidates( ), } ) - # Independence is compared on the model FAMILY, not the exact id: a - # `gpt-5.5` author admitting a `gpt-5.6-sol` reviewer is the same-family - # review this requirement exists to prevent (cc-4dcd7873f71a). The durable - # ledger marker keeps its `same-model` spelling, which older records - # already carry; it now means "shares the author's model family". - author_family = model_family(meta["model"]) - eligible: list[dict[str, str]] = [] - for reviewer in validated: - model_is_separate = model_family(reviewer["model"]) != author_family - if model_is_separate or allow_same_model: - if not model_is_separate: - reviewer["model_independence"] = "same-model" - eligible.append(reviewer) - if eligible: - return eligible - fail( - "reviewer model policy found no configured reviewer outside the model " - f"family of {meta['model']!r}" - ) + return validated def review_output_schema( @@ -3594,7 +3602,7 @@ def make_prompt( You are using the same model as the author and may share the author's blind spots and priors. Compensate explicitly: attack the change adversarially, try to falsify the author's claims rather than confirm them, and default to reporting a finding when uncertain. """ - return f"""You are the independent merge-gate reviewer for a pull request. + prompt = f"""You are the independent merge-gate reviewer for a pull request. {same_model_warning}Review exact head {snapshot_value['head_sha']} against exact base {snapshot_value['base_sha']}. Perform a rigorous release-readiness review of the full diff and the PR's own claims. Do not trust the PR description or a previous clean run. @@ -3645,6 +3653,20 @@ def make_prompt( Bounded durable-finding lifecycle metadata and proof digests: {json.dumps(projection, indent=2, sort_keys=True)} """ + # The shared prompt is intentionally byte-identical for every Codex-family + # reviewer. Cross-family models receive only this appended clarification: + # the exact-SHA verdict check below remains the authority and is not + # weakened to accommodate a reviewer that omitted its required literals. + if model_family(config["model"]) != "openai": + prompt += f""" +REPRODUCTION COMMAND FORMAT - EXACT REQUIREMENT: +The literal string you place in `executed_reproduction.command` MUST contain, verbatim, both +full 40-character SHAs: exact base {snapshot_value['base_sha']} and exact head {snapshot_value['head_sha']}. +Example: bash .crosscheck/reproductions/repro.sh {snapshot_value['base_sha']} {snapshot_value['head_sha']} +A command that omits either SHA, abbreviates it, or references it through a shell variable is +refused and the entire review is discarded as UNREVIEWED. +""" + return prompt def reviewer_timeout() -> int: @@ -5298,6 +5320,94 @@ def timings_crosscheck(home: Path, task_id: str) -> int: return 0 +def latest_review_family( + data: Path, +) -> tuple[str, str, str] | None: + """Return the latest recorded run's time, task id, and review family.""" + + if not data.exists() and not data.is_symlink(): + return None + require( + data.is_dir() and not data.is_symlink(), + f"crosscheck data root is not a directory: {data}", + ) + try: + task_dirs = sorted(data.iterdir(), key=lambda path: path.name) + except OSError as exc: + fail(f"crosscheck data root inspection failed at {data}: {exc}") + latest: tuple[str, str, str] | None = None + for task_dir in task_dirs: + try: + is_task_dir = task_dir.is_dir() and not task_dir.is_symlink() + except OSError as exc: + fail(f"crosscheck task data inspection failed at {task_dir}: {exc}") + if not is_task_dir: + continue + ledger_path = task_dir / "crosscheck-ledger.json" + if not ledger_path.exists() and not ledger_path.is_symlink(): + continue + raw = read_json( + ledger_path, + "findings ledger", + maximum_bytes=MAX_LEDGER_BYTES, + maximum_items=262_144, + ) + require(isinstance(raw, dict), "existing findings ledger must be an object") + task_id = require_string(raw.get("task_id"), "ledger.task_id") + require( + ID_RE.fullmatch(task_id) is not None, + f"ledger task_id is invalid at {ledger_path}", + ) + url = require_string(raw.get("pull_request"), "ledger.pull_request") + ledger = validate_ledger(raw, task_id, url) + if not ledger["runs"]: + continue + run = ledger["runs"][-1] + reviewer = run.get("reviewer") + family = ( + reviewer.get("review_family_mode") or "none" + if isinstance(reviewer, dict) + else "none" + ) + candidate = (run["at"], task_id, family) + if latest is None or candidate[:2] > latest[:2]: + latest = candidate + return latest + + +def status_crosscheck(home: Path) -> int: + """Print the configured serving family and latest durable run family. + + This is a read-only operator view. It deliberately takes no task lock and + creates no state, so checking whether the primary lane or fallback is at + the front of the roster cannot interfere with an in-flight review. + """ + + try: + roster = reviewer_roster(home) + relaxation = "on" if same_model_review_enabled(home) else "off" + data = Path(environment_value("FM_DATA_OVERRIDE", str(home / "data"))) + latest = latest_review_family(data) + except CrosscheckError as exc: + tool_fail(f"status preflight failed: {exc}") + serving = roster[0] + if cross_family_lane_for_model(serving["model"]) is not None: + lane = "cross-family serving" + else: + lane = "codex fallback active" + print( + f"crosscheck lane: {lane} " + f"({serving['harness']} {serving['model']}, roster entry 1)" + ) + print(f"crosscheck same-model relaxation: {relaxation}") + if latest is None: + print("crosscheck last review family: none") + else: + at, task_id, family = latest + print(f"crosscheck last review family: {family} ({task_id} at {at})") + return 0 + + def load_azure_crosscheck_adapter(root: Path) -> Any: """Load the dedicated Azure review/ledger adapter without weakening local review.""" @@ -5379,6 +5489,7 @@ def build_parser() -> argparse.ArgumentParser: command.add_argument("pr_url") timings = subparsers.add_parser("timings") timings.add_argument("task_id") + subparsers.add_parser("status") merge = subparsers.add_parser("merge") merge.add_argument("task_id") merge.add_argument("pr_url") @@ -5414,9 +5525,10 @@ def assert_supported_interpreter() -> None: def main() -> int: args = build_parser().parse_args() - if ID_RE.fullmatch(args.task_id) is None: + task_id = getattr(args, "task_id", None) + if task_id is not None and ID_RE.fullmatch(task_id) is None: print( - f"CROSSCHECK TOOL-FAILURE: task id validation rejected {args.task_id!r}", + f"CROSSCHECK TOOL-FAILURE: task id validation rejected {task_id!r}", file=sys.stderr, ) return 1 @@ -5433,6 +5545,10 @@ def main() -> int: home = Path(environment_value("FM_HOME", str(root))).resolve() state = Path(environment_value("FM_STATE_OVERRIDE", str(home / "state"))) try: + if args.command == "status": + # Read-only, so it takes no run lock and creates no state: asking + # which review family is serving must never interfere with a run. + return status_crosscheck(home) if args.command == "timings": # Read-only, so it takes no run lock and creates no state: asking # where the time went must never block or be blocked by a review. diff --git a/bin/fm-crosscheck.sh b/bin/fm-crosscheck.sh index c3824ce4a7d..62165010509 100755 --- a/bin/fm-crosscheck.sh +++ b/bin/fm-crosscheck.sh @@ -4,6 +4,7 @@ # Usage: # fm-crosscheck.sh run # fm-crosscheck.sh verify +# fm-crosscheck.sh status # fm-crosscheck.sh timings # fm-crosscheck.sh merge [--allow-queue] # @@ -13,6 +14,8 @@ # and claims document to be clear, and prints only the reviewed SHA. # `timings` is the read-only C1 breakdown: it prints the per-phase duration # table every recorded run carries, takes no lock, and changes nothing. +# `status` is the read-only R6 family view: it prints the serving roster family, +# same-model relaxation, and latest durable review family without taking a lock. # `merge` repeats that verification and is the sole entrypoint to the private # exact-SHA GitHub merge or merge-queue primitive. # diff --git a/docs/crosscheck.md b/docs/crosscheck.md index fcda7772260..3ba4fbf9504 100644 --- a/docs/crosscheck.md +++ b/docs/crosscheck.md @@ -84,6 +84,12 @@ The accepted profiles are Pi at xhigh on every registered cross-family model (to Reviewer independence is compared on the model FAMILY, not the exact id, so a `gpt-5.5` author is not admitted a `gpt-5.6-sol` reviewer (finding cc-4dcd7873f71a); an unrecognized model remains its own family. Absent reviewer configuration, unavailable reviewer credentials, or model-policy mismatch produces `CROSSCHECK TOOL-FAILURE` and a nonzero exit before reviewer launch. +The lock-free status read reports whether the roster's first serving family is the cross-family primary or the Codex fallback, the current `crosscheck-same-model` setting, and the latest durable run's `review_family_mode` when a run exists. + +```sh +bin/fm-crosscheck.sh status +``` + Crosscheck requires Python 3.11 or newer and refuses to run on anything older. This is a safety floor rather than a style preference: the bounded-read layer rejects hostile JSON integers by relying on CPython's integer/string conversion limit, which first exists in 3.11, and on an older interpreter that rejection silently stops happening while every banner the gate prints reads exactly the same. Stock macOS `python3` is 3.9, so `bin/fm-crosscheck.sh` resolves a supported sibling interpreter instead of assuming `python3` qualifies, and `bin/fm-crosscheck.py` enforces the same minimum itself so a direct invocation cannot bypass it. diff --git a/tests/fm-crosscheck.test.sh b/tests/fm-crosscheck.test.sh index 4ddc9bb0f9e..319f0057d2e 100755 --- a/tests/fm-crosscheck.test.sh +++ b/tests/fm-crosscheck.test.sh @@ -1054,6 +1054,233 @@ select_cross_family_reviewer() { EOF } +test_non_codex_prompt_addendum_preserves_codex_prompt_bytes() { + "$CROSSCHECK_PYTHON" - "$CROSSCHECK_PY" <<'PY' \ + || fail "the lane-specific exact-SHA prompt addendum regressed" +import hashlib +import importlib.util +import sys + +spec = importlib.util.spec_from_file_location("fm_crosscheck", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +assert spec.loader is not None +spec.loader.exec_module(module) + +base_sha = "b" * 40 +head_sha = "a" * 40 +snapshot = { + "base_sha": base_sha, + "head_sha": head_sha, + "claims_document": "fixture claims", +} +ledger = {"findings": []} +codex_prompt = module.make_prompt( + snapshot, + ledger, + {"account_selector": "CODEX_HOME", "model": "gpt-5.6-sol"}, +) +# Golden bytes captured immediately before the lane addendum landed. This +# makes an accidental edit to the shared Codex prompt observable even when a +# refactor leaves the cross-family assertions below green. +assert len(codex_prompt.encode("utf-8")) == 5404, len(codex_prompt.encode("utf-8")) +assert hashlib.sha256(codex_prompt.encode("utf-8")).hexdigest() == ( + "1fd424d1d16c7d09f3b0da232591559e4ebc33786a1fb5ae861aaaa7fe951bbe" +) + +addendum = f""" +REPRODUCTION COMMAND FORMAT - EXACT REQUIREMENT: +The literal string you place in `executed_reproduction.command` MUST contain, verbatim, both +full 40-character SHAs: exact base {base_sha} and exact head {head_sha}. +Example: bash .crosscheck/reproductions/repro.sh {base_sha} {head_sha} +A command that omits either SHA, abbreviates it, or references it through a shell variable is +refused and the entire review is discarded as UNREVIEWED. +""" +pi_codex_prompt = module.make_prompt( + snapshot, + ledger, + {"account_selector": "PI_CODING_AGENT_DIR", "model": "gpt-5.6-sol"}, +) +cross_family_prompt = module.make_prompt( + snapshot, + ledger, + { + "account_selector": "PI_CODING_AGENT_DIR", + "model": "accounts/fireworks/models/glm-5p2", + }, +) +assert cross_family_prompt == pi_codex_prompt + addendum +assert base_sha in addendum and head_sha in addendum +print("PROMPT ADDENDUM OK") +PY + pass "the non-Codex addendum names both full SHAs without changing Codex prompt bytes" +} + +test_full_sha_verdict_gate_is_not_relaxed_by_the_prompt_addendum() { + "$CROSSCHECK_PYTHON" - "$CROSSCHECK_PY" "$TMP_ROOT" <<'PY' \ + || fail "the full-SHA verdict gate was weakened or lost its mutation pin" +import importlib.util +from pathlib import Path +import subprocess +import sys + +spec = importlib.util.spec_from_file_location("fm_crosscheck", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +assert spec.loader is not None +spec.loader.exec_module(module) + +review_dir = Path(sys.argv[2]) / "full-sha-gate-pin" +review_dir.mkdir() +(review_dir / "app.txt").write_text("reviewed\n", encoding="utf-8") +subprocess.run(["git", "-C", str(review_dir), "init", "-q", "-b", "main"], check=True) +subprocess.run(["git", "-C", str(review_dir), "add", "app.txt"], check=True) + + +class EvidenceExecutor: + def validate_declared_paths(self, paths, *, receipt_path): + assert paths == {".crosscheck/reproductions/repro.sh"}, paths + assert receipt_path == ".crosscheck/reproductions/repro.receipt" + + +base_sha = "b" * 40 +head_sha = "a" * 40 +verdict = { + "schema": module.REVIEW_SCHEMA, + "head_sha": head_sha, + "executing_account_home": "/reviewer/account", + "execution_home": "/reviewer/home", + "executed_reproduction": { + "test_path": ".crosscheck/reproductions/repro.sh", + # Otherwise-valid shape, but neither required literal SHA is present. + # Deleting the implementation's full-SHA check makes this validation + # return successfully and therefore makes this mutation pin fail. + "command": "bash .crosscheck/reproductions/repro.sh BASE HEAD", + "expected_exit": 0, + "output_contains": "REPRODUCED", + "receipt_path": ".crosscheck/reproductions/repro.receipt", + "receipt_contains": "REPRODUCED", + }, + "summary": "review complete", + "citations": [{"path": "app.txt", "line": 1}], + "finding_updates": [], + "new_findings": [], + "suspicions": [], +} +try: + module.validate_review_shape( + verdict, + {"base_sha": base_sha, "head_sha": head_sha}, + review_dir, + { + "executing_account_home": "/reviewer/account", + "execution_home": "/reviewer/home", + }, + evidence_executor=EvidenceExecutor(), + ) +except module.CrosscheckError as exc: + assert str(exc) == ( + "reviewer verdict executed reproduction command must name the exact " + "base and head SHAs" + ), str(exc) +else: + raise AssertionError("a verdict command omitting both full SHAs was accepted") +print("FULL SHA GATE PINNED") +PY + pass "the full-SHA verdict gate remains independently mutation-pinned" +} + +test_status_reports_serving_family_relaxation_and_latest_run() { + local primary fallback primary_out fallback_out flipped_out state_path + primary="$TMP_ROOT/status-primary" + fallback="$TMP_ROOT/status-fallback" + # Durable task ids survive historical directory renames. Status reads the + # validated ledger id and must not reinterpret the containing directory as + # part of the ledger schema. + mkdir -p "$primary/home/config" "$primary/lane-home" "$primary/codex-home" \ + "$primary/data/historical-directory-name" "$fallback/home/config" "$fallback/lane-home" \ + "$fallback/codex-home" "$fallback/data/task-fallback" + cat > "$primary/home/config/crosscheck-reviewer.json" < "$fallback/home/config/crosscheck-reviewer.json" < "$primary/home/config/crosscheck-same-model" + printf 'on\n' > "$fallback/home/config/crosscheck-same-model" + "$CROSSCHECK_PYTHON" - \ + "$primary/data/historical-directory-name/crosscheck-ledger.json" task-primary \ + 2026-08-21T10:00:00Z accounts/fireworks/models/glm-5p2 cross-family-primary \ + "$fallback/data/task-fallback/crosscheck-ledger.json" task-fallback \ + 2026-08-21T11:00:00Z gpt-5.6-sol codex-fallback <<'PY' +import json +from pathlib import Path +import sys + +for offset in (0, 5): + path, task_id, at, model, family = sys.argv[1 + offset:6 + offset] + sha = "a" * 40 + value = { + "schema": "firstmate.crosscheck-ledger.v2", + "task_id": task_id, + "pull_request": "https://github.com/ruby-dlee/firstmate/pull/72", + "findings": [], + "runs": [{ + "at": at, + "head_sha": sha, + "base_sha": sha, + "base_branch_sha": sha, + "claims_sha256": "0" * 64, + "reviewer": {"model": model, "review_family_mode": family}, + "state": "tool-failure", + "summary": "status fixture", + "citations": [], + "updated_findings": [], + "new_findings": [], + "active_blockers": [], + "suspicions": [], + }], + } + Path(path).write_text(json.dumps(value), encoding="utf-8") +PY + + state_path="$primary/read-only-state" + primary_out=$(FM_HOME="$primary/home" FM_DATA_OVERRIDE="$primary/data" \ + FM_STATE_OVERRIDE="$state_path" \ + "$CROSSCHECK_PYTHON" "$CROSSCHECK_PY" status) \ + || fail "status refused the cross-family-serving fixture" + [ "$primary_out" = "crosscheck lane: cross-family serving (pi accounts/fireworks/models/glm-5p2, roster entry 1) +crosscheck same-model relaxation: off +crosscheck last review family: cross-family-primary (task-primary at 2026-08-21T10:00:00Z)" ] \ + || fail "cross-family-serving status was unexpected: $primary_out" + assert_absent "$state_path" "the read-only status command created its state root" + + fallback_out=$(FM_HOME="$fallback/home" FM_DATA_OVERRIDE="$fallback/data" \ + FM_STATE_OVERRIDE="$fallback/read-only-state" \ + "$CROSSCHECK_PYTHON" "$CROSSCHECK_PY" status) \ + || fail "status refused the fallback-active fixture" + [ "$fallback_out" = "crosscheck lane: codex fallback active (codex gpt-5.6-sol, roster entry 1) +crosscheck same-model relaxation: on +crosscheck last review family: codex-fallback (task-fallback at 2026-08-21T11:00:00Z)" ] \ + || fail "fallback-active status was unexpected: $fallback_out" + assert_absent "$fallback/read-only-state" \ + "the fallback status read created its state root" + + printf 'on\n' > "$primary/home/config/crosscheck-same-model" + flipped_out=$(FM_HOME="$primary/home" FM_DATA_OVERRIDE="$primary/data" \ + "$CROSSCHECK_PYTHON" "$CROSSCHECK_PY" status) \ + || fail "status refused the flipped same-model setting" + assert_contains "$flipped_out" 'crosscheck same-model relaxation: on' \ + "flipping same-model did not flip the status read" + [ "${flipped_out/crosscheck same-model relaxation: on/crosscheck same-model relaxation: off}" = "$primary_out" ] \ + || fail "flipping same-model changed more than the status policy line" + pass "status reads the roster family, same-model policy, and latest durable run without a lock" +} + test_reviewer_policy_profiles_and_independence() { "$CROSSCHECK_PYTHON" - "$CROSSCHECK_PY" "$TMP_ROOT" <<'PY' \ || fail "reviewer policy profiles or independence validation regressed" @@ -5213,6 +5440,9 @@ PY if [ -n "${FM_TEST_CASE:-}" ]; then case "$FM_TEST_CASE" in + test_non_codex_prompt_addendum_preserves_codex_prompt_bytes|\ + test_full_sha_verdict_gate_is_not_relaxed_by_the_prompt_addendum|\ + test_status_reports_serving_family_relaxation_and_latest_run|\ test_reviewer_policy_profiles_and_independence|\ test_same_model_relaxation_does_not_require_author_identity|\ test_reviewer_binary_never_resolves_from_working_directory|\ @@ -5331,6 +5561,9 @@ if [ "${FM_TEST_FOCUSED:-}" = review-round-3 ]; then fi test_launcher_requires_supported_python +test_non_codex_prompt_addendum_preserves_codex_prompt_bytes +test_full_sha_verdict_gate_is_not_relaxed_by_the_prompt_addendum +test_status_reports_serving_family_relaxation_and_latest_run test_reviewer_policy_profiles_and_independence test_same_model_relaxation_does_not_require_author_identity test_reviewer_binary_never_resolves_from_working_directory