diff --git a/bin/fm-crosscheck-azure-model-guest.sh b/bin/fm-crosscheck-azure-model-guest.sh index d2507a4adda..3ad768a3ab2 100755 --- a/bin/fm-crosscheck-azure-model-guest.sh +++ b/bin/fm-crosscheck-azure-model-guest.sh @@ -99,7 +99,23 @@ reviewer = request["reviewer"] identity = request["identity"] if "sha256:" + hashlib.sha256(source.read_bytes()).hexdigest() != identity["credential_archive_digest"]: raise SystemExit("model guest: credential archive digest mismatch") -if reviewer["harness"] == "pi" and reviewer["model"] == "FW-GLM-5.2": +# R6 cross-family lane registry, mirroring CROSS_FAMILY_LANES in +# bin/fm-crosscheck.py: model -> (provider slot, pinned chat-completions +# base URL, non-secret executing identity, pinned model-level compat). +CROSS_FAMILY_LANES = { + "accounts/fireworks/models/glm-5p2": ( + "fireworks-glm", + "https://api.fireworks.ai/inference/v1", + "fireworks-glm:api.fireworks.ai/accounts/fireworks/models/glm-5p2", + {}, + ), +} +lane = ( + CROSS_FAMILY_LANES.get(reviewer["model"]) + if reviewer["harness"] == "pi" + else None +) +if lane is not None: expected_name = "models.json" elif reviewer["harness"] in {"codex", "pi"}: expected_name = "auth.json" @@ -128,32 +144,56 @@ expected = { if manifest != expected or manifest["credential_digest"] != identity["credential_digest"]: raise SystemExit("model guest: credential manifest identity mismatch") credential = json.loads(credential_bytes) -if reviewer["harness"] == "pi" and reviewer["model"] == "FW-GLM-5.2": - # R6 GLM lane: the api-key credential must stay inside the pinned - # chat-completions endpoint allowlist, and the executing identity is - # the non-secret Foundry resource/deployment binding. +if lane is not None: + # R6 cross-family lane: the api-key credential must stay inside that + # lane's pinned chat-completions endpoint allowlist, and the executing + # identity is the non-secret provider host/model binding. + slot, allowed_base_url, account, allowed_compat = lane providers = credential.get("providers") if isinstance(credential, dict) else None entry = ( - providers.get("azure-glm") - if isinstance(providers, dict) and set(providers) == {"azure-glm"} + providers.get(slot) + if isinstance(providers, dict) and set(providers) == {slot} else None ) base_url = entry.get("baseUrl") if isinstance(entry, dict) else None - if base_url != "https://aif-fm7c799d-eus01.cognitiveservices.azure.com/openai/v1": - raise SystemExit("model guest: GLM credential endpoint allowlist mismatch") + if base_url != allowed_base_url: + raise SystemExit("model guest: cross-family credential endpoint allowlist mismatch") + # pi composes provider-level compat/headers and a modelOverrides layer into + # the effective model, so the provider's keys are allowlisted rather than + # individually refused. + if set(entry) - {"baseUrl", "api", "apiKey", "models"}: + raise SystemExit("model guest: cross-family credential provider-level field override") # pi gives model-level baseUrl/api precedence over the provider level, # so any model entry carrying either field escapes the provider pin. models = entry.get("models") if isinstance(entry, dict) else None for model_entry in (models if isinstance(models, list) else []): if isinstance(model_entry, dict) and ("baseUrl" in model_entry or "api" in model_entry): - raise SystemExit("model guest: GLM credential model-level endpoint override") - account = "azure-glm:aif-fm7c799d-eus01/FW-GLM-5.2" + raise SystemExit("model guest: cross-family credential model-level endpoint override") + # `compat` keys change how pi frames the request and reads the + # response, so the lane owns them exactly. + if isinstance(model_entry, dict) and model_entry.get("compat", {}) != allowed_compat: + raise SystemExit("model guest: cross-family credential model-level compat override") elif reviewer["harness"] == "codex": + # The PREFIXED identity, byte-identical to + # `account_identity_from_credential` on the host. This is the third place + # that derivation exists (host reader, host archive gate, and here), and + # it is the one that cannot import the others because the guest ships as a + # self-contained script onto a VM. It disagreed by exactly this prefix + # once already: the host digests `codex:` while this derived the bare + # ``, so the comparison below could never be equal and the refusal + # fired INSIDE a booted, paid VM instead of during staging. Any change to + # the host rule must be mirrored here, and + # `model_guest_executing_account_unit` in tests/fm-crosscheck-azure.test.sh + # executes this exact block against the host readers to prove they agree. tokens = credential.get("tokens") if isinstance(credential, dict) else None - account = tokens.get("account_id") if isinstance(tokens, dict) else None + raw = tokens.get("account_id") if isinstance(tokens, dict) else None + account = "codex:" + raw.strip() if isinstance(raw, str) and raw.strip() else None else: entry = credential.get("openai-codex") if isinstance(credential, dict) else None - account = entry.get("accountId") if isinstance(entry, dict) else None + raw = entry.get("accountId") if isinstance(entry, dict) else None + account = ( + "openai-codex:" + raw.strip() if isinstance(raw, str) and raw.strip() else None + ) if not isinstance(account, str) or "sha256:" + hashlib.sha256(account.encode()).hexdigest() != identity["reviewer_account_digest"]: raise SystemExit("model guest: credential executing account mismatch") path = destination / expected_name @@ -183,11 +223,11 @@ case "$HARNESS" in ;; pi) export PI_CODING_AGENT_DIR="$ACCOUNT" - # The model decides the provider slot (R6): the GLM deployment runs on - # the azure-glm Foundry provider, the gpt fallback family stays on + # The model decides the provider slot (R6): each cross-family deployment + # runs on its own provider slot, the gpt fallback family stays on # openai-codex, and an unmapped model refuses rather than guessing. case "$MODEL" in - FW-GLM-5.2) PI_PROVIDER=azure-glm ;; + accounts/fireworks/models/glm-5p2) PI_PROVIDER=fireworks-glm ;; gpt-5.6-sol) PI_PROVIDER=openai-codex ;; *) echo "model guest: no Pi provider mapping for model $MODEL" >&2; exit 125 ;; esac diff --git a/bin/fm-crosscheck-azure.py b/bin/fm-crosscheck-azure.py index 94d6a5728a2..ddc41c37f35 100755 --- a/bin/fm-crosscheck-azure.py +++ b/bin/fm-crosscheck-azure.py @@ -218,12 +218,19 @@ def preflight_reviewer_credential(core: Any, config: dict[str, str]) -> dict[str same interval. """ - if config["harness"] == "pi" and config["model"] == GLM_REVIEWER_MODEL: - # The GLM lane authenticates with a Foundry api-key models.json, - # which declares no expiry, so the preflight that matters is the - # shape/allowlist inspection itself. A refusal there is already the - # core tool failure the roster uses to rotate reviewers. - core.inspect_pi_glm_credential(Path(config["account_home"])) + preflight_lane = ( + cross_family_lane_for_model(config["model"]) + if config["harness"] == "pi" + else None + ) + if preflight_lane is not None: + # A cross-family lane authenticates with an api-key + # models.json, which declares no expiry, so the preflight that matters + # is the shape/allowlist inspection itself. A refusal there is already + # the core tool failure the roster uses to rotate reviewers. + core.inspect_pi_cross_family_credential( + Path(config["account_home"]), preflight_lane + ) return { "profile": config["account_home"], "harness": "pi", @@ -232,7 +239,10 @@ def preflight_reviewer_credential(core: Any, config: dict[str, str]) -> dict[str "expires_at": None, "expires_in_seconds": None, "refresh_expires_at": None, - "detail": "GLM Foundry api-key credential declares no expiry", + "detail": ( + f"{preflight_lane['slot']} api-key credential declares no " + "expiry" + ), } expiry = load_credential_expiry() record = expiry.inspect_profile( @@ -571,21 +581,25 @@ def require_model_image_attests_harness( return attested -# R6 (docs/azure-requirements.md): these pins must equal the constants in -# bin/fm-crosscheck.py; tests/fm-crosscheck-azure.test.sh enforces the -# equality. The GLM lane binds exactly one Foundry resource + deployment and -# exactly one chat-completions endpoint; the interim claude reviewer lane -# and its provider host are retired. -GLM_REVIEWER_MODEL = "FW-GLM-5.2" -GLM_PROVIDER_SLOT = "azure-glm" -GLM_FOUNDRY_RESOURCE = "aif-fm7c799d-eus01" -GLM_PROVIDER_HOST = "aif-fm7c799d-eus01.cognitiveservices.azure.com" -GLM_ALLOWED_BASE_URL = ( - "https://aif-fm7c799d-eus01.cognitiveservices.azure.com/openai/v1" -) -GLM_REVIEWER_ACCOUNT_IDENTITY = ( - GLM_PROVIDER_SLOT + ":" + GLM_FOUNDRY_RESOURCE + "/" + GLM_REVIEWER_MODEL -) +# R6 (docs/azure-requirements.md): this registry must equal +# `CROSS_FAMILY_LANES` in bin/fm-crosscheck.py; +# tests/fm-crosscheck-azure.test.sh enforces the equality as a whole, so a lane +# added on one side and not the other is a test failure rather than a silently +# divergent allowlist. Each lane binds exactly one provider slot + model and +# exactly one chat-completions endpoint; the interim claude reviewer lane and +# its provider host are retired. +CROSS_FAMILY_LANE_API = "openai-completions" +CROSS_FAMILY_LANES = { + "fireworks-glm": { + "slot": "fireworks-glm", + "model": "accounts/fireworks/models/glm-5p2", + "api": CROSS_FAMILY_LANE_API, + "compat": {}, + "host": "api.fireworks.ai", + "base_url": "https://api.fireworks.ai/inference/v1", + "family_aliases": frozenset({"glm5p2", "glm52", "glm5point2"}), + }, +} HARNESS_PROVIDER_HOSTS = { "codex": "chatgpt.com", @@ -593,22 +607,50 @@ def require_model_image_attests_harness( } +def cross_family_lane_for_model(reviewer_model: Any) -> dict[str, str] | None: + """Return the registered cross-family lane one reviewer model belongs to. + + Mirrors `cross_family_lane_for_model` in bin/fm-crosscheck.py, including + its exact matching rule: a lane model id can itself contain slashes, so + the comparison is against the id or the `/` form pi records, + never a suffix. The lane is keyed on the model, never on anything the + credential file supplies. + """ + + if not isinstance(reviewer_model, str): + return None + candidate = reviewer_model.strip() + for lane in CROSS_FAMILY_LANES.values(): + if candidate in (lane["model"], lane["slot"] + "/" + lane["model"]): + return lane + return None + + +def cross_family_account_identity(lane: dict[str, str]) -> str: + return lane["slot"] + ":" + lane["host"] + "/" + lane["model"] + + def effective_provider_host( azure: dict[str, Any], reviewer_harness: str, reviewer_model: str ) -> str: """One exact model-egress host per review, decided by the reviewer model - first: a GLM review binds the pinned Foundry host and refuses any other - configured host. For the codex-family fallback, explicit config wins, - else the reviewer harness names its provider.""" - if reviewer_harness == "pi" and reviewer_model == GLM_REVIEWER_MODEL: + first: a cross-family review binds that lane's pinned provider host and + refuses any other configured host. For the codex-family fallback, explicit + config wins, else the reviewer harness names its provider.""" + lane = ( + cross_family_lane_for_model(reviewer_model) + if reviewer_harness == "pi" + else None + ) + if lane is not None: host = azure.get("provider_host") - if host and host != GLM_PROVIDER_HOST: + if host and host != lane["host"]: raise AzureCrosscheckError( - "Azure Crosscheck GLM reviews bind exactly one provider host " - f"({GLM_PROVIDER_HOST}); refusing configured provider_host " - f"{host!r}" + f"Azure Crosscheck {lane['slot']} reviews bind exactly one " + f"provider host ({lane['host']}); refusing configured " + f"provider_host {host!r}" ) - return GLM_PROVIDER_HOST + return lane["host"] host = azure.get("provider_host") if host: return host @@ -670,13 +712,19 @@ def inspect_reviewer_credential( source, identifier = core.inspect_codex_credential(account_home) credential = account_home / "auth.json" account_identity = core.account_identity(config["harness"], account_home) - elif config["harness"] == "pi" and config["model"] == GLM_REVIEWER_MODEL: - # R6 GLM lane: the credential is the api-key models.json and the - # executing identity is the non-secret Foundry resource/deployment + elif ( + config["harness"] == "pi" + and cross_family_lane_for_model(config["model"]) is not None + ): + # R6 cross-family lane: the credential is the api-key models.json and + # the executing identity is the non-secret provider host/model # binding, because an api key names no upstream account. - source, identifier = core.inspect_pi_glm_credential(account_home) + lane = cross_family_lane_for_model(config["model"]) + source, identifier = core.inspect_pi_cross_family_credential( + account_home, lane + ) credential = account_home / "models.json" - account_identity = core.GLM_REVIEWER_ACCOUNT_IDENTITY + account_identity = core.cross_family_account_identity(lane) elif config["harness"] == "pi": source, identifier = core.inspect_pi_credential(account_home) credential = account_home / "auth.json" @@ -695,12 +743,35 @@ def inspect_reviewer_credential( return credential, source, identifier, account_identity +def require_stable_reviewer_credential( + core: Any, config: dict[str, str], admitted: tuple[Any, str, str, str] +) -> None: + """Re-prove the reviewer credential has not changed since admission. + + Extracted so it can be DRIVEN by a test. It raises + `core.CrosscheckToolError`, not a bare `AzureCrosscheckError`, and the + class is the whole point: this is the TOCTOU refusal, the most + security-relevant one in the staging region, and `AzureCrosscheckError` is + a plain `RuntimeError` that none of the persisting handlers catch. Raised + as the bare class, a credential swapped between admission and staging + would leave no ledger, no report and no data directory - the fleet would + see the swap as nothing at all. + """ + + reproved = inspect_reviewer_credential(core, config) + if reproved != admitted: + raise core.CrosscheckToolError( + "reviewer credential identity changed before exact staging" + ) + + def create_credential_archive( destination: Path, credential: Path, identity: dict[str, str], config: dict[str, str], reviewer_account_identity: str, + core: Any, ) -> tuple[str, str]: """Package the reviewer credential for one-way copy-in at boot. @@ -726,28 +797,49 @@ def create_credential_archive( ) from exc if len(credential_bytes) > MAX_CONFIG_BYTES: raise AzureCrosscheckError("reviewer credential exceeds its byte bound") - glm_profile = ( - config["harness"] == "pi" and config["model"] == GLM_REVIEWER_MODEL + archive_lane = ( + cross_family_lane_for_model(config["model"]) + if config["harness"] == "pi" + else None ) try: parsed = json.loads(credential_bytes) except (json.JSONDecodeError, UnicodeError) as exc: raise AzureCrosscheckError("reviewer credential is malformed") from exc - if glm_profile: - # The archived GLM credential must stay inside the R6 endpoint - # allowlist, and its executing identity is the non-secret Foundry - # resource/deployment binding - never the api key or a digest of it. + if archive_lane is not None: + # The archived cross-family credential must stay inside that lane's R6 + # endpoint allowlist, and its executing identity is the non-secret + # provider host/model binding - never the api key or a digest of it. + slot = archive_lane["slot"] providers = parsed.get("providers") if isinstance(parsed, dict) else None entry = ( - providers.get(GLM_PROVIDER_SLOT) - if isinstance(providers, dict) and set(providers) == {GLM_PROVIDER_SLOT} + providers.get(slot) + if isinstance(providers, dict) and set(providers) == {slot} else None ) + # Pi composes provider-level compat/headers and a modelOverrides layer + # into the effective model, so the archive gate allowlists the + # provider's keys rather than naming fields to refuse + # (cc-ca5848b19ac3). The allowlists come from CORE, not a local copy: + # a hardcoded set here drifted weaker than the inspector's within one + # change - it applied no model-level allowlist and never checked + # `api`, so an archived credential could carry `openai-responses`, + # which R6 forbids outright. + if isinstance(entry, dict) and set(entry) - core.PI_PROVIDER_ALLOWED_KEYS: + raise AzureCrosscheckError( + f"archived {slot} reviewer credential carries provider-level " + "fields the lane does not pin" + ) + if isinstance(entry, dict) and entry.get("api") != archive_lane["api"]: + raise AzureCrosscheckError( + f"archived {slot} reviewer credential does not pin api " + f"{archive_lane['api']!r}" + ) base_url = entry.get("baseUrl") if isinstance(entry, dict) else None - if base_url != GLM_ALLOWED_BASE_URL: + if base_url != archive_lane["base_url"]: raise AzureCrosscheckError( - "archived GLM reviewer credential is not bound to the pinned " - f"R6 Foundry endpoint {GLM_ALLOWED_BASE_URL}" + f"archived {slot} reviewer credential is not bound to the " + f"pinned R6 provider endpoint {archive_lane['base_url']}" ) # pi gives model-level baseUrl/api precedence over the provider # level, so a model entry carrying either field would escape the @@ -759,17 +851,38 @@ def create_credential_archive( "baseUrl" in model_entry or "api" in model_entry ): raise AzureCrosscheckError( - "archived GLM reviewer credential carries a model-level " - "baseUrl/api override that escapes the pinned R6 Foundry " - "endpoint" + f"archived {slot} reviewer credential carries a " + "model-level baseUrl/api override that escapes the pinned " + "R6 provider endpoint" ) - archived_identity = GLM_REVIEWER_ACCOUNT_IDENTITY - elif config["harness"] == "codex": - tokens = parsed.get("tokens") if isinstance(parsed, dict) else None - archived_identity = tokens.get("account_id") if isinstance(tokens, dict) else None - elif config["harness"] == "pi": - entry = parsed.get("openai-codex") if isinstance(parsed, dict) else None - archived_identity = entry.get("accountId") if isinstance(entry, dict) else None + if ( + isinstance(model_entry, dict) + and model_entry.get("compat", {}) != archive_lane["compat"] + ): + raise AzureCrosscheckError( + f"archived {slot} reviewer credential carries a " + "model-level compat that is not the pinned lane compat" + ) + if isinstance(model_entry, dict) and ( + set(model_entry) - core.PI_MODEL_ALLOWED_KEYS + ): + raise AzureCrosscheckError( + f"archived {slot} reviewer credential carries model-level " + "fields the lane does not pin" + ) + archived_identity = cross_family_account_identity(archive_lane) + elif config["harness"] in {"codex", "pi"}: + # ONE derivation, shared with `account_identity`. Deriving it here a + # second time is what broke this lane: this branch returned the bare + # account id while the admitted identity carried a `codex:` / + # `openai-codex:` prefix, so the comparison below could never be + # equal and no codex-family compartment review could ever run. + try: + archived_identity = core.account_identity_from_credential( + config["harness"], parsed, str(credential) + ) + except core.CrosscheckToolError as exc: + raise AzureCrosscheckError(str(exc)) from exc else: raise AzureCrosscheckError( "Azure Crosscheck has no credential-archive lane for reviewer " @@ -785,7 +898,9 @@ def create_credential_archive( "harness": config["harness"], "model": config["model"], "effort": config["effort"], - "credential_name": "models.json" if glm_profile else "auth.json", + "credential_name": ( + "models.json" if archive_lane is not None else "auth.json" + ), "credential_digest": digest_bytes(credential_bytes), } payload = { @@ -1902,18 +2017,32 @@ def _run_azure_review_in_lane( credential_path = work / "credential.tar.gz" result_path = work / "result.json" with measured_phase(phase_timer, "stage"): - credential_archive_digest, credential_digest = create_credential_archive( - credential_path, - credential, - identity, + # A raw AzureCrosscheckError here escapes to main()'s catch-all + # OUTSIDE the window whose handlers persist a run, so this class of + # refusal used to leave no ledger, no report and no data/ directory + # at all - the live codex-family identity refusal was invisible + # afterwards. Converting it to a tool failure is the same treatment + # the model-image attestation refusal above already gets: it is + # recorded, and the roster rotates to the next reviewer account. + try: + ( + credential_archive_digest, + credential_digest, + ) = create_credential_archive( + credential_path, + credential, + identity, + config, + reviewer_account_identity, + core, + ) + except AzureCrosscheckError as exc: + raise core.CrosscheckToolError(str(exc)) from exc + require_stable_reviewer_credential( + core, config, - reviewer_account_identity, + (credential, source, identifier, reviewer_account_identity), ) - reproved = inspect_reviewer_credential(core, config) - if reproved != (credential, source, identifier, reviewer_account_identity): - raise AzureCrosscheckError( - "reviewer credential identity changed before exact staging" - ) identity.update( { "credential_archive_digest": credential_archive_digest, @@ -2160,13 +2289,15 @@ def validate_azure_reviewer_record( raise RuntimeError(f"{label}.reviewer Azure deployment identity is malformed") if not re.fullmatch(r"[0-9a-f]{64}", identity["claims_sha256"]): raise RuntimeError(f"{label}.reviewer Azure claims digest is malformed") - if ( - identity["reviewer_harness"] == "pi" - and identity["reviewer_model"] == GLM_REVIEWER_MODEL - and identity["provider_host"] != GLM_PROVIDER_HOST - ): + recorded_lane = ( + cross_family_lane_for_model(identity["reviewer_model"]) + if identity["reviewer_harness"] == "pi" + else None + ) + if recorded_lane is not None and identity["provider_host"] != recorded_lane["host"]: raise RuntimeError( - f"{label}.reviewer GLM provider host is not the pinned R6 Foundry endpoint" + f"{label}.reviewer {recorded_lane['slot']} provider host is not " + "the pinned R6 provider endpoint" ) generation = digest_bytes( canonical_bytes({field: identity[field] for field in generation_fields}) diff --git a/bin/fm-crosscheck-slack.py b/bin/fm-crosscheck-slack.py index c72275c9e49..d0a44e6adc1 100755 --- a/bin/fm-crosscheck-slack.py +++ b/bin/fm-crosscheck-slack.py @@ -714,11 +714,11 @@ def matched_agent_prefix(branch: str, prefixes: tuple[str, ...]) -> str | None: def lane_name(reviewer: Any) -> str: """Name the lane that produced a ledger run, for the R6/R10 visibility rule. - Prefers an explicit lane marker recorded by the crosscheck ledger (the - sibling GLM/degraded-mode work); when the run predates that marker, the - lane is derived from the reviewer profile that ran: a GLM model is the - primary lane and the retained pi/codex roster is the recorded degraded - fallback. + Prefers an explicit lane marker recorded by the crosscheck ledger, then + the durable `review_family_mode` provenance the crosscheck gate records on + every run; when the run predates both, the lane is derived from the + reviewer profile that ran: a GLM model is the primary lane of that era and + the retained pi/codex roster is the recorded degraded fallback. """ if not isinstance(reviewer, dict): @@ -731,6 +731,13 @@ def lane_name(reviewer: Any) -> str: return name model = str(reviewer.get("model") or "") harness = str(reviewer.get("harness") or "") + family = reviewer.get("review_family_mode") + # The crosscheck gate binds this marker to the reviewer model in both + # directions, so it names the lane without guessing from the model string. + if family in {"cross-family-primary", "glm-primary"}: + return f"{model.rsplit('/', 1)[-1]} primary" if model else "cross-family primary" + if family == "codex-fallback": + return "pi-codex fallback (degraded)" lowered = model.lower() if "glm" in lowered: return "GLM-5.2 primary" if "5.2" in lowered else f"{model} primary" diff --git a/bin/fm-crosscheck.py b/bin/fm-crosscheck.py index 1d5178fefab..3e74afe64b4 100755 --- a/bin/fm-crosscheck.py +++ b/bin/fm-crosscheck.py @@ -52,36 +52,135 @@ # opts in with this exact mode; current task metadata never sets it. LEGACY_AUTHOR_ADMISSION_MODE = "legacy-author-admission" -# R6 (docs/azure-requirements.md): the primary Crosscheck reviewer family is -# GLM-5.2 served from the fleet's own Azure AI Foundry resource through the -# Fireworks partner lane, driven by Pi through a custom `azure-glm` provider. -# The endpoint is an ALLOWLIST of exactly one chat-completions base URL: any -# other baseUrl - including any Responses API surface - is refused by name. -# The reviewer identity binds the Foundry resource + deployment, never the -# api key or anything derived from it. -GLM_REVIEWER_MODEL = "FW-GLM-5.2" -GLM_PROVIDER_SLOT = "azure-glm" -GLM_PROVIDER_API = "openai-completions" -GLM_FOUNDRY_RESOURCE = "aif-fm7c799d-eus01" -GLM_PROVIDER_HOST = "aif-fm7c799d-eus01.cognitiveservices.azure.com" -GLM_ALLOWED_BASE_URL = ( - "https://aif-fm7c799d-eus01.cognitiveservices.azure.com/openai/v1" -) -# Non-secret executing identity for the api-key GLM lane: an api key names no -# upstream account, so the reviewer identity is the resource/deployment pair. -GLM_REVIEWER_ACCOUNT_IDENTITY = ( - GLM_PROVIDER_SLOT + ":" + GLM_FOUNDRY_RESOURCE + "/" + GLM_REVIEWER_MODEL -) +# R6 (docs/azure-requirements.md): the primary Crosscheck reviewer family is a +# NAMED cross-family lane served from the fleet's own Azure AI Foundry resource +# and driven by Pi through a custom provider. Authors run on the OpenAI family, +# so any lane below is outside it, which is what R6 actually requires; no +# single partner model is baked into the gate. +# +# Every registered lane is a complete, code-reviewed ENDPOINT ALLOWLIST entry: +# the deployment name, the Pi provider slot, the chat-completions api surface, +# the Foundry resource, and the ONE accepted base URL. Substituting the serving +# lane among registered lanes is a config change (the roster names the model); +# admitting a NEW endpoint stays a reviewed code change on purpose, because the +# allowlist is the security control - a credential file must never be able to +# introduce an endpoint the policy never named. +# +# The reviewer identity binds the Foundry resource + deployment, never the api +# key or anything derived from it. +CROSS_FAMILY_LANE_API = "openai-completions" +# pi's OpenAI-completions client honors two MODEL-LEVEL knobs that outrank the +# provider level: `baseUrl`/`api` (dist/api provider composer) and the per-model +# `compat` object. `compat` is not cosmetic - `supportsFinishReason: false` +# would blunt the truncated-verdict refusal this gate depends on - so each lane +# declares the EXACT compat its credential may carry and the inspector refuses +# anything else, the same treatment baseUrl and api already get. +CROSS_FAMILY_LANES = { + # The direct Fireworks account. Reaching GLM-5.2 through Azure AI Foundry's + # Fireworks partner lane is impossible on this subscription: partner models + # are Marketplace SaaS offers and a credit-only "Microsoft Azure + # Sponsorship" subscription cannot purchase them, so `FW-GLM-5.2` returned + # HTTP 500 `invalid_model_endpoint_authentication` on every request. Going + # direct bypasses Azure Marketplace. The evidence and citation are in + # docs/azure-requirements.md R6. + # + # The pinned model id, not the `accounts/fireworks/routers/glm-5p2-fast` + # router: a router may re-point to a different serving variant, and the + # reviewer identity this gate records has to name an exact model. Router + # latency is not worth an unattributable reviewer. + "fireworks-glm": { + "slot": "fireworks-glm", + "model": "accounts/fireworks/models/glm-5p2", + "api": CROSS_FAMILY_LANE_API, + "compat": {}, + "host": "api.fireworks.ai", + "base_url": "https://api.fireworks.ai/inference/v1", + # Every spelling of THIS MODEL that an author could be recorded under, + # normalized. The family screen keys on the final path segment, so + # `z-ai/glm-5.2` or a bare `GLM-5.2` would otherwise read as a + # different family and take the GLM reviewer with no same-model + # marker - the original bug reached through a vendor alias. Aliases + # are used ONLY by `model_family`; lane selection stays exact. + # + # An alias list is inherently INCOMPLETE and is not a security + # boundary. Sibling and successor ids - `glm-5.2-flash`, `glm-4.6`, + # and likewise `chatgpt-4o-latest` or `sonnet-5` for the prefix + # families - remain their own family, as does any id normalizing to + # the empty string. Every one of those is refused at ADMISSION today, + # because `allowed_profiles` accepts only registered models, so the + # gap costs a refusal rather than a same-family review. Extend the set + # when a lane's model gains a spelling the fleet actually authors on. + "family_aliases": frozenset({"glm5p2", "glm52", "glm5point2"}), + }, +} # The model decides the Pi provider slot. An unmapped model is refused rather # than guessed, so a roster typo can never route a review to a provider the # policy never named. PI_MODEL_PROVIDERS = { - GLM_REVIEWER_MODEL: GLM_PROVIDER_SLOT, + **{lane["model"]: lane["slot"] for lane in CROSS_FAMILY_LANES.values()}, "gpt-5.6-sol": "openai-codex", } +# Model-family classification for the reviewer independence screen. Comparing +# exact model IDs was not enough: a `gpt-5.5` author admitted a `gpt-5.6-sol` +# codex fallback with no same-model marker, which is same-family review of the +# kind R6 exists to prevent (crosscheck finding cc-4dcd7873f71a, reproduced +# 2026-08-21). A registered cross-family lane is its own family; the prefixes +# below name the families the fleet actually authors on. An unrecognized model +# stays its own family, which preserves the previous behavior for anything not +# listed while strictly tightening it for everything that is. +# Allowlisted credential shape for a cross-family lane's models.json. Pi +# composes an effective model from the provider layer, the model entry, and a +# `modelOverrides` layer, and several composed fields (`compat`, `headers`, +# `baseUrl`, `api`) change where the request goes or how its completion is +# read. Only these keys may appear; anything else refuses by name. +PI_PROVIDER_ALLOWED_KEYS = {"baseUrl", "api", "apiKey", "models"} +PI_MODEL_ALLOWED_KEYS = { + "id", + "name", + "reasoning", + "input", + "cost", + "contextWindow", + "maxTokens", + "compat", +} +# Prefixes are matched against a NORMALIZED identity (lowercased, separators +# removed), so they carry no separator of their own. Requiring a literal dash +# meant `gpt5.6-sol` read as its own family and would have been admitted a +# `gpt-5.6-sol` reviewer - the same defect as cc-4dcd7873f71a, one alias away. +AUTHOR_MODEL_FAMILY_PREFIXES = ( + ("gpt", "openai"), + ("o1", "openai"), + ("o3", "openai"), + ("o4", "openai"), + ("codex", "openai"), + ("claude", "anthropic"), +) + + +def normalize_model_identity(identity: str) -> str: + """Fold a model id to the form the FAMILY screen compares. + + Lowercased with separators removed, so `GLM-5.2`, `glm 5.2` and `glm_5.2` + are one string. Used ONLY by `model_family`, never by lane selection: the + safe error differs between them, and folding two ids together is safe only + where the consequence is refusing a reviewer. + """ + + return re.sub(r"[^a-z0-9]", "", identity.lower()) # Review family provenance recorded in every run's ledger reviewer record. +REVIEW_FAMILY_CROSS_FAMILY_PRIMARY = "cross-family-primary" +# Legacy provenance value: runs recorded before the lane registry landed named +# the Azure GLM lane directly. Durable ledgers still carry it, so validation +# accepts it - bound, as before, to exactly that lane's model, which is no +# longer a registered lane and so can never be claimed by a new run. REVIEW_FAMILY_GLM_PRIMARY = "glm-primary" +LEGACY_GLM_PRIMARY_MODEL = "FW-GLM-5.2" REVIEW_FAMILY_CODEX_FALLBACK = "codex-fallback" +REVIEW_FAMILY_PRIMARY_MODES = { + REVIEW_FAMILY_CROSS_FAMILY_PRIMARY, + REVIEW_FAMILY_GLM_PRIMARY, +} # C1 (docs/azure-requirements.md): every run records where its wall clock went. # The local lane owns the first four; the Azure compartment lane additionally @@ -404,20 +503,54 @@ def account_identity(harness: str, account_home: Path) -> str: """Stable upstream executing-account identity for one reviewer home. The returned string never carries token material: Codex and Pi expose - explicit upstream account ids. The GLM api-key lane never reaches this - reader - its identity is the non-secret Foundry resource/deployment - binding (GLM_REVIEWER_ACCOUNT_IDENTITY), because an api key names no + explicit upstream account ids. The api-key cross-family lanes never reach + this reader - their identity is the non-secret Foundry resource/deployment + binding (`cross_family_account_identity`), because an api key names no upstream account. """ home = Path(account_home).resolve() if harness == "codex": - credential = read_json( - home / "auth.json", - "Codex executing-account credential", - maximum_bytes=1024 * 1024, - maximum_items=256, - ) + label = "Codex executing-account credential" + elif harness == "pi": + label = "Pi executing-account credential" + else: + # The claude reader (a refresh-token digest over .credentials.json) + # left with the retired claude reviewer lane (R6); no crosscheck + # profile can reach it, so an unknown harness refuses by name. + tool_fail(f"no executing-account identity reader for harness {harness!r}") + credential = read_json( + home / "auth.json", + label, + maximum_bytes=1024 * 1024, + maximum_items=256, + ) + return account_identity_from_credential(harness, credential, str(home)) + + +def account_identity_from_credential( + harness: str, credential: Any, source: str +) -> str: + """Derive the executing-account identity from one PARSED credential. + + This is the SINGLE derivation of that identity, and it exists because + there used to be two. `account_identity` reads a reviewer home's + `auth.json` and calls this; the Azure credential archive parses the bytes + it is about to package and calls the SAME function, then compares. + + When the two derivations were written separately they disagreed by a + literal prefix: this one returns `codex:` / `openai-codex:` while + the archive returned the bare ``. The archive's + `archived_identity != reviewer_account_identity` refusal was therefore + structurally always true, and NO codex-family compartment review could + ever run - a live run refused at that line before any billable resource. + Only the cross-family branch passed, because both sides there read one + shared constant, which is exactly why it went unnoticed. Keep it one + function; do not "fix" a future mismatch by making the comparison lenient + or by stripping prefixes in a third place. + """ + + if harness == "codex": tokens = credential.get("tokens") if isinstance(credential, dict) else None account = tokens.get("account_id") if isinstance(tokens, dict) else None if isinstance(account, str) and account.strip(): @@ -428,26 +561,13 @@ def account_identity(harness: str, account_home: Path) -> str: # crosscheck finding cc-36d5b5cfcb2a). A credential without an # upstream account id exposes no stable executing-account identity # and is refused. - tool_fail( - f"Codex credential at {home} exposes no executing account identity" - ) + tool_fail(f"Codex credential at {source} exposes no executing account identity") if harness == "pi": - credentials = read_json( - home / "auth.json", - "Pi executing-account credential", - maximum_bytes=1024 * 1024, - maximum_items=256, - ) - credential = ( - credentials.get("openai-codex") if isinstance(credentials, dict) else None - ) - account = credential.get("accountId") if isinstance(credential, dict) else None + entry = credential.get("openai-codex") if isinstance(credential, dict) else None + account = entry.get("accountId") if isinstance(entry, dict) else None if isinstance(account, str) and account.strip(): return "openai-codex:" + account.strip() - tool_fail(f"Pi credential at {home} exposes no executing account identity") - # The claude reader (a refresh-token digest over .credentials.json) left - # with the retired claude reviewer lane (R6); no crosscheck profile can - # reach it, so an unknown harness refuses by name. + tool_fail(f"Pi credential at {source} exposes no executing account identity") tool_fail(f"no executing-account identity reader for harness {harness!r}") @@ -550,12 +670,85 @@ def inspect_pi_credential(account_home: Path) -> tuple[str, str]: return "pi-openai-codex-oauth-file", str(credential_file) +def cross_family_lane_for_model(model: Any) -> dict[str, str] | None: + """Return the registered cross-family lane a reviewer model belongs to. + + Matching is EXACT against the lane's model id, or against the + `/` form pi records. It is deliberately not a suffix + or `model_identity` comparison: a lane model id can itself contain slashes + (`accounts/fireworks/models/glm-5p2`), so a loose rule would either miss + the lane or admit an unrelated model that happens to end the same way. + + A model outside CROSS_FAMILY_LANES is not a cross-family reviewer: it is + either the codex-family fallback or an unmapped model that + `pi_provider_for_model` refuses. Returning None rather than guessing keeps + the fallback lane and the primary lane from ever blurring together. + """ + + if not isinstance(model, str): + return None + candidate = model.strip() + for lane in CROSS_FAMILY_LANES.values(): + if candidate in (lane["model"], lane["slot"] + "/" + lane["model"]): + return lane + return None + + +def model_family(model: Any) -> str: + """Classify one model into the family the independence screen compares. + + Reviewer independence is a FAMILY property, not a version-string one. + Comparing exact ids let a `gpt-5.5` author be reviewed by `gpt-5.6-sol` + with no same-model marker recorded (cc-4dcd7873f71a). + + Lane membership is judged MORE loosely here than in + `cross_family_lane_for_model`, and deliberately in the opposite direction. + That function picks a credential and a provider, so it must match exactly + or refuse. This one decides whether two models are the same family, where + the safe error is to say yes: pi records a model as + `/`, so the SAME lane model reached through some + other author-side slot must not read as a different family. Matching only + the registry's own slot let exactly that through - a + `some-slot/accounts/fireworks/models/glm-5p2` author was admitted the + `fireworks-glm` reviewer with no relaxation and no degraded marker + (cc-5ec330d3c74d). Any model whose final segment matches a lane's final + segment is now that lane's family. An unrelated model that happens to end + the same way is classified together with the lane and refused, which is + the direction that fails closed. + """ + + exact = cross_family_lane_for_model(model) + if exact is not None: + return "cross-family:" + exact["slot"] + identity = model_identity(model if isinstance(model, str) else "") + normalized = normalize_model_identity(identity) + for lane in CROSS_FAMILY_LANES.values(): + if normalized in lane["family_aliases"]: + return "cross-family:" + lane["slot"] + for prefix, family in AUTHOR_MODEL_FAMILY_PREFIXES: + if normalized.startswith(prefix): + return family + return "model:" + normalized + + +def cross_family_account_identity(lane: dict[str, str]) -> str: + """Non-secret executing identity for one api-key cross-family lane. + + An api key names no upstream account, so the reviewer identity is the + provider slot plus the pinned endpoint host and model. It neither contains + nor is derived from the key. + """ + + return lane["slot"] + ":" + lane["host"] + "/" + lane["model"] + + def pi_provider_for_model(model: str) -> str: """Return the exact Pi provider slot the reviewer model executes on. - The mapping is explicit, not heuristic: FW-GLM-5.2 runs on the R6 - `azure-glm` Foundry provider and the gpt fallback family stays on - `openai-codex`. Any model outside the table refuses by name. + The mapping is explicit, not heuristic: every registered cross-family + deployment runs on its own R6 Foundry provider slot and the gpt fallback + family stays on `openai-codex`. Any model outside the table refuses by + name. """ provider = PI_MODEL_PROVIDERS.get(model) @@ -567,68 +760,97 @@ def pi_provider_for_model(model: str) -> str: return provider -def inspect_pi_glm_credential(account_home: Path) -> tuple[str, str]: - """Validate the api-key models.json credential of one GLM reviewer home. +def inspect_pi_cross_family_credential( + account_home: Path, lane: dict[str, str] +) -> tuple[str, str]: + """Validate the api-key models.json credential of one cross-family home. + + A cross-family reviewer's account home is a dedicated Pi agent dir whose + credential is `models.json` carrying exactly that lane's custom provider. + The endpoint is an allowlist of exactly the lane's registered base URL + (chat completions only; any configuration reaching for a Responses API + surface is refused). The returned credential identifier is a non-secret + binding of the Foundry resource + deployment + pinned endpoint; it neither + contains nor is derived from the api key. - A GLM reviewer's account home is a dedicated Pi agent dir whose credential - is `models.json` carrying exactly the `azure-glm` custom provider. The - endpoint is an allowlist of exactly GLM_ALLOWED_BASE_URL (chat completions - only; any configuration reaching for a Responses API surface is refused). - The returned credential identifier is a non-secret binding of the Foundry - resource + deployment + pinned endpoint; it neither contains nor is derived - from the api key. + The lane comes from the code-side registry, never from the credential + file, so a models.json can only satisfy or fail the pin - never move it. """ + slot = lane["slot"] + allowed_base_url = lane["base_url"] credential_file = account_home.resolve() / "models.json" try: metadata = credential_file.lstat() except OSError as exc: tool_fail( - "GLM reviewer credential inspection failed at " + f"{slot} reviewer credential inspection failed at " f"{credential_file}: {exc}" ) if not stat.S_ISREG(metadata.st_mode) or credential_file.is_symlink(): tool_fail( - "GLM reviewer credential inspection requires a regular " + f"{slot} reviewer credential inspection requires a regular " f"non-symlink file at {credential_file}" ) try: document = read_json( credential_file, - "GLM reviewer credential", + f"{slot} reviewer credential", maximum_bytes=1024 * 1024, maximum_items=4096, ) except CrosscheckError as exc: tool_fail(str(exc)) - providers = document.get("providers") if isinstance(document, dict) else None - if not isinstance(providers, dict) or set(providers) != {GLM_PROVIDER_SLOT}: + if not isinstance(document, dict) or set(document) != {"providers"}: tool_fail( - f"GLM reviewer credential at {credential_file} must declare " - f"exactly the {GLM_PROVIDER_SLOT} provider" + f"{slot} reviewer credential at {credential_file} must be exactly " + 'a {"providers": ...} document' ) - provider = providers[GLM_PROVIDER_SLOT] + providers = document.get("providers") + if not isinstance(providers, dict) or set(providers) != {slot}: + tool_fail( + f"{slot} reviewer credential at {credential_file} must declare " + f"exactly the {slot} provider" + ) + provider = providers[slot] if not isinstance(provider, dict): tool_fail( - f"GLM reviewer credential at {credential_file} has a malformed " - f"{GLM_PROVIDER_SLOT} provider entry" + f"{slot} reviewer credential at {credential_file} has a malformed " + f"{slot} provider entry" + ) + # Pi composes an effective model from SEVERAL layers, not just the model + # entry: `compat: mergeCompat(providerConfig.compat, definition.compat)` + # and a topmost `modelOverrides[]` layer that can carry `compat` + # and `headers` (dist/core/provider-composer.js). Naming the fields to + # refuse one at a time missed both of those and let a credential turn off + # `supportsFinishReason` behind the lane's pin (cc-ca5848b19ac3). The + # provider is therefore an ALLOWLIST of keys: anything this gate has not + # reasoned about is refused rather than composed. + unexpected = set(provider) - PI_PROVIDER_ALLOWED_KEYS + if unexpected: + tool_fail( + f"{slot} reviewer credential at {credential_file} carries " + f"provider-level fields the lane does not pin: " + f"{', '.join(sorted(unexpected))}; pi composes provider-level " + "compat, headers and modelOverrides into the effective model, so " + "only the pinned fields may appear" ) base_url = provider.get("baseUrl") - if base_url != GLM_ALLOWED_BASE_URL: + if base_url != allowed_base_url: tool_fail( - f"GLM reviewer endpoint allowlist refused baseUrl {base_url!r}; " - f"the only accepted endpoint is {GLM_ALLOWED_BASE_URL}" + f"{slot} reviewer endpoint allowlist refused baseUrl " + f"{base_url!r}; the only accepted endpoint is {allowed_base_url}" ) - if provider.get("api") != GLM_PROVIDER_API: + if provider.get("api") != lane["api"]: tool_fail( - f"GLM reviewer credential at {credential_file} must pin api " - f"{GLM_PROVIDER_API!r} (chat completions only; a Responses API " + f"{slot} reviewer credential at {credential_file} must pin api " + f"{lane['api']!r} (chat completions only; a Responses API " "configuration is refused)" ) api_key = provider.get("apiKey") if not isinstance(api_key, str) or not api_key.strip(): tool_fail( - f"GLM reviewer credential is unusable at {credential_file}: " + f"{slot} reviewer credential is unusable at {credential_file}: " "no api key material" ) models = provider.get("models") @@ -646,26 +868,50 @@ def inspect_pi_glm_credential(account_home: Path) -> tuple[str, str]: for entry in model_entries: if "baseUrl" in entry or "api" in entry: tool_fail( - f"GLM reviewer credential at {credential_file} carries a " + f"{slot} reviewer credential at {credential_file} carries a " "model-level baseUrl/api override; pi gives model-level " "fields precedence over the provider, so the pinned " "provider-level endpoint must own both" ) - if GLM_REVIEWER_MODEL not in [entry.get("id") for entry in model_entries]: + # Same allowlist reasoning one layer down: a model entry may only + # carry descriptive fields plus the lane's own pinned compat. + unexpected = set(entry) - PI_MODEL_ALLOWED_KEYS + if unexpected: + tool_fail( + f"{slot} reviewer credential at {credential_file} carries " + f"model-level fields the lane does not pin: " + f"{', '.join(sorted(unexpected))}" + ) + # `compat` keys weaken this gate's own defenses + # (`supportsFinishReason: false` would blunt the truncated-verdict + # refusal), so every entry must carry exactly the lane's declared + # compat and nothing else. + if entry.get("compat", {}) != lane["compat"]: + tool_fail( + f"{slot} reviewer credential at {credential_file} carries a " + "model-level compat that is not the pinned lane compat " + f"{json.dumps(lane['compat'], sort_keys=True)}; compat keys " + "change how pi frames the request and reads the response, so " + "the lane owns them" + ) + if lane["model"] not in [entry.get("id") for entry in model_entries]: tool_fail( - f"GLM reviewer credential at {credential_file} does not declare " - f"the {GLM_REVIEWER_MODEL} deployment" + f"{slot} reviewer credential at {credential_file} does not " + f"declare the {lane['model']} deployment" ) binding = hashlib.sha256( ( - GLM_FOUNDRY_RESOURCE + lane["host"] + "/" - + GLM_REVIEWER_MODEL + + lane["model"] + "\n" - + GLM_ALLOWED_BASE_URL + + allowed_base_url ).encode("utf-8") ).hexdigest() - return "pi-azure-glm-models-file", "glm-foundry-binding:" + binding + return ( + "pi-" + slot + "-models-file", + "provider-binding:" + slot + ":" + binding, + ) def model_identity(model: str) -> str: @@ -2849,22 +3095,37 @@ def validate_ledger(value: Any, task_id: str, url: str) -> dict[str, Any]: family in { None, - REVIEW_FAMILY_GLM_PRIMARY, + *REVIEW_FAMILY_PRIMARY_MODES, REVIEW_FAMILY_CODEX_FALLBACK, }, f"{label}.reviewer.review_family_mode is invalid", ) if family is not None: # The family marker is bound to the model, so a forged record - # cannot claim glm-primary for a codex-family review or hide - # a fallback behind the primary label. + # cannot claim a primary lane for a codex-family review or + # hide a fallback behind the primary label. Each marker names + # the exact set of models allowed to carry it: + # + # cross-family-primary -> a currently registered lane + # glm-primary (legacy) -> only the retired Azure GLM model, + # which is no longer registered, so + # no new run can claim it + # codex-fallback -> neither of the above reviewer_model = reviewer.get("model") - model_is_glm = ( + lane = cross_family_lane_for_model(reviewer_model) + is_legacy_glm = ( isinstance(reviewer_model, str) - and model_identity(reviewer_model) == GLM_REVIEWER_MODEL + and model_identity(reviewer_model) + == LEGACY_GLM_PRIMARY_MODEL ) + if family == REVIEW_FAMILY_CROSS_FAMILY_PRIMARY: + matches = lane is not None + elif family == REVIEW_FAMILY_GLM_PRIMARY: + matches = is_legacy_glm + else: + matches = lane is None and not is_legacy_glm require( - (family == REVIEW_FAMILY_GLM_PRIMARY) == model_is_glm, + matches, f"{label}.reviewer.review_family_mode does not match the " "reviewer model", ) @@ -3049,14 +3310,14 @@ def reviewer_candidates( isinstance(reviewers, list) and reviewers, "reviewer configuration.reviewers must be a nonempty array", ) - # R6: GLM-5.2 on the fleet's own Foundry resource is the PRIMARY review - # family. The pi-codex/codex profiles remain only as the dormant fallback - # lane; selecting one is recorded as a degraded mode in the ledger and - # announced loudly at run time. The interim claude reviewer lane is - # retired: a claude profile is refused here by the same exact-profile - # message as any other unlisted profile. + # R6: a registered cross-family lane on the fleet's own Foundry resource is + # the PRIMARY review family. The pi-codex/codex profiles remain only as the + # dormant fallback lane; selecting one is recorded as a degraded mode in + # the ledger and announced loudly at run time. The interim claude reviewer + # lane is retired: a claude profile is refused here by the same + # exact-profile message as any other unlisted profile. allowed_profiles = { - ("pi", GLM_REVIEWER_MODEL, "xhigh"), + *((("pi", lane["model"], "xhigh")) for lane in CROSS_FAMILY_LANES.values()), ("codex", "gpt-5.6-sol", "xhigh"), ("pi", "gpt-5.6-sol", "xhigh"), } @@ -3091,19 +3352,25 @@ def reviewer_candidates( "model": model, "effort": effort, "account_home": str(account_home.resolve()), - # Durable review-family provenance: GLM is the primary lane; - # every codex-family profile is the recorded fallback. + # Durable review-family provenance: a registered cross-family + # lane is the primary; every codex-family profile is the + # recorded fallback. "review_family_mode": ( - REVIEW_FAMILY_GLM_PRIMARY - if model == GLM_REVIEWER_MODEL + REVIEW_FAMILY_CROSS_FAMILY_PRIMARY + if cross_family_lane_for_model(model) is not None else REVIEW_FAMILY_CODEX_FALLBACK ), } ) - author_model = model_identity(meta["model"]) + # Independence is compared on the model FAMILY, not the exact id: a + # `gpt-5.5` author admitting a `gpt-5.6-sol` reviewer is the same-family + # review this requirement exists to prevent (cc-4dcd7873f71a). The durable + # ledger marker keeps its `same-model` spelling, which older records + # already carry; it now means "shares the author's model family". + author_family = model_family(meta["model"]) eligible: list[dict[str, str]] = [] for reviewer in validated: - model_is_separate = model_identity(reviewer["model"]) != author_model + model_is_separate = model_family(reviewer["model"]) != author_family if model_is_separate or allow_same_model: if not model_is_separate: reviewer["model_independence"] = "same-model" @@ -3111,8 +3378,8 @@ def reviewer_candidates( if eligible: return eligible fail( - "reviewer model policy found no configured reviewer on a different model " - f"from {meta['model']!r}" + "reviewer model policy found no configured reviewer outside the model " + f"family of {meta['model']!r}" ) @@ -3486,6 +3753,70 @@ def pi_reviewer_command() -> list[str]: return [str(resolved_entrypoint)] +# A verdict is a bare JSON object. Chat models routinely present one inside a +# Markdown code fence instead, which is a formatting habit rather than a +# different verdict. Measured, not assumed: asked for this gate's exact review +# instruction and schema, GLM-5.2 returns "```json\n{...}\n```", and a bare +# parse of that fails with `Expecting value: line 1 column 1 (char 0)` - byte +# for byte the failure that cost the lane its first review attempt. +# +# The rule is "EXACTLY ONE complete fenced block in the message", which is +# deterministic and never asks the gate to choose: with one block there is +# nothing to pick between, so surrounding prose is harmless and unwrapping is +# safe. Several complete blocks refuse, because then the gate WOULD be +# choosing which one was the verdict. Requiring the fence to span the whole +# message instead would re-break the lane the first time a model prefaces its +# answer with a sentence, and the point of a registry-driven lane is that the +# next model does not need the prompt re-tuned. +# +# This tolerates a WRAPPER, never a TRUNCATED verdict. A truncated verdict +# never closes its fence, so it yields zero complete blocks, falls through to +# the bare text, and still fails to parse - and `stopReason` refuses it one +# step earlier regardless. Both remain pinned by tests. +PI_FENCED_BLOCK_RE = re.compile( + r"```[A-Za-z0-9_+.-]*[ \t]*\r?\n(?P.*?)\r?\n?```", + re.DOTALL, +) + + +def pi_verdict_body(final_text: str) -> str: + """Return the JSON body of a Pi reviewer's final assistant text. + + An UNTERMINATED fence anywhere in the message refuses outright, before the + block count is even consulted. That ordering is the whole safety property. + A truncated verdict fence contributes ZERO complete blocks, so a model that + emitted any complete fence earlier in the same message - a draft, an + example, a quoted snippet - left the count at exactly one, and this + returned THAT EARLIER BLOCK as the verdict while silently discarding the + truncated real one. `stopReason` is `stop` in that shape (the exact live + condition seen on attempt 3), and the parse SUCCEEDS on the wrong block, so + nothing else downstream catches it: a superseded draft gets certified as + the review. That is strictly worse than the failure it replaced, which at + least failed loudly. + """ + + stripped = final_text.strip() + # An odd number of fence markers means one was opened and never closed. + # Refusing on the marker count rather than on "no complete block found" + # is what makes a preceding complete fence unable to rescue a truncated + # one; returning the raw text sends it to the parser, which fails. + if stripped.count("```") % 2: + return stripped + blocks = PI_FENCED_BLOCK_RE.findall(stripped) + if len(blocks) != 1: + return stripped + # The block must be the ONLY JSON-bearing content in the message. An even + # fence count is not enough on its own: a COMPLETE example fence followed + # by a truncated BARE verdict also counts one block, and unwrapping there + # would certify the example and discard the real answer. Prose carries no + # braces, so this still tolerates a wrapper while refusing every shape + # where a second candidate verdict exists. + remainder = PI_FENCED_BLOCK_RE.sub("", stripped, count=1) + if "{" in remainder or "}" in remainder: + return stripped + return blocks[0].strip() + + def pi_review_result(output: str) -> tuple[dict[str, Any], int]: turn_count = 0 agent_ended = False @@ -3568,10 +3899,19 @@ def pi_review_result(output: str) -> tuple[dict[str, Any], int]: ) if final_text is None or not final_text.strip(): tool_fail("Pi reviewer completed without a verdict artifact") + body = pi_verdict_body(final_text) try: - verdict = json.loads(final_text) + verdict = json.loads(body) except (json.JSONDecodeError, ValueError, RecursionError) as exc: - tool_fail(f"Pi reviewer returned a malformed verdict artifact: {exc}") + # The offending text is bounded and repr-escaped: it is reviewer + # output, so it must never be able to inject lines into an operator's + # log, and without it "malformed verdict artifact" names no cause at + # all - the defect that made the first cross-family lane failure + # unreadable. + tool_fail( + f"Pi reviewer returned a malformed verdict artifact: {exc}; " + f"final assistant text began {body[:240]!r}" + ) if not isinstance(verdict, dict): tool_fail("Pi reviewer verdict artifact must be an object") return verdict, turn_count @@ -3615,12 +3955,31 @@ def run_reviewer( config["account_selector"] = "CODEX_HOME" else: execution_home = prepare_pi_execution_home(protocol_dir, account_home) - # The model decides the credential shape as well as the provider: the - # GLM lane authenticates through the api-key models.json custom - # provider, while the codex-family fallback keeps its OAuth auth.json. - if pi_provider_for_model(config["model"]) == GLM_PROVIDER_SLOT: - credential_source, credential_identifier = inspect_pi_glm_credential( - account_home + # The model decides the credential shape as well as the provider: a + # cross-family lane authenticates through the api-key models.json + # custom provider, while the codex-family fallback keeps its OAuth + # auth.json. The provider slot must be the lane's own slot: without + # that check a mapping edit could route a cross-family review onto the + # author's own family while the ledger still recorded the cross-family + # model. + cross_family_lane = cross_family_lane_for_model(config["model"]) + if cross_family_lane is not None: + # Defensive only, and STRUCTURALLY UNREACHABLE today: + # PI_MODEL_PROVIDERS is built as {lane["model"]: lane["slot"]}, so + # the two cannot disagree unless someone hand-writes an entry. + # Kept as a cheap consistency assertion, but no documentation + # claims it as a control, because an unreachable guard is not one. + require( + pi_provider_for_model(config["model"]) + == cross_family_lane["slot"], + "reviewer provider mapping does not match the cross-family " + f"lane registered for model {config['model']!r}", + ) + ( + credential_source, + credential_identifier, + ) = inspect_pi_cross_family_credential( + account_home, cross_family_lane ) else: credential_source, credential_identifier = inspect_pi_credential( @@ -4265,8 +4624,8 @@ def render_report(ledger: dict[str, Any], run: dict[str, Any]) -> str: ): lines.extend( [ - "Review family: **CODEX FALLBACK** (degraded; the GLM-5.2 " - "primary lane did not serve this run).", + "Review family: **CODEX FALLBACK** (degraded; no cross-family " + "primary lane served this run).", "", ] ) @@ -4624,7 +4983,7 @@ def persist(run: dict[str, Any]) -> None: print( "CROSSCHECK DEGRADED: codex-family fallback reviewer " f"{config['harness']} {config['model']} is standing in " - f"for the GLM-5.2 primary lane; {relaxation}", + f"for the cross-family primary lane; {relaxation}", file=sys.stderr, ) remaining = len(candidates) - position - 1 diff --git a/docs/azure-crosscheck.md b/docs/azure-crosscheck.md index 049d09aef73..4adb8c1e999 100644 --- a/docs/azure-crosscheck.md +++ b/docs/azure-crosscheck.md @@ -77,15 +77,15 @@ The reviewer credential is staged as a short-lived exact-object capability. It exists only in the model compartment and is removed before result publication. The macOS Keychain is never copied. -### GLM-5.2 primary reviewer (R6) +### Cross-family primary reviewer (R6) -The primary review family is GLM-5.2 on the fleet's own Azure AI Foundry resource through the Fireworks partner lane, driven by Pi as the `FW-GLM-5.2` deployment on the `azure-glm` custom provider. -For that profile the packaged compartment credential is the api-key `models.json` (not a codex `auth.json`), pinned to exactly `https://aif-fm7c799d-eus01.cognitiveservices.azure.com/openai/v1` - chat completions only; any other baseUrl, including a Responses API surface, refuses before staging. +The primary review family is a registered cross-family lane, driven by Pi as that lane's model on its own custom provider slot. Today's registry is the single lane `fireworks-glm` (`accounts/fireworks/models/glm-5p2`, GLM-5.2 direct from Fireworks). The Azure Foundry partner lane it replaced is unusable on this subscription: see R6 in docs/azure-requirements.md. +For that profile the packaged compartment credential is the api-key `models.json` (not a codex `auth.json`), pinned to exactly `https://api.fireworks.ai/inference/v1` - chat completions only; any other baseUrl, including a Responses API surface, refuses before staging. The archive gate also refuses a model-level `compat` that is not the lane's pinned compat, so a credential cannot weaken the truncated-verdict refusal on its way into a compartment. pi gives model-level `baseUrl`/`api` fields precedence over the provider level, so the inspection, the archive gate, and the model guest all refuse a model entry carrying either field; the pinned provider level owns both. -`effective_provider_host` is model-aware: a GLM review derives `aif-fm7c799d-eus01.cognitiveservices.azure.com` as its single egress host and refuses a conflicting configured `provider_host`, while the codex-family fallback keeps its `chatgpt.com` derivation. +`effective_provider_host` is model-aware: a cross-family review derives its own lane's host, today `api.fireworks.ai`, as its single egress host and refuses a conflicting configured `provider_host`, while the codex-family fallback keeps its `chatgpt.com` derivation. The executing identity is the non-secret Foundry resource/deployment binding (an api key names no upstream account); the api key and anything derived from it never enter identity, ledger, or output. The interim claude reviewer lane is retired end to end: no `api.anthropic.com` host derivation, no `.credentials.json` packaging or boot copy, and no claude launch branch in the model guest. -Honest limit, corrected 2026-08-20: this Azure-compartment GLM lane does not run today because it is switched off, not because the image lacks `pi`. `$FM_HOME/config/crosscheck-azure.json` exists and carries `"enabled": false`, set by an operator on 2026-08-20; that flag, not an image rebake, is what stands between this lane and a run. The executable GLM lane today is the local Pi reviewer. +Honest limit, corrected twice. This Azure-compartment lane does not run today, and until 2026-08-21 both reasons given for that were wrong. It is not that the image lacks `pi` (stale), and it is not ONLY that the lane is switched off. `$FM_HOME/config/crosscheck-azure.json` carries `"enabled": false`, set by an operator on 2026-08-20 - but flipping that flag would NOT have restored the codex-family path, because a second, independent, in-code blocker sat behind it: the archive gate derived the executing-account identity a second time and returned the BARE account id while the admitted identity carried a `codex:` / `openai-codex:` prefix, so `archived_identity != reviewer_account_identity` was structurally always true. A live run refused there at 04:43Z, before any billable resource. NO codex-family compartment review has ever run; every one of the resource group's historical `fm-crosscheck-model-*` deployments is on the cross-family path, which passed only because both sides there read one shared value. Fixed 2026-08-21, and the FIRST attempt at that fix was incomplete in a way worth recording: it made the host reader and the host archive gate share one derivation, but the model guest carried a THIRD copy that still derived the bare account id, so the refusal simply moved from staging into a booted, paid VM. All three now derive the prefixed identity; the guest cannot import the others because it ships self-contained onto the VM, so `model_guest_executing_account_unit` EXECUTES the guest's own credential block against the host readers to prove they agree, and is red on either the two- or the three-derivation form. The executable cross-family lane today is still the local Pi reviewer. The earlier reading of this limit said the built image carries no `pi` binary and needed a rebake. That was measured on 2026-08-16 against gallery version `1.0.1786915905`, whose source managed image `img-fm7c799d-ccm-1.0.0` was built on 2026-08-13 from the pre-Pi declaration and carries no `pi-tarball-sha256` tag (M29 in the owner's mutation ledger, `firstmate-azure-full-completion-mutation-ledger.md`, which lives outside this repository rather than in it). It was already stale when it was written here: `model_image_id` has named `1.0.1787092687` since 2026-08-18T22:45Z. That current version was published 2026-08-18T22:38:08Z from managed image `img-fm7c799d-ccm-1.0.1787091895`, which carries `pi-tarball-sha256` `a69a1859...` and `node-tarball-sha256` `d60acfe0...`, matching `docs/azure-crosscheck/model-image-closure.json` for `pi-coding-agent` 0.84.1 and Node v22.23.2. Only a build from the Pi-carrying declaration writes those tags, its Image Builder run succeeded, and that declaration asserts `/usr/local/bin/pi --version` against the tracked version twice under `set -eu`, before and after the credential purge, so a build that reached distribution cannot have omitted `pi`. What remains unproven is a Pi review actually completing on this image, which is a separate claim from the binary being present. Both readings were guesses about an image that admission never inspected. It does now: the harness attestation guard described under Operator setup reads `pi-tarball-sha256` and `node-tarball-sha256` off the configured image before any model VM exists, so the next time this question is asked the lane answers it from the image rather than from a document, and a wrong `model_image_id` is refused for free instead of discovered on a paid VM. diff --git a/docs/azure-requirements.md b/docs/azure-requirements.md index eca5c2e6b52..39de9f98bbe 100644 --- a/docs/azure-requirements.md +++ b/docs/azure-requirements.md @@ -254,24 +254,150 @@ Acceptance: concurrent crewmates run on distinct pi profiles with no account col ## R6. Crosscheck reviews outside the author's model family -Status: BUILT 2026-08-20; no GLM review has ever completed. The lane below was built and merged -(#264) and the deployment is live, but the primary reviewer is 0 for 6. Six GLM attempts are -recorded against PR #220 in the crosscheck ledger on 2026-08-20, and all six ended in -`tool-failure` with no citations and no execution proof; a seventh attempt, against PR #266, sits -in the archived ledger beside it and also failed. GLM has produced no verdict at all. The one -verdict this lane has produced came from the pi-codex fallback reviewer. - -The roster was GLM-only when those attempts ran. As of 2026-08-20 it is not: the operator restored -the pi-codex fallback entries alongside GLM with `config/crosscheck-same-model` on, the sanctioned -degraded mode while GLM cannot finish a review, so crosscheck can return verdicts again. It returns -them from the fallback, which means codex-authored work is being reviewed by its own family again -for the duration, exactly the degradation this requirement exists to remove. - -The deployment carries two per-minute limits, not one: 25,000 tokens and 25 requests (`FW-GLM-5.2`, -DataZoneStandard, capacity 25). The 429 body names neither, reading only that requests to -`FW-GLM-5.2` in eastus have "exceeded rate limit". Attributing the blocker to the token-per-minute -limit specifically is therefore inference rather than measurement. Raising either is an owner -action in the Foundry portal; the Microsoft.Quota API does not cover Cognitive Services. +Status: NOT DONE. **No cross-family review has ever completed, and none completed for this change +either.** The acceptance is two completed reviews, not correct wiring: a codex-authored change AND +a claude-authored change each reviewed by a GLM-backed reviewer, with bound reviewer identity and +the same evidence discipline as the codex lane. Zero of those two exist. What landed is the lane +being executable at all, which it previously was not, plus the fixes below. Read "Where the lane +actually stands" before treating any of it as finished. + +Context: see "The Azure Foundry Fireworks lane is unusable on this subscription" and "The lane is a +named registry, now serving GLM-5.2 direct from Fireworks" below, both 2026-08-20. GLM never +completed a review through Azure and on this subscription never can; it now serves through a direct +Fireworks account instead. + +The earlier record stands as history: the lane was built and merged (#264), the `FW-GLM-5.2` +deployment is live, and the primary reviewer went 0 for 6 against PR #220 plus a seventh failed +attempt against PR #266, all `tool-failure` with no citations and no execution proof. The one +verdict this requirement's lane had produced before today came from the pi-codex fallback +reviewer, which is same-family review for codex-authored work and exactly the degradation this +requirement exists to remove. + +The quota reading in the earlier draft was wrong about the cause. The deployment does carry two +per-minute limits (25,000 tokens and 25 requests, `FW-GLM-5.2`, DataZoneStandard, capacity 25) and +one attempt did record a 429, but neither limit is what stops this lane. The measured cause is +below. + +### The Azure Foundry Fireworks lane is unusable on this subscription (2026-08-20, root-caused) + +This is the durable finding. It is not a quota, a route, a region, a credential, or a vendor +problem. It is Azure Marketplace billing. + +Measured, not inferred: + +- Every request to the Foundry deployment `FW-GLM-5.2` returns HTTP 500 + `invalid_model_endpoint_authentication` ("Failed to authenticate to backend endpoint") in roughly + 0.2 seconds, at every input size. +- The credential is fine. A bad key returns 401. These 500s carry fully computed, DECREMENTING + `x-ratelimit-*` headers, which only happens after the caller is authenticated and metered. The + failure is one hop past us, Foundry to Fireworks. +- Not a route problem: `services.ai.azure.com/models/chat/completions` and + `cognitiveservices.azure.com/openai/v1/chat/completions` fail identically, with both Bearer and + api-key auth. +- Not region- or resource-specific: a brand-new AIServices account in eastus2 with a fresh + deployment failed identically. +- The clean discriminator is the PUBLISHER, not the vendor's models. On the same account, same key, + same subscription, `DeepSeek-V4-Pro` (publisher DeepSeek) and `Kimi-K2.7-Code` (publisher + MoonshotAI) both returned HTTP 200 on the same endpoint. Every `FW-*` deployment is published by + Fireworks AI through Marketplace and fails. + +The mechanism, with a citable source. Microsoft Learn, "Foundry Models from partners and +community" +(https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-from-partners) +states that Student, Visual Studio Enterprise and Free-credit subscriptions cannot purchase +software-as-a-service offers in Marketplace, and lists as unsupported both subscriptions without an +active pay-as-you-go billing method (student, free trial, startup credit-based) and sponsored +subscriptions that only use Azure credits. This subscription is named "Microsoft Azure +Sponsorship". Partner models require Azure Marketplace, meaning a `Microsoft.SaaS` resource plus an +accepted `Microsoft.MarketplaceOrdering` agreement; BOTH were verified to be ZERO on this +subscription even after the owner deployed `FW-GLM-5.2` through the Foundry portal. The deployment +exists, but the marketplace linkage to the publisher backend can never be established, which is +what `invalid_model_endpoint_authentication` reports. + +GLM-5.2 exists only in the Fireworks flavor in the Foundry catalog, which is why GLM specifically +was unreachable there, and why `FW-Kimi-K2.7-Code` would have failed for the same reason while +`Kimi-K2.7-Code` worked. + +The two remedies were owner-owned: put a payment method on the subscription (the doc notes a card +on file is charged instead of credits), or reach the model without Azure Marketplace. The owner +took the second. This section exists so nobody re-litigates the Azure partner lane in three weeks. + +### The lane is a named registry, now serving GLM-5.2 direct from Fireworks (2026-08-20) + +The owner opened a direct Fireworks account, which bypasses Azure Marketplace entirely, so GLM-5.2 +is the reviewer again on its merits rather than on what Azure would serve. + +`bin/fm-crosscheck.py` carries `CROSS_FAMILY_LANES`, a code-side registry of vetted reviewer lanes. +Each entry is a complete endpoint allowlist entry: deployment/model id, Pi provider slot, +chat-completions api surface, endpoint host, the one accepted base URL, and the exact model-level +`compat` the credential may carry. The roster (`$FM_HOME/config/crosscheck-reviewer.json`) selects +the serving lane by naming the model, so substituting among registered lanes is a config change. +Admitting a NEW endpoint stays a reviewed code change on purpose: the allowlist is the security +control, and a credential file must never be able to introduce an endpoint the policy never named. + +The registered lane: + +| field | value | +|---|---| +| provider slot | `fireworks-glm` | +| model | `accounts/fireworks/models/glm-5p2` | +| endpoint | `https://api.fireworks.ai/inference/v1` (chat completions only) | +| api | `openai-completions` | +| pinned model-level compat | none | +| declared cost | input 1.45, output 4.69 per MILLION tokens | +| live check 2026-08-20 | HTTP 200, `finish_reason: stop`, streaming 1.56s with usage | + +The PINNED model id, not the `accounts/fireworks/routers/glm-5p2-fast` router. The router is +faster, but a router may re-point to a different serving variant, and the reviewer identity this +gate records has to name an exact model. An unattributable reviewer is worth more than a few +seconds. Latency is not the constraint on a multi-minute review. + +Reviewer identity binds the provider slot, the pinned host, and the model +(`fireworks-glm:api.fireworks.ai/accounts/fireworks/models/glm-5p2`), and the recorded credential +identifier is a digest of host + model + endpoint. Neither contains nor is derived from the api +key: two credentials differing only in `apiKey` produce byte-identical identifiers. + +The Azure `azure-glm` slot, the `FW-GLM-5.2` deployment, and the two Azure R6 attempt deployments +(`Kimi-K2.7-Code`, `DeepSeek-V4-Pro`) are retired. Nothing in the code or config points at them. +The legacy `glm-primary` ledger provenance value stays readable for durable records already +carrying it, bound to exactly that retired model, which is no longer registered and so can never be +claimed by a new run. + +`config/crosscheck-same-model` is `off`. That relaxation existed only because no cross-family lane +could finish a review, so the codex fallback had to be allowed to review codex-authored work. With +a working non-OpenAI primary, leaving it on would mean a transient provider hiccup silently drops +back to same-family review, which is the exact defect this requirement removes. With it off, the +fallback still serves any author outside the codex family, and a codex-authored PR whose primary is +down FAILS CLOSED rather than being self-reviewed. That single-primary risk is accepted +deliberately; flipping the relaxation back on is an operator act, not a silent degradation. + +Reviewer independence is now a FAMILY comparison, not an exact model-id one. The first completed +cross-family review of this work found that a `gpt-5.5` author admitted a `gpt-5.6-sol` codex +reviewer with no same-model marker recorded, which is same-family review of exactly the kind this +requirement exists to prevent (crosscheck finding cc-4dcd7873f71a, reproduced 2026-08-21). A +registered lane is its own family; `gpt-*`/`o1-`/`o3-`/`o4-`/`codex-*` are one OpenAI family and +`claude-*` one Anthropic family; an unrecognized model stays its own family, so nothing that +previously passed starts failing while everything recognized is strictly tightened. + +**Truncation is a failed review, not a verdict.** GLM-5.2 is a reasoning model and will spend the +output budget on reasoning first: measured live at `max_tokens=600`, the response came back +`finish_reason: length` with EMPTY visible content, while 4000 completed cleanly. The lane +therefore declares `maxTokens` 32000, and a truncated turn is refused on two independent grounds +that both already existed: the Pi stream parser refuses any final assistant turn whose `stopReason` +is not `stop` (pi maps `finish_reason: length` to `stopReason: "length"`), and the verdict must +then parse as JSON against the review schema, which truncated output cannot. Both are pinned by +tests, including an end-to-end case that emits a COMPLETE, schema-valid clear verdict with only the +stop reason set to `length` and requires the run to record `tool-failure` with no citations. + +**The USD budget control is inert, and R10 must stop claiming it.** The declared cost is now real +(1.45 / 4.69 per million; pi's unit convention is per million, confirmed in its own source at +`@earendil-works/pi-ai/dist/models.js`, `usage.cost.input = (rates.input / 1000000) * usage.input`, +so the owner-set values are correct as written). But `bin/fm-crosscheck.py` records no token usage +at all: there is no `prompt_tokens`/`completion_tokens` handling anywhere in it, and the ledger +reviewer record carries no usage or cost field. A `daily_budget_usd` control therefore has nothing +to meter regardless of what the cost field says. Fixing that is not in this change; C3's daily +Cost-Management bound remains the only guard, with its own caveat that it is a backstop on recorded +spend rather than a real-time meter. The Work list below is retained as the record of what was asked for; the state of each item is in "What landed" or "Still owed" below. @@ -312,6 +438,9 @@ author fleet consume. The pi-codex roster (which R5 records as proven at the ros R9 still owes the live proof) is retained as a dormant fallback behind a config flip, never deleted; every review must name the lane that produced it, and a status read must show whether GLM is serving or the fallback is active, so a silent fallback is impossible. +(Correction, 2026-08-21: "behind a config flip" held for the LOCAL fallback only. For the +Azure-compartment lane it was false - the codex-family path was additionally broken in code and no +flip would have restored it. See the compartment bullet under "Still owed".) Fallback operation is a recorded degradation, not free service restoration: with the fallback active, codex-authored work is reviewed by its own family again (the flip therefore includes `config/crosscheck-same-model` on for the duration, which the policy screen otherwise refuses), @@ -382,39 +511,205 @@ What landed, 2026-08-20 (#264, plus #268 for a defect the live runs exposed): afterwards; the operator has since restored the fallback entries again, as the status above records. This run is still the only verdict this requirement's lane has produced. +### The acceptance sentence is not evidenceable as written, and is amended (2026-08-21) + +R6's acceptance says "a codex-authored change AND a claude-authored change are each reviewed by a +GLM-backed reviewer". **The system cannot evidence that sentence, and this must be said before any +run is recorded against it.** + +What the code actually does, traced rather than assumed: + +- Eligibility derives from `model` in the task meta and NEVER from `harness` + (`bin/fm-crosscheck.py`, `model_family(meta["model"])`). `parse_meta` requires `harness` to be + present and non-empty, then nothing in reviewer selection reads it. +- The ledger records **no author identity at all**. The one call site passes + `author_account_identity=""`, with an inline comment saying task metadata carries no upstream + authorship account record. +- Git carries no harness signal either, by design: the no-self-attribution rule means no trailer + ever distinguishes a claude crewmate from a codex one. + +So the task meta is the only authorship input, and a task meta is a DECLARATION, not a record. +Anyone can write `harness=claude` over any change. "A claude-authored change" is therefore not a +checkable property of any artifact this system produces. + +**Amendment, and the reading to use.** The acceptance becomes what the system can actually +evidence, which is also what the 2026-08-19 amendment's own reasoning needs ("one reviewer family +outside both author families satisfies the paradigm for EVERY author"): + +> The cross-family lane completes a review end to end, and the family screen admits that reviewer +> against BOTH a codex-model author and a non-codex-model author, with the ledger recording the +> reviewer model, the review family mode, and whether the `crosscheck-same-model` relaxation was +> required. + +Every clause there is checkable from the ledger. Nothing in it claims to know who wrote the code. +This is deliberately weaker than the original sentence, and it is weaker in the only direction +available: the original was never provable, so leaving it in place would have meant marking R6 DONE +on an assertion. + +Making authorship genuinely recordable is the alternative, and it is a real change rather than a +doc edit: it needs an authorship identity captured at task creation and carried into the ledger, +which is the same `author_account_identity` field the Azure adapter's same-account refusal is +already waiting on. Worth doing, out of scope here, and the amendment above does not depend on it. + +**Declaring a codex author is the SAFE error; declaring a claude author is the dangerous one.** A +codex declaration can only narrow eligibility: against a codex-family reviewer it forces the +`crosscheck-same-model` relaxation to appear in the record, and with the relaxation off it fails +closed. A claude declaration widens eligibility silently and makes the run print "relaxation was +not required" and record clean cross-family separation that may never have happened. When the real +author is unknown, declare codex and take the louder record. + +**Correction to this program's own artifacts.** The two runs in +`$FM_HOME/data/crossfamily-r6-281/` were driven by a hand-written two-line meta asserting +`harness=claude` / `model=claude-opus-5` over a PR whose real author harness is not recorded +anywhere. Those runs are real reviews and their findings were real and acted on, but their +authorship claim is a declaration, and **they must not be counted toward the acceptance above**. +Later probe runs declare a codex author instead, per the asymmetry. + +### Where the lane actually stands, 2026-08-21 + +Stated plainly, because wiring being right is not the acceptance. + +**Has a cross-family review run end to end and produced a verdict? NO.** Not once, on any lane, +ever. Three attempts against PR #281 on the direct Fireworks lane, each failing FURTHER along than +the last: + +| # | outcome | +|---|---| +| 1 | GLM produced a COMPLETE turn; final text was not bare JSON, refused as `malformed verdict artifact`. Fell through to the codex fallback, which reached a real blocking verdict | +| 2 | `Pi reviewer: bounded command timed out after 1800 seconds` | +| 3 | GLM produced a real verdict naming the exact head and the exact account home, in BARE JSON, refused as `Unterminated string starting at: line 1 column 3045` | +| 4 | GLM completed a full 424s review and produced a COMPLETE, SCHEMA-VALID verdict. Refused `UNREVIEWED` because the reviewer's own `executed_reproduction.command` did not name both SHAs | +| 5 | Identical to 4, same refusal. So attempt 4 was not a lucky run and this failure is not variance | + +Attempt 1 was never a hang: the model went through pi, completed a turn, and produced text, and was +rejected at the LAST step on output SHAPE. That fact also disproved the suspect this document +previously named. Measured since: + +- **`reasoning_content` in streamed deltas is NOT the problem, and that Work item is answered.** + Streaming the review-shaped prompt reaches `[DONE]` in 21-24s carrying 390 and 867 reasoning + deltas, at default and `high` reasoning effort, with a maximum inter-chunk gap of 3.4s. A large + reasoning stream streams fine. +- **The output shape was a Markdown fence, confirmed byte for byte.** Asked for this gate's exact + review instruction and schema, GLM returns ` ```json\n{...}\n``` `, and a bare parse of that fails + with `Expecting value: line 1 column 1 (char 0)` - precisely attempt 1's recorded error. The + extractor now unwraps exactly one complete fenced block, and attempt 3 duly got past it. +- **The remaining failure is a TRUNCATED verdict carrying a SUCCESSFUL stop reason.** Attempt 3's + JSON breaks mid-string at char 3044 while pi reported `stopReason: stop`, so the stop-reason + guard never fired and only the JSON parse caught it. That is the strongest possible argument for + keeping BOTH refusal grounds rather than treating either as redundant, and both stay pinned by + tests. +- **It is not the token cap.** Both `max_completion_tokens` (what pi sends for this provider) and + `max_tokens` are honored at 20000 and both return complete, parseable verdicts of 3212 and 3775 + completion tokens; the lane declares `maxTokens` 32000. + +Attempt 4 is the milestone: **the lane executes a complete review and produces a schema-valid +verdict.** The ledger recorded a genuine cross-family reviewer record against a codex-model author +- `model: accounts/fireworks/models/glm-5p2`, `review_family_mode: cross-family-primary`, NO +`model_independence` marker (so clean family separation with no relaxation required), +`credential_source: pi-fireworks-glm-models-file`, and the non-secret +`credential_identifier: provider-binding:fireworks-glm:d2a164ff...`, over 424s of reviewer time. +Under the amended acceptance above, the family-screen clause is now EVIDENCED for a codex-model +author. + +What remains is NOT transport, shape, or policy wiring. It is the reviewer obeying the review's own +evidence discipline: the gate refused the verdict because the model's `executed_reproduction` +command did not name both the base and head SHAs, which the prompt requires. That is the gate +working exactly as designed, and it must NOT be relaxed to get a green verdict - a lane that earns +its first verdict by lowering the evidence bar would be worth less than no lane. Attempt 5 repeated attempt 4 exactly, so this is +REPRODUCED behavior rather than a flaky run: GLM-5.2 reliably omits the SHAs from its reproduction +command. The remaining risk is therefore instruction compliance by this model on the evidence +contract, plus the intermittent truncation attempt 3 showed, and neither is a reason to weaken a +refusal. + +The obvious next step is to strengthen the reproduction-command INSTRUCTION rather than the check, +which is legitimate prompt work and not a relaxation. It is deliberately NOT done here: that prompt +is shared with the codex lane, which currently satisfies the clause, and changing a contract that +every merge depends on to accommodate one model is a decision to take deliberately rather than at +the end of a long session. + +**What #281 closes:** + +- The lane is executable at all. `azure-glm` / `FW-GLM-5.2` on main are dead references: the Foundry + account `aif-fm7c799d-eus01` has ZERO deployments as of 2026-08-21, so the pre-existing lane could + not have served a review under any circumstances. +- Two reproduced high-severity policy bypasses, both found by a real completed review of the branch + (codex-family fallback lane) and both fixed with tests: provider-qualified authors bypassing + family separation (cc-4dcd7873f71a and cc-5ec330d3c74d) and the model-level `compat` pin missing + the provider and `modelOverrides` layers pi also composes (cc-ca5848b19ac3). +- Operator documentation that told captains to provision an `openai-codex` `auth.json` for every Pi + reviewer, which misprovisions a cross-family lane home (cc-769d7eba2ded). +- The startup-credit item above, retired as moot. + +**What #281 does NOT close, and must not be read as closing:** + +- The acceptance itself. Zero completed cross-family reviews; the requirement needs two, over two + different author families. +- The status command answering whether the cross-family lane is serving or the fallback is active. + It still does not exist; `bin/fm-crosscheck.py` exposes `run`, `verify`, `merge` and `timings`. + This one is substantive rather than cosmetic, and it bites right now: the fallback IS active, and + the only thing that says so is a stderr line at run time. +- Review guards sized to the model's context window. Two of the three the Work list names DO + exist and are stronger than asked: the findings schema is strict (`additionalProperties: false`, + enum'd severities, `maxItems` caps), and citations are validated before filing by escape check, + `git ls-files --error-unmatch` tracked-at-head check, and a line-in-range check. The third, a + per-review context cap actually SIZED to the serving model, does not exist: + `MAX_LEDGER_PROMPT_BYTES` (64,000) and `MAX_PROJECTED_FINDINGS` (512) are fixed constants, and + nothing reads the lane's declared `contextWindow`. This change does not re-size them. +- A per-review spend meter. The provider slot declares real per-million costs (input 1.45, output + 4.69) instead of the old slot's zeros, but that number lives in pi's own config and this change + does NOT make the crosscheck ledger's cost non-zero, because the ledger records no token usage at + all. R10's `daily_budget_usd` therefore still has nothing to bind to. + Still owed, and honestly so: -- The live end-to-end GLM review. The six attempts did not all die of the quota, and the record - should not be read as if they did. Attempts 0 and 1 (05:54Z) died before reaching the provider - at all, on a local harness fault in an operator-authored instrumentation shim whose `/dev/fd` - redirect was refused under the reviewer sandbox. Attempts 2, 3 and 4 (05:55Z to 05:59Z) each - recorded only `Pi reviewer emitted a turn after agent completion`, the parser defect #268 then - fixed (pi continues a retried attempt after `agent_end` via `auto_retry_start`, and the stream - parser read that continuation as a turn after completion, masking whatever the provider had - actually returned). Because the parser masked it, no retained artifact records what killed those - three. Attempt 6 (06:25Z), two minutes after #268 landed, is the only review run anywhere that - records an actual 429. The remaining ledger slot, index 5 at 06:08Z, is not a GLM run at all: it - is the pi-codex fallback demonstration. What is measured rather than inferred is the traffic - shape: in the deployment's metrics the 06:00Z hour shows 35 model requests and 23 client errors. - One real GLM review end to end closes this item, and the quota is the leading suspect for what - stands in the way, not an established cause. -- The startup-credit decrement check. Deployment metrics for `aif-fm7c799d-eus01` record 727,136 - tokens on 2026-08-20: 515,965 in the 04:00Z hour, 135,911 in 05:00Z, 75,260 in 06:00Z. (An - earlier draft of this section reported roughly 510K for the day; that was the 04:00Z hour alone.) - Cost Management shows no charge against the resource yet. That absence carries no information - either way at this range: C3 records that Cost Management actual lags hours, which is why its - own bound is a backstop on recorded spend. This needs the portal's cost view. -- The Azure-compartment GLM lane, which is switched off rather than unbuildable. The serving lane +- CLOSED as far as GLM is concerned, and the earlier reading of it was wrong. This item asked for + a live end-to-end GLM review; no such review is obtainable on this subscription, for the + Marketplace reason root-caused above. The historical detail stands: attempts 0 and 1 (05:54Z) + died before reaching the provider at all, on a local harness fault in an operator-authored + instrumentation shim whose `/dev/fd` redirect was refused under the reviewer sandbox; attempts + 2, 3 and 4 (05:55Z to 05:59Z) each recorded only `Pi reviewer emitted a turn after agent + completion`, the parser defect #268 then fixed, which masked whatever the provider actually + returned; attempt 6 (06:25Z) is the only review run anywhere that records an actual 429; ledger + slot 5 at 06:08Z is the pi-codex fallback demonstration, not a GLM run. The quota was named as + the leading suspect. It was not the cause. The acceptance now rests on a completed review from a + registered, reachable cross-family lane instead. +- The startup-credit decrement check is MOOT, retired rather than left standing. It asked for a + small live spend confirming the charge decrements Azure startup credit. That premise died with + the lane move. Fireworks pay-per-token ON FOUNDRY billed as Azure consumption; Fireworks DIRECT + bills a Fireworks account and touches no Azure credit at all, so no charge for this lane can ever + appear in Azure Cost Management. Nobody should go hunting for one. The historical Azure numbers + are kept only as a record of what the dead lane consumed: deployment metrics for + `aif-fm7c799d-eus01` recorded 727,136 tokens on 2026-08-20 (515,965 in the 04:00Z hour, 135,911 + in 05:00Z, 75,260 in 06:00Z; an earlier draft reported roughly 510K for the day, which was the + 04:00Z hour alone). That resource now has ZERO deployments, verified 2026-08-21, so it can serve + nothing. If a spend signal is still wanted it is a Fireworks-side number, and the meter it would + need does not exist here either: see the spend bullet below. +- The Azure-compartment lane. "Switched off rather than unbuildable" was TOO KIND, and this is the + second time this requirement has had to retract a merely-disabled claim without anyone reaching + the code (the first was the stale `pi`-binary reason). `enabled: false` was MASKING an + independent in-code blocker: the compartment archive gate derived the executing-account identity + separately from `account_identity` and compared a bare account id against a prefixed one, so the + codex-family path refused ITSELF and no codex-family compartment review has ever run. Fixed + 2026-08-21 with a single shared derivation and a test that is red on the old code; the switch is + still off, so the lane still does not run. Recorded because the first fix was itself incomplete: + it unified the two HOST derivations while the model guest kept a third, moving the refusal from + staging into a booted, paid VM. The guest is covered by an executing test now rather than + substring assertions, which is why that was invisible. The serving lane today is the local pi reviewer. The reason recorded here previously, that the `fm-ccm` image carries no `pi` binary and needs a rebake, is stale and is corrected below. -- A spend signal for the new primary reviewer. The GLM provider entry declares `cost` as zeros for - `input`, `output`, `cacheRead` and `cacheWrite` (`tests/fm-crosscheck.test.sh:1046`), so a GLM - review prices at zero and the crosscheck ledger records no per-review cost for it. That is the - same ledger R10's `daily_budget_usd` waits on, and the reason it does not bind today. It leaves - C3's daily bound as the only guard over this lane's spend, and C3's own caveat is that the bound - is a backstop on Cost-Management-recorded spend rather than a real-time meter. +- A spend signal for the primary reviewer, HALF closed and honestly so. The cost declaration is no + longer fake: the Fireworks lane declares input 1.45 and output 4.69 per million, and pi's unit + convention is per million (`@earendil-works/pi-ai/dist/models.js`: + `usage.cost.input = (rates.input / 1000000) * usage.input`), so those values are correct as + written. What is still missing is the meter. `bin/fm-crosscheck.py` records no token usage at + all - no `prompt_tokens`/`completion_tokens` handling anywhere in it, and no usage or cost field + on the ledger reviewer record - so R10's `daily_budget_usd` has nothing to bind to whatever the + cost field says. R10 must stop describing that as a control it has. C3's daily bound remains the + only guard over this lane's spend, and C3's own caveat is that the bound is a backstop on + Cost-Management-recorded spend rather than a real-time meter. - Three items from the Work list above that neither landed nor were separately tracked. The status - command that answers whether GLM is serving or the fallback is active does not exist: + command that answers whether a cross-family lane is serving or the fallback is active does not + exist: `bin/fm-crosscheck.py` exposes `run`, `verify` and `merge` and nothing else. This one is substantive rather than cosmetic, because the stated reason for it was that a silent fallback must be impossible, and the fallback is active right now. Second, that pi tolerates diff --git a/docs/configuration.md b/docs/configuration.md index 13517cae603..7c7096861da 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -106,18 +106,18 @@ The gate does not reproduce CI's concurrent shard fan-out because it does not se `config/crosscheck-reviewer.json` selects the local dedicated account pool for the PR merge-gate reviewer. It is gitignored and is not inferred from lane metadata or ambient provider configuration. The current schema has one nonempty `reviewers` array, whose entries require exactly `harness`, `model`, `effort`, and `account_home`. -The accepted policy profiles are Pi `FW-GLM-5.2` at `xhigh` effort (the sole primary review family, R6) plus Codex `gpt-5.6-sol` and Pi `gpt-5.6-sol` at `xhigh` effort as the loud degraded fallback family; Claude is never an eligible Crosscheck reviewer (the interim claude lane is retired). +The accepted policy profiles are Pi at `xhigh` effort on every registered cross-family model (today `accounts/fireworks/models/glm-5p2`; the primary review family, R6) plus Codex `gpt-5.6-sol` and Pi `gpt-5.6-sol` at `xhigh` effort as the loud degraded fallback family; Claude is never an eligible Crosscheck reviewer (the interim claude lane is retired). Every reviewer `account_home` must be an existing absolute directory. [`crosscheck.md`](crosscheck.md) owns the reviewer-independence policy, including the removal of author-identity checks and the legacy admission workaround. The optional local, gitignored `config/crosscheck-same-model` file contains exactly `on` or `off`, defaults to `off` when absent, and is read fresh for every reviewer selection. -`on` relaxes only the default model-separation screen and makes the weaker mode explicit in the prompt and durable evidence. +`on` relaxes only the default model-FAMILY separation screen and makes the weaker mode explicit in the prompt and durable evidence. Independence is a family comparison: `gpt-5.5` and `gpt-5.6-sol` are one family, so a version bump never buys independence. Invalid values and unsafe file shapes fail closed. Crosscheck runs eligible entries in configured order, advancing to the next only when a reviewer could not reach its provider. List more than one entry per supported client so a single unavailable account cannot block the gate. Codex binds both `CODEX_HOME` and `HOME` to the selected reviewer path and sets `project_doc_max_bytes=0` so reviewed-repository `AGENTS.md` files cannot supply reviewer instructions. -Pi creates a disposable private `HOME` whose `.pi/agent` resolves to the selected reviewer path, binds `PI_CODING_AGENT_DIR` to it, requires an `openai-codex` OAuth credential in its `auth.json` before launch, resolves an npm-installed Pi entrypoint with its sibling Node runtime before reviewer `PATH` can substitute another interpreter, and uses `--no-context-files` so reviewed-repository context files cannot supply reviewer instructions. +Pi creates a disposable private `HOME` whose `.pi/agent` resolves to the selected reviewer path, binds `PI_CODING_AGENT_DIR` to it, requires the credential its lane calls for before launch (an `openai-codex` OAuth entry in `auth.json` for the codex-family fallback, or an api-key `models.json` declaring exactly that lane's provider slot for a cross-family lane), resolves an npm-installed Pi entrypoint with its sibling Node runtime before reviewer `PATH` can substitute another interpreter, and uses `--no-context-files` so reviewed-repository context files cannot supply reviewer instructions. The verdict and its Bash-created receipt must report the executing reviewer selector and private `HOME`. An absent or invalid reviewer file blocks Crosscheck and merge. See [`crosscheck.md`](crosscheck.md) for the example file, reviewer capture control, evidence rules, and operator flow. diff --git a/docs/crosscheck-slack.md b/docs/crosscheck-slack.md index 7b6878421c8..0978b9e4a8c 100644 --- a/docs/crosscheck-slack.md +++ b/docs/crosscheck-slack.md @@ -14,7 +14,7 @@ GitHub pull-request link. The bot validates the repository against the allowlist, checks the submitter's daily meter, acks in thread ("Review started"), runs `bin/fm-crosscheck.sh run ` as a bounded subprocess, and posts the findings as a thread reply on the engineer's own -message, naming the lane that produced the review ("GLM-5.2 primary" or +message, naming the lane that produced the review (the cross-family primary deployment or "pi-codex fallback (degraded)"). Tool failures are posted honestly as failures, never as verdicts. Cursor Bugbot continues to run for engineers' pull requests; this lane complements it. diff --git a/docs/crosscheck.md b/docs/crosscheck.md index c477ac6d8b2..fcda7772260 100644 --- a/docs/crosscheck.md +++ b/docs/crosscheck.md @@ -21,9 +21,9 @@ The file is local and gitignored at `config/crosscheck-reviewer.json`. "reviewers": [ { "harness": "pi", - "model": "FW-GLM-5.2", + "model": "accounts/fireworks/models/glm-5p2", "effort": "xhigh", - "account_home": "/absolute/path/to/the/glm/pi/agent/home" + "account_home": "/absolute/path/to/the/cross-family/pi/agent/home" }, { "harness": "codex", @@ -36,10 +36,15 @@ The file is local and gitignored at `config/crosscheck-reviewer.json`. ``` Crosscheck resolves configured reviewer homes in order and keeps entries that satisfy its reviewer-profile and model policies. -GLM-5.2 is the sole primary review family (R6, docs/azure-requirements.md): the Pi `FW-GLM-5.2` profile runs on the fleet's own Azure AI Foundry deployment through the `azure-glm` custom provider, whose credential is an api-key `models.json` in a dedicated Pi agent dir - never a codex `auth.json`. -The GLM endpoint is an allowlist of exactly `https://aif-fm7c799d-eus01.cognitiveservices.azure.com/openai/v1` (chat completions only; any other baseUrl, including a Responses API surface, is refused by name), and the recorded reviewer identity binds the Foundry resource + deployment - never the api key or anything derived from it. +A registered cross-family lane is the primary review family (R6, docs/azure-requirements.md). `bin/fm-crosscheck.py` carries `CROSS_FAMILY_LANES`, a code-side registry of vetted reviewer lanes; each entry pins a model id, a Pi provider slot, the chat-completions api surface, the endpoint host, the one accepted base URL, and the exact model-level `compat` the credential may carry. Today's registry is the single lane `fireworks-glm` (`accounts/fireworks/models/glm-5p2`, GLM-5.2 direct from Fireworks). The roster picks the serving lane by naming the model, so substituting among registered lanes is a config change; admitting a new endpoint is a reviewed code change, because the allowlist is the control. +Each lane's credential is an api-key `models.json` in a dedicated Pi agent dir declaring exactly that lane's provider slot - never a codex `auth.json`, and never a second provider. +Each lane's endpoint is an allowlist of exactly its registered base URL, today `https://api.fireworks.ai/inference/v1` (chat completions only; any other baseUrl, including a Responses API surface, is refused by name), and the recorded reviewer identity binds the provider slot + host + model - never the api key or anything derived from it. +`compat` is the other model-level object pi honors, and some of its keys weaken this gate's own defenses (`supportsFinishReason: false` would blunt the truncated-verdict refusal), so the lane owns it exactly: the registered lane declares no compat, and a credential carrying any is refused. +A truncated reviewer turn is a FAILED review, never a verdict. The stream parser refuses a final assistant turn whose `stopReason` is not `stop` (pi maps `finish_reason: length` to `stopReason: "length"`). The claim that a truncated verdict "cannot parse either" was FALSE and is corrected: a truncated verdict contributes no complete fenced block, so any complete fence earlier in the same message left the count at one and the gate certified THAT block instead, with `stopReason: stop` and a successful parse. The verdict extractor therefore refuses outright when any fence is left unterminated, and unwraps a single fenced block only when it is the ONLY JSON-bearing content in the message. +Both rules are load-bearing and neither is redundant: the unterminated-fence rule alone catches a verdict truncated immediately after its opening fence (no braces to detect), and the brace rule alone catches a complete example fence followed by a truncated bare verdict (an even marker count). +Two availability costs are accepted deliberately, and both FAIL CLOSED as a `tool-failure` that rotates the roster rather than as a wrong verdict: a legitimate verdict whose own string content contains a triple backtick refuses on the marker count, and a verdict accompanied by brace-bearing prose refuses on the remainder check. In a repository whose docs and tests are full of fences that is a real cost, paid to make a superseded block impossible to certify. Because pi's provider composer gives model-level `baseUrl`/`api` fields precedence over the provider level, the allowlist also refuses any model entry carrying either field - the pinned provider level must own both, even when an override repeats the pinned values. -The pi-codex/codex `gpt-5.6-sol` profiles remain only as the dormant fallback family: a run they serve prints a loud `CROSSCHECK DEGRADED` warning naming whether the `crosscheck-same-model` relaxation was required, and its ledger reviewer record carries `review_family_mode: codex-fallback` (GLM runs record `glm-primary`) with the readable report labeling the run `CODEX FALLBACK`. +The pi-codex/codex `gpt-5.6-sol` profiles remain only as the dormant fallback family: a run they serve prints a loud `CROSSCHECK DEGRADED` warning naming whether the `crosscheck-same-model` relaxation was required, and its ledger reviewer record carries `review_family_mode: codex-fallback` (cross-family runs record `cross-family-primary`; durable ledgers written before the registry landed carry the legacy `glm-primary`, which stays bound to exactly that lane's model) with the readable report labeling the run `CODEX FALLBACK`. Firstmate authors also run on codex-family models, so the fallback usually needs that recorded same-model degraded state. Claude is never an eligible Crosscheck reviewer; the interim claude lane is retired, and a claude profile is refused with the exact-profile message before any reviewer machinery runs. Model separation is mandatory by default. @@ -56,15 +61,15 @@ The former `config/crosscheck-legacy-author-admissions.json` path existed only t Crosscheck then binds the provider's executing credential selector to that exact reviewer path and requires the verdict plus a Bash-created receipt to report the selector and actual private `HOME`. That proves which dedicated reviewer home executed the review without comparing it to an author account. Every reviewer disables reviewed-repository instruction discovery at launch: Codex sets `project_doc_max_bytes=0`, and Pi uses `--no-context-files`. -Pi is launched through the resolved installed executable at `xhigh` with JSON event output, an ephemeral session, and only the read and Bash-capable review tools; the model decides the provider slot through an explicit mapping (`FW-GLM-5.2` on `azure-glm`, `gpt-5.6-sol` on `openai-codex`) that refuses an unmapped model rather than guessing. +Pi is launched through the resolved installed executable at `xhigh` with JSON event output, an ephemeral session, and only the read and Bash-capable review tools; the model decides the provider slot through an explicit mapping derived from the lane registry (each registered model on its own slot, `gpt-5.6-sol` on `openai-codex`) that refuses an unmapped model rather than guessing. For the installed npm entrypoint, Crosscheck also resolves Pi's sibling Node runtime before launch instead of allowing the reviewer environment's `PATH` to substitute another interpreter. That pin recognizes every `env`-based Node shebang, including `#!/usr/bin/env -S node --flag`, and preserves the flags; an `env` shebang naming no interpreter fails closed rather than silently falling back to `PATH`. Its event stream must contain at least one completed turn, end with a successful `stop` assistant turn, and complete the agent before Crosscheck accepts that terminal turn's JSON verdict. -Pi credential provisioning is a captain-owned prerequisite: the selected `account_home` must contain a usable `openai-codex` OAuth entry in `auth.json`, and Firstmate does not create or copy that credential. -Because Pi selects only the default `openai-codex` slot and reviewer launches disable extension discovery, a Pi reviewer home holds exactly one account; a multi-slot Pi home reviews as its default slot, not as whichever slot has capacity. +Pi credential provisioning is a captain-owned prerequisite, and its shape depends on the lane: a codex-family fallback home must contain a usable `openai-codex` OAuth entry in `auth.json`, while a cross-family lane home must instead contain an api-key `models.json` declaring exactly that lane's provider slot and no `auth.json` is required. Firstmate does not create or copy either credential. +Because reviewer launches disable extension discovery, a Pi reviewer home holds exactly one account and exactly one provider; a multi-provider Pi home is refused for a cross-family lane and reviews as its default slot for the codex fallback, not as whichever slot has capacity. -Pi is a third client, not extra capacity. -It authenticates against the same upstream OpenAI accounts the Codex reviewer uses, so a Codex account at its usage limit is equally unavailable through Pi, and a Pi reviewer does not route around an exhausted Codex account. +Pi on the codex-family fallback is a third client, not extra capacity. +On that lane it authenticates against the same upstream OpenAI accounts the Codex reviewer uses, so a Codex account at its usage limit is equally unavailable through Pi, and a Pi reviewer does not route around an exhausted Codex account. A cross-family lane is a different provider entirely and shares none of that capacity. What Pi adds is an independent client path and a reviewer that is separate from a Claude author by construction. A usage-limited reviewer account records a `tool-failure`, never a verdict about code, and Crosscheck then advances to the next independent entry rather than refusing the merge. Failover is limited to faults that prevented a verdict: a launch failure, an unusable credential, a provider that was never reached, or an exhausted account. @@ -75,7 +80,8 @@ Every candidate passed the configured reviewer-profile and model policy. Reviewer credential inspection still proves that the selected reviewer home can execute its configured client, but it makes no claim about the author. Model identity compares the model itself, not the recorded string: Pi records `/`, so `openai-codex-2/gpt-5.6-sol` is the same model as a Codex reviewer's plain `gpt-5.6-sol`. That canonical identity is screened out by default and is what marks a selected review as same-model when the explicit relaxation is on. -The accepted profiles are Pi `FW-GLM-5.2` xhigh (the primary family) plus Codex `gpt-5.6-sol` xhigh and Pi `gpt-5.6-sol` xhigh (the loud degraded fallback family). +The accepted profiles are Pi at xhigh on every registered cross-family model (today `accounts/fireworks/models/glm-5p2`) as the primary family, plus Codex `gpt-5.6-sol` xhigh and Pi `gpt-5.6-sol` xhigh as the loud degraded fallback family. +Reviewer independence is compared on the model FAMILY, not the exact id, so a `gpt-5.5` author is not admitted a `gpt-5.6-sol` reviewer (finding cc-4dcd7873f71a); an unrecognized model remains its own family. Absent reviewer configuration, unavailable reviewer credentials, or model-policy mismatch produces `CROSSCHECK TOOL-FAILURE` and a nonzero exit before reviewer launch. Crosscheck requires Python 3.11 or newer and refuses to run on anything older. @@ -300,7 +306,7 @@ PR claims are delimited as untrusted data, and the reviewer is directed to ignor Later reviewers receive only a bounded projection of finding IDs, lifecycle state, severity, exact-head clearance, and proof digests. Finding prose, reproduction output, test output, and lifecycle notes remain durable in the ledger but are never reinjected into a later reviewer prompt. The Codex path pins `gpt-5.6-sol`, xhigh reasoning, noninteractive approval, an independent `CODEX_HOME`, the same account-bound `HOME`, and the exact review checkout. -The Pi path pins the roster model on its mapped provider (`FW-GLM-5.2` on `azure-glm`, `gpt-5.6-sol` on `openai-codex`), xhigh reasoning, an independent `PI_CODING_AGENT_DIR`, a disposable private `HOME`, extension and context isolation, and JSON event output. +The Pi path pins the roster model on its mapped provider (each registered cross-family model on its own slot, `gpt-5.6-sol` on `openai-codex`), xhigh reasoning, an independent `PI_CODING_AGENT_DIR`, a disposable private `HOME`, extension and context isolation, and JSON event output. An unavailable reviewer binary, sandbox, reviewer credential binding, verdict-level execution proof, or exact remote PR head records a `tool-failure` attempt when the live head is already known, and otherwise emits the same tool-failure class without fabricating a ledger run. A ledger that cannot be read is the one stop that cannot record itself: appending a run to a file that failed to parse would risk destroying the durable findings it still holds, so the ledger is left exactly as it is and only the readable `crosscheck.md` report is rewritten, naming the parse failure so the cause is on disk rather than only in the exit status of a run nobody kept. A reviewer that never reached its provider is also a `tool-failure` rather than an `unreviewed` attempt, and is the case that fails over. @@ -364,7 +370,7 @@ Most of `tests/fm-crosscheck.test.sh` is hermetic coverage using observed-shape Its `test_installed_sandbox_denies_shared_private_tmp` case is the exception: it invokes the real installed `/usr/bin/sandbox-exec` and verifies the generated proof profile denies shared host temporary state. Its `test_pytest_runner_resolves_through_a_uv_aware_ladder` case is the named regression for runner-name resolution: it pins monorepo uv-project discovery, the skipped uv rung outside a project, the unchanged absent-runner refusal, and pytest's retained node-id support. Its `test_missing_author_identity_reaches_normal_verdict` case is the named regression for a Pi lane without a captured account identity: the review reaches an ordinary clear verdict without an identity warning or downgrade. -Its `test_claude_reviewer_profile_is_retired` case proves the standing rule that Claude is not an accepted reviewer profile and never launches, and its `test_glm_reviewer_executes_bound_policy_profile`, `test_glm_credential_binding_is_key_independent`, and `test_codex_fallback_family_is_loud_and_recorded` cases pin the GLM primary lane's provider mapping, key-independent endpoint binding, and the loud durable fallback marker. +Its `test_claude_reviewer_profile_is_retired` case proves the standing rule that Claude is not an accepted reviewer profile and never launches, and its `test_cross_family_reviewer_executes_bound_policy_profile`, `test_truncated_cross_family_verdict_is_never_a_verdict`, `test_cross_family_credential_binding_is_key_independent`, `test_cross_family_family_marker_is_bound_to_the_reviewer_model`, and `test_codex_fallback_family_is_loud_and_recorded` cases pin every registered lane's provider mapping, the refusal of a truncated verdict, key-independent endpoint binding, model-bound family provenance, and the loud durable fallback marker. Its `test_same_model_relaxation_does_not_require_author_identity` case proves the explicit model-policy relaxation does not revive an author-account precondition. `tests/fm-spawn-dispatch-profile.test.sh` separately proves a failed Pi identity capture remains nonfatal and the lane still launches. Its `test_typescript_jest_mutation_proof_can_clear` and `test_inadequate_typescript_jest_coverage_stays_blocking` cases prove that package-governed Jest coverage can certify a TypeScript fix while a named Jest test that stays green under mutation keeps the finding blocking. diff --git a/tests/fm-crosscheck-azure.test.sh b/tests/fm-crosscheck-azure.test.sh index c78874162ee..25293c59f32 100755 --- a/tests/fm-crosscheck-azure.test.sh +++ b/tests/fm-crosscheck-azure.test.sh @@ -84,20 +84,28 @@ for marker in ( guest_source = guest.read_text(encoding="utf-8") assert "--disable shell_tool" in guest_source assert "--no-tools" in guest_source -# R6: the model decides the Pi provider slot inside the guest too, the GLM -# credential is the models.json shape bound to the exact Foundry endpoint, -# and the interim claude launch/boot-copy lane is gone entirely. -assert "azure-glm" in guest_source -assert "FW-GLM-5.2" in guest_source +# R6: the model decides the Pi provider slot inside the guest too, every +# registered cross-family credential is the models.json shape bound to that +# lane's exact endpoint, and the interim claude launch/boot-copy lane is gone +# entirely. The guest is checked against the core registry rather than a +# hardcoded name, so a lane added in one place and not the other fails here. +import importlib.util + +core_spec = importlib.util.spec_from_file_location("fm_crosscheck_core", core) +core_module = importlib.util.module_from_spec(core_spec) +core_spec.loader.exec_module(core_module) +assert core_module.CROSS_FAMILY_LANES, "the core lane registry is empty" +for lane in core_module.CROSS_FAMILY_LANES.values(): + assert lane["slot"] in guest_source, lane + assert lane["model"] in guest_source, lane + assert lane["base_url"] in guest_source, lane assert "models.json" in guest_source -assert ( - "https://aif-fm7c799d-eus01.cognitiveservices.azure.com/openai/v1" - in guest_source -) assert "no Pi provider mapping for model" in guest_source # The guest refuses model-level baseUrl/api overrides, which pi would give -# precedence over the pinned provider-level endpoint. +# precedence over the pinned provider-level endpoint, and the model-level +# compat object, whose keys change how pi frames and reads the response. assert "model-level endpoint override" in guest_source +assert "model-level compat override" in guest_source assert "claude" not in guest_source.lower() assert "AZURE_CLIENT_SECRET" in guest_source assert "DOCKER_HOST" in guest_source @@ -179,7 +187,196 @@ PY pass "Azure selection is explicit, local-default, and unsafe config fails closed" } -glm_provider_host_unit() { +model_guest_executing_account_unit() { + python3 - "$MODEL_GUEST" "$CORE" "$ADAPTER" <<'PY' \ + || fail "model guest credential contract failed" +import hashlib +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +import tarfile +import tempfile + +guest_path, core_path, adapter_path = map(Path, sys.argv[1:4]) +core_spec = importlib.util.spec_from_file_location("fm_crosscheck", core_path) +core = importlib.util.module_from_spec(core_spec) +core_spec.loader.exec_module(core) +adapter_spec = importlib.util.spec_from_file_location("azure_crosscheck", adapter_path) +adapter = importlib.util.module_from_spec(adapter_spec) +adapter_spec.loader.exec_module(adapter) + +# EXECUTE the guest's own credential block, extracted from the shipped bytes +# rather than reimplemented, so this covers behavior instead of substrings. +# Substring assertions are why the guest could derive a different executing +# account from the host for as long as it did. +source = guest_path.read_text(encoding="utf-8") +marker = 'python3 - "$CREDENTIAL" "$ACCOUNT" "$INPUT" <<\'PY\'\n' +start = source.index(marker) + len(marker) +end = source.index("\nPY\n", start) +guest_block = source[start:end] +assert "reviewer_account_digest" in guest_block, "extracted the wrong guest block" + +SCHEMA = adapter.SCHEMA + + +def run_guest(harness, model, credential_document, account_identity, + manifest_override=None): + """Drive the REAL guest block over a real archive, as the VM would.""" + root = Path(tempfile.mkdtemp()) + credential_bytes = json.dumps(credential_document).encode() + name = "models.json" if core.cross_family_lane_for_model(model) else "auth.json" + identity = {"review_generation": "0123456789abcdef01234567"} + material = { + "schema": SCHEMA, + "review_generation": identity["review_generation"], + "harness": harness, + "model": model, + "effort": "xhigh", + "credential_name": name, + "credential_digest": adapter.digest_bytes(credential_bytes), + } + material.update(manifest_override or {}) + payload = { + "manifest.json": adapter.canonical_bytes(material) + b"\n", + name: credential_bytes, + } + archive = root / "credential.tar.gz" + with tarfile.open(archive, "w:gz", format=tarfile.PAX_FORMAT) as handle: + for member_name, content in payload.items(): + info = tarfile.TarInfo(member_name) + info.size = len(content) + handle.addfile(info, __import__("io").BytesIO(content)) + request = { + "schema": SCHEMA, + "reviewer": {"harness": harness, "model": model, "effort": "xhigh"}, + "identity": { + "review_generation": identity["review_generation"], + "credential_archive_digest": adapter.digest_bytes(archive.read_bytes()), + "credential_digest": material["credential_digest"], + "reviewer_account_digest": adapter.digest_bytes( + account_identity.encode("utf-8") + ), + }, + } + request_path = root / "request.json" + request_path.write_text(json.dumps(request), encoding="utf-8") + destination = root / "account" + destination.mkdir() + script = root / "guest_block.py" + script.write_text(guest_block, encoding="utf-8") + result = subprocess.run( + [sys.executable, str(script), str(archive), str(destination), + str(request_path)], + capture_output=True, text=True, timeout=120, + ) + return result, destination / name + + +# The identity the HOST admits, derived by the host's single reader. +for harness, document, key in ( + ("codex", {"tokens": {"account_id": "acct_ABC123"}}, "auth.json"), + ("pi", {"openai-codex": {"accountId": "acct_ABC123"}}, "auth.json"), +): + home = Path(tempfile.mkdtemp()) + (home / "auth.json").write_text(json.dumps(document), encoding="utf-8") + admitted = core.account_identity(harness, home) + assert ":" in admitted, admitted + + # REGRESSION: the guest used to derive the BARE account id while the host + # digests the PREFIXED one, so this refused inside a booted, paid VM and + # no codex-family compartment review could ever run. Red on that code. + result, landed = run_guest(harness, "gpt-5.6-sol", document, admitted) + assert result.returncode == 0, ( + harness, result.returncode, result.stdout, result.stderr + ) + assert landed.is_file(), landed + print(f"GUEST ACCEPTED {harness} with the host-admitted identity {admitted}") + + # A different account still refuses, so agreement was not bought by + # dropping the check. + result, _ = run_guest(harness, "gpt-5.6-sol", document, admitted + "-other") + assert result.returncode != 0, (harness, result.stdout) + assert "credential executing account mismatch" in (result.stdout + result.stderr) + print(f"GUEST REFUSED {harness} with a foreign executing account") + + # A credential carrying no account id refuses rather than landing None. + result, _ = run_guest(harness, "gpt-5.6-sol", {}, admitted) + assert result.returncode != 0, (harness, result.stdout) + print(f"GUEST REFUSED {harness} with no account id in the credential") + +# The guest's MANIFEST identity check binds the archive to the REQUEST, not +# just the credential to the account. Removing it is a real behavior change +# rather than a no-op: a forged manifest is otherwise admitted, and the +# reviewer effort or model the compartment actually runs stops matching what +# the host admitted. +for label, override in ( + ("forged effort", {"effort": "low"}), + ("forged model", {"model": "gpt-4o-mini"}), + ("forged harness", {"harness": "claude"}), + ("forged review generation", {"review_generation": "ffffffffffffffffffffffff"}), + ("forged credential name", {"credential_name": "models.json"}), + ("forged credential digest", {"credential_digest": "sha256:" + "0" * 64}), +): + result, _ = run_guest( + "codex", + "gpt-5.6-sol", + {"tokens": {"account_id": "acct_ABC123"}}, + core.account_identity( + "codex", + (lambda h: (h.mkdir(exist_ok=True), (h / "auth.json").write_text( + json.dumps({"tokens": {"account_id": "acct_ABC123"}}), encoding="utf-8" + ), h)[-1])(Path(tempfile.mkdtemp()) / "home"), + ), + manifest_override=override, + ) + assert result.returncode != 0, (label, result.stdout) + combined = result.stdout + result.stderr + assert ( + "credential manifest identity mismatch" in combined + or "credential archive shape mismatch" in combined + ), (label, combined) + print(f"GUEST REFUSED a manifest with a {label}") + +# The cross-family lane keeps its own non-secret identity and still lands. +lane = next(iter(core.CROSS_FAMILY_LANES.values())) +lane_document = {"providers": {lane["slot"]: { + "baseUrl": lane["base_url"], "api": lane["api"], "apiKey": "k", + "models": [{"id": lane["model"], "name": "n"}], +}}} +result, landed = run_guest( + "pi", lane["model"], lane_document, core.cross_family_account_identity(lane) +) +assert result.returncode == 0, (result.stdout, result.stderr) +assert landed.name == "models.json", landed +print(f"GUEST ACCEPTED the {lane['slot']} lane credential") + +# A foreign endpoint in the archived credential still refuses in the guest. +foreign = json.loads(json.dumps(lane_document)) +foreign["providers"][lane["slot"]]["baseUrl"] = "https://evil.example/v1" +result, _ = run_guest( + "pi", lane["model"], foreign, core.cross_family_account_identity(lane) +) +assert result.returncode != 0, result.stdout +print("GUEST REFUSED a foreign endpoint in the archived credential") + +# REGISTRATION COMPLETENESS: a lane in the registry with no `case "$MODEL"` +# dispatch arm passes every substring assertion and dies at runtime with exit +# 125. Registering a lane touches four places; pin all of them. +for registered in core.CROSS_FAMILY_LANES.values(): + assert f'{registered["model"]}) PI_PROVIDER={registered["slot"]} ;;' in source, ( + f"lane {registered['slot']} has no provider dispatch arm in the guest" + ) + assert f'"{registered["model"]}": (' in source, ( + f"lane {registered['slot']} is absent from the guest lane table" + ) +print("GUEST dispatches every registered lane") +PY + pass "the model guest derives the host's executing account, refuses foreign ones, and dispatches every registered lane" +} + +cross_family_provider_host_unit() { python3 - "$ADAPTER" "$CORE" <<'PY' || fail "model-aware provider host derivation failed" import importlib.util import sys @@ -191,28 +388,46 @@ core_spec = importlib.util.spec_from_file_location("fm_crosscheck", sys.argv[2]) core = importlib.util.module_from_spec(core_spec) core_spec.loader.exec_module(core) -# The adapter and the core pin one identical R6 endpoint binding. -assert module.GLM_REVIEWER_MODEL == core.GLM_REVIEWER_MODEL == "FW-GLM-5.2" -assert module.GLM_PROVIDER_HOST == core.GLM_PROVIDER_HOST -assert module.GLM_ALLOWED_BASE_URL == core.GLM_ALLOWED_BASE_URL -assert module.GLM_REVIEWER_ACCOUNT_IDENTITY == core.GLM_REVIEWER_ACCOUNT_IDENTITY -assert module.GLM_PROVIDER_HOST == "aif-fm7c799d-eus01.cognitiveservices.azure.com" - -# The model decides the host: a GLM review derives the exact Foundry host, -# refuses a conflicting configured host, and the codex-family fallback keeps -# its existing derivation. The retired claude harness derives nothing. -assert module.effective_provider_host({}, "pi", "FW-GLM-5.2") == module.GLM_PROVIDER_HOST -assert module.effective_provider_host( - {"provider_host": module.GLM_PROVIDER_HOST}, "pi", "FW-GLM-5.2" -) == module.GLM_PROVIDER_HOST -try: - module.effective_provider_host( - {"provider_host": "api.example.com"}, "pi", "FW-GLM-5.2" +# The adapter and the core pin ONE identical R6 lane registry. Comparing the +# whole registry rather than a handful of constants means a lane added on one +# side and not the other is a failure here rather than a divergent allowlist. +assert module.CROSS_FAMILY_LANES == core.CROSS_FAMILY_LANES, ( + module.CROSS_FAMILY_LANES, + core.CROSS_FAMILY_LANES, +) +assert module.CROSS_FAMILY_LANES["fireworks-glm"]["model"] == ( + "accounts/fireworks/models/glm-5p2" +) +for lane in module.CROSS_FAMILY_LANES.values(): + # Per-lane consistency, NOT one hardcoded host. Asserting a single host for + # every lane meant any genuinely new lane failed HERE first, so the + # registration-completeness guard in the model-guest unit - the one this + # repo advertises for that job - never got to run. + assert lane["base_url"].startswith("https://" + lane["host"] + "/"), lane + assert "://" not in lane["host"] and "/" not in lane["host"], lane + assert module.cross_family_account_identity(lane) == ( + core.cross_family_account_identity(lane) ) -except module.AzureCrosscheckError as exc: - assert "bind exactly one provider host" in str(exc), str(exc) -else: - raise AssertionError("a GLM review accepted a foreign provider host") + +# The model decides the host: every cross-family review derives its own exact +# Foundry host, refuses a conflicting configured host, and the codex-family +# fallback keeps its existing derivation. The retired claude harness derives +# nothing. +for lane in module.CROSS_FAMILY_LANES.values(): + assert module.effective_provider_host({}, "pi", lane["model"]) == lane["host"] + assert module.effective_provider_host( + {"provider_host": lane["host"]}, "pi", lane["model"] + ) == lane["host"] + try: + module.effective_provider_host( + {"provider_host": "api.example.com"}, "pi", lane["model"] + ) + except module.AzureCrosscheckError as exc: + assert "bind exactly one provider host" in str(exc), str(exc) + else: + raise AssertionError( + "a cross-family review accepted a foreign provider host" + ) assert module.effective_provider_host({}, "pi", "gpt-5.6-sol") == "chatgpt.com" assert module.effective_provider_host({}, "codex", "gpt-5.6-sol") == "chatgpt.com" assert module.effective_provider_host( @@ -226,7 +441,8 @@ except module.AzureCrosscheckError as exc: else: raise AssertionError("the retired claude harness still derives a provider host") -# The GLM identity record must carry the pinned host through validation. +# The cross-family identity record must carry the pinned host through +# validation. import copy identity = { "home_binding": "sha256:" + "1" * 64, @@ -242,7 +458,7 @@ identity = { "provider_host": "api.example.com", "provider_port": "443", "reviewer_harness": "pi", - "reviewer_model": "FW-GLM-5.2", + "reviewer_model": "accounts/fireworks/models/glm-5p2", "reviewer_effort": "xhigh", "reviewer_account_digest": "sha256:" + "2" * 64, "ledger_digest": "sha256:" + "f" * 64, @@ -261,7 +477,7 @@ identity.update({ reviewer = { "execution_mode": "azure-compartment-v1", "harness": "pi", - "model": "FW-GLM-5.2", + "model": "accounts/fireworks/models/glm-5p2", "effort": "xhigh", "reviewer_account_identity_sha256": "2" * 64, "azure_identity": identity, @@ -270,15 +486,17 @@ run = {"head_sha": "a" * 40, "base_sha": "b" * 40, "claims_sha256": "c" * 64} try: module.validate_azure_reviewer_record(reviewer, run, "run") except RuntimeError as exc: - assert "GLM provider host is not the pinned R6 Foundry endpoint" in str(exc), str(exc) + assert "fireworks-glm provider host is not the pinned R6 provider endpoint" in str(exc), str(exc) else: - raise AssertionError("a GLM ledger record with a foreign provider host validated") + raise AssertionError( + "a cross-family ledger record with a foreign provider host validated" + ) PY pass "the reviewer model derives the exact Foundry host and the claude host lane is retired" } -glm_credential_lane_unit() { - python3 - "$ADAPTER" "$CORE" <<'PY' || fail "GLM Azure credential lane contract failed" +cross_family_credential_lane_unit() { + python3 - "$ADAPTER" "$CORE" <<'PY' || fail "cross-family Azure credential lane contract failed" import importlib.util import json from pathlib import Path @@ -293,15 +511,19 @@ core_spec = importlib.util.spec_from_file_location("fm_crosscheck", sys.argv[2]) core = importlib.util.module_from_spec(core_spec) core_spec.loader.exec_module(core) -PINNED = "https://aif-fm7c799d-eus01.cognitiveservices.azure.com/openai/v1" +PINNED = "https://api.fireworks.ai/inference/v1" +LANE = module.CROSS_FAMILY_LANES["fireworks-glm"] +SLOT = LANE["slot"] +MODEL = LANE["model"] -def models_json(base_url=PINNED, api_key="glm-key-material", model_extra=None): - model = {"id": "FW-GLM-5.2", "name": "GLM 5.2"} +def models_json(base_url=PINNED, api_key="lane-key-material", model_extra=None, + slot=SLOT, model_id=MODEL): + model = {"id": model_id, "name": "cross-family reviewer"} model.update(model_extra or {}) return json.dumps({ "providers": { - "azure-glm": { + slot: { "baseUrl": base_url, "api": "openai-completions", "apiKey": api_key, @@ -313,12 +535,12 @@ def models_json(base_url=PINNED, api_key="glm-key-material", model_extra=None): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) - home = root / "glm-home" + home = root / "lane-home" home.mkdir() (home / "models.json").write_text(models_json(), encoding="utf-8") config = { "harness": "pi", - "model": "FW-GLM-5.2", + "model": MODEL, "effort": "xhigh", "account_home": str(home), } @@ -326,16 +548,16 @@ with tempfile.TemporaryDirectory() as temporary: module.inspect_reviewer_credential(core, config) ) assert credential == home.resolve() / "models.json", credential - assert source == "pi-azure-glm-models-file", source - assert identifier.startswith("glm-foundry-binding:"), identifier - assert account_identity == module.GLM_REVIEWER_ACCOUNT_IDENTITY + assert source == "pi-" + SLOT + "-models-file", source + assert identifier.startswith("provider-binding:" + SLOT + ":"), identifier + assert account_identity == module.cross_family_account_identity(LANE) # The packaged compartment credential is the models.json under the same # allowlist pin, and its archived identity is the non-secret binding. identity = {"review_generation": "0123456789abcdef01234567"} archive_path = root / "credential.tar.gz" archive_digest, credential_digest = module.create_credential_archive( - archive_path, credential, identity, config, account_identity + archive_path, credential, identity, config, account_identity, core ) assert archive_digest.startswith("sha256:") with tarfile.open(archive_path, "r:gz") as archive: @@ -343,13 +565,13 @@ with tempfile.TemporaryDirectory() as temporary: assert names == {"manifest.json", "models.json"}, names manifest = json.loads(archive.extractfile("manifest.json").read()) assert manifest["credential_name"] == "models.json", manifest - assert manifest["model"] == "FW-GLM-5.2", manifest + assert manifest["model"] == MODEL, manifest # A credential outside the endpoint allowlist never enters the archive. foreign = root / "foreign-home" foreign.mkdir() (foreign / "models.json").write_text( - models_json(base_url="https://aif-other.cognitiveservices.azure.com/openai/v1"), + models_json(base_url="https://api.fireworks.ai.evil.example/inference/v1"), encoding="utf-8", ) try: @@ -359,11 +581,100 @@ with tempfile.TemporaryDirectory() as temporary: identity, config, account_identity, + core, ) except module.AzureCrosscheckError as exc: - assert "pinned R6 Foundry endpoint" in str(exc), str(exc) + assert "pinned R6 provider endpoint" in str(exc), str(exc) else: - raise AssertionError("a foreign-endpoint GLM credential was archived") + raise AssertionError( + "a foreign-endpoint cross-family credential was archived" + ) + + # An unexpected provider slot is still the wrong slot for this review: the + # lane is keyed on the reviewer model, never on what the credential file + # declares about itself. + swapped = root / "swapped-slot-home" + swapped.mkdir() + (swapped / "models.json").write_text( + models_json(slot="openai-codex"), encoding="utf-8" + ) + try: + module.create_credential_archive( + root / "swapped.tar.gz", + swapped / "models.json", + identity, + config, + account_identity, + core, + ) + except module.AzureCrosscheckError as exc: + assert "pinned R6 provider endpoint" in str(exc), str(exc) + else: + raise AssertionError("a foreign-slot cross-family credential was archived") + + # The archive gate's allowlists come from CORE, not a local copy. A + # hardcoded set drifted weaker than the inspector inside one change: no + # model-level allowlist and no `api` check, so an archived credential + # could carry `openai-responses`, which R6 forbids outright. + assert module.__dict__.get("PI_PROVIDER_ALLOWED_KEYS") is None, ( + "the archive gate must reference core's allowlist, not keep its own" + ) + for label, mutate in ( + ("responses api", lambda d: d["providers"][SLOT].__setitem__("api", "openai-responses")), + ("provider compat", lambda d: d["providers"][SLOT].__setitem__("compat", {"supportsFinishReason": False})), + ("modelOverrides", lambda d: d["providers"][SLOT].__setitem__("modelOverrides", {MODEL: {"compat": {}}})), + ("provider headers", lambda d: d["providers"][SLOT].__setitem__("headers", {"x": "1"})), + ("model-level extra", lambda d: d["providers"][SLOT]["models"][0].__setitem__("headers", {"x": "1"})), + ): + drifted = json.loads(models_json()) + mutate(drifted) + drift_home = root / ("drift-" + label.replace(" ", "-")) + drift_home.mkdir() + (drift_home / "models.json").write_text(json.dumps(drifted), encoding="utf-8") + try: + module.create_credential_archive( + root / ("drift-" + label.replace(" ", "-") + ".tar.gz"), + drift_home / "models.json", + identity, + config, + account_identity, + core, + ) + except module.AzureCrosscheckError: + pass + else: + raise AssertionError(f"the archive gate admitted {label}") + # The core inspector refuses the same shape, so the two agree. + try: + core.inspect_pi_cross_family_credential(drift_home, LANE) + except core.CrosscheckToolError: + pass + else: + raise AssertionError(f"the core inspector admitted {label}") + print("ARCHIVE GATE and CORE INSPECTOR agree on every drifted credential shape") + + # The archive gate owns the model-level compat pin too, not just the + # inspector: a compat that weakens the truncation guard never ships into + # a compartment. + compat_home = root / "compat-home" + compat_home.mkdir() + (compat_home / "models.json").write_text( + models_json(model_extra={"compat": {"supportsFinishReason": False}}), + encoding="utf-8", + ) + try: + module.create_credential_archive( + root / "compat.tar.gz", + compat_home / "models.json", + identity, + config, + account_identity, + core, + ) + except module.AzureCrosscheckError as exc: + assert "model-level compat" in str(exc), str(exc) + else: + raise AssertionError("a model-level compat override was archived") # pi gives MODEL-level baseUrl/api precedence over the provider level, so # a credential keeping the pinned endpoint at provider level while @@ -385,6 +696,7 @@ with tempfile.TemporaryDirectory() as temporary: identity, config, account_identity, + core, ) except module.AzureCrosscheckError as exc: assert "model-level" in str(exc), str(exc) @@ -399,6 +711,81 @@ with tempfile.TemporaryDirectory() as temporary: else: raise AssertionError("a model-level endpoint override passed inspection") + # REGRESSION: the codex-family compartment path used to refuse itself. + # `account_identity` returned "codex:" / "openai-codex:" while the + # archive derived the BARE "" separately, so + # `archived_identity != reviewer_account_identity` was structurally always + # true and no codex-family compartment review could ever run. Only the + # cross-family branch passed, because both sides there read one shared + # constant. This drives the REAL readers end to end for BOTH branches: + # it fails on the two-derivation code and passes on the shared one. + for harness, document in ( + ("codex", {"tokens": {"account_id": "acct-codex-1"}}), + ("pi", {"openai-codex": {"accountId": "acct-pi-1"}}), + ): + family_home = root / ("family-home-" + harness) + family_home.mkdir() + (family_home / "auth.json").write_text(json.dumps(document), encoding="utf-8") + admitted = core.account_identity(harness, family_home) + # The prefix is the whole point: the identity is not a bare account id. + assert admitted.split(":", 1)[1] in {"acct-codex-1", "acct-pi-1"}, admitted + assert ":" in admitted and not admitted.startswith(":"), admitted + family_config = { + "harness": harness, + "model": "gpt-5.6-sol", + "effort": "xhigh", + "account_home": str(family_home), + } + archive_digest, _ = module.create_credential_archive( + root / ("family-" + harness + ".tar.gz"), + family_home / "auth.json", + identity, + family_config, + admitted, + core, + ) + assert archive_digest.startswith("sha256:"), archive_digest + # A genuinely different account still refuses, so the fix did not make + # the comparison lenient. + try: + module.create_credential_archive( + root / ("family-wrong-" + harness + ".tar.gz"), + family_home / "auth.json", + identity, + family_config, + admitted + "-other", + core, + ) + except module.AzureCrosscheckError as exc: + assert "differs from the admitted executing account" in str(exc), str(exc) + else: + raise AssertionError("a foreign executing account was archived") + + # TOCTOU: a credential swapped between admission and staging must refuse + # as a TOOL FAILURE, not a bare AzureCrosscheckError. The class is the + # control: AzureCrosscheckError is a plain RuntimeError that none of the + # persisting handlers catch, so raised as that class the swap would leave + # no ledger, no report and no data directory at all. + assert not issubclass(module.AzureCrosscheckError, core.CrosscheckToolError), ( + "the two classes must stay distinguishable for this test to mean anything" + ) + stable = module.inspect_reviewer_credential(core, config) + module.require_stable_reviewer_credential(core, config, stable) + try: + module.require_stable_reviewer_credential( + core, config, (stable[0], stable[1], stable[2], "openai-codex:swapped") + ) + except core.CrosscheckToolError as exc: + assert "identity changed before exact staging" in str(exc), str(exc) + except module.AzureCrosscheckError as exc: + raise AssertionError( + "the TOCTOU refusal raised a class the persisting handlers ignore: " + + str(exc) + ) + else: + raise AssertionError("a swapped reviewer credential passed the TOCTOU re-proof") + print("TOCTOU refusal raises a persisted tool failure, not a vanishing error") + # The retired claude harness has no Azure credential lane at all. claude_home = root / "claude-home" claude_home.mkdir() @@ -418,8 +805,8 @@ with tempfile.TemporaryDirectory() as temporary: else: raise AssertionError("the retired claude credential lane still packages a profile") - # The GLM preflight accepts the api-key credential without an expiry - # reader and still refuses a missing credential loudly. + # The cross-family preflight accepts the api-key credential without an + # expiry reader and still refuses a missing credential loudly. record = module.preflight_reviewer_credential(core, config) assert record["state"] == "usable", record assert record["credential"] == "models.json", record @@ -430,11 +817,11 @@ with tempfile.TemporaryDirectory() as temporary: core, {**config, "account_home": str(missing)} ) except core.CrosscheckToolError as exc: - assert "GLM reviewer credential inspection failed" in str(exc), str(exc) + assert SLOT + " reviewer credential inspection failed" in str(exc), str(exc) else: - raise AssertionError("a missing GLM credential passed preflight") + raise AssertionError("a missing cross-family credential passed preflight") PY - pass "the Azure GLM credential lane packages models.json under the endpoint allowlist and the claude lane is gone" + pass "the Azure cross-family credential lane packages models.json under each lane's endpoint allowlist and the claude lane is gone" } identity_outcome_unit() { @@ -549,7 +936,7 @@ PY } account_and_cleanup_identity_unit() { - python3 - "$ADAPTER" <<'PY' || fail "Azure account and cleanup identity contract failed" + python3 - "$ADAPTER" "$CORE" <<'PY' || fail "Azure account and cleanup identity contract failed" import importlib.util import json from pathlib import Path @@ -559,6 +946,9 @@ import tempfile spec = importlib.util.spec_from_file_location("azure_crosscheck", sys.argv[1]) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) +core_spec = importlib.util.spec_from_file_location("fm_crosscheck", sys.argv[2]) +core = importlib.util.module_from_spec(core_spec) +core_spec.loader.exec_module(core) common = { "home": Path("/home/firstmate"), "task_id": "review-one", @@ -584,8 +974,8 @@ common = { }, "ledger": {"schema": "firstmate.crosscheck-ledger.v2", "findings": [], "runs": []}, } -first = module.review_identity(**common, reviewer_account_identity="account-one") -second = module.review_identity(**common, reviewer_account_identity="account-two") +first = module.review_identity(**common, reviewer_account_identity="openai-codex:account-one") +second = module.review_identity(**common, reviewer_account_identity="openai-codex:account-two") assert first["reviewer_account_digest"] != second["reviewer_account_digest"] assert first["review_generation"] != second["review_generation"] with tempfile.TemporaryDirectory() as temporary: @@ -593,12 +983,16 @@ with tempfile.TemporaryDirectory() as temporary: credential = root / "auth.json" credential.write_text(json.dumps({"openai-codex":{"accountId":"account-one"}}), encoding="utf-8") archive_digest, credential_digest = module.create_credential_archive( - root / "credential.tar.gz", credential, first, common["config"], "account-one" + root / "credential.tar.gz", credential, first, common["config"], + "openai-codex:account-one", + core, ) assert archive_digest.startswith("sha256:") and credential_digest.startswith("sha256:") try: module.create_credential_archive( - root / "wrong.tar.gz", credential, first, common["config"], "account-two" + root / "wrong.tar.gz", credential, first, common["config"], + "openai-codex:account-two", + core, ) except module.AzureCrosscheckError as exc: assert "differs" in str(exc) @@ -608,7 +1002,9 @@ with tempfile.TemporaryDirectory() as temporary: linked.symlink_to(credential) try: module.create_credential_archive( - root / "linked.tar.gz", linked, first, common["config"], "account-one" + root / "linked.tar.gz", linked, first, common["config"], + "openai-codex:account-one", + core, ) except module.AzureCrosscheckError as exc: assert "symlink" in str(exc) @@ -1538,8 +1934,9 @@ PY static_contract parameter_contract_unit adapter_mode_unit -glm_provider_host_unit -glm_credential_lane_unit +cross_family_provider_host_unit +cross_family_credential_lane_unit +model_guest_executing_account_unit identity_outcome_unit account_and_cleanup_identity_unit bridge_security_unit diff --git a/tests/fm-crosscheck-slack.test.sh b/tests/fm-crosscheck-slack.test.sh index 0b6cbf9f057..0fdc13156b7 100755 --- a/tests/fm-crosscheck-slack.test.sh +++ b/tests/fm-crosscheck-slack.test.sh @@ -511,13 +511,14 @@ test_completed_review_names_the_lane_and_writes_gate_metadata() { before=$(fixture_run_count) event="$TMP_ROOT/event-clear.json" write_event "$event" C0TESTCHAN U0ALICE 1755640004.000100 "$GOOD_PR_TEXT" - FM_FIXTURE_REVIEWER_JSON='{"harness":"pi","model":"FW-GLM-5.2","effort":"xhigh","account_home":"/tmp/x"}' \ + FM_FIXTURE_REVIEWER_JSON='{"harness":"pi","model":"accounts/fireworks/models/glm-5p2","effort":"xhigh","account_home":"/tmp/x","review_family_mode":"cross-family-primary"}' \ run_mention ev-clear-1 "$CONFIG_MAIN" "$event" clear \ || fail "clear review errored: $RUN_MENTION_OUTPUT" assert_contains "$RUN_MENTION_OUTPUT" "action: completed:clear" "expected completed:clear" reply=$(last_post_text) assert_contains "$reply" "Crosscheck CLEAR" "verdict reply missing state" - assert_contains "$reply" "Lane: GLM-5.2 primary" "verdict reply did not name the GLM lane" + assert_contains "$reply" "Lane: glm-5p2 primary" \ + "verdict reply did not name the cross-family lane" assert_contains "$reply" "crosscheck.md" "verdict reply did not point at the full report" assert_contains "$reply" "Host report path for the operator:" \ "report path was not labeled as host-local" @@ -525,7 +526,7 @@ test_completed_review_names_the_lane_and_writes_gate_metadata() { [ "$after" = $((before + 1)) ] || fail "expected exactly one crosscheck invocation" assert_grep "Review started" "$POSTS" "review start ack missing" assert_grep "hourglass" "$REACTS" "ack reaction missing" - pass "a completed review posts threaded findings naming the GLM lane (fixture verified gate metadata)" + pass "a completed review posts threaded findings naming the cross-family lane (fixture verified gate metadata)" } test_lane_naming_covers_fallback_and_explicit_marker() { diff --git a/tests/fm-crosscheck.test.sh b/tests/fm-crosscheck.test.sh index 74aefca78cd..4ddc9bb0f9e 100755 --- a/tests/fm-crosscheck.test.sh +++ b/tests/fm-crosscheck.test.sh @@ -105,8 +105,8 @@ JSON "project=$repo" \ "kind=ship" \ "mode=no-mistakes" \ - "harness=codex" \ - "model=gpt-5.5" \ + "harness=claude" \ + "model=claude-opus-5" \ "account_home=$case_dir/author-home" cat > "$case_dir/reviewer.json" < "$case_dir/reviewer.json" < "$destination" < "$case_dir/reviewer.json" </, so the SAME +# lane model reached through some OTHER author-side slot must not read as a +# different family. Matching only the registry's own slot admitted exactly +# that reviewer with no relaxation and no degraded marker. +LANE = next(iter(module.CROSS_FAMILY_LANES.values())) +for disguise in ( + LANE["model"], + LANE["slot"] + "/" + LANE["model"], + "some-other-slot/" + LANE["model"], + "author-side-slot/glm-5p2", + "glm-5p2", +): + assert module.model_family(disguise) == "cross-family:" + LANE["slot"], disguise +for slot_prefix in ("some-other-slot/", "author-side-slot/"): + disguised_author = { + "harness": "pi", + "model": slot_prefix + LANE["model"], + "account_home": str(homes["author-home"]), + } + write_config([reviewer("pi", LANE["model"], "xhigh", "lane-home")]) + bypass = expect_refused(disguised_author, "outside the model family") + print(f"REFUSED provider-qualified same-model author {slot_prefix}: {bypass}") +# The looser family rule must NOT loosen lane/credential selection, which +# still has to match exactly or refuse. +assert module.cross_family_lane_for_model("some-other-slot/" + LANE["model"]) is None +assert module.cross_family_lane_for_model("glm-5p2") is None + +# And an unrecognized author model stays its own family, so nothing that used +# to pass silently starts failing. +assert module.model_family("mystery-1") != module.model_family("mystery-2") +assert module.model_family("gpt-5.5") == module.model_family("gpt-5.6-sol") == "openai" +assert module.model_family("claude-opus-5") == "anthropic" +assert module.model_family("openai-codex-2/gpt-5.6-sol") == "openai" +# The family screen must not depend on a literal separator: `gpt5.6-sol` +# read as its own family and would have been admitted a `gpt-5.6-sol` +# reviewer, which is cc-4dcd7873f71a one alias away. +for spelling in ("gpt5.6-sol", "GPT-5.6-SOL", "gpt_5.6_sol", "Codex-Mini"): + assert module.model_family(spelling) == "openai", spelling +assert module.model_family("CLAUDE-OPUS-5") == "anthropic" +# And a lane model reached under ANY vendor alias is the lane's family, so an +# author on GLM-5.2 via a non-Fireworks id cannot take the GLM reviewer with +# no same-model marker. +for alias in ("z-ai/glm-5.2", "GLM-5.2", "glm_5.2", "glm-5p2"): + assert module.model_family(alias) == "cross-family:" + LANE["slot"], alias + # Lane SELECTION stays exact - the alias must not pick a credential. + assert module.cross_family_lane_for_model(alias) is None, alias +aliased_author = { "harness": "pi", - "model": "azure-glm/FW-GLM-5.2", + "model": "z-ai/glm-5.2", "account_home": str(homes["author-home"]), } -write_config([reviewer("pi", "FW-GLM-5.2", "xhigh", "glm-home")]) -glm_same_model = expect_refused(glm_author, "different model") -print(f"REFUSED same-model GLM reviewer: {glm_same_model}") +write_config([reviewer("pi", LANE["model"], "xhigh", "lane-home")]) +aliased = expect_refused(aliased_author, "outside the model family") +print(f"REFUSED aliased same-model author: {aliased}") write_config([reviewer("pi", "gpt-5.6-sol", "xhigh", "pi-home")]) selected = module.reviewer_candidates(root, claude_author)[0] @@ -1216,8 +1282,8 @@ same_model_author = { "account_home": str(homes["author-home"]), } write_config([reviewer("pi", "gpt-5.6-sol", "xhigh", "pi-home")]) -same_model = expect_refused(same_model_author, "different model") -print(f"REFUSED shared-model: {same_model}") +same_model = expect_refused(same_model_author, "outside the model family") +print(f"REFUSED shared-family: {same_model}") write_config( [ @@ -1344,9 +1410,9 @@ def expect_refused_exact(account_home, expected, label): # Absent and explicit-off configuration preserve the shipped cross-model rule. -expect_refused(distinct, "different model", "same-model-default-off") +expect_refused(distinct, "outside the model family", "same-model-default-off") mode_path.write_text("off\n", encoding="utf-8") -expect_refused(distinct, "different model", "same-model-explicit-off") +expect_refused(distinct, "outside the model family", "same-model-explicit-off") mode_path.write_text("on\n", encoding="utf-8") for account_home in (aliased, opaque, distinct): @@ -1523,6 +1589,91 @@ expect_tool_failure( [assistant_turn(verdict_text, "length")], "stopReason='length'", ) + +# One Markdown fence around the WHOLE verdict is a presentation habit, not a +# different verdict, and GLM-5.2 through Pi produces exactly that. It is +# unwrapped; everything looser still refuses, because each looser shape is one +# where the reviewer said more than one thing. +for fence in ("```json\n%s\n```", "```\n%s\n```", "```JSON \n%s\n```"): + fenced, count = module.pi_review_result( + event_stream([assistant_turn(fence % verdict_text, "stop")]) + ) + assert fenced == {"verdict": "clear"}, fenced + assert count == 1 + +# Exactly ONE complete fenced block leaves nothing to choose between, so +# surrounding prose is harmless and the block is unwrapped. +for label, body in ( + ("prose before the fence", "Here is my verdict:\n```json\n%s\n```" % verdict_text), + ("prose after the fence", "```json\n%s\n```\nHope that helps." % verdict_text), + ("prose both sides", "Verdict:\n```\n%s\n```\nDone." % verdict_text), +): + wrapped, count = module.pi_review_result( + event_stream([assistant_turn(body, "stop")]) + ) + assert wrapped == {"verdict": "clear"}, (label, wrapped) + assert count == 1 + +# Several complete blocks WOULD make the gate choose, so they refuse. An +# unterminated fence yields no complete block and still fails to parse, which +# is what keeps a wrapper tolerance from becoming a truncation tolerance. +for label, body in ( + ("two fences", "```json\n%s\n```\n```json\n%s\n```" % (verdict_text, verdict_text)), + ("unterminated fence", "```json\n%s" % verdict_text), + ("truncated fenced verdict", '```json\n{"verdict": "cle'), + ("truncated bare verdict", '{"verdict": "cle'), + # A truncated verdict fence contributes ZERO complete blocks, so ANY + # complete fence earlier in the message - a draft, an example, a quoted + # snippet - used to leave the count at one and get certified in place of + # the real, truncated verdict. stopReason is `stop` here and the parse + # SUCCEEDS on the wrong block, so nothing else downstream catches it. + ( + "draft fence then truncated verdict fence", + 'Draft:\n```json\n{"verdict": "clear", "findings": []}\n```\n' + 'Final verdict:\n```json\n{"verdict": "blocking", "summary": "BLOCKING: the cre', + ), + ( + "example fence then truncated bare verdict", + 'For example:\n```json\n{"verdict": "clear"}\n```\n{"verdict": "block', + ), + ( + "two complete fences then a truncated third", + "```\n%s\n```\n```\n%s\n```\n```json\n{\"verdict\": \"bl" % (verdict_text, verdict_text), + ), + # The shape ONLY the odd-marker rule catches, and the reason that rule is + # not redundant with the brace-remainder rule: the real verdict is + # truncated immediately after its OPENING fence, so it contributes no + # braces at all. Markers 3 (odd), complete blocks 1, remainder brace-free + # - without the marker count the example below gets certified as the + # verdict. It is a plausible truncation point and it is exactly the + # failure this lane exists to close. + ( + "example fence then a verdict truncated at its opening fence", + '```json\n{"verdict": "clear", "findings": []}\n```\nFinal verdict:\n```json\n', + ), + ( + "example fence then a verdict truncated inside its fence header", + '```json\n{"verdict": "clear"}\n```\nHere is the real one:\n```js', + ), +): + expect_tool_failure( + label, + [assistant_turn(body, "stop")], + "malformed verdict artifact", + ) + +# The refusal names the offending text, bounded and repr-escaped so reviewer +# output can never inject a line into an operator's log. +try: + module.pi_review_result( + event_stream([assistant_turn("not json at all\nsecond line", "stop")]) + ) +except module.CrosscheckToolError as exc: + assert "final assistant text began" in str(exc), str(exc) + assert "\n" not in str(exc), repr(str(exc)) + assert "not json at all" in str(exc), str(exc) +else: + raise AssertionError("a non-JSON verdict artifact was accepted") expect_tool_failure( "nonterminal tool-use turn", [assistant_turn(verdict_text, "toolUse")], @@ -1826,8 +1977,8 @@ test_missing_author_identity_reaches_normal_verdict() { record=$(make_case missing-author-identity-normal-verdict) IFS=$'\t' read -r case_dir base head <<< "$record" sed -i.bak \ - -e 's/harness=codex/harness=pi/' \ - -e 's#model=gpt-5.5#model=openai-codex-5/gpt-5.5#' \ + -e 's/harness=claude/harness=pi/' \ + -e 's#model=claude-opus-5#model=fireworks-glm/accounts/fireworks/models/glm-5p2#' \ -e '/^account_home=/d' \ "$case_dir/state/task-x1.meta" rm "$case_dir/state/task-x1.meta.bak" @@ -1866,7 +2017,7 @@ EOF expect_code 1 "$rc" "retired claude reviewer profile" assert_grep 'CROSSCHECK TOOL-FAILURE: reviewer preflight failed' \ "$case_dir/err" "the retired claude profile was not refused at reviewer preflight" - assert_grep 'must be codex gpt-5.6-sol xhigh or pi FW-GLM-5.2 xhigh or pi gpt-5.6-sol xhigh' \ + assert_grep 'must be codex gpt-5.6-sol xhigh or pi accounts/fireworks/models/glm-5p2 xhigh or pi gpt-5.6-sol xhigh' \ "$case_dir/err" "the retired claude profile was not refused with the exact profile message" assert_absent "$case_dir/fakebin/claude" "Claude reviewer machinery was installed by the fixture" assert_absent "$case_dir/pi.log" "a reviewer launched despite the retired profile" @@ -1874,49 +2025,114 @@ EOF pass "the retired claude reviewer profile is refused before any reviewer machinery runs" } -test_glm_reviewer_executes_bound_policy_profile() { - local record case_dir base head output - record=$(make_case glm-reviewer) - IFS=$'\t' read -r case_dir base head <<< "$record" - select_glm_reviewer "$case_dir" - output=$(FM_TEST_PI_BIN=pi PATH="$case_dir/fakebin:$PATH" \ - FM_TEST_PI_EXPECT_PROVIDER=azure-glm FM_TEST_PI_EXPECT_MODEL=FW-GLM-5.2 \ - run_case "$case_dir" "$base" "$head" clear run 2> "$case_dir/err") \ - || fail "GLM reviewer did not complete" - assert_contains "$output" 'crosscheck clear' \ - "GLM reviewer did not earn a clear result" - assert_grep '--mode json --provider azure-glm --model FW-GLM-5.2 --thinking xhigh --tools read,bash,grep,find,ls --no-session' \ - "$case_dir/pi.log" \ - "GLM reviewer was not invoked on the azure-glm provider with its pinned model, effort, and tools" - assert_no_grep 'CROSSCHECK DEGRADED' "$case_dir/err" \ - "the GLM primary lane announced a degraded fallback" - python3 -c ' +test_cross_family_reviewer_executes_bound_policy_profile() { + local record case_dir base head output slot model lanes + # EVERY registered cross-family lane must execute on its own provider slot + # and record its own non-secret binding: the lane is data, not a hardcoded + # model, so the case is driven from the registry itself and a lane added + # there is covered here without touching this test. + lanes=$("$CROSSCHECK_PYTHON" - "$CROSSCHECK_PY" <<'PY' +import importlib.util +import sys + +spec = importlib.util.spec_from_file_location("fm_crosscheck", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +assert spec.loader is not None +spec.loader.exec_module(module) +for lane in module.CROSS_FAMILY_LANES.values(): + print(lane["slot"], lane["model"]) +PY +) + [ -n "$lanes" ] || fail "the cross-family lane registry is empty" + while read -r slot model; do + [ -n "$slot" ] || continue + record=$(make_case "cross-family-reviewer-$slot") + IFS=$'\t' read -r case_dir base head <<< "$record" + select_cross_family_reviewer "$case_dir" "$slot" "$model" + output=$(FM_TEST_PI_BIN=pi PATH="$case_dir/fakebin:$PATH" \ + FM_TEST_PI_EXPECT_PROVIDER="$slot" FM_TEST_PI_EXPECT_MODEL="$model" \ + run_case "$case_dir" "$base" "$head" clear run 2> "$case_dir/err") \ + || fail "$model reviewer did not complete" + assert_contains "$output" 'crosscheck clear' \ + "$model reviewer did not earn a clear result" + assert_grep "--mode json --provider $slot --model $model --thinking xhigh --tools read,bash,grep,find,ls --no-session" \ + "$case_dir/pi.log" \ + "$model reviewer was not invoked on the $slot provider with its pinned model, effort, and tools" + assert_no_grep 'CROSSCHECK DEGRADED' "$case_dir/err" \ + "the $model primary lane announced a degraded fallback" + python3 -c ' import hashlib, json, sys value = json.load(open(sys.argv[1])) +slot, model = sys.argv[3], sys.argv[4] reviewer = value["runs"][-1]["reviewer"] assert reviewer["harness"] == "pi" -assert reviewer["model"] == "FW-GLM-5.2" -assert reviewer["review_family_mode"] == "glm-primary" +assert reviewer["model"] == model +assert reviewer["review_family_mode"] == "cross-family-primary" assert reviewer["account_home"] == sys.argv[2] assert reviewer["executing_account_home"] == sys.argv[2] assert reviewer["account_selector"] == "PI_CODING_AGENT_DIR" -assert reviewer["credential_source"] == "pi-azure-glm-models-file" +assert reviewer["credential_source"] == "pi-" + slot + "-models-file" binding = hashlib.sha256( - b"aif-fm7c799d-eus01/FW-GLM-5.2\n" - b"https://aif-fm7c799d-eus01.cognitiveservices.azure.com/openai/v1" + ("api.fireworks.ai/" + model + "\n" + "https://api.fireworks.ai/inference/v1").encode() ).hexdigest() -assert reviewer["credential_identifier"] == "glm-foundry-binding:" + binding +assert reviewer["credential_identifier"] == "provider-binding:" + slot + ":" + binding assert reviewer["execution_proof"]["actual_exit"] == 0 -' "$case_dir/data/task-x1/crosscheck-ledger.json" "$case_dir/pi-home" \ - || fail "GLM review did not record its bound provider, family mode, and non-secret credential binding" - assert_no_grep 'CODEX FALLBACK' "$case_dir/data/task-x1/crosscheck.md" \ - "a GLM primary review rendered the degraded fallback marker" - pass "the GLM reviewer executes on the azure-glm provider with a non-secret Foundry binding" +' "$case_dir/data/task-x1/crosscheck-ledger.json" "$case_dir/pi-home" "$slot" "$model" \ + || fail "$model review did not record its bound provider, family mode, and non-secret credential binding" + assert_no_grep 'CODEX FALLBACK' "$case_dir/data/task-x1/crosscheck.md" \ + "a $model primary review rendered the degraded fallback marker" + done <<< "$lanes" + pass "every registered cross-family reviewer executes on its own provider slot with a non-secret binding" } -test_glm_credential_binding_is_key_independent() { +test_truncated_cross_family_verdict_is_never_a_verdict() { + local record case_dir base head rc + # GLM-5.2 is a reasoning model: at a tight output budget it spends the whole + # allowance on reasoning and the visible verdict is cut off. Measured live + # against the pinned Fireworks deployment on 2026-08-20: max_tokens=600 + # returned finish_reason=length with empty visible content, while 4000 + # completed. pi maps that finish_reason to stopReason "length". + # + # The body below is a COMPLETE, schema-valid clear verdict. Only the stop + # reason says it was truncated. The run must therefore refuse because a + # truncated turn was ADMITTED as a verdict, not because any text differed: + # a silently truncated verdict is a wrong verdict, and every merge rests on + # this lane. + record=$(make_case truncated-cross-family-verdict) + IFS=$'\t' read -r case_dir base head <<< "$record" + select_cross_family_reviewer "$case_dir" + set +e + FM_TEST_PI_BIN=pi PATH="$case_dir/fakebin:$PATH" \ + FM_TEST_PI_EXPECT_PROVIDER=fireworks-glm \ + FM_TEST_PI_EXPECT_MODEL=accounts/fireworks/models/glm-5p2 \ + FM_TEST_PI_STOP_REASON=length \ + run_case "$case_dir" "$base" "$head" clear run \ + > "$case_dir/out" 2> "$case_dir/err" + rc=$? + set -e + expect_code 1 "$rc" "truncated cross-family verdict" + assert_no_grep 'crosscheck clear' "$case_dir/out" \ + "a truncated reviewer turn was accepted as a clear verdict" + assert_grep "stopReason='length'" "$case_dir/err" \ + "the truncated reviewer turn was not refused by its stop reason" + "$CROSSCHECK_PYTHON" - "$case_dir/data/task-x1/crosscheck-ledger.json" <<'PY' \ + || fail "a truncated review was recorded as anything but a tool failure" +import json +import sys + +value = json.load(open(sys.argv[1])) +run = value["runs"][-1] +assert run["state"] == "tool-failure", run +assert not run["citations"], run +assert "reviewer" not in run or "execution_proof" not in run["reviewer"], run +PY + pass "a truncated reviewer turn is a failed review, never a verdict" +} + +test_cross_family_credential_binding_is_key_independent() { "$CROSSCHECK_PYTHON" - "$CROSSCHECK_PY" "$TMP_ROOT" <<'PY' \ - || fail "GLM credential allowlist or key-independent binding regressed" + || fail "cross-family credential allowlist or key-independent binding regressed" import importlib.util import json from pathlib import Path @@ -1927,20 +2143,27 @@ module = importlib.util.module_from_spec(spec) assert spec.loader is not None spec.loader.exec_module(module) -root = Path(sys.argv[2]) / "glm-credential-binding" +root = Path(sys.argv[2]) / "cross-family-credential-binding" root.mkdir() -PINNED = "https://aif-fm7c799d-eus01.cognitiveservices.azure.com/openai/v1" +PINNED = "https://api.fireworks.ai/inference/v1" +LANE = module.CROSS_FAMILY_LANES["fireworks-glm"] +SLOT = LANE["slot"] +MODEL = LANE["model"] +assert LANE["base_url"] == PINNED, LANE def write_home(name, api_key="key-one", base_url=PINNED, api="openai-completions", - model_id="FW-GLM-5.2", extra_provider=False, model_extra=None): + model_id=MODEL, extra_provider=False, model_extra=None, + slot=SLOT, body=None): + # `name` may contain no path separators; lane ids do, so callers pass + # plain names. home = root / name home.mkdir() - model = {"id": model_id, "name": "GLM 5.2", "reasoning": True} + model = {"id": model_id, "name": "cross-family reviewer", "reasoning": True} model.update(model_extra or {}) providers = { - "azure-glm": { + slot: { "baseUrl": base_url, "api": api, "apiKey": api_key, @@ -1948,24 +2171,27 @@ def write_home(name, api_key="key-one", base_url=PINNED, api="openai-completions } } if extra_provider: - providers["another"] = dict(providers["azure-glm"]) + providers["another"] = dict(providers[slot]) (home / "models.json").write_text( - json.dumps({"providers": providers}), encoding="utf-8" + json.dumps({"providers": providers}) if body is None else body, + encoding="utf-8", ) return home -def expect_tool_failure(home, expected): +def expect_tool_failure(home, expected, lane=LANE): try: - module.inspect_pi_glm_credential(home) + module.inspect_pi_cross_family_credential(home, lane) except module.CrosscheckToolError as exc: assert expected in str(exc), str(exc) - return - raise AssertionError("unusable GLM credential was accepted: " + expected) + return str(exc) + raise AssertionError("unusable cross-family credential was accepted: " + expected) -# The provider mapping is explicit and refuses unmapped models. -assert module.pi_provider_for_model("FW-GLM-5.2") == "azure-glm" +# The provider mapping is explicit, covers every registered lane, and refuses +# unmapped models. +for lane in module.CROSS_FAMILY_LANES.values(): + assert module.pi_provider_for_model(lane["model"]) == lane["slot"], lane assert module.pi_provider_for_model("gpt-5.6-sol") == "openai-codex" try: module.pi_provider_for_model("mystery-model") @@ -1974,50 +2200,67 @@ except module.CrosscheckToolError as exc: else: raise AssertionError("an unmapped Pi model was routed to a guessed provider") +# The lane lookup is keyed on the model, tolerates pi's provider-slot prefix, +# and never claims a codex-family model. Matching is EXACT: a lane model id +# contains slashes, so a suffix rule would admit an unrelated model that +# happens to end the same way. +assert module.cross_family_lane_for_model(MODEL) is LANE +assert module.cross_family_lane_for_model(SLOT + "/" + MODEL) is LANE +assert module.cross_family_lane_for_model("gpt-5.6-sol") is None +assert module.cross_family_lane_for_model("openai-codex-2/gpt-5.6-sol") is None +assert module.cross_family_lane_for_model("glm-5p2") is None +assert module.cross_family_lane_for_model("evil/models/glm-5p2") is None +assert module.cross_family_lane_for_model(None) is None + # Two credentials differing ONLY in api key must expose the identical # non-secret identifier: the binding is resource+deployment+endpoint and is # never derived from the key. first_key, second_key = "key-one-material", "key-two-material" -source_one, identifier_one = module.inspect_pi_glm_credential( - write_home("key-one-home", api_key=first_key) +source_one, identifier_one = module.inspect_pi_cross_family_credential( + write_home("key-one-home", api_key=first_key), LANE ) -source_two, identifier_two = module.inspect_pi_glm_credential( - write_home("key-two-home", api_key=second_key) +source_two, identifier_two = module.inspect_pi_cross_family_credential( + write_home("key-two-home", api_key=second_key), LANE ) -assert source_one == source_two == "pi-azure-glm-models-file" +assert source_one == source_two == "pi-" + SLOT + "-models-file" assert identifier_one == identifier_two, (identifier_one, identifier_two) import hashlib for key in (first_key, second_key): assert key not in identifier_one assert hashlib.sha256(key.encode()).hexdigest() not in identifier_one -expected = "glm-foundry-binding:" + hashlib.sha256( - ("aif-fm7c799d-eus01/FW-GLM-5.2\n" + PINNED).encode() +expected = "provider-binding:" + SLOT + ":" + hashlib.sha256( + ("api.fireworks.ai/" + MODEL + "\n" + PINNED).encode() ).hexdigest() assert identifier_one == expected, identifier_one -# The endpoint is an allowlist with an exact refusal. +# The identity binds the pinned host and model, never the key. +assert module.cross_family_account_identity(LANE) == ( + SLOT + ":api.fireworks.ai/" + MODEL +) + +# The endpoint is an allowlist with an exact refusal, per lane. wrong = write_home( "wrong-endpoint-home", - base_url="https://aif-other.cognitiveservices.azure.com/openai/v1", + base_url="https://api.fireworks.ai.evil.example/inference/v1", ) -expect_tool_failure( +print("REFUSED foreign endpoint: " + expect_tool_failure( wrong, - "GLM reviewer endpoint allowlist refused baseUrl " - "'https://aif-other.cognitiveservices.azure.com/openai/v1'; " + SLOT + " reviewer endpoint allowlist refused baseUrl " + "'https://api.fireworks.ai.evil.example/inference/v1'; " "the only accepted endpoint is " + PINNED, -) +)) # Chat completions only: a Responses-API-shaped configuration is refused. -expect_tool_failure( +print("REFUSED responses api: " + expect_tool_failure( write_home("responses-home", api="openai-responses"), "chat completions only", -) +)) # pi's provider composer gives MODEL-level baseUrl/api precedence over the # provider level (dist/core/provider-composer.js), so a credential keeping # the pinned endpoint at provider level while smuggling an override inside # the model entry must refuse - this is the exact exploit shape. -expect_tool_failure( +print("REFUSED model-level baseUrl+api: " + expect_tool_failure( write_home( "model-override-home", model_extra={ @@ -2026,36 +2269,212 @@ expect_tool_failure( }, ), "model-level baseUrl/api override", -) +)) +# Each field alone is enough: a model entry needs only ONE of them to +# outrank the provider-level pin. +print("REFUSED model-level baseUrl alone: " + expect_tool_failure( + write_home( + "model-baseurl-only-home", + model_extra={"baseUrl": "https://evil.example/openai/v1"}, + ), + "model-level baseUrl/api override", +)) +print("REFUSED model-level api alone: " + expect_tool_failure( + write_home("model-api-only-home", model_extra={"api": "openai-responses"}), + "model-level baseUrl/api override", +)) # Even an override repeating the pinned values refuses: the provider level # must own both fields, and equality today says nothing about tomorrow's # rotation of the pin. -expect_tool_failure( +print("REFUSED model-level repeat of the pin: " + expect_tool_failure( write_home( "model-repeat-home", model_extra={"baseUrl": PINNED, "api": "openai-completions"}, ), "model-level baseUrl/api override", -) +)) -# The credential must declare exactly the azure-glm provider. -expect_tool_failure( +# The credential must declare exactly this lane's provider slot. +print("REFUSED pooled providers: " + expect_tool_failure( write_home("pooled-home", extra_provider=True), - "exactly the azure-glm provider", -) + "exactly the " + SLOT + " provider", +)) +# An unexpected provider slot is refused: the lane comes from the code +# registry, so a credential can never select its own. +print("REFUSED unexpected provider slot: " + expect_tool_failure( + write_home("foreign-slot-home", slot="openai-codex", model_id=MODEL), + "exactly the " + SLOT + " provider", +)) +print("REFUSED retired azure slot: " + expect_tool_failure( + write_home("retired-slot-home", slot="azure-glm", model_id=MODEL), + "exactly the " + SLOT + " provider", +)) + +# `compat` is the other model-level object pi honors, and some of its keys +# weaken this gate's own defenses, so the lane owns it exactly. The pinned +# lane declares none, so any compat at all refuses. +assert LANE["compat"] == {}, LANE +print("REFUSED model-level compat weakening the truncation guard: " + + expect_tool_failure( + write_home("compat-finish-home", + model_extra={"compat": {"supportsFinishReason": False}}), + "model-level compat that is not the pinned lane compat", + )) +print("REFUSED any model-level compat when the lane pins none: " + + expect_tool_failure( + write_home("compat-any-home", + model_extra={"compat": {"supportsDeveloperRole": False}}), + "model-level compat that is not the pinned lane compat", + )) + +# cc-ca5848b19ac3: pi composes the effective model from MORE than the model +# entry - `mergeCompat(providerConfig.compat, definition.compat)` plus a +# topmost `modelOverrides[]` layer carrying compat and headers +# (dist/core/provider-composer.js). Refusing named fields one at a time missed +# both, so the credential shape is an allowlist at every layer. +def write_raw(name, provider_extra=None, document_extra=None): + home = root / name + home.mkdir() + provider = { + "baseUrl": PINNED, + "api": "openai-completions", + "apiKey": "key-one", + "models": [{"id": MODEL, "name": "cross-family reviewer"}], + } + provider.update(provider_extra or {}) + document = {"providers": {SLOT: provider}} + document.update(document_extra or {}) + (home / "models.json").write_text(json.dumps(document), encoding="utf-8") + return home + + +for label, provider_extra in ( + ("provider-level compat", {"compat": {"supportsFinishReason": False}}), + ("modelOverrides compat", {"modelOverrides": {MODEL: {"compat": {"supportsFinishReason": False}}}}), + ("provider-level headers", {"headers": {"x-injected": "1"}}), + ("modelOverrides headers", {"modelOverrides": {MODEL: {"headers": {"x-injected": "1"}}}}), +): + print("REFUSED " + label + ": " + expect_tool_failure( + write_raw("provider-" + label.replace(" ", "-"), provider_extra=provider_extra), + "provider-level fields the lane does not pin", + )) +# An unexpected model-level field is refused by the same allowlist. +print("REFUSED unexpected model-level field: " + expect_tool_failure( + write_home("model-extra-home", model_extra={"headers": {"x-injected": "1"}}), + "model-level fields the lane does not pin", +)) +# And a stray top-level key beside `providers` is refused too. +print("REFUSED stray top-level key: " + expect_tool_failure( + write_raw("top-level-home", document_extra={"modelOverrides": {}}), + 'must be exactly a {"providers": ...} document', +)) +# The shape the operator actually provisions still passes. +module.inspect_pi_cross_family_credential(write_raw("clean-home"), LANE) +print("ACCEPTED the pinned credential shape") # The deployment id must be present. -expect_tool_failure( +print("REFUSED missing deployment: " + expect_tool_failure( write_home("wrong-model-home", model_id="other-model"), - "does not declare the FW-GLM-5.2 deployment", -) + "does not declare the " + MODEL + " deployment", +)) + +# A malformed models file fails closed rather than reading as an empty one. +print("REFUSED malformed models file: " + expect_tool_failure( + write_home("malformed-home", body="{not json"), + "reviewer credential", +)) # A missing models.json is refused by name. missing = root / "missing-home" missing.mkdir() -expect_tool_failure(missing, "GLM reviewer credential inspection failed at") +print("REFUSED missing models file: " + expect_tool_failure( + missing, SLOT + " reviewer credential inspection failed at" +)) PY - pass "the GLM credential pins the endpoint allowlist and binds identity without the api key" + pass "the cross-family credential pins each lane's endpoint allowlist and binds identity without the api key" +} + +test_cross_family_family_marker_is_bound_to_the_reviewer_model() { + "$CROSSCHECK_PYTHON" - "$CROSSCHECK_PY" "$TMP_ROOT" <<'PY' \ + || fail "review_family_mode is no longer bound to the reviewer model" +import importlib.util +import sys + +spec = importlib.util.spec_from_file_location("fm_crosscheck", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +assert spec.loader is not None +spec.loader.exec_module(module) + +BASE = { + "state": "cannot-certify", + "at": "2026-08-20T00:00:00Z", + "head_sha": "a" * 40, + "base_sha": "b" * 40, + "claims_sha256": "c" * 64, + "summary": "s", + "citations": [], + "updated_findings": [], + "new_findings": [], + "active_blockers": [], + "suspicions": [], +} + + +URL = "https://github.com/o/r/pull/1" + + +def ledger(model, family): + run = dict(BASE) + run["reviewer"] = {"model": model, "review_family_mode": family} + return { + "schema": module.SCHEMA, + "task_id": "task-x1", + "pull_request": URL, + "findings": [], + "runs": [run], + } + + +def expect_refused(model, family, expected="does not match the reviewer model"): + try: + module.validate_ledger(ledger(model, family), "task-x1", URL) + except module.CrosscheckError as exc: + assert expected in str(exc), str(exc) + return str(exc) + raise AssertionError(f"validate_ledger admitted {family!r} for {model!r}") + + +# Every registered cross-family lane may claim the primary marker. +for lane in module.CROSS_FAMILY_LANES.values(): + module.validate_ledger( + ledger(lane["model"], "cross-family-primary"), "task-x1", URL + ) + module.validate_ledger( + ledger(lane["slot"] + "/" + lane["model"], "cross-family-primary"), + "task-x1", + URL, + ) +# The codex fallback may not. +print("REFUSED forged primary: " + expect_refused("gpt-5.6-sol", "cross-family-primary")) +# And a cross-family reviewer may not hide behind the fallback marker. +LANE_MODEL = next(iter(module.CROSS_FAMILY_LANES.values()))["model"] +print("REFUSED hidden primary: " + expect_refused(LANE_MODEL, "codex-fallback")) + +# The legacy glm-primary value stays readable for durable ledgers written +# before the registry landed, bound to exactly the retired Azure lane model +# that recorded it. It is not a synonym for any primary, and because that +# model is no longer registered, a NEW run can never claim it either. +module.validate_ledger(ledger("FW-GLM-5.2", "glm-primary"), "task-x1", URL) +module.validate_ledger(ledger("azure-glm/FW-GLM-5.2", "glm-primary"), "task-x1", URL) +assert module.cross_family_lane_for_model("FW-GLM-5.2") is None +print("REFUSED legacy marker on the live lane: " + expect_refused( + LANE_MODEL, "glm-primary" +)) +print("REFUSED unknown family: " + expect_refused( + LANE_MODEL, "glm-5p2-primary", "review_family_mode is invalid" +)) +PY + pass "review_family_mode stays bound to the reviewer model in both directions across every lane" } test_codex_fallback_family_is_loud_and_recorded() { @@ -2070,7 +2489,7 @@ test_codex_fallback_family_is_loud_and_recorded() { || fail "fallback reviewer did not complete" assert_contains "$output" 'crosscheck clear' \ "fallback reviewer did not produce a verdict" - assert_grep 'CROSSCHECK DEGRADED: codex-family fallback reviewer pi gpt-5.6-sol is standing in for the GLM-5.2 primary lane; crosscheck-same-model relaxation was not required' \ + assert_grep 'CROSSCHECK DEGRADED: codex-family fallback reviewer pi gpt-5.6-sol is standing in for the cross-family primary lane; crosscheck-same-model relaxation was not required' \ "$case_dir/err" \ "the codex-family fallback did not announce itself with the exact degraded warning" python3 -c ' @@ -2092,8 +2511,8 @@ assert "model_independence" not in reviewer, reviewer mkdir -p "$case_dir/home/config" printf 'on\n' > "$case_dir/home/config/crosscheck-same-model" sed -i.bak \ - -e 's/harness=codex/harness=pi/' \ - -e 's#model=gpt-5.5#model=openai-codex-5/gpt-5.6-sol#' \ + -e 's/harness=claude/harness=pi/' \ + -e 's#model=claude-opus-5#model=openai-codex-5/gpt-5.6-sol#' \ -e '/^account_home=/d' \ "$case_dir/state/task-x1.meta" rm "$case_dir/state/task-x1.meta.bak" @@ -2101,7 +2520,7 @@ assert "model_independence" not in reviewer, reviewer || fail "same-model fallback reviewer did not complete" assert_contains "$output" 'crosscheck clear' \ "same-model fallback reviewer did not produce a verdict" - assert_grep 'CROSSCHECK DEGRADED: codex-family fallback reviewer codex gpt-5.6-sol is standing in for the GLM-5.2 primary lane; crosscheck-same-model relaxation was required' \ + assert_grep 'CROSSCHECK DEGRADED: codex-family fallback reviewer codex gpt-5.6-sol is standing in for the cross-family primary lane; crosscheck-same-model relaxation was required' \ "$case_dir/err" \ "the same-model fallback did not name the required relaxation in its warning" python3 -c ' @@ -2161,19 +2580,26 @@ def ledger_with(model, family): } -# Honest pairings load; the field also remains optional for older ledgers. +# Honest pairings load; the field also remains optional for older ledgers, +# and the legacy glm-primary value stays readable for durable records. for model, family in ( + ("accounts/fireworks/models/glm-5p2", "cross-family-primary"), + ("fireworks-glm/accounts/fireworks/models/glm-5p2", "cross-family-primary"), ("FW-GLM-5.2", "glm-primary"), - ("gpt-5.6-sol", "codex-fallback"), ("azure-glm/FW-GLM-5.2", "glm-primary"), + ("gpt-5.6-sol", "codex-fallback"), ("gpt-5.6-sol", None), ): module.validate_ledger(ledger_with(model, family), "task-x1", URL) -# Forged pairings refuse, in both directions. +# Forged pairings refuse, in both directions, and the legacy value cannot be +# reused as a synonym for a different lane. for model, family in ( + ("gpt-5.6-sol", "cross-family-primary"), ("gpt-5.6-sol", "glm-primary"), - ("FW-GLM-5.2", "codex-fallback"), + ("accounts/fireworks/models/glm-5p2", "codex-fallback"), + ("accounts/fireworks/models/glm-5p2", "glm-primary"), + ("FW-GLM-5.2", "cross-family-primary"), ): try: module.validate_ledger(ledger_with(model, family), "task-x1", URL) @@ -2192,8 +2618,8 @@ test_same_model_review_is_adversarial_and_durable() { mkdir -p "$case_dir/home/config" printf 'on\n' > "$case_dir/home/config/crosscheck-same-model" sed -i.bak \ - -e 's/harness=codex/harness=pi/' \ - -e 's#model=gpt-5.5#model=openai-codex-5/gpt-5.6-sol#' \ + -e 's/harness=claude/harness=pi/' \ + -e 's#model=claude-opus-5#model=openai-codex-5/gpt-5.6-sol#' \ -e '/^account_home=/d' \ "$case_dir/state/task-x1.meta" rm "$case_dir/state/task-x1.meta.bak" @@ -2442,7 +2868,7 @@ assert reviewer["credential_identifier"] == sys.argv[2] # account identities: this case exists to exercise the launch-time Codex # credential preflight, and a same-provider pair would now be refused at # selection before the reviewer is ever bound. - sed -i.bak -e 's/harness=codex/harness=claude/' -e 's/model=gpt-5.5/model=claude-opus-5/' \ + sed -i.bak -e 's/harness=claude/harness=claude/' -e 's/model=claude-opus-5/model=claude-opus-5/' \ "$case_dir/state/task-x1.meta" rm "$case_dir/state/task-x1.meta.bak" set +e @@ -3966,7 +4392,7 @@ test_reviewer_configuration_failures_are_tool_failures() { case "$mode" in absent) rm "$case_dir/reviewer.json" ;; same-model) - sed -i.bak 's/model=gpt-5.5/model=gpt-5.6-sol/' "$case_dir/state/task-x1.meta" + sed -i.bak -e 's/harness=claude/harness=codex/' -e 's/model=claude-opus-5/model=gpt-5.6-sol/' "$case_dir/state/task-x1.meta" rm "$case_dir/state/task-x1.meta.bak" ;; esac @@ -4798,8 +5224,10 @@ if [ -n "${FM_TEST_CASE:-}" ]; then test_clear_review_uses_policy_contract|\ test_missing_author_identity_reaches_normal_verdict|\ test_claude_reviewer_profile_is_retired|\ - test_glm_reviewer_executes_bound_policy_profile|\ - test_glm_credential_binding_is_key_independent|\ + test_cross_family_reviewer_executes_bound_policy_profile|\ + test_truncated_cross_family_verdict_is_never_a_verdict|\ + test_cross_family_credential_binding_is_key_independent|\ + test_cross_family_family_marker_is_bound_to_the_reviewer_model|\ test_codex_fallback_family_is_loud_and_recorded|\ test_same_model_review_is_adversarial_and_durable|\ test_empty_runtime_overrides_use_home_defaults|\ @@ -4915,8 +5343,10 @@ test_pi_reviewer_failures_are_tool_failures test_clear_review_uses_policy_contract test_missing_author_identity_reaches_normal_verdict test_claude_reviewer_profile_is_retired -test_glm_reviewer_executes_bound_policy_profile -test_glm_credential_binding_is_key_independent +test_cross_family_reviewer_executes_bound_policy_profile +test_truncated_cross_family_verdict_is_never_a_verdict +test_cross_family_credential_binding_is_key_independent +test_cross_family_family_marker_is_bound_to_the_reviewer_model test_codex_fallback_family_is_loud_and_recorded test_same_model_review_is_adversarial_and_durable test_empty_runtime_overrides_use_home_defaults