diff --git a/README.md b/README.md index 6f965e1..31222cc 100644 --- a/README.md +++ b/README.md @@ -832,16 +832,21 @@ flightrecorder schemas --check runs/promotion_release_record.json `promotion-decision` is side-effect free. It emits an alias-update receipt only when every required artifact is present and fingerprinted, every gate passes, the rollback target is declared by a valid rollback receipt, license status is -known, cards have no TODO/TBD/unsupported-claim markers, and eval movement shows -no task-completion regressions, new critical failures, forbidden actions, secret -exposure, contract drift, or unverified contracts. +known, cards have no TODO/TBD/unsupported-claim markers, the serving/eval report +proves `base`, `trace-only`, `frontier`, `champion`, and `candidate` arms on the +same held-out scenario set, and eval movement shows no task-completion +regressions, new critical failures, forbidden actions, secret exposure, contract +drift, or unverified contracts. `promotion-rollback-receipt` is also side-effect free: it fingerprints the model registry, proves the rollback target is registered, and blocks when the target no longer matches the current champion before promotion. `--promotion-policy` makes the required artifact contract and zero-tolerance limits explicit. A policy may document or tighten governance expectations, but it cannot relax the default blockers for missing artifacts, unknown license, -unsafe eval movement, unsupported claims, rollback, cards, or validation. +unsafe eval movement, unsupported claims, rollback, cards, required comparison +arms, or validation. Use an `eval-summary` or `serving_demo_run` artifact for +`--serving-report`; generic pass/fail serving receipts block promotion unless +they declare the required arms and identical held-out coverage. `promotion-alias-apply` performs the guarded registry write after validating that receipt. The model registry must use `hfr.model_registry.v1`, register all alias targets, expose aliases as an object, and have a missing or list-valued diff --git a/TRAINING_PIPELINE.md b/TRAINING_PIPELINE.md index 74304d5..af1afa6 100644 --- a/TRAINING_PIPELINE.md +++ b/TRAINING_PIPELINE.md @@ -399,10 +399,12 @@ flightrecorder schemas --check runs/promotion_release_record.json The decision blocks promotion on missing evidence, unknown license status, redaction or safety failure, missing cards, missing rollback metadata, failed -rollback receipts, eval mismatch, task-completion regression, new critical -failures, secret exposure, forbidden actions, and unsupported card claims. A -passing decision is still side-effect free: it authorizes an alias-update -receipt, leaving the actual registry write to a later guarded step. +rollback receipts, eval mismatch, missing `base`/`trace-only`/`frontier`/ +`champion`/`candidate` comparison arms, non-identical held-out scenarios, +task-completion regression, new critical failures, secret exposure, forbidden +actions, and unsupported card claims. A passing decision is still side-effect +free: it authorizes an alias-update receipt, leaving the actual registry write +to a later guarded step. `promotion-rollback-receipt` is side-effect free: it fingerprints the model registry, proves the rollback target is registered, and blocks when the target no longer matches the current champion before promotion. @@ -410,7 +412,10 @@ no longer matches the current champion before promotion. decision/release artifact contract, allowed model classes, zero-tolerance eval limits, required forbidden-rule blockers, license, rollback, card, and validation requirements. Policy files can make expectations reviewable but -cannot relax the default promotion blockers. +cannot relax the default promotion blockers or drop required comparison arms. +Use `eval-summary` or `serving_demo_run` output for `--serving-report`; a +generic pass/fail serving receipt blocks promotion unless it declares every +required arm and identical held-out coverage. `promotion-alias-apply` is that guarded write: it revalidates the promotion decision, requires a `hfr.model_registry.v1` registry with registered `candidate`, `champion`, and `rollback` targets, verifies the live champion diff --git a/examples/promotion_policy.demo.json b/examples/promotion_policy.demo.json index ea2e488..0eb8c76 100644 --- a/examples/promotion_policy.demo.json +++ b/examples/promotion_policy.demo.json @@ -39,6 +39,13 @@ "compare_gate", "release_notes" ], + "required_comparison_arms": [ + "base", + "trace-only", + "frontier", + "champion", + "candidate" + ], "require_accepted_terms": true, "require_artifact_validation": true, "require_known_license": true, diff --git a/flightrecorder/governance.py b/flightrecorder/governance.py index e4f2c32..cb6f462 100644 --- a/flightrecorder/governance.py +++ b/flightrecorder/governance.py @@ -56,6 +56,7 @@ "allowed_candidate_classes", "allowed_champion_classes", "limits", + "required_comparison_arms", "forbid_new_critical_rules", "forbid_regressed_rules", "require_known_license", @@ -73,6 +74,7 @@ "max_rule_regressions": 0, } PROMOTION_POLICY_REQUIRED_FORBIDDEN_RULES = ("forbidden_actions", "secret_exposure") +PROMOTION_POLICY_REQUIRED_COMPARISON_ARMS = ("base", "trace-only", "frontier", "champion", "candidate") _JSON_ARTIFACT_ROLES = { "evidence_bundle": EVIDENCE_BUNDLE_SCHEMA_VERSION, "promotion_ledger_gate": PROMOTION_LEDGER_GATE_SCHEMA_VERSION, @@ -204,6 +206,8 @@ def build_promotion_decision( _add_schema_check(checks, "trainer_launch_check", json_artifacts.get("trainer_launch_check")) for role in _PASSED_JSON_ROLES: _add_passed_json_check(checks, role, json_artifacts.get(role)) + comparison_arms = _comparison_arm_summary(json_artifacts.get("serving_report"), policy["required_comparison_arms"]) + _add_comparison_arm_checks(checks, comparison_arms, policy["required_comparison_arms"]) compare_metrics = _metrics_object(json_artifacts.get("compare_gate")) limits = policy["limits"] @@ -257,7 +261,7 @@ def build_promotion_decision( failed_checks = sum(1 for check in checks if not check["passed"]) passed = failed_checks == 0 - metrics = _decision_metrics(checks, compare_metrics, policy) + metrics = _decision_metrics(checks, compare_metrics, policy, comparison_arms) decision = { "readiness": "ready" if passed else "blocked", "recommendation": "apply_alias_update" if passed else "block_promotion", @@ -287,6 +291,7 @@ def build_promotion_decision( "checks": checks, "artifacts": artifacts, "policy": _promotion_policy_output(policy, policy_artifact), + "comparison_arms": comparison_arms, "metrics": metrics, "alias_update": _alias_update(passed, candidate_id, champion_id, rollback_id or ""), "notes": [ @@ -1245,6 +1250,7 @@ def _default_promotion_policy() -> dict[str, Any]: "allowed_candidate_classes": sorted(MODEL_CLASSES), "allowed_champion_classes": sorted(MODEL_CLASSES), "limits": dict(PROMOTION_POLICY_DEFAULT_LIMITS), + "required_comparison_arms": list(PROMOTION_POLICY_REQUIRED_COMPARISON_ARMS), "forbid_new_critical_rules": list(PROMOTION_POLICY_REQUIRED_FORBIDDEN_RULES), "forbid_regressed_rules": list(PROMOTION_POLICY_REQUIRED_FORBIDDEN_RULES), "requirements": { @@ -1279,6 +1285,7 @@ def _load_promotion_policy(path: Path | None, preserve_paths: bool) -> dict[str, policy["release_required_artifacts"] = _policy_string_list(payload, "release_required_artifacts", parse_errors) policy["allowed_candidate_classes"] = _policy_string_list(payload, "allowed_candidate_classes", parse_errors) policy["allowed_champion_classes"] = _policy_string_list(payload, "allowed_champion_classes", parse_errors) + policy["required_comparison_arms"] = _policy_string_list(payload, "required_comparison_arms", parse_errors) policy["forbid_new_critical_rules"] = _policy_string_list(payload, "forbid_new_critical_rules", parse_errors) policy["forbid_regressed_rules"] = _policy_string_list(payload, "forbid_regressed_rules", parse_errors) limits = payload.get("limits") @@ -1389,6 +1396,22 @@ def _add_promotion_policy_checks( scope={"field": "champion_class"}, summary="champion class is allowed by the promotion policy", ) + required_arms = set(policy.get("required_comparison_arms", [])) + missing_default_arms = sorted(set(PROMOTION_POLICY_REQUIRED_COMPARISON_ARMS) - required_arms) + unknown_arms = sorted(required_arms - MODEL_CLASSES) + _add_check( + checks, + "promotion_policy_comparison_arms_complete", + not missing_default_arms and not unknown_arms, + actual={ + "required_comparison_arms": sorted(required_arms), + "missing_default_arms": missing_default_arms, + "unknown_arms": unknown_arms, + }, + expected={"required_comparison_arms": list(PROMOTION_POLICY_REQUIRED_COMPARISON_ARMS)}, + scope={"artifact_role": "promotion_policy"}, + summary="promotion policy requires base, trace-only, frontier, champion, and candidate comparison arms", + ) limits = policy.get("limits") if isinstance(policy.get("limits"), dict) else {} relaxed_limits = { field_name: limits.get(field_name) @@ -1464,6 +1487,7 @@ def _promotion_policy_output(policy: dict[str, Any], artifact: dict[str, Any] | "allowed_candidate_classes": list(policy.get("allowed_candidate_classes", [])), "allowed_champion_classes": list(policy.get("allowed_champion_classes", [])), "limits": dict(policy.get("limits", {})), + "required_comparison_arms": list(policy.get("required_comparison_arms", [])), "forbid_new_critical_rules": list(policy.get("forbid_new_critical_rules", [])), "forbid_regressed_rules": list(policy.get("forbid_regressed_rules", [])), "requirements": dict(policy.get("requirements", {})), @@ -1592,6 +1616,91 @@ def _add_card_claims_check(checks: list[dict[str, Any]], role: str, path: Path | ) +def _comparison_arm_summary(payload: dict[str, Any] | None, required_arms: list[str]) -> dict[str, Any]: + arms = sorted(_comparison_arm_labels(payload)) + heldout_identical = _comparison_arms_heldout_identical(payload) + required = sorted(dict.fromkeys(required_arms)) + return { + "required": required, + "evidenced": arms, + "missing": sorted(set(required) - set(arms)), + "extra": sorted(set(arms) - set(required)), + "heldout_identical": heldout_identical, + "source_schema_version": payload.get("schema_version") if isinstance(payload, dict) else None, + } + + +def _comparison_arm_labels(payload: dict[str, Any] | None) -> set[str]: + if not isinstance(payload, dict): + return set() + labels: set[str] = set() + for field_name in ("arm", "candidate_arm"): + value = payload.get(field_name) + if isinstance(value, str) and value: + labels.add(value) + labels.update(_arm_labels_from_rows(payload.get("arms"))) + heldout = payload.get("heldout_scenarios") if isinstance(payload.get("heldout_scenarios"), dict) else {} + labels.update(_arm_labels_from_rows(heldout.get("arms"))) + scenario_sets = payload.get("scenario_sets") + if isinstance(scenario_sets, dict): + labels.update(str(label) for label in scenario_sets if isinstance(label, str) and label) + return labels + + +def _arm_labels_from_rows(rows: Any) -> set[str]: + labels: set[str] = set() + if not isinstance(rows, list): + return labels + for row in rows: + if not isinstance(row, dict): + continue + for field_name in ("label", "name", "arm", "id"): + value = row.get(field_name) + if isinstance(value, str) and value: + labels.add(value) + break + return labels + + +def _comparison_arms_heldout_identical(payload: dict[str, Any] | None) -> bool | None: + if not isinstance(payload, dict): + return None + heldout = payload.get("heldout_scenarios") if isinstance(payload.get("heldout_scenarios"), dict) else {} + if isinstance(heldout.get("cross_arm_claims_allowed"), bool): + return heldout["cross_arm_claims_allowed"] + if isinstance(heldout.get("identical"), bool): + return heldout["identical"] + if isinstance(payload.get("same_scenario_ids"), bool): + return payload["same_scenario_ids"] + return None + + +def _add_comparison_arm_checks( + checks: list[dict[str, Any]], + comparison_arms: dict[str, Any], + required_arms: list[str], +) -> None: + missing = comparison_arms.get("missing") if isinstance(comparison_arms.get("missing"), list) else [] + _add_check( + checks, + "required_comparison_arms_present", + not missing, + actual={"evidenced": comparison_arms.get("evidenced", []), "missing": missing}, + expected={"required": sorted(dict.fromkeys(required_arms))}, + scope={"artifact_role": "serving_report"}, + summary="serving/eval report proves required base, trace-only, frontier, champion, and candidate arms", + ) + _add_check( + checks, + "comparison_arms_identical_heldout", + comparison_arms.get("heldout_identical") is True, + actual={"heldout_identical": comparison_arms.get("heldout_identical")}, + expected={"heldout_identical": True}, + scope={"artifact_role": "serving_report"}, + summary="required comparison arms use identical held-out scenarios before promotion claims are trusted", + ) + + def _add_max_count_check(checks: list[dict[str, Any]], check_id: str, value: Any, maximum: int) -> None: actual = _int_value(value) _add_check( @@ -1651,13 +1760,22 @@ def _add_check( checks.append(check) -def _decision_metrics(checks: list[dict[str, Any]], compare_metrics: dict[str, Any], policy: dict[str, Any]) -> dict[str, Any]: +def _decision_metrics( + checks: list[dict[str, Any]], + compare_metrics: dict[str, Any], + policy: dict[str, Any], + comparison_arms: dict[str, Any], +) -> dict[str, Any]: return { "check_count": len(checks), "failed_check_count": sum(1 for check in checks if not check["passed"]), "required_artifact_count": len(PROMOTION_DECISION_REQUIRED_ARTIFACTS), "policy_required_artifact_count": len(policy.get("required_artifacts", [])), "policy_release_required_artifact_count": len(policy.get("release_required_artifacts", [])), + "required_comparison_arm_count": len(policy.get("required_comparison_arms", [])), + "evidenced_comparison_arm_count": len(comparison_arms.get("evidenced", [])) + if isinstance(comparison_arms.get("evidenced"), list) + else 0, "task_completion_regression_count": _int_value(compare_metrics.get("task_completion_regression_count")), "baseline_win_count": _int_value(compare_metrics.get("baseline_win_count")), "contract_drift_count": _int_value(compare_metrics.get("contract_drift_count")), diff --git a/flightrecorder/schemas/manifest.json b/flightrecorder/schemas/manifest.json index 42e04e4..9663380 100644 --- a/flightrecorder/schemas/manifest.json +++ b/flightrecorder/schemas/manifest.json @@ -508,7 +508,7 @@ { "artifact": "promotion_policy", "artifact_schema_version": "hfr.promotion_policy.v1", - "description": "Promotion governance policy declaring required evidence, model classes, safety limits, forbidden-rule blockers, and release artifact contracts.", + "description": "Promotion governance policy declaring required evidence, model classes, comparison arms, safety limits, forbidden-rule blockers, and release artifact contracts.", "filename": "promotion_policy.v1.schema.json", "id": "https://schemas.hermes-flight-recorder.dev/promotion_policy.v1.schema.json", "name": "promotion_policy" diff --git a/flightrecorder/schemas/promotion_policy.v1.schema.json b/flightrecorder/schemas/promotion_policy.v1.schema.json index fffb673..487a325 100644 --- a/flightrecorder/schemas/promotion_policy.v1.schema.json +++ b/flightrecorder/schemas/promotion_policy.v1.schema.json @@ -54,6 +54,15 @@ ], "type": "object" }, + "required_comparison_arms": { + "items": { + "enum": ["base", "candidate", "champion", "frontier", "trace-only"], + "type": "string" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + }, "release_required_artifacts": { "items": { "enum": [ @@ -107,6 +116,7 @@ "allowed_candidate_classes", "allowed_champion_classes", "limits", + "required_comparison_arms", "forbid_new_critical_rules", "forbid_regressed_rules", "require_known_license", diff --git a/flightrecorder/validation.py b/flightrecorder/validation.py index 32e2319..90056d4 100644 --- a/flightrecorder/validation.py +++ b/flightrecorder/validation.py @@ -52,6 +52,7 @@ PROMOTION_CARDS_SCHEMA_VERSION, PROMOTION_DECISION_REQUIRED_ARTIFACTS, PROMOTION_DECISION_SCHEMA_VERSION, + PROMOTION_POLICY_REQUIRED_COMPARISON_ARMS, PROMOTION_POLICY_SCHEMA_VERSION, PROMOTION_ROLLBACK_RECEIPT_SCHEMA_VERSION, PROMOTION_RELEASE_RECORD_REQUIRED_ARTIFACTS, @@ -8328,6 +8329,7 @@ def _validate_raw_promotion_policy(policy: dict[str, Any], target: ValidationTar _validate_policy_role_list(policy.get("release_required_artifacts"), target, "promotion_policy.release_required_artifacts") _validate_policy_class_list(policy.get("allowed_candidate_classes"), target, "promotion_policy.allowed_candidate_classes") _validate_policy_class_list(policy.get("allowed_champion_classes"), target, "promotion_policy.allowed_champion_classes") + _validate_policy_comparison_arm_list(policy.get("required_comparison_arms"), target, "promotion_policy.required_comparison_arms") _validate_policy_limits(policy.get("limits"), target, "promotion_policy.limits") _validate_policy_forbidden_rules(policy.get("forbid_new_critical_rules"), target, "promotion_policy.forbid_new_critical_rules") _validate_policy_forbidden_rules(policy.get("forbid_regressed_rules"), target, "promotion_policy.forbid_regressed_rules") @@ -8362,6 +8364,7 @@ def _validate_promotion_policy_section(value: Any, target: ValidationTarget, sou ) _validate_policy_class_list(value.get("allowed_candidate_classes"), target, f"{label}.allowed_candidate_classes") _validate_policy_class_list(value.get("allowed_champion_classes"), target, f"{label}.allowed_champion_classes") + _validate_policy_comparison_arm_list(value.get("required_comparison_arms"), target, f"{label}.required_comparison_arms") _validate_policy_limits(value.get("limits"), target, f"{label}.limits") _validate_policy_forbidden_rules(value.get("forbid_new_critical_rules"), target, f"{label}.forbid_new_critical_rules") _validate_policy_forbidden_rules(value.get("forbid_regressed_rules"), target, f"{label}.forbid_regressed_rules") @@ -8392,6 +8395,15 @@ def _validate_policy_class_list(value: Any, target: ValidationTarget, label: str target.errors.append(f"{label} contains unknown model classes: {unknown!r}.") +def _validate_policy_comparison_arm_list(value: Any, target: ValidationTarget, label: str) -> None: + _validate_policy_class_list(value, target, label) + if not _is_string_list(value): + return + missing = sorted(set(PROMOTION_POLICY_REQUIRED_COMPARISON_ARMS) - set(value)) + if missing: + target.errors.append(f"{label} must include required comparison arms: {missing!r}.") + + def _validate_policy_limits(value: Any, target: ValidationTarget, label: str) -> None: if not isinstance(value, dict): target.errors.append(f"{label} must be an object.") @@ -8523,6 +8535,9 @@ def _validate_promotion_decision_metrics(value: Any, checks: list[Any], target: "policy_release_required_artifact_count": ( len(policy_obj.get("release_required_artifacts", [])) if isinstance(policy_obj.get("release_required_artifacts"), list) else 0 ), + "required_comparison_arm_count": ( + len(policy_obj.get("required_comparison_arms", [])) if isinstance(policy_obj.get("required_comparison_arms"), list) else 0 + ), } for field_name, expected in expected_fields.items(): if value.get(field_name) != expected: @@ -8534,6 +8549,7 @@ def _validate_promotion_decision_metrics(value: Any, checks: list[Any], target: "unverified_contract_count", "new_critical_failure_count", "rule_regression_count", + "evidenced_comparison_arm_count", ): if not _is_non_negative_int(value.get(field_name)): target.errors.append(f"promotion_decision.metrics.{field_name} must be a non-negative integer.") diff --git a/tests/test_promotion_decision.py b/tests/test_promotion_decision.py index 28284f7..fd457ef 100644 --- a/tests/test_promotion_decision.py +++ b/tests/test_promotion_decision.py @@ -379,6 +379,55 @@ def test_promotion_decision_blocks_incomplete_policy_contract(self): self.assertFalse(decision["alias_update"]["authorized"]) self.assertEqual(run_cli(["validate", "--promotion-decision", str(decision_path), "--strict"]), 0) + def test_promotion_decision_blocks_policy_that_omits_comparison_arm(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + artifacts = write_governance_artifacts(root) + decision_path = root / "promotion_decision.json" + policy_path = write_promotion_policy( + root, + required_comparison_arms=["base", "trace-only", "champion", "candidate"], + ) + + code = run_cli(promotion_decision_args(artifacts, decision_path, promotion_policy=policy_path)) + + self.assertEqual(code, 1) + decision = json.loads(decision_path.read_text(encoding="utf-8")) + self.assertIn("promotion_policy_comparison_arms_complete", failed_check_ids(decision)) + self.assertFalse(decision["alias_update"]["authorized"]) + self.assertEqual(run_cli(["validate", "--promotion-decision", str(decision_path), "--strict"]), 1) + + def test_promotion_decision_blocks_missing_required_comparison_arm(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + artifacts = write_governance_artifacts(root) + write_serving_report(root, arms=["base", "trace-only", "champion", "candidate"]) + decision_path = root / "promotion_decision.json" + + code = run_cli(promotion_decision_args(artifacts, decision_path)) + + self.assertEqual(code, 1) + decision = json.loads(decision_path.read_text(encoding="utf-8")) + self.assertIn("required_comparison_arms_present", failed_check_ids(decision)) + self.assertIn("frontier", decision["comparison_arms"]["missing"]) + self.assertFalse(decision["alias_update"]["authorized"]) + self.assertEqual(run_cli(["validate", "--promotion-decision", str(decision_path), "--strict"]), 0) + + def test_promotion_decision_blocks_nonidentical_comparison_arms(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + artifacts = write_governance_artifacts(root) + write_serving_report(root, heldout_identical=False) + decision_path = root / "promotion_decision.json" + + code = run_cli(promotion_decision_args(artifacts, decision_path)) + + self.assertEqual(code, 1) + decision = json.loads(decision_path.read_text(encoding="utf-8")) + self.assertIn("comparison_arms_identical_heldout", failed_check_ids(decision)) + self.assertFalse(decision["alias_update"]["authorized"]) + self.assertEqual(run_cli(["validate", "--promotion-decision", str(decision_path), "--strict"]), 0) + def test_promotion_decision_blocks_eval_regressions_and_secret_exposure(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -462,8 +511,7 @@ def write_governance_artifacts(root: Path, *, license_status: str = "known", com redaction_check.write_text(json.dumps({"passed": True}), encoding="utf-8") safety_gate = root / "safety_gate.json" safety_gate.write_text(json.dumps({"passed": True}), encoding="utf-8") - serving_report = root / "serving_report.json" - serving_report.write_text(json.dumps({"passed": True}), encoding="utf-8") + serving_report = write_serving_report(root) return { "evidence_bundle": evidence_bundle, "promotion_ledger_gate": promotion_ledger_gate, @@ -479,6 +527,53 @@ def write_governance_artifacts(root: Path, *, license_status: str = "known", com } +def write_serving_report( + root: Path, + *, + arms: list[str] | None = None, + heldout_identical: bool = True, +) -> Path: + arm_labels = arms or required_comparison_arms() + serving_report = root / "serving_report.json" + serving_report.write_text( + json.dumps( + { + "schema_version": "hfr.eval_summary.v1", + "passed": True, + "governance_ready": True, + "arms": [ + { + "label": arm, + "scenario_count": 2, + "scenario_ids": ["heldout-1", "heldout-2"], + "blocking_reasons": [], + } + for arm in arm_labels + ], + "heldout_scenarios": { + "status": "identical" if heldout_identical else "mismatched", + "identical": heldout_identical, + "cross_arm_claims_allowed": heldout_identical, + "arms": [ + { + "label": arm, + "scenario_count": 2, + "scenario_ids": ["heldout-1", "heldout-2"] if heldout_identical else [f"{arm}-only"], + } + for arm in arm_labels + ], + "blocking_reasons": [] if heldout_identical else ["heldout_scenario_set_mismatch"], + }, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + return serving_report + + def write_training_export(root: Path) -> Path: training_export = root / "training_export" training_export.mkdir() @@ -529,6 +624,7 @@ def write_promotion_policy( policy_id: str = "strict-local-policy", required_artifacts: list[str] | None = None, release_required_artifacts: list[str] | None = None, + required_comparison_arms: list[str] | None = None, ) -> Path: policy_path = root / filename policy_path.write_text( @@ -539,6 +635,7 @@ def write_promotion_policy( "description": "Strict local promotion policy for tests.", "required_artifacts": required_artifacts or promotion_decision_required_artifacts(), "release_required_artifacts": release_required_artifacts or promotion_release_required_artifacts(), + "required_comparison_arms": required_comparison_arms or required_comparison_arms_default(), "allowed_candidate_classes": ["base", "candidate", "champion", "frontier", "trace-only"], "allowed_champion_classes": ["base", "candidate", "champion", "frontier", "trace-only"], "limits": { @@ -593,6 +690,14 @@ def promotion_release_required_artifacts() -> list[str]: ] +def required_comparison_arms() -> list[str]: + return ["base", "trace-only", "frontier", "champion", "candidate"] + + +def required_comparison_arms_default() -> list[str]: + return required_comparison_arms() + + def promotion_alias_apply_args(registry_path: Path, decision_path: Path, receipt_path: Path) -> list[str]: return [ "promotion-alias-apply",