diff --git a/bin/fm-azure-validation-guest.sh b/bin/fm-azure-validation-guest.sh index 8c02ac073d4..287a37541bf 100755 --- a/bin/fm-azure-validation-guest.sh +++ b/bin/fm-azure-validation-guest.sh @@ -305,13 +305,30 @@ auth_home_pull() { elif [ "${pulled:-0}" -eq 0 ]; then # First-ever boot against an empty share: proceed, but leave a durable # marker so the operator knows one interactive auth is still needed. - printf 'auth share %s was empty at %s; interactive provider auth is needed once\n' \ - "$AUTH_SHARE" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$STATE/auth-needed" - chmod 0600 "$STATE/auth-needed" + { printf 'auth share %s was empty at %s; interactive provider auth is needed once\n' \ + "$AUTH_SHARE" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$STATE/auth-needed" \ + && chmod 0600 "$STATE/auth-needed"; } || : echo "validation guest: auth share is empty; interactive auth marker written" >&2 else - rm -f "$STATE/auth-needed" + rm -f "$STATE/auth-needed" || : fi + # From here on this cell owns auth the share has not seen, whether it came + # from the share, from the seeded bundle after a failed pull, or from a first + # interactive auth against an empty share. The owed marker is durable on the + # worktree disk, so a cell that dies before its clean shutdown - the only + # place the push runs - carries the skipped write-back into its report + # instead of losing it silently. Naming the actual origin keeps the marker + # from asserting a pull that did not happen. + if [ "$pull_rc" -ne 0 ]; then + owed_origin="the seeded bundle after a failed pull from share $AUTH_SHARE" + elif [ "${pulled:-0}" -eq 0 ]; then + owed_origin="a first interactive auth against empty share $AUTH_SHARE" + else + owed_origin="a pull from share $AUTH_SHARE" + fi + { printf 'this cell owns auth from %s at %s; a write-back to %s is owed\n' \ + "$owed_origin" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$AUTH_SHARE" >"$STATE/auth-push-owed" \ + && chmod 0600 "$STATE/auth-push-owed"; } || : } auth_home_push() { @@ -320,7 +337,17 @@ auth_home_push() { >>"$LOGS/auth-sync-a$ATTEMPT.log" 2>&1 push_rc=$? set -e - [ "$push_rc" -eq 0 ] || echo "validation guest: auth-home push failed; refreshed tokens stay cell-local" >&2 + if [ "$push_rc" -eq 0 ]; then + rm -f "$STATE/auth-push-owed" "$STATE/auth-push-failed" || : + return 0 + fi + # A warning on stderr dies with the guest. The share is now stale, every + # later boot starts from an older credential, and only a durable marker + # carried into the operator report says so. + { printf 'auth-home push to share %s failed with status %s at %s; refreshed tokens stay cell-local and the share is stale\n' \ + "$AUTH_SHARE" "$push_rc" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$STATE/auth-push-failed" \ + && chmod 0600 "$STATE/auth-push-failed"; } || : + echo "validation guest: auth-home push failed; refreshed tokens stay cell-local" >&2 } if [ "$MODE" = start ]; then @@ -979,6 +1006,11 @@ REPORT=$STATE/report.md if [ -f "$STATE/auth-needed" ]; then printf -- "- Auth: \`interactive provider auth needed once (auth share empty)\`\n" fi + if [ -f "$STATE/auth-push-failed" ]; then + printf -- "- Auth write-back: \`FAILED - refreshed tokens stayed cell-local and %s is stale\`\n" "$AUTH_SHARE" + elif [ -f "$STATE/auth-push-owed" ]; then + printf -- "- Auth write-back: \`SKIPPED - an attempt ended without reaching its clean-shutdown push to %s\`\n" "$AUTH_SHARE" + fi } >"$REPORT" RESULT_ARCHIVE=$BOOTSTRAP/result.tar.gz diff --git a/bin/fm-azure-validation.py b/bin/fm-azure-validation.py index 6d99cb18cea..392e206aa44 100755 --- a/bin/fm-azure-validation.py +++ b/bin/fm-azure-validation.py @@ -32,6 +32,7 @@ TEMPLATE = ROOT / "docs" / "azure-validation" / "cell.json" GUEST = ROOT / "bin" / "fm-azure-validation-guest.sh" SHARD_BRIDGE = ROOT / "bin" / "fm-azure-validation-shard-bridge.py" +CREDENTIAL_EXPIRY = ROOT / "bin" / "fm-credential-expiry.py" CONTAINER = "validation-shards" SCHEMA = "fm.azure-validation/v1" RESULT_SCHEMA = "fm.azure-validation-result/v1" @@ -975,6 +976,170 @@ def runner_module(): return _RUNNER_MODULE +_CREDENTIAL_EXPIRY_MODULE = None + + +def credential_expiry_module(): + global _CREDENTIAL_EXPIRY_MODULE + if _CREDENTIAL_EXPIRY_MODULE is None: + spec = importlib.util.spec_from_file_location( + "credential_expiry_module", str(CREDENTIAL_EXPIRY) + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + _CREDENTIAL_EXPIRY_MODULE = module + return _CREDENTIAL_EXPIRY_MODULE + + +# What the fm-auth-home share actually is, verified against both consumers: +# the guest's auth_home_pull copies the WHOLE share into one cell home and the +# guest exports CODEX_HOME=$HOME/.codex or CLAUDE_CONFIG_DIR=$HOME/.claude, so +# the share is exactly one home-shaped tree holding at most one codex profile +# and one claude profile. Crosscheck reviewers never read the share at all; +# they receive a per-review credential archive. There is therefore no consumer +# for a multi-profile layout, and inventing one would write bytes nothing +# reads. Seeding keeps the layout the consumers already expect. +# +# Only the credential file itself is uploaded. Sessions, history, caches, and +# project state are cell-local by design and have no reason to sit on a shared +# Azure Files share. +AUTH_HOME_LAYOUT = { + "codex": (".codex", "auth.json"), + "claude": (".claude", ".credentials.json"), +} + + +def auth_seed_targets(args): + """Resolve the requested harness/profile pairs for one seeding run.""" + + selected = [] + for harness in PROVIDERS: + value = getattr(args, harness, None) + if not value: + continue + profile = Path(value).expanduser() + if not profile.is_dir() or profile.is_symlink(): + raise ValidationError( + "{} profile must be an existing non-symlink directory: {}".format( + harness, profile + ) + ) + directory, credential = AUTH_HOME_LAYOUT[harness] + source = profile.resolve() / credential + if not source.is_file() or source.is_symlink(): + raise ValidationError( + "{} profile holds no regular {} to seed: {}".format( + harness, credential, source + ) + ) + selected.append({ + "harness": harness, + "profile": profile.resolve(), + "source": source, + "share_directory": directory, + "share_path": "{}/{}".format(directory, credential), + }) + if not selected: + raise ValidationError( + "auth-seed requires at least one of --codex or --claude" + ) + return selected + + +def auth_seed_preflight(targets): + """Refuse to publish a credential the cells cannot authenticate with. + + Seeding exists to carry a freshly re-authenticated profile onto the share. + A profile whose access token is already dead would be uploaded, pulled by + every later boot, and fail there instead of here, so it is refused with + its own expiry named. + """ + + expiry = credential_expiry_module() + for target in targets: + record = expiry.inspect_profile( + target["profile"], harness=target["harness"] + ) + try: + expiry.require_state(record, "usable", "fm-auth-home seed") + except expiry.CredentialExpiryError as exc: + raise ValidationError( + "{}; re-authenticate that profile and seed again".format(exc) + ) + target["expiry"] = record + return targets + + +def auth_seed(env, args): + """Publish selected local credentials onto the persistent auth share. + + Plan is local and touches no Azure. Apply requires the exact subscription + plus an explicit seed confirmation, uploads each credential to the exact + path its consumer reads, and then re-reads the share to prove the upload. + """ + + targets = auth_seed_preflight(auth_seed_targets(args)) + share = auth_share_name() + if not share: + raise ValidationError("FM_AZURE_AUTH_SHARE is empty; the auth-home sync is disabled") + if not args.apply: + print("auth-seed plan (no Azure call made)") + print(" share: {}".format(share)) + for target in targets: + print(" {} {} -> {}".format( + target["harness"], target["source"], target["share_path"] + )) + print(" state {} expires {}".format( + target["expiry"]["state"], target["expiry"]["expires_at"] + )) + print(" apply with: --apply --confirm-seed --confirm-subscription ") + return + if not args.confirm_seed: + raise ValidationError("auth-seed --apply requires --confirm-seed") + if args.confirm_subscription != env["subscription"]: + raise ValidationError("auth-seed --apply requires the exact --confirm-subscription") + scope_gate(env) + backup = ["--auth-mode", "login", "--enable-file-backup-request-intent"] + for target in targets: + az_command(env, [ + "storage", "directory", "create", + "--account-name", env["storage"], + "--share-name", share, + "--name", target["share_directory"], + ] + backup) + _, code, detail = az_command(env, [ + "storage", "file", "upload", + "--account-name", env["storage"], + "--share-name", share, + "--source", str(target["source"]), + "--path", target["share_path"], + ] + backup, check=False) + if code != 0: + raise ValidationError("auth-seed upload failed for {}: {}".format( + target["share_path"], detail + )) + # An accepted upload is not proof: re-read the share and require the + # exact byte count, so a truncated or replaced object is caught here + # instead of at the next cell boot. + published, code, detail = az_command(env, [ + "storage", "file", "show", + "--account-name", env["storage"], + "--share-name", share, + "--path", target["share_path"], + ] + backup, check=False) + expected = target["source"].stat().st_size + published_size = ((published or {}).get("properties") or {}).get("contentLength") + if code != 0 or published_size != expected: + raise ValidationError( + "auth-seed could not prove {} landed at its expected {} bytes: {}".format( + target["share_path"], expected, detail or published_size + ) + ) + print("auth-seed published {} ({} bytes, expires {})".format( + target["share_path"], expected, target["expiry"]["expires_at"] + )) + + def lifecycle_command(env, arguments): command_env = os.environ.copy() # The allocator store is fenced to its home identity, so the operator's @@ -2862,6 +3027,12 @@ def parser(): retain_parser.add_argument("--confirm-retain", action="store_true") retain_parser.add_argument("--confirm-subscription") commands.add_parser("queue") + seed_parser = commands.add_parser("auth-seed") + seed_parser.add_argument("--codex") + seed_parser.add_argument("--claude") + seed_parser.add_argument("--apply", action="store_true") + seed_parser.add_argument("--confirm-seed", action="store_true") + seed_parser.add_argument("--confirm-subscription") pure = commands.add_parser("pure-check", help=argparse.SUPPRESS) pure.add_argument("--fixture", required=True) return root @@ -2874,6 +3045,10 @@ def main(): pure_check(args) return 0 cloud = args.command in ("dispatch", "drive", "observe", "collect", "respond", "replace", "close", "retain-failure") + # Planning a seed is a purely local credential read; only the upload + # needs a cloud scope, so a plan works without Azure environment. + if args.command == "auth-seed" and args.apply: + cloud = True env = environment(require_cloud=cloud) if args.command == "submit": submit(env, args) @@ -2897,6 +3072,8 @@ def main(): queue(env) elif args.command == "status": status(env, args) + elif args.command == "auth-seed": + auth_seed(env, args) return 0 except ValidationError as exc: print("AZURE VALIDATION FAILED: {}".format(exc), file=sys.stderr) diff --git a/bin/fm-azure-validation.sh b/bin/fm-azure-validation.sh index 34be3d93f19..f6e60d4ede0 100755 --- a/bin/fm-azure-validation.sh +++ b/bin/fm-azure-validation.sh @@ -42,19 +42,28 @@ # fm-azure-validation.sh retain-failure --cell --confirm-retain \ # --confirm-subscription # fm-azure-validation.sh queue +# fm-azure-validation.sh auth-seed [--codex ] [--claude ] +# [--apply --confirm-seed --confirm-subscription ] +# +# auth-seed publishes a locally re-authenticated credential onto the +# fm-auth-home share so cells stop booting with a dead token. Without --apply +# it plans locally and makes no Azure call. It refuses any profile whose +# credential is not usable now (bin/fm-credential-expiry.py owns that +# judgement) and uploads only the credential file, into the one home-shaped +# layout the guest actually reads. set -euo pipefail SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) usage() { - sed -n '2,44p' "$0" | sed 's/^# \{0,1\}//' + sed -n '2,54p' "$0" | sed 's/^# \{0,1\}//' } case "${1:-}" in help|-h|--help|"") usage ;; - submit|dispatch|drive|observe|collect|status|respond|replace|close|retain-failure|queue) + submit|dispatch|drive|observe|collect|status|respond|replace|close|retain-failure|queue|auth-seed) exec python3 "$SCRIPT_DIR/fm-azure-validation.py" "$@" ;; *) diff --git a/bin/fm-credential-expiry.py b/bin/fm-credential-expiry.py new file mode 100755 index 00000000000..2905fe67ce3 --- /dev/null +++ b/bin/fm-credential-expiry.py @@ -0,0 +1,501 @@ +#!/usr/bin/env python3 +"""Report whether one local account profile's provider credential is usable. + +This module is the single owner of firstmate's provider-credential expiry +question. It answers exactly one thing for one local account profile +directory: can that credential still authenticate, and until when. It never +logs in, never refreshes, never mutates a profile, and never emits, returns, +or persists token material - only the profile path, harness, credential file +name, classified state, and expiry instants. + +Why this exists: staging a dead credential onto a cloud compartment costs a +real VM and returns a tool failure instead of a verdict. Every path that +stages a credential is expected to call this first and refuse before it +provisions anything. + +Credential shapes, read exactly as the providers write them: + + codex /auth.json tokens.access_token (JWT `exp`, seconds), + tokens.refresh_token, or a non-expiring + OPENAI_API_KEY + pi /auth.json openai-codex.expires (milliseconds), + openai-codex.refresh + claude /.credentials.json + claudeAiOauth.expiresAt (milliseconds), + claudeAiOauth.refreshToken, + claudeAiOauth.refreshTokenExpiresAt + +States, most usable first: + + usable the credential authenticates now and still will after the + caller's margin; an API-key credential declares no expiry and + is usable until the provider revokes it + refreshable the access token is expired or expires inside the margin, but + refresh material is present and is not provably dead. A refresh + needs the provider's auth host, so a caller whose network + allowlist excludes that host must NOT accept this state + expired the access token is dead and no refresh can revive it: refresh + material is absent, or its own declared expiry has passed + unusable no credential to classify: absent, a symlink, not a regular + file, over the byte bound, malformed, or carrying no token + material at all + +`refreshable` is deliberately distinct from `usable`. Firstmate has no token +refresh anywhere - no job, no timer, no call site - so nothing on the host +turns a `refreshable` profile into a `usable` one. Only an interactive +provider login, or the provider CLI reaching its own auth host from wherever +the profile runs, does that. + +Usage: + fm-credential-expiry.py report [--json] [--margin-seconds N] + [--pool-root DIR] [...] + fm-credential-expiry.py check [--harness H] [--margin-seconds N] + [--min-state usable|refreshable] + +`report` with no profile arguments walks the Agent Fleet account pool +(`~/.local/share/agent-fleet/accounts//`) and prints one row per +profile. `check` exits 0 when the named profile meets `--min-state` (default +`usable`) and 1 with an operator message when it does not. +""" + +from __future__ import annotations + +import argparse +import base64 +import binascii +import datetime +import json +import os +from pathlib import Path +import stat +import sys +import time +from typing import Any + + +# One provider credential is small; anything larger is not a credential we +# know how to read and is refused rather than parsed. +MAX_CREDENTIAL_BYTES = 1024 * 1024 + +# Default headroom a credential must still hold to count as usable. Callers +# that know their own deadline should pass it instead: a token that outlives +# the preflight but not the run is the failure this module exists to stop. +DEFAULT_MARGIN_SECONDS = 900 + +HARNESS_CREDENTIAL_FILE = { + "codex": "auth.json", + "pi": "auth.json", + "claude": ".credentials.json", +} + +POOL_VENDORS = ("codex", "pi", "claude") + +STATE_RANK = {"unusable": 0, "expired": 1, "refreshable": 2, "usable": 3} +STATE_ORDER = ("unusable", "expired", "refreshable", "usable") + +DEFAULT_POOL_ROOT = "~/.local/share/agent-fleet/accounts" + + +class CredentialExpiryError(RuntimeError): + """One profile's credential does not meet the caller's required state.""" + + +def _utc(seconds: float | None) -> str | None: + if seconds is None: + return None + moment = datetime.datetime.fromtimestamp(seconds, datetime.timezone.utc) + return moment.replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _epoch_seconds(value: Any, *, milliseconds: bool) -> float | None: + """Return a positive epoch instant, or None when the field is unusable. + + A zero or negative stamp is not an instant. Claude writes `expiresAt: 0` + for a profile whose access token was cleared, so zero must read as + "no live access token", never as "expired in 1970" and never as absent. + """ + + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + seconds = float(value) / 1000.0 if milliseconds else float(value) + if seconds <= 0: + return None + return seconds + + +def _jwt_expiry(token: Any) -> float | None: + """Read the `exp` claim of a JWT without verifying or retaining it.""" + + if not isinstance(token, str): + return None + parts = token.split(".") + if len(parts) != 3: + return None + payload = parts[1] + payload += "=" * (-len(payload) % 4) + try: + claims = json.loads(base64.urlsafe_b64decode(payload)) + except (binascii.Error, ValueError, UnicodeDecodeError): + return None + if not isinstance(claims, dict): + return None + return _epoch_seconds(claims.get("exp"), milliseconds=False) + + +def _present(value: Any) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def detect_harness(profile: Path) -> str | None: + """Name the harness that owns this profile from its credential shape.""" + + claude = profile / HARNESS_CREDENTIAL_FILE["claude"] + if claude.is_file() and not claude.is_symlink(): + return "claude" + auth = profile / "auth.json" + if not auth.is_file() or auth.is_symlink(): + return None + try: + value = json.loads(auth.read_bytes()[:MAX_CREDENTIAL_BYTES]) + except (OSError, ValueError): + return None + if not isinstance(value, dict): + return None + if isinstance(value.get("openai-codex"), dict): + return "pi" + if "tokens" in value or "OPENAI_API_KEY" in value or "auth_mode" in value: + return "codex" + return None + + +def _read_credential(path: Path) -> tuple[dict[str, Any] | None, str]: + try: + metadata = path.lstat() + except OSError as exc: + return None, f"credential is unreadable at {path}: {exc.strerror or exc}" + if not stat.S_ISREG(metadata.st_mode): + return None, f"credential is not a regular non-symlink file at {path}" + if metadata.st_size > MAX_CREDENTIAL_BYTES: + return None, f"credential exceeds its {MAX_CREDENTIAL_BYTES}-byte bound at {path}" + try: + raw = path.read_bytes() + except OSError as exc: + return None, f"credential is unreadable at {path}: {exc.strerror or exc}" + try: + value = json.loads(raw) + except (ValueError, UnicodeDecodeError): + return None, f"credential is malformed JSON at {path}" + if not isinstance(value, dict): + return None, f"credential is not a JSON object at {path}" + return value, "" + + +def _codex_facts(value: dict[str, Any]) -> dict[str, Any]: + tokens = value.get("tokens") if isinstance(value.get("tokens"), dict) else {} + access = tokens.get("access_token") + refresh = tokens.get("refresh_token") + api_key = value.get("OPENAI_API_KEY") + return { + "never_expires": _present(api_key) and not _present(access), + "has_access": _present(access), + "access_expires_at": _jwt_expiry(access), + "has_refresh": _present(refresh), + "refresh_expires_at": None, + } + + +def _pi_facts(value: dict[str, Any]) -> dict[str, Any]: + entry = value.get("openai-codex") + entry = entry if isinstance(entry, dict) else {} + access = entry.get("access") + return { + "never_expires": False, + "has_access": _present(access), + "access_expires_at": ( + _epoch_seconds(entry.get("expires"), milliseconds=True) + or _jwt_expiry(access) + ), + "has_refresh": _present(entry.get("refresh")), + "refresh_expires_at": None, + } + + +def _claude_facts(value: dict[str, Any]) -> dict[str, Any]: + oauth = value.get("claudeAiOauth") + oauth = oauth if isinstance(oauth, dict) else {} + return { + "never_expires": False, + "has_access": _present(oauth.get("accessToken")), + "access_expires_at": _epoch_seconds(oauth.get("expiresAt"), milliseconds=True), + "has_refresh": _present(oauth.get("refreshToken")), + "refresh_expires_at": _epoch_seconds( + oauth.get("refreshTokenExpiresAt"), milliseconds=True + ), + } + + +_FACT_READERS = {"codex": _codex_facts, "pi": _pi_facts, "claude": _claude_facts} + + +def inspect_profile( + profile: str | os.PathLike[str], + *, + harness: str | None = None, + now: float | None = None, + margin_seconds: float = DEFAULT_MARGIN_SECONDS, +) -> dict[str, Any]: + """Classify one profile's provider credential without touching a network. + + The returned record carries the profile path, harness, credential file + name, state, both expiry instants, and a human detail line. It never + carries token material, and no field is derived from one. + """ + + moment = time.time() if now is None else float(now) + path = Path(profile).expanduser() + try: + path = path.resolve() + except OSError: + path = path.absolute() + resolved_harness = harness or detect_harness(path) + record: dict[str, Any] = { + "profile": str(path), + "harness": resolved_harness or "", + "credential": "", + "state": "unusable", + "expires_at": None, + "expires_in_seconds": None, + "refresh_expires_at": None, + "detail": "", + } + if resolved_harness is None: + record["detail"] = f"no recognizable provider credential in {path}" + return record + if resolved_harness not in HARNESS_CREDENTIAL_FILE: + record["detail"] = f"no credential reader for harness {resolved_harness!r}" + return record + name = HARNESS_CREDENTIAL_FILE[resolved_harness] + record["credential"] = name + value, problem = _read_credential(path / name) + if value is None: + record["detail"] = problem + return record + + facts = _FACT_READERS[resolved_harness](value) + record["expires_at"] = _utc(facts["access_expires_at"]) + record["refresh_expires_at"] = _utc(facts["refresh_expires_at"]) + if facts["access_expires_at"] is not None: + record["expires_in_seconds"] = int(facts["access_expires_at"] - moment) + + if facts["never_expires"]: + record["state"] = "usable" + record["detail"] = ( + f"{resolved_harness} api-key credential declares no expiry" + ) + return record + if not facts["has_access"] and not facts["has_refresh"]: + record["detail"] = ( + f"{resolved_harness} credential at {path / name} carries no token material" + ) + return record + + deadline = moment + max(0.0, float(margin_seconds)) + access_live = ( + facts["has_access"] + and facts["access_expires_at"] is not None + and facts["access_expires_at"] > deadline + ) + if access_live: + record["state"] = "usable" + record["detail"] = ( + f"{resolved_harness} access token is valid through {record['expires_at']}" + ) + return record + + # An access token with no declared expiry cannot be proved dead. Refusing + # it would be a false refusal, so it is reported as refreshable with the + # missing expiry named rather than silently promoted to usable. + if facts["has_access"] and facts["access_expires_at"] is None: + record["state"] = "refreshable" + record["detail"] = ( + f"{resolved_harness} access token declares no expiry; its liveness " + "cannot be proved from the profile" + ) + return record + + refresh_dead = facts["refresh_expires_at"] is not None and ( + facts["refresh_expires_at"] <= moment + ) + if facts["has_refresh"] and not refresh_dead: + record["state"] = "refreshable" + expiry = record["expires_at"] or "an unrecorded instant" + # Distinguish the two ways a credential lands here. A token that is + # alive now but dies inside the margin is not expired, and calling it + # expired contradicts `report`, which shows the same profile as usable + # with a future expiry. + if facts.get("access_expires_at") is not None and facts["access_expires_at"] > moment: + record["detail"] = ( + f"{resolved_harness} access token expires at {expiry}, inside the " + "window this caller needs it for; refresh material is present but " + "firstmate never refreshes it" + ) + else: + record["detail"] = ( + f"{resolved_harness} access token expired at {expiry}; refresh " + "material is present but firstmate never refreshes it" + ) + return record + + record["state"] = "expired" + if not facts["has_refresh"]: + record["detail"] = ( + f"{resolved_harness} access token expired at " + f"{record['expires_at'] or 'an unrecorded instant'} and the profile " + "holds no refresh material" + ) + else: + record["detail"] = ( + f"{resolved_harness} access token expired at " + f"{record['expires_at'] or 'an unrecorded instant'} and its refresh " + f"material expired at {record['refresh_expires_at']}" + ) + return record + + +def state_meets(state: str, minimum: str) -> bool: + return STATE_RANK.get(state, 0) >= STATE_RANK[minimum] + + +def require_state( + record: dict[str, Any], minimum: str, label: str +) -> dict[str, Any]: + """Raise unless one inspected profile meets the caller's minimum state. + + The message names the profile, the state, and the expiry, so an operator + reads what to re-authenticate without opening the credential. + """ + + if minimum not in STATE_RANK: + raise CredentialExpiryError(f"unknown minimum credential state {minimum!r}") + if state_meets(record["state"], minimum): + return record + raise CredentialExpiryError( + f"{label} credential preflight refused: profile {record['profile']} is " + f"{record['state']} (required {minimum} or better): {record['detail']}" + ) + + +def pool_profiles(pool_root: str | os.PathLike[str]) -> list[Path]: + root = Path(pool_root).expanduser() + found: list[Path] = [] + for vendor in POOL_VENDORS: + directory = root / vendor + if not directory.is_dir(): + continue + for entry in sorted(directory.iterdir()): + if entry.is_dir() and not entry.is_symlink(): + found.append(entry) + return found + + +def _render_table(records: list[dict[str, Any]]) -> str: + header = ("STATE", "HARNESS", "EXPIRES", "PROFILE") + rows = [ + ( + record["state"], + record["harness"] or "-", + record["expires_at"] or "-", + record["profile"], + ) + for record in records + ] + widths = [ + max(len(header[index]), *(len(row[index]) for row in rows)) + if rows + else len(header[index]) + for index in range(len(header)) + ] + lines = [" ".join(header[i].ljust(widths[i]) for i in range(len(header))).rstrip()] + for row in rows: + lines.append( + " ".join(row[i].ljust(widths[i]) for i in range(len(row))).rstrip() + ) + return "\n".join(lines) + + +def _command_report(args: argparse.Namespace) -> int: + profiles = [Path(value) for value in args.profile] or pool_profiles(args.pool_root) + records = [ + inspect_profile(profile, margin_seconds=args.margin_seconds) + for profile in profiles + ] + if args.json: + print(json.dumps({"profiles": records}, indent=2, sort_keys=True)) + elif not records: + print("no account profiles found", file=sys.stderr) + else: + print(_render_table(records)) + return 0 + + +def _command_check(args: argparse.Namespace) -> int: + record = inspect_profile( + args.profile, + harness=args.harness, + margin_seconds=args.margin_seconds, + ) + try: + require_state(record, args.min_state, "account profile") + except CredentialExpiryError as exc: + print(str(exc), file=sys.stderr) + return 1 + print(f"{record['state']}: {record['detail']}") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="fm-credential-expiry.py", + description="Classify local provider credentials by expiry, never printing token material.", + ) + commands = parser.add_subparsers(dest="command", required=True) + + report = commands.add_parser("report", help="report every named or pooled profile") + report.add_argument("profile", nargs="*", help="account profile directories") + report.add_argument("--json", action="store_true", help="emit a JSON record set") + report.add_argument( + "--margin-seconds", + type=float, + default=DEFAULT_MARGIN_SECONDS, + help="headroom a credential must hold to count as usable", + ) + report.add_argument( + "--pool-root", + default=DEFAULT_POOL_ROOT, + help="Agent Fleet account pool scanned when no profile is named", + ) + report.set_defaults(handler=_command_report) + + check = commands.add_parser("check", help="refuse one profile below a minimum state") + check.add_argument("profile", help="account profile directory") + check.add_argument("--harness", choices=sorted(HARNESS_CREDENTIAL_FILE)) + check.add_argument( + "--margin-seconds", type=float, default=DEFAULT_MARGIN_SECONDS + ) + # Deliberately not the full STATE_ORDER: `check --min-state unusable` is a + # gate that cannot refuse anything, which is worse than no gate because it + # reads like one. + check.add_argument( + "--min-state", choices=["usable", "refreshable"], default="usable" + ) + check.set_defaults(handler=_command_check) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + return int(args.handler(args)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bin/fm-crosscheck-azure.py b/bin/fm-crosscheck-azure.py index 8beef791504..0706a33da34 100755 --- a/bin/fm-crosscheck-azure.py +++ b/bin/fm-crosscheck-azure.py @@ -60,6 +60,7 @@ RUNNER_CONTROLLER = ROOT / "bin" / "fm-azure-runner.py" RUNNER_GUEST = ROOT / "bin" / "fm-azure-runner-guest.sh" RUNNER_EXECUTOR = ROOT / "bin" / "fm-azure-runner-exec.py" +CREDENTIAL_EXPIRY = ROOT / "bin" / "fm-credential-expiry.py" class AzureCrosscheckError(RuntimeError): @@ -156,6 +157,63 @@ def load_tool_bridge() -> Any: ) +def load_credential_expiry() -> Any: + return load_module( + CREDENTIAL_EXPIRY, + "firstmate_credential_expiry", + "provider credential expiry preflight", + ) + + +# The interval between a granted lane and the reviewer's first token: scope +# verification, VM create, boot, and bundle upload. `poll_model_run` budgets +# it on the run deadline and the credential margin budgets it on the token, +# so both read the same number. +PROVISIONING_ALLOWANCE_SECONDS = 900 + + +def preflight_reviewer_credential(core: Any, config: dict[str, str]) -> dict[str, Any]: + """Refuse a dead reviewer credential before any billable compartment. + + The model compartment's egress allowlist is Azure DNS plus exactly one + provider API host (docs/azure-crosscheck/network-policy.json), and a + provider auth host is not on it. A CLI inside the compartment therefore + cannot refresh an expired token, so `refreshable` is not recoverable + there: the credential must already authenticate and must still do so + after the review deadline. Raising the core tool failure lets the + reviewer roster skip this account and try the next one, which is the + same treatment any other environment fault gets. + + The margin covers the review, not the wait in front of it, so this is + called twice: once to fail fast, and once after the lane is held, which + is the call that actually stands between a dead token and a paid VM. + + It also covers the gap between the check and the reviewer's first token: + scope verification, VM create, boot, and bundle upload all happen after + the lane is granted. `poll_model_run` already budgets that gap, so the + margin reuses its constant rather than inventing a second estimate of the + same interval. + """ + + expiry = load_credential_expiry() + record = expiry.inspect_profile( + config["account_home"], + harness=config["harness"], + margin_seconds=bounded_environment_integer( + "FM_CROSSCHECK_REVIEWER_TIMEOUT_SECONDS", 1800, 30, MAX_REVIEW_SECONDS + ) + + PROVISIONING_ALLOWANCE_SECONDS, + ) + try: + expiry.require_state(record, "usable", "Azure Crosscheck reviewer") + except expiry.CredentialExpiryError as exc: + raise core.CrosscheckToolError( + f"{exc}; re-authenticate that account before another review " + "(no model compartment, lane, or staged object was created)" + ) from exc + return record + + def azure_review_enabled(home: Path) -> bool: explicit = os.environ.get("FM_CROSSCHECK_EXECUTION_MODE") if explicit: @@ -949,7 +1007,7 @@ def submit_model_run( def poll_model_run( config: dict[str, Any], command_id: str, timeout_seconds: int ) -> tuple[str, str]: - deadline = time.monotonic() + timeout_seconds + 900 + deadline = time.monotonic() + timeout_seconds + PROVISIONING_ALLOWANCE_SECONDS url = "https://management.azure.com" + command_id + "?api-version=2024-03-01&$expand=instanceView" while time.monotonic() < deadline: value, rc, detail = az(config, ["rest", "--method", "get", "--url", url], check=False) @@ -1459,11 +1517,25 @@ def run_azure_review( until a lane frees, in exact submission order. The lane index selects the reviewer SKU deterministically so concurrent reviewers spread families. """ + # Expiry first: a dead credential must cost nothing. This runs before any + # Azure call and before any staged object, so an already-expired reviewer + # is skipped instead of provisioning a VM that dies with an unrefreshable + # session. + preflight_reviewer_credential(core, config) probe = runtime_config(home) lane, lane_handle = acquire_review_lane( home, probe["lanes"], probe["queue_wait_seconds"] ) try: + # The check above bounded nothing but its own instant. acquire_review_lane + # blocks in FIFO order for up to queue_wait_seconds - 7200 by default and + # 86400 at the maximum - which is far longer than the review margin, so a + # credential admitted as usable can be long dead by the time a lane frees. + # Under load, which is exactly when spend is highest, the first check is + # the one that proves nothing. This second check is the one that gates + # spend: every billable action happens after it, and it costs one local + # file read. + preflight_reviewer_credential(core, config) return _run_azure_review_in_lane( core=core, root=root, home=home, task_id=task_id, pr_url=pr_url, review_dir=review_dir, proof_root=proof_root, diff --git a/docs/azure-crosscheck.md b/docs/azure-crosscheck.md index 6f604028ad5..f0b69a1052f 100644 --- a/docs/azure-crosscheck.md +++ b/docs/azure-crosscheck.md @@ -56,6 +56,17 @@ A force-push changes the live head and invalidates the ordinary exact-head ledge A stale claims document invalidates the claims match. A wrong account, model, generation, VM, boot, request, transport, or cleanup identity cannot become a clear run. +## Reviewer credential preflight + +The model compartment's egress allowlist is Azure-provided DNS plus the exact provider API endpoint, and a provider auth host is not on it. +A reviewer CLI inside the compartment therefore cannot refresh an expired session, so a dead credential buys a real VM and returns a tool failure instead of a verdict. + +Every review runs `bin/fm-credential-expiry.py` against the selected reviewer's account home twice: once before the FIFO lane wait, before any Azure call and before any staged object, and again once the lane is held. +The second check is the one that gates spend. The margin covers the review, not the queue in front of it, and `FM_AZURE_CROSSCHECK_QUEUE_WAIT_SECONDS` bounds that queue at 7200 seconds by default and 86400 at its maximum, so a credential admitted before the wait can be long dead by the time a lane frees - under load, which is when spend is highest. +The credential must be `usable` and must still be usable after the review deadline (`FM_CROSSCHECK_REVIEWER_TIMEOUT_SECONDS`); `refreshable` is refused because it is not recoverable inside the compartment. +A refusal is an ordinary tool failure, so the roster records the account and rotates to the next policy-screened reviewer rather than ending the review. +The preflight reads expiry instants and account paths only, and never emits token material. + ## Model compartment The model VM uses a separately built and reviewed exact Azure image resource ID supplied through `FM_CROSSCHECK_AZURE_MODEL_IMAGE_ID`. diff --git a/docs/azure-validation.md b/docs/azure-validation.md index 2e5701fd47c..1436c0186f9 100644 --- a/docs/azure-validation.md +++ b/docs/azure-validation.md @@ -107,6 +107,28 @@ A result schema `fm.azure-validation-result/v1` repeats the home, task, task gen A passed result additionally requires a full PR URL, CI-green marker, exact remote-current head, and the complete independent behavior-shard receipt set. Wrong-head, wrong-run, wrong-disk, wrong-VM, wrong-boot, stale, partial, or malformed results are retained and refused. +## Persistent auth home and expiry + +The `fm-auth-home` Azure Files share is exactly one home-shaped tree, not a profile pool. +The guest's `auth_home_pull` copies the whole share into one cell home and then exports `CODEX_HOME=$HOME/.codex` or `CLAUDE_CONFIG_DIR=$HOME/.claude`, so the only paths any consumer reads are `.codex/auth.json` and `.claude/.credentials.json`. +Azure Crosscheck reviewers never read the share at all; they receive a per-review credential archive. +A multi-profile layout has no consumer and is not created. + +`bin/fm-credential-expiry.py` owns the question of whether one local account profile's credential is usable, and until when. +It classifies a profile as `usable`, `refreshable`, `expired`, or `unusable` from the provider's own credential file, never emits token material, and never logs in or refreshes. +`refreshable` means the access token is dead but refresh material survives; firstmate has no token refresh anywhere, so only an interactive login, or the provider CLI reaching its own auth host from wherever the profile runs, turns `refreshable` into `usable`. + +`bin/fm-azure-validation.sh auth-seed` publishes a locally re-authenticated credential onto the share, so an operator can replace a dead share credential without waiting for a cell to fail on it. +It gates what goes onto the share, not what comes off it: `dispatch_cell` does not re-check the share before creating a cell, so a credential that expires between seedings still reaches a booted cell. +It plans locally with no Azure call, refuses any profile that is not `usable`, uploads only the credential file into the layout above, and re-reads the share to prove the exact byte count landed. +`--apply` additionally requires `--confirm-seed` and the exact `--confirm-subscription`. + +The clean-shutdown `auth_home_push` is the only write-back. +A pull now leaves a durable `auth-push-owed` marker on the worktree disk naming where the cell's credential actually came from, a failed push leaves `auth-push-failed`, and a successful push clears both. +Because a successful push clears the owed marker, an earlier skipped write-back is reported only while the share is still stale, which is the window in which it matters. +Every marker write is best-effort: the guest runs under `set -euo pipefail`, and a note to the operator must never abort a run that has already been paid for. +The cell report surfaces the surviving marker the same way it surfaces `auth-needed`, so a stale share is an operator signal rather than a stderr line that died with the guest. + ## Credential lease Credential bytes never enter the repository bundle, runtime bundle, request JSON, local state JSON, Azure tags, ARM parameters, snapshots, reports, command logs, shard requests, shard responses, or identity-less command VMs. diff --git a/docs/scripts.md b/docs/scripts.md index b359dffb6a8..f897b2512de 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -71,6 +71,7 @@ The shared no-mistakes gate refusal used by every directly invocable mutating co | `fm-azure-pilot.sh` | Validate, preview, apply, inspect, recover, or explicitly remove the private Azure foundation | | `fm-azure-runner.sh` | Run one credential-free exact repository command on one private disposable Azure VM | | `fm-azure-validation.sh` | Queue and control exact-head no-mistakes runs in isolated elastic Azure cells | +| `fm-credential-expiry.py` | Classify one account profile's provider credential by expiry without emitting token material | | `fm-azure-validation-shard-bridge.py` | Exchange exact behavior/lint requests and independent Azure runner receipts inside one cell | | `fm-nm-step-liveness.sh` | Read a no-mistakes step's processes as alive, dead, or graded unknown | | `fm-tangle-lib.sh` | Shared default-branch resolution and primary-checkout tangle classification | diff --git a/tests/behavior-test-durations.tsv b/tests/behavior-test-durations.tsv index 9207ca15ac4..eeedf3d2345 100644 --- a/tests/behavior-test-durations.tsv +++ b/tests/behavior-test-durations.tsv @@ -52,6 +52,7 @@ 1000 tests/fm-cloud-state.test.sh 263 tests/fm-composer-ghost.test.sh 30 tests/fm-composer-lib.test.sh +4000 tests/fm-credential-expiry.test.sh 10055 tests/fm-crew-state.test.sh 5000 tests/fm-crosscheck-azure.test.sh 94232 tests/fm-crosscheck.test.sh diff --git a/tests/fm-credential-expiry.test.sh b/tests/fm-credential-expiry.test.sh new file mode 100755 index 00000000000..b1791d99d5b --- /dev/null +++ b/tests/fm-credential-expiry.test.sh @@ -0,0 +1,698 @@ +#!/usr/bin/env bash +# shellcheck source=tests/test-entry.sh +. "$(dirname "$0")/test-entry.sh" +# Provider credential expiry classification, the Azure Crosscheck reviewer +# preflight that must refuse before any compartment exists, the fm-auth-home +# seeding contract, and the guest's durable auth write-back markers. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +EXPIRY="$ROOT/bin/fm-credential-expiry.py" +ADAPTER="$ROOT/bin/fm-crosscheck-azure.py" +CORE="$ROOT/bin/fm-crosscheck.py" +VALIDATION="$ROOT/bin/fm-azure-validation.sh" +GUEST="$ROOT/bin/fm-azure-validation-guest.sh" + +# Every fixture credential carries this exact byte string in place of token +# material, so any output that leaks a token leaks this marker with it. +FIXTURE_TOKEN_MARKER=fmtestsecrettokenmarker + +make_profiles() { + # Build one fixture pool: //, each in a known state. + python3 - "$1" "$FIXTURE_TOKEN_MARKER" <<'PY' || fail "credential fixture build failed" +import base64 +import json +import pathlib +import sys +import time + +root = pathlib.Path(sys.argv[1]) +marker = sys.argv[2] +now = time.time() + + +def jwt(expiry): + payload = base64.urlsafe_b64encode( + json.dumps({"exp": int(expiry)}).encode("utf-8") + ).decode("ascii").rstrip("=") + return "h." + payload + "." + marker + + +def write(vendor, name, credential, value): + directory = root / vendor / name + directory.mkdir(parents=True) + (directory / credential).write_text( + json.dumps(value, indent=2) + "\n", encoding="utf-8" + ) + + +def codex(tokens): + return {"OPENAI_API_KEY": None, "auth_mode": "chatgpt", "tokens": tokens} + + +write("codex", "live", "auth.json", codex({ + "access_token": jwt(now + 86400), "refresh_token": marker, + "id_token": jwt(now + 86400), "account_id": "acct-live", +})) +write("codex", "stale", "auth.json", codex({ + "access_token": jwt(now - 3600), "refresh_token": marker, + "id_token": jwt(now - 3600), "account_id": "acct-stale", +})) +write("codex", "expiring", "auth.json", codex({ + "access_token": jwt(now + 1200), "refresh_token": marker, + "id_token": jwt(now + 1200), "account_id": "acct-expiring", +})) +# Outlives the review margin (1800) but not the provisioning that happens +# before the review starts, so only the allowance refuses it. +write("codex", "provisioning", "auth.json", codex({ + "access_token": jwt(now + 2100), "refresh_token": marker, + "id_token": jwt(now + 2100), "account_id": "acct-provisioning", +})) +write("codex", "orphan", "auth.json", codex({ + "access_token": jwt(now - 3600), "refresh_token": "", + "id_token": jwt(now - 3600), "account_id": "acct-orphan", +})) +write("codex", "apikey", "auth.json", { + "OPENAI_API_KEY": marker, "auth_mode": "apikey", "tokens": None, +}) +write("codex", "malformed", "auth.json", "not-an-object") + +write("pi", "live", "auth.json", {"openai-codex": { + "type": "oauth", "access": jwt(now + 86400), "refresh": marker, + "accountId": "acct-pi-live", "expires": int((now + 86400) * 1000), +}}) +write("pi", "stale", "auth.json", {"openai-codex": { + "type": "oauth", "access": jwt(now - 3600), "refresh": marker, + "accountId": "acct-pi-stale", "expires": int((now - 3600) * 1000), +}}) + +write("claude", "live", ".credentials.json", {"claudeAiOauth": { + "accessToken": marker, "refreshToken": marker, + "expiresAt": int((now + 43200) * 1000), + "refreshTokenExpiresAt": int((now + 864000) * 1000), +}}) +write("claude", "stale", ".credentials.json", {"claudeAiOauth": { + "accessToken": marker, "refreshToken": marker, + "expiresAt": int((now - 3600) * 1000), + "refreshTokenExpiresAt": int((now + 864000) * 1000), +}}) +write("claude", "dead", ".credentials.json", {"claudeAiOauth": { + "accessToken": marker, "refreshToken": marker, + "expiresAt": int((now - 864000) * 1000), + "refreshTokenExpiresAt": int((now - 3600) * 1000), +}}) +# A zeroed stamp is Claude's "no instant recorded", not an instant in 1970. +# Reading zero as a real timestamp would declare live refresh material dead. +write("claude", "zeroedrefresh", ".credentials.json", {"claudeAiOauth": { + "accessToken": marker, "refreshToken": marker, + "expiresAt": int((now - 3600) * 1000), + "refreshTokenExpiresAt": 0, +}}) +write("claude", "clearedaccess", ".credentials.json", {"claudeAiOauth": { + "accessToken": marker, "refreshToken": marker, + "expiresAt": 0, + "refreshTokenExpiresAt": int((now + 864000) * 1000), +}}) +write("claude", "blanked", ".credentials.json", {"claudeAiOauth": { + "accessToken": "", "refreshToken": "", + "expiresAt": 0, + "refreshTokenExpiresAt": int((now + 864000) * 1000), +}}) +(root / "claude" / "empty").mkdir(parents=True) +PY +} + +classification_unit() { + local work + work=$(fm_test_tmproot fm-credential-expiry-classify) + make_profiles "$work/pool" + python3 - "$EXPIRY" "$work/pool" <<'PY' || fail "credential classification contract failed" +import importlib.util +import pathlib +import sys + +spec = importlib.util.spec_from_file_location("expiry", sys.argv[1]) +expiry = importlib.util.module_from_spec(spec) +spec.loader.exec_module(expiry) +pool = pathlib.Path(sys.argv[2]) + +expected = { + ("codex", "live"): "usable", + ("codex", "stale"): "refreshable", + # No refresh material: nothing can revive this one, so it is provably dead + # rather than merely stale. + ("codex", "orphan"): "expired", + # An API key declares no expiry and must not be classified by a clock. + ("codex", "apikey"): "usable", + ("codex", "malformed"): "unusable", + ("pi", "live"): "usable", + ("pi", "stale"): "refreshable", + ("claude", "live"): "usable", + ("claude", "stale"): "refreshable", + # The refresh token's own declared expiry has passed: provably dead. + ("claude", "dead"): "expired", + # A zeroed refresh stamp records no instant at all, so the refresh cannot + # be proved dead and this is stale, not expired. + ("claude", "zeroedrefresh"): "refreshable", + # A cleared access stamp is not an expiry in 1970; the live refresh token + # still makes this recoverable by an interactive login. + ("claude", "clearedaccess"): "refreshable", + # Blanked token strings carry no material to classify at all. + ("claude", "blanked"): "unusable", + ("claude", "empty"): "unusable", +} +for (vendor, name), state in expected.items(): + record = expiry.inspect_profile(pool / vendor / name) + assert record["state"] == state, (vendor, name, record["state"], state) + assert record["profile"] == str((pool / vendor / name).resolve()) + +# Harness detection reads the credential shape, not the directory name. +assert expiry.inspect_profile(pool / "pi" / "live")["harness"] == "pi" +assert expiry.inspect_profile(pool / "codex" / "live")["harness"] == "codex" +assert expiry.inspect_profile(pool / "claude" / "live")["harness"] == "claude" + +# The margin is the caller's own deadline: a token that survives the preflight +# but not the run is exactly the failure this module exists to stop. +live = pool / "codex" / "live" +assert expiry.inspect_profile(live, margin_seconds=0)["state"] == "usable" +assert expiry.inspect_profile(live, margin_seconds=172800)["state"] == "refreshable" + +# A symlinked credential is never followed. +linked = pool / "codex" / "linked" +linked.mkdir() +(linked / "auth.json").symlink_to(pool / "codex" / "live" / "auth.json") +assert expiry.inspect_profile(linked, harness="codex")["state"] == "unusable" + +# require_state names the profile and the refusal without opening the file. +record = expiry.inspect_profile(pool / "codex" / "stale") +try: + expiry.require_state(record, "usable", "unit") +except expiry.CredentialExpiryError as exc: + assert str(pool / "codex" / "stale") in str(exc) + assert "refreshable" in str(exc) +else: + raise AssertionError("require_state admitted a refreshable profile as usable") +expiry.require_state(record, "refreshable", "unit") +PY + pass "credential states classify usable, refreshable, expired, and unusable from the real provider shapes" +} + +cli_unit() { + local work out code + work=$(fm_test_tmproot fm-credential-expiry-cli) + make_profiles "$work/pool" + + out=$(python3 "$EXPIRY" report --pool-root "$work/pool" 2>&1) + assert_contains "$out" "usable" "pool report omitted a usable profile" + assert_contains "$out" "refreshable" "pool report omitted a refreshable profile" + assert_contains "$out" "$work/pool/codex/live" "pool report omitted a scanned profile path" + # The report is an operator artifact: it must never carry token material. + assert_not_contains "$out" "$FIXTURE_TOKEN_MARKER" "pool report leaked token material" + + out=$(python3 "$EXPIRY" report --json --pool-root "$work/pool" 2>&1) + assert_not_contains "$out" "$FIXTURE_TOKEN_MARKER" "JSON report leaked token material" + assert_contains "$out" '"expires_at"' "JSON report omitted the expiry instant" + + code=0 + out=$(python3 "$EXPIRY" check "$work/pool/codex/live" 2>&1) || code=$? + expect_code 0 "$code" "check refused a usable profile" + code=0 + out=$(python3 "$EXPIRY" check "$work/pool/codex/stale" 2>&1) || code=$? + expect_code 1 "$code" "check admitted a refreshable profile at the usable default" + assert_contains "$out" "refreshable" "check refusal did not name the state" + assert_not_contains "$out" "$FIXTURE_TOKEN_MARKER" "check refusal leaked token material" + code=0 + out=$(python3 "$EXPIRY" check --min-state refreshable "$work/pool/codex/stale" 2>&1) || code=$? + expect_code 0 "$code" "check refused a refreshable profile at the refreshable minimum" + # `check` is a gate. A minimum state that nothing can fall below is a gate + # that cannot refuse, which reads like one and is not. + code=0 + "$ROOT/bin/fm-credential-expiry.py" check --min-state unusable "$work/pool/codex/live" >/dev/null 2>&1 || code=$? + expect_code 2 "$code" "check accepted a minimum state it can never refuse" + + pass "the expiry CLI reports and gates profiles without ever emitting token material" +} + +crosscheck_preflight_unit() { + local work + work=$(fm_test_tmproot fm-credential-expiry-crosscheck) + make_profiles "$work/pool" + mkdir -p "$work/home" + # Enough Azure scope that a refused reviewer is refused by the preflight and + # not by an absent environment: without it, runtime_config raises before a + # lane is ever taken and the no-lane assertion below holds identically + # whether the preflight exists or not. + # + # It is deliberately not a COMPLETE scope. verify_scope_and_foundation still + # raises inside the lane before any `az` runs, so this unit cannot observe + # Azure CLI calls at all and does not claim to; the lane is what it proves. + FM_AZURE_TENANT_ID=11111111-1111-4111-8111-111111111111 \ + FM_AZURE_SUBSCRIPTION_ID=11111111-1111-4111-8111-111111111111 \ + FM_AZURE_NAMING_PREFIX=fmtest FM_AZURE_STORAGE_NAME=stfmtest \ + FM_AZURE_DEPLOYMENT_GENERATION=gen-1 FM_AZURE_OWNER_TAG=owner \ + FM_AZURE_RUNNER_OPERATOR_OBJECT_ID=11111111-1111-4111-8111-111111111111 \ + FM_CROSSCHECK_AZURE_MODEL_IMAGE_ID="/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/rg/providers/Microsoft.Compute/galleries/g/images/i/versions/1.0.0" \ + python3 - "$ADAPTER" "$CORE" "$work" <<'PY' || fail "Azure Crosscheck reviewer preflight contract failed" +import fcntl +import importlib.util +import inspect +import pathlib +import subprocess +import sys + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +adapter = load("adapter", sys.argv[1]) +# The real core module supplies the real CrosscheckToolError, so the refusal +# is proved against the class the reviewer roster actually catches to rotate. +core = load("core", sys.argv[2]) +work = pathlib.Path(sys.argv[3]) +pool = work / "pool" +home = work / "home" + +# The preflight runs before the FIFO lane wait, before runtime_config, and +# before any staged object - and again once the lane is held, because the lane +# wait can outlast the credential. +source = inspect.getsource(adapter.run_azure_review) +assert source.index("preflight_reviewer_credential") < source.index("acquire_review_lane") +assert source.index("preflight_reviewer_credential") < source.index("runtime_config") +assert source.rindex("preflight_reviewer_credential") > source.index("acquire_review_lane") + + +def review(profile, harness): + return adapter.run_azure_review( + core=core, root=home, home=home, task_id="t", pr_url="https://example.invalid/pr/1", + review_dir=home, proof_root=home, snapshot_value={}, ledger={}, + config={"harness": harness, "account_home": str(profile), + "model": "m", "effort": "xhigh"}, + author_account_identity="", + ) + + +# An expired reviewer is refused as a tool failure, which is what makes the +# roster skip this account instead of ending the review. +try: + review(pool / "codex" / "orphan", "codex") +except core.CrosscheckToolError as exc: + message = str(exc) + assert str(pool / "codex" / "orphan") in message, message + assert "expired" in message, message + assert "no model compartment" in message, message +else: + raise AssertionError("an expired reviewer reached the Azure compartment path") + +# The compartment's egress allowlist is Azure DNS plus one provider API host, +# so a CLI inside it can never reach an auth host: refreshable is not +# recoverable there and must be refused too. +try: + review(pool / "claude" / "stale", "claude") +except core.CrosscheckToolError as exc: + assert "refreshable" in str(exc), str(exc) +else: + raise AssertionError("a refreshable reviewer reached the Azure compartment path") + +# A credential alive right now but dead before the review deadline is refused +# too. This is the only fixture the review margin participates in: without it +# the margin could be zeroed and every assertion here would still pass. +try: + review(pool / "codex" / "expiring", "codex") +except core.CrosscheckToolError as exc: + # 1200s of life is more than the module default margin (900) and less than + # the caller's review margin (1800), so only the caller's own margin + # refuses it: dropping that argument makes this fixture pass. + assert "acct" not in str(exc) + assert "refreshable" in str(exc) or "expired" in str(exc), str(exc) +else: + raise AssertionError("a reviewer that dies mid-review reached the Azure compartment path") + +# A credential that outlives the review itself but not the VM create, boot, and +# bundle upload in front of it still cannot finish, so the margin covers that +# gap too. Without the allowance this profile is admitted and buys a VM. +try: + review(pool / "codex" / "provisioning", "codex") +except core.CrosscheckToolError as exc: + assert "refreshable" in str(exc) or "expired" in str(exc), str(exc) +else: + raise AssertionError("a reviewer that dies during provisioning reached the Azure compartment path") + +# Nothing above took a lane. +lanes = adapter.lane_root(home) +assert not lanes.exists() or not any(lanes.iterdir()), "a refused reviewer held a review lane" + +# Positive control for that assertion: a usable credential must get PAST the +# preflight and reach the lane. Without this, the check above would be satisfied +# by a code path that never reaches a lane at all, and would stay green with the +# whole feature deleted. +try: + review(pool / "codex" / "live", "codex") +except Exception: + pass +assert lanes.exists() and any(lanes.iterdir()), ( + "a usable reviewer never reached a lane, so the refusal assertion proves nothing" +) + +# The second preflight, the one that gates spend, is proved by expiring the +# credential during the lane wait: the pre-lane call saw a live token, so only +# a check behind the lane can refuse this. +expiring_home = pool / "codex" / "secondgate" +expiring_home.mkdir(parents=True) +live_body = (pool / "codex" / "live" / "auth.json").read_text(encoding="utf-8") +dead_body = (pool / "codex" / "stale" / "auth.json").read_text(encoding="utf-8") +(expiring_home / "auth.json").write_text(live_body, encoding="utf-8") + +# Lane occupancy is an flock on a lock file that acquire_review_lane creates +# once and never unlinks, so a held lane and a released one produce identical +# directory listings. Probe the lock itself: a non-blocking flock from a +# separate process succeeds only if nothing still holds it. +def lanes_held(): + held = [] + if not lanes.exists(): + return held + for lock in sorted(lanes.glob("lane-*.lock")): + probe = subprocess.run( + [sys.executable, "-c", + "import fcntl,sys\n" + "handle = open(sys.argv[1], 'a+')\n" + "try:\n" + " fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)\n" + "except OSError:\n" + " raise SystemExit(3)\n" + "raise SystemExit(0)\n", + str(lock)], + capture_output=True, + ) + if probe.returncode == 3: + held.append(lock.name) + return held + + +# Self-check: the probe must actually observe a held lane, or every assertion +# built on it is vacuous. +_probe_lane = lanes / "lane-probe.lock" +lanes.mkdir(parents=True, exist_ok=True) +_probe_handle = open(_probe_lane, "a+") +fcntl.flock(_probe_handle, fcntl.LOCK_EX | fcntl.LOCK_NB) +assert "lane-probe.lock" in lanes_held(), "the lane occupancy probe cannot see a held lane" +_probe_handle.close() +assert "lane-probe.lock" not in lanes_held(), "the lane occupancy probe reports a released lane as held" +_probe_lane.unlink() + +real_acquire = adapter.acquire_review_lane + + +def expire_during_wait(*args, **kwargs): + # Stand in for a queue wait that outlasts the token. + (expiring_home / "auth.json").write_text(dead_body, encoding="utf-8") + return real_acquire(*args, **kwargs) + + +adapter.acquire_review_lane = expire_during_wait +try: + try: + review(expiring_home, "codex") + except core.CrosscheckToolError as exc: + assert "expired" in str(exc), str(exc) + else: + raise AssertionError( + "a credential that died during the lane wait was never re-checked behind the lane" + ) +finally: + adapter.acquire_review_lane = real_acquire + +# That refusal released its lane rather than leaking it. +still_held = lanes_held() +assert not still_held, "the second preflight leaked its review lane: %r" % (still_held,) +PY + pass "an expired, unrefreshable, or mid-review-expiring Azure reviewer is refused before any lane or compartment" +} + +auth_seed_unit() { + local work out code + work=$(fm_test_tmproot fm-credential-expiry-seed) + make_profiles "$work/pool" + mkdir -p "$work/home" + + code=0 + out=$(FM_HOME=$work/home "$VALIDATION" auth-seed --codex "$work/pool/codex/live" 2>&1) || code=$? + expect_code 0 "$code" "auth-seed refused a usable codex profile" + # The share layout is the one the guest actually reads; a plan that named a + # different path would publish bytes nothing pulls. + assert_contains "$out" ".codex/auth.json" "auth-seed planned the wrong share path" + assert_contains "$out" "no Azure call made" "auth-seed plan did not declare itself local" + assert_not_contains "$out" "$FIXTURE_TOKEN_MARKER" "auth-seed plan leaked token material" + + code=0 + out=$(FM_HOME=$work/home "$VALIDATION" auth-seed --claude "$work/pool/claude/live" 2>&1) || code=$? + expect_code 0 "$code" "auth-seed refused a usable claude profile" + assert_contains "$out" ".claude/.credentials.json" "auth-seed planned the wrong claude share path" + + code=0 + out=$(FM_HOME=$work/home "$VALIDATION" auth-seed --codex "$work/pool/codex/stale" 2>&1) || code=$? + expect_code 1 "$code" "auth-seed published a credential the cells cannot authenticate with" + assert_contains "$out" "re-authenticate that profile" "auth-seed refusal did not tell the operator what to do" + + code=0 + out=$(FM_HOME=$work/home "$VALIDATION" auth-seed 2>&1) || code=$? + expect_code 1 "$code" "auth-seed accepted a run with no profile selected" + + # A complete Azure scope is present for the apply refusals below, so they + # prove the confirmation gate rather than an absent environment. + local uuid_a=11111111-1111-4111-8111-111111111111 + local uuid_b=22222222-2222-4222-8222-222222222222 + seed_env() { + env FM_HOME="$work/home" \ + FM_AZURE_TENANT_ID=$uuid_a FM_AZURE_SUBSCRIPTION_ID=$uuid_a \ + FM_AZURE_NAMING_PREFIX=fmtest FM_AZURE_STORAGE_NAME=stfmtest \ + FM_AZURE_DEPLOYMENT_GENERATION=gen-1 FM_AZURE_OWNER_TAG=owner \ + FM_AZURE_RUNNER_OPERATOR_OBJECT_ID=$uuid_a \ + "$VALIDATION" "$@" + } + + code=0 + out=$(seed_env auth-seed --codex "$work/pool/codex/live" --apply 2>&1) || code=$? + expect_code 1 "$code" "auth-seed applied without its explicit confirmation" + assert_contains "$out" "confirm-seed" "auth-seed apply refusal did not name the missing confirmation" + + code=0 + out=$(seed_env auth-seed --codex "$work/pool/codex/live" --apply --confirm-seed \ + --confirm-subscription "$uuid_b" 2>&1) || code=$? + expect_code 1 "$code" "auth-seed applied against a subscription the operator did not confirm" + assert_contains "$out" "confirm-subscription" "auth-seed apply refusal did not name the wrong subscription confirmation" + + # A dead credential is refused before any confirmation is even considered, + # so an operator cannot confirm their way onto the share with a stale token. + code=0 + out=$(seed_env auth-seed --codex "$work/pool/codex/stale" --apply --confirm-seed \ + --confirm-subscription "$uuid_a" 2>&1) || code=$? + expect_code 1 "$code" "auth-seed applied a credential the cells cannot authenticate with" + assert_contains "$out" "re-authenticate that profile" "confirmed auth-seed apply skipped the expiry preflight" + + python3 - "$ROOT/bin/fm-azure-validation.py" "$GUEST" <<'PY' || fail "auth-seed layout contract failed" +import pathlib +import sys + +host = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8") +guest = pathlib.Path(sys.argv[2]).read_text(encoding="utf-8") +# One home-shaped share: the seeding layout and the guest's pull/push set and +# provider-home exports must name the same two directories, or seeding writes +# where nothing reads. +assert '"codex": (".codex", "auth.json")' in host +assert '"claude": (".claude", ".credentials.json")' in host +assert 'AUTH_DIRS = (".codex", ".claude")' in guest +assert 'CODEX_HOME=%s\\n' in guest and '$HOME_DIR/.codex' in guest +assert '$HOME_DIR/.claude' in guest +PY + pass "auth-seed plans the exact layout the guest reads and refuses a credential the cells cannot use" +} + +guest_writeback_markers_unit() { + local work + work=$(fm_test_tmproot fm-credential-expiry-guest) + mkdir -p "$work/state" "$work/logs" "$work/fakebin" "$work/home" + + # Drive the guest's real auth-sync functions in isolation. The auth-sync + # helper is PATH-shimmed so pull and push outcomes are chosen by the test + # rather than by an Azure Files share. + python3 - "$GUEST" "$work/functions.sh" <<'EXTRACT' || fail "guest auth function extraction failed" +import pathlib +import sys + +guest = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8") +start = guest.index("auth_home_pull() {") +end = guest.index('\nif [ "$MODE" = start ]; then', start) +pathlib.Path(sys.argv[2]).write_text(guest[start:end] + "\n", encoding="utf-8") +EXTRACT + + cat >"$work/fakebin/python3" <<'PYSHIM' +#!/bin/sh +# Stand in for the guest's auth-sync helper. FM_TEST_AUTH_SYNC_RC picks the +# push outcome and FM_TEST_AUTH_SYNC_COUNT is the pulled-file count the pull +# reports. FM_TEST_AUTH_PULL_RC picks the pull outcome separately, because the +# guest takes a different branch when the pull itself fails and that branch +# cannot be reached while the pull is hardwired to succeed. +if [ "$2" = pull ]; then + if [ "${FM_TEST_AUTH_PULL_RC:-0}" -ne 0 ]; then + exit "${FM_TEST_AUTH_PULL_RC}" + fi + printf '%s\n' "${FM_TEST_AUTH_SYNC_COUNT:-1}" + exit 0 +fi +exit "${FM_TEST_AUTH_SYNC_RC:-0}" +PYSHIM + chmod +x "$work/fakebin/python3" + + cat >"$work/drive.sh" <<'DRIVER' +# The guest runs under `set -euo pipefail`. Driving these functions under +# anything weaker cannot observe a marker write that aborts the run, which is +# exactly the failure this unit exists to catch. +set -euo pipefail +AUTH_SYNC=/dev/null +STORAGE_ACCOUNT=acct +AUTH_SHARE=fm-auth-home +IDENTITY_CLIENT_ID=cid +HOME_DIR=$FM_TEST_AUTH_WORK/home +STATE=$FM_TEST_AUTH_WORK/state +LOGS=$FM_TEST_AUTH_WORK/logs +ATTEMPT=1 +. "$FM_TEST_AUTH_WORK/functions.sh" +auth_home_pull +[ "$FM_TEST_AUTH_MODE" = pull-only ] || auth_home_push +# Stands in for everything the guest does after auth: packaging the report, +# uploading the result blob, and echoing the completion marker the caller +# blocks on. If a marker write can abort the run, this line is what disappears. +echo FM-TEST-RUN-REACHED-END +DRIVER + + drive_auth() { + env PATH="$work/fakebin:$PATH" \ + FM_TEST_AUTH_SYNC_RC="$1" FM_TEST_AUTH_SYNC_COUNT="$2" \ + FM_TEST_AUTH_MODE="$3" FM_TEST_AUTH_PULL_RC="${4:-0}" \ + FM_TEST_AUTH_WORK="$work" \ + bash "$work/drive.sh" + } + + rm -f "$work/state"/auth-* + drive_auth 0 1 pull-only + assert_present "$work/state/auth-push-owed" "a pulled auth home recorded no owed write-back" + + rm -f "$work/state"/auth-* + # A push that fails must leave a durable marker, not just a dead stderr line. + drive_auth 1 1 push + assert_present "$work/state/auth-push-failed" "a failed auth write-back left no durable marker" + assert_present "$work/state/auth-push-owed" "a failed auth write-back cleared the owed marker" + assert_grep "fm-auth-home" "$work/state/auth-push-failed" "the failure marker did not name the stale share" + + rm -f "$work/state"/auth-* + drive_auth 0 1 push + assert_absent "$work/state/auth-push-failed" "a successful write-back left a stale failure marker" + assert_absent "$work/state/auth-push-owed" "a successful write-back left the owed marker behind" + + rm -f "$work/state"/auth-* + # The empty-share case keeps its existing interactive-auth marker. + drive_auth 0 0 push + assert_present "$work/state/auth-needed" "an empty auth share left no interactive-auth marker" + + # A marker is a note to the operator; the run outranks it. With an unwritable + # state directory every marker write fails, and under the guest's own + # `set -euo pipefail` that must not stop the run: the pull marker would abort + # before the run starts, and the push marker would abort a completed run + # before it packages its result or echoes its completion marker. + rm -f "$work/state"/auth-* + chmod 0500 "$work/state" + code=0 + out=$(drive_auth 1 1 push 2>/dev/null) || code=$? + chmod 0700 "$work/state" + expect_code 0 "$code" "an unwritable state directory turned an auth marker into a run-killer" + assert_contains "$out" "FM-TEST-RUN-REACHED-END" \ + "a failed auth marker write stopped the run before it could package its result" + + # Clearing a marker is the same bookkeeping as writing one, and it fails the + # same way. Seeding a marker BEFORE making the directory unwritable is what + # reaches the `rm -f` paths at all: with no pre-existing file, `rm -f` never + # has anything to fail on and the cleanup side goes untested. + rm -f "$work/state"/auth-* + : >"$work/state/auth-needed" + : >"$work/state/auth-push-owed" + chmod 0500 "$work/state" + code=0 + out=$(drive_auth 1 1 push 2>/dev/null) || code=$? + chmod 0700 "$work/state" + expect_code 0 "$code" "a stale marker plus an unwritable state directory killed the run before it started" + assert_contains "$out" "FM-TEST-RUN-REACHED-END" \ + "clearing a stale marker stopped the run before it started" + + # The same on the far side of a SUCCESSFUL push, which is the expensive case: + # the run is finished and paid for, and only the marker cleanup is left. + rm -f "$work/state"/auth-* + : >"$work/state/auth-push-owed" + : >"$work/state/auth-push-failed" + chmod 0500 "$work/state" + code=0 + out=$(drive_auth 0 1 push 2>/dev/null) || code=$? + chmod 0700 "$work/state" + expect_code 0 "$code" "a successful push died clearing its markers after the run was already paid for" + assert_contains "$out" "FM-TEST-RUN-REACHED-END" \ + "a successful push stopped the run while clearing its markers" + + # The empty-share marker is written on a different branch from the two above, + # so an unwritable directory has to reach that branch too. + rm -f "$work/state"/auth-* + chmod 0500 "$work/state" + code=0 + out=$(drive_auth 0 0 push 2>/dev/null) || code=$? + chmod 0700 "$work/state" + expect_code 0 "$code" "an unwritable state directory turned the empty-share marker into a run-killer" + assert_contains "$out" "FM-TEST-RUN-REACHED-END" \ + "a failed empty-share marker write stopped the run" + + # Same for the pull-side marker, which sits before the run rather than after + # it: a failure there must not stop the run from starting. + rm -f "$work/state"/auth-* + chmod 0500 "$work/state" + code=0 + out=$(drive_auth 0 1 push 2>/dev/null) || code=$? + chmod 0700 "$work/state" + expect_code 0 "$code" "an unwritable state directory stopped the run before it started" + assert_contains "$out" "FM-TEST-RUN-REACHED-END" \ + "a failed owed-marker write stopped the run before it started" + + # The owed marker must not claim a pull that did not happen. + rm -f "$work/state"/auth-* + drive_auth 0 1 pull-only 1 + assert_grep "seeded bundle" "$work/state/auth-push-owed" "the owed marker claimed a pull that failed" + rm -f "$work/state"/auth-* + drive_auth 0 0 pull-only + assert_grep "empty share" "$work/state/auth-push-owed" "the owed marker claimed a pull from an empty share" + + python3 - "$GUEST" <<'REPORTCONTRACT' || fail "guest auth report contract failed" +import pathlib +import re +import sys + +guest = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8") +report = guest[guest.index("REPORT=$STATE/report.md"):] +# Both durable markers must reach the operator report, the same way the +# existing empty-share marker does. +assert '[ -f "$STATE/auth-push-failed" ]' in report, report[:2000] +assert '[ -f "$STATE/auth-push-owed" ]' in report, report[:2000] +assert len(re.findall(r"Auth write-back:", report)) == 2, report[:2000] +assert "FAILED" in report and "SKIPPED" in report +# The report is composed after the push decides, or it prints stale state. +assert guest.index("\nauth_home_push\n") < guest.index("REPORT=$STATE/report.md") +REPORTCONTRACT + pass "a failed or incomplete auth write-back leaves a durable marker and reaches the operator report" +} + +classification_unit +cli_unit +crosscheck_preflight_unit +auth_seed_unit +guest_writeback_markers_unit diff --git a/tests/test-capabilities.tsv b/tests/test-capabilities.tsv index 942cb4a9050..203d3cc4a2d 100644 --- a/tests/test-capabilities.tsv +++ b/tests/test-capabilities.tsv @@ -43,6 +43,7 @@ fm-checkout-return-boundary.test.sh hermetic fm-cloud-state.test.sh hermetic fm-composer-ghost.test.sh hermetic fm-composer-lib.test.sh hermetic +fm-credential-expiry.test.sh hermetic fm-crew-state.test.sh hermetic fm-crosscheck-azure.test.sh hermetic fm-crosscheck.test.sh hermetic