diff --git a/AGENTS.md b/AGENTS.md index b15533fed53..99cb84c86d5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -88,6 +88,7 @@ config/secondmate-harness PRIMARY launch ` [] []`; LOCA config/account-routing-mode `off|observe|enforce`; direct account directories for new observe/enforce launches, legacy recovery for existing managed metadata; LOCAL, gitignored, default off, inherited (docs/configuration.md "Agent Fleet account routing") config/secondmate-account-pool optional Agent Fleet pool the PRIMARY uses for SECONDMATE launches when routing is enabled; LOCAL, gitignored; selection-only and NOT inherited Direct account-directory launch covers ship/scout crewmates and secondmate launches; a secondmate binds only the selected account, never the ship/scout worktree-identity contract. +config/azure-worker-account-home optional canonical absolute Pi home used only as the Azure worker credential pool; LOCAL, gitignored; absent falls back to the primary Pi coding-agent home for compatibility config/backlog-backend backlog backend override; LOCAL, gitignored; absent or "tasks-axi" = default tasks-axi backend, "manual" = force routine backlog updates to hand-editing; inherited by secondmate homes (section 10) config/backend new-task runtime override; LOCAL, gitignored, not inherited; absent auto-detects herdr/cmux then tmux, explicit zellij/Orca only, rejects codex-app; tmux reference, herdr/zellij/cmux experimental, Orca legacy (docs/tmux-backend.md, docs/herdr-backend.md, docs/zellij-backend.md, docs/orca-backend.md, docs/cmux-backend.md, docs/codex-app-backend.md) config/cmux-socket-password optional cmux control-socket password; LOCAL, gitignored; read fresh on every cmux CLI call and passed through without ever overriding an operator's own ambient CMUX_SOCKET_PASSWORD when absent (docs/cmux-backend.md "Setup") diff --git a/bin/fm-pi-refresh.py b/bin/fm-pi-refresh.py index 4ecca9bda65..2ba58713686 100755 --- a/bin/fm-pi-refresh.py +++ b/bin/fm-pi-refresh.py @@ -727,6 +727,13 @@ def command_run_once(args: argparse.Namespace) -> int: now = time.time() try: code = _run_once(args) + azure_source = scheduled_azure_source(args) + if code == 0 and azure_source is not None: + azure_args = argparse.Namespace(**vars(args)) + azure_args.source = str(azure_source) + azure_args.destination_root = str(scheduler_state_root() / "azure-account-homes") + azure_args.backup_root = str(scheduler_state_root() / "azure-pool-backups") + code = _run_once(azure_args) except RefreshError as exc: record_heartbeat( kind="attention" if exc.attention else "failed", detail=str(exc), now=now @@ -740,6 +747,34 @@ def command_run_once(args: argparse.Namespace) -> int: return code +def scheduled_azure_source(args: argparse.Namespace) -> Path | None: + config_value = getattr(args, "azure_home_config", None) + if not getattr(args, "scheduled", False) or not config_value: + return None + config = Path(config_value) + try: + metadata = config.lstat() + except FileNotFoundError: + return None + except OSError as exc: + fail(f"Azure Pi pool config is unreadable at {config}: {exc.strerror}") + if not stat.S_ISREG(metadata.st_mode) or config.is_symlink(): + fail(f"Azure Pi pool config must be a regular non-symlink file at {config}") + try: + lines = config.read_text(encoding="utf-8").splitlines() + except OSError as exc: + fail(f"Azure Pi pool config cannot be read at {config}: {exc.strerror}") + if len(lines) != 1 or not lines[0]: + fail(f"Azure Pi pool config must contain exactly one path at {config}") + home = Path(lines[0]) + if not home.is_absolute() or home != home.resolve(): + fail(f"Azure Pi pool config must name a canonical absolute directory at {config}") + source = home / "auth.json" + if source.expanduser().resolve() == Path(args.source).expanduser().resolve(): + return None + return source + + def _run_once(args: argparse.Namespace) -> int: if args.all and args.slot: fail("--all and --slot name different selections; pass one") @@ -1146,6 +1181,12 @@ def scheduler_job(interval: int, state_root: Path, nonce: str) -> dict[str, Any] "run-once", "--all", "--scheduled", + "--azure-home-config", + str( + Path(os.environ.get("FM_HOME") or BIN_DIR.parent).resolve() + / "config" + / "azure-worker-account-home" + ), ], "EnvironmentVariables": { "PATH": search, @@ -1467,6 +1508,7 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="record the heartbeat, which only launchd's own invocation can do", ) + run.add_argument("--azure-home-config", help=argparse.SUPPRESS) run.set_defaults(handler=command_run_once) install = commands.add_parser( diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index debb6e10b39..7474cb2cb39 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -39,12 +39,13 @@ # account_home and worktree_git_dir_identity bindings that lifecycle derives # its request from, and window= stays empty so local endpoint probes fail # closed. Cloud spawns run ENTIRELY on the pi-codex runtime: harness -# dispatch and claude profile routing are bypassed, account_home is the pi -# coding-agent directory, and pi's extension owns multi-profile selection on -# the worker. With the switch off (default) spawns stay byte-identical to -# the local path. Secondmate and account-recovery spawns always stay local, -# and --backend, raw launch commands, or a non-pi harness cannot be combined -# with cloud placement. +# dispatch and claude profile routing are bypassed. account_home comes from +# config/azure-worker-account-home when that file exists, otherwise from the +# pi coding-agent directory for backward compatibility; pi's extension owns +# multi-profile selection on the worker. With the switch off (default), +# spawns stay byte-identical to the local path. Secondmate and +# account-recovery spawns always stay local, and --backend, raw launch +# commands, or a non-pi harness cannot be combined with cloud placement. # FM_SPAWN_PARENT_TASK / FM_SPAWN_PARENT_TASK_GENERATION mark a cloud spawn as # a SECONDMATE COMPARTMENT CHILD: both are forwarded to the lifecycle request # as --parent-task/--parent-task-generation, where the controller's fan-out, @@ -611,6 +612,7 @@ if [ "$SPAWN_CLOUD" = azure ] && [ "$STATE" != "$TASK_HOME/state" ]; then exit 1 fi CLOUD_ACCOUNT_HOME= +CLOUD_ACCOUNT_MIN_HEADROOM_SECONDS=43200 CLOUD_PLACEMENT_STATE= CLOUD_WORKER_LAUNCH= RESUME_META= @@ -627,6 +629,80 @@ SPAWN_PREFLIGHT_ID=${POS[0]:-} spawn_idpart=${SPAWN_PREFLIGHT_ID%%=*} SPAWN_PREFLIGHT_BATCH=0 +resolve_cloud_account_home() { + local config_file="$CONFIG/azure-worker-account-home" configured canonical lines + if [ -e "$config_file" ] || [ -L "$config_file" ]; then + [ -f "$config_file" ] && [ ! -L "$config_file" ] || { + echo "error: config/azure-worker-account-home must be a regular non-symlink file" >&2 + return 1 + } + lines=$(awk 'END { print NR }' "$config_file" 2>/dev/null) || return 1 + [ "$lines" = 1 ] || { + echo "error: config/azure-worker-account-home must contain exactly one line" >&2 + return 1 + } + IFS= read -r configured < "$config_file" || { + echo "error: cannot read config/azure-worker-account-home" >&2 + return 1 + } + case "$configured" in + /*) ;; + *) + echo "error: config/azure-worker-account-home must name an absolute path" >&2 + return 1 + ;; + esac + [ -d "$configured" ] && [ ! -L "$configured" ] || { + echo "error: cloud placement account home '$configured' is not a real directory" >&2 + return 1 + } + canonical=$(CDPATH='' cd -- "$configured" 2>/dev/null && pwd -P) || return 1 + [ "$canonical" = "$configured" ] || { + echo "error: config/azure-worker-account-home must name its canonical physical directory ($canonical)" >&2 + return 1 + } + printf '%s\n' "$configured" + return 0 + fi + printf '%s\n' "${PI_CODING_AGENT_DIR:-$HOME/.pi/agent}" +} + +validate_cloud_account_pool() { + python3 - "$SCRIPT_DIR/fm-pi-account-home.py" "$CLOUD_ACCOUNT_HOME/auth.json" <<'PY' +import importlib.util +import sys + +module_path, source = sys.argv[1:] +spec = importlib.util.spec_from_file_location("fm_pi_account_home", module_path) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +pool = module.read_pool(module.Path(source)) +if not pool: + module.fail("Azure Pi pool must contain at least one profile") +expected = ["openai-codex"] + [ + f"openai-codex-{index}" for index in range(2, len(pool) + 1) +] +if sorted(pool) != sorted(expected): + module.fail("Azure Pi pool profiles must be gap-free: " + ", ".join(expected)) +faults = {name: module.entry_faults(pool[name]) for name in expected} +broken = [f"{name}: {', '.join(items)}" for name, items in faults.items() if items] +if broken: + module.fail("Azure Pi pool has unusable profile shapes: " + "; ".join(broken)) +accounts = [pool[name]["accountId"].strip() for name in expected] +if len(set(accounts)) != len(accounts): + module.fail("Azure Pi pool profiles must name distinct upstream accounts") +PY +} + +if [ "$SPAWN_CLOUD" = azure ]; then + CLOUD_ACCOUNT_HOME=$(resolve_cloud_account_home) || exit 1 + [ -d "$CLOUD_ACCOUNT_HOME" ] || { + echo "error: cloud placement account home '$CLOUD_ACCOUNT_HOME' is not a directory" >&2 + exit 1 + } + validate_cloud_account_pool || exit 1 +fi + release_secondmate_home_lifecycle_locks() { [ -z "${TASK_HOME_LIFECYCLE_LOCK:-}" ] \ || fm_account_lifecycle_lock_release "$TASK_HOME_LIFECYCLE_LOCK" >/dev/null 2>&1 || true @@ -2693,7 +2769,7 @@ launch_template() { # revisit this if pi ever ships a question tool that can park a secondmate - # fm-watch.sh skips stale-pane wakes for kind=secondmate, so a parked # secondmate would not trip stale detection. - printf '%s' '__AGENT__ --approve __MODELFLAG____EFFORTFLAG__-e __PITURNEND__ -e __PIWATCH__ "$(cat __BRIEF__)"' + printf '%s' '__AGENT__ --approve --fast __MODELFLAG____EFFORTFLAG__-e __PITURNEND__ -e __PIWATCH__ "$(cat __BRIEF__)"' else # --exclude-tools is a plain denylist over built-in, extension, and custom # tool names (pi 0.84.0 filters it as a Set, ignoring names that are not @@ -2701,7 +2777,7 @@ launch_template() { # crewmate's contract is to run autonomously and report through its status # file, so a tool that halts the run to ask a question nobody is watching is # never the right behavior here. - printf '%s' '__AGENT__ --approve --exclude-tools ask_question __MODELFLAG____EFFORTFLAG__-e __PIEXT__ "$(cat __BRIEF__)"' + printf '%s' '__AGENT__ --approve --exclude-tools ask_question --fast __MODELFLAG____EFFORTFLAG__-e __PIEXT__ "$(cat __BRIEF__)"' fi ;; # grok (Grok Build TUI): a positional prompt starts the supervised interactive @@ -4091,7 +4167,7 @@ if [ "$SPAWN_CLOUD" = azure ]; then # NOT reused - its paths exist only on this machine. --print keeps the run # bounded and non-interactive under the supervisor's device-null stdin. # shellcheck disable=SC2016 # single quotes are deliberate: $(cat ...) expands on the worker, not here - CLOUD_WORKER_LAUNCH="env PI_CODING_AGENT_DIR=/mnt/account/pi-agent pi --print --approve --exclude-tools ask_question ${MODELFLAG}${EFFORTFLAG}"'"$(cat /mnt/task/.fm-task/brief.md)"' + CLOUD_WORKER_LAUNCH="env PI_CODING_AGENT_DIR=/mnt/account/pi-agent pi --print --approve --exclude-tools ask_question --fast ${MODELFLAG}${EFFORTFLAG}"'"$(cat /mnt/task/.fm-task/brief.md)"' # A secondmate compartment has no single worker entrypoint: its session # legs are built and dispatched by fm-secondmate-cloud-monitor.sh, so the # crewmate launch string above is never persisted or executed for it. @@ -4468,6 +4544,17 @@ spawn_cloud_bind_leased_account() { # echo "error: leased provider-account home '$leased' holds no credential for $ID" >&2 return 1 } + # Azure's hard VM shutdown is six hours after creation. Require twice that + # much access-token headroom before staging so the guest cannot reach Pi's + # automatic OAuth refresh path and rotate a refresh token independently of + # the controller-owned pool. The host refresh scheduler is the sole refresh + # authority; a stale pool slot is handed back rather than copied to Azure. + "$SCRIPT_DIR/fm-credential-expiry.py" check --harness pi \ + --margin-seconds "$CLOUD_ACCOUNT_MIN_HEADROOM_SECONDS" \ + --min-state usable "$leased" >/dev/null || { + echo "error: leased provider-account credential lacks twelve hours of access-token headroom for $ID" >&2 + return 1 + } # Exactly one provider slot, checked by shape and never by content: a home # carrying more than one is the pool, and the guest would pick the first. python3 - "$leased/auth.json" <<'PY' || return 1 @@ -4591,7 +4678,7 @@ spawn_cloud_persist_convergence_artifacts() { cp "$SCRIPT_DIR/fm-secondmate-session.py" "$STATE/$ID.cloud-payload/fm-secondmate-session.py" || exit 1 cp "$SCRIPT_DIR/fm-secondmate-spawn.pi-ext.ts" "$STATE/$ID.cloud-payload/fm-secondmate-spawn.pi-ext.ts" || exit 1 fi - CLOUD_ACCOUNT_SOURCE=${PI_CODING_AGENT_DIR:-$HOME/.pi/agent} + CLOUD_ACCOUNT_SOURCE=$CLOUD_ACCOUNT_HOME if [ ! -f "$CLOUD_ACCOUNT_SOURCE/auth.json" ]; then echo "error: cloud account source lacks auth.json at $CLOUD_ACCOUNT_SOURCE" >&2 exit 1 @@ -4820,9 +4907,10 @@ META_WINDOW=$T # Cloud placement persists the exact identities the elastic worker lifecycle # derives its bindings from (docs/azure-workers.md "Queue request"): the # physical worktree Git-dir identity and the provider account home. The cloud -# lane is pi-codex only, so the account home is always the pi coding-agent -# directory - never a claude/codex profile home, and never a local -# account-directory rotation (that machinery is bypassed entirely for cloud). +# lane is pi-codex only, so the account home is the dedicated Azure worker Pi +# pool when configured, otherwise the legacy pi coding-agent directory - never +# a claude/codex profile home, and never a local account-directory rotation +# (that machinery is bypassed entirely for cloud). if [ "$SPAWN_CLOUD" = azure ]; then [ "$DIRECT_ACCOUNT_ROUTING" != 1 ] || { echo "error: cloud placement must not reach direct account-directory routing; refusing to mix profile machinery into the worker lane" >&2 @@ -4834,7 +4922,6 @@ if [ "$SPAWN_CLOUD" = azure ]; then exit 1 } fi - CLOUD_ACCOUNT_HOME=${PI_CODING_AGENT_DIR:-$HOME/.pi/agent} [ -d "$CLOUD_ACCOUNT_HOME" ] || { echo "error: cloud placement account home '$CLOUD_ACCOUNT_HOME' is not a directory" >&2 exit 1 diff --git a/docs/azure-requirements.md b/docs/azure-requirements.md index 5d7b055c0c9..ad41970de34 100644 --- a/docs/azure-requirements.md +++ b/docs/azure-requirements.md @@ -64,7 +64,8 @@ digest `1f238e42...`, and an outcome bundle of one commit `r5-accept-readme-v3-20260822` (`asg-00000020`) and `r5-accept-package-v3-20260822` (`asg-00000021`) each returned exit 0, `timed_out false`, and `outcome_commits 0` for their read-only briefs. Evidence and paths are in R2/R3 and R5. -Placement across distinct upstream accounts is R5, and is now proven live there. +Placement across distinct upstream accounts is R5, where the former single-profile placement +residual is closed and now proven live. ## R2/R3. Secondmates run in Azure, and can spawn crewmates in Azure @@ -344,8 +345,9 @@ reaches `close` with its worktree disk released". Status: DONE, met live on 2026-08-22. -Crosscheck on the pi fleet is done at the roster level: eight pi profiles across eight distinct -upstream accounts, projected into single-profile account homes by `bin/fm-pi-account-home.py`, +Crosscheck on the pi fleet is done at the roster level. The current operating split is six Azure +worker profiles across six distinct upstream accounts plus one separate local Firstmate profile; +Azure profiles are projected into single-profile account homes by `bin/fm-pi-account-home.py`, with the roster repointed and read back through the real `bin/fm-crosscheck.py` reader and policy screen. Under the second 2026-08-19 amendment this roster is now the dormant crosscheck fallback; the pi fleet's primary duties are authors and no-mistakes. @@ -358,6 +360,9 @@ into a controller-owned account home, and `bin/fm-spawn.sh` narrows the staged p to it, so the worker receives exactly one account rather than the pooled `auth.json`. An exhausted pool refuses by name, listing every leased profile and the task holding it. Mechanics are owned by `docs/azure-workers.md` ("Provider-account placement across the Pi fleet"). +The host is also the sole OAuth refresh authority: staging requires twelve hours of access-token +headroom against a six-hour worker shutdown deadline, so a guest cannot live long enough to rotate +the copied refresh token independently of the Azure pool. Proven locally against a fixture provider by `tests/fm-worker-placement.test.sh` (eight concurrent placements racing the controller lock take eight distinct upstream accounts, read back from @@ -367,10 +372,11 @@ account) and end to end through the real `bin/fm-spawn.sh` by `tests/fm-spawn-cl asserts the staged credential is the leased single profile. Operational consequence the owner must know: concurrent placements are now bounded by -`min(FM_AZURE_WORKER_MAX, distinct upstream accounts in the pool)`. With eight Pi accounts the -ninth concurrent placement refuses although MAX_WORKERS is 16, and compartments compete in the -same pool. Sixteen crewmates never could run on eight accounts without sharing one; they used -to do it silently. Raising the ceiling means adding profiles on distinct accounts. +`min(FM_AZURE_WORKER_MAX, distinct upstream accounts in the pool)`. With the current six-account +Azure pool the seventh concurrent placement refuses although MAX_WORKERS is 16, and compartments +compete in the same pool. Sixteen crewmates cannot run on six accounts without sharing one. +Raising the ceiling means adding profiles on distinct accounts to the Azure pool; adding profiles +to the separate local Firstmate pool does not change Azure capacity. Acceptance: concurrent crewmates run on distinct pi profiles with no account collision. diff --git a/docs/azure-workers.md b/docs/azure-workers.md index c0faa838877..375bae405a3 100644 --- a/docs/azure-workers.md +++ b/docs/azure-workers.md @@ -36,9 +36,11 @@ The account binding must be a high-entropy digest produced by the account lease The lease owner is the controller: it derives the binding from the profile's upstream account identity (see the placement section below), never from the profile's local slot name. The local slot name (`openai-codex-2` and the like) DOES appear in bounded status, because an operator has to be able to see which profile a task holds; it is a local label for a pool slot, not the upstream account identity, and it never reaches an Azure tag. +The host is the only OAuth refresh authority. Before a leased single-profile home is staged, `fm-spawn.sh` requires twelve hours of access-token headroom, twice the worker VM's six-hour hard shutdown window. A stale slot is withdrawn and refused instead of being copied to a guest that could reach Pi's automatic refresh path and rotate the pool's refresh token independently. + ## Provider-account placement across the Pi fleet -The task metadata names the provider-account POOL this task may draw from (the cloud lane's is the Pi coding-agent home); WHICH profile of that pool the placement gets is decided by the controller, because that decision has to exclude every other concurrent placement and no task-local document can see them. +The task metadata names the provider-account POOL this task may draw from. The cloud lane reads the canonical absolute directory in `config/azure-worker-account-home` when present and otherwise falls back to the primary Pi coding-agent home for compatibility. This lets a local Firstmate keep a separate Pi login while Azure owns a disjoint worker fleet. WHICH profile of the selected pool the placement gets is decided by the controller, because that decision has to exclude every other concurrent placement and no task-local document can see them. Selection happens inside `command_request`, in the same lock hold and the same `save_state` that writes the queue entry, so selection and the lease are one act. The queue entry IS the lease: the set of leased accounts is derived from the non-complete queue, never from a second ledger, so there is no state a crash can leave in which an account is held by something the queue does not show. @@ -225,7 +227,7 @@ The blob name carries the request digest, so a later execute against the same wo The controller never infers safe deletion from a terminal chat line, a missing VM, elapsed time, or budget pressure. The ordinary Firstmate owners first remove the endpoint, publish the report, prove landed work, release the provider account, and complete their normal cleanup checks. `authority-receipt` invokes `bin/fm-worker-authority.py`, which reads the ordinary task metadata, endpoint backend oracle, completion-report contract, Git landing graph, account task/home binding, and clean exact worktree root rather than accepting operator-entered digests. -For Azure placement, the task's `account_home` is the vendor-neutral Pi pool, so account authority instead requires the task-recorded selected profile and single-profile home to equal the controller queue's exact lease, then reads its owner-private credential without following links and reproduces the controller-owned upstream-account binding. +For Azure placement, the task's `account_home` is the vendor-neutral Pi pool selected above, so account authority instead requires the task-recorded selected profile and single-profile home to equal the controller queue's exact lease, then reads its owner-private credential without following links and reproduces the controller-owned upstream-account binding. It produces an `fm.worker-release/v2` bundle with five independently canonical `fm.worker-authority/v1` receipts for endpoint absence, report validity, landed work, account ownership, and writable-worktree cleanliness, plus the exact home, task, generations, cloud instance, account, worktree, repository, and every resource identity. When the CONTROLLER-OWNED worker role is `secondmate`, the same five receipts carry compartment evidence semantics: endpoint proves the compartment monitor pane absent through the same backend oracle; report proves the session closeout - the monitor's terminal status file, the chained close ack in its durable state, and the ordered `completion.md` contract; landing proves every chained outbox bundle landed into the local secondmate home worktree (or provably none) by REACHABILITY - each collected bundle's own tip commit must be an ancestor of the home worktree's HEAD, which also descends from the assignment's exact starting repository generation; account is unchanged; and worktree proves the home quiesced - exact repository root, no uncommitted or untracked work - while staying advisory for children, whose refusal `command_release` owns. Which semantics apply is never decided by the task metadata alone: the worker record's `role` and the metadata's `kind` must agree, and a disagreement in either direction refuses, so flipping one local metadata line can never move an ordinary author worker onto the compartment lane and release work that was never landed. diff --git a/tests/fm-pi-refresh.test.sh b/tests/fm-pi-refresh.test.sh index c77d1ab969f..576aeea3d85 100755 --- a/tests/fm-pi-refresh.test.sh +++ b/tests/fm-pi-refresh.test.sh @@ -199,7 +199,9 @@ assert job["RunAtLoad"] is True, job assert job["StartInterval"] == 900, job arguments = job["ProgramArguments"] assert str(pathlib.Path(sys.argv[2]).resolve()) in arguments, arguments -assert arguments[-3:] == ["run-once", "--all", "--scheduled"], arguments +assert arguments[-5:-2] == ["run-once", "--all", "--scheduled"], arguments +assert arguments[-2] == "--azure-home-config", arguments +assert pathlib.Path(arguments[-1]).is_absolute(), arguments environment = job["EnvironmentVariables"] for name in ("PATH", "HOME", "FM_PI_BIN", "FM_PI_NODE_BIN", "FM_PI_REFRESH_STATE_ROOT", "FM_PI_REFRESH_ACTIVATION_NONCE", @@ -387,6 +389,47 @@ PY pass "the schedule is machine-global, stamps only for launchd, and reports absent, orphaned, foreign, unloaded, unproven, stale, failing and attention apart from healthy" } +scheduled_azure_pool_contract() { + local work primary azure config out code + work=$(fm_test_tmproot fm-pi-refresh-azure-pool) + primary=$work/primary + azure=$work/azure + config=$work/azure-worker-account-home + mkdir -p "$primary" "$azure" + python3 - "$primary/auth.json" "$azure/auth.json" "$MARKER" <<'PY' +import json +import sys +import time + +primary, azure, marker = sys.argv[1:] +day = 86400 * 1000 +now = time.time() * 1000 +json.dump({ + "openai-codex": { + "type": "oauth", "access": marker + ".primary.access", + "refresh": marker + ".primary.refresh", "accountId": "primary-account", + "expires": now + 9 * day, + } +}, open(primary, "w")) +json.dump({ + "openai-codex": { + "type": "oauth", "access": marker + ".azure.access", + "refresh": "", "accountId": "azure-account", "expires": now + day, + } +}, open(azure, "w")) +PY + printf '%s\n' "$azure" > "$config" + + code=0 + out=$(python3 "$TOOL" run-once --source "$primary/auth.json" --all --scheduled \ + --azure-home-config "$config" --backup-root "$work/backups" \ + --destination-root "$work/homes" 2>&1) || code=$? + expect_code 1 "$code" "the scheduled run ignored an Azure pool that needs interactive login" + assert_contains "$out" "interactive login" "the Azure pool refusal did not reach the scheduler result" + assert_not_contains "$out" "$MARKER" "the two-pool scheduled refusal leaked credential material" + pass "the machine scheduler owns a separately configured Azure Pi pool" +} + selection_contract() { local work pool out code work=$(fm_test_tmproot fm-pi-refresh-select) @@ -809,4 +852,5 @@ republish_contract pool_integrity_contract adapter_outcome_contract adapter_store_contract +scheduled_azure_pool_contract scheduler_contract diff --git a/tests/fm-spawn-cloud.test.sh b/tests/fm-spawn-cloud.test.sh index 9283408de1f..ba38c9885ec 100755 --- a/tests/fm-spawn-cloud.test.sh +++ b/tests/fm-spawn-cloud.test.sh @@ -19,18 +19,19 @@ TMP_ROOT=$(fm_test_tmproot fm-spawn-cloud) # pi agent home, and placement leases ONE profile out of it (R5), so the fixture # has to look like the real pool: several profiles, each a complete oauth # credential shape naming a distinct upstream account. -fm_spawn_cloud_write_pi_pool() { # - python3 - "$1" <<'PY' +fm_spawn_cloud_write_pi_pool() { # [account prefix] + python3 - "$1" "${2:-fixture}" <<'PY' import json import sys +prefix = sys.argv[2] pool = {} -for index in range(1, 5): +for index in range(1, 7): name = "openai-codex" if index == 1 else "openai-codex-{}".format(index) pool[name] = { - "type": "oauth", "access": "fixture-access-{}".format(index), - "refresh": "fixture-refresh-{}".format(index), - "accountId": "fixture-account-{}".format(index), + "type": "oauth", "access": "{}-access-{}".format(prefix, index), + "refresh": "{}-refresh-{}".format(prefix, index), + "accountId": "{}-account-{}".format(prefix, index), "expires": 4102444800000, } with open(sys.argv[1], "w", encoding="utf-8") as handle: @@ -486,6 +487,7 @@ test_cloud_spawn_places_worker_and_runs_the_entrypoint() { assert_grep 'placement=azure' "$meta" "the cloud spawn did not record placement=azure" assert_grep 'harness=pi' "$meta" "the cloud spawn did not record the pi-codex runtime" assert_grep "account_home=$CASE_DIR/pi-agent-home" "$meta" "the cloud spawn did not record the pi coding-agent account home" + assert_grep '--fast' "$HOME_DIR/state/$id.cloud-entrypoint" "the cloud worker entrypoint did not force Fast Mode" assert_grep 'worktree_git_dir_identity=' "$meta" "the cloud spawn did not record the worktree Git-dir identity" assert_grep 'worktree_git_dir=' "$meta" "the cloud spawn did not record the worktree Git dir" # R5: the controller leased ONE profile out of that pool, and the credential @@ -558,6 +560,113 @@ PY pass "cloud spawn persists lifecycle bindings, assigns a worker, and runs the entrypoint remotely" } +test_cloud_spawn_uses_the_dedicated_azure_account_pool() { + local record id out status meta azure_home + id=cloud-pool-c2b + record=$(make_cloud_case dedicated-account-pool "$id") + read_cloud_case "$record" + azure_home="$CASE_DIR/azure-pi-agent-home" + mkdir -p "$azure_home" + fm_spawn_cloud_write_pi_pool "$azure_home/auth.json" azure + printf '%s\n' "$azure_home" > "$HOME_DIR/config/azure-worker-account-home" + out=$(run_cloud_spawn "$CASE_DIR" "$HOME_DIR" "$WORKTREE_DIR" "$FAKEBIN_DIR" "$id" "$PROJECT_DIR") + status=$? + expect_code 0 "$status" "a cloud spawn with a dedicated account pool should succeed: $out" + meta="$HOME_DIR/state/$id.meta" + assert_grep "account_home=$azure_home" "$meta" "the cloud spawn ignored config/azure-worker-account-home" + python3 - "$meta" "$HOME_DIR/state/azure-workers/controller.json" \ + "$CASE_DIR/pi-agent-home/auth.json" "$azure_home/auth.json" "$id" <<'PY' \ + || fail "the controller did not lease from the dedicated Azure pool" +import json +from pathlib import Path +import sys + +meta_path, controller_path, local_path, azure_path, task = sys.argv[1:] +meta = {} +for line in open(meta_path, encoding="utf-8"): + if "=" in line: + key, value = line.rstrip("\n").split("=", 1) + meta[key] = value +state = json.load(open(controller_path, encoding="utf-8")) +item = next(entry for entry in state["queue"].values() if entry["task"] == task) +local_pool = json.load(open(local_path, encoding="utf-8")) +azure_pool = json.load(open(azure_path, encoding="utf-8")) +assert item["account_pool_home"] == meta["account_home"] +assert item["account_pool_home"] == str(Path(azure_path).parent) +assert azure_pool[item["account_profile"]]["accountId"].startswith("azure-account-") +assert azure_pool[item["account_profile"]]["accountId"] != local_pool[item["account_profile"]]["accountId"] +PY + pass "config/azure-worker-account-home isolates Azure credentials from the primary Pi home" +} + +test_cloud_spawn_refuses_an_unsafe_azure_account_pool_path() { + local record id out status + id=cloud-pool-bad-c2c + record=$(make_cloud_case unsafe-account-pool "$id") + read_cloud_case "$record" + printf '%s\n' relative/pi-agent-home > "$HOME_DIR/config/azure-worker-account-home" + out=$(run_cloud_spawn "$CASE_DIR" "$HOME_DIR" "$WORKTREE_DIR" "$FAKEBIN_DIR" "$id" "$PROJECT_DIR") + status=$? + expect_code 1 "$status" "a relative Azure account-pool path should fail closed: $out" + assert_contains "$out" "must name an absolute path" "the refusal did not explain the account-pool path contract: $out" + assert_absent "$HOME_DIR/state/$id.meta" "an unsafe Azure account-pool path still wrote task metadata" + pass "an unsafe Azure account-pool path fails closed before spawn mutation" +} + +test_cloud_spawn_refuses_a_gap_in_the_azure_account_pool() { + local record id out status azure_home + id=cloud-pool-gap-c2e + record=$(make_cloud_case incomplete-account-pool "$id") + read_cloud_case "$record" + azure_home="$CASE_DIR/azure-pi-agent-home" + mkdir -p "$azure_home" + fm_spawn_cloud_write_pi_pool "$azure_home/auth.json" azure + python3 - "$azure_home/auth.json" <<'PY' +import json +import sys +path = sys.argv[1] +pool = json.load(open(path, encoding="utf-8")) +del pool["openai-codex-4"] +with open(path, "w", encoding="utf-8") as handle: + json.dump(pool, handle) +PY + printf '%s\n' "$azure_home" > "$HOME_DIR/config/azure-worker-account-home" + out=$(run_cloud_spawn "$CASE_DIR" "$HOME_DIR" "$WORKTREE_DIR" "$FAKEBIN_DIR" "$id" "$PROJECT_DIR") + status=$? + expect_code 1 "$status" "a gapped Azure account pool should fail closed: $out" + assert_contains "$out" "profiles must be gap-free" "the refusal did not explain the profile-numbering contract: $out" + assert_absent "$HOME_DIR/state/$id.meta" "a gapped Azure pool still wrote task metadata" + pass "Azure placement requires gap-free account numbering while allowing pool growth" +} + +test_cloud_spawn_refuses_a_credential_that_could_refresh_on_the_guest() { + local record id out status + id=cloud-expiring-c2d + record=$(make_cloud_case expiring-account "$id") + read_cloud_case "$record" + python3 - "$CASE_DIR/pi-agent-home/auth.json" <<'PY' +import json +import sys +import time + +path = sys.argv[1] +pool = json.load(open(path, encoding="utf-8")) +pool["openai-codex"]["expires"] = int((time.time() + 3600) * 1000) +with open(path, "w", encoding="utf-8") as handle: + json.dump(pool, handle, sort_keys=True, indent=2) +PY + out=$(run_cloud_spawn "$CASE_DIR" "$HOME_DIR" "$WORKTREE_DIR" "$FAKEBIN_DIR" "$id" "$PROJECT_DIR") + status=$? + expect_code 1 "$status" "a credential that could refresh during the worker lifetime should fail closed: $out" + assert_contains "$out" "lacks twelve hours of access-token headroom" \ + "the refusal did not explain the guest refresh boundary: $out" + assert_absent "$HOME_DIR/state/$id.cloud-account/auth.json" \ + "a near-expiry credential was left staged after refusal" + assert_no_grep "\"task\":\"$id\"" "$HOME_DIR/state/azure-workers/controller.json" \ + "the refused near-expiry credential kept its provider-account lease" + pass "cloud staging prevents a guest from becoming a second OAuth refresh authority" +} + test_cloud_spawn_stays_durably_queued_without_admission() { local record id out status meta id=cloud-queue-c3 @@ -1743,6 +1852,10 @@ test_compartment_child_withdraw_removes_the_staged_credential() { test_cloud_switch_off_keeps_the_local_path_and_metadata_shape test_cloud_spawn_places_worker_and_runs_the_entrypoint +test_cloud_spawn_uses_the_dedicated_azure_account_pool +test_cloud_spawn_refuses_an_unsafe_azure_account_pool_path +test_cloud_spawn_refuses_a_gap_in_the_azure_account_pool +test_cloud_spawn_refuses_a_credential_that_could_refresh_on_the_guest test_monitor_lands_the_outcome_bundle test_monitor_reports_an_already_landed_outcome test_monitor_reports_a_crewmate_that_never_committed diff --git a/tests/fm-spawn-dispatch-profile.test.sh b/tests/fm-spawn-dispatch-profile.test.sh index 1cda7d1ff78..89e1daf6270 100755 --- a/tests/fm-spawn-dispatch-profile.test.sh +++ b/tests/fm-spawn-dispatch-profile.test.sh @@ -464,7 +464,7 @@ test_pi_omits_invalid_max_effort() { expect_code 0 "$status" "pi spawn with max effort should not pass an invalid flag" assert_meta_profile "$HOME_DIR/state/$id.meta" pi sonnet max launch=$(cat "$LAUNCH_LOG") - assert_contains "$launch" "pi --approve --exclude-tools ask_question --model 'sonnet' -e" \ + assert_contains "$launch" "pi --approve --exclude-tools ask_question --fast --model 'sonnet' -e" \ "pi launch did not thread model after the autonomy flags" assert_not_contains "$launch" "--thinking" "pi launch must omit --thinking max because the CLI rejects it" pass "pi threads model and omits unsupported max effort" @@ -487,7 +487,7 @@ test_pi_crewmate_carries_autonomy_flags() { status=$? expect_code 0 "$status" "pi spawn with model and effort should succeed: $out" launch=$(cat "$LAUNCH_LOG") - assert_contains "$launch" "pi --approve --exclude-tools ask_question --model 'sonnet' --thinking 'high' -e" \ + assert_contains "$launch" "pi --approve --exclude-tools ask_question --fast --model 'sonnet' --thinking 'high' -e" \ "pi launch with model/effort did not render the autonomy flags ahead of the profile flags" # Without model or effort: both placeholders expand to nothing, so --approve and @@ -499,7 +499,7 @@ test_pi_crewmate_carries_autonomy_flags() { status=$? expect_code 0 "$status" "pi spawn without model or effort should succeed: $out" launch=$(cat "$LAUNCH_LOG") - assert_contains "$launch" "pi --approve --exclude-tools ask_question -e" \ + assert_contains "$launch" "pi --approve --exclude-tools ask_question --fast -e" \ "pi launch without model/effort did not render well-formed autonomy flags" assert_not_contains "$launch" "--model" "bare pi launch must not carry a --model flag" assert_not_contains "$launch" "--thinking" "bare pi launch must not carry a --thinking flag" @@ -530,7 +530,7 @@ test_pi_secondmate_approves_without_excluding_tools() { expect_code 0 "$status" "pi secondmate spawn should succeed: $out" assert_contains "$out" "spawned $id harness=pi kind=secondmate" "secondmate did not launch on pi" launch=$(cat "$LAUNCH_LOG") - assert_contains "$launch" "pi --approve -e" \ + assert_contains "$launch" "pi --approve --fast -e" \ "pi secondmate launch did not carry --approve ahead of its extensions" assert_not_contains "$launch" "--exclude-tools" \ "pi secondmate launch must keep its question tool; --exclude-tools is crewmate-only"