diff --git a/bin/fm-spawn-cloud-monitor.sh b/bin/fm-spawn-cloud-monitor.sh index 6d32123f5a8..cde4c235823 100755 --- a/bin/fm-spawn-cloud-monitor.sh +++ b/bin/fm-spawn-cloud-monitor.sh @@ -110,11 +110,41 @@ reclaim_stale_dispatch() { rm -f "$DISPATCH_MARKER" } +# account_directory_is_single_slot: the staged account directory holds exactly +# one provider slot, which is the only shape that may ride to a worker. +# A pooled directory would hand every signed-in account to the guest and let pi +# resolve the first slot - a shared-account placement whatever the queue says. +# The spawn writes this directory exactly once, from the single account the +# controller leased; this monitor outlives the spawn and dispatches on its own, +# so the shape is re-checked at the point of USE and not only where it is written. +account_directory_is_single_slot() { + python3 - "$STATE/$ID.cloud-account/auth.json" <<'ACCOUNTSLOTS' +import json +import sys + +try: + with open(sys.argv[1], encoding="utf-8") as handle: + parsed = json.load(handle) +except (OSError, ValueError): + raise SystemExit(1) +raise SystemExit(0 if isinstance(parsed, dict) and len(parsed) == 1 else 1) +ACCOUNTSLOTS +} dispatch_converged_execute() { + local assignment + # BEFORE the claim, deliberately. The claim is the exactly-once marker shared + # with the spawn: standing down after taking it would leave both owners + # believing the other dispatched, and nothing ever would. An account + # directory that is not yet the leased single account means the spawn has not + # finished narrowing it, so this poll simply does not claim and the next one + # retries. + if [ -d "$STATE/$ID.cloud-account" ] && ! account_directory_is_single_slot; then + echo "cloud-crewmate $ID: staged account directory is not one leased provider slot yet; not dispatching this poll" + return 0 + fi # Claim first (O_EXCL): if the spawn process already dispatched, or a prior # monitor iteration did, stand down. The whole dispatch runs in a subshell # so the sourced persisted environment never leaks into the monitor loop. - local assignment (set -C; : > "$DISPATCH_MARKER") 2>/dev/null || return 0 assignment=$(assignment_generation) || assignment= if [ -z "$assignment" ]; then diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index dcc9284cdb6..3044c17a6d9 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -4083,6 +4083,7 @@ if [ "$SPAWN_CLOUD" = azure ]; then # new monitor can never observe them. rm -f "$STATE/$ID.cloud-entrypoint" "$STATE/$ID.cloud-env" \ "$STATE/$ID.cloud-execute-dispatched" "$STATE/$ID.cloud-worktree" \ + "$STATE/$ID.worker-request.out" \ "$STATE/$ID.worker-result.json" "$STATE/$ID.worker-execute.log" rm -rf "$STATE/$ID.cloud-payload" "$STATE/$ID.cloud-account" if [ "$KIND" = secondmate ]; then @@ -4398,6 +4399,99 @@ spawn_cloud_record_assignment() { # fi fm_account_meta_lock_release "$lock" || return 1 } +# spawn_cloud_bind_leased_account: narrow the staged provider credential to the +# ONE Pi profile the controller leased for this placement (R5). +# +# The controller is the only selector: it picks a free profile under its own +# lock, in the same act that writes the queue entry that IS the lease, and +# prints the single-profile account home it projected (bin/fm-pi-account-home.py +# writes it; nothing here re-derives a home or re-implements a projection). +# This function reads that path back and makes it the credential the worker +# actually receives, so the lease is not a paper lease: without it the pooled +# auth.json would ride to the guest and every concurrent crewmate would resolve +# to the pool's first slot - one account, N workers, which is the collision R5 +# exists to remove. +# +# It refuses rather than falling back. No leased path, no credential at the +# leased path, or a leased credential carrying more than one provider slot all +# stop the placement, because each of those is "we do not know which account +# this worker will use". +spawn_cloud_bind_leased_account() { # + local out=$1 leased line tmp profile + if spawn_test_lab_enabled && [ "${FM_TEST_CLOUD_ACCOUNT_BIND_FAIL:-0}" = 1 ]; then + # Test-only: the lease-handback path below has no other injection point, + # and an untested handback is how a pool quietly shrinks to zero. + echo "error: test-only provider-account bind failure for $ID" >&2 + return 1 + fi + leased= + profile= + while IFS= read -r line; do + case "$line" in + 'account-home /'*) leased=${line#account-home } ;; + 'account-profile '*) profile=${line#account-profile } ;; + esac + done < "$out" + # The controller reports the slot name separately BECAUSE the projected home + # is keyed on the lease identity rather than the slot name; reading the name + # off the path's last component would be reading the wrong thing. + [ -n "$profile" ] || { + echo "error: the controller named no leased provider-account profile for $ID" >&2 + return 1 + } + [ -n "$leased" ] || { + echo "error: the controller named no leased provider-account home for $ID; refusing to stage a pooled credential" >&2 + return 1 + } + [ -d "$leased" ] && [ -f "$leased/auth.json" ] || { + echo "error: leased provider-account home '$leased' holds no credential for $ID" >&2 + return 1 + } + # Exactly one provider slot, checked by shape and never by content: a home + # carrying more than one is the pool, and the guest would pick the first. + python3 - "$leased/auth.json" <<'PY' || return 1 +import json +import sys + +try: + with open(sys.argv[1], encoding="utf-8") as handle: + parsed = json.load(handle) +except (OSError, ValueError): + print("error: leased provider-account credential is unreadable", file=sys.stderr) + raise SystemExit(1) +if not isinstance(parsed, dict) or len(parsed) != 1: + print( + "error: leased provider-account credential does not hold exactly one provider slot", + file=sys.stderr, + ) + raise SystemExit(1) +PY + install -d -m 0700 "$STATE/$ID.cloud-account" || return 1 + tmp=$(mktemp "$STATE/$ID.cloud-account/.auth.XXXXXX") || return 1 + cp "$leased/auth.json" "$tmp" || { rm -f "$tmp"; return 1; } + chmod 0600 "$tmp" || { rm -f "$tmp"; return 1; } + # Renamed, not written in place: a crash mid-copy must never leave the staged + # credential truncated, which would fail the guest's digest check with no + # clue why. + mv "$tmp" "$STATE/$ID.cloud-account/auth.json" || { rm -f "$tmp"; return 1; } + spawn_cloud_record_account_placement "$profile" "$leased" || return 1 + echo "fm-spawn: $ID placed on pi profile $profile ($leased)" >&2 +} +spawn_cloud_record_account_placement() { # + local profile=$1 leased=$2 lock + lock=$(fm_account_meta_lock_acquire "$STATE" "$ID") || return 1 + if [ "$(fm_account_meta_value "$STATE/$ID.meta" generation_id)" = "$SPAWN_GENERATION_ID" ] \ + && [ -z "$(fm_account_meta_value "$STATE/$ID.meta" worker_account_profile)" ]; then + { + printf 'worker_account_profile=%s\n' "$profile" + printf 'worker_account_home=%s\n' "$leased" + } >> "$STATE/$ID.meta" || { + fm_account_meta_lock_release "$lock" >/dev/null 2>&1 || true + return 1 + } + fi + fm_account_meta_lock_release "$lock" || return 1 +} # spawn_cloud_dispatch: after metadata install, drive the elastic worker # lifecycle; the local Herdr endpoint holds only the tracking monitor. The # request is durable; if admission leaves it queued (budget/quota/cost @@ -4472,8 +4566,16 @@ spawn_cloud_persist_convergence_artifacts() { echo "error: cloud account source lacks auth.json at $CLOUD_ACCOUNT_SOURCE" >&2 exit 1 fi - cp "$CLOUD_ACCOUNT_SOURCE/auth.json" "$STATE/$ID.cloud-account/auth.json" || exit 1 - chmod 0600 "$STATE/$ID.cloud-account/auth.json" + # The POOLED auth.json is deliberately NOT copied here. This runs BEFORE the + # request creates the lease, and the tracking monitor pane already exists and + # is already polling: a crash, kill, or plain slow reconcile between here and + # the narrowing would leave every signed-in account staged in a directory the + # monitor is willing to dispatch as --account-dir. The account directory is + # therefore written exactly once, by spawn_cloud_bind_leased_account, after + # the controller has said which single account this placement leased. That + # removes the window rather than guarding it. + # settings.json is pi CONFIGURATION, not credential material, so it is staged + # here with the rest of the payload. if [ -f "$CLOUD_ACCOUNT_SOURCE/settings.json" ]; then cp "$CLOUD_ACCOUNT_SOURCE/settings.json" "$STATE/$ID.cloud-account/settings.json" || exit 1 chmod 0600 "$STATE/$ID.cloud-account/settings.json" @@ -4518,7 +4620,7 @@ spawn_cloud_claim_execute_dispatch() { (set -C; : > "$STATE/$ID.cloud-execute-dispatched") 2>/dev/null } spawn_cloud_dispatch() { - local owner_kind=primary assignment wall role_args parent_args task_home_args + local owner_kind=primary assignment wall role_args parent_args task_home_args request_report CLOUD_PLACEMENT_STATE=queued # owner_kind is a property of the home the TASK belongs to, not of the home # that owns the money document. They are the same directory everywhere @@ -4557,11 +4659,18 @@ spawn_cloud_dispatch() { # directory against the marker plus the primary's own registry. task_home_args=() [ "$TASK_HOME" = "$FM_HOME" ] || task_home_args=(--task-home "$TASK_HOME") + # The request's STDOUT carries the leased provider-account home (R5), so it is + # captured rather than folded into stderr; stderr still flows through + # untouched, and the captured lines are echoed on for the operator either way. + request_report="$STATE/$ID.worker-request.out" + rm -f "$request_report" spawn_cloud_lifecycle request \ --task "$ID" --task-generation "$SPAWN_GENERATION_ID" \ --owner-kind "$owner_kind" ${role_args[@]+"${role_args[@]}"} \ ${parent_args[@]+"${parent_args[@]}"} \ - ${task_home_args[@]+"${task_home_args[@]}"} --eligible >&2 || { + ${task_home_args[@]+"${task_home_args[@]}"} --eligible > "$request_report" || { + cat "$request_report" >&2 2>/dev/null || true + rm -f "$request_report" # No durable queue entry exists, so the convergence artifacts have no # owner; remove them (including the copied provider credential) with the # rolled-back spawn. @@ -4582,6 +4691,31 @@ spawn_cloud_dispatch() { echo "error: cloud worker request was refused for $ID" >&2 return 1 } + cat "$request_report" >&2 + if spawn_test_lab_enabled && [ "${FM_TEST_CLOUD_ABORT_AFTER_REQUEST:-0}" = 1 ]; then + # Test-only: die exactly in the window between the durable lease and the + # narrowing, with the tracking monitor already live. This is the window the + # pool used to be staged in, and the only way to assert it is empty is to + # stop the process inside it. + kill -9 $$ + fi + spawn_cloud_bind_leased_account "$request_report" || { + # The queue entry exists and is the LEASE on a provider account. A spawn + # that cannot bind that account must hand it back rather than leave it held + # by work that will never run: an orphaned lease shrinks the pool by one + # every time this happens. The entry is still `queued` here (reconcile has + # not run), which is exactly what withdraw accepts, and withdraw also + # removes the staged credential. + if spawn_cloud_lifecycle withdraw --task "$ID" \ + --task-generation "$SPAWN_GENERATION_ID" --confirm-withdraw \ + --confirm-subscription "${FM_AZURE_SUBSCRIPTION_ID:-}" >&2; then + echo "notice: released the provider-account lease for $ID with its withdrawn request" >&2 + else + echo "error: the provider-account lease for $ID is still held by its queued request; withdraw it with bin/fm-worker-lifecycle.sh withdraw --task $ID --task-generation $SPAWN_GENERATION_ID" >&2 + fi + echo "error: cloud placement for $ID could not be bound to its leased provider account" >&2 + return 1 + } if ! spawn_cloud_lifecycle reconcile --apply \ --confirm-subscription "${FM_AZURE_SUBSCRIPTION_ID:-}" --json \ > "$STATE/$ID.worker-reconcile.json" 2>&1; then diff --git a/bin/fm-worker-lifecycle.py b/bin/fm-worker-lifecycle.py index 45194b05c34..fccf9950121 100755 --- a/bin/fm-worker-lifecycle.py +++ b/bin/fm-worker-lifecycle.py @@ -32,6 +32,12 @@ ROOT = Path(__file__).resolve().parent.parent AZURE_PROVIDER = ROOT / "bin" / "fm-azure-worker-provider.py" +# The ONE implementation of "what is a Pi profile", "which upstream account is +# it", and "how is a single-profile account home written". Placement imports it +# rather than re-deriving any of the three: a second implementation of an +# account home is exactly how a credential stager and its remover once resolved +# different directories and leaked a credential. +PI_ACCOUNT_HOME_TOOL = ROOT / "bin" / "fm-pi-account-home.py" STATE_SCHEMA = "fm.worker-lifecycle/v1" # The scalar pending_action slot this schema carried is superseded by the # per-slot pending_actions map. The sentinel is deliberately a string an OLD @@ -356,6 +362,16 @@ def environment(): "daily_bound_override": daily_override, "idle_release_seconds": idle_release, "provider_argv": provider_argv, + # Where placement writes the single-profile account homes it leases. + # CONTROLLER-owned, under the same state directory as the document that + # records the lease, and deliberately NOT the shared crosscheck roster + # under ~/.local/share/agent-fleet/accounts/pi: those homes belong to + # the reviewer lane, and a placement rewriting one mid-review would + # swap a running reviewer's credential underneath it. It also makes the + # root follow FM_HOME, so a fixture home cannot write into a real one. + "pi_account_root": Path(os.environ.get( + "FM_PI_ACCOUNT_HOME_ROOT", str(state_dir / "accounts") + )).expanduser(), } @@ -675,6 +691,25 @@ def verify_request(request): "task home is owned by compartment child requests only") if not isinstance(task_home, str) or not task_home.startswith("/") or len(task_home) > 4096: raise LifecycleError("worker request task home must be one absolute path") + pool_home = request.get("account_pool_home") + if pool_home is not None and ( + not isinstance(pool_home, str) or not pool_home.startswith("/") + or len(pool_home) > 4096 + ): + raise LifecycleError("worker request account pool home must be one absolute path") + profile = request.get("account_profile") + account_home = request.get("account_home") + if (profile is None) != (account_home is None): + # The pair IS the lease record. Half of it would let a reader see a + # leased profile with no home to stage, or a staged home no exclusion + # covers; both read as "placed" while one of the two is missing. + raise LifecycleError( + "account_profile and account_home travel together or not at all") + if profile is not None: + require_id("account_profile", profile) + if not isinstance(account_home, str) or not account_home.startswith("/") \ + or len(account_home) > 4096: + raise LifecycleError("worker request account home must be one absolute path") if request.get("eligible") is not True: raise LifecycleError("worker request must be explicitly eligible") @@ -683,12 +718,223 @@ def active_queue_items(state): return [item for item in state["queue"].values() if item.get("status") != "complete"] +_PI_PROJECTION = {} + + +def pi_projection(): + """The projection tool, loaded as a module, not re-implemented. + + Placement needs three things from it and takes all three from the one + implementation: `read_pool` (what profiles exist and are they shaped like a + credential), `account_digest` (which upstream account is this, by digest + and never by token material), and `write_home`/`prepare_root` (how a + single-profile account home is written, owner-only, atomically). + """ + module = _PI_PROJECTION.get("module") + if module is not None: + return module + import importlib.util + + spec = importlib.util.spec_from_file_location( + "fm_pi_account_home", str(PI_ACCOUNT_HOME_TOOL)) + if spec is None or spec.loader is None: + raise LifecycleError( + "the Pi account-home projection tool is unavailable at {}".format( + PI_ACCOUNT_HOME_TOOL)) + module = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(module) + except Exception as exc: # noqa: BLE001 - any import failure is a refusal + raise LifecycleError( + "the Pi account-home projection tool could not be loaded from {}: {}".format( + PI_ACCOUNT_HOME_TOOL, type(exc).__name__)) + _PI_PROJECTION["module"] = module + return module + + +def placement_account_binding(account_digest): + """The lease identity, keyed on the UPSTREAM ACCOUNT. + + Not the profile name and not the account-home path, both of which are + handles that can point at the same account. Eight profiles map to eight + accounts today; nothing enforces that, and a re-login can point two slots + at one account. The thing a concurrent crewmate actually contends for - + the rate limit, the ban, the session - belongs to the account, so the + account is the unit of exclusion. `account_digest` is already a truncated + SHA-256 of the upstream account id and never carries token material. + """ + return digest_value({"provider": "pi", "upstream_account": account_digest}) + + +def leased_placement_accounts(state): + """Which upstream accounts non-complete queue work already holds. + + Derived from the queue, never from a separate ledger: the queue entry IS + the lease, so there is no second document that a crash could leave holding + a profile nothing owns. + """ + held = {} + for item in state["queue"].values(): + if item.get("status") == "complete": + continue + binding = item.get("account_binding") + if isinstance(binding, str): + held.setdefault(binding, (item.get("account_profile"), item.get("task"))) + return held + + +def select_placement_account(env, state, pool_home, task): + """Lease one free Pi profile for this placement, under the controller lock. + + Called from `command_request` inside the same lock hold and the same + `save_state` that writes the queue entry, so selection and the lease are + one atomic act: no window exists in which a profile is held by anything + that is not a queue entry. + + Fails closed at every step. An unreadable pool, a pool of unusable + credentials, and an exhausted pool all raise by name; none of them falls + through to a shared or arbitrary profile, because a silent fallthrough + here is an account collision with extra steps. + """ + projection = pi_projection() + pool_file = Path(pool_home) / "auth.json" + try: + pool = projection.read_pool(pool_file) + except projection.ProjectionError as exc: + raise LifecycleError( + "provider-account placement pool is unusable: {}; a worker is never " + "placed on an unidentified account".format(exc)) + if not pool: + raise LifecycleError( + "provider-account placement pool at {} declares no profile".format(pool_file)) + held = leased_placement_accounts(state) + if len(pool) == 1: + # Already a single-profile account home - the exact shape every Pi + # consumer reads, and what the projection tool produces. There is + # nothing to select and nothing to write: lease it in place. Its SHAPE + # is deliberately not screened here, because projecting is the only + # operation that needs a writable credential and "is this credential + # still good" has one owner, bin/fm-credential-expiry.py. What IS + # required is the one thing exclusion depends on: an upstream account + # this lease can name. + name, entry = next(iter(pool.items())) + digest = projection.account_digest(entry) if isinstance(entry, dict) else "none" + if digest == "none": + raise LifecycleError( + "provider-account home at {} names no upstream account, so a placement " + "on it could not be excluded from any other".format(pool_file)) + binding = placement_account_binding(digest) + if binding in held: + raise LifecycleError( + "provider-account placement is exhausted: the single account home {} is " + "already leased ({} holds profile {}); refusing to place {} on a shared " + "upstream account. Sign in additional Pi profiles on distinct upstream " + "accounts to raise the ceiling, or release the placement above with " + "`bin/fm-worker-lifecycle.sh withdraw`".format( + pool_file, held[binding][1] or "unknown-task", + held[binding][0] or name, task)) + return { + "account_profile": name, + "account_home": str(Path(pool_home)), + "account_binding": binding, + } + usable = [] + faults = {} + for name in sorted(pool): + entry_faults = projection.entry_faults(pool[name]) + if entry_faults: + faults[name] = "; ".join(entry_faults) + continue + usable.append(name) + if not usable: + raise LifecycleError( + "provider-account placement pool at {} holds no projectable profile ({})".format( + pool_file, + ", ".join("{}: {}".format(name, faults[name]) for name in sorted(faults)))) + bindings = {} + for name in usable: + digest = projection.account_digest(pool[name]) + if digest == "none": + # entry_faults already requires a non-blank accountId, so this is + # unreachable through the loop above; refuse rather than mint a + # lease identity that names no account. + raise LifecycleError( + "Pi profile {} exposes no upstream account identity".format(name)) + # Deliberately setdefault, not assignment: two profiles that resolve to + # ONE upstream account are one lease, and the first name in sorted + # order owns it. That is the whole reason the unit is the account. + bindings.setdefault(placement_account_binding(digest), name) + for binding, name in sorted(bindings.items(), key=lambda pair: pair[1]): + if binding in held: + continue + root = Path(env["pi_account_root"]).resolve() + # The projected home is keyed on the LEASE IDENTITY, never on the + # profile's local slot name. The two must be the same function of the + # pool or the projection is not injective over live leases, and the + # slot name is not: an operator re-logging slot `openai-codex` from one + # upstream account to another gives two placements two distinct + # bindings (correctly, they ARE two accounts) that both project into + # `accounts/openai-codex`, so the second write silently replaces the + # credential the first placement's still-live lease points at. The + # queue then reports two accounts while the disk holds one. Keying on + # the binding makes the directory name and the exclusion key the same + # string, so that state is unrepresentable. + destination = root / binding + for entry in state["queue"].values(): + if entry.get("status") == "complete": + continue + if entry.get("account_home") == str(destination): + # Unreachable while the two keys agree, because a binding that + # is free by definition is named by no live entry. Kept as the + # second line: if a future change re-keys the projection, this + # refuses instead of clobbering a live placement's credential. + raise LifecycleError( + "refusing to project Pi profile {} over the account home a live " + "placement already holds ({} holds {})".format( + name, entry.get("task") or "an unnamed task", destination)) + try: + projection.prepare_root(root) + credential = projection.write_home(destination, pool[name]) + except projection.ProjectionError as exc: + raise LifecycleError( + "Pi profile {} could not be projected into its account home: {}".format( + name, exc)) + except OSError as exc: + raise LifecycleError( + "Pi profile {} could not be projected into its account home: {}".format( + name, exc.strerror or exc)) + return { + "account_profile": name, + "account_home": str(Path(credential).parent), + "account_binding": binding, + } + raise LifecycleError( + "provider-account placement is exhausted: all {} distinct upstream accounts in {} " + "are leased ({}); refusing to place {} on a shared upstream account. Sign in " + "additional Pi profiles on distinct upstream accounts to raise the ceiling, or " + "release a placement above with `bin/fm-worker-lifecycle.sh withdraw`".format( + len(bindings), pool_file, + ", ".join( + "{} -> {}".format(name, held[binding][1] or "unknown-task") + for binding, name in sorted(bindings.items(), key=lambda pair: pair[1]) + ), + task)) + + def ensure_unique_bindings(state, candidate, ignore_key=None): for key, item in state["queue"].items(): if key == ignore_key or item.get("status") == "complete": continue if item.get("account_binding") == candidate["account_binding"]: - raise LifecycleError("provider-account lease binding is already owned by another queued or active task") + # The account-collision screen. It is the SAME screen selection + # already respected; keeping it means a hand-edited queue, a + # replayed old binary, or a broken selector still cannot seat two + # concurrent tasks on one upstream account. + raise LifecycleError( + "provider-account lease binding is already owned by another queued or " + "active task ({} holds profile {})".format( + item.get("task") or "an unnamed task", + item.get("account_profile") or "an unnamed profile")) if item.get("worktree_binding") == candidate["worktree_binding"]: raise LifecycleError("writable worktree binding is already owned by another queued or active task") @@ -2273,6 +2519,23 @@ def status_projection(env, state, inventory=None): "family_observed_plus_reserved_vcpus": family_committed, "shared_headroom_vcpus": SHARED_HEADROOM_VCPUS, "compartments": compartment_projection(state), + # Who holds which provider account right now. Read straight off the + # queue, because the queue entry IS the lease: a profile that shows + # here and nowhere else does not exist. + "account_placements": sorted( + ( + { + "task": item.get("task"), + "task_generation": item.get("task_generation"), + "status": item.get("status"), + "account_profile": item.get("account_profile"), + "account_home": item.get("account_home"), + } + for item in state["queue"].values() + if item.get("status") != "complete" and item.get("account_profile") + ), + key=lambda entry: (entry["account_profile"], entry["task"] or ""), + ), "idle_cooldown_seconds": env["cooldown_seconds"], "warm_idle_target": env["warm_idle"], "retained_disks": retained_disks, @@ -2324,6 +2587,12 @@ def print_status(status, json_output): if "session_legs" in compartment: line += " legs={}".format(compartment["session_legs"]) print(line) + for placement in status.get("account_placements") or []: + print("account-placement: profile={} task={}@{} status={} home={}".format( + placement["account_profile"], placement["task"], + placement["task_generation"], placement["status"], + placement["account_home"], + )) if status["pending_mutations"]: print("pending-mutations: {}".format(json.dumps( status["pending_mutations"], sort_keys=True, separators=(",", ":")))) @@ -2580,7 +2849,14 @@ def exactly(key): raise LifecycleError("ordinary worktree Git-directory identity differs") return { "home_binding": home_binding(origin), - "account_binding": digest_value({"task": task, "account_home": str(account_home)}), + # The POOL, not the lease. The task's own metadata proves which provider + # account source this task is entitled to draw from; WHICH profile of + # that pool it gets is decided by the controller under its lock, because + # that decision has to exclude every other concurrent placement and no + # task-local document can see them. The account lease identity + # (`account_binding`) is minted from the selected profile in + # `command_request`. + "account_pool_home": str(account_home), "worktree_binding": digest_value({"worktree": str(worktree), "git_dir": str(git_dir)}), "repository_binding": hashlib.sha256(head.encode("ascii")).hexdigest(), "repository_generation": head, @@ -2822,6 +3098,11 @@ def command_request(env, args): else: bindings = authoritative_request_bindings( env, args.task, args.task_generation, task_home=task_home) + # DURABLE on the item, not consumed here: the queue entry should record + # which provider-account pool its lease was drawn from, so an audit of a + # placement never has to re-read a task metadata file that teardown may + # already have removed. + pool_home = bindings.get("account_pool_home") item = { "schema": REQUEST_SCHEMA, "task": args.task, @@ -2842,22 +3123,45 @@ def command_request(env, args): # digest: authority_home reads it back to tell fm-worker-authority.py # where this task's own state/.meta lives. item["task_home"] = str(task_home) - verify_request(item) + if pool_home is None: + # The asserted-bindings lane already carries an account_binding of its + # own, so there is nothing to select; it is reachable only with the + # test env var AND a fixture provider (checked above). + verify_request(item) key = request_key(item["task"], item["task_generation"]) with controller_lock(env): state = load_state(env) existing = state["queue"].get(key) if existing is not None: + # Replay reuses the SAME profile, because the lease it took is this + # very entry. Selection happens only on the branch that creates a + # new entry, so a replayed request cannot consume a second account. identity_fields = ( - "schema", "task", "task_generation", "home_binding", "account_binding", + "schema", "task", "task_generation", "home_binding", "worktree_binding", "repository_binding", "repository_generation", "owner_kind", "role", "eligible", "discretionary", "parent_task", "parent_task_generation", "task_home", + "account_pool_home", ) + if pool_home is None: + identity_fields += ("account_binding",) if any(existing.get(field) != item.get(field) for field in identity_fields): raise LifecycleError("task generation already exists with different queue identity") + if existing.get("account_home"): + # The profile NAME is reported separately because the home is + # keyed on the lease identity, not on the name: the caller can + # no longer read the profile off the path's last component. + print("account-profile {}".format(existing.get("account_profile") or "")) + print("account-home {}".format(existing["account_home"])) print("request already exists with exact identity") return + if pool_home is not None: + # Selection and the lease are ONE act under ONE lock over ONE + # document: the queue entry written below IS the lease, so no + # window exists where a profile is held by something the queue does + # not show, and no concurrent request can read the same free set. + item.update(select_placement_account(env, state, pool_home, item["task"])) + verify_request(item) ensure_unique_bindings(state, item) if item.get("parent_task") is not None: if task_home is not None: @@ -2880,6 +3184,14 @@ def command_request(env, args): parent_worker = state["workers"][str(state["queue"][parent_key].get("slot"))] parent_worker["children_total"] = int(parent_worker.get("children_total", 0)) + 1 save_state(env, state) + if item.get("account_home"): + # The caller stages the provider credential from the home the + # controller leased, so the leased profile is the ONE credential that + # reaches the worker. Printed as a path and a slot name, never as + # contents; the home is keyed on the lease identity, so the slot name + # is not recoverable from the path. + print("account-profile {}".format(item.get("account_profile") or "")) + print("account-home {}".format(item["account_home"])) print("queued {} generation {} for one isolated author worker".format(item["task"], item["task_generation"])) diff --git a/docs/azure-requirements.md b/docs/azure-requirements.md index 1a7ac5609d2..557bcdff76f 100644 --- a/docs/azure-requirements.md +++ b/docs/azure-requirements.md @@ -159,11 +159,33 @@ upstream accounts, projected into single-profile account homes by `bin/fm-pi-acc with the roster repointed and read back through the real `bin/fm-crosscheck.py` reader and policy screen. Under the second 2026-08-19 amendment this roster is now the dormant crosscheck fallback; the pi fleet's primary duties are authors and no-mistakes. -Crewmate placement is not wired: the cell image carries pi, but placement does not select across -the eight profiles. - -Work: multi-profile account selection for crewmate and worker placement, reusing the projection -tool and the account-lease identity already present in the worker request path. +Crewmate placement now selects across the pool. The controller chooses one free profile inside the +same lock hold and the same durable write that creates the queue entry, so selection and the lease +are one act and the queue entry IS the lease; the unit of exclusion is the UPSTREAM ACCOUNT rather +than the profile name, because two profiles can be re-logged into one account and what a crewmate +contends for belongs to the account. The chosen profile is projected with `bin/fm-pi-account-home.py` +into a controller-owned account home, and `bin/fm-spawn.sh` narrows the staged provider credential +to it, so the worker receives exactly one account rather than the pooled `auth.json`. An exhausted +pool refuses by name, listing every leased profile and the task holding it. Mechanics are owned by +`docs/azure-workers.md` ("Provider-account placement across the Pi fleet"). + +Proven locally against a fixture provider by `tests/fm-worker-placement.test.sh` (eight concurrent +placements racing the controller lock take eight distinct upstream accounts, read back from +`controller.json`; exhaustion refuses; a compartment child, its compartment and an ordinary crewmate +hold three distinct accounts; killing placements between selection and the durable lease orphans no +account) and end to end through the real `bin/fm-spawn.sh` by `tests/fm-spawn-cloud.test.sh`, which +asserts the staged credential is the leased single profile. + +Operational consequence the owner must know: concurrent placements are now bounded by +`min(FM_AZURE_WORKER_MAX, distinct upstream accounts in the pool)`. With eight Pi accounts the +ninth concurrent placement refuses although MAX_WORKERS is 16, and compartments compete in the +same pool. Sixteen crewmates never could run on eight accounts without sharing one; they used +to do it silently. Raising the ceiling means adding profiles on distinct accounts. + +Still owed for DONE: the acceptance sentence on real compute. No leg of this has run against a live +Azure worker, so "concurrent crewmates RUN on distinct pi profiles" is proven up to the credential +the worker is handed and no further; what the guest agent does with that credential is unobserved +here, and R9 (no crewmate has returned an outcome) still gates it. Acceptance: concurrent crewmates run on distinct pi profiles with no account collision. diff --git a/docs/azure-workers.md b/docs/azure-workers.md index 7cb0ac01760..3d32d77fa7b 100644 --- a/docs/azure-workers.md +++ b/docs/azure-workers.md @@ -33,6 +33,36 @@ The caller supplies only task, task generation, owner kind, and eligibility; the Caller-supplied bindings are unsupported outside the hermetic test backstop. Raw provider-account identity never appears in bounded status or Azure tags. The account binding must be a high-entropy digest produced by the account lease owner, not a digest of a guessable profile name. +The lease owner is the controller: it derives the binding from the profile's upstream account identity (see the placement section below), never from the profile's local slot name. +The local slot name (`openai-codex-2` and the like) DOES appear in bounded status, because an operator has to be able to see which profile a task holds; it is a local label for a pool slot, not the upstream account identity, and it never reaches an Azure tag. + +## Provider-account placement across the Pi fleet + +The task metadata names the provider-account POOL this task may draw from (the cloud lane's is the Pi coding-agent home); WHICH profile of that pool the placement gets is decided by the controller, because that decision has to exclude every other concurrent placement and no task-local document can see them. + +Selection happens inside `command_request`, in the same lock hold and the same `save_state` that writes the queue entry, so selection and the lease are one act. +The queue entry IS the lease: the set of leased accounts is derived from the non-complete queue, never from a second ledger, so there is no state a crash can leave in which an account is held by something the queue does not show. +Selection is deterministic - the first free profile in the pool's sorted (lexicographic) name order - and replaying the same task generation reuses the same profile, because the replay path short-circuits on the existing entry before selecting anything. + +The unit of exclusion is the UPSTREAM ACCOUNT, not the profile name and not the account-home path. +Eight profiles map to eight accounts today, but nothing enforces that: a re-login can point two slots at one account, and what a concurrent crewmate actually contends for - the rate limit, the ban, the session - belongs to the account. +So `account_binding` is a digest over the profile's upstream account identity (itself a SHA-256 digest of the account id, never token material), two profiles resolving to one account are ONE lease, and the duplicate-account screen below is the same screen selection already respected. + +A pool holding more than one profile is projected: the controller writes the chosen profile's single-profile account home with `bin/fm-pi-account-home.py`, under its OWN state directory (`$FM_HOME/state/azure-workers/accounts/`, overridable with `FM_PI_ACCOUNT_HOME_ROOT`) and deliberately not the shared crosscheck roster, which belongs to the reviewer lane and must not be rewritten under a running reviewer. +The directory is keyed on the LEASE IDENTITY, never on the profile's local slot name, because the projection key and the exclusion key must be the same function of the pool. The slot name is not: re-logging one slot from one upstream account to another yields two placements with two correct, distinct bindings that would both project into one `accounts/` directory, and the second write would replace the credential the first placement's still-live lease points at, leaving the queue reporting two accounts while the disk held one. A second, defensive refusal also declines to project over an account home a live queue entry still names. +A home already holding exactly one profile is that single-profile home already, and is leased in place with nothing written; its credential shape is not screened there, because "is this credential still good" has one owner, `bin/fm-credential-expiry.py`. +`request` prints the leased profile and account home, and `bin/fm-spawn.sh` writes the staged account directory exactly once, from that home, after the lease exists. +The pooled `auth.json` is never staged: the payload step deliberately does not copy it, because that step runs BEFORE the lease is created and while the tracking monitor pane is already polling, so a crash there would otherwise leave every signed-in account in a directory the monitor is willing to dispatch as `--account-dir`. The window is removed rather than guarded. +As defence in depth at the point of USE, `bin/fm-spawn-cloud-monitor.sh` re-checks that the staged account directory holds exactly one provider slot before it dispatches, and does so BEFORE taking the shared exactly-once dispatch marker so a not-yet-narrowed directory simply retries on the next poll instead of wedging both owners. +Staging the pool would put every signed-in account on the guest and let Pi resolve the first slot, which is a shared-account placement whatever the queue records. + +Every failure refuses by name and none of them falls through to a shared or arbitrary profile: an unreadable or empty pool, a pool whose profiles are all unprojectable, a home naming no upstream account, and an exhausted pool (which names each leased profile and the task holding it). +Bounded status projects the live placements - profile, task generation, status, and account home - from `controller.json` alone. + +**The pool is now a concurrency ceiling, and it is lower than the worker ceiling.** Concurrent placements are bounded by `min(FM_AZURE_WORKER_MAX, distinct upstream accounts in the pool)`: with the fleet's eight Pi accounts, the ninth concurrent placement refuses even though `MAX_WORKERS` is 16 and quota, budget and capacity would all admit it. +That is the requirement, not a regression - sixteen crewmates never could run on eight accounts without sharing one, they just used to do it silently - but it does halve the effective author parallelism, and compartments compete in the same pool: one compartment plus its four children consumes five of the eight before an ordinary crewmate is placed. +Raising the ceiling means adding signed-in profiles on distinct upstream accounts to the pool, not raising a knob here. +A compartment child contends in the same document as an ordinary crewmate, because `FM_HOME` still names the primary's controller for both, so the two can never be handed one account. The controller rejects duplicate active account or writable-worktree bindings. A general request has role `author`, is explicitly eligible, and is owned by either the primary or a secondmate; a secondmate-owned author request may carry a parent compartment pair, which marks it as a compartment child and arms the child bounds. diff --git a/tests/behavior-test-durations.tsv b/tests/behavior-test-durations.tsv index df60a8af6a0..583f6513859 100644 --- a/tests/behavior-test-durations.tsv +++ b/tests/behavior-test-durations.tsv @@ -97,7 +97,7 @@ 6911 tests/fm-session-start.test.sh 28460 tests/fm-spawn-backlog.test.sh 2094 tests/fm-spawn-batch.test.sh -12000 tests/fm-spawn-cloud.test.sh +16000 tests/fm-spawn-cloud.test.sh 42452 tests/fm-spawn-dispatch-profile.test.sh 20000 tests/fm-spawn-provision.test.sh 42 tests/fm-stow-contract.test.sh @@ -118,6 +118,7 @@ 3000 tests/fm-worker-authority-secondmate.test.sh 12000 tests/fm-worker-lifecycle.test.sh 1000 tests/fm-worker-outcome-transport.test.sh +26000 tests/fm-worker-placement.test.sh 1000 tests/fm-worker-supervisor.test.sh 23744 tests/fm-x-mode.test.sh 250 tests/lavish-repair.test.sh diff --git a/tests/fm-secondmate-cloud-monitor.test.sh b/tests/fm-secondmate-cloud-monitor.test.sh index ae38a2112c1..e8ae59f0458 100755 --- a/tests/fm-secondmate-cloud-monitor.test.sh +++ b/tests/fm-secondmate-cloud-monitor.test.sh @@ -1995,12 +1995,48 @@ PY pass "a compartment child's queue item bindings equal the local authoritative mints, field by field" } +# fm_secondmate_write_pi_pool - a fixture Pi +# credential pool shaped as bin/fm-pi-account-home.py requires of a projectable +# profile, with one distinct upstream account per slot. +fm_secondmate_write_pi_pool() { # [account-scope] + python3 - "$1" "$2" "${3:-shared}" <<'FMPIPOOL' +import json +import sys + +scope = sys.argv[3] +pool = {} +for index in range(1, int(sys.argv[2]) + 1): + name = "openai-codex" if index == 1 else "openai-codex-{}".format(index) + pool[name] = { + "type": "oauth", "access": "fixture-access-{}".format(index), + "refresh": "fixture-refresh-{}".format(index), + # Scoped so pools written for different tasks never name the same + # upstream account: placement leases the ACCOUNT, so colliding ids + # across fixtures would read as exhaustion rather than as a fixture bug. + "accountId": "fixture-account-{}-{}".format(scope, index), + "expires": 4102444800000, + } +with open(sys.argv[1], "w", encoding="utf-8") as handle: + json.dump(pool, handle, sort_keys=True, indent=2) +FMPIPOOL + chmod 600 "$1" +} + # make_lifecycle_task - the ordinary local # authorities a real request derives its bindings from. make_lifecycle_task() { local world=$1 home=$2 task=$3 generation=$4 fm_git_init_commit "$world/$task-wt" mkdir -p "$world/$task-account" + # The task's provider-account POOL. Placement leases one profile out of the + # pool named by this task's own metadata and refuses a directory it cannot + # identify an upstream account in, so an empty directory is not a usable + # account authority any more. + # Distinct accounts PER TASK, not a shared pair: every lifecycle task in this + # suite draws from its own pool, and identical accountId values across tasks + # would make a future third concurrent task hit exhaustion and look + # mysterious. The task name seeds the account ids so they cannot collide. + fm_secondmate_write_pi_pool "$world/$task-account/auth.json" 2 "$task" python3 - "$home/state/$task.meta" "$generation" "$world/$task-wt" "$world/$task-account" "$task" <<'PY' import os import pathlib @@ -2744,8 +2780,12 @@ setup_spawn_world() { SP_PANE="$SP_DIR/pane.txt" mkdir -p "$SP_HOME/projects" "$SP_HOME/data" "$SP_HOME/state" "$SP_DIR/pi-agent-home" chmod 755 "$SP_DIR" - printf '{"openai-codex":{"accountId":"fixture-account"}}\n' > "$SP_DIR/pi-agent-home/auth.json" - chmod 600 "$SP_DIR/pi-agent-home/auth.json" + # MORE THAN ONE ACCOUNT, deliberately. This lane places a compartment AND a + # child of that compartment, and a placement now leases one upstream account + # exclusively. Against a single-account pool the child refuses as exhausted - + # which is the invariant working, not a regression - so a one-profile fixture + # here could only ever have passed while that invariant was unenforced. + fm_secondmate_write_pi_pool "$SP_DIR/pi-agent-home/auth.json" 3 fm_git_init_commit "$SP_HOME/projects/alpha" fm_git_add_origin "$SP_HOME/projects/alpha" "$SP_DIR/remotes/alpha.git" # direct-PR mode: an untagged project defaults to no-mistakes mode, whose diff --git a/tests/fm-spawn-cloud.test.sh b/tests/fm-spawn-cloud.test.sh index e7bce10c5c2..0525a47c38c 100755 --- a/tests/fm-spawn-cloud.test.sh +++ b/tests/fm-spawn-cloud.test.sh @@ -14,6 +14,30 @@ set -u SPAWN="$ROOT/bin/fm-spawn.sh" TMP_ROOT=$(fm_test_tmproot fm-spawn-cloud) + +# The fixture Pi credential POOL. The cloud lane's account home is the pooled +# pi agent home, and placement leases ONE profile out of it (R5), so the fixture +# has to look like the real pool: several profiles, each a complete oauth +# credential shape naming a distinct upstream account. +fm_spawn_cloud_write_pi_pool() { # + python3 - "$1" <<'PY' +import json +import sys + +pool = {} +for index in range(1, 5): + name = "openai-codex" if index == 1 else "openai-codex-{}".format(index) + pool[name] = { + "type": "oauth", "access": "fixture-access-{}".format(index), + "refresh": "fixture-refresh-{}".format(index), + "accountId": "fixture-account-{}".format(index), + "expires": 4102444800000, + } +with open(sys.argv[1], "w", encoding="utf-8") as handle: + json.dump(pool, handle, sort_keys=True, indent=2) +PY + chmod 600 "$1" +} SUB=11111111-1111-4111-8111-111111111111 # --- fixtures --------------------------------------------------------------- @@ -349,8 +373,7 @@ make_cloud_case() { # Cloud dispatch packages the pi provider-account material for the worker's # encrypted account disk; the hermetic account home carries a fixture # credential so the persist step has something real to digest. - printf '{"openai-codex":{"accountId":"fixture-account"}}\n' > "$case_dir/pi-agent-home/auth.json" - chmod 600 "$case_dir/pi-agent-home/auth.json" + fm_spawn_cloud_write_pi_pool "$case_dir/pi-agent-home/auth.json" printf '%s\n' codex > "$home/config/crew-harness" printf '%s\n' manual > "$home/config/backlog-backend" fm_git_init_commit "$project" @@ -452,6 +475,40 @@ test_cloud_spawn_places_worker_and_runs_the_entrypoint() { assert_grep "account_home=$CASE_DIR/pi-agent-home" "$meta" "the cloud spawn did not record the pi coding-agent account home" assert_grep 'worktree_git_dir_identity=' "$meta" "the cloud spawn did not record the worktree Git-dir identity" assert_grep 'worktree_git_dir=' "$meta" "the cloud spawn did not record the worktree Git dir" + # R5: the controller leased ONE profile out of that pool, and the credential + # this worker actually receives is that profile's alone. Staging the pool + # would put four accounts on the guest and let pi pick the first slot, which + # is a shared-account placement no matter what the queue records. + assert_grep 'worker_account_profile=' "$meta" "the cloud spawn did not record its leased provider-account profile" + assert_grep "worker_account_home=$HOME_DIR/state/azure-workers/accounts/" "$meta" \ + "the cloud spawn did not record the controller-projected account home it was placed on" + python3 - "$meta" "$HOME_DIR/state/$id.cloud-account/auth.json" \ + "$HOME_DIR/state/azure-workers/controller.json" "$id" <<'PY' \ + || fail "the staged provider credential is not the leased single profile" +import json +import sys + +meta_path, staged_path, controller_path, task = sys.argv[1:] +meta = {} +for line in open(meta_path, encoding="utf-8"): + if "=" in line: + key, value = line.rstrip("\n").split("=", 1) + meta[key] = value +staged = json.load(open(staged_path, encoding="utf-8")) +assert list(staged) == ["openai-codex"], sorted(staged) +leased = json.load(open(meta["worker_account_home"] + "/auth.json", encoding="utf-8")) +assert staged == leased, "the staged credential is not the leased profile's" +state = json.load(open(controller_path, encoding="utf-8")) +item = next(entry for entry in state["queue"].values() if entry["task"] == task) +assert item["account_profile"] == meta["worker_account_profile"], (item, meta) +assert item["account_home"] == meta["worker_account_home"], (item, meta) +assert item["account_pool_home"] == meta["account_home"], (item, meta) +# The pool it was drawn from really did hold more than one account, so this +# proves a SELECTION happened rather than there being nothing to choose. +pool = json.load(open(meta["account_home"] + "/auth.json", encoding="utf-8")) +assert len(pool) > 1, sorted(pool) +assert leased["openai-codex"]["accountId"] == pool[item["account_profile"]]["accountId"], item +PY # Herdr tracking endpoint: every cloud crewmate registers a real Herdr # endpoint running the cloud monitor, with ZERO tmux involvement anywhere # in the cloud lane. @@ -563,7 +620,23 @@ test_cloud_spawn_fails_closed_when_the_lifecycle_refuses_the_request() { # FM_SPAWN_CLOUD=azure without the FM_AZURE_* identity environment: the # lifecycle refuses the request, so the spawn must roll back rather than # leave a lane that exists nowhere. - out=$(FM_SPAWN_CLOUD=azure run_spawn "$CASE_DIR" "$HOME_DIR" "$WORKTREE_DIR" "$FAKEBIN_DIR" "$id" "$PROJECT_DIR") + # + # The identity variables are unset EXPLICITLY and the provider is pinned to + # the case fixture, rather than trusting the ambient environment not to carry + # them. An operator shell exports the real subscription, tenant, resource + # group and image; inherited here, this unit's request would be admitted and + # its reconcile would reach the real Azure adapter, which is how a test that + # believes it is exercising a fake creates a billable VM tagged with this + # unit's own hardcoded fixture id. + out=$( + unset FM_AZURE_SUBSCRIPTION_ID FM_AZURE_TENANT_ID \ + FM_AZURE_DEPLOYMENT_GENERATION FM_AZURE_OWNER_TAG FM_AZURE_NAMING_PREFIX \ + FM_AZURE_RESOURCE_GROUP FM_AZURE_STORAGE_NAME FM_AZURE_WORKER_STATE_DIR \ + FM_AZURE_VM_IMAGE_ID FM_AZURE_WORKER_IMAGE_ID + FM_WORKER_PROVIDER_COMMAND="python3 $CASE_DIR/provider.py" \ + FM_SPAWN_CLOUD=azure \ + run_spawn "$CASE_DIR" "$HOME_DIR" "$WORKTREE_DIR" "$FAKEBIN_DIR" "$id" "$PROJECT_DIR" + ) status=$? expect_code 1 "$status" "a cloud spawn whose worker request is refused should fail: $out" assert_contains "$out" "cloud worker request was refused" "the refusal did not surface the request failure: $out" @@ -572,6 +645,88 @@ test_cloud_spawn_fails_closed_when_the_lifecycle_refuses_the_request() { pass "a refused worker request rolls the spawn back instead of stranding the task" } +test_the_pool_is_never_staged_in_the_request_to_narrow_window() { + local record id out status + id=cloud-window-c15 + record=$(make_cloud_case narrow-window "$id") + read_cloud_case "$record" + # THE WINDOW: the durable lease exists and the tracking monitor pane is + # already polling, but the spawn has not yet narrowed the account directory. + # The spawn is killed exactly there. If the pooled auth.json were staged + # before the lease (as it used to be), every signed-in account would be + # sitting in a directory the monitor is willing to dispatch as --account-dir. + # Written to a FILE, never captured through a command substitution. The spawn + # is SIGKILLed here, and a command substitution would keep blocking on the + # pipe until every surviving descendant (the tracking pane among them) closed + # its copy - a hang that reads as a silent suite death rather than a failure. + FM_ACCOUNT_DIRECTORY_TEST_LAB=firstmate-account-directory-test-lab-v1 \ + FM_TEST_CLOUD_ABORT_AFTER_REQUEST=1 \ + run_cloud_spawn "$CASE_DIR" "$HOME_DIR" "$WORKTREE_DIR" "$FAKEBIN_DIR" "$id" "$PROJECT_DIR" \ + > "$CASE_DIR/window-spawn.log" 2>&1 /dev/null || true) + test "$status" -ne 0 || fail "the spawn was supposed to die inside the window: $out" + python3 - "$HOME_DIR/state/azure-workers/controller.json" "$HOME_DIR/state/$id.cloud-account" "$id" \ + "$CASE_DIR/pi-agent-home/auth.json" <<'PY2' || fail "the pooled credential reached the window" +import json +import pathlib +import sys + +controller, account_dir, task, pool_path = sys.argv[1:] +state = json.load(open(controller, encoding="utf-8")) +live = [item for item in state["queue"].values() + if item.get("task") == task and item.get("status") != "complete"] +# The lease really is durable at this point, so the window is real and not +# something the test skipped past. +assert live, "no durable lease existed, so this never entered the window" +pool = json.load(open(pool_path, encoding="utf-8")) +assert len(pool) > 1, sorted(pool) +staged = pathlib.Path(account_dir) / "auth.json" +if staged.exists(): + parsed = json.load(open(staged, encoding="utf-8")) + assert isinstance(parsed, dict) and len(parsed) == 1, ( + "the pooled credential was staged inside the window", sorted(parsed)) +print("# window state: lease durable, staged slots = {}".format( + sorted(json.load(open(staged, encoding="utf-8"))) if staged.exists() else "no auth.json at all")) +PY2 + pass "the pooled credential is never staged in the window between the durable lease and the narrowing" +} + +test_a_spawn_that_cannot_bind_its_leased_account_hands_it_back() { + local record id out status + id=cloud-bindfail-c14 + record=$(make_cloud_case bind-failure "$id") + read_cloud_case "$record" + # The queue entry IS the provider-account lease. A spawn that gets past the + # request but cannot bind the account it was handed must give the lease back, + # or the pool loses one account every time this happens and eventually + # refuses every placement. + out=$(FM_ACCOUNT_DIRECTORY_TEST_LAB=firstmate-account-directory-test-lab-v1 \ + FM_TEST_CLOUD_ACCOUNT_BIND_FAIL=1 \ + run_cloud_spawn "$CASE_DIR" "$HOME_DIR" "$WORKTREE_DIR" "$FAKEBIN_DIR" "$id" "$PROJECT_DIR") + status=$? + expect_code 1 "$status" "a spawn that cannot bind its leased account should fail: $out" + assert_contains "$out" "could not be bound to its leased provider account" \ + "the failure did not name the account binding step: $out" + assert_contains "$out" "released the provider-account lease for $id" \ + "the spawn did not hand the provider-account lease back: $out" + python3 - "$HOME_DIR/state/azure-workers/controller.json" "$id" <<'PY' \ + || fail "the unbindable placement kept holding its provider account" +import json +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +state = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {"queue": {}} +live = [item for item in state.get("queue", {}).values() + if item.get("status") != "complete"] +assert not [item for item in live if item.get("task") == sys.argv[2]], live +# Nothing else holds an account either, so the whole pool is free again. +assert not [item for item in live if item.get("account_profile")], live +PY + pass "a spawn that cannot bind its leased provider account hands the lease back instead of orphaning it" +} + test_cloud_switch_refuses_non_pi_harness() { local record id out status id=cloud-hn-c10 @@ -945,8 +1100,7 @@ make_child_case() { # -> record "$sub/data" "$sub/projects" "$sub/state" "$sub/treehouse-pools" \ "$case_dir/codex-home" "$case_dir/pi-agent-home" chmod 755 "$case_dir" - printf '{"openai-codex":{"accountId":"fixture-account"}}\n' > "$case_dir/pi-agent-home/auth.json" - chmod 600 "$case_dir/pi-agent-home/auth.json" + fm_spawn_cloud_write_pi_pool "$case_dir/pi-agent-home/auth.json" printf '%s\n' codex > "$primary/config/crew-harness" printf '%s\n' manual > "$primary/config/backlog-backend" fm_git_init_commit "$project" @@ -1146,6 +1300,8 @@ test_monitor_stands_down_when_dispatch_already_claimed test_cloud_spawn_config_file_default_and_env_override test_cloud_spawn_refuses_unknown_switch_value test_cloud_spawn_fails_closed_when_the_lifecycle_refuses_the_request +test_a_spawn_that_cannot_bind_its_leased_account_hands_it_back +test_the_pool_is_never_staged_in_the_request_to_narrow_window test_cloud_switch_refuses_non_pi_harness test_cloud_switch_refuses_explicit_backend test_compartment_child_spawn_splits_the_task_home_from_the_money_document diff --git a/tests/fm-worker-lifecycle.test.sh b/tests/fm-worker-lifecycle.test.sh index eb855128d66..63429252f6c 100755 --- a/tests/fm-worker-lifecycle.test.sh +++ b/tests/fm-worker-lifecycle.test.sh @@ -3885,11 +3885,32 @@ def seed_home(path, marker_id): return path +def pi_pool(root, profiles=8): + """The fixture Pi credential pool every placed task draws from. + + Shaped exactly as bin/fm-pi-account-home.py requires of a projectable + profile, with one distinct upstream account per slot, because placement + leases the ACCOUNT and refuses anything it cannot identify. + """ + home = root / "pi-pool" + home.mkdir(parents=True, exist_ok=True) + pool = {} + for index in range(1, profiles + 1): + name = "openai-codex" if index == 1 else "openai-codex-{}".format(index) + pool[name] = { + "type": "oauth", "access": "fixture-access-{}".format(index), + "refresh": "fixture-refresh-{}".format(index), + "accountId": "fixture-account-{}".format(index), + "expires": 4102444800000, + } + (home / "auth.json").write_text(json.dumps(pool, sort_keys=True, indent=2)) + return home + + def task_meta(home, task, generation): """The ordinary local authorities authoritative_request_bindings reads.""" worktree = make_repo(root / "worktrees" / task) - account = root / "accounts" / task - account.mkdir(parents=True, exist_ok=True) + account = pi_pool(root) git_dir = worktree / ".git" identity = "{}:{}".format(os.stat(git_dir).st_dev, os.stat(git_dir).st_ino) (home / "state" / (task + ".meta")).write_text( @@ -3975,10 +3996,27 @@ assert item["home_binding"] == hashlib.sha256( assert state["home_binding"] == hashlib.sha256( str(primary.resolve()).encode()).hexdigest(), state["home_binding"] assert item["home_binding"] != state["home_binding"] -# The bindings are real mints from the task home's own authorities. -assert item["account_binding"] == hashlib.sha256(json.dumps( - {"account_home": str(account.resolve()), "task": "child-1"}, - sort_keys=True, separators=(",", ":")).encode()).hexdigest(), item +# The bindings are real mints from the task home's own authorities, and the +# account lease is the CONTROLLER's placement decision over that home's pool: +# the lowest free profile, bound to its upstream account and nothing else. +assert item["account_profile"] == "openai-codex", item +expected_binding = hashlib.sha256(json.dumps( + {"provider": "pi", "upstream_account": hashlib.sha256( + b"fixture-account-1").hexdigest()[:16]}, + sort_keys=True, separators=(",", ":")).encode()).hexdigest() +assert item["account_binding"] == expected_binding, item +# The projected home is keyed on the LEASE IDENTITY, not the slot name: the +# projection key and the exclusion key have to be the same function of the pool +# or a re-logged slot silently overwrites a live placement's credential. +assert item["account_home"] == str( + primary / "state" / "azure-workers" / "accounts" / expected_binding), item +# The leased home is a SINGLE-profile home, projected by the one projection +# tool: a pooled home would put every signed-in account on the worker and let +# the guest pick the first slot, which is the collision this requirement removes. +leased = json.loads((Path(item["account_home"]) / "auth.json").read_text()) +assert list(leased) == ["openai-codex"], sorted(leased) +assert leased["openai-codex"]["accountId"] == "fixture-account-1", item +assert account.resolve() == (Path(str(root)) / "pi-pool").resolve(), account assert item["worktree_binding"] == hashlib.sha256(json.dumps( {"git_dir": str((worktree / ".git").resolve()), "worktree": str(worktree.resolve())}, sort_keys=True, separators=(",", ":")).encode()).hexdigest(), item @@ -4140,6 +4178,19 @@ subprocess.run(["git", "commit", "-q", "-m", "fixture", "--no-gpg-sign"], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) account = root / "account" account.mkdir(parents=True, exist_ok=True) +# The task's provider-account POOL, with more than one profile so the golden +# covers the R5 lane that actually selects and projects; a home with no pool +# refuses, which is why the golden seeds a real one. +(account / "auth.json").write_text(json.dumps({ + "openai-codex": { + "type": "oauth", "access": "fixture-access-1", "refresh": "fixture-refresh-1", + "accountId": "fixture-account-1", "expires": 4102444800000, + }, + "openai-codex-2": { + "type": "oauth", "access": "fixture-access-2", "refresh": "fixture-refresh-2", + "accountId": "fixture-account-2", "expires": 4102444800000, + }, +}, sort_keys=True, indent=2)) git_dir = worktree / ".git" (home / "state" / "local-crew.meta").write_text( "generation_id=gen-local\nworktree={}\naccount_home={}\naccount_task=local-crew\n" @@ -4174,7 +4225,19 @@ expected = { # IS FM_HOME. --task-home changes where that identity is read from, never # what it is here. "home_binding": hashlib.sha256(str(home.resolve()).encode("utf-8")).hexdigest(), - "account_binding": digest({"task": "local-crew", "account_home": str(account.resolve())}), + # The account lease is the CONTROLLER's placement over this task's pool: + # keyed on the upstream account, with the leased profile and its + # single-profile home recorded beside it. + "account_binding": digest({ + "provider": "pi", + "upstream_account": hashlib.sha256(b"fixture-account-1").hexdigest()[:16], + }), + "account_profile": "openai-codex", + "account_home": str(home / "state" / "azure-workers" / "accounts" / digest({ + "provider": "pi", + "upstream_account": hashlib.sha256(b"fixture-account-1").hexdigest()[:16], + })), + "account_pool_home": str(account.resolve()), "worktree_binding": digest( {"worktree": str(worktree.resolve()), "git_dir": str(git_dir.resolve())}), "repository_binding": hashlib.sha256(head.encode("ascii")).hexdigest(), @@ -4378,6 +4441,15 @@ subprocess.run(["git", "clone", "-q", str(origin), str(worktree)], check=True, # The account directory the real account helper validates, and the completion # report the report authority requires - both in the SECONDMATE's home. account = Path(env["FM_ACCOUNT_DIRECTORY_ROOT"]) / "codex" / "1" +# The same directory is this task's provider-account POOL, exactly as the real +# cloud lane's account_home (the Pi coding-agent home) is: placement leases one +# profile out of it and refuses a home it cannot identify an account in. +(account / "auth.json").write_text(json.dumps({ + "openai-codex": { + "type": "oauth", "access": "fixture-access-1", "refresh": "fixture-refresh-1", + "accountId": "fixture-account-1", "expires": 4102444800000, + }, +}, sort_keys=True, indent=2)) (sub / "data" / "child-1").mkdir(parents=True, exist_ok=True) (sub / "data" / "child-1" / "completion.md").write_text( "# child-1\n\n## Summary\ns\n\n## What changed\nc\n\n## Verification\nv\n\n" @@ -4556,10 +4628,31 @@ def make_repo(path): return path +def pi_pool(root, profiles=8): + """The fixture Pi credential pool every placed task draws from. + + Shaped exactly as bin/fm-pi-account-home.py requires of a projectable + profile, with one distinct upstream account per slot, because placement + leases the ACCOUNT and refuses anything it cannot identify. + """ + home = root / "pi-pool" + home.mkdir(parents=True, exist_ok=True) + pool = {} + for index in range(1, profiles + 1): + name = "openai-codex" if index == 1 else "openai-codex-{}".format(index) + pool[name] = { + "type": "oauth", "access": "fixture-access-{}".format(index), + "refresh": "fixture-refresh-{}".format(index), + "accountId": "fixture-account-{}".format(index), + "expires": 4102444800000, + } + (home / "auth.json").write_text(json.dumps(pool, sort_keys=True, indent=2)) + return home + + def task_meta(home, task, generation): worktree = make_repo(root / "worktrees" / task) - account = root / "accounts" / task - account.mkdir(parents=True, exist_ok=True) + account = pi_pool(root) git_dir = worktree / ".git" (home / "state" / (task + ".meta")).write_text( "generation_id={}\nworktree={}\naccount_home={}\naccount_task={}\n" @@ -4722,6 +4815,28 @@ git_env.update({ }) +def pi_pool(root, profiles=8): + """The fixture Pi credential pool every placed task draws from. + + Shaped exactly as bin/fm-pi-account-home.py requires of a projectable + profile, with one distinct upstream account per slot, because placement + leases the ACCOUNT and refuses anything it cannot identify. + """ + home = root / "pi-pool" + home.mkdir(parents=True, exist_ok=True) + pool = {} + for index in range(1, profiles + 1): + name = "openai-codex" if index == 1 else "openai-codex-{}".format(index) + pool[name] = { + "type": "oauth", "access": "fixture-access-{}".format(index), + "refresh": "fixture-refresh-{}".format(index), + "accountId": "fixture-account-{}".format(index), + "expires": 4102444800000, + } + (home / "auth.json").write_text(json.dumps(pool, sort_keys=True, indent=2)) + return home + + def task_meta(home, task, generation): """A COMPLETE set of local authorities, so that when a home rule is deleted the request is genuinely ADMITTED rather than failing later on a missing @@ -4735,8 +4850,7 @@ def task_meta(home, task, generation): (worktree / "README.md").write_text("fixture\n") subprocess.run(argv, cwd=str(worktree), env=git_env, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) - account = root / "accounts" / task - account.mkdir(parents=True, exist_ok=True) + account = pi_pool(root) git_dir = worktree / ".git" (home / "state").mkdir(parents=True, exist_ok=True) (home / "state" / (task + ".meta")).write_text( diff --git a/tests/fm-worker-placement.test.sh b/tests/fm-worker-placement.test.sh new file mode 100755 index 00000000000..dda68c1d00f --- /dev/null +++ b/tests/fm-worker-placement.test.sh @@ -0,0 +1,768 @@ +#!/usr/bin/env bash +# shellcheck source=tests/test-entry.sh +. "$(dirname "$0")/test-entry.sh" +# Behavior: multi-profile provider-account placement for crewmate and worker +# requests (R5). Concurrent placements land on DISTINCT upstream accounts, an +# exhausted pool refuses by name instead of sharing one, a crashed placement +# leaves a lease the queue still shows, and a compartment child contends for the +# same pool as an ordinary crewmate. +# +# EVERY unit runs against an explicit fixture provider AND a fake `az` that +# records any invocation and fails; each unit asserts the fixture was the one +# used and that no cloud call was attempted. A suite here that silently resolved +# the real Azure provider would create billable infrastructure. +set -u + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P) +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +CONTROLLER="$ROOT/bin/fm-worker-lifecycle.py" +SUB=11111111-1111-4111-8111-111111111111 + +# --- fixtures --------------------------------------------------------------- + +# A provider that records every operation it is asked for and answers the +# minimum the controller verifies. It is the ONLY provider any unit may reach; +# `placement_world` asserts that after every run. +write_recording_provider() { + cat >"$1" <<'PY' +#!/usr/bin/env python3 +import hashlib +import json +import os +from pathlib import Path +import sys + +request = json.loads(sys.stdin.read()) +controller = request["controller"] +log = Path(os.environ["PROVIDER_CALL_LOG"]) +with log.open("a", encoding="utf-8") as handle: + handle.write(request["operation"] + "\n") +response = { + "schema": "fm.worker-provider-response/v1", + "operation": request["operation"], + "controller": controller, +} +if request["operation"] == "inventory": + # The SKU/family table is read from the controller itself rather than + # copied, so a plan change cannot leave this fixture quietly admitting + # against families the real one no longer has. + import importlib.util + + spec = importlib.util.spec_from_file_location( + "fm_worker_lifecycle", os.environ["CONTROLLER_PATH"]) + lifecycle = importlib.util.module_from_spec(spec) + spec.loader.exec_module(lifecycle) + families = {family for _, family in lifecycle.SKU_PLAN.values()} + response["inventory"] = { + "schema": "fm.worker-provider-inventory/v1", + "workers": [], + "capacity_reservations": [], + "conflicts": [], + "metrics": { + "actual_usd": 1.0, "forecast_usd": 2.0, + "regional_limit_vcpus": 128, "regional_used_vcpus": 0, + "specialized_active_vcpus": 0, "specialized_active_by_family": {}, + "family_limit_vcpus": {family: 40 for family in families}, + "family_used_vcpus": {family: 0 for family in families}, + "family_free_vcpus": {family: 40 for family in families}, + "sku_hourly_usd": {sku: 0.25 for sku, _ in lifecycle.SKU_PLAN.values()}, + }, + } +elif request["operation"] == "mutate": + action = request["action"] + # The real provider recomputes this over the whole action and refuses a + # mismatch. A fixture that skipped it would be MORE permissive than the + # callee it stands in for, and would hide a controller bug. + expected = hashlib.sha256(json.dumps( + {name: value for name, value in action.items() if name != "idempotency_key"}, + sort_keys=True, separators=(",", ":")).encode()).hexdigest() + if action["idempotency_key"] != expected: + sys.stderr.write("FIXTURE PROVIDER REFUSED: idempotency key is not exact\n") + raise SystemExit(1) + if action["type"] != "create": + sys.stderr.write( + "FIXTURE PROVIDER REFUSED: this suite only creates, never {}\n".format( + action["type"])) + raise SystemExit(1) + bindings = action["bindings"] + secondmate = action.get("role") == "secondmate" + tags = { + "workload": "firstmate", + "firstmate-role": "secondmate-compartment" if secondmate else "worker", + "deployment-generation": action["deployment_generation"], + "cleanup-owner": action["owner"], "worker-slot": str(action["slot"]), + "home-binding": bindings["home_binding"], "task-binding": bindings["task"], + "task-generation": bindings["task_generation"], + "assignment-generation": bindings["assignment_generation"], + "account-binding": bindings["account_binding"], + "worktree-binding": bindings["worktree_binding"], + "repository-binding": bindings["repository_binding"], + "repository-generation": bindings["repository_generation"], + "nested-team": "forbidden", "browser-profile": "forbidden", + } + if secondmate: + tags.update({"agent-capacity": "one-home-scoped-secondmate", "child-launcher": "absent"}) + else: + tags.update({ + "agent-capacity": "one-task-scoped-crewmate", + "secondmate-placement": "forbidden", + }) + serial = "{}-{}".format(action["cloud_generation"], action["idempotency_key"][:8]) + resources = {} + for kind in ( + "vm", "nic", "os-disk", "task-disk", "account-disk", "identity", + "role-assignment", "state-container", "monitor-extension", + "bootstrap-command", "task-command", "ttl-schedule", "global-reservation", + "staging-request", "staging-result", + ): + resources[kind] = { + "id": "/fixture/slot/{}/{}".format(action["slot"], kind), + "immutable_id": "{}-{}".format(kind, serial), + "etag": "etag-{}".format(serial), "tags": dict(tags), + } + resources["vm"]["power_state"] = "VM running" + for kind in ("nic", "os-disk", "task-disk", "account-disk", "monitor-extension", + "bootstrap-command", "task-command", "ttl-schedule"): + resources[kind]["attached_to"] = resources["vm"]["id"] + for kind in ("monitor-extension", "bootstrap-command", "task-command"): + resources[kind]["provisioning_state"] = "Succeeded" + resources["ttl-schedule"].update({"status": "Enabled", "deadline": "2300"}) + for kind in ("global-reservation", "staging-request", "staging-result"): + resources[kind].update({"digest": "f" * 64, "length": 1}) + response["result"] = { + "idempotency_key": action["idempotency_key"], "action": "create", + "worker": {"slot": action["slot"], "resources": resources}, + } +else: + response["result"] = {"status": "refused", "reason": "fixture provider performs no mutation"} +print(json.dumps(response, sort_keys=True, separators=(",", ":"))) +PY + chmod +x "$1" +} + +# A fake `az` that can only record and fail. If any unit ever reaches the real +# Azure adapter, this is the tripwire. +write_forbidden_az() { + mkdir -p "$1" + cat >"$1/az" <<'SH' +#!/bin/sh +printf '%s\n' "$*" >> "$AZ_CALL_LOG" +echo "az is forbidden inside the placement suite" >&2 +exit 1 +SH + chmod +x "$1/az" +} + +# placement_world - a hermetic controller home whose +# tasks all draw from one Pi pool of distinct upstream accounts. +placement_world() { + # The scratch variable is deliberately NOT named like the caller's: bash + # locals are dynamically scoped, so a local named `world` here would shadow + # the caller's `world` and printf -v would set this frame's copy instead. + local target=$1 prefix=$2 profiles=$3 fm_placement_root + fm_test_tmproot_into fm_placement_root "$prefix" || return 1 + mkdir -p "$fm_placement_root/home/state" "$fm_placement_root/home/data" \ + "$fm_placement_root/fakebin" + write_recording_provider "$fm_placement_root/provider.py" + write_forbidden_az "$fm_placement_root/fakebin" + python3 - "$fm_placement_root/pool/auth.json" "$profiles" <<'PY' || return 1 +import json +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +path.parent.mkdir(parents=True, exist_ok=True) +pool = {} +for index in range(1, int(sys.argv[2]) + 1): + name = "openai-codex" if index == 1 else "openai-codex-{}".format(index) + pool[name] = { + "type": "oauth", "access": "fixture-access-{}".format(index), + "refresh": "fixture-refresh-{}".format(index), + "accountId": "fixture-account-{}".format(index), + "expires": 4102444800000, + } +path.write_text(json.dumps(pool, sort_keys=True, indent=2), encoding="utf-8") +PY + printf -v "$target" '%s' "$fm_placement_root" +} + +# placement_task - the ordinary local +# authorities a real request mints from, all pointed at the one pool. +placement_task() { + local world=$1 home=$2 task=$3 generation=$4 + fm_git_init_commit "$world/wt-$task" >/dev/null || return 1 + python3 - "$home/state/$task.meta" "$generation" "$world/wt-$task" "$world/pool" "$task" <<'PY' +import os +import pathlib +import sys + +meta, generation, worktree, pool, task = sys.argv[1:] +worktree = str(pathlib.Path(worktree).resolve()) +git_dir = os.path.join(worktree, ".git") +stat = os.stat(git_dir) +pathlib.Path(meta).write_text( + "generation_id={}\nworktree={}\naccount_home={}\naccount_task={}\n" + "worktree_git_dir_identity={}:{}\n".format( + generation, worktree, str(pathlib.Path(pool).resolve()), task, + stat.st_dev, stat.st_ino), + encoding="utf-8") +PY +} + +run_placement() { # + # The fixture provider is passed EXPLICITLY on every call, never inherited: + # a run that fell back to the packaged Azure adapter would talk to a real + # subscription. `az` is additionally shadowed by the recording tripwire. + local world=$1 + shift + env -u FM_AZURE_RESOURCE_GROUP -u FM_AZURE_STORAGE_NAME \ + FM_HOME="$world/home" \ + FM_AZURE_SUBSCRIPTION_ID="$SUB" \ + FM_AZURE_DEPLOYMENT_GENERATION=dep-one \ + FM_AZURE_OWNER_TAG=owner \ + FM_AZURE_NAMING_PREFIX=fmtest \ + FM_AZURE_WORKER_IDLE_COOLDOWN_SECONDS=0 \ + FM_WORKER_PROVIDER_COMMAND="python3 $world/provider.py" \ + PROVIDER_CALL_LOG="$world/provider-calls.log" \ + CONTROLLER_PATH="$CONTROLLER" \ + AZ_CALL_LOG="$world/az-calls.log" \ + PATH="$world/fakebin:$PATH" \ + python3 "$CONTROLLER" "$@" +} + +# Backgrounding a shell FUNCTION gives $! the subshell's pid, and a kill sent +# there leaves the controller running as its child - the kill would land on a +# process that was never doing the work. `exec` replaces the subshell, so $! is +# the controller itself. +run_placement_exec() { # + local world=$1 + shift + exec env -u FM_AZURE_RESOURCE_GROUP -u FM_AZURE_STORAGE_NAME \ + FM_HOME="$world/home" \ + FM_AZURE_SUBSCRIPTION_ID="$SUB" \ + FM_AZURE_DEPLOYMENT_GENERATION=dep-one \ + FM_AZURE_OWNER_TAG=owner \ + FM_AZURE_NAMING_PREFIX=fmtest \ + FM_AZURE_WORKER_IDLE_COOLDOWN_SECONDS=0 \ + FM_WORKER_PROVIDER_COMMAND="python3 $world/provider.py" \ + PROVIDER_CALL_LOG="$world/provider-calls.log" \ + CONTROLLER_PATH="$CONTROLLER" \ + AZ_CALL_LOG="$world/az-calls.log" \ + PATH="$world/fakebin:$PATH" \ + python3 "$CONTROLLER" "$@" +} + +assert_no_cloud_call() { #