From 55c512ba9c449c4660150e579b40486b70b7491c Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Mon, 13 Jul 2026 20:35:06 -0400 Subject: [PATCH 001/103] feat: add default-off agent account routing --- .../skills/secondmate-provisioning/SKILL.md | 22 +- .gitignore | 2 + AGENTS.md | 17 +- bin/fm-account-routing-lib.sh | 252 +++++++++++++ bin/fm-account-session-sync.sh | 98 +++++ bin/fm-bootstrap.sh | 30 +- bin/fm-config-inherit-lib.sh | 2 +- bin/fm-dispatch-select.sh | 93 ++++- bin/fm-spawn.sh | 347 +++++++++++++++--- bin/fm-teardown.sh | 34 ++ bin/fm-watch.sh | 6 + docs/architecture.md | 8 +- docs/configuration.md | 42 ++- docs/scripts.md | 8 +- tests/fm-account-routing.test.sh | 344 +++++++++++++++++ tests/fm-bootstrap.test.sh | 9 + tests/fm-dispatch-select.test.sh | 97 +++++ tests/fm-gotmp.test.sh | 2 + tests/fm-teardown.test.sh | 75 ++++ 19 files changed, 1417 insertions(+), 71 deletions(-) create mode 100644 bin/fm-account-routing-lib.sh create mode 100755 bin/fm-account-session-sync.sh create mode 100755 tests/fm-account-routing.test.sh diff --git a/.agents/skills/secondmate-provisioning/SKILL.md b/.agents/skills/secondmate-provisioning/SKILL.md index c8febd24d1e..ba4dcfa5df4 100644 --- a/.agents/skills/secondmate-provisioning/SKILL.md +++ b/.agents/skills/secondmate-provisioning/SKILL.md @@ -3,7 +3,7 @@ name: secondmate-provisioning description: >- Agent-only reference for persistent secondmate setup and retirement. Use when creating, seeding, validating, launching, recovering, handing backlog to, pushing inherited config into, or retiring a secondmate home, or when editing data/secondmates.md. - Covers home leases, transactional seeding, project clone restrictions, secondmate harness pins, inherited config push, idle charter, handoff helper, and teardown safety. + Covers home leases, transactional seeding, project clone restrictions, secondmate harness and account-pool pins, inherited config push, idle charter, handoff helper, and teardown safety. user-invocable: false metadata: internal: true @@ -71,14 +71,19 @@ When the file's tokens do apply, an explicit per-spawn `--model` or `--effort` f Because this resolves from the file on every spawn, the pin is durable across every respawn (recovery, `/updatefirstmate`, restart) exactly like the harness axis itself - e.g. `config/secondmate-harness` containing `claude opus` keeps a secondmate pinned to Opus even if the primary's own default model later changes. This is secondmate-only: crewmate/scout model resolution is untouched by this file. +`config/secondmate-account-pool` is the primary's optional Agent Fleet pool for launching secondmate agents when account routing is already enabled. +The file contains one non-secret dynamic pool id, is re-read on every spawn, and does not activate account routing by itself. +An explicit `--account-pool` or `--account-profile` overrides it for that spawn. +This pool is not inherited into the secondmate home because it governs the primary's secondmate launch, while `config/account-routing-mode` is inherited so the secondmate applies the same off/observe/enforce policy to its own crewmates. + This section is the single owner of the secondmate sync and inheritable-config propagation contract; `AGENTS.md` sections 3 and 4 point here. Before launch, `fm-spawn.sh --secondmate` locally fast-forwards the home to the primary firstmate checkout's current default-branch commit when it is safe; dirty, diverged, or in-flight homes launch unchanged with a warning. The locked session-start bootstrap sweep runs the same guarded fast-forward for every live secondmate home, discovered from `state/.meta` records with `kind=secondmate` (`data/secondmates.md` only backfills `home=` for older records). That no-fetch path is a purely local fast-forward of tracked files, never an origin fetch, and it never touches the gitignored operational dirs, so a secondmate's backlog, projects, and in-flight work are never disturbed; a linked worktree advances immediately, while a standalone clone that lacks the target receives firstmate updates through `/updatefirstmate`'s origin refresh. -The same launch and the same locked bootstrap sweep also propagate the primary's declared inheritable local config, currently `config/crew-dispatch.json`, `config/crew-harness`, and `config/backlog-backend`, into the secondmate home's `config/`. +The same launch and the same locked bootstrap sweep also propagate the primary's declared inheritable local config, currently `config/crew-dispatch.json`, `config/crew-harness`, `config/backlog-backend`, and `config/account-routing-mode`, into the secondmate home's `config/`. Because `config/` is gitignored, that propagation is a separate, primary-authoritative copy independent of the tracked-files fast-forward: it re-converges every live home whether or not its tracked files advanced, and it touches only the declared items. Inheritance copies the literal `config/crew-harness` file, so a secondmate's own crewmates use the primary's crewmate harness only when it names a concrete adapter such as `codex`; an unset or `default` value has nothing concrete to inherit, and the secondmate's own crewmates fall back to the secondmate's own or detected harness instead. -`config/secondmate-harness` is not inherited because it is only the primary's knob for launching secondmate agents. +`config/secondmate-harness` and `config/secondmate-account-pool` are not inherited because they are only the primary's knobs for launching secondmate agents. No reread nudge is needed at spawn or respawn because the agent reads `AGENTS.md` fresh on launch; only the bootstrap sweep's `NUDGE_SECONDMATES:` case (a RUNNING home whose instruction surface advanced) needs one. For already-live secondmates, use `bin/fm-config-push.sh` to push a mid-session inherited-config change without running the tracked-file fast-forward or nudging the agents. It uses the same live-home discovery and propagation helper as bootstrap and reports each item as `pushed`, `unchanged`, `skipped`, or `error`. @@ -118,7 +123,14 @@ Do not hand off `local-only` items. ## Recovery -For `kind=secondmate` meta with no window, treat the secondmate as a dead persistent direct report and respawn it with: +For managed `kind=secondmate` meta with `account_profile=`, preserve the sticky account and provider session by recovering it with: + +```sh +bin/fm-spawn.sh --resume-account +``` + +This path requires the recorded Agent Fleet SessionStart mapping, uses `lease recover` rather than new-task quota selection, and refuses to create a fresh prompted session when mapping or sticky metadata is missing or mismatched. +For unmanaged `kind=secondmate` meta with no window, treat the secondmate as a dead persistent direct report and respawn it with: ```sh bin/fm-spawn.sh --secondmate @@ -126,7 +138,7 @@ bin/fm-spawn.sh --secondmate Use the recorded `home=` in meta. If meta is missing but `data/secondmates.md` still registers the secondmate, respawn from the registry entry and its persistent on-disk home. -Respawn re-resolves the secondmate harness from current config, uses the same guarded pre-launch sync, and re-propagates inheritable config, so recovered secondmates converge to the primary firstmate version and local dispatch, crew-harness, and backlog-backend settings whenever their home can be cleanly fast-forwarded. +Respawn re-resolves the secondmate harness from current config, uses the same guarded pre-launch sync, and re-propagates inheritable config, so recovered secondmates converge to the primary firstmate version and local dispatch, crew-harness, backlog-backend, and account-routing settings whenever their home can be cleanly fast-forwarded. If the secondmate is already running and only inherited config changed, prefer `bin/fm-config-push.sh` over respawning. Do not reconstruct a secondmate's whole tree from the main home. diff --git a/.gitignore b/.gitignore index 5ed2da0c32a..09d38e81ac4 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ __pycache__/ .env config/crew-harness config/crew-dispatch.json +config/account-routing-mode +config/secondmate-account-pool config/secondmate-harness config/backlog-backend config/backend diff --git a/AGENTS.md b/AGENTS.md index 6cbeb6b98c5..c1e5b267c5e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,6 +77,8 @@ bin/ helper scripts, committed; read each script's header before config/crew-harness crewmate harness override; LOCAL, gitignored; absent or "default" = same as firstmate. Inherited as the literal file: a concrete primary adapter value also controls a secondmate home's own crewmates (section 4) config/crew-dispatch.json optional crewmate dispatch profiles; LOCAL, gitignored; firstmate-maintained but human-editable natural-language rules that choose a per-task harness/model/effort profile (section 4). Inherited by secondmate homes config/secondmate-harness harness the PRIMARY uses to launch SECONDMATE agents, optionally followed by a model and effort token on the same line (" [] []"; section 4); LOCAL, gitignored; absent or "default" harness falls back to config/crew-harness then firstmate's own. The primary's own setting; NOT inherited into secondmate homes (secondmates do not spawn secondmates) +config/account-routing-mode optional Agent Fleet routing policy (`off`, `observe`, or `enforce`); LOCAL, gitignored; default off; inherited by secondmate homes (docs/configuration.md "Agent Fleet account routing") +config/secondmate-account-pool optional Agent Fleet pool the PRIMARY uses for SECONDMATE launches when routing is enabled; LOCAL, gitignored; selection-only and NOT inherited config/backlog-backend backlog backend override; LOCAL, gitignored; absent or "tasks-axi" = default tasks-axi backend, "manual" = force routine backlog updates to hand-editing; inherited by secondmate homes (section 10) config/backend runtime session-provider backend override for new tasks; LOCAL, gitignored; absent = falls through to runtime auto-detection (the runtime firstmate itself is executing inside), then tmux; tmux is the verified reference backend (docs/tmux-backend.md), while herdr, zellij, orca, and cmux are experimental spawn backends (docs/herdr-backend.md, docs/zellij-backend.md, docs/orca-backend.md, docs/cmux-backend.md) - herdr and cmux can also be selected by runtime auto-detection, zellij and orca never are (always explicit), and codex-app is not accepted; see docs/codex-app-backend.md; not inherited into secondmate homes config/cmux-socket-password optional cmux control-socket password; LOCAL, gitignored; read fresh on every cmux CLI call and passed through without ever overriding an operator's own ambient CMUX_SOCKET_PASSWORD when absent (docs/cmux-backend.md "Setup") @@ -191,7 +193,7 @@ Pick the single best-fit rule using your own judgment. This is explicitly not first-match: weigh all rules, their `when` text, and their `why` rationales against the actual task. For a chosen rule with a single-object `use`, or an array `use` with no `select`, resolve the first profile directly. For a chosen rule with `select: "quota-balanced"`, pipe the full rule JSON to `bin/fm-dispatch-select.sh` and use the compact JSON profile it prints. -Extract that chosen concrete profile `(harness, model, effort)` and pass it to `bin/fm-spawn.sh` with explicit `--harness`, `--model`, and `--effort` flags for the axes that are set. +Extract that chosen concrete profile `(harness, model, effort, account_pool, account_profile)` and pass it to `bin/fm-spawn.sh` with explicit `--harness`, `--model`, `--effort`, `--account-pool`, and `--account-profile` flags for the axes that are set. If no rule fits, use `default`. If `default` is absent, fall back to `config/crew-harness` through `bin/fm-harness.sh crew`, exactly as the static path did before dispatch profiles, but still pass that resolved harness explicitly. This is enforced: when `config/crew-dispatch.json` exists, `bin/fm-spawn.sh` refuses crewmate and scout launches that do not include an explicit harness (`--harness `, a positional adapter name, or a raw launch command). @@ -199,7 +201,8 @@ That refusal is the consultation backstop, so the rules are never silently skipp The requirement is gated only on the file's presence; when the file is absent, `fm-spawn.sh` keeps resolving the crewmate harness from `config/crew-harness` as before. Secondmate launches are exempt because they resolve through `fm-harness.sh secondmate`, not the crewmate dispatch-profile rules. -`quota-balanced` selection is deterministic and owned by `bin/fm-dispatch-select.sh`; its header documents the general-window rules, freshness margin, and every fallback, and it degrades to the first array element whenever quota data is unusable. +`quota-balanced` selection is deterministic and owned by `bin/fm-dispatch-select.sh`; its header documents the general-window rules, Agent Fleet pool-summary path, freshness margin, and every fallback, and it degrades to the first array element whenever quota data is unusable. +When candidates carry account pools, the selector uses only Agent Fleet summaries and never falls through to quota-axi's default-account data; pass its selected pool to spawn for the atomic concrete-profile lease. Quota trouble must never block dispatch. Precedence, highest first: @@ -222,7 +225,7 @@ An explicit per-spawn harness still overrides either kind, and every secondmate `config/secondmate-harness` can also pin a model/effort for the secondmate agent in one line (` [] []`); format, accessors, and inheritance exceptions live in `secondmate-provisioning` (load before creating/seeding/launching/recovering a secondmate). -`config/crew-dispatch.json`, `config/crew-harness`, and `config/backlog-backend` are inherited into every secondmate home; `config/secondmate-harness` is not, because secondmates never spawn secondmates. +`config/crew-dispatch.json`, `config/crew-harness`, `config/backlog-backend`, and `config/account-routing-mode` are inherited into every secondmate home; `config/secondmate-harness` and `config/secondmate-account-pool` are primary-owned launch knobs and are not inherited. `secondmate-provisioning` owns the propagation timing, mechanism, the literal-file inheritance nuance, and `bin/fm-config-push.sh`. Each adapter splits into mechanics and knowledge. @@ -246,8 +249,9 @@ Reconcile reality with your records before doing anything else, working from the Do not sweep every `fm-*` tmux window, herdr tab, zellij tab, Orca terminal, or cmux workspace across all sessions during recovery; another firstmate home's child endpoints may share that namespace and are not this home's orphans. 5. If the digest reports a recorded direct-report's endpoint as `dead` (or a meta has no `window=`), reconcile it through its meta as described below. 6. For meta with no window, or an endpoint the digest reported dead, reconcile by kind. + If meta records `account_profile=`, recover the sticky Agent Fleet task with `bin/fm-spawn.sh --resume-account`; never run new-task selection or replay the brief as a new provider session, and surface a missing/mismatched SessionStart mapping as a blocker. For ordinary crewmates, check the recorded backend metadata first; use `treehouse status` for treehouse-backed tasks, and the recorded `orca_worktree_id=`/`terminal=` for Orca tasks. - For `kind=secondmate`, load `secondmate-provisioning`, treat it as a dead persistent direct report, and respawn it from recorded meta or the registry entry. + For an unmanaged `kind=secondmate`, load `secondmate-provisioning`, treat it as a dead persistent direct report, and respawn it from recorded meta or the registry entry. 7. Do not reconstruct a secondmate's whole tree from the main home. The main firstmate reconciles only direct reports. Each secondmate is a firstmate in its own home, so it reconciles only work that is already its own and then idles; it never creates new work during recovery. @@ -416,13 +420,16 @@ Load `harness-adapters` before spawning or recovering any direct report so trust ```sh bin/fm-spawn.sh projects/ # uses the active crewmate harness only when no crew-dispatch.json is active bin/fm-spawn.sh projects/ --harness codex --model gpt-5.5 --effort high # explicit profile axes +bin/fm-spawn.sh projects/ --harness codex --account-pool codex-crew # dynamic Agent Fleet account pool +bin/fm-spawn.sh projects/ --harness claude --account-profile claude-2 # explicit Agent Fleet profile +bin/fm-spawn.sh --resume-account # sticky managed recovery; never a fresh prompt bin/fm-spawn.sh projects/ --backend # explicit runtime backend (docs/configuration.md "Runtime backend") bin/fm-spawn.sh projects/ --scout # scout task; records kind=scout in meta bin/fm-spawn.sh [] --secondmate # launch or recover a persistent secondmate in its home bin/fm-spawn.sh =projects/ =projects/ [--scout] # batch: one call, several tasks ``` -Batch dispatch spawns each `id=repo` pair through the same single-task path, with shared `--scout`, `--harness`, `--model`, `--effort`, and `--backend` flags applying to all; one failed pair does not stop the rest, and the batch exits non-zero. +Batch dispatch spawns each `id=repo` pair through the same single-task path, with shared `--scout`, `--harness`, `--model`, `--effort`, `--backend`, `--account-pool`, `--account-profile`, and `--no-account-routing` flags applying to all; one failed pair does not stop the rest, and the batch exits non-zero. When `config/crew-dispatch.json` exists, include an explicit resolved harness for every crewmate or scout spawn or batch after consulting the dispatch rules (section 4). `bin/fm-spawn.sh`'s header owns the full resolution contract: harness and runtime-backend resolution order, spawn-capable backends and the `codex-app` rejection, verified launch templates, delivery-mode resolution, recorded meta fields, and per-harness turn-end hook installation. A backend spawn refusal - a missing dependency, an unauthenticated socket, or a version gate - must be surfaced to the captain as a blocker; never silently retry the spawn on a different backend to work around it. diff --git a/bin/fm-account-routing-lib.sh b/bin/fm-account-routing-lib.sh new file mode 100644 index 00000000000..f88fdbefe42 --- /dev/null +++ b/bin/fm-account-routing-lib.sh @@ -0,0 +1,252 @@ +# shellcheck shell=bash +# Agent Fleet account-routing helpers shared by spawn, recovery, supervision, +# and teardown. +# +# This file owns Firstmate's shell-side Agent Fleet contract. +# It consumes only `agent-fleet --format json contract` version 1 commands and +# never reads Agent Fleet state, profile homes, provider credentials, or quota +# caches directly. +# +# Routing mode precedence is: +# 1. an explicit per-spawn account pool/profile (enforce for that spawn), or +# --no-account-routing (off for that spawn); +# 2. FM_ACCOUNT_ROUTING; +# 3. config/account-routing-mode; +# 4. off. +# Valid modes are off, observe, and enforce. +# Off does not invoke Agent Fleet. +# Observe performs only `choose --dry-run`, never creates a lease, and never +# changes the provider launch or task metadata. +# Enforce atomically reserves one profile before endpoint creation and fails +# closed on every Agent Fleet or validation error. +# +# FM_AGENT_FLEET_BIN may name a deterministic fake or a pinned candidate in +# tests/labs. Otherwise `agent-fleet` is resolved from PATH. + +fm_account_shell_quote() { + printf "'" + printf '%s' "$1" | sed "s/'/'\\\\''/g" + printf "'" +} + +fm_account_valid_id() { + case "$1" in + ''|*[!A-Za-z0-9._-]*|.*|-*) return 1 ;; + esac + return 0 +} + +fm_account_fleet_bin() { + if [ -n "${FM_AGENT_FLEET_BIN:-}" ]; then + [ -x "$FM_AGENT_FLEET_BIN" ] || { + echo "error: FM_AGENT_FLEET_BIN is not executable: $FM_AGENT_FLEET_BIN" >&2 + return 1 + } + printf '%s\n' "$FM_AGENT_FLEET_BIN" + return 0 + fi + command -v agent-fleet 2>/dev/null || { + echo "error: agent-fleet is required for account routing" >&2 + return 1 + } +} + +fm_account_read_single_value() { # + local file=$1 value extra + [ -f "$file" ] || return 1 + value=$(sed -e 's/[[:space:]]*#.*$//' -e '/^[[:space:]]*$/d' "$file" | head -1 | tr -d '[:space:]') + extra=$(sed -e 's/[[:space:]]*#.*$//' -e '/^[[:space:]]*$/d' "$file" | sed -n '2p') + [ -z "$extra" ] || { + echo "error: $file must contain exactly one value" >&2 + return 2 + } + [ -n "$value" ] || return 1 + printf '%s\n' "$value" +} + +fm_account_resolve_mode() { # + local config=$1 explicit=$2 disabled=$3 value source + if [ "$disabled" = 1 ]; then + printf 'off\n' + return 0 + fi + if [ "$explicit" = 1 ]; then + printf 'enforce\n' + return 0 + fi + if [ -n "${FM_ACCOUNT_ROUTING:-}" ]; then + value=$FM_ACCOUNT_ROUTING + source=FM_ACCOUNT_ROUTING + elif value=$(fm_account_read_single_value "$config/account-routing-mode" 2>/dev/null); then + source=config/account-routing-mode + else + value=off + source=default + fi + case "$value" in + off|observe|enforce) printf '%s\n' "$value" ;; + *) echo "error: invalid account routing mode '$value' from $source (expected off, observe, or enforce)" >&2; return 1 ;; + esac +} + +fm_account_secondmate_pool() { # + local value + value=$(fm_account_read_single_value "$1/secondmate-account-pool") || return $? + fm_account_valid_id "$value" || { + echo "error: invalid account pool '$value' in config/secondmate-account-pool" >&2 + return 2 + } + printf '%s\n' "$value" +} + +fm_account_default_pool() { # + case "$1" in + claude|codex) printf '%s-crew\n' "$1" ;; + *) return 1 ;; + esac +} + +fm_account_json_field() { #