diff --git a/bin/backends/tmux.sh b/bin/backends/tmux.sh index 9eed5f3ec3..81d5f260ac 100644 --- a/bin/backends/tmux.sh +++ b/bin/backends/tmux.sh @@ -62,11 +62,16 @@ fm_backend_tmux_send_text_submit() { # # firstmate itself runs inside tmux, else ensure a dedicated detached # "firstmate" session exists. Mirrors fm-spawn.sh's container-ensure block; # prints the resolved session name. +# +# The probe asks for "=firstmate" - the same exact form every consumer of the +# returned name then targets. A bare -t falls back to prefix and then fnmatch, +# so a live look-alike (firstmate-lab, firstmate2) would answer for a session +# that does not exist, and the name handed back would address nothing. fm_backend_tmux_container_ensure() { if [ -n "${TMUX:-}" ]; then tmux display-message -p '#S' else - tmux has-session -t firstmate 2>/dev/null || tmux new-session -d -s firstmate + tmux has-session -t "=firstmate" 2>/dev/null || tmux new-session -d -s firstmate printf 'firstmate' fi } @@ -86,13 +91,96 @@ fm_backend_tmux_container_ensure() { # treehouse cd's into the worktree, which would break name-based targeting. # The returned window id lets callers target the window even if its name is ever # lost, so worktree discovery cannot fall back to the active client's window. +# +# The session is targeted as "=": tmux's bare -t falls back to +# prefix and then fnmatch resolution, so a recorded session name that no longer +# exists would silently resolve to an unrelated live session whose name merely +# starts with it, and the duplicate check would then be answered by - and the +# window created in - that other session. fm_backend_tmux_create_task() { # -> prints window id local ses=$1 wname=$2 proj_abs=$3 wid - if tmux list-windows -t "$ses" -F '#{window_name}' | grep -qx "$wname"; then + if tmux list-windows -t "=$ses" -F '#{window_name}' | grep -qx "$wname"; then echo "error: window $ses:$wname already exists" >&2 return 1 fi - wid=$(tmux new-window -dP -F '#{window_id}' -t "$ses:" -n "$wname" -c "$proj_abs") || return 1 + wid=$(tmux new-window -dP -F '#{window_id}' -t "=$ses:" -n "$wname" -c "$proj_abs") || return 1 + tmux set-window-option -t "$wid" automatic-rename off 2>/dev/null || true + tmux set-window-option -t "$wid" allow-rename off 2>/dev/null || true + printf '%s\n' "$wid" +} + +# Recreate an authoritatively missing task endpoint at its recorded address. +# The caller retains the existing worktree and task record; this function only +# creates the shell endpoint that can host the replacement agent. The recorded +# session is looked up as "=" so an unrelated live session that merely +# shares its prefix can never absorb the replacement window; when the recorded +# session is genuinely gone, a session of exactly that name is started instead. +fm_backend_tmux_recreate_task() { # -> prints window id + local target=$1 worktree=$2 session window state wid + case "$target" in + *:*:*) + echo "error: tmux task endpoint '$target' is malformed" >&2 + return 1 + ;; + *:*) ;; + *) + echo "error: tmux task endpoint '$target' is malformed" >&2 + return 1 + ;; + esac + session=${target%%:*} + window=${target#*:} + [ -n "$session" ] && [ -n "$window" ] || { + echo "error: tmux task endpoint '$target' is malformed" >&2 + return 1 + } + state=$(fm_backend_tmux_agent_state "$target") + [ "$state" = missing ] || { + echo "error: tmux task endpoint '$target' reads '$state', not missing; refusing to create a duplicate endpoint" >&2 + return 1 + } + if tmux has-session -t "=$session" 2>/dev/null; then + fm_backend_tmux_create_task "$session" "$window" "$worktree" + return + fi + wid=$(tmux new-session -dP -F '#{window_id}' -s "$session" -n "$window" -c "$worktree") || return 1 + tmux set-window-option -t "$wid" automatic-rename off 2>/dev/null || true + tmux set-window-option -t "$wid" allow-rename off 2>/dev/null || true + printf '%s\n' "$wid" +} + +# Recreate an authoritatively missing task endpoint at its recorded address. +# The caller retains the existing worktree and task record; this function only +# creates the shell endpoint that can host the replacement agent. +fm_backend_tmux_recreate_task() { # -> prints window id + local target=$1 worktree=$2 session window state wid + case "$target" in + *:*:*) + echo "error: tmux task endpoint '$target' is malformed" >&2 + return 1 + ;; + *:*) ;; + *) + echo "error: tmux task endpoint '$target' is malformed" >&2 + return 1 + ;; + esac + session=${target%%:*} + window=${target#*:} + [ -n "$session" ] && [ -n "$window" ] || { + echo "error: tmux task endpoint '$target' is malformed" >&2 + return 1 + } + state=$(fm_backend_tmux_agent_state "$target") + [ "$state" = missing ] || { + echo "error: tmux task endpoint '$target' reads '$state', not missing; refusing to create a duplicate endpoint" >&2 + return 1 + } + if tmux has-session -t "$session" 2>/dev/null; then + fm_backend_tmux_create_task "$session" "$window" "$worktree" + return + fi + wid=$(tmux new-session -dP -F '#{window_id}' -s "$session" -n "$window" -c "$worktree") || return 1 tmux set-window-option -t "$wid" automatic-rename off 2>/dev/null || true tmux set-window-option -t "$wid" allow-rename off 2>/dev/null || true printf '%s\n' "$wid" @@ -246,6 +334,90 @@ fm_backend_tmux_foreground_argv0s() { # done } +# fm_backend_tmux_inspect_endpoint: the one owner of how a recorded +# ":" endpoint is looked up - the target-shape parse, the +# exact-session inventory read, and the reading of tmux's own refusals. The +# liveness verdict and the recovery grade below are both derived from this +# single observation, so they cannot drift into disagreeing about what was +# asked of tmux or what tmux answered. +# +# The session is targeted as "=": a bare -t resolves by exact +# match, then prefix, then fnmatch, so an unrelated live session sharing the +# recorded name's prefix would otherwise answer in its place. +# +# Sets FM_BACKEND_TMUX_INSPECT_RESULT to one of: +# malformed the target is not a single : pair +# listed a reachable server's session inventory names the window +# window-absent a reachable server's session inventory omits the window +# session-absent a reachable server reports the recorded session is gone +# unreachable no server could be reached on the socket at all +# unreadable tmux failed for some other reason +# plus FM_BACKEND_TMUX_INSPECT_SOCKET (the socket tmux itself named, whenever it +# named one) and FM_BACKEND_TMUX_INSPECT_RESPONSE (tmux's own first line, or a +# description of what the inventory showed). +FM_BACKEND_TMUX_INSPECT_RESULT= +FM_BACKEND_TMUX_INSPECT_SOCKET= +FM_BACKEND_TMUX_INSPECT_RESPONSE= +fm_backend_tmux_inspect_endpoint() { # + local target=$1 session window windows inventory_status rest + FM_BACKEND_TMUX_INSPECT_RESULT=malformed + FM_BACKEND_TMUX_INSPECT_SOCKET= + FM_BACKEND_TMUX_INSPECT_RESPONSE= + case "$target" in + *:*:*|'':*|*:'') FM_BACKEND_TMUX_INSPECT_RESPONSE="endpoint '$target' is malformed"; return 0 ;; + *:*) ;; + *) FM_BACKEND_TMUX_INSPECT_RESPONSE="endpoint '$target' is malformed"; return 0 ;; + esac + session=${target%%:*} + window=${target#*:} + if windows=$(LC_ALL=C tmux list-windows -t "=$session" -F '#{window_name}' 2>&1); then + inventory_status=0 + else + inventory_status=$? + fi + if [ "$inventory_status" -eq 0 ]; then + FM_BACKEND_TMUX_INSPECT_SOCKET=$(tmux display-message -p '#{socket_path}' 2>/dev/null) || true + if printf '%s\n' "$windows" | grep -Fqx "$window"; then + FM_BACKEND_TMUX_INSPECT_RESULT=listed + FM_BACKEND_TMUX_INSPECT_RESPONSE="session inventory read; it lists $window, so the endpoint is present" + else + FM_BACKEND_TMUX_INSPECT_RESULT='window-absent' + FM_BACKEND_TMUX_INSPECT_RESPONSE="session inventory read; it does not list $window" + fi + return 0 + fi + FM_BACKEND_TMUX_INSPECT_RESPONSE=$(printf '%s\n' "$windows" | head -n 1) + [ -n "$FM_BACKEND_TMUX_INSPECT_RESPONSE" ] \ + || FM_BACKEND_TMUX_INSPECT_RESPONSE="tmux list-windows exited $inventory_status with no message" + case "$windows" in + *"can't find session:"*) + FM_BACKEND_TMUX_INSPECT_RESULT='session-absent' + FM_BACKEND_TMUX_INSPECT_SOCKET=$(tmux display-message -p '#{socket_path}' 2>/dev/null) || true + ;; + *"no server running on "*) + FM_BACKEND_TMUX_INSPECT_RESULT=unreachable + rest=${windows#*"no server running on "} + FM_BACKEND_TMUX_INSPECT_SOCKET=${rest%%$'\n'*} + ;; + *"error connecting to "*" (No such file or directory)"|*"error connecting to "*" (Connection refused)") + FM_BACKEND_TMUX_INSPECT_RESULT=unreachable + rest=${windows#*"error connecting to "} + rest=${rest%%$'\n'*} + FM_BACKEND_TMUX_INSPECT_SOCKET=${rest% (*} + ;; + *"error connecting to "*) + FM_BACKEND_TMUX_INSPECT_RESULT=unreadable + rest=${windows#*"error connecting to "} + rest=${rest%%$'\n'*} + FM_BACKEND_TMUX_INSPECT_SOCKET=${rest% (*} + ;; + *) + FM_BACKEND_TMUX_INSPECT_RESULT=unreadable + ;; + esac + return 0 +} + # fm_backend_tmux_agent_state: recovery-grade harness-agent state for one # recorded target. See bin/fm-backend.sh's fm_backend_agent_state for the # shared state vocabulary and docs/tmux-backend.md "Agent liveness probe" for @@ -263,35 +435,14 @@ fm_backend_tmux_foreground_argv0s() { # # authoritative for the negative verdicts, since it is the only source that can # distinguish a truly idle pane from a rewritten process title. fm_backend_tmux_agent_state() { # - local target=$1 comm session window windows inventory_status + local target=$1 comm local foreground argv0s name fg_seen=0 fg_shell=0 fg_other=0 - case "$target" in - *:*:*|'':*|*:'') printf 'unreadable'; return 0 ;; - *:*) ;; + fm_backend_tmux_inspect_endpoint "$target" + case "$FM_BACKEND_TMUX_INSPECT_RESULT" in + listed) ;; + window-absent|session-absent|unreachable) printf 'missing'; return 0 ;; *) printf 'unreadable'; return 0 ;; esac - session=${target%%:*} - window=${target#*:} - if windows=$(LC_ALL=C tmux list-windows -t "$session" -F '#{window_name}' 2>&1); then - inventory_status=0 - else - inventory_status=$? - fi - if [ "$inventory_status" -ne 0 ]; then - case "$windows" in - *"can't find session:"*|*"no server running on "*|*"error connecting to "*" (No such file or directory)"|*"error connecting to "*" (Connection refused)") - printf 'missing' - ;; - *) - printf 'unreadable' - ;; - esac - return 0 - fi - if ! printf '%s\n' "$windows" | grep -Fqx "$window"; then - printf 'missing' - return 0 - fi foreground=$(fm_backend_tmux_foreground_comms "$target") while IFS= read -r name; do @@ -346,6 +497,43 @@ EOF esac } +# fm_backend_tmux_missing_grade: how much a `missing` verdict actually proves. +# See bin/fm-backend.sh's fm_backend_missing_grade for the shared grade +# vocabulary. Two very different observations both read `missing` above: +# +# strong a REACHABLE server answered about the recorded session - either a +# successful inventory that omits the recorded window, or "can't +# find session", which only a live server can say. The window is +# gone from the server that would host it, so nothing can still be +# running there. +# ambiguous anything else. The server or its socket could not be reached at +# all, which cannot distinguish a wiped runtime from a server this +# process is simply not looking at (a different TMUX_TMPDIR, socket +# name, or user) - or the inventory came back and DOES list the +# window, which contradicts the missing verdict outright, since the +# two are separate reads and the endpoint can be restored between +# them. +# +# Strong is therefore reachable only from an observation that positively +# accounts for the exact recorded window, never from a read that merely +# succeeded. Sets FM_BACKEND_TMUX_MISSING_GRADE, plus the socket tmux itself +# named and the response it gave, so a caller can put the concrete evidence in +# front of a human instead of a bare verdict. +FM_BACKEND_TMUX_MISSING_GRADE= +FM_BACKEND_TMUX_MISSING_SOCKET= +FM_BACKEND_TMUX_MISSING_RESPONSE= +# shellcheck disable=SC2034 # Read by callers after fm_backend_tmux_missing_grade returns. +fm_backend_tmux_missing_grade() { # + fm_backend_tmux_inspect_endpoint "$1" + FM_BACKEND_TMUX_MISSING_SOCKET=$FM_BACKEND_TMUX_INSPECT_SOCKET + FM_BACKEND_TMUX_MISSING_RESPONSE=$FM_BACKEND_TMUX_INSPECT_RESPONSE + case "$FM_BACKEND_TMUX_INSPECT_RESULT" in + window-absent|session-absent) FM_BACKEND_TMUX_MISSING_GRADE=strong ;; + *) FM_BACKEND_TMUX_MISSING_GRADE=ambiguous ;; + esac + return 0 +} + # Backward-compatible three-state view for callers that only need a yes/no # agent verdict. The detailed state contract is owned by fm_backend_agent_state. fm_backend_tmux_agent_alive() { # diff --git a/bin/fm-backend.sh b/bin/fm-backend.sh index 2882f4a6af..54ec0ca8fb 100644 --- a/bin/fm-backend.sh +++ b/bin/fm-backend.sh @@ -879,9 +879,11 @@ fm_backend_target_exists() { # [expected-label] # ambiguous - the endpoint exists but its process cannot be attributed. # unreadable - a target or inventory read failed or contradicted itself. # unverified - this backend has no recovery classifier. -# Only `dead` and `missing` license recovery. The tmux adapter requires a -# successful session inventory and returns `missing` only when it omits the -# exact window; the Herdr adapter reuses its husk +# Only `dead` and `missing` license recovery. The tmux adapter returns `missing` +# when a reachable server's inventory omits the exact window, when it reports the +# session gone, and when its socket cannot be reached at all - see +# fm_backend_missing_grade below for which of those may create an endpoint; the +# Herdr adapter reuses its husk # classifier. Zellij remains unverified because its secondmate ghost-tab and # agent-process recovery path has not been empirically validated. Orca and cmux # do not support secondmate spawns. @@ -895,6 +897,56 @@ fm_backend_agent_state() { # esac } +# fm_backend_missing_grade: how much a `missing` verdict proves, for the one +# caller that acts on it destructively enough to need the distinction - creating +# a replacement endpoint. `missing` means "the recorded endpoint is not there", +# which comes from two observations of very different strength: +# strong a reachable runtime answered and the endpoint is absent from it, +# so nothing can still be running at that address. +# ambiguous the runtime itself could not be reached, so its silence cannot +# distinguish a wiped endpoint from one this process cannot see. +# Only `strong` licenses creating an endpoint without a human decision. Sets +# FM_BACKEND_MISSING_GRADE plus FM_BACKEND_MISSING_SOCKET and +# FM_BACKEND_MISSING_RESPONSE - the concrete address consulted and the answer it +# gave - so the caller can hand a human the evidence rather than a verdict. It +# sets variables instead of printing because command substitution would discard +# them. An unrecognized backend grades ambiguous: never having asked is the +# weakest evidence of all. +FM_BACKEND_MISSING_GRADE= +FM_BACKEND_MISSING_SOCKET= +FM_BACKEND_MISSING_RESPONSE= +# shellcheck disable=SC2034 # Read by callers after fm_backend_missing_grade returns. +fm_backend_missing_grade() { # + local backend=$1 target=$2 + FM_BACKEND_MISSING_GRADE=ambiguous + FM_BACKEND_MISSING_SOCKET= + FM_BACKEND_MISSING_RESPONSE= + fm_backend_source "$backend" || { + FM_BACKEND_MISSING_RESPONSE="backend '$backend' could not be loaded" + return 0 + } + case "$backend" in + tmux) + fm_backend_tmux_missing_grade "$target" + FM_BACKEND_MISSING_GRADE=$FM_BACKEND_TMUX_MISSING_GRADE + FM_BACKEND_MISSING_SOCKET=$FM_BACKEND_TMUX_MISSING_SOCKET + FM_BACKEND_MISSING_RESPONSE=$FM_BACKEND_TMUX_MISSING_RESPONSE + ;; + herdr) + # Herdr reaches `missing` only from a SUCCESSFUL pane read that reported a + # structurally gone pane; every failed or unexpected API read already + # classifies `unreadable`, so a missing verdict here is always strong. + FM_BACKEND_MISSING_GRADE=strong + FM_BACKEND_MISSING_SOCKET=$target + FM_BACKEND_MISSING_RESPONSE="pane read succeeded and reported the pane structurally gone" + ;; + *) + FM_BACKEND_MISSING_RESPONSE="backend '$backend' has no recovery-grade classifier" + ;; + esac + return 0 +} + # Backward-compatible three-state view for existing callers. An # authoritatively missing endpoint is confidently not a live agent, while every # ambiguous, unreadable, or unverified result stays unknown. diff --git a/bin/fm-control.sh b/bin/fm-control.sh index 4196d3095c..f15e5e34e4 100755 --- a/bin/fm-control.sh +++ b/bin/fm-control.sh @@ -6,6 +6,7 @@ # fm-control.sh exit # fm-control.sh relaunch [--harness ] [--model ] # [--effort ] +# [--confirm-endpoint-gone] # (--note | --note-file ) # # Why this exists, and how it differs from fm-send.sh. bin/fm-send.sh is the @@ -32,8 +33,8 @@ # the backend's recovery-grade classifier reports the agent gone. # Already-stopped is success (idempotent). # relaunch Transactionally replace the running agent with a new one, in the -# SAME endpoint and SAME worktree, on the same or a newly chosen -# harness/model/effort - so switching harness is one ordinary use +# SAME worktree and normally the same endpoint, on the same or a +# newly chosen harness/model/effort - so switching harness is one ordinary use # of this verb. With no explicit axis, a secondmate re-resolves its # durable config/secondmate-harness pin (harness plus its optional # model and effort tokens) exactly as any other respawn does, while @@ -44,10 +45,21 @@ # inherits the local copy but none of the conversation; a # secondmate reconciles its own home's records at startup, so its # standing charter is never rewritten. -# Records a durable checkpoint and that note, exits the old agent, -# then delegates the launch to its single owner, -# bin/fm-spawn.sh --relaunch. A failure before publication keeps -# the prior durable record in place and reports the concrete +# Records a durable checkpoint and that note, exits the old agent +# when one exists, then delegates the launch to its single owner, +# bin/fm-spawn.sh --relaunch. When the backend reports the +# recorded endpoint missing from a RUNTIME IT REACHED, the launch +# owner rechecks that verdict and creates a fresh endpoint in the +# same worktree. When the runtime itself could not be reached, the +# missing verdict is ambiguous: relaunch stops with the socket it +# consulted and the answer it got, changes nothing, and names +# --confirm-endpoint-gone as the continuation once a human has +# confirmed the runtime is genuinely gone. Confirmation only +# licenses endpoint creation while the endpoint still classifies +# missing; a live agent or any contradicting recheck still +# refuses. Ambiguous and unreadable agent states refuse before the +# note or durable record changes. A failure before publication +# keeps the prior durable record in place and reports the concrete # state; it never leaves a half-transitioned task claiming to be # running. # @@ -82,6 +94,9 @@ # than reported as successful blind. # - An ambiguous or unreadable endpoint state refuses; only a positively # classified state acts. +# - Creating a replacement endpoint requires a missing verdict from a runtime +# that ANSWERED. An unreachable runtime cannot prove no agent holds the +# worktree, so it is a human decision rather than an automatic recovery. # # Environment knobs (all bounded waits, seconds): # FM_CONTROL_POLL poll interval for postcondition waits (0.5) @@ -150,6 +165,8 @@ CONTROL_LOCK= CONTROL_LOCK_HELD=0 RELAUNCH_ACTIVE=0 RELAUNCH_PHASE=start +RELAUNCH_RECREATE_ENDPOINT=0 +RELAUNCH_PRIOR_TARGET= control_cleanup() { local status=$? @@ -192,6 +209,7 @@ MODEL_SET=0 EFFORT_SET=0 NOTE= NOTE_SET=0 +CONFIRM_ENDPOINT_GONE=0 want_value= for a in "$@"; do if [ -n "$want_value" ]; then @@ -221,6 +239,7 @@ for a in "$@"; do --effort=*) NEW_EFFORT=${a#--effort=}; EFFORT_SET=1 ;; --note) want_value=note ;; --note=*) NOTE=${a#--note=}; NOTE_SET=1 ;; + --confirm-endpoint-gone) CONFIRM_ENDPOINT_GONE=1 ;; --note-file) want_value=note-file ;; --note-file=*) [ -f "${a#--note-file=}" ] || die "--note-file '${a#--note-file=}' is not a readable file" @@ -234,7 +253,8 @@ done if [ "$VERB" != relaunch ]; then [ "$HARNESS_SET" = 0 ] && [ "$MODEL_SET" = 0 ] && [ "$EFFORT_SET" = 0 ] && [ "$NOTE_SET" = 0 ] \ - || die "--harness, --model, --effort, and --note apply to 'relaunch' only" + && [ "$CONFIRM_ENDPOINT_GONE" = 0 ] \ + || die "--harness, --model, --effort, --note, and --confirm-endpoint-gone apply to 'relaunch' only" fi [ "$HARNESS_SET" = 0 ] || [ -n "$NEW_HARNESS" ] || die "--harness requires a non-empty value" [ "$MODEL_SET" = 0 ] || [ -n "$NEW_MODEL" ] || die "--model requires a non-empty value" @@ -568,6 +588,10 @@ relaunch_rollback() { journal_write "failed:$RELAUNCH_PHASE" "rollback=prior-record-kept-agent-dead" || true echo "error: $ID's agent stopped but relaunch did not reach replacement launch; no agent is running, and its work plus progress note are preserved at $WT" >&2 ;; + missing) + journal_write "failed:$RELAUNCH_PHASE" "rollback=prior-record-kept-endpoint-missing" || true + echo "error: $ID's endpoint is still missing and relaunch did not reach replacement launch; its durable record, work, and progress note are preserved" >&2 + ;; *) journal_write "failed:$RELAUNCH_PHASE" "rollback=none-agent-state-$state" || true echo "error: relaunch of $ID failed while stopping the old agent and its state is '$state'; the durable record and progress note were retained for recovery" >&2 @@ -590,7 +614,11 @@ relaunch_rollback() { echo "error: $ID was relaunched on $TARGET_HARNESS but no running agent could be confirmed; its work is preserved at $WT" >&2 else journal_write "failed:$RELAUNCH_PHASE" "rollback=prior-record-kept" || true - echo "error: $ID's agent was stopped but the replacement did not launch; no agent is running, and its work plus the recorded progress note are preserved at $WT" >&2 + if [ "$RELAUNCH_RECREATE_ENDPOINT" = 1 ]; then + echo "error: $ID's missing endpoint could not be recreated; its prior durable record, work, and progress note are preserved at $WT" >&2 + else + echo "error: $ID's agent was stopped but the replacement did not launch; no agent is running, and its work plus the recorded progress note are preserved at $WT" >&2 + fi fi ;; esac @@ -765,12 +793,65 @@ record_note() { esac } +# classify_missing_endpoint: decide whether a `missing` verdict may create a +# replacement endpoint. `missing` is not one observation but two, and only one +# of them proves that no agent can still be holding the worktree: +# +# strong a runtime ANSWERED and the endpoint is not in it. Nothing runs +# there; recreate automatically, exactly as a wiped runtime needs. +# ambiguous the runtime could not be reached at all. Its silence is equally +# consistent with "the endpoint is gone" and "this process is +# looking at the wrong socket", and the second reading would put a +# second agent on a worktree the first one is still working in. +# +# Ambiguity is not a dead end here - a wiped runtime is exactly the case that +# needs recovery, and it is a human who can walk over and see whether anything +# is still running. So it stops with the concrete address consulted and the +# answer it gave, changes nothing, and names the continuation. The confirmation +# grants no standing authority: it is consumed here, only while the endpoint is +# still classified missing, and every downstream recheck (fm-spawn's own state +# read and the backend's create-time recheck, both under the lifecycle locks) +# still runs and still refuses a contradicting verdict. +classify_missing_endpoint() { + local socket detail + fm_backend_missing_grade "$BACKEND" "$T" + socket=${FM_BACKEND_MISSING_SOCKET:-unknown} + detail=${FM_BACKEND_MISSING_RESPONSE:-no response recorded} + if [ "$FM_BACKEND_MISSING_GRADE" = strong ]; then + RELAUNCH_RECREATE_ENDPOINT=1 + return 0 + fi + if [ "$CONFIRM_ENDPOINT_GONE" = 1 ]; then + echo "notice: recreating $ID's endpoint $T on confirmation that its runtime is gone ($BACKEND at $socket answered: $detail); the replacement launch rechecks that verdict under the lifecycle locks and still refuses if anything contradicts it" >&2 + RELAUNCH_RECREATE_ENDPOINT=1 + return 0 + fi + { + echo "error: task $ID's endpoint $T reads missing, but only because its $BACKEND runtime could not be reached: $socket answered '$detail'. That cannot tell a wiped runtime apart from a runtime this process is not looking at, and recreating the endpoint on the second reading would put a second agent into $WT while the first one is still working there." + echo "Nothing was changed: $ID's durable record, its progress note, and every uncommitted change in $WT are exactly as they were, and no endpoint was created." + echo "To continue, confirm the runtime is genuinely gone - no $BACKEND runtime anywhere holds $T, and no agent is still working in $WT - then re-run this command with --confirm-endpoint-gone. That confirmation only applies while the endpoint still classifies missing; a live agent, or any recheck that contradicts it under the lifecycle locks, still refuses." + } >&2 + exit 1 +} + do_relaunch() { - local exit_result state note_line + local exit_result state note_line outcome_suffix local -a spawn_args require_state_verified_backend relaunch resolve_relaunch_profile + RELAUNCH_PRIOR_TARGET=$T + state=$(agent_state) + case "$state" in + alive|dead) + [ "$CONFIRM_ENDPOINT_GONE" = 0 ] \ + || echo "notice: --confirm-endpoint-gone is not applied: task $ID's endpoint $T reads '$state', so this relaunch replaces the agent in the endpoint it already has" >&2 + ;; + missing) classify_missing_endpoint ;; + *) + die "task $ID's endpoint reads '$state', not alive, dead, or authoritatively missing; refusing to relaunch from an ambiguous endpoint state or change its durable record" + ;; + esac case "$KIND" in ship|scout) @@ -803,8 +884,12 @@ do_relaunch() { record_note journal_write noted "${CHECKPOINT_LINES[@]}" "$note_line" - journal_write stopping "${CHECKPOINT_LINES[@]}" "$note_line" - exit_result=$(do_exit) + if [ "$RELAUNCH_RECREATE_ENDPOINT" = 1 ]; then + exit_result='endpoint-missing' + else + journal_write stopping "${CHECKPOINT_LINES[@]}" "$note_line" + exit_result=$(do_exit) + fi journal_write exited "${CHECKPOINT_LINES[@]}" "$note_line" "exit_result=$exit_result" # The launch owner (fm-spawn --relaunch) clears the previous incarnation's @@ -812,6 +897,7 @@ do_relaunch() { RELAUNCH_TX="${BASHPID:-$$}.$(date -u +%Y%m%dT%H%M%SZ).$RANDOM" journal_write launching "${CHECKPOINT_LINES[@]}" "$note_line" "relaunch_tx=$RELAUNCH_TX" spawn_args=("$ID" --relaunch --harness "$TARGET_HARNESS") + [ "$RELAUNCH_RECREATE_ENDPOINT" = 0 ] || spawn_args+=(--recreate-endpoint) [ "$TARGET_MODEL" = default ] || spawn_args+=(--model "$TARGET_MODEL") [ "$TARGET_EFFORT" = default ] || spawn_args+=(--effort "$TARGET_EFFORT") if FM_CONTROL_RELAUNCH_TX="$RELAUNCH_TX" \ @@ -823,14 +909,25 @@ do_relaunch() { die "the replacement agent for $ID could not be launched on $TARGET_HARNESS" fi + fm_backend_validate_task_endpoint "$META" "$ID" \ + || die "the replacement for $ID published endpoint metadata that cannot be validated" + [ "$FM_BACKEND_VALIDATED_BACKEND" = "$BACKEND" ] \ + || die "the replacement for $ID changed backend from $BACKEND to $FM_BACKEND_VALIDATED_BACKEND during relaunch" + T=$FM_BACKEND_VALIDATED_TARGET + state=$(wait_agent_state "$LAUNCH_WAIT" alive) || { die "the replacement agent for $ID did not come up within ${LAUNCH_WAIT}s (endpoint reads '$state')" } RELAUNCH_AGENT_CONFIRMED=1 - journal_write complete "${CHECKPOINT_LINES[@]}" "$note_line" "exit_result=$exit_result" + journal_write complete "${CHECKPOINT_LINES[@]}" "$note_line" \ + "exit_result=$exit_result" "previous_endpoint=$RELAUNCH_PRIOR_TARGET" RELAUNCH_ACTIVE=0 - echo "relaunched $ID harness=$TARGET_HARNESS from=$PRIOR_RECORDED_HARNESS model=$TARGET_MODEL effort=$TARGET_EFFORT backend=$BACKEND endpoint=$T worktree=$WT" + outcome_suffix= + if [ "$RELAUNCH_RECREATE_ENDPOINT" = 1 ]; then + outcome_suffix=" endpoint-recreated previous-endpoint=$RELAUNCH_PRIOR_TARGET" + fi + echo "relaunched $ID harness=$TARGET_HARNESS from=$PRIOR_RECORDED_HARNESS model=$TARGET_MODEL effort=$TARGET_EFFORT backend=$BACKEND endpoint=$T worktree=$WT$outcome_suffix" } # --- verbs ------------------------------------------------------------------ diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index cfb25f0058..5d6683f4c9 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -16,9 +16,12 @@ # loud one-line deviation notice is printed and the spawn continues. # no-mistakes-prod-only is a registry policy rather than a task mode and is # refused as a flag value. -# fm-spawn.sh --relaunch [--harness ] [--model ] [--effort ] +# fm-spawn.sh --relaunch [--recreate-endpoint] [--harness ] [--model ] [--effort ] # --relaunch launches a replacement agent for an EXISTING task into that -# task's own recorded endpoint and worktree instead of creating either. It is +# task's own recorded worktree, ordinarily reusing its endpoint. The control +# plane alone adds --recreate-endpoint after it authoritatively classifies the +# recorded endpoint missing; fm-spawn rechecks that verdict under the task's +# lifecycle locks before creating a replacement shell endpoint. It is # the launch half of the control plane (bin/fm-control.sh relaunch), which # owns the checkpoint, the progress note, stopping the previous agent, and the # transaction; call fm-control rather than this flag directly unless you are @@ -28,10 +31,11 @@ # positional, and batch pairs are all refused alongside it; only harness, # model, and effort may change, which is what makes a harness switch one # ordinary relaunch. It refuses unless the recorded endpoint is positively -# agent-free on a backend with a recovery-grade agent-state classifier (tmux -# or herdr), refuses unless the endpoint's shell is sitting in the recorded -# worktree, and clears the previous harness's per-task wiring before arming -# the new incarnation. +# agent-free, or --recreate-endpoint accompanies a positively missing verdict, +# on a backend with a recovery-grade agent-state classifier (tmux or herdr). +# It refuses unless the adopted or recreated endpoint's shell is sitting in +# the recorded worktree, and clears the previous harness's per-task wiring +# before arming the new incarnation. # --harness is the explicit per-spawn harness/profile adapter. The old # positional harness arg still works for back-compat. # --model and --effort are concrete profile @@ -283,6 +287,7 @@ MODE_SET=0 YOLO_SET=0 TRACEPARENT_SET=0 RELAUNCH=0 +RECREATE_ENDPOINT=0 POS=() want_value= for a in "$@"; do @@ -307,6 +312,7 @@ for a in "$@"; do --scout) KIND=scout; KIND_SET=1 ;; --secondmate) KIND=secondmate; KIND_SET=1 ;; --relaunch) RELAUNCH=1 ;; + --recreate-endpoint) RECREATE_ENDPOINT=1 ;; --harness) want_value=harness ;; --harness=*) HARNESS_ARG=${a#--harness=}; HARNESS_SET=1 ;; --model) want_value=model ;; @@ -350,16 +356,22 @@ case "$EFFORT" in *) echo "error: --effort must be one of low, medium, high, xhigh, max" >&2; exit 1 ;; esac -# --relaunch reuses an existing task's endpoint, worktree, project, and kind, -# so every axis this block resolves for a fresh spawn instead comes from that -# task's own durable record below. Contradicting it on the command line is a -# refusal rather than a silently-ignored flag. +# --relaunch reuses an existing task's worktree, project, and kind. It normally +# reuses the endpoint too; --recreate-endpoint is the control plane's narrow +# recovery request for an endpoint the backend authoritatively reports missing. +# Every identity axis still comes from the task's own durable record below. +# Contradicting it on the command line is a refusal rather than a silently- +# ignored flag. if [ "$RELAUNCH" -eq 1 ]; then [ "$BACKEND_SET" -eq 0 ] || { echo "error: --relaunch reuses the task's recorded backend; --backend cannot override it" >&2; exit 1; } [ "$KIND_SET" -eq 0 ] || { echo "error: --relaunch reuses the task's recorded kind; --scout/--secondmate cannot override it" >&2; exit 1; } [ "$MODE_SET" -eq 0 ] || { echo "error: --relaunch reuses the task's recorded delivery mode; --mode cannot override it" >&2; exit 1; } [ "$YOLO_SET" -eq 0 ] || { echo "error: --relaunch reuses the task's recorded yolo posture; --yolo cannot override it" >&2; exit 1; } else + [ "$RECREATE_ENDPOINT" -eq 0 ] || { + echo "error: --recreate-endpoint applies only to --relaunch" >&2 + exit 1 + } # Delivery contract (AGENTS.md section 7). A ship task's mode and yolo are # firstmate's per-task decision, so they are required and closed-set validated # here rather than resolved from the project registry. Scouts deliver a report @@ -716,6 +728,10 @@ spawn_abort_cleanup() { fi fi fi + if [ "$RECREATE_ENDPOINT" = 1 ] && [ "$HERDR_PROJECTION_ABORT_CLEANUP" = 1 ]; then + HERDR_PROJECTION_ABORT_CLEANUP=0 + echo "warning: retaining the recreated Herdr shell endpoint after relaunch failure; fm-control never closes endpoints or discards work" >&2 + fi if [ "$HERDR_PROJECTION_ABORT_CLEANUP" = 1 ] \ && [ "$HERDR_PRESENTATION_ORDER_LOCK_HELD" != 1 ]; then if ! spawn_herdr_presentation_order_lock_acquire "${HERDR_PROJECTION_ABORT_SESSION:-}"; then @@ -974,9 +990,9 @@ PROJ= ARG3= FIRSTMATE_HOME= -# --relaunch adoption: every identity axis comes from the task's own validated +# --relaunch recovery: every identity axis comes from the task's own validated # durable record, never from the command line, so a relaunch can only ever -# re-launch the task it names. The endpoint identity check is the same shared +# relaunch the task it names. The endpoint identity check is the same shared # validation teardown uses, so a malformed, ambiguous, or foreign record # refuses here exactly as it refuses there. RELAUNCH_PRIOR_HARNESS= @@ -1002,11 +1018,23 @@ if [ "$RELAUNCH" -eq 1 ]; then echo "error: backend '$BACKEND' has no recovery-grade agent-state classifier, so a relaunch cannot prove the previous agent exited; refusing rather than risking two agents in one endpoint" >&2 exit 1 } - RELAUNCH_STATE=$(fm_backend_agent_state "$BACKEND" "$RELAUNCH_TARGET") - [ "$RELAUNCH_STATE" = dead ] || { - echo "error: task $ID's endpoint reads '$RELAUNCH_STATE'; a relaunch requires a positively agent-free endpoint (stop the agent first with bin/fm-control.sh $ID exit)" >&2 + if [ "$RECREATE_ENDPOINT" -eq 1 ] \ + && { [ "$SPAWN_CONTROL_PARENT" -ne 1 ] || [ -z "${FM_CONTROL_RELAUNCH_TX:-}" ]; }; then + echo "error: --recreate-endpoint is authorized only by bin/fm-control.sh after its locked missing-state checkpoint" >&2 exit 1 - } + fi + RELAUNCH_STATE=$(fm_backend_agent_state "$BACKEND" "$RELAUNCH_TARGET") + if [ "$RECREATE_ENDPOINT" -eq 1 ]; then + [ "$RELAUNCH_STATE" = missing ] || { + echo "error: task $ID's endpoint reads '$RELAUNCH_STATE', not missing; refusing to create a duplicate endpoint" >&2 + exit 1 + } + else + [ "$RELAUNCH_STATE" = dead ] || { + echo "error: task $ID's endpoint reads '$RELAUNCH_STATE'; a relaunch requires a positively agent-free endpoint (stop the agent first with bin/fm-control.sh $ID exit), or an authoritative missing-endpoint recovery from fm-control" >&2 + exit 1 + } + fi RELAUNCH_PRIOR_HARNESS=$(fm_meta_get "$RELAUNCH_META" harness) KIND=$(fm_meta_get "$RELAUNCH_META" kind) [ -n "$KIND" ] || KIND=ship @@ -1028,10 +1056,10 @@ if [ "$RELAUNCH" -eq 1 ]; then } fi if [ "$BACKEND" = herdr ]; then - HERDR_SES=$(fm_meta_get "$RELAUNCH_META" herdr_session) - HERDR_WORKSPACE_ID=$(fm_meta_get "$RELAUNCH_META" herdr_workspace_id) - HERDR_TAB_ID=$(fm_meta_get "$RELAUNCH_META" herdr_tab_id) - HERDR_PANE_ID=$(fm_meta_get "$RELAUNCH_META" herdr_pane_id) + RELAUNCH_HERDR_SESSION=$(fm_meta_get "$RELAUNCH_META" herdr_session) + RELAUNCH_HERDR_WORKSPACE_ID=$(fm_meta_get "$RELAUNCH_META" herdr_workspace_id) + RELAUNCH_HERDR_TAB_ID=$(fm_meta_get "$RELAUNCH_META" herdr_tab_id) + RELAUNCH_HERDR_PANE_ID=$(fm_meta_get "$RELAUNCH_META" herdr_pane_id) fi # With no explicit harness, a relaunch reuses the harness already recorded # for this task. It must NOT fall through to the fresh-spawn config @@ -1691,6 +1719,25 @@ BRIEF_REAL="$BRIEF_DIR_REAL/$(basename "$BRIEF")" # once here so every downstream comparison uses the same physical form # (docs/herdr-backend.md "Known gaps"). PROJ_ABS_REAL=$(cd "$PROJ_ABS" 2>/dev/null && pwd -P) || PROJ_ABS_REAL="$PROJ_ABS" +ENDPOINT_CWD=$PROJ_ABS +if [ "$RECREATE_ENDPOINT" -eq 1 ]; then + # Same carve-out as the endpoint-adopting relaunch path below: a secondmate's + # home already resolved WT through validate_firstmate_home_for_spawn, the same + # validation a fresh secondmate spawn uses, so it must not be overwritten with + # the raw recorded value. Every other kind takes the recorded worktree. + [ "$KIND" = secondmate ] || WT=$RELAUNCH_WT + ENDPOINT_CWD=$WT + if [ "$BACKEND" = herdr ]; then + HERDR_SES=$RELAUNCH_HERDR_SESSION + HERDR_SESSION=$HERDR_SES + export HERDR_SESSION + fi +elif [ "$RELAUNCH" -eq 1 ] && [ "$BACKEND" = herdr ]; then + HERDR_SES=$RELAUNCH_HERDR_SESSION + HERDR_WORKSPACE_ID=$RELAUNCH_HERDR_WORKSPACE_ID + HERDR_TAB_ID=$RELAUNCH_HERDR_TAB_ID + HERDR_PANE_ID=$RELAUNCH_HERDR_PANE_ID +fi real_path_or_raw() { # local path=$1 real @@ -1846,7 +1893,7 @@ herdr_projection_existing_meta_allows_flat() { # } W="fm-$ID" -if [ "$RELAUNCH" -eq 1 ]; then +if [ "$RELAUNCH" -eq 1 ] && [ "$RECREATE_ENDPOINT" -eq 0 ]; then # Adopt the recorded endpoint instead of creating one. This is what keeps a # relaunch a REPLACEMENT rather than a second copy of the task: no new # terminal, no second worktree, and every uncommitted change left exactly @@ -1860,15 +1907,21 @@ if [ "$RELAUNCH" -eq 1 ]; then else case "$BACKEND" in tmux) - SES=$(fm_backend_tmux_container_ensure) - T="$SES:$W" - # #134 robustness (tmux): fm_backend_tmux_create_task captures a stable window - # id and pins the window name (automatic-rename/allow-rename off) so a captain's + if [ "$RECREATE_ENDPOINT" -eq 1 ]; then + T=$RELAUNCH_TARGET + SES=${T%%:*} + WID=$(fm_backend_tmux_recreate_task "$T" "$ENDPOINT_CWD") || exit 1 + else + SES=$(fm_backend_tmux_container_ensure) + T="$SES:$W" + WID=$(fm_backend_tmux_create_task "$SES" "$W" "$ENDPOINT_CWD") || exit 1 + fi + # #134 robustness (tmux): the create helpers capture a stable window id and + # pin the window name (automatic-rename/allow-rename off) so a captain's # non-default tmux config cannot rename the window away from fm- once # treehouse cd's into the worktree. WT_TARGET carries that stable id for the # rename-critical worktree-detection steps below; the persisted window= handle # stays $T (the name form), which is safe now that rename is disabled. - WID=$(fm_backend_tmux_create_task "$SES" "$W" "$PROJ_ABS") || exit 1 WT_TARGET="$WID" ;; herdr) @@ -1920,7 +1973,7 @@ case "$BACKEND" in FM_HOME="$HERDR_LABEL_HOME" fm_backend_herdr_projection_reclaim_task \ "$HERDR_SES" "$HERDR_PRESENTATION_JOURNAL" "$ID" "$HERDR_LABEL_HOME" \ "$HERDR_RECOVERY_WORKSPACE_ID" "$HERDR_RECOVERY_TAB_ID" "$HERDR_RECOVERY_PANE_ID" \ - "$HERDR_PARENT_LABEL" "$W" "$PROJ_ABS" + "$HERDR_PARENT_LABEL" "$W" "$ENDPOINT_CWD" HERDR_RECLAIM_STATUS=$? set -e case "$HERDR_RECLAIM_STATUS" in @@ -1974,7 +2027,7 @@ case "$BACKEND" in HERDR_PROJECTION_ID=$(fm_backend_herdr_projection_journal_create "$STATE" "$ID") || exit 1 HERDR_PROJECTION_LABEL=$(fm_backend_herdr_projection_workspace_label "$ID" "$HERDR_PROJECTION_ID") if ! FM_HOME="$HERDR_LABEL_HOME" fm_backend_herdr_projection_create_task \ - "$PROJ_ABS" "$HERDR_PROJECTION_LABEL" "$W"; then + "$ENDPOINT_CWD" "$HERDR_PROJECTION_LABEL" "$W"; then if [ "${FM_BACKEND_HERDR_PROJECTION_CLEANUP_SAFE:-0}" = 1 ]; then HERDR_PROJECTION_ABORT_CLEANUP=1 HERDR_PROJECTION_ABORT_SESSION=$FM_BACKEND_HERDR_PROJECTION_SESSION @@ -2016,7 +2069,7 @@ case "$BACKEND" in fi fi if [ "$HERDR_PROJECTED" -ne 1 ]; then - HERDR_CONTAINER_RAW=$(FM_HOME="$HERDR_LABEL_HOME" fm_backend_herdr_container_ensure "$PROJ_ABS" "$HERDR_LAUNCHER_RELATIONSHIP") || exit 1 + HERDR_CONTAINER_RAW=$(FM_HOME="$HERDR_LABEL_HOME" fm_backend_herdr_container_ensure "$ENDPOINT_CWD" "$HERDR_LAUNCHER_RELATIONSHIP") || exit 1 # fm_backend_herdr_container_ensure echoes ":\t" # (the second field empty when this call ADOPTED a pre-existing workspace # rather than creating a fresh one). Split on the guaranteed single tab @@ -2027,7 +2080,7 @@ case "$BACKEND" in HERDR_SEEDED_DEFAULT_TAB_ID=${HERDR_CONTAINER_RAW#*$'\t'} HERDR_SES=${CONTAINER%%:*} HERDR_WORKSPACE_ID=${CONTAINER#*:} - HERDR_TASK_IDS=$(FM_HOME="$HERDR_LABEL_HOME" fm_backend_herdr_create_task "$CONTAINER" "$W" "$PROJ_ABS" "$HERDR_SEEDED_DEFAULT_TAB_ID") || exit 1 + HERDR_TASK_IDS=$(FM_HOME="$HERDR_LABEL_HOME" fm_backend_herdr_create_task "$CONTAINER" "$W" "$ENDPOINT_CWD" "$HERDR_SEEDED_DEFAULT_TAB_ID") || exit 1 read -r HERDR_TAB_ID HERDR_PANE_ID <` or `error connecting to ` - or the grade's own read no longer showed the window absent. | Stops and asks. Nothing is created, stopped, or written. | +| **alive** | A verified harness agent is running. | Never recreates an endpoint. The ordinary relaunch replaces the agent in the endpoint it already has. | + +An unreachable socket says only that *this process* cannot see a runtime there. +It cannot distinguish a wiped runtime from a server behind a different `TMUX_TMPDIR`, socket name, or user - and on the second reading, recreating the endpoint would start a second agent in a worktree the first one is still working in. +The liveness verdict and the grade are two separate reads of the runtime, so the grade is taken from what its own read saw rather than from the earlier `missing`: an inventory that now lists the window grades ambiguous rather than quoting tmux as authority for the absence of a window that is sitting right there. + +So an ambiguous verdict is surfaced as a decision rather than a dead end - a wiped runtime is exactly the case that needs recovery, and a human can see what a socket probe cannot. +The refusal names the exact socket consulted and the backend's own response, states that the durable record, progress note, and worktree are untouched, and names the continuation: confirm no runtime anywhere holds that endpoint and no agent is still working in the worktree, then re-run with `--confirm-endpoint-gone`. + +That confirmation grants no standing authority. +It is consumed by the one invocation that carries it, and only while the endpoint still classifies missing. +`fm-spawn --relaunch` re-reads the state under the lifecycle locks and refuses anything but `missing`, and the backend's create step classifies the address once more before creating anything, so a runtime that has come back - or an endpoint that reads alive - still refuses. +Passing `--confirm-endpoint-gone` at an alive or agent-free endpoint applies nothing: it is reported as not applied, and the relaunch proceeds as the ordinary in-place replacement. + ### Failure and rollback -- A refusal **before** the agent is stopped leaves the durable record and the instructions byte-identical. +- A preflight refusal **before** the progress note is recorded leaves the durable record and the instructions byte-identical. - A launch failure **after** the agent is stopped restores the prior durable record, keeps the progress note so a later recovery still has it, marks the journal `failed:launching`, and reports plainly that no agent is running and where the work is preserved. +- A missing-endpoint recreation failure before publication likewise keeps the prior durable record and progress note, with the existing worktree untouched. + An endpoint the recovery already created is kept too, not cleaned up on the way out, because this plane closes no endpoint even one it opened moments earlier. - If the launch owner already published the new record but no running agent can be confirmed, the new record is kept: the task is recorded on the new harness with no agent confirmed, which is exactly what recovery reconciles. Rewriting it back to the old harness would be a second, worse inaccuracy. @@ -98,7 +129,10 @@ Switching harness is therefore one ordinary relaunch rather than a separate mech zellij, orca, and cmux are refused rather than reported as successful blind. - An ambiguous or unreadable endpoint state refuses. Only a positively classified state acts. -- `fm-spawn --relaunch` independently refuses unless the recorded endpoint is positively agent-free and its shell is sitting in the recorded worktree, so a replacement can never join a live agent or start outside the copy holding the work. +- Creating a replacement endpoint requires a `missing` verdict from a runtime that **answered**. + An unreachable runtime cannot prove that no agent holds the worktree, so it becomes a human decision (`--confirm-endpoint-gone`) rather than an automatic recovery. +- `fm-spawn --relaunch` independently requires either a positively agent-free recorded endpoint or an explicit missing-endpoint recovery whose `missing` verdict it rechecks under the lifecycle locks. + It verifies the adopted or recreated shell is sitting in the recorded worktree, so a replacement can never join a live agent or start outside the copy holding the work. ## Capability matrix @@ -118,5 +152,6 @@ The empirical basis for each adapter's value is the `harness-adapters` skill's v ## Verification - `tests/fm-control.test.sh` - the adapter contract for every verified harness, the backend capability matrix, exact-id scoping, the closed verb list, the busy, idle, dead, and idempotent lifecycle cases, and marker non-regression, all against a stubbed session provider. -- `tests/fm-control-relaunch.test.sh` - the relaunch transaction: identity preservation, harness switching, the progress note, checkpoint refusals, and rollback after a failed launch. +- `tests/fm-control-relaunch.test.sh` - the relaunch transaction: identity preservation, harness switching, the progress note, checkpoint refusals, rollback after a failed launch, and the missing-endpoint decision - strong recovery into the recorded worktree with its uncommitted work intact, the ambiguous decision that changes nothing before and after `--confirm-endpoint-gone`, a confirmation that still refuses a live endpoint, a secondmate recovering into its validated home, and repeat recovery that never creates a duplicate endpoint. - `tests/fm-control-herdr-smoke.test.sh` - the second state-verified backend against the real herdr binary, on an isolated throwaway lab session. +- `tests/fm-backend-tmux-smoke.test.sh` - the missing grade and endpoint recreation against a real tmux server on a private socket, which is where the answers quoted in the decision table above come from. diff --git a/tests/fm-backend-tmux-smoke.test.sh b/tests/fm-backend-tmux-smoke.test.sh index aa1e07c326..9622688425 100755 --- a/tests/fm-backend-tmux-smoke.test.sh +++ b/tests/fm-backend-tmux-smoke.test.sh @@ -169,5 +169,153 @@ state=$(fm_backend_agent_state tmux "$TARGET") fm_backend_tmux_kill "$TARGET" || fail "fm_backend_tmux_kill on an already-dead target must stay best-effort (never fail)" pass "real tmux: kill removes the window and the readable session inventory authoritatively classifies it missing" +# --- how much that missing verdict actually proves -------------------------- +# +# The same `missing` word covers a reachable server that answered and an +# unreachable socket that could not. Only the first proves nothing is running +# at the address, and only the first may recreate the endpoint automatically. + +fm_backend_tmux_missing_grade "$TARGET" +[ "$FM_BACKEND_TMUX_MISSING_GRADE" = strong ] \ + || fail "a missing window in a REACHABLE session should grade strong, got '$FM_BACKEND_TMUX_MISSING_GRADE'" +[ -n "$FM_BACKEND_TMUX_MISSING_SOCKET" ] \ + || fail "a strong missing grade should still name the socket it consulted" + +# A private TMUX_TMPDIR with no server on it is the real unreachable case: tmux +# answers about the socket, never about the window. +unreachable_tmpdir=$(mktemp -d) || fail "could not stage an empty tmux socket dir" +# tmux reports the physically resolved socket path, and the OS temp dir reaches +# it through a symlink on macOS. +unreachable_real=$(cd "$unreachable_tmpdir" && pwd -P) +( + export TMUX_TMPDIR="$unreachable_tmpdir" + unset TMUX + fm_backend_tmux_missing_grade "$TARGET" + [ "$FM_BACKEND_TMUX_MISSING_GRADE" = ambiguous ] \ + || fail "an unreachable tmux socket should grade ambiguous, got '$FM_BACKEND_TMUX_MISSING_GRADE'" + case "$FM_BACKEND_TMUX_MISSING_SOCKET" in + "$unreachable_real"/*) : ;; + *) fail "the ambiguous grade should name the exact socket consulted, got '$FM_BACKEND_TMUX_MISSING_SOCKET'" ;; + esac + [ -n "$FM_BACKEND_TMUX_MISSING_RESPONSE" ] \ + || fail "the ambiguous grade should carry the backend's own response" + state=$(fm_backend_agent_state tmux "$TARGET") + [ "$state" = missing ] \ + || fail "an unreachable socket still classifies missing (that is what the grade exists to qualify), got '$state'" +) || exit 1 +rm -rf "$unreachable_tmpdir" +pass "real tmux: a missing verdict grades strong only when a reachable server answered, and ambiguous when the socket could not be reached" + +# --- recreate the exact missing endpoint ------------------------------------ + +recreated_id=$(fm_backend_tmux_recreate_task "$TARGET" "$HOME") \ + || fail "fm_backend_tmux_recreate_task failed to restore the missing task window" +[ -n "$recreated_id" ] || fail "fm_backend_tmux_recreate_task returned no stable window id" +tmux list-windows -t "$SESSION" -F '#{window_name}' | grep -qx "$WINDOW" \ + || fail "the recreated task window is not visible at its recorded address" +[ "$(tmux display-message -p -t "$recreated_id" '#{pane_current_path}')" = "$HOME" ] \ + || fail "the recreated endpoint did not start in the requested existing worktree" +state=$(fm_backend_agent_state tmux "$TARGET") +[ "$state" = dead ] \ + || fail "the recreated shell endpoint should classify as agent-free, got '$state'" +if fm_backend_tmux_recreate_task "$TARGET" "$HOME" 2>/dev/null; then + fail "fm_backend_tmux_recreate_task should refuse while the endpoint already exists" +fi +pass "real tmux: a missing task endpoint is recreated once at its recorded address and worktree" + +# --- a read that merely SUCCEEDED never grades strong ------------------------ +# +# The liveness verdict and the grade are two separate tmux reads, so the window +# can come back between them (an operator restoring it by hand). The grade must +# account for the exact window in the inventory it read, not treat a successful +# read as proof of absence - otherwise it hands a human an authoritative quote +# about a window that is sitting right there. + +fm_backend_tmux_missing_grade "$TARGET" +[ "$FM_BACKEND_TMUX_MISSING_GRADE" != strong ] \ + || fail "an inventory that LISTS the recorded window must never grade strong" +case "$FM_BACKEND_TMUX_MISSING_RESPONSE" in + *"does not list"*) + fail "the grade quotes tmux as not listing a window the same inventory does list: $FM_BACKEND_TMUX_MISSING_RESPONSE" + ;; +esac +[ -n "$FM_BACKEND_TMUX_MISSING_RESPONSE" ] \ + || fail "a non-strong grade should still carry the response it read" +pass "real tmux: a successful inventory that lists the recorded window is not strong evidence of its absence" + +# --- a recorded session name is never resolved to a look-alike --------------- +# +# tmux resolves a bare `-t ` by exact match, then prefix, then fnmatch. +# "smok" is a prefix of the live "smoke" session, so a bare lookup would report +# smoke's inventory as if it were smok's and graft the replacement window into +# smoke - recovering the task into a session that was never its address. + +PREFIX_SESSION="smok" +PREFIX_WINDOW="fm-smoke2" +PREFIX_TARGET="$PREFIX_SESSION:$PREFIX_WINDOW" + +tmux has-session -t "=$PREFIX_SESSION" 2>/dev/null \ + && fail "the look-alike fixture requires '$PREFIX_SESSION' to be absent" + +state=$(fm_backend_agent_state tmux "$PREFIX_TARGET") +[ "$state" = missing ] \ + || fail "an absent session with a live prefix-sibling should classify missing, got '$state'" +fm_backend_tmux_missing_grade "$PREFIX_TARGET" +case "$FM_BACKEND_TMUX_MISSING_RESPONSE" in + *"does not list"*) + fail "the missing evidence claims a session inventory that only the look-alike '$SESSION' could have answered: $FM_BACKEND_TMUX_MISSING_RESPONSE" + ;; +esac +# A live server that reports the recorded session gone is the ordinary +# killed-session recovery, and the contract recreates it without asking a human. +[ "$FM_BACKEND_TMUX_MISSING_GRADE" = strong ] \ + || fail "a live server reporting the recorded session absent should grade strong, got '$FM_BACKEND_TMUX_MISSING_GRADE'" +[ -n "$FM_BACKEND_TMUX_MISSING_SOCKET" ] \ + || fail "a strong grade from a live server should name the socket it consulted" + +recreated_id=$(fm_backend_tmux_recreate_task "$PREFIX_TARGET" "$HOME") \ + || fail "fm_backend_tmux_recreate_task failed for a session that no longer exists" +tmux has-session -t "=$PREFIX_SESSION" 2>/dev/null \ + || fail "recreating an endpoint whose session is gone should start a session named exactly '$PREFIX_SESSION'" +tmux list-windows -t "=$PREFIX_SESSION" -F '#{window_name}' | grep -qx "$PREFIX_WINDOW" \ + || fail "the replacement window is not in its recorded session" +if tmux list-windows -t "=$SESSION" -F '#{window_name}' | grep -qx "$PREFIX_WINDOW"; then + fail "the replacement window was grafted into the look-alike session '$SESSION'" +fi +[ "$(tmux display-message -p -t "$recreated_id" '#{pane_current_path}')" = "$HOME" ] \ + || fail "the replacement endpoint did not start in the requested worktree" +tmux kill-session -t "=$PREFIX_SESSION" >/dev/null 2>&1 || true +pass "real tmux: a recorded session is matched exactly, so a prefix look-alike never answers for it or absorbs its endpoint" + +# --- the container probe and the windows created in it agree ---------------- +# +# Whatever fm_backend_tmux_container_ensure hands back is immediately targeted +# exactly by fm_backend_tmux_create_task, so its own existence probe has to be +# exact too. A live "firstmate-lab" that merely prefix-matches must not be +# mistaken for the dedicated session: the name would then address nothing and +# every spawn into it would fail. + +tmux new-session -d -s firstmate-lab || fail "could not stage the look-alike container session" +tmux has-session -t "=firstmate" 2>/dev/null \ + && fail "the container fixture requires no session named exactly 'firstmate'" +( + # container-ensure only reaches the dedicated-session branch outside tmux. + unset TMUX + container=$(fm_backend_tmux_container_ensure) \ + || fail "fm_backend_tmux_container_ensure failed while a look-alike session was live" + [ "$container" = firstmate ] \ + || fail "fm_backend_tmux_container_ensure resolved '$container', expected the dedicated 'firstmate'" + tmux has-session -t "=$container" 2>/dev/null \ + || fail "fm_backend_tmux_container_ensure returned '$container' without ensuring that session exists" + fm_backend_tmux_create_task "$container" fm-container1 "$HOME" >/dev/null \ + || fail "a task window could not be created in the container container-ensure resolved" + tmux list-windows -t "=firstmate" -F '#{window_name}' | grep -qx fm-container1 \ + || fail "the task window is not in the dedicated container session" + if tmux list-windows -t "=firstmate-lab" -F '#{window_name}' | grep -qx fm-container1; then + fail "the task window was created in the look-alike session 'firstmate-lab'" + fi +) || exit 1 +pass "real tmux: container-ensure creates and returns the dedicated session a live look-alike would otherwise stand in for" + cleanup_all trap - EXIT diff --git a/tests/fm-control-relaunch.test.sh b/tests/fm-control-relaunch.test.sh index 9a7b4285ba..3e2c472576 100755 --- a/tests/fm-control-relaunch.test.sh +++ b/tests/fm-control-relaunch.test.sh @@ -17,6 +17,12 @@ # 6. fm-spawn --relaunch refuses on its own: a live agent, a contradicting # flag, an extra positional, or a backend that cannot prove the previous # agent exited. +# 7. A missing endpoint reported by a runtime that ANSWERED is recreated in +# the recorded worktree; a missing verdict from an UNREACHABLE runtime +# surfaces a decision that changes nothing until it is confirmed, and that +# confirmation never reaches a live endpoint; ambiguous agent state refuses +# unchanged, a secondmate recovers into its own validated home, and repeat +# recovery never creates a duplicate endpoint. set -u # shellcheck source=tests/lib.sh @@ -96,8 +102,13 @@ case "${1:-}" in fi exit 0 ;; display-message) + if [ -f "$D/unreachable" ]; then + printf 'no server running on %s\n' "$(cat "$D/socket")" >&2 + exit 1 + fi for a in "$@"; do case "$a" in + *socket_path*) cat "$D/socket"; exit 0 ;; *cursor_y*) printf '1\n'; exit 0 ;; *pane_current_command*) cat "$D/command"; printf '\n'; exit 0 ;; *pane_current_path*) @@ -110,7 +121,54 @@ case "${1:-}" in done printf 'fakepane\n'; exit 0 ;; capture-pane) printf '╭────╮\n│ │\n╰────╯\n'; exit 0 ;; - list-windows) [ -f "$D/windows" ] && cat "$D/windows"; exit 0 ;; + list-windows) + # $D/unreachable models a tmux server this process cannot reach at all - + # the socket is not there, so tmux answers about the SOCKET, never about + # the window. Creating a session below starts a server again. + if [ -f "$D/unreachable" ]; then + printf 'no server running on %s\n' "$(cat "$D/socket")" >&2 + exit 1 + fi + [ -f "$D/windows" ] && cat "$D/windows"; exit 0 ;; + has-session) + [ ! -f "$D/unreachable" ] || exit 1 + [ -s "$D/sessions" ]; exit $? ;; + new-session) + rm -f "$D/unreachable" + printf 'new-session\n' >> "$D/lifecycle" + session= + window= + start_dir= + while [ $# -gt 0 ]; do + case "$1" in + -s) shift; session=$1 ;; + -n) shift; window=$1 ;; + -c) shift; start_dir=$1 ;; + esac + shift + done + printf '%s\n' "$session" > "$D/sessions" + printf '%s\n' "$window" > "$D/windows" + # -c is where the endpoint's shell actually opens, so it becomes the pane's + # reported cwd from here on. + if [ -n "$start_dir" ]; then + printf '%s' "$start_dir" > "$D/created-cwd" + printf '%s' "$start_dir" > "$D/cwd" + fi + printf '@replacement\n' + exit 0 ;; + set-window-option) exit 0 ;; + new-window) + printf 'new-window\n' >> "$D/lifecycle" + while [ $# -gt 0 ]; do + case "$1" in + -n) shift; printf '%s\n' "$1" > "$D/windows" ;; + -c) shift; printf '%s' "$1" > "$D/created-cwd"; printf '%s' "$1" > "$D/cwd" ;; + esac + shift + done + printf '@replacement\n' + exit 0 ;; esac exit 0 SH @@ -128,6 +186,10 @@ new_case() { mkdir -p "$dir/home/state" "$dir/home/data" "$dir/fake" : > "$dir/fake/literal" : > "$dir/fake/keys" + : > "$dir/fake/lifecycle" + : > "$dir/fake/sessions" + : > "$dir/fake/created-cwd" + printf '/tmp/fm-fake-tmux/default\n' > "$dir/fake/socket" printf 'claude' > "$dir/fake/command" printf 'claude' > "$dir/fake/becomes" printf '%s\n' "fm-$id" > "$dir/fake/windows" @@ -1312,6 +1374,253 @@ test_spawn_relaunch_refuses_a_pane_outside_the_worktree() { pass "fm-spawn --relaunch: refuses to start a replacement outside the copy holding the work" } +test_missing_endpoint_relaunch_creates_a_fresh_endpoint_without_touching_work() { + local dir out rc before_status before_diff + dir=$(new_case missing-endpoint rl36) + add_ship_task "$dir" rl36 claude + printf 'tracked work in progress\n' >> "$dir/wt/README.md" + printf 'preserve exactly\n' > "$dir/wt/uncommitted.txt" + before_status=$(git -C "$dir/wt" status --porcelain=v1 --untracked-files=all) + before_diff=$(git -C "$dir/wt" diff --binary) + : > "$dir/fake/windows" + + out=$(run_control "$dir" rl36 relaunch --note "runtime wiped the endpoint"); rc=$? + + expect_code 0 "$rc" "an authoritatively missing endpoint should be recreated"$'\n'"$out" + assert_contains "$out" "previous-endpoint=fmses:fm-rl36" \ + "the outcome should distinguish the replaced missing endpoint" + [ "$(meta_field "$dir" rl36 window)" = "fmses:fm-rl36" ] \ + || fail "the durable record should name the fresh endpoint" + [ "$(git -C "$dir/wt" status --porcelain=v1 --untracked-files=all)" = "$before_status" ] \ + || fail "relaunch must preserve the worktree status exactly" + [ "$(git -C "$dir/wt" diff --binary)" = "$before_diff" ] \ + || fail "relaunch must preserve tracked changes exactly" + [ "$(cat "$dir/wt/uncommitted.txt")" = "preserve exactly" ] \ + || fail "relaunch must preserve untracked file contents exactly" + [ "$(cat "$dir/fake/created-cwd")" = "$dir/wt" ] \ + || fail "the fresh endpoint must open in the recorded worktree, got '$(cat "$dir/fake/created-cwd")'" + pass "fm-control relaunch: a missing endpoint is recreated without touching work" +} + +# A secondmate's worktree is its own firstmate home, and that home is resolved +# and validated on its own - the same validation a fresh secondmate spawn uses - +# rather than adopted verbatim from the recorded worktree= value. Recreation +# must honor that carve-out, so the recorded path here reaches the home through +# a symlink: the raw value must reach neither the endpoint the replacement opens +# in nor the record the recovery publishes. +test_secondmate_missing_endpoint_recreation_uses_its_validated_home() { + local dir home out rc validated_home + dir=$(new_case sm-missing-endpoint sm8) + home="$dir/home" + mkdir -p "$home/config" + printf 'claude\n' > "$home/config/secondmate-harness" + fm_git_worktree "$dir/proj" "$dir/smhome" sm-branch + mkdir -p "$dir/smhome/state" "$dir/smhome/data" "$dir/smhome/bin" + printf 'sm8\n' > "$dir/smhome/.fm-secondmate-home" + printf '# charter\n' > "$dir/smhome/data/charter.md" + printf '# agents\n' > "$dir/smhome/AGENTS.md" + printf 'uncommitted secondmate work\n' > "$dir/smhome/in-progress.txt" + ln -s "$dir/smhome" "$dir/smlink" + validated_home=$(cd "$dir/smhome" && pwd -P) + { + echo "window=fmses:fm-sm8" + echo "endpoint_task_id=sm8" + echo "worktree=$dir/smlink" + echo "project=$dir/smlink" + echo "harness=claude" + echo "kind=secondmate" + echo "mode=secondmate" + echo "yolo=off" + echo "model=default" + echo "effort=default" + echo "home=$dir/smlink" + echo "projects=" + } > "$home/state/sm8.meta" + # An empty window inventory from a runtime that answered: the endpoint is + # authoritatively gone, so recovery recreates it. + : > "$dir/fake/windows" + printf '%s' "$dir/smlink" > "$dir/fake/cwd" + + out=$(run_control "$dir" sm8 relaunch); rc=$? + + expect_code 0 "$rc" "a secondmate's missing endpoint should be recreated"$'\n'"$out" + assert_contains "$out" "endpoint-recreated previous-endpoint=fmses:fm-sm8" \ + "the outcome should distinguish the recreated secondmate endpoint" + [ "$(cat "$dir/fake/created-cwd")" = "$validated_home" ] \ + || fail "the recreated endpoint must open in the validated secondmate home, got '$(cat "$dir/fake/created-cwd")'" + [ "$(meta_field "$dir" sm8 worktree)" = "$validated_home" ] \ + || fail "the published record must name the validated home, got '$(meta_field "$dir" sm8 worktree)'" + [ "$(cat "$dir/smhome/in-progress.txt")" = "uncommitted secondmate work" ] \ + || fail "recovery must preserve the secondmate home's uncommitted work exactly" + [ "$(cat "$dir/smhome/data/charter.md")" = "# charter" ] \ + || fail "recovery must never rewrite a secondmate's standing charter" + pass "fm-control relaunch: recreating a secondmate's endpoint honors its validated home over the raw recorded worktree" +} + +test_spawn_recreate_endpoint_requires_the_control_transaction() { + local dir out rc + dir=$(new_case direct-missing-endpoint rl39) + add_ship_task "$dir" rl39 claude + : > "$dir/fake/windows" + + out=$(run_spawn "$dir" rl39 --relaunch --recreate-endpoint --harness claude); rc=$? + + expect_code 1 "$rc" "direct spawn must not acquire missing-endpoint recreation authority" + assert_contains "$out" "authorized only by bin/fm-control.sh" \ + "the refusal should name the control transaction that owns recreation" + [ -z "$(cat "$dir/fake/lifecycle")" ] \ + || fail "a direct spawn must not create a missing endpoint" + pass "fm-spawn --relaunch: only fm-control's locked transaction may recreate an endpoint" +} + +test_ambiguous_endpoint_state_refuses_without_changing_the_record() { + local dir out rc before_meta before_brief + dir=$(new_case ambiguous-endpoint rl37) + add_ship_task "$dir" rl37 claude + printf 'node' > "$dir/fake/command" + before_meta=$(cat "$dir/home/state/rl37.meta") + before_brief=$(cat "$dir/home/data/rl37/brief.md") + + out=$(run_control "$dir" rl37 relaunch --note "must not land"); rc=$? + + expect_code 1 "$rc" "an ambiguous endpoint must fail closed" + assert_contains "$out" "ambiguous" "the refusal should distinguish ambiguity from a missing endpoint" + [ "$(cat "$dir/home/state/rl37.meta")" = "$before_meta" ] \ + || fail "an ambiguous endpoint must preserve the durable record byte-for-byte" + [ "$(cat "$dir/home/data/rl37/brief.md")" = "$before_brief" ] \ + || fail "an ambiguous endpoint must preserve the instructions byte-for-byte" + [ -z "$(cat "$dir/fake/literal")" ] \ + || fail "an ambiguous endpoint must receive no lifecycle input" + [ -z "$(cat "$dir/fake/lifecycle")" ] \ + || fail "an ambiguous endpoint must not create a replacement endpoint" + pass "fm-control relaunch: ambiguous endpoint state fails closed without changing the task" +} + +# A `missing` verdict that came from an UNREACHABLE runtime proves only that +# this process cannot see a server on that socket. Recreating on that evidence +# would put a second agent into a worktree the first one may still be working +# in, so it is a decision for a human rather than an automatic recovery - and +# equally, not a dead end, because a wiped runtime is exactly what needs +# recovering. These three pin both halves of that, plus the guarantee that the +# confirmation can never reach a live endpoint. +test_unreachable_runtime_surfaces_a_decision_instead_of_recreating() { + local dir out rc before_meta before_brief before_status + dir=$(new_case unreachable-endpoint rl40) + add_ship_task "$dir" rl40 claude + printf 'uncommitted work\n' > "$dir/wt/in-progress.txt" + : > "$dir/fake/unreachable" + before_meta=$(cat "$dir/home/state/rl40.meta") + before_brief=$(cat "$dir/home/data/rl40/brief.md") + before_status=$(git -C "$dir/wt" status --porcelain=v1 --untracked-files=all) + + out=$(run_control "$dir" rl40 relaunch --note "the runtime host rebooted"); rc=$? + + expect_code 1 "$rc" "an unreachable runtime must not recreate the endpoint on its own"$'\n'"$out" + assert_contains "$out" "/tmp/fm-fake-tmux/default" \ + "the decision should name the exact socket that was consulted" + assert_contains "$out" "no server running on" \ + "the decision should quote the backend's own response" + assert_contains "$out" "--confirm-endpoint-gone" \ + "the decision should name the explicit human-confirmed continuation" + [ "$(cat "$dir/home/state/rl40.meta")" = "$before_meta" ] \ + || fail "the decision must preserve the durable record byte-for-byte" + [ "$(cat "$dir/home/data/rl40/brief.md")" = "$before_brief" ] \ + || fail "the decision must preserve the instructions byte-for-byte" + [ "$(git -C "$dir/wt" status --porcelain=v1 --untracked-files=all)" = "$before_status" ] \ + || fail "the decision must preserve the worktree exactly" + [ -z "$(cat "$dir/fake/lifecycle")" ] \ + || fail "no endpoint may be created before the decision is made" + [ -z "$(cat "$dir/fake/literal")" ] && [ -z "$(cat "$dir/fake/keys")" ] \ + || fail "no lifecycle input may be delivered before the decision is made" + [ ! -e "$dir/home/state/rl40.control-relaunch" ] \ + || fail "an undecided relaunch must not open a durable transaction journal" + pass "fm-control relaunch: an unreachable runtime surfaces a decision that recreates nothing and changes nothing" +} + +test_confirmed_unreachable_runtime_recovers_the_same_task() { + local dir out rc before_status before_diff + dir=$(new_case confirm-unreachable rl41) + add_ship_task "$dir" rl41 claude + printf 'tracked work in progress\n' >> "$dir/wt/README.md" + printf 'preserve exactly\n' > "$dir/wt/uncommitted.txt" + before_status=$(git -C "$dir/wt" status --porcelain=v1 --untracked-files=all) + before_diff=$(git -C "$dir/wt" diff --binary) + : > "$dir/fake/unreachable" + + out=$(run_control "$dir" rl41 relaunch --note "runtime is gone"); rc=$? + expect_code 1 "$rc" "the undecided run should stop for a decision"$'\n'"$out" + [ -z "$(cat "$dir/fake/lifecycle")" ] || fail "the undecided run must create nothing" + + # Nothing was reconciled in between: the same command, plus the confirmation, + # is the whole continuation. That is what makes the refusal a decision rather + # than a terminal state. + out=$(run_control "$dir" rl41 relaunch --note "runtime is gone" --confirm-endpoint-gone); rc=$? + + expect_code 0 "$rc" "a confirmed unreachable runtime should recover the task"$'\n'"$out" + assert_contains "$out" "endpoint-recreated previous-endpoint=fmses:fm-rl41" \ + "the outcome should distinguish the recreated endpoint" + [ "$(cat "$dir/fake/lifecycle")" = "new-session" ] \ + || fail "confirmed recovery should create exactly one fresh endpoint" + [ "$(meta_field "$dir" rl41 window)" = "fmses:fm-rl41" ] \ + || fail "the durable record should name the recovered endpoint" + [ "$(git -C "$dir/wt" status --porcelain=v1 --untracked-files=all)" = "$before_status" ] \ + || fail "confirmed recovery must preserve the worktree status exactly" + [ "$(git -C "$dir/wt" diff --binary)" = "$before_diff" ] \ + || fail "confirmed recovery must preserve tracked changes exactly" + [ "$(cat "$dir/wt/uncommitted.txt")" = "preserve exactly" ] \ + || fail "confirmed recovery must preserve untracked file contents exactly" + pass "fm-control relaunch: confirming a gone runtime recovers the task without touching its work" +} + +test_confirmation_never_recreates_an_endpoint_that_reads_alive() { + local dir out rc + dir=$(new_case confirm-alive rl42) + add_ship_task "$dir" rl42 claude + + out=$(run_control "$dir" rl42 relaunch --note "carry on" --confirm-endpoint-gone); rc=$? + + expect_code 0 "$rc" "an alive endpoint should still relaunch in place"$'\n'"$out" + assert_contains "$out" "not applied" \ + "the confirmation should be reported as not applied at a live endpoint" + assert_not_contains "$out" "endpoint-recreated" \ + "a live endpoint must never be reported as recreated" + [ -z "$(cat "$dir/fake/lifecycle")" ] \ + || fail "a confirmation must never create an endpoint beside a live agent" + [ "$(meta_field "$dir" rl42 window)" = "fmses:fm-rl42" ] \ + || fail "the live endpoint must be reused, not replaced" + assert_grep "/exit" "$dir/fake/literal" \ + "the live agent must still be stopped by the ordinary relaunch path" + pass "fm-control relaunch: a human confirmation cannot recreate an endpoint that reads alive" +} + +test_missing_endpoint_relaunch_is_repeat_safe() { + local dir first second rc + dir=$(new_case repeat-missing-endpoint rl38) + add_ship_task "$dir" rl38 claude + : > "$dir/fake/windows" + + first=$(run_control "$dir" rl38 relaunch --note "first recovery"); rc=$? + expect_code 0 "$rc" "the first recovery should recreate the missing endpoint"$'\n'"$first" + second=$(run_control "$dir" rl38 relaunch --note "repeat recovery"); rc=$? + expect_code 0 "$rc" "a repeated relaunch should replace the running worker without duplicating it"$'\n'"$second" + + [ "$(cat "$dir/fake/windows")" = "fm-rl38" ] \ + || fail "repeat relaunch must leave exactly one endpoint for the task" + [ "$(cat "$dir/fake/lifecycle")" = "new-session" ] \ + || fail "repeat relaunch must create a fresh endpoint only for the missing incarnation" + [ "$(meta_field "$dir" rl38 window)" = "fmses:fm-rl38" ] \ + || fail "repeat relaunch must retain the replacement endpoint identity" + pass "fm-control relaunch: repeating recovery never creates a duplicate worker endpoint" +} + +test_ambiguous_endpoint_state_refuses_without_changing_the_record +test_spawn_recreate_endpoint_requires_the_control_transaction +test_missing_endpoint_relaunch_creates_a_fresh_endpoint_without_touching_work +test_secondmate_missing_endpoint_recreation_uses_its_validated_home +test_unreachable_runtime_surfaces_a_decision_instead_of_recreating +test_confirmed_unreachable_runtime_recovers_the_same_task +test_confirmation_never_recreates_an_endpoint_that_reads_alive +test_missing_endpoint_relaunch_is_repeat_safe test_same_harness_relaunch_keeps_identity_and_reuses_the_endpoint test_relaunch_preserves_durable_task_metadata test_relaunch_serializes_concurrent_durable_metadata_publication