From c86b81a5d6052e361daa800ef9eccd5c716d04b9 Mon Sep 17 00:00:00 2001 From: Nikita Guzenko Date: Sat, 15 Aug 2026 16:26:54 +0000 Subject: [PATCH] feat(bin): add session-independent fleet liveness guardian Add bin/fm-liveness-guardian.sh, a systemd-user-timer supervision backstop that runs below every session and keeps each firstmate home's watcher alive so tasks keep being handled when a director's session stops re-arming. Per guarded home (main + local secondmates, discovered from the main home's kind=secondmate meta records) it classifies supervision with the fleet's own model-aware predicates and acts only on a genuine lapse: - healthy/idle: no-op (never double-arms). - session-less lapsed home: re-arm a real watcher via a self-surviving systemd-run --user transient unit (never a fire-and-forget shell &), which restores the beacon and captures wakes durably. - dead local secondmate: relaunch the director through the sanctioned fm-spawn --secondmate path, rate-limited, escalate on a crash loop. - live-but-lapsed session: never re-arm (that would mask a wedge and defeat the confirmation clock) and never relaunch a live session; escalate a likely wedge only after it stays lapsed past a threshold longer than any legitimate turn. - remote secondmate: reported and skipped. It exports XDG_RUNTIME_DIR/DBUS_SESSION_BUS_ADDRESS before any systemctl --user call, uses absolute paths, runs cleanly under env -i, never touches the shared no-mistakes daemon, and logs one machine-scannable line per real event (silent when healthy). It deliberately does not put directors into unattended daemon mode, which is a fleet-behavior change left to the captain. Ships systemd .service/.timer templates under dist/systemd/, a colocated test exercising every classification under env -i against disposable mock homes only, and docs/liveness-guardian.md plus scripts.md and watcher-continuity.md pointers. --- bin/fm-liveness-guardian.sh | 536 ++++++++++++++++++++++ dist/systemd/fm-liveness-guardian.service | 24 + dist/systemd/fm-liveness-guardian.timer | 18 + docs/documentation-audiences.json | 4 + docs/liveness-guardian.md | 81 ++++ docs/scripts.md | 1 + docs/watcher-continuity.md | 1 + tests/fm-liveness-guardian.test.sh | 291 ++++++++++++ 8 files changed, 956 insertions(+) create mode 100755 bin/fm-liveness-guardian.sh create mode 100644 dist/systemd/fm-liveness-guardian.service create mode 100644 dist/systemd/fm-liveness-guardian.timer create mode 100644 docs/liveness-guardian.md create mode 100755 tests/fm-liveness-guardian.test.sh diff --git a/bin/fm-liveness-guardian.sh b/bin/fm-liveness-guardian.sh new file mode 100755 index 0000000000..ee708cf374 --- /dev/null +++ b/bin/fm-liveness-guardian.sh @@ -0,0 +1,536 @@ +#!/usr/bin/env bash +# fm-liveness-guardian.sh - session-independent fleet supervision backstop. +# +# Every firstmate home keeps its own watcher (bin/fm-watch.sh) alive through a +# harness-owned continuity mechanism: Claude's Stop auto-arm, Cursor's stop park, +# Pi/OpenCode extensions, or a persistent tracked arm (docs/watcher-continuity.md). +# Each of those depends on a LIVE session in that home. When a director's session +# stops taking turns while its armed watcher dies (observed 2026-08-15: several +# homes' beacons stale for thousands of seconds while the panes stayed open), the +# harness has no Stop to re-arm on, so supervision stays down and incoming wakes +# (new Telegram-routed tasks, crew status) are never surfaced or handled until a +# human notices. This guardian is the layer BELOW every session: it runs from a +# systemd user timer with no session in the loop, and for each guarded home it +# classifies supervision and repairs a genuine lapse through the SANCTIONED paths +# only - never by faking a beacon, never with a fire-and-forget shell `&`. +# +# What it does NOT do, on purpose: +# - It never touches state/.last-watcher-beat directly. A fresh beacon is +# produced ONLY by launching a real watcher (docs/watcher-continuity.md's +# invariant "no helper process can make a wedged watcher appear healthy"). +# - It never double-arms: a home whose model-aware verdict is already healthy +# is left completely alone, and the watcher singleton lock plus fm-watch-arm's +# own attach make a concurrent arm converge on one watcher. +# - It never kills live crew, discards unlanded work, force-tears-down, or +# touches the shared no-mistakes daemon. Its only repair verbs are re-arm +# (launch a real watcher self-survivingly), relaunch a genuinely dead LOCAL +# secondmate director through the sanctioned spawn path, and escalate. +# - It does not put directors into unattended sub-supervisor daemon mode. That +# is the only design that makes wakes SELF-HANDLE with zero session, and it is +# a fleet-behavior change (the /afk daemon; AGENTS.md section 8). This guardian +# deliberately leaves that choice to the captain (docs/liveness-guardian.md +# "Boundary and the unattended-handling decision"). +# +# Repair is HANDLING-honest, not just beacon-honest. Re-arming restores the +# beacon and captures every wake durably in state/.wake-queue (fm-watch.sh +# enqueues before advancing suppression), so nothing is lost; but a surfaced wake +# is HANDLED by the home's own live session. So the guardian relaunches a dead +# local director (restoring a handler) and escalates a home it cannot restore +# rather than masking it behind a fresh beacon. +# +# systemd-timer environment. Runs under `env -i` from the timer with a bare +# environment, so it exports XDG_RUNTIME_DIR and DBUS_SESSION_BUS_ADDRESS before +# any `systemctl --user`/`systemd-run --user` call (see +# ~/.claude/rules/shell-failure-patterns.md #9), sets FM_HOME per guarded home, +# and uses absolute paths throughout. +# +# Usage: +# fm-liveness-guardian.sh one repair pass over every guarded home +# fm-liveness-guardian.sh --list classify every home and print the plan; +# repairs nothing (safe dry run) +# fm-liveness-guardian.sh --home guard exactly one home (repair pass) +# fm-liveness-guardian.sh --help +# +# Homes to guard: the MAIN home (FM_GUARDIAN_MAIN_HOME, else FM_HOME) plus every +# LOCAL secondmate discovered from the main home's state/*.meta records with +# kind=secondmate (their recorded home=), with data/secondmates.md as the fallback +# for a missing home=. Remote secondmates (remote_host set) are reported and +# skipped: a local timer cannot repair another host. FM_GUARDIAN_HOMES (a +# colon- or newline-separated list of absolute home paths) overrides discovery. +# +# Environment knobs (all optional): +# FM_GUARDIAN_MAIN_HOME main home to guard and to discover secondmates from +# (default: FM_HOME, else error - it cannot be guessed) +# FM_GUARDIAN_HOMES explicit ':'/newline home list; overrides discovery +# FM_GUARDIAN_GRACE beacon-staleness threshold, seconds (default: the same +# FM_GUARD_GRACE the fleet uses, 300) +# FM_GUARDIAN_RELAUNCH_MIN_INTERVAL min seconds between relaunches of one home +# (crash-loop backstop; default 900) +# FM_GUARDIAN_ESCALATE_MIN_INTERVAL min seconds between escalations of one home +# (default 1800) +# FM_GUARDIAN_WEDGE_ESCALATE_SECS how long a live session must stay +# continuously unsupervised before it is escalated as a +# possible wedge (default 1800; larger than any +# legitimate long turn, so a busy session is never +# mislabeled) +# FM_GUARDIAN_LOG append-only log path (default +# $XDG_STATE_HOME/fm-liveness-guardian.log, i.e. +# ~/.local/state/fm-liveness-guardian.log) +# FM_GUARDIAN_STATE_DIR guardian's own rate-limit/confirmation state dir +# (default $XDG_STATE_HOME/fm-liveness-guardian) +# FM_GUARDIAN_MAIN_MODEL supervision model for the main home when it cannot be +# read from durable state (default: autoarm - the safe +# choice: fresh beacon = healthy, so it never disturbs a +# healthy home and only ever acts on a genuinely stale +# beacon) +# FM_GUARDIAN_ARM_LAUNCHER test/override seam. A command invoked as +# ` ` that must +# launch (with FM_HOME=) in a way +# that SURVIVES this process. Default: an internal +# `systemd-run --user` transient unit named . +# FM_GUARDIAN_RELAUNCH_CMD test/override seam for the dead-director relaunch. +# Invoked as ` `. +# Default: the sanctioned `
/bin/fm-spawn.sh +# --secondmate` (the same relaunch bootstrap uses). +# FM_GUARDIAN_ESCALATE_CMD test/override seam for escalation. Invoked as +# ` `. Default: append a captain +# signal line to the main home's backlog-adjacent escalation +# log and the guardian log (a human path, never a merge). +# FM_GUARDIAN_DRY_RUN 1 = classify and PRINT the planned repair commands +# (including the exact self-surviving systemd-run) but +# execute none. --list implies this. +set -u + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# The guardian's own predicates come from its own tracked libs; each is called +# with an explicit per-home / argument, so guarding a home never +# depends on the guardian's own FM_HOME. +FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" +# Resolve the main home BEFORE sourcing the libs (which default FM_HOME/STATE and +# mkdir the state dir at source time). Pointing the guardian's own lib state at +# the main home keeps sourcing from creating a stray state/ in the repo checkout. +GUARDIAN_MAIN="${FM_GUARDIAN_MAIN_HOME:-${FM_HOME:-}}" +export FM_HOME="${GUARDIAN_MAIN:-$FM_ROOT}" +# shellcheck source=bin/fm-wake-lib.sh +. "$SCRIPT_DIR/fm-wake-lib.sh" +# shellcheck source=bin/fm-supervision-lib.sh +. "$SCRIPT_DIR/fm-supervision-lib.sh" +# shellcheck source=bin/fm-session-lock-lib.sh +. "$SCRIPT_DIR/fm-session-lock-lib.sh" +# shellcheck source=bin/fm-secondmate-registry-lib.sh +. "$SCRIPT_DIR/fm-secondmate-registry-lib.sh" + +GRACE=${FM_GUARDIAN_GRACE:-${FM_GUARD_GRACE:-300}} +case "$GRACE" in ''|*[!0-9]*) GRACE=300 ;; esac +RELAUNCH_MIN_INTERVAL=${FM_GUARDIAN_RELAUNCH_MIN_INTERVAL:-900} +case "$RELAUNCH_MIN_INTERVAL" in ''|*[!0-9]*) RELAUNCH_MIN_INTERVAL=900 ;; esac +ESCALATE_MIN_INTERVAL=${FM_GUARDIAN_ESCALATE_MIN_INTERVAL:-1800} +case "$ESCALATE_MIN_INTERVAL" in ''|*[!0-9]*) ESCALATE_MIN_INTERVAL=1800 ;; esac +# How long a live session must stay continuously unsupervised before the guardian +# calls it a wedge. Larger than any legitimate long turn (a slow build, a +# no-mistakes run), so a busy session is never mislabeled a wedge. +WEDGE_ESCALATE_SECS=${FM_GUARDIAN_WEDGE_ESCALATE_SECS:-1800} +case "$WEDGE_ESCALATE_SECS" in ''|*[!0-9]*) WEDGE_ESCALATE_SECS=1800 ;; esac +MAIN_MODEL=${FM_GUARDIAN_MAIN_MODEL:-autoarm} + +XDG_STATE_HOME_DEFAULT="${XDG_STATE_HOME:-$HOME/.local/state}" +LOG=${FM_GUARDIAN_LOG:-$XDG_STATE_HOME_DEFAULT/fm-liveness-guardian.log} +GSTATE=${FM_GUARDIAN_STATE_DIR:-$XDG_STATE_HOME_DEFAULT/fm-liveness-guardian} + +DRY_RUN=0 +case "${FM_GUARDIAN_DRY_RUN:-0}" in 1|true|TRUE|yes|YES) DRY_RUN=1 ;; esac + +# --- systemd user-bus environment (bare env from the timer) ------------------- +# Harmless when already set from an interactive shell; required from cron/timer. +export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" +export DBUS_SESSION_BUS_ADDRESS="${DBUS_SESSION_BUS_ADDRESS:-unix:path=$XDG_RUNTIME_DIR/bus}" + +now() { date +%s; } + +# One machine-scannable, human-scannable line per real event. Silent for healthy +# and idle homes: a quiet log means a healthy fleet. +log_line() { # + local level=$1 home=$2 class=$3 action=$4 detail=$5 ts + ts=$(date '+%Y-%m-%dT%H:%M:%S%z' 2>/dev/null || now) + mkdir -p "$(dirname "$LOG")" 2>/dev/null || true + printf '%s\t%s\thome=%s\tclass=%s\taction=%s\t%s\n' \ + "$ts" "$level" "$home" "$class" "$action" "$detail" >> "$LOG" 2>/dev/null || true +} + +# Stable, filesystem- and unit-name-safe key for a home path. +home_key() { # + printf 'fm-fleet-%s' "$(printf '%s' "$1" | cksum | tr -cd '0-9')" +} + +# --- rate limiting ------------------------------------------------------------ +# A stamp file per (home,kind) records the last action time. rate_ok returns 0 +# when at least seconds have elapsed since the last stamp (or none exists). +rate_stamp_path() { # + printf '%s/%s.%s' "$GSTATE" "$(home_key "$1")" "$2" +} + +rate_ok() { # + local stamp last age + stamp=$(rate_stamp_path "$1" "$2") + [ -f "$stamp" ] || return 0 + last=$(cat "$stamp" 2>/dev/null || echo 0) + case "$last" in ''|*[!0-9]*) return 0 ;; esac + age=$(( $(now) - last )) + [ "$age" -ge "$3" ] +} + +rate_touch() { # + local stamp + stamp=$(rate_stamp_path "$1" "$2") + mkdir -p "$GSTATE" 2>/dev/null || true + printf '%s\n' "$(now)" > "$stamp" 2>/dev/null || true +} + +rate_clear() { # + rm -f "$(rate_stamp_path "$1" "$2")" 2>/dev/null || true +} + +# Age of a stamp in seconds, or a large sentinel when it does not exist (so a +# first observation never reads as "long ago"). +stamp_age() { # + local stamp last + stamp=$(rate_stamp_path "$1" "$2") + [ -f "$stamp" ] || { echo -1; return; } + last=$(cat "$stamp" 2>/dev/null || echo 0) + case "$last" in ''|*[!0-9]*) echo -1; return ;; esac + echo $(( $(now) - last )) +} + +# --- home discovery ----------------------------------------------------------- +# Enumerate absolute home paths, one per line, deduplicated in first-seen order. +# Each line is "\t" where role is main|secondmate|remote:. +discover_homes() { + local main data meta id home host seen + seen="" + + if [ -n "${FM_GUARDIAN_HOMES:-}" ]; then + printf '%s\n' "$FM_GUARDIAN_HOMES" | tr ':' '\n' | while IFS= read -r home; do + [ -n "$home" ] || continue + printf '%s\tmain\n' "$home" + done + return 0 + fi + + main=$GUARDIAN_MAIN + if [ -z "$main" ]; then + echo "error: no home to guard - set FM_GUARDIAN_MAIN_HOME, FM_HOME, or FM_GUARDIAN_HOMES" >&2 + return 2 + fi + printf '%s\tmain\n' "$main" + seen="|$main|" + + data="$main/data" + # Live homes come from the main home's kind=secondmate meta records; the + # registry is only a fallback for a missing home= (fm-config-push.sh contract). + for meta in "$main"/state/*.meta; do + [ -e "$meta" ] || continue + grep -qx 'kind=secondmate' "$meta" 2>/dev/null || continue + id=$(basename "$meta" .meta) + host=$(sed -n 's/^remote_host=//p' "$meta" 2>/dev/null | head -1) + home=$(sed -n 's/^home=//p' "$meta" 2>/dev/null | head -1) + if [ -z "$home" ] && [ -f "$data/secondmates.md" ]; then + home=$(secondmate_registry_field "$data/secondmates.md" "$id" home 2>/dev/null || true) + fi + [ -n "$home" ] || continue + case "$seen" in *"|$home|"*) continue ;; esac + seen="$seen$home|" + if [ -n "$host" ]; then + printf '%s\tremote:%s\n' "$home" "$host" + else + printf '%s\tsecondmate:%s\n' "$home" "$id" + fi + done +} + +# --- per-home supervision model ---------------------------------------------- +# Resolve the supervision model for a home from durable evidence, so the +# guardian's verdict matches how that home is actually supervised even though the +# guardian itself runs under no harness. +resolve_model() { # + local home=$1 role=$2 harness lock_role meta id + # A secondmate records its launch harness in the main home's meta. + case "$role" in + secondmate:*) + id=${role#secondmate:} + meta="$GUARDIAN_MAIN/state/$id.meta" + harness=$(sed -n 's/^harness=//p' "$meta" 2>/dev/null | head -1) + ;; + esac + if [ -z "${harness:-}" ]; then + # The last watcher's lock role is durable evidence of the auto-arm model. + lock_role=$(cat "$home/state/.watch.lock/role" 2>/dev/null || true) + [ "$lock_role" = autoarm ] && { printf 'autoarm\n'; return 0; } + fi + case "${harness:-}" in + claude|cursor) printf 'autoarm\n'; return 0 ;; + pi|pi-signed) printf 'extension\n'; return 0 ;; + '') ;; + *) printf 'persistent\n'; return 0 ;; + esac + # Main home (or unresolved): the configured safe default. + printf '%s\n' "$MAIN_MODEL" +} + +# --- session liveness --------------------------------------------------------- +# A home's director session is alive when its session lock (state/.lock) names a +# live harness process. env-i friendly: only ps/kill, no backend needed. +session_alive() { # + local lock_pid + lock_pid=$(cat "$1/state/.lock" 2>/dev/null || true) + case "$lock_pid" in ''|*[!0-9]*) return 1 ;; esac + fm_harness_pid_alive "$lock_pid" +} + +# --- repair primitives -------------------------------------------------------- +# Re-arm a real watcher for through a SELF-SURVIVING systemd transient +# unit, never a shell `&`. Idempotent: a still-active unit means an arm is +# already running, and fm-watch-arm.sh attaches to an already-healthy watcher +# instead of starting a second one. +systemd_run_arm() { # + local unit=$1 home=$2 arm=$3 + if [ "$DRY_RUN" -eq 1 ]; then + printf 'DRY-RUN arm: systemd-run --user --unit=%s --property=CollectMode=inactive-or-failed --setenv=FM_HOME=%s %s\n' \ + "$unit" "$home" "$arm" + return 0 + fi + if ! command -v systemd-run >/dev/null 2>&1; then + echo "systemd-run unavailable" >&2 + return 1 + fi + # Clear any lingering failed unit of this name so the transient can be reused. + systemctl --user reset-failed "$unit" >/dev/null 2>&1 || true + # An already-active unit (systemd-run exits non-zero on name collision) means a + # watcher is already armed for this home - treat as success, do not double-arm. + if systemctl --user is-active "$unit" >/dev/null 2>&1; then + return 0 + fi + systemd-run --user \ + --unit="$unit" \ + --property=CollectMode=inactive-or-failed \ + --property=Restart=no \ + --setenv=FM_HOME="$home" \ + --setenv=FM_GUARD_GRACE="$GRACE" \ + "$arm" >/dev/null 2>&1 +} + +arm_watcher() { # + local home=$1 arm unit + arm="$home/bin/fm-watch-arm.sh" + [ -x "$arm" ] || arm="$SCRIPT_DIR/fm-watch-arm.sh" + unit=$(home_key "$home") + # A dry run only ever PRINTS the plan, never consulting a live launcher seam. + if [ "$DRY_RUN" -eq 0 ] && [ -n "${FM_GUARDIAN_ARM_LAUNCHER:-}" ]; then + "$FM_GUARDIAN_ARM_LAUNCHER" "$unit" "$home" "$arm" + return $? + fi + systemd_run_arm "$unit" "$home" "$arm" +} + +# Relaunch a genuinely dead LOCAL secondmate director through the sanctioned +# spawn path (the same relaunch the bootstrap secondmate-liveness sweep uses), +# never by discarding its unlanded work. +relaunch_director() { # + local home=$1 id=$2 main=$3 cmd + if [ "$DRY_RUN" -eq 1 ]; then + printf 'DRY-RUN relaunch: %s/bin/fm-spawn.sh %s --secondmate (FM_HOME=%s)\n' "$main" "$id" "$main" + return 0 + fi + if [ -n "${FM_GUARDIAN_RELAUNCH_CMD:-}" ]; then + "$FM_GUARDIAN_RELAUNCH_CMD" "$home" "$id" "$main" + return $? + fi + cmd="$main/bin/fm-spawn.sh" + [ -x "$cmd" ] || { echo "spawn script missing: $cmd" >&2; return 1; } + FM_HOME="$main" "$cmd" "$id" --secondmate >/dev/null 2>&1 +} + +# Escalate to the captain: a human path (log ERROR + optional signal command), +# never a merge or any irreversible action. +escalate() { # + local home=$1 summary=$2 + if [ -n "${FM_GUARDIAN_ESCALATE_CMD:-}" ] && [ "$DRY_RUN" -eq 0 ]; then + "$FM_GUARDIAN_ESCALATE_CMD" "$home" "$summary" || true + fi + log_line ERROR "$home" - escalate "detail=$summary" +} + +# --- classification + repair -------------------------------------------------- +# Returns via stdout one classification token for --list; performs repair in a +# normal pass. +guard_home() { + local home=$1 role=$2 state watch model verdict_ok verdict_reason id detail + + state="$home/state" + watch="$home/bin/fm-watch.sh" + + # A remote secondmate's state lives on another host, so this check precedes the + # local-state-dir check: a local timer cannot repair another host. + case "$role" in + remote:*) + printf 'home=%s\tclass=remote-skip\taction=none\treason=remote-host-%s\n' "$home" "${role#remote:}" + return 0 + ;; + esac + + if [ ! -d "$state" ]; then + printf 'home=%s\tclass=skip\taction=none\treason=no-state-dir\n' "$home" + return 0 + fi + + # Does this home need a watcher at all? (in-flight meta / x-watch / event source) + if ! fm_supervision_needed "$state" "$GRACE"; then + [ "$DRY_RUN" -eq 1 ] || rate_clear "$home" lapse-since + printf 'home=%s\tclass=idle\taction=none\treason=no-supervision-needed\n' "$home" + return 0 + fi + + # Model-aware "is supervision healthy right now" verdict, using the home's own + # model so a between-turns auto-arm home reads healthy while a genuinely stale + # beacon reads as a lapse. + model=$(resolve_model "$home" "$role") + FM_SUPERVISION_MODEL="$model" fm_watcher_supervision_verdict "$state" "$watch" "$GRACE" "$home" "$home" + verdict_ok=$FM_WATCHER_VERDICT_OK + verdict_reason=$FM_WATCHER_VERDICT_REASON + + if [ "$verdict_ok" = true ]; then + # Recovery resets the wedge-confirmation clock for the next episode. + [ "$DRY_RUN" -eq 1 ] || rate_clear "$home" lapse-since + printf 'home=%s\tclass=healthy\taction=none\treason=watcher-fresh-model-%s\n' "$home" "$model" + return 0 + fi + + # Genuinely lapsed. Decide the repair by whether a director session is alive. + if session_alive "$home"; then + # A live session whose supervision lapsed is EITHER a legitimate long turn + # (a slow build, a no-mistakes run) that will re-arm itself at its next turn + # boundary, OR an asleep/wedged session whose watcher died with no Stop left to + # re-arm on. The guardian deliberately does NOT re-arm here: a guardian-owned + # watcher would freshen the beacon and mask the wedge (and defeat the very + # confirmation clock below), while re-arm gives an asleep session no handler + # anyway - only its own harness or a relaunch can. Wakes are not lost: their + # signal/status files persist and a watcher scans them once the session resumes + # or is relaunched. So the guardian starts a confirmation clock and, only when a + # live session stays continuously unsupervised past WEDGE_ESCALATE_SECS (longer + # than any legitimate long turn), escalates it as a likely wedge rather than + # relaunching a live session out from under in-flight work. + if [ "$DRY_RUN" -eq 1 ]; then + printf 'home=%s\tclass=lapsed\taction=confirm-wedge\treason=%s-model-%s-session-alive\n' "$home" "$verdict_reason" "$model" + return 0 + fi + if [ "$(stamp_age "$home" lapse-since)" -lt 0 ]; then + rate_touch "$home" lapse-since + log_line INFO "$home" lapsed confirm-wedge "reason=$verdict_reason model=$model session=alive note=clock-started" + elif [ "$(stamp_age "$home" lapse-since)" -ge "$WEDGE_ESCALATE_SECS" ] \ + && rate_ok "$home" escalate "$ESCALATE_MIN_INTERVAL"; then + escalate "$home" "director session alive but supervision has stayed lapsed for over ${WEDGE_ESCALATE_SECS}s (reason=$verdict_reason); likely wedged - nudge or relaunch it" + rate_touch "$home" escalate + fi + return 0 + fi + + # Session is DEAD. Clear any stale lapse-since confirmation from a prior live + # episode so it does not carry into this dead one. + [ "$DRY_RUN" -eq 1 ] || rate_clear "$home" lapse-since + + case "$role" in + secondmate:*) + id=${role#secondmate:} + if [ "$DRY_RUN" -eq 1 ]; then + printf 'home=%s\tclass=dead\taction=relaunch\treason=%s-session-dead-id-%s\n' "$home" "$verdict_reason" "$id" + relaunch_director "$home" "$id" "$GUARDIAN_MAIN" | sed "s#^# #" + return 0 + fi + # Crash-loop backstop: do not hammer a home that keeps dying. Escalate instead. + if ! rate_ok "$home" relaunch "$RELAUNCH_MIN_INTERVAL"; then + if rate_ok "$home" escalate "$ESCALATE_MIN_INTERVAL"; then + escalate "$home" "dead director $id relaunched too recently (< ${RELAUNCH_MIN_INTERVAL}s); not relaunching again - investigate the crash loop" + rate_touch "$home" escalate + fi + log_line WARN "$home" dead relaunch-suppressed "id=$id reason=rate-limited" + return 0 + fi + rate_touch "$home" relaunch + if relaunch_director "$home" "$id" "$GUARDIAN_MAIN"; then + log_line INFO "$home" dead relaunch "id=$id reason=$verdict_reason" + else + escalate "$home" "sanctioned relaunch of dead director $id FAILED (reason=$verdict_reason)" + fi + ;; + *) + # Main home with no live session: the guardian must not spawn a competing + # firstmate (a fleet-behavior/architecture choice). Capture wakes durably by + # re-arming a watcher, and escalate so a human starts a session. + detail="reason=$verdict_reason" + if [ "$DRY_RUN" -eq 1 ]; then + printf 'home=%s\tclass=dead\taction=rearm+escalate\treason=%s-main-no-session\n' "$home" "$verdict_reason" + arm_watcher "$home" | sed "s#^# #" + return 0 + fi + if arm_watcher "$home"; then + log_line INFO "$home" dead rearm "$detail note=main-home-no-session-wakes-captured-durably" + rate_touch "$home" rearm + fi + if rate_ok "$home" escalate "$ESCALATE_MIN_INTERVAL"; then + escalate "$home" "main firstmate home needs supervision but no session is running; start a session to handle queued work ($detail)" + rate_touch "$home" escalate + fi + ;; + esac +} + +usage() { + cat <<'EOF' +fm-liveness-guardian.sh - session-independent fleet supervision backstop. + +Runs from a systemd user timer. For each guarded home (main + local secondmates) +it classifies supervision and repairs a genuine lapse through sanctioned paths +only: re-arm a real watcher self-survivingly, relaunch a dead local director, or +escalate. It never fakes a beacon, never double-arms, never disturbs in-flight +work, and never touches the no-mistakes daemon. + +usage: + fm-liveness-guardian.sh one repair pass over every guarded home + fm-liveness-guardian.sh --list classify + print the plan; repairs nothing + fm-liveness-guardian.sh --home guard exactly one home (repair pass) + fm-liveness-guardian.sh --help + +Homes: the main home (FM_GUARDIAN_MAIN_HOME, else FM_HOME) plus every local +secondmate discovered from the main home's kind=secondmate meta records. Remote +secondmates are reported and skipped. FM_GUARDIAN_HOMES overrides discovery. + +See the script header and docs/liveness-guardian.md for the full environment +knobs, the classification matrix, and the install step. +EOF +} + +main() { + local one_home="" + while [ "$#" -gt 0 ]; do + case "$1" in + -h|--help) usage; exit 0 ;; + --list|--dry-run) DRY_RUN=1 ;; + --home) shift; one_home=${1:-} ; [ -n "$one_home" ] || { echo "error: --home needs a path" >&2; exit 2; } ;; + *) echo "error: unknown argument: $1 (valid: --list, --home , --help)" >&2; exit 2 ;; + esac + shift + done + + if [ -n "$one_home" ]; then + guard_home "$one_home" main + return 0 + fi + + local homes rc + homes=$(discover_homes) || { rc=$?; return "$rc"; } + printf '%s\n' "$homes" | while IFS=$'\t' read -r home role; do + [ -n "$home" ] || continue + guard_home "$home" "$role" + done +} + +main "$@" diff --git a/dist/systemd/fm-liveness-guardian.service b/dist/systemd/fm-liveness-guardian.service new file mode 100644 index 0000000000..af75e85340 --- /dev/null +++ b/dist/systemd/fm-liveness-guardian.service @@ -0,0 +1,24 @@ +# Firstmate fleet liveness guardian - systemd USER service template. +# +# Install: see docs/liveness-guardian.md "Install". Copy this file and the timer +# into ~/.config/systemd/user/, edit FM_GUARDIAN_MAIN_HOME below if this box's +# main firstmate home is not ~/firstmate, then enable the .timer (never this +# .service directly - the timer drives it). +# +# This is a short read-mostly oneshot. When it finds a genuinely lapsed home it +# re-arms a real watcher through its OWN transient `systemd-run --user` unit, +# which the user manager owns independently, so that watcher survives this +# oneshot exiting. This service never touches the shared no-mistakes daemon. +[Unit] +Description=Firstmate fleet liveness guardian (session-independent supervision backstop) +Documentation=file:%h/firstmate/docs/liveness-guardian.md + +[Service] +Type=oneshot +# Edit this to the main firstmate home on this box if it is not ~/firstmate. +Environment=FM_GUARDIAN_MAIN_HOME=%h/firstmate +# Absolute path to the guardian; adjust if the repo lives elsewhere. +ExecStart=%h/firstmate/bin/fm-liveness-guardian.sh +TimeoutStartSec=120 +# A failing pass must be loud in the journal, not silently swallowed. +SuccessExitStatus=0 diff --git a/dist/systemd/fm-liveness-guardian.timer b/dist/systemd/fm-liveness-guardian.timer new file mode 100644 index 0000000000..f5af34ef5f --- /dev/null +++ b/dist/systemd/fm-liveness-guardian.timer @@ -0,0 +1,18 @@ +# Firstmate fleet liveness guardian - systemd USER timer template. +# +# Drives fm-liveness-guardian.service on a short cadence so a lapsed home is +# repaired within roughly one interval and the re-arm gap after any wake stays +# bounded. Install and enable per docs/liveness-guardian.md "Install". +[Unit] +Description=Run the Firstmate fleet liveness guardian periodically + +[Timer] +# First pass shortly after boot/login, then every 2 minutes. +OnBootSec=2min +OnUnitActiveSec=2min +AccuracySec=15s +# Catch up one missed pass after the machine was asleep/off. +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/docs/documentation-audiences.json b/docs/documentation-audiences.json index 64dea78dc6..459610e076 100644 --- a/docs/documentation-audiences.json +++ b/docs/documentation-audiences.json @@ -368,6 +368,10 @@ "path": "docs/watcher-continuity.md", "audience": "operator-current" }, + { + "path": "docs/liveness-guardian.md", + "audience": "operator-current" + }, { "path": "docs/wedge-alarm.md", "audience": "operator-current" diff --git a/docs/liveness-guardian.md b/docs/liveness-guardian.md new file mode 100644 index 0000000000..419d24f2bb --- /dev/null +++ b/docs/liveness-guardian.md @@ -0,0 +1,81 @@ +# Fleet liveness guardian + +`bin/fm-liveness-guardian.sh` is a session-independent supervision backstop that runs from a systemd user timer, below every session, and keeps each firstmate home's watcher alive so tasks keep being handled without a human noticing a lapse. + +Read the script header and `bin/fm-liveness-guardian.sh --help` for the exact flags, environment knobs, and paths. +This page owns the mechanism, the classification matrix, the boundary, the install step, and how to verify it live. + +## Why it exists + +Each home's watcher (`bin/fm-watch.sh`) is kept armed by a harness-owned continuity mechanism that depends on a live session in that home: Claude's Stop auto-arm, Cursor's stop park, the Pi/OpenCode extensions, or a persistent tracked arm ([`watcher-continuity.md`](watcher-continuity.md)). +When a director's session stops taking turns while its armed watcher dies, the harness has no Stop to re-arm on, so supervision stays down: the beacon (`state/.last-watcher-beat`) goes stale far past `FM_GUARD_GRACE`, and incoming wakes such as new Telegram-routed tasks or crew status are never surfaced until a human notices. +This was observed live on 2026-08-15 with several homes' beacons stale for thousands of seconds while their panes stayed open. +The guardian is the layer that repairs that with no session in the loop. + +## What it does per home + +For each guarded home the guardian classifies supervision with the fleet's own model-aware predicates and acts only on a genuine lapse. + +| Home state | Classification | Action | +| --- | --- | --- | +| No in-flight work, no relay poll, no event source | `idle` | none | +| Needs supervision, watcher beacon fresh for its model | `healthy` | none (never double-arms) | +| Needs supervision, beacon stale, director session alive | `lapsed` | never re-arm and never relaunch a live session; start a confirmation clock and escalate a likely wedge only after it stays lapsed past `FM_GUARDIAN_WEDGE_ESCALATE_SECS` | +| Needs supervision, beacon stale, local secondmate session dead | `dead` | relaunch the director through the sanctioned spawn path (rate-limited), else escalate | +| Needs supervision, beacon stale, main home has no session | `dead` | re-arm to capture wakes durably and escalate a human to start a session | +| Remote secondmate (another host) | `remote-skip` | none (a local timer cannot repair another host) | + +Repair is honest about handling, not just about the beacon. +Re-arming a session-less home launches a real watcher, so the beacon is genuinely fresh and every wake is captured durably in `state/.wake-queue`; the guardian never fakes a beacon. +It deliberately does not re-arm a home whose session is still alive: a guardian-owned watcher would freshen that beacon and mask a wedged session, and re-arm gives an asleep session no handler anyway. +Because a surfaced wake is handled by the home's own live session, the guardian relaunches a dead local director to restore a handler and escalates a home it cannot safely restore rather than masking it behind a fresh beacon. +Wakes are not lost while a live session is only escalated: their signal and status files persist, and a watcher scans them once the session resumes or is relaunched. + +## Safety properties + +- It re-arms only through a self-surviving `systemd-run --user` transient unit, never a fire-and-forget shell `&`, which the `bin/fm-watch-arm.sh` header documents as the exact mistake that silently killed supervision for about thirty minutes. +- It never double-arms: a healthy home is left completely alone, and the watcher singleton lock plus `fm-watch-arm.sh`'s own attach make any concurrent arm converge on one watcher. +- It never kills live crew, discards unlanded work, or force-tears-down; the only relaunch path is the sanctioned `fm-spawn.sh --secondmate`, which reconciles rather than discards. +- It never stops, restarts, or touches the shared no-mistakes daemon. +- Relaunches and escalations are rate-limited per home, so a crash-looping home is escalated instead of hammered. +- It exports `XDG_RUNTIME_DIR` and `DBUS_SESSION_BUS_ADDRESS` before any `systemctl --user` call and uses absolute paths, so it runs correctly from the bare environment a timer provides. + +## Boundary and the unattended-handling decision + +The guardian keeps supervision armed and relaunches dead directors, but it does not make wakes self-handle with zero session. +The only design that self-handles routine wakes without any session is the away-mode sub-supervisor daemon ([`AGENTS.md`](../AGENTS.md) section 8), which is a fleet-behavior change: it batches escalations, changes the token tradeoff, and injects into panes. +The guardian deliberately leaves that choice to the captain rather than imposing it. +If a director is genuinely wedged rather than merely unsupervised, the guardian escalates it instead of masking it, so the captain can decide whether to adopt unattended daemon mode fleet-wide as a follow-up. + +## Install + +The guardian is inert until installed as a systemd user timer. +Templates live in `dist/systemd/`. +Firstmate installs and enables it on the live box; do not enable it from a task worktree. + +```sh +mkdir -p ~/.config/systemd/user +cp /dist/systemd/fm-liveness-guardian.service ~/.config/systemd/user/ +cp /dist/systemd/fm-liveness-guardian.timer ~/.config/systemd/user/ +# Edit FM_GUARDIAN_MAIN_HOME in the .service if the main home is not ~/firstmate. +loginctl enable-linger "$USER" # so user units run with no active login session +systemctl --user daemon-reload +systemctl --user enable --now fm-liveness-guardian.timer +``` + +`loginctl enable-linger` is required: without it user units do not run while no one is logged in, which is exactly when the guardian matters. + +Escalations go to the guardian log by default. +To also raise an active captain-facing signal (a Telegram ping, a checkpoint), set `FM_GUARDIAN_ESCALATE_CMD` in the `.service` to a command invoked as ` `; see the script header for the seam contract. + +## Verify it live + +```sh +systemctl --user list-timers fm-liveness-guardian.timer # next/last fire +systemctl --user status fm-liveness-guardian.service # last pass result +/bin/fm-liveness-guardian.sh --list # classify every home, repair nothing +tail -f ~/.local/state/fm-liveness-guardian.log # actions and escalations; silent when healthy +``` + +A quiet log is a healthy fleet: the guardian writes a line only when it re-arms, relaunches, escalates, or errors. +To confirm the repair path end to end, watch `list-timers` fire while a home's beacon is genuinely stale, then confirm that home's beacon becomes fresh and a `systemd-run` transient unit (`systemctl --user list-units 'fm-fleet-*'`) is briefly present. diff --git a/docs/scripts.md b/docs/scripts.md index 484911c380..387518fbf8 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -69,6 +69,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-procevent-when.sh` | Fire a trust-bound deterministic action at most once when its registered condition holds, then wake with the outcome | | `fm-gate-refuse-lib.sh` | Shared no-mistakes gate-context refusal for fleet lifecycle entrypoints | | `fm-watch-arm.sh` | Verified home-scoped watcher arm wrapper with loud cycle endings and bounded lifecycle ledger | +| `fm-liveness-guardian.sh` | Session-independent systemd-timer supervision backstop: re-arm a lapsed watcher self-survivingly, relaunch a dead local director, or escalate (docs/liveness-guardian.md) | | `fm-watch-checkpoint.sh` | Run one bounded foreground watcher checkpoint for Codex-style supervision | | `fm-watch.sh` | Singleton-safe always-on watcher: absorb benign wakes, queue and exit on actionable ones | | `fm-inactive-reconcile.sh` | Reconcile long-inactive direct crewmate terminal outcomes without forge access | diff --git a/docs/watcher-continuity.md b/docs/watcher-continuity.md index 1a94ec0ede..64166bacd2 100644 --- a/docs/watcher-continuity.md +++ b/docs/watcher-continuity.md @@ -2,6 +2,7 @@ The watcher remains intentionally one-shot: one actionable reason closes one watcher cycle. Must-work continuity now lives above that process boundary instead of depending on the model remembering a re-arm step. +Every mechanism below is session-owned; the session-independent backstop that repairs a home whose session stopped re-arming is [`liveness-guardian.md`](liveness-guardian.md). ## Ownership diff --git a/tests/fm-liveness-guardian.test.sh b/tests/fm-liveness-guardian.test.sh new file mode 100755 index 0000000000..d9fa2d0785 --- /dev/null +++ b/tests/fm-liveness-guardian.test.sh @@ -0,0 +1,291 @@ +#!/usr/bin/env bash +# Behavior tests for the session-independent fleet liveness guardian +# (bin/fm-liveness-guardian.sh, docs/liveness-guardian.md). +# +# The guardian runs from a systemd user timer with no session in the loop, so +# every case here exercises it against disposable mock homes and NEVER against a +# live fleet home. The real self-surviving arm (systemd-run) and the real +# secondmate relaunch (fm-spawn) are exercised only through their override seams +# so the tests neither require a user bus nor spawn a real agent; the seams are +# the guardian's own public interface, not private internals. The adversarial +# arm case uses a faithful watcher stand-in (touch the beacon, exactly what a real +# watcher does at the top of every poll) launched through the same seam, so the +# assertion is the real observable outcome - a fresh beacon and no double-arm - +# not a mocked verdict. +# shellcheck disable=SC2016 # single quotes are deliberate in the stub heredocs: $1/$2 expand inside the stub child, not here +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +GUARD="$ROOT/bin/fm-liveness-guardian.sh" +TMP_ROOT=$(fm_test_tmproot fm-liveness-guardian) +fm_git_identity fmtest fmtest@example.invalid + +FAKEBIN=$(fm_fakebin "$TMP_ROOT/fakebin") +# A live "harness" process for session-alive cases: a process whose comm is +# `claude` satisfies fm_harness_pid_alive. +ln -s /bin/bash "$FAKEBIN/claude" + +# Track background processes started for live-session fixtures so teardown reaps +# them even on a hard failure. The fixture is a bash (comm=claude) that spawns a +# `sleep` child, so teardown must kill the CHILD too (pkill -P) - otherwise the +# orphaned sleep lingers in the test's process group and the runner waits on it. +# pkill -P is portable across Linux and macOS; setsid is not. +FIXTURE_PIDS=() +cleanup() { + local p + for p in "${FIXTURE_PIDS[@]:-}"; do + [ -n "$p" ] || continue + pkill -P "$p" 2>/dev/null || true + kill "$p" 2>/dev/null || true + done + fm_test_cleanup +} +trap cleanup EXIT +trap 'cleanup; exit 130' INT +trap 'cleanup; exit 143' TERM + +# A guardian-scoped scratch state/log dir so no case pollutes another. +GSTATE="$TMP_ROOT/gstate" +GLOG="$TMP_ROOT/guardian.log" + +# The faithful arm launcher stub: records the invocation and freshens the beacon +# exactly as a real watcher's first poll iteration does. Records per call. +ARM_STUB="$TMP_ROOT/fake-arm-launcher.sh" +cat > "$ARM_STUB" <<'SH' +#!/usr/bin/env bash +# args: +set -u +printf '%s\n' "$2" >> "$ARM_INVOCATIONS" +mkdir -p "$2/state" +touch "$2/state/.last-watcher-beat" +SH +chmod +x "$ARM_STUB" + +RELAUNCH_STUB="$TMP_ROOT/fake-relaunch.sh" +cat > "$RELAUNCH_STUB" <<'SH' +#!/usr/bin/env bash +# args: +set -u +printf '%s\t%s\n' "$1" "$2" >> "$RELAUNCH_INVOCATIONS" +SH +chmod +x "$RELAUNCH_STUB" + +ESCALATE_STUB="$TMP_ROOT/fake-escalate.sh" +cat > "$ESCALATE_STUB" <<'SH' +#!/usr/bin/env bash +# args: +set -u +printf '%s\t%s\n' "$1" "$2" >> "$ESCALATE_INVOCATIONS" +SH +chmod +x "$ESCALATE_STUB" + +# make_home : a minimal firstmate home (just a state dir; the guardian falls +# back to the repo's own fm-watch-arm for the real path and uses seams in tests). +make_home() { + mkdir -p "$1/state" +} + +# needs_work : give the home an in-flight task so it needs supervision. +needs_work() { + : > "$1/state/task-$RANDOM.meta" +} + +# stale_beacon : a beacon far past any grace. +stale_beacon() { + touch -d '2020-01-01' "$1/state/.last-watcher-beat" +} + +# fresh_beacon : a beacon at "now". +fresh_beacon() { + touch "$1/state/.last-watcher-beat" +} + +# live_session : write a live claude pid into the session lock. The `; :` +# keeps bash from exec-optimizing a single `sleep` into place (which would leave +# comm=sleep, not claude) so fm_harness_pid_alive recognizes a live harness. +live_session() { + "$FAKEBIN/claude" -c 'sleep 3000; :' & + local pid=$! + FIXTURE_PIDS+=("$pid") + printf '%s\n' "$pid" > "$1/state/.lock" +} + +# new_ledgers : own a fresh, empty invocation ledger for the three seams IN +# THE PARENT shell (run_guardian runs in a $(...) subshell, so it cannot set +# these for the caller). Each pass that must be counted separately gets its own. +new_ledgers() { + ARM_INVOCATIONS="$TMP_ROOT/arm.$1.log" + RELAUNCH_INVOCATIONS="$TMP_ROOT/relaunch.$1.log" + ESCALATE_INVOCATIONS="$TMP_ROOT/escalate.$1.log" + : > "$ARM_INVOCATIONS"; : > "$RELAUNCH_INVOCATIONS"; : > "$ESCALATE_INVOCATIONS" + export ARM_INVOCATIONS RELAUNCH_INVOCATIONS ESCALATE_INVOCATIONS +} + +# run_guardian [args...]: run one guardian pass with the seams wired. +# Echoes stdout. The caller must have called new_ledgers first. +run_guardian() { + local main=$1 + shift + FM_HOME="$main" \ + FM_GUARDIAN_MAIN_HOME="$main" \ + FM_GUARDIAN_STATE_DIR="$GSTATE" \ + FM_GUARDIAN_LOG="$GLOG" \ + FM_GUARDIAN_WEDGE_ESCALATE_SECS="${TEST_WEDGE_SECS:-1800}" \ + FM_GUARDIAN_ARM_LAUNCHER="$ARM_STUB" \ + FM_GUARDIAN_RELAUNCH_CMD="$RELAUNCH_STUB" \ + FM_GUARDIAN_ESCALATE_CMD="$ESCALATE_STUB" \ + bash "$GUARD" "$@" +} + +# Count non-empty lines. `grep -c` prints 0 AND exits 1 on no match, so capture +# once and normalize, never `grep -c ... || echo 0` (which double-prints 0). +count_lines() { + [ -f "$1" ] || { echo 0; return; } + local n + n=$(grep -c . "$1" 2>/dev/null) || n=0 + echo "$n" +} + +# --------------------------------------------------------------------------- +# Case A: an idle home (no in-flight work) is healthy - no action, ever. +# --------------------------------------------------------------------------- +A="$TMP_ROOT/A/main"; make_home "$A"; stale_beacon "$A" +new_ledgers A +out=$(run_guardian "$A" --list) +assert_contains "$out" "class=idle" "idle home with no work must classify idle" +assert_contains "$out" "action=none" "idle home takes no action" +pass "idle home classifies idle with no action" + +# --------------------------------------------------------------------------- +# Case B: a needing home with a FRESH beacon is healthy - no arm, beacon +# untouched (the guardian never re-touches a healthy beacon). +# --------------------------------------------------------------------------- +B="$TMP_ROOT/B/main"; make_home "$B"; needs_work "$B"; fresh_beacon "$B" +before=$(stat -c %Y "$B/state/.last-watcher-beat") +new_ledgers B +out=$(run_guardian "$B") +assert_absent "$B/state/.watch.lock" "healthy home must not be armed" +[ "$(count_lines "$ARM_INVOCATIONS")" -eq 0 ] || fail "healthy home must not invoke the arm launcher" +after=$(stat -c %Y "$B/state/.last-watcher-beat") +[ "$before" = "$after" ] || fail "guardian must not re-touch a healthy home's beacon" +pass "healthy home is a no-op (no arm, no beacon touch)" + +# --------------------------------------------------------------------------- +# Case C (adversarial): a lapsed home with NO session in the loop and a stale +# beacon is re-armed to a FRESH beacon; a second pass sees it healthy and does +# NOT double-arm. This is the brief's "re-arm ... with no interactive session in +# the loop" scenario. +# --------------------------------------------------------------------------- +C="$TMP_ROOT/C/main"; make_home "$C"; needs_work "$C"; stale_beacon "$C" # no .lock => no session +# Confirm the precondition the brief simulates: beacon stale past grace. +age_before=$(( $(date +%s) - $(stat -c %Y "$C/state/.last-watcher-beat") )) +[ "$age_before" -gt 300 ] || fail "precondition: beacon must be stale past grace" +new_ledgers C1 +out=$(run_guardian "$C") +[ "$(count_lines "$ARM_INVOCATIONS")" -eq 1 ] || fail "lapsed session-less home must be armed exactly once (got $(count_lines "$ARM_INVOCATIONS"))" +age_after=$(( $(date +%s) - $(stat -c %Y "$C/state/.last-watcher-beat") )) +[ "$age_after" -lt 300 ] || fail "after re-arm the beacon must be fresh (age ${age_after}s)" +# Second pass: now healthy -> no second arm. +new_ledgers C2 +out=$(run_guardian "$C") +[ "$(count_lines "$ARM_INVOCATIONS")" -eq 0 ] || fail "healthy home must NOT be re-armed on the next pass (no double-arm)" +pass "lapsed session-less home re-arms to a fresh beacon; healthy home is not double-armed" + +# --------------------------------------------------------------------------- +# Case I: a live-but-lapsed session (a long turn, or a wedge) is NEVER re-armed +# (a guardian watcher would mask the wedge and defeat the confirmation clock) and +# is NOT escalated until it has stayed lapsed past the wedge threshold. +# --------------------------------------------------------------------------- +I="$TMP_ROOT/I/main"; make_home "$I"; needs_work "$I"; stale_beacon "$I"; live_session "$I" +TEST_WEDGE_SECS=0 # any persisted lapse counts as a wedge, to drive the second pass +export TEST_WEDGE_SECS +new_ledgers I1 +run_guardian "$I" >/dev/null +[ "$(count_lines "$ARM_INVOCATIONS")" -eq 0 ] || fail "a live session must never be re-armed (that masks a wedge)" +[ "$(count_lines "$ESCALATE_INVOCATIONS")" -eq 0 ] || fail "first observation of a lapse must not escalate (long turns recover)" +# Beacon must remain stale (the guardian did not freshen a live session's beacon). +age_i=$(( $(date +%s) - $(stat -c %Y "$I/state/.last-watcher-beat") )) +[ "$age_i" -gt 300 ] || fail "guardian must not freshen a live-but-lapsed session's beacon" +# Second pass: the lapse has now persisted past the (zeroed) threshold -> escalate. +new_ledgers I2 +run_guardian "$I" >/dev/null +[ "$(count_lines "$ARM_INVOCATIONS")" -eq 0 ] || fail "a persisted live lapse must still never be re-armed" +[ "$(count_lines "$ESCALATE_INVOCATIONS")" -ge 1 ] || fail "a live session lapsed past the wedge threshold must be escalated" +unset TEST_WEDGE_SECS +pass "live-but-lapsed session is never re-armed; escalated only after the wedge threshold" + +# --------------------------------------------------------------------------- +# Case D: a genuinely DEAD local secondmate director is relaunched through the +# sanctioned seam, and the relaunch is rate-limited on an immediate second pass. +# --------------------------------------------------------------------------- +D="$TMP_ROOT/D/main"; make_home "$D"; fresh_beacon "$D" # main healthy, focus on the secondmate +SM="$TMP_ROOT/D/sm1-home"; make_home "$SM"; needs_work "$SM"; stale_beacon "$SM" +# Register the secondmate in the main home's state as a LOCAL kind=secondmate. +fm_write_secondmate_meta "$D/state/sm1.meta" "$SM" "firstmate:fm-sm1" alpha claude +new_ledgers D1 +out=$(run_guardian "$D") +[ "$(count_lines "$RELAUNCH_INVOCATIONS")" -eq 1 ] || fail "dead secondmate must be relaunched once (got $(count_lines "$RELAUNCH_INVOCATIONS"))" +assert_grep "$SM" "$RELAUNCH_INVOCATIONS" "relaunch must target the secondmate home" +# Immediate second pass: rate-limited, no second relaunch. +new_ledgers D2 +out=$(run_guardian "$D") +[ "$(count_lines "$RELAUNCH_INVOCATIONS")" -eq 0 ] || fail "relaunch must be rate-limited on an immediate second pass" +pass "dead secondmate director is relaunched once, then rate-limited" + +# --------------------------------------------------------------------------- +# Case E: a REMOTE secondmate is reported and skipped (a local timer cannot +# repair another host). +# --------------------------------------------------------------------------- +E="$TMP_ROOT/E/main"; make_home "$E"; fresh_beacon "$E" +RM_HOME="/remote/opt/firstmate-remote" +fm_write_meta "$E/state/rm1.meta" \ + "kind=secondmate" "home=$RM_HOME" "remote_host=box.example.invalid" "harness=claude" +new_ledgers E +out=$(run_guardian "$E" --list) +assert_contains "$out" "class=remote-skip" "remote secondmate must be reported as remote-skip" +pass "remote secondmate is reported and skipped" + +# --------------------------------------------------------------------------- +# Case F: the DEFAULT arm transport is a self-surviving systemd-run, never a +# fire-and-forget shell '&' (constraint 1), asserted through the --list plan. +# --------------------------------------------------------------------------- +F="$TMP_ROOT/F/main"; make_home "$F"; needs_work "$F"; stale_beacon "$F" +# --list without the launcher seam so the DEFAULT transport is printed. +out=$(FM_HOME="$F" FM_GUARDIAN_MAIN_HOME="$F" FM_GUARDIAN_STATE_DIR="$GSTATE" \ + FM_GUARDIAN_LOG="$GLOG" bash "$GUARD" --list) +assert_contains "$out" "systemd-run --user" "default arm must use a self-surviving systemd-run unit" +assert_not_contains "$out" "nohup" "default arm must not use nohup" +case "$out" in + *' &'*) fail "default arm must never use a fire-and-forget shell '&'" ;; +esac +pass "default arm transport is a self-surviving systemd-run, not a shell '&'" + +# --------------------------------------------------------------------------- +# Case G: runs cleanly under a bare `env -i` systemd-timer environment. +# --------------------------------------------------------------------------- +G="$TMP_ROOT/G/main"; make_home "$G"; needs_work "$G"; stale_beacon "$G" +out=$(env -i HOME="$HOME" PATH="/usr/bin:/bin" \ + FM_HOME="$G" FM_GUARDIAN_MAIN_HOME="$G" FM_GUARDIAN_STATE_DIR="$GSTATE" \ + FM_GUARDIAN_LOG="$GLOG" \ + bash "$GUARD" --list 2>&1) +rc=$? +expect_code 0 "$rc" "guardian must exit 0 under env -i" +assert_contains "$out" "home=$G" "guardian must classify the home under env -i" +pass "runs cleanly under a bare env -i environment" + +# --------------------------------------------------------------------------- +# Case H: a DEAD main home (needs work, stale beacon, no session) is re-armed to +# capture wakes durably AND escalated - it is never relaunched (the guardian +# must not spawn a competing firstmate). +# --------------------------------------------------------------------------- +H="$TMP_ROOT/H/main"; make_home "$H"; needs_work "$H"; stale_beacon "$H" +new_ledgers H +out=$(run_guardian "$H") +[ "$(count_lines "$ARM_INVOCATIONS")" -eq 1 ] || fail "dead main home must be re-armed to capture wakes durably" +[ "$(count_lines "$ESCALATE_INVOCATIONS")" -ge 1 ] || fail "dead main home must be escalated to a human" +[ "$(count_lines "$RELAUNCH_INVOCATIONS")" -eq 0 ] || fail "guardian must never relaunch/spawn the main firstmate" +pass "dead main home is re-armed and escalated, never relaunched" + +pass "fm-liveness-guardian: all cases passed"