diff --git a/.agents/skills/afk/SKILL.md b/.agents/skills/afk/SKILL.md index 2b68f29ed9..776e7fede0 100644 --- a/.agents/skills/afk/SKILL.md +++ b/.agents/skills/afk/SKILL.md @@ -229,7 +229,7 @@ the operational prefix lets firstmate distinguish it from a real captain message ## Stale-artifact lifecycle -Treat `state/.subsuper-escalations`, its `.since` sidecar, and `state/.subsuper-inject-wedged` as session-scoped delivery artifacts, not as the durable work record. +Treat `state/.subsuper-escalations`, its `.since` sidecar, `state/.subsuper-inject-wedged`, and the bounded-re-injection cap markers `state/.subsuper-inject-failed` and `state/.subsuper-inject-capped` as session-scoped delivery artifacts, not as the durable work record. Always enter through `bin/fm-afk-launch.sh`, which clears prior-session artifacts only for a fresh entry and preserves the current session's buffer on refresh. Always exit through `bin/fm-afk-launch.sh stop`, which keeps `state/.afk` present through the daemon's shutdown flush and clears it last. `docs/herdr-backend.md` "Away-mode supervisor support" owns the current mechanism, and `docs/verification/runtime-backends.md` "Away-mode transport" owns active evidence. diff --git a/bin/fm-afk-launch.sh b/bin/fm-afk-launch.sh index 5df2a9d991..b283de3376 100755 --- a/bin/fm-afk-launch.sh +++ b/bin/fm-afk-launch.sh @@ -360,11 +360,13 @@ fm_afk_launch_restore_backup() { # rm -f "$FM_AFK_LAUNCH_STATE/.afk" \ "$FM_AFK_LAUNCH_STATE/.subsuper-escalations" \ "$FM_AFK_LAUNCH_STATE/.subsuper-escalations.since" \ - "$FM_AFK_LAUNCH_STATE/.subsuper-inject-wedged" || result=1 + "$FM_AFK_LAUNCH_STATE/.subsuper-inject-wedged" \ + "$FM_AFK_LAUNCH_STATE/.subsuper-inject-failed" \ + "$FM_AFK_LAUNCH_STATE/.subsuper-inject-capped" || result=1 if [ "$had_afk" -eq 1 ]; then cp "$backup/.afk" "$FM_AFK_LAUNCH_STATE/.afk" || result=1 fi - for artifact in .subsuper-escalations .subsuper-escalations.since .subsuper-inject-wedged; do + for artifact in .subsuper-escalations .subsuper-escalations.since .subsuper-inject-wedged .subsuper-inject-failed .subsuper-inject-capped; do if [ -e "$backup/$artifact" ]; then cp -p "$backup/$artifact" "$FM_AFK_LAUNCH_STATE/$artifact" || result=1 fi @@ -487,7 +489,7 @@ fm_afk_launch_start() { had_afk=1 cp "$FM_AFK_LAUNCH_STATE/.afk" "$backup/.afk" || { rm -rf "$backup"; return 1; } fi - for artifact in .subsuper-escalations .subsuper-escalations.since .subsuper-inject-wedged; do + for artifact in .subsuper-escalations .subsuper-escalations.since .subsuper-inject-wedged .subsuper-inject-failed .subsuper-inject-capped; do if [ -e "$FM_AFK_LAUNCH_STATE/$artifact" ]; then cp -p "$FM_AFK_LAUNCH_STATE/$artifact" "$backup/$artifact" || { rm -rf "$backup"; return 1; } fi @@ -545,7 +547,7 @@ fm_afk_launch_start_native() { had_afk=1 cp "$FM_AFK_LAUNCH_STATE/.afk" "$backup/.afk" || { rm -rf "$backup"; return 1; } fi - for artifact in .subsuper-escalations .subsuper-escalations.since .subsuper-inject-wedged; do + for artifact in .subsuper-escalations .subsuper-escalations.since .subsuper-inject-wedged .subsuper-inject-failed .subsuper-inject-capped; do if [ -e "$FM_AFK_LAUNCH_STATE/$artifact" ]; then cp -p "$FM_AFK_LAUNCH_STATE/$artifact" "$backup/$artifact" || { rm -rf "$backup"; return 1; } fi diff --git a/bin/fm-afk-return.sh b/bin/fm-afk-return.sh index b38c1e07c4..68ae234b00 100755 --- a/bin/fm-afk-return.sh +++ b/bin/fm-afk-return.sh @@ -124,7 +124,9 @@ clear_delivery_artifacts() { rm -f \ "$STATE/.subsuper-escalations" \ "$STATE/.subsuper-escalations.since" \ - "$STATE/.subsuper-inject-wedged" + "$STATE/.subsuper-inject-wedged" \ + "$STATE/.subsuper-inject-failed" \ + "$STATE/.subsuper-inject-capped" } return_guard() { diff --git a/bin/fm-afk-start.sh b/bin/fm-afk-start.sh index e86c54f170..bfb9c4adb0 100755 --- a/bin/fm-afk-start.sh +++ b/bin/fm-afk-start.sh @@ -63,7 +63,9 @@ fm_afk_clear_stale_artifacts() { # local state=$1 rm -f "$state/.subsuper-escalations" \ "$state/.subsuper-escalations.since" \ - "$state/.subsuper-inject-wedged" 2>/dev/null + "$state/.subsuper-inject-wedged" \ + "$state/.subsuper-inject-failed" \ + "$state/.subsuper-inject-capped" 2>/dev/null } daemon_lock_owner() { diff --git a/bin/fm-supervise-daemon.sh b/bin/fm-supervise-daemon.sh index ff123a7faf..9ff142d468 100755 --- a/bin/fm-supervise-daemon.sh +++ b/bin/fm-supervise-daemon.sh @@ -51,7 +51,11 @@ # Buffered escalation delivery also has a max-defer alarm: if a digest stays # undelivered past FM_MAX_DEFER_SECS, the daemon retries a normal flush and # writes state/.subsuper-inject-wedged and attempts a configurable active -# alert if submit still cannot be confirmed. +# alert if submit still cannot be confirmed. Re-injection is then hard-capped +# per buffered content (see escalate_flush_capped): a submit-confirmation +# false-negative costs at most one alarm, never an all-night re-injection +# loop. The cap bounds retry COUNT only; the buffer stays durable, so nothing +# is lost after queue publication and return catch-up still surfaces it. # - Cheap heartbeat catch-all: every HEARTBEAT_SCAN_SECS the daemon greps all # state/*.status for a captain-relevant line the per-wake classifier might # have missed (e.g. a status verb outside CAPTAIN_RE) and escalates it. @@ -640,10 +644,12 @@ escalate_add() { # } # Flush the escalation buffer as ONE batched, single-line digest to the -# supervisor pane. Returns 0 on successful inject (or empty buffer), non-zero on -# inject failure (buffer preserved for retry / catch-up). +# supervisor pane. Returns 0 on successful inject (or empty buffer). On failure +# the buffer is preserved for retry / catch-up and the exit code is inject_msg's +# verbatim (1 = deferred before typing, 2 = typed but submit unconfirmed), so +# escalate_flush_capped can count only the typed-unconfirmed case toward the cap. escalate_flush() { # - local state=$1 buf item n msg + local state=$1 buf item n msg irc buf="$state/.subsuper-escalations" [ -s "$buf" ] || return 0 n=$(wc -l < "$buf" 2>/dev/null || echo 0) @@ -652,8 +658,93 @@ escalate_flush() { # # Single-line wrapper: no embedded newlines (inject_msg also collapses as a # safety net, but keeping the source single-line makes the intent explicit). msg=$(printf 'Supervisor escalate (%s event(s)): %s (pre-read; re-arm not needed — watcher daemon-managed)' "$n" "$msg") - if inject_msg "$msg" "$state"; then : > "$buf"; rm -f "${buf}.since" "$state/.subsuper-inject-wedged"; return 0; fi - return 1 + inject_msg "$msg" "$state"; irc=$? + if [ "$irc" -eq 0 ]; then : > "$buf"; rm -f "${buf}.since" "$state/.subsuper-inject-wedged"; return 0; fi + return "$irc" +} + +# --- bounded re-injection cap (composer-confirmation false-negative guard) --- +# A composer-confirmation false-negative (a digest that lands and starts a real +# turn while the idle composer is briefly misread as still holding text) makes +# escalate_flush report the submit unconfirmed even though delivery happened. +# Without a bound, housekeeping's per-tick batch flush would re-inject the SAME +# digest every tick indefinitely (the overnight ~10h re-injection incident). The +# cap bounds how many times ONE unchanged buffered content is TYPED into the +# pane: the first typed attempt, plus the single max-defer escape retry that +# fires the wedge alarm; after that alarm the content is CAPPED and never typed +# again until it CHANGES (a genuinely new escalation arrives) or delivery is +# later positively confirmed. It bounds typed-attempt COUNT only - the buffer +# stays durable throughout, so nothing is lost after queue publication and return +# catch-up still surfaces it, and the wedge alarm still fires once per episode. +# +# ONLY a typed-but-unconfirmed attempt (inject_msg/escalate_flush exit 2) counts +# toward the cap. A pre-typing DEFERRAL (exit 1: pane busy, composer holds a +# human's half-typed line, afk off) types nothing, costs nothing, and MUST keep +# retrying every tick so the digest lands the moment the pane frees - conflating +# it with a real failure would strand the escalation (the Scenario A regression). +# +# Two content-addressed markers, both keyed to the exact buffered bytes: +# .subsuper-inject-failed content whose last TYPED flush was unconfirmed; the +# per-tick batch flush skips it (the max-defer escape +# still gets its single forced retry). +# .subsuper-inject-capped content whose max-defer escape has already alarmed; +# every flush path skips it until it changes. +# A successful escalate_flush clears the buffer and its .since/wedged markers; +# escalate_flush_capped then clears both cap markers, so a later positively +# confirmed delivery fully restores normal injection. + +# Stable hash of the buffered escalation content; fails (non-zero) if empty. +_inject_content_hash() { # + local buf=$1/.subsuper-escalations + [ -s "$buf" ] || return 1 + _hash_text "$(cat "$buf" 2>/dev/null)" +} + +# Clear the cap tracking (both markers). Called when the buffer drains or a flush +# is positively confirmed. +reset_inject_cap() { # + rm -f "$1/.subsuper-inject-failed" "$1/.subsuper-inject-capped" +} + +# Mark the current buffered content as capped (its max-defer escape has fired the +# wedge alarm), so no flush path re-injects it until it changes or drains. +cap_inject_content() { # + local h + h=$(_inject_content_hash "$1") || return 0 + printf '%s' "$h" > "$1/.subsuper-inject-capped" +} + +# Bounded wrapper around escalate_flush. is "normal" (the per-tick batch +# flush) or "force" (the single max-defer escape retry, which ignores the +# already-failed marker but still honors the hard cap). Return codes: +# 0 delivered (buffer and all cap markers cleared) +# 1 TYPED but submit unconfirmed - the cappable false-negative; this exact +# content recorded as failed. The caller may raise the wedge alarm and cap. +# 2 HELD without any typed attempt because this content is already capped, or +# already failed on a prior normal tick (the max-defer escape owns retry). +# 3 DEFERRED before typing (pane busy / composer pending / afk off). Nothing +# was typed, no marker set: retry naturally next tick. Never a cap trip. +# Never re-types capped content. +escalate_flush_capped() { # + local state=$1 mode=${2:-normal} h capped failed frc + [ -s "$state/.subsuper-escalations" ] || { reset_inject_cap "$state"; return 0; } + h=$(_inject_content_hash "$state") || { reset_inject_cap "$state"; return 0; } + capped=$(cat "$state/.subsuper-inject-capped" 2>/dev/null || true) + if [ "$h" = "$capped" ]; then + return 2 # Wedge alarm already fired for this exact content; hold durably. + fi + if [ "$mode" != force ]; then + failed=$(cat "$state/.subsuper-inject-failed" 2>/dev/null || true) + if [ "$h" = "$failed" ]; then + return 2 # Already typed+failed this content; the max-defer escape owns the retry. + fi + fi + escalate_flush "$state"; frc=$? + case "$frc" in + 0) reset_inject_cap "$state"; return 0 ;; # confirmed delivery + 2) printf '%s' "$h" > "$state/.subsuper-inject-failed"; return 1 ;; # typed, unconfirmed + *) return 3 ;; # deferred before typing: nothing to cap, retry next tick + esac } # --- backend-independent active wedge alert --------------------------------- @@ -949,8 +1040,9 @@ _oldest_line_age() { # -> seconds since the oldest buffered item first ar # 1) batch flush: if the escalation buffer's oldest content is older than # ESCALATE_BATCH_SECS (or batching is disabled), inject one digest. # 1b) max-defer escape: if the buffer is STILL undelivered past MAX_DEFER_SECS, -# attempt one normal delivery; if it cannot confirm, raise the wedge alarm. -# Never silently defer forever. +# attempt one forced delivery; if it cannot confirm, raise the wedge alarm +# once and CAP this content so no later tick re-injects or re-alarms it (the +# buffer stays durable). Never silently defer forever, never loop forever. # 2) stale recheck: for each pending stale marker past STALE_ESCALATE_SECS, # re-peek the pane; still idle -> escalate (wedge); resumed -> clear marker. # 2b) pause re-surface: for each declared-pause marker past PAUSE_RESURFACE_SECS, @@ -959,17 +1051,19 @@ _oldest_line_age() { # -> seconds since the oldest buffered item first ar # 3) heartbeat scan: every HEARTBEAT_SCAN_SECS, grep state/*.status for a # captain-relevant line the per-wake classifier missed and escalate it. housekeeping() { # - local state=$1 now due f key task win marker age last max_defer oldest pause_secs + local state=$1 now due f key task win marker age last max_defer oldest pause_secs rc h failed now=$(_now) migrate_watcher_pause_markers "$state" - # (1) batch flush + # (1) batch flush (bounded: content whose normal flush already failed is not + # re-injected each tick - escalate_flush_capped holds it for the max-defer + # escape below or a positively confirmed retry, never a per-tick re-inject). if [ "${FM_ESCALATE_BATCH_SECS:-$ESCALATE_BATCH_SECS_DEFAULT}" -le 0 ]; then - escalate_flush "$state" || true + escalate_flush_capped "$state" normal || true else due=$(_oldest_line_age "$state/.subsuper-escalations") if [ "$due" -ge "${FM_ESCALATE_BATCH_SECS:-$ESCALATE_BATCH_SECS_DEFAULT}" ]; then - escalate_flush "$state" || true + escalate_flush_capped "$state" normal || true fi fi @@ -984,12 +1078,31 @@ housekeeping() { # # and waits. if [ "$oldest" -ge "$max_defer" ] \ && [ "$(_file_age "$state/.subsuper-inject-wedged")" -ge "$max_defer" ]; then - if escalate_flush "$state"; then - log "inject recovered: max-defer flush succeeded after ${oldest}s undelivered" - rm -f "$state/.subsuper-inject-wedged" - else - inject_wedge_alarm "$state" "$oldest" - fi + # The single escape retry for this content. force ignores the per-tick + # failed marker but still honors the hard cap, so once the wedge alarm has + # fired for a typed-but-unconfirmed content the flush returns 2 (held) and + # neither re-types nor re-alarms - a confirmation false-negative costs at + # most this one alarm, not an all-night re-injection loop. + escalate_flush_capped "$state" force; rc=$? + case "$rc" in + 0) log "inject recovered: max-defer flush succeeded after ${oldest}s undelivered" + rm -f "$state/.subsuper-inject-wedged" ;; + 1) inject_wedge_alarm "$state" "$oldest" + cap_inject_content "$state" ;; # typed+unconfirmed: alarm, then cap re-typing + 2) : ;; # already capped for this content: hold durably, do not re-alarm + 3) inject_wedge_alarm "$state" "$oldest" + # Deferred before typing on this escape. If this exact content was + # already typed-but-unconfirmed on a prior flush (its failed marker + # matches), the deferral is that false-negative's own residue (its + # swallowed text still sitting in the composer), so cap it - the wedge + # alarm has fired and re-typing would just repeat the loop. A content + # that was NEVER typed (a genuine pane-busy / half-typed human line) + # has no failed marker: leave it uncapped so it keeps retrying and + # lands the moment the pane frees. + h=$(_inject_content_hash "$state" 2>/dev/null || true) + failed=$(cat "$state/.subsuper-inject-failed" 2>/dev/null || true) + if [ -n "$h" ] && [ "$h" = "$failed" ]; then cap_inject_content "$state"; fi ;; + esac fi fi @@ -1095,10 +1208,17 @@ window_for_task() { # [state] # --- injection -------------------------------------------------------------- # inject_msg: send one escalation digest to the supervisor pane. -# Returns 0 on successful inject (or empty buffer), non-zero if the pane is -# gone, the supervisor is busy, afk is inactive, or the verified submit cannot -# be confirmed after bounded retries. On non-zero the caller preserves -# the buffer so the escalation survives for the next cycle or the catch-up flush. +# Returns 0 on successful inject (or empty buffer). On failure the caller +# preserves the buffer so the escalation survives for the next cycle or the +# catch-up flush, and the exit code distinguishes WHY, because the re-injection +# cap treats the two cases differently (see escalate_flush_capped): +# 1 DEFERRED before typing anything - pane gone, supervisor busy, afk +# inactive, or composer not confirmed-empty. Nothing was typed, so this is +# free to retry every tick and must NEVER count toward the cap. +# 2 TYPED but the submit could not be confirmed after bounded retries - the +# digest was actually sent into the composer. A persistent verdict here is +# the confirmation false-negative that drove the overnight re-injection +# loop, so ONLY this case counts toward the hard cap. # # Submit model: # - TYPE ONCE, then submit with Enter. Never retype the digest: a swallowed @@ -1168,7 +1288,7 @@ inject_msg() { # [state] return 0 # Backend confirmed the submit. fi log "inject failed: submit unconfirmed after $retries retries (verdict=$verdict, text may be in composer)" - return 1 + return 2 # TYPED but submit unconfirmed - the cappable false-negative. } # --- INJECT_SKIP prefix match (literal prefixes, no regex) ------------------ @@ -1233,7 +1353,7 @@ handle_wake() { # # housekeeping re-escalates the same pane as a false wedge later. [ "$kind" = "stale" ] && stale_marker_remove "$arg" "$state" mark_escalated_seen "$kind" "$arg" "$state" - [ "${FM_ESCALATE_BATCH_SECS:-$ESCALATE_BATCH_SECS_DEFAULT}" -le 0 ] && { escalate_flush "$state" || true; } + [ "${FM_ESCALATE_BATCH_SECS:-$ESCALATE_BATCH_SECS_DEFAULT}" -le 0 ] && { escalate_flush_capped "$state" normal || true; } ;; pause) # Declared external-wait pause: record a pause marker (long re-surface diff --git a/docs/configuration.md b/docs/configuration.md index e4aef1a7cc..7e0a00a937 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -113,7 +113,7 @@ Beyond the durable `state/.subsuper-inject-wedged` marker and the tmux status-li `config/wedge-alarm` (local, gitignored) lists channel directives, one per non-empty, non-comment line; every listed non-`off` channel fires, best-effort. `FM_WEDGE_ALARM_CHANNEL` overrides the file with a single directive. Directives are `off` (a position-independent kill switch that disables every active alert), `auto`/`default`, `osascript` (macOS Notification Center banner), `herdr` (herdr UI notification), and `command:` (run `` via `sh -c`, summary on `$1` and stdin). -An absent file means `auto`, i.e. default-on on macOS: the alarm exists precisely so a wedged away-mode primary is never silent, and it fires at most once per max-defer window after a genuine wedge. +An absent file means `auto`, i.e. default-on on macOS: the alarm exists precisely so a wedged away-mode primary is never silent, and per the bounded-re-injection cap in [`wedge-alarm.md`](wedge-alarm.md) it fires at most once per wedge episode for a given buffered escalation. A missing or failing channel logs and falls through to the next, never crashing the daemon. See [`wedge-alarm.md`](wedge-alarm.md) for the current channel reference, [`verification/supervision.md`](verification/supervision.md#wedge-alarm-channels) for active evidence, and [`examples/wedge-alarm`](examples/wedge-alarm) for a copyable config. diff --git a/docs/wedge-alarm.md b/docs/wedge-alarm.md index cfee3784b4..8567d53e45 100644 --- a/docs/wedge-alarm.md +++ b/docs/wedge-alarm.md @@ -5,6 +5,14 @@ When injection cannot confirm a submit past `FM_MAX_DEFER_SECS`, `inject_wedge_a The active alert is pane-independent because a tmux status-line flash has no cross-backend equivalent and cannot reach an unattended captain reliably. The durable marker and tmux flash remain as additional signals. +## Bounded re-injection + +Re-injection of an undelivered digest is hard-capped so a submit-confirmation false-negative (the digest lands and starts a real turn, but the idle composer is briefly misread as still holding text) cannot become an all-night re-injection loop. +The daemon attempts the initial delivery and, once past `FM_MAX_DEFER_SECS`, at most one escape retry; if that still cannot confirm, it raises the wedge alarm exactly once for that buffered content and then holds without re-injecting or re-alarming it. +The cap bounds the retry count only: the buffered escalation stays durable, so nothing is lost and return catch-up still surfaces it. +The cap is per buffered content, never a permanent mute: a genuinely distinct escalation is attempted again and raises its own single alarm, and any positively confirmed delivery clears the cap so normal injection resumes. +This is backend-independent because the cap is tracked in state files, not in any backend's pane surface. + ## Channels `config/wedge-alarm` is local and gitignored. @@ -19,7 +27,7 @@ It lists channel directives, one per non-empty, non-comment line, and every list - `command:` runs `` through `sh -c` with the alarm summary as `$1` and on stdin, allowing delivery to a phone or pager service. An absent `config/wedge-alarm` behaves as `auto`, which is default-on on macOS. -This is deliberate because the alarm fires only after a genuine max-defer wedge and is rate-limited to at most once per max-defer window. +This is deliberate because the alarm fires only after a genuine max-defer wedge and, per the bounded-re-injection cap above, at most once per wedge episode for a given buffered content. Each channel is best-effort. A missing binary or non-zero exit logs a warning and continues to the next channel without crashing the daemon loop. @@ -35,5 +43,5 @@ When the daemon is sourced as a library, that seam defaults to `discard`, so a t `tests/wake-helpers.sh` replaces it with a recorder when a suite needs to assert channel selection and summary propagation. Production leaves the seam unset and uses the configured real channels. -`tests/fm-daemon.test.sh` covers directive parsing, rate limiting, timeout and process-group cleanup, argv-safe dispatch, channel fallback, and safe `command:` summary delivery. +`tests/fm-daemon.test.sh` covers directive parsing, rate limiting, timeout and process-group cleanup, argv-safe dispatch, channel fallback, safe `command:` summary delivery, and the bounded-re-injection cap under persistent submit non-confirmation. [`verification/supervision.md`](verification/supervision.md#wedge-alarm-channels) records the bounded manual macOS and Herdr channel proof. diff --git a/tests/fm-daemon.test.sh b/tests/fm-daemon.test.sh index 2fe02fb431..774c174c74 100755 --- a/tests/fm-daemon.test.sh +++ b/tests/fm-daemon.test.sh @@ -1149,6 +1149,95 @@ test_max_defer_empty_swallow_types_once_and_alarms() { pass "max-defer on an empty stuck pane types once, alarms, and preserves the buffer" } +# The overnight incident: a submit-confirmation false-negative (the digest lands +# and starts a real turn, but the idle composer is misread as still pending) +# makes every housekeeping tick's batch flush re-inject the SAME digest forever. +# Drive persistent non-confirmation across many ticks and assert re-injection is +# hard-capped (initial attempt + at most one max-defer escape), the wedge alarm +# fires exactly once, and the buffer stays durable for return catch-up. +test_persistent_nonconfirmation_caps_reinjection() { + local dir state fakebin sent alert i injects alerts + dir=$(make_bordered_case reinject-cap) + state="$dir/state"; fakebin="$dir/fakebin" + sent="$dir/sent.log"; : > "$sent" + alert="$dir/alert.log"; : > "$alert" + touch "$dir/.swallow" + escalate_add "$state" "needs-decision: pick A" + echo $(( $(date +%s) - 600 )) > "$state/.subsuper-escalations.since" + afk_enter "$state" + # A prior test in this sourced process may have advanced the alarm-notify + # throttle epoch; reset it so this episode's single alarm is not suppressed. + WEDGE_ALARM_LAST_EPOCH=0 + # Persistent submit non-confirmation: the digest "lands" (composer reset to + # bordered-empty each tick) but the fake's persist-swallow leaves the typed + # text so the submit-ack keeps reading pending - the confirmation + # false-negative that drove the loop. Without the cap this types once per tick. + for i in $(seq 1 8); do + printf '╭─────╮\n│ > │\n╰─────╯\n' > "$dir/composer" + PATH="$fakebin:$PATH" FM_FAKE_COMPOSER="$dir/composer" FM_FAKE_SENT="$sent" \ + FM_FAKE_SWALLOW="$dir/.swallow" FM_FAKE_PERSIST_SWALLOW=1 FM_INJECT_CONFIRM_SLEEP=0.05 \ + FM_WEDGE_ALARM_LOG="$alert" FM_WEDGE_ALARM_CHANNEL=osascript \ + FM_ESCALATE_BATCH_SECS=1 FM_MAX_DEFER_SECS=1 housekeeping "$state" + done + injects=$(grep -c 'Supervisor escalate' "$sent" 2>/dev/null || true) + alerts=$(grep -c . "$alert" 2>/dev/null || true) + [ "$injects" -le 2 ] \ + || fail "re-injection not capped: typed the digest $injects times across 8 ticks (expected <= 2)" + [ "$injects" -ge 1 ] || fail "cap suppressed the initial delivery attempt entirely" + [ "$alerts" -eq 1 ] \ + || fail "wedge alarm fired $alerts times across 8 ticks (expected exactly 1)" + [ -s "$state/.subsuper-inject-wedged" ] \ + || fail "no durable wedge marker after persistent non-confirmation" + [ -s "$state/.subsuper-escalations" ] \ + || fail "buffer lost after capping (must stay durable for return catch-up)" + pass "persistent submit non-confirmation caps re-injection (<= 2), alarms once, keeps the buffer durable" +} + +# The cap is per-buffered-content, never a permanent daemon mute. A genuinely +# distinct escalation arriving after a wedge cap must be attempted again, and +# once delivery is positively confirmed (pane affirmatively empty, submit +# confirmed) every cap/wedge marker clears so normal injection fully resumes. +test_new_escalation_after_cap_rearms_and_confirmed_flush_resumes() { + local dir state fakebin sent i capped_injects after + dir=$(make_bordered_case reinject-cap-rearm) + state="$dir/state"; fakebin="$dir/fakebin" + sent="$dir/sent.log"; : > "$sent" + touch "$dir/.swallow" + escalate_add "$state" "needs-decision: pick A" + echo $(( $(date +%s) - 600 )) > "$state/.subsuper-escalations.since" + afk_enter "$state" + WEDGE_ALARM_LAST_EPOCH=0 + # Drive the first content to the capped state. + for i in 1 2 3; do + printf '╭─────╮\n│ > │\n╰─────╯\n' > "$dir/composer" + PATH="$fakebin:$PATH" FM_FAKE_COMPOSER="$dir/composer" FM_FAKE_SENT="$sent" \ + FM_FAKE_SWALLOW="$dir/.swallow" FM_FAKE_PERSIST_SWALLOW=1 FM_INJECT_CONFIRM_SLEEP=0.05 \ + FM_WEDGE_ALARM_CHANNEL=off \ + FM_ESCALATE_BATCH_SECS=1 FM_MAX_DEFER_SECS=1 housekeeping "$state" + done + [ -e "$state/.subsuper-inject-capped" ] \ + || fail "content was not capped after persistent non-confirmation" + capped_injects=$(grep -c 'Supervisor escalate' "$sent" 2>/dev/null || true) + # A genuinely distinct escalation arrives and the pane is now healthy (submit + # confirms empty). The cap must lift for the changed content, deliver, and + # clear every marker. + rm -f "$dir/.swallow" + escalate_add "$state" "blocked: db credential expired" + printf '╭─────╮\n│ > │\n╰─────╯\n' > "$dir/composer" + PATH="$fakebin:$PATH" FM_FAKE_COMPOSER="$dir/composer" FM_FAKE_SENT="$sent" \ + FM_INJECT_CONFIRM_SLEEP=0.05 FM_WEDGE_ALARM_CHANNEL=off \ + FM_ESCALATE_BATCH_SECS=1 FM_MAX_DEFER_SECS=1 housekeeping "$state" + after=$(grep -c 'Supervisor escalate' "$sent" 2>/dev/null || true) + [ "$after" -gt "$capped_injects" ] \ + || fail "cap suppressed a genuinely new escalation (was $capped_injects, now $after)" + [ ! -s "$state/.subsuper-escalations" ] \ + || fail "confirmed flush of the re-armed content did not clear the buffer" + [ ! -e "$state/.subsuper-inject-capped" ] || fail "cap marker survived a positively confirmed flush" + [ ! -e "$state/.subsuper-inject-failed" ] || fail "failed marker survived a positively confirmed flush" + [ ! -e "$state/.subsuper-inject-wedged" ] || fail "wedge marker survived a positively confirmed flush" + pass "a distinct escalation re-arms the cap and a confirmed flush resumes normal injection" +} + test_max_defer_flushes_empty_idle_pane() { local dir state fakebin sent dir=$(make_bordered_case maxdefer-recover) @@ -1891,6 +1980,8 @@ test_pane_input_pending_bordered_with_text_is_pending test_submit_ack_confirms_on_bordered_empty_composer test_submit_ack_reports_pending_on_persistent_swallow test_max_defer_empty_swallow_types_once_and_alarms +test_persistent_nonconfirmation_caps_reinjection +test_new_escalation_after_cap_rearms_and_confirmed_flush_resumes test_max_defer_flushes_empty_idle_pane test_max_defer_pending_composer_alarms_without_typing test_normal_flush_clears_stale_wedge_marker