From 5f19b906b50539e6ae6d7779a3b685b35d4e100b Mon Sep 17 00:00:00 2001 From: Rodrigo Campos Date: Thu, 13 Aug 2026 23:56:18 -0400 Subject: [PATCH 1/6] fix(bin): stop treating cancelled or empty CI as green A checks-passed claim is now done only when at least one check completed with success and none are pending, failed, cancelled, skipped, or never-run. Zero checks is a distinct non-success state. --- AGENTS.md | 2 +- bin/fm-ci-verdict-lib.sh | 68 ++++++++++++++++++++++++ bin/fm-crew-state.sh | 78 +++++++++++++++++++--------- bin/fm-test-run.sh | 5 +- docs/architecture.md | 2 +- docs/scripts.md | 1 + tests/fm-ci-verdict-lib.test.sh | 73 ++++++++++++++++++++++++++ tests/fm-crew-state.test.sh | 92 +++++++++++++++++++++++++++------ 8 files changed, 278 insertions(+), 43 deletions(-) create mode 100755 bin/fm-ci-verdict-lib.sh create mode 100755 tests/fm-ci-verdict-lib.test.sh diff --git a/AGENTS.md b/AGENTS.md index bd40813bf7..2e55687569 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -346,7 +346,7 @@ Require the matching `resolved` event, forbid `--yes`, and require the worker to Resume fleet supervision immediately after the decision lands. Judge validation by the current-code-matched run step through `bin/fm-crew-state.sh`, not by shell liveness or the last status event. -Running, fixing, or CI states remain working; parked approval or fix-review states require the worker to follow the active gate help; passed or checks-passed is done; failed or cancelled is failed. +Running, fixing, or CI states remain working; parked approval or fix-review states require the worker to follow the active gate help; passed is done; a checks-passed claim is done only when `bin/fm-crew-state.sh` scores at least one successful check and no pending, failed, cancelled, skipped, or never-run conclusions; failed or cancelled is failed. A worker hand-editing, committing, aborting, or restarting during an active validation run duplicates pipeline ownership outside the supersession sequence above; steer it back to the gate response flow. The worker reports the PR when CI first becomes green rather than waiting for merge monitoring to finish. diff --git a/bin/fm-ci-verdict-lib.sh b/bin/fm-ci-verdict-lib.sh new file mode 100755 index 0000000000..95383723bb --- /dev/null +++ b/bin/fm-ci-verdict-lib.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Shared CI-ready scoring for a set of check conclusions. +# +# ONE owner of the rule that a green verdict is valid only when at least one +# check completed with success AND none are pending, failed, cancelled, +# skipped, or never-run. Zero checks is a distinct non-success state, never +# passed. Used by current-state reporting so a cancelled-only or empty check +# set cannot be read as checks-passed. +# +# Public interface: fm_ci_ready_verdict [conclusion...] +# Reads conclusions from the arguments, or from stdin (one per line) when +# none are given. Blank lines are ignored. +# Prints exactly one of: passed | empty | not-passed +# Sourced by callers; also runnable as a command so tests drive the same +# public function through an executable interface. + +fm_ci_verdict_trim() { + local s=${1:-} + s="${s#"${s%%[![:space:]]*}"}" + s="${s%"${s##*[![:space:]]}"}" + printf '%s' "$s" +} + +# 0 if $1 is a completed success conclusion. +fm_ci_verdict_is_success() { + case "$1" in + SUCCESS|success) return 0 ;; + *) return 1 ;; + esac +} + +fm_ci_ready_verdict() { + local raw conclusion saw_success=0 + if [ "$#" -gt 0 ]; then + for raw in "$@"; do + conclusion=$(fm_ci_verdict_trim "$raw") + [ -n "$conclusion" ] || continue + if fm_ci_verdict_is_success "$conclusion"; then + saw_success=1 + continue + fi + printf 'not-passed' + return 0 + done + else + while IFS= read -r raw || [ -n "$raw" ]; do + conclusion=$(fm_ci_verdict_trim "$raw") + [ -n "$conclusion" ] || continue + if fm_ci_verdict_is_success "$conclusion"; then + saw_success=1 + continue + fi + printf 'not-passed' + return 0 + done + fi + if [ "$saw_success" = 1 ]; then + printf 'passed' + else + printf 'empty' + fi +} + +if [ "${BASH_SOURCE[0]}" = "$0" ]; then + set -u + fm_ci_ready_verdict "$@" + printf '\n' +fi diff --git a/bin/fm-crew-state.sh b/bin/fm-crew-state.sh index 2cb290373c..c571f134f9 100755 --- a/bin/fm-crew-state.sh +++ b/bin/fm-crew-state.sh @@ -30,11 +30,15 @@ # diverged from it, invalidates attribution. # The run-step is AUTHORITATIVE: running/fixing -> working, ci -> working, # awaiting_approval/fix_review -> parked (with gate findings), terminal -# passed/checks-passed -> done, failed/cancelled -> failed. EXCEPT: while -# the active step is ci, `axi status` alone cannot tell "still waiting on -# checks" from "checks green, waiting on merge" (see nm_ci_checks_state) - -# a ci-step log-tail check overrides working -> done once checks read -# green, so a green PR is never silently read as still-validating. +# passed -> done, failed/cancelled -> failed. A checks-passed outcome is +# done only when the CI-ready scorer in bin/fm-ci-verdict-lib.sh confirms +# at least one successful check and no pending, failed, cancelled, +# skipped, or never-run conclusions; zero checks is a distinct non-success +# state, never done. EXCEPT: while the active step is ci, `axi status` +# alone cannot tell "still waiting on checks" from "checks green, waiting +# on merge" (see nm_ci_checks_state) - a ci-step log-tail check overrides +# working -> done only when that same scorer reads green, so a cancelled +# or empty check set is never silently read as still-validating-and-green. # 3. Reconcile the status log: if its last line says needs-decision/blocked but # the run-step shows the run moved on, the log is deterministically stale and # is flagged superseded. A genuinely parked run plus a needs-decision log @@ -66,6 +70,8 @@ STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" . "$SCRIPT_DIR/fm-busy-lib.sh" # shellcheck source=bin/fm-nm-run-lib.sh . "$SCRIPT_DIR/fm-nm-run-lib.sh" +# shellcheck source=bin/fm-ci-verdict-lib.sh +. "$SCRIPT_DIR/fm-ci-verdict-lib.sh" ID=${1:-} [ -n "$ID" ] || { echo "usage: fm-crew-state.sh " >&2; exit 2; } @@ -281,26 +287,37 @@ nm_effective_ci_step_status() { # actually merged (or failed/cancelled if closed). `axi status`'s steps[] table # never distinguishes "still waiting on checks" from "checks green, waiting on # merge": both read as plain `ci,running,...`. The only place that transition is -# recorded is the ci step's own log text, e.g. "all CI checks passed - still -# monitoring until merged or closed" or "no CI checks reported - still -# monitoring until merged or closed" (verified against 360+ real run logs under -# ~/.no-mistakes/logs/*/ci.log on the installed v1.32.2 binary, including the -# actual PR #252 run). Reads the ci step's log tail via `axi logs` and scans it -# for the MOST RECENT recognized marker (the log is append-only/chronological, -# so the last match is current): green with nothing red after it means CI is -# green right now, still only waiting on merge/close. +# recorded is the ci step's own log text. Reads the ci step's log tail via +# `axi logs` and scores the MOST RECENT recognized marker (the log is +# append-only/chronological, so the last match is current). Green is valid +# only when bin/fm-ci-verdict-lib.sh sees at least one success and no +# pending, failed, cancelled, skipped, or never-run conclusions. Zero checks +# is a distinct non-success state, never green. +nm_ci_marker_conclusion() { + case "$1" in + *"all CI checks passed"*) printf 'SUCCESS' ;; + *"CI check cancelled"*|"CI checks were cancelled"*|"cancelled without"*) printf 'CANCELLED' ;; + *"checks failed"*|"CI failures"*|"issues detected"*) printf 'FAILURE' ;; + *"CI checks running"*|"waiting for checks"*|"base branch advanced"*"re-arming CI monitor timeout"*) printf 'PENDING' ;; + *"no CI checks reported"*|"repository declares no CI"*) ;; + esac +} + nm_ci_checks_state() { - local run_id log_tail marker + local run_id log_tail marker conclusion verdict run_id=$(strip_quotes "$(nm_field id)") [ -n "$run_id" ] || { printf 'unknown'; return; } log_tail=$(nm_run axi logs --step ci --run "$run_id") || true [ -n "$log_tail" ] || { printf 'unknown'; return; } marker=$(printf '%s\n' "$log_tail" \ - | grep -E 'CI checks passed|no CI checks reported - still monitoring|no CI checks reported yet|checks failed|issues detected|CI checks running|base branch advanced.*re-arming CI monitor timeout' \ + | grep -E 'CI checks passed|no CI checks reported|repository declares no CI|checks failed|issues detected|CI checks running|base branch advanced.*re-arming CI monitor timeout|CI check cancelled|CI checks were cancelled|cancelled without' \ | tail -1) - case "$marker" in - *"checks passed"*|*"no CI checks reported - still monitoring"*) printf 'green' ;; - *"no CI checks reported yet"*|*"checks failed"*|*"issues detected"*|*"CI checks running"*|*"base branch advanced"*"re-arming CI monitor timeout"*) printf 'not-ready' ;; + [ -n "$marker" ] || { printf 'unknown'; return; } + conclusion=$(nm_ci_marker_conclusion "$marker") + verdict=$(fm_ci_ready_verdict "$conclusion") + case "$verdict" in + passed) printf 'green' ;; + empty|not-passed) printf 'not-ready' ;; *) printf 'unknown' ;; esac } @@ -448,7 +465,23 @@ if [ "$HAVE_RUN" = 1 ]; then if [ -n "$outcome" ]; then case "$outcome" in passed) RUN_STATE="done"; RUN_DETAIL="run passed: PR merged/closed" ;; - checks-passed) RUN_STATE="done"; RUN_DETAIL="checks green: PR ready for review" ;; + checks-passed) + CI_LOG_STATE=$(nm_ci_checks_state) + case "$CI_LOG_STATE" in + green) + RUN_STATE="done" + RUN_DETAIL="checks green: PR ready for review" + ;; + not-ready) + RUN_STATE=blocked + RUN_DETAIL="run reported checks-passed without green check evidence" + ;; + *) + RUN_STATE=unknown + RUN_DETAIL="run reported checks-passed but CI evidence is unavailable" + ;; + esac + ;; failed) RUN_STATE=failed; RUN_DETAIL="run failed" ;; cancelled) RUN_STATE=failed; RUN_DETAIL="run cancelled" ;; *) RUN_STATE=unknown; RUN_DETAIL="outcome: $outcome" ;; @@ -496,10 +529,7 @@ if [ "$HAVE_RUN" = 1 ]; then fi fi - if [ "$RUN_STATE" = working ] && log_reports_ci_ready; then - if [ "$RUN_SOURCE" = coarse ]; then - emit "done" status-log "$(status_line_note "$LOG_LINE")${SEP}run still monitoring PR" - fi + if [ "$RUN_STATE" = working ] && log_reports_ci_ready && [ "$RUN_SOURCE" != coarse ]; then [ -n "$CI_STEP_STATUS" ] || CI_STEP_STATUS=$(nm_effective_ci_step_status) if [ "$RUN_STATUS" = fixing ]; then CI_LOG_STATE=not-ready @@ -508,7 +538,7 @@ if [ "$HAVE_RUN" = 1 ]; then elif [ "$CI_STEP_STATUS" = fixing ]; then CI_LOG_STATE=not-ready fi - if [ "$CI_LOG_STATE" != not-ready ]; then + if [ "$CI_LOG_STATE" = green ]; then emit "done" status-log "$(status_line_note "$LOG_LINE")${SEP}run still monitoring PR" fi fi diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 4ca26c865e..2ee6488e70 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -135,7 +135,7 @@ family_for_basename() { fm-arm-pretool-check.test.sh|fm-ask-user-authority.test.sh|\ fm-brief.test.sh|fm-vendor-auth-probe.test.sh|\ fm-calm-pi-extension.test.sh|fm-cd-pretool-check.test.sh|\ - fm-classify-decision-key.test.sh|\ + fm-ci-verdict-lib.test.sh|fm-classify-decision-key.test.sh|\ fm-composer-ghost.test.sh|fm-composer-lib.test.sh|\ fm-crew-state.test.sh|fm-decision-hold-lifecycle.test.sh|\ fm-documentation-audiences.test.sh|fm-ensure-agents-md.test.sh|fm-grok-harness.test.sh|\ @@ -938,6 +938,9 @@ families_for_changed_path() { printf '%s\n' pure-contract-unit printf '%s\n' pr-forge ;; + bin/fm-ci-verdict-lib.sh) + printf '%s\n' pure-contract-unit + ;; bin/fm-composer-lib.sh) # The shared shape catalogue is vendor-rendered signal; a change to it # re-selects the live guard (fm-composer-matrix-live-e2e) alongside the diff --git a/docs/architecture.md b/docs/architecture.md index afca3208d7..404047564d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -44,7 +44,7 @@ Any direct or remaining historical annotation prints every status line unread at `bin/fm-crew-state.sh ` is the cheap current-state read for an actionable heartbeat review: it attributes a no-mistakes run, active or terminal, only when it matches the crew's branch and current code identity, then keeps that run-step authoritative even if the pane has closed. The script header owns the exact run-head ancestry rules. During no-mistakes' `ci` monitor phase, it also reads the ci step log tail because `axi status` reports both "still waiting on checks" and "checks green, waiting on merge" as `ci,running`. -The most recent recognized ci log marker wins, so checks-green monitoring reports done while a later re-arm, failed-check, or issue marker returns the crew to working. +The most recent recognized ci log marker wins, and `bin/fm-ci-verdict-lib.sh` scores it: only a completed success with no pending, failed, cancelled, skipped, or never-run conclusions reports done, while a cancelled, empty, or later re-arm, failed-check, or issue marker returns the crew to working. Only when no matching run exists does it consult semantic busy state; exact busy reports working, exact idle permits fallback to a status-log event whose verb maps to a recognized run-state, and unknown or a dead pane stays unknown instead of trusting a stale log. Decision-only events such as `resolved` never become current state or leak their prose into the current-state detail. In that status-log fallback, a declared external wait reports the distinct `paused` state with its reason. diff --git a/docs/scripts.md b/docs/scripts.md index 484911c380..8dfe029872 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -79,6 +79,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-supervise-daemon.sh` | Presence-gated away-mode sub-supervisor: self-handle routine wakes, guard injection by the detected primary harness, escalate batched digests, alert on failed delivery | | `fm-crew-state.sh` | Print one deterministic current-state line for a crew | | `fm-nm-run-lib.sh` | Shared branch-and-code-identity attribution for no-mistakes runs | +| `fm-ci-verdict-lib.sh` | Score a CI check-conclusion set so empty or cancelled results cannot read as passed | | `fm-tangle-lib.sh` | Shared default-branch resolution and primary-checkout tangle classification | | `fm-timeout-lib.sh` | Single owner of hard-bounded command execution and its fallback watchdog | | `fm-timing-lib.sh` | Single owner of the deferred network stage's per-step elapsed-time records, inert unless a run asks for them | diff --git a/tests/fm-ci-verdict-lib.test.sh b/tests/fm-ci-verdict-lib.test.sh new file mode 100755 index 0000000000..1a8a4bf14a --- /dev/null +++ b/tests/fm-ci-verdict-lib.test.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Behavior tests for bin/fm-ci-verdict-lib.sh - the CI-ready scoring rule. +# +# A green verdict is valid only when at least one check completed with success +# AND none are pending, failed, cancelled, skipped, or never-run. These cases +# drive the public command, never the implementation source: +# (a) cancelled-only conclusions are not passed +# (b) zero checks is empty, not passed +# (c) a genuinely all-success set is passed +# (d) mixed success-plus-non-success is not passed +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +VERDICT="$ROOT/bin/fm-ci-verdict-lib.sh" + +score() { + "$VERDICT" "$@" +} + +test_cancelled_only_is_not_passed() { + local out + out=$(score CANCELLED) + [ "$out" = not-passed ] || fail "cancelled-only scored as '$out', want not-passed" + out=$(score CANCELED) + [ "$out" = not-passed ] || fail "canceled-only scored as '$out', want not-passed" + out=$(score SUCCESS CANCELLED) + [ "$out" = not-passed ] || fail "success+cancelled scored as '$out', want not-passed" + [ "$out" != passed ] || fail "cancelled conclusions must never score as passed" + pass "cancelled-only and mixed-cancelled conclusions are not passed" +} + +test_zero_checks_is_not_passed() { + local out + out=$(score) + [ "$out" = empty ] || fail "zero checks scored as '$out', want empty" + [ "$out" != passed ] || fail "zero checks must never score as passed" + out=$(printf '\n\n' | "$VERDICT") + [ "$out" = empty ] || fail "blank-only input scored as '$out', want empty" + [ "$out" != passed ] || fail "blank-only input must never score as passed" + pass "zero checks is empty, not passed" +} + +test_all_success_is_passed() { + local out + out=$(score SUCCESS) + [ "$out" = passed ] || fail "one success scored as '$out', want passed" + out=$(score SUCCESS SUCCESS SUCCESS) + [ "$out" = passed ] || fail "all-success scored as '$out', want passed" + pass "a genuinely all-success set is passed" +} + +test_skipped_pending_failed_never_run_are_not_passed() { + local out word + for word in SKIPPED NEUTRAL FAILURE ERROR TIMED_OUT ACTION_REQUIRED \ + STARTUP_FAILURE STALE QUEUED PENDING IN_PROGRESS WAITING REQUESTED \ + NEVER never-run; do + out=$(score "$word") + [ "$out" = not-passed ] || fail "$word scored as '$out', want not-passed" + [ "$out" != passed ] || fail "$word must never score as passed" + out=$(score SUCCESS "$word") + [ "$out" = not-passed ] || fail "success+$word scored as '$out', want not-passed" + done + pass "skipped, pending, failed, and never-run conclusions are not passed" +} + +test_cancelled_only_is_not_passed +test_zero_checks_is_not_passed +test_all_success_is_passed +test_skipped_pending_failed_never_run_are_not_passed + +echo "all fm-ci-verdict-lib tests passed" diff --git a/tests/fm-crew-state.test.sh b/tests/fm-crew-state.test.sh index 8f986b6139..46313ad9bf 100755 --- a/tests/fm-crew-state.test.sh +++ b/tests/fm-crew-state.test.sh @@ -13,6 +13,8 @@ # (b) needs-decision/blocked log + resumed run = SUPERSEDED -> run-step # (c) genuine parked run + needs-decision log = NOT superseded -> run-step # (d) terminal run-step (passed/failed) is authoritative -> run-step +# checks-passed is done only with scored green evidence; cancelled, +# skipped, never-run, or zero checks stay non-success # (e) cross-branch attribution: this branch's own run found via list lookup # (f) no run + semantic busy -> pane # (g) no run + semantic idle falls to the status-log verb -> status-log @@ -278,6 +280,19 @@ outcome: passed EOF } +run_checks_passed() { # + cat < cat < "$d/state/feat-ci.status" FM_FAKE_AXI_STATUS="$(run_ci_monitoring fm/feat-ci)" local out; out=$(run_crew_state "$d" feat-ci) - assert_contains "$out" "state: done" "ci-ready status log -> done" - assert_contains "$out" "source: status-log" "ci-ready state comes from the status log" - assert_contains "$out" "checks green" "ci-ready detail preserves the report" - assert_not_contains "$out" "state: working" "ci-ready is not hidden by monitoring run" - pass "ci-ready status log beats monitoring run" + assert_contains "$out" "state: working" "uncorroborated ci-ready status log stays working" + assert_not_contains "$out" "state: done" "a checks-green claim without evidence must not be done" + assert_not_contains "$out" "checks green" "a checks-green claim without evidence must not read as checks green" + pass "uncorroborated ci-ready status log does not beat a monitoring run" } # Regression for the PR #252 incident: the crew's own status log never got a @@ -499,7 +513,7 @@ test_top_level_ci_checks_green_surfaces_done() { pass "top-level ci status uses ci log green marker" } -test_ci_monitoring_no_checks_terminal_surfaces_done() { +test_ci_monitoring_no_checks_terminal_is_not_green() { reset_fakes local d; d=$(new_case ci-nochecks) make_repo_on_branch "$d/wt" fm/feat-cinochecks @@ -508,9 +522,53 @@ test_ci_monitoring_no_checks_terminal_surfaces_done() { FM_FAKE_AXI_STATUS="$(run_ci_monitoring fm/feat-cinochecks)" FM_FAKE_CI_LOGS="no CI checks reported - still monitoring until merged or closed" local out; out=$(run_crew_state "$d" feat-cinochecks) - assert_contains "$out" "state: done" "terminal no-checks ci-monitor run -> done" - assert_contains "$out" "checks green" "terminal no-checks ci-monitor detail mentions checks green" - pass "terminal no-checks ci-monitor marker surfaces done" + assert_contains "$out" "state: working" "terminal no-checks ci-monitor run -> working" + assert_not_contains "$out" "state: done" "zero checks must not read as done" + assert_not_contains "$out" "checks green" "zero checks must not read as checks green" + pass "terminal no-checks ci-monitor marker is not green" +} + +test_ci_monitoring_cancelled_only_is_not_green() { + reset_fakes + local d; d=$(new_case ci-cancelled) + make_repo_on_branch "$d/wt" fm/feat-cicancelled + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-cicancelled.meta" "window=fm:fm-feat-cicancelled" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_ci_monitoring fm/feat-cicancelled)" + FM_FAKE_CI_LOGS="CI checks were cancelled without reporting a verdict" + local out; out=$(run_crew_state "$d" feat-cicancelled) + assert_contains "$out" "state: working" "cancelled-only ci-monitor run -> working" + assert_not_contains "$out" "state: done" "cancelled checks must not read as done" + assert_not_contains "$out" "checks green" "cancelled checks must not read as checks green" + pass "cancelled-only ci-monitor marker is not green" +} + +test_checks_passed_without_green_evidence_is_not_done() { + reset_fakes + local d; d=$(new_case checks-passed-empty) + make_repo_on_branch "$d/wt" fm/feat-cpassedempty + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-cpassedempty.meta" "window=fm:fm-feat-cpassedempty" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_checks_passed fm/feat-cpassedempty)" + FM_FAKE_CI_LOGS="no CI checks reported - still monitoring until merged or closed" + local out; out=$(run_crew_state "$d" feat-cpassedempty) + assert_not_contains "$out" "state: done" "checks-passed with zero checks must not be done" + assert_not_contains "$out" "checks green" "checks-passed with zero checks must not read as checks green" + pass "a checks-passed claim with zero check evidence is not done" +} + +test_checks_passed_with_success_evidence_is_done() { + reset_fakes + local d; d=$(new_case checks-passed-green) + make_repo_on_branch "$d/wt" fm/feat-cpassedgreen + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-cpassedgreen.meta" "window=fm:fm-feat-cpassedgreen" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_checks_passed fm/feat-cpassedgreen)" + FM_FAKE_CI_LOGS="all CI checks passed - still monitoring until merged or closed" + local out; out=$(run_crew_state "$d" feat-cpassedgreen) + assert_contains "$out" "state: done" "corroborated checks-passed -> done" + assert_contains "$out" "checks green" "corroborated checks-passed names checks green" + pass "a checks-passed claim corroborated by success evidence is done" } test_ci_monitoring_green_then_rearm_stays_working() { @@ -754,10 +812,9 @@ EOF )" FM_FAKE_CI_LOGS="CI checks running, waiting for results..." local out; out=$(run_crew_state "$d" feat-coarseready) - assert_contains "$out" "state: done" "coarse ready status -> done" - assert_contains "$out" "source: status-log" "coarse ready status remains status-log sourced" - assert_not_contains "$out" "state: working" "coarse ready status must not be suppressed by another branch log" - pass "coarse run does not probe another branch's ci log" + assert_contains "$out" "state: working" "coarse ready status without evidence stays working" + assert_not_contains "$out" "state: done" "another branch's pending log must not corroborate a checks-green claim" + pass "coarse run does not treat another branch's ci log as this crew's green evidence" } # A different-branch run with NO matching runs-list row must NOT be @@ -1315,10 +1372,13 @@ test_stale_blocked_superseded test_genuine_parked_not_superseded test_scalar_gate_parked_not_superseded test_gate_block_parked_not_superseded -test_ci_ready_done_log_beats_monitoring_run +test_ci_ready_done_log_without_evidence_stays_working test_ci_monitoring_checks_green_surfaces_done test_top_level_ci_checks_green_surfaces_done -test_ci_monitoring_no_checks_terminal_surfaces_done +test_ci_monitoring_no_checks_terminal_is_not_green +test_ci_monitoring_cancelled_only_is_not_green +test_checks_passed_without_green_evidence_is_not_done +test_checks_passed_with_success_evidence_is_done test_ci_monitoring_green_then_rearm_stays_working test_ci_monitoring_no_checks_yet_stays_working test_ci_monitoring_still_waiting_stays_working From 3a96144cdc1befdb70cb301532b09c665d97ab75 Mon Sep 17 00:00:00 2001 From: Rodrigo Campos Date: Fri, 14 Aug 2026 00:26:28 -0400 Subject: [PATCH 2/6] no-mistakes(review): sync CI marker patterns and fail coarse completed closed --- bin/fm-ci-verdict-lib.sh | 32 ++++++---------- bin/fm-crew-state.sh | 19 ++++++---- tests/fm-crew-state.test.sh | 76 +++++++++++++++++++++++++++++++++++-- 3 files changed, 97 insertions(+), 30 deletions(-) diff --git a/bin/fm-ci-verdict-lib.sh b/bin/fm-ci-verdict-lib.sh index 95383723bb..9df7423b11 100755 --- a/bin/fm-ci-verdict-lib.sh +++ b/bin/fm-ci-verdict-lib.sh @@ -31,29 +31,21 @@ fm_ci_verdict_is_success() { fm_ci_ready_verdict() { local raw conclusion saw_success=0 - if [ "$#" -gt 0 ]; then - for raw in "$@"; do - conclusion=$(fm_ci_verdict_trim "$raw") - [ -n "$conclusion" ] || continue - if fm_ci_verdict_is_success "$conclusion"; then - saw_success=1 - continue - fi - printf 'not-passed' - return 0 - done - else + if [ "$#" -eq 0 ]; then while IFS= read -r raw || [ -n "$raw" ]; do - conclusion=$(fm_ci_verdict_trim "$raw") - [ -n "$conclusion" ] || continue - if fm_ci_verdict_is_success "$conclusion"; then - saw_success=1 - continue - fi - printf 'not-passed' - return 0 + set -- "$@" "$raw" done fi + for raw in "$@"; do + conclusion=$(fm_ci_verdict_trim "$raw") + [ -n "$conclusion" ] || continue + if fm_ci_verdict_is_success "$conclusion"; then + saw_success=1 + continue + fi + printf 'not-passed' + return 0 + done if [ "$saw_success" = 1 ]; then printf 'passed' else diff --git a/bin/fm-crew-state.sh b/bin/fm-crew-state.sh index c571f134f9..df59de9704 100755 --- a/bin/fm-crew-state.sh +++ b/bin/fm-crew-state.sh @@ -296,10 +296,10 @@ nm_effective_ci_step_status() { nm_ci_marker_conclusion() { case "$1" in *"all CI checks passed"*) printf 'SUCCESS' ;; - *"CI check cancelled"*|"CI checks were cancelled"*|"cancelled without"*) printf 'CANCELLED' ;; - *"checks failed"*|"CI failures"*|"issues detected"*) printf 'FAILURE' ;; - *"CI checks running"*|"waiting for checks"*|"base branch advanced"*"re-arming CI monitor timeout"*) printf 'PENDING' ;; - *"no CI checks reported"*|"repository declares no CI"*) ;; + *"CI check cancelled"*|*"CI checks were cancelled"*|*"cancelled without"*) printf 'CANCELLED' ;; + *"checks failed"*|*"CI failures"*|*"issues detected"*) printf 'FAILURE' ;; + *"CI checks running"*|*"waiting for checks"*|*"base branch advanced"*"re-arming CI monitor timeout"*) printf 'PENDING' ;; + *"no CI checks reported"*|*"repository declares no CI"*) ;; esac } @@ -310,7 +310,7 @@ nm_ci_checks_state() { log_tail=$(nm_run axi logs --step ci --run "$run_id") || true [ -n "$log_tail" ] || { printf 'unknown'; return; } marker=$(printf '%s\n' "$log_tail" \ - | grep -E 'CI checks passed|no CI checks reported|repository declares no CI|checks failed|issues detected|CI checks running|base branch advanced.*re-arming CI monitor timeout|CI check cancelled|CI checks were cancelled|cancelled without' \ + | grep -E 'CI checks passed|no CI checks reported|repository declares no CI|checks failed|CI failures|issues detected|CI checks running|waiting for checks|base branch advanced.*re-arming CI monitor timeout|CI check cancelled|CI checks were cancelled|cancelled without' \ | tail -1) [ -n "$marker" ] || { printf 'unknown'; return; } conclusion=$(nm_ci_marker_conclusion "$marker") @@ -440,15 +440,20 @@ if [ "$HAVE_RUN" = 1 ]; then RUN_STATUS="" if [ "$RUN_SOURCE" = coarse ]; then # No step/gate detail is available from the plain runs list - only ever - # true/working, done, or failed. A crew genuinely parked at a gate still + # working, failed, or unknown. A crew genuinely parked at a gate still # gets full detail once `axi status` reports its own branch again (e.g. # once its own step is the most-recently-touched one), and its own # needs-decision/blocked status-log append (a captain-relevant VERB) is # surfaced through signal_reason_is_actionable regardless of this # coarse-vs-full distinction, so a real gate is never silently missed. + # A `completed` row carries no outcome and no check evidence, and the + # ci-log scorer cannot run here ($RUN_OUT holds another run's id), so it + # cannot distinguish a merged run from one that claimed checks-passed over + # cancelled or zero checks. Reporting unknown keeps the same invariant the + # scorer enforces on the full path: done requires scored green evidence. case "$COARSE_STATUS" in running) RUN_STATE=working; RUN_DETAIL="validating (background run)" ;; - completed) RUN_STATE="done"; RUN_DETAIL="run completed" ;; + completed) RUN_STATE=unknown; RUN_DETAIL="run completed without check evidence" ;; failed) RUN_STATE=failed; RUN_DETAIL="run failed" ;; cancelled) RUN_STATE=failed; RUN_DETAIL="run cancelled" ;; *) RUN_STATE=unknown; RUN_DETAIL="runs list status: $COARSE_STATUS" ;; diff --git a/tests/fm-crew-state.test.sh b/tests/fm-crew-state.test.sh index 46313ad9bf..7f0034713c 100755 --- a/tests/fm-crew-state.test.sh +++ b/tests/fm-crew-state.test.sh @@ -543,6 +543,47 @@ test_ci_monitoring_cancelled_only_is_not_green() { pass "cancelled-only ci-monitor marker is not green" } +# Real ci.log lines carry a timestamp/indent prefix, and a red marker can land +# AFTER a green one. Both the marker selector and the conclusion classifier must +# see the prefixed later line, or the earlier green wins. +test_prefixed_red_marker_after_green_is_not_green() { + reset_fakes + local d; d=$(new_case ci-prefixed-red) + make_repo_on_branch "$d/wt" fm/feat-ciprefixed + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-ciprefixed.meta" "window=fm:fm-feat-ciprefixed" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_ci_monitoring fm/feat-ciprefixed)" + FM_FAKE_CI_LOGS="$(cat <<'EOF' +[12:00:03] all CI checks passed - still monitoring until merged or closed +[12:00:04] CI checks were cancelled without reporting a verdict +EOF +)" + local out; out=$(run_crew_state "$d" feat-ciprefixed) + assert_contains "$out" "state: working" "a later prefixed cancelled marker overrides the earlier green" + assert_not_contains "$out" "state: done" "prefixed cancelled marker must not read as done" + pass "prefixed red marker after green is not green" +} + +# "CI failures" is a red phrase the classifier recognizes; the marker selector +# must recognize it too, or the last green line stays the selected marker. +test_ci_failures_phrase_after_green_is_not_green() { + reset_fakes + local d; d=$(new_case ci-failures-phrase) + make_repo_on_branch "$d/wt" fm/feat-cifailures + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-cifailures.meta" "window=fm:fm-feat-cifailures" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_ci_monitoring fm/feat-cifailures)" + FM_FAKE_CI_LOGS="$(cat <<'EOF' +all CI checks passed - still monitoring until merged or closed +CI failures: 2 jobs red +EOF +)" + local out; out=$(run_crew_state "$d" feat-cifailures) + assert_contains "$out" "state: working" "a later CI failures line overrides the earlier green" + assert_not_contains "$out" "state: done" "CI failures must not read as done" + pass "CI failures marker after green is not green" +} + test_checks_passed_without_green_evidence_is_not_done() { reset_fakes local d; d=$(new_case checks-passed-empty) @@ -810,13 +851,39 @@ test_coarse_run_does_not_probe_other_branch_ci_log_for_ready_status() { running fm/feat-coarseready ${short} 2026-07-02 22:05 EOF )" - FM_FAKE_CI_LOGS="CI checks running, waiting for results..." + # The other branch's log is GREEN on purpose: any code that probed it would + # score green and emit done, so this assertion fails on a regression. A + # pending fake would pass either way and guard nothing. + FM_FAKE_CI_LOGS="all CI checks passed - still monitoring until merged or closed" local out; out=$(run_crew_state "$d" feat-coarseready) - assert_contains "$out" "state: working" "coarse ready status without evidence stays working" - assert_not_contains "$out" "state: done" "another branch's pending log must not corroborate a checks-green claim" + assert_contains "$out" "state: working" "coarse ready status without own evidence stays working" + assert_not_contains "$out" "state: done" "another branch's green log must not corroborate a checks-green claim" pass "coarse run does not treat another branch's ci log as this crew's green evidence" } +# A coarse `completed` row carries no outcome and no check evidence, and the +# ci-log scorer cannot run for it, so it must not reach done. +test_coarse_completed_row_is_not_done() { + reset_fakes + local d short; d=$(new_case coarse-completed) + make_repo_on_branch "$d/wt" fm/feat-coarsedone + short=$(git -C "$d/wt" rev-parse --short=7 HEAD) + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-coarsedone.meta" "window=fm:fm-feat-coarsedone" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_ci_monitoring fm/other-crew)" + FM_FAKE_RUNS_LIST="$(cat < Date: Fri, 14 Aug 2026 00:34:37 -0400 Subject: [PATCH 3/6] no-mistakes(review): reject outcomeless completed run and unblock verdict test stdin --- bin/fm-crew-state.sh | 7 +++++-- tests/fm-ci-verdict-lib.test.sh | 2 +- tests/fm-crew-state.test.sh | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/bin/fm-crew-state.sh b/bin/fm-crew-state.sh index df59de9704..753b72cb8e 100755 --- a/bin/fm-crew-state.sh +++ b/bin/fm-crew-state.sh @@ -30,7 +30,10 @@ # diverged from it, invalidates attribution. # The run-step is AUTHORITATIVE: running/fixing -> working, ci -> working, # awaiting_approval/fix_review -> parked (with gate findings), terminal -# passed -> done, failed/cancelled -> failed. A checks-passed outcome is +# passed -> done, failed/cancelled -> failed. A terminal run that carries +# no outcome at all (a bare `completed` status, whether from `axi status` +# or the coarse runs list) reads unknown, never done: there is no verdict +# to score. A checks-passed outcome is # done only when the CI-ready scorer in bin/fm-ci-verdict-lib.sh confirms # at least one successful check and no pending, failed, cancelled, # skipped, or never-run conclusions; zero checks is a distinct non-success @@ -510,7 +513,7 @@ if [ "$HAVE_RUN" = 1 ]; then case "$status" in ci) RUN_STATE=working; RUN_DETAIL="ci running" ;; running|fixing) RUN_STATE=working; RUN_DETAIL="validating ($status)" ;; - completed) RUN_STATE="done"; RUN_DETAIL="run completed" ;; + completed) RUN_STATE=unknown; RUN_DETAIL="run completed without an outcome" ;; failed) RUN_STATE=failed; RUN_DETAIL="run failed" ;; cancelled) RUN_STATE=failed; RUN_DETAIL="run cancelled" ;; "") RUN_STATE=working; RUN_DETAIL="run active" ;; diff --git a/tests/fm-ci-verdict-lib.test.sh b/tests/fm-ci-verdict-lib.test.sh index 1a8a4bf14a..a48554b206 100755 --- a/tests/fm-ci-verdict-lib.test.sh +++ b/tests/fm-ci-verdict-lib.test.sh @@ -16,7 +16,7 @@ set -u VERDICT="$ROOT/bin/fm-ci-verdict-lib.sh" score() { - "$VERDICT" "$@" + "$VERDICT" "$@" < /dev/null } test_cancelled_only_is_not_passed() { diff --git a/tests/fm-crew-state.test.sh b/tests/fm-crew-state.test.sh index 7f0034713c..e30d99c2a1 100755 --- a/tests/fm-crew-state.test.sh +++ b/tests/fm-crew-state.test.sh @@ -306,6 +306,20 @@ outcome: failed EOF } +# A completed run whose answer carries no outcome field at all: terminal status +# word, zero verdict evidence. +run_completed_no_outcome() { # + cat < cat </dev/null + fm_write_meta "$d/state/feat-cnoout.meta" "window=fm:fm-feat-cnoout" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_completed_no_outcome fm/feat-cnoout)" + FM_FAKE_CI_LOGS="all CI checks passed - still monitoring until merged or closed" + local out; out=$(run_crew_state "$d" feat-cnoout) + assert_contains "$out" "source: run-step" "completed-without-outcome run is still attributed" + assert_not_contains "$out" "state: done" "a completed run with no outcome must not be done" + assert_contains "$out" "run completed without an outcome" "the missing outcome is named" + pass "a completed run with no outcome is not done" +} + test_ci_monitoring_green_then_rearm_stays_working() { reset_fakes local d; d=$(new_case ci-green-then-rearm) @@ -1448,6 +1479,7 @@ test_prefixed_red_marker_after_green_is_not_green test_ci_failures_phrase_after_green_is_not_green test_checks_passed_without_green_evidence_is_not_done test_checks_passed_with_success_evidence_is_done +test_completed_without_outcome_is_not_done test_ci_monitoring_green_then_rearm_stays_working test_ci_monitoring_no_checks_yet_stays_working test_ci_monitoring_still_waiting_stays_working From 9eb7125c197e34904857c7c5b05a65ceafde9b9c Mon Sep 17 00:00:00 2001 From: Rodrigo Campos Date: Fri, 14 Aug 2026 01:17:33 -0400 Subject: [PATCH 4/6] no-mistakes(test): verify cancelled checks-passed guard; watcher flake is environmental --- tests/fm-crew-state.test.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/fm-crew-state.test.sh b/tests/fm-crew-state.test.sh index e30d99c2a1..297189ce0c 100755 --- a/tests/fm-crew-state.test.sh +++ b/tests/fm-crew-state.test.sh @@ -612,6 +612,25 @@ test_checks_passed_without_green_evidence_is_not_done() { pass "a checks-passed claim with zero check evidence is not done" } +# The cancelled half of the same rule. The ci-monitoring cancelled case above +# reads working even on the old code (an unrecognized marker scored unknown, so +# nothing overrode working), so only this terminal-outcome case actually pins +# "a cancelled-only run cannot produce checks-passed" from behavior. +test_checks_passed_cancelled_only_is_not_done() { + reset_fakes + local d; d=$(new_case checks-passed-cancelled) + make_repo_on_branch "$d/wt" fm/feat-cpassedcancel + make_fakebin "$d" >/dev/null + fm_write_meta "$d/state/feat-cpassedcancel.meta" "window=fm:fm-feat-cpassedcancel" "worktree=$d/wt" "kind=ship" + FM_FAKE_AXI_STATUS="$(run_checks_passed fm/feat-cpassedcancel)" + FM_FAKE_CI_LOGS="[12:00:04] CI checks were cancelled without reporting a verdict" + local out; out=$(run_crew_state "$d" feat-cpassedcancel) + assert_not_contains "$out" "state: done" "checks-passed over cancelled checks must not be done" + assert_not_contains "$out" "checks green" "cancelled checks must not read as checks green" + assert_contains "$out" "without green check evidence" "the missing green evidence is named" + pass "a checks-passed claim over cancelled-only checks is not done" +} + test_checks_passed_with_success_evidence_is_done() { reset_fakes local d; d=$(new_case checks-passed-green) @@ -1478,6 +1497,7 @@ test_ci_monitoring_cancelled_only_is_not_green test_prefixed_red_marker_after_green_is_not_green test_ci_failures_phrase_after_green_is_not_green test_checks_passed_without_green_evidence_is_not_done +test_checks_passed_cancelled_only_is_not_done test_checks_passed_with_success_evidence_is_done test_completed_without_outcome_is_not_done test_ci_monitoring_green_then_rearm_stays_working From 3c32b24b2057df5d0fb17bc900a6cd1d81fca23c Mon Sep 17 00:00:00 2001 From: Rodrigo Campos Date: Sat, 15 Aug 2026 00:52:00 -0400 Subject: [PATCH 5/6] no-mistakes(document): document scored CI verdict and outcomeless run states --- AGENTS.md | 2 +- docs/architecture.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 2e55687569..eaafbfc8e1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -346,7 +346,7 @@ Require the matching `resolved` event, forbid `--yes`, and require the worker to Resume fleet supervision immediately after the decision lands. Judge validation by the current-code-matched run step through `bin/fm-crew-state.sh`, not by shell liveness or the last status event. -Running, fixing, or CI states remain working; parked approval or fix-review states require the worker to follow the active gate help; passed is done; a checks-passed claim is done only when `bin/fm-crew-state.sh` scores at least one successful check and no pending, failed, cancelled, skipped, or never-run conclusions; failed or cancelled is failed. +Running, fixing, or CI states remain working; parked approval or fix-review states require the worker to follow the active gate help; passed is done; a checks-passed claim is done only when `bin/fm-crew-state.sh` scores at least one successful check and no pending, failed, cancelled, skipped, or never-run conclusions; a run that only reports completion, with no outcome to score, is unknown rather than done; failed or cancelled is failed. A worker hand-editing, committing, aborting, or restarting during an active validation run duplicates pipeline ownership outside the supersession sequence above; steer it back to the gate response flow. The worker reports the PR when CI first becomes green rather than waiting for merge monitoring to finish. diff --git a/docs/architecture.md b/docs/architecture.md index 404047564d..b7c55101b2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -45,6 +45,8 @@ Any direct or remaining historical annotation prints every status line unread at The script header owns the exact run-head ancestry rules. During no-mistakes' `ci` monitor phase, it also reads the ci step log tail because `axi status` reports both "still waiting on checks" and "checks green, waiting on merge" as `ci,running`. The most recent recognized ci log marker wins, and `bin/fm-ci-verdict-lib.sh` scores it: only a completed success with no pending, failed, cancelled, skipped, or never-run conclusions reports done, while a cancelled, empty, or later re-arm, failed-check, or issue marker returns the crew to working. +A terminal `checks-passed` outcome is scored through that same owner, so a claim the scorer does not read as green reports blocked, and a missing or unreadable ci log reports unknown, instead of done. +A terminal run that carries no outcome at all, including the coarse runs-list `completed` row that has no step, gate, or check evidence to score, reads unknown rather than done. Only when no matching run exists does it consult semantic busy state; exact busy reports working, exact idle permits fallback to a status-log event whose verb maps to a recognized run-state, and unknown or a dead pane stays unknown instead of trusting a stale log. Decision-only events such as `resolved` never become current state or leak their prose into the current-state detail. In that status-log fallback, a declared external wait reports the distinct `paused` state with its reason. From 8ae568005b06505aa975a2e1ff3dd959f121089e Mon Sep 17 00:00:00 2001 From: Rodrigo Campos Date: Mon, 17 Aug 2026 08:53:43 -0400 Subject: [PATCH 6/6] no-mistakes: apply CI fixes --- bin/fm-remote-job-worker.sh | 23 ++++++++++++++++++- tests/fm-remote-job.test.sh | 46 +++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/bin/fm-remote-job-worker.sh b/bin/fm-remote-job-worker.sh index 6046fdda36..e3b528261b 100755 --- a/bin/fm-remote-job-worker.sh +++ b/bin/fm-remote-job-worker.sh @@ -143,6 +143,21 @@ worker_recover_quarantine() { # rm -f -- "$WORKER_LOCK/quarantine" } +# An owner that died between mktemp and mv leaves one of its own private temp +# entries inside the lock directory, and rmdir can never reclaim a lock that +# still holds one. Only this worker's own temp shapes are removable, and only as +# plain files, so a proven-stale lock stays reclaimable after any unclean death +# without widening what reclaim may delete. +worker_clear_lock_residue() { + local entry + for entry in "$WORKER_LOCK"/.pid.* "$WORKER_LOCK"/.start.* \ + "$WORKER_LOCK"/.command.* "$WORKER_LOCK"/.quarantine.*; do + [ -e "$entry" ] || [ -L "$entry" ] || continue + [ ! -L "$entry" ] && [ -f "$entry" ] || return 1 + rm -f -- "$entry" || return 1 + done +} + worker_acquire_lock() { local account_home=$1 attempt=0 while [ "$attempt" -lt 150 ]; do @@ -164,6 +179,7 @@ worker_acquire_lock() { fi [ ! -L "$WORKER_LOCK/pid" ] && [ ! -L "$WORKER_LOCK/start" ] && [ ! -L "$WORKER_LOCK/command" ] || return 1 rm -f -- "$WORKER_LOCK/pid" "$WORKER_LOCK/start" "$WORKER_LOCK/command" || return 1 + worker_clear_lock_residue || return 1 rmdir "$WORKER_LOCK" || return 1 done return 1 @@ -291,8 +307,13 @@ worker_stop_active_execution() { WORKER_ACTIVE_JOB= } +# A replacement stop signals the whole worker tree and the supervisor signals +# this child again, so a second delivery always lands. Ignoring it for the +# shutdown sequence - rather than restoring the default disposition, which lets +# that second signal kill this process between publishing and clearing guarded +# ownership - is what keeps the sequence atomic and the lock free of residue. worker_shutdown() { - trap - HUP INT TERM + trap '' HUP INT TERM worker_publish_quarantine || { worker_error "cannot guard worker ownership for shutdown" trap worker_shutdown HUP INT TERM diff --git a/tests/fm-remote-job.test.sh b/tests/fm-remote-job.test.sh index f2ef8ce643..a09ccc5f54 100755 --- a/tests/fm-remote-job.test.sh +++ b/tests/fm-remote-job.test.sh @@ -264,6 +264,52 @@ fm_remote_job_ensure_worker "$REMOTE_ROOT" "$ACCOUNT_HOME" || fail "$FM_REMOTE_J NEW_WORKER_PID=$(cat "$STATE_ROOT/worker.pid") pass "worker identity binds the canonical configured code root" +# Every replacement stop signals the whole worker tree, and the supervisor in +# that tree signals the serving child again, so a second delivery always lands +# on a child that is already shutting down. A child that dies inside its own +# shutdown leaves ownership behind, and the replacement then has to reclaim it +# instead of serving. +STOPPED_WORKER_PID=$NEW_WORKER_PID +STOPPED_WORKER_PGID=$(fm_remote_job_worker_process_group "$STOPPED_WORKER_PID") \ + || fail "the repeated-stop fixture could not resolve an isolated worker group" +kill -TERM -- "-$STOPPED_WORKER_PGID" 2>/dev/null || true +for _ in $(seq 1 100); do + kill -0 "$STOPPED_WORKER_PID" 2>/dev/null || break + kill -TERM "$STOPPED_WORKER_PID" 2>/dev/null || true + sleep 0.003 +done +wait "$STOPPED_WORKER_PID" 2>/dev/null || true +for _ in $(seq 1 200); do + kill -0 "$STOPPED_WORKER_PID" 2>/dev/null || break + sleep 0.05 +done +kill -0 "$STOPPED_WORKER_PID" 2>/dev/null && fail "the repeatedly signalled worker did not stop" +assert_absent "$STATE_ROOT/worker.lock" "a repeatedly signalled shutdown left worker ownership held" +assert_absent "$STATE_ROOT/worker.ready" "a repeatedly signalled shutdown left a readiness heartbeat behind" +assert_absent "$STATE_ROOT/worker.pid" "a repeatedly signalled shutdown left a worker pid behind" +fm_remote_job_ensure_worker "$REMOTE_ROOT" "$ACCOUNT_HOME" || fail "$FM_REMOTE_JOB_ERROR" +NEW_WORKER_PID=$(cat "$STATE_ROOT/worker.pid") +pass "a repeated stop signal never leaves worker ownership behind" + +# An owner that died between mktemp and mv leaves one of its own private temp +# entries inside the ownership lock. rmdir can never reclaim a lock that still +# holds one, so without residue clearing every replacement worker exits and the +# account reads as a worker that never becomes ready. +RESIDUE_WORKER_PID=$NEW_WORKER_PID +fm_remote_job_stop_worker_tree "$RESIDUE_WORKER_PID" \ + || fail "the lock residue fixture could not stop the worker" +mkdir -p "$STATE_ROOT/worker.lock" +LOCK_RESIDUE="$STATE_ROOT/worker.lock/.pid.aB3xY9" +printf '%s\n' "$RESIDUE_WORKER_PID" > "$LOCK_RESIDUE" +touch -t 200001010000 "$STATE_ROOT/worker.lock" +fm_remote_job_ensure_worker "$REMOTE_ROOT" "$ACCOUNT_HOME" || fail "$FM_REMOTE_JOB_ERROR" +NEW_WORKER_PID=$(cat "$STATE_ROOT/worker.pid") +kill -0 "$NEW_WORKER_PID" 2>/dev/null || fail "the reclaiming worker is not running" +assert_absent "$LOCK_RESIDUE" "reclaim left a dead owner's private lock residue behind" +fm_remote_job_worker_identity_matches "$REMOTE_ROOT" "$ACCOUNT_HOME" \ + || fail "the reclaiming worker did not publish the current code identity" +pass "a lock left with a dead owner's private residue is still reclaimed" + CRASHED_WORKER_PID=$NEW_WORKER_PID kill -KILL "$CRASHED_WORKER_PID" wait "$CRASHED_WORKER_PID" 2>/dev/null || true