ci: split behavior coverage between local validation and required CI

.no-mistakes.yaml

@@ -18,20 +18,17 @@ disable_project_settings: true
 # the locked Agent Fleet ruff check that .github/workflows/ci.yml also invokes.
 # The shell definition stays single-owned, so local and CI cannot diverge
 # (parity asserted by tests/fm-lint.test.sh).
-# The test command mirrors the source checks in .github/workflows/ci.yml:
-# route every behavior test through tests/run.sh's sealed admission boundary,
-# then run Agent Fleet's locked pytest and compileall checks, and fail if any
-# check exits non-zero. The e2e tests need tmux on PATH, which the firstmate
-# environment provides.
-# Local and CI cannot diverge on the admission a test receives: CI fans the same
-# inventory across eight shards, and bin/fm-behavior-shards.sh admits every path
-# it selects through tests/run.sh, so both routes cross this same boundary.
-# This command keeps the single serial runner rather than reproducing that
-# concurrent fan-out locally, because the two routes differ in exactly one way
-# that matters here: CI shards select the explicit non-Herdr path
-# (FM_TEST_SKIP_HERDR=1) because their disposable image carries no Herdr, while
-# this gate never skips, so every real-Herdr declaration provisions an owned
-# Herdr lab and the serial route holds those labs to one at a time.
+# The ordinary local test command derives every herdr-lab and herdr-mixed file
+# from tests/test-capabilities.tsv, admits those files through tests/run.sh's
+# sealed safety and lab-ownership boundary, and runs them serially on the
+# real-Herdr path before Agent Fleet's locked pytest and compileall checks.
+# It deliberately does not duplicate hermetic-only behavior files on the Mac.
+# The required Behavior tests CI job owns the complete behavior inventory:
+# eight isolated runners admit every selected path through tests/run.sh, run
+# every hermetic body plus the hermetic portion of the mixed file, and verify
+# the exact complete executed-manifest union after all shards finish.
+# Local real-Herdr execution plus required CI is therefore the complete
+# behavioral coverage contract; local pre-push does not claim all 123 files.
 # The dispatch wrapper is local by default.
 # An operator may opt an exact command class into the private Azure runner; once
 # selected remote, a cloud or result failure is returned without local fallback.

bin/fm-azure-runner-dispatch.sh

@@ -299,9 +299,9 @@ if [ -n "${FM_AZURE_RUNNER_REMOTE_CLASSES:-}" ] && [ "$ROUTING_STATE" != absent
   fi
 fi
 
-# The no-mistakes test owner needs to choose between its ordinary full-local
-# suite and its remote-non-Herdr/local-Herdr split before it has a command to
-# hand this wrapper. Inspection shares the exact routing and disagreement
+# The no-mistakes test owner needs to choose between its ordinary local Herdr
+# host set and its remote-non-Herdr/local-Herdr split before it has a command
+# to hand this wrapper. Inspection shares the exact routing and disagreement
 # checks above but never consumes the routing budget. A later real dispatch
 # is required to present the returned binding, revalidates it under the stable
 # lock, and is the only consumer.

bin/fm-no-mistakes-test-command.sh

@@ -1,18 +1,21 @@
 #!/usr/bin/env bash
-# Own firstmate's no-mistakes test command, preserving the ordinary full local
-# suite while splitting only an explicitly Azure-selected test class into an
-# uncredentialed heavy Linux shard plus the required local Herdr lifecycle set.
-# A remote failure is never rerun locally.
+# Own firstmate's no-mistakes test command.
+# Ordinary local validation runs the capability-derived real-Herdr host set;
+# required CI owns the complete behavior inventory across isolated runners.
+# An explicitly Azure-selected test class retains its existing split, and a
+# remote failure is never rerun locally.
 set -euo pipefail
 
 ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)
 DISPATCH="$ROOT/bin/fm-azure-runner-dispatch.sh"
 
-run_full() {
+run_local_required() {
   command -v tmux >/dev/null || { echo "tmux is required for e2e tests" >&2; return 1; }
   tmux -V
+  printf 'no-mistakes: local host set files=%s source=tests/test-capabilities.tsv; complete behavior inventory is required in CI\n' \
+    "${#herdr_tests[@]}"
   local rc=0
-  "$ROOT/tests/run.sh" || rc=1
+  "$ROOT/tests/run.sh" "${herdr_tests[@]}" || rc=1
   uv run --directory "$ROOT/tools/agent-fleet" --locked pytest || rc=1
   uv run --directory "$ROOT/tools/agent-fleet" --locked python -m compileall -q src || rc=1
   return "$rc"
@@ -23,7 +26,7 @@ run_full() {
 # it here. This validates every present routing-file field and explicit local
 # recovery without spending a dispatch slot; only the real dispatch below may
 # consume one. Any malformed or disagreeing authority exits before either the
-# full-local suite or one half of the split can start.
+# local host set or one half of the split can start.
 set +e
 selection_output=$("$DISPATCH" --inspect-selection test)
 selection_rc=$?
@@ -33,6 +36,18 @@ selection_count=$(printf '%s\n' "$selection_output" | grep -c '^selection=' || t
 [ "$selection_count" -eq 1 ] \
   || { echo "azure-runner test selection inspection returned no unique decision" >&2; exit 1; }
 selection=$(printf '%s\n' "$selection_output" | sed -n 's/^selection=//p')
+
+# tests/test-capabilities.tsv is the only owner of the local host set.
+# tests/run.sh verifies that registry against every behavior file before it
+# admits these entries, then serializes their owned Herdr labs.
+herdr_tests=()
+while IFS=$'\t' read -r script capability; do
+  case "$capability" in
+    herdr-lab|herdr-mixed) herdr_tests+=("$ROOT/tests/$script") ;;
+  esac
+done < <(grep -v '^#' "$ROOT/tests/test-capabilities.tsv")
+[ "${#herdr_tests[@]}" -gt 0 ] || { echo "Herdr test inventory is unexpectedly empty" >&2; exit 1; }
+
 case "$selection" in
   local)
     reason_count=$(printf '%s\n' "$selection_output" | grep -c '^reason=' || true)
@@ -42,7 +57,7 @@ case "$selection" in
     [ -n "$reason" ] \
       || { echo "azure-runner local test selection returned an empty reason" >&2; exit 1; }
     printf 'azure-runner: class=test executed LOCALLY (%s)\n' "$reason" >&2
-    run_full
+    run_local_required
     exit $?
     ;;
   remote) ;;
@@ -58,14 +73,6 @@ selection_binding=$(printf '%s\n' "$selection_output" | sed -n 's/^selection_bin
 [[ "$selection_binding" =~ ^sha256:[0-9a-f]{64}$ ]] \
   || { echo "azure-runner remote test selection returned an invalid selection binding" >&2; exit 1; }
 
-herdr_tests=()
-while IFS=$'\t' read -r script capability; do
-  case "$capability" in
-    herdr-lab|herdr-mixed) herdr_tests+=("$ROOT/tests/$script") ;;
-  esac
-done < <(grep -v '^#' "$ROOT/tests/test-capabilities.tsv")
-[ "${#herdr_tests[@]}" -gt 0 ] || { echo "Herdr test inventory is unexpectedly empty" >&2; exit 1; }
-
 # The Azure shard excludes real Herdr by explicit sealed-suite admission while
 # the local shard runs every Herdr declaration through its owned guarded lab.
 # They run concurrently and report independently into this one command step.

docs/azure-runner.md

@@ -314,11 +314,11 @@ The resolver opens with no-follow semantics and reads only the exact device/inod
 Each selected dispatch consumes one `max_dispatches` slot under a stable sibling lock and durably records the new count before the runner starts.
 Delete the exact run file after the run; expiry and the dispatch budget remain fail-closed backstops, not garbage collection.
 
-The no-mistakes test owner must choose between its ordinary complete local suite and the remote-non-Herdr/local-Herdr split before it has a payload to dispatch.
+The no-mistakes test owner must choose between its ordinary capability-derived local Herdr set and the remote-non-Herdr/local-Herdr split before it has a payload to dispatch.
 It asks `bin/fm-azure-runner-dispatch.sh --inspect-selection test` for that decision.
 Inspection applies the same complete routing-document, local-recovery, and dual-authority checks but never consumes a dispatch slot, and returns an exact selection binding.
 The subsequent real remote dispatch must present that binding, refuses if the route was deleted, unselected, or replaced, revalidates under the stable lock, and is the only budget consumer.
-A local inspection writes the exact local reason into the test step's own stderr before the full suite starts, while a malformed present authority refuses before either local or remote tests start.
+A local inspection writes the exact local reason into the test step's own stderr before the local host set starts, while a malformed present authority refuses before either local or remote tests start.
 
 The selected daemon step then asks the resolver to read the landed Azure host configuration from the provenance-checked regular file `~/.fm-azure/fleet.env`.
 That file must supply `FM_AZURE_TENANT_ID`, `FM_AZURE_SUBSCRIPTION_ID`, `FM_AZURE_NAMING_PREFIX`, `FM_AZURE_STORAGE_NAME`, `FM_AZURE_OWNER_TAG`, `FM_AZURE_DEPLOYMENT_GENERATION`, and `FM_AZURE_BLOB_PE_NIC_RESOURCE_GUID`.
@@ -347,7 +347,7 @@ explicit local opt-out.
 The lint payload preserves the tracked shell owner and locked Agent Fleet command unchanged inside the dispatched argv.
 For a validation-owned feature branch, the caller passes its exact current `refs/heads/<branch>` identity plus the one-ref private snapshot bundle.
 The runner binds and privately stages that unpushed commit, while a changed local bundle/head/tree or public default base refuses before compute creation.
-The ordinary test path preserves the existing complete local command.
+The ordinary test path runs the capability-derived real-Herdr host set locally and leaves complete behavior-inventory verification to required CI, as defined in [`configuration.md`](configuration.md#gate-defaults-no-mistakesyaml).
 When the `test` class is explicitly remote, `bin/fm-no-mistakes-test-command.sh` runs the sealed non-Herdr behavior inventory plus locked Agent Fleet checks on one Azure VM while every real-Herdr declaration runs through owned guarded labs on the Mac; a failed Azure shard is never replayed locally.
 Model review, document generation that requires a model, fixes, Git mutation, push, PR creation, CI monitoring, and gate decisions remain in no-mistakes' existing owner.
 A configured uncredentialed documentation command may use this runner like any other command, but this bridge never moves a model document step by implication.

docs/configuration.md

@@ -94,12 +94,15 @@ See [`wedge-alarm.md`](wedge-alarm.md) for the channel reference and macOS verif
 
 ## Gate defaults (.no-mistakes.yaml)
 
-The tracked `.no-mistakes.yaml` keeps test evidence outside the repo and preserves `bin/fm-no-mistakes-test-command.sh` as the complete ordinary local test owner.
+The tracked `.no-mistakes.yaml` keeps test evidence outside the repo and preserves `bin/fm-no-mistakes-test-command.sh` as the ordinary local test owner.
 Inside an admitted Azure validation cell, the trusted default-branch command string instead invokes the root-owned bridge from `docs/azure-validation.md`, which runs lint and requested behavior shards on separate credential-free Azure command VMs without exposing the cell's provider or GitHub lease.
 That evidence policy is specific to the firstmate repo: target projects may legitimately commit `.no-mistakes/evidence/` from their own no-mistakes pipeline, but firstmate keeps `.no-mistakes/` local and CI rejects tracked entries under that path.
-That command requires `tmux` on `PATH`, prints `tmux -V`, routes behavior tests through `tests/run.sh`, and fails if any script exits non-zero.
-It intentionally runs the complete behavior-test inventory serially while [`.github/workflows/ci.yml`](../.github/workflows/ci.yml) uses the duration-balanced sharding owned by [`bin/fm-behavior-shards.sh`](../bin/fm-behavior-shards.sh); both routes cross the same sealed admission boundary instead of delegating the test step to an agent, so local-CI parity holds on what admission a test receives.
-The gate does not reproduce CI's concurrent shard fan-out because it does not select the explicit non-Herdr path: every real-Herdr declaration provisions an owned lab here, and the serial route holds those labs to one at a time.
+The ordinary local command requires `tmux` on `PATH`, prints `tmux -V`, derives every `herdr-lab` and `herdr-mixed` path directly from [`tests/test-capabilities.tsv`](../tests/test-capabilities.tsv), and sends those files in one serial invocation through [`tests/run.sh`](../tests/run.sh) before running the locked Agent Fleet pytest and compileall checks.
+It does not maintain a second file list or duplicate hermetic-only behavior files on the Mac.
+The required `Behavior tests` job in [`.github/workflows/ci.yml`](../.github/workflows/ci.yml) uses the duration-balanced sharding owned by [`bin/fm-behavior-shards.sh`](../bin/fm-behavior-shards.sh) to run and verify the complete behavior inventory across eight isolated runners.
+CI selects the explicit non-Herdr path because its disposable image carries no Herdr, so it runs every hermetic body and the hermetic portion of the mixed file while the local host admits the complete `herdr-lab` and `herdr-mixed` set through owned labs.
+The exact behavioral coverage contract is the union of that capability-derived local host set and the required CI executed-manifest union, not a claim that all 123 files run serially before push.
+Both routes cross the same sealed admission boundary, and the local serial route holds real-Herdr labs to one at a time.
 
 ## Crosscheck reviewer
 

tests/fm-azure-runner.test.sh

@@ -762,10 +762,18 @@ exit 0
 SH
   cat >"$root/tests/run.sh" <<SH
 #!/bin/sh
-printf '%s\n' "\$@" >>"$root/local-runs"
+{
+  printf 'call'
+  for arg do printf '\t%s' "\$arg"; done
+  printf '\n'
+} >>"$root/local-runs"
 exit 0
 SH
-  printf 'fm-fixture-herdr.test.sh\therdr-lab\n' >"$root/tests/test-capabilities.tsv"
+  cat >"$root/tests/test-capabilities.tsv" <<'TSV'
+fm-fixture-hermetic.test.sh	hermetic
+fm-fixture-herdr.test.sh	herdr-lab
+fm-fixture-mixed.test.sh	herdr-mixed
+TSV
   chmod +x "$root/bin/"* "$root/tests/run.sh"
 }
 
@@ -1000,6 +1008,21 @@ print(json.load(open(sys.argv[1])).get("dispatched", 0))
 PY
   }
 
+  assert_capability_derived_local_host_set() {
+    local log=$1
+    [ -s "$log" ] || fail "the capability-derived local host set did not cross tests/run.sh"
+    [ "$(wc -l < "$log" | tr -d ' ')" -eq 1 ] \
+      || fail "the local host set did not run in one serial tests/run.sh invocation"
+    grep -q 'fm-fixture-herdr.test.sh' "$log" \
+      || fail "the local host set omitted a herdr-lab registry row"
+    grep -q 'fm-fixture-mixed.test.sh' "$log" \
+      || fail "the local host set omitted a herdr-mixed registry row"
+    ! grep -q 'fm-fixture-hermetic.test.sh' "$log" \
+      || fail "the local host set duplicated a hermetic-only registry row before CI"
+    ! grep -q -- '--skip-herdr' "$log" \
+      || fail "the local host set selected the non-Herdr path"
+  }
+
   # The daemon-owned production path has no FM_AZURE_RUNNER_REMOTE_CLASSES.
   # A valid per-run selection must still choose the split lane, and inspection
   # must not spend a second budget slot before the real dispatch consumes one.
@@ -1011,10 +1034,7 @@ PY
   [ -f "$fixture/captured" ] || fail "a per-run test selection never reached the fixture runner"
   assert_contains "$out" "selected REMOTE resource-class=behavior-heavy" \
     "the per-run test selection emitted no remote-selection proof"
-  grep -q "fm-fixture-herdr.test.sh" "$fixture/local-runs" \
-    || fail "the per-run test selection lost the local Herdr shard"
-  grep -q -- "--skip-herdr" "$fixture/local-runs" 2>/dev/null \
-    && fail "the remotely selected non-Herdr shard also ran locally"
+  assert_capability_derived_local_host_set "$fixture/local-runs"
   [ "$(routing_dispatch_count)" -eq 1 ] \
     || fail "non-consuming test inspection did not leave exactly one durable dispatch spend"
 
@@ -1094,30 +1114,30 @@ SH
   [ ! -e "$fixture/captured" ] && [ ! -e "$fixture/local-runs" ] \
     || fail "the real test owner executed after a malformed-routing refusal"
 
-  # A valid document that does not select test preserves the complete ordinary
-  # local suite and records why the test class ran locally in the step log.
+  # A valid document that does not select test preserves the ordinary
+  # capability-derived host set and records why the class ran locally.
   write_test_routing '{"classes":{"lint":"validation-standard"}}'
   rm -f "$fixture/captured" "$fixture/local-runs"
   out=$(cd "$gatewt" && env HOME="$anchor" PATH="$fakebin:$PATH" \
     "$fixture/bin/fm-no-mistakes-test-command.sh" 2>&1) \
-    || fail "an unselected per-run test class did not preserve full local execution"
+    || fail "an unselected per-run test class did not preserve local host execution"
   [ ! -e "$fixture/captured" ] || fail "an unselected per-run test class reached the runner"
-  [ -e "$fixture/local-runs" ] || fail "an unselected per-run test class did not run locally"
+  assert_capability_derived_local_host_set "$fixture/local-runs"
   assert_contains "$out" "executed LOCALLY (routing=present-not-selected, env=absent)" \
     "an unselected per-run test class emitted no local-execution proof"
   [ "$(routing_dispatch_count)" -eq 0 ] \
     || fail "an unselected per-run test class spent a dispatch budget slot"
 
-  # Explicit recovery over a valid selected document runs the ordinary full
-  # suite locally, says so, and consumes no remote budget.
+  # Explicit recovery over a valid selected document runs the ordinary local
+  # host set, says so, and consumes no remote budget.
   write_test_routing '{}'
   rm -f "$fixture/captured" "$fixture/local-runs"
   out=$(cd "$gatewt" && env HOME="$anchor" PATH="$fakebin:$PATH" \
     FM_AZURE_RUNNER_LOCAL_RECOVERY_CLASSES=test \
     "$fixture/bin/fm-no-mistakes-test-command.sh" 2>&1) \
-    || fail "explicit per-run test recovery did not preserve full local execution"
+    || fail "explicit per-run test recovery did not preserve local host execution"
   [ ! -e "$fixture/captured" ] || fail "explicit per-run test recovery reached the runner"
-  [ -e "$fixture/local-runs" ] || fail "explicit per-run test recovery did not run locally"
+  assert_capability_derived_local_host_set "$fixture/local-runs"
   assert_contains "$out" "executed LOCALLY (explicit local recovery)" \
     "explicit per-run test recovery emitted no local-execution proof"
   [ "$(routing_dispatch_count)" -eq 0 ] \
@@ -1142,8 +1162,7 @@ assert value("--confirm-subscription")==sys.argv[4]
 assert value("--resource-class")=="behavior-heavy"
 assert any("tests/run.sh --skip-herdr" in item for item in argv), "the Azure shard lost the non-Herdr suite"
 PY
-  grep -q "fm-fixture-herdr.test.sh" "$fixture/local-runs" \
-    || fail "the local Herdr shard did not run its declared inventory"
+  assert_capability_derived_local_host_set "$fixture/local-runs"
 
   # An underivable subscription fails the step closed: exit 1, exact refusal,
   # no host fallback for the non-Herdr suite.