diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 8fa1f30c56..69fe9a8491 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -81,6 +81,7 @@ bin/fm-test-run.sh --lane portable-serial # portable serial remainder (watcher
bin/fm-test-run.sh --list-lanes # discover exact lane names, including the current CI serial shards
bin/fm-test-run.sh --check-coverage # prove portable shards + serial + serial shards + Herdr equal the full inventory
bin/fm-test-run.sh --all # deliberate complete regression (optional local full walk; not no-mistakes Test)
+bin/fm-test-run.sh --compare-commits --output-dir /tmp/fm-test-partition # bounded detached base/head inventories and failure partition
bin/fm-test-isolation-proof.sh --list # proven parallel candidate set (Phase 2 owner)
bin/fm-test-isolation-proof.sh --jobs 4 --json /tmp/fm-isolation-proof.json # re-run concurrent isolation proof only
[ "$(readlink CLAUDE.md)" = "AGENTS.md" ]
@@ -90,6 +91,7 @@ tmp=$(mktemp -d) && printf 'done: smoke\n' > "$tmp/smoke.status" && FM_STATE_OVE
`bin/fm-test-run.sh` is the single owner of behavior-suite selection, portable CI lane composition, optional local `--jobs` for the proven-isolated set only, per-script timing markers, family totals, the coverage guard, and the optional JSON timing artifact.
Its header and `--help` own the flags, family labels, lanes, and changed-file map; this section only documents the entry points.
+Its commit-comparison mode gives independent gates per-script hard bounds, detached local copies, independently reconciled inventories, honest measured-outcome coverage, and retry-confirmed transition buckets for inherited failures, timeouts, regressions, genuine fixes, coverage erosion, and flakes.
`bin/fm-test-isolation-proof.sh` remains the single owner of the Phase 2 concurrent isolation proof and the exact proven candidate set; see `docs/fm-test-isolation-proof.md`.
Portable shard balance evidence lives in `docs/fm-test-portable-shards.md`.
Local no-mistakes Test stays intent-targeted and must not wire `commands.test` to `--all` or a `tests/*.test.sh` walk.
diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh
index 4ca26c865e..8450565181 100755
--- a/bin/fm-test-run.sh
+++ b/bin/fm-test-run.sh
@@ -4,7 +4,7 @@
# timing markers, and the complete-regression coverage guard.
#
# Selection modes (exactly one of: --all, --family, --changed, --lane,
-# --proven-isolated, or script paths):
+# --proven-isolated, --compare-commits, or script paths):
# fm-test-run.sh --all
# fm-test-run.sh --family
# fm-test-run.sh --changed [--base ]
@@ -12,6 +12,7 @@
# fm-test-run.sh --lane portable-serial-of (one CI serial shard)
# fm-test-run.sh --proven-isolated
# fm-test-run.sh tests/.test.sh [more scripts...]
+# fm-test-run.sh --compare-commits --output-dir
#
# Inspection (no execution):
# fm-test-run.sh --list --all
@@ -24,6 +25,18 @@
# Aggregation (no suite execution):
# fm-test-run.sh --aggregate-json [more lane.json...]
#
+# Base/head partition (isolated execution):
+# fm-test-run.sh --compare-commits --output-dir [--script-timeout ]
+# Resolves both refs once, checks each out detached in an independent local
+# clone, and writes base.json, head.json, and partition.json. Every independently
+# enumerated script receives passed, failed, timed_out, skipped, or errored;
+# an interrupted in-flight row remains running. Each script runs in its own
+# process group under a hard bound (default: 300s). Candidate introduced and
+# regressed transitions are re-run once on both sides before classification.
+# The command exits non-zero for confirmed introduced/regressed outcomes,
+# deleted or skipped coverage, inventory refusal, or checkout-integrity error;
+# inherited failures and explicitly recorded flakes do not fail by themselves.
+#
# Options:
# --json write a deterministic timing artifact after the run
# --list print selected script paths (one per line) and exit 0
@@ -42,6 +55,9 @@
# selected script is in the proven-isolated set
# (bin/fm-test-isolation-proof.sh --list). Cap is 8. Stateful
# families never schedule under --jobs.
+# --script-timeout N
+# per-script bound in seconds for --compare-commits (default: 300).
+# --output-dir D fresh artifact directory for --compare-commits.
# -h, --help print this header
#
# Per-script machine-parseable markers (stdout):
@@ -79,6 +95,10 @@ LIST_FAMILIES=0
LIST_LANES=0
CHECK_COVERAGE=0
AGGREGATE_OUT=
+COMPARE_BASE=
+COMPARE_HEAD=
+COMPARE_OUTPUT_DIR=
+SCRIPT_TIMEOUT=300
FAMILY=
LANE=
BASE_REF=origin/main
@@ -1193,6 +1213,625 @@ with open(out, "w", encoding="utf-8") as fh:
PY
}
+compare_commits() {
+ local base_ref=$1 head_ref=$2 output_dir=$3 script_timeout=$4
+ command -v python3 >/dev/null 2>&1 || die "--compare-commits requires python3"
+ python3 - "$ROOT" "$base_ref" "$head_ref" "$output_dir" "$script_timeout" <<'PY'
+import collections
+import json
+import os
+import pathlib
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+
+root = pathlib.Path(sys.argv[1]).resolve()
+base_ref, head_ref = sys.argv[2], sys.argv[3]
+output_dir = pathlib.Path(sys.argv[4]).resolve()
+timeout_seconds = int(sys.argv[5])
+active_proc = None
+inventories = []
+confirmations = {}
+temp_root = None
+termination = None
+
+
+def git(*args, cwd=root, check=True):
+ return subprocess.run(
+ ["git", *args], cwd=cwd, check=check, text=True,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ )
+
+
+def resolve_commit(ref):
+ try:
+ return git("rev-parse", "--verify", f"{ref}^{{commit}}").stdout.strip()
+ except subprocess.CalledProcessError as exc:
+ raise RuntimeError(f"commit not found: {ref}: {exc.stderr.strip()}") from exc
+
+
+def atomic_json(path, value):
+ path.parent.mkdir(parents=True, exist_ok=True)
+ stage = path.with_name(path.name + ".tmp")
+ stage.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+ os.replace(stage, path)
+
+
+def checkout_state(copy):
+ head = git("rev-parse", "HEAD", cwd=copy).stdout.strip()
+ detached = git("symbolic-ref", "-q", "HEAD", cwd=copy, check=False).returncode != 0
+ clean = git("status", "--porcelain", "--untracked-files=all", cwd=copy).stdout == ""
+ return head, detached, clean
+
+
+def inventory_summary(rows):
+ counts = collections.Counter(row["outcome"] for row in rows)
+ return {
+ "total": len(rows),
+ "passed": counts["passed"],
+ "failed": counts["failed"],
+ "timed_out": counts["timed_out"],
+ "skipped": counts["skipped"],
+ "errored": counts["errored"],
+ "running": counts["running"],
+ }
+
+
+def reconcile(doc):
+ expected = doc["expected_scripts"]
+ discovered = doc["discovered_scripts"]
+ inventoried = [row["path"] for row in doc["scripts"]]
+ missing = sorted(set(expected) - set(inventoried))
+ extra = sorted(set(inventoried) - set(expected))
+ missing_from_discovery = sorted(set(expected) - set(discovered))
+ extra_in_discovery = sorted(set(discovered) - set(expected))
+ duplicates = sorted(path for path, count in collections.Counter(inventoried).items() if count != 1)
+ ok = bool(
+ expected == discovered == inventoried
+ and not missing and not extra
+ and not missing_from_discovery and not extra_in_discovery
+ and not duplicates
+ )
+ doc["reconciliation"] = {
+ "accounted": ok,
+ "expected_count": len(expected),
+ "discovered_count": len(discovered),
+ "inventory_count": len(inventoried),
+ "missing": missing,
+ "extra": extra,
+ "missing_from_discovery": missing_from_discovery,
+ "extra_in_discovery": extra_in_discovery,
+ "duplicates": duplicates,
+ }
+ doc["summary"] = inventory_summary(doc["scripts"])
+ return ok
+
+
+def write_inventory(doc):
+ reconcile(doc)
+ atomic_json(output_dir / f"{doc['label']}.json", doc)
+
+
+def first_meaningful_line(path):
+ with path.open(encoding="utf-8", errors="replace") as stream:
+ for line in stream:
+ if line.strip():
+ return line.strip()
+ return ""
+
+
+def stop_process_group(proc):
+ try:
+ os.killpg(proc.pid, signal.SIGTERM)
+ except ProcessLookupError:
+ return
+ try:
+ proc.wait(timeout=0.2)
+ return
+ except subprocess.TimeoutExpired:
+ pass
+ try:
+ os.killpg(proc.pid, signal.SIGKILL)
+ except ProcessLookupError:
+ pass
+ proc.wait()
+
+
+def tracked_test_scripts(copy, commit):
+ tree = subprocess.run(
+ ["git", "ls-tree", "-rz", "--name-only", commit, "--", "tests"],
+ cwd=copy, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ ).stdout
+ scripts = set()
+ for raw_path in tree.split(b"\0"):
+ if not raw_path:
+ continue
+ path = os.fsdecode(raw_path)
+ parts = pathlib.PurePosixPath(path).parts
+ if len(parts) >= 2 and parts[0] == "tests" and parts[1].endswith(".test.sh"):
+ scripts.add(f"tests/{parts[1]}")
+ return sorted(scripts)
+
+
+def make_inventory(label, requested_ref, commit, copy):
+ expected_scripts = tracked_test_scripts(copy, commit)
+ discovered_scripts = sorted(
+ str(path.relative_to(copy))
+ for path in (copy / "tests").glob("*.test.sh")
+ if path.is_file()
+ )
+ rows = []
+ for script in expected_scripts:
+ rows.append({
+ "path": script,
+ "outcome": "errored",
+ "exit_code": None,
+ "duration_ms": 0,
+ "detail": "not_run: runner interrupted before execution",
+ "log": f"{label}/logs/{pathlib.Path(script).name}.log",
+ })
+ head, detached, clean = checkout_state(copy)
+ doc = {
+ "schema_version": 2,
+ "label": label,
+ "requested_ref": requested_ref,
+ "commit": commit,
+ "timeout_seconds": timeout_seconds,
+ "checkout": {
+ "path": str(copy),
+ "initial_head": head,
+ "final_head": None,
+ "detached": detached,
+ "clean_before": clean,
+ "clean_after": None,
+ "invariant_ok": head == commit and detached and clean,
+ },
+ "expected_scripts": expected_scripts,
+ "discovered_scripts": discovered_scripts,
+ "scripts": rows,
+ }
+ write_inventory(doc)
+ return doc
+
+
+def run_inventory(doc, copy):
+ global active_proc
+ compromised = not doc["checkout"]["invariant_ok"]
+ for row in doc["scripts"]:
+ if compromised:
+ row["detail"] = "not_run: checkout invariant was already violated"
+ write_inventory(doc)
+ continue
+
+ before_head, before_detached, before_clean = checkout_state(copy)
+ if before_head != doc["commit"] or not before_detached or not before_clean:
+ row["detail"] = (
+ "not_run: checkout invariant violated before script "
+ f"(head={before_head}, detached={before_detached}, clean={before_clean})"
+ )
+ compromised = True
+ doc["checkout"]["invariant_ok"] = False
+ write_inventory(doc)
+ continue
+
+ script_path = copy / row["path"]
+ log_path = output_dir / row["log"]
+ log_path.parent.mkdir(parents=True, exist_ok=True)
+ if not script_path.is_file():
+ row["detail"] = "not_run: tracked test path is not a regular file"
+ write_inventory(doc)
+ continue
+
+ env = os.environ.copy()
+ for name in (
+ "FM_HOME", "FM_STATE_OVERRIDE", "FM_DATA_OVERRIDE", "FM_ROOT_OVERRIDE",
+ "FM_PROJECTS_OVERRIDE", "FM_CONFIG_OVERRIDE", "FM_BACKEND",
+ ):
+ env.pop(name, None)
+ script_tmp = pathlib.Path(tempfile.mkdtemp(prefix=f"fm-compare-{doc['label']}-"))
+ env["TMPDIR"] = str(script_tmp)
+ env["TMP"] = str(script_tmp)
+ started = time.monotonic()
+ try:
+ with log_path.open("wb") as log_stream:
+ proc = subprocess.Popen(
+ ["/bin/bash", row["path"]], cwd=copy, env=env,
+ stdout=log_stream, stderr=subprocess.STDOUT,
+ start_new_session=True,
+ )
+ active_proc = proc
+ row["outcome"] = "running"
+ row["detail"] = "in flight"
+ row["exit_code"] = None
+ write_inventory(doc)
+ try:
+ returncode = proc.wait(timeout=timeout_seconds)
+ except subprocess.TimeoutExpired:
+ stop_process_group(proc)
+ returncode = 124
+ row["outcome"] = "timed_out"
+ row["detail"] = f"per-script timeout after {timeout_seconds}s"
+ else:
+ if returncode == 0:
+ if first_meaningful_line(log_path).startswith("skip:"):
+ row["outcome"] = "skipped"
+ row["detail"] = "first meaningful output line is a gate skip"
+ else:
+ row["outcome"] = "passed"
+ row["detail"] = "exit 0"
+ elif returncode < 0:
+ row["outcome"] = "errored"
+ row["detail"] = f"terminated by signal {-returncode}"
+ else:
+ row["outcome"] = "failed"
+ row["detail"] = f"exit {returncode}"
+ row["exit_code"] = returncode
+ active_proc = None
+ except OSError as exc:
+ row["outcome"] = "errored"
+ row["detail"] = f"runner error: {exc}"
+ row["exit_code"] = None
+ finally:
+ active_proc = None
+ row["duration_ms"] = max(0, int((time.monotonic() - started) * 1000))
+ shutil.rmtree(script_tmp, ignore_errors=True)
+
+ after_head, after_detached, after_clean = checkout_state(copy)
+ if after_head != doc["commit"] or not after_detached or not after_clean:
+ row["outcome"] = "errored"
+ row["detail"] = (
+ "checkout invariant violated by script "
+ f"(head={after_head}, detached={after_detached}, clean={after_clean})"
+ )
+ compromised = True
+ doc["checkout"]["invariant_ok"] = False
+ write_inventory(doc)
+ print(
+ "FM_TEST_COMPARE_ROW "
+ f"side={doc['label']} script={row['path']} outcome={row['outcome']} "
+ f"exit={row['exit_code']} duration_ms={row['duration_ms']}",
+ flush=True,
+ )
+
+ final_head, final_detached, final_clean = checkout_state(copy)
+ doc["checkout"]["final_head"] = final_head
+ doc["checkout"]["clean_after"] = final_clean
+ doc["checkout"]["invariant_ok"] = bool(
+ doc["checkout"]["invariant_ok"]
+ and final_head == doc["commit"] and final_detached and final_clean
+ )
+ write_inventory(doc)
+
+
+def run_confirmation(doc, copy, path, attempt):
+ global active_proc
+ if path not in doc["expected_scripts"]:
+ confirmations[path][doc["label"]].append("absent")
+ return "absent"
+ script_path = copy / path
+ if not script_path.is_file():
+ confirmations[path][doc["label"]].append("errored")
+ return "errored"
+ before_head, before_detached, before_clean = checkout_state(copy)
+ if before_head != doc["commit"] or not before_detached or not before_clean:
+ doc["checkout"]["invariant_ok"] = False
+ write_inventory(doc)
+ confirmations[path][doc["label"]].append("errored")
+ return "errored"
+ log_path = output_dir / doc["label"] / "confirmations" / (
+ f"{pathlib.Path(path).name}.attempt-{attempt}.log"
+ )
+ log_path.parent.mkdir(parents=True, exist_ok=True)
+ env = os.environ.copy()
+ for name in (
+ "FM_HOME", "FM_STATE_OVERRIDE", "FM_DATA_OVERRIDE", "FM_ROOT_OVERRIDE",
+ "FM_PROJECTS_OVERRIDE", "FM_CONFIG_OVERRIDE", "FM_BACKEND",
+ ):
+ env.pop(name, None)
+ script_tmp = pathlib.Path(tempfile.mkdtemp(prefix=f"fm-confirm-{doc['label']}-"))
+ env["TMPDIR"] = str(script_tmp)
+ env["TMP"] = str(script_tmp)
+ outcome = "errored"
+ confirmations[path][doc["label"]].append("running")
+ write_current_partition()
+ try:
+ with log_path.open("wb") as log_stream:
+ proc = subprocess.Popen(
+ ["/bin/bash", path], cwd=copy, env=env,
+ stdout=log_stream, stderr=subprocess.STDOUT,
+ start_new_session=True,
+ )
+ active_proc = proc
+ try:
+ returncode = proc.wait(timeout=timeout_seconds)
+ except subprocess.TimeoutExpired:
+ stop_process_group(proc)
+ outcome = "timed_out"
+ else:
+ if returncode == 0:
+ outcome = (
+ "skipped" if first_meaningful_line(log_path).startswith("skip:")
+ else "passed"
+ )
+ elif returncode < 0:
+ outcome = "errored"
+ else:
+ outcome = "failed"
+ except OSError:
+ outcome = "errored"
+ finally:
+ active_proc = None
+ shutil.rmtree(script_tmp, ignore_errors=True)
+ after_head, after_detached, after_clean = checkout_state(copy)
+ if after_head != doc["commit"] or not after_detached or not after_clean:
+ doc["checkout"]["invariant_ok"] = False
+ outcome = "errored"
+ write_inventory(doc)
+ confirmations[path][doc["label"]][-1] = outcome
+ write_current_partition()
+ print(
+ "FM_TEST_COMPARE_CONFIRMATION "
+ f"side={doc['label']} script={path} attempt={attempt} outcome={outcome}",
+ flush=True,
+ )
+ return outcome
+
+
+def outcome_map(doc):
+ return {row["path"]: row["outcome"] for row in doc["scripts"]}
+
+
+def transition_entry(path, base_outcome, head_outcome, confirmation=None):
+ entry = {
+ "path": path,
+ "base_outcome": base_outcome,
+ "head_outcome": head_outcome,
+ }
+ if confirmation is not None:
+ entry["base_observations"] = confirmation["base"]
+ entry["head_observations"] = confirmation["head"]
+ return entry
+
+
+def make_partition(base, head):
+ base_rows = outcome_map(base)
+ head_rows = outcome_map(head)
+ all_paths = sorted(set(base["expected_scripts"]) | set(head["expected_scripts"]))
+ inherited = []
+ inherited_timeouts = []
+ introduced = []
+ regressed = []
+ fixed = []
+ now_passing = []
+ no_longer_measured = []
+ flaky = []
+ comparable_outcomes = 0
+ for path in all_paths:
+ base_outcome = base_rows.get(path, "absent")
+ head_outcome = head_rows.get(path, "absent")
+ confirmation = confirmations.get(path)
+ entry = transition_entry(path, base_outcome, head_outcome, confirmation)
+ if base_outcome in {"passed", "failed"} and head_outcome in {"passed", "failed"}:
+ comparable_outcomes += 1
+ if base_outcome == head_outcome:
+ if base_outcome in {"failed", "errored"}:
+ inherited.append(entry)
+ elif base_outcome == "timed_out":
+ inherited_timeouts.append(entry)
+ continue
+ if head_outcome == "absent" and base_outcome != "absent":
+ no_longer_measured.append(entry)
+ continue
+ if head_outcome == "skipped" and base_outcome not in {"absent", "skipped"}:
+ no_longer_measured.append(entry)
+ continue
+ if base_outcome == "failed" and head_outcome == "passed":
+ fixed.append(entry)
+ continue
+ if head_outcome == "passed" and base_outcome in {"timed_out", "errored", "skipped"}:
+ now_passing.append(entry)
+ continue
+ candidate_bucket = None
+ if base_outcome == "absent" and head_outcome in {"failed", "timed_out", "errored"}:
+ candidate_bucket = introduced
+ elif base_outcome != "absent" and head_outcome in {"failed", "timed_out", "errored"}:
+ candidate_bucket = regressed
+ if candidate_bucket is not None:
+ retry_outcomes = [] if confirmation is None else (
+ confirmation["base"][1:] + confirmation["head"][1:]
+ )
+ if (
+ confirmation is not None
+ and confirmation["head"][-1] == "passed"
+ and not any(outcome in {"errored", "running", "skipped", "timed_out"}
+ for outcome in retry_outcomes)
+ ):
+ flaky.append(entry)
+ else:
+ candidate_bucket.append(entry)
+ comparison_total = len(all_paths)
+ return {
+ "schema_version": 2,
+ "base_commit": base["commit"],
+ "head_commit": head["commit"],
+ "inventory_reconciled": bool(
+ base["reconciliation"]["accounted"]
+ and head["reconciliation"]["accounted"]
+ ),
+ "checkout_invariants_ok": bool(
+ base["checkout"]["invariant_ok"]
+ and head["checkout"]["invariant_ok"]
+ ),
+ "inherited_failures": inherited,
+ "inherited_timeouts": inherited_timeouts,
+ "head_introduced_failures": introduced,
+ "regressed": regressed,
+ "fixed": fixed,
+ "now_passing": now_passing,
+ "no_longer_measured": no_longer_measured,
+ "flaky_transitions": flaky,
+ "summary": {
+ "inherited": len(inherited),
+ "inherited_timeouts": len(inherited_timeouts),
+ "head_introduced": len(introduced),
+ "regressed": len(regressed),
+ "fixed": len(fixed),
+ "now_passing": len(now_passing),
+ "no_longer_measured": len(no_longer_measured),
+ "flaky": len(flaky),
+ "compared_outcomes": comparable_outcomes,
+ "comparison_total": comparison_total,
+ "coverage": f"{comparable_outcomes}/{comparison_total}",
+ },
+ }
+
+
+def confirm_regression_candidates(partition, base, head, copies):
+ paths = sorted(
+ row["path"]
+ for bucket in (partition["head_introduced_failures"], partition["regressed"])
+ for row in bucket
+ )
+ base_rows = outcome_map(base)
+ head_rows = outcome_map(head)
+ for path in paths:
+ observation = {
+ "base": [base_rows.get(path, "absent")],
+ "head": [head_rows.get(path, "absent")],
+ }
+ confirmations[path] = observation
+ run_confirmation(base, copies["base"], path, 1)
+ run_confirmation(head, copies["head"], path, 1)
+
+
+def current_partition():
+ if len(inventories) == 2:
+ partition = make_partition(*inventories)
+ else:
+ partition = {
+ "schema_version": 2,
+ "base_commit": base_commit,
+ "head_commit": head_commit,
+ "inventory_reconciled": False,
+ "checkout_invariants_ok": False,
+ "incomplete": True,
+ "inventories_available": len(inventories),
+ }
+ if termination is not None:
+ partition["termination"] = termination
+ return partition
+
+
+def write_current_partition():
+ atomic_json(output_dir / "partition.json", current_partition())
+
+
+def handle_signal(signum, _frame):
+ global termination
+ termination = {
+ "complete": False,
+ "signal": signal.Signals(signum).name,
+ "exit_code": 128 + signum,
+ }
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
+ if active_proc is not None:
+ stop_process_group(active_proc)
+ for doc in inventories:
+ write_inventory(doc)
+ write_current_partition()
+ raise SystemExit(128 + signum)
+
+
+base_commit = None
+head_commit = None
+try:
+ if timeout_seconds < 1:
+ raise RuntimeError("--script-timeout must be a positive integer")
+ if output_dir.exists():
+ if not output_dir.is_dir():
+ raise RuntimeError(f"--output-dir must be a directory: {output_dir}")
+ if any(output_dir.iterdir()):
+ raise RuntimeError(f"--output-dir must be absent or empty: {output_dir}")
+ output_dir.mkdir(parents=True, exist_ok=True)
+ base_commit = resolve_commit(base_ref)
+ head_commit = resolve_commit(head_ref)
+ signal.signal(signal.SIGINT, handle_signal)
+ signal.signal(signal.SIGTERM, handle_signal)
+ temp_root = pathlib.Path(tempfile.mkdtemp(prefix="fm-test-compare-"))
+ copies = {}
+ for label, commit in (("base", base_commit), ("head", head_commit)):
+ copy = temp_root / label
+ clone = subprocess.run(
+ ["git", "clone", "--no-local", "--no-checkout", "--quiet", str(root), str(copy)],
+ text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ )
+ if clone.returncode != 0:
+ raise RuntimeError(f"could not create isolated {label} copy: {clone.stderr.strip()}")
+ checkout = git("checkout", "--detach", "--quiet", commit, cwd=copy, check=False)
+ if checkout.returncode != 0:
+ raise RuntimeError(f"could not detach {label} at {commit}: {checkout.stderr.strip()}")
+ copies[label] = copy
+
+ base = make_inventory("base", base_ref, base_commit, copies["base"])
+ head = make_inventory("head", head_ref, head_commit, copies["head"])
+ inventories.extend((base, head))
+ run_inventory(base, copies["base"])
+ run_inventory(head, copies["head"])
+ partition = make_partition(base, head)
+ write_current_partition()
+ confirm_regression_candidates(partition, base, head, copies)
+ partition = make_partition(base, head)
+ write_current_partition()
+ for doc in (base, head):
+ summary = doc["summary"]
+ print(
+ "FM_TEST_COMPARE_INVENTORY "
+ f"side={doc['label']} commit={doc['commit']} total={summary['total']} "
+ f"passed={summary['passed']} failed={summary['failed']} "
+ f"timed_out={summary['timed_out']} skipped={summary['skipped']} "
+ f"errored={summary['errored']} running={summary['running']} "
+ f"accounted={str(doc['reconciliation']['accounted']).lower()} "
+ f"checkout_ok={str(doc['checkout']['invariant_ok']).lower()}",
+ flush=True,
+ )
+ summary = partition["summary"]
+ print(
+ "FM_TEST_COMPARE_PARTITION "
+ f"inherited={summary['inherited']} inherited_timeouts={summary['inherited_timeouts']} "
+ f"head_introduced={summary['head_introduced']} regressed={summary['regressed']} "
+ f"fixed={summary['fixed']} now_passing={summary['now_passing']} "
+ f"no_longer_measured={summary['no_longer_measured']} flaky={summary['flaky']} "
+ f"compared_outcomes={summary['compared_outcomes']} coverage={summary['coverage']} "
+ f"output_dir={output_dir}",
+ flush=True,
+ )
+ valid = partition["inventory_reconciled"] and partition["checkout_invariants_ok"]
+ refused = bool(
+ partition["head_introduced_failures"]
+ or partition["regressed"]
+ or partition["no_longer_measured"]
+ or not valid
+ )
+ raise SystemExit(1 if refused else 0)
+except BaseException as exc:
+ for doc in inventories:
+ write_inventory(doc)
+ if output_dir.exists() and output_dir.is_dir():
+ write_current_partition()
+ if isinstance(exc, SystemExit):
+ raise
+ print(f"fm-test-run: compare failed: {exc}", file=sys.stderr)
+ raise SystemExit(2)
+finally:
+ if temp_root is not None:
+ shutil.rmtree(temp_root, ignore_errors=True)
+PY
+}
+
while [ "$#" -gt 0 ]; do
case "$1" in
--all)
@@ -1236,6 +1875,14 @@ while [ "$#" -gt 0 ]; do
MODE=changed
shift
;;
+ --compare-commits)
+ [ -z "$MODE" ] || die "only one selection mode is allowed"
+ [ "$#" -gt 2 ] || die "--compare-commits requires "
+ MODE=compare
+ COMPARE_BASE=$2
+ COMPARE_HEAD=$3
+ shift 3
+ ;;
--base)
[ "$#" -gt 1 ] || die "--base requires a git ref"
BASE_REF=$2
@@ -1254,6 +1901,24 @@ while [ "$#" -gt 0 ]; do
JSON_PATH=${1#--json=}
shift
;;
+ --script-timeout)
+ [ "$#" -gt 1 ] || die "--script-timeout requires a positive integer"
+ SCRIPT_TIMEOUT=$2
+ shift 2
+ ;;
+ --script-timeout=*)
+ SCRIPT_TIMEOUT=${1#--script-timeout=}
+ shift
+ ;;
+ --output-dir)
+ [ "$#" -gt 1 ] || die "--output-dir requires a path"
+ COMPARE_OUTPUT_DIR=$2
+ shift 2
+ ;;
+ --output-dir=*)
+ COMPARE_OUTPUT_DIR=${1#--output-dir=}
+ shift
+ ;;
--jobs)
[ "$#" -gt 1 ] || die "--jobs requires a positive integer"
JOBS=$2
@@ -1358,6 +2023,17 @@ if [ "${MODE:-}" = "aggregate" ]; then
exit 0
fi
+if [ "${MODE:-}" = "compare" ]; then
+ [ "$LIST_ONLY" -eq 0 ] || die "--list cannot be combined with --compare-commits"
+ [ -n "$COMPARE_OUTPUT_DIR" ] || die "--compare-commits requires --output-dir "
+ case "$SCRIPT_TIMEOUT" in
+ ''|*[!0-9]*) die "--script-timeout must be a positive integer" ;;
+ esac
+ [ "$SCRIPT_TIMEOUT" -ge 1 ] || die "--script-timeout must be >= 1"
+ compare_commits "$COMPARE_BASE" "$COMPARE_HEAD" "$COMPARE_OUTPUT_DIR" "$SCRIPT_TIMEOUT"
+ exit $?
+fi
+
case "$JOBS" in
''|*[!0-9]*) die "--jobs must be a positive integer" ;;
esac
diff --git a/docs/scripts.md b/docs/scripts.md
index 484911c380..1450be455c 100644
--- a/docs/scripts.md
+++ b/docs/scripts.md
@@ -31,7 +31,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize
| `fm-install-herdr.sh` | Install CI's exact-version Herdr pin with official asset URL, SHA-256, and protocol checks |
| `fm-install-treehouse.sh`| Install CI's exact-version Treehouse pin for real-Herdr E2E that needs spawn worktrees |
| `fm-herdr-ci-cleanup.sh` | Snapshot and tear down only job-owned `fm-lab-*` sessions in the Herdr CI lane |
-| `fm-test-run.sh` | Behavior-test runner: selection, portable lanes, proven-isolated `--jobs`, coverage guard, timing/JSON |
+| `fm-test-run.sh` | Behavior-test runner: selection, portable lanes, proven-isolated `--jobs`, coverage guard, timing/JSON, and isolated base/head comparison |
| `fm-test-isolation-proof.sh` | Concurrent isolation proof and proven-isolated candidate set owner |
| `fm-ensure-agents-md.sh` | Ensure a project's real `AGENTS.md`, its `CLAUDE.md` symlink, and the canonical self-governance section |
| `fm-guard.sh` | Warn on primary-checkout tangles, pending queued wakes, and unhealthy supervision |
diff --git a/tests/fm-test-run.test.sh b/tests/fm-test-run.test.sh
index 21bdd69ba5..6b302fa646 100755
--- a/tests/fm-test-run.test.sh
+++ b/tests/fm-test-run.test.sh
@@ -1,7 +1,8 @@
#!/usr/bin/env bash
# Contract tests for bin/fm-test-run.sh - the single owner of behavior suite
# selection, portable lane composition, proven-isolated --jobs, timing markers,
-# JSON artifacts, coverage guard, and aggregate exit status.
+# JSON artifacts, coverage guard, aggregate exit status, and bounded detached
+# base/head failure partitioning.
#
# These tests intentionally exercise the runner with fixtures, --list, and
# focused scheduler checks, not the complete Firstmate suite.
@@ -669,6 +670,675 @@ assert len(doc["scripts"])==3
pass "aggregate-json merges lane timing artifacts"
}
+init_compare_fixture_repo() {
+ local repo=$1
+ mkdir -p "$repo/bin" "$repo/tests"
+ cp "$RUNNER" "$repo/bin/fm-test-run.sh"
+ chmod +x "$repo/bin/fm-test-run.sh"
+ cat >"$repo/tests/aa-pass.test.sh" <<'SH'
+#!/usr/bin/env bash
+echo "ok - pass"
+SH
+ cat >"$repo/tests/ab-inherited.test.sh" <<'SH'
+#!/usr/bin/env bash
+echo "not ok - inherited"
+exit 1
+SH
+ cat >"$repo/tests/ac-fixed.test.sh" <<'SH'
+#!/usr/bin/env bash
+echo "not ok - base failure"
+exit 1
+SH
+ cat >"$repo/tests/ad-introduced.test.sh" <<'SH'
+#!/usr/bin/env bash
+echo "ok - base pass"
+SH
+ cat >"$repo/tests/ae-skip.test.sh" <<'SH'
+#!/usr/bin/env bash
+echo "skip: optional fixture unavailable"
+SH
+ cat >"$repo/tests/af-hang.test.sh" <<'SH'
+#!/usr/bin/env bash
+trap '' TERM
+sleep 30
+SH
+ chmod +x "$repo"/tests/*.test.sh
+ git -C "$repo" init -q
+ git -C "$repo" add bin tests
+ git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm base
+}
+
+test_compare_commits_partitions_and_bounds_every_script() {
+ local tmp repo base head rc out
+ tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare.XXXXXX")
+ repo="$tmp/repo"
+ init_compare_fixture_repo "$repo"
+ base=$(git -C "$repo" rev-parse HEAD)
+ cat >"$repo/tests/ac-fixed.test.sh" <<'SH'
+#!/usr/bin/env bash
+echo "ok - fixed"
+SH
+ cat >"$repo/tests/ad-introduced.test.sh" <<'SH'
+#!/usr/bin/env bash
+echo "not ok - introduced"
+exit 1
+SH
+ chmod +x "$repo/tests/ac-fixed.test.sh" "$repo/tests/ad-introduced.test.sh"
+ git -C "$repo" add tests
+ git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm head
+ head=$(git -C "$repo" rev-parse HEAD)
+
+ set +e
+ out=$(cd "$repo" && bin/fm-test-run.sh --compare-commits "$base" "$head" \
+ --script-timeout 1 --output-dir "$tmp/result" 2>"$tmp/err")
+ rc=$?
+ set -e
+ [ "$rc" -eq 1 ] || { rm -rf "$tmp"; fail "introduced failure must make compare exit 1 (got $rc): $(cat "$tmp/err")"; }
+ assert_contains "$out" "FM_TEST_COMPARE_INVENTORY side=base" "base terminal inventory marker"
+ assert_contains "$out" "FM_TEST_COMPARE_INVENTORY side=head" "head terminal inventory marker"
+ assert_contains "$out" "FM_TEST_COMPARE_PARTITION inherited=1 inherited_timeouts=1 head_introduced=0 regressed=1 fixed=1" \
+ "outcome-transition partition"
+ if ! python3 - "$tmp/result/base.json" "$tmp/result/head.json" "$tmp/result/partition.json" <<'PY'
+import json, sys
+base, head, partition = (json.load(open(path, encoding="utf-8")) for path in sys.argv[1:])
+assert base["reconciliation"]["accounted"] is True, base
+assert head["reconciliation"]["accounted"] is True, head
+assert base["checkout"]["invariant_ok"] is True, base
+assert head["checkout"]["invariant_ok"] is True, head
+assert base["summary"] == {"errored": 0, "failed": 2, "passed": 2, "running": 0, "skipped": 1, "timed_out": 1, "total": 6}, base
+assert head["summary"] == {"errored": 0, "failed": 2, "passed": 2, "running": 0, "skipped": 1, "timed_out": 1, "total": 6}, head
+base_rows = {row["path"]: row["outcome"] for row in base["scripts"]}
+head_rows = {row["path"]: row["outcome"] for row in head["scripts"]}
+assert base_rows["tests/af-hang.test.sh"] == "timed_out", base_rows
+assert head_rows["tests/af-hang.test.sh"] == "timed_out", head_rows
+assert [row["path"] for row in partition["inherited_failures"]] == [
+ "tests/ab-inherited.test.sh"
+], partition
+assert [row["path"] for row in partition["inherited_timeouts"]] == ["tests/af-hang.test.sh"], partition
+assert partition["head_introduced_failures"] == [], partition
+assert [row["path"] for row in partition["regressed"]] == ["tests/ad-introduced.test.sh"], partition
+assert [row["path"] for row in partition["fixed"]] == ["tests/ac-fixed.test.sh"], partition
+assert partition["summary"]["coverage"] == "4/6", partition
+PY
+ then
+ rm -rf "$tmp"
+ fail "compare JSON inventories or partition are wrong"
+ fi
+ rm -rf "$tmp"
+ pass "commit comparison bounds a deliberate hang and emits complete diffable inventories"
+}
+
+test_compare_commits_accounts_for_script_lost_after_discovery() {
+ local tmp repo commit rc
+ tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare-missing.XXXXXX")
+ repo="$tmp/repo"
+ mkdir -p "$repo/bin" "$repo/tests"
+ cp "$RUNNER" "$repo/bin/fm-test-run.sh"
+ chmod +x "$repo/bin/fm-test-run.sh"
+ cat >"$repo/tests/aa-delete-next.test.sh" <<'SH'
+#!/usr/bin/env bash
+rm tests/zz-victim.test.sh
+SH
+ cat >"$repo/tests/zz-victim.test.sh" <<'SH'
+#!/usr/bin/env bash
+echo "ok - should never run from a corrupted checkout"
+SH
+ chmod +x "$repo"/tests/*.test.sh
+ git -C "$repo" init -q
+ git -C "$repo" add bin tests
+ git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm fixture
+ commit=$(git -C "$repo" rev-parse HEAD)
+
+ set +e
+ (cd "$repo" && bin/fm-test-run.sh --compare-commits "$commit" "$commit" \
+ --script-timeout 1 --output-dir "$tmp/result" >"$tmp/out" 2>"$tmp/err")
+ rc=$?
+ set -e
+ [ "$rc" -eq 1 ] || { rm -rf "$tmp"; fail "checkout corruption must make compare exit 1 (got $rc)"; }
+ if ! python3 - "$tmp/result/base.json" "$tmp/result/head.json" "$tmp/result/partition.json" <<'PY'
+import json, sys
+base, head, partition = (json.load(open(path, encoding="utf-8")) for path in sys.argv[1:])
+for doc in (base, head):
+ assert doc["discovered_scripts"] == ["tests/aa-delete-next.test.sh", "tests/zz-victim.test.sh"], doc
+ assert [row["path"] for row in doc["scripts"]] == doc["discovered_scripts"], doc
+ assert doc["reconciliation"] == {
+ "accounted": True, "discovered_count": 2, "duplicates": [],
+ "expected_count": 2, "extra": [], "extra_in_discovery": [],
+ "inventory_count": 2, "missing": [], "missing_from_discovery": []
+ }, doc
+ assert all(row["outcome"] == "errored" for row in doc["scripts"]), doc
+ assert doc["checkout"]["invariant_ok"] is False, doc
+assert partition["inventory_reconciled"] is True, partition
+assert partition["checkout_invariants_ok"] is False, partition
+PY
+ then
+ rm -rf "$tmp"
+ fail "lost-script accounting did not fail closed"
+ fi
+ rm -rf "$tmp"
+ pass "a script lost after discovery remains inventoried as errored and checkout drift fails loudly"
+}
+
+test_compare_commits_rechecks_a_first_only_failure() {
+ local tmp repo marker base head rc fail_detail
+ tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare-flake.XXXXXX")
+ repo="$tmp/repo"
+ marker="$tmp/head-failed-once"
+ mkdir -p "$repo/bin" "$repo/tests"
+ cp "$RUNNER" "$repo/bin/fm-test-run.sh"
+ chmod +x "$repo/bin/fm-test-run.sh"
+ cat >"$repo/tests/aa-flaky.test.sh" <<'SH'
+#!/usr/bin/env bash
+echo "ok - stable base"
+SH
+ chmod +x "$repo/tests/aa-flaky.test.sh"
+ git -C "$repo" init -q
+ git -C "$repo" add bin tests
+ git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm base
+ base=$(git -C "$repo" rev-parse HEAD)
+ cat >"$repo/tests/aa-flaky.test.sh" <"$marker"
+ echo "not ok - first-only environmental failure"
+ exit 1
+fi
+echo "ok - retry passes"
+SH
+ chmod +x "$repo/tests/aa-flaky.test.sh"
+ git -C "$repo" add tests
+ git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm head
+ head=$(git -C "$repo" rev-parse HEAD)
+
+ set +e
+ (cd "$repo" && bin/fm-test-run.sh --compare-commits "$base" "$head" \
+ --script-timeout 1 --output-dir "$tmp/result" >"$tmp/out" 2>"$tmp/err")
+ rc=$?
+ set -e
+ [ "$rc" -eq 0 ] || {
+ fail_detail=$(cat "$tmp/err" 2>/dev/null || true)
+ rm -rf "$tmp"
+ fail "a first-only failure must not be a regression (got $rc): $fail_detail"
+ }
+ python3 - "$tmp/result/partition.json" <<'PY' || {
+import json, sys
+partition = json.load(open(sys.argv[1], encoding="utf-8"))
+assert partition["head_introduced_failures"] == [], partition
+assert partition["regressed"] == [], partition
+assert [row["path"] for row in partition["flaky_transitions"]] == ["tests/aa-flaky.test.sh"], partition
+row = partition["flaky_transitions"][0]
+assert row["base_observations"] == ["passed", "passed"], row
+assert row["head_observations"] == ["failed", "passed"], row
+PY
+ rm -rf "$tmp"
+ fail "first-only failure was not recorded as flaky"
+ }
+ rm -rf "$tmp"
+ pass "a first-only failure is rechecked and recorded as flaky, not introduced"
+}
+
+test_compare_commits_refuses_an_inconclusive_retry() {
+ local tmp repo marker base head rc
+ tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare-inconclusive.XXXXXX")
+ repo="$tmp/repo"
+ marker="$tmp/head-failed-once"
+ mkdir -p "$repo/bin" "$repo/tests"
+ cp "$RUNNER" "$repo/bin/fm-test-run.sh"
+ chmod +x "$repo/bin/fm-test-run.sh"
+ cat >"$repo/tests/aa-inconclusive.test.sh" <<'SH'
+#!/usr/bin/env bash
+echo "ok - stable base"
+SH
+ chmod +x "$repo/tests/aa-inconclusive.test.sh"
+ git -C "$repo" init -q
+ git -C "$repo" add bin tests
+ git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm base
+ base=$(git -C "$repo" rev-parse HEAD)
+ cat >"$repo/tests/aa-inconclusive.test.sh" <"$marker"
+ echo "not ok - first head failure"
+ exit 1
+fi
+trap '' TERM
+sleep 30
+SH
+ chmod +x "$repo/tests/aa-inconclusive.test.sh"
+ git -C "$repo" add tests
+ git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm head
+ head=$(git -C "$repo" rev-parse HEAD)
+
+ set +e
+ (cd "$repo" && bin/fm-test-run.sh --compare-commits "$base" "$head" \
+ --script-timeout 1 --output-dir "$tmp/result" >"$tmp/out" 2>"$tmp/err")
+ rc=$?
+ set -e
+ [ "$rc" -eq 1 ] || { rm -rf "$tmp"; fail "inconclusive retry must exit 1 (got $rc)"; }
+ python3 - "$tmp/result/partition.json" <<'PY' || {
+import json, sys
+partition = json.load(open(sys.argv[1], encoding="utf-8"))
+assert partition["head_introduced_failures"] == [], partition
+assert partition["flaky_transitions"] == [], partition
+assert partition["regressed"] == [{
+ "base_observations": ["passed", "passed"],
+ "base_outcome": "passed",
+ "head_observations": ["failed", "timed_out"],
+ "head_outcome": "failed",
+ "path": "tests/aa-inconclusive.test.sh",
+}], partition
+PY
+ rm -rf "$tmp"
+ fail "inconclusive retry was not retained as a refusing regression"
+ }
+ rm -rf "$tmp"
+ pass "an inconclusive retry remains a regression"
+}
+
+test_compare_commits_reports_timeout_to_failure_as_regressed() {
+ local tmp repo base head rc
+ tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare-regressed.XXXXXX")
+ repo="$tmp/repo"
+ mkdir -p "$repo/bin" "$repo/tests"
+ cp "$RUNNER" "$repo/bin/fm-test-run.sh"
+ chmod +x "$repo/bin/fm-test-run.sh"
+ cat >"$repo/tests/aa-transition.test.sh" <<'SH'
+#!/usr/bin/env bash
+trap '' TERM
+sleep 30
+SH
+ chmod +x "$repo/tests/aa-transition.test.sh"
+ git -C "$repo" init -q
+ git -C "$repo" add bin tests
+ git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm base
+ base=$(git -C "$repo" rev-parse HEAD)
+ cat >"$repo/tests/aa-transition.test.sh" <<'SH'
+#!/usr/bin/env bash
+echo "not ok - assertion now reaches a real failure"
+exit 1
+SH
+ git -C "$repo" add tests
+ git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm head
+ head=$(git -C "$repo" rev-parse HEAD)
+
+ set +e
+ (cd "$repo" && bin/fm-test-run.sh --compare-commits "$base" "$head" \
+ --script-timeout 1 --output-dir "$tmp/result" >"$tmp/out" 2>"$tmp/err")
+ rc=$?
+ set -e
+ [ "$rc" -eq 1 ] || { rm -rf "$tmp"; fail "timeout-to-failure regression must exit 1 (got $rc)"; }
+ assert_contains "$(cat "$tmp/out")" "regressed=1" "timeout-to-failure regression headline"
+ python3 - "$tmp/result/partition.json" <<'PY' || {
+import json, sys
+partition = json.load(open(sys.argv[1], encoding="utf-8"))
+assert partition["inherited_failures"] == [], partition
+assert partition["regressed"] == [{
+ "base_observations": ["timed_out", "timed_out"],
+ "base_outcome": "timed_out",
+ "head_observations": ["failed", "failed"],
+ "head_outcome": "failed",
+ "path": "tests/aa-transition.test.sh",
+}], partition
+PY
+ rm -rf "$tmp"
+ fail "timeout-to-failure transition was not classified as regressed"
+ }
+ rm -rf "$tmp"
+ pass "timeout-to-failure is a confirmed regression and exits non-zero"
+}
+
+test_compare_commits_never_calls_deleted_or_skipped_tests_fixed() {
+ local tmp repo base head rc
+ tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare-erosion.XXXXXX")
+ repo="$tmp/repo"
+ mkdir -p "$repo/bin" "$repo/tests"
+ cp "$RUNNER" "$repo/bin/fm-test-run.sh"
+ chmod +x "$repo/bin/fm-test-run.sh"
+ for script in aa-deleted bb-skipped cc-fixed; do
+ cat >"$repo/tests/$script.test.sh" <<'SH'
+#!/usr/bin/env bash
+echo "not ok - base failure"
+exit 1
+SH
+ done
+ chmod +x "$repo"/tests/*.test.sh
+ git -C "$repo" init -q
+ git -C "$repo" add bin tests
+ git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm base
+ base=$(git -C "$repo" rev-parse HEAD)
+ git -C "$repo" rm -q tests/aa-deleted.test.sh
+ cat >"$repo/tests/bb-skipped.test.sh" <<'SH'
+#!/usr/bin/env bash
+echo "skip: head no longer measures this test"
+SH
+ cat >"$repo/tests/cc-fixed.test.sh" <<'SH'
+#!/usr/bin/env bash
+echo "ok - genuinely fixed"
+SH
+ git -C "$repo" add tests
+ git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm head
+ head=$(git -C "$repo" rev-parse HEAD)
+
+ set +e
+ (cd "$repo" && bin/fm-test-run.sh --compare-commits "$base" "$head" \
+ --script-timeout 1 --output-dir "$tmp/result" >"$tmp/out" 2>"$tmp/err")
+ rc=$?
+ set -e
+ [ "$rc" -eq 1 ] || { rm -rf "$tmp"; fail "deleted or skipped coverage must exit 1 (got $rc)"; }
+ python3 - "$tmp/result/partition.json" <<'PY' || {
+import json, sys
+partition = json.load(open(sys.argv[1], encoding="utf-8"))
+assert [row["path"] for row in partition["fixed"]] == ["tests/cc-fixed.test.sh"], partition
+assert [row["path"] for row in partition["no_longer_measured"]] == [
+ "tests/aa-deleted.test.sh", "tests/bb-skipped.test.sh",
+], partition
+assert partition["summary"]["fixed"] == 1, partition
+assert partition["summary"]["no_longer_measured"] == 2, partition
+PY
+ rm -rf "$tmp"
+ fail "deleted or skipped tests were confused with genuine fixes"
+ }
+ rm -rf "$tmp"
+ pass "only failure-to-pass is fixed; deletion and skip are coverage erosion"
+}
+
+test_compare_commits_refuses_an_independent_enumeration_mismatch() {
+ local tmp repo commit rc
+ tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare-enumeration.XXXXXX")
+ repo="$tmp/repo"
+ mkdir -p "$repo/bin" "$repo/tests"
+ cp "$RUNNER" "$repo/bin/fm-test-run.sh"
+ chmod +x "$repo/bin/fm-test-run.sh"
+ cat >"$repo/tests/aa-real.test.sh" <<'SH'
+#!/usr/bin/env bash
+echo "ok - real test"
+SH
+ chmod +x "$repo/tests/aa-real.test.sh"
+ ln -s ./nowhere-nothing "$repo/tests/zz-ghost.test.sh"
+ git -C "$repo" init -q
+ git -C "$repo" add bin tests
+ git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm fixture
+ commit=$(git -C "$repo" rev-parse HEAD)
+
+ set +e
+ (cd "$repo" && bin/fm-test-run.sh --compare-commits "$commit" "$commit" \
+ --script-timeout 1 --output-dir "$tmp/result" >"$tmp/out" 2>"$tmp/err")
+ rc=$?
+ set -e
+ [ "$rc" -eq 1 ] || { rm -rf "$tmp"; fail "enumeration mismatch must exit 1 (got $rc)"; }
+ assert_contains "$(cat "$tmp/out")" "accounted=false" "enumeration mismatch inventory marker"
+ python3 - "$tmp/result/base.json" "$tmp/result/head.json" "$tmp/result/partition.json" <<'PY' || {
+import json, sys
+base, head, partition = (json.load(open(path, encoding="utf-8")) for path in sys.argv[1:])
+for doc in (base, head):
+ assert doc["expected_scripts"] == ["tests/aa-real.test.sh", "tests/zz-ghost.test.sh"], doc
+ assert doc["discovered_scripts"] == ["tests/aa-real.test.sh"], doc
+ assert doc["reconciliation"]["accounted"] is False, doc
+ assert doc["reconciliation"]["missing_from_discovery"] == ["tests/zz-ghost.test.sh"], doc
+ rows = {row["path"]: row for row in doc["scripts"]}
+ assert rows["tests/zz-ghost.test.sh"]["outcome"] == "errored", rows
+ assert rows["tests/zz-ghost.test.sh"]["detail"] == "not_run: tracked test path is not a regular file", rows
+assert partition["inventory_reconciled"] is False, partition
+PY
+ rm -rf "$tmp"
+ fail "independent enumeration mismatch did not fail closed"
+ }
+ rm -rf "$tmp"
+ pass "tracked test paths missing from discovery make accounted refuse"
+}
+
+test_compare_commits_headline_exposes_measured_coverage() {
+ local tmp repo commit rc out
+ tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare-coverage.XXXXXX")
+ repo="$tmp/repo"
+ mkdir -p "$repo/bin" "$repo/tests"
+ cp "$RUNNER" "$repo/bin/fm-test-run.sh"
+ chmod +x "$repo/bin/fm-test-run.sh"
+ cat >"$repo/tests/aa-pass.test.sh" <<'SH'
+#!/usr/bin/env bash
+echo "ok - measured"
+SH
+ for script in bb-timeout cc-timeout; do
+ cat >"$repo/tests/$script.test.sh" <<'SH'
+#!/usr/bin/env bash
+trap '' TERM
+sleep 30
+SH
+ done
+ chmod +x "$repo"/tests/*.test.sh
+ git -C "$repo" init -q
+ git -C "$repo" add bin tests
+ git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm fixture
+ commit=$(git -C "$repo" rev-parse HEAD)
+
+ set +e
+ out=$(cd "$repo" && bin/fm-test-run.sh --compare-commits "$commit" "$commit" \
+ --script-timeout 1 --output-dir "$tmp/result" 2>"$tmp/err")
+ rc=$?
+ set -e
+ [ "$rc" -eq 0 ] || { rm -rf "$tmp"; fail "inherited timeouts alone must remain inspectable (got $rc)"; }
+ assert_contains "$out" "inherited_timeouts=2" "timeout-specific inherited count"
+ assert_contains "$out" "compared_outcomes=1 coverage=1/3" "honest compared-outcome coverage"
+ rm -rf "$tmp"
+ pass "headline distinguishes timeouts from one genuinely compared outcome"
+}
+
+test_compare_commits_preflight_errors_exit_two_without_tracebacks() {
+ local tmp repo commit rc err
+ tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare-preflight.XXXXXX")
+ repo="$tmp/repo"
+ mkdir -p "$repo/bin" "$repo/tests"
+ cp "$RUNNER" "$repo/bin/fm-test-run.sh"
+ chmod +x "$repo/bin/fm-test-run.sh"
+ cat >"$repo/tests/aa-pass.test.sh" <<'SH'
+#!/usr/bin/env bash
+echo "ok - pass"
+SH
+ chmod +x "$repo/tests/aa-pass.test.sh"
+ git -C "$repo" init -q
+ git -C "$repo" add bin tests
+ git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm fixture
+ commit=$(git -C "$repo" rev-parse HEAD)
+
+ set +e
+ (cd "$repo" && bin/fm-test-run.sh --compare-commits definitely-missing "$commit" \
+ --script-timeout 1 --output-dir "$tmp/bad-ref" >"$tmp/out" 2>"$tmp/err")
+ rc=$?
+ set -e
+ err=$(cat "$tmp/err")
+ [ "$rc" -eq 2 ] || { rm -rf "$tmp"; fail "bad ref must exit 2 (got $rc)"; }
+ assert_contains "$err" "fm-test-run: compare failed: commit not found: definitely-missing" "bad-ref reason"
+ case "$err" in *Traceback*) rm -rf "$tmp"; fail "bad ref must not emit a traceback" ;; esac
+
+ : >"$tmp/not-a-directory"
+ set +e
+ (cd "$repo" && bin/fm-test-run.sh --compare-commits "$commit" "$commit" \
+ --script-timeout 1 --output-dir "$tmp/not-a-directory" >"$tmp/out2" 2>"$tmp/err2")
+ rc=$?
+ set -e
+ err=$(cat "$tmp/err2")
+ [ "$rc" -eq 2 ] || { rm -rf "$tmp"; fail "non-directory output path must exit 2 (got $rc)"; }
+ assert_contains "$err" "fm-test-run: compare failed: --output-dir must be a directory" "output-dir reason"
+ case "$err" in *Traceback*) rm -rf "$tmp"; fail "bad output dir must not emit a traceback" ;; esac
+ rm -rf "$tmp"
+ pass "bad refs and output paths are infrastructure errors with exact reasons"
+}
+
+test_compare_commits_signal_records_inflight_and_cleans_clones() {
+ local tmp repo commit run_dir runner_pid python_pid rc clone_path signal_name expected_rc i
+ tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare-signal.XXXXXX")
+ repo="$tmp/repo"
+ mkdir -p "$repo/bin" "$repo/tests"
+ cp "$RUNNER" "$repo/bin/fm-test-run.sh"
+ chmod +x "$repo/bin/fm-test-run.sh"
+ cat >"$repo/tests/aa-slow.test.sh" <<'SH'
+#!/usr/bin/env bash
+trap '' TERM
+sleep 30
+SH
+ chmod +x "$repo/tests/aa-slow.test.sh"
+ git -C "$repo" init -q
+ git -C "$repo" add bin tests
+ git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm fixture
+ commit=$(git -C "$repo" rev-parse HEAD)
+
+ for signal_name in TERM INT; do
+ run_dir="$tmp/result-$signal_name"
+ (cd "$repo" && exec bin/fm-test-run.sh --compare-commits "$commit" "$commit" \
+ --script-timeout 60 --output-dir "$run_dir" >"$tmp/out-$signal_name" 2>"$tmp/err-$signal_name") &
+ runner_pid=$!
+ clone_path=
+ i=0
+ while [ "$i" -lt 100 ]; do
+ i=$((i + 1))
+ if [ -f "$run_dir/base.json" ]; then
+ clone_path=$(python3 - "$run_dir/base.json" <<'PY'
+import json, sys
+doc = json.load(open(sys.argv[1], encoding="utf-8"))
+row = doc["scripts"][0]
+if row["outcome"] == "running" and row["detail"] == "in flight":
+ print(doc["checkout"]["path"])
+PY
+)
+ [ -n "$clone_path" ] && break
+ fi
+ sleep 0.05
+ done
+ [ -n "$clone_path" ] || {
+ kill -TERM "$runner_pid" 2>/dev/null || true
+ wait "$runner_pid" 2>/dev/null || true
+ rm -rf "$tmp"
+ fail "$signal_name fixture never recorded its in-flight script"
+ }
+ python_pid=$(ps -axo pid=,ppid=,command= | awk -v result="$run_dir" \
+ 'index($0, result) && $0 ~ /[Pp]ython/ { print $1; exit }')
+ # Bash may exec its final foreground command, in which case the recorded
+ # runner PID is already the Python compare worker.
+ [ -n "$python_pid" ] || python_pid=$runner_pid
+ kill -"$signal_name" "$python_pid"
+ set +e
+ wait "$runner_pid"
+ rc=$?
+ set -e
+ case "$signal_name" in
+ TERM) expected_rc=143 ;;
+ INT) expected_rc=130 ;;
+ esac
+ [ "$rc" -eq "$expected_rc" ] || { rm -rf "$tmp"; fail "$signal_name must exit $expected_rc (got $rc)"; }
+ python3 - "$run_dir/base.json" "$run_dir/partition.json" "$signal_name" "$expected_rc" <<'PY' || {
+import json, sys
+base = json.load(open(sys.argv[1], encoding="utf-8"))
+partition = json.load(open(sys.argv[2], encoding="utf-8"))
+row = base["scripts"][0]
+assert row["outcome"] == "running", row
+assert row["detail"] == "in flight", row
+assert partition["termination"] == {
+ "complete": False, "exit_code": int(sys.argv[4]), "signal": f"SIG{sys.argv[3]}"
+}, partition
+PY
+ rm -rf "$tmp"
+ fail "$signal_name artifacts did not preserve in-flight and termination evidence"
+ }
+ [ ! -e "$(dirname "$clone_path")" ] || { rm -rf "$tmp"; fail "$signal_name leaked isolated clones at $(dirname "$clone_path")"; }
+ done
+ rm -rf "$tmp"
+ pass "SIGTERM and SIGINT preserve partial evidence and remove isolated clones"
+}
+
+test_compare_commits_signal_records_inflight_confirmation() {
+ local tmp repo marker base head run_dir runner_pid python_pid rc clone_root i
+ tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare-confirmation-signal.XXXXXX")
+ repo="$tmp/repo"
+ marker="$tmp/base-confirmation-started"
+ mkdir -p "$repo/bin" "$repo/tests"
+ cp "$RUNNER" "$repo/bin/fm-test-run.sh"
+ chmod +x "$repo/bin/fm-test-run.sh"
+ cat >"$repo/tests/aa-transition.test.sh" <"$marker"
+echo "ok - initial base run"
+SH
+ chmod +x "$repo/tests/aa-transition.test.sh"
+ git -C "$repo" init -q
+ git -C "$repo" add bin tests
+ git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm base
+ base=$(git -C "$repo" rev-parse HEAD)
+ cat >"$repo/tests/aa-transition.test.sh" <<'SH'
+#!/usr/bin/env bash
+echo "not ok - head regression"
+exit 1
+SH
+ chmod +x "$repo/tests/aa-transition.test.sh"
+ git -C "$repo" add tests
+ git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm head
+ head=$(git -C "$repo" rev-parse HEAD)
+ run_dir="$tmp/result"
+ (cd "$repo" && exec bin/fm-test-run.sh --compare-commits "$base" "$head" \
+ --script-timeout 60 --output-dir "$run_dir" >"$tmp/out" 2>"$tmp/err") &
+ runner_pid=$!
+ clone_root=
+ i=0
+ while [ "$i" -lt 100 ]; do
+ i=$((i + 1))
+ if [ -f "$run_dir/partition.json" ]; then
+ clone_root=$(python3 - "$run_dir/base.json" "$run_dir/partition.json" <<'PY'
+import json, sys
+base = json.load(open(sys.argv[1], encoding="utf-8"))
+partition = json.load(open(sys.argv[2], encoding="utf-8"))
+rows = partition.get("regressed", [])
+if rows and rows[0].get("base_observations") == ["passed", "running"] and rows[0].get("head_observations") == ["failed"]:
+ print(__import__("pathlib").Path(base["checkout"]["path"]).parent)
+PY
+)
+ [ -n "$clone_root" ] && break
+ fi
+ sleep 0.05
+ done
+ [ -n "$clone_root" ] || {
+ kill -TERM "$runner_pid" 2>/dev/null || true
+ wait "$runner_pid" 2>/dev/null || true
+ rm -rf "$tmp"
+ fail "confirmation fixture never recorded its running observation"
+ }
+ python_pid=$(ps -axo pid=,ppid=,command= | awk -v result="$run_dir" \
+ 'index($0, result) && $0 ~ /[Pp]ython/ { print $1; exit }')
+ [ -n "$python_pid" ] || python_pid=$runner_pid
+ kill -TERM "$python_pid"
+ set +e
+ wait "$runner_pid"
+ rc=$?
+ set -e
+ [ "$rc" -eq 143 ] || { rm -rf "$tmp"; fail "SIGTERM during confirmation must exit 143 (got $rc)"; }
+ python3 - "$run_dir/partition.json" <<'PY' || {
+import json, sys
+partition = json.load(open(sys.argv[1], encoding="utf-8"))
+assert partition["termination"] == {"complete": False, "exit_code": 143, "signal": "SIGTERM"}, partition
+rows = partition["regressed"]
+assert rows == [{
+ "base_observations": ["passed", "running"],
+ "base_outcome": "passed",
+ "head_observations": ["failed"],
+ "head_outcome": "failed",
+ "path": "tests/aa-transition.test.sh",
+}], partition
+PY
+ rm -rf "$tmp"
+ fail "SIGTERM artifacts did not preserve the in-flight confirmation"
+ }
+ [ ! -e "$clone_root" ] || { rm -rf "$tmp"; fail "SIGTERM leaked isolated clones at $clone_root"; }
+ rm -rf "$tmp"
+ pass "SIGTERM during confirmation preserves a running observation"
+}
+
+if [ -n "${FM_TEST_RUN_ONLY:-}" ]; then
+ "$FM_TEST_RUN_ONLY"
+ exit $?
+fi
+
test_list_all_exact_suite_coverage
test_family_selection
test_single_script_selection
@@ -686,3 +1356,14 @@ test_portable_serial_shard_lane_refusals
test_jobs_requires_proven_isolated
test_jobs_parallel_scheduler_and_failure_propagation
test_aggregate_json
+test_compare_commits_partitions_and_bounds_every_script
+test_compare_commits_accounts_for_script_lost_after_discovery
+test_compare_commits_rechecks_a_first_only_failure
+test_compare_commits_refuses_an_inconclusive_retry
+test_compare_commits_reports_timeout_to_failure_as_regressed
+test_compare_commits_never_calls_deleted_or_skipped_tests_fixed
+test_compare_commits_refuses_an_independent_enumeration_mismatch
+test_compare_commits_headline_exposes_measured_coverage
+test_compare_commits_preflight_errors_exit_two_without_tracebacks
+test_compare_commits_signal_records_inflight_and_cleans_clones
+test_compare_commits_signal_records_inflight_confirmation