From 85d2ee73616e64e64408ff8a39357c6852b1b7da Mon Sep 17 00:00:00 2001 From: 420tombombadil Date: Thu, 6 Aug 2026 22:57:07 -0600 Subject: [PATCH 1/5] feat(test): add bounded base-head partition runner --- CONTRIBUTING.md | 2 + bin/fm-test-run.sh | 433 ++++++++++++++++++++++++++++++++++++++ tests/fm-test-run.test.sh | 150 ++++++++++++- 3 files changed, 584 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8fa1f30c56..9049488f6a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -81,6 +81,7 @@ bin/fm-test-run.sh --lane portable-serial # portable serial remainder (watcher bin/fm-test-run.sh --list-lanes # discover exact lane names, including the current CI serial shards bin/fm-test-run.sh --check-coverage # prove portable shards + serial + serial shards + Herdr equal the full inventory bin/fm-test-run.sh --all # deliberate complete regression (optional local full walk; not no-mistakes Test) +bin/fm-test-run.sh --compare-commits --output-dir /tmp/fm-test-partition # bounded detached base/head inventories and failure partition bin/fm-test-isolation-proof.sh --list # proven parallel candidate set (Phase 2 owner) bin/fm-test-isolation-proof.sh --jobs 4 --json /tmp/fm-isolation-proof.json # re-run concurrent isolation proof only [ "$(readlink CLAUDE.md)" = "AGENTS.md" ] @@ -90,6 +91,7 @@ tmp=$(mktemp -d) && printf 'done: smoke\n' > "$tmp/smoke.status" && FM_STATE_OVE `bin/fm-test-run.sh` is the single owner of behavior-suite selection, portable CI lane composition, optional local `--jobs` for the proven-isolated set only, per-script timing markers, family totals, the coverage guard, and the optional JSON timing artifact. Its header and `--help` own the flags, family labels, lanes, and changed-file map; this section only documents the entry points. +Its commit-comparison mode gives independent gates per-script hard bounds, detached local copies, reconciled terminal inventories, and a mechanical inherited/introduced/fixed partition. `bin/fm-test-isolation-proof.sh` remains the single owner of the Phase 2 concurrent isolation proof and the exact proven candidate set; see `docs/fm-test-isolation-proof.md`. Portable shard balance evidence lives in `docs/fm-test-portable-shards.md`. Local no-mistakes Test stays intent-targeted and must not wire `commands.test` to `--all` or a `tests/*.test.sh` walk. diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 4ca26c865e..e4401bb931 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -12,6 +12,7 @@ # fm-test-run.sh --lane portable-serial-of (one CI serial shard) # fm-test-run.sh --proven-isolated # fm-test-run.sh tests/.test.sh [more scripts...] +# fm-test-run.sh --compare-commits --output-dir # # Inspection (no execution): # fm-test-run.sh --list --all @@ -24,6 +25,16 @@ # Aggregation (no suite execution): # fm-test-run.sh --aggregate-json [more lane.json...] # +# Base/head partition (isolated execution): +# fm-test-run.sh --compare-commits --output-dir [--script-timeout ] +# Resolves both refs once, checks each out detached in an independent local +# clone, and writes base.json, head.json, and partition.json. Every discovered +# script receives one of: passed, failed, timed_out, skipped, errored. Each +# script runs in its own process group under a hard bound (default: 300s). +# The command exits non-zero only for head-introduced failures or an inventory +# or checkout-integrity error; inherited failures remain visible but do not by +# themselves fail the comparison. +# # Options: # --json write a deterministic timing artifact after the run # --list print selected script paths (one per line) and exit 0 @@ -42,6 +53,9 @@ # selected script is in the proven-isolated set # (bin/fm-test-isolation-proof.sh --list). Cap is 8. Stateful # families never schedule under --jobs. +# --script-timeout N +# per-script bound in seconds for --compare-commits (default: 300). +# --output-dir D fresh artifact directory for --compare-commits. # -h, --help print this header # # Per-script machine-parseable markers (stdout): @@ -79,6 +93,10 @@ LIST_FAMILIES=0 LIST_LANES=0 CHECK_COVERAGE=0 AGGREGATE_OUT= +COMPARE_BASE= +COMPARE_HEAD= +COMPARE_OUTPUT_DIR= +SCRIPT_TIMEOUT=300 FAMILY= LANE= BASE_REF=origin/main @@ -1193,6 +1211,384 @@ with open(out, "w", encoding="utf-8") as fh: PY } +compare_commits() { + local base_ref=$1 head_ref=$2 output_dir=$3 script_timeout=$4 + command -v python3 >/dev/null 2>&1 || die "--compare-commits requires python3" + python3 - "$ROOT" "$base_ref" "$head_ref" "$output_dir" "$script_timeout" <<'PY' +import collections +import json +import os +import pathlib +import shutil +import signal +import subprocess +import sys +import tempfile +import time + +root = pathlib.Path(sys.argv[1]).resolve() +base_ref, head_ref = sys.argv[2], sys.argv[3] +output_dir = pathlib.Path(sys.argv[4]).resolve() +timeout_seconds = int(sys.argv[5]) + + +def git(*args, cwd=root, check=True): + return subprocess.run( + ["git", *args], cwd=cwd, check=check, text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + + +def resolve_commit(ref): + try: + return git("rev-parse", "--verify", f"{ref}^{{commit}}").stdout.strip() + except subprocess.CalledProcessError as exc: + raise RuntimeError(f"commit not found: {ref}: {exc.stderr.strip()}") from exc + + +def atomic_json(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + stage = path.with_name(path.name + ".tmp") + stage.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(stage, path) + + +def checkout_state(copy): + head = git("rev-parse", "HEAD", cwd=copy).stdout.strip() + detached = git("symbolic-ref", "-q", "HEAD", cwd=copy, check=False).returncode != 0 + clean = git("status", "--porcelain", "--untracked-files=all", cwd=copy).stdout == "" + return head, detached, clean + + +def inventory_summary(rows): + counts = collections.Counter(row["outcome"] for row in rows) + return { + "total": len(rows), + "passed": counts["passed"], + "failed": counts["failed"], + "timed_out": counts["timed_out"], + "skipped": counts["skipped"], + "errored": counts["errored"], + } + + +def reconcile(doc): + discovered = doc["discovered_scripts"] + inventoried = [row["path"] for row in doc["scripts"]] + missing = sorted(set(discovered) - set(inventoried)) + extra = sorted(set(inventoried) - set(discovered)) + duplicates = sorted(path for path, count in collections.Counter(inventoried).items() if count != 1) + ok = discovered == inventoried and not missing and not extra and not duplicates + doc["reconciliation"] = { + "accounted": ok, + "discovered_count": len(discovered), + "inventory_count": len(inventoried), + "missing": missing, + "extra": extra, + "duplicates": duplicates, + } + doc["summary"] = inventory_summary(doc["scripts"]) + return ok + + +def write_inventory(doc): + reconcile(doc) + atomic_json(output_dir / f"{doc['label']}.json", doc) + + +def first_meaningful_line(path): + with path.open(encoding="utf-8", errors="replace") as stream: + for line in stream: + if line.strip(): + return line.strip() + return "" + + +def stop_process_group(proc): + try: + os.killpg(proc.pid, signal.SIGTERM) + except ProcessLookupError: + return + try: + proc.wait(timeout=0.2) + return + except subprocess.TimeoutExpired: + pass + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.wait() + + +def make_inventory(label, requested_ref, commit, copy): + scripts = sorted( + str(path.relative_to(copy)) + for path in (copy / "tests").glob("*.test.sh") + if path.is_file() + ) + rows = [] + for script in scripts: + rows.append({ + "path": script, + "outcome": "errored", + "exit_code": None, + "duration_ms": 0, + "detail": "not_run: runner interrupted before execution", + "log": f"{label}/logs/{pathlib.Path(script).name}.log", + }) + head, detached, clean = checkout_state(copy) + doc = { + "schema_version": 1, + "label": label, + "requested_ref": requested_ref, + "commit": commit, + "timeout_seconds": timeout_seconds, + "checkout": { + "path": str(copy), + "initial_head": head, + "final_head": None, + "detached": detached, + "clean_before": clean, + "clean_after": None, + "invariant_ok": head == commit and detached and clean, + }, + "discovered_scripts": scripts, + "scripts": rows, + } + write_inventory(doc) + return doc + + +def run_inventory(doc, copy): + compromised = not doc["checkout"]["invariant_ok"] + for row in doc["scripts"]: + if compromised: + row["detail"] = "not_run: checkout invariant was already violated" + write_inventory(doc) + continue + + before_head, before_detached, before_clean = checkout_state(copy) + if before_head != doc["commit"] or not before_detached or not before_clean: + row["detail"] = ( + "not_run: checkout invariant violated before script " + f"(head={before_head}, detached={before_detached}, clean={before_clean})" + ) + compromised = True + doc["checkout"]["invariant_ok"] = False + write_inventory(doc) + continue + + script_path = copy / row["path"] + log_path = output_dir / row["log"] + log_path.parent.mkdir(parents=True, exist_ok=True) + if not script_path.is_file(): + row["detail"] = "not_run: discovered script is missing at execution" + compromised = True + doc["checkout"]["invariant_ok"] = False + write_inventory(doc) + continue + + env = os.environ.copy() + for name in ( + "FM_HOME", "FM_STATE_OVERRIDE", "FM_DATA_OVERRIDE", "FM_ROOT_OVERRIDE", + "FM_PROJECTS_OVERRIDE", "FM_CONFIG_OVERRIDE", "FM_BACKEND", + ): + env.pop(name, None) + script_tmp = pathlib.Path(tempfile.mkdtemp(prefix=f"fm-compare-{doc['label']}-")) + env["TMPDIR"] = str(script_tmp) + env["TMP"] = str(script_tmp) + started = time.monotonic() + try: + with log_path.open("wb") as log_stream: + proc = subprocess.Popen( + ["/bin/bash", row["path"]], cwd=copy, env=env, + stdout=log_stream, stderr=subprocess.STDOUT, + start_new_session=True, + ) + try: + returncode = proc.wait(timeout=timeout_seconds) + except subprocess.TimeoutExpired: + stop_process_group(proc) + returncode = 124 + row["outcome"] = "timed_out" + row["detail"] = f"per-script timeout after {timeout_seconds}s" + else: + if returncode == 0: + if first_meaningful_line(log_path).startswith("skip:"): + row["outcome"] = "skipped" + row["detail"] = "first meaningful output line is a gate skip" + else: + row["outcome"] = "passed" + row["detail"] = "exit 0" + elif returncode < 0: + row["outcome"] = "errored" + row["detail"] = f"terminated by signal {-returncode}" + else: + row["outcome"] = "failed" + row["detail"] = f"exit {returncode}" + row["exit_code"] = returncode + except OSError as exc: + row["outcome"] = "errored" + row["detail"] = f"runner error: {exc}" + row["exit_code"] = None + finally: + row["duration_ms"] = max(0, int((time.monotonic() - started) * 1000)) + shutil.rmtree(script_tmp, ignore_errors=True) + + after_head, after_detached, after_clean = checkout_state(copy) + if after_head != doc["commit"] or not after_detached or not after_clean: + row["outcome"] = "errored" + row["detail"] = ( + "checkout invariant violated by script " + f"(head={after_head}, detached={after_detached}, clean={after_clean})" + ) + compromised = True + doc["checkout"]["invariant_ok"] = False + write_inventory(doc) + print( + "FM_TEST_COMPARE_ROW " + f"side={doc['label']} script={row['path']} outcome={row['outcome']} " + f"exit={row['exit_code']} duration_ms={row['duration_ms']}", + flush=True, + ) + + final_head, final_detached, final_clean = checkout_state(copy) + doc["checkout"]["final_head"] = final_head + doc["checkout"]["clean_after"] = final_clean + doc["checkout"]["invariant_ok"] = bool( + doc["checkout"]["invariant_ok"] + and final_head == doc["commit"] and final_detached and final_clean + ) + write_inventory(doc) + + +def failure_rows(doc): + return { + row["path"]: row for row in doc["scripts"] + if row["outcome"] in {"failed", "timed_out", "errored"} + } + + +def make_partition(base, head): + base_failures = failure_rows(base) + head_failures = failure_rows(head) + inherited = [] + introduced = [] + fixed = [] + for path in sorted(set(base_failures) & set(head_failures)): + inherited.append({ + "path": path, + "base_outcome": base_failures[path]["outcome"], + "head_outcome": head_failures[path]["outcome"], + }) + for path in sorted(set(head_failures) - set(base_failures)): + introduced.append({ + "path": path, + "base_outcome": next( + (row["outcome"] for row in base["scripts"] if row["path"] == path), + "absent", + ), + "head_outcome": head_failures[path]["outcome"], + }) + for path in sorted(set(base_failures) - set(head_failures)): + fixed.append({ + "path": path, + "base_outcome": base_failures[path]["outcome"], + "head_outcome": next( + (row["outcome"] for row in head["scripts"] if row["path"] == path), + "absent", + ), + }) + return { + "schema_version": 1, + "base_commit": base["commit"], + "head_commit": head["commit"], + "inventory_reconciled": bool( + base["reconciliation"]["accounted"] + and head["reconciliation"]["accounted"] + ), + "checkout_invariants_ok": bool( + base["checkout"]["invariant_ok"] + and head["checkout"]["invariant_ok"] + ), + "inherited_failures": inherited, + "head_introduced_failures": introduced, + "failures_fixed_by_head": fixed, + "summary": { + "inherited": len(inherited), + "head_introduced": len(introduced), + "fixed_by_head": len(fixed), + }, + } + + +if timeout_seconds < 1: + raise SystemExit("fm-test-run: --script-timeout must be a positive integer") +if output_dir.exists() and any(output_dir.iterdir()): + raise SystemExit(f"fm-test-run: --output-dir must be absent or empty: {output_dir}") +output_dir.mkdir(parents=True, exist_ok=True) + +base_commit = resolve_commit(base_ref) +head_commit = resolve_commit(head_ref) +temp_root = pathlib.Path(tempfile.mkdtemp(prefix="fm-test-compare-")) +inventories = [] +try: + copies = {} + for label, commit in (("base", base_commit), ("head", head_commit)): + copy = temp_root / label + clone = subprocess.run( + ["git", "clone", "--no-local", "--no-checkout", "--quiet", str(root), str(copy)], + text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + if clone.returncode != 0: + raise RuntimeError(f"could not create isolated {label} copy: {clone.stderr.strip()}") + checkout = git("checkout", "--detach", "--quiet", commit, cwd=copy, check=False) + if checkout.returncode != 0: + raise RuntimeError(f"could not detach {label} at {commit}: {checkout.stderr.strip()}") + copies[label] = copy + + base = make_inventory("base", base_ref, base_commit, copies["base"]) + head = make_inventory("head", head_ref, head_commit, copies["head"]) + inventories.extend((base, head)) + run_inventory(base, copies["base"]) + run_inventory(head, copies["head"]) + partition = make_partition(base, head) + atomic_json(output_dir / "partition.json", partition) + for doc in (base, head): + summary = doc["summary"] + print( + "FM_TEST_COMPARE_INVENTORY " + f"side={doc['label']} commit={doc['commit']} total={summary['total']} " + f"passed={summary['passed']} failed={summary['failed']} " + f"timed_out={summary['timed_out']} skipped={summary['skipped']} " + f"errored={summary['errored']} accounted={str(doc['reconciliation']['accounted']).lower()} " + f"checkout_ok={str(doc['checkout']['invariant_ok']).lower()}", + flush=True, + ) + summary = partition["summary"] + print( + "FM_TEST_COMPARE_PARTITION " + f"inherited={summary['inherited']} head_introduced={summary['head_introduced']} " + f"fixed_by_head={summary['fixed_by_head']} output_dir={output_dir}", + flush=True, + ) + valid = partition["inventory_reconciled"] and partition["checkout_invariants_ok"] + raise SystemExit(1 if partition["head_introduced_failures"] or not valid else 0) +except BaseException as exc: + for doc in inventories: + write_inventory(doc) + if len(inventories) == 2: + atomic_json(output_dir / "partition.json", make_partition(*inventories)) + if isinstance(exc, SystemExit): + raise + print(f"fm-test-run: compare failed: {exc}", file=sys.stderr) + raise SystemExit(2) +finally: + shutil.rmtree(temp_root, ignore_errors=True) +PY +} + while [ "$#" -gt 0 ]; do case "$1" in --all) @@ -1236,6 +1632,14 @@ while [ "$#" -gt 0 ]; do MODE=changed shift ;; + --compare-commits) + [ -z "$MODE" ] || die "only one selection mode is allowed" + [ "$#" -gt 2 ] || die "--compare-commits requires " + MODE=compare + COMPARE_BASE=$2 + COMPARE_HEAD=$3 + shift 3 + ;; --base) [ "$#" -gt 1 ] || die "--base requires a git ref" BASE_REF=$2 @@ -1254,6 +1658,24 @@ while [ "$#" -gt 0 ]; do JSON_PATH=${1#--json=} shift ;; + --script-timeout) + [ "$#" -gt 1 ] || die "--script-timeout requires a positive integer" + SCRIPT_TIMEOUT=$2 + shift 2 + ;; + --script-timeout=*) + SCRIPT_TIMEOUT=${1#--script-timeout=} + shift + ;; + --output-dir) + [ "$#" -gt 1 ] || die "--output-dir requires a path" + COMPARE_OUTPUT_DIR=$2 + shift 2 + ;; + --output-dir=*) + COMPARE_OUTPUT_DIR=${1#--output-dir=} + shift + ;; --jobs) [ "$#" -gt 1 ] || die "--jobs requires a positive integer" JOBS=$2 @@ -1358,6 +1780,17 @@ if [ "${MODE:-}" = "aggregate" ]; then exit 0 fi +if [ "${MODE:-}" = "compare" ]; then + [ "$LIST_ONLY" -eq 0 ] || die "--list cannot be combined with --compare-commits" + [ -n "$COMPARE_OUTPUT_DIR" ] || die "--compare-commits requires --output-dir " + case "$SCRIPT_TIMEOUT" in + ''|*[!0-9]*) die "--script-timeout must be a positive integer" ;; + esac + [ "$SCRIPT_TIMEOUT" -ge 1 ] || die "--script-timeout must be >= 1" + compare_commits "$COMPARE_BASE" "$COMPARE_HEAD" "$COMPARE_OUTPUT_DIR" "$SCRIPT_TIMEOUT" + exit $? +fi + case "$JOBS" in ''|*[!0-9]*) die "--jobs must be a positive integer" ;; esac diff --git a/tests/fm-test-run.test.sh b/tests/fm-test-run.test.sh index 21bdd69ba5..55f7ab40f3 100755 --- a/tests/fm-test-run.test.sh +++ b/tests/fm-test-run.test.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # Contract tests for bin/fm-test-run.sh - the single owner of behavior suite # selection, portable lane composition, proven-isolated --jobs, timing markers, -# JSON artifacts, coverage guard, and aggregate exit status. +# JSON artifacts, coverage guard, aggregate exit status, and bounded detached +# base/head failure partitioning. # # These tests intentionally exercise the runner with fixtures, --list, and # focused scheduler checks, not the complete Firstmate suite. @@ -669,6 +670,151 @@ assert len(doc["scripts"])==3 pass "aggregate-json merges lane timing artifacts" } +init_compare_fixture_repo() { + local repo=$1 + mkdir -p "$repo/bin" "$repo/tests" + cp "$RUNNER" "$repo/bin/fm-test-run.sh" + chmod +x "$repo/bin/fm-test-run.sh" + cat >"$repo/tests/aa-pass.test.sh" <<'SH' +#!/usr/bin/env bash +echo "ok - pass" +SH + cat >"$repo/tests/ab-inherited.test.sh" <<'SH' +#!/usr/bin/env bash +echo "not ok - inherited" +exit 1 +SH + cat >"$repo/tests/ac-fixed.test.sh" <<'SH' +#!/usr/bin/env bash +echo "not ok - base failure" +exit 1 +SH + cat >"$repo/tests/ad-introduced.test.sh" <<'SH' +#!/usr/bin/env bash +echo "ok - base pass" +SH + cat >"$repo/tests/ae-skip.test.sh" <<'SH' +#!/usr/bin/env bash +echo "skip: optional fixture unavailable" +SH + cat >"$repo/tests/af-hang.test.sh" <<'SH' +#!/usr/bin/env bash +trap '' TERM +sleep 30 +SH + chmod +x "$repo"/tests/*.test.sh + git -C "$repo" init -q + git -C "$repo" add bin tests + git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm base +} + +test_compare_commits_partitions_and_bounds_every_script() { + local tmp repo base head rc out + tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare.XXXXXX") + repo="$tmp/repo" + init_compare_fixture_repo "$repo" + base=$(git -C "$repo" rev-parse HEAD) + cat >"$repo/tests/ac-fixed.test.sh" <<'SH' +#!/usr/bin/env bash +echo "ok - fixed" +SH + cat >"$repo/tests/ad-introduced.test.sh" <<'SH' +#!/usr/bin/env bash +echo "not ok - introduced" +exit 1 +SH + chmod +x "$repo/tests/ac-fixed.test.sh" "$repo/tests/ad-introduced.test.sh" + git -C "$repo" add tests + git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm head + head=$(git -C "$repo" rev-parse HEAD) + + set +e + out=$(cd "$repo" && bin/fm-test-run.sh --compare-commits "$base" "$head" \ + --script-timeout 1 --output-dir "$tmp/result" 2>"$tmp/err") + rc=$? + set -e + [ "$rc" -eq 1 ] || { rm -rf "$tmp"; fail "introduced failure must make compare exit 1 (got $rc): $(cat "$tmp/err")"; } + assert_contains "$out" "FM_TEST_COMPARE_INVENTORY side=base" "base terminal inventory marker" + assert_contains "$out" "FM_TEST_COMPARE_INVENTORY side=head" "head terminal inventory marker" + assert_contains "$out" "FM_TEST_COMPARE_PARTITION inherited=2 head_introduced=1 fixed_by_head=1" \ + "mechanical failure partition" + if ! python3 - "$tmp/result/base.json" "$tmp/result/head.json" "$tmp/result/partition.json" <<'PY' +import json, sys +base, head, partition = (json.load(open(path, encoding="utf-8")) for path in sys.argv[1:]) +assert base["reconciliation"]["accounted"] is True, base +assert head["reconciliation"]["accounted"] is True, head +assert base["checkout"]["invariant_ok"] is True, base +assert head["checkout"]["invariant_ok"] is True, head +assert base["summary"] == {"errored": 0, "failed": 2, "passed": 2, "skipped": 1, "timed_out": 1, "total": 6}, base +assert head["summary"] == {"errored": 0, "failed": 2, "passed": 2, "skipped": 1, "timed_out": 1, "total": 6}, head +base_rows = {row["path"]: row["outcome"] for row in base["scripts"]} +head_rows = {row["path"]: row["outcome"] for row in head["scripts"]} +assert base_rows["tests/af-hang.test.sh"] == "timed_out", base_rows +assert head_rows["tests/af-hang.test.sh"] == "timed_out", head_rows +assert [row["path"] for row in partition["inherited_failures"]] == [ + "tests/ab-inherited.test.sh", "tests/af-hang.test.sh" +], partition +assert [row["path"] for row in partition["head_introduced_failures"]] == ["tests/ad-introduced.test.sh"], partition +assert [row["path"] for row in partition["failures_fixed_by_head"]] == ["tests/ac-fixed.test.sh"], partition +PY + then + rm -rf "$tmp" + fail "compare JSON inventories or partition are wrong" + fi + rm -rf "$tmp" + pass "commit comparison bounds a deliberate hang and emits complete diffable inventories" +} + +test_compare_commits_accounts_for_script_lost_after_discovery() { + local tmp repo commit rc + tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare-missing.XXXXXX") + repo="$tmp/repo" + mkdir -p "$repo/bin" "$repo/tests" + cp "$RUNNER" "$repo/bin/fm-test-run.sh" + chmod +x "$repo/bin/fm-test-run.sh" + cat >"$repo/tests/aa-delete-next.test.sh" <<'SH' +#!/usr/bin/env bash +rm tests/zz-victim.test.sh +SH + cat >"$repo/tests/zz-victim.test.sh" <<'SH' +#!/usr/bin/env bash +echo "ok - should never run from a corrupted checkout" +SH + chmod +x "$repo"/tests/*.test.sh + git -C "$repo" init -q + git -C "$repo" add bin tests + git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm fixture + commit=$(git -C "$repo" rev-parse HEAD) + + set +e + (cd "$repo" && bin/fm-test-run.sh --compare-commits "$commit" "$commit" \ + --script-timeout 1 --output-dir "$tmp/result" >"$tmp/out" 2>"$tmp/err") + rc=$? + set -e + [ "$rc" -eq 1 ] || { rm -rf "$tmp"; fail "checkout corruption must make compare exit 1 (got $rc)"; } + if ! python3 - "$tmp/result/base.json" "$tmp/result/head.json" "$tmp/result/partition.json" <<'PY' +import json, sys +base, head, partition = (json.load(open(path, encoding="utf-8")) for path in sys.argv[1:]) +for doc in (base, head): + assert doc["discovered_scripts"] == ["tests/aa-delete-next.test.sh", "tests/zz-victim.test.sh"], doc + assert [row["path"] for row in doc["scripts"]] == doc["discovered_scripts"], doc + assert doc["reconciliation"] == { + "accounted": True, "discovered_count": 2, "duplicates": [], + "extra": [], "inventory_count": 2, "missing": [] + }, doc + assert all(row["outcome"] == "errored" for row in doc["scripts"]), doc + assert doc["checkout"]["invariant_ok"] is False, doc +assert partition["inventory_reconciled"] is True, partition +assert partition["checkout_invariants_ok"] is False, partition +PY + then + rm -rf "$tmp" + fail "lost-script accounting did not fail closed" + fi + rm -rf "$tmp" + pass "a script lost after discovery remains inventoried as errored and checkout drift fails loudly" +} + test_list_all_exact_suite_coverage test_family_selection test_single_script_selection @@ -686,3 +832,5 @@ test_portable_serial_shard_lane_refusals test_jobs_requires_proven_isolated test_jobs_parallel_scheduler_and_failure_propagation test_aggregate_json +test_compare_commits_partitions_and_bounds_every_script +test_compare_commits_accounts_for_script_lost_after_discovery From 1236825ab2f52a530178f38ad3d8e06ed5b92a22 Mon Sep 17 00:00:00 2001 From: 420tombombadil Date: Fri, 7 Aug 2026 03:58:31 -0600 Subject: [PATCH 2/5] fix(test): make commit suite partition sound --- CONTRIBUTING.md | 2 +- bin/fm-test-run.sh | 363 +++++++++++++++++++++++++++------- tests/fm-test-run.test.sh | 400 +++++++++++++++++++++++++++++++++++++- 3 files changed, 690 insertions(+), 75 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9049488f6a..69fe9a8491 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -91,7 +91,7 @@ tmp=$(mktemp -d) && printf 'done: smoke\n' > "$tmp/smoke.status" && FM_STATE_OVE `bin/fm-test-run.sh` is the single owner of behavior-suite selection, portable CI lane composition, optional local `--jobs` for the proven-isolated set only, per-script timing markers, family totals, the coverage guard, and the optional JSON timing artifact. Its header and `--help` own the flags, family labels, lanes, and changed-file map; this section only documents the entry points. -Its commit-comparison mode gives independent gates per-script hard bounds, detached local copies, reconciled terminal inventories, and a mechanical inherited/introduced/fixed partition. +Its commit-comparison mode gives independent gates per-script hard bounds, detached local copies, independently reconciled inventories, honest measured-outcome coverage, and retry-confirmed transition buckets for inherited failures, timeouts, regressions, genuine fixes, coverage erosion, and flakes. `bin/fm-test-isolation-proof.sh` remains the single owner of the Phase 2 concurrent isolation proof and the exact proven candidate set; see `docs/fm-test-isolation-proof.md`. Portable shard balance evidence lives in `docs/fm-test-portable-shards.md`. Local no-mistakes Test stays intent-targeted and must not wire `commands.test` to `--all` or a `tests/*.test.sh` walk. diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index e4401bb931..c910d896a5 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -28,12 +28,14 @@ # Base/head partition (isolated execution): # fm-test-run.sh --compare-commits --output-dir [--script-timeout ] # Resolves both refs once, checks each out detached in an independent local -# clone, and writes base.json, head.json, and partition.json. Every discovered -# script receives one of: passed, failed, timed_out, skipped, errored. Each -# script runs in its own process group under a hard bound (default: 300s). -# The command exits non-zero only for head-introduced failures or an inventory -# or checkout-integrity error; inherited failures remain visible but do not by -# themselves fail the comparison. +# clone, and writes base.json, head.json, and partition.json. Every independently +# enumerated script receives passed, failed, timed_out, skipped, or errored; +# an interrupted in-flight row remains running. Each script runs in its own +# process group under a hard bound (default: 300s). Candidate introduced and +# regressed transitions are re-run once on both sides before classification. +# The command exits non-zero for confirmed introduced/regressed outcomes, +# deleted or skipped coverage, inventory refusal, or checkout-integrity error; +# inherited failures and explicitly recorded flakes do not fail by themselves. # # Options: # --json write a deterministic timing artifact after the run @@ -1230,6 +1232,11 @@ root = pathlib.Path(sys.argv[1]).resolve() base_ref, head_ref = sys.argv[2], sys.argv[3] output_dir = pathlib.Path(sys.argv[4]).resolve() timeout_seconds = int(sys.argv[5]) +active_proc = None +inventories = [] +confirmations = {} +temp_root = None +termination = None def git(*args, cwd=root, check=True): @@ -1269,22 +1276,34 @@ def inventory_summary(rows): "timed_out": counts["timed_out"], "skipped": counts["skipped"], "errored": counts["errored"], + "running": counts["running"], } def reconcile(doc): + expected = doc["expected_scripts"] discovered = doc["discovered_scripts"] inventoried = [row["path"] for row in doc["scripts"]] - missing = sorted(set(discovered) - set(inventoried)) - extra = sorted(set(inventoried) - set(discovered)) + missing = sorted(set(expected) - set(inventoried)) + extra = sorted(set(inventoried) - set(expected)) + missing_from_discovery = sorted(set(expected) - set(discovered)) + extra_in_discovery = sorted(set(discovered) - set(expected)) duplicates = sorted(path for path, count in collections.Counter(inventoried).items() if count != 1) - ok = discovered == inventoried and not missing and not extra and not duplicates + ok = bool( + expected == discovered == inventoried + and not missing and not extra + and not missing_from_discovery and not extra_in_discovery + and not duplicates + ) doc["reconciliation"] = { "accounted": ok, + "expected_count": len(expected), "discovered_count": len(discovered), "inventory_count": len(inventoried), "missing": missing, "extra": extra, + "missing_from_discovery": missing_from_discovery, + "extra_in_discovery": extra_in_discovery, "duplicates": duplicates, } doc["summary"] = inventory_summary(doc["scripts"]) @@ -1321,14 +1340,31 @@ def stop_process_group(proc): proc.wait() +def tracked_test_scripts(copy, commit): + tree = subprocess.run( + ["git", "ls-tree", "-rz", "--name-only", commit, "--", "tests"], + cwd=copy, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ).stdout + scripts = set() + for raw_path in tree.split(b"\0"): + if not raw_path: + continue + path = os.fsdecode(raw_path) + parts = pathlib.PurePosixPath(path).parts + if len(parts) >= 2 and parts[0] == "tests" and parts[1].endswith(".test.sh"): + scripts.add(f"tests/{parts[1]}") + return sorted(scripts) + + def make_inventory(label, requested_ref, commit, copy): - scripts = sorted( + expected_scripts = tracked_test_scripts(copy, commit) + discovered_scripts = sorted( str(path.relative_to(copy)) for path in (copy / "tests").glob("*.test.sh") if path.is_file() ) rows = [] - for script in scripts: + for script in expected_scripts: rows.append({ "path": script, "outcome": "errored", @@ -1339,7 +1375,7 @@ def make_inventory(label, requested_ref, commit, copy): }) head, detached, clean = checkout_state(copy) doc = { - "schema_version": 1, + "schema_version": 2, "label": label, "requested_ref": requested_ref, "commit": commit, @@ -1353,7 +1389,8 @@ def make_inventory(label, requested_ref, commit, copy): "clean_after": None, "invariant_ok": head == commit and detached and clean, }, - "discovered_scripts": scripts, + "expected_scripts": expected_scripts, + "discovered_scripts": discovered_scripts, "scripts": rows, } write_inventory(doc) @@ -1361,6 +1398,7 @@ def make_inventory(label, requested_ref, commit, copy): def run_inventory(doc, copy): + global active_proc compromised = not doc["checkout"]["invariant_ok"] for row in doc["scripts"]: if compromised: @@ -1383,9 +1421,7 @@ def run_inventory(doc, copy): log_path = output_dir / row["log"] log_path.parent.mkdir(parents=True, exist_ok=True) if not script_path.is_file(): - row["detail"] = "not_run: discovered script is missing at execution" - compromised = True - doc["checkout"]["invariant_ok"] = False + row["detail"] = "not_run: tracked test path is not a regular file" write_inventory(doc) continue @@ -1406,6 +1442,11 @@ def run_inventory(doc, copy): stdout=log_stream, stderr=subprocess.STDOUT, start_new_session=True, ) + active_proc = proc + row["outcome"] = "running" + row["detail"] = "in flight" + row["exit_code"] = None + write_inventory(doc) try: returncode = proc.wait(timeout=timeout_seconds) except subprocess.TimeoutExpired: @@ -1428,11 +1469,13 @@ def run_inventory(doc, copy): row["outcome"] = "failed" row["detail"] = f"exit {returncode}" row["exit_code"] = returncode + active_proc = None except OSError as exc: row["outcome"] = "errored" row["detail"] = f"runner error: {exc}" row["exit_code"] = None finally: + active_proc = None row["duration_ms"] = max(0, int((time.monotonic() - started) * 1000)) shutil.rmtree(script_tmp, ignore_errors=True) @@ -1463,45 +1506,143 @@ def run_inventory(doc, copy): write_inventory(doc) -def failure_rows(doc): - return { - row["path"]: row for row in doc["scripts"] - if row["outcome"] in {"failed", "timed_out", "errored"} +def run_confirmation(doc, copy, path, attempt): + global active_proc + if path not in doc["expected_scripts"]: + return "absent" + script_path = copy / path + if not script_path.is_file(): + return "errored" + before_head, before_detached, before_clean = checkout_state(copy) + if before_head != doc["commit"] or not before_detached or not before_clean: + doc["checkout"]["invariant_ok"] = False + write_inventory(doc) + return "errored" + log_path = output_dir / doc["label"] / "confirmations" / ( + f"{pathlib.Path(path).name}.attempt-{attempt}.log" + ) + log_path.parent.mkdir(parents=True, exist_ok=True) + env = os.environ.copy() + for name in ( + "FM_HOME", "FM_STATE_OVERRIDE", "FM_DATA_OVERRIDE", "FM_ROOT_OVERRIDE", + "FM_PROJECTS_OVERRIDE", "FM_CONFIG_OVERRIDE", "FM_BACKEND", + ): + env.pop(name, None) + script_tmp = pathlib.Path(tempfile.mkdtemp(prefix=f"fm-confirm-{doc['label']}-")) + env["TMPDIR"] = str(script_tmp) + env["TMP"] = str(script_tmp) + outcome = "errored" + try: + with log_path.open("wb") as log_stream: + proc = subprocess.Popen( + ["/bin/bash", path], cwd=copy, env=env, + stdout=log_stream, stderr=subprocess.STDOUT, + start_new_session=True, + ) + active_proc = proc + try: + returncode = proc.wait(timeout=timeout_seconds) + except subprocess.TimeoutExpired: + stop_process_group(proc) + outcome = "timed_out" + else: + if returncode == 0: + outcome = ( + "skipped" if first_meaningful_line(log_path).startswith("skip:") + else "passed" + ) + elif returncode < 0: + outcome = "errored" + else: + outcome = "failed" + except OSError: + outcome = "errored" + finally: + active_proc = None + shutil.rmtree(script_tmp, ignore_errors=True) + after_head, after_detached, after_clean = checkout_state(copy) + if after_head != doc["commit"] or not after_detached or not after_clean: + doc["checkout"]["invariant_ok"] = False + outcome = "errored" + write_inventory(doc) + print( + "FM_TEST_COMPARE_CONFIRMATION " + f"side={doc['label']} script={path} attempt={attempt} outcome={outcome}", + flush=True, + ) + return outcome + + +def outcome_map(doc): + return {row["path"]: row["outcome"] for row in doc["scripts"]} + + +def transition_entry(path, base_outcome, head_outcome, confirmation=None): + entry = { + "path": path, + "base_outcome": base_outcome, + "head_outcome": head_outcome, } + if confirmation is not None: + entry["base_observations"] = confirmation["base"] + entry["head_observations"] = confirmation["head"] + return entry def make_partition(base, head): - base_failures = failure_rows(base) - head_failures = failure_rows(head) + base_rows = outcome_map(base) + head_rows = outcome_map(head) + all_paths = sorted(set(base["expected_scripts"]) | set(head["expected_scripts"])) inherited = [] + inherited_timeouts = [] introduced = [] + regressed = [] fixed = [] - for path in sorted(set(base_failures) & set(head_failures)): - inherited.append({ - "path": path, - "base_outcome": base_failures[path]["outcome"], - "head_outcome": head_failures[path]["outcome"], - }) - for path in sorted(set(head_failures) - set(base_failures)): - introduced.append({ - "path": path, - "base_outcome": next( - (row["outcome"] for row in base["scripts"] if row["path"] == path), - "absent", - ), - "head_outcome": head_failures[path]["outcome"], - }) - for path in sorted(set(base_failures) - set(head_failures)): - fixed.append({ - "path": path, - "base_outcome": base_failures[path]["outcome"], - "head_outcome": next( - (row["outcome"] for row in head["scripts"] if row["path"] == path), - "absent", - ), - }) + now_passing = [] + no_longer_measured = [] + flaky = [] + comparable_outcomes = 0 + for path in all_paths: + base_outcome = base_rows.get(path, "absent") + head_outcome = head_rows.get(path, "absent") + confirmation = confirmations.get(path) + entry = transition_entry(path, base_outcome, head_outcome, confirmation) + if base_outcome in {"passed", "failed"} and head_outcome in {"passed", "failed"}: + comparable_outcomes += 1 + if base_outcome == head_outcome: + if base_outcome in {"failed", "errored"}: + inherited.append(entry) + elif base_outcome == "timed_out": + inherited_timeouts.append(entry) + continue + if head_outcome == "absent" and base_outcome != "absent": + no_longer_measured.append(entry) + continue + if head_outcome == "skipped" and base_outcome not in {"absent", "skipped"}: + no_longer_measured.append(entry) + continue + if base_outcome == "failed" and head_outcome == "passed": + fixed.append(entry) + continue + if head_outcome == "passed" and base_outcome in {"timed_out", "errored", "skipped"}: + now_passing.append(entry) + continue + candidate_bucket = None + if base_outcome == "absent" and head_outcome in {"failed", "timed_out", "errored"}: + candidate_bucket = introduced + elif base_outcome != "absent" and head_outcome in {"failed", "timed_out", "errored"}: + candidate_bucket = regressed + if candidate_bucket is not None: + if confirmation is not None and ( + len(set(confirmation["base"])) != 1 + or len(set(confirmation["head"])) != 1 + ): + flaky.append(entry) + else: + candidate_bucket.append(entry) + comparison_total = len(all_paths) return { - "schema_version": 1, + "schema_version": 2, "base_commit": base["commit"], "head_commit": head["commit"], "inventory_reconciled": bool( @@ -1513,27 +1654,102 @@ def make_partition(base, head): and head["checkout"]["invariant_ok"] ), "inherited_failures": inherited, + "inherited_timeouts": inherited_timeouts, "head_introduced_failures": introduced, - "failures_fixed_by_head": fixed, + "regressed": regressed, + "fixed": fixed, + "now_passing": now_passing, + "no_longer_measured": no_longer_measured, + "flaky_transitions": flaky, "summary": { "inherited": len(inherited), + "inherited_timeouts": len(inherited_timeouts), "head_introduced": len(introduced), - "fixed_by_head": len(fixed), + "regressed": len(regressed), + "fixed": len(fixed), + "now_passing": len(now_passing), + "no_longer_measured": len(no_longer_measured), + "flaky": len(flaky), + "compared_outcomes": comparable_outcomes, + "comparison_total": comparison_total, + "coverage": f"{comparable_outcomes}/{comparison_total}", }, } -if timeout_seconds < 1: - raise SystemExit("fm-test-run: --script-timeout must be a positive integer") -if output_dir.exists() and any(output_dir.iterdir()): - raise SystemExit(f"fm-test-run: --output-dir must be absent or empty: {output_dir}") -output_dir.mkdir(parents=True, exist_ok=True) +def confirm_regression_candidates(partition, base, head, copies): + paths = sorted( + row["path"] + for bucket in (partition["head_introduced_failures"], partition["regressed"]) + for row in bucket + ) + base_rows = outcome_map(base) + head_rows = outcome_map(head) + for path in paths: + observation = { + "base": [base_rows.get(path, "absent")], + "head": [head_rows.get(path, "absent")], + } + observation["base"].append(run_confirmation(base, copies["base"], path, 1)) + observation["head"].append(run_confirmation(head, copies["head"], path, 1)) + confirmations[path] = observation + + +def current_partition(): + if len(inventories) == 2: + partition = make_partition(*inventories) + else: + partition = { + "schema_version": 2, + "base_commit": base_commit, + "head_commit": head_commit, + "inventory_reconciled": False, + "checkout_invariants_ok": False, + "incomplete": True, + "inventories_available": len(inventories), + } + if termination is not None: + partition["termination"] = termination + return partition + + +def write_current_partition(): + atomic_json(output_dir / "partition.json", current_partition()) + + +def handle_signal(signum, _frame): + global termination + termination = { + "complete": False, + "signal": signal.Signals(signum).name, + "exit_code": 128 + signum, + } + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + if active_proc is not None: + stop_process_group(active_proc) + for doc in inventories: + write_inventory(doc) + write_current_partition() + raise SystemExit(128 + signum) -base_commit = resolve_commit(base_ref) -head_commit = resolve_commit(head_ref) -temp_root = pathlib.Path(tempfile.mkdtemp(prefix="fm-test-compare-")) -inventories = [] + +base_commit = None +head_commit = None try: + if timeout_seconds < 1: + raise RuntimeError("--script-timeout must be a positive integer") + if output_dir.exists(): + if not output_dir.is_dir(): + raise RuntimeError(f"--output-dir must be a directory: {output_dir}") + if any(output_dir.iterdir()): + raise RuntimeError(f"--output-dir must be absent or empty: {output_dir}") + output_dir.mkdir(parents=True, exist_ok=True) + base_commit = resolve_commit(base_ref) + head_commit = resolve_commit(head_ref) + signal.signal(signal.SIGINT, handle_signal) + signal.signal(signal.SIGTERM, handle_signal) + temp_root = pathlib.Path(tempfile.mkdtemp(prefix="fm-test-compare-")) copies = {} for label, commit in (("base", base_commit), ("head", head_commit)): copy = temp_root / label @@ -1554,7 +1770,10 @@ try: run_inventory(base, copies["base"]) run_inventory(head, copies["head"]) partition = make_partition(base, head) - atomic_json(output_dir / "partition.json", partition) + write_current_partition() + confirm_regression_candidates(partition, base, head, copies) + partition = make_partition(base, head) + write_current_partition() for doc in (base, head): summary = doc["summary"] print( @@ -1562,30 +1781,42 @@ try: f"side={doc['label']} commit={doc['commit']} total={summary['total']} " f"passed={summary['passed']} failed={summary['failed']} " f"timed_out={summary['timed_out']} skipped={summary['skipped']} " - f"errored={summary['errored']} accounted={str(doc['reconciliation']['accounted']).lower()} " + f"errored={summary['errored']} running={summary['running']} " + f"accounted={str(doc['reconciliation']['accounted']).lower()} " f"checkout_ok={str(doc['checkout']['invariant_ok']).lower()}", flush=True, ) summary = partition["summary"] print( "FM_TEST_COMPARE_PARTITION " - f"inherited={summary['inherited']} head_introduced={summary['head_introduced']} " - f"fixed_by_head={summary['fixed_by_head']} output_dir={output_dir}", + f"inherited={summary['inherited']} inherited_timeouts={summary['inherited_timeouts']} " + f"head_introduced={summary['head_introduced']} regressed={summary['regressed']} " + f"fixed={summary['fixed']} now_passing={summary['now_passing']} " + f"no_longer_measured={summary['no_longer_measured']} flaky={summary['flaky']} " + f"compared_outcomes={summary['compared_outcomes']} coverage={summary['coverage']} " + f"output_dir={output_dir}", flush=True, ) valid = partition["inventory_reconciled"] and partition["checkout_invariants_ok"] - raise SystemExit(1 if partition["head_introduced_failures"] or not valid else 0) + refused = bool( + partition["head_introduced_failures"] + or partition["regressed"] + or partition["no_longer_measured"] + or not valid + ) + raise SystemExit(1 if refused else 0) except BaseException as exc: for doc in inventories: write_inventory(doc) - if len(inventories) == 2: - atomic_json(output_dir / "partition.json", make_partition(*inventories)) + if output_dir.exists() and output_dir.is_dir(): + write_current_partition() if isinstance(exc, SystemExit): raise print(f"fm-test-run: compare failed: {exc}", file=sys.stderr) raise SystemExit(2) finally: - shutil.rmtree(temp_root, ignore_errors=True) + if temp_root is not None: + shutil.rmtree(temp_root, ignore_errors=True) PY } diff --git a/tests/fm-test-run.test.sh b/tests/fm-test-run.test.sh index 55f7ab40f3..182dd604fd 100755 --- a/tests/fm-test-run.test.sh +++ b/tests/fm-test-run.test.sh @@ -736,8 +736,8 @@ SH [ "$rc" -eq 1 ] || { rm -rf "$tmp"; fail "introduced failure must make compare exit 1 (got $rc): $(cat "$tmp/err")"; } assert_contains "$out" "FM_TEST_COMPARE_INVENTORY side=base" "base terminal inventory marker" assert_contains "$out" "FM_TEST_COMPARE_INVENTORY side=head" "head terminal inventory marker" - assert_contains "$out" "FM_TEST_COMPARE_PARTITION inherited=2 head_introduced=1 fixed_by_head=1" \ - "mechanical failure partition" + assert_contains "$out" "FM_TEST_COMPARE_PARTITION inherited=1 inherited_timeouts=1 head_introduced=0 regressed=1 fixed=1" \ + "outcome-transition partition" if ! python3 - "$tmp/result/base.json" "$tmp/result/head.json" "$tmp/result/partition.json" <<'PY' import json, sys base, head, partition = (json.load(open(path, encoding="utf-8")) for path in sys.argv[1:]) @@ -745,17 +745,20 @@ assert base["reconciliation"]["accounted"] is True, base assert head["reconciliation"]["accounted"] is True, head assert base["checkout"]["invariant_ok"] is True, base assert head["checkout"]["invariant_ok"] is True, head -assert base["summary"] == {"errored": 0, "failed": 2, "passed": 2, "skipped": 1, "timed_out": 1, "total": 6}, base -assert head["summary"] == {"errored": 0, "failed": 2, "passed": 2, "skipped": 1, "timed_out": 1, "total": 6}, head +assert base["summary"] == {"errored": 0, "failed": 2, "passed": 2, "running": 0, "skipped": 1, "timed_out": 1, "total": 6}, base +assert head["summary"] == {"errored": 0, "failed": 2, "passed": 2, "running": 0, "skipped": 1, "timed_out": 1, "total": 6}, head base_rows = {row["path"]: row["outcome"] for row in base["scripts"]} head_rows = {row["path"]: row["outcome"] for row in head["scripts"]} assert base_rows["tests/af-hang.test.sh"] == "timed_out", base_rows assert head_rows["tests/af-hang.test.sh"] == "timed_out", head_rows assert [row["path"] for row in partition["inherited_failures"]] == [ - "tests/ab-inherited.test.sh", "tests/af-hang.test.sh" + "tests/ab-inherited.test.sh" ], partition -assert [row["path"] for row in partition["head_introduced_failures"]] == ["tests/ad-introduced.test.sh"], partition -assert [row["path"] for row in partition["failures_fixed_by_head"]] == ["tests/ac-fixed.test.sh"], partition +assert [row["path"] for row in partition["inherited_timeouts"]] == ["tests/af-hang.test.sh"], partition +assert partition["head_introduced_failures"] == [], partition +assert [row["path"] for row in partition["regressed"]] == ["tests/ad-introduced.test.sh"], partition +assert [row["path"] for row in partition["fixed"]] == ["tests/ac-fixed.test.sh"], partition +assert partition["summary"]["coverage"] == "4/6", partition PY then rm -rf "$tmp" @@ -800,7 +803,8 @@ for doc in (base, head): assert [row["path"] for row in doc["scripts"]] == doc["discovered_scripts"], doc assert doc["reconciliation"] == { "accounted": True, "discovered_count": 2, "duplicates": [], - "extra": [], "inventory_count": 2, "missing": [] + "expected_count": 2, "extra": [], "extra_in_discovery": [], + "inventory_count": 2, "missing": [], "missing_from_discovery": [] }, doc assert all(row["outcome"] == "errored" for row in doc["scripts"]), doc assert doc["checkout"]["invariant_ok"] is False, doc @@ -815,6 +819,379 @@ PY pass "a script lost after discovery remains inventoried as errored and checkout drift fails loudly" } +test_compare_commits_rechecks_a_first_only_failure() { + local tmp repo marker base head rc fail_detail + tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare-flake.XXXXXX") + repo="$tmp/repo" + marker="$tmp/head-failed-once" + mkdir -p "$repo/bin" "$repo/tests" + cp "$RUNNER" "$repo/bin/fm-test-run.sh" + chmod +x "$repo/bin/fm-test-run.sh" + cat >"$repo/tests/aa-flaky.test.sh" <<'SH' +#!/usr/bin/env bash +echo "ok - stable base" +SH + chmod +x "$repo/tests/aa-flaky.test.sh" + git -C "$repo" init -q + git -C "$repo" add bin tests + git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm base + base=$(git -C "$repo" rev-parse HEAD) + cat >"$repo/tests/aa-flaky.test.sh" <"$marker" + echo "not ok - first-only environmental failure" + exit 1 +fi +echo "ok - retry passes" +SH + chmod +x "$repo/tests/aa-flaky.test.sh" + git -C "$repo" add tests + git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm head + head=$(git -C "$repo" rev-parse HEAD) + + set +e + (cd "$repo" && bin/fm-test-run.sh --compare-commits "$base" "$head" \ + --script-timeout 1 --output-dir "$tmp/result" >"$tmp/out" 2>"$tmp/err") + rc=$? + set -e + [ "$rc" -eq 0 ] || { + fail_detail=$(cat "$tmp/err" 2>/dev/null || true) + rm -rf "$tmp" + fail "a first-only failure must not be a regression (got $rc): $fail_detail" + } + python3 - "$tmp/result/partition.json" <<'PY' || { +import json, sys +partition = json.load(open(sys.argv[1], encoding="utf-8")) +assert partition["head_introduced_failures"] == [], partition +assert partition["regressed"] == [], partition +assert [row["path"] for row in partition["flaky_transitions"]] == ["tests/aa-flaky.test.sh"], partition +row = partition["flaky_transitions"][0] +assert row["base_observations"] == ["passed", "passed"], row +assert row["head_observations"] == ["failed", "passed"], row +PY + rm -rf "$tmp" + fail "first-only failure was not recorded as flaky" + } + rm -rf "$tmp" + pass "a first-only failure is rechecked and recorded as flaky, not introduced" +} + +test_compare_commits_reports_timeout_to_failure_as_regressed() { + local tmp repo base head rc + tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare-regressed.XXXXXX") + repo="$tmp/repo" + mkdir -p "$repo/bin" "$repo/tests" + cp "$RUNNER" "$repo/bin/fm-test-run.sh" + chmod +x "$repo/bin/fm-test-run.sh" + cat >"$repo/tests/aa-transition.test.sh" <<'SH' +#!/usr/bin/env bash +trap '' TERM +sleep 30 +SH + chmod +x "$repo/tests/aa-transition.test.sh" + git -C "$repo" init -q + git -C "$repo" add bin tests + git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm base + base=$(git -C "$repo" rev-parse HEAD) + cat >"$repo/tests/aa-transition.test.sh" <<'SH' +#!/usr/bin/env bash +echo "not ok - assertion now reaches a real failure" +exit 1 +SH + git -C "$repo" add tests + git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm head + head=$(git -C "$repo" rev-parse HEAD) + + set +e + (cd "$repo" && bin/fm-test-run.sh --compare-commits "$base" "$head" \ + --script-timeout 1 --output-dir "$tmp/result" >"$tmp/out" 2>"$tmp/err") + rc=$? + set -e + [ "$rc" -eq 1 ] || { rm -rf "$tmp"; fail "timeout-to-failure regression must exit 1 (got $rc)"; } + assert_contains "$(cat "$tmp/out")" "regressed=1" "timeout-to-failure regression headline" + python3 - "$tmp/result/partition.json" <<'PY' || { +import json, sys +partition = json.load(open(sys.argv[1], encoding="utf-8")) +assert partition["inherited_failures"] == [], partition +assert partition["regressed"] == [{ + "base_observations": ["timed_out", "timed_out"], + "base_outcome": "timed_out", + "head_observations": ["failed", "failed"], + "head_outcome": "failed", + "path": "tests/aa-transition.test.sh", +}], partition +PY + rm -rf "$tmp" + fail "timeout-to-failure transition was not classified as regressed" + } + rm -rf "$tmp" + pass "timeout-to-failure is a confirmed regression and exits non-zero" +} + +test_compare_commits_never_calls_deleted_or_skipped_tests_fixed() { + local tmp repo base head rc + tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare-erosion.XXXXXX") + repo="$tmp/repo" + mkdir -p "$repo/bin" "$repo/tests" + cp "$RUNNER" "$repo/bin/fm-test-run.sh" + chmod +x "$repo/bin/fm-test-run.sh" + for script in aa-deleted bb-skipped cc-fixed; do + cat >"$repo/tests/$script.test.sh" <<'SH' +#!/usr/bin/env bash +echo "not ok - base failure" +exit 1 +SH + done + chmod +x "$repo"/tests/*.test.sh + git -C "$repo" init -q + git -C "$repo" add bin tests + git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm base + base=$(git -C "$repo" rev-parse HEAD) + git -C "$repo" rm -q tests/aa-deleted.test.sh + cat >"$repo/tests/bb-skipped.test.sh" <<'SH' +#!/usr/bin/env bash +echo "skip: head no longer measures this test" +SH + cat >"$repo/tests/cc-fixed.test.sh" <<'SH' +#!/usr/bin/env bash +echo "ok - genuinely fixed" +SH + git -C "$repo" add tests + git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm head + head=$(git -C "$repo" rev-parse HEAD) + + set +e + (cd "$repo" && bin/fm-test-run.sh --compare-commits "$base" "$head" \ + --script-timeout 1 --output-dir "$tmp/result" >"$tmp/out" 2>"$tmp/err") + rc=$? + set -e + [ "$rc" -eq 1 ] || { rm -rf "$tmp"; fail "deleted or skipped coverage must exit 1 (got $rc)"; } + python3 - "$tmp/result/partition.json" <<'PY' || { +import json, sys +partition = json.load(open(sys.argv[1], encoding="utf-8")) +assert [row["path"] for row in partition["fixed"]] == ["tests/cc-fixed.test.sh"], partition +assert [row["path"] for row in partition["no_longer_measured"]] == [ + "tests/aa-deleted.test.sh", "tests/bb-skipped.test.sh", +], partition +assert partition["summary"]["fixed"] == 1, partition +assert partition["summary"]["no_longer_measured"] == 2, partition +PY + rm -rf "$tmp" + fail "deleted or skipped tests were confused with genuine fixes" + } + rm -rf "$tmp" + pass "only failure-to-pass is fixed; deletion and skip are coverage erosion" +} + +test_compare_commits_refuses_an_independent_enumeration_mismatch() { + local tmp repo commit rc + tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare-enumeration.XXXXXX") + repo="$tmp/repo" + mkdir -p "$repo/bin" "$repo/tests" + cp "$RUNNER" "$repo/bin/fm-test-run.sh" + chmod +x "$repo/bin/fm-test-run.sh" + cat >"$repo/tests/aa-real.test.sh" <<'SH' +#!/usr/bin/env bash +echo "ok - real test" +SH + chmod +x "$repo/tests/aa-real.test.sh" + ln -s ./nowhere-nothing "$repo/tests/zz-ghost.test.sh" + git -C "$repo" init -q + git -C "$repo" add bin tests + git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm fixture + commit=$(git -C "$repo" rev-parse HEAD) + + set +e + (cd "$repo" && bin/fm-test-run.sh --compare-commits "$commit" "$commit" \ + --script-timeout 1 --output-dir "$tmp/result" >"$tmp/out" 2>"$tmp/err") + rc=$? + set -e + [ "$rc" -eq 1 ] || { rm -rf "$tmp"; fail "enumeration mismatch must exit 1 (got $rc)"; } + assert_contains "$(cat "$tmp/out")" "accounted=false" "enumeration mismatch inventory marker" + python3 - "$tmp/result/base.json" "$tmp/result/head.json" "$tmp/result/partition.json" <<'PY' || { +import json, sys +base, head, partition = (json.load(open(path, encoding="utf-8")) for path in sys.argv[1:]) +for doc in (base, head): + assert doc["expected_scripts"] == ["tests/aa-real.test.sh", "tests/zz-ghost.test.sh"], doc + assert doc["discovered_scripts"] == ["tests/aa-real.test.sh"], doc + assert doc["reconciliation"]["accounted"] is False, doc + assert doc["reconciliation"]["missing_from_discovery"] == ["tests/zz-ghost.test.sh"], doc + rows = {row["path"]: row for row in doc["scripts"]} + assert rows["tests/zz-ghost.test.sh"]["outcome"] == "errored", rows + assert rows["tests/zz-ghost.test.sh"]["detail"] == "not_run: tracked test path is not a regular file", rows +assert partition["inventory_reconciled"] is False, partition +PY + rm -rf "$tmp" + fail "independent enumeration mismatch did not fail closed" + } + rm -rf "$tmp" + pass "tracked test paths missing from discovery make accounted refuse" +} + +test_compare_commits_headline_exposes_measured_coverage() { + local tmp repo commit rc out + tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare-coverage.XXXXXX") + repo="$tmp/repo" + mkdir -p "$repo/bin" "$repo/tests" + cp "$RUNNER" "$repo/bin/fm-test-run.sh" + chmod +x "$repo/bin/fm-test-run.sh" + cat >"$repo/tests/aa-pass.test.sh" <<'SH' +#!/usr/bin/env bash +echo "ok - measured" +SH + for script in bb-timeout cc-timeout; do + cat >"$repo/tests/$script.test.sh" <<'SH' +#!/usr/bin/env bash +trap '' TERM +sleep 30 +SH + done + chmod +x "$repo"/tests/*.test.sh + git -C "$repo" init -q + git -C "$repo" add bin tests + git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm fixture + commit=$(git -C "$repo" rev-parse HEAD) + + set +e + out=$(cd "$repo" && bin/fm-test-run.sh --compare-commits "$commit" "$commit" \ + --script-timeout 1 --output-dir "$tmp/result" 2>"$tmp/err") + rc=$? + set -e + [ "$rc" -eq 0 ] || { rm -rf "$tmp"; fail "inherited timeouts alone must remain inspectable (got $rc)"; } + assert_contains "$out" "inherited_timeouts=2" "timeout-specific inherited count" + assert_contains "$out" "compared_outcomes=1 coverage=1/3" "honest compared-outcome coverage" + rm -rf "$tmp" + pass "headline distinguishes timeouts from one genuinely compared outcome" +} + +test_compare_commits_preflight_errors_exit_two_without_tracebacks() { + local tmp repo commit rc err + tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare-preflight.XXXXXX") + repo="$tmp/repo" + mkdir -p "$repo/bin" "$repo/tests" + cp "$RUNNER" "$repo/bin/fm-test-run.sh" + chmod +x "$repo/bin/fm-test-run.sh" + cat >"$repo/tests/aa-pass.test.sh" <<'SH' +#!/usr/bin/env bash +echo "ok - pass" +SH + chmod +x "$repo/tests/aa-pass.test.sh" + git -C "$repo" init -q + git -C "$repo" add bin tests + git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm fixture + commit=$(git -C "$repo" rev-parse HEAD) + + set +e + (cd "$repo" && bin/fm-test-run.sh --compare-commits definitely-missing "$commit" \ + --script-timeout 1 --output-dir "$tmp/bad-ref" >"$tmp/out" 2>"$tmp/err") + rc=$? + set -e + err=$(cat "$tmp/err") + [ "$rc" -eq 2 ] || { rm -rf "$tmp"; fail "bad ref must exit 2 (got $rc)"; } + assert_contains "$err" "fm-test-run: compare failed: commit not found: definitely-missing" "bad-ref reason" + case "$err" in *Traceback*) rm -rf "$tmp"; fail "bad ref must not emit a traceback" ;; esac + + : >"$tmp/not-a-directory" + set +e + (cd "$repo" && bin/fm-test-run.sh --compare-commits "$commit" "$commit" \ + --script-timeout 1 --output-dir "$tmp/not-a-directory" >"$tmp/out2" 2>"$tmp/err2") + rc=$? + set -e + err=$(cat "$tmp/err2") + [ "$rc" -eq 2 ] || { rm -rf "$tmp"; fail "non-directory output path must exit 2 (got $rc)"; } + assert_contains "$err" "fm-test-run: compare failed: --output-dir must be a directory" "output-dir reason" + case "$err" in *Traceback*) rm -rf "$tmp"; fail "bad output dir must not emit a traceback" ;; esac + rm -rf "$tmp" + pass "bad refs and output paths are infrastructure errors with exact reasons" +} + +test_compare_commits_signal_records_inflight_and_cleans_clones() { + local tmp repo commit run_dir runner_pid python_pid rc clone_path signal_name expected_rc i + tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare-signal.XXXXXX") + repo="$tmp/repo" + mkdir -p "$repo/bin" "$repo/tests" + cp "$RUNNER" "$repo/bin/fm-test-run.sh" + chmod +x "$repo/bin/fm-test-run.sh" + cat >"$repo/tests/aa-slow.test.sh" <<'SH' +#!/usr/bin/env bash +trap '' TERM +sleep 30 +SH + chmod +x "$repo/tests/aa-slow.test.sh" + git -C "$repo" init -q + git -C "$repo" add bin tests + git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm fixture + commit=$(git -C "$repo" rev-parse HEAD) + + for signal_name in TERM INT; do + run_dir="$tmp/result-$signal_name" + (cd "$repo" && exec bin/fm-test-run.sh --compare-commits "$commit" "$commit" \ + --script-timeout 60 --output-dir "$run_dir" >"$tmp/out-$signal_name" 2>"$tmp/err-$signal_name") & + runner_pid=$! + clone_path= + i=0 + while [ "$i" -lt 100 ]; do + i=$((i + 1)) + if [ -f "$run_dir/base.json" ]; then + clone_path=$(python3 - "$run_dir/base.json" <<'PY' +import json, sys +doc = json.load(open(sys.argv[1], encoding="utf-8")) +row = doc["scripts"][0] +if row["outcome"] == "running" and row["detail"] == "in flight": + print(doc["checkout"]["path"]) +PY +) + [ -n "$clone_path" ] && break + fi + sleep 0.05 + done + [ -n "$clone_path" ] || { + kill -TERM "$runner_pid" 2>/dev/null || true + wait "$runner_pid" 2>/dev/null || true + rm -rf "$tmp" + fail "$signal_name fixture never recorded its in-flight script" + } + python_pid=$(ps -axo pid=,ppid=,command= | awk -v result="$run_dir" \ + 'index($0, result) && $0 ~ /[Pp]ython/ { print $1; exit }') + # Bash may exec its final foreground command, in which case the recorded + # runner PID is already the Python compare worker. + [ -n "$python_pid" ] || python_pid=$runner_pid + kill -"$signal_name" "$python_pid" + set +e + wait "$runner_pid" + rc=$? + set -e + case "$signal_name" in + TERM) expected_rc=143 ;; + INT) expected_rc=130 ;; + esac + [ "$rc" -eq "$expected_rc" ] || { rm -rf "$tmp"; fail "$signal_name must exit $expected_rc (got $rc)"; } + python3 - "$run_dir/base.json" "$run_dir/partition.json" "$signal_name" "$expected_rc" <<'PY' || { +import json, sys +base = json.load(open(sys.argv[1], encoding="utf-8")) +partition = json.load(open(sys.argv[2], encoding="utf-8")) +row = base["scripts"][0] +assert row["outcome"] == "running", row +assert row["detail"] == "in flight", row +assert partition["termination"] == { + "complete": False, "exit_code": int(sys.argv[4]), "signal": f"SIG{sys.argv[3]}" +}, partition +PY + rm -rf "$tmp" + fail "$signal_name artifacts did not preserve in-flight and termination evidence" + } + [ ! -e "$(dirname "$clone_path")" ] || { rm -rf "$tmp"; fail "$signal_name leaked isolated clones at $(dirname "$clone_path")"; } + done + rm -rf "$tmp" + pass "SIGTERM and SIGINT preserve partial evidence and remove isolated clones" +} + +if [ -n "${FM_TEST_RUN_ONLY:-}" ]; then + "$FM_TEST_RUN_ONLY" + exit $? +fi + test_list_all_exact_suite_coverage test_family_selection test_single_script_selection @@ -834,3 +1211,10 @@ test_jobs_parallel_scheduler_and_failure_propagation test_aggregate_json test_compare_commits_partitions_and_bounds_every_script test_compare_commits_accounts_for_script_lost_after_discovery +test_compare_commits_rechecks_a_first_only_failure +test_compare_commits_reports_timeout_to_failure_as_regressed +test_compare_commits_never_calls_deleted_or_skipped_tests_fixed +test_compare_commits_refuses_an_independent_enumeration_mismatch +test_compare_commits_headline_exposes_measured_coverage +test_compare_commits_preflight_errors_exit_two_without_tracebacks +test_compare_commits_signal_records_inflight_and_cleans_clones From 8bd7ccea7532ebbe4c0651ba99ba146a1a1f0ed8 Mon Sep 17 00:00:00 2001 From: 420tombombadil Date: Fri, 14 Aug 2026 13:50:24 -0600 Subject: [PATCH 3/5] no-mistakes(review): Preserve running confirmation state on interruption --- bin/fm-test-run.sh | 17 ++++++-- tests/fm-test-run.test.sh | 90 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 3 deletions(-) diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index c910d896a5..3c5d345b68 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -1509,14 +1509,17 @@ def run_inventory(doc, copy): def run_confirmation(doc, copy, path, attempt): global active_proc if path not in doc["expected_scripts"]: + confirmations[path][doc["label"]].append("absent") return "absent" script_path = copy / path if not script_path.is_file(): + confirmations[path][doc["label"]].append("errored") return "errored" before_head, before_detached, before_clean = checkout_state(copy) if before_head != doc["commit"] or not before_detached or not before_clean: doc["checkout"]["invariant_ok"] = False write_inventory(doc) + confirmations[path][doc["label"]].append("errored") return "errored" log_path = output_dir / doc["label"] / "confirmations" / ( f"{pathlib.Path(path).name}.attempt-{attempt}.log" @@ -1532,6 +1535,8 @@ def run_confirmation(doc, copy, path, attempt): env["TMPDIR"] = str(script_tmp) env["TMP"] = str(script_tmp) outcome = "errored" + confirmations[path][doc["label"]].append("running") + write_current_partition() try: with log_path.open("wb") as log_stream: proc = subprocess.Popen( @@ -1565,6 +1570,8 @@ def run_confirmation(doc, copy, path, attempt): doc["checkout"]["invariant_ok"] = False outcome = "errored" write_inventory(doc) + confirmations[path][doc["label"]][-1] = outcome + write_current_partition() print( "FM_TEST_COMPARE_CONFIRMATION " f"side={doc['label']} script={path} attempt={attempt} outcome={outcome}", @@ -1633,7 +1640,11 @@ def make_partition(base, head): elif base_outcome != "absent" and head_outcome in {"failed", "timed_out", "errored"}: candidate_bucket = regressed if candidate_bucket is not None: - if confirmation is not None and ( + if confirmation is not None and "running" in ( + confirmation["base"] + confirmation["head"] + ): + candidate_bucket.append(entry) + elif confirmation is not None and ( len(set(confirmation["base"])) != 1 or len(set(confirmation["head"])) != 1 ): @@ -1690,9 +1701,9 @@ def confirm_regression_candidates(partition, base, head, copies): "base": [base_rows.get(path, "absent")], "head": [head_rows.get(path, "absent")], } - observation["base"].append(run_confirmation(base, copies["base"], path, 1)) - observation["head"].append(run_confirmation(head, copies["head"], path, 1)) confirmations[path] = observation + run_confirmation(base, copies["base"], path, 1) + run_confirmation(head, copies["head"], path, 1) def current_partition(): diff --git a/tests/fm-test-run.test.sh b/tests/fm-test-run.test.sh index 182dd604fd..75b9ec5c97 100755 --- a/tests/fm-test-run.test.sh +++ b/tests/fm-test-run.test.sh @@ -1187,6 +1187,95 @@ PY pass "SIGTERM and SIGINT preserve partial evidence and remove isolated clones" } +test_compare_commits_signal_records_inflight_confirmation() { + local tmp repo marker base head run_dir runner_pid python_pid rc clone_root i + tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare-confirmation-signal.XXXXXX") + repo="$tmp/repo" + marker="$tmp/base-confirmation-started" + mkdir -p "$repo/bin" "$repo/tests" + cp "$RUNNER" "$repo/bin/fm-test-run.sh" + chmod +x "$repo/bin/fm-test-run.sh" + cat >"$repo/tests/aa-transition.test.sh" <"$marker" +echo "ok - initial base run" +SH + chmod +x "$repo/tests/aa-transition.test.sh" + git -C "$repo" init -q + git -C "$repo" add bin tests + git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm base + base=$(git -C "$repo" rev-parse HEAD) + cat >"$repo/tests/aa-transition.test.sh" <<'SH' +#!/usr/bin/env bash +echo "not ok - head regression" +exit 1 +SH + chmod +x "$repo/tests/aa-transition.test.sh" + git -C "$repo" add tests + git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm head + head=$(git -C "$repo" rev-parse HEAD) + run_dir="$tmp/result" + (cd "$repo" && exec bin/fm-test-run.sh --compare-commits "$base" "$head" \ + --script-timeout 60 --output-dir "$run_dir" >"$tmp/out" 2>"$tmp/err") & + runner_pid=$! + clone_root= + i=0 + while [ "$i" -lt 100 ]; do + i=$((i + 1)) + if [ -f "$run_dir/partition.json" ]; then + clone_root=$(python3 - "$run_dir/base.json" "$run_dir/partition.json" <<'PY' +import json, sys +base = json.load(open(sys.argv[1], encoding="utf-8")) +partition = json.load(open(sys.argv[2], encoding="utf-8")) +rows = partition.get("regressed", []) +if rows and rows[0].get("base_observations") == ["passed", "running"] and rows[0].get("head_observations") == ["failed"]: + print(__import__("pathlib").Path(base["checkout"]["path"]).parent) +PY +) + [ -n "$clone_root" ] && break + fi + sleep 0.05 + done + [ -n "$clone_root" ] || { + kill -TERM "$runner_pid" 2>/dev/null || true + wait "$runner_pid" 2>/dev/null || true + rm -rf "$tmp" + fail "confirmation fixture never recorded its running observation" + } + python_pid=$(ps -axo pid=,ppid=,command= | awk -v result="$run_dir" \ + 'index($0, result) && $0 ~ /[Pp]ython/ { print $1; exit }') + [ -n "$python_pid" ] || python_pid=$runner_pid + kill -TERM "$python_pid" + set +e + wait "$runner_pid" + rc=$? + set -e + [ "$rc" -eq 143 ] || { rm -rf "$tmp"; fail "SIGTERM during confirmation must exit 143 (got $rc)"; } + python3 - "$run_dir/partition.json" <<'PY' || { +import json, sys +partition = json.load(open(sys.argv[1], encoding="utf-8")) +assert partition["termination"] == {"complete": False, "exit_code": 143, "signal": "SIGTERM"}, partition +rows = partition["regressed"] +assert rows == [{ + "base_observations": ["passed", "running"], + "base_outcome": "passed", + "head_observations": ["failed"], + "head_outcome": "failed", + "path": "tests/aa-transition.test.sh", +}], partition +PY + rm -rf "$tmp" + fail "SIGTERM artifacts did not preserve the in-flight confirmation" + } + [ ! -e "$clone_root" ] || { rm -rf "$tmp"; fail "SIGTERM leaked isolated clones at $clone_root"; } + rm -rf "$tmp" + pass "SIGTERM during confirmation preserves a running observation" +} + if [ -n "${FM_TEST_RUN_ONLY:-}" ]; then "$FM_TEST_RUN_ONLY" exit $? @@ -1218,3 +1307,4 @@ test_compare_commits_refuses_an_independent_enumeration_mismatch test_compare_commits_headline_exposes_measured_coverage test_compare_commits_preflight_errors_exit_two_without_tracebacks test_compare_commits_signal_records_inflight_and_cleans_clones +test_compare_commits_signal_records_inflight_confirmation From 26d0eac7fa82da0a17a907789ca07fa736464cc1 Mon Sep 17 00:00:00 2001 From: 420tombombadil Date: Fri, 14 Aug 2026 19:30:23 -0600 Subject: [PATCH 4/5] no-mistakes(review): Refuse inconclusive candidate retries --- bin/fm-test-run.sh | 15 +++++----- tests/fm-test-run.test.sh | 59 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 3c5d345b68..6da50118c6 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -1640,13 +1640,14 @@ def make_partition(base, head): elif base_outcome != "absent" and head_outcome in {"failed", "timed_out", "errored"}: candidate_bucket = regressed if candidate_bucket is not None: - if confirmation is not None and "running" in ( - confirmation["base"] + confirmation["head"] - ): - candidate_bucket.append(entry) - elif confirmation is not None and ( - len(set(confirmation["base"])) != 1 - or len(set(confirmation["head"])) != 1 + retry_outcomes = [] if confirmation is None else ( + confirmation["base"][1:] + confirmation["head"][1:] + ) + if ( + confirmation is not None + and confirmation["head"][-1] == "passed" + and not any(outcome in {"errored", "running", "skipped", "timed_out"} + for outcome in retry_outcomes) ): flaky.append(entry) else: diff --git a/tests/fm-test-run.test.sh b/tests/fm-test-run.test.sh index 75b9ec5c97..6b302fa646 100755 --- a/tests/fm-test-run.test.sh +++ b/tests/fm-test-run.test.sh @@ -877,6 +877,64 @@ PY pass "a first-only failure is rechecked and recorded as flaky, not introduced" } +test_compare_commits_refuses_an_inconclusive_retry() { + local tmp repo marker base head rc + tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare-inconclusive.XXXXXX") + repo="$tmp/repo" + marker="$tmp/head-failed-once" + mkdir -p "$repo/bin" "$repo/tests" + cp "$RUNNER" "$repo/bin/fm-test-run.sh" + chmod +x "$repo/bin/fm-test-run.sh" + cat >"$repo/tests/aa-inconclusive.test.sh" <<'SH' +#!/usr/bin/env bash +echo "ok - stable base" +SH + chmod +x "$repo/tests/aa-inconclusive.test.sh" + git -C "$repo" init -q + git -C "$repo" add bin tests + git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm base + base=$(git -C "$repo" rev-parse HEAD) + cat >"$repo/tests/aa-inconclusive.test.sh" <"$marker" + echo "not ok - first head failure" + exit 1 +fi +trap '' TERM +sleep 30 +SH + chmod +x "$repo/tests/aa-inconclusive.test.sh" + git -C "$repo" add tests + git -C "$repo" -c user.name=test -c user.email=test@example.invalid commit -qm head + head=$(git -C "$repo" rev-parse HEAD) + + set +e + (cd "$repo" && bin/fm-test-run.sh --compare-commits "$base" "$head" \ + --script-timeout 1 --output-dir "$tmp/result" >"$tmp/out" 2>"$tmp/err") + rc=$? + set -e + [ "$rc" -eq 1 ] || { rm -rf "$tmp"; fail "inconclusive retry must exit 1 (got $rc)"; } + python3 - "$tmp/result/partition.json" <<'PY' || { +import json, sys +partition = json.load(open(sys.argv[1], encoding="utf-8")) +assert partition["head_introduced_failures"] == [], partition +assert partition["flaky_transitions"] == [], partition +assert partition["regressed"] == [{ + "base_observations": ["passed", "passed"], + "base_outcome": "passed", + "head_observations": ["failed", "timed_out"], + "head_outcome": "failed", + "path": "tests/aa-inconclusive.test.sh", +}], partition +PY + rm -rf "$tmp" + fail "inconclusive retry was not retained as a refusing regression" + } + rm -rf "$tmp" + pass "an inconclusive retry remains a regression" +} + test_compare_commits_reports_timeout_to_failure_as_regressed() { local tmp repo base head rc tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-compare-regressed.XXXXXX") @@ -1301,6 +1359,7 @@ test_aggregate_json test_compare_commits_partitions_and_bounds_every_script test_compare_commits_accounts_for_script_lost_after_discovery test_compare_commits_rechecks_a_first_only_failure +test_compare_commits_refuses_an_inconclusive_retry test_compare_commits_reports_timeout_to_failure_as_regressed test_compare_commits_never_calls_deleted_or_skipped_tests_fixed test_compare_commits_refuses_an_independent_enumeration_mismatch From ec9af5874366c6e2c38c69530ee6013b5c79364d Mon Sep 17 00:00:00 2001 From: 420tombombadil Date: Fri, 14 Aug 2026 19:36:02 -0600 Subject: [PATCH 5/5] no-mistakes(document): Document Suite R2 comparison mode --- bin/fm-test-run.sh | 2 +- docs/scripts.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 6da50118c6..8450565181 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -4,7 +4,7 @@ # timing markers, and the complete-regression coverage guard. # # Selection modes (exactly one of: --all, --family, --changed, --lane, -# --proven-isolated, or script paths): +# --proven-isolated, --compare-commits, or script paths): # fm-test-run.sh --all # fm-test-run.sh --family # fm-test-run.sh --changed [--base ] diff --git a/docs/scripts.md b/docs/scripts.md index 484911c380..1450be455c 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -31,7 +31,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-install-herdr.sh` | Install CI's exact-version Herdr pin with official asset URL, SHA-256, and protocol checks | | `fm-install-treehouse.sh`| Install CI's exact-version Treehouse pin for real-Herdr E2E that needs spawn worktrees | | `fm-herdr-ci-cleanup.sh` | Snapshot and tear down only job-owned `fm-lab-*` sessions in the Herdr CI lane | -| `fm-test-run.sh` | Behavior-test runner: selection, portable lanes, proven-isolated `--jobs`, coverage guard, timing/JSON | +| `fm-test-run.sh` | Behavior-test runner: selection, portable lanes, proven-isolated `--jobs`, coverage guard, timing/JSON, and isolated base/head comparison | | `fm-test-isolation-proof.sh` | Concurrent isolation proof and proven-isolated candidate set owner | | `fm-ensure-agents-md.sh` | Ensure a project's real `AGENTS.md`, its `CLAUDE.md` symlink, and the canonical self-governance section | | `fm-guard.sh` | Warn on primary-checkout tangles, pending queued wakes, and unhealthy supervision |