From ab6e61ae2c85185787abf817634cddb5389c2fbc Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Mon, 24 Aug 2026 15:06:04 -0400 Subject: [PATCH 01/13] fix(azure): route detached gate tests privately --- bin/fm-azure-runner-agent-fleet-install.py | 212 +++++++++++++++++++++ bin/fm-azure-runner-dispatch.sh | 35 +++- bin/fm-azure-runner-guest.sh | 40 ++-- bin/fm-azure-runner.py | 98 +++++++++- bin/fm-azure-runner.sh | 1 + bin/fm-no-mistakes-test-command.sh | 4 +- docs/azure-runner.md | 30 ++- docs/azure-validation.md | 3 + tests/fm-azure-runner.test.sh | 187 +++++++++++++++++- tests/fm-nm-step-liveness.test.sh | 40 +++- 10 files changed, 599 insertions(+), 51 deletions(-) create mode 100755 bin/fm-azure-runner-agent-fleet-install.py diff --git a/bin/fm-azure-runner-agent-fleet-install.py b/bin/fm-azure-runner-agent-fleet-install.py new file mode 100755 index 00000000000..405dac39160 --- /dev/null +++ b/bin/fm-azure-runner-agent-fleet-install.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""Install the exact locked Agent Fleet source into a prepared offline venv. + +The Azure runner has no package network while repository code executes and its +wheelhouse intentionally contains only locked registry dependencies. Agent +Fleet has no runtime dependencies, so this trusted installer copies the exact +snapshot package into the fresh venv, writes ordinary distribution metadata, +and creates the release-local console entrypoint without invoking an unsealed +PEP 517 build backend. +""" + +import base64 +import csv +import hashlib +import io +import os +from pathlib import Path +import re +import stat +import sys +import sysconfig + + +class InstallError(RuntimeError): + pass + + +def real_subdirectory(root, parts, label): + current = root + for part in parts: + current = current / part + try: + metadata = current.lstat() + except OSError as exc: + raise InstallError("{} is unavailable: {}".format(label, exc)) + if not stat.S_ISDIR(metadata.st_mode): + raise InstallError("{} must have real directory ancestry".format(label)) + return current + + +def regular_file(path, label): + try: + metadata = path.lstat() + except OSError as exc: + raise InstallError("{} is unavailable: {}".format(label, exc)) + if not stat.S_ISREG(metadata.st_mode): + raise InstallError("{} must be a regular non-link file".format(label)) + return path + + +def write_new(path, content, mode=0o644): + path.parent.mkdir(parents=True, exist_ok=True) + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + descriptor = os.open(str(path), flags, mode) + with os.fdopen(descriptor, "wb") as handle: + handle.write(content) + os.chmod(path, mode) + + +def record_digest(path): + digest = base64.urlsafe_b64encode(hashlib.sha256(path.read_bytes()).digest()).rstrip(b"=") + return "sha256=" + digest.decode("ascii") + + +def copy_package(source, destination): + if destination.exists() or destination.is_symlink(): + raise InstallError("Agent Fleet package destination already exists") + destination.mkdir(parents=True, mode=0o755) + copied = [] + for child in sorted(source.rglob("*")): + relative = child.relative_to(source) + metadata = child.lstat() + target = destination / relative + if stat.S_ISLNK(metadata.st_mode): + raise InstallError("Agent Fleet source contains a link: {}".format(relative)) + if stat.S_ISDIR(metadata.st_mode): + target.mkdir(mode=0o755) + continue + if not stat.S_ISREG(metadata.st_mode): + raise InstallError("Agent Fleet source contains a non-regular entry: {}".format(relative)) + target.parent.mkdir(parents=True, exist_ok=True) + write_new(target, child.read_bytes()) + copied.append(target) + if not copied: + raise InstallError("Agent Fleet source package is empty") + return copied + + +def install(project, venv): + if sys.version_info < (3, 11): + raise InstallError("Agent Fleet requires Python 3.11 or newer") + project = project.resolve() + venv = venv.resolve() + if Path(sys.prefix).resolve() != venv: + raise InstallError("installer must run with the exact target venv interpreter") + + pyproject_path = regular_file(project / "pyproject.toml", "Agent Fleet pyproject") + lock_path = regular_file(project / "uv.lock", "Agent Fleet lock") + source = real_subdirectory(project, ("src", "agent_fleet"), "Agent Fleet package source") + + try: + pyproject_text = pyproject_path.read_text(encoding="utf-8") + lock_text = lock_path.read_text(encoding="utf-8") + except OSError as exc: + raise InstallError("Agent Fleet project metadata is unreadable: {}".format(exc)) + + project_match = re.search(r"(?ms)^\[project\]\n(.*?)(?=^\[|\Z)", pyproject_text) + scripts_match = re.search(r"(?ms)^\[project\.scripts\]\n(.*?)(?=^\[|\Z)", pyproject_text) + if project_match is None or scripts_match is None: + raise InstallError("Agent Fleet project tables are absent") + project_table = project_match.group(1) + scripts_table = scripts_match.group(1) + name_match = re.search(r'^name = "([^"]+)"$', project_table, re.MULTILINE) + version_match = re.search(r'^version = "([^"]+)"$', project_table, re.MULTILINE) + dependencies_match = re.search(r"^dependencies = (.+)$", project_table, re.MULTILINE) + name = name_match.group(1) if name_match else None + version = version_match.group(1) if version_match else None + if name != "agent-fleet" or version is None or not re.fullmatch(r"[0-9]+(?:\.[0-9]+){2}", version): + raise InstallError("Agent Fleet project identity is not exact") + script_lines = [line.strip() for line in scripts_table.splitlines() if line.strip()] + if script_lines != ['agent-fleet = "agent_fleet.cli:main"']: + raise InstallError("Agent Fleet console entrypoint declaration is not exact") + if dependencies_match is None or dependencies_match.group(1).strip() != "[]": + raise InstallError("Agent Fleet gained runtime dependencies outside the sealed offline closure") + + locked = [] + for block in lock_text.split("[[package]]")[1:]: + locked_name = re.search(r'^name = "([^"]+)"$', block, re.MULTILINE) + if locked_name and locked_name.group(1) == name: + locked.append(block) + if len(locked) != 1: + raise InstallError("Agent Fleet lock does not contain one exact project record") + locked_block = locked[0] + if ( + not re.search(r'^version = "{}"$'.format(re.escape(version)), locked_block, re.MULTILINE) + or not re.search(r'^source = \{ editable = "\." \}$', locked_block, re.MULTILINE) + or re.search(r"^dependencies = \[", locked_block, re.MULTILINE) + ): + raise InstallError("Agent Fleet lock does not bind the exact editable project without runtime dependencies") + + purelib = Path(sysconfig.get_path("purelib")).resolve() + scripts_dir = Path(sysconfig.get_path("scripts")).resolve() + if venv not in purelib.parents or venv not in scripts_dir.parents: + raise InstallError("target interpreter paths escape the exact venv") + + package_destination = purelib / "agent_fleet" + dist_info = purelib / "agent_fleet-{}.dist-info".format(version) + entrypoint = scripts_dir / "agent-fleet" + if dist_info.exists() or dist_info.is_symlink() or entrypoint.exists() or entrypoint.is_symlink(): + raise InstallError("Agent Fleet project or console entrypoint is already installed") + + installed = copy_package(source, package_destination) + dist_info.mkdir(mode=0o755) + metadata = ( + "Metadata-Version: 2.3\n" + "Name: agent-fleet\n" + "Version: {}\n" + "Summary: Machine-global account profile routing for local agent CLIs\n" + "Requires-Python: >=3.11\n" + "\n" + ).format(version).encode("utf-8") + write_new(dist_info / "METADATA", metadata) + write_new(dist_info / "WHEEL", b"Wheel-Version: 1.0\nGenerator: fm-azure-runner\nRoot-Is-Purelib: true\nTag: py3-none-any\n") + write_new(dist_info / "entry_points.txt", b"[console_scripts]\nagent-fleet = agent_fleet.cli:main\n") + write_new(dist_info / "INSTALLER", b"fm-azure-runner\n") + + python_path = venv / "bin" / "python" + if not python_path.exists(): + raise InstallError("target venv has no Python entrypoint") + entrypoint_bytes = ( + "#!{}\n" + "import sys\n" + "from agent_fleet.cli import main\n" + "if __name__ == '__main__':\n" + " sys.exit(main())\n" + ).format(python_path).encode("utf-8") + write_new(entrypoint, entrypoint_bytes, mode=0o755) + + record_rows = [] + for path in sorted(installed + [ + dist_info / "METADATA", + dist_info / "WHEEL", + dist_info / "entry_points.txt", + dist_info / "INSTALLER", + entrypoint, + ]): + relative = path.relative_to(venv).as_posix() + record_rows.append((relative, record_digest(path), str(path.stat().st_size))) + record_path = dist_info / "RECORD" + record_rows.append((record_path.relative_to(venv).as_posix(), "", "")) + output = io.StringIO(newline="") + writer = csv.writer(output, lineterminator="\n") + writer.writerows(record_rows) + write_new(record_path, output.getvalue().encode("utf-8")) + return entrypoint + + +def main(): + if len(sys.argv) != 3: + print("usage: fm-azure-runner-agent-fleet-install.py ", file=sys.stderr) + return 2 + try: + entrypoint = install(Path(sys.argv[1]), Path(sys.argv[2])) + except InstallError as exc: + print("agent-fleet offline install failed: {}".format(exc), file=sys.stderr) + return 125 + print("agent-fleet offline install: {}".format(entrypoint)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bin/fm-azure-runner-dispatch.sh b/bin/fm-azure-runner-dispatch.sh index d1bc1da7bae..edae1723b13 100755 --- a/bin/fm-azure-runner-dispatch.sh +++ b/bin/fm-azure-runner-dispatch.sh @@ -443,13 +443,34 @@ if [ -z "$CONFIRM" ]; then CONFIRM=$FM_AZURE_SUBSCRIPTION_ID fi +SOURCE_ARGUMENTS=() +if [ "$ROUTING_STATE" = selected ]; then + # A per-run no-mistakes step executes from the gate's detached, pipeline-owned + # snapshot. Give that exact HEAD a deterministic private bundle ref derived + # from the already-proved run id instead of guessing or pushing a task branch. + # The direct bundle takes the ordinary standalone shared-capacity path; it is + # not a validation-cell child and carries no parent reservation. + [ -n "$ROUTING_RUN_ID" ] \ + || refuse "selected per-run routing has no exact no-mistakes run identity" + SOURCE_ARGUMENTS=( + --source-ref "refs/heads/fm-no-mistakes/$ROUTING_RUN_ID" + --private-snapshot-from-head + ) +fi + printf 'azure-runner: class=%s selected REMOTE resource-class=%s source=%s (dispatching)\n' \ "$COMMAND_CLASS" "$RESOURCE_CLASS" "$SELECTION_SOURCE" >&2 -exec "$SCRIPT_DIR/fm-azure-runner.sh" run \ - --confirm-run \ - --confirm-subscription "$CONFIRM" \ - --task "$TASK" \ - --generation "$GENERATION" \ - --resource-class "$RESOURCE_CLASS" \ - -- "$@" +RUNNER_ARGUMENTS=( + run + --confirm-run + --confirm-subscription "$CONFIRM" + --task "$TASK" + --generation "$GENERATION" + --resource-class "$RESOURCE_CLASS" +) +if [ "$ROUTING_STATE" = selected ]; then + RUNNER_ARGUMENTS+=("${SOURCE_ARGUMENTS[@]}") +fi +RUNNER_ARGUMENTS+=(-- "$@") +exec "$SCRIPT_DIR/fm-azure-runner.sh" "${RUNNER_ARGUMENTS[@]}" diff --git a/bin/fm-azure-runner-guest.sh b/bin/fm-azure-runner-guest.sh index fa98a1e5313..0a3eddad923 100755 --- a/bin/fm-azure-runner-guest.sh +++ b/bin/fm-azure-runner-guest.sh @@ -18,10 +18,11 @@ OUTPUT_BLOB=${output_blob:-} INPUT_BLOB=${input_blob:-} IDENTITY_CLIENT_ID=${identity_client_id:-} EXECUTOR_B64=${executor_b64:-} +AGENT_FLEET_INSTALLER_B64=${agent_fleet_installer_b64:-} unset request_b64 vm_resource_id vm_instance_id guest_digest storage_account -unset container output_blob input_blob identity_client_id executor_b64 -for bound in "$REQUEST_B64" "$VM_RESOURCE_ID" "$VM_INSTANCE_ID" "$GUEST_DIGEST" "$STORAGE_ACCOUNT" "$CONTAINER" "$OUTPUT_BLOB" "$INPUT_BLOB" "$IDENTITY_CLIENT_ID" "$EXECUTOR_B64"; do - [ -n "$bound" ] || { echo "guest bootstrap: expected ten bound parameters" >&2; exit 125; } +unset container output_blob input_blob identity_client_id executor_b64 agent_fleet_installer_b64 +for bound in "$REQUEST_B64" "$VM_RESOURCE_ID" "$VM_INSTANCE_ID" "$GUEST_DIGEST" "$STORAGE_ACCOUNT" "$CONTAINER" "$OUTPUT_BLOB" "$INPUT_BLOB" "$IDENTITY_CLIENT_ID" "$EXECUTOR_B64" "$AGENT_FLEET_INSTALLER_B64"; do + [ -n "$bound" ] || { echo "guest bootstrap: expected eleven bound parameters" >&2; exit 125; } done unset bound case "$GUEST_DIGEST" in sha256:[0-9a-f][0-9a-f]*) ;; *) echo "guest bootstrap: bad protocol digest" >&2; exit 125 ;; esac @@ -169,22 +170,26 @@ rm -rf "$BASE" install -d -m 0700 -o root -g root "$BASE" REQUEST=$BASE/request.json EXECUTOR=$BASE/runner-exec.py +AGENT_FLEET_INSTALLER=$BASE/agent-fleet-install.py printf '%s' "$REQUEST_B64" | base64 -d >"$REQUEST" printf '%s' "$EXECUTOR_B64" | base64 -d >"$EXECUTOR" -unset REQUEST_B64 EXECUTOR_B64 +printf '%s' "$AGENT_FLEET_INSTALLER_B64" | base64 -d >"$AGENT_FLEET_INSTALLER" +unset REQUEST_B64 EXECUTOR_B64 AGENT_FLEET_INSTALLER_B64 -python3 - "$REQUEST" "$EXECUTOR" "$GUEST_DIGEST" <<'PY' +python3 - "$REQUEST" "$EXECUTOR" "$AGENT_FLEET_INSTALLER" "$GUEST_DIGEST" <<'PY' import hashlib, json, pathlib, sys -request_path, executor_path = map(pathlib.Path, sys.argv[1:3]) +request_path, executor_path, installer_path = map(pathlib.Path, sys.argv[1:4]) request = json.loads(request_path.read_text(encoding="utf-8")) unsigned = dict(request); supplied = unsigned.pop("request_digest", None) canonical = json.dumps(unsigned, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode() if supplied != "sha256:" + hashlib.sha256(canonical).hexdigest(): raise SystemExit("guest bootstrap: request digest mismatch") if "sha256:" + hashlib.sha256(executor_path.read_bytes()).hexdigest() != request["protocol"]["executor_digest"]: raise SystemExit("guest bootstrap: executor digest mismatch") -if request["protocol"]["guest_digest"] != sys.argv[3]: raise SystemExit("guest bootstrap: guest digest mismatch") +if "sha256:" + hashlib.sha256(installer_path.read_bytes()).hexdigest() != request["protocol"]["agent_fleet_installer_digest"]: raise SystemExit("guest bootstrap: Agent Fleet installer digest mismatch") +if request["protocol"]["guest_digest"] != sys.argv[4]: raise SystemExit("guest bootstrap: guest digest mismatch") repo = request["repository"] -if repo.get("source_mode") not in ("public-github-https", "private-parent-bundle", "private-exact-bundle") or not repo.get("remote", "").startswith("https://github.com/"): raise SystemExit("guest bootstrap: source mode mismatch") -if repo.get("source_mode") in ("private-parent-bundle", "private-exact-bundle"): +private_modes = ("private-parent-bundle", "private-exact-bundle", "private-direct-bundle") +if repo.get("source_mode") not in ("public-github-https",) + private_modes or not repo.get("remote", "").startswith("https://github.com/"): raise SystemExit("guest bootstrap: source mode mismatch") +if repo.get("source_mode") in private_modes: if not repo.get("input_blob") or not repo.get("snapshot_digest") or not repo.get("snapshot_bytes"): raise SystemExit("guest bootstrap: private snapshot binding is incomplete") else: if repo.get("input_blob") is not None or repo.get("snapshot_bytes") != 0: raise SystemExit("guest bootstrap: public source carries private staging") @@ -234,7 +239,7 @@ runuser -u fmrunner -- git -C /work/repo remote add origin "$REMOTE" # repository as dubious (CVE-2022-24765); scope the exception through the # environment exactly as the validation cell guest does. export GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=safe.directory GIT_CONFIG_VALUE_0=/work/repo -if [ "$SOURCE_MODE" = private-parent-bundle ] || [ "$SOURCE_MODE" = private-exact-bundle ]; then +if [ "$SOURCE_MODE" = private-parent-bundle ] || [ "$SOURCE_MODE" = private-exact-bundle ] || [ "$SOURCE_MODE" = private-direct-bundle ]; then [ "$INPUT_BLOB" = "$(read_request repository.input_blob)" ] || { echo "guest bootstrap: private snapshot blob mismatch" >&2; exit 125; } SNAPSHOT=$BASE/snapshot.bundle TOKEN_FILE=$BASE/input-token @@ -260,11 +265,11 @@ if [ "$SOURCE_MODE" = private-parent-bundle ] || [ "$SOURCE_MODE" = private-exac rm -f /work/snapshot.bundle elif [ "$SOURCE_REF" != none ] && [ "$SOURCE_HEAD" = "$COMMIT" ]; then [ "$INPUT_BLOB" = none ] || { echo "guest bootstrap: public source received a private snapshot blob" >&2; exit 125; } - run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch --depth=1 origin "$SOURCE_REF" + run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch origin "$SOURCE_REF" [ "$(git -C /work/repo rev-parse FETCH_HEAD)" = "$COMMIT" ] || { echo "guest bootstrap: source ref moved after admission" >&2; exit 125; } else [ "$INPUT_BLOB" = none ] || { echo "guest bootstrap: public source received a private snapshot blob" >&2; exit 125; } - run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch --depth=1 origin "$COMMIT" + run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch origin "$COMMIT" fi runuser -u fmrunner -- git -C /work/repo checkout --detach "$COMMIT" >/dev/null python3 - "$REQUEST" <<'PY' >"$BASE/source-ancestors" @@ -274,12 +279,12 @@ PY while IFS= read -r ancestor; do [ -n "$ancestor" ] || continue if [ "$SOURCE_MODE" = public-github-https ]; then - run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch --depth=1 origin "$ancestor" + run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch origin "$ancestor" [ "$(git -C /work/repo rev-parse FETCH_HEAD)" = "$ancestor" ] || { echo "guest bootstrap: source ancestor identity mismatch" >&2; exit 125; } git -C /work/repo cat-file -e "$ancestor^{commit}" || { echo "guest bootstrap: source ancestor is absent" >&2; exit 125; } fi done <"$BASE/source-ancestors" -if [ "$SOURCE_MODE" = private-parent-bundle ] || [ "$SOURCE_MODE" = private-exact-bundle ]; then +if [ "$SOURCE_MODE" = private-parent-bundle ] || [ "$SOURCE_MODE" = private-exact-bundle ] || [ "$SOURCE_MODE" = private-direct-bundle ]; then /usr/bin/python3 "$EXECUTOR" --verify-private-source-ancestors "$REQUEST" /work/repo fi [ "$(git -C /work/repo rev-parse HEAD)" = "$COMMIT" ] && [ "$(git -C /work/repo rev-parse 'HEAD^{tree}')" = "$TREE" ] || { echo "guest bootstrap: source identity mismatch" >&2; exit 125; } @@ -296,13 +301,15 @@ if [ "$DEFAULT_REF" != none ] && [ "$DEFAULT_HEAD" != none ]; then case "$DEFAULT_REF" in refs/heads/?*) DEFAULT_NAME=${DEFAULT_REF#refs/heads/} ;; *) DEFAULT_NAME= ;; esac if [ -n "$DEFAULT_NAME" ]; then if ! git -C /work/repo cat-file -e "$DEFAULT_HEAD^{commit}" 2>/dev/null; then - run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch --depth=1 origin "$DEFAULT_HEAD" + run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch origin "$DEFAULT_HEAD" [ "$(git -C /work/repo rev-parse FETCH_HEAD)" = "$DEFAULT_HEAD" ] || { echo "guest bootstrap: default head identity mismatch" >&2; exit 125; } fi runuser -u fmrunner -- git -C /work/repo update-ref "refs/remotes/origin/$DEFAULT_NAME" "$DEFAULT_HEAD" runuser -u fmrunner -- git -C /work/repo symbolic-ref refs/remotes/origin/HEAD "refs/remotes/origin/$DEFAULT_NAME" fi fi +[ "$(git -C /work/repo rev-parse --is-shallow-repository)" = false ] \ + || { echo "guest bootstrap: sealed source graph is shallow" >&2; exit 125; } fetch_exact() { local url=$1 path=$2 bytes=$3 digest=$4 redirects=${5:-no} args=(); [ "$redirects" != yes ] || args+=(--location); run_bootstrap_network curl --fail --silent --show-error "${args[@]}" --connect-timeout 30 --max-time 300 --max-filesize "$bytes" --output "$path" "$url"; [ "$(stat -c %s "$path")" = "$bytes" ] && [ "sha256:$(sha256sum "$path" | awk '{print $1}')" = "$digest" ] || { echo "guest bootstrap: pinned download mismatch" >&2; exit 125; }; } # The golden image stages the pinned archives under /opt/fm-tools; a copy @@ -341,6 +348,9 @@ if [ "$LOCK_DIGEST" != None ]; then cd /work/repo runuser -u fmrunner -- /work/home/.fm-runner-tools/uv/uv venv --python /usr/bin/python3 /work/repo/tools/agent-fleet/.venv >/dev/null runuser -u fmrunner -- env UV_OFFLINE=1 UV_NO_INDEX=1 /work/home/.fm-runner-tools/uv/uv pip install --python /work/repo/tools/agent-fleet/.venv/bin/python --offline --no-index --find-links /work/home/.fm-runner-tools/wheelhouse pytest ruff >/dev/null + /work/repo/tools/agent-fleet/.venv/bin/python "$AGENT_FLEET_INSTALLER" /work/repo/tools/agent-fleet /work/repo/tools/agent-fleet/.venv >/dev/null + chown -R fmrunner:fmrunner /work/repo/tools/agent-fleet/.venv + runuser -u fmrunner -- /work/repo/tools/agent-fleet/.venv/bin/agent-fleet --help >/dev/null elif [ -s "$BASE/wheels.tsv" ]; then echo "guest bootstrap: unbound Python wheels" >&2 exit 125 diff --git a/bin/fm-azure-runner.py b/bin/fm-azure-runner.py index 3a1f148d342..7f73e9b37c4 100755 --- a/bin/fm-azure-runner.py +++ b/bin/fm-azure-runner.py @@ -38,6 +38,7 @@ TEMPLATE = ROOT / "docs" / "azure-runner" / "invocation.json" GUEST = ROOT / "bin" / "fm-azure-runner-guest.sh" EXECUTOR = ROOT / "bin" / "fm-azure-runner-exec.py" +AGENT_FLEET_INSTALLER = ROOT / "bin" / "fm-azure-runner-agent-fleet-install.py" WORKER_LIFECYCLE = ROOT / "bin" / "fm-worker-lifecycle.py" CONTAINER = "validation-shards" CONTROL_CONTAINER = "runner-control" @@ -752,6 +753,45 @@ def locked_python_manifest(repo): +def verify_self_contained_private_bundle(bundle, commit, source_ref): + heads = git(ROOT, "bundle", "list-heads", str(bundle)).stdout.splitlines() + expected_head = "{} {}".format(commit, source_ref) + if heads != [expected_head]: + raise RunnerError("private snapshot must contain only the exact source-ref head") + with tempfile.TemporaryDirectory(prefix="fm-azure-bundle-verify-") as temporary: + verification_repo = Path(temporary) / "repo.git" + run(["git", "init", "--bare", str(verification_repo)]) + run(["git", "-C", str(verification_repo), "bundle", "verify", str(bundle)]) + run([ + "git", "-C", str(verification_repo), "fetch", "--no-tags", str(bundle), + "+{}:refs/fm-azure-runner/verified".format(source_ref), + ]) + verified = git( + verification_repo, "rev-parse", "--verify", "refs/fm-azure-runner/verified" + ).stdout.strip() + shallow = git(verification_repo, "rev-parse", "--is-shallow-repository").stdout.strip() + if verified != commit or shallow != "false": + raise RunnerError("private snapshot source graph is incomplete or has the wrong head") + + +def create_private_snapshot_from_head(repo, destination, commit, source_ref): + shallow = git(repo, "rev-parse", "--is-shallow-repository").stdout.strip() + if shallow != "false": + raise RunnerError("direct private snapshot requires a complete non-shallow source graph") + with tempfile.TemporaryDirectory(prefix="fm-azure-bundle-stage-") as temporary: + staging_repo = Path(temporary) / "repo.git" + run(["git", "init", "--bare", str(staging_repo)]) + run([ + "git", "-C", str(staging_repo), "fetch", "--no-tags", "--force", str(repo), + "+{}:{}".format(commit, source_ref), + ]) + staged = git(staging_repo, "rev-parse", "--verify", source_ref).stdout.strip() + if staged != commit: + raise RunnerError("direct private snapshot staging changed the exact source head") + run(["git", "-C", str(staging_repo), "bundle", "create", str(destination), source_ref]) + verify_self_contained_private_bundle(destination, commit, source_ref) + + def tree_digest(repo, relative): path = repo / relative if not path.exists(): @@ -802,15 +842,18 @@ def prepare(env, args, parent_state=None): branch = git(repo, "symbolic-ref", "--quiet", "--short", "HEAD", check=False) if ( branch.returncode != 0 - and args.public_ref is None - and not args.private_snapshot_bundle + and getattr(args, "public_ref", None) is None + and getattr(args, "source_ref", None) is None ): raise RunnerError( - "repository must be on a named committed branch unless an exact public ref or private snapshot is supplied" + "repository must be on a named committed branch unless an exact source ref is supplied" ) commit = git(repo, "rev-parse", "HEAD").stdout.strip() remote = git(repo, "remote", "get-url", "origin").stdout.strip() private_snapshot_source = None + private_snapshot_from_head = bool(getattr(args, "private_snapshot_from_head", False)) + if args.private_snapshot_bundle and private_snapshot_from_head: + raise RunnerError("choose one private snapshot input: bundle or exact HEAD") if args.private_snapshot_bundle: private_snapshot_arg = Path(args.private_snapshot_bundle) private_snapshot_source = private_snapshot_arg.resolve() @@ -846,6 +889,16 @@ def prepare(env, args, parent_state=None): source_ancestors=source_ancestors, private_source=private_snapshot_source is not None, ) + if private_snapshot_from_head: + if args.capacity_parent or not args.source_ref: + raise RunnerError("direct private HEAD snapshot requires one exact source ref and no capacity parent") + public = public_origin_proof( + repo, remote, commit, + source_ref=args.source_ref, + source_ancestors=source_ancestors, + private_source=True, + ) + private_snapshot_requested = private_snapshot_source is not None or private_snapshot_from_head tree = public["tree"] task = require_identifier("task", args.task) @@ -875,12 +928,19 @@ def prepare(env, args, parent_state=None): private_snapshot_path = None private_snapshot_digest = None private_snapshot_bytes = 0 - if private_snapshot_source is not None: + if private_snapshot_requested: private_snapshot_path = payload_dir / "snapshot.bundle" - shutil.copyfile(str(private_snapshot_source), str(private_snapshot_path)) + if private_snapshot_from_head: + create_private_snapshot_from_head( + repo, private_snapshot_path, commit, args.source_ref + ) + else: + shutil.copyfile(str(private_snapshot_source), str(private_snapshot_path)) os.chmod(private_snapshot_path, 0o600) private_snapshot_digest = "sha256:" + sha256_file(private_snapshot_path) private_snapshot_bytes = private_snapshot_path.stat().st_size + if private_snapshot_bytes > MAX_STAGING_INPUT_BYTES: + raise RunnerError("private snapshot exceeds the one-GiB staging bound") source_identity = { "remote": remote, "default_ref": public["default_ref"], @@ -958,7 +1018,11 @@ def prepare(env, args, parent_state=None): ( "private-parent-bundle" if args.capacity_parent - else "private-exact-bundle" + else ( + "private-direct-bundle" + if private_snapshot_from_head + else "private-exact-bundle" + ) ) if private_snapshot_path else "public-github-https" @@ -988,6 +1052,7 @@ def prepare(env, args, parent_state=None): "shellcheck_archive_digest": "sha256:8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198", "uv_archive_digest": "sha256:440c4215b171e64061d65d16a23753dd25c29a7f7b1b0446c9e9aed0fa372f27", "agent_fleet_python": locked_python, + "agent_fleet_installer_digest": "sha256:" + sha256_file(AGENT_FLEET_INSTALLER), }, "created_at": iso_utc(prepared_at), "compute_deallocation_deadline": iso_utc(expires_at), @@ -1069,7 +1134,9 @@ def reprove_public_request(state): repository = state["request"]["repository"] repo = Path(state["repository_root"]).resolve() source_mode = repository.get("source_mode") - private_source = source_mode in ("private-parent-bundle", "private-exact-bundle") + private_source = source_mode in ( + "private-parent-bundle", "private-exact-bundle", "private-direct-bundle", + ) expected = { "remote": repository["remote"], "default_ref": repository["default_ref"], @@ -2645,9 +2712,13 @@ def create_run_command(env, state): current_guest_digest = "sha256:" + sha256_file(GUEST) if current_guest_digest != state["request"]["protocol"]["guest_digest"]: raise RunnerError("trusted guest protocol changed after request preparation") + current_installer_digest = "sha256:" + sha256_file(AGENT_FLEET_INSTALLER) + if current_installer_digest != state["request"]["protocol"]["agent_fleet_installer_digest"]: + raise RunnerError("trusted Agent Fleet installer changed after request preparation") script = GUEST.read_text(encoding="utf-8") request_b64 = base64.b64encode(Path(state["input_path"]).read_bytes()).decode("ascii") executor_b64 = base64.b64encode(EXECUTOR.read_bytes()).decode("ascii") + agent_fleet_installer_b64 = base64.b64encode(AGENT_FLEET_INSTALLER.read_bytes()).decode("ascii") properties = { "location": "eastus", "tags": ownership_tags(env, state), @@ -2664,6 +2735,7 @@ def create_run_command(env, state): {"name": "input_blob", "value": state["staging"].get("input_blob") or "none"}, {"name": "identity_client_id", "value": env["controller_identity_client_id"]}, {"name": "executor_b64", "value": executor_b64}, + {"name": "agent_fleet_installer_b64", "value": agent_fleet_installer_b64}, ], "asyncExecution": False, "timeoutInSeconds": state["request"]["limits"]["wall_seconds"] + 1200, @@ -3265,7 +3337,11 @@ def retry(env, old_state, args): private_source = repository.get("source_mode") in ( "private-parent-bundle", "private-exact-bundle", + "private-direct-bundle", ) + private_parent_source = repository.get("source_mode") == "private-parent-bundle" + private_exact_source = repository.get("source_mode") == "private-exact-bundle" + private_direct_source = repository.get("source_mode") == "private-direct-bundle" selected_source_ref = ( repository["source_ref"] if repository["source_ref"] != repository["default_ref"] @@ -3278,9 +3354,10 @@ def retry(env, old_state, args): args.source_ref = None args.public_ref = selected_source_ref args.public_ancestor = list(repository.get("source_ancestors", [])) + args.private_snapshot_from_head = private_direct_source args.private_snapshot_bundle = ( str(Path(old_state["input_path"]).parent / "snapshot.bundle") - if private_source + if private_parent_source or private_exact_source else None ) args.wall_seconds = old_state["request"]["limits"]["wall_seconds"] @@ -3353,6 +3430,11 @@ def add_request_arguments(parser, require_command=True): "--private-snapshot-bundle", help="exact parent-cell Git bundle staged privately for an unpushed validation head", ) + parser.add_argument( + "--private-snapshot-from-head", + action="store_true", + help="seal the exact clean non-shallow HEAD into a direct one-ref private bundle", + ) parser.add_argument("--dependency", action="append", default=[]) parser.add_argument("--artifact", action="append", default=[]) if require_command: diff --git a/bin/fm-azure-runner.sh b/bin/fm-azure-runner.sh index c6cc0809c0f..ae875343d3e 100755 --- a/bin/fm-azure-runner.sh +++ b/bin/fm-azure-runner.sh @@ -31,6 +31,7 @@ # fm-azure-runner.sh run --confirm-run --confirm-subscription \ # [--confirm-cost-admission-mode commissioning-bounded] \ # --task --generation --resource-class \ +# [--source-ref refs/heads/] [--private-snapshot-from-head] \ # [--wall-seconds N] [--dependency ]... \ # [--artifact ]... -- # fm-azure-runner.sh resume --invocation diff --git a/bin/fm-no-mistakes-test-command.sh b/bin/fm-no-mistakes-test-command.sh index 06875d0c6bd..a5dac3b536c 100755 --- a/bin/fm-no-mistakes-test-command.sh +++ b/bin/fm-no-mistakes-test-command.sh @@ -78,7 +78,9 @@ selection_binding=$(printf '%s\n' "$selection_output" | sed -n 's/^selection_bin # They run concurrently and report independently into this one command step. # shellcheck disable=SC2016 # The command expands its variables inside the Azure guest shell. "$DISPATCH" --require-selection-binding "$selection_binding" test -- \ - "$ROOT/bin/fm-azure-runner-command.sh" bash -c ' + "$ROOT/bin/fm-azure-runner-command.sh" env \ + FM_TEST_HOST_CAPABILITIES_ABSENT=real-tmux-server,passwordless-root-escalation,system-openat-binding,origin-egress \ + bash -c ' command -v tmux >/dev/null || { echo "tmux is required for e2e tests" >&2; exit 1; } tmux -V rc=0 diff --git a/docs/azure-runner.md b/docs/azure-runner.md index 8aa944cee6f..d7d2da73956 100644 --- a/docs/azure-runner.md +++ b/docs/azure-runner.md @@ -31,13 +31,15 @@ A present file that is unreadable, partial, malformed, expired, exhausted, wrong Once either selector validly chooses remote execution, the command never falls back to the Mac after any cloud, identity, quota, staging, execution, or integrity failure. `FM_AZURE_RUNNER_LOCAL_RECOVERY_CLASSES` is the only explicit local recovery selection. -No billable resource was created while implementing this code. -The first live invocation remains blocked until the foundation and this code are reviewed, landed, explicitly applied, and approved for the exact subscription. +C2 no-mistakes run `01M0T2PJB0EZX8AA94WQV0WXA9` exposed the remaining production blocker before VM creation because its gate worktree was detached. +Direct diagnostic invocations `azr-0a4691b5bbae` and `azr-949b35e79a57` both returned command exit 1 and safely cleaned their invocation compute to zero. +Those attempts are failed evidence, not acceptance. +A new billable acceptance remains blocked until this exact repair is on public `main` and the operator explicitly approves the exact subscription and run. ## Request and snapshot contract `prepare` refuses tracked changes, untracked files, and any origin identity other than a credential-free GitHub HTTPS URL. -It also refuses a detached HEAD unless the caller supplies an exact public source ref or a digest-bound private bundle for that detached commit. +It also refuses a detached HEAD unless the caller supplies an exact source ref whose public proof or sealed private bundle head equals that detached commit. By default the candidate must be reachable from a freshly advertised and fetched `refs/heads/main` default head. The explicit `--source-ref refs/heads/` seam alone requires the candidate commit to be the exact freshly advertised and fetched head of that public branch; it never accepts an ancestor, stale tracking ref, tag, or changed remote head. An explicit `--public-ref` may instead name only an advertised branch head or `refs/pull//head`, and the candidate must equal that ref's exact fetched head; mutable pull merge refs and unsafe ref shapes are refused, and the two source-ref seams are mutually exclusive. @@ -45,8 +47,11 @@ A caller may additionally bind one or more exact `--public-ancestor` commits; pu Crosscheck evidence for a private GitHub repository supplies `--private-snapshot-bundle` without a parent reservation: the trusted host packages its clean exact-head review checkout, binds the authenticated PR ref and base ancestor, and stages only that digest-bound Git bundle so the evidence VM receives no GitHub credential. When that arbitrary repository does not contain Firstmate's Agent Fleet lock, the `crosscheck-tool` class uses only the sealed base toolchain and records an empty Python-wheel closure instead of requiring Firstmate-specific files. An Azure validation cell additionally supplies `--private-snapshot-bundle` with its parent-cell reservation so an unpushed pipeline-fix head can execute without prematurely changing the task branch on GitHub. -Both private modes bind one exact source ref/head, a one-ref Git bundle, digest, size, and private staging object; the validation-cell mode additionally binds its parent cell and freshly proves the public origin's trusted default ref/head. -The public proof runs in a fresh bare repository with system/global Git configuration, credentials, prompts, extra HTTP headers, and file transport disabled; private Crosscheck proof revalidates the clean checkout and bundle identities locally immediately before compute creation and retry. +The per-run no-mistakes route instead supplies `--private-snapshot-from-head` with the deterministic private ref `refs/heads/fm-no-mistakes/`. +That direct mode seals the exact clean detached gate HEAD and its complete non-shallow ancestry into one self-contained bundle without guessing or pushing a task branch and without claiming a validation-cell parent. +All three private modes bind one exact source ref/head, a one-ref Git bundle, digest, size, and private staging object. +The parent mode additionally binds its exact cell and reservation. +The public proof runs in a fresh bare repository with system/global Git configuration, credentials, prompts, extra HTTP headers, and file transport disabled; all modes repeat their exact public/private source proof immediately before compute creation and retry. No live worktree, primary home, provider account home, browser profile, or peer storage is mounted or copied. The canonical `fm.azure-command/v1` request binds these fields: @@ -72,7 +77,8 @@ The fixed root bootstrap installs a hard-coded Ubuntu transport and Linux test-t Before each package operation it waits up to three minutes for the standard apt/dpkg locks, and apt carries the same bounded dpkg timeout, so normal image maintenance can finish without turning into a repository-command failure while a stuck lock still fails closed. Repository code cannot alter that privileged package list, all package and staging traffic is shaped to one megabit per second and ends before deny-all command networking starts, and invocation evidence must record the resolved package/image versions during real acceptance. The request records the exact-size, checksum-pinned ShellCheck 0.11.0 and uv 0.9.10 releases plus, when the snapshot contains the Agent Fleet `uv.lock`, the complete Linux x86_64 pytest/ruff wheel closure selected from that lock. -When that lock is present, trusted root verifies the lock, archive, file set, sizes, and hashes, creates the Agent Fleet environment with an empty cache and networking disabled, and forces repository `uv run --locked` commands to use that already-synchronized offline environment. +When that lock is present, trusted root verifies the lock, archive, file set, sizes, and hashes, creates the Agent Fleet environment with an empty cache and networking disabled, and installs the exact locked Agent Fleet source plus its release-local `agent-fleet` console entrypoint before repository code starts. +The command wrapper then forces repository `uv run --locked` commands to use that synchronized offline environment through `UV_NO_SYNC=1`. ## Private control and VM boundary @@ -149,6 +155,7 @@ The validation-cell dispatcher may request up to eight mixed-family invocations A validation-owned invocation carries an exact `capacity-parent` cell id and complete parent vCPU reservation so the cell's pre-reserved processor shape and the child VM inventory cannot be double-counted or mistaken for unrelated capacity. Before admission and again before VM creation, the runner proves one live parent cell with the exact owner, deployment generation, home, lifecycle, id, and processor-reservation tags, then refuses a child beyond the reserved `(vCPUs - 8) / 4` slots. Each child still receives its own durable first-day reservation for its direct compute, storage, network, monitoring, and control meters; only the already-accounted $210 shared foundation reserve is omitted from that child reservation. +A direct private no-mistakes bundle has no validation parent and therefore takes the ordinary standalone shared-capacity reservation and complete foundation cost bound exactly once. Immediately before reservation and again immediately before VM creation, the controller proves the exact subscription/resource-group IDs and owner/generation tags for the named foundation storage account, zero-data admission-control account/container and ETag, controller UAMI and its sole exact effective container role including inherited/group expansion, VNet and address space, validation and private-endpoint subnets, complete NSG rule set, NAT and bound Standard public IP, blob private endpoint and endpoint NIC, named approved blob connection, private-DNS zone, VNet link, zone group/config names, and private-access properties. It also proves current SKU capabilities and restrictions, current East US regional and selected-family free vCPU quota, month-to-date actual cost, forecast cost, the exact unambiguous Linux on-demand Consumption retail meter (never Spot, Low Priority, Windows, dev/test, reservation, or savings pricing), and active runner count. @@ -348,10 +355,15 @@ local fallback, and `FM_AZURE_RUNNER_LOCAL_RECOVERY_CLASSES` remains the only explicit local opt-out. The lint payload preserves the tracked shell owner and locked Agent Fleet command unchanged inside the dispatched argv. -For a validation-owned feature branch, the caller passes its exact current `refs/heads/` identity plus the one-ref private snapshot bundle. -The runner binds and privately stages that unpushed commit, while a changed local bundle/head/tree or public default base refuses before compute creation. +For the per-run no-mistakes route, the dispatcher binds the detached gate HEAD to `refs/heads/fm-no-mistakes/` and asks the runner to build the complete one-ref private bundle inside its protected payload directory. +That source ref is a deterministic run identity, not a guess at the author's branch. +The direct route uses ordinary shared-capacity accounting and carries no validation-cell parent. +For a validation-owned feature branch, the caller passes its exact current `refs/heads/` identity plus the one-ref private snapshot bundle and parent reservation. +The runner binds and privately stages either unpushed commit, while a changed local bundle/head/tree or public default base refuses before compute creation. The ordinary test path runs the capability-derived real-Herdr host set locally and leaves complete behavior-inventory verification to required CI, as defined in [`configuration.md`](configuration.md#gate-defaults-no-mistakesyaml). When the `test` class is explicitly remote, `bin/fm-no-mistakes-test-command.sh` runs the sealed non-Herdr behavior inventory plus locked Agent Fleet checks on one Azure VM while every real-Herdr declaration runs through owned guarded labs on the Mac; a failed Azure shard is never replayed locally. +The remote command declares exactly `real-tmux-server,passwordless-root-escalation,system-openat-binding,origin-egress` absent and emits the existing loud per-unit skips rather than adding failures to the skip inventory. +The guest materializes complete source and default-branch history, so admitted non-Herdr fixtures retain their parent and historical-path graph instead of operating from a shallow root. Model review, document generation that requires a model, fixes, Git mutation, push, PR creation, CI monitoring, and gate decisions remain in no-mistakes' existing owner. A configured uncredentialed documentation command may use this runner like any other command, but this bridge never moves a model document step by implication. @@ -373,6 +385,8 @@ bin/fm-azure-runner.sh cost The implementation tests use fake Azure APIs and local executor fixtures only. They create no Azure resource and incur no charge. +Acceptance for the per-run no-mistakes repair remains blocked until this exact code lands on public `main` and one fresh routed `test=behavior-heavy` run passes on real Azure compute with exact remote execution proof and zero cleanup. +Do not spend on an unlanded build, and do not weaken the foundation's `require_landed_code` gate to accelerate that acceptance. Real usability remains unclaimed until the approved deployment performs this bounded acceptance: 1. Record Mac CPU, memory, swap, process count, and responsive interactive latency before launch. diff --git a/docs/azure-validation.md b/docs/azure-validation.md index cba71c97656..c95cbdffa1f 100644 --- a/docs/azure-validation.md +++ b/docs/azure-validation.md @@ -38,6 +38,7 @@ No operation stops, starts, updates, or inspects a local legacy no-mistakes daem Remote Herdr is not part of this path. Later Herdr proxy tabs may display state, but validation submission, ask-user responses, recovery, evidence collection, and cleanup use Azure Resource Manager, Managed Run Command, and the private storage endpoint. +The runner's separate per-run no-mistakes routing path may seal a detached gate HEAD as a direct private bundle with ordinary standalone capacity accounting; that path is not a validation cell or a child of one. No Azure resource was created while implementing this feature. Live Azure acceptance of these cells runs from released public main under separate explicit billable authorization; the first live Stage C pipeline acceptance has occurred and is recorded in the Live acceptance record section, while the full multi-leg checklist in the Live acceptance section remains unperformed. @@ -272,6 +273,7 @@ It does not run the command locally. The dispatcher assigns each shard the exact SKU and pre-reserved constituent id from the admitted shape plan, avoiding the control cell's family, and sets the runner concurrency ceiling to eight. Each request uses the runner's exact private-parent snapshot mode, so a pipeline-owned fix commit can run before the no-mistakes push step without executing locally or prematurely mutating the remote task branch. The one-ref Git bundle, current source ref/head/tree, digest, size, and private input blob are all bound to the parent cell and command request, while the runner independently re-proves the public trusted default base. +This parent-bound shard mode remains distinct from the direct per-run no-mistakes bundle in `docs/azure-runner.md`, which must not claim or require validation-cell capacity. A new child starts through `runner run` with an explicit invocation, confirmation, source ref, private bundle, and parent-cell reservation; only an already recorded child uses `runner resume`, so a missing VM cannot turn a prepared record into duplicate execution. The one-shot runner still re-proves live family quota, regional quota, rate, budget, private network, image, command bounds, and global admission under its own contract. Every shard therefore receives a separate VM, OS disk, process namespace, port space, lock space, temp root, terminal-server space, boot id, and VM instance id. @@ -484,6 +486,7 @@ A missing ETag, changed instance, foreign tag, foreign principal, extra role ass Focused fake-cloud and static tests do not claim real Azure usability. The first live Stage C pipeline acceptance has occurred, as described in the Live acceptance record section, while the full multi-leg acceptance checklist below remains unperformed. +The separate direct per-run runner repair is not accepted by those cell results and remains blocked until its exact code lands on public `main` and a fresh routed no-mistakes run passes on real Azure compute with zero cleanup. After this stack is released to public main and the operator has explicit billable authorization, the full checklist acceptance must run from that released main and record all of these results: 1. Record Mac wall time, CPU, memory, swap, process count, and interactive latency before admission. diff --git a/tests/fm-azure-runner.test.sh b/tests/fm-azure-runner.test.sh index 000ac9ba64c..e3e45a350ac 100755 --- a/tests/fm-azure-runner.test.sh +++ b/tests/fm-azure-runner.test.sh @@ -9,6 +9,7 @@ HOST="$ROOT/bin/fm-azure-runner.py" RUNNER="$ROOT/bin/fm-azure-runner.sh" GUEST="$ROOT/bin/fm-azure-runner-guest.sh" EXECUTOR="$ROOT/bin/fm-azure-runner-exec.py" +AGENT_FLEET_INSTALLER="$ROOT/bin/fm-azure-runner-agent-fleet-install.py" TEMPLATE="$ROOT/docs/azure-runner/invocation.json" SUB=11111111-1111-4111-8111-111111111111 TENANT=22222222-2222-4222-8222-222222222222 @@ -135,9 +136,9 @@ PY } static_private_controller_contract() { - python3 - "$TEMPLATE" "$HOST" "$GUEST" <<'PY' || fail "private controller static contract failed" + python3 - "$TEMPLATE" "$HOST" "$GUEST" "$AGENT_FLEET_INSTALLER" <<'PY' || fail "private controller static contract failed" import json, pathlib, sys -template=json.loads(pathlib.Path(sys.argv[1]).read_text()); host=pathlib.Path(sys.argv[2]).read_text(); guest=pathlib.Path(sys.argv[3]).read_text() +template=json.loads(pathlib.Path(sys.argv[1]).read_text()); host=pathlib.Path(sys.argv[2]).read_text(); guest=pathlib.Path(sys.argv[3]).read_text(); installer=pathlib.Path(sys.argv[4]).read_text() vm=next(r for r in template["resources"] if r["type"]=="Microsoft.Compute/virtualMachines") nic=next(r for r in template["resources"] if r["type"]=="Microsoft.Network/networkInterfaces") assert template["parameters"]["controllerIdentityId"]["type"] == "string" @@ -182,7 +183,20 @@ assert input_token_at < guest.index('rm -f "$TOKEN_FILE"',input_token_at) < run_ assert '/usr/bin/python3 "$EXECUTOR"' in guest assert "https://files.pythonhosted.org/packages/*.whl" in guest assert 'repository"].get("source_ancestors", [])' in guest +assert 'git -C /work/repo fetch --depth=1' not in guest +assert 'git -C /work/repo rev-parse --is-shallow-repository' in guest assert 'fetch_exact "$url"' in guest and '--location' not in guest[guest.index('while IFS=$\'\\t\' read -r url'):guest.index('done <"$BASE/wheels.tsv"')] +assert 'AGENT_FLEET_INSTALLER_B64=${agent_fleet_installer_b64:-}' in guest +assert 'agent_fleet_installer_digest' in guest and 'Agent Fleet installer digest mismatch' in guest +pip_at=guest.index('pip install --python /work/repo/tools/agent-fleet/.venv/bin/python') +install_at=guest.index('/.venv/bin/python "$AGENT_FLEET_INSTALLER"',pip_at) +assert pip_at < install_at < guest.index('/.venv/bin/agent-fleet --help',install_at) < guest.index('systemd-run --quiet') +assert 'FM_TEST_HOST_CAPABILITIES_ABSENT' not in guest +assert 'dependencies_match.group(1).strip() != "[]"' in installer +assert 'Agent Fleet lock does not bind the exact editable project' in installer +assert 'from agent_fleet.cli import main' in installer +assert 'private-direct-bundle' in host and 'private_snapshot_from_head' in host +assert 'verify_self_contained_private_bundle' in host assert "protectedParameters" not in host assert "generate-sas" not in host assert "controller_identity_client_id" in host @@ -316,8 +330,8 @@ repo=pathlib.Path(sys.argv[2]); bundle=pathlib.Path(sys.argv[3]); state_dir=path env={"state_dir":state_dir,"home_binding":"sha256:"+"a"*64,"deployment_generation":"gen","prefix":"fmtest","subscription":"11111111-1111-4111-8111-111111111111","resource_group":"rg","owner":"owner","cost_admission_mode":m.STRICT_COST_ADMISSION_MODE,"cell_ordinal":None} m.ensure_state_dirs(env) head=m.git(repo,"rev-parse","HEAD").stdout.strip(); tree=m.git(repo,"rev-parse","HEAD^{tree}").stdout.strip() -m.public_origin_proof=lambda *_a,**_k:{"remote":"https://github.com/Ruby-Labs/cloud-host-owner.git","default_ref":"refs/heads/main","default_head":"d"*40,"source_ref":"refs/heads/topic","source_head":head,"tree":tree} -args=argparse.Namespace(repo=str(repo),task="azv-aaaaaaaaaaaa-s1",generation="round-aaaaaaaaaaaa",resource_class="behavior-heavy",source_ref="refs/heads/topic",private_snapshot_bundle=str(bundle),capacity_parent="azv-aaaaaaaaaaaa",capacity_reservation_vcpus=40,wall_seconds=None,dependency=[],artifact=[],command=["true"],invocation="azr-aaaaaaaaaaaa") +m.public_origin_proof=lambda *_a,**k:{"remote":"https://github.com/Ruby-Labs/cloud-host-owner.git","default_ref":"refs/heads/main","default_head":"d"*40,"source_ref":k.get("source_ref") or "refs/heads/main","source_head":head,"tree":tree} +args=argparse.Namespace(repo=str(repo),task="azv-aaaaaaaaaaaa-s1",generation="round-aaaaaaaaaaaa",resource_class="behavior-heavy",source_ref="refs/heads/topic",public_ref=None,public_ancestor=[],private_snapshot_bundle=str(bundle),private_snapshot_from_head=False,capacity_parent="azv-aaaaaaaaaaaa",capacity_reservation_vcpus=40,wall_seconds=None,dependency=[],artifact=[],command=["true"],invocation="azr-aaaaaaaaaaaa") state=m.prepare(env,args) r=state["request"]["repository"] assert r["source_mode"]=="private-parent-bundle" and r["source_head"]==head and r["tree"]==tree @@ -367,8 +381,31 @@ args.private_snapshot_bundle=str(plain_bundle) plain_state=m.prepare(env,args) assert plain_state["request"]["repository"]["source_head"]==plain_head assert plain_state["request"]["protocol"]["agent_fleet_python"]=={"lock_digest":None,"wheels":[]} +# The direct no-mistakes route seals a detached, unpushed HEAD without a +# validation parent. Its deterministic private ref is identity, not a guessed +# task branch, and the bundle must carry complete ancestry into an empty repo. +args.repo=str(repo); args.resource_class="behavior-heavy"; args.command=["true"] +args.private_snapshot_bundle=None; args.private_snapshot_from_head=True +args.capacity_parent=None; args.capacity_reservation_vcpus=None +args.source_ref="refs/heads/fm-no-mistakes/01BX5ZZKBKACTAV9WEVGEMMVRZ" +args.invocation="azr-eeeeeeeeeeee"; args.task="nm-01BX5ZZKBKACTAV9WEVGEMMVRZ" +(repo/"second").write_text("second\n"); m.run(["git","-C",str(repo),"add","second"]); m.run(["git","-C",str(repo),"commit","-qm","second"]) +head=m.git(repo,"rev-parse","HEAD").stdout.strip(); parent=m.git(repo,"rev-parse","HEAD^1").stdout.strip() +m.public_origin_proof=lambda *_a,**k:{"remote":"https://github.com/Ruby-Labs/cloud-host-owner.git","default_ref":"refs/heads/main","default_head":"d"*40,"source_ref":k.get("source_ref"),"source_head":head,"tree":m.git(repo,"rev-parse","HEAD^{tree}").stdout.strip()} +m.run(["git","-C",str(repo),"checkout","-q","--detach",head]) +direct=m.prepare(env,args); direct_repo=direct["request"]["repository"] +assert direct_repo["source_mode"]=="private-direct-bundle" +assert direct_repo["source_ref"]==args.source_ref and direct_repo["source_head"]==head +assert direct["request"]["capacity_parent"] is None +assert direct["request"]["capacity_fence"] is None +sealed=pathlib.Path(direct["input_path"]).parent/"snapshot.bundle" +assert m.git(repo,"bundle","list-heads",str(sealed)).stdout.splitlines()==[head+" "+args.source_ref] +verify=repo.parent/"verify.git"; m.run(["git","init","--bare",str(verify)]); m.run(["git","-C",str(verify),"bundle","verify",str(sealed)]); m.run(["git","-C",str(verify),"fetch",str(sealed),args.source_ref]) +assert m.git(verify,"rev-parse","FETCH_HEAD").stdout.strip()==head +assert m.git(verify,"rev-parse","FETCH_HEAD^1").stdout.strip()==parent +assert m.git(verify,"rev-parse","--is-shallow-repository").stdout.strip()=="false" PY - pass "private snapshot preparation binds both parent-cell and credentialless exact-checkout bundle modes" + pass "private prepare binds parent, exact-checkout, and detached direct source graphs" } private_snapshot_ancestor_verification() { @@ -421,6 +458,58 @@ PY pass "private bundle modes verify ancestors locally without contacting origin" } +agent_fleet_offline_install_contract() { + local tmp project python out rc + fm_test_tmproot_into tmp fm-azure-agent-fleet-install + project="$tmp/project" + mkdir -p "$project/src" + cp "$ROOT/tools/agent-fleet/pyproject.toml" "$ROOT/tools/agent-fleet/uv.lock" "$project/" + cp -R "$ROOT/tools/agent-fleet/src/agent_fleet" "$project/src/" + python="$ROOT/tools/agent-fleet/.venv/bin/python" + [ -x "$python" ] || fail "the provisioned Agent Fleet Python is unavailable" + "$python" -m venv --without-pip "$project/.venv" \ + || fail "could not create the hermetic Agent Fleet installer venv" + "$project/.venv/bin/python" "$AGENT_FLEET_INSTALLER" "$project" "$project/.venv" >/dev/null \ + || fail "the locked Agent Fleet project could not be installed offline" + "$project/.venv/bin/agent-fleet" --help >/dev/null \ + || fail "the offline Agent Fleet console entrypoint is not runnable" + "$project/.venv/bin/python" - "$project/.venv/bin/agent-fleet" <<'PY' \ + || fail "the offline Agent Fleet project is not release-local" +import pathlib, stat, sys +from agent_fleet.providers import agent_fleet_entrypoint_path +expected = pathlib.Path(sys.argv[1]).absolute() +actual = agent_fleet_entrypoint_path() +assert actual == expected, (actual, expected) +metadata = expected.lstat() +assert stat.S_ISREG(metadata.st_mode) and metadata.st_mode & 0o111 +PY + sed 's/^dependencies = \[\]$/dependencies = ["requests"]/' "$project/pyproject.toml" \ + >"$project/pyproject.changed" + mv "$project/pyproject.changed" "$project/pyproject.toml" + "$python" -m venv --without-pip "$project/changed-venv" >/dev/null + rc=0 + out=$("$project/changed-venv/bin/python" "$AGENT_FLEET_INSTALLER" \ + "$project" "$project/changed-venv" 2>&1) || rc=$? + [ "$rc" -eq 125 ] \ + || fail "the offline Agent Fleet installer accepted an unsealed runtime dependency" + assert_contains "$out" "gained runtime dependencies" \ + "the offline Agent Fleet refusal did not name its closure change" + cp "$ROOT/tools/agent-fleet/pyproject.toml" "$project/pyproject.toml" + mkdir -p "$tmp/linked-source" + cp -R "$ROOT/tools/agent-fleet/src/agent_fleet" "$tmp/linked-source/" + rm -rf "$project/src" + ln -s "$tmp/linked-source" "$project/src" + "$python" -m venv --without-pip "$project/linked-venv" >/dev/null + rc=0 + out=$("$project/linked-venv/bin/python" "$AGENT_FLEET_INSTALLER" \ + "$project" "$project/linked-venv" 2>&1) || rc=$? + [ "$rc" -eq 125 ] \ + || fail "the offline Agent Fleet installer followed linked source ancestry" + assert_contains "$out" "real directory ancestry" \ + "the linked Agent Fleet source refusal did not name its ancestry defect" + pass "the locked Agent Fleet project and console entrypoint are installed into the offline venv before execution" +} + executor_credential_adversary() { local tmp repo request output uid gid fm_test_tmproot_into tmp fm-azure-exec-adversary @@ -988,6 +1077,13 @@ printf '%s\n' "\$@" >"$root/captured" printf 'FM_AZURE_RUNNER_STATE_DIR=%s\n' "\${FM_AZURE_RUNNER_STATE_DIR:-}" printf 'FM_AZURE_SHARED_CAPACITY_STATE_DIR=%s\n' "\${FM_AZURE_SHARED_CAPACITY_STATE_DIR:-}" } >"$root/captured-env" +if [ "\${FM_TEST_FIXTURE_EXECUTE_REMOTE:-0}" = 1 ]; then + while [ "\$#" -gt 0 ] && [ "\${1:-}" != -- ]; do shift; done + [ "\${1:-}" = -- ] || exit 98 + shift + ( cd "$root" && HOME="\${FM_TEST_FIXTURE_REMOTE_HOME:?}" FM_AZURE_RUNNER=1 "\$@" ) + exit \$? +fi exit 0 SH cat >"$root/tests/run.sh" <>"$root/local-runs" +if [ "\${1:-}" = --skip-herdr ]; then + [ "\${FM_TEST_HOST_CAPABILITIES_ABSENT:-}" = real-tmux-server,passwordless-root-escalation,system-openat-binding,origin-egress ] || exit 96 + printf 'FM_HOST_CAPABILITY_DECLARATION absent=%s\n' "\$FM_TEST_HOST_CAPABILITIES_ABSENT" >>"$root/remote-runs" +fi exit 0 SH cat >"$root/tests/test-capabilities.tsv" <<'TSV' @@ -1180,13 +1280,14 @@ PY no_mistakes_test_step_offload_contract() { local tmp fixture gatewt gate_head fakebin anchor fmhome routing expires rc out - local mutation mutation_action replacement + local mutation mutation_action replacement remote_home project python fm_test_tmproot_into tmp fm-azure-runner-test-step-offload fixture="$tmp/fixture" make_dispatch_fixture "$fixture" gatewt="$tmp/nm-home/worktrees/19543ae8611e/$NM_RUN_FIXTURE" make_ambient_worktree "$gatewt" gate_head=$(git -C "$gatewt" rev-parse HEAD) + git -C "$gatewt" checkout -q --detach "$gate_head" fakebin="$tmp/fakebin" mkdir -p "$fakebin" cat >"$fakebin/tmux" <<'SH' @@ -1267,6 +1368,72 @@ PY assert_capability_derived_local_host_set "$fixture/local-runs" [ "$(routing_dispatch_count)" -eq 1 ] \ || fail "non-consuming test inspection did not leave exactly one durable dispatch spend" + grep -qx -- '--private-snapshot-from-head' "$fixture/captured" \ + || fail "the detached per-run gate did not select an exact private HEAD snapshot" + grep -qx -- "refs/heads/fm-no-mistakes/$NM_RUN_FIXTURE" "$fixture/captured" \ + || fail "the detached per-run gate did not bind its deterministic private source ref" + ! grep -qx -- '--capacity-parent' "$fixture/captured" \ + || fail "the direct per-run gate incorrectly required a validation-cell parent" + + # Execute the exact production remote argv in a hermetic fixture. The fake uv + # keeps this focused on orchestration while requiring the release-local Agent + # Fleet entrypoint that all 416 incident setup errors could not find. + remote_home="$tmp/remote-home" + mkdir -p "$remote_home/.fm-runner-tools/bin" "$remote_home/.fm-runner-tools/uv" \ + "$remote_home/.fm-runner-tools/wheelhouse" "$fixture/tools/agent-fleet/src" + cp "$ROOT/tools/agent-fleet/pyproject.toml" "$ROOT/tools/agent-fleet/uv.lock" \ + "$fixture/tools/agent-fleet/" + cp -R "$ROOT/tools/agent-fleet/src/agent_fleet" "$fixture/tools/agent-fleet/src/" + python="$ROOT/tools/agent-fleet/.venv/bin/python" + "$python" -m venv --without-pip "$fixture/tools/agent-fleet/.venv" >/dev/null \ + || fail "the remote-command fixture could not create its Agent Fleet venv" + "$fixture/tools/agent-fleet/.venv/bin/python" "$AGENT_FLEET_INSTALLER" \ + "$fixture/tools/agent-fleet" "$fixture/tools/agent-fleet/.venv" >/dev/null \ + || fail "the remote-command fixture could not install Agent Fleet offline" + cat >"$remote_home/.fm-runner-tools/bin/shellcheck" <<'SH' +#!/bin/sh +printf '%s\n' 'ShellCheck - shell script analysis tool' 'version: 0.11.0' +SH + cat >"$remote_home/.fm-runner-tools/uv/uv" <>"$fixture/remote-uv-runs" +[ "\${1:-}" = run ] && [ "\${2:-}" = --directory ] && [ "\${4:-}" = --locked ] || exit 95 +case "\${5:-}" in + pytest) + [ -x "\$3/.venv/bin/agent-fleet" ] || exit 93 + "\$3/.venv/bin/agent-fleet" --help >/dev/null + "\$3/.venv/bin/python" -c 'from agent_fleet.providers import agent_fleet_entrypoint_path; assert agent_fleet_entrypoint_path().is_file()' + ;; + python) + shift 5 + "tools/agent-fleet/.venv/bin/python" "\$@" + ;; + *) exit 92 ;; +esac +SH + chmod +x "$remote_home/.fm-runner-tools/bin/shellcheck" \ + "$remote_home/.fm-runner-tools/uv/uv" + write_test_routing '{}' + rm -f "$fixture/captured" "$fixture/local-runs" "$fixture/remote-runs" "$fixture/remote-uv-runs" + out=$(cd "$gatewt" && env HOME="$anchor" PATH="$fakebin:$PATH" \ + FM_TEST_FIXTURE_EXECUTE_REMOTE=1 FM_TEST_FIXTURE_REMOTE_HOME="$remote_home" \ + "$fixture/bin/fm-no-mistakes-test-command.sh" 2>&1) \ + || fail "the full remote no-mistakes command failed in its hermetic fixture" + assert_contains "$out" "selected REMOTE resource-class=behavior-heavy" \ + "the hermetic full command lost its remote routing selection" + grep -Fqx \ + "FM_HOST_CAPABILITY_DECLARATION absent=real-tmux-server,passwordless-root-escalation,system-openat-binding,origin-egress" \ + "$fixture/remote-runs" \ + || fail "the production remote command lost the exact four-name capability declaration" + grep -Fqx "run --directory tools/agent-fleet --locked pytest" \ + "$fixture/remote-uv-runs" \ + || fail "the production remote command did not run locked Agent Fleet pytest" + grep -Fqx "run --directory tools/agent-fleet --locked python -m compileall -q src" \ + "$fixture/remote-uv-runs" \ + || fail "the production remote command did not run locked Agent Fleet compileall" + pass "the full production remote command runs hermetically with four named absences and the offline Agent Fleet entrypoint" # Inspection is only a planning read. The real heavy dispatch must carry an # exact binding from that read so deletion, unselection, or replacement in @@ -1390,6 +1557,8 @@ assert value("--task")==sys.argv[2] assert value("--generation")==sys.argv[3] assert value("--confirm-subscription")==sys.argv[4] assert value("--resource-class")=="behavior-heavy" +declaration="FM_TEST_HOST_CAPABILITIES_ABSENT=real-tmux-server,passwordless-root-escalation,system-openat-binding,origin-egress" +assert declaration in argv, "the Azure shard lost its reviewed Linux host-capability declaration" assert any("tests/run.sh --skip-herdr" in item for item in argv), "the Azure shard lost the non-Herdr suite" PY assert_capability_derived_local_host_set "$fixture/local-runs" @@ -1417,6 +1586,7 @@ storage_network_access_contract prepare_contract private_snapshot_prepare_contract private_snapshot_ancestor_verification +agent_fleet_offline_install_contract executor_credential_adversary linux_systemd_drop_integration spend_ledger_unit @@ -1483,6 +1653,11 @@ required = ( "immutable Azure `vm_instance_id`", "verified guest `boot_id`", "The earlier `selected REMOTE ... (dispatching)` line proves only selection", + "`--private-snapshot-from-head`", + "`refs/heads/fm-no-mistakes/`", + "ordinary standalone shared-capacity reservation", + "real-tmux-server,passwordless-root-escalation,system-openat-binding,origin-egress", + "one fresh routed `test=behavior-heavy` run passes on real Azure compute", ) missing = [needle for needle in required if needle not in document] assert not missing, missing diff --git a/tests/fm-nm-step-liveness.test.sh b/tests/fm-nm-step-liveness.test.sh index 7a258ab8dce..32c93fb4968 100755 --- a/tests/fm-nm-step-liveness.test.sh +++ b/tests/fm-nm-step-liveness.test.sh @@ -228,13 +228,42 @@ pass "a missing run id is a usage error" # (j1) The exact incident signature: a step whose log is quiet, whose agent_pid # is empty and whose round reads `starting` - i.e. nothing in `axi status` says # it is alive - is still reported ALIVE while its processes are working. -# A one-shot call has no stored baseline. The fixture turns over children during -# the short in-invocation sample, matching a suite loop beginning new units. -( cd "$WT" && exec bash -c 'while :; do sleep 0.2; done' ) & +# A one-shot call has no stored baseline, so membership turnover is its positive +# signal. The original fixture hoped `sleep 0.2` would turn over within the +# one-second sample. Under the loaded Azure C2 run that child stayed scheduled +# for the whole sample and the assertion saw stable presence, even though the +# probe was correct to refuse `alive`. Drive one exact post-baseline membership +# change through the test barrier instead. This preserves the same positive +# signal without relying on scheduler timing. +( cd "$WT" && exec sleep 120 ) & LIVE=$! STARTED_PIDS="$STARTED_PIDS $LIVE" SNAP_J="$TMP_ROOT/snap-j1" -out=$(FM_NM_SNAP_DIR="$SNAP_J" "$PROBE" "$RUN_ID" --worktree "$WT" --sample 1) +LIVE_BARRIER="$TMP_ROOT/live-barrier" +LIVE_OUT="$TMP_ROOT/live.out" +mkdir -p "$LIVE_BARRIER" +FM_NM_SNAP_DIR="$SNAP_J" \ + FM_NM_TEST_BARRIER_DIR="$LIVE_BARRIER" \ + FM_NM_TEST_BARRIER_PHASE=before-progress-sample \ + "$PROBE" "$RUN_ID" --worktree "$WT" --sample 1 >"$LIVE_OUT" & +LIVE_PROBE=$! +STARTED_PIDS="$STARTED_PIDS $LIVE_PROBE" +wait_for_barrier "$LIVE_BARRIER" "$LIVE_PROBE" \ + || fail "the one-shot liveness probe never captured its initial membership" +( cd "$WT" && exec sleep 120 ) & +LIVE_SUCCESSOR=$! +STARTED_PIDS="$STARTED_PIDS $LIVE_SUCCESSOR" +out="" +for _ in 1 2 3 4 5 6 7 8 9 10; do + out=$("$PROBE" "$RUN_ID" --worktree "$WT" --sample 0) + case "$out" in *"procs: 2"*) break ;; esac + sleep 0.2 +done +assert_contains "$out" "procs: 2" \ + "the barrier-controlled successor was not observable before the sample" +printf 'before-progress-sample\n' >"$LIVE_BARRIER/release" +wait "$LIVE_PROBE" || fail "the one-shot liveness probe failed after release" +out=$(cat "$LIVE_OUT") [ "$(verdict_of "$out")" = alive ] \ || fail "REGRESSION: a one-shot working step with no prior sample must read alive, got: $out" assert_contains "$out" "process membership changed in 1s" \ @@ -244,8 +273,7 @@ pass "regression: a one-shot call proves a quiet working step alive from child t # (j2) A momentary gap between units of work must NOT read dead. The scan barrier # proves the first scan was empty before the successor is allowed to appear, so # this cannot pass merely because a load-delayed first scan found the successor. -kill_tree "$LIVE" -wait "$LIVE" 2>/dev/null || true +kill_started GAP_BARRIER="$TMP_ROOT/gap-barrier" mkdir -p "$GAP_BARRIER" GAP_OUT="$TMP_ROOT/gap.out" From 49af0fea5645c96cae935c80a8a0ec00d86eae40 Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Mon, 24 Aug 2026 15:10:52 -0400 Subject: [PATCH 02/13] no-mistakes(review): Replace Azure source assertions with behavioral coverage --- tests/fm-azure-runner.test.sh | 28 +++++----------------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/tests/fm-azure-runner.test.sh b/tests/fm-azure-runner.test.sh index e3e45a350ac..d6896896e29 100755 --- a/tests/fm-azure-runner.test.sh +++ b/tests/fm-azure-runner.test.sh @@ -136,9 +136,9 @@ PY } static_private_controller_contract() { - python3 - "$TEMPLATE" "$HOST" "$GUEST" "$AGENT_FLEET_INSTALLER" <<'PY' || fail "private controller static contract failed" + python3 - "$TEMPLATE" "$HOST" "$GUEST" <<'PY' || fail "private controller static contract failed" import json, pathlib, sys -template=json.loads(pathlib.Path(sys.argv[1]).read_text()); host=pathlib.Path(sys.argv[2]).read_text(); guest=pathlib.Path(sys.argv[3]).read_text(); installer=pathlib.Path(sys.argv[4]).read_text() +template=json.loads(pathlib.Path(sys.argv[1]).read_text()); host=pathlib.Path(sys.argv[2]).read_text(); guest=pathlib.Path(sys.argv[3]).read_text() vm=next(r for r in template["resources"] if r["type"]=="Microsoft.Compute/virtualMachines") nic=next(r for r in template["resources"] if r["type"]=="Microsoft.Network/networkInterfaces") assert template["parameters"]["controllerIdentityId"]["type"] == "string" @@ -183,20 +183,7 @@ assert input_token_at < guest.index('rm -f "$TOKEN_FILE"',input_token_at) < run_ assert '/usr/bin/python3 "$EXECUTOR"' in guest assert "https://files.pythonhosted.org/packages/*.whl" in guest assert 'repository"].get("source_ancestors", [])' in guest -assert 'git -C /work/repo fetch --depth=1' not in guest -assert 'git -C /work/repo rev-parse --is-shallow-repository' in guest assert 'fetch_exact "$url"' in guest and '--location' not in guest[guest.index('while IFS=$\'\\t\' read -r url'):guest.index('done <"$BASE/wheels.tsv"')] -assert 'AGENT_FLEET_INSTALLER_B64=${agent_fleet_installer_b64:-}' in guest -assert 'agent_fleet_installer_digest' in guest and 'Agent Fleet installer digest mismatch' in guest -pip_at=guest.index('pip install --python /work/repo/tools/agent-fleet/.venv/bin/python') -install_at=guest.index('/.venv/bin/python "$AGENT_FLEET_INSTALLER"',pip_at) -assert pip_at < install_at < guest.index('/.venv/bin/agent-fleet --help',install_at) < guest.index('systemd-run --quiet') -assert 'FM_TEST_HOST_CAPABILITIES_ABSENT' not in guest -assert 'dependencies_match.group(1).strip() != "[]"' in installer -assert 'Agent Fleet lock does not bind the exact editable project' in installer -assert 'from agent_fleet.cli import main' in installer -assert 'private-direct-bundle' in host and 'private_snapshot_from_head' in host -assert 'verify_self_contained_private_bundle' in host assert "protectedParameters" not in host assert "generate-sas" not in host assert "controller_identity_client_id" in host @@ -465,8 +452,7 @@ agent_fleet_offline_install_contract() { mkdir -p "$project/src" cp "$ROOT/tools/agent-fleet/pyproject.toml" "$ROOT/tools/agent-fleet/uv.lock" "$project/" cp -R "$ROOT/tools/agent-fleet/src/agent_fleet" "$project/src/" - python="$ROOT/tools/agent-fleet/.venv/bin/python" - [ -x "$python" ] || fail "the provisioned Agent Fleet Python is unavailable" + python=$(command -v python3) "$python" -m venv --without-pip "$project/.venv" \ || fail "could not create the hermetic Agent Fleet installer venv" "$project/.venv/bin/python" "$AGENT_FLEET_INSTALLER" "$project" "$project/.venv" >/dev/null \ @@ -819,6 +805,7 @@ assert len(reserve_calls)==2 and sleeps assert all(command[command.index("--reservation-id")+1]=="azr-aaaaaaaaaaaa" for command in reserve_calls) assert len({command[command.index("--fence-binding")+1] for command in reserve_calls})==1 assert reserve_calls[-1][reserve_calls[-1].index("--sku-family")+1]=="StandardDasv7Family" +assert reserve_calls[-1][reserve_calls[-1].index("--role")+1]=="validation" assert state["shared_capacity_reservation"]["status"]=="reserved" # A non-capacity queue refusal is immediate and releases its exact row. state.pop("shared_capacity_reservation") @@ -1384,7 +1371,7 @@ PY cp "$ROOT/tools/agent-fleet/pyproject.toml" "$ROOT/tools/agent-fleet/uv.lock" \ "$fixture/tools/agent-fleet/" cp -R "$ROOT/tools/agent-fleet/src/agent_fleet" "$fixture/tools/agent-fleet/src/" - python="$ROOT/tools/agent-fleet/.venv/bin/python" + python=$(command -v python3) "$python" -m venv --without-pip "$fixture/tools/agent-fleet/.venv" >/dev/null \ || fail "the remote-command fixture could not create its Agent Fleet venv" "$fixture/tools/agent-fleet/.venv/bin/python" "$AGENT_FLEET_INSTALLER" \ @@ -1653,11 +1640,6 @@ required = ( "immutable Azure `vm_instance_id`", "verified guest `boot_id`", "The earlier `selected REMOTE ... (dispatching)` line proves only selection", - "`--private-snapshot-from-head`", - "`refs/heads/fm-no-mistakes/`", - "ordinary standalone shared-capacity reservation", - "real-tmux-server,passwordless-root-escalation,system-openat-binding,origin-egress", - "one fresh routed `test=behavior-heavy` run passes on real Azure compute", ) missing = [needle for needle in required if needle not in document] assert not missing, missing From 3342d2d262a54472cf2d67e6de6a0cb1dcf7c5e0 Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Mon, 24 Aug 2026 17:49:17 -0400 Subject: [PATCH 03/13] no-mistakes(test): Align teardown fixture with authoritative HEAD --- tests/fm-teardown-suite.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/fm-teardown-suite.sh b/tests/fm-teardown-suite.sh index d96e0990e48..2ed209c85bd 100644 --- a/tests/fm-teardown-suite.sh +++ b/tests/fm-teardown-suite.sh @@ -6597,6 +6597,9 @@ SH git -C "$firstmate_source" fetch --quiet "$ROOT" "$firstmate_tip" git -C "$firstmate_source" checkout --quiet -b "$default" FETCH_HEAD git -C "$case_dir/project" remote set-url origin "$firstmate_source" + git -C "$case_dir/project" fetch --quiet "$ROOT" "$firstmate_tip" + git -C "$case_dir/wt" reset --quiet --hard "$firstmate_tip" + git -C "$case_dir/project" update-ref "refs/remotes/origin/$default" "$firstmate_tip" PATH=$original_path tip=$(git -C "$source" rev-parse refs/remotes/origin/main) git -C "$clone" remote set-url origin https://example.com/repository.git From 35d72f68a97672bfd7506a0b985b555136f737c6 Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Mon, 24 Aug 2026 23:40:12 -0400 Subject: [PATCH 04/13] fix(validation): bound local behavior proof --- docs/configuration.md | 2 +- tests/fm-azure-runner.test.sh | 6 ++++-- tests/fm-teardown-suite.sh | 5 +++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index fc3d7b37f81..ffb117a93d4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -101,7 +101,7 @@ The ordinary local command requires `tmux` on `PATH`, prints `tmux -V`, derives It does not maintain a second file list or duplicate hermetic-only behavior files on the Mac. The required `Behavior tests` job in [`.github/workflows/ci.yml`](../.github/workflows/ci.yml) uses the duration-balanced sharding owned by [`bin/fm-behavior-shards.sh`](../bin/fm-behavior-shards.sh) to run and verify the complete behavior inventory across eight isolated runners. CI selects the explicit non-Herdr path because its disposable image carries no Herdr, so it runs every hermetic body and the hermetic portion of the mixed file while the local host admits the complete `herdr-lab` and `herdr-mixed` set through owned labs. -The exact behavioral coverage contract is the union of that capability-derived local host set and the required CI executed-manifest union, not a claim that all 123 files run serially before push. +The exact behavioral coverage contract is the union of that capability-derived local host set and the required CI executed-manifest union, not a claim that all behavior files run serially before push. Both routes cross the same sealed admission boundary, and the local serial route holds real-Herdr labs to one at a time. ## Crosscheck reviewer diff --git a/tests/fm-azure-runner.test.sh b/tests/fm-azure-runner.test.sh index d6896896e29..25851d970e0 100755 --- a/tests/fm-azure-runner.test.sh +++ b/tests/fm-azure-runner.test.sh @@ -452,7 +452,8 @@ agent_fleet_offline_install_contract() { mkdir -p "$project/src" cp "$ROOT/tools/agent-fleet/pyproject.toml" "$ROOT/tools/agent-fleet/uv.lock" "$project/" cp -R "$ROOT/tools/agent-fleet/src/agent_fleet" "$project/src/" - python=$(command -v python3) + python=$(uv python find '>=3.11') \ + || fail "could not resolve Python 3.11+ for the hermetic Agent Fleet installer" "$python" -m venv --without-pip "$project/.venv" \ || fail "could not create the hermetic Agent Fleet installer venv" "$project/.venv/bin/python" "$AGENT_FLEET_INSTALLER" "$project" "$project/.venv" >/dev/null \ @@ -1371,7 +1372,8 @@ PY cp "$ROOT/tools/agent-fleet/pyproject.toml" "$ROOT/tools/agent-fleet/uv.lock" \ "$fixture/tools/agent-fleet/" cp -R "$ROOT/tools/agent-fleet/src/agent_fleet" "$fixture/tools/agent-fleet/src/" - python=$(command -v python3) + python=$(uv python find '>=3.11') \ + || fail "could not resolve Python 3.11+ for the remote-command fixture" "$python" -m venv --without-pip "$fixture/tools/agent-fleet/.venv" >/dev/null \ || fail "the remote-command fixture could not create its Agent Fleet venv" "$fixture/tools/agent-fleet/.venv/bin/python" "$AGENT_FLEET_INSTALLER" \ diff --git a/tests/fm-teardown-suite.sh b/tests/fm-teardown-suite.sh index 2ed209c85bd..bf796d28a51 100644 --- a/tests/fm-teardown-suite.sh +++ b/tests/fm-teardown-suite.sh @@ -6933,6 +6933,11 @@ test_secondmate_registry_updates_are_locked_and_literal() { pass "secondmate registry updates are serialized and compare ids literally" } +if [ "${FM_TEST_FOCUSED:-}" = network-authority ]; then + test_secondmate_network_fetches_pin_validated_addresses + exit 0 +fi + if [ "${FM_TEST_FOCUSED:-}" = tasktmp-safety ]; then test_teardown_removes_safe_tasktmp_and_accepts_absence test_teardown_refuses_unsafe_tasktmp_metadata From a54fa281ba92750f9f9ffffb05f04735f9e4374c Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Tue, 25 Aug 2026 00:16:09 -0400 Subject: [PATCH 05/13] no-mistakes(test): Reap watcher subprocesses and parallelize local validation --- bin/fm-no-mistakes-test-command.sh | 26 ++++++++++++++++----- bin/fm-supervise-daemon.sh | 13 ++++++++++- bin/fm-watch.sh | 13 +++-------- tests/fm-azure-runner.test.sh | 21 +++++++++++++++++ tests/fm-watcher-lock.test.sh | 36 ++++++++++++++++++++++++++++++ 5 files changed, 93 insertions(+), 16 deletions(-) diff --git a/bin/fm-no-mistakes-test-command.sh b/bin/fm-no-mistakes-test-command.sh index a5dac3b536c..a923996d426 100755 --- a/bin/fm-no-mistakes-test-command.sh +++ b/bin/fm-no-mistakes-test-command.sh @@ -14,11 +14,27 @@ run_local_required() { tmux -V printf 'no-mistakes: local host set files=%s source=tests/test-capabilities.tsv; complete behavior inventory is required in CI\n' \ "${#herdr_tests[@]}" - local rc=0 - "$ROOT/tests/run.sh" "${herdr_tests[@]}" || rc=1 - uv run --directory "$ROOT/tools/agent-fleet" --locked pytest || rc=1 - uv run --directory "$ROOT/tools/agent-fleet" --locked python -m compileall -q src || rc=1 - return "$rc" + local lane_dir herdr_pid agent_fleet_pid herdr_rc=0 agent_fleet_rc=0 + lane_dir=$(mktemp -d "${TMPDIR:-/tmp}/fm-local-test-lanes.XXXXXX") || return 1 + "$ROOT/tests/run.sh" "${herdr_tests[@]}" >"$lane_dir/herdr.log" 2>&1 & + herdr_pid=$! + ( + rc=0 + uv run --directory "$ROOT/tools/agent-fleet" --locked pytest || rc=1 + uv run --directory "$ROOT/tools/agent-fleet" --locked python -m compileall -q src || rc=1 + exit "$rc" + ) >"$lane_dir/agent-fleet.log" 2>&1 & + agent_fleet_pid=$! + wait "$herdr_pid" || herdr_rc=$? + wait "$agent_fleet_pid" || agent_fleet_rc=$? + printf '%s\n' '== local Herdr lane ==' && cat "$lane_dir/herdr.log" + printf '%s\n' '== local Agent Fleet lane ==' && cat "$lane_dir/agent-fleet.log" + rm -rf "$lane_dir" + if [ "$herdr_rc" -ne 0 ] || [ "$agent_fleet_rc" -ne 0 ]; then + printf 'no-mistakes local test lanes failed: herdr=%s agent-fleet=%s\n' \ + "$herdr_rc" "$agent_fleet_rc" >&2 + return 1 + fi } # A daemon step inherits no FM_* selection variables from the operator. Ask the diff --git a/bin/fm-supervise-daemon.sh b/bin/fm-supervise-daemon.sh index 2cf08fc1b05..f0b35f84776 100755 --- a/bin/fm-supervise-daemon.sh +++ b/bin/fm-supervise-daemon.sh @@ -1314,6 +1314,17 @@ trim_log() { # classifiers above are sourceable for unit tests (tests/fm-daemon.test.sh). # ============================================================================ +fm_super_stop_watcher() { # + local watcher_pid=$1 child current_parent + while read -r child; do + [ -n "$child" ] || continue + current_parent=$(ps -o ppid= -p "$child" 2>/dev/null | tr -d '[:space:]') + [ "$current_parent" = "$watcher_pid" ] || continue + kill -TERM "$child" 2>/dev/null || true + done < <(ps -axo pid=,ppid= | awk -v parent="$watcher_pid" '$2 == parent { print $1 }') + kill -TERM "$watcher_pid" 2>/dev/null || true +} + fm_super_main() { local STATE DELIVERY BACKEND TARGET backend_source target_source STATE="$(_state_root)" @@ -1457,7 +1468,7 @@ fm_super_main() { escalate_flush "$STATE" 2>/dev/null || true fi if [ -n "${WATCHER_PID:-}" ]; then - kill "$WATCHER_PID" 2>/dev/null || true + fm_super_stop_watcher "$WATCHER_PID" wait "$WATCHER_PID" 2>/dev/null || true fi if [ -n "${CUR_TMP:-}" ]; then diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 1f111990871..7f99b3f1f63 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -69,6 +69,8 @@ mkdir -p "$STATE" # shellcheck source=bin/fm-wake-lib.sh . "$SCRIPT_DIR/fm-wake-lib.sh" +# shellcheck source=bin/fm-process-tree-lib.sh +. "$SCRIPT_DIR/fm-process-tree-lib.sh" # Shared wake classifier (captain-relevant verbs + signal/stale/heartbeat # predicates), the SAME library the away-mode daemon uses, so the triage policy # has one definition. @@ -739,16 +741,7 @@ scan_signals() { } run_bounded() { # [args...] - local seconds=$1 - shift - if command -v timeout >/dev/null 2>&1; then - timeout --kill-after=1 "$seconds" "$@" - elif command -v gtimeout >/dev/null 2>&1; then - gtimeout --kill-after=1 "$seconds" "$@" - else - # shellcheck disable=SC2016 # single quotes are deliberate: Perl expands its own variables. - perl -e 'my $t = shift; my $pid = fork; die "fork failed" unless defined $pid; if (!$pid) { setpgrp(0, 0); exec @ARGV } local $SIG{ALRM} = sub { kill "TERM", -$pid; select undef, undef, undef, 0.2; kill "KILL", -$pid; exit 124 }; alarm $t; waitpid $pid, 0; exit($? >> 8)' "$seconds" "$@" - fi + fm_run_bounded "$@" } run_check() { diff --git a/tests/fm-azure-runner.test.sh b/tests/fm-azure-runner.test.sh index 25851d970e0..1ded4f7e85e 100755 --- a/tests/fm-azure-runner.test.sh +++ b/tests/fm-azure-runner.test.sh @@ -1076,6 +1076,14 @@ exit 0 SH cat >"$root/tests/run.sh" <"$fakebin/uv" <<'SH' #!/bin/sh +if [ -n "${FM_TEST_CONCURRENCY_DIR:-}" ] && [ "${5:-}" = pytest ]; then + touch "$FM_TEST_CONCURRENCY_DIR/agent-fleet.started" + i=0 + while [ ! -e "$FM_TEST_CONCURRENCY_DIR/herdr.started" ]; do + i=$((i + 1)); [ "$i" -lt 100 ] || exit 95 + sleep 0.05 + done +fi exit 0 SH chmod +x "$fakebin/tmux" "$fakebin/uv" @@ -1504,13 +1520,18 @@ SH # capability-derived host set and records why the class ran locally. write_test_routing '{"classes":{"lint":"validation-standard"}}' rm -f "$fixture/captured" "$fixture/local-runs" + concurrency_dir="$tmp/local-concurrency" + mkdir -p "$concurrency_dir" out=$(cd "$gatewt" && env HOME="$anchor" PATH="$fakebin:$PATH" \ + FM_TEST_CONCURRENCY_DIR="$concurrency_dir" \ "$fixture/bin/fm-no-mistakes-test-command.sh" 2>&1) \ || fail "an unselected per-run test class did not preserve local host execution" [ ! -e "$fixture/captured" ] || fail "an unselected per-run test class reached the runner" assert_capability_derived_local_host_set "$fixture/local-runs" assert_contains "$out" "executed LOCALLY (routing=present-not-selected, env=absent)" \ "an unselected per-run test class emitted no local-execution proof" + [ -e "$concurrency_dir/herdr.started" ] && [ -e "$concurrency_dir/agent-fleet.started" ] \ + || fail "the local Herdr and Agent Fleet lanes did not overlap" [ "$(routing_dispatch_count)" -eq 0 ] \ || fail "an unselected per-run test class spent a dispatch budget slot" diff --git a/tests/fm-watcher-lock.test.sh b/tests/fm-watcher-lock.test.sh index a1fee474bb8..7783c8b982e 100755 --- a/tests/fm-watcher-lock.test.sh +++ b/tests/fm-watcher-lock.test.sh @@ -14,6 +14,8 @@ WATCH="$ROOT/bin/fm-watch.sh" WATCH_ARM="$ROOT/bin/fm-watch-arm.sh" DRAIN="$ROOT/bin/fm-wake-drain.sh" LIB="$ROOT/bin/fm-wake-lib.sh" +# shellcheck source=bin/fm-supervise-daemon.sh +. "$ROOT/bin/fm-supervise-daemon.sh" fm_test_tmproot_into TMP_ROOT fm-watcher-lock-tests @@ -961,6 +963,34 @@ test_pid_identity_is_locale_invariant() { pass "fm_pid_identity is locale-invariant across LC_ALL/LC_TIME" } +test_watcher_bounded_command_reaped_on_owner_shutdown() { + local dir marker owner child i + dir=$(make_case bounded-owner-shutdown) + marker="$dir/child.pid" + FM_STATE_OVERRIDE="$dir/state" bash -c ' + . "$1" + run_bounded 30 bash -c '\''echo "$$" > "$1"; sleep 30'\'' _ "$2" + ' _ "$WATCH" "$marker" & + owner=$! + i=0 + while [ "$i" -lt 50 ] && [ ! -s "$marker" ]; do + sleep 0.1 + i=$((i + 1)) + done + [ -s "$marker" ] || { kill "$owner" 2>/dev/null || true; wait "$owner" 2>/dev/null || true; fail "bounded watcher command never started"; } + child=$(cat "$marker") + fm_super_stop_watcher "$owner" + wait "$owner" 2>/dev/null || true + i=0 + while [ "$i" -lt 50 ] && is_live_non_zombie "$child"; do + sleep 0.1 + i=$((i + 1)) + done + ! is_live_non_zombie "$child" \ + || { kill -KILL "$child" 2>/dev/null || true; fail "watcher shutdown orphaned bounded child $child"; } + pass "watcher shutdown terminates and reaps its bounded command tree" +} + if [ "${FM_TEST_FOCUSED:-}" = self-evict ]; then test_watcher_self_evicts_on_lock_takeover exit 0 @@ -971,8 +1001,14 @@ if [ "${FM_TEST_FOCUSED:-}" = stale-steal-chain ]; then exit 0 fi +if [ "${FM_TEST_FOCUSED:-}" = bounded-owner-shutdown ]; then + test_watcher_bounded_command_reaped_on_owner_shutdown + exit 0 +fi + test_singleton_start test_pid_identity_is_locale_invariant +test_watcher_bounded_command_reaped_on_owner_shutdown test_stale_watch_lock_reclaimed test_live_stale_watch_lock_is_actionable test_guard_warnings From 0daace4b88da28c10813b4216937aea868b86a44 Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Tue, 25 Aug 2026 00:38:56 -0400 Subject: [PATCH 06/13] no-mistakes(document): Correct Azure capability documentation counts --- bin/fm-azure-validation-shard-bridge.py | 2 +- bin/fm-supervise-daemon.sh | 15 ++++++++++++++- docs/azure-requirements.md | 22 +++++++--------------- docs/azure-validation.md | 2 +- docs/test-isolation.md | 2 +- tests/fm-watcher-lock.test.sh | 7 ++++++- 6 files changed, 30 insertions(+), 20 deletions(-) diff --git a/bin/fm-azure-validation-shard-bridge.py b/bin/fm-azure-validation-shard-bridge.py index 44178a2e600..0e2004b61f5 100755 --- a/bin/fm-azure-validation-shard-bridge.py +++ b/bin/fm-azure-validation-shard-bridge.py @@ -48,7 +48,7 @@ # that executes these shards sets PrivateNetwork=yes, # RestrictAddressFamilies=AF_UNIX and IPAddressDeny=any, so bin/fm-teardown.sh's # secondmate upstream-authority probe can never resolve or reach the origin -# remote's host. That skips THIRTY-THREE units, the whole secondmate +# remote's host. That skips THIRTY-SEVEN units, the whole secondmate # teardown/retirement family in tests/fm-teardown-suite.sh. The set was # enumerated to convergence - both teardown files run to completion with the # network off, 143 of 143 cases - and those units are SKIPPED in the cell, not diff --git a/bin/fm-supervise-daemon.sh b/bin/fm-supervise-daemon.sh index f0b35f84776..7f9df7db028 100755 --- a/bin/fm-supervise-daemon.sh +++ b/bin/fm-supervise-daemon.sh @@ -1315,13 +1315,26 @@ trim_log() { # ============================================================================ fm_super_stop_watcher() { # - local watcher_pid=$1 child current_parent + local watcher_pid=$1 child current_parent i children="" while read -r child; do [ -n "$child" ] || continue current_parent=$(ps -o ppid= -p "$child" 2>/dev/null | tr -d '[:space:]') [ "$current_parent" = "$watcher_pid" ] || continue + children="${children}${children:+ }$child" kill -TERM "$child" 2>/dev/null || true done < <(ps -axo pid=,ppid= | awk -v parent="$watcher_pid" '$2 == parent { print $1 }') + # A bounded-command owner traps TERM so it can terminate and reap its own + # process group. Keep its watcher parent alive long enough to wait for that + # owner; killing both at once reparents the cleanup owner to launchd/init. + for child in $children; do + i=0 + while [ "$i" -lt 50 ]; do + current_parent=$(ps -o ppid= -p "$child" 2>/dev/null | tr -d '[:space:]') + [ "$current_parent" = "$watcher_pid" ] || break + sleep 0.1 + i=$((i + 1)) + done + done kill -TERM "$watcher_pid" 2>/dev/null || true } diff --git a/docs/azure-requirements.md b/docs/azure-requirements.md index 314938bb3e3..5f3670434cb 100644 --- a/docs/azure-requirements.md +++ b/docs/azure-requirements.md @@ -250,19 +250,11 @@ neither was the receipts strand: approval markers), alongside 377 passing units. Until those units skipped loudly off macOS, no intent could reach a green test step there. - Partially closed. The retained shard responses under - `$FM_HOME/state/azure-validation/shards/azv-36b2726cbcf3/*/response/` are the measurement, and - they name eleven failing test files, not three. Three classes are genuine host capabilities the - cell does not have, and those are now gated: a real tmux server it can create windows in - (`server exited unexpectedly` on the shard-2 and shard-4 workers), passwordless sudo with - `systemd-run` (`Linux systemd integration requires passwordless sudo`), and the `/usr/bin/cpp` - binding `bin/fm-account-directory.sh` needs before it can validate any Claude quota-axi - Keychain approval marker (`system openat binding unavailable`). Fifteen units across six test - files are bound to those three capabilities in `tests/host-capabilities.tsv`; the cell declares - the three absences by name in `bin/fm-azure-validation-shard-bridge.py`, and - `tests/host-capability-gate.sh` turns each into a loud `FM_HOST_CAPABILITY_SKIP`. The gate - refuses that declaration on Darwin, so macOS coverage is unchanged and cannot be switched off, - and CI declares nothing, so its coverage is unchanged too. + Partially closed. + The retained shard responses under `$FM_HOME/state/azure-validation/shards/azv-36b2726cbcf3/*/response/` are the measurement, and they name eleven failing test files, not three. + Four classes are genuine host capabilities the cell does not have, and those are now gated: a real tmux server it can create windows in (`server exited unexpectedly` on the shard-2 and shard-4 workers), passwordless sudo with `systemd-run` (`Linux systemd integration requires passwordless sudo`), the `/usr/bin/cpp` binding `bin/fm-account-directory.sh` needs before it can validate any Claude quota-axi Keychain approval marker (`system openat binding unavailable`), and outbound reach to the origin remote's host (`origin-egress`). + Fifty-two units across seven test files are bound to those four capabilities in `tests/host-capabilities.tsv`; the cell declares the four absences by name in `bin/fm-azure-validation-shard-bridge.py`, and `tests/host-capability-gate.sh` turns each into a loud `FM_HOST_CAPABILITY_SKIP`. + The gate refuses that declaration on Darwin, so macOS coverage is unchanged and cannot be switched off, and CI declares nothing, so its coverage is unchanged too. The other five failing files have now been MEASURED rather than inferred, by running the whole sealed suite in a local reproduction of the cell's own package closure (Ubuntu 24.04 @@ -297,11 +289,11 @@ neither was the receipts strand: iteration: the suite invokes every case through a single choke point, `run_partitioned_test`, so running each case in a subshell there reports every failure in one run instead of stopping at the first. Both files were run to completion with the network off - 143 of 143 cases - and - the result is exactly 33 units, all in the secondmate teardown/retirement family. An earlier + the result is exactly 37 units, all in the secondmate teardown/retirement family. An earlier one-at-a-time iteration had found only 19 and had not converged; the difference is why the partial set was not shipped. - BE CLEAR ABOUT WHAT THIS COSTS. Those 33 units are SKIPPED in the cell, not preserved by some + BE CLEAR ABOUT WHAT THIS COSTS. Those 37 units are SKIPPED in the cell, not preserved by some other route. The cell does not verify secondmate teardown or retirement authority at all: not the landed-work refusals, not the registry locking, not the network-authority pinning, not the child quiescence ordering. macOS and CI still run every one of them, and CI is where that diff --git a/docs/azure-validation.md b/docs/azure-validation.md index c95cbdffa1f..4aec21ba815 100644 --- a/docs/azure-validation.md +++ b/docs/azure-validation.md @@ -261,7 +261,7 @@ Each request binds the cell, round, shard index/count, branch, current head, tre The fixed behavior command is the existing sealed `bin/fm-behavior-shards.sh --run 8` route with the explicit Azure/Linux non-Herdr selection already used by isolated CI. It also carries the cell's own host-capability declaration, `FM_TEST_HOST_CAPABILITIES_ABSENT`, naming by name the four sealed-suite capabilities a shard worker cannot provide: a real tmux server it can create windows in, passwordless sudo with `systemd-run`, the `/usr/bin/cpp` binding `bin/fm-account-directory.sh` needs before it can validate a Claude quota-axi Keychain approval marker, and outbound reach to the origin remote's host (`origin-egress`). Be exact about what the fourth one costs, because it is much the largest and a skip must never read as coverage. -The runner unit that executes these shards sets `PrivateNetwork=yes`, `RestrictAddressFamilies=AF_UNIX` and `IPAddressDeny=any`, so `bin/fm-teardown.sh`'s secondmate upstream-authority probe can never resolve or reach the origin host, and `origin-egress` therefore skips THIRTY-THREE units - the whole secondmate teardown/retirement family in `tests/fm-teardown-suite.sh`. +The runner unit that executes these shards sets `PrivateNetwork=yes`, `RestrictAddressFamilies=AF_UNIX` and `IPAddressDeny=any`, so `bin/fm-teardown.sh`'s secondmate upstream-authority probe can never resolve or reach the origin host, and `origin-egress` therefore skips THIRTY-SEVEN units - the whole secondmate teardown/retirement family in `tests/fm-teardown-suite.sh`. The cell verifies none of that family: not the landed-work refusals, not the registry locking, not the network-authority pinning, not the child quiescence ordering. That coverage lives on macOS and in CI, which is where any change to `bin/fm-teardown.sh` must be proven. The set was enumerated to convergence rather than sampled: both teardown files run to completion with the network off, 143 of 143 cases, and `docs/azure-requirements.md` R4 owns the full account. diff --git a/docs/test-isolation.md b/docs/test-isolation.md index 14922339473..ab6c2677bd3 100644 --- a/docs/test-isolation.md +++ b/docs/test-isolation.md @@ -54,7 +54,7 @@ Herdr is not the only thing a sealed-suite host can lack. `tests/host-capabilities.tsv` declares every other host capability the suite depends on, the platforms that can provide it, and the exact units bound to it, and `tests/host-capability-gate.sh` is the only door to that set, through `fm_require_host_capability "