diff --git a/.no-mistakes.yaml b/.no-mistakes.yaml index 9e86ba057c6..e234565d4de 100644 --- a/.no-mistakes.yaml +++ b/.no-mistakes.yaml @@ -21,7 +21,8 @@ disable_project_settings: true # The ordinary local test command derives every herdr-lab and herdr-mixed file # from tests/test-capabilities.tsv, admits those files through tests/run.sh's # sealed safety and lab-ownership boundary, and runs them serially on the -# real-Herdr path before Agent Fleet's locked pytest and compileall checks. +# real-Herdr path concurrently with Agent Fleet's locked pytest and compileall +# checks. # It deliberately does not duplicate hermetic-only behavior files on the Mac. # The required Behavior tests CI job owns the complete behavior inventory: # eight isolated runners admit every selected path through tests/run.sh, run @@ -36,6 +37,8 @@ disable_project_settings: true # bypasses worktree scripts and delegates lint plus every behavior shard to the # root-owned bridge, so command children receive neither provider nor GitHub # credentials and never execute on the credentialed coordinator VM. +# tests/fm-azure-runner.test.sh executes this test command in an isolated fixture +# and proves that its two branches exclusively reach those respective owners. commands: lint: 'if [ "${FM_AZURE_VALIDATION_CELL:-0}" = 1 ]; then exec "$FM_AZURE_VALIDATION_SHARD_BRIDGE" lint -- bin/fm-azure-runner-command.sh bash -c ''bin/fm-lint.sh && uv run --directory tools/agent-fleet --locked ruff check .''; else exec bin/fm-azure-runner-dispatch.sh lint -- bin/fm-azure-runner-command.sh bash -c ''bin/fm-lint.sh && uv run --directory tools/agent-fleet --locked ruff check .''; fi' test: 'if [ "${FM_AZURE_VALIDATION_CELL:-0}" = 1 ]; then exec "$FM_AZURE_VALIDATION_SHARD_BRIDGE" behavior --count "${FM_AZURE_VALIDATION_SHARD_COUNT:-8}"; else exec bin/fm-no-mistakes-test-command.sh; fi' diff --git a/bin/fm-azure-runner-agent-fleet-install.py b/bin/fm-azure-runner-agent-fleet-install.py new file mode 100755 index 00000000000..405dac39160 --- /dev/null +++ b/bin/fm-azure-runner-agent-fleet-install.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""Install the exact locked Agent Fleet source into a prepared offline venv. + +The Azure runner has no package network while repository code executes and its +wheelhouse intentionally contains only locked registry dependencies. Agent +Fleet has no runtime dependencies, so this trusted installer copies the exact +snapshot package into the fresh venv, writes ordinary distribution metadata, +and creates the release-local console entrypoint without invoking an unsealed +PEP 517 build backend. +""" + +import base64 +import csv +import hashlib +import io +import os +from pathlib import Path +import re +import stat +import sys +import sysconfig + + +class InstallError(RuntimeError): + pass + + +def real_subdirectory(root, parts, label): + current = root + for part in parts: + current = current / part + try: + metadata = current.lstat() + except OSError as exc: + raise InstallError("{} is unavailable: {}".format(label, exc)) + if not stat.S_ISDIR(metadata.st_mode): + raise InstallError("{} must have real directory ancestry".format(label)) + return current + + +def regular_file(path, label): + try: + metadata = path.lstat() + except OSError as exc: + raise InstallError("{} is unavailable: {}".format(label, exc)) + if not stat.S_ISREG(metadata.st_mode): + raise InstallError("{} must be a regular non-link file".format(label)) + return path + + +def write_new(path, content, mode=0o644): + path.parent.mkdir(parents=True, exist_ok=True) + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + descriptor = os.open(str(path), flags, mode) + with os.fdopen(descriptor, "wb") as handle: + handle.write(content) + os.chmod(path, mode) + + +def record_digest(path): + digest = base64.urlsafe_b64encode(hashlib.sha256(path.read_bytes()).digest()).rstrip(b"=") + return "sha256=" + digest.decode("ascii") + + +def copy_package(source, destination): + if destination.exists() or destination.is_symlink(): + raise InstallError("Agent Fleet package destination already exists") + destination.mkdir(parents=True, mode=0o755) + copied = [] + for child in sorted(source.rglob("*")): + relative = child.relative_to(source) + metadata = child.lstat() + target = destination / relative + if stat.S_ISLNK(metadata.st_mode): + raise InstallError("Agent Fleet source contains a link: {}".format(relative)) + if stat.S_ISDIR(metadata.st_mode): + target.mkdir(mode=0o755) + continue + if not stat.S_ISREG(metadata.st_mode): + raise InstallError("Agent Fleet source contains a non-regular entry: {}".format(relative)) + target.parent.mkdir(parents=True, exist_ok=True) + write_new(target, child.read_bytes()) + copied.append(target) + if not copied: + raise InstallError("Agent Fleet source package is empty") + return copied + + +def install(project, venv): + if sys.version_info < (3, 11): + raise InstallError("Agent Fleet requires Python 3.11 or newer") + project = project.resolve() + venv = venv.resolve() + if Path(sys.prefix).resolve() != venv: + raise InstallError("installer must run with the exact target venv interpreter") + + pyproject_path = regular_file(project / "pyproject.toml", "Agent Fleet pyproject") + lock_path = regular_file(project / "uv.lock", "Agent Fleet lock") + source = real_subdirectory(project, ("src", "agent_fleet"), "Agent Fleet package source") + + try: + pyproject_text = pyproject_path.read_text(encoding="utf-8") + lock_text = lock_path.read_text(encoding="utf-8") + except OSError as exc: + raise InstallError("Agent Fleet project metadata is unreadable: {}".format(exc)) + + project_match = re.search(r"(?ms)^\[project\]\n(.*?)(?=^\[|\Z)", pyproject_text) + scripts_match = re.search(r"(?ms)^\[project\.scripts\]\n(.*?)(?=^\[|\Z)", pyproject_text) + if project_match is None or scripts_match is None: + raise InstallError("Agent Fleet project tables are absent") + project_table = project_match.group(1) + scripts_table = scripts_match.group(1) + name_match = re.search(r'^name = "([^"]+)"$', project_table, re.MULTILINE) + version_match = re.search(r'^version = "([^"]+)"$', project_table, re.MULTILINE) + dependencies_match = re.search(r"^dependencies = (.+)$", project_table, re.MULTILINE) + name = name_match.group(1) if name_match else None + version = version_match.group(1) if version_match else None + if name != "agent-fleet" or version is None or not re.fullmatch(r"[0-9]+(?:\.[0-9]+){2}", version): + raise InstallError("Agent Fleet project identity is not exact") + script_lines = [line.strip() for line in scripts_table.splitlines() if line.strip()] + if script_lines != ['agent-fleet = "agent_fleet.cli:main"']: + raise InstallError("Agent Fleet console entrypoint declaration is not exact") + if dependencies_match is None or dependencies_match.group(1).strip() != "[]": + raise InstallError("Agent Fleet gained runtime dependencies outside the sealed offline closure") + + locked = [] + for block in lock_text.split("[[package]]")[1:]: + locked_name = re.search(r'^name = "([^"]+)"$', block, re.MULTILINE) + if locked_name and locked_name.group(1) == name: + locked.append(block) + if len(locked) != 1: + raise InstallError("Agent Fleet lock does not contain one exact project record") + locked_block = locked[0] + if ( + not re.search(r'^version = "{}"$'.format(re.escape(version)), locked_block, re.MULTILINE) + or not re.search(r'^source = \{ editable = "\." \}$', locked_block, re.MULTILINE) + or re.search(r"^dependencies = \[", locked_block, re.MULTILINE) + ): + raise InstallError("Agent Fleet lock does not bind the exact editable project without runtime dependencies") + + purelib = Path(sysconfig.get_path("purelib")).resolve() + scripts_dir = Path(sysconfig.get_path("scripts")).resolve() + if venv not in purelib.parents or venv not in scripts_dir.parents: + raise InstallError("target interpreter paths escape the exact venv") + + package_destination = purelib / "agent_fleet" + dist_info = purelib / "agent_fleet-{}.dist-info".format(version) + entrypoint = scripts_dir / "agent-fleet" + if dist_info.exists() or dist_info.is_symlink() or entrypoint.exists() or entrypoint.is_symlink(): + raise InstallError("Agent Fleet project or console entrypoint is already installed") + + installed = copy_package(source, package_destination) + dist_info.mkdir(mode=0o755) + metadata = ( + "Metadata-Version: 2.3\n" + "Name: agent-fleet\n" + "Version: {}\n" + "Summary: Machine-global account profile routing for local agent CLIs\n" + "Requires-Python: >=3.11\n" + "\n" + ).format(version).encode("utf-8") + write_new(dist_info / "METADATA", metadata) + write_new(dist_info / "WHEEL", b"Wheel-Version: 1.0\nGenerator: fm-azure-runner\nRoot-Is-Purelib: true\nTag: py3-none-any\n") + write_new(dist_info / "entry_points.txt", b"[console_scripts]\nagent-fleet = agent_fleet.cli:main\n") + write_new(dist_info / "INSTALLER", b"fm-azure-runner\n") + + python_path = venv / "bin" / "python" + if not python_path.exists(): + raise InstallError("target venv has no Python entrypoint") + entrypoint_bytes = ( + "#!{}\n" + "import sys\n" + "from agent_fleet.cli import main\n" + "if __name__ == '__main__':\n" + " sys.exit(main())\n" + ).format(python_path).encode("utf-8") + write_new(entrypoint, entrypoint_bytes, mode=0o755) + + record_rows = [] + for path in sorted(installed + [ + dist_info / "METADATA", + dist_info / "WHEEL", + dist_info / "entry_points.txt", + dist_info / "INSTALLER", + entrypoint, + ]): + relative = path.relative_to(venv).as_posix() + record_rows.append((relative, record_digest(path), str(path.stat().st_size))) + record_path = dist_info / "RECORD" + record_rows.append((record_path.relative_to(venv).as_posix(), "", "")) + output = io.StringIO(newline="") + writer = csv.writer(output, lineterminator="\n") + writer.writerows(record_rows) + write_new(record_path, output.getvalue().encode("utf-8")) + return entrypoint + + +def main(): + if len(sys.argv) != 3: + print("usage: fm-azure-runner-agent-fleet-install.py ", file=sys.stderr) + return 2 + try: + entrypoint = install(Path(sys.argv[1]), Path(sys.argv[2])) + except InstallError as exc: + print("agent-fleet offline install failed: {}".format(exc), file=sys.stderr) + return 125 + print("agent-fleet offline install: {}".format(entrypoint)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bin/fm-azure-runner-dispatch.sh b/bin/fm-azure-runner-dispatch.sh index d1bc1da7bae..edae1723b13 100755 --- a/bin/fm-azure-runner-dispatch.sh +++ b/bin/fm-azure-runner-dispatch.sh @@ -443,13 +443,34 @@ if [ -z "$CONFIRM" ]; then CONFIRM=$FM_AZURE_SUBSCRIPTION_ID fi +SOURCE_ARGUMENTS=() +if [ "$ROUTING_STATE" = selected ]; then + # A per-run no-mistakes step executes from the gate's detached, pipeline-owned + # snapshot. Give that exact HEAD a deterministic private bundle ref derived + # from the already-proved run id instead of guessing or pushing a task branch. + # The direct bundle takes the ordinary standalone shared-capacity path; it is + # not a validation-cell child and carries no parent reservation. + [ -n "$ROUTING_RUN_ID" ] \ + || refuse "selected per-run routing has no exact no-mistakes run identity" + SOURCE_ARGUMENTS=( + --source-ref "refs/heads/fm-no-mistakes/$ROUTING_RUN_ID" + --private-snapshot-from-head + ) +fi + printf 'azure-runner: class=%s selected REMOTE resource-class=%s source=%s (dispatching)\n' \ "$COMMAND_CLASS" "$RESOURCE_CLASS" "$SELECTION_SOURCE" >&2 -exec "$SCRIPT_DIR/fm-azure-runner.sh" run \ - --confirm-run \ - --confirm-subscription "$CONFIRM" \ - --task "$TASK" \ - --generation "$GENERATION" \ - --resource-class "$RESOURCE_CLASS" \ - -- "$@" +RUNNER_ARGUMENTS=( + run + --confirm-run + --confirm-subscription "$CONFIRM" + --task "$TASK" + --generation "$GENERATION" + --resource-class "$RESOURCE_CLASS" +) +if [ "$ROUTING_STATE" = selected ]; then + RUNNER_ARGUMENTS+=("${SOURCE_ARGUMENTS[@]}") +fi +RUNNER_ARGUMENTS+=(-- "$@") +exec "$SCRIPT_DIR/fm-azure-runner.sh" "${RUNNER_ARGUMENTS[@]}" diff --git a/bin/fm-azure-runner-guest.sh b/bin/fm-azure-runner-guest.sh index fa98a1e5313..0a3eddad923 100755 --- a/bin/fm-azure-runner-guest.sh +++ b/bin/fm-azure-runner-guest.sh @@ -18,10 +18,11 @@ OUTPUT_BLOB=${output_blob:-} INPUT_BLOB=${input_blob:-} IDENTITY_CLIENT_ID=${identity_client_id:-} EXECUTOR_B64=${executor_b64:-} +AGENT_FLEET_INSTALLER_B64=${agent_fleet_installer_b64:-} unset request_b64 vm_resource_id vm_instance_id guest_digest storage_account -unset container output_blob input_blob identity_client_id executor_b64 -for bound in "$REQUEST_B64" "$VM_RESOURCE_ID" "$VM_INSTANCE_ID" "$GUEST_DIGEST" "$STORAGE_ACCOUNT" "$CONTAINER" "$OUTPUT_BLOB" "$INPUT_BLOB" "$IDENTITY_CLIENT_ID" "$EXECUTOR_B64"; do - [ -n "$bound" ] || { echo "guest bootstrap: expected ten bound parameters" >&2; exit 125; } +unset container output_blob input_blob identity_client_id executor_b64 agent_fleet_installer_b64 +for bound in "$REQUEST_B64" "$VM_RESOURCE_ID" "$VM_INSTANCE_ID" "$GUEST_DIGEST" "$STORAGE_ACCOUNT" "$CONTAINER" "$OUTPUT_BLOB" "$INPUT_BLOB" "$IDENTITY_CLIENT_ID" "$EXECUTOR_B64" "$AGENT_FLEET_INSTALLER_B64"; do + [ -n "$bound" ] || { echo "guest bootstrap: expected eleven bound parameters" >&2; exit 125; } done unset bound case "$GUEST_DIGEST" in sha256:[0-9a-f][0-9a-f]*) ;; *) echo "guest bootstrap: bad protocol digest" >&2; exit 125 ;; esac @@ -169,22 +170,26 @@ rm -rf "$BASE" install -d -m 0700 -o root -g root "$BASE" REQUEST=$BASE/request.json EXECUTOR=$BASE/runner-exec.py +AGENT_FLEET_INSTALLER=$BASE/agent-fleet-install.py printf '%s' "$REQUEST_B64" | base64 -d >"$REQUEST" printf '%s' "$EXECUTOR_B64" | base64 -d >"$EXECUTOR" -unset REQUEST_B64 EXECUTOR_B64 +printf '%s' "$AGENT_FLEET_INSTALLER_B64" | base64 -d >"$AGENT_FLEET_INSTALLER" +unset REQUEST_B64 EXECUTOR_B64 AGENT_FLEET_INSTALLER_B64 -python3 - "$REQUEST" "$EXECUTOR" "$GUEST_DIGEST" <<'PY' +python3 - "$REQUEST" "$EXECUTOR" "$AGENT_FLEET_INSTALLER" "$GUEST_DIGEST" <<'PY' import hashlib, json, pathlib, sys -request_path, executor_path = map(pathlib.Path, sys.argv[1:3]) +request_path, executor_path, installer_path = map(pathlib.Path, sys.argv[1:4]) request = json.loads(request_path.read_text(encoding="utf-8")) unsigned = dict(request); supplied = unsigned.pop("request_digest", None) canonical = json.dumps(unsigned, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode() if supplied != "sha256:" + hashlib.sha256(canonical).hexdigest(): raise SystemExit("guest bootstrap: request digest mismatch") if "sha256:" + hashlib.sha256(executor_path.read_bytes()).hexdigest() != request["protocol"]["executor_digest"]: raise SystemExit("guest bootstrap: executor digest mismatch") -if request["protocol"]["guest_digest"] != sys.argv[3]: raise SystemExit("guest bootstrap: guest digest mismatch") +if "sha256:" + hashlib.sha256(installer_path.read_bytes()).hexdigest() != request["protocol"]["agent_fleet_installer_digest"]: raise SystemExit("guest bootstrap: Agent Fleet installer digest mismatch") +if request["protocol"]["guest_digest"] != sys.argv[4]: raise SystemExit("guest bootstrap: guest digest mismatch") repo = request["repository"] -if repo.get("source_mode") not in ("public-github-https", "private-parent-bundle", "private-exact-bundle") or not repo.get("remote", "").startswith("https://github.com/"): raise SystemExit("guest bootstrap: source mode mismatch") -if repo.get("source_mode") in ("private-parent-bundle", "private-exact-bundle"): +private_modes = ("private-parent-bundle", "private-exact-bundle", "private-direct-bundle") +if repo.get("source_mode") not in ("public-github-https",) + private_modes or not repo.get("remote", "").startswith("https://github.com/"): raise SystemExit("guest bootstrap: source mode mismatch") +if repo.get("source_mode") in private_modes: if not repo.get("input_blob") or not repo.get("snapshot_digest") or not repo.get("snapshot_bytes"): raise SystemExit("guest bootstrap: private snapshot binding is incomplete") else: if repo.get("input_blob") is not None or repo.get("snapshot_bytes") != 0: raise SystemExit("guest bootstrap: public source carries private staging") @@ -234,7 +239,7 @@ runuser -u fmrunner -- git -C /work/repo remote add origin "$REMOTE" # repository as dubious (CVE-2022-24765); scope the exception through the # environment exactly as the validation cell guest does. export GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=safe.directory GIT_CONFIG_VALUE_0=/work/repo -if [ "$SOURCE_MODE" = private-parent-bundle ] || [ "$SOURCE_MODE" = private-exact-bundle ]; then +if [ "$SOURCE_MODE" = private-parent-bundle ] || [ "$SOURCE_MODE" = private-exact-bundle ] || [ "$SOURCE_MODE" = private-direct-bundle ]; then [ "$INPUT_BLOB" = "$(read_request repository.input_blob)" ] || { echo "guest bootstrap: private snapshot blob mismatch" >&2; exit 125; } SNAPSHOT=$BASE/snapshot.bundle TOKEN_FILE=$BASE/input-token @@ -260,11 +265,11 @@ if [ "$SOURCE_MODE" = private-parent-bundle ] || [ "$SOURCE_MODE" = private-exac rm -f /work/snapshot.bundle elif [ "$SOURCE_REF" != none ] && [ "$SOURCE_HEAD" = "$COMMIT" ]; then [ "$INPUT_BLOB" = none ] || { echo "guest bootstrap: public source received a private snapshot blob" >&2; exit 125; } - run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch --depth=1 origin "$SOURCE_REF" + run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch origin "$SOURCE_REF" [ "$(git -C /work/repo rev-parse FETCH_HEAD)" = "$COMMIT" ] || { echo "guest bootstrap: source ref moved after admission" >&2; exit 125; } else [ "$INPUT_BLOB" = none ] || { echo "guest bootstrap: public source received a private snapshot blob" >&2; exit 125; } - run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch --depth=1 origin "$COMMIT" + run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch origin "$COMMIT" fi runuser -u fmrunner -- git -C /work/repo checkout --detach "$COMMIT" >/dev/null python3 - "$REQUEST" <<'PY' >"$BASE/source-ancestors" @@ -274,12 +279,12 @@ PY while IFS= read -r ancestor; do [ -n "$ancestor" ] || continue if [ "$SOURCE_MODE" = public-github-https ]; then - run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch --depth=1 origin "$ancestor" + run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch origin "$ancestor" [ "$(git -C /work/repo rev-parse FETCH_HEAD)" = "$ancestor" ] || { echo "guest bootstrap: source ancestor identity mismatch" >&2; exit 125; } git -C /work/repo cat-file -e "$ancestor^{commit}" || { echo "guest bootstrap: source ancestor is absent" >&2; exit 125; } fi done <"$BASE/source-ancestors" -if [ "$SOURCE_MODE" = private-parent-bundle ] || [ "$SOURCE_MODE" = private-exact-bundle ]; then +if [ "$SOURCE_MODE" = private-parent-bundle ] || [ "$SOURCE_MODE" = private-exact-bundle ] || [ "$SOURCE_MODE" = private-direct-bundle ]; then /usr/bin/python3 "$EXECUTOR" --verify-private-source-ancestors "$REQUEST" /work/repo fi [ "$(git -C /work/repo rev-parse HEAD)" = "$COMMIT" ] && [ "$(git -C /work/repo rev-parse 'HEAD^{tree}')" = "$TREE" ] || { echo "guest bootstrap: source identity mismatch" >&2; exit 125; } @@ -296,13 +301,15 @@ if [ "$DEFAULT_REF" != none ] && [ "$DEFAULT_HEAD" != none ]; then case "$DEFAULT_REF" in refs/heads/?*) DEFAULT_NAME=${DEFAULT_REF#refs/heads/} ;; *) DEFAULT_NAME= ;; esac if [ -n "$DEFAULT_NAME" ]; then if ! git -C /work/repo cat-file -e "$DEFAULT_HEAD^{commit}" 2>/dev/null; then - run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch --depth=1 origin "$DEFAULT_HEAD" + run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch origin "$DEFAULT_HEAD" [ "$(git -C /work/repo rev-parse FETCH_HEAD)" = "$DEFAULT_HEAD" ] || { echo "guest bootstrap: default head identity mismatch" >&2; exit 125; } fi runuser -u fmrunner -- git -C /work/repo update-ref "refs/remotes/origin/$DEFAULT_NAME" "$DEFAULT_HEAD" runuser -u fmrunner -- git -C /work/repo symbolic-ref refs/remotes/origin/HEAD "refs/remotes/origin/$DEFAULT_NAME" fi fi +[ "$(git -C /work/repo rev-parse --is-shallow-repository)" = false ] \ + || { echo "guest bootstrap: sealed source graph is shallow" >&2; exit 125; } fetch_exact() { local url=$1 path=$2 bytes=$3 digest=$4 redirects=${5:-no} args=(); [ "$redirects" != yes ] || args+=(--location); run_bootstrap_network curl --fail --silent --show-error "${args[@]}" --connect-timeout 30 --max-time 300 --max-filesize "$bytes" --output "$path" "$url"; [ "$(stat -c %s "$path")" = "$bytes" ] && [ "sha256:$(sha256sum "$path" | awk '{print $1}')" = "$digest" ] || { echo "guest bootstrap: pinned download mismatch" >&2; exit 125; }; } # The golden image stages the pinned archives under /opt/fm-tools; a copy @@ -341,6 +348,9 @@ if [ "$LOCK_DIGEST" != None ]; then cd /work/repo runuser -u fmrunner -- /work/home/.fm-runner-tools/uv/uv venv --python /usr/bin/python3 /work/repo/tools/agent-fleet/.venv >/dev/null runuser -u fmrunner -- env UV_OFFLINE=1 UV_NO_INDEX=1 /work/home/.fm-runner-tools/uv/uv pip install --python /work/repo/tools/agent-fleet/.venv/bin/python --offline --no-index --find-links /work/home/.fm-runner-tools/wheelhouse pytest ruff >/dev/null + /work/repo/tools/agent-fleet/.venv/bin/python "$AGENT_FLEET_INSTALLER" /work/repo/tools/agent-fleet /work/repo/tools/agent-fleet/.venv >/dev/null + chown -R fmrunner:fmrunner /work/repo/tools/agent-fleet/.venv + runuser -u fmrunner -- /work/repo/tools/agent-fleet/.venv/bin/agent-fleet --help >/dev/null elif [ -s "$BASE/wheels.tsv" ]; then echo "guest bootstrap: unbound Python wheels" >&2 exit 125 diff --git a/bin/fm-azure-runner.py b/bin/fm-azure-runner.py index 3a1f148d342..7638d7224d5 100755 --- a/bin/fm-azure-runner.py +++ b/bin/fm-azure-runner.py @@ -38,6 +38,7 @@ TEMPLATE = ROOT / "docs" / "azure-runner" / "invocation.json" GUEST = ROOT / "bin" / "fm-azure-runner-guest.sh" EXECUTOR = ROOT / "bin" / "fm-azure-runner-exec.py" +AGENT_FLEET_INSTALLER = ROOT / "bin" / "fm-azure-runner-agent-fleet-install.py" WORKER_LIFECYCLE = ROOT / "bin" / "fm-worker-lifecycle.py" CONTAINER = "validation-shards" CONTROL_CONTAINER = "runner-control" @@ -752,6 +753,45 @@ def locked_python_manifest(repo): +def verify_self_contained_private_bundle(bundle, commit, source_ref): + heads = git(ROOT, "bundle", "list-heads", str(bundle)).stdout.splitlines() + expected_head = "{} {}".format(commit, source_ref) + if heads != [expected_head]: + raise RunnerError("private snapshot must contain only the exact source-ref head") + with tempfile.TemporaryDirectory(prefix="fm-azure-bundle-verify-") as temporary: + verification_repo = Path(temporary) / "repo.git" + run(["git", "init", "--bare", str(verification_repo)]) + run(["git", "-C", str(verification_repo), "bundle", "verify", str(bundle)]) + run([ + "git", "-C", str(verification_repo), "fetch", "--no-tags", str(bundle), + "+{}:refs/fm-azure-runner/verified".format(source_ref), + ]) + verified = git( + verification_repo, "rev-parse", "--verify", "refs/fm-azure-runner/verified" + ).stdout.strip() + shallow = git(verification_repo, "rev-parse", "--is-shallow-repository").stdout.strip() + if verified != commit or shallow != "false": + raise RunnerError("private snapshot source graph is incomplete or has the wrong head") + + +def create_private_snapshot_from_head(repo, destination, commit, source_ref): + shallow = git(repo, "rev-parse", "--is-shallow-repository").stdout.strip() + if shallow != "false": + raise RunnerError("direct private snapshot requires a complete non-shallow source graph") + with tempfile.TemporaryDirectory(prefix="fm-azure-bundle-stage-") as temporary: + staging_repo = Path(temporary) / "repo.git" + run(["git", "init", "--bare", str(staging_repo)]) + run([ + "git", "-C", str(staging_repo), "fetch", "--no-tags", "--force", str(repo), + "+{}:{}".format(commit, source_ref), + ]) + staged = git(staging_repo, "rev-parse", "--verify", source_ref).stdout.strip() + if staged != commit: + raise RunnerError("direct private snapshot staging changed the exact source head") + run(["git", "-C", str(staging_repo), "bundle", "create", str(destination), source_ref]) + verify_self_contained_private_bundle(destination, commit, source_ref) + + def tree_digest(repo, relative): path = repo / relative if not path.exists(): @@ -802,15 +842,18 @@ def prepare(env, args, parent_state=None): branch = git(repo, "symbolic-ref", "--quiet", "--short", "HEAD", check=False) if ( branch.returncode != 0 - and args.public_ref is None - and not args.private_snapshot_bundle + and getattr(args, "public_ref", None) is None + and getattr(args, "source_ref", None) is None ): raise RunnerError( - "repository must be on a named committed branch unless an exact public ref or private snapshot is supplied" + "repository must be on a named committed branch unless an exact source ref is supplied" ) commit = git(repo, "rev-parse", "HEAD").stdout.strip() remote = git(repo, "remote", "get-url", "origin").stdout.strip() private_snapshot_source = None + private_snapshot_from_head = bool(getattr(args, "private_snapshot_from_head", False)) + if args.private_snapshot_bundle and private_snapshot_from_head: + raise RunnerError("choose one private snapshot input: bundle or exact HEAD") if args.private_snapshot_bundle: private_snapshot_arg = Path(args.private_snapshot_bundle) private_snapshot_source = private_snapshot_arg.resolve() @@ -831,7 +874,16 @@ def prepare(env, args, parent_state=None): if getattr(args, "public_ref", None) and args.source_ref: raise RunnerError("choose one exact source identity: --source-ref or --public-ref") source_ancestors = tuple(getattr(args, "public_ancestor", None) or ()) - if private_snapshot_source is not None and not args.capacity_parent: + if private_snapshot_from_head: + if args.capacity_parent or not args.source_ref: + raise RunnerError("direct private HEAD snapshot requires one exact source ref and no capacity parent") + public = public_origin_proof( + repo, remote, commit, + source_ref=args.source_ref, + source_ancestors=source_ancestors, + private_source=True, + ) + elif private_snapshot_source is not None and not args.capacity_parent: public = private_bundle_origin_proof( repo, remote, @@ -846,6 +898,7 @@ def prepare(env, args, parent_state=None): source_ancestors=source_ancestors, private_source=private_snapshot_source is not None, ) + private_snapshot_requested = private_snapshot_source is not None or private_snapshot_from_head tree = public["tree"] task = require_identifier("task", args.task) @@ -875,12 +928,19 @@ def prepare(env, args, parent_state=None): private_snapshot_path = None private_snapshot_digest = None private_snapshot_bytes = 0 - if private_snapshot_source is not None: + if private_snapshot_requested: private_snapshot_path = payload_dir / "snapshot.bundle" - shutil.copyfile(str(private_snapshot_source), str(private_snapshot_path)) + if private_snapshot_from_head: + create_private_snapshot_from_head( + repo, private_snapshot_path, commit, args.source_ref + ) + else: + shutil.copyfile(str(private_snapshot_source), str(private_snapshot_path)) os.chmod(private_snapshot_path, 0o600) private_snapshot_digest = "sha256:" + sha256_file(private_snapshot_path) private_snapshot_bytes = private_snapshot_path.stat().st_size + if private_snapshot_bytes > MAX_STAGING_INPUT_BYTES: + raise RunnerError("private snapshot exceeds the one-GiB staging bound") source_identity = { "remote": remote, "default_ref": public["default_ref"], @@ -958,7 +1018,11 @@ def prepare(env, args, parent_state=None): ( "private-parent-bundle" if args.capacity_parent - else "private-exact-bundle" + else ( + "private-direct-bundle" + if private_snapshot_from_head + else "private-exact-bundle" + ) ) if private_snapshot_path else "public-github-https" @@ -988,6 +1052,7 @@ def prepare(env, args, parent_state=None): "shellcheck_archive_digest": "sha256:8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198", "uv_archive_digest": "sha256:440c4215b171e64061d65d16a23753dd25c29a7f7b1b0446c9e9aed0fa372f27", "agent_fleet_python": locked_python, + "agent_fleet_installer_digest": "sha256:" + sha256_file(AGENT_FLEET_INSTALLER), }, "created_at": iso_utc(prepared_at), "compute_deallocation_deadline": iso_utc(expires_at), @@ -1069,7 +1134,9 @@ def reprove_public_request(state): repository = state["request"]["repository"] repo = Path(state["repository_root"]).resolve() source_mode = repository.get("source_mode") - private_source = source_mode in ("private-parent-bundle", "private-exact-bundle") + private_source = source_mode in ( + "private-parent-bundle", "private-exact-bundle", "private-direct-bundle", + ) expected = { "remote": repository["remote"], "default_ref": repository["default_ref"], @@ -2645,9 +2712,13 @@ def create_run_command(env, state): current_guest_digest = "sha256:" + sha256_file(GUEST) if current_guest_digest != state["request"]["protocol"]["guest_digest"]: raise RunnerError("trusted guest protocol changed after request preparation") + current_installer_digest = "sha256:" + sha256_file(AGENT_FLEET_INSTALLER) + if current_installer_digest != state["request"]["protocol"]["agent_fleet_installer_digest"]: + raise RunnerError("trusted Agent Fleet installer changed after request preparation") script = GUEST.read_text(encoding="utf-8") request_b64 = base64.b64encode(Path(state["input_path"]).read_bytes()).decode("ascii") executor_b64 = base64.b64encode(EXECUTOR.read_bytes()).decode("ascii") + agent_fleet_installer_b64 = base64.b64encode(AGENT_FLEET_INSTALLER.read_bytes()).decode("ascii") properties = { "location": "eastus", "tags": ownership_tags(env, state), @@ -2664,6 +2735,7 @@ def create_run_command(env, state): {"name": "input_blob", "value": state["staging"].get("input_blob") or "none"}, {"name": "identity_client_id", "value": env["controller_identity_client_id"]}, {"name": "executor_b64", "value": executor_b64}, + {"name": "agent_fleet_installer_b64", "value": agent_fleet_installer_b64}, ], "asyncExecution": False, "timeoutInSeconds": state["request"]["limits"]["wall_seconds"] + 1200, @@ -3265,7 +3337,11 @@ def retry(env, old_state, args): private_source = repository.get("source_mode") in ( "private-parent-bundle", "private-exact-bundle", + "private-direct-bundle", ) + private_parent_source = repository.get("source_mode") == "private-parent-bundle" + private_exact_source = repository.get("source_mode") == "private-exact-bundle" + private_direct_source = repository.get("source_mode") == "private-direct-bundle" selected_source_ref = ( repository["source_ref"] if repository["source_ref"] != repository["default_ref"] @@ -3278,9 +3354,10 @@ def retry(env, old_state, args): args.source_ref = None args.public_ref = selected_source_ref args.public_ancestor = list(repository.get("source_ancestors", [])) + args.private_snapshot_from_head = private_direct_source args.private_snapshot_bundle = ( str(Path(old_state["input_path"]).parent / "snapshot.bundle") - if private_source + if private_parent_source or private_exact_source else None ) args.wall_seconds = old_state["request"]["limits"]["wall_seconds"] @@ -3353,6 +3430,11 @@ def add_request_arguments(parser, require_command=True): "--private-snapshot-bundle", help="exact parent-cell Git bundle staged privately for an unpushed validation head", ) + parser.add_argument( + "--private-snapshot-from-head", + action="store_true", + help="seal the exact clean non-shallow HEAD into a direct one-ref private bundle", + ) parser.add_argument("--dependency", action="append", default=[]) parser.add_argument("--artifact", action="append", default=[]) if require_command: diff --git a/bin/fm-azure-runner.sh b/bin/fm-azure-runner.sh index c6cc0809c0f..ae875343d3e 100755 --- a/bin/fm-azure-runner.sh +++ b/bin/fm-azure-runner.sh @@ -31,6 +31,7 @@ # fm-azure-runner.sh run --confirm-run --confirm-subscription \ # [--confirm-cost-admission-mode commissioning-bounded] \ # --task --generation --resource-class \ +# [--source-ref refs/heads/] [--private-snapshot-from-head] \ # [--wall-seconds N] [--dependency ]... \ # [--artifact ]... -- # fm-azure-runner.sh resume --invocation diff --git a/bin/fm-azure-validation-shard-bridge.py b/bin/fm-azure-validation-shard-bridge.py index 44178a2e600..0e2004b61f5 100755 --- a/bin/fm-azure-validation-shard-bridge.py +++ b/bin/fm-azure-validation-shard-bridge.py @@ -48,7 +48,7 @@ # that executes these shards sets PrivateNetwork=yes, # RestrictAddressFamilies=AF_UNIX and IPAddressDeny=any, so bin/fm-teardown.sh's # secondmate upstream-authority probe can never resolve or reach the origin -# remote's host. That skips THIRTY-THREE units, the whole secondmate +# remote's host. That skips THIRTY-SEVEN units, the whole secondmate # teardown/retirement family in tests/fm-teardown-suite.sh. The set was # enumerated to convergence - both teardown files run to completion with the # network off, 143 of 143 cases - and those units are SKIPPED in the cell, not diff --git a/bin/fm-no-mistakes-test-command.sh b/bin/fm-no-mistakes-test-command.sh index 06875d0c6bd..a923996d426 100755 --- a/bin/fm-no-mistakes-test-command.sh +++ b/bin/fm-no-mistakes-test-command.sh @@ -14,11 +14,27 @@ run_local_required() { tmux -V printf 'no-mistakes: local host set files=%s source=tests/test-capabilities.tsv; complete behavior inventory is required in CI\n' \ "${#herdr_tests[@]}" - local rc=0 - "$ROOT/tests/run.sh" "${herdr_tests[@]}" || rc=1 - uv run --directory "$ROOT/tools/agent-fleet" --locked pytest || rc=1 - uv run --directory "$ROOT/tools/agent-fleet" --locked python -m compileall -q src || rc=1 - return "$rc" + local lane_dir herdr_pid agent_fleet_pid herdr_rc=0 agent_fleet_rc=0 + lane_dir=$(mktemp -d "${TMPDIR:-/tmp}/fm-local-test-lanes.XXXXXX") || return 1 + "$ROOT/tests/run.sh" "${herdr_tests[@]}" >"$lane_dir/herdr.log" 2>&1 & + herdr_pid=$! + ( + rc=0 + uv run --directory "$ROOT/tools/agent-fleet" --locked pytest || rc=1 + uv run --directory "$ROOT/tools/agent-fleet" --locked python -m compileall -q src || rc=1 + exit "$rc" + ) >"$lane_dir/agent-fleet.log" 2>&1 & + agent_fleet_pid=$! + wait "$herdr_pid" || herdr_rc=$? + wait "$agent_fleet_pid" || agent_fleet_rc=$? + printf '%s\n' '== local Herdr lane ==' && cat "$lane_dir/herdr.log" + printf '%s\n' '== local Agent Fleet lane ==' && cat "$lane_dir/agent-fleet.log" + rm -rf "$lane_dir" + if [ "$herdr_rc" -ne 0 ] || [ "$agent_fleet_rc" -ne 0 ]; then + printf 'no-mistakes local test lanes failed: herdr=%s agent-fleet=%s\n' \ + "$herdr_rc" "$agent_fleet_rc" >&2 + return 1 + fi } # A daemon step inherits no FM_* selection variables from the operator. Ask the @@ -78,7 +94,9 @@ selection_binding=$(printf '%s\n' "$selection_output" | sed -n 's/^selection_bin # They run concurrently and report independently into this one command step. # shellcheck disable=SC2016 # The command expands its variables inside the Azure guest shell. "$DISPATCH" --require-selection-binding "$selection_binding" test -- \ - "$ROOT/bin/fm-azure-runner-command.sh" bash -c ' + "$ROOT/bin/fm-azure-runner-command.sh" env \ + FM_TEST_HOST_CAPABILITIES_ABSENT=real-tmux-server,passwordless-root-escalation,system-openat-binding,origin-egress \ + bash -c ' command -v tmux >/dev/null || { echo "tmux is required for e2e tests" >&2; exit 1; } tmux -V rc=0 diff --git a/bin/fm-supervise-daemon.sh b/bin/fm-supervise-daemon.sh index 2cf08fc1b05..7f9df7db028 100755 --- a/bin/fm-supervise-daemon.sh +++ b/bin/fm-supervise-daemon.sh @@ -1314,6 +1314,30 @@ trim_log() { # classifiers above are sourceable for unit tests (tests/fm-daemon.test.sh). # ============================================================================ +fm_super_stop_watcher() { # + local watcher_pid=$1 child current_parent i children="" + while read -r child; do + [ -n "$child" ] || continue + current_parent=$(ps -o ppid= -p "$child" 2>/dev/null | tr -d '[:space:]') + [ "$current_parent" = "$watcher_pid" ] || continue + children="${children}${children:+ }$child" + kill -TERM "$child" 2>/dev/null || true + done < <(ps -axo pid=,ppid= | awk -v parent="$watcher_pid" '$2 == parent { print $1 }') + # A bounded-command owner traps TERM so it can terminate and reap its own + # process group. Keep its watcher parent alive long enough to wait for that + # owner; killing both at once reparents the cleanup owner to launchd/init. + for child in $children; do + i=0 + while [ "$i" -lt 50 ]; do + current_parent=$(ps -o ppid= -p "$child" 2>/dev/null | tr -d '[:space:]') + [ "$current_parent" = "$watcher_pid" ] || break + sleep 0.1 + i=$((i + 1)) + done + done + kill -TERM "$watcher_pid" 2>/dev/null || true +} + fm_super_main() { local STATE DELIVERY BACKEND TARGET backend_source target_source STATE="$(_state_root)" @@ -1457,7 +1481,7 @@ fm_super_main() { escalate_flush "$STATE" 2>/dev/null || true fi if [ -n "${WATCHER_PID:-}" ]; then - kill "$WATCHER_PID" 2>/dev/null || true + fm_super_stop_watcher "$WATCHER_PID" wait "$WATCHER_PID" 2>/dev/null || true fi if [ -n "${CUR_TMP:-}" ]; then diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 1f111990871..7f99b3f1f63 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -69,6 +69,8 @@ mkdir -p "$STATE" # shellcheck source=bin/fm-wake-lib.sh . "$SCRIPT_DIR/fm-wake-lib.sh" +# shellcheck source=bin/fm-process-tree-lib.sh +. "$SCRIPT_DIR/fm-process-tree-lib.sh" # Shared wake classifier (captain-relevant verbs + signal/stale/heartbeat # predicates), the SAME library the away-mode daemon uses, so the triage policy # has one definition. @@ -739,16 +741,7 @@ scan_signals() { } run_bounded() { # [args...] - local seconds=$1 - shift - if command -v timeout >/dev/null 2>&1; then - timeout --kill-after=1 "$seconds" "$@" - elif command -v gtimeout >/dev/null 2>&1; then - gtimeout --kill-after=1 "$seconds" "$@" - else - # shellcheck disable=SC2016 # single quotes are deliberate: Perl expands its own variables. - perl -e 'my $t = shift; my $pid = fork; die "fork failed" unless defined $pid; if (!$pid) { setpgrp(0, 0); exec @ARGV } local $SIG{ALRM} = sub { kill "TERM", -$pid; select undef, undef, undef, 0.2; kill "KILL", -$pid; exit 124 }; alarm $t; waitpid $pid, 0; exit($? >> 8)' "$seconds" "$@" - fi + fm_run_bounded "$@" } run_check() { diff --git a/docs/azure-requirements.md b/docs/azure-requirements.md index 314938bb3e3..5f3670434cb 100644 --- a/docs/azure-requirements.md +++ b/docs/azure-requirements.md @@ -250,19 +250,11 @@ neither was the receipts strand: approval markers), alongside 377 passing units. Until those units skipped loudly off macOS, no intent could reach a green test step there. - Partially closed. The retained shard responses under - `$FM_HOME/state/azure-validation/shards/azv-36b2726cbcf3/*/response/` are the measurement, and - they name eleven failing test files, not three. Three classes are genuine host capabilities the - cell does not have, and those are now gated: a real tmux server it can create windows in - (`server exited unexpectedly` on the shard-2 and shard-4 workers), passwordless sudo with - `systemd-run` (`Linux systemd integration requires passwordless sudo`), and the `/usr/bin/cpp` - binding `bin/fm-account-directory.sh` needs before it can validate any Claude quota-axi - Keychain approval marker (`system openat binding unavailable`). Fifteen units across six test - files are bound to those three capabilities in `tests/host-capabilities.tsv`; the cell declares - the three absences by name in `bin/fm-azure-validation-shard-bridge.py`, and - `tests/host-capability-gate.sh` turns each into a loud `FM_HOST_CAPABILITY_SKIP`. The gate - refuses that declaration on Darwin, so macOS coverage is unchanged and cannot be switched off, - and CI declares nothing, so its coverage is unchanged too. + Partially closed. + The retained shard responses under `$FM_HOME/state/azure-validation/shards/azv-36b2726cbcf3/*/response/` are the measurement, and they name eleven failing test files, not three. + Four classes are genuine host capabilities the cell does not have, and those are now gated: a real tmux server it can create windows in (`server exited unexpectedly` on the shard-2 and shard-4 workers), passwordless sudo with `systemd-run` (`Linux systemd integration requires passwordless sudo`), the `/usr/bin/cpp` binding `bin/fm-account-directory.sh` needs before it can validate any Claude quota-axi Keychain approval marker (`system openat binding unavailable`), and outbound reach to the origin remote's host (`origin-egress`). + Fifty-two units across seven test files are bound to those four capabilities in `tests/host-capabilities.tsv`; the cell declares the four absences by name in `bin/fm-azure-validation-shard-bridge.py`, and `tests/host-capability-gate.sh` turns each into a loud `FM_HOST_CAPABILITY_SKIP`. + The gate refuses that declaration on Darwin, so macOS coverage is unchanged and cannot be switched off, and CI declares nothing, so its coverage is unchanged too. The other five failing files have now been MEASURED rather than inferred, by running the whole sealed suite in a local reproduction of the cell's own package closure (Ubuntu 24.04 @@ -297,11 +289,11 @@ neither was the receipts strand: iteration: the suite invokes every case through a single choke point, `run_partitioned_test`, so running each case in a subshell there reports every failure in one run instead of stopping at the first. Both files were run to completion with the network off - 143 of 143 cases - and - the result is exactly 33 units, all in the secondmate teardown/retirement family. An earlier + the result is exactly 37 units, all in the secondmate teardown/retirement family. An earlier one-at-a-time iteration had found only 19 and had not converged; the difference is why the partial set was not shipped. - BE CLEAR ABOUT WHAT THIS COSTS. Those 33 units are SKIPPED in the cell, not preserved by some + BE CLEAR ABOUT WHAT THIS COSTS. Those 37 units are SKIPPED in the cell, not preserved by some other route. The cell does not verify secondmate teardown or retirement authority at all: not the landed-work refusals, not the registry locking, not the network-authority pinning, not the child quiescence ordering. macOS and CI still run every one of them, and CI is where that diff --git a/docs/azure-runner.md b/docs/azure-runner.md index 8aa944cee6f..d7d2da73956 100644 --- a/docs/azure-runner.md +++ b/docs/azure-runner.md @@ -31,13 +31,15 @@ A present file that is unreadable, partial, malformed, expired, exhausted, wrong Once either selector validly chooses remote execution, the command never falls back to the Mac after any cloud, identity, quota, staging, execution, or integrity failure. `FM_AZURE_RUNNER_LOCAL_RECOVERY_CLASSES` is the only explicit local recovery selection. -No billable resource was created while implementing this code. -The first live invocation remains blocked until the foundation and this code are reviewed, landed, explicitly applied, and approved for the exact subscription. +C2 no-mistakes run `01M0T2PJB0EZX8AA94WQV0WXA9` exposed the remaining production blocker before VM creation because its gate worktree was detached. +Direct diagnostic invocations `azr-0a4691b5bbae` and `azr-949b35e79a57` both returned command exit 1 and safely cleaned their invocation compute to zero. +Those attempts are failed evidence, not acceptance. +A new billable acceptance remains blocked until this exact repair is on public `main` and the operator explicitly approves the exact subscription and run. ## Request and snapshot contract `prepare` refuses tracked changes, untracked files, and any origin identity other than a credential-free GitHub HTTPS URL. -It also refuses a detached HEAD unless the caller supplies an exact public source ref or a digest-bound private bundle for that detached commit. +It also refuses a detached HEAD unless the caller supplies an exact source ref whose public proof or sealed private bundle head equals that detached commit. By default the candidate must be reachable from a freshly advertised and fetched `refs/heads/main` default head. The explicit `--source-ref refs/heads/` seam alone requires the candidate commit to be the exact freshly advertised and fetched head of that public branch; it never accepts an ancestor, stale tracking ref, tag, or changed remote head. An explicit `--public-ref` may instead name only an advertised branch head or `refs/pull//head`, and the candidate must equal that ref's exact fetched head; mutable pull merge refs and unsafe ref shapes are refused, and the two source-ref seams are mutually exclusive. @@ -45,8 +47,11 @@ A caller may additionally bind one or more exact `--public-ancestor` commits; pu Crosscheck evidence for a private GitHub repository supplies `--private-snapshot-bundle` without a parent reservation: the trusted host packages its clean exact-head review checkout, binds the authenticated PR ref and base ancestor, and stages only that digest-bound Git bundle so the evidence VM receives no GitHub credential. When that arbitrary repository does not contain Firstmate's Agent Fleet lock, the `crosscheck-tool` class uses only the sealed base toolchain and records an empty Python-wheel closure instead of requiring Firstmate-specific files. An Azure validation cell additionally supplies `--private-snapshot-bundle` with its parent-cell reservation so an unpushed pipeline-fix head can execute without prematurely changing the task branch on GitHub. -Both private modes bind one exact source ref/head, a one-ref Git bundle, digest, size, and private staging object; the validation-cell mode additionally binds its parent cell and freshly proves the public origin's trusted default ref/head. -The public proof runs in a fresh bare repository with system/global Git configuration, credentials, prompts, extra HTTP headers, and file transport disabled; private Crosscheck proof revalidates the clean checkout and bundle identities locally immediately before compute creation and retry. +The per-run no-mistakes route instead supplies `--private-snapshot-from-head` with the deterministic private ref `refs/heads/fm-no-mistakes/`. +That direct mode seals the exact clean detached gate HEAD and its complete non-shallow ancestry into one self-contained bundle without guessing or pushing a task branch and without claiming a validation-cell parent. +All three private modes bind one exact source ref/head, a one-ref Git bundle, digest, size, and private staging object. +The parent mode additionally binds its exact cell and reservation. +The public proof runs in a fresh bare repository with system/global Git configuration, credentials, prompts, extra HTTP headers, and file transport disabled; all modes repeat their exact public/private source proof immediately before compute creation and retry. No live worktree, primary home, provider account home, browser profile, or peer storage is mounted or copied. The canonical `fm.azure-command/v1` request binds these fields: @@ -72,7 +77,8 @@ The fixed root bootstrap installs a hard-coded Ubuntu transport and Linux test-t Before each package operation it waits up to three minutes for the standard apt/dpkg locks, and apt carries the same bounded dpkg timeout, so normal image maintenance can finish without turning into a repository-command failure while a stuck lock still fails closed. Repository code cannot alter that privileged package list, all package and staging traffic is shaped to one megabit per second and ends before deny-all command networking starts, and invocation evidence must record the resolved package/image versions during real acceptance. The request records the exact-size, checksum-pinned ShellCheck 0.11.0 and uv 0.9.10 releases plus, when the snapshot contains the Agent Fleet `uv.lock`, the complete Linux x86_64 pytest/ruff wheel closure selected from that lock. -When that lock is present, trusted root verifies the lock, archive, file set, sizes, and hashes, creates the Agent Fleet environment with an empty cache and networking disabled, and forces repository `uv run --locked` commands to use that already-synchronized offline environment. +When that lock is present, trusted root verifies the lock, archive, file set, sizes, and hashes, creates the Agent Fleet environment with an empty cache and networking disabled, and installs the exact locked Agent Fleet source plus its release-local `agent-fleet` console entrypoint before repository code starts. +The command wrapper then forces repository `uv run --locked` commands to use that synchronized offline environment through `UV_NO_SYNC=1`. ## Private control and VM boundary @@ -149,6 +155,7 @@ The validation-cell dispatcher may request up to eight mixed-family invocations A validation-owned invocation carries an exact `capacity-parent` cell id and complete parent vCPU reservation so the cell's pre-reserved processor shape and the child VM inventory cannot be double-counted or mistaken for unrelated capacity. Before admission and again before VM creation, the runner proves one live parent cell with the exact owner, deployment generation, home, lifecycle, id, and processor-reservation tags, then refuses a child beyond the reserved `(vCPUs - 8) / 4` slots. Each child still receives its own durable first-day reservation for its direct compute, storage, network, monitoring, and control meters; only the already-accounted $210 shared foundation reserve is omitted from that child reservation. +A direct private no-mistakes bundle has no validation parent and therefore takes the ordinary standalone shared-capacity reservation and complete foundation cost bound exactly once. Immediately before reservation and again immediately before VM creation, the controller proves the exact subscription/resource-group IDs and owner/generation tags for the named foundation storage account, zero-data admission-control account/container and ETag, controller UAMI and its sole exact effective container role including inherited/group expansion, VNet and address space, validation and private-endpoint subnets, complete NSG rule set, NAT and bound Standard public IP, blob private endpoint and endpoint NIC, named approved blob connection, private-DNS zone, VNet link, zone group/config names, and private-access properties. It also proves current SKU capabilities and restrictions, current East US regional and selected-family free vCPU quota, month-to-date actual cost, forecast cost, the exact unambiguous Linux on-demand Consumption retail meter (never Spot, Low Priority, Windows, dev/test, reservation, or savings pricing), and active runner count. @@ -348,10 +355,15 @@ local fallback, and `FM_AZURE_RUNNER_LOCAL_RECOVERY_CLASSES` remains the only explicit local opt-out. The lint payload preserves the tracked shell owner and locked Agent Fleet command unchanged inside the dispatched argv. -For a validation-owned feature branch, the caller passes its exact current `refs/heads/` identity plus the one-ref private snapshot bundle. -The runner binds and privately stages that unpushed commit, while a changed local bundle/head/tree or public default base refuses before compute creation. +For the per-run no-mistakes route, the dispatcher binds the detached gate HEAD to `refs/heads/fm-no-mistakes/` and asks the runner to build the complete one-ref private bundle inside its protected payload directory. +That source ref is a deterministic run identity, not a guess at the author's branch. +The direct route uses ordinary shared-capacity accounting and carries no validation-cell parent. +For a validation-owned feature branch, the caller passes its exact current `refs/heads/` identity plus the one-ref private snapshot bundle and parent reservation. +The runner binds and privately stages either unpushed commit, while a changed local bundle/head/tree or public default base refuses before compute creation. The ordinary test path runs the capability-derived real-Herdr host set locally and leaves complete behavior-inventory verification to required CI, as defined in [`configuration.md`](configuration.md#gate-defaults-no-mistakesyaml). When the `test` class is explicitly remote, `bin/fm-no-mistakes-test-command.sh` runs the sealed non-Herdr behavior inventory plus locked Agent Fleet checks on one Azure VM while every real-Herdr declaration runs through owned guarded labs on the Mac; a failed Azure shard is never replayed locally. +The remote command declares exactly `real-tmux-server,passwordless-root-escalation,system-openat-binding,origin-egress` absent and emits the existing loud per-unit skips rather than adding failures to the skip inventory. +The guest materializes complete source and default-branch history, so admitted non-Herdr fixtures retain their parent and historical-path graph instead of operating from a shallow root. Model review, document generation that requires a model, fixes, Git mutation, push, PR creation, CI monitoring, and gate decisions remain in no-mistakes' existing owner. A configured uncredentialed documentation command may use this runner like any other command, but this bridge never moves a model document step by implication. @@ -373,6 +385,8 @@ bin/fm-azure-runner.sh cost The implementation tests use fake Azure APIs and local executor fixtures only. They create no Azure resource and incur no charge. +Acceptance for the per-run no-mistakes repair remains blocked until this exact code lands on public `main` and one fresh routed `test=behavior-heavy` run passes on real Azure compute with exact remote execution proof and zero cleanup. +Do not spend on an unlanded build, and do not weaken the foundation's `require_landed_code` gate to accelerate that acceptance. Real usability remains unclaimed until the approved deployment performs this bounded acceptance: 1. Record Mac CPU, memory, swap, process count, and responsive interactive latency before launch. diff --git a/docs/azure-validation.md b/docs/azure-validation.md index cba71c97656..4aec21ba815 100644 --- a/docs/azure-validation.md +++ b/docs/azure-validation.md @@ -38,6 +38,7 @@ No operation stops, starts, updates, or inspects a local legacy no-mistakes daem Remote Herdr is not part of this path. Later Herdr proxy tabs may display state, but validation submission, ask-user responses, recovery, evidence collection, and cleanup use Azure Resource Manager, Managed Run Command, and the private storage endpoint. +The runner's separate per-run no-mistakes routing path may seal a detached gate HEAD as a direct private bundle with ordinary standalone capacity accounting; that path is not a validation cell or a child of one. No Azure resource was created while implementing this feature. Live Azure acceptance of these cells runs from released public main under separate explicit billable authorization; the first live Stage C pipeline acceptance has occurred and is recorded in the Live acceptance record section, while the full multi-leg checklist in the Live acceptance section remains unperformed. @@ -260,7 +261,7 @@ Each request binds the cell, round, shard index/count, branch, current head, tre The fixed behavior command is the existing sealed `bin/fm-behavior-shards.sh --run 8` route with the explicit Azure/Linux non-Herdr selection already used by isolated CI. It also carries the cell's own host-capability declaration, `FM_TEST_HOST_CAPABILITIES_ABSENT`, naming by name the four sealed-suite capabilities a shard worker cannot provide: a real tmux server it can create windows in, passwordless sudo with `systemd-run`, the `/usr/bin/cpp` binding `bin/fm-account-directory.sh` needs before it can validate a Claude quota-axi Keychain approval marker, and outbound reach to the origin remote's host (`origin-egress`). Be exact about what the fourth one costs, because it is much the largest and a skip must never read as coverage. -The runner unit that executes these shards sets `PrivateNetwork=yes`, `RestrictAddressFamilies=AF_UNIX` and `IPAddressDeny=any`, so `bin/fm-teardown.sh`'s secondmate upstream-authority probe can never resolve or reach the origin host, and `origin-egress` therefore skips THIRTY-THREE units - the whole secondmate teardown/retirement family in `tests/fm-teardown-suite.sh`. +The runner unit that executes these shards sets `PrivateNetwork=yes`, `RestrictAddressFamilies=AF_UNIX` and `IPAddressDeny=any`, so `bin/fm-teardown.sh`'s secondmate upstream-authority probe can never resolve or reach the origin host, and `origin-egress` therefore skips THIRTY-SEVEN units - the whole secondmate teardown/retirement family in `tests/fm-teardown-suite.sh`. The cell verifies none of that family: not the landed-work refusals, not the registry locking, not the network-authority pinning, not the child quiescence ordering. That coverage lives on macOS and in CI, which is where any change to `bin/fm-teardown.sh` must be proven. The set was enumerated to convergence rather than sampled: both teardown files run to completion with the network off, 143 of 143 cases, and `docs/azure-requirements.md` R4 owns the full account. @@ -272,6 +273,7 @@ It does not run the command locally. The dispatcher assigns each shard the exact SKU and pre-reserved constituent id from the admitted shape plan, avoiding the control cell's family, and sets the runner concurrency ceiling to eight. Each request uses the runner's exact private-parent snapshot mode, so a pipeline-owned fix commit can run before the no-mistakes push step without executing locally or prematurely mutating the remote task branch. The one-ref Git bundle, current source ref/head/tree, digest, size, and private input blob are all bound to the parent cell and command request, while the runner independently re-proves the public trusted default base. +This parent-bound shard mode remains distinct from the direct per-run no-mistakes bundle in `docs/azure-runner.md`, which must not claim or require validation-cell capacity. A new child starts through `runner run` with an explicit invocation, confirmation, source ref, private bundle, and parent-cell reservation; only an already recorded child uses `runner resume`, so a missing VM cannot turn a prepared record into duplicate execution. The one-shot runner still re-proves live family quota, regional quota, rate, budget, private network, image, command bounds, and global admission under its own contract. Every shard therefore receives a separate VM, OS disk, process namespace, port space, lock space, temp root, terminal-server space, boot id, and VM instance id. @@ -484,6 +486,7 @@ A missing ETag, changed instance, foreign tag, foreign principal, extra role ass Focused fake-cloud and static tests do not claim real Azure usability. The first live Stage C pipeline acceptance has occurred, as described in the Live acceptance record section, while the full multi-leg acceptance checklist below remains unperformed. +The separate direct per-run runner repair is not accepted by those cell results and remains blocked until its exact code lands on public `main` and a fresh routed no-mistakes run passes on real Azure compute with zero cleanup. After this stack is released to public main and the operator has explicit billable authorization, the full checklist acceptance must run from that released main and record all of these results: 1. Record Mac wall time, CPU, memory, swap, process count, and interactive latency before admission. diff --git a/docs/configuration.md b/docs/configuration.md index fc3d7b37f81..6da7b6867c9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -97,11 +97,12 @@ See [`wedge-alarm.md`](wedge-alarm.md) for the channel reference and macOS verif The tracked `.no-mistakes.yaml` keeps test evidence outside the repo and preserves `bin/fm-no-mistakes-test-command.sh` as the ordinary local test owner. Inside an admitted Azure validation cell, the trusted default-branch command string instead invokes the root-owned bridge from `docs/azure-validation.md`, which runs lint and requested behavior shards on separate credential-free Azure command VMs without exposing the cell's provider or GitHub lease. That evidence policy is specific to the firstmate repo: target projects may legitimately commit `.no-mistakes/evidence/` from their own no-mistakes pipeline, but firstmate keeps `.no-mistakes/` local and CI rejects tracked entries under that path. -The ordinary local command requires `tmux` on `PATH`, prints `tmux -V`, derives every `herdr-lab` and `herdr-mixed` path directly from [`tests/test-capabilities.tsv`](../tests/test-capabilities.tsv), and sends those files in one serial invocation through [`tests/run.sh`](../tests/run.sh) before running the locked Agent Fleet pytest and compileall checks. +The ordinary local command requires `tmux` on `PATH`, prints `tmux -V`, derives every `herdr-lab` and `herdr-mixed` path directly from [`tests/test-capabilities.tsv`](../tests/test-capabilities.tsv), and runs two concurrent lanes: one serial [`tests/run.sh`](../tests/run.sh) invocation for those files and one locked Agent Fleet pytest-and-compileall invocation. It does not maintain a second file list or duplicate hermetic-only behavior files on the Mac. The required `Behavior tests` job in [`.github/workflows/ci.yml`](../.github/workflows/ci.yml) uses the duration-balanced sharding owned by [`bin/fm-behavior-shards.sh`](../bin/fm-behavior-shards.sh) to run and verify the complete behavior inventory across eight isolated runners. CI selects the explicit non-Herdr path because its disposable image carries no Herdr, so it runs every hermetic body and the hermetic portion of the mixed file while the local host admits the complete `herdr-lab` and `herdr-mixed` set through owned labs. -The exact behavioral coverage contract is the union of that capability-derived local host set and the required CI executed-manifest union, not a claim that all 123 files run serially before push. +The exact behavioral coverage contract is the 123-file union of that capability-derived local host set and the required CI executed-manifest union across eight shards, not a claim that all 123 files run serially before push. +The tracked pytest proof adapter `tests/test_azure_proof_contracts.py::test_local_required_ci_coverage_contract` executes the public `tests/fm-azure-runner.test.sh` behavior suite, which fails if a hermetic-only registry entry is added to the production local selector or if `.no-mistakes.yaml` stops routing ordinary local tests to `bin/fm-no-mistakes-test-command.sh` and admitted validation-cell tests to the required shard bridge. Both routes cross the same sealed admission boundary, and the local serial route holds real-Herdr labs to one at a time. ## Crosscheck reviewer diff --git a/docs/test-isolation.md b/docs/test-isolation.md index 14922339473..ab6c2677bd3 100644 --- a/docs/test-isolation.md +++ b/docs/test-isolation.md @@ -54,7 +54,7 @@ Herdr is not the only thing a sealed-suite host can lack. `tests/host-capabilities.tsv` declares every other host capability the suite depends on, the platforms that can provide it, and the exact units bound to it, and `tests/host-capability-gate.sh` is the only door to that set, through `fm_require_host_capability "