diff --git a/.no-mistakes.yaml b/.no-mistakes.yaml index 9e86ba057c6..e234565d4de 100644 --- a/.no-mistakes.yaml +++ b/.no-mistakes.yaml @@ -21,7 +21,8 @@ disable_project_settings: true # The ordinary local test command derives every herdr-lab and herdr-mixed file # from tests/test-capabilities.tsv, admits those files through tests/run.sh's # sealed safety and lab-ownership boundary, and runs them serially on the -# real-Herdr path before Agent Fleet's locked pytest and compileall checks. +# real-Herdr path concurrently with Agent Fleet's locked pytest and compileall +# checks. # It deliberately does not duplicate hermetic-only behavior files on the Mac. # The required Behavior tests CI job owns the complete behavior inventory: # eight isolated runners admit every selected path through tests/run.sh, run @@ -36,6 +37,8 @@ disable_project_settings: true # bypasses worktree scripts and delegates lint plus every behavior shard to the # root-owned bridge, so command children receive neither provider nor GitHub # credentials and never execute on the credentialed coordinator VM. +# tests/fm-azure-runner.test.sh executes this test command in an isolated fixture +# and proves that its two branches exclusively reach those respective owners. commands: lint: 'if [ "${FM_AZURE_VALIDATION_CELL:-0}" = 1 ]; then exec "$FM_AZURE_VALIDATION_SHARD_BRIDGE" lint -- bin/fm-azure-runner-command.sh bash -c ''bin/fm-lint.sh && uv run --directory tools/agent-fleet --locked ruff check .''; else exec bin/fm-azure-runner-dispatch.sh lint -- bin/fm-azure-runner-command.sh bash -c ''bin/fm-lint.sh && uv run --directory tools/agent-fleet --locked ruff check .''; fi' test: 'if [ "${FM_AZURE_VALIDATION_CELL:-0}" = 1 ]; then exec "$FM_AZURE_VALIDATION_SHARD_BRIDGE" behavior --count "${FM_AZURE_VALIDATION_SHARD_COUNT:-8}"; else exec bin/fm-no-mistakes-test-command.sh; fi' diff --git a/bin/fm-azure-runner-agent-fleet-install.py b/bin/fm-azure-runner-agent-fleet-install.py new file mode 100755 index 00000000000..405dac39160 --- /dev/null +++ b/bin/fm-azure-runner-agent-fleet-install.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""Install the exact locked Agent Fleet source into a prepared offline venv. + +The Azure runner has no package network while repository code executes and its +wheelhouse intentionally contains only locked registry dependencies. Agent +Fleet has no runtime dependencies, so this trusted installer copies the exact +snapshot package into the fresh venv, writes ordinary distribution metadata, +and creates the release-local console entrypoint without invoking an unsealed +PEP 517 build backend. +""" + +import base64 +import csv +import hashlib +import io +import os +from pathlib import Path +import re +import stat +import sys +import sysconfig + + +class InstallError(RuntimeError): + pass + + +def real_subdirectory(root, parts, label): + current = root + for part in parts: + current = current / part + try: + metadata = current.lstat() + except OSError as exc: + raise InstallError("{} is unavailable: {}".format(label, exc)) + if not stat.S_ISDIR(metadata.st_mode): + raise InstallError("{} must have real directory ancestry".format(label)) + return current + + +def regular_file(path, label): + try: + metadata = path.lstat() + except OSError as exc: + raise InstallError("{} is unavailable: {}".format(label, exc)) + if not stat.S_ISREG(metadata.st_mode): + raise InstallError("{} must be a regular non-link file".format(label)) + return path + + +def write_new(path, content, mode=0o644): + path.parent.mkdir(parents=True, exist_ok=True) + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + descriptor = os.open(str(path), flags, mode) + with os.fdopen(descriptor, "wb") as handle: + handle.write(content) + os.chmod(path, mode) + + +def record_digest(path): + digest = base64.urlsafe_b64encode(hashlib.sha256(path.read_bytes()).digest()).rstrip(b"=") + return "sha256=" + digest.decode("ascii") + + +def copy_package(source, destination): + if destination.exists() or destination.is_symlink(): + raise InstallError("Agent Fleet package destination already exists") + destination.mkdir(parents=True, mode=0o755) + copied = [] + for child in sorted(source.rglob("*")): + relative = child.relative_to(source) + metadata = child.lstat() + target = destination / relative + if stat.S_ISLNK(metadata.st_mode): + raise InstallError("Agent Fleet source contains a link: {}".format(relative)) + if stat.S_ISDIR(metadata.st_mode): + target.mkdir(mode=0o755) + continue + if not stat.S_ISREG(metadata.st_mode): + raise InstallError("Agent Fleet source contains a non-regular entry: {}".format(relative)) + target.parent.mkdir(parents=True, exist_ok=True) + write_new(target, child.read_bytes()) + copied.append(target) + if not copied: + raise InstallError("Agent Fleet source package is empty") + return copied + + +def install(project, venv): + if sys.version_info < (3, 11): + raise InstallError("Agent Fleet requires Python 3.11 or newer") + project = project.resolve() + venv = venv.resolve() + if Path(sys.prefix).resolve() != venv: + raise InstallError("installer must run with the exact target venv interpreter") + + pyproject_path = regular_file(project / "pyproject.toml", "Agent Fleet pyproject") + lock_path = regular_file(project / "uv.lock", "Agent Fleet lock") + source = real_subdirectory(project, ("src", "agent_fleet"), "Agent Fleet package source") + + try: + pyproject_text = pyproject_path.read_text(encoding="utf-8") + lock_text = lock_path.read_text(encoding="utf-8") + except OSError as exc: + raise InstallError("Agent Fleet project metadata is unreadable: {}".format(exc)) + + project_match = re.search(r"(?ms)^\[project\]\n(.*?)(?=^\[|\Z)", pyproject_text) + scripts_match = re.search(r"(?ms)^\[project\.scripts\]\n(.*?)(?=^\[|\Z)", pyproject_text) + if project_match is None or scripts_match is None: + raise InstallError("Agent Fleet project tables are absent") + project_table = project_match.group(1) + scripts_table = scripts_match.group(1) + name_match = re.search(r'^name = "([^"]+)"$', project_table, re.MULTILINE) + version_match = re.search(r'^version = "([^"]+)"$', project_table, re.MULTILINE) + dependencies_match = re.search(r"^dependencies = (.+)$", project_table, re.MULTILINE) + name = name_match.group(1) if name_match else None + version = version_match.group(1) if version_match else None + if name != "agent-fleet" or version is None or not re.fullmatch(r"[0-9]+(?:\.[0-9]+){2}", version): + raise InstallError("Agent Fleet project identity is not exact") + script_lines = [line.strip() for line in scripts_table.splitlines() if line.strip()] + if script_lines != ['agent-fleet = "agent_fleet.cli:main"']: + raise InstallError("Agent Fleet console entrypoint declaration is not exact") + if dependencies_match is None or dependencies_match.group(1).strip() != "[]": + raise InstallError("Agent Fleet gained runtime dependencies outside the sealed offline closure") + + locked = [] + for block in lock_text.split("[[package]]")[1:]: + locked_name = re.search(r'^name = "([^"]+)"$', block, re.MULTILINE) + if locked_name and locked_name.group(1) == name: + locked.append(block) + if len(locked) != 1: + raise InstallError("Agent Fleet lock does not contain one exact project record") + locked_block = locked[0] + if ( + not re.search(r'^version = "{}"$'.format(re.escape(version)), locked_block, re.MULTILINE) + or not re.search(r'^source = \{ editable = "\." \}$', locked_block, re.MULTILINE) + or re.search(r"^dependencies = \[", locked_block, re.MULTILINE) + ): + raise InstallError("Agent Fleet lock does not bind the exact editable project without runtime dependencies") + + purelib = Path(sysconfig.get_path("purelib")).resolve() + scripts_dir = Path(sysconfig.get_path("scripts")).resolve() + if venv not in purelib.parents or venv not in scripts_dir.parents: + raise InstallError("target interpreter paths escape the exact venv") + + package_destination = purelib / "agent_fleet" + dist_info = purelib / "agent_fleet-{}.dist-info".format(version) + entrypoint = scripts_dir / "agent-fleet" + if dist_info.exists() or dist_info.is_symlink() or entrypoint.exists() or entrypoint.is_symlink(): + raise InstallError("Agent Fleet project or console entrypoint is already installed") + + installed = copy_package(source, package_destination) + dist_info.mkdir(mode=0o755) + metadata = ( + "Metadata-Version: 2.3\n" + "Name: agent-fleet\n" + "Version: {}\n" + "Summary: Machine-global account profile routing for local agent CLIs\n" + "Requires-Python: >=3.11\n" + "\n" + ).format(version).encode("utf-8") + write_new(dist_info / "METADATA", metadata) + write_new(dist_info / "WHEEL", b"Wheel-Version: 1.0\nGenerator: fm-azure-runner\nRoot-Is-Purelib: true\nTag: py3-none-any\n") + write_new(dist_info / "entry_points.txt", b"[console_scripts]\nagent-fleet = agent_fleet.cli:main\n") + write_new(dist_info / "INSTALLER", b"fm-azure-runner\n") + + python_path = venv / "bin" / "python" + if not python_path.exists(): + raise InstallError("target venv has no Python entrypoint") + entrypoint_bytes = ( + "#!{}\n" + "import sys\n" + "from agent_fleet.cli import main\n" + "if __name__ == '__main__':\n" + " sys.exit(main())\n" + ).format(python_path).encode("utf-8") + write_new(entrypoint, entrypoint_bytes, mode=0o755) + + record_rows = [] + for path in sorted(installed + [ + dist_info / "METADATA", + dist_info / "WHEEL", + dist_info / "entry_points.txt", + dist_info / "INSTALLER", + entrypoint, + ]): + relative = path.relative_to(venv).as_posix() + record_rows.append((relative, record_digest(path), str(path.stat().st_size))) + record_path = dist_info / "RECORD" + record_rows.append((record_path.relative_to(venv).as_posix(), "", "")) + output = io.StringIO(newline="") + writer = csv.writer(output, lineterminator="\n") + writer.writerows(record_rows) + write_new(record_path, output.getvalue().encode("utf-8")) + return entrypoint + + +def main(): + if len(sys.argv) != 3: + print("usage: fm-azure-runner-agent-fleet-install.py ", file=sys.stderr) + return 2 + try: + entrypoint = install(Path(sys.argv[1]), Path(sys.argv[2])) + except InstallError as exc: + print("agent-fleet offline install failed: {}".format(exc), file=sys.stderr) + return 125 + print("agent-fleet offline install: {}".format(entrypoint)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bin/fm-azure-runner-dispatch.sh b/bin/fm-azure-runner-dispatch.sh index d1bc1da7bae..edae1723b13 100755 --- a/bin/fm-azure-runner-dispatch.sh +++ b/bin/fm-azure-runner-dispatch.sh @@ -443,13 +443,34 @@ if [ -z "$CONFIRM" ]; then CONFIRM=$FM_AZURE_SUBSCRIPTION_ID fi +SOURCE_ARGUMENTS=() +if [ "$ROUTING_STATE" = selected ]; then + # A per-run no-mistakes step executes from the gate's detached, pipeline-owned + # snapshot. Give that exact HEAD a deterministic private bundle ref derived + # from the already-proved run id instead of guessing or pushing a task branch. + # The direct bundle takes the ordinary standalone shared-capacity path; it is + # not a validation-cell child and carries no parent reservation. + [ -n "$ROUTING_RUN_ID" ] \ + || refuse "selected per-run routing has no exact no-mistakes run identity" + SOURCE_ARGUMENTS=( + --source-ref "refs/heads/fm-no-mistakes/$ROUTING_RUN_ID" + --private-snapshot-from-head + ) +fi + printf 'azure-runner: class=%s selected REMOTE resource-class=%s source=%s (dispatching)\n' \ "$COMMAND_CLASS" "$RESOURCE_CLASS" "$SELECTION_SOURCE" >&2 -exec "$SCRIPT_DIR/fm-azure-runner.sh" run \ - --confirm-run \ - --confirm-subscription "$CONFIRM" \ - --task "$TASK" \ - --generation "$GENERATION" \ - --resource-class "$RESOURCE_CLASS" \ - -- "$@" +RUNNER_ARGUMENTS=( + run + --confirm-run + --confirm-subscription "$CONFIRM" + --task "$TASK" + --generation "$GENERATION" + --resource-class "$RESOURCE_CLASS" +) +if [ "$ROUTING_STATE" = selected ]; then + RUNNER_ARGUMENTS+=("${SOURCE_ARGUMENTS[@]}") +fi +RUNNER_ARGUMENTS+=(-- "$@") +exec "$SCRIPT_DIR/fm-azure-runner.sh" "${RUNNER_ARGUMENTS[@]}" diff --git a/bin/fm-azure-runner-exec.py b/bin/fm-azure-runner-exec.py index 3de17bcdf9f..80710ef0d59 100755 --- a/bin/fm-azure-runner-exec.py +++ b/bin/fm-azure-runner-exec.py @@ -20,7 +20,7 @@ RESULT_SCHEMA = "fm.azure-command-result/v1" -PRIVATE_SOURCE_MODES = ("private-parent-bundle", "private-exact-bundle") +PRIVATE_SOURCE_MODES = ("private-parent-bundle", "private-exact-bundle", "private-direct-bundle") def fail(message): diff --git a/bin/fm-azure-runner-guest.sh b/bin/fm-azure-runner-guest.sh index fa98a1e5313..0a3eddad923 100755 --- a/bin/fm-azure-runner-guest.sh +++ b/bin/fm-azure-runner-guest.sh @@ -18,10 +18,11 @@ OUTPUT_BLOB=${output_blob:-} INPUT_BLOB=${input_blob:-} IDENTITY_CLIENT_ID=${identity_client_id:-} EXECUTOR_B64=${executor_b64:-} +AGENT_FLEET_INSTALLER_B64=${agent_fleet_installer_b64:-} unset request_b64 vm_resource_id vm_instance_id guest_digest storage_account -unset container output_blob input_blob identity_client_id executor_b64 -for bound in "$REQUEST_B64" "$VM_RESOURCE_ID" "$VM_INSTANCE_ID" "$GUEST_DIGEST" "$STORAGE_ACCOUNT" "$CONTAINER" "$OUTPUT_BLOB" "$INPUT_BLOB" "$IDENTITY_CLIENT_ID" "$EXECUTOR_B64"; do - [ -n "$bound" ] || { echo "guest bootstrap: expected ten bound parameters" >&2; exit 125; } +unset container output_blob input_blob identity_client_id executor_b64 agent_fleet_installer_b64 +for bound in "$REQUEST_B64" "$VM_RESOURCE_ID" "$VM_INSTANCE_ID" "$GUEST_DIGEST" "$STORAGE_ACCOUNT" "$CONTAINER" "$OUTPUT_BLOB" "$INPUT_BLOB" "$IDENTITY_CLIENT_ID" "$EXECUTOR_B64" "$AGENT_FLEET_INSTALLER_B64"; do + [ -n "$bound" ] || { echo "guest bootstrap: expected eleven bound parameters" >&2; exit 125; } done unset bound case "$GUEST_DIGEST" in sha256:[0-9a-f][0-9a-f]*) ;; *) echo "guest bootstrap: bad protocol digest" >&2; exit 125 ;; esac @@ -169,22 +170,26 @@ rm -rf "$BASE" install -d -m 0700 -o root -g root "$BASE" REQUEST=$BASE/request.json EXECUTOR=$BASE/runner-exec.py +AGENT_FLEET_INSTALLER=$BASE/agent-fleet-install.py printf '%s' "$REQUEST_B64" | base64 -d >"$REQUEST" printf '%s' "$EXECUTOR_B64" | base64 -d >"$EXECUTOR" -unset REQUEST_B64 EXECUTOR_B64 +printf '%s' "$AGENT_FLEET_INSTALLER_B64" | base64 -d >"$AGENT_FLEET_INSTALLER" +unset REQUEST_B64 EXECUTOR_B64 AGENT_FLEET_INSTALLER_B64 -python3 - "$REQUEST" "$EXECUTOR" "$GUEST_DIGEST" <<'PY' +python3 - "$REQUEST" "$EXECUTOR" "$AGENT_FLEET_INSTALLER" "$GUEST_DIGEST" <<'PY' import hashlib, json, pathlib, sys -request_path, executor_path = map(pathlib.Path, sys.argv[1:3]) +request_path, executor_path, installer_path = map(pathlib.Path, sys.argv[1:4]) request = json.loads(request_path.read_text(encoding="utf-8")) unsigned = dict(request); supplied = unsigned.pop("request_digest", None) canonical = json.dumps(unsigned, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode() if supplied != "sha256:" + hashlib.sha256(canonical).hexdigest(): raise SystemExit("guest bootstrap: request digest mismatch") if "sha256:" + hashlib.sha256(executor_path.read_bytes()).hexdigest() != request["protocol"]["executor_digest"]: raise SystemExit("guest bootstrap: executor digest mismatch") -if request["protocol"]["guest_digest"] != sys.argv[3]: raise SystemExit("guest bootstrap: guest digest mismatch") +if "sha256:" + hashlib.sha256(installer_path.read_bytes()).hexdigest() != request["protocol"]["agent_fleet_installer_digest"]: raise SystemExit("guest bootstrap: Agent Fleet installer digest mismatch") +if request["protocol"]["guest_digest"] != sys.argv[4]: raise SystemExit("guest bootstrap: guest digest mismatch") repo = request["repository"] -if repo.get("source_mode") not in ("public-github-https", "private-parent-bundle", "private-exact-bundle") or not repo.get("remote", "").startswith("https://github.com/"): raise SystemExit("guest bootstrap: source mode mismatch") -if repo.get("source_mode") in ("private-parent-bundle", "private-exact-bundle"): +private_modes = ("private-parent-bundle", "private-exact-bundle", "private-direct-bundle") +if repo.get("source_mode") not in ("public-github-https",) + private_modes or not repo.get("remote", "").startswith("https://github.com/"): raise SystemExit("guest bootstrap: source mode mismatch") +if repo.get("source_mode") in private_modes: if not repo.get("input_blob") or not repo.get("snapshot_digest") or not repo.get("snapshot_bytes"): raise SystemExit("guest bootstrap: private snapshot binding is incomplete") else: if repo.get("input_blob") is not None or repo.get("snapshot_bytes") != 0: raise SystemExit("guest bootstrap: public source carries private staging") @@ -234,7 +239,7 @@ runuser -u fmrunner -- git -C /work/repo remote add origin "$REMOTE" # repository as dubious (CVE-2022-24765); scope the exception through the # environment exactly as the validation cell guest does. export GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=safe.directory GIT_CONFIG_VALUE_0=/work/repo -if [ "$SOURCE_MODE" = private-parent-bundle ] || [ "$SOURCE_MODE" = private-exact-bundle ]; then +if [ "$SOURCE_MODE" = private-parent-bundle ] || [ "$SOURCE_MODE" = private-exact-bundle ] || [ "$SOURCE_MODE" = private-direct-bundle ]; then [ "$INPUT_BLOB" = "$(read_request repository.input_blob)" ] || { echo "guest bootstrap: private snapshot blob mismatch" >&2; exit 125; } SNAPSHOT=$BASE/snapshot.bundle TOKEN_FILE=$BASE/input-token @@ -260,11 +265,11 @@ if [ "$SOURCE_MODE" = private-parent-bundle ] || [ "$SOURCE_MODE" = private-exac rm -f /work/snapshot.bundle elif [ "$SOURCE_REF" != none ] && [ "$SOURCE_HEAD" = "$COMMIT" ]; then [ "$INPUT_BLOB" = none ] || { echo "guest bootstrap: public source received a private snapshot blob" >&2; exit 125; } - run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch --depth=1 origin "$SOURCE_REF" + run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch origin "$SOURCE_REF" [ "$(git -C /work/repo rev-parse FETCH_HEAD)" = "$COMMIT" ] || { echo "guest bootstrap: source ref moved after admission" >&2; exit 125; } else [ "$INPUT_BLOB" = none ] || { echo "guest bootstrap: public source received a private snapshot blob" >&2; exit 125; } - run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch --depth=1 origin "$COMMIT" + run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch origin "$COMMIT" fi runuser -u fmrunner -- git -C /work/repo checkout --detach "$COMMIT" >/dev/null python3 - "$REQUEST" <<'PY' >"$BASE/source-ancestors" @@ -274,12 +279,12 @@ PY while IFS= read -r ancestor; do [ -n "$ancestor" ] || continue if [ "$SOURCE_MODE" = public-github-https ]; then - run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch --depth=1 origin "$ancestor" + run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch origin "$ancestor" [ "$(git -C /work/repo rev-parse FETCH_HEAD)" = "$ancestor" ] || { echo "guest bootstrap: source ancestor identity mismatch" >&2; exit 125; } git -C /work/repo cat-file -e "$ancestor^{commit}" || { echo "guest bootstrap: source ancestor is absent" >&2; exit 125; } fi done <"$BASE/source-ancestors" -if [ "$SOURCE_MODE" = private-parent-bundle ] || [ "$SOURCE_MODE" = private-exact-bundle ]; then +if [ "$SOURCE_MODE" = private-parent-bundle ] || [ "$SOURCE_MODE" = private-exact-bundle ] || [ "$SOURCE_MODE" = private-direct-bundle ]; then /usr/bin/python3 "$EXECUTOR" --verify-private-source-ancestors "$REQUEST" /work/repo fi [ "$(git -C /work/repo rev-parse HEAD)" = "$COMMIT" ] && [ "$(git -C /work/repo rev-parse 'HEAD^{tree}')" = "$TREE" ] || { echo "guest bootstrap: source identity mismatch" >&2; exit 125; } @@ -296,13 +301,15 @@ if [ "$DEFAULT_REF" != none ] && [ "$DEFAULT_HEAD" != none ]; then case "$DEFAULT_REF" in refs/heads/?*) DEFAULT_NAME=${DEFAULT_REF#refs/heads/} ;; *) DEFAULT_NAME= ;; esac if [ -n "$DEFAULT_NAME" ]; then if ! git -C /work/repo cat-file -e "$DEFAULT_HEAD^{commit}" 2>/dev/null; then - run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch --depth=1 origin "$DEFAULT_HEAD" + run_bootstrap_network runuser -u fmrunner -- git -C /work/repo fetch origin "$DEFAULT_HEAD" [ "$(git -C /work/repo rev-parse FETCH_HEAD)" = "$DEFAULT_HEAD" ] || { echo "guest bootstrap: default head identity mismatch" >&2; exit 125; } fi runuser -u fmrunner -- git -C /work/repo update-ref "refs/remotes/origin/$DEFAULT_NAME" "$DEFAULT_HEAD" runuser -u fmrunner -- git -C /work/repo symbolic-ref refs/remotes/origin/HEAD "refs/remotes/origin/$DEFAULT_NAME" fi fi +[ "$(git -C /work/repo rev-parse --is-shallow-repository)" = false ] \ + || { echo "guest bootstrap: sealed source graph is shallow" >&2; exit 125; } fetch_exact() { local url=$1 path=$2 bytes=$3 digest=$4 redirects=${5:-no} args=(); [ "$redirects" != yes ] || args+=(--location); run_bootstrap_network curl --fail --silent --show-error "${args[@]}" --connect-timeout 30 --max-time 300 --max-filesize "$bytes" --output "$path" "$url"; [ "$(stat -c %s "$path")" = "$bytes" ] && [ "sha256:$(sha256sum "$path" | awk '{print $1}')" = "$digest" ] || { echo "guest bootstrap: pinned download mismatch" >&2; exit 125; }; } # The golden image stages the pinned archives under /opt/fm-tools; a copy @@ -341,6 +348,9 @@ if [ "$LOCK_DIGEST" != None ]; then cd /work/repo runuser -u fmrunner -- /work/home/.fm-runner-tools/uv/uv venv --python /usr/bin/python3 /work/repo/tools/agent-fleet/.venv >/dev/null runuser -u fmrunner -- env UV_OFFLINE=1 UV_NO_INDEX=1 /work/home/.fm-runner-tools/uv/uv pip install --python /work/repo/tools/agent-fleet/.venv/bin/python --offline --no-index --find-links /work/home/.fm-runner-tools/wheelhouse pytest ruff >/dev/null + /work/repo/tools/agent-fleet/.venv/bin/python "$AGENT_FLEET_INSTALLER" /work/repo/tools/agent-fleet /work/repo/tools/agent-fleet/.venv >/dev/null + chown -R fmrunner:fmrunner /work/repo/tools/agent-fleet/.venv + runuser -u fmrunner -- /work/repo/tools/agent-fleet/.venv/bin/agent-fleet --help >/dev/null elif [ -s "$BASE/wheels.tsv" ]; then echo "guest bootstrap: unbound Python wheels" >&2 exit 125 diff --git a/bin/fm-azure-runner.py b/bin/fm-azure-runner.py index 3a1f148d342..7638d7224d5 100755 --- a/bin/fm-azure-runner.py +++ b/bin/fm-azure-runner.py @@ -38,6 +38,7 @@ TEMPLATE = ROOT / "docs" / "azure-runner" / "invocation.json" GUEST = ROOT / "bin" / "fm-azure-runner-guest.sh" EXECUTOR = ROOT / "bin" / "fm-azure-runner-exec.py" +AGENT_FLEET_INSTALLER = ROOT / "bin" / "fm-azure-runner-agent-fleet-install.py" WORKER_LIFECYCLE = ROOT / "bin" / "fm-worker-lifecycle.py" CONTAINER = "validation-shards" CONTROL_CONTAINER = "runner-control" @@ -752,6 +753,45 @@ def locked_python_manifest(repo): +def verify_self_contained_private_bundle(bundle, commit, source_ref): + heads = git(ROOT, "bundle", "list-heads", str(bundle)).stdout.splitlines() + expected_head = "{} {}".format(commit, source_ref) + if heads != [expected_head]: + raise RunnerError("private snapshot must contain only the exact source-ref head") + with tempfile.TemporaryDirectory(prefix="fm-azure-bundle-verify-") as temporary: + verification_repo = Path(temporary) / "repo.git" + run(["git", "init", "--bare", str(verification_repo)]) + run(["git", "-C", str(verification_repo), "bundle", "verify", str(bundle)]) + run([ + "git", "-C", str(verification_repo), "fetch", "--no-tags", str(bundle), + "+{}:refs/fm-azure-runner/verified".format(source_ref), + ]) + verified = git( + verification_repo, "rev-parse", "--verify", "refs/fm-azure-runner/verified" + ).stdout.strip() + shallow = git(verification_repo, "rev-parse", "--is-shallow-repository").stdout.strip() + if verified != commit or shallow != "false": + raise RunnerError("private snapshot source graph is incomplete or has the wrong head") + + +def create_private_snapshot_from_head(repo, destination, commit, source_ref): + shallow = git(repo, "rev-parse", "--is-shallow-repository").stdout.strip() + if shallow != "false": + raise RunnerError("direct private snapshot requires a complete non-shallow source graph") + with tempfile.TemporaryDirectory(prefix="fm-azure-bundle-stage-") as temporary: + staging_repo = Path(temporary) / "repo.git" + run(["git", "init", "--bare", str(staging_repo)]) + run([ + "git", "-C", str(staging_repo), "fetch", "--no-tags", "--force", str(repo), + "+{}:{}".format(commit, source_ref), + ]) + staged = git(staging_repo, "rev-parse", "--verify", source_ref).stdout.strip() + if staged != commit: + raise RunnerError("direct private snapshot staging changed the exact source head") + run(["git", "-C", str(staging_repo), "bundle", "create", str(destination), source_ref]) + verify_self_contained_private_bundle(destination, commit, source_ref) + + def tree_digest(repo, relative): path = repo / relative if not path.exists(): @@ -802,15 +842,18 @@ def prepare(env, args, parent_state=None): branch = git(repo, "symbolic-ref", "--quiet", "--short", "HEAD", check=False) if ( branch.returncode != 0 - and args.public_ref is None - and not args.private_snapshot_bundle + and getattr(args, "public_ref", None) is None + and getattr(args, "source_ref", None) is None ): raise RunnerError( - "repository must be on a named committed branch unless an exact public ref or private snapshot is supplied" + "repository must be on a named committed branch unless an exact source ref is supplied" ) commit = git(repo, "rev-parse", "HEAD").stdout.strip() remote = git(repo, "remote", "get-url", "origin").stdout.strip() private_snapshot_source = None + private_snapshot_from_head = bool(getattr(args, "private_snapshot_from_head", False)) + if args.private_snapshot_bundle and private_snapshot_from_head: + raise RunnerError("choose one private snapshot input: bundle or exact HEAD") if args.private_snapshot_bundle: private_snapshot_arg = Path(args.private_snapshot_bundle) private_snapshot_source = private_snapshot_arg.resolve() @@ -831,7 +874,16 @@ def prepare(env, args, parent_state=None): if getattr(args, "public_ref", None) and args.source_ref: raise RunnerError("choose one exact source identity: --source-ref or --public-ref") source_ancestors = tuple(getattr(args, "public_ancestor", None) or ()) - if private_snapshot_source is not None and not args.capacity_parent: + if private_snapshot_from_head: + if args.capacity_parent or not args.source_ref: + raise RunnerError("direct private HEAD snapshot requires one exact source ref and no capacity parent") + public = public_origin_proof( + repo, remote, commit, + source_ref=args.source_ref, + source_ancestors=source_ancestors, + private_source=True, + ) + elif private_snapshot_source is not None and not args.capacity_parent: public = private_bundle_origin_proof( repo, remote, @@ -846,6 +898,7 @@ def prepare(env, args, parent_state=None): source_ancestors=source_ancestors, private_source=private_snapshot_source is not None, ) + private_snapshot_requested = private_snapshot_source is not None or private_snapshot_from_head tree = public["tree"] task = require_identifier("task", args.task) @@ -875,12 +928,19 @@ def prepare(env, args, parent_state=None): private_snapshot_path = None private_snapshot_digest = None private_snapshot_bytes = 0 - if private_snapshot_source is not None: + if private_snapshot_requested: private_snapshot_path = payload_dir / "snapshot.bundle" - shutil.copyfile(str(private_snapshot_source), str(private_snapshot_path)) + if private_snapshot_from_head: + create_private_snapshot_from_head( + repo, private_snapshot_path, commit, args.source_ref + ) + else: + shutil.copyfile(str(private_snapshot_source), str(private_snapshot_path)) os.chmod(private_snapshot_path, 0o600) private_snapshot_digest = "sha256:" + sha256_file(private_snapshot_path) private_snapshot_bytes = private_snapshot_path.stat().st_size + if private_snapshot_bytes > MAX_STAGING_INPUT_BYTES: + raise RunnerError("private snapshot exceeds the one-GiB staging bound") source_identity = { "remote": remote, "default_ref": public["default_ref"], @@ -958,7 +1018,11 @@ def prepare(env, args, parent_state=None): ( "private-parent-bundle" if args.capacity_parent - else "private-exact-bundle" + else ( + "private-direct-bundle" + if private_snapshot_from_head + else "private-exact-bundle" + ) ) if private_snapshot_path else "public-github-https" @@ -988,6 +1052,7 @@ def prepare(env, args, parent_state=None): "shellcheck_archive_digest": "sha256:8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198", "uv_archive_digest": "sha256:440c4215b171e64061d65d16a23753dd25c29a7f7b1b0446c9e9aed0fa372f27", "agent_fleet_python": locked_python, + "agent_fleet_installer_digest": "sha256:" + sha256_file(AGENT_FLEET_INSTALLER), }, "created_at": iso_utc(prepared_at), "compute_deallocation_deadline": iso_utc(expires_at), @@ -1069,7 +1134,9 @@ def reprove_public_request(state): repository = state["request"]["repository"] repo = Path(state["repository_root"]).resolve() source_mode = repository.get("source_mode") - private_source = source_mode in ("private-parent-bundle", "private-exact-bundle") + private_source = source_mode in ( + "private-parent-bundle", "private-exact-bundle", "private-direct-bundle", + ) expected = { "remote": repository["remote"], "default_ref": repository["default_ref"], @@ -2645,9 +2712,13 @@ def create_run_command(env, state): current_guest_digest = "sha256:" + sha256_file(GUEST) if current_guest_digest != state["request"]["protocol"]["guest_digest"]: raise RunnerError("trusted guest protocol changed after request preparation") + current_installer_digest = "sha256:" + sha256_file(AGENT_FLEET_INSTALLER) + if current_installer_digest != state["request"]["protocol"]["agent_fleet_installer_digest"]: + raise RunnerError("trusted Agent Fleet installer changed after request preparation") script = GUEST.read_text(encoding="utf-8") request_b64 = base64.b64encode(Path(state["input_path"]).read_bytes()).decode("ascii") executor_b64 = base64.b64encode(EXECUTOR.read_bytes()).decode("ascii") + agent_fleet_installer_b64 = base64.b64encode(AGENT_FLEET_INSTALLER.read_bytes()).decode("ascii") properties = { "location": "eastus", "tags": ownership_tags(env, state), @@ -2664,6 +2735,7 @@ def create_run_command(env, state): {"name": "input_blob", "value": state["staging"].get("input_blob") or "none"}, {"name": "identity_client_id", "value": env["controller_identity_client_id"]}, {"name": "executor_b64", "value": executor_b64}, + {"name": "agent_fleet_installer_b64", "value": agent_fleet_installer_b64}, ], "asyncExecution": False, "timeoutInSeconds": state["request"]["limits"]["wall_seconds"] + 1200, @@ -3265,7 +3337,11 @@ def retry(env, old_state, args): private_source = repository.get("source_mode") in ( "private-parent-bundle", "private-exact-bundle", + "private-direct-bundle", ) + private_parent_source = repository.get("source_mode") == "private-parent-bundle" + private_exact_source = repository.get("source_mode") == "private-exact-bundle" + private_direct_source = repository.get("source_mode") == "private-direct-bundle" selected_source_ref = ( repository["source_ref"] if repository["source_ref"] != repository["default_ref"] @@ -3278,9 +3354,10 @@ def retry(env, old_state, args): args.source_ref = None args.public_ref = selected_source_ref args.public_ancestor = list(repository.get("source_ancestors", [])) + args.private_snapshot_from_head = private_direct_source args.private_snapshot_bundle = ( str(Path(old_state["input_path"]).parent / "snapshot.bundle") - if private_source + if private_parent_source or private_exact_source else None ) args.wall_seconds = old_state["request"]["limits"]["wall_seconds"] @@ -3353,6 +3430,11 @@ def add_request_arguments(parser, require_command=True): "--private-snapshot-bundle", help="exact parent-cell Git bundle staged privately for an unpushed validation head", ) + parser.add_argument( + "--private-snapshot-from-head", + action="store_true", + help="seal the exact clean non-shallow HEAD into a direct one-ref private bundle", + ) parser.add_argument("--dependency", action="append", default=[]) parser.add_argument("--artifact", action="append", default=[]) if require_command: diff --git a/bin/fm-azure-runner.sh b/bin/fm-azure-runner.sh index c6cc0809c0f..ae875343d3e 100755 --- a/bin/fm-azure-runner.sh +++ b/bin/fm-azure-runner.sh @@ -31,6 +31,7 @@ # fm-azure-runner.sh run --confirm-run --confirm-subscription \ # [--confirm-cost-admission-mode commissioning-bounded] \ # --task --generation --resource-class \ +# [--source-ref refs/heads/] [--private-snapshot-from-head] \ # [--wall-seconds N] [--dependency ]... \ # [--artifact ]... -- # fm-azure-runner.sh resume --invocation diff --git a/bin/fm-azure-validation-shard-bridge.py b/bin/fm-azure-validation-shard-bridge.py index 44178a2e600..0e2004b61f5 100755 --- a/bin/fm-azure-validation-shard-bridge.py +++ b/bin/fm-azure-validation-shard-bridge.py @@ -48,7 +48,7 @@ # that executes these shards sets PrivateNetwork=yes, # RestrictAddressFamilies=AF_UNIX and IPAddressDeny=any, so bin/fm-teardown.sh's # secondmate upstream-authority probe can never resolve or reach the origin -# remote's host. That skips THIRTY-THREE units, the whole secondmate +# remote's host. That skips THIRTY-SEVEN units, the whole secondmate # teardown/retirement family in tests/fm-teardown-suite.sh. The set was # enumerated to convergence - both teardown files run to completion with the # network off, 143 of 143 cases - and those units are SKIPPED in the cell, not diff --git a/bin/fm-azure-worker-provider.py b/bin/fm-azure-worker-provider.py index 7a2912479a4..8330c76a14f 100755 --- a/bin/fm-azure-worker-provider.py +++ b/bin/fm-azure-worker-provider.py @@ -17,6 +17,7 @@ exact-assignment gate enforces it today. """ +import base64 import contextlib import datetime as dt import email.utils @@ -427,7 +428,7 @@ def immutable_id(kind, value): role = value.get("roleDefinitionId") or properties.get("roleDefinitionId") return "{}|{}".format(principal, role) if principal and role else None if kind == "state-container": - return value.get("etag") or properties.get("etag") or value.get("version") + return value.get("id") if kind in MUTABLE_PROVISIONING_CHILD_KINDS: # Azure mutates provisioningState during ordinary VM lifecycle # transitions (including Succeeded -> Updating after deallocation). @@ -1310,11 +1311,17 @@ def recorded_exact( # path stays fixed. Task commands and staging request/result blobs # also bind changing execution content through request/result # digests, so their path identity is the ownership fence here. + legacy_state_container = ( + kind == "state-container" + and current.get("immutable_id") == current.get("id") + and prior.get("id") == current.get("id") + ) if ( kind not in skip_immutable and kind not in MUTABLE_PROVISIONING_CHILD_KINDS and kind not in ("staging-request", "staging-result") and current.get("immutable_id") != prior.get("immutable_id") + and not legacy_state_container ): raise ProviderIdentityRefusal( "{} immutable identity differs from the recorded assignment".format(kind) @@ -1423,6 +1430,7 @@ def blob_identity_digest(value, volatile_fields=()): def upload_json_blob( controller, account, container, name, value, tags, overwrite=False, volatile_fields=(), + if_match=None, ): payload = canonical_bytes(value) + b"\n" digest = hashlib.sha256(payload).hexdigest() @@ -1435,12 +1443,66 @@ def upload_json_blob( metadata = dict(tags_to_metadata(tags)) metadata["content_digest"] = digest metadata["identity_digest"] = identity - _, rc, stderr = az(controller, [ + upload_args = [ "storage", "blob", "upload", "--auth-mode", "login", "--account-name", account, "--container-name", container, "--name", name, "--file", path, "--overwrite", "true" if overwrite else "false", "--metadata", - ] + ["{}={}".format(key, value) for key, value in sorted(metadata.items())], check=False) + ] + ["{}={}".format(key, value) for key, value in sorted(metadata.items())] + if if_match is not None: + upload_args += ["--if-match", if_match] + _, rc, stderr = az(controller, upload_args, check=False) if rc != 0: + condition_error = str(stderr).lower() + if if_match is not None and ( + "conditionnotmet" in condition_error + or "condition specified" in condition_error + ): + current, show_rc, show_stderr = az(controller, [ + "storage", "blob", "show", "--auth-mode", "login", + "--account-name", account, "--container-name", container, + "--name", name, + ], check=False) + if show_rc != 0 or not isinstance(current, dict): + raise ProviderError( + "conditionally written worker staging blob is unreadable: {}".format( + show_stderr + ) + ) + current_properties = (current or {}).get("properties") or {} + current_etag = (current or {}).get("etag") or current_properties.get("etag") + if not current_etag: + raise ProviderError( + "conditionally written worker staging blob is unreadable: {}".format( + show_stderr + ) + ) + current_fd, current_path = tempfile.mkstemp( + prefix="fm-worker-current-blob-", suffix=".json" + ) + os.close(current_fd) + os.chmod(current_path, 0o600) + try: + _, download_rc, download_stderr = az(controller, [ + "storage", "blob", "download", "--auth-mode", "login", + "--account-name", account, "--container-name", container, + "--name", name, "--file", current_path, "--overwrite", "true", + "--if-match", current_etag, + ], check=False) + if download_rc != 0: + raise ProviderError( + "conditionally written worker staging blob changed during read: {}".format( + download_stderr + ) + ) + current_payload = Path(current_path).read_bytes() + if ( + len(current_payload) == len(payload) + and hashlib.sha256(current_payload).hexdigest() == digest + ): + return digest + finally: + with contextlib.suppress(FileNotFoundError): + Path(current_path).unlink() # A create-once blob that already carries exactly these bytes is # this same action replaying after a lost or timed-out response, # which must converge rather than wedge. Different bytes under the @@ -2308,7 +2370,10 @@ def create_or_resume(controller, action): raise ProviderError("visible worker belongs to another task or generation") elif reuse: recorded_exact( - action, existing, allow_missing=("vm", "nic", "os-disk"), + action, existing, allow_missing=( + "vm", "nic", "os-disk", "monitor-extension", + "bootstrap-command", "task-command", "ttl-schedule", + ), allow_previous_cloud_generation=True, ) else: @@ -2601,6 +2666,30 @@ def build_execute_script(action): request = action["request"] request_json = json.dumps(request, sort_keys=True, separators=(",", ":")) bindings = action["bindings"] + supervisor_prelude = "" + supervisor_command = "/usr/local/libexec/fm-worker-supervisor" + if request.get("existing_task_disk"): + try: + supervisor_body = (ROOT / "bin" / "fm-worker-supervisor.py").read_bytes() + except OSError as exc: + raise ProviderError( + "existing task-disk recovery supervisor is unreadable: {}".format(exc) + ) from None + supervisor_digest = hashlib.sha256(supervisor_body).hexdigest() + if request.get("supervisor_sha256") != supervisor_digest: + raise ProviderError("existing task-disk recovery supervisor binding differs") + supervisor_path = "/var/lib/firstmate-worker/recovery-supervisor-{}.py".format( + supervisor_digest + ) + supervisor_prelude = """printf '%s' '{body}' | /usr/bin/base64 --decode > '{path}' +[ "$(/usr/bin/sha256sum '{path}' | /usr/bin/awk '{{print $1}}')" = '{digest}' ] +chmod 0700 '{path}' +""".format( + body=base64.b64encode(supervisor_body).decode("ascii"), + path=supervisor_path, + digest=supervisor_digest, + ) + supervisor_command = "/usr/bin/python3 '{}'".format(supervisor_path) return """set -eu umask 077 install -d -m 0700 /var/lib/firstmate-worker @@ -2612,14 +2701,15 @@ def build_execute_script(action): export FM_WORKER_WORKTREE_BINDING='{worktree}' FM_WORKER_REPOSITORY_BINDING='{repository}' export FM_WORKER_REPOSITORY_GENERATION='{repository_generation}' FM_WORKER_CLOUD_INSTANCE_ID='{cloud}' export FM_WORKER_WORKTREE=/mnt/task FM_WORKER_ACCOUNT_HOME=/mnt/account -/usr/local/libexec/fm-worker-supervisor execute --request /var/lib/firstmate-worker/request.json --result /var/lib/firstmate-worker/result.json +{supervisor_prelude}{supervisor_command} execute --request /var/lib/firstmate-worker/request.json --result /var/lib/firstmate-worker/result.json printf 'FM-WORKER-RESULT:%s\\n' "$(cat /var/lib/firstmate-worker/result.json)" """.format( request=request_json, home=bindings["home_binding"], task=bindings["task"], task_generation=bindings["task_generation"], generation_line=execute_generation_line(action), worktree=bindings["worktree_binding"], repository=bindings["repository_binding"], repository_generation=bindings["repository_generation"], - cloud=action["cloud_instance_id"], + cloud=action["cloud_instance_id"], supervisor_prelude=supervisor_prelude, + supervisor_command=supervisor_command, ) @@ -2647,16 +2737,39 @@ def initial_execute_staging_pair(action): } +def blob_content_is_exact(resource, value): + payload = canonical_bytes(value) + b"\n" + return ( + resource.get("digest") == hashlib.sha256(payload).hexdigest() + and resource.get("length") == len(payload) + ) + + def initial_execute_staging_is_exact(action, resources): - for kind, value in initial_execute_staging_pair(action).items(): - payload = canonical_bytes(value) + b"\n" - resource = resources.get(kind) or {} - if ( - resource.get("digest") != hashlib.sha256(payload).hexdigest() - or resource.get("length") != len(payload) - ): - return False - return True + # The result must still carry its assignment ETag. The request may carry + # that ETag or the exact request bytes from this action: the latter is a + # retry after Azure applied the conditional write but lost its response. + expected = action["resources"] + request_current = resources.get("staging-request") or {} + request_prior = expected.get("staging-request") or {} + if ( + not request_current.get("id") + or request_current.get("id") != request_prior.get("id") + or not request_current.get("immutable_id") + or ( + request_current.get("immutable_id") != request_prior.get("immutable_id") + and not blob_content_is_exact(request_current, action["request"]) + ) + ): + return False + result_current = resources.get("staging-result") or {} + result_prior = expected.get("staging-result") or {} + return bool( + result_current.get("id") + and result_current.get("id") == result_prior.get("id") + and result_current.get("immutable_id") + and result_current.get("immutable_id") == result_prior.get("immutable_id") + ) def run_command_execution_binding(live): @@ -2784,7 +2897,15 @@ def execute_terminal_disposition(controller, action, resources): def persist_execute_result(controller, action, names, tags, execution): request = action["request"] - if request.get("outcome_expected") and execution.get("outcome_present"): + storage = os.environ.get("FM_AZURE_STORAGE_NAME", "") + assignment_etag = ( + (action.get("resources") or {}).get("staging-result") or {} + ).get("immutable_id") + if not assignment_etag: + raise ProviderError("staging-result assignment ETag is absent") + if request.get("outcome_expected") and ( + execution.get("outcome_present") or execution.get("return_present") + ): outcome_target = action.get("outcome_dir") if not outcome_target: raise ProviderError("execution collected an outcome with no controller directory to land it in") @@ -2805,13 +2926,14 @@ def persist_execute_result(controller, action, names, tags, execution): if not isinstance(bytes_claim, int) or isinstance(bytes_claim, bool) or not 0 < bytes_claim <= MAX_OUTCOME_BYTES: raise ProviderError("execution outcome size is malformed or unbounded") download_outcome_bundle( - controller, os.environ.get("FM_AZURE_STORAGE_NAME", ""), names["state-container"], + controller, storage, names["state-container"], outcome_blob_name(request["request_digest"]), digest_claim, bytes_claim, Path(outcome_target) / "outcome.bundle", ) upload_json_blob( - controller, os.environ.get("FM_AZURE_STORAGE_NAME", ""), names["state-container"], + controller, storage, names["state-container"], names["staging-result"], execution, tags, overwrite=True, + if_match=assignment_etag, ) @@ -2838,6 +2960,7 @@ def mutate_execute(controller, action): upload_json_blob( controller, os.environ.get("FM_AZURE_STORAGE_NAME", ""), names["state-container"], names["staging-request"], request, tags, overwrite=True, + if_match=(action.get("resources") or {}).get("staging-request", {}).get("immutable_id"), ) # Crewmate payload plane: the digest-bound request carries only manifests; # the archives ride private blobs and reach the guest over short-lived diff --git a/bin/fm-checkout-refresh.sh b/bin/fm-checkout-refresh.sh index 013fd910692..ae2abf9cdd3 100755 --- a/bin/fm-checkout-refresh.sh +++ b/bin/fm-checkout-refresh.sh @@ -1601,9 +1601,15 @@ try: not mentions_home or len(program_arguments) != 4 or not os.path.isabs(program_arguments[0]) - or program_arguments[1:] != [script, "run-once", "--scheduled"] + or program_arguments[2:] != ["run-once", "--scheduled"] ): raise OSError("incomplete launch agent identity") + if program_arguments[1] != script: + print( + "checkout-refresh: audit: LaunchAgent script differs " + f"expected={script} observed={program_arguments[1]}; adopting observed namespace", + file=sys.stderr, + ) label = entry.name[:-6] if authoritative_label != label: raise OSError("launch agent filename and Label differ") diff --git a/bin/fm-cloud-result.py b/bin/fm-cloud-result.py new file mode 100755 index 00000000000..fae18f4f659 --- /dev/null +++ b/bin/fm-cloud-result.py @@ -0,0 +1,595 @@ +#!/usr/bin/env python3 +"""Localize one provider-neutral cloud worker return bundle. + +The worker result and its digest-verified Git bundle are the transport record. +This command copies only the task's authorized report/status/visual/scratch +artifacts, reconstructs the ordinary ship branch without overwriting local +divergence, and appends one truthful terminal status. Re-running the same +command converges on the same files, refs, branch, and status line. +""" + +import argparse +import hashlib +import io +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import tarfile +import tempfile +import unicodedata + + +RESULT_SCHEMA = "fm.worker-execution-result/v1" +RETURN_SCHEMA = "fm.worker-return/v1" +HEX40 = re.compile(r"^[0-9a-f]{40}$") +HEX64 = re.compile(r"^[0-9a-f]{64}$") +SAFE_TASK = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$") +SAFE_GENERATION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$") +REQUIRED_SECTIONS = ( + "Summary", "What changed", "Verification", "Visual evidence", "Artifacts", "Follow-ups", +) +MAX_RESULT_BYTES = 8 * 1024 * 1024 +MAX_BUNDLE_BYTES = 256 * 1024 * 1024 +MAX_ARTIFACT_BYTES = 128 * 1024 * 1024 +MAX_VISUAL_BYTES = 20 * 1024 * 1024 +MAX_VISUAL_ENTRIES = 512 +STATUS_LINE = re.compile(r"^(working|needs-decision|blocked|paused|resolved|done|failed):\s+\S.*$") +TERMINAL_LINE = re.compile(r"^(done|failed):") + + +class ReturnError(RuntimeError): + pass + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode() + + +def sha256(body): + return hashlib.sha256(body).hexdigest() + + +def read_regular(path, label, limit): + if path.is_symlink() or not path.is_file(): + raise ReturnError("{} is absent or redirected: {}".format(label, path)) + size = path.stat().st_size + if size <= 0 or size > limit: + raise ReturnError("{} has an invalid byte count: {}".format(label, size)) + return path.read_bytes() + + +def read_result(path, task, generation, assignment): + body = read_regular(path, "worker result", MAX_RESULT_BYTES) + try: + result = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ReturnError("worker result is truncated or corrupt: {}".format(exc)) + if not isinstance(result, dict) or result.get("schema") != RESULT_SCHEMA: + raise ReturnError("worker result schema is not supported") + supplied = result.get("result_digest") + unsigned = dict(result) + unsigned.pop("result_digest", None) + if not HEX64.fullmatch(str(supplied)) or supplied != sha256(canonical(unsigned)): + raise ReturnError("worker result digest is not exact") + expected = { + "task": task, + "task_generation": generation, + "assignment_generation": assignment, + } + for field, value in expected.items(): + if result.get(field) != value: + raise ReturnError("worker result {} binding differs".format(field)) + if result.get("return_present") is not True: + raise ReturnError("worker result has no authorized return bundle") + for field in ("request_digest", "return_manifest_sha256", "outcome_sha256"): + if not HEX64.fullmatch(str(result.get(field))): + raise ReturnError("worker result {} is malformed".format(field)) + for field in ("repository_generation", "return_commit", "outcome_tip"): + if not HEX40.fullmatch(str(result.get(field))): + raise ReturnError("worker result {} is malformed".format(field)) + commits = result.get("outcome_commits") + if not isinstance(commits, int) or isinstance(commits, bool) or commits < 0: + raise ReturnError("worker result outcome commit count is malformed") + outcome_bytes = result.get("outcome_bytes") + if not isinstance(outcome_bytes, int) or isinstance(outcome_bytes, bool) or outcome_bytes <= 0: + raise ReturnError("worker result outcome byte count is malformed") + if result.get("outcome_present") is not (commits > 0): + raise ReturnError("worker result outcome presence differs from its commit count") + if not isinstance(result.get("outcome_uncommitted_changes"), bool): + raise ReturnError("worker result working-tree disposition is malformed") + if not isinstance(result.get("exit_code"), int) or isinstance(result.get("exit_code"), bool): + raise ReturnError("worker result exit code is malformed") + if not isinstance(result.get("timed_out"), bool): + raise ReturnError("worker result timeout disposition is malformed") + expected_return_ref = "refs/fm-return/{}".format(result["request_digest"][:32]) + if result.get("return_ref") != expected_return_ref: + raise ReturnError("worker result return ref is not exact") + return result + + +def git(worktree, *arguments, input_bytes=None, check=True): + completed = subprocess.run( + ["git", "-C", str(worktree), *arguments], input=input_bytes, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, + ) + if check and completed.returncode != 0: + raise ReturnError( + "git {} failed: {}".format( + arguments[0] if arguments else "command", + completed.stderr.decode("utf-8", errors="replace").strip()[-500:], + ) + ) + return completed + + +def meta_values(path): + body = read_regular(path, "task metadata", 1024 * 1024) + try: + lines = body.decode("utf-8").splitlines() + except UnicodeDecodeError as exc: + raise ReturnError("task metadata is not UTF-8: {}".format(exc)) + values = {} + for line in lines: + if "=" in line: + key, value = line.split("=", 1) + values.setdefault(key, []).append(value) + return values + + +def exactly(values, key): + found = values.get(key, []) + if len(found) != 1 or not found[0]: + raise ReturnError("task metadata {} is not exact".format(key)) + return found[0] + + +def fetch_return_refs(worktree, bundle, result, task, generation): + body = read_regular(bundle, "worker return bundle", MAX_BUNDLE_BYTES) + if len(body) != result.get("outcome_bytes") or sha256(body) != result.get("outcome_sha256"): + raise ReturnError("worker return bundle differs from the digest-bound result") + git(worktree, "bundle", "verify", str(bundle)) + listed = git(worktree, "bundle", "list-heads", str(bundle)).stdout.decode().splitlines() + heads = {} + for line in listed: + parts = line.split(" ", 1) + if len(parts) == 2: + heads[parts[1]] = parts[0] + return_ref = result.get("return_ref") + if heads.get(return_ref) != result["return_commit"]: + raise ReturnError("worker return ref does not bind the declared artifact commit") + namespace = "refs/fm-cloud-return/{}/{}".format(task, result["request_digest"][:32]) + artifact_ref = namespace + "/artifacts" + git(worktree, "fetch", "--quiet", "--no-tags", str(bundle), "+{}:{}".format(return_ref, artifact_ref)) + outcome_ref = "refs/fm-outcome/{}".format(result["request_digest"][:32]) + outcome_custody_ref = namespace + "/outcome" + if result.get("outcome_commits", 0): + if heads.get(outcome_ref) != result["outcome_tip"]: + raise ReturnError("worker outcome ref does not bind the declared outcome tip") + git(worktree, "fetch", "--quiet", "--no-tags", str(bundle), "+{}:{}".format(outcome_ref, outcome_custody_ref)) + return artifact_ref, outcome_custody_ref + + +def object_bytes(worktree, commit, name, limit=MAX_ARTIFACT_BYTES): + shown = git(worktree, "show", "{}:{}".format(commit, name), check=False) + if shown.returncode != 0: + raise ReturnError("worker return bundle lacks authorized artifact {}".format(name)) + if len(shown.stdout) > limit: + raise ReturnError("worker return artifact {} exceeds its byte bound".format(name)) + return shown.stdout + + +def read_manifest(worktree, result, task, generation, assignment): + body = object_bytes(worktree, result["return_commit"], "manifest.json", 1024 * 1024) + if sha256(body) != result["return_manifest_sha256"]: + raise ReturnError("worker return manifest differs from the digest-bound result") + try: + manifest = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ReturnError("worker return manifest is truncated or corrupt: {}".format(exc)) + if not isinstance(manifest, dict) or manifest.get("schema") != RETURN_SCHEMA: + raise ReturnError("worker return manifest schema is not supported") + expected = { + "task": task, + "task_generation": generation, + "assignment_generation": assignment, + "request_digest": result["request_digest"], + "repository_generation": result["repository_generation"], + "outcome_commits": result.get("outcome_commits", 0), + "outcome_tip": result["outcome_tip"], + } + for field, value in expected.items(): + if manifest.get(field) != value: + raise ReturnError("worker return manifest {} binding differs".format(field)) + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, dict) or any(not isinstance(item, dict) for item in artifacts.values()): + raise ReturnError("worker return manifest artifacts are malformed") + allowed = {"report.md", "status.log", "visuals.tar", "scratch.patch", "scratch-untracked.tar"} + if not set(artifacts).issubset(allowed): + raise ReturnError("worker return manifest names an unauthorized artifact") + bodies = {} + for name, descriptor in artifacts.items(): + body = object_bytes(worktree, result["return_commit"], name) + if descriptor.get("bytes") != len(body) or descriptor.get("sha256") != sha256(body): + raise ReturnError("worker return artifact {} is truncated or corrupt".format(name)) + bodies[name] = body + return manifest, bodies + + +def substantive_report(body): + try: + text = body.decode("utf-8") + except UnicodeDecodeError: + return False, "report is not UTF-8" + sections = {name: [] for name in REQUIRED_SECTIONS} + seen = [] + current = None + fenced = False + for raw in text.splitlines(): + stripped = raw.strip() + if stripped.startswith("```") or stripped.startswith("~~~"): + fenced = not fenced + if current: + sections[current].append("") + continue + if not fenced and stripped.startswith("## ") and not stripped.startswith("### "): + heading = stripped[3:] + if heading in sections: + expected_index = len(seen) + if expected_index >= len(REQUIRED_SECTIONS) or heading != REQUIRED_SECTIONS[expected_index]: + return False, "required report sections are duplicated or out of order" + seen.append(heading) + current = heading + else: + current = None + continue + if current is not None: + sections[current].append(raw) + missing = [name for name in REQUIRED_SECTIONS if name not in seen] + empty = [] + for name, lines in sections.items(): + if not any( + any(character.isalnum() or unicodedata.category(character).startswith("S") for character in line) + for line in lines + ): + empty.append(name) + if missing or empty: + return False, "missing={} empty={}".format(",".join(missing) or "none", ",".join(empty) or "none") + return True, "" + + +def atomic_write(path, body): + if path.parent.is_symlink() or not path.parent.is_dir(): + raise ReturnError("local artifact parent is redirected: {}".format(path.parent)) + if path.exists(): + if path.is_symlink() or not path.is_file(): + raise ReturnError("local artifact destination is redirected: {}".format(path)) + if path.read_bytes() == body: + return + raise ReturnError("local artifact destination diverged: {}".format(path)) + descriptor, temporary = tempfile.mkstemp(prefix=".fm-cloud-return-", dir=str(path.parent)) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "wb") as handle: + handle.write(body) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + finally: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + + +def check_directory(path): + if path.is_symlink() or (path.exists() and not path.is_dir()): + raise ReturnError("local artifact directory is redirected: {}".format(path)) + + +def ensure_directory(path): + check_directory(path) + if not path.exists(): + os.mkdir(str(path), 0o700) + check_directory(path) + + +def physical_state_directory(argument): + state = Path(os.path.abspath(str(argument))) + if state.name != "state": + raise ReturnError("task state directory is unavailable") + current = Path(state.anchor) + for part in state.parts[1:]: + current = current / part + if current.is_symlink() or not current.is_dir(): + raise ReturnError("task state directory is redirected or unavailable") + if state.resolve() != state or state.parent.resolve() != state.parent: + raise ReturnError("task state directory is redirected or unavailable") + return state + + +def read_visuals(body): + total = 0 + count = 0 + entries = [] + try: + with tarfile.open(fileobj=io.BytesIO(body), mode="r:") as archive: + for member in archive.getmembers(): + count += 1 + if count > MAX_VISUAL_ENTRIES or not member.isreg(): + raise ReturnError("worker visual artifact archive is unsafe") + relative = Path(member.name) + if relative.is_absolute() or ".." in relative.parts or not relative.parts: + raise ReturnError("worker visual artifact path is unsafe") + content = archive.extractfile(member).read() + total += len(content) + if total > MAX_VISUAL_BYTES: + raise ReturnError("worker visual artifacts exceed their byte bound") + # The archive path includes data//visuals. Keep only the + # part beneath visuals at the authorized local destination. + try: + visual_index = relative.parts.index("visuals") + except ValueError: + raise ReturnError("worker visual artifact is outside the authorized visual root") + target_relative = Path(*relative.parts[visual_index + 1:]) + if not target_relative.parts: + raise ReturnError("worker visual artifact has no file name") + entries.append((target_relative, content)) + except tarfile.TarError as exc: + raise ReturnError("worker visual artifact archive is corrupt: {}".format(exc)) + return entries + + +def check_visual_directories(destination, entries): + check_directory(destination) + if not destination.exists(): + return + for relative, _content in entries: + current = destination + for part in relative.parent.parts: + current = current / part + check_directory(current) + if not current.exists(): + break + + +def extract_visuals(entries, destination): + ensure_directory(destination) + for relative, content in entries: + current = destination + for part in relative.parent.parts: + current = current / part + ensure_directory(current) + atomic_write(destination / relative, content) + + +def branch_custody(worktree, task, result, kind): + base = result["repository_generation"] + tip = result["outcome_tip"] + commits = result["outcome_commits"] + if git(worktree, "merge-base", "--is-ancestor", base, tip, check=False).returncode != 0: + raise ReturnError("returned outcome tip does not descend from the dispatched generation") + counted = git(worktree, "rev-list", "--count", "{}..{}".format(base, tip)) + if int(counted.stdout.decode().strip()) != commits: + raise ReturnError("returned outcome commit count differs from its Git history") + if kind == "scout": + if commits: + raise ReturnError("a scout returned project commits; they remain in the custody ref") + return + if commits <= 0: + return + branch = "refs/heads/fm/{}".format(task) + existing = git(worktree, "rev-parse", "--verify", branch, check=False) + if existing.returncode == 0: + branch_head = existing.stdout.decode().strip() + if branch_head not in (base, tip) and git( + worktree, "merge-base", "--is-ancestor", tip, branch_head, check=False, + ).returncode != 0: + raise ReturnError("local task branch diverged from the returned outcome") + else: + branch_head = None + head = git(worktree, "rev-parse", "HEAD").stdout.decode().strip() + if head not in (base, tip, branch_head): + raise ReturnError("local worktree diverged from the dispatched generation") + if git(worktree, "status", "--porcelain=v1", "--untracked-files=all").stdout.strip(): + raise ReturnError("local worktree is dirty; returned work remains in the custody ref") + if branch_head is None: + created = git(worktree, "update-ref", branch, tip, "0" * 40, check=False) + if created.returncode != 0: + raise ReturnError("returned task branch could not be created at the exact outcome tip") + branch_head = tip + switched = git(worktree, "checkout", "--quiet", "fm/{}".format(task), check=False) + if switched.returncode != 0: + raise ReturnError("returned task branch exists but cannot be checked out in its worktree") + checked_out_head = git(worktree, "rev-parse", "HEAD").stdout.decode().strip() + if checked_out_head != branch_head: + raise ReturnError("returned task branch moved while it was being checked out") + advanced = git(worktree, "merge", "--quiet", "--ff-only", tip, check=False) + if advanced.returncode != 0: + raise ReturnError("returned task branch could not fast-forward to the exact outcome tip") + final_branch = git(worktree, "symbolic-ref", "--quiet", "HEAD", check=False) + if final_branch.returncode != 0 or final_branch.stdout.decode().strip() != branch: + raise ReturnError("returned task branch is not checked out in its worktree") + final_head = git(worktree, "rev-parse", "HEAD").stdout.decode().strip() + if git(worktree, "merge-base", "--is-ancestor", tip, final_head, check=False).returncode != 0: + raise ReturnError("returned outcome is not reachable from the task branch") + if git(worktree, "status", "--porcelain=v1", "--untracked-files=all").stdout.strip(): + raise ReturnError("returned task branch was not cleanly materialized in its worktree") + + +def merge_status(local_path, raw_status, terminal): + existing = "" + if local_path.exists(): + if local_path.is_symlink() or not local_path.is_file(): + raise ReturnError("local status trail is redirected") + existing = local_path.read_text(encoding="utf-8") + merged = [line for line in existing.splitlines() if line != terminal] + if raw_status: + try: + remote_lines = raw_status.decode("utf-8").splitlines() + except UnicodeDecodeError as exc: + raise ReturnError("returned status trail is not UTF-8: {}".format(exc)) + for line in remote_lines: + line = line.strip() + if not line: + continue + if not STATUS_LINE.fullmatch(line): + raise ReturnError("returned status trail contains a malformed event") + if TERMINAL_LINE.match(line): + continue + if line not in merged: + merged.append(line) + merged.append(terminal) + body = "".join(line + "\n" for line in merged) + if body == existing: + return + descriptor, temporary = tempfile.mkstemp(prefix=".fm-cloud-status-", dir=str(local_path.parent)) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(body) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, local_path) + finally: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + + +def collect(args): + if not SAFE_TASK.fullmatch(args.task): + raise ReturnError("task identity is malformed") + if not SAFE_GENERATION.fullmatch(args.task_generation): + raise ReturnError("task generation identity is malformed") + if not SAFE_GENERATION.fullmatch(args.assignment_generation): + raise ReturnError("assignment identity is malformed") + state = physical_state_directory(args.state) + home = state.parent + values = meta_values(state / (args.task + ".meta")) + if exactly(values, "generation_id") != args.task_generation: + raise ReturnError("task metadata generation differs") + kind = exactly(values, "kind") + if kind not in ("ship", "scout"): + raise ReturnError("only ship and scout returns are supported") + placement = exactly(values, "placement") + if placement != "azure": + raise ReturnError("task is not an Azure placement") + worktree_text = read_regular(state / (args.task + ".cloud-worktree"), "cloud worktree pointer", 64 * 1024) + try: + worktree = Path(worktree_text.decode("utf-8").strip()).resolve() + except UnicodeDecodeError as exc: + raise ReturnError("cloud worktree pointer is not UTF-8: {}".format(exc)) + recorded_worktree = Path(exactly(values, "worktree")).resolve() + if worktree != recorded_worktree: + raise ReturnError("cloud worktree pointer differs from task metadata") + if not worktree.is_dir() or Path(git(worktree, "rev-parse", "--show-toplevel").stdout.decode().strip()).resolve() != worktree: + raise ReturnError("cloud worktree pointer does not name the exact repository root") + result = read_result( + state / (args.task + ".worker-result.json"), args.task, + args.task_generation, args.assignment_generation, + ) + bundle = state / (args.task + ".cloud-outcome") / "outcome.bundle" + fetch_return_refs(worktree, bundle, result, args.task, args.task_generation) + manifest, artifacts = read_manifest( + worktree, result, args.task, args.task_generation, args.assignment_generation, + ) + if manifest.get("kind") != kind or manifest.get("report_required") is not True: + raise ReturnError("worker return task contract differs from local task metadata") + expected_report = "data/{}/{}".format(args.task, "completion.md" if kind == "ship" else "report.md") + expected_status = "state/{}.status".format(args.task) + if manifest.get("report_path") != expected_report or manifest.get("status_path") != expected_status: + raise ReturnError("worker return authorized paths differ from the local task contract") + data_root = home / "data" + data_dir = data_root / args.task + check_directory(data_root) + check_directory(data_dir) + visual_entries = None + if "visuals.tar" in artifacts: + visual_entries = read_visuals(artifacts["visuals.tar"]) + check_visual_directories(data_dir / "visuals", visual_entries) + ensure_directory(data_root) + ensure_directory(data_dir) + report = artifacts.get("report.md") + report_valid = False + report_reason = "worker returned no report" + if report is not None: + report_valid, report_reason = substantive_report(report) + if not report_valid: + atomic_write(data_dir / "cloud-return-report.invalid.md", report) + if not report_valid: + raise ReturnError("required worker report was absent or invalid: {}".format(report_reason)) + report_target = data_dir / ("completion.md" if kind == "ship" else "report.md") + if report_target.exists(): + if report_target.is_symlink() or not report_target.is_file(): + raise ReturnError("local report destination is redirected") + local_report = report_target.read_bytes() + local_valid, _local_reason = substantive_report(local_report) + if not local_valid: + raise ReturnError("local report destination diverged with an invalid report") + report = local_report + else: + atomic_write(report_target, report) + atomic_write(data_dir / "cloud-return.json", canonical(manifest) + b"\n") + if "status.log" in artifacts: + atomic_write(data_dir / "cloud-status.log", artifacts["status.log"]) + if "scratch.patch" in artifacts: + atomic_write(data_dir / "cloud-scratch.patch", artifacts["scratch.patch"]) + if "scratch-untracked.tar" in artifacts: + atomic_write(data_dir / "cloud-scratch-untracked.tar", artifacts["scratch-untracked.tar"]) + if visual_entries is not None: + extract_visuals(visual_entries, data_dir / "visuals") + branch_custody(worktree, args.task, result, kind) + succeeded = ( + result.get("exit_code") == 0 and result.get("timed_out") is False + and not result.get("outcome_error") + and (kind == "scout" or not result.get("outcome_uncommitted_changes")) + and (kind != "ship" or int(result.get("outcome_commits", 0)) > 0) + ) + if succeeded: + terminal = "done: cloud outcome returned to local custody" + else: + reasons = [] + if result.get("timed_out"): + reasons.append("worker timed out") + elif result.get("exit_code") != 0: + reasons.append("worker exited {}".format(result.get("exit_code"))) + if result.get("outcome_error"): + reasons.append("return collection reported an error") + if kind == "ship" and result.get("outcome_uncommitted_changes"): + reasons.append("uncommitted scratch was retained") + if kind == "ship" and int(result.get("outcome_commits", 0)) <= 0: + reasons.append("ship returned no commits") + terminal = "failed: {}".format("; ".join(reasons) or "cloud outcome was incomplete") + merge_status(state / (args.task + ".status"), artifacts.get("status.log"), terminal) + print(terminal) + + +def parser(): + value = argparse.ArgumentParser() + sub = value.add_subparsers(dest="command", required=True) + collect_parser = sub.add_parser("collect") + collect_parser.add_argument("--state", required=True) + collect_parser.add_argument("--task", required=True) + collect_parser.add_argument("--task-generation", required=True) + collect_parser.add_argument("--assignment-generation", required=True) + return value + + +def main(argv=None): + args = parser().parse_args(argv) + if args.command == "collect": + collect(args) + return 0 + raise ReturnError("unknown command") + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (ReturnError, OSError, ValueError) as exc: + print("CLOUD RETURN REFUSED: {}".format(exc), file=sys.stderr) + raise SystemExit(2) diff --git a/bin/fm-cloud-state-lib.sh b/bin/fm-cloud-state-lib.sh index 1c06f00c4d3..87b61460fba 100755 --- a/bin/fm-cloud-state-lib.sh +++ b/bin/fm-cloud-state-lib.sh @@ -58,7 +58,8 @@ fm_cloud_state_dispatch_paths() { # # returned artifacts. fm_cloud_state_result_paths() { # printf '%s\n' "$1/$2.worker-request.out" "$1/$2.worker-result.json" \ - "$1/$2.worker-execute.log" "$1/$2.worker-reconcile.json" + "$1/$2.worker-execute.log" "$1/$2.worker-reconcile.json" \ + "$1/$2.worker-release.json" } # The leased local worktree pointer. It used to be excluded from the task-end diff --git a/bin/fm-credential-expiry.py b/bin/fm-credential-expiry.py index 66db9d27422..6cb1662089d 100755 --- a/bin/fm-credential-expiry.py +++ b/bin/fm-credential-expiry.py @@ -239,6 +239,32 @@ def _claude_facts(value: dict[str, Any]) -> dict[str, Any]: _FACT_READERS = {"codex": _codex_facts, "pi": _pi_facts, "claude": _claude_facts} +def credential_usable_through( + value: dict[str, Any], + *, + harness: str, + deadline: float, +) -> bool: + """Whether one already-read credential remains usable past a deadline. + + Stagers that select one entry from a pooled credential call this before + writing a snapshot. Credential interpretation stays here with the expiry + owner, while token material stays in the caller's already-private memory. + """ + + reader = _FACT_READERS.get(harness) + if reader is None or not isinstance(value, dict): + return False + facts = reader(value) + if facts["never_expires"]: + return True + return bool( + facts["has_access"] + and facts["access_expires_at"] is not None + and facts["access_expires_at"] > float(deadline) + ) + + def inspect_profile( profile: str | os.PathLike[str], *, @@ -302,10 +328,8 @@ def inspect_profile( return record deadline = moment + max(0.0, float(margin_seconds)) - access_live = ( - facts["has_access"] - and facts["access_expires_at"] is not None - and facts["access_expires_at"] > deadline + access_live = credential_usable_through( + value, harness=resolved_harness, deadline=deadline ) if access_live: record["state"] = "usable" diff --git a/bin/fm-no-mistakes-runtime b/bin/fm-no-mistakes-runtime new file mode 100755 index 00000000000..151aed5071c --- /dev/null +++ b/bin/fm-no-mistakes-runtime @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Build the credential-free Pi runtime used by the Azure no-mistakes worker. +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) +exec python3 "$SCRIPT_DIR/fm-no-mistakes-runtime.py" "$@" diff --git a/bin/fm-no-mistakes-runtime.py b/bin/fm-no-mistakes-runtime.py new file mode 100755 index 00000000000..436bc243416 --- /dev/null +++ b/bin/fm-no-mistakes-runtime.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 +"""Build and verify the sealed Pi runtime for Azure no-mistakes workers.""" + +import argparse +import gzip +import hashlib +import io +import json +import os +from pathlib import Path +import re +import stat +import subprocess +import tarfile +import tempfile + + +SCHEMA = "fm.azure-validation-runtime/v1" +PROVIDER = "pi" +NM_VERSION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+-]{0,127}$") +HEX40 = re.compile(r"^[0-9a-f]{40}$") +PACKAGE_VERSION = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(?:[-+][A-Za-z0-9.-]+)?$") +MAX_FILES = 20000 +MAX_FILE_BYTES = 512 * 1024 * 1024 +MAX_TOTAL_BYTES = 2 * 1024 * 1024 * 1024 +MANIFEST_FIELDS = { + "schema", "provider", "no_mistakes_version", "no_mistakes_source_commit", + "owner_decision_protocol", "no_mistakes_path", "provider_path", "gh_path", + "node_path", "gh_axi_path", "gh_axi_entrypoint", "gh_axi_closure", "files", +} +DENIED_BASENAMES = { + ".env", ".netrc", ".npmrc", "auth.json", "credentials.json", + "credentials", "id_rsa", "id_ed25519", +} + +PI_WRAPPER = b'''#!/bin/sh +set -eu +runtime_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd -P) +agent_dir=${PI_CODING_AGENT_DIR:-"$HOME/pi-agent"} +export PI_CODING_AGENT_DIR=$agent_dir +config_dir=$agent_dir/extensions/pi-openai-fast-mode +mkdir -p "$config_dir" +cp "$runtime_root/config/pi-openai-fast-mode.json" "$config_dir/config.json" +exec "$runtime_root/bin/node" "$runtime_root/lib/pi/dist/cli.js" \ + --no-extensions \ + --extension "$runtime_root/extensions/pi-openai-fast-mode/src/index.ts" \ + --extension "$runtime_root/extensions/fast-mode-all-codex-accounts.ts" \ + --extension "$runtime_root/extensions/pi-ketch/src/index.ts" "$@" +''' + +FAST_CONFIG = b'''{ + "enabled": true, + "targets": [ + {"provider":"openai-codex","model":"gpt-5.4","serviceTier":"priority"}, + {"provider":"openai-codex","model":"gpt-5.5","serviceTier":"priority"}, + {"provider":"openai-codex","model":"gpt-5.6","serviceTier":"priority"}, + {"provider":"openai-codex","model":"gpt-5.6-sol","serviceTier":"priority"}, + {"provider":"openai-codex","model":"gpt-5.6-terra","serviceTier":"priority"}, + {"provider":"openai-codex","model":"gpt-5.6-luna","serviceTier":"priority"} + ] +} +''' + + +class RuntimeError(ValueError): + pass + + +def digest(body): + return "sha256:" + hashlib.sha256(body).hexdigest() + + +def linux_amd64_elf(body): + return ( + len(body) >= 20 and body[:4] == b"\x7fELF" and body[4] == 2 + and body[5] == 1 and int.from_bytes(body[18:20], "little") == 62 + ) + + +def safe_relative(value): + parts = Path(value).parts + return bool(parts) and not value.startswith("/") and all( + part not in ("", ".", "..") for part in parts + ) + + +def read_regular(path, label, executable=False, linux=False, enforce_linux=True): + path = Path(path).expanduser() + try: + observed = os.lstat(path) + except OSError as exc: + raise RuntimeError("{} is unreadable: {}".format(label, exc)) + if not stat.S_ISREG(observed.st_mode) or observed.st_nlink != 1: + raise RuntimeError("{} must be one regular, unlinked file".format(label)) + if executable and observed.st_mode & 0o111 == 0: + raise RuntimeError("{} must be executable".format(label)) + if observed.st_size > MAX_FILE_BYTES: + raise RuntimeError("{} exceeds 512 MiB".format(label)) + body = path.read_bytes() + if len(body) != observed.st_size or os.lstat(path)[:4] != observed[:4]: + raise RuntimeError("{} changed while it was read".format(label)) + if linux and enforce_linux and not linux_amd64_elf(body[:20]): + raise RuntimeError("{} must be a Linux amd64 ELF artifact".format(label)) + return body + + +def record(name, body, mode): + if not safe_relative(name): + raise RuntimeError("runtime member path is unsafe: {}".format(name)) + if Path(name).name.lower() in DENIED_BASENAMES: + raise RuntimeError("runtime member path is credential-like: {}".format(name)) + return {"name": name, "body": body, "mode": mode, "digest": digest(body)} + + +def package_records(root, expected_name, destination, label, enforce_linux=True): + root = Path(root).expanduser() + if root.is_symlink() or not root.is_dir(): + raise RuntimeError("{} must be one package directory".format(label)) + package_body = read_regular(root / "package.json", label + " package.json") + try: + metadata = json.loads(package_body) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("{} package.json is malformed: {}".format(label, exc)) + if ( + not isinstance(metadata, dict) or metadata.get("name") != expected_name + or not PACKAGE_VERSION.fullmatch(str(metadata.get("version", ""))) + ): + raise RuntimeError("{} package identity/version is not exact".format(label)) + output = [] + for directory, directories, files in os.walk(root, followlinks=False): + directories.sort() + files.sort() + base = Path(directory) + for name in list(directories): + child = base / name + if child.is_symlink(): + if name == ".bin": + directories.remove(name) + continue + raise RuntimeError("{} contains a redirected directory".format(label)) + for name in files: + source = base / name + relative = source.relative_to(root).as_posix() + if source.is_symlink(): + if "/.bin/" in "/{}/".format(relative): + continue + raise RuntimeError("{} contains a redirected file".format(label)) + body = read_regular(source, "{} {}".format(label, relative)) + if enforce_linux and source.suffix == ".node" and not linux_amd64_elf(body[:20]): + raise RuntimeError("{} native module is not Linux amd64: {}".format(label, relative)) + output.append(record(destination + "/" + relative, body, 0o644)) + return output, metadata["version"] + + +def build(args, enforce_linux=True): + if not NM_VERSION.fullmatch(args.no_mistakes_version): + raise RuntimeError("--no-mistakes-version is malformed") + if not HEX40.fullmatch(args.no_mistakes_source_commit): + raise RuntimeError("--no-mistakes-source-commit must be exact 40-hex") + output = Path(args.output).expanduser().absolute() + if not output.parent.is_dir() or output.exists() or output.is_symlink(): + raise RuntimeError("output parent must exist and output must be absent") + records = [ + record("bin/no-mistakes", read_regular( + args.no_mistakes, "no-mistakes", True, True, enforce_linux), 0o755), + record("bin/node", read_regular( + args.node, "Node", True, True, enforce_linux), 0o755), + record("bin/pi", PI_WRAPPER, 0o755), + record("config/pi-openai-fast-mode.json", FAST_CONFIG, 0o644), + record("extensions/fast-mode-all-codex-accounts.ts", read_regular( + args.fast_mode_fleet_extension, "fleet fast-mode extension"), 0o644), + ] + pi_records, pi_version = package_records( + args.pi_package, "@earendil-works/pi-coding-agent", "lib/pi", "Pi", enforce_linux) + fast_records, fast_version = package_records( + args.fast_mode_package, "pi-openai-fast-mode", + "extensions/pi-openai-fast-mode", "Pi fast mode", enforce_linux) + ketch_records, ketch_version = package_records( + args.ketch_package, "pi-ketch", "extensions/pi-ketch", "Pi Ketch", enforce_linux) + records.extend(pi_records + fast_records + ketch_records) + names = [item["name"] for item in records] + if len(names) > MAX_FILES or len(names) != len(set(names)): + raise RuntimeError("runtime file inventory is duplicated or unbounded") + if "lib/pi/dist/cli.js" not in names: + raise RuntimeError("Pi package lacks dist/cli.js") + for required in ( + "extensions/pi-openai-fast-mode/src/index.ts", + "extensions/pi-ketch/src/index.ts", + ): + if required not in names: + raise RuntimeError("extension package lacks {}".format(required)) + records.sort(key=lambda item: item["name"]) + manifest = { + "schema": SCHEMA, + "provider": PROVIDER, + "no_mistakes_version": args.no_mistakes_version, + "no_mistakes_source_commit": args.no_mistakes_source_commit, + "owner_decision_protocol": "fm.azure-validation-owner-decision/v1", + "no_mistakes_path": "bin/no-mistakes", + "provider_path": "bin/pi", + "gh_path": "", + "node_path": "bin/node", + "gh_axi_path": "", + "gh_axi_entrypoint": "", + "gh_axi_closure": [], + "files": [{"path": item["name"], "digest": item["digest"]} for item in records], + } + manifest_body = json.dumps(manifest, sort_keys=True, indent=1).encode() + b"\n" + if len(records) + 1 > MAX_FILES or sum(len(item["body"]) for item in records) > MAX_TOTAL_BYTES: + raise RuntimeError("runtime exceeds its bounded inventory") + fd, temporary_name = tempfile.mkstemp(prefix="." + output.name + ".", dir=output.parent) + temporary = Path(temporary_name) + try: + with os.fdopen(fd, "wb") as raw: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw, compresslevel=9, mtime=0) as zipped: + with tarfile.open(fileobj=zipped, mode="w:", format=tarfile.PAX_FORMAT) as archive: + for name, body, mode in [("runtime.json", manifest_body, 0o644)] + [ + (item["name"], item["body"], item["mode"]) for item in records + ]: + info = tarfile.TarInfo(name) + info.size = len(body) + info.mode = mode + info.uid = info.gid = 0 + info.uname = info.gname = "root" + info.mtime = 0 + archive.addfile(info, io.BytesIO(body)) + raw.flush() + os.fsync(raw.fileno()) + verify(temporary, enforce_linux) + smoke_pi_runtime(temporary) + os.link(temporary, output, follow_symlinks=False) + finally: + temporary.unlink(missing_ok=True) + bundle_digest = hashlib.sha256(output.read_bytes()).hexdigest() + print("NO-MISTAKES PI RUNTIME BUILT output={} sha256={} pi={} fast={} ketch={}".format( + output, bundle_digest, pi_version, fast_version, ketch_version)) + return manifest + + +def verify(path, enforce_linux=True): + source = Path(path) + if source.is_symlink() or not source.is_file() or source.stat().st_size > 1024**3: + raise RuntimeError("runtime archive is absent, redirected, or oversized") + try: + with tarfile.open(source, "r:gz") as archive: + members = archive.getmembers() + if not members or len(members) > MAX_FILES: + raise RuntimeError("runtime member inventory is empty or unbounded") + bodies = {} + modes = {} + for member in members: + if not member.isreg() or not safe_relative(member.name) or member.name in bodies: + raise RuntimeError("runtime contains unsafe or duplicate members") + handle = archive.extractfile(member) + body = handle.read() if handle else b"" + bodies[member.name] = body + modes[member.name] = member.mode + except (OSError, tarfile.TarError) as exc: + raise RuntimeError("runtime archive is unreadable: {}".format(exc)) + try: + manifest = json.loads(bodies.pop("runtime.json")) + except (KeyError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("runtime manifest is unreadable: {}".format(exc)) + if ( + not isinstance(manifest, dict) or set(manifest) != MANIFEST_FIELDS + or manifest.get("schema") != SCHEMA or manifest.get("provider") != PROVIDER + or not NM_VERSION.fullmatch(str(manifest.get("no_mistakes_version", ""))) + or not HEX40.fullmatch(str(manifest.get("no_mistakes_source_commit", ""))) + or manifest.get("owner_decision_protocol") != "fm.azure-validation-owner-decision/v1" + or manifest.get("provider_path") != "bin/pi" + or manifest.get("node_path") != "bin/node" + or manifest.get("no_mistakes_path") != "bin/no-mistakes" + or manifest.get("gh_path") != "" or manifest.get("gh_axi_path") != "" + or manifest.get("gh_axi_entrypoint") != "" or manifest.get("gh_axi_closure") != [] + ): + raise RuntimeError("runtime manifest identity is not the exact Pi worker schema") + declared = manifest.get("files") + if not isinstance(declared, list) or any( + not isinstance(item, dict) or set(item) != {"path", "digest"} + or not isinstance(item.get("path"), str) or not safe_relative(item["path"]) + or not isinstance(item.get("digest"), str) + or not re.fullmatch(r"sha256:[0-9a-f]{64}", item["digest"]) + for item in declared + ): + raise RuntimeError("runtime file inventory is malformed") + expected = {item["path"]: item["digest"] for item in declared} + if len(expected) != len(declared) or set(expected) != set(bodies): + raise RuntimeError("runtime manifest does not inventory every byte") + if any(expected[name] != digest(body) for name, body in bodies.items()): + raise RuntimeError("runtime member digest differs") + if any(Path(name).name.lower() in DENIED_BASENAMES for name in bodies): + raise RuntimeError("runtime contains a credential-like path") + if any(modes.get(name, 0) & 0o111 == 0 for name in ("bin/no-mistakes", "bin/node", "bin/pi")): + raise RuntimeError("runtime executable is not executable") + if enforce_linux and any(not linux_amd64_elf(bodies[name][:20]) for name in ("bin/no-mistakes", "bin/node")): + raise RuntimeError("runtime executable is not Linux amd64") + return manifest + + +def smoke_pi_runtime(path): + """Start the staged Pi CLI under the staged Node without account material.""" + with tempfile.TemporaryDirectory(prefix="fm-pi-runtime-smoke.") as directory: + root = Path(directory) / "runtime" + home = Path(directory) / "home" + root.mkdir(mode=0o700) + home.mkdir(mode=0o700) + with tarfile.open(path, "r:gz") as archive: + for member in archive.getmembers(): + if member.name == "runtime.json": + continue + handle = archive.extractfile(member) + body = handle.read() if handle else b"" + target = root.joinpath(*Path(member.name).parts) + target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + target.write_bytes(body) + target.chmod(member.mode) + environment = { + "HOME": str(home), + "PI_CODING_AGENT_DIR": str(home / "pi-agent"), + "PATH": str(root / "bin") + ":/usr/bin:/bin", + "LANG": "C.UTF-8", + "LC_ALL": "C.UTF-8", + } + try: + completed = subprocess.run( + [str(root / "bin/pi"), "--version"], env=environment, + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=30, check=False, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise RuntimeError("staged Pi CLI did not start under bundled Node: {}".format(exc)) + if completed.returncode != 0: + detail = completed.stderr.decode("utf-8", errors="replace")[-500:].strip() + raise RuntimeError( + "staged Pi CLI did not start under bundled Node: exit={} {}".format( + completed.returncode, detail)) + + +def parser(): + item = argparse.ArgumentParser(prog="fm-no-mistakes-runtime") + item.add_argument("--no-mistakes", required=True) + item.add_argument("--node", required=True) + item.add_argument("--pi-package", required=True) + item.add_argument("--fast-mode-package", required=True) + item.add_argument("--fast-mode-fleet-extension", required=True) + item.add_argument("--ketch-package", required=True) + item.add_argument("--no-mistakes-version", required=True) + item.add_argument("--no-mistakes-source-commit", required=True) + item.add_argument("--output", required=True) + return item + + +def main(): + try: + build(parser().parse_args()) + except RuntimeError as exc: + raise SystemExit("NO-MISTAKES PI RUNTIME FAILED: {}".format(exc)) + + +if __name__ == "__main__": + main() diff --git a/bin/fm-no-mistakes-test-command.sh b/bin/fm-no-mistakes-test-command.sh index 06875d0c6bd..7fc70b99dd2 100755 --- a/bin/fm-no-mistakes-test-command.sh +++ b/bin/fm-no-mistakes-test-command.sh @@ -14,11 +14,27 @@ run_local_required() { tmux -V printf 'no-mistakes: local host set files=%s source=tests/test-capabilities.tsv; complete behavior inventory is required in CI\n' \ "${#herdr_tests[@]}" - local rc=0 - "$ROOT/tests/run.sh" "${herdr_tests[@]}" || rc=1 - uv run --directory "$ROOT/tools/agent-fleet" --locked pytest || rc=1 - uv run --directory "$ROOT/tools/agent-fleet" --locked python -m compileall -q src || rc=1 - return "$rc" + local lane_dir herdr_pid agent_fleet_pid herdr_rc=0 agent_fleet_rc=0 + lane_dir=$(mktemp -d "${TMPDIR:-/tmp}/fm-local-test-lanes.XXXXXX") || return 1 + "$ROOT/tests/run.sh" "${herdr_tests[@]}" >"$lane_dir/herdr.log" 2>&1 & + herdr_pid=$! + ( + rc=0 + uv run --directory "$ROOT/tools/agent-fleet" --locked pytest || rc=1 + uv run --directory "$ROOT/tools/agent-fleet" --locked python -m compileall -q src || rc=1 + exit "$rc" + ) >"$lane_dir/agent-fleet.log" 2>&1 & + agent_fleet_pid=$! + wait "$herdr_pid" || herdr_rc=$? + wait "$agent_fleet_pid" || agent_fleet_rc=$? + printf '%s\n' '== local Herdr lane ==' && cat "$lane_dir/herdr.log" + printf '%s\n' '== local Agent Fleet lane ==' && cat "$lane_dir/agent-fleet.log" + rm -rf "$lane_dir" + if [ "$herdr_rc" -ne 0 ] || [ "$agent_fleet_rc" -ne 0 ]; then + printf 'no-mistakes local test lanes failed: herdr=%s agent-fleet=%s\n' \ + "$herdr_rc" "$agent_fleet_rc" >&2 + return 1 + fi } # A daemon step inherits no FM_* selection variables from the operator. Ask the @@ -78,7 +94,9 @@ selection_binding=$(printf '%s\n' "$selection_output" | sed -n 's/^selection_bin # They run concurrently and report independently into this one command step. # shellcheck disable=SC2016 # The command expands its variables inside the Azure guest shell. "$DISPATCH" --require-selection-binding "$selection_binding" test -- \ - "$ROOT/bin/fm-azure-runner-command.sh" bash -c ' + bin/fm-azure-runner-command.sh env \ + FM_TEST_HOST_CAPABILITIES_ABSENT=real-tmux-server,passwordless-root-escalation,system-openat-binding,origin-egress \ + bash -c ' command -v tmux >/dev/null || { echo "tmux is required for e2e tests" >&2; exit 1; } tmux -V rc=0 diff --git a/bin/fm-no-mistakes-worker b/bin/fm-no-mistakes-worker new file mode 100755 index 00000000000..b52815fb172 --- /dev/null +++ b/bin/fm-no-mistakes-worker @@ -0,0 +1,706 @@ +#!/usr/bin/env python3 +"""Firstmate-owned Azure transport for one no-mistakes pipeline step.""" + +import argparse +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import time + + +CONFIG_SCHEMA = "fm.no-mistakes-worker-wrapper-config/v1" +REQUEST_SCHEMA = "no-mistakes.firstmate-worker-request/v1" +RESULT_SCHEMA = "no-mistakes.firstmate-worker-result/v1" +STEP_SCHEMA = "no-mistakes.worker-step-outcome/v1" +RETURN_SCHEMA = "fm.no-mistakes-worker-return/v1" +FIRSTMATE_RETURN = "fm.worker-return-contract/v1" +HEX40 = re.compile(r"^[0-9a-f]{40}$") +HEX64 = re.compile(r"^[0-9a-f]{64}$") +SAFE_TEXT = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$") +MAX_JSON = 1024 * 1024 +REQUEST_FIELDS = { + "schema", "job_id", "run_id", "step_result_id", "step", "kind", "round", + "desired_head_sha", "input_digest", "runtime_identity", "owner_decision_head", + "desired_generation", "attempt", "lease_fence", "lease_owner", "source_ref", + "source_bundle_sha256", "source_bundle_size", "guest_argv", + "expected_result_schema", "expected_firstmate_return", +} +CONFIG_FIELDS = { + "schema", "fm_home", "account_pool_home", "runtime_bundle", + "runtime_bundle_sha256", "lifecycle_path", "lifecycle_source_commit", "lifecycle_env", + "assignment_timeout_seconds", "cleanup_timeout_seconds", "poll_seconds", + "wall_seconds", +} +LIFECYCLE_ENV_KEYS = { + "FM_AZURE_SUBSCRIPTION_ID", "FM_AZURE_DEPLOYMENT_GENERATION", + "FM_AZURE_OWNER_TAG", "FM_AZURE_NAMING_PREFIX", "FM_AZURE_RESOURCE_GROUP", + "FM_AZURE_STORAGE_NAME", "FM_AZURE_LOCATION", "FM_AZURE_WORKER_STATE_DIR", + "FM_AZURE_WORKER_MAX", "FM_AZURE_WORKER_IDLE_COOLDOWN_SECONDS", + "FM_AZURE_WORKER_IDLE_RELEASE_SECONDS", "FM_AZURE_WORKER_POLICY_PHASE", + "FM_AZURE_WORKER_STEADY_TARGET_USD", "FM_AZURE_WORKER_COMMISSIONING_CEILING_USD", + "FM_AZURE_WORKER_ADMISSION_HOURS", "FM_AZURE_WORKER_HOUR_PLANNING_THRESHOLD", + "FM_AZURE_WORKER_DAILY_BOUND_USD", "FM_AZURE_WORKER_WARM_IDLE", + "FM_AZURE_SECONDMATE_MAX", "FM_SECONDMATE_CHILD_MAX", "FM_SECONDMATE_CHILD_TOTAL", + "FM_PI_ACCOUNT_HOME_ROOT", "AZURE_CONFIG_DIR", +} + + +class WrapperError(RuntimeError): + def __init__(self, message, category="wrapper_failure", retryable=True): + super().__init__(message) + self.category = category + self.retryable = retryable + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode() + + +def sha256_bytes(value): + return hashlib.sha256(value).hexdigest() + + +def sha256_file(path): + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def read_json(path, label, fields): + path = Path(path) + if path.is_symlink() or not path.is_file() or not 0 < path.stat().st_size <= MAX_JSON: + raise WrapperError("{} is absent, redirected, or oversized".format(label), "input_invalid", False) + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise WrapperError("{} is unreadable: {}".format(label, exc), "input_invalid", False) + if not isinstance(value, dict) or set(value) != fields: + raise WrapperError("{} fields are not the exact schema".format(label), "input_invalid", False) + return value + + +def validate_config(value): + if value["schema"] != CONFIG_SCHEMA: + raise WrapperError("wrapper config schema is unsupported", "config_invalid", False) + for field in ("fm_home", "account_pool_home", "runtime_bundle", "lifecycle_path"): + path = Path(value[field]) if isinstance(value[field], str) else Path("") + if not path.is_absolute() or str(path) != str(path.resolve()): + raise WrapperError("config {} must be one clean absolute path".format(field), "config_invalid", False) + if not HEX64.fullmatch(str(value["runtime_bundle_sha256"])): + raise WrapperError("runtime bundle digest is malformed", "config_invalid", False) + if not HEX40.fullmatch(str(value["lifecycle_source_commit"])): + raise WrapperError("lifecycle source commit is malformed", "config_invalid", False) + env = value["lifecycle_env"] + if not isinstance(env, dict) or not env or not set(env) <= LIFECYCLE_ENV_KEYS: + raise WrapperError("lifecycle environment contains unsupported fields", "config_invalid", False) + required = { + "FM_AZURE_SUBSCRIPTION_ID", "FM_AZURE_DEPLOYMENT_GENERATION", + "FM_AZURE_OWNER_TAG", "FM_AZURE_NAMING_PREFIX", "FM_AZURE_STORAGE_NAME", + } + if not required <= set(env) or any( + not isinstance(item, str) or not item or "\x00" in item for item in env.values() + ): + raise WrapperError("lifecycle environment is incomplete or malformed", "config_invalid", False) + for field, low, high in ( + ("assignment_timeout_seconds", 1, 7200), + ("cleanup_timeout_seconds", 1, 7200), + ("poll_seconds", 1, 60), + ("wall_seconds", 1, 21600), + ): + number = value[field] + if not isinstance(number, int) or isinstance(number, bool) or not low <= number <= high: + raise WrapperError("config {} is outside its bound".format(field), "config_invalid", False) + + +def validate_request(value): + if ( + value["schema"] != REQUEST_SCHEMA + or value["expected_result_schema"] != RESULT_SCHEMA + or value["expected_firstmate_return"] != FIRSTMATE_RETURN + or value["kind"] not in ("review", "repair", "test") + or value["step"] not in ("review", "test") + or (value["kind"] == "review" and value["step"] != "review") + or (value["kind"] == "test" and value["step"] != "test") + or value["source_ref"] != "HEAD" + or not HEX40.fullmatch(str(value["desired_head_sha"])) + or not HEX64.fullmatch(str(value["input_digest"])) + or not HEX64.fullmatch(str(value["runtime_identity"])) + or not HEX64.fullmatch(str(value["source_bundle_sha256"])) + ): + raise WrapperError("worker request identity is malformed", "input_invalid", False) + for field in ("job_id", "run_id", "step_result_id", "lease_owner"): + if not isinstance(value[field], str) or not SAFE_TEXT.fullmatch(value[field]): + raise WrapperError("worker request {} is malformed".format(field), "input_invalid", False) + for field in ("round", "desired_generation", "attempt", "lease_fence"): + if not isinstance(value[field], int) or isinstance(value[field], bool) or value[field] < 0: + raise WrapperError("worker request {} is malformed".format(field), "input_invalid", False) + argv = value["guest_argv"] + expected = [ + "no-mistakes", "worker", "run", "--role", value["kind"], + "--brief", "brief.md", "--result", "outcome.json", + ] + if argv != expected: + raise WrapperError("worker guest argv is not the supported exact role", "input_invalid", False) + if ( + not isinstance(value["source_bundle_size"], int) + or isinstance(value["source_bundle_size"], bool) + or not 0 < value["source_bundle_size"] <= 512 * 1024 * 1024 + ): + raise WrapperError("source bundle size is outside its bound", "input_invalid", False) + + +def regular(path, label, maximum=None, executable=False): + path = Path(path) + if path.is_symlink() or not path.is_file(): + raise WrapperError("{} is unavailable or redirected".format(label), "input_invalid", False) + stat = path.stat() + if stat.st_mode & 0o022 or (maximum is not None and not 0 < stat.st_size <= maximum): + raise WrapperError("{} permissions or size are unsafe".format(label), "input_invalid", False) + if executable and not os.access(path, os.X_OK): + raise WrapperError("{} is not executable".format(label), "config_invalid", False) + return path + + +def atomic_bytes(path, body): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + fd, temporary = tempfile.mkstemp(prefix=".{}-".format(path.name), dir=str(path.parent)) + try: + os.fchmod(fd, 0o600) + with os.fdopen(fd, "wb") as handle: + handle.write(body) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + finally: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + + +def atomic_json(path, value): + atomic_bytes(path, canonical(value) + b"\n") + + +def run(argv, env, timeout, cwd=None, category="infrastructure"): + try: + result = subprocess.run( + argv, cwd=cwd, env=env, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout, check=False, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise WrapperError("command failed: {}".format(exc), category, True) + if result.returncode != 0: + detail = result.stderr.decode("utf-8", errors="replace")[-800:].strip() + raise WrapperError("command refused: {}".format(detail), category, True) + return result.stdout.decode("utf-8", errors="strict") + + +def lifecycle(config, *arguments, timeout=900): + env = { + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "HOME": os.environ.get("HOME", "/nonexistent"), + "TMPDIR": os.environ.get("TMPDIR", "/tmp"), + "LANG": "C", "LC_ALL": "C", "GIT_TERMINAL_PROMPT": "0", + "FM_HOME": config["fm_home"], + **config["lifecycle_env"], + } + return run([str(verified_lifecycle_source(config)), *arguments], env, timeout) + + +def verified_lifecycle_source(config): + lifecycle_path = regular(config["lifecycle_path"], "lifecycle executable", executable=True) + probe = subprocess.run( + ["git", "-C", str(lifecycle_path.parent), "rev-parse", "--show-toplevel"], + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, + ) + if probe.returncode != 0: + raise WrapperError("lifecycle executable is not in a Git source checkout", "config_invalid", False) + try: + root = Path(probe.stdout.decode("utf-8", errors="strict").strip()).resolve() + relative = lifecycle_path.resolve().relative_to(root).as_posix() + except (UnicodeDecodeError, ValueError): + raise WrapperError("lifecycle source checkout identity is malformed", "config_invalid", False) + head = subprocess.run( + ["git", "-C", str(root), "rev-parse", "HEAD"], stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, + ) + status = subprocess.run( + ["git", "-C", str(root), "status", "--porcelain=v1", "--untracked-files=all"], + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, + ) + tracked = subprocess.run( + ["git", "-C", str(root), "ls-files", "--error-unmatch", "--", relative], + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, + ) + observed = head.stdout.decode("utf-8", errors="replace").strip() + if ( + head.returncode != 0 or observed != config["lifecycle_source_commit"] + or status.returncode != 0 or status.stdout + or tracked.returncode != 0 + ): + raise WrapperError("lifecycle source checkout is not the exact clean configured commit", "config_invalid", False) + return lifecycle_path + + +def last_json(output, label): + for line in reversed(output.splitlines()): + try: + value = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(value, dict): + return value + raise WrapperError("{} returned no JSON object".format(label), "infrastructure", True) + + +def verify_payload(request, payload): + payload = Path(payload) + if payload.is_symlink() or not payload.is_dir(): + raise WrapperError("payload directory is unavailable", "input_invalid", False) + entries = sorted(entry.name for entry in payload.iterdir()) + if entries != ["brief.md", "repo.bundle"]: + raise WrapperError("payload must contain exactly brief.md and repo.bundle", "input_invalid", False) + brief = regular(payload / "brief.md", "brief", 1024 * 1024) + bundle = regular(payload / "repo.bundle", "source bundle", 512 * 1024 * 1024) + if bundle.stat().st_size != request["source_bundle_size"] or sha256_file(bundle) != request["source_bundle_sha256"]: + raise WrapperError("source bundle bytes differ from the request", "source_invalid", False) + heads = subprocess.run( + ["git", "bundle", "list-heads", str(bundle)], stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, + ) + line = heads.stdout.decode("utf-8", errors="replace").strip() + if heads.returncode != 0 or line != "{} HEAD".format(request["desired_head_sha"]): + raise WrapperError("source bundle does not contain exactly bound HEAD", "source_invalid", False) + if not brief.read_bytes() or sha256_bytes(brief.read_bytes()) != request["input_digest"]: + raise WrapperError("brief bytes differ from the request", "input_invalid", False) + return bundle, brief + + +def prepare_task(config, request, bundle, brief, runtime): + identity = sha256_bytes(canonical(request)) + task = "nmw-{}".format(identity[:24]) + generation = "nmg-{}".format(identity[24:48]) + root = Path(config["fm_home"]) / "state" / "no-mistakes-workers" / task / generation + root.mkdir(parents=True, exist_ok=True, mode=0o700) + os.chmod(root, 0o700) + repo = root / "repo" + if not repo.exists(): + run(["git", "clone", "--quiet", str(bundle), str(repo)], os.environ.copy(), 600) + head = run(["git", "-C", str(repo), "rev-parse", "HEAD"], os.environ.copy(), 60).strip() + status = run( + ["git", "-C", str(repo), "status", "--porcelain=v1", "--untracked-files=all"], + os.environ.copy(), 60, + ) + if head != request["desired_head_sha"] or status: + raise WrapperError("durable service worktree moved or is dirty", "source_invalid", False) + git_dir = Path(run( + ["git", "-C", str(repo), "rev-parse", "--git-dir"], os.environ.copy(), 60, + ).strip()) + if not git_dir.is_absolute(): + git_dir = (repo / git_dir).resolve() + meta = Path(config["fm_home"]) / "state" / (task + ".meta") + meta_body = ( + "generation_id={}\nworktree={}\naccount_home={}\naccount_task={}\n" + "worktree_git_dir_identity={}:{}\n" + ).format( + generation, repo, config["account_pool_home"], task, + os.stat(git_dir).st_dev, os.stat(git_dir).st_ino, + ).encode() + if meta.exists() and meta.read_bytes() != meta_body: + raise WrapperError("durable service metadata differs", "stale_identity", False) + atomic_bytes(meta, meta_body) + staged = root / "payload" + staged.mkdir(exist_ok=True, mode=0o700) + for source, name in ((bundle, "repo.bundle"), (brief, "brief.md"), (runtime, "runtime.tar.gz")): + target = staged / name + if target.exists(): + if sha256_file(target) != sha256_file(source): + raise WrapperError("durable staged {} differs".format(name), "stale_identity", False) + else: + shutil.copyfile(source, target) + target.chmod(0o600) + if sha256_file(staged / "runtime.tar.gz") != config["runtime_bundle_sha256"]: + raise WrapperError("durable staged runtime differs from config", "config_invalid", False) + return task, generation, root, staged + + +def request_assignment(config, task, generation): + subscription = config["lifecycle_env"]["FM_AZURE_SUBSCRIPTION_ID"] + lifecycle( + config, "request", "--task", task, "--task-generation", generation, + "--owner-kind", "primary", "--role", "no-mistakes", "--eligible", "--required", + timeout=120, + ) + deadline = time.monotonic() + config["assignment_timeout_seconds"] + while time.monotonic() < deadline: + lifecycle( + config, "reconcile", "--apply", "--confirm-subscription", subscription, + timeout=min(900, max(1, int(deadline - time.monotonic()))), + ) + status = last_json(lifecycle(config, "status", "--json", timeout=120), "lifecycle status") + matches = [ + item for item in status.get("account_placements", []) + if item.get("task") == task and item.get("task_generation") == generation + ] + if len(matches) == 1 and matches[0].get("status") == "assigned": + item = matches[0] + if ( + not isinstance(item.get("assignment_generation"), str) + or not isinstance(item.get("account_home"), str) + or not Path(item["account_home"]).is_dir() + ): + raise WrapperError("assigned account placement is incomplete", "infrastructure", True) + return item["assignment_generation"], item["account_home"] + time.sleep(config["poll_seconds"]) + raise WrapperError("Azure worker assignment timed out", "assignment_timeout", True) + + +def extract_service_return(root, execution, request): + bundle = root / "outcome" / "outcome.bundle" + if execution.get("service_return_present") is not True or not bundle.is_file(): + raise WrapperError("guest produced no structured step outcome", "guest_execution", True) + if ( + not HEX64.fullmatch(str(execution.get("outcome_sha256"))) + or sha256_file(bundle) != execution["outcome_sha256"] + ): + raise WrapperError("service outcome bundle digest differs", "malformed_result", False) + scratch = root / "returned.git" + if scratch.exists(): + shutil.rmtree(scratch) + run( + ["git", "clone", "--bare", "--quiet", str(root / "repo"), str(scratch)], + os.environ.copy(), 300, + ) + return_ref = execution.get("return_ref") + return_commit = execution.get("return_commit") + if not isinstance(return_ref, str) or not HEX40.fullmatch(str(return_commit)): + raise WrapperError("service return identity is malformed", "malformed_result", False) + run( + ["git", "-C", str(scratch), "fetch", "--quiet", str(bundle), + "+{}:refs/service-return".format(return_ref)], + os.environ.copy(), 300, + ) + fetched = run( + ["git", "-C", str(scratch), "rev-parse", "refs/service-return^{commit}"], + os.environ.copy(), 60, + ).strip() + if fetched != return_commit: + raise WrapperError("service return commit differs", "malformed_result", False) + manifest_body = run( + ["git", "-C", str(scratch), "show", "refs/service-return:manifest.json"], + os.environ.copy(), 60, + ).encode() + step_body = run( + ["git", "-C", str(scratch), "show", "refs/service-return:step-outcome.json"], + os.environ.copy(), 60, + ).encode() + if ( + sha256_bytes(manifest_body) != execution.get("return_manifest_sha256") + or sha256_bytes(step_body) != execution.get("step_outcome_sha256") + ): + raise WrapperError("service return artifact digest differs", "malformed_result", False) + try: + manifest = json.loads(manifest_body) + step = json.loads(step_body) + except json.JSONDecodeError as exc: + raise WrapperError("service return JSON is malformed: {}".format(exc), "malformed_result", False) + manifest_fields = { + "schema", "task", "task_generation", "assignment_generation", "request_digest", + "repository_generation", "outcome_commits", "outcome_tip", "step_outcome_sha256", + } + if not isinstance(manifest, dict) or set(manifest) != manifest_fields or manifest["schema"] != RETURN_SCHEMA: + raise WrapperError("service return manifest is not the closed schema", "malformed_result", False) + if ( + manifest.get("request_digest") != execution.get("request_digest") + or manifest.get("repository_generation") != request["desired_head_sha"] + or manifest.get("outcome_tip") != execution.get("outcome_tip") + or manifest.get("step_outcome_sha256") != execution.get("step_outcome_sha256") + or not isinstance(manifest.get("outcome_commits"), int) + or isinstance(manifest.get("outcome_commits"), bool) + or manifest["outcome_commits"] < 0 + ): + raise WrapperError("service return manifest binding differs", "malformed_result", False) + step_fields = { + "schema", "step", "needs_approval", "auto_fixable", "exit_code", "skipped", + "skip_remaining", + } + allowed_optional = {"findings_json", "fix_summary", "review_approved_head_sha", "quality_outcome"} + if ( + not isinstance(step, dict) or not step_fields <= set(step) + or not set(step) - step_fields <= allowed_optional + ): + raise WrapperError("step outcome fields are not the closed schema", "malformed_result", False) + output_head = manifest["outcome_tip"] + expected_step = request["step"] + if ( + step.get("schema") != STEP_SCHEMA or step.get("step") != expected_step + or not isinstance(step.get("exit_code"), int) or isinstance(step.get("exit_code"), bool) + or not 0 <= step["exit_code"] <= 255 + or any(not isinstance(step.get(field), bool) for field in ( + "needs_approval", "auto_fixable", "skipped", "skip_remaining")) + or step.get("review_approved_head_sha", "") != ( + "" if expected_step == "test" else output_head) + or not HEX40.fullmatch(str(output_head)) + ): + raise WrapperError("step outcome identity or head binding differs", "malformed_result", False) + findings = step.get("findings_json", "") + summary = step.get("fix_summary", "") + try: + if findings: + json.loads(findings) + except json.JSONDecodeError: + raise WrapperError("step outcome findings are not JSON", "malformed_result", False) + if ( + not isinstance(findings, str) or len(findings.encode()) > 512 * 1024 or "\x00" in findings + or ((step["needs_approval"] or step["auto_fixable"]) and not findings) + or not isinstance(summary, str) or len(summary.encode()) > 256 + or any(character in summary for character in "\r\n\x00") + ): + raise WrapperError("step outcome semantic fields are malformed", "malformed_result", False) + quality = step.get("quality_outcome") + if quality is not None: + quality_fields = { + "fix_attempt_id", "root_id", "classification", "fixed_head_sha", + "observed_head_sha", "evidence_digest", "evidence_provenance", + } + if ( + request["kind"] != "repair" or expected_step != "review" + or not isinstance(quality, dict) or set(quality) != quality_fields + or quality.get("classification") not in ( + "clean_fix", "same_root_followup", "introduced_regression", "primary_handoff") + or quality.get("fixed_head_sha") != output_head + or quality.get("observed_head_sha") != output_head + or quality.get("evidence_provenance") != "semantic_rereview" + or not isinstance(quality.get("fix_attempt_id"), str) + or not quality["fix_attempt_id"].startswith("review-fix-") + or len(quality["fix_attempt_id"]) > 64 + or any(character in quality["fix_attempt_id"] for character in "\r\n\x00") + or not isinstance(quality.get("root_id"), str) or len(quality["root_id"]) > 128 + or any(character in quality["root_id"] for character in "\r\n\x00") + or not isinstance(quality.get("evidence_digest"), str) + or not quality["evidence_digest"].startswith("sha256:") + or not HEX64.fullmatch(quality["evidence_digest"][7:]) + ): + raise WrapperError("semantic quality outcome is malformed or unauthorized", "malformed_result", False) + return bundle, scratch, manifest, step_body, output_head + + +def result_base(request): + return { + "schema": RESULT_SCHEMA, "job_id": request["job_id"], "run_id": request["run_id"], + "step_result_id": request["step_result_id"], "step": request["step"], + "kind": request["kind"], + "round": request["round"], "desired_head_sha": request["desired_head_sha"], + "input_digest": request["input_digest"], "runtime_identity": request["runtime_identity"], + "owner_decision_head": request["owner_decision_head"], + "desired_generation": request["desired_generation"], "attempt": request["attempt"], + "lease_fence": request["lease_fence"], "lease_owner": request["lease_owner"], + "source_bundle_sha256": request["source_bundle_sha256"], + } + + +def make_failure(request, error): + return { + **result_base(request), "outcome": "failed", "output_head_sha": "", + "error_category": error.category, "retryable": error.retryable, + } + + +def make_success(request, root, service_bundle, scratch, manifest, step_body, output_head): + result = { + **result_base(request), "outcome": "succeeded", "output_head_sha": output_head, + "step_outcome_sha256": sha256_bytes(step_body), + } + if request["kind"] != "repair": + if output_head != request["desired_head_sha"]: + raise WrapperError("read-only worker changed the exact head", "stale_result", False) + return result, None + if output_head == request["desired_head_sha"]: + raise WrapperError("repair worker returned no descendant commit", "stale_result", False) + outcome_ref = "refs/fm-outcome/{}".format(manifest["request_digest"][:32]) + run( + ["git", "-C", str(scratch), "fetch", "--quiet", str(service_bundle), + "+{}:{}".format(outcome_ref, outcome_ref)], + os.environ.copy(), 300, + ) + ancestor = subprocess.run( + ["git", "-C", str(scratch), "merge-base", "--is-ancestor", + request["desired_head_sha"], output_head], + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, + ) + if ancestor.returncode != 0: + raise WrapperError("repair head is not a descendant", "stale_result", False) + outward_ref = "refs/heads/no-mistakes/azure/{}".format( + sha256_bytes(canonical(request))[:20]) + run(["git", "-C", str(scratch), "update-ref", outward_ref, output_head], os.environ.copy(), 60) + outward = root / "external-outcome.bundle" + if outward.exists(): + outward.unlink() + run( + ["git", "-C", str(scratch), "bundle", "create", str(outward), + outward_ref, "^{}".format(request["desired_head_sha"])], + os.environ.copy(), 300, + ) + result["return_ref"] = outward_ref + result["return_bundle_sha256"] = sha256_bytes(outward.read_bytes()) + return result, outward + + +def cleanup_service(config, task, generation, assignment, request_digest): + subscription = config["lifecycle_env"]["FM_AZURE_SUBSCRIPTION_ID"] + lifecycle( + config, "service-complete", "--task", task, "--task-generation", generation, + "--assignment-generation", assignment, "--request-digest", request_digest, + "--confirm-subscription", subscription, timeout=120, + ) + deadline = time.monotonic() + config["cleanup_timeout_seconds"] + while time.monotonic() < deadline: + lifecycle( + config, "reconcile", "--apply", "--confirm-subscription", subscription, + timeout=min(900, max(1, int(deadline - time.monotonic()))), + ) + status = last_json(lifecycle(config, "status", "--json", timeout=120), "lifecycle status") + matches = [ + item for item in status.get("account_placements", []) + if item.get("task") == task and item.get("task_generation") == generation + ] + if not matches: + return + time.sleep(config["poll_seconds"]) + raise WrapperError("Azure worker cleanup timed out", "cleanup_timeout", True) + + +def execute(args): + config = read_json(args.config, "wrapper config", CONFIG_FIELDS) + validate_config(config) + regular(args.config, "wrapper config", MAX_JSON) + verified_lifecycle_source(config) + runtime = regular(config["runtime_bundle"], "runtime bundle", 1024 * 1024 * 1024) + if sha256_file(runtime) != config["runtime_bundle_sha256"]: + raise WrapperError("runtime bundle bytes differ from config", "config_invalid", False) + request = read_json(args.request, "worker request", REQUEST_FIELDS) + validate_request(request) + bundle, brief = verify_payload(request, args.payload) + task, generation, root, staged = prepare_task(config, request, bundle, brief, runtime) + lock = root / ".lock" + with open(lock, "a+", encoding="utf-8") as handle: + os.chmod(lock, 0o600) + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + candidate_path = root / "candidate.json" + if candidate_path.is_file(): + cached = json.loads(candidate_path.read_text(encoding="utf-8")) + assignment = cached["assignment_generation"] + request_digest = cached["request_digest"] + result = cached["result"] + else: + assignment, account_home = request_assignment(config, task, generation) + outcome_dir = root / "outcome" + outcome_dir.mkdir(exist_ok=True, mode=0o700) + output = lifecycle( + config, "execute", "--task", task, "--task-generation", generation, + "--assignment-generation", assignment, "--wall-seconds", str(config["wall_seconds"]), + "--payload-dir", str(staged), "--account-dir", account_home, + "--outcome-dir", str(outcome_dir), "--confirm-execute", + "--confirm-subscription", config["lifecycle_env"]["FM_AZURE_SUBSCRIPTION_ID"], + "--", *request["guest_argv"], timeout=config["wall_seconds"] + 1800, + ) + execution = last_json(output, "worker execution") + request_digest = execution.get("request_digest") + if not HEX64.fullmatch(str(request_digest)): + raise WrapperError("worker execution request digest is malformed", "malformed_result", False) + try: + service_bundle, scratch, manifest, step_body, output_head = extract_service_return( + root, execution, request) + result, outward = make_success( + request, root, service_bundle, scratch, manifest, step_body, output_head) + atomic_bytes(root / "external-step-outcome.json", step_body) + if outward is not None: + pass + except WrapperError as error: + result = make_failure(request, error) + cached = { + "schema": "fm.no-mistakes-worker-candidate/v1", + "assignment_generation": assignment, + "request_digest": request_digest, + "result": result, + } + atomic_json(candidate_path, cached) + cached_step = root / "external-step-outcome.json" + cached_outcome = root / "external-outcome.bundle" + if result.get("outcome") == "succeeded": + if not cached_step.is_file(): + raise WrapperError("cached step outcome is absent", "malformed_result", False) + shutil.copyfile(cached_step, args.step_outcome) + Path(args.step_outcome).chmod(0o600) + if request["kind"] == "repair": + if not cached_outcome.is_file(): + raise WrapperError("cached repair bundle is absent", "malformed_result", False) + shutil.copyfile(cached_outcome, args.outcome) + Path(args.outcome).chmod(0o600) + cleanup_service(config, task, generation, assignment, request_digest) + atomic_json(args.result, result) + + +def parser(): + top = argparse.ArgumentParser(description="Run one no-mistakes step through Firstmate Azure workers") + top.add_argument("--config", required=True) + sub = top.add_subparsers(dest="command", required=True) + command = sub.add_parser("execute") + command.add_argument("--request", required=True) + command.add_argument("--payload", required=True) + command.add_argument("--result", required=True) + command.add_argument("--outcome", required=True) + command.add_argument("--step-outcome", required=True) + return top + + +def validate_cli_paths(args): + values = [ + args.config, args.request, args.payload, args.result, args.outcome, args.step_outcome, + ] + resolved = [] + for value in values: + path = Path(value) + if not path.is_absolute() or str(path) != str(path.resolve()): + raise WrapperError("wrapper paths must be clean and absolute", "input_invalid", False) + if path.is_symlink(): + raise WrapperError("wrapper paths must not be redirects", "input_invalid", False) + resolved.append(str(path)) + if len(resolved) != len(set(resolved)): + raise WrapperError("wrapper paths must be distinct", "input_invalid", False) + + +def main(): + args = parser().parse_args() + if args.command != "execute": + raise SystemExit(2) + try: + validate_cli_paths(args) + except WrapperError as error: + print("NO-MISTAKES WORKER REFUSED: {}".format(error), file=os.sys.stderr) + raise SystemExit(2) + try: + execute(args) + except WrapperError as error: + try: + request = read_json(args.request, "worker request", REQUEST_FIELDS) + validate_request(request) + atomic_json(args.result, make_failure(request, error)) + return + except WrapperError: + print("NO-MISTAKES WORKER REFUSED: {}".format(error), file=os.sys.stderr) + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/bin/fm-spawn-cloud-monitor.sh b/bin/fm-spawn-cloud-monitor.sh index cde4c235823..997f4c506c9 100755 --- a/bin/fm-spawn-cloud-monitor.sh +++ b/bin/fm-spawn-cloud-monitor.sh @@ -5,8 +5,9 @@ # local endpoint that keeps it visible and reapable in the same Herdr # workspace as local crewmates. It renders the durable lifecycle state # (queue/assignment from the worker controller, then the bounded execute log) -# and exits when the digest-bound result lands, so the pane's lifetime tracks -# the crewmate's remote lifetime instead of ending at spawn time. +# and exits only after the digest-bound result is in local custody and its +# assignment has released, so endpoint loss can never strand account or +# capacity ownership behind an otherwise successful worker exit. # # Convergence duty: when the spawn-time reconcile left the request queued # (transient admission evidence), a LATER reconcile assigns the worker after @@ -20,6 +21,8 @@ set -u SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) +# shellcheck source=bin/fm-cloud-state-lib.sh +. "$SCRIPT_DIR/fm-cloud-state-lib.sh" ID=${1:?task id} GENERATION=${2:?task generation id} FM_HOME=${FM_HOME:?FM_HOME is required} @@ -175,6 +178,9 @@ dispatch_converged_execute() { # outcome_expected exists to prevent; create the directory instead. install -d -m 0700 "$STATE/$ID.cloud-outcome" 2>/dev/null || true payload_args+=(--outcome-dir "$STATE/$ID.cloud-outcome") + case "${FM_SPAWN_CLOUD_RETURN_KIND:-}" in + ship|scout) payload_args+=(--return-kind "$FM_SPAWN_CLOUD_RETURN_KIND") ;; + esac fi nohup env FM_HOME="$FM_HOME" FM_STATE_OVERRIDE="$STATE" \ "$SCRIPT_DIR/fm-worker-lifecycle.sh" execute \ @@ -206,6 +212,63 @@ elif value is not None: PY } +finalize_authorized_return() { + local assignment status proof lifecycle + assignment=$(result_field assignment_generation) + [ -n "$assignment" ] || { + echo "cloud-crewmate $ID: authorized return has no assignment generation" + return 1 + } + if ! python3 "$SCRIPT_DIR/fm-cloud-result.py" collect \ + --state "$STATE" --task "$ID" --task-generation "$GENERATION" \ + --assignment-generation "$assignment"; then + echo "cloud-crewmate $ID: authorized return is not in local custody yet; retaining the assignment for retry" + return 1 + fi + lifecycle=${FM_CLOUD_RETURN_LIFECYCLE_COMMAND:-$SCRIPT_DIR/fm-worker-lifecycle.sh} + status=$(queue_status) + proof=$STATE/$ID.worker-release.json + case "$status" in + assigned) + if ! FM_HOME="$FM_HOME" FM_STATE_OVERRIDE="$STATE" "$lifecycle" authority-receipt \ + --task "$ID" --task-generation "$GENERATION" \ + --assignment-generation "$assignment" --output "$proof"; then + echo "cloud-crewmate $ID: local custody is established but release authority is not ready; retrying" + return 1 + fi + if ! FM_HOME="$FM_HOME" FM_STATE_OVERRIDE="$STATE" "$lifecycle" release \ + --task "$ID" --task-generation "$GENERATION" --proof-file "$proof"; then + echo "cloud-crewmate $ID: local custody is established but release recording failed; retrying" + return 1 + fi + status=releasing + ;; + releasing) : ;; + complete) + fm_cloud_state_remove "$STATE" "$ID" + echo "cloud-crewmate $ID: return is local and the worker assignment is released" + return 0 + ;; + *) + echo "cloud-crewmate $ID: return is local but controller status is '$status'; retaining artifacts for retry" + return 1 + ;; + esac + if ! FM_HOME="$FM_HOME" FM_STATE_OVERRIDE="$STATE" "$lifecycle" reconcile --apply \ + --confirm-subscription "${FM_AZURE_SUBSCRIPTION_ID:-}" >/dev/null; then + echo "cloud-crewmate $ID: worker release convergence failed; retrying without replaying the task" + return 1 + fi + status=$(queue_status) + if [ "$status" != complete ]; then + echo "cloud-crewmate $ID: worker release is '$status'; retrying until account and capacity are free" + return 1 + fi + fm_cloud_state_remove "$STATE" "$ID" + echo "cloud-crewmate $ID: return is local and the worker assignment is released" + return 0 +} + land_outcome_bundle() { # Landing v1: the crewmate committed on the worker's copy of the leased # worktree and the bundle came home digest-verified. The landing authority @@ -288,6 +351,13 @@ while :; do if [ -s "$RESULT" ]; then echo "cloud-crewmate $ID: worker result landed" python3 -m json.tool "$RESULT" 2>/dev/null | head -40 || cat "$RESULT" + if [ -n "$(result_field return_present)" ]; then + if finalize_authorized_return; then + exit 0 + fi + sleep "$INTERVAL" + continue + fi land_outcome_bundle exit 0 fi diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 7474cb2cb39..68e875ac73b 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -41,8 +41,10 @@ # closed. Cloud spawns run ENTIRELY on the pi-codex runtime: harness # dispatch and claude profile routing are bypassed. account_home comes from # config/azure-worker-account-home when that file exists, otherwise from the -# pi coding-agent directory for backward compatibility; pi's extension owns -# multi-profile selection on the worker. With the switch off (default), +# pi coding-agent directory for backward compatibility. The controller +# load-balances usable profiles and gives each assignment one private +# single-profile snapshot; the worker never receives the canonical pool. +# With the switch off (default), # spawns stay byte-identical to the local path. Secondmate and # account-recovery spawns always stay local, and --backend, raw launch # commands, or a non-pi harness cannot be combined with cloud placement. @@ -688,9 +690,6 @@ faults = {name: module.entry_faults(pool[name]) for name in expected} broken = [f"{name}: {', '.join(items)}" for name, items in faults.items() if items] if broken: module.fail("Azure Pi pool has unusable profile shapes: " + "; ".join(broken)) -accounts = [pool[name]["accountId"].strip() for name in expected] -if len(set(accounts)) != len(accounts): - module.fail("Azure Pi pool profiles must name distinct upstream accounts") PY } @@ -4496,28 +4495,23 @@ spawn_cloud_record_assignment() { # fi fm_account_meta_lock_release "$lock" || return 1 } -# spawn_cloud_bind_leased_account: narrow the staged provider credential to the -# ONE Pi profile the controller leased for this placement (R5). +# spawn_cloud_bind_account_snapshot: stage the ONE profile snapshot selected +# for this assignment. # -# The controller is the only selector: it picks a free profile under its own -# lock, in the same act that writes the queue entry that IS the lease, and -# prints the single-profile account home it projected (bin/fm-pi-account-home.py -# writes it; nothing here re-derives a home or re-implements a projection). -# This function reads that path back and makes it the credential the worker -# actually receives, so the lease is not a paper lease: without it the pooled -# auth.json would ride to the guest and every concurrent crewmate would resolve -# to the pool's first slot - one account, N workers, which is the collision R5 -# exists to remove. +# The controller load-balances profiles under its lock, checks the canonical +# profile's twelve-hour headroom, and writes a request-private single-profile +# home through bin/fm-pi-account-home.py. This function rechecks that snapshot +# immediately before copying it into the task-private staging directory. The +# pooled auth.json never reaches the guest, and assignments using the same +# profile never share writable homes. # -# It refuses rather than falling back. No leased path, no credential at the -# leased path, or a leased credential carrying more than one provider slot all -# stop the placement, because each of those is "we do not know which account -# this worker will use". -spawn_cloud_bind_leased_account() { # +# It refuses rather than falling back. A missing path, credential, or exact +# one-slot shape means the selected snapshot cannot be proved. +spawn_cloud_bind_account_snapshot() { # local out=$1 leased line tmp profile if spawn_test_lab_enabled && [ "${FM_TEST_CLOUD_ACCOUNT_BIND_FAIL:-0}" = 1 ]; then - # Test-only: the lease-handback path below has no other injection point, - # and an untested handback is how a pool quietly shrinks to zero. + # Test-only: the projection-cleanup path below has no other injection + # point, and an untested failure could leave credential snapshots behind. echo "error: test-only provider-account bind failure for $ID" >&2 return 1 fi @@ -4529,19 +4523,19 @@ spawn_cloud_bind_leased_account() { # 'account-profile '*) profile=${line#account-profile } ;; esac done < "$out" - # The controller reports the slot name separately BECAUSE the projected home - # is keyed on the lease identity rather than the slot name; reading the name - # off the path's last component would be reading the wrong thing. + # The controller reports the slot name separately because the projected home + # is keyed on the assignment-private projection binding, not the reusable + # profile or account identity. [ -n "$profile" ] || { - echo "error: the controller named no leased provider-account profile for $ID" >&2 + echo "error: the controller named no provider-account snapshot profile for $ID" >&2 return 1 } [ -n "$leased" ] || { - echo "error: the controller named no leased provider-account home for $ID; refusing to stage a pooled credential" >&2 + echo "error: the controller named no assignment-private provider-account home for $ID; refusing to stage a pooled credential" >&2 return 1 } [ -d "$leased" ] && [ -f "$leased/auth.json" ] || { - echo "error: leased provider-account home '$leased' holds no credential for $ID" >&2 + echo "error: provider-account snapshot home '$leased' holds no credential for $ID" >&2 return 1 } # Azure's hard VM shutdown is six hours after creation. Require twice that @@ -4552,7 +4546,7 @@ spawn_cloud_bind_leased_account() { # "$SCRIPT_DIR/fm-credential-expiry.py" check --harness pi \ --margin-seconds "$CLOUD_ACCOUNT_MIN_HEADROOM_SECONDS" \ --min-state usable "$leased" >/dev/null || { - echo "error: leased provider-account credential lacks twelve hours of access-token headroom for $ID" >&2 + echo "error: provider-account snapshot lacks twelve hours of access-token headroom for $ID" >&2 return 1 } # Exactly one provider slot, checked by shape and never by content: a home @@ -4565,11 +4559,11 @@ try: with open(sys.argv[1], encoding="utf-8") as handle: parsed = json.load(handle) except (OSError, ValueError): - print("error: leased provider-account credential is unreadable", file=sys.stderr) + print("error: provider-account snapshot credential is unreadable", file=sys.stderr) raise SystemExit(1) if not isinstance(parsed, dict) or len(parsed) != 1: print( - "error: leased provider-account credential does not hold exactly one provider slot", + "error: provider-account snapshot does not hold exactly one provider slot", file=sys.stderr, ) raise SystemExit(1) @@ -4661,7 +4655,23 @@ spawn_cloud_persist_convergence_artifacts() { exit 1 } if [ -f "$DATA/$ID/brief.md" ]; then - cp "$DATA/$ID/brief.md" "$STATE/$ID.cloud-payload/brief.md" || exit 1 + if [ "$KIND" = secondmate ]; then + cp "$DATA/$ID/brief.md" "$STATE/$ID.cloud-payload/brief.md" || exit 1 + else + # The generated brief names the task home's authorized report, visual, + # and status paths. Those host paths do not exist on a worker. Rewrite + # only the exact task-home prefix into the fixed return staging root; + # the guest collector later accepts only the digest-bound task paths, + # so this does not authorize arbitrary worker files to come home. + python3 - "$DATA/$ID/brief.md" "$STATE/$ID.cloud-payload/brief.md" "$TASK_HOME" <<'PY' || exit 1 +from pathlib import Path +import sys +source, destination, task_home = sys.argv[1:] +body = Path(source).read_bytes() +prefix = task_home.encode() +Path(destination).write_bytes(body.replace(prefix, b"/mnt/task/.fm-return")) +PY + fi elif [ "$KIND" = secondmate ] && [ -f "$WT/data/charter.md" ]; then # A secondmate's standing brief is its persistent charter in the home; # the compartment payload carries a copy so the cloud agent's brief is @@ -4684,12 +4694,12 @@ spawn_cloud_persist_convergence_artifacts() { exit 1 fi # The POOLED auth.json is deliberately NOT copied here. This runs BEFORE the - # request creates the lease, and the tracking monitor pane already exists and + # request creates the assignment-private projection, and the tracking monitor pane already exists and # is already polling: a crash, kill, or plain slow reconcile between here and # the narrowing would leave every signed-in account staged in a directory the # monitor is willing to dispatch as --account-dir. The account directory is - # therefore written exactly once, by spawn_cloud_bind_leased_account, after - # the controller has said which single account this placement leased. That + # therefore written exactly once, by spawn_cloud_bind_account_snapshot, after + # the controller has said which single profile snapshot this placement owns. That # removes the window rather than guarding it. # settings.json is pi CONFIGURATION, not credential material, so it is staged # here with the rest of the payload. @@ -4699,6 +4709,8 @@ spawn_cloud_persist_convergence_artifacts() { fi { printf 'export FM_SPAWN_CLOUD_WALL_SECONDS=%q\n' "$wall" + [ "$KIND" = secondmate ] \ + || printf 'export FM_SPAWN_CLOUD_RETURN_KIND=%q\n' "$KIND" [ -z "${FM_WORKER_PROVIDER_COMMAND:-}" ] \ || printf 'export FM_WORKER_PROVIDER_COMMAND=%q\n' "$FM_WORKER_PROVIDER_COMMAND" if [ "$KIND" = secondmate ]; then @@ -4776,7 +4788,7 @@ spawn_cloud_dispatch() { # directory against the marker plus the primary's own registry. task_home_args=() [ "$TASK_HOME" = "$FM_HOME" ] || task_home_args=(--task-home "$TASK_HOME") - # The request's STDOUT carries the leased provider-account home (R5), so it is + # The request's STDOUT carries the assignment-private snapshot home, so it is # captured rather than folded into stderr; stderr still flows through # untouched, and the captured lines are echoed on for the operator either way. request_report="$STATE/$ID.worker-request.out" @@ -4788,11 +4800,17 @@ spawn_cloud_dispatch() { ${task_home_args[@]+"${task_home_args[@]}"} --eligible > "$request_report" || { cat "$request_report" >&2 2>/dev/null || true rm -f "$request_report" - # No durable queue entry exists, so the convergence artifacts have no - # owner; remove them (including the copied provider credential) with the - # rolled-back spawn. $STATE is the directory this spawn just staged into, - # which on the compartment-child lane is the secondmate's and not the - # primary's, so the rollback needs no resolution of its own. + # Most refusals precede insertion, but a crash-resumable projection failure + # leaves a durable `projecting` owner by design. Withdraw that exact + # generation when present so its private snapshot is removed; an ordinary + # pre-insertion refusal simply makes this best-effort probe fail harmlessly. + if spawn_cloud_lifecycle withdraw --task "$ID" \ + --task-generation "$SPAWN_GENERATION_ID" --confirm-withdraw \ + --confirm-subscription "${FM_AZURE_SUBSCRIPTION_ID:-}" >/dev/null 2>&1; then + echo "notice: removed the incomplete provider-account projection for $ID" >&2 + fi + # Remove task-local convergence artifacts from the home this spawn staged + # into. On the compartment-child lane that is the secondmate's home. fm_cloud_state_remove_generation "$STATE" "$ID" # The outcome directory is NOT transport. When the monitor cannot # fast-forward it tells the operator the bundle is "kept for manual @@ -4811,27 +4829,25 @@ spawn_cloud_dispatch() { } cat "$request_report" >&2 if spawn_test_lab_enabled && [ "${FM_TEST_CLOUD_ABORT_AFTER_REQUEST:-0}" = 1 ]; then - # Test-only: die exactly in the window between the durable lease and the + # Test-only: die exactly in the window between the durable projection and the # narrowing, with the tracking monitor already live. This is the window the # pool used to be staged in, and the only way to assert it is empty is to # stop the process inside it. kill -9 $$ fi - spawn_cloud_bind_leased_account "$request_report" || { - # The queue entry exists and is the LEASE on a provider account. A spawn - # that cannot bind that account must hand it back rather than leave it held - # by work that will never run: an orphaned lease shrinks the pool by one - # every time this happens. The entry is still `queued` here (reconcile has - # not run), which is exactly what withdraw accepts, and withdraw also - # removes the staged credential. + spawn_cloud_bind_account_snapshot "$request_report" || { + # The queue entry owns one assignment-private projection. A spawn that + # cannot stage it withdraws while still queued, which removes that exact + # projection and the task-private staged credential without touching any + # same-profile placement. if spawn_cloud_lifecycle withdraw --task "$ID" \ --task-generation "$SPAWN_GENERATION_ID" --confirm-withdraw \ --confirm-subscription "${FM_AZURE_SUBSCRIPTION_ID:-}" >&2; then - echo "notice: released the provider-account lease for $ID with its withdrawn request" >&2 + echo "notice: removed the provider-account projection for $ID with its withdrawn request" >&2 else - echo "error: the provider-account lease for $ID is still held by its queued request; withdraw it with bin/fm-worker-lifecycle.sh withdraw --task $ID --task-generation $SPAWN_GENERATION_ID" >&2 + echo "error: the provider-account projection for $ID is still owned by its queued request; withdraw it with bin/fm-worker-lifecycle.sh withdraw --task $ID --task-generation $SPAWN_GENERATION_ID" >&2 fi - echo "error: cloud placement for $ID could not be bound to its leased provider account" >&2 + echo "error: cloud placement for $ID could not stage its provider-account snapshot" >&2 return 1 } if ! spawn_cloud_lifecycle reconcile --apply \ @@ -4868,7 +4884,7 @@ spawn_cloud_dispatch() { --task "$ID" --task-generation "$SPAWN_GENERATION_ID" \ --assignment-generation "$assignment" --wall-seconds "$wall" \ --payload-dir "$STATE/$ID.cloud-payload" --account-dir "$STATE/$ID.cloud-account" \ - --outcome-dir "$STATE/$ID.cloud-outcome" \ + --outcome-dir "$STATE/$ID.cloud-outcome" --return-kind "$KIND" \ --confirm-execute --confirm-subscription "${FM_AZURE_SUBSCRIPTION_ID:-}" \ -- /bin/bash -lc "$CLOUD_WORKER_LAUNCH" \ > "$STATE/$ID.worker-result.json" 2> "$STATE/$ID.worker-execute.log" < /dev/null & diff --git a/bin/fm-supervise-daemon.sh b/bin/fm-supervise-daemon.sh index 2cf08fc1b05..7f9df7db028 100755 --- a/bin/fm-supervise-daemon.sh +++ b/bin/fm-supervise-daemon.sh @@ -1314,6 +1314,30 @@ trim_log() { # classifiers above are sourceable for unit tests (tests/fm-daemon.test.sh). # ============================================================================ +fm_super_stop_watcher() { # + local watcher_pid=$1 child current_parent i children="" + while read -r child; do + [ -n "$child" ] || continue + current_parent=$(ps -o ppid= -p "$child" 2>/dev/null | tr -d '[:space:]') + [ "$current_parent" = "$watcher_pid" ] || continue + children="${children}${children:+ }$child" + kill -TERM "$child" 2>/dev/null || true + done < <(ps -axo pid=,ppid= | awk -v parent="$watcher_pid" '$2 == parent { print $1 }') + # A bounded-command owner traps TERM so it can terminate and reap its own + # process group. Keep its watcher parent alive long enough to wait for that + # owner; killing both at once reparents the cleanup owner to launchd/init. + for child in $children; do + i=0 + while [ "$i" -lt 50 ]; do + current_parent=$(ps -o ppid= -p "$child" 2>/dev/null | tr -d '[:space:]') + [ "$current_parent" = "$watcher_pid" ] || break + sleep 0.1 + i=$((i + 1)) + done + done + kill -TERM "$watcher_pid" 2>/dev/null || true +} + fm_super_main() { local STATE DELIVERY BACKEND TARGET backend_source target_source STATE="$(_state_root)" @@ -1457,7 +1481,7 @@ fm_super_main() { escalate_flush "$STATE" 2>/dev/null || true fi if [ -n "${WATCHER_PID:-}" ]; then - kill "$WATCHER_PID" 2>/dev/null || true + fm_super_stop_watcher "$WATCHER_PID" wait "$WATCHER_PID" 2>/dev/null || true fi if [ -n "${CUR_TMP:-}" ]; then diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 1f111990871..c30bf98f783 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -739,16 +739,47 @@ scan_signals() { } run_bounded() { # [args...] - local seconds=$1 + local seconds=$1 status shift if command -v timeout >/dev/null 2>&1; then - timeout --kill-after=1 "$seconds" "$@" + timeout --kill-after=1 "$seconds" "$@" & elif command -v gtimeout >/dev/null 2>&1; then - gtimeout --kill-after=1 "$seconds" "$@" + gtimeout --kill-after=1 "$seconds" "$@" & else # shellcheck disable=SC2016 # single quotes are deliberate: Perl expands its own variables. - perl -e 'my $t = shift; my $pid = fork; die "fork failed" unless defined $pid; if (!$pid) { setpgrp(0, 0); exec @ARGV } local $SIG{ALRM} = sub { kill "TERM", -$pid; select undef, undef, undef, 0.2; kill "KILL", -$pid; exit 124 }; alarm $t; waitpid $pid, 0; exit($? >> 8)' "$seconds" "$@" + perl -MPOSIX=:sys_wait_h -MErrno=EINTR -e ' + my $t = shift; + my $pid = fork; + die "fork failed" unless defined $pid; + if (!$pid) { setpgrp(0, 0); exec @ARGV } + sub terminate { + my ($status) = @_; + alarm 0; + kill "TERM", -$pid; + for (1 .. 10) { + my $waited = waitpid $pid, WNOHANG; + exit $status if $waited == $pid; + select undef, undef, undef, 0.1; + } + kill "KILL", -$pid; + my $waited; + do { $waited = waitpid $pid, 0 } while ($waited == -1 && $! == EINTR); + exit $status; + } + local $SIG{ALRM} = sub { terminate(124) }; + local $SIG{HUP} = sub { terminate(129) }; + local $SIG{INT} = sub { terminate(130) }; + local $SIG{TERM} = sub { terminate(143) }; + alarm $t; + my $waited; + do { $waited = waitpid $pid, 0 } while ($waited == -1 && $! == EINTR); + exit($? >> 8); + ' "$seconds" "$@" & fi + ACTIVE_BOUNDED_PID=$! + if wait "$ACTIVE_BOUNDED_PID"; then status=0; else status=$?; fi + ACTIVE_BOUNDED_PID= + return "$status" } run_check() { @@ -1063,7 +1094,19 @@ if ! fm_lock_try_acquire "$WATCH_LOCK"; then fi exit 0 fi -trap 'fm_lock_release "$WATCH_LOCK"' EXIT +ACTIVE_BOUNDED_PID= +watcher_cleanup() { + trap - EXIT TERM INT + if [ -n "${ACTIVE_BOUNDED_PID:-}" ]; then + kill "$ACTIVE_BOUNDED_PID" 2>/dev/null || true + wait "$ACTIVE_BOUNDED_PID" 2>/dev/null || true + ACTIVE_BOUNDED_PID= + fi + fm_lock_release "$WATCH_LOCK" +} +trap watcher_cleanup EXIT +trap 'watcher_cleanup; exit 143' TERM +trap 'watcher_cleanup; exit 130' INT # This watcher's own pid, as recorded in the lock by fm_lock_claim (which writes # ${BASHPID:-$$} from this same main shell). Read directly, never via a command # substitution, so it matches the stored holder pid for the self-eviction check. diff --git a/bin/fm-worker-authority.py b/bin/fm-worker-authority.py index 4c8853ddc46..4deaa87c8d7 100755 --- a/bin/fm-worker-authority.py +++ b/bin/fm-worker-authority.py @@ -11,6 +11,7 @@ import stat import subprocess import sys +import unicodedata ROOT = Path(__file__).resolve().parent.parent @@ -133,13 +134,29 @@ def endpoint_evidence(home, task, values): text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env={**os.environ, "FM_HOME": str(home), "FM_ROOT": str(home)}, ) - if result.returncode != 0 or result.stdout.strip() != "absent": - raise AuthorityError("endpoint authority did not prove the exact task endpoint absent") - return "{}\0{}\0{}\0absent".format(backend, target, expected).encode() - - -def report_evidence(home, task): - path = home / "data" / task / "completion.md" + endpoint_state = result.stdout.strip() + if result.returncode == 0 and endpoint_state == "absent": + return "{}\0{}\0{}\0absent".format(backend, target, expected).encode() + # A cloud endpoint is only the local tracking monitor. Once the exact + # digest-bound return has reached local custody it has no guest process or + # steering authority left, and release must not wait on its own process to + # disappear before it can free the remote lease. The terminal status is + # produced only after report and branch custody, and unknown still refuses. + placement = values.get("placement", []) + if result.returncode == 0 and endpoint_state == "present" and placement == ["azure"]: + status = home / "state" / (task + ".status") + if status.is_symlink() or not status.is_file(): + raise AuthorityError("cloud endpoint authority has no local terminal custody status") + lines = [line.strip() for line in status.read_text(encoding="utf-8").splitlines() if line.strip()] + if not lines or not re.match(r"^(done|failed):", lines[-1]): + raise AuthorityError("cloud endpoint authority has no local terminal custody status") + return "{}\0{}\0{}\0cloud-return-localized".format(backend, target, expected).encode() + raise AuthorityError("endpoint authority did not prove the exact task endpoint absent or return-localized") + + +def report_evidence(home, task, kind="ship"): + report_name = "report.md" if kind == "scout" else "completion.md" + path = home / "data" / task / report_name if path.is_symlink() or not path.is_file() or path.stat().st_size > 16 * 1024 * 1024: raise AuthorityError("completion report authority is absent, redirected, or oversized") content = path.read_bytes() @@ -147,6 +164,14 @@ def report_evidence(home, task): positions = [text.find(heading) for heading in REQUIRED_HEADINGS] if any(position < 0 for position in positions) or positions != sorted(positions): raise AuthorityError("completion report authority lacks the exact ordered contract headings") + for index, heading in enumerate(REQUIRED_HEADINGS): + start = positions[index] + len(heading) + end = positions[index + 1] if index + 1 < len(positions) else len(text) + if not any( + character.isalnum() or unicodedata.category(character).startswith("S") + for character in text[start:end] + ): + raise AuthorityError("completion report authority has an empty required section") return content @@ -162,6 +187,125 @@ def worktree_evidence(task, values): return "{}\0{}\0{}".format(worktree, common.resolve(), git(worktree, "rev-parse", "HEAD")).encode(), worktree +def cloud_return_evidence(home, task, generation, assignment, kind, worktree, repository_generation): + """Prove local custody before releasing remote worker capacity. + + This is intentionally not ordinary forge landing. The returned commits + remain protected by the ordinary task branch and later teardown gate; this + receipt proves only that deleting the worker cannot delete the last copy. + """ + result_path = home / "state" / (task + ".worker-result.json") + bundle_path = home / "state" / (task + ".cloud-outcome") / "outcome.bundle" + manifest_path = home / "data" / task / "cloud-return.json" + for path, label, limit in ( + (result_path, "result", 8 * 1024 * 1024), + (bundle_path, "bundle", 256 * 1024 * 1024), + (manifest_path, "manifest", 1024 * 1024), + ): + if path.is_symlink() or not path.is_file() or not 0 < path.stat().st_size <= limit: + raise AuthorityError("cloud return {} custody is absent, redirected, or oversized".format(label)) + result = json.loads(result_path.read_text(encoding="utf-8")) + if result.get("schema") != "fm.worker-execution-result/v1": + raise AuthorityError("cloud return result schema is not supported") + unsigned = dict(result) + supplied = unsigned.pop("result_digest", None) + if supplied != digest(unsigned): + raise AuthorityError("cloud return result digest is not exact") + expected = { + "task": task, + "task_generation": generation, + "assignment_generation": assignment, + "repository_generation": repository_generation, + "return_present": True, + } + for field, value in expected.items(): + if result.get(field) != value: + raise AuthorityError("cloud return result {} binding differs".format(field)) + bundle = bundle_path.read_bytes() + if result.get("outcome_bytes") != len(bundle) or result.get("outcome_sha256") != hashlib.sha256(bundle).hexdigest(): + raise AuthorityError("cloud return bundle differs from the digest-bound result") + manifest = manifest_path.read_bytes() + if result.get("return_manifest_sha256") != hashlib.sha256(manifest).hexdigest(): + raise AuthorityError("cloud return manifest differs from the digest-bound result") + manifest_value = json.loads(manifest.decode("utf-8")) + manifest_expected = { + "schema": "fm.worker-return/v1", + "task": task, + "task_generation": generation, + "assignment_generation": assignment, + "request_digest": result.get("request_digest"), + "repository_generation": repository_generation, + "kind": kind, + "report_required": True, + "report_path": "data/{}/{}".format( + task, "report.md" if kind == "scout" else "completion.md", + ), + "status_path": "state/{}.status".format(task), + "visuals_path": "data/{}/visuals".format(task), + "branch": "" if kind == "scout" else "fm/{}".format(task), + "outcome_commits": result.get("outcome_commits"), + "outcome_tip": result.get("outcome_tip"), + "uncommitted_changes": result.get("outcome_uncommitted_changes"), + } + for field, value in manifest_expected.items(): + if manifest_value.get(field) != value: + raise AuthorityError("cloud return manifest {} binding differs".format(field)) + artifacts = manifest_value.get("artifacts") + if not isinstance(artifacts, dict): + raise AuthorityError("cloud return manifest artifact bindings are malformed") + status_path = home / "state" / (task + ".status") + if status_path.is_symlink() or not status_path.is_file(): + raise AuthorityError("cloud return has no local terminal status") + status_lines = [line.strip() for line in status_path.read_text(encoding="utf-8").splitlines() if line.strip()] + if not status_lines or not re.match(r"^(done|failed):", status_lines[-1]): + raise AuthorityError("cloud return has no local terminal status") + commits = result.get("outcome_commits") + if not isinstance(commits, int) or isinstance(commits, bool) or commits < 0: + raise AuthorityError("cloud return commit count is malformed") + tip = result.get("outcome_tip") + if not isinstance(tip, str) or not re.fullmatch(r"[0-9a-f]{40}", tip): + raise AuthorityError("cloud return outcome tip is malformed") + if kind == "ship" and commits: + branch = "refs/heads/fm/{}".format(task) + branch_head = git(worktree, "rev-parse", "--verify", branch) + if subprocess.run(["git", "-C", str(worktree), "merge-base", "--is-ancestor", tip, branch_head]).returncode != 0: + raise AuthorityError("cloud return commit is not reachable from the required task branch") + if git(worktree, "symbolic-ref", "--quiet", "HEAD") != branch: + raise AuthorityError("cloud return task branch is not checked out") + uncommitted = result.get("outcome_uncommitted_changes") + if not isinstance(uncommitted, bool): + raise AuthorityError("cloud return working-tree disposition is malformed") + if uncommitted: + scratch_destinations = { + "scratch.patch": home / "data" / task / "cloud-scratch.patch", + "scratch-untracked.tar": home / "data" / task / "cloud-scratch-untracked.tar", + } + declared = [name for name in scratch_destinations if name in artifacts] + if not declared: + raise AuthorityError("cloud return reports uncommitted work without retained scratch custody") + for name in declared: + descriptor = artifacts[name] + path = scratch_destinations[name] + if not isinstance(descriptor, dict): + raise AuthorityError("cloud return scratch custody descriptor is malformed") + if path.is_symlink() or not path.is_file() or path.stat().st_size > 128 * 1024 * 1024: + raise AuthorityError("cloud return scratch custody is absent, redirected, or oversized") + body = path.read_bytes() + if ( + descriptor.get("bytes") != len(body) + or descriptor.get("sha256") != hashlib.sha256(body).hexdigest() + ): + raise AuthorityError("cloud return scratch custody differs from the manifest") + return canonical({ + "result_digest": supplied, + "bundle_sha256": result["outcome_sha256"], + "manifest_sha256": result["return_manifest_sha256"], + "outcome_tip": tip, + "outcome_commits": commits, + "terminal": status_lines[-1], + }) + + def landing_evidence(worktree, repository_generation): # Only the canonical origin remote proves landing; a scratch or fork # remote-tracking ref must not count, and an unpushed local default @@ -873,7 +1017,7 @@ def main(): if worker["assignment_generation"] != args.assignment_generation: raise AuthorityError("worker assignment generation differs") kind_entries = values.get("kind", []) - if len(kind_entries) > 1: + if len(kind_entries) != 1: raise AuthorityError("task metadata kind identity is not exact") # WHICH evidence semantics apply is a release-safety decision, so it may # not rest on the task metadata alone: `kind` is a local, operator-writable @@ -885,6 +1029,16 @@ def main(): # released. Both directions refuse, fail closed, before any evidence runs. meta_kind = kind_entries[0] if kind_entries else "" worker_role = worker.get("role", "author") + worker_placement = worker.get("placement") + metadata_placement = values.get("placement", []) + if worker_placement == "azure": + if metadata_placement != ["azure"]: + raise AuthorityError("task metadata placement differs from the controller-owned worker placement") + elif worker_placement is None: + if metadata_placement: + raise AuthorityError("task metadata placement has no controller-owned worker authority") + else: + raise AuthorityError("controller-owned worker placement is unsupported") if meta_kind == "secondmate" and worker_role != "secondmate": raise AuthorityError( "task metadata claims a secondmate compartment but the controller-owned worker " @@ -893,6 +1047,8 @@ def main(): raise AuthorityError( "the controller-owned worker role is secondmate but the task metadata kind is " "{!r}; ordinary evidence is refused for a compartment".format(meta_kind)) + if worker_role != "secondmate" and meta_kind not in ("ship", "scout"): + raise AuthorityError("task metadata kind is not an exact ship or scout authority") if worker_role == "secondmate": # The secondmate compartment evidence mode (design B.7): same bundle, # same five receipt names, compartment semantics. The bundle still @@ -904,9 +1060,16 @@ def main(): home, args.task, worker, home_worktree, worker["bindings"]["repository_generation"]) else: worktree_info, worktree = worktree_evidence(args.task, values) - report_authority = lambda: report_evidence(home, args.task) - landing_authority = lambda: landing_evidence( - worktree, worker["bindings"]["repository_generation"]) + ordinary_kind = meta_kind if meta_kind in ("ship", "scout") else "ship" + report_authority = lambda: report_evidence(home, args.task, ordinary_kind) + if worker_placement == "azure": + landing_authority = lambda: cloud_return_evidence( + home, args.task, generation, args.assignment_generation, + ordinary_kind, worktree, worker["bindings"]["repository_generation"], + ) + else: + landing_authority = lambda: landing_evidence( + worktree, worker["bindings"]["repository_generation"]) authorities = { "endpoint": receipt("endpoint", args.task, generation, args.assignment_generation, endpoint_evidence(home, args.task, values)), "report": receipt("report", args.task, generation, args.assignment_generation, report_authority()), diff --git a/bin/fm-worker-lifecycle.py b/bin/fm-worker-lifecycle.py index cc7ff69bf79..990dfa30198 100755 --- a/bin/fm-worker-lifecycle.py +++ b/bin/fm-worker-lifecycle.py @@ -23,6 +23,7 @@ from pathlib import Path import re import shlex +import stat import subprocess import sys import tempfile @@ -32,12 +33,18 @@ ROOT = Path(__file__).resolve().parent.parent AZURE_PROVIDER = ROOT / "bin" / "fm-azure-worker-provider.py" +WORKER_SUPERVISOR = ROOT / "bin" / "fm-worker-supervisor.py" # The ONE implementation of "what is a Pi profile", "which upstream account is # it", and "how is a single-profile account home written". Placement imports it # rather than re-deriving any of the three: a second implementation of an # account home is exactly how a credential stager and its remover once resolved # different directories and leaked a credential. PI_ACCOUNT_HOME_TOOL = ROOT / "bin" / "fm-pi-account-home.py" +CREDENTIAL_EXPIRY_TOOL = ROOT / "bin" / "fm-credential-expiry.py" +# Azure's hard worker shutdown is six hours. Every canonical Pi profile must +# retain twice that headroom before the controller writes an assignment-private +# snapshot, so a guest never reaches the refresh path before the VM is dark. +CLOUD_ACCOUNT_MIN_HEADROOM_SECONDS = 12 * 60 * 60 LEGACY_STATE_SCHEMA = "fm.worker-lifecycle/v1" STATE_SCHEMA = "fm.worker-lifecycle/v2" # The scalar pending_action slot this schema carried is superseded by the @@ -376,13 +383,12 @@ def environment(): "daily_bound_override": daily_override, "idle_release_seconds": idle_release, "provider_argv": provider_argv, - # Where placement writes the single-profile account homes it leases. + # Where placement writes assignment-private single-profile snapshots. # CONTROLLER-owned, under the same state directory as the document that - # records the lease, and deliberately NOT the shared crosscheck roster - # under ~/.local/share/agent-fleet/accounts/pi: those homes belong to - # the reviewer lane, and a placement rewriting one mid-review would - # swap a running reviewer's credential underneath it. It also makes the - # root follow FM_HOME, so a fixture home cannot write into a real one. + # records each projection, and deliberately NOT a shared reviewer or + # worker pool home. Reusing one upstream profile therefore never makes + # two assignments share writable storage. The root follows FM_HOME so + # a fixture home cannot write into a real one. "pi_account_root": Path(os.environ.get( "FM_PI_ACCOUNT_HOME_ROOT", str(state_dir / "accounts") )).expanduser(), @@ -634,6 +640,9 @@ def load_state(env): state.setdefault("executions", {}) state.setdefault("pending_actions", {}) state.setdefault("revision", 0) + for worker in state["workers"].values(): + if isinstance(worker, dict): + worker.setdefault("placement", "azure") legacy = state.get("pending_action") if legacy is not None and legacy != LEGACY_PENDING_SENTINEL and not isinstance(legacy, dict): # The old binary refused this shape loudly; paving it over with the @@ -700,8 +709,8 @@ def verify_request(request): for field in ("home_binding", "account_binding", "worktree_binding", "repository_binding"): require_binding(field, request.get(field)) role = request.get("role") - if role not in ("author", "secondmate"): - raise LifecycleError("worker request role must be author or secondmate") + if role not in ("author", "secondmate", "no-mistakes"): + raise LifecycleError("worker request role must be author, secondmate, or no-mistakes") if request.get("owner_kind") not in ("primary", "secondmate"): raise LifecycleError("worker request owner_kind must be primary or secondmate") if role == "secondmate" and request.get("owner_kind") != "primary": @@ -711,6 +720,8 @@ def verify_request(request): raise LifecycleError( "a secondmate compartment is requested only by the primary; " "secondmates own author crewmates, never another secondmate") + if role == "no-mistakes" and request.get("owner_kind") != "primary": + raise LifecycleError("a no-mistakes worker is requested only by the primary") parent = request.get("parent_task") parent_generation = request.get("parent_task_generation") if (parent is None) != (parent_generation is None): @@ -742,14 +753,15 @@ def verify_request(request): raise LifecycleError("worker request account pool home must be one absolute path") profile = request.get("account_profile") account_home = request.get("account_home") - if (profile is None) != (account_home is None): - # The pair IS the lease record. Half of it would let a reader see a - # leased profile with no home to stage, or a staged home no exclusion - # covers; both read as "placed" while one of the two is missing. + projection_binding = request.get("account_projection_binding") + if (profile is None) != (account_home is None) or ( + profile is None and projection_binding is not None + ): raise LifecycleError( - "account_profile and account_home travel together or not at all") + "account_profile, account_home, and account_projection_binding travel together") if profile is not None: require_id("account_profile", profile) + require_binding("account_projection_binding", projection_binding) if not isinstance(account_home, str) or not account_home.startswith("/") \ or len(account_home) > 4096: raise LifecycleError("worker request account home must be one absolute path") @@ -795,51 +807,97 @@ def pi_projection(): return module +_CREDENTIAL_EXPIRY = {} + + +def credential_expiry(): + """Load the credential-expiry owner without copying its token semantics.""" + module = _CREDENTIAL_EXPIRY.get("module") + if module is not None: + return module + import importlib.util + + spec = importlib.util.spec_from_file_location( + "fm_credential_expiry", str(CREDENTIAL_EXPIRY_TOOL)) + if spec is None or spec.loader is None: + raise LifecycleError( + "the credential-expiry tool is unavailable at {}".format( + CREDENTIAL_EXPIRY_TOOL)) + module = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(module) + except Exception as exc: # noqa: BLE001 - any import failure is a refusal + raise LifecycleError( + "the credential-expiry tool could not be loaded from {}: {}".format( + CREDENTIAL_EXPIRY_TOOL, type(exc).__name__)) + _CREDENTIAL_EXPIRY["module"] = module + return module + + def placement_account_binding(account_digest): - """The lease identity, keyed on the UPSTREAM ACCOUNT. - - Not the profile name and not the account-home path, both of which are - handles that can point at the same account. Eight profiles map to eight - accounts today; nothing enforces that, and a re-login can point two slots - at one account. The thing a concurrent crewmate actually contends for - - the rate limit, the ban, the session - belongs to the account, so the - account is the unit of exclusion. `account_digest` is already a truncated - SHA-256 of the upstream account id and never carries token material. + """The reusable, non-secret identity of one upstream Pi account. + + The binding stays stable and travels through queue, worker, Azure, release, + and status records, but it is no longer an exclusion key. Multiple + assignments may use immutable snapshots of the same canonical profile + while retaining this exact provider-quota identity. """ return digest_value({"provider": "pi", "upstream_account": account_digest}) -def leased_placement_accounts(state): - """Which upstream accounts non-complete queue work already holds. +def leased_placement_accounts(state, pool_home=None): + """Active load per reusable profile/account pair, derived from the queue. - Derived from the queue, never from a separate ledger: the queue entry IS - the lease, so there is no second document that a crash could leave holding - a profile nothing owns. + The compatibility name remains because callers already know it, but the + result is load evidence rather than an exclusive lease set. Pool identity + is part of the key so equal local slot names in separate canonical homes do + not distort each other's selection. """ - held = {} + loads = {} for item in state["queue"].values(): if item.get("status") == "complete": continue + item_pool = item.get("account_pool_home") + profile = item.get("account_profile") binding = item.get("account_binding") - if isinstance(binding, str): - held.setdefault(binding, (item.get("account_profile"), item.get("task"))) - return held + if pool_home is not None and item_pool != str(pool_home): + continue + if not all(isinstance(value, str) and value for value in ( + item_pool, profile, binding, + )): + continue + key = (item_pool, profile, binding) + load = loads.setdefault(key, {"active": 0, "tasks": []}) + load["active"] += 1 + load["tasks"].append(item.get("task")) + return loads -def select_placement_account(env, state, pool_home, task): - """Lease one free Pi profile for this placement, under the controller lock. +def placement_projection_binding(item, profile, account_binding): + """Stable assignment-private writable-home identity for one request.""" + return digest_value({ + "provider": "pi", + "home_binding": item["home_binding"], + "task": item["task"], + "task_generation": item["task_generation"], + "account_pool_home": item["account_pool_home"], + "account_profile": profile, + "account_binding": account_binding, + }) - Called from `command_request` inside the same lock hold and the same - `save_state` that writes the queue entry, so selection and the lease are - one atomic act: no window exists in which a profile is held by anything - that is not a queue entry. - Fails closed at every step. An unreadable pool, a pool of unusable - credentials, and an exhausted pool all raise by name; none of them falls - through to a shared or arbitrary profile, because a silent fallthrough - here is an account collision with extra steps. +def select_placement_account(env, state, pool_home, item): + """Snapshot the least-loaded usable Pi profile for one placement. + + Selection and queue ownership remain inside one controller lock hold, and + projection begins only after that owner is durable. Reusable upstream + bindings are load evidence, while the writable projection is keyed by the + exact task generation and selected + profile so no two assignments ever share a home. """ projection = pi_projection() + expiry = credential_expiry() + pool_home = str(pool_home) pool_file = Path(pool_home) / "auth.json" try: pool = projection.read_pool(pool_file) @@ -850,136 +908,218 @@ def select_placement_account(env, state, pool_home, task): if not pool: raise LifecycleError( "provider-account placement pool at {} declares no profile".format(pool_file)) - held = leased_placement_accounts(state) - if len(pool) == 1: - # Already a single-profile account home - the exact shape every Pi - # consumer reads, and what the projection tool produces. There is - # nothing to select and nothing to write: lease it in place. Its SHAPE - # is deliberately not screened here, because projecting is the only - # operation that needs a writable credential and "is this credential - # still good" has one owner, bin/fm-credential-expiry.py. What IS - # required is the one thing exclusion depends on: an upstream account - # this lease can name. - name, entry = next(iter(pool.items())) - digest = projection.account_digest(entry) if isinstance(entry, dict) else "none" - if digest == "none": - raise LifecycleError( - "provider-account home at {} names no upstream account, so a placement " - "on it could not be excluded from any other".format(pool_file)) - binding = placement_account_binding(digest) - if binding in held: - raise LifecycleError( - "provider-account placement is exhausted: the single account home {} is " - "already leased ({} holds profile {}); refusing to place {} on a shared " - "upstream account. Sign in additional Pi profiles on distinct upstream " - "accounts to raise the ceiling, or release the placement above with " - "`bin/fm-worker-lifecycle.sh withdraw`".format( - pool_file, held[binding][1] or "unknown-task", - held[binding][0] or name, task)) - return { - "account_profile": name, - "account_home": str(Path(pool_home)), - "account_binding": binding, - } - usable = [] + + loads = leased_placement_accounts(state, pool_home) + deadline = time.time() + CLOUD_ACCOUNT_MIN_HEADROOM_SECONDS + candidates = [] faults = {} for name in sorted(pool): - entry_faults = projection.entry_faults(pool[name]) + entry = pool[name] + entry_faults = projection.entry_faults(entry) if entry_faults: faults[name] = "; ".join(entry_faults) continue - usable.append(name) - if not usable: + digest = projection.account_digest(entry) + if digest == "none": + faults[name] = "exposes no upstream account identity" + continue + # The expiry owner interprets the credential in memory BEFORE any + # assignment snapshot is written. A guest cannot refresh the + # canonical profile, and a token that would need refresh inside the + # worker lifetime never reaches a projection at all. + shaped = {projection.CONSUMER_KEY: entry} + if not expiry.credential_usable_through( + shaped, harness="pi", deadline=deadline, + ): + faults[name] = "lacks twelve hours of access-token headroom" + continue + binding = placement_account_binding(digest) + active = loads.get((pool_home, name, binding), {}).get("active", 0) + candidates.append((active, name, binding)) + if not candidates: raise LifecycleError( - "provider-account placement pool at {} holds no projectable profile ({})".format( + "provider-account placement pool at {} holds no usable profile ({})".format( pool_file, ", ".join("{}: {}".format(name, faults[name]) for name in sorted(faults)))) - bindings = {} - for name in usable: - digest = projection.account_digest(pool[name]) - if digest == "none": - # entry_faults already requires a non-blank accountId, so this is - # unreachable through the loop above; refuse rather than mint a - # lease identity that names no account. - raise LifecycleError( - "Pi profile {} exposes no upstream account identity".format(name)) - # Deliberately setdefault, not assignment: two profiles that resolve to - # ONE upstream account are one lease, and the first name in sorted - # order owns it. That is the whole reason the unit is the account. - bindings.setdefault(placement_account_binding(digest), name) - for binding, name in sorted(bindings.items(), key=lambda pair: pair[1]): - if binding in held: + + # Least active first, then the stable local profile label, then the stable + # upstream digest. Every usable profile is represented before any is + # reused, and equal loads converge on the same choice after a restart. + _, name, binding = min(candidates) + projection_binding = placement_projection_binding(item, name, binding) + root = Path(env["pi_account_root"]).resolve() + destination = root / projection_binding + for active in state["queue"].values(): + if active.get("status") == "complete": continue - root = Path(env["pi_account_root"]).resolve() - # The projected home is keyed on the LEASE IDENTITY, never on the - # profile's local slot name. The two must be the same function of the - # pool or the projection is not injective over live leases, and the - # slot name is not: an operator re-logging slot `openai-codex` from one - # upstream account to another gives two placements two distinct - # bindings (correctly, they ARE two accounts) that both project into - # `accounts/openai-codex`, so the second write silently replaces the - # credential the first placement's still-live lease points at. The - # queue then reports two accounts while the disk holds one. Keying on - # the binding makes the directory name and the exclusion key the same - # string, so that state is unrepresentable. - destination = root / binding - for entry in state["queue"].values(): - if entry.get("status") == "complete": - continue - if entry.get("account_home") == str(destination): - # Unreachable while the two keys agree, because a binding that - # is free by definition is named by no live entry. Kept as the - # second line: if a future change re-keys the projection, this - # refuses instead of clobbering a live placement's credential. - raise LifecycleError( - "refusing to project Pi profile {} over the account home a live " - "placement already holds ({} holds {})".format( - name, entry.get("task") or "an unnamed task", destination)) - try: - projection.prepare_root(root) - credential = projection.write_home(destination, pool[name]) - except projection.ProjectionError as exc: - raise LifecycleError( - "Pi profile {} could not be projected into its account home: {}".format( - name, exc)) - except OSError as exc: + if ( + active.get("account_projection_binding") == projection_binding + or active.get("account_home") == str(destination) + ): raise LifecycleError( - "Pi profile {} could not be projected into its account home: {}".format( - name, exc.strerror or exc)) - return { - "account_profile": name, - "account_home": str(Path(credential).parent), - "account_binding": binding, - } - raise LifecycleError( - "provider-account placement is exhausted: all {} distinct upstream accounts in {} " - "are leased ({}); refusing to place {} on a shared upstream account. Sign in " - "additional Pi profiles on distinct upstream accounts to raise the ceiling, or " - "release a placement above with `bin/fm-worker-lifecycle.sh withdraw`".format( - len(bindings), pool_file, - ", ".join( - "{} -> {}".format(name, held[binding][1] or "unknown-task") - for binding, name in sorted(bindings.items(), key=lambda pair: pair[1]) - ), - task)) + "assignment-private provider projection is already owned by {}".format( + active.get("task") or "an unnamed task")) + return { + "account_profile": name, + "account_home": str(destination), + "account_binding": binding, + "account_projection_binding": projection_binding, + } + + +def write_placement_snapshot(env, item): + """Write or replay one queue-owned canonical-profile snapshot.""" + projection = pi_projection() + expiry = credential_expiry() + pool_home = item.get("account_pool_home") + profile = item.get("account_profile") + if not isinstance(pool_home, str) or not isinstance(profile, str): + raise LifecycleError("provider-account snapshot source identity is unavailable") + pool_file = Path(pool_home) / "auth.json" + try: + pool = projection.read_pool(pool_file) + except projection.ProjectionError as exc: + raise LifecycleError( + "provider-account snapshot source is unusable: {}".format(exc)) + entry = pool.get(profile) + if entry is None or projection.entry_faults(entry): + raise LifecycleError( + "selected Pi profile {} is no longer usable for its snapshot".format(profile)) + account_digest = projection.account_digest(entry) + if placement_account_binding(account_digest) != item.get("account_binding"): + raise LifecycleError( + "selected Pi profile {} changed upstream identity before snapshot".format(profile)) + if not expiry.credential_usable_through( + {projection.CONSUMER_KEY: entry}, harness="pi", + deadline=time.time() + CLOUD_ACCOUNT_MIN_HEADROOM_SECONDS, + ): + raise LifecycleError( + "selected Pi profile {} lacks twelve hours of access-token headroom".format( + profile)) + projected = placement_projection_path(env, item) + if projected is None: + raise LifecycleError("assignment-private provider projection identity is unavailable") + root, destination = projected + try: + projection.prepare_root(root) + credential = projection.write_home(destination, entry) + except projection.ProjectionError as exc: + raise LifecycleError( + "Pi profile {} could not be snapshotted into its assignment-private home: {}".format( + profile, exc)) + except OSError as exc: + raise LifecycleError( + "Pi profile {} could not be snapshotted into its assignment-private home: {}".format( + profile, exc.strerror or exc)) + if str(Path(credential).parent) != item["account_home"]: + raise LifecycleError("provider-account snapshot destination changed identity") def ensure_unique_bindings(state, candidate, ignore_key=None): + """Refuse shared writable custody while allowing reusable account identity.""" for key, item in state["queue"].items(): if key == ignore_key or item.get("status") == "complete": continue - if item.get("account_binding") == candidate["account_binding"]: - # The account-collision screen. It is the SAME screen selection - # already respected; keeping it means a hand-edited queue, a - # replayed old binary, or a broken selector still cannot seat two - # concurrent tasks on one upstream account. + candidate_projection = candidate.get("account_projection_binding") + if candidate_projection is not None and ( + item.get("account_projection_binding") == candidate_projection + or item.get("account_home") == candidate.get("account_home") + ): raise LifecycleError( - "provider-account lease binding is already owned by another queued or " - "active task ({} holds profile {})".format( - item.get("task") or "an unnamed task", - item.get("account_profile") or "an unnamed profile")) + "assignment-private provider projection is already owned by another " + "queued or active task") if item.get("worktree_binding") == candidate["worktree_binding"]: - raise LifecycleError("writable worktree binding is already owned by another queued or active task") + raise LifecycleError( + "writable worktree binding is already owned by another queued or active task") + + +def placement_projection_path(env, item): + """Return one new-style projection path, never a legacy shared home.""" + binding = item.get("account_projection_binding") + account_home = item.get("account_home") + if binding is None: + # Existing assignments created before reusable snapshots may point at a + # canonical single-profile home or an account-keyed projection. This + # release must never infer ownership of either and delete it. + return None + require_binding("account projection binding", binding) + root = Path(env["pi_account_root"]).resolve() + expected = root / binding + if account_home != str(expected): + raise LifecycleError( + "assignment-private provider projection path differs from its binding") + return root, expected + + +def cleanup_placement_projection(env, item): + """Remove exactly one assignment-private snapshot without traversing peers.""" + projected = placement_projection_path(env, item) + if projected is None: + return + root, destination = projected + try: + root_info = root.lstat() + except FileNotFoundError: + return + except OSError as exc: + raise LifecycleError( + "provider projection root is unreadable during cleanup: {}".format( + exc.strerror or exc)) + if not stat.S_ISDIR(root_info.st_mode) or root.is_symlink(): + raise LifecycleError("provider projection root changed identity during cleanup") + try: + destination_info = destination.lstat() + except FileNotFoundError: + return + except OSError as exc: + raise LifecycleError( + "assignment-private provider projection is unreadable during cleanup: {}".format( + exc.strerror or exc)) + if ( + not stat.S_ISDIR(destination_info.st_mode) + or destination.is_symlink() + or destination_info.st_uid != os.geteuid() + or destination_info.st_mode & (stat.S_IWGRP | stat.S_IWOTH) + ): + raise LifecycleError( + "assignment-private provider projection changed identity during cleanup") + try: + entries = list(os.scandir(destination)) + except OSError as exc: + raise LifecycleError( + "assignment-private provider projection cannot be inventoried: {}".format( + exc.strerror or exc)) + if len(entries) > 8: + raise LifecycleError( + "assignment-private provider projection holds unexpected cleanup state") + for entry in entries: + if entry.name != "auth.json" and not ( + entry.name.startswith(".auth-") and entry.name.endswith(".tmp") + ): + raise LifecycleError( + "assignment-private provider projection holds unexpected cleanup state") + info = entry.stat(follow_symlinks=False) + if ( + not stat.S_ISREG(info.st_mode) + or stat.S_ISLNK(info.st_mode) + or info.st_uid != os.geteuid() + or info.st_mode & (stat.S_IRWXG | stat.S_IRWXO) + ): + raise LifecycleError( + "assignment-private provider credential changed identity during cleanup") + try: + for entry in entries: + os.unlink(entry.path) + os.rmdir(destination) + descriptor = os.open(str(root), os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + except OSError as exc: + raise LifecycleError( + "assignment-private provider projection cleanup failed: {}".format( + exc.strerror or exc)) def provider_action_timeout(action): @@ -1791,6 +1931,7 @@ def create_worker_record(env, state, slot, item, reservation): sku, family = SKU_PLAN[slot] record = { "slot": slot, + "placement": "azure", "role": item.get("role", "author"), "sku": sku, "sku_family": family, @@ -1969,7 +2110,15 @@ def apply_pending(env, action, result): claimed = state["pending_actions"].get(slot) if not isinstance(claimed, dict) or claimed.get("idempotency_key") != action["idempotency_key"]: raise LifecycleError("durable claim for slot {} is no longer this action".format(slot)) + worker = state.get("workers", {}).get(slot) or {} + queue_key = worker.get("queue_key") apply_result_transactionally(env, state, action, result) + if action.get("type") == "reset" and queue_key is not None: + # Provider reset already proved cloud-side absence. Retire only this + # queue owner's host snapshot before making completion durable. A + # crash after unlink but before save is an idempotent missing-path + # retry; no same-profile peer path is ever inventoried. + cleanup_placement_projection(env, state["queue"].get(queue_key) or {}) state["pending_actions"].pop(slot, None) save_state(env, state) return state @@ -2129,7 +2278,8 @@ def apply_action_result(env, state, action, result): ): if execution.get(field) != expected: raise LifecycleError("provider execution {} binding differs".format(field)) - if (action.get("request") or {}).get("outcome_expected"): + execution_request = action.get("request") or {} + if execution_request.get("outcome_expected"): # A worker whose pinned supervisor predates the outcome contract # would run the command and answer with no outcome fields at all. # Refusing here turns that version skew into a visible failure @@ -2138,6 +2288,63 @@ def apply_action_result(env, state, action, result): raise LifecycleError( "provider execution reports no outcome disposition for a landing task" ) + if execution_request.get("return_contract"): + if execution.get("return_present") is not True: + raise LifecycleError( + "provider execution reports no authorized return artifact bundle" + ) + expected_return_ref = "refs/fm-return/{}".format( + action["request_digest"][:32] + ) + if execution.get("return_ref") != expected_return_ref: + raise LifecycleError("provider execution return ref is not exact") + for field in ("return_commit", "outcome_tip"): + if not re.fullmatch(r"[0-9a-f]{40}", str(execution.get(field))): + raise LifecycleError( + "provider execution return {} is malformed".format(field) + ) + if not re.fullmatch( + r"[0-9a-f]{64}", str(execution.get("return_manifest_sha256")) + ): + raise LifecycleError( + "provider execution return return_manifest_sha256 is malformed" + ) + commits = execution.get("outcome_commits") + if not isinstance(commits, int) or isinstance(commits, bool) or commits < 0: + raise LifecycleError("provider execution outcome commit count is malformed") + if execution.get("outcome_present") is not (commits > 0): + raise LifecycleError( + "provider execution outcome presence differs from its commit count" + ) + if not isinstance(execution.get("outcome_uncommitted_changes"), bool): + raise LifecycleError( + "provider execution working-tree disposition is malformed" + ) + if execution_request.get("service_return_contract"): + present = execution.get("service_return_present") + if not isinstance(present, bool): + raise LifecycleError( + "provider execution reports no no-mistakes service return disposition") + if present: + if execution.get("return_present") is not True: + raise LifecycleError("no-mistakes service artifact has no return bundle") + expected_return_ref = "refs/fm-return/{}".format( + action["request_digest"][:32]) + if execution.get("return_ref") != expected_return_ref: + raise LifecycleError("no-mistakes service return ref is not exact") + for field in ( + "return_commit", "outcome_tip", "return_manifest_sha256", + "step_outcome_sha256", + ): + if not re.fullmatch(r"[0-9a-f]{40}" if field in ( + "return_commit", "outcome_tip") else r"[0-9a-f]{64}", + str(execution.get(field)), + ): + raise LifecycleError( + "no-mistakes service return {} is malformed".format(field)) + elif execution.get("step_outcome_sha256") not in (None, ""): + raise LifecycleError( + "absent no-mistakes service return asserted a step outcome digest") state["executions"][action["request_digest"]] = execution worker["last_execution_digest"] = supplied worker["last_execution_at"] = iso_utc() @@ -2554,6 +2761,46 @@ def status_projection(env, state, inventory=None): for worker in state["workers"].values() if worker.get("idle_deallocated_at") and not worker.get("released_at") ] + selected_loads = leased_placement_accounts(state) + profile_loads = {} + for (_, profile, binding), load in selected_loads.items(): + aggregate = profile_loads.setdefault( + (profile, binding), {"active": 0, "tasks": []}) + aggregate["active"] += load["active"] + aggregate["tasks"].extend(load["tasks"]) + account_placements = [] + for item in state["queue"].values(): + if item.get("status") == "complete" or not item.get("account_profile"): + continue + load_key = (item.get("account_profile"), item.get("account_binding")) + account_placements.append({ + "task": item.get("task"), + "task_generation": item.get("task_generation"), + "status": item.get("status"), + "account_profile": item.get("account_profile"), + "account_binding": item.get("account_binding"), + "account_home": item.get("account_home"), + "account_projection_binding": item.get("account_projection_binding"), + "assignment_generation": ( + state["workers"].get(str(item.get("slot")), {}).get("assignment_generation") + if item.get("slot") is not None else None + ), + "profile_active_load": profile_loads.get(load_key, {}).get("active", 0), + }) + account_placements.sort(key=lambda entry: ( + entry["account_profile"], entry.get("account_binding") or "", + entry["task"] or "", + )) + account_profile_loads = [ + { + "account_profile": profile, + "account_binding": binding, + "active_placements": load["active"], + } + for (profile, binding), load in sorted( + profile_loads.items(), key=lambda pair: (pair[0][0], pair[0][1]) + ) + ] return { "schema": "fm.worker-status/v1", "daily_bound_usd": env["daily_bound_usd"], @@ -2592,23 +2839,11 @@ def status_projection(env, state, inventory=None): "family_observed_plus_reserved_vcpus": family_committed, "shared_headroom_vcpus": SHARED_HEADROOM_VCPUS, "compartments": compartment_projection(state), - # Who holds which provider account right now. Read straight off the - # queue, because the queue entry IS the lease: a profile that shows - # here and nowhere else does not exist. - "account_placements": sorted( - ( - { - "task": item.get("task"), - "task_generation": item.get("task_generation"), - "status": item.get("status"), - "account_profile": item.get("account_profile"), - "account_home": item.get("account_home"), - } - for item in state["queue"].values() - if item.get("status") != "complete" and item.get("account_profile") - ), - key=lambda entry: (entry["account_profile"], entry["task"] or ""), - ), + # Every placement remains visible even when several use one upstream + # identity. The reusable digest and per-profile active load expose + # provider-quota pressure without raw account identity or token bytes. + "account_placements": account_placements, + "account_profile_loads": account_profile_loads, "idle_cooldown_seconds": env["cooldown_seconds"], "warm_idle_target": env["warm_idle"], "retained_disks": retained_disks, @@ -2660,12 +2895,22 @@ def print_status(status, json_output): if "session_legs" in compartment: line += " legs={}".format(compartment["session_legs"]) print(line) - for placement in status.get("account_placements") or []: - print("account-placement: profile={} task={}@{} status={} home={}".format( - placement["account_profile"], placement["task"], - placement["task_generation"], placement["status"], - placement["account_home"], + for load in status.get("account_profile_loads") or []: + print("account-profile-load: profile={} active={} account-binding={}".format( + load["account_profile"], load["active_placements"], + load["account_binding"], )) + for placement in status.get("account_placements") or []: + print( + "account-placement: profile={} load={} task={}@{} status={} " + "account-binding={} projection={} home={}".format( + placement["account_profile"], placement["profile_active_load"], + placement["task"], placement["task_generation"], + placement["status"], placement["account_binding"], + placement.get("account_projection_binding") or "legacy", + placement["account_home"], + ) + ) if status["pending_mutations"]: print("pending-mutations: {}".format(json.dumps( status["pending_mutations"], sort_keys=True, separators=(",", ":")))) @@ -2704,7 +2949,8 @@ def parser(): request.add_argument("--repository-binding", help=argparse.SUPPRESS) request.add_argument("--repository-generation", help=argparse.SUPPRESS) request.add_argument("--owner-kind", choices=("primary", "secondmate"), required=True) - request.add_argument("--role", choices=("author", "secondmate"), default="author") + request.add_argument( + "--role", choices=("author", "secondmate", "no-mistakes"), default="author") request.add_argument("--parent-task", default=None) request.add_argument("--parent-task-generation", default=None) request.add_argument( @@ -2715,7 +2961,7 @@ def parser(): request.add_argument("--required", action="store_true", help="mark non-discretionary recovery/landing work") withdraw_parser = sub.add_parser( - "withdraw", help="retire one exact queued request no worker ever took") + "withdraw", help="retire one exact projecting/queued request no worker ever took") withdraw_parser.add_argument("--task", required=True) withdraw_parser.add_argument("--task-generation", required=True) withdraw_parser.add_argument("--confirm-withdraw", action="store_true") @@ -2811,6 +3057,11 @@ def parser(): execute.add_argument("--payload-dir", default=None) execute.add_argument("--account-dir", default=None) execute.add_argument("--outcome-dir", default=None) + execute.add_argument("--return-kind", choices=("ship", "scout"), default=None) + execute.add_argument( + "--existing-task-disk", action="store_true", + help="continue or collect an assigned task disk without replacing its repository", + ) execute.add_argument("--confirm-execute", action="store_true") execute.add_argument("--confirm-subscription", required=True) execute.add_argument("argv", nargs=argparse.REMAINDER) @@ -2826,6 +3077,16 @@ def parser(): release.add_argument("--task-generation", required=True) release.add_argument("--proof-file", required=True) + service_complete = sub.add_parser( + "service-complete", + help="release one no-mistakes service worker after its exact execution is recorded", + ) + service_complete.add_argument("--task", required=True) + service_complete.add_argument("--task-generation", required=True) + service_complete.add_argument("--assignment-generation", required=True) + service_complete.add_argument("--request-digest", required=True) + service_complete.add_argument("--confirm-subscription", required=True) + resume = sub.add_parser("resume", help="reattach exact retained dirty task capacity") resume.add_argument("--task", required=True) resume.add_argument("--task-generation", required=True) @@ -2937,12 +3198,10 @@ def exactly(key): raise LifecycleError("ordinary worktree Git-directory identity differs") return { "home_binding": home_binding(origin), - # The POOL, not the lease. The task's own metadata proves which provider - # account source this task is entitled to draw from; WHICH profile of - # that pool it gets is decided by the controller under its lock, because - # that decision has to exclude every other concurrent placement and no - # task-local document can see them. The account lease identity - # (`account_binding`) is minted from the selected profile in + # The POOL, not one snapshot. The task metadata proves which canonical + # host-owned source it may draw from; the controller selects the + # least-loaded usable profile under its lock and mints a reusable + # account binding plus an assignment-private projection in # `command_request`. "account_pool_home": str(account_home), "worktree_binding": digest_value({"worktree": str(worktree), "git_dir": str(git_dir)}), @@ -3186,10 +3445,9 @@ def command_request(env, args): else: bindings = authoritative_request_bindings( env, args.task, args.task_generation, task_home=task_home) - # DURABLE on the item, not consumed here: the queue entry should record - # which provider-account pool its lease was drawn from, so an audit of a - # placement never has to re-read a task metadata file that teardown may - # already have removed. + # DURABLE on the item, not consumed here: the queue entry records which + # canonical provider-account pool its immutable snapshot came from, so an + # audit never has to re-read task metadata teardown may already have removed. pool_home = bindings.get("account_pool_home") item = { "schema": REQUEST_SCHEMA, @@ -3221,9 +3479,9 @@ def command_request(env, args): state = load_state(env) existing = state["queue"].get(key) if existing is not None: - # Replay reuses the SAME profile, because the lease it took is this - # very entry. Selection happens only on the branch that creates a - # new entry, so a replayed request cannot consume a second account. + # Replay reuses the SAME profile and assignment-private snapshot, + # because selection runs only on the branch that creates this + # entry. A replay cannot consume another load-balanced placement. identity_fields = ( "schema", "task", "task_generation", "home_binding", "worktree_binding", "repository_binding", "repository_generation", @@ -3235,21 +3493,30 @@ def command_request(env, args): identity_fields += ("account_binding",) if any(existing.get(field) != item.get(field) for field in identity_fields): raise LifecycleError("task generation already exists with different queue identity") + if existing.get("status") == "projecting": + # A crash may interrupt the snapshot write, but never before + # the queue owns its exact path. The same request resumes from + # that state, retaining profile, upstream binding, and private + # projection identity. + write_placement_snapshot(env, existing) + existing["status"] = "queued" + existing["projected_at"] = iso_utc() + save_state(env, state) if existing.get("account_home"): - # The profile NAME is reported separately because the home is - # keyed on the lease identity, not on the name: the caller can - # no longer read the profile off the path's last component. + # The profile name is reported separately because the home is + # keyed on an assignment-private projection binding, not on the + # reusable local label or upstream account digest. print("account-profile {}".format(existing.get("account_profile") or "")) print("account-home {}".format(existing["account_home"])) print("request already exists with exact identity") return if pool_home is not None: - # Selection and the lease are ONE act under ONE lock over ONE - # document: the queue entry written below IS the lease, so no - # window exists where a profile is held by something the queue does - # not show, and no concurrent request can read the same free set. - item.update(select_placement_account(env, state, pool_home, item["task"])) - verify_request(item) + # Selection is one act under one lock over one document. The + # durable `projecting` state is saved before credential bytes are + # written, so a crash leaves a resumable owner rather than an + # orphan snapshot. + item.update(select_placement_account(env, state, pool_home, item)) + verify_request(item) ensure_unique_bindings(state, item) if item.get("parent_task") is not None: if task_home is not None: @@ -3260,24 +3527,31 @@ def command_request(env, args): if item.get("role") == "secondmate": active_compartments = sum( 1 for entry in state["queue"].values() - if entry.get("role") == "secondmate" and entry.get("status") != "complete" + if entry.get("role") == "secondmate" + and entry.get("status") != "complete" ) if active_compartments >= env["secondmate_max"]: raise LifecycleError( "secondmate compartment cap reached ({} active, cap {})".format( active_compartments, env["secondmate_max"])) + if pool_home is not None: + item["status"] = "projecting" state["queue"][key] = item if item.get("parent_task") is not None: parent_key = request_key(item["parent_task"], item["parent_task_generation"]) parent_worker = state["workers"][str(state["queue"][parent_key].get("slot"))] parent_worker["children_total"] = int(parent_worker.get("children_total", 0)) + 1 save_state(env, state) + if pool_home is not None: + write_placement_snapshot(env, item) + item["status"] = "queued" + item["projected_at"] = iso_utc() + save_state(env, state) if item.get("account_home"): - # The caller stages the provider credential from the home the - # controller leased, so the leased profile is the ONE credential that - # reaches the worker. Printed as a path and a slot name, never as - # contents; the home is keyed on the lease identity, so the slot name - # is not recoverable from the path. + # The caller stages the provider credential from this request's + # assignment-private snapshot. Printed as a path and slot name, never + # as contents; another placement using the same profile has another + # path and cleanup authority. print("account-profile {}".format(item.get("account_profile") or "")) print("account-home {}".format(item["account_home"])) print("queued {} generation {} for one isolated author worker".format(item["task"], item["task_generation"])) @@ -3816,6 +4090,11 @@ def command_capacity_retire_fence(env, args): "fm-secondmate-session.py", "fm-secondmate-spawn.pi-ext.ts", ) +NO_MISTAKES_PAYLOAD_FILE_BOUNDS = { + **PAYLOAD_FILE_BOUNDS, + "runtime.tar.gz": 1024 * 1024 * 1024, +} +NO_MISTAKES_PAYLOAD_REQUIRED = PAYLOAD_REQUIRED + ("runtime.tar.gz",) ACCOUNT_TOTAL_BOUND = 1024 * 1024 @@ -3829,6 +4108,8 @@ def payload_contract(role): """ if role == "secondmate": return COMPARTMENT_PAYLOAD_FILE_BOUNDS, COMPARTMENT_PAYLOAD_REQUIRED + if role == "no-mistakes": + return NO_MISTAKES_PAYLOAD_FILE_BOUNDS, NO_MISTAKES_PAYLOAD_REQUIRED return PAYLOAD_FILE_BOUNDS, PAYLOAD_REQUIRED @@ -3886,8 +4167,14 @@ def command_execute(env, args): raise LifecycleError("execution wall deadline must be between 1 and 21600 seconds") if (args.payload_dir is None) != (args.account_dir is None): raise LifecycleError("payload and account staging directories travel together or not at all") - if args.outcome_dir is not None and args.payload_dir is None: - raise LifecycleError("an outcome can only be collected from a staged repository") + if args.existing_task_disk and args.payload_dir is not None: + raise LifecycleError("existing task-disk recovery cannot replace payload or account state") + if args.outcome_dir is not None and args.payload_dir is None and not args.existing_task_disk: + raise LifecycleError("an outcome can only be collected from a staged repository or an explicitly retained repository") + if args.return_kind is not None and args.outcome_dir is None: + raise LifecycleError("an authorized task return requires an outcome directory") + if args.existing_task_disk and (args.outcome_dir is None or args.return_kind is None): + raise LifecycleError("existing task-disk recovery requires an authorized return outcome") if args.outcome_dir is not None: outcome_root = Path(args.outcome_dir) if outcome_root.is_symlink() or not outcome_root.is_dir(): @@ -3938,11 +4225,40 @@ def command_execute(env, args): if payload_manifest is not None: request["payload_files"] = payload_manifest request["account_files"] = account_manifest + if args.existing_task_disk: + try: + supervisor_body = WORKER_SUPERVISOR.read_bytes() + except OSError as exc: + raise LifecycleError( + "existing task-disk recovery supervisor is unreadable: {}".format(exc) + ) from None + request["existing_task_disk"] = True + request["supervisor_sha256"] = hashlib.sha256(supervisor_body).hexdigest() if args.outcome_dir is not None: # Digest-bound, so withholding the staging URL downstream cannot # silently turn a landing task into a fire-and-forget one: the # guest refuses instead. request["outcome_expected"] = True + if args.return_kind is not None: + report_name = "completion.md" if args.return_kind == "ship" else "report.md" + request["return_contract"] = { + "schema": "fm.worker-return-contract/v1", + "kind": args.return_kind, + "report_required": True, + "report_path": "data/{}/{}".format(args.task, report_name), + "status_path": "state/{}.status".format(args.task), + "visuals_path": "data/{}/visuals".format(args.task), + "branch": "fm/{}".format(args.task) if args.return_kind == "ship" else "", + } + if worker.get("role") == "no-mistakes": + if args.outcome_dir is None: + raise LifecycleError("a no-mistakes execution requires an outcome directory") + request["worker_role"] = "no-mistakes" + request["service_return_contract"] = { + "schema": "fm.no-mistakes-worker-return/v1", + "step_outcome_path": "outcome.json", + "step_outcome_max_bytes": 1024 * 1024, + } request["request_digest"] = digest_value(request) existing = state["executions"].get(request["request_digest"]) if existing is not None: @@ -4043,8 +4359,69 @@ def command_release(env, args): print("release proofs recorded; only exact idle capacity is now eligible for deallocation and reset") +def command_service_complete(env, args): + """Release a service assignment from execution evidence the lifecycle owns. + + Ordinary crewmates return through task reports and landing receipts. A + no-mistakes worker instead returns a digest-bound service envelope to its + controller, so asking it to manufacture ordinary task authority would be + ceremony and, worse, a false claim. This narrow role-owned boundary only + accepts an execution already stored under the exact assigned worker. + """ + if args.confirm_subscription != env["subscription"]: + raise LifecycleError( + "--confirm-subscription must exactly match FM_AZURE_SUBSCRIPTION_ID") + request_digest = require_binding("execution request digest", args.request_digest) + with controller_lock(env): + state = load_state(env) + key = request_key( + require_id("task", args.task), + require_id("task generation", args.task_generation), + ) + item = state["queue"].get(key) + worker = state["workers"].get(str((item or {}).get("slot"))) + if item is None or item.get("status") != "assigned" or worker is None: + raise LifecycleError("service completion requires one exact assigned worker") + if item.get("role") != "no-mistakes" or worker.get("role") != "no-mistakes": + raise LifecycleError("service completion is owned by no-mistakes workers only") + if worker.get("assignment_generation") != args.assignment_generation: + raise LifecycleError("service completion assignment generation is not exact") + execution = state["executions"].get(request_digest) + if not isinstance(execution, dict): + raise LifecycleError("service completion has no exact recorded execution") + if ( + execution.get("request_digest") != request_digest + or execution.get("result_digest") != worker.get("last_execution_digest") + or execution.get("assignment_generation") != args.assignment_generation + ): + raise LifecycleError("service completion execution identity differs") + proof = { + "schema": "fm.worker-service-release/v1", + **worker["bindings"], + "assignment_generation": args.assignment_generation, + "cloud_instance_id": worker["cloud_instance_id"], + "resources": worker["resources"], + "request_digest": request_digest, + "result_digest": execution["result_digest"], + "verdict": "proved", + } + proof["proof_digest"] = digest_value(proof) + held = worker.get("release_proof") + if held is not None: + if held != proof: + raise LifecycleError("worker already has a different service release proof") + print("service release proof already recorded with exact identity") + return + worker["release_proof"] = proof + worker["released_at"] = iso_utc() + worker["phase"] = "release-proved" + item["status"] = "releasing" + save_state(env, state) + print("service execution proved; exact idle capacity is eligible for cleanup") + + def command_withdraw(env, args): - """Retire a queued request that no worker ever took. + """Retire a projecting or queued request that no worker ever took. A task can finish, be cancelled, or be superseded locally long before any cloud capacity is built for it, and its queue entry then keeps counting as @@ -4052,6 +4429,9 @@ def command_withdraw(env, args): already done. That is not merely wasted spend. Re-running a task that has side effects outside this fleet, posting or sending or filing, repeats them. + It also retires a `projecting` request whose process died after the queue + took ownership but before the assignment-private snapshot completed. + `release` cannot cover this: it requires an ASSIGNED item with a durable worker owner and a release proof describing that worker. An entry that was never assigned has neither, so before this command the only way to clear one @@ -4072,8 +4452,8 @@ def command_withdraw(env, args): if item is None: raise LifecycleError("withdraw requires one exact queued task generation") status = item.get("status") - if status != "queued": - # Anything past `queued` has cloud capacity or a live assignment + if status not in ("queued", "projecting"): + # Anything past the queue has cloud capacity or a live assignment # behind it, and dropping the entry would strand that worker with no # queue owner. Those go out through release. # `release` only accepts `assigned`, so pointing a `complete` or @@ -4113,6 +4493,11 @@ def command_withdraw(env, args): pending.get("type", "provider"), slot_key) ) withdrawn = item + # This request never held provider capacity, so its projection is the + # only controller-owned assignment artifact. Remove exactly that + # private directory before deleting its durable owner; a same-profile + # request lives at another projection binding and is not inspected. + cleanup_placement_projection(env, withdrawn) del state["queue"][key] save_state(env, state) # A machine-readable receipt naming the exact entry that was deleted. The @@ -4858,6 +5243,8 @@ def main(argv=None): command_proof_template(env, args) elif args.command == "release": command_release(env, args) + elif args.command == "service-complete": + command_service_complete(env, args) elif args.command == "withdraw": command_withdraw(env, args) elif args.command == "surrender": diff --git a/bin/fm-worker-lifecycle.sh b/bin/fm-worker-lifecycle.sh index 4bb2b918bac..7784382a5cc 100755 --- a/bin/fm-worker-lifecycle.sh +++ b/bin/fm-worker-lifecycle.sh @@ -33,7 +33,7 @@ # fm-worker-lifecycle.sh capacity-reserve # fm-worker-lifecycle.sh capacity-reserve-shape # fm-worker-lifecycle.sh capacity-release -# fm-worker-lifecycle.sh execute -- +# fm-worker-lifecycle.sh execute [--existing-task-disk --return-kind --outcome-dir ] -- # fm-worker-lifecycle.sh authority-receipt --output # fm-worker-lifecycle.sh proof-template --task --task-generation # fm-worker-lifecycle.sh release --task --task-generation --proof-file diff --git a/bin/fm-worker-supervisor.py b/bin/fm-worker-supervisor.py index b648d908776..63961228fe3 100755 --- a/bin/fm-worker-supervisor.py +++ b/bin/fm-worker-supervisor.py @@ -83,10 +83,73 @@ def read_request(path): outcome_expected = request.get("outcome_expected", False) if not isinstance(outcome_expected, bool): raise SupervisorError("execution outcome expectation is malformed") + existing_task_disk = request.get("existing_task_disk", False) + if not isinstance(existing_task_disk, bool): + raise SupervisorError("execution existing task-disk disposition is malformed") + if existing_task_disk: + if "payload_files" in request or "account_files" in request: + raise SupervisorError("existing task-disk recovery cannot replace staged state") + supervisor_digest = request.get("supervisor_sha256") + if not isinstance(supervisor_digest, str) or not HEX.fullmatch(supervisor_digest): + raise SupervisorError("existing task-disk recovery supervisor binding is malformed") + try: + running_digest = hashlib.sha256(Path(__file__).read_bytes()).hexdigest() + except OSError as exc: + raise SupervisorError("existing task-disk recovery supervisor is unreadable: {}".format(exc)) + if running_digest != supervisor_digest: + raise SupervisorError("existing task-disk recovery supervisor binding differs") + elif "supervisor_sha256" in request: + raise SupervisorError("ordinary execution cannot select a recovery supervisor") + return_contract = request.get("return_contract") + if return_contract is not None: + if not isinstance(return_contract, dict) or set(return_contract) != { + "schema", "kind", "report_required", "report_path", "status_path", + "visuals_path", "branch", + }: + raise SupervisorError("execution return contract is malformed") + if return_contract.get("schema") != "fm.worker-return-contract/v1": + raise SupervisorError("execution return contract schema is not supported") + if return_contract.get("kind") not in ("ship", "scout"): + raise SupervisorError("execution return kind is not supported") + if return_contract.get("report_required") is not True: + raise SupervisorError("execution return contract must require its task report") + for field in ("report_path", "status_path", "visuals_path"): + value = return_contract.get(field) + if ( + not isinstance(value, str) or not value or value.startswith("/") + or ".." in Path(value).parts or "\x00" in value + ): + raise SupervisorError("execution return {} is unsafe".format(field)) + branch = return_contract.get("branch") + if return_contract["kind"] == "ship": + if not isinstance(branch, str) or not branch.startswith("fm/") or not SAFE_ID.fullmatch(branch[3:]): + raise SupervisorError("execution return branch is malformed") + elif branch != "": + raise SupervisorError("a scout return contract must not name a task branch") + if not outcome_expected: + raise SupervisorError("execution return contract requires the outcome transport") + if existing_task_disk and (return_contract is None or not outcome_expected): + raise SupervisorError("existing task-disk recovery requires an authorized return outcome") + worker_role = request.get("worker_role", "author") + if worker_role not in ("author", "no-mistakes"): + raise SupervisorError("execution worker role is not supported") + service_contract = request.get("service_return_contract") + if worker_role == "no-mistakes": + if service_contract != { + "schema": "fm.no-mistakes-worker-return/v1", + "step_outcome_path": "outcome.json", + "step_outcome_max_bytes": 1024 * 1024, + }: + raise SupervisorError("no-mistakes service return contract is not exact") + if not outcome_expected or not isinstance(request.get("payload_files"), dict): + raise SupervisorError("no-mistakes execution requires staged outcome transport") + elif service_contract is not None: + raise SupervisorError("ordinary execution cannot select a service return contract") if outcome_expected: - # An outcome is bundled out of the staged repository, so the request - # that arms it must also be the one that stages that repository. - if not isinstance(request.get("payload_files"), dict): + # An ordinary outcome is bundled from the repository this request + # stages. Explicit recovery instead binds the already-assigned task + # disk and must never replace the repository it exists to preserve. + if not existing_task_disk and not isinstance(request.get("payload_files"), dict): raise SupervisorError("an outcome cannot be collected without a staged repository") # The URL is an unbound protected parameter; refusing here means a # control-plane actor cannot silently downgrade a landing task into a @@ -134,6 +197,11 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): MAX_ARCHIVE_BYTES = 600 * 1024 * 1024 MAX_OUTCOME_BYTES = 256 * 1024 * 1024 +MAX_RETURN_REPORT_BYTES = 16 * 1024 * 1024 +MAX_RETURN_STATUS_BYTES = 4 * 1024 * 1024 +MAX_RETURN_VISUAL_BYTES = 20 * 1024 * 1024 +MAX_RETURN_VISUAL_ENTRIES = 512 +MAX_RETURN_SCRATCH_BYTES = 128 * 1024 * 1024 # Every bounded step that runs OUTSIDE the wall, named once and used at the # call site, so the budget below is the same number the code actually spends. @@ -232,8 +300,36 @@ def extract_staged_archive(body, manifest, target, label): def stage_payload(request, worktree, account_home): - """Materialize the crewmate payload: repository from its bundle, task - files, and the provider-account material, all digest-verified.""" + """Materialize a new payload or bind an explicitly retained task disk.""" + if request.get("existing_task_disk"): + repo = worktree / "repo" + if repo.is_symlink() or not repo.is_dir(): + raise SupervisorError("existing task-disk repository is unavailable or redirected") + top = subprocess.run( + ["git", "-C", str(repo), "rev-parse", "--show-toplevel"], + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=GIT_HEAD_TIMEOUT, check=False, + ) + if top.returncode != 0 or Path(top.stdout.decode().strip()).resolve() != repo.resolve(): + raise SupervisorError("existing task-disk repository is not the exact repository root") + lineage = subprocess.run( + [ + "git", "-C", str(repo), "merge-base", "--is-ancestor", + request["repository_generation"], "HEAD", + ], + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=GIT_HEAD_TIMEOUT, check=False, + ) + if lineage.returncode != 0: + raise SupervisorError("existing task-disk repository lost its dispatched lineage") + readable = subprocess.run( + ["git", "-C", str(repo), "status", "--porcelain"], + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=GIT_STATUS_TIMEOUT, check=False, + ) + if readable.returncode != 0: + raise SupervisorError("existing task-disk working tree is unreadable") + return repo payload_manifest = request.get("payload_files") account_manifest = request.get("account_files") if payload_manifest is None and account_manifest is None: @@ -246,19 +342,9 @@ def stage_payload(request, worktree, account_home): extract_staged_archive(fetch_archive("account"), account_manifest, account_target, "account") repo = worktree / "repo" if repo.exists(): - # Staging runs when no executed marker exists, which is USUALLY the - # debris of an interrupted earlier staging. - # - # KNOWN GAP, not fixed here: it is not always. The executed marker - # lives on the disposable OS disk while /mnt/task is retained, so a - # resume (which replaces VM, NIC and OS disk and reattaches the task - # disk) destroys the marker while the previous run's commits survive - # here, and this removes them. A guard that merely refused was tried - # and reverted: it preserved the commits on a disk with no reader, - # since there is no collect-only mode, and wedged every later dispatch - # until a reset deleted them anyway. Closing it properly needs a way - # to collect a retained outcome without re-executing, which belongs - # with the release-receipt work (D6) rather than here. + # Explicit retained-disk recovery returned above and can never reach + # this remover. Ordinary staging gets here only for the repository its + # own payload is replacing, typically debris from interrupted staging. if repo.is_symlink() or not repo.is_dir(): raise SupervisorError("staged repository target is not a removable directory") shutil.rmtree(repo) @@ -281,9 +367,116 @@ def stage_payload(request, worktree, account_home): ) if head.returncode != 0 or head.stdout.decode().strip() != request["repository_generation"]: raise SupervisorError("staged repository head differs from the bound repository generation") + if request.get("worker_role") == "no-mistakes": + stage_no_mistakes_runtime(staging / "runtime.tar.gz", worktree / ".fm-runtime") return repo +def stage_no_mistakes_runtime(source, target, enforce_linux=True): + """Extract and re-verify the sealed credential-free runtime in the guest.""" + if source.is_symlink() or not source.is_file(): + raise SupervisorError("no-mistakes runtime bundle is unavailable or redirected") + if target.exists(): + if target.is_symlink() or not target.is_dir(): + raise SupervisorError("no-mistakes runtime target is unsafe") + shutil.rmtree(target) + target.mkdir(mode=0o700) + extracted = {} + total = 0 + try: + with tarfile.open(source, mode="r:gz") as archive: + members = archive.getmembers() + if not members or len(members) > 4096: + raise SupervisorError("no-mistakes runtime member inventory is unbounded") + for member in members: + parts = Path(member.name).parts + if ( + not member.isreg() or not parts or member.name.startswith("/") + or any(part in ("", ".", "..") for part in parts) + or member.mode not in (0o644, 0o755) + or member.name in extracted + ): + raise SupervisorError( + "no-mistakes runtime member is unsafe: {}".format(member.name)) + handle = archive.extractfile(member) + body = handle.read() if handle else b"" + total += len(body) + if total > 2 * 1024 * 1024 * 1024: + raise SupervisorError("no-mistakes runtime expands beyond its bound") + destination = target.joinpath(*parts) + destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + destination.write_bytes(body) + destination.chmod(member.mode) + extracted[member.name] = body + except tarfile.TarError as exc: + raise SupervisorError("no-mistakes runtime archive is malformed: {}".format(exc)) + try: + manifest = json.loads(extracted["runtime.json"].decode("utf-8")) + except (KeyError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise SupervisorError("no-mistakes runtime manifest is unreadable: {}".format(exc)) + records = manifest.get("files") if isinstance(manifest, dict) else None + manifest_fields = { + "schema", "provider", "no_mistakes_version", "no_mistakes_source_commit", + "owner_decision_protocol", "no_mistakes_path", "provider_path", "gh_path", + "node_path", "gh_axi_path", "gh_axi_entrypoint", "gh_axi_closure", "files", + } + if ( + not isinstance(manifest, dict) + or set(manifest) != manifest_fields + or manifest.get("schema") != "fm.azure-validation-runtime/v1" + or manifest.get("provider") != "pi" + or manifest.get("no_mistakes_path") != "bin/no-mistakes" + or manifest.get("provider_path") != "bin/pi" + or manifest.get("node_path") != "bin/node" + or manifest.get("gh_path") != "" + or manifest.get("gh_axi_path") != "" + or manifest.get("gh_axi_entrypoint") != "" + or manifest.get("gh_axi_closure") != [] + or manifest.get("owner_decision_protocol") != "fm.azure-validation-owner-decision/v1" + or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._+-]{0,127}", str(manifest.get("no_mistakes_version", ""))) + or not re.fullmatch(r"[0-9a-f]{40}", str(manifest.get("no_mistakes_source_commit", ""))) + or not isinstance(records, list) or not records + ): + raise SupervisorError("no-mistakes runtime manifest identity is not exact") + expected = {"runtime.json"} + for record in records: + if not isinstance(record, dict) or set(record) != {"path", "digest"}: + raise SupervisorError("no-mistakes runtime file record is malformed") + path = record.get("path") + digest_claim = record.get("digest") + if ( + not isinstance(path, str) or path in expected or path not in extracted + or Path(path).name.lower() in { + ".env", ".netrc", ".npmrc", "auth.json", "credentials.json", + "credentials", "id_rsa", "id_ed25519", + } + or not isinstance(digest_claim, str) or not digest_claim.startswith("sha256:") + or hashlib.sha256(extracted[path]).hexdigest() != digest_claim[7:] + ): + raise SupervisorError("no-mistakes runtime file inventory differs") + expected.add(path) + required_executables = ("bin/no-mistakes", "bin/node", "bin/pi") + if ( + set(extracted) != expected + or any(not os.access(target / path, os.X_OK) for path in required_executables) + or "lib/pi/dist/cli.js" not in extracted + or "extensions/pi-openai-fast-mode/src/index.ts" not in extracted + or "extensions/fast-mode-all-codex-accounts.ts" not in extracted + or "extensions/pi-ketch/src/index.ts" not in extracted + ): + raise SupervisorError("no-mistakes runtime is not exactly inventoried and executable") + if enforce_linux: + for path in ("bin/no-mistakes", "bin/node"): + header = extracted[path][:20] + if not ( + len(header) == 20 and header[:4] == b"\x7fELF" + and header[4] == 2 and header[5] == 1 + and int.from_bytes(header[18:20], "little") == 62 + ): + raise SupervisorError( + "no-mistakes runtime {} is not Linux amd64".format(path)) + + def git_in(repo, *arguments, timeout=BUNDLE_CREATE_TIMEOUT): return subprocess.run( ["git", "-C", str(repo), *arguments], @@ -292,6 +485,142 @@ def git_in(repo, *arguments, timeout=BUNDLE_CREATE_TIMEOUT): ) +def _safe_return_file(root, relative, limit): + path = root / relative + try: + relative_parts = Path(relative).parts + current = root + for part in relative_parts: + current = current / part + if current.is_symlink(): + raise SupervisorError("returned artifact is redirected: {}".format(relative)) + if not path.is_file(): + return None + body = path.read_bytes() + except OSError as exc: + raise SupervisorError("returned artifact is unreadable: {}: {}".format(relative, exc)) + if len(body) > limit: + raise SupervisorError("returned artifact exceeds its byte bound: {}".format(relative)) + return body + + +def _deterministic_tar(root, relative_paths, byte_limit, entry_limit): + """Archive already-authorized relative paths without following redirects.""" + output = io.BytesIO() + total = 0 + with tarfile.open(fileobj=output, mode="w") as archive: + for relative in sorted(relative_paths): + if len(relative_paths) > entry_limit: + raise SupervisorError("returned artifact archive has too many entries") + source = root / relative + current = root + for part in Path(relative).parts: + current = current / part + if current.is_symlink(): + raise SupervisorError("returned artifact archive contains a redirect: {}".format(relative)) + if not source.is_file(): + raise SupervisorError("returned artifact archive entry is not a regular file: {}".format(relative)) + body = source.read_bytes() + total += len(body) + if total > byte_limit: + raise SupervisorError("returned artifact archive exceeds its byte bound") + info = tarfile.TarInfo(relative) + info.size = len(body) + info.mode = 0o600 + info.mtime = 0 + info.uid = info.gid = 0 + info.uname = info.gname = "" + archive.addfile(info, io.BytesIO(body)) + return output.getvalue() + + +def _visual_archive(return_root, relative): + root = return_root / relative + if not root.exists(): + return None + if root.is_symlink() or not root.is_dir(): + raise SupervisorError("returned visual evidence root is redirected or not a directory") + paths = [] + for directory, names, files in os.walk(root, followlinks=False): + names.sort() + files.sort() + directory_path = Path(directory) + if directory_path.is_symlink(): + raise SupervisorError("returned visual evidence contains a redirected directory") + for name in files: + source = directory_path / name + paths.append(str(source.relative_to(return_root))) + if not paths: + return None + return _deterministic_tar( + return_root, paths, MAX_RETURN_VISUAL_BYTES, MAX_RETURN_VISUAL_ENTRIES, + ) + + +def _scratch_artifacts(repo): + patch = git_in(repo, "diff", "--binary", "HEAD", timeout=GIT_STATUS_TIMEOUT) + if patch.returncode != 0: + raise SupervisorError("returned scratch diff is unreadable") + listed = git_in( + repo, "ls-files", "-z", "--others", "--exclude-standard", + timeout=GIT_STATUS_TIMEOUT, + ) + if listed.returncode != 0: + raise SupervisorError("returned untracked scratch is unreadable") + try: + untracked = [item.decode("utf-8") for item in listed.stdout.split(b"\0") if item] + except UnicodeDecodeError as exc: + raise SupervisorError("returned untracked scratch path is not UTF-8: {}".format(exc)) + archive = None + if untracked: + archive = _deterministic_tar( + repo, untracked, MAX_RETURN_SCRATCH_BYTES, MAX_RETURN_VISUAL_ENTRIES, + ) + if len(patch.stdout) > MAX_RETURN_SCRATCH_BYTES: + raise SupervisorError("returned scratch diff exceeds its byte bound") + return patch.stdout or None, archive + + +def _hash_blob(repo, body): + result = subprocess.run( + ["git", "-C", str(repo), "hash-object", "-w", "--stdin"], input=body, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=GIT_HEAD_TIMEOUT, check=False, + ) + if result.returncode != 0: + raise SupervisorError("returned artifact could not be stored in the repository") + return result.stdout.decode().strip() + + +def _return_commit(repo, base, artifacts, request): + entries = [] + for name, body in sorted(artifacts.items()): + entries.append("100644 blob {}\t{}\n".format(_hash_blob(repo, body), name)) + tree = subprocess.run( + ["git", "-C", str(repo), "mktree"], input="".join(entries).encode(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=GIT_HEAD_TIMEOUT, check=False, + ) + if tree.returncode != 0: + raise SupervisorError("returned artifact tree could not be created") + environment = dict(os.environ) + environment.update({ + "GIT_AUTHOR_NAME": "Firstmate worker return", + "GIT_AUTHOR_EMAIL": "worker-return@localhost", + "GIT_COMMITTER_NAME": "Firstmate worker return", + "GIT_COMMITTER_EMAIL": "worker-return@localhost", + "GIT_AUTHOR_DATE": "@0 +0000", + "GIT_COMMITTER_DATE": "@0 +0000", + }) + committed = subprocess.run( + ["git", "-C", str(repo), "commit-tree", tree.stdout.decode().strip(), "-p", base], + input=("Firstmate worker return {}\n".format(request["request_digest"])).encode(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=GIT_HEAD_TIMEOUT, + check=False, env=environment, + ) + if committed.returncode != 0: + raise SupervisorError("returned artifact commit could not be created") + return committed.stdout.decode().strip() + + def outcome_bundle_path(request, worktree): """Where the collected bundle lives on the RETAINED task disk. @@ -365,28 +694,164 @@ def collect_outcome(request, repo, worktree_root): commits = int(counted.stdout.decode().strip()) except ValueError: raise SupervisorError("outcome commit count is not a number") - if commits == 0: - # A crewmate that edited without committing looks identical here, so - # the result says so explicitly rather than reading as "nothing to do". - dirty = git_in(repo, "status", "--porcelain", timeout=GIT_STATUS_TIMEOUT) - if dirty.returncode != 0: - # Unknown is not clean. Reporting False here would render an - # unreadable tree as a tidy read-only task, which is the exact - # confusion this field exists to prevent. - raise SupervisorError( - "outcome working-tree state is unreadable: {}".format( - dirty.stderr.decode("utf-8", errors="replace")[-200:] - ) + dirty = git_in(repo, "status", "--porcelain", timeout=GIT_STATUS_TIMEOUT) + if dirty.returncode != 0: + # Unknown is not clean. Reporting False here would render an + # unreadable tree as a tidy read-only task, which is the exact + # confusion this field exists to prevent. + raise SupervisorError( + "outcome working-tree state is unreadable: {}".format( + dirty.stderr.decode("utf-8", errors="replace")[-200:] ) + ) + contract = request.get("return_contract") + service_contract = request.get("service_return_contract") + if commits == 0 and contract is None and service_contract is None: return { "outcome_present": False, "outcome_sha256": "", "outcome_bytes": 0, "outcome_commits": 0, "outcome_sink": "", "outcome_uncommitted_changes": bool(dirty.stdout.strip()), } + bundle = outcome_bundle_path(request, worktree_root) bundle.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + bundle_refs = [] + returned = {} + if contract is not None: + return_root = worktree_root / ".fm-return" + report = _safe_return_file(return_root, contract["report_path"], MAX_RETURN_REPORT_BYTES) + status = _safe_return_file(return_root, contract["status_path"], MAX_RETURN_STATUS_BYTES) + visuals = _visual_archive(return_root, contract["visuals_path"]) + scratch_patch, scratch_untracked = _scratch_artifacts(repo) + artifact_bodies = {} + artifact_sources = { + "report.md": (report, contract["report_path"]), + "status.log": (status, contract["status_path"]), + "visuals.tar": (visuals, contract["visuals_path"]), + "scratch.patch": (scratch_patch, "git-diff"), + "scratch-untracked.tar": (scratch_untracked, "git-untracked"), + } + manifest_artifacts = {} + for name, (body, source) in artifact_sources.items(): + if body is None: + continue + artifact_bodies[name] = body + manifest_artifacts[name] = { + "source": source, "bytes": len(body), + "sha256": hashlib.sha256(body).hexdigest(), + } + manifest = { + "schema": "fm.worker-return/v1", + "task": request["task"], + "task_generation": request["task_generation"], + "assignment_generation": request["assignment_generation"], + "request_digest": request["request_digest"], + "repository_generation": base, + "kind": contract["kind"], + "branch": contract["branch"], + "report_required": contract["report_required"], + "report_path": contract["report_path"], + "status_path": contract["status_path"], + "visuals_path": contract["visuals_path"], + "outcome_commits": commits, + "outcome_tip": git_in(repo, "rev-parse", "HEAD", timeout=GIT_HEAD_TIMEOUT).stdout.decode().strip(), + "uncommitted_changes": bool(dirty.stdout.strip()), + "artifacts": manifest_artifacts, + } + manifest_body = canonical(manifest) + b"\n" + artifact_bodies["manifest.json"] = manifest_body + return_commit = _return_commit(repo, base, artifact_bodies, request) + return_ref = "refs/fm-return/{}".format(request["request_digest"][:32]) + updated = git_in(repo, "update-ref", return_ref, return_commit, timeout=GIT_HEAD_TIMEOUT) + if updated.returncode != 0: + raise SupervisorError("returned artifact ref could not be created") + bundle_refs.append(return_ref) + if commits: + outcome_ref = "refs/fm-outcome/{}".format(request["request_digest"][:32]) + updated = git_in(repo, "update-ref", outcome_ref, manifest["outcome_tip"], timeout=GIT_HEAD_TIMEOUT) + if updated.returncode != 0: + raise SupervisorError("returned outcome ref could not be created") + bundle_refs.append(outcome_ref) + # The local repository already has the exact dispatched generation. + # Excluding it keeps the bundle to the artifact commit plus only the + # project commits this assignment added, rather than retransmitting + # arbitrary repository history. + bundle_refs.append("^{}".format(base)) + returned = { + "return_present": True, + "return_ref": return_ref, + "return_commit": return_commit, + "return_manifest_sha256": hashlib.sha256(manifest_body).hexdigest(), + "outcome_tip": manifest["outcome_tip"], + } + elif service_contract is not None: + outcome_path = repo / service_contract["step_outcome_path"] + outcome_body = None + if outcome_path.exists(): + if outcome_path.is_symlink() or not outcome_path.is_file(): + raise SupervisorError("no-mistakes step outcome is redirected or not regular") + outcome_body = outcome_path.read_bytes() + if not outcome_body or len(outcome_body) > service_contract["step_outcome_max_bytes"]: + raise SupervisorError("no-mistakes step outcome is empty or oversized") + if outcome_body is not None: + manifest = { + "schema": "fm.no-mistakes-worker-return/v1", + "task": request["task"], + "task_generation": request["task_generation"], + "assignment_generation": request["assignment_generation"], + "request_digest": request["request_digest"], + "repository_generation": base, + "outcome_commits": commits, + "outcome_tip": git_in( + repo, "rev-parse", "HEAD", timeout=GIT_HEAD_TIMEOUT + ).stdout.decode().strip(), + "step_outcome_sha256": hashlib.sha256(outcome_body).hexdigest(), + } + manifest_body = canonical(manifest) + b"\n" + return_commit = _return_commit( + repo, base, + {"manifest.json": manifest_body, "step-outcome.json": outcome_body}, + request, + ) + return_ref = "refs/fm-return/{}".format(request["request_digest"][:32]) + if git_in( + repo, "update-ref", return_ref, return_commit, timeout=GIT_HEAD_TIMEOUT + ).returncode != 0: + raise SupervisorError("no-mistakes service return ref could not be created") + bundle_refs.append(return_ref) + if commits: + outcome_ref = "refs/fm-outcome/{}".format(request["request_digest"][:32]) + if git_in( + repo, "update-ref", outcome_ref, manifest["outcome_tip"], + timeout=GIT_HEAD_TIMEOUT, + ).returncode != 0: + raise SupervisorError("no-mistakes outcome ref could not be created") + bundle_refs.append(outcome_ref) + bundle_refs.append("^{}".format(base)) + returned = { + "return_present": True, + "return_ref": return_ref, + "return_commit": return_commit, + "return_manifest_sha256": hashlib.sha256(manifest_body).hexdigest(), + "outcome_tip": manifest["outcome_tip"], + "service_return_present": True, + "step_outcome_sha256": manifest["step_outcome_sha256"], + } + elif commits: + bundle_refs.append("{}..HEAD".format(base)) + returned = {"service_return_present": False, "step_outcome_sha256": ""} + else: + return { + "outcome_present": False, "outcome_sha256": "", "outcome_bytes": 0, + "outcome_commits": 0, "outcome_sink": "", + "outcome_uncommitted_changes": bool(dirty.stdout.strip()), + "service_return_present": False, "step_outcome_sha256": "", + } + elif commits: + bundle_refs.append("{}..HEAD".format(base)) + created = git_in( - repo, "bundle", "create", str(bundle), "{}..HEAD".format(base), + repo, "bundle", "create", str(bundle), *bundle_refs, timeout=BUNDLE_CREATE_TIMEOUT, ) if created.returncode != 0 or not bundle.is_file(): @@ -400,11 +865,13 @@ def collect_outcome(request, repo, worktree_root): body = bundle.read_bytes() sink = put_outcome_blob(body) return { - "outcome_present": True, + "outcome_present": commits > 0, "outcome_sha256": hashlib.sha256(body).hexdigest(), "outcome_bytes": len(body), "outcome_commits": commits, "outcome_sink": sink, + "outcome_uncommitted_changes": bool(dirty.stdout.strip()), + **returned, } @@ -418,7 +885,7 @@ def replay_outcome_upload(request, worktree, recorded): the crewmate's commits die with the VM. A failure here must not stop the replay from answering, so it is reported and swallowed. """ - if not recorded.get("outcome_present"): + if not (recorded.get("outcome_present") or recorded.get("return_present")): return False retained = outcome_bundle_path(request, worktree) try: @@ -457,9 +924,14 @@ def write_atomic(path, value): def execute(request, worktree, worktree_root): + path = "/usr/local/bin:/usr/bin:/bin" + if request.get("worker_role") == "no-mistakes": + path = str((worktree_root / ".fm-runtime" / "bin").resolve()) + ":" + path + account_home = Path(os.environ.get("FM_WORKER_ACCOUNT_HOME", "/nonexistent")).resolve() safe_env = { - "HOME": str(Path(os.environ.get("FM_WORKER_ACCOUNT_HOME", "/nonexistent")).resolve()), - "PATH": "/usr/local/bin:/usr/bin:/bin", + "HOME": str(account_home), + "PI_CODING_AGENT_DIR": str(account_home / "pi-agent"), + "PATH": path, "LANG": "C.UTF-8", "LC_ALL": "C.UTF-8", "GIT_TERMINAL_PROMPT": "0", @@ -484,8 +956,8 @@ def execute(request, worktree, worktree_root): stderr, stderr_truncated = bounded(stderr) # NOTHING after the task command may raise. Its effects already happened, # so any escape here means no executed marker is written, and the next - # dispatch both re-runs the command and (through stage_payload's rmtree of - # the staged repository) destroys the commits the first run produced. + # ordinary dispatch both re-runs the command and (through stage_payload's + # rmtree of the staged repository) destroys the commits the first run produced. # Every post-command failure is therefore recorded in the digest-bound # result instead of raised, whatever its exception class: a full disk # reaches the stream write, a git timeout or MemoryError reaches the diff --git a/docs/azure-no-mistakes-worker-config.example.json b/docs/azure-no-mistakes-worker-config.example.json new file mode 100644 index 00000000000..76154e9a6dd --- /dev/null +++ b/docs/azure-no-mistakes-worker-config.example.json @@ -0,0 +1,20 @@ +{ + "schema": "fm.no-mistakes-worker-wrapper-config/v1", + "fm_home": "/absolute/path/to/firstmate-home", + "account_pool_home": "/absolute/path/to/azure-pi-account-pool", + "runtime_bundle": "/absolute/path/to/no-mistakes-pi-runtime.tar.gz", + "runtime_bundle_sha256": "replace-with-64-lowercase-hex", + "lifecycle_path": "/absolute/path/to/firstmate/bin/fm-worker-lifecycle.sh", + "lifecycle_source_commit": "replace-with-exact-40-hex-firstmate-commit", + "lifecycle_env": { + "FM_AZURE_SUBSCRIPTION_ID": "replace-with-subscription-uuid", + "FM_AZURE_DEPLOYMENT_GENERATION": "replace-with-deployment-generation", + "FM_AZURE_OWNER_TAG": "replace-with-owner-tag", + "FM_AZURE_NAMING_PREFIX": "replace-with-naming-prefix", + "FM_AZURE_STORAGE_NAME": "replace-with-storage-account" + }, + "assignment_timeout_seconds": 1800, + "cleanup_timeout_seconds": 1800, + "poll_seconds": 5, + "wall_seconds": 3600 +} diff --git a/docs/azure-pilot/main.json b/docs/azure-pilot/main.json index 852dcbc2946..b184861eb38 100644 --- a/docs/azure-pilot/main.json +++ b/docs/azure-pilot/main.json @@ -315,7 +315,8 @@ "aggregate-worker-hour-planning-threshold": "3500", "landing-worker-hour-reserve": "400" }, - "budgetName": "[format('bud-{0}-monthly', parameters('namingPrefix'))]" + "budgetName": "[format('bud-{0}-monthly', parameters('namingPrefix'))]", + "resourcesDeploymentName": "[format('firstmate-pilot-resources-{0}', uniqueString(deployment().name))]" }, "resources": [ { @@ -328,7 +329,7 @@ { "type": "Microsoft.Resources/deployments", "apiVersion": "2022-09-01", - "name": "firstmate-pilot-resources", + "name": "[variables('resourcesDeploymentName')]", "resourceGroup": "[variables('safeResourceGroupName')]", "dependsOn": [ "[resourceId('Microsoft.Resources/resourceGroups', variables('safeResourceGroupName'))]" @@ -2266,11 +2267,11 @@ }, "blobPrivateEndpointNicId": { "type": "string", - "value": "[reference('firstmate-pilot-resources').outputs.blobPrivateEndpointNicId.value]" + "value": "[reference(variables('resourcesDeploymentName')).outputs.blobPrivateEndpointNicId.value]" }, "blobPrivateEndpointNicResourceGuid": { "type": "string", - "value": "[reference('firstmate-pilot-resources').outputs.blobPrivateEndpointNicResourceGuid.value]" + "value": "[reference(variables('resourcesDeploymentName')).outputs.blobPrivateEndpointNicResourceGuid.value]" } } } diff --git a/docs/azure-requirements.md b/docs/azure-requirements.md index 31f9290eef5..b930b2ddb08 100644 --- a/docs/azure-requirements.md +++ b/docs/azure-requirements.md @@ -64,8 +64,7 @@ digest `1f238e42...`, and an outcome bundle of one commit `r5-accept-readme-v3-20260822` (`asg-00000020`) and `r5-accept-package-v3-20260822` (`asg-00000021`) each returned exit 0, `timed_out false`, and `outcome_commits 0` for their read-only briefs. Evidence and paths are in R2/R3 and R5. -Placement across distinct upstream accounts is R5, where the former single-profile placement -residual is closed and now proven live. +Provider-profile placement is R5, where live Pi execution and deterministic reusable assignment-private snapshots are recorded separately. ## R2/R3. Secondmates run in Azure, and can spawn crewmates in Azure @@ -250,19 +249,11 @@ neither was the receipts strand: approval markers), alongside 377 passing units. Until those units skipped loudly off macOS, no intent could reach a green test step there. - Partially closed. The retained shard responses under - `$FM_HOME/state/azure-validation/shards/azv-36b2726cbcf3/*/response/` are the measurement, and - they name eleven failing test files, not three. Three classes are genuine host capabilities the - cell does not have, and those are now gated: a real tmux server it can create windows in - (`server exited unexpectedly` on the shard-2 and shard-4 workers), passwordless sudo with - `systemd-run` (`Linux systemd integration requires passwordless sudo`), and the `/usr/bin/cpp` - binding `bin/fm-account-directory.sh` needs before it can validate any Claude quota-axi - Keychain approval marker (`system openat binding unavailable`). Fifteen units across six test - files are bound to those three capabilities in `tests/host-capabilities.tsv`; the cell declares - the three absences by name in `bin/fm-azure-validation-shard-bridge.py`, and - `tests/host-capability-gate.sh` turns each into a loud `FM_HOST_CAPABILITY_SKIP`. The gate - refuses that declaration on Darwin, so macOS coverage is unchanged and cannot be switched off, - and CI declares nothing, so its coverage is unchanged too. + Partially closed. + The retained shard responses under `$FM_HOME/state/azure-validation/shards/azv-36b2726cbcf3/*/response/` are the measurement, and they name eleven failing test files, not three. + Four classes are genuine host capabilities the cell does not have, and those are now gated: a real tmux server it can create windows in (`server exited unexpectedly` on the shard-2 and shard-4 workers), passwordless sudo with `systemd-run` (`Linux systemd integration requires passwordless sudo`), the `/usr/bin/cpp` binding `bin/fm-account-directory.sh` needs before it can validate any Claude quota-axi Keychain approval marker (`system openat binding unavailable`), and outbound reach to the origin remote's host (`origin-egress`). + Fifty-two units across seven test files are bound to those four capabilities in `tests/host-capabilities.tsv`; the cell declares the four absences by name in `bin/fm-azure-validation-shard-bridge.py`, and `tests/host-capability-gate.sh` turns each into a loud `FM_HOST_CAPABILITY_SKIP`. + The gate refuses that declaration on Darwin, so macOS coverage is unchanged and cannot be switched off, and CI declares nothing, so its coverage is unchanged too. The other five failing files have now been MEASURED rather than inferred, by running the whole sealed suite in a local reproduction of the cell's own package closure (Ubuntu 24.04 @@ -297,11 +288,11 @@ neither was the receipts strand: iteration: the suite invokes every case through a single choke point, `run_partitioned_test`, so running each case in a subshell there reports every failure in one run instead of stopping at the first. Both files were run to completion with the network off - 143 of 143 cases - and - the result is exactly 33 units, all in the secondmate teardown/retirement family. An earlier + the result is exactly 37 units, all in the secondmate teardown/retirement family. An earlier one-at-a-time iteration had found only 19 and had not converged; the difference is why the partial set was not shipped. - BE CLEAR ABOUT WHAT THIS COSTS. Those 33 units are SKIPPED in the cell, not preserved by some + BE CLEAR ABOUT WHAT THIS COSTS. Those 37 units are SKIPPED in the cell, not preserved by some other route. The cell does not verify secondmate teardown or retirement authority at all: not the landed-work refusals, not the registry locking, not the network-authority pinning, not the child quiescence ordering. macOS and CI still run every one of them, and CI is where that @@ -345,40 +336,24 @@ reaches `close` with its worktree disk released". Status: DONE, met live on 2026-08-22. -Crosscheck on the pi fleet is done at the roster level. The current operating split is six Azure -worker profiles across six distinct upstream accounts plus one separate local Firstmate profile; -Azure profiles are projected into single-profile account homes by `bin/fm-pi-account-home.py`, -with the roster repointed and read back through the real `bin/fm-crosscheck.py` reader and -policy screen. Under the second 2026-08-19 amendment this roster is now the dormant crosscheck -fallback; the pi fleet's primary duties are authors and no-mistakes. -Crewmate placement now selects across the pool. The controller chooses one free profile inside the -same lock hold and the same durable write that creates the queue entry, so selection and the lease -are one act and the queue entry IS the lease; the unit of exclusion is the UPSTREAM ACCOUNT rather -than the profile name, because two profiles can be re-logged into one account and what a crewmate -contends for belongs to the account. The chosen profile is projected with `bin/fm-pi-account-home.py` -into a controller-owned account home, and `bin/fm-spawn.sh` narrows the staged provider credential -to it, so the worker receives exactly one account rather than the pooled `auth.json`. An exhausted -pool refuses by name, listing every leased profile and the task holding it. Mechanics are owned by -`docs/azure-workers.md` ("Provider-account placement across the Pi fleet"). -The host is also the sole OAuth refresh authority: staging requires twelve hours of access-token -headroom against a six-hour worker shutdown deadline, so a guest cannot live long enough to rotate -the copied refresh token independently of the Azure pool. - -Proven locally against a fixture provider by `tests/fm-worker-placement.test.sh` (eight concurrent -placements racing the controller lock take eight distinct upstream accounts, read back from -`controller.json`; exhaustion refuses; a compartment child, its compartment and an ordinary crewmate -hold three distinct accounts; killing placements between selection and the durable lease orphans no -account) and end to end through the real `bin/fm-spawn.sh` by `tests/fm-spawn-cloud.test.sh`, which -asserts the staged credential is the leased single profile. - -Operational consequence the owner must know: concurrent placements are now bounded by -`min(FM_AZURE_WORKER_MAX, distinct upstream accounts in the pool)`. With the current six-account -Azure pool the seventh concurrent placement refuses although MAX_WORKERS is 16, and compartments -compete in the same pool. Sixteen crewmates cannot run on six accounts without sharing one. -Raising the ceiling means adding profiles on distinct accounts to the Azure pool; adding profiles -to the separate local Firstmate pool does not change Azure capacity. - -Acceptance: concurrent crewmates run on distinct pi profiles with no account collision. +The Pi multi-profile requirement applies to ordinary Codex/Pi workers, nested supervisors, and no-mistakes author work. +Crosscheck's primary reviewer is GLM 5.2 through the separately staged `fireworks-glm` provider credential in its Azure model compartment. +Pi is only the bounded guest CLI in that compartment, so Crosscheck consumes neither a Codex/Pi worker profile nor a worker slot. +No-mistakes runs in its separate Azure runner environment. +Both specialized lanes still consume the shared 40-vCPU specialized envelope and 128-vCPU regional ledger. + +Crewmate placement now load-balances the least-active usable profile with a stable tie-break, then reuses profiles through the independent sixteen-worker ceiling. +The reusable `account_binding` remains a digest of upstream identity and is visible with per-profile active load, but it is not an exclusive lease. +Every task generation receives a distinct writable projection keyed by an assignment-private binding, and the pooled `auth.json` never reaches a guest. +The controller durably owns an interrupted projection before writing credential bytes, so exact replay keeps the same profile and path and `withdraw` can clean it. +The host remains the sole OAuth refresh authority because the canonical profile must retain twelve hours of headroom before snapshot and the guest VM shuts down within six hours. +Mechanics are owned by `docs/azure-workers.md` under "Provider-account placement across the Pi fleet". + +`tests/fm-worker-placement.test.sh` admits sixteen concurrent requests from three profiles, proves balanced loads of six/five/five, distinct projection homes, exact replay, interrupted-write recovery, same-profile cleanup isolation, host-only refresh, and concurrent specialized reservations. +`tests/fm-spawn-cloud.test.sh` proves the staged credential is the one selected snapshot and repeats the twelve-hour preflight before use. +The release lane deliberately does not run a sixteen-VM live campaign; C2's bounded live acceptance owner consumes this deterministic result separately. + +Acceptance: concurrent Codex/Pi workers use digest-bound reusable profile snapshots without sharing writable account homes, while specialized no-mistakes and GLM Crosscheck capacity remains outside the worker/profile ceiling. Met on real compute, which is what an earlier revision of this section still owed. The tracked evidence `simultaneous_assignments` array is derived from one controller snapshot holding four assignments simultaneously in `assigned` state, on four slots and four distinct upstream account bindings: @@ -392,9 +367,9 @@ The tracked evidence `simultaneous_assignments` array is derived from one contro The two ordinary crewmates executed successfully: each returned exit 0, `timed_out false` and `outcome_commits 0` for its read-only inspection brief, with result digests `77c43f95...` and `55151c48...` (`executions[1]` and `executions[2]`). Each record in `release_proofs` records the `account` authority `proved` against exactly the binding above, so what the worker held is proved on release rather than only at staging. -No account collision appears anywhere in that snapshot: four live workers, four bindings. +That historical snapshot used four live workers with four bindings, so it proves the Pi placement path but does not exercise reusable same-profile snapshots. -What this does NOT prove, stated so nobody reads more into it: four concurrent accounts were exercised, not the pool of eight, and the two ordinary briefs were read-only inspections. +What this does NOT prove, stated so nobody reads more into it: four concurrent accounts were exercised, not profile reuse at sixteen, and the two ordinary briefs were read-only inspections. The committing leg is the compartment child, recorded in R2/R3. These limits are machine-readable in `limitations`. @@ -1133,6 +1108,10 @@ The first of the three changes landed on 2026-08-19: a provider mutation applies commits only on success, and is refused if its effects reach outside the one compartment its slot owns. The later per-slot pending map and lock-discipline changes completed that work. +The profile ceiling defect is also corrected in this implementation: sixteen ordinary worker/supervisor requests can reuse fewer host-owned Pi profiles through assignment-private snapshots. +No-mistakes reservations and `fireworks-glm` Crosscheck model/tool/verifier reservations do not consume those sixteen worker slots or any Codex/Pi worker profile. +They still share specialized-envelope, regional, exact-family, and spend admission with the worker fleet. +`tests/fm-worker-placement.test.sh` proves that separation deterministically, while the live concurrent campaign remains the acceptance owed here. Crosscheck model admission now retains the shipped four-lane FIFO model while transient exact-family or shared-capacity pressure polls one durable allocator reservation identity within the configured queue wait. Timeout releases that exact queued identity, non-capacity refusals remain immediate, and reviewer credentials are rechecked after admission before staging or billable compute. diff --git a/docs/azure-runner.md b/docs/azure-runner.md index 8aa944cee6f..d7d2da73956 100644 --- a/docs/azure-runner.md +++ b/docs/azure-runner.md @@ -31,13 +31,15 @@ A present file that is unreadable, partial, malformed, expired, exhausted, wrong Once either selector validly chooses remote execution, the command never falls back to the Mac after any cloud, identity, quota, staging, execution, or integrity failure. `FM_AZURE_RUNNER_LOCAL_RECOVERY_CLASSES` is the only explicit local recovery selection. -No billable resource was created while implementing this code. -The first live invocation remains blocked until the foundation and this code are reviewed, landed, explicitly applied, and approved for the exact subscription. +C2 no-mistakes run `01M0T2PJB0EZX8AA94WQV0WXA9` exposed the remaining production blocker before VM creation because its gate worktree was detached. +Direct diagnostic invocations `azr-0a4691b5bbae` and `azr-949b35e79a57` both returned command exit 1 and safely cleaned their invocation compute to zero. +Those attempts are failed evidence, not acceptance. +A new billable acceptance remains blocked until this exact repair is on public `main` and the operator explicitly approves the exact subscription and run. ## Request and snapshot contract `prepare` refuses tracked changes, untracked files, and any origin identity other than a credential-free GitHub HTTPS URL. -It also refuses a detached HEAD unless the caller supplies an exact public source ref or a digest-bound private bundle for that detached commit. +It also refuses a detached HEAD unless the caller supplies an exact source ref whose public proof or sealed private bundle head equals that detached commit. By default the candidate must be reachable from a freshly advertised and fetched `refs/heads/main` default head. The explicit `--source-ref refs/heads/` seam alone requires the candidate commit to be the exact freshly advertised and fetched head of that public branch; it never accepts an ancestor, stale tracking ref, tag, or changed remote head. An explicit `--public-ref` may instead name only an advertised branch head or `refs/pull//head`, and the candidate must equal that ref's exact fetched head; mutable pull merge refs and unsafe ref shapes are refused, and the two source-ref seams are mutually exclusive. @@ -45,8 +47,11 @@ A caller may additionally bind one or more exact `--public-ancestor` commits; pu Crosscheck evidence for a private GitHub repository supplies `--private-snapshot-bundle` without a parent reservation: the trusted host packages its clean exact-head review checkout, binds the authenticated PR ref and base ancestor, and stages only that digest-bound Git bundle so the evidence VM receives no GitHub credential. When that arbitrary repository does not contain Firstmate's Agent Fleet lock, the `crosscheck-tool` class uses only the sealed base toolchain and records an empty Python-wheel closure instead of requiring Firstmate-specific files. An Azure validation cell additionally supplies `--private-snapshot-bundle` with its parent-cell reservation so an unpushed pipeline-fix head can execute without prematurely changing the task branch on GitHub. -Both private modes bind one exact source ref/head, a one-ref Git bundle, digest, size, and private staging object; the validation-cell mode additionally binds its parent cell and freshly proves the public origin's trusted default ref/head. -The public proof runs in a fresh bare repository with system/global Git configuration, credentials, prompts, extra HTTP headers, and file transport disabled; private Crosscheck proof revalidates the clean checkout and bundle identities locally immediately before compute creation and retry. +The per-run no-mistakes route instead supplies `--private-snapshot-from-head` with the deterministic private ref `refs/heads/fm-no-mistakes/`. +That direct mode seals the exact clean detached gate HEAD and its complete non-shallow ancestry into one self-contained bundle without guessing or pushing a task branch and without claiming a validation-cell parent. +All three private modes bind one exact source ref/head, a one-ref Git bundle, digest, size, and private staging object. +The parent mode additionally binds its exact cell and reservation. +The public proof runs in a fresh bare repository with system/global Git configuration, credentials, prompts, extra HTTP headers, and file transport disabled; all modes repeat their exact public/private source proof immediately before compute creation and retry. No live worktree, primary home, provider account home, browser profile, or peer storage is mounted or copied. The canonical `fm.azure-command/v1` request binds these fields: @@ -72,7 +77,8 @@ The fixed root bootstrap installs a hard-coded Ubuntu transport and Linux test-t Before each package operation it waits up to three minutes for the standard apt/dpkg locks, and apt carries the same bounded dpkg timeout, so normal image maintenance can finish without turning into a repository-command failure while a stuck lock still fails closed. Repository code cannot alter that privileged package list, all package and staging traffic is shaped to one megabit per second and ends before deny-all command networking starts, and invocation evidence must record the resolved package/image versions during real acceptance. The request records the exact-size, checksum-pinned ShellCheck 0.11.0 and uv 0.9.10 releases plus, when the snapshot contains the Agent Fleet `uv.lock`, the complete Linux x86_64 pytest/ruff wheel closure selected from that lock. -When that lock is present, trusted root verifies the lock, archive, file set, sizes, and hashes, creates the Agent Fleet environment with an empty cache and networking disabled, and forces repository `uv run --locked` commands to use that already-synchronized offline environment. +When that lock is present, trusted root verifies the lock, archive, file set, sizes, and hashes, creates the Agent Fleet environment with an empty cache and networking disabled, and installs the exact locked Agent Fleet source plus its release-local `agent-fleet` console entrypoint before repository code starts. +The command wrapper then forces repository `uv run --locked` commands to use that synchronized offline environment through `UV_NO_SYNC=1`. ## Private control and VM boundary @@ -149,6 +155,7 @@ The validation-cell dispatcher may request up to eight mixed-family invocations A validation-owned invocation carries an exact `capacity-parent` cell id and complete parent vCPU reservation so the cell's pre-reserved processor shape and the child VM inventory cannot be double-counted or mistaken for unrelated capacity. Before admission and again before VM creation, the runner proves one live parent cell with the exact owner, deployment generation, home, lifecycle, id, and processor-reservation tags, then refuses a child beyond the reserved `(vCPUs - 8) / 4` slots. Each child still receives its own durable first-day reservation for its direct compute, storage, network, monitoring, and control meters; only the already-accounted $210 shared foundation reserve is omitted from that child reservation. +A direct private no-mistakes bundle has no validation parent and therefore takes the ordinary standalone shared-capacity reservation and complete foundation cost bound exactly once. Immediately before reservation and again immediately before VM creation, the controller proves the exact subscription/resource-group IDs and owner/generation tags for the named foundation storage account, zero-data admission-control account/container and ETag, controller UAMI and its sole exact effective container role including inherited/group expansion, VNet and address space, validation and private-endpoint subnets, complete NSG rule set, NAT and bound Standard public IP, blob private endpoint and endpoint NIC, named approved blob connection, private-DNS zone, VNet link, zone group/config names, and private-access properties. It also proves current SKU capabilities and restrictions, current East US regional and selected-family free vCPU quota, month-to-date actual cost, forecast cost, the exact unambiguous Linux on-demand Consumption retail meter (never Spot, Low Priority, Windows, dev/test, reservation, or savings pricing), and active runner count. @@ -348,10 +355,15 @@ local fallback, and `FM_AZURE_RUNNER_LOCAL_RECOVERY_CLASSES` remains the only explicit local opt-out. The lint payload preserves the tracked shell owner and locked Agent Fleet command unchanged inside the dispatched argv. -For a validation-owned feature branch, the caller passes its exact current `refs/heads/` identity plus the one-ref private snapshot bundle. -The runner binds and privately stages that unpushed commit, while a changed local bundle/head/tree or public default base refuses before compute creation. +For the per-run no-mistakes route, the dispatcher binds the detached gate HEAD to `refs/heads/fm-no-mistakes/` and asks the runner to build the complete one-ref private bundle inside its protected payload directory. +That source ref is a deterministic run identity, not a guess at the author's branch. +The direct route uses ordinary shared-capacity accounting and carries no validation-cell parent. +For a validation-owned feature branch, the caller passes its exact current `refs/heads/` identity plus the one-ref private snapshot bundle and parent reservation. +The runner binds and privately stages either unpushed commit, while a changed local bundle/head/tree or public default base refuses before compute creation. The ordinary test path runs the capability-derived real-Herdr host set locally and leaves complete behavior-inventory verification to required CI, as defined in [`configuration.md`](configuration.md#gate-defaults-no-mistakesyaml). When the `test` class is explicitly remote, `bin/fm-no-mistakes-test-command.sh` runs the sealed non-Herdr behavior inventory plus locked Agent Fleet checks on one Azure VM while every real-Herdr declaration runs through owned guarded labs on the Mac; a failed Azure shard is never replayed locally. +The remote command declares exactly `real-tmux-server,passwordless-root-escalation,system-openat-binding,origin-egress` absent and emits the existing loud per-unit skips rather than adding failures to the skip inventory. +The guest materializes complete source and default-branch history, so admitted non-Herdr fixtures retain their parent and historical-path graph instead of operating from a shallow root. Model review, document generation that requires a model, fixes, Git mutation, push, PR creation, CI monitoring, and gate decisions remain in no-mistakes' existing owner. A configured uncredentialed documentation command may use this runner like any other command, but this bridge never moves a model document step by implication. @@ -373,6 +385,8 @@ bin/fm-azure-runner.sh cost The implementation tests use fake Azure APIs and local executor fixtures only. They create no Azure resource and incur no charge. +Acceptance for the per-run no-mistakes repair remains blocked until this exact code lands on public `main` and one fresh routed `test=behavior-heavy` run passes on real Azure compute with exact remote execution proof and zero cleanup. +Do not spend on an unlanded build, and do not weaken the foundation's `require_landed_code` gate to accelerate that acceptance. Real usability remains unclaimed until the approved deployment performs this bounded acceptance: 1. Record Mac CPU, memory, swap, process count, and responsive interactive latency before launch. diff --git a/docs/azure-validation.md b/docs/azure-validation.md index cba71c97656..4aec21ba815 100644 --- a/docs/azure-validation.md +++ b/docs/azure-validation.md @@ -38,6 +38,7 @@ No operation stops, starts, updates, or inspects a local legacy no-mistakes daem Remote Herdr is not part of this path. Later Herdr proxy tabs may display state, but validation submission, ask-user responses, recovery, evidence collection, and cleanup use Azure Resource Manager, Managed Run Command, and the private storage endpoint. +The runner's separate per-run no-mistakes routing path may seal a detached gate HEAD as a direct private bundle with ordinary standalone capacity accounting; that path is not a validation cell or a child of one. No Azure resource was created while implementing this feature. Live Azure acceptance of these cells runs from released public main under separate explicit billable authorization; the first live Stage C pipeline acceptance has occurred and is recorded in the Live acceptance record section, while the full multi-leg checklist in the Live acceptance section remains unperformed. @@ -260,7 +261,7 @@ Each request binds the cell, round, shard index/count, branch, current head, tre The fixed behavior command is the existing sealed `bin/fm-behavior-shards.sh --run 8` route with the explicit Azure/Linux non-Herdr selection already used by isolated CI. It also carries the cell's own host-capability declaration, `FM_TEST_HOST_CAPABILITIES_ABSENT`, naming by name the four sealed-suite capabilities a shard worker cannot provide: a real tmux server it can create windows in, passwordless sudo with `systemd-run`, the `/usr/bin/cpp` binding `bin/fm-account-directory.sh` needs before it can validate a Claude quota-axi Keychain approval marker, and outbound reach to the origin remote's host (`origin-egress`). Be exact about what the fourth one costs, because it is much the largest and a skip must never read as coverage. -The runner unit that executes these shards sets `PrivateNetwork=yes`, `RestrictAddressFamilies=AF_UNIX` and `IPAddressDeny=any`, so `bin/fm-teardown.sh`'s secondmate upstream-authority probe can never resolve or reach the origin host, and `origin-egress` therefore skips THIRTY-THREE units - the whole secondmate teardown/retirement family in `tests/fm-teardown-suite.sh`. +The runner unit that executes these shards sets `PrivateNetwork=yes`, `RestrictAddressFamilies=AF_UNIX` and `IPAddressDeny=any`, so `bin/fm-teardown.sh`'s secondmate upstream-authority probe can never resolve or reach the origin host, and `origin-egress` therefore skips THIRTY-SEVEN units - the whole secondmate teardown/retirement family in `tests/fm-teardown-suite.sh`. The cell verifies none of that family: not the landed-work refusals, not the registry locking, not the network-authority pinning, not the child quiescence ordering. That coverage lives on macOS and in CI, which is where any change to `bin/fm-teardown.sh` must be proven. The set was enumerated to convergence rather than sampled: both teardown files run to completion with the network off, 143 of 143 cases, and `docs/azure-requirements.md` R4 owns the full account. @@ -272,6 +273,7 @@ It does not run the command locally. The dispatcher assigns each shard the exact SKU and pre-reserved constituent id from the admitted shape plan, avoiding the control cell's family, and sets the runner concurrency ceiling to eight. Each request uses the runner's exact private-parent snapshot mode, so a pipeline-owned fix commit can run before the no-mistakes push step without executing locally or prematurely mutating the remote task branch. The one-ref Git bundle, current source ref/head/tree, digest, size, and private input blob are all bound to the parent cell and command request, while the runner independently re-proves the public trusted default base. +This parent-bound shard mode remains distinct from the direct per-run no-mistakes bundle in `docs/azure-runner.md`, which must not claim or require validation-cell capacity. A new child starts through `runner run` with an explicit invocation, confirmation, source ref, private bundle, and parent-cell reservation; only an already recorded child uses `runner resume`, so a missing VM cannot turn a prepared record into duplicate execution. The one-shot runner still re-proves live family quota, regional quota, rate, budget, private network, image, command bounds, and global admission under its own contract. Every shard therefore receives a separate VM, OS disk, process namespace, port space, lock space, temp root, terminal-server space, boot id, and VM instance id. @@ -484,6 +486,7 @@ A missing ETag, changed instance, foreign tag, foreign principal, extra role ass Focused fake-cloud and static tests do not claim real Azure usability. The first live Stage C pipeline acceptance has occurred, as described in the Live acceptance record section, while the full multi-leg acceptance checklist below remains unperformed. +The separate direct per-run runner repair is not accepted by those cell results and remains blocked until its exact code lands on public `main` and a fresh routed no-mistakes run passes on real Azure compute with zero cleanup. After this stack is released to public main and the operator has explicit billable authorization, the full checklist acceptance must run from that released main and record all of these results: 1. Record Mac wall time, CPU, memory, swap, process count, and interactive latency before admission. diff --git a/docs/azure-worker-runtime.md b/docs/azure-worker-runtime.md index 752285c7f23..62034d6ed16 100644 --- a/docs/azure-worker-runtime.md +++ b/docs/azure-worker-runtime.md @@ -1,9 +1,7 @@ # Azure worker crewmate runtime and payload plane (design) -Status: PARTLY BUILT. D1-D3 (runtime image, payload plane, worker-coordinate -entrypoint) and D5 (landing v1) have shipped; docs/azure-workers.md is the -authority for their built behavior and this document keeps only the decisions -and what remains. D6 (release receipts before cloud teardown) is still open. +Status: BUILT, with the post-2026-08-25 return path awaiting its bounded live ship and scout acceptance. +D1-D3 (runtime image, payload plane, worker-coordinate entrypoint), D5 (landing v1), and D6 (provider-neutral local custody plus release) have shipped in code; docs/azure-workers.md is the authority for their built behavior and this document keeps the design history. This document owns the design for the pieces between "a worker executes a digest-bound argv" (proven live 2026-08-17 on vm-fm7c799d-wkr-01) and "a real pi crewmate does task work in the cloud". @@ -15,10 +13,9 @@ data-disk preparation (/mnt/account lun0, /mnt/task lun1), bounded execute with digest-bound fm.worker-execution-result/v1, TTL, Herdr tracking endpoint with queued-spawn convergence (PR 227). -Items 1 to 5 below were the original gap list. They are now BUILT (1 to 4 by -D1-D3, 5 by D5) and docs/azure-workers.md owns their behavior; they are kept -here only so the decisions that follow read against the problem they solved. -Item 6 is the one still open. +Items 1 to 6 below were the original gap list. +They are now BUILT (1 to 4 by D1-D3, 5 by D5, and 6 by D6), and docs/azure-workers.md owns their behavior. +They are kept here only so the decisions that follow read against the problem they solved. 1. BUILT. Runtime: pi and node did not exist on the worker. Workers booted the raw Canonical Ubuntu 24.04 base; bootstrap installed only the supervisor. @@ -34,18 +31,10 @@ Item 6 is the one still open. worker. The crewmate's commits ride home as a digest-verified bundle and the local side keeps the landing authority. Direct push from a worker remains explicitly deferred. -6a. OPEN. Resume destroys uncollected commits: the executed marker is on the - disposable OS disk and /mnt/task is retained, so a resume reattaches a task - disk holding the previous run's commits to a worker with no marker. The next - execute re-stages and removes them. The retained outcome bundle survives but - is unreachable, because the only re-upload path is the marker branch and - there is no collect-only mode. A refusing guard was tried and reverted (it - wedged the slot and lost the work anyway); the fix needs collect-only, which - belongs with D6. -6. OPEN. Release-after-teardown: bin/fm-worker-authority.py needs the task meta - and worktree that bin/fm-teardown.sh deletes in the same pass that removes - the endpoint, so an ordinarily torn-down cloud task can never produce its - release receipts (observed live: cloud-smoke-20260817 rode its TTL out). +6a. BUILT. An explicit `execute --existing-task-disk` recovery binds the already-assigned repository without staging over it, runs one bounded continuation or no-op collection command, and returns the same authorized report, status, commit, and scratch bundle. + The request binds the exact landed recovery-supervisor digest, and the adapter executes those bytes without replacing the assignment's originally bootstrapped supervisor. +6. BUILT. Release no longer waits for ordinary teardown. + The monitor localizes the digest-bound return, reconstructs the ship branch or retains scout scratch, writes the required report and terminal status, mints the cloud-custody release receipt while the exact tracking endpoint is finishing, and retries release/reconcile until the assignment is complete. ## Decisions @@ -65,10 +54,8 @@ is inert until the parameter is supplied) and point it at the new gallery version. Rationale: workers are egress-sealed by posture; per-boot registry downloads are both a provenance hole and a boot-time failure mode, while the image is a cache of a recorded provenance chain (the bake doc's own model). -The pi extension pack (multi-pass and friends) is NOT staged: those exist to -rotate a human's many local accounts; a worker runs one leased account. The -multi-pass OAuth-refresh patch is therefore also not needed on workers; the -bounded wall (max 6h) sits inside a fresh access token's life. +Ordinary author workers do not stage the local Pi extension pack because one worker runs one leased account. +The specialized no-mistakes role instead uses the sealed runtime documented in `docs/azure-workers.md`, which includes only fast mode and Ketch and still excludes multi-pass and OAuth material. ### D2. One payload archive per assignment, over the private staging lane @@ -123,12 +110,12 @@ already puts them. Direct push from workers (with a scoped deploy token) is explicitly deferred; it changes the custody story and should be its own decision when the soak data says the round-trip is too slow. -### D6. Cloud teardown produces release receipts before destroying evidence +### D6. The return monitor releases cloud capacity after local custody -For placement=azure tasks, fm-teardown.sh runs authority-receipt (which needs -the live meta and worktree) BEFORE removing them, stores the -fm.worker-release/v2 bundle under state/, then calls release and a reconcile -so the slot deallocates inside its cooldown instead of riding the TTL. +For `placement=azure` tasks, the digest-bound return carries the authorized report, status, visuals, repository commits, and scratch in one provider-neutral Git bundle. +The local monitor validates and publishes those bytes, reconstructs the required `fm/` branch for ship work, synthesizes a truthful terminal status, and then mints `fm.worker-release/v2` while the task metadata and worktree still exist. +The receipt's landing authority proves local custody, not remote forge landing, so remote worker capacity can be released before no-mistakes and the later ordinary teardown continue locally. +Release and reconcile are idempotent retries, and local credential staging is removed only after the controller queue entry is complete. ## Sequencing @@ -140,13 +127,14 @@ so the slot deallocates inside its cooldown instead of riding the TTL. 3. Flip: workerImageId parameter supplied; one live crewmate smoke on a real task; the crosscheck lane reviews the whole stack (it gates its own producer now). -4. Landing: outcome bundle + monitor fetch + local fast-forward (D5, SHIPPED); the - push stays where it always was, in the ordinary local landing flow. Then D6. +4. Landing: authorized return bundle, exact task-branch reconstruction, local report/status publication, and release after custody (D5/D6, BUILT). + The push stays where it always was, in the ordinary local landing flow. 5. Wide soak (8 then 16 lanes) only after 3 and 4 hold. ## Non-goals -- No pi extension pack, fast-mode, or account rotation on workers. +- No account rotation or OAuth material inside a worker runtime bundle. +- Ordinary author workers keep their existing extension-free runtime; the no-mistakes role has only its sealed fast-mode and Ketch extensions. - No public egress from workers; every transfer stays on the private lane. - No captain-on-cloud changes; that is its own phase with its own custody design (setup-token) and is tracked outside this document. diff --git a/docs/azure-workers.md b/docs/azure-workers.md index f236af11d70..8f90ca1a1ac 100644 --- a/docs/azure-workers.md +++ b/docs/azure-workers.md @@ -6,10 +6,10 @@ The private foundation and one-shot command substrate remain owned by [Azure pil ## Boundary and topology -A general worker VM runs one task-scoped crewmate plus the minimal machine supervisor required for lifecycle, event delivery, steering, and recovery. -It never runs Firstmate, a secondmate, another supervisor, a nested team, a browser profile, validation, policy review, or a child-worker launcher. -Persistent secondmates remain on trusted control-plane capacity and may request one task worker without moving into it. -The primary Firstmate requests workers for tasks with no matching secondmate. +One worker-ceiling assignment runs either one task-scoped crewmate or one bounded secondmate compartment plus the minimal machine supervisor required for lifecycle, event delivery, steering, and recovery. +A secondmate compartment requests children only through the host controller and has no child-worker launcher or authority to create compute directly. +No assignment runs Firstmate, an additional nested supervisor, a browser profile, validation, or policy review. +The primary Firstmate requests ordinary workers for tasks with no matching secondmate and may request up to the separately bounded secondmate-compartment count within the same sixteen-assignment ceiling. Validation, review, browser, and networkless verification use their separate single-purpose compartments. The controller is provider-neutral at the queue and state-machine boundary. @@ -32,41 +32,51 @@ The state file and lock are owner-only, atomically replaced, directory-synced, a The caller supplies only task, task generation, owner kind, and eligibility; the controller reads the ordinary task metadata, canonical account home/task owner, exact worktree root, physical Git-directory identity, and HEAD to derive the home, provider-account, writable-worktree, repository, and repository-generation bindings. Caller-supplied bindings are unsupported outside the hermetic test backstop. Raw provider-account identity never appears in bounded status or Azure tags. -The account binding must be a high-entropy digest produced by the account lease owner, not a digest of a guessable profile name. -The lease owner is the controller: it derives the binding from the profile's upstream account identity (see the placement section below), never from the profile's local slot name. -The local slot name (`openai-codex-2` and the like) DOES appear in bounded status, because an operator has to be able to see which profile a task holds; it is a local label for a pool slot, not the upstream account identity, and it never reaches an Azure tag. +The controller derives one high-entropy `account_binding` from the selected profile's upstream account identity, never from a guessable local profile label. +That reusable digest remains visible in bounded placement/load status and Azure assignment bindings so provider quota pressure can be correlated without exposing identity or token material. +The local profile label (`openai-codex-2` and the like) also appears in bounded status but never in an Azure tag. -The host is the only OAuth refresh authority. Before a leased single-profile home is staged, `fm-spawn.sh` requires twelve hours of access-token headroom, twice the worker VM's six-hour hard shutdown window. A stale slot is withdrawn and refused instead of being copied to a guest that could reach Pi's automatic refresh path and rotate the pool's refresh token independently. +The host is the only OAuth refresh authority. +The controller uses `bin/fm-credential-expiry.py` to require twelve hours of canonical-profile access-token headroom before every snapshot, which is twice the worker VM's six-hour hard shutdown window. +`bin/fm-spawn.sh` repeats the same check against the immutable snapshot immediately before task staging. +A stale profile is excluded before snapshot creation, so no guest lives long enough to refresh its copied credential and no guest can change the canonical pool. ## Provider-account placement across the Pi fleet -The task metadata names the provider-account POOL this task may draw from. The cloud lane reads the canonical absolute directory in `config/azure-worker-account-home` when present and otherwise falls back to the primary Pi coding-agent home for compatibility. This lets a local Firstmate keep a separate Pi login while Azure owns a disjoint worker fleet. WHICH profile of the selected pool the placement gets is decided by the controller, because that decision has to exclude every other concurrent placement and no task-local document can see them. +Task metadata names the canonical provider-account pool this task may draw from. +The cloud lane reads the absolute directory in `config/azure-worker-account-home` when present and otherwise falls back to the primary Pi coding-agent home for compatibility. +This lets local Firstmate and Azure worker credentials remain disjoint. -Selection happens inside `command_request`, in the same lock hold and the same `save_state` that writes the queue entry, so selection and the lease are one act. -The queue entry IS the lease: the set of leased accounts is derived from the non-complete queue, never from a second ledger, so there is no state a crash can leave in which an account is held by something the queue does not show. -Selection is deterministic - the first free profile in the pool's sorted (lexicographic) name order - and replaying the same task generation reuses the same profile, because the replay path short-circuits on the existing entry before selecting anything. +Selection occurs under the controller lock and counts every non-complete placement by canonical pool, local profile label, and upstream account binding. +It chooses the least-active usable profile with profile label and account binding as stable tie-breakers, so every usable profile is represented before one is reused. +An upstream account binding is load identity rather than exclusion authority, and multiple simultaneous workers may carry immutable snapshots of the same profile or account. +Replaying one exact task generation never selects again and retains its profile, account binding, and assignment-private projection binding. -The unit of exclusion is the UPSTREAM ACCOUNT, not the profile name and not the account-home path. -Eight profiles map to eight accounts today, but nothing enforces that: a re-login can point two slots at one account, and what a concurrent crewmate actually contends for - the rate limit, the ban, the session - belongs to the account. -So `account_binding` is a digest over the profile's upstream account identity (itself a SHA-256 digest of the account id, never token material), two profiles resolving to one account are ONE lease, and the duplicate-account screen below is the same screen selection already respected. +Every new placement computes a projection binding over its home, task, task generation, canonical pool, selected profile, and upstream account binding. +The controller records the request in durable `projecting` state before credential bytes are written under `$FM_HOME/state/azure-workers/accounts/` or the configured `FM_PI_ACCOUNT_HOME_ROOT`. +A crash during projection therefore leaves a resumable and withdrawable queue owner rather than an unowned credential directory. +A single-profile canonical pool is still copied into an assignment-private projection and is never used in place. +No writable directory is keyed only by profile label or upstream account identity. -A pool holding more than one profile is projected: the controller writes the chosen profile's single-profile account home with `bin/fm-pi-account-home.py`, under its OWN state directory (`$FM_HOME/state/azure-workers/accounts/`, overridable with `FM_PI_ACCOUNT_HOME_ROOT`) and deliberately not the shared crosscheck roster, which belongs to the reviewer lane and must not be rewritten under a running reviewer. -The directory is keyed on the LEASE IDENTITY, never on the profile's local slot name, because the projection key and the exclusion key must be the same function of the pool. The slot name is not: re-logging one slot from one upstream account to another yields two placements with two correct, distinct bindings that would both project into one `accounts/` directory, and the second write would replace the credential the first placement's still-live lease points at, leaving the queue reporting two accounts while the disk held one. A second, defensive refusal also declines to project over an account home a live queue entry still names. -A home already holding exactly one profile is that single-profile home already, and is leased in place with nothing written; its credential shape is not screened there, because "is this credential still good" has one owner, `bin/fm-credential-expiry.py`. -`request` prints the leased profile and account home, and `bin/fm-spawn.sh` writes the staged account directory exactly once, from that home, after the lease exists. -The pooled `auth.json` is never staged: the payload step deliberately does not copy it, because that step runs BEFORE the lease is created and while the tracking monitor pane is already polling, so a crash there would otherwise leave every signed-in account in a directory the monitor is willing to dispatch as `--account-dir`. The window is removed rather than guarded. -As defence in depth at the point of USE, `bin/fm-spawn-cloud-monitor.sh` re-checks that the staged account directory holds exactly one provider slot before it dispatches, and does so BEFORE taking the shared exactly-once dispatch marker so a not-yet-narrowed directory simply retries on the next poll instead of wedging both owners. -Staging the pool would put every signed-in account on the guest and let Pi resolve the first slot, which is a shared-account placement whatever the queue records. +`bin/fm-pi-account-home.py` writes one fixed-key single-profile snapshot into that private projection. +`request` prints only the selected profile label and its assignment-private home, and `bin/fm-spawn.sh` copies that credential once into the task's own staged account directory. +The pooled `auth.json` is never staged. +As defence in depth at use, `bin/fm-spawn-cloud-monitor.sh` checks that the staged account directory holds exactly one provider slot before taking the exactly-once dispatch marker. -Every failure refuses by name and none of them falls through to a shared or arbitrary profile: an unreadable or empty pool, a pool whose profiles are all unprojectable, a home naming no upstream account, and an exhausted pool (which names each leased profile and the task holding it). -Bounded status projects the live placements - profile, task generation, status, and account home - from `controller.json` alone. +`withdraw` accepts both queued and interrupted `projecting` requests and removes only the exact projection binding the queue entry owns. +Provider reset removes the same exact projection only after release and cloud-side cleanup are proved. +Cleanup inventories no sibling path, so one assignment cannot replace, inspect, or delete another assignment that uses the same profile. +A legacy entry with no projection binding is never inferred to own and delete a shared home. -**The pool is now a concurrency ceiling, and it is lower than the worker ceiling.** Concurrent placements are bounded by `min(FM_AZURE_WORKER_MAX, distinct upstream accounts in the pool)`: with the fleet's eight Pi accounts, the ninth concurrent placement refuses even though `MAX_WORKERS` is 16 and quota, budget and capacity would all admit it. -That is the requirement, not a regression - sixteen crewmates never could run on eight accounts without sharing one, they just used to do it silently - but it does halve the effective author parallelism, and compartments compete in the same pool: one compartment plus its four children consumes five of the eight before an ordinary crewmate is placed. -Raising the ceiling means adding signed-in profiles on distinct upstream accounts to the pool, not raising a knob here. -A compartment child contends in the same document as an ordinary crewmate, because `FM_HOME` still names the primary's controller for both, so the two can never be handed one account. +Every failure refuses by name, including an unreadable pool, no usable profile with twelve-hour headroom, a changed upstream identity during projection replay, and a conflicting private projection or worktree. +Bounded status lists every placement with profile, task generation, reusable account binding, projection binding, private home, and current per-profile/account active load. -The controller rejects duplicate active account or writable-worktree bindings. +The worker/supervisor software ceiling remains the independent `FM_AZURE_WORKER_MAX` maximum of sixteen even when fewer than sixteen Pi profiles or upstream accounts are available. +Ordinary author workers and nested secondmate supervisors share that ceiling and the load-balanced Pi worker pool. +No-mistakes uses its separate Azure runner, and Crosscheck uses its separately credentialed `fireworks-glm` model plus networkless tool/verifier compartments. +Those specialized lanes consume the shared 40-vCPU specialized envelope and 128-vCPU regional accounting but consume no worker slot and no Codex/Pi worker profile. + +The controller permits duplicate active upstream account bindings and rejects duplicate assignment-private projection, account-home, or writable-worktree bindings. A general request has role `author`, is explicitly eligible, and is owned by either the primary or a secondmate; a secondmate-owned author request may carry a parent compartment pair, which marks it as a compartment child and arms the child bounds. A `secondmate` role request stands up a secondmate compartment, is requested only by the primary, and is capped by `FM_AZURE_SECONDMATE_MAX`. A compartment cannot mint that parent pair itself: its agent only emits a bounded `fm.secondmate-child-request/v1` on the outbox, and the compartment monitor validates it locally and then stamps the pair onto an ordinary `bin/fm-spawn.sh` spawn run as the secondmate, so every compartment child is admitted by the one controller under the child bounds. @@ -87,7 +97,7 @@ The `bounded` in the compartment invariant is per message, not aggregate: each r Bounded status additionally projects every live compartment - task, status, slot, active and lifetime children counts, and the durable assignment TTL anchor - from `controller.json` fields only, never from the leg state the compartment monitor owns locally. The same task generation and exact identity is idempotent, while a changed identity under the same task generation refuses. An assigned request stays in the queue until its ordinary release proof is accepted and every exact cloud resource is safely reset. -A request that never reached assignment leaves the queue by `withdraw`: it accepts an entry still in `queued`, refuses anything a worker owns or a pending provider action names, requires `--confirm-withdraw` and `--confirm-subscription`, touches no capacity, and removes the per-task cloud state including the staged provider credential. +A request that never reached assignment leaves the queue by `withdraw`: it accepts `projecting` or `queued`, refuses anything a worker owns or a pending provider action names, requires `--confirm-withdraw` and `--confirm-subscription`, touches no capacity, and removes the exact provider projection plus per-task cloud state including the staged credential. Release remains the only exit for work that ever held capacity. Operator surrender is not a second exit: it mints that release proof for the one case where the ordinary authorities are unrecoverable, under its own refusal-first gates (below). Therefore a truly empty queue also means there is no active task worker and desired worker compute is zero. @@ -190,7 +200,7 @@ All fifteen resource kinds are recorded by complete resource ID and immutable pr Every taggable resource is additionally bound to deployment owner, slot, home, task, task generation, assignment generation, cloud generation, account digest, worktree digest, repository digest, and repository generation. The container carries the equivalent exact metadata. -No two active tasks share a VM, account lease, browser profile, or writable task disk. +No two active tasks share a VM, writable account home, browser profile, or writable task disk, even when their reusable upstream account bindings match. The worker has exactly one slot identity and its NIC has no public-IP relation. The general worker contract forbids a browser profile rather than allocating one. The OS disk is disposable, while the account and task disks detach from VM deletion and remain encrypted by the guest contract. @@ -209,25 +219,43 @@ A visible VM with another task or assignment binding refuses instead of being ad ## Outcome collection and landing A crewmate on a worker holds no forge or provider credential, so the work comes home as bytes, not as a push: nothing on the worker pushes anywhere, and the local side keeps the landing authority. -`execute --outcome-dir` records `outcome_expected` inside the digest-bound execution request and the provider mints one short-lived user-delegation SAS with create/write on exactly one blob name, delivered as a protected Run Command parameter. +`execute --outcome-dir --return-kind ` records both `outcome_expected` and one closed `fm.worker-return-contract/v1` inside the digest-bound execution request. +The contract authorizes only that task's report, status trail, visual directory, required ship branch, and repository scratch. +The spawned brief rewrites only the exact task-home prefix to `/mnt/task/.fm-return`, so its authorized completion paths exist on the guest without making arbitrary guest paths returnable. +The provider mints one short-lived user-delegation SAS with create/write on exactly one blob name, delivered as a protected Run Command parameter. That SAS scopes the credential, not the guest: the worker identity already holds Storage Blob Data Contributor on its whole state container, so what makes a landing safe is the digest in the signed result, not the narrowness of the SAS. Because the expectation is digest-bound, a stripped parameter cannot silently downgrade a landing task: the guest refuses before the argv runs. -After the bounded execute, the supervisor counts the commits the crewmate added over the bound repository generation, bundles exactly those commits, refuses a bundle over 256 MiB, uploads it, and records `outcome_present`, `outcome_commits`, `outcome_sha256`, `outcome_bytes`, and `outcome_error` inside the signed result. +After the bounded execute, the supervisor counts the commits the crewmate added over the bound repository generation and records tracked and untracked scratch without applying it locally. +It places the authorized report, status, visuals, and scratch in one synthetic Git artifact commit, places project commits on a distinct outcome ref, and creates one bundle containing both refs while excluding the already-bound repository generation. +The result records `return_present`, the exact return ref and commit, the manifest digest, the project outcome tip, and the existing bundle digest and byte fields. +A scout with no commits therefore still returns its report and scratch, while a ship's project history and completion artifacts remain distinguishable inside one provider-neutral blob. A collection failure never aborts the result: the command has already had its effects, so the failure is recorded instead, which both blocks an unverifiable landing and stops a replay from running the command a second time. -A worker whose pinned supervisor predates this contract answers with no outcome disposition at all, and the controller refuses that result rather than reporting a task whose commits silently never came home. +A worker whose pinned supervisor predates this contract answers with no return disposition, and the controller refuses that result rather than reporting a task whose commits or required deliverables silently never came home. The controller downloads the blob only after the digest-bound result commits to its bytes, verifies size and SHA-256, and stores it in the requesting task's outcome directory. -The tracking monitor then fast-forwards the leased local worktree, but only when that worktree still sits on the dispatched generation and is clean; otherwise the verified bundle is kept and its path reported. -Landing authority, push, and release receipts stay exactly where the ordinary local flow already puts them. -The blob name carries the request digest, so a later execute against the same worker cannot overwrite an outcome the controller has not collected yet. Reset deletes the inbound staging archives by name and the outcome blobs as part of removing the whole state container. +`bin/fm-cloud-result.py` independently validates the result, bundle, refs, manifest, and each artifact digest before publishing anything locally. +For ship work it creates or fast-forwards `fm/` and checks that branch out only when the leased worktree and any existing task branch have not diverged; a divergence keeps the fetched custody ref and never overwrites local work. +For scout work it leaves the scratch worktree on its dispatched generation and stores any returned patch and untracked archive with the report. +A missing or invalid authored report retains the assignment and produces no generated report or local terminal authority; collection and release resume only after the actual returned report and branch or scratch custody validate. +Replaying the same result converges on the same refs, files, branch, and one terminal status line. +An assigned worker whose earlier supervisor could not return its task disk uses the explicit `execute --existing-task-disk` recovery lane. +That request carries no payload or account archive, binds the exact landed recovery-supervisor digest, and runs one bounded continuation or no-op collection command against the retained repository instead of replacing it. +The Azure adapter executes those bound supervisor bytes from a task-command-local recovery path without changing the originally bootstrapped supervisor, while every alternate provider must preserve the same request binding and retained-disk semantics. +A missing repository, lost dispatched lineage, unreadable working tree, payload-restaging attempt, or recovery request without an authorized return refuses before the command runs, because continuing after any of them could delete or misattribute unlanded task-disk work. +The blob name carries the request digest, so a later execute against the same worker cannot overwrite an outcome the controller has not collected yet. +Reset deletes the inbound staging archives by name and the outcome blobs as part of removing the whole state container. ## Release, reset, and cooldown The controller never infers safe deletion from a terminal chat line, a missing VM, elapsed time, or budget pressure. -The ordinary Firstmate owners first remove the endpoint, publish the report, prove landed work, release the provider account, and complete their normal cleanup checks. -`authority-receipt` invokes `bin/fm-worker-authority.py`, which reads the ordinary task metadata, endpoint backend oracle, completion-report contract, Git landing graph, account task/home binding, and clean exact worktree root rather than accepting operator-entered digests. -For Azure placement, the task's `account_home` is the vendor-neutral Pi pool selected above, so account authority instead requires the task-recorded selected profile and single-profile home to equal the controller queue's exact lease, then reads its owner-private credential without following links and reproduces the controller-owned upstream-account binding. +The ordinary Firstmate owners first establish the task's required local authorities, then release the provider account and complete their normal cleanup checks. +For a local placement, that still means endpoint absence, report publication, and forge-reachable landing before release. +For an Azure author return, the exact local tracking endpoint may still be alive only while it performs this finalization: the digest-bound result has ended remote execution, and the endpoint receipt accepts that narrow return-localized state after the terminal status exists. +The Azure landing receipt proves local custody rather than forge landing: the return bundle and manifest match the result, the required report and terminal status exist, ship commits are reachable from the checked-out `fm/` branch, and any declared uncommitted scratch has a retained artifact. +This releases billable remote capacity without weakening the ordinary later teardown gate, which still protects the unpushed local task branch until it reaches a remote or default branch. +`authority-receipt` invokes `bin/fm-worker-authority.py`, which reads the ordinary task metadata, endpoint backend oracle, completion-report contract, Git landing or cloud-return custody graph, account task/home binding, and clean exact worktree root rather than accepting operator-entered digests. +For Azure placement, the task's `account_home` is the canonical Pi pool selected above, so account authority requires the task-recorded profile and assignment-private home to equal the controller queue's exact snapshot record, then reads its owner-private credential without following links and reproduces the reusable upstream-account binding. It produces an `fm.worker-release/v2` bundle with five independently canonical `fm.worker-authority/v1` receipts for endpoint absence, report validity, landed work, account ownership, and writable-worktree cleanliness, plus the exact home, task, generations, cloud instance, account, worktree, repository, and every resource identity. When the CONTROLLER-OWNED worker role is `secondmate`, the same five receipts carry compartment evidence semantics: endpoint proves the compartment monitor pane absent through the same backend oracle; report proves the session closeout - the monitor's terminal status file, the chained close ack in its durable state, and the ordered `completion.md` contract; landing proves every chained outbox bundle landed into the local secondmate home worktree (or provably none) by REACHABILITY - each collected bundle's own tip commit must be an ancestor of the home worktree's HEAD, which also descends from the assignment's exact starting repository generation; account is unchanged; and worktree proves the home quiesced - exact repository root, no uncommitted or untracked work - while staying advisory for children, whose refusal `command_release` owns. Which semantics apply is never decided by the task metadata alone: the worker record's `role` and the metadata's `kind` must agree, and a disagreement in either direction refuses, so flipping one local metadata line can never move an ordinary author worker onto the compartment lane and release work that was never landed. @@ -252,6 +280,8 @@ The compartment bundle is the identical `fm.worker-release/v2` shape and verifie `proof-template` remains diagnostic only: its placeholders are deliberately invalid and hand-filling them is unsupported. A missing, stale, malformed, or conflicting receipt retains everything. +The tracking monitor retries collection, receipt minting, release recording, and reconcile by re-running the same idempotent steps until the queue entry is complete. +Only then does it remove the locally staged provider credential and convergence files; the credential-free outcome bundle and task report remain. After an exact release receipt, reconcile deallocates the VM promptly. Azure deallocation stops compute billing but not disks, NICs, public foundation meters, monitoring, or storage operations. After deallocation it deletes the named execute and bootstrap Run Commands and monitor extension before the VM, then proves VM absence and disk/NIC detach before deleting NIC and OS disk; the TTL remains enabled until those proofs complete and is deleted last among compute children. @@ -314,6 +344,71 @@ The Azure adapter re-verifies the full live assignment before invoking the minim Its read-only inventory uses Azure CLI 2.88-compatible role-assignment syntax (`--all` without `--resource-group`), requires private Entra blob reads for reservation/request/result identities, and uses one unambiguous primary Linux on-demand USD Consumption meter while excluding Spot, Low Priority, Windows, dev/test, reservation, and savings offers. It never forwards arbitrary shell text, provider credentials, or a hosted control endpoint. +## No-mistakes worker wrapper + +`bin/fm-no-mistakes-worker` is the Firstmate-owned high-level transport used by the no-mistakes coordinator. +Its only supported invocation is: + +```sh +bin/fm-no-mistakes-worker --config '' execute \ + --request '' \ + --payload '' \ + --result '' \ + --outcome '' \ + --step-outcome '' +``` + +The request, result, and semantic step outcome use `no-mistakes.firstmate-worker-request/v1`, `no-mistakes.firstmate-worker-result/v1`, and `no-mistakes.worker-step-outcome/v1` respectively. +The request and result echo the canonical `step` (`review` or `test`) separately from job `kind`; a repair may repair either step, and the semantic artifact follows `step`, so a test repair can never assert a review-approved head. +They also echo the caller's lowercase SHA-256 `runtime_identity`, which binds the exact wrapper bytes, private wrapper-config bytes, and transport protocol into the job's content-addressed input; a changed runtime is a new job identity, never a replay under mutable code. +The caller payload contains exactly `repo.bundle` and `brief.md`; the wrapper verifies both against the request, stages the configured digest-bound credential-free `runtime.tar.gz`, and submits the request's exact argv without a shell. +The owner-private config uses `fm.no-mistakes-worker-wrapper-config/v1` and names the Firstmate home, canonical Pi account pool home, sealed runtime path and digest, lifecycle executable and exact clean Firstmate source commit, bounded assignment/cleanup/wall times, and the non-secret lifecycle environment. +The wrapper rechecks that source commit and clean tracked lifecycle closure before every lifecycle call, and re-verifies the staged guest runtime against its configured digest after copying it, so neither an ordinary Firstmate update nor a path replacement can silently change an admitted job. + +`docs/azure-no-mistakes-worker-config.example.json` is the copy-and-fill wrapper config template. + +Build the credential-free runtime on Linux amd64 from the exact custom no-mistakes binary, a compatible Linux Node binary, an installed `@earendil-works/pi-coding-agent` package closure, and the exact fast-mode and Ketch packages: + +```sh +bin/fm-no-mistakes-runtime \ + --no-mistakes /absolute/linux-amd64/no-mistakes \ + --node /absolute/linux-amd64/node \ + --pi-package /absolute/linux-pi-package/@earendil-works/pi-coding-agent \ + --fast-mode-package /absolute/extensions/pi-openai-fast-mode \ + --fast-mode-fleet-extension /absolute/fast-mode-all-codex-accounts.ts \ + --ketch-package /absolute/extensions/pi-ketch \ + --no-mistakes-version '' \ + --no-mistakes-source-commit '' \ + --output /absolute/no-mistakes-pi-runtime.tar.gz +``` + +The builder refuses host-native or non-amd64 Node and no-mistakes artifacts, redirected package members, native modules from another platform, credential-shaped files, duplicate paths, and unbounded inputs. + +It emits the established `fm.azure-validation-runtime/v1` manifest with a digest for every byte and pins Pi, fast mode, and Ketch by their package bytes and versions. + +Use an npm installation made on the target Linux amd64 build host as `--pi-package`; never point the builder at `~/.pi/agent`, an account pool, or a directory containing `auth.json`. + +The worker lifecycle selects the least-loaded usable account and projects only that account into the assignment-private HOME as the fixed `openai-codex` profile. + +The guest does not load multi-pass or choose an account. + +The sealed `bin/pi` launcher uses the bundled Node, disables ambient extension discovery, pins fast mode and Ketch from the verified runtime, and reads OAuth only from `$HOME/pi-agent/auth.json`. + +The builder and hermetic staged-runtime execution are proven locally; a real Azure no-mistakes run is still required before claiming the packaged Linux closure live. +It never names an account profile: `fm-worker-lifecycle` selects the least-loaded usable profile under its controller lock and creates an assignment-private projection. + +The dedicated `no-mistakes` lifecycle role admits only `repo.bundle`, `brief.md`, and `runtime.tar.gz`. +The guest verifies the runtime's exact file inventory, runs the role command with that runtime on `PATH`, and returns the bounded semantic artifact through an execution-owned `fm.no-mistakes-worker-return/v1` bundle. +The wrapper verifies the semantic bytes and head binding before writing the controller-facing result; a process exit, missing outcome, malformed outcome, or changed read-only head is a failed result, never `CLEAR` by inference. +Repair results return one digest-bound single-ref bundle whose head must descend from the requested head, while review and test return no code bundle and must keep the exact requested head. +The wrapper records a retryable local candidate before cleanup, releases through `service-complete` only after the lifecycle owns the exact execution result, and replays the candidate after a lost response instead of executing the step again. +No caller chooses an Azure account, sees a credential, invokes `fm-azure-runner.sh`, or bypasses lifecycle cleanup. + +The failed retained proof `azr-763d70ab8206` used the generic Azure runner, reached runtime dependency installation, then exited 125 at `guest bootstrap: isolated executor failed` without any structured result. +That is not evidence about a no-mistakes verdict. +This wrapper avoids that failure shape by using the worker supervisor's digest-bound execution record and returns a closed failed envelope when the guest produces no semantic artifact. +Live Azure usability remains unclaimed until this exact wrapper/runtime/lifecycle path completes a billable zero-to-zero proof. + ## Reconcile and bounded status A read-only plan is the default: @@ -341,7 +436,7 @@ The next controller process replays that exact action before considering new wor `status` is local and bounded by default, while `status --live` refreshes Azure and cost evidence. The output includes author and specialized queue depths, desired and actual active workers, all five classification counts, assignment generations, worker-hours and warning threshold, actual and forecast spend, active policy phase and limit, the 128-vCPU regional ceiling plus observed and observed-plus-reserved usage, exact-family observed-plus-reserved commitments, the 64-vCPU author plan, active and reserved specialized capacity, 22-vCPU shared headroom, cooldown, warm target, retained-disk count, and the last ten cleanup refusals. -It omits subscription IDs, resource IDs, account identities, account digests, worktree digests, private addresses, credentials, and secrets. +It omits subscription IDs, resource IDs, raw account identities, worktree digests, private addresses, credentials, and secrets; the high-entropy reusable account binding is deliberately visible with profile load. ## Operator policy and overrides @@ -357,7 +452,7 @@ Supported policy changes are deliberately narrow: - `FM_AZURE_WORKER_MAX` may lower the software cap but may never exceed sixteen. - `--required` admits already-authorized recovery or landing demand through cost pressure, but never through unreadable telemetry, quota, identity, or cleanup proofs. -There is no force-adopt, force-delete, delete-by-age, kill-for-budget, public-network, shared-account, shared-worktree, shared-browser, warm-filesystem, or hosted-form override. +There is no force-adopt, force-delete, delete-by-age, kill-for-budget, public-network, shared-account-home, shared-worktree, shared-browser, warm-filesystem, or hosted-form override. A human who needs a different destructive or security boundary must change and review this contract rather than bypass it at runtime. ## Lavish @@ -374,7 +469,7 @@ Run `bin/fm-worker-lifecycle.sh acceptance-plan` for the concise checklist. The complete acceptance must record all of these outcomes: -1. Start from zero and run at least three representative tasks in parallel on three distinct VMs, account bindings, and task disks. +1. Start from zero and run at least three representative tasks in parallel on distinct VMs, assignment-private account projections, and task disks, including two tasks that reuse one upstream binding. 2. Finish one task while another compatible task waits and prove that deallocate, VM/NIC/OS deletion, released account/task/identity/container reset, and a new assignment generation occur before the waiting task starts. 3. Drain every request and prove desired and active worker compute reach zero after cooldown. 4. Prove every disposable VM, NIC, OS disk, extension, Run Command, TTL, staging request/result, and active global reservation is absent, while only explicitly retained ambiguous unfinished disks remain. @@ -385,7 +480,7 @@ The complete acceptance must record all of these outcomes: 9. Execute one representative private command through the pinned supervisor, collect its exact result, close the real endpoint, validate/publish the report, prove landing, account release, and clean worktree through `authority-receipt`, then record bounded status plus actual and forecast cost evidence before, during, and after the exercise. Every acceptance leg needs a positive control that proves the check detects the unsafe state. -Positive controls include a planted public-IP relation, a foreign immutable ID, a stale task generation, a duplicated account digest, a duplicated worktree digest, a deliberately retained dirty disk, a repeated provider action, a forecast above policy, and planted cross-task files, processes, sockets, browser data, cloud identities, or credentials. +Positive controls include a planted public-IP relation, a foreign immutable ID, a stale task generation, a duplicated projection binding, a duplicated worktree digest, a deliberately retained dirty disk, a repeated provider action, a forecast above policy, and planted cross-task files, processes, sockets, browser data, cloud identities, or credentials. The unsafe fixture must be isolated, must trigger the expected refusal or probe failure, and must be removed without weakening the production check. A failed leg keeps cloud-default author use unaccepted. diff --git a/docs/configuration.md b/docs/configuration.md index fc3d7b37f81..6da7b6867c9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -97,11 +97,12 @@ See [`wedge-alarm.md`](wedge-alarm.md) for the channel reference and macOS verif The tracked `.no-mistakes.yaml` keeps test evidence outside the repo and preserves `bin/fm-no-mistakes-test-command.sh` as the ordinary local test owner. Inside an admitted Azure validation cell, the trusted default-branch command string instead invokes the root-owned bridge from `docs/azure-validation.md`, which runs lint and requested behavior shards on separate credential-free Azure command VMs without exposing the cell's provider or GitHub lease. That evidence policy is specific to the firstmate repo: target projects may legitimately commit `.no-mistakes/evidence/` from their own no-mistakes pipeline, but firstmate keeps `.no-mistakes/` local and CI rejects tracked entries under that path. -The ordinary local command requires `tmux` on `PATH`, prints `tmux -V`, derives every `herdr-lab` and `herdr-mixed` path directly from [`tests/test-capabilities.tsv`](../tests/test-capabilities.tsv), and sends those files in one serial invocation through [`tests/run.sh`](../tests/run.sh) before running the locked Agent Fleet pytest and compileall checks. +The ordinary local command requires `tmux` on `PATH`, prints `tmux -V`, derives every `herdr-lab` and `herdr-mixed` path directly from [`tests/test-capabilities.tsv`](../tests/test-capabilities.tsv), and runs two concurrent lanes: one serial [`tests/run.sh`](../tests/run.sh) invocation for those files and one locked Agent Fleet pytest-and-compileall invocation. It does not maintain a second file list or duplicate hermetic-only behavior files on the Mac. The required `Behavior tests` job in [`.github/workflows/ci.yml`](../.github/workflows/ci.yml) uses the duration-balanced sharding owned by [`bin/fm-behavior-shards.sh`](../bin/fm-behavior-shards.sh) to run and verify the complete behavior inventory across eight isolated runners. CI selects the explicit non-Herdr path because its disposable image carries no Herdr, so it runs every hermetic body and the hermetic portion of the mixed file while the local host admits the complete `herdr-lab` and `herdr-mixed` set through owned labs. -The exact behavioral coverage contract is the union of that capability-derived local host set and the required CI executed-manifest union, not a claim that all 123 files run serially before push. +The exact behavioral coverage contract is the 123-file union of that capability-derived local host set and the required CI executed-manifest union across eight shards, not a claim that all 123 files run serially before push. +The tracked pytest proof adapter `tests/test_azure_proof_contracts.py::test_local_required_ci_coverage_contract` executes the public `tests/fm-azure-runner.test.sh` behavior suite, which fails if a hermetic-only registry entry is added to the production local selector or if `.no-mistakes.yaml` stops routing ordinary local tests to `bin/fm-no-mistakes-test-command.sh` and admitted validation-cell tests to the required shard bridge. Both routes cross the same sealed admission boundary, and the local serial route holds real-Herdr labs to one at a time. ## Crosscheck reviewer diff --git a/docs/evidence/azure-breaktest-live-v1-private-acceptance.md b/docs/evidence/azure-breaktest-live-v1-private-acceptance.md new file mode 100644 index 00000000000..28dc28eb1ae --- /dev/null +++ b/docs/evidence/azure-breaktest-live-v1-private-acceptance.md @@ -0,0 +1,3 @@ +# Private Azure composite acceptance marker + +This private-only commit exists solely to exercise the bounded no-mistakes Azure route against integration tip `7f4828ff0a2d66498d9933cd84d23002aaf4f715`. diff --git a/docs/test-isolation.md b/docs/test-isolation.md index 14922339473..ab6c2677bd3 100644 --- a/docs/test-isolation.md +++ b/docs/test-isolation.md @@ -54,7 +54,7 @@ Herdr is not the only thing a sealed-suite host can lack. `tests/host-capabilities.tsv` declares every other host capability the suite depends on, the platforms that can provide it, and the exact units bound to it, and `tests/host-capability-gate.sh` is the only door to that set, through `fm_require_host_capability "