chore: merge current main into CI cleanup

Author: ruby-dlee

bin/fm-no-mistakes-worker

@@ -4,6 +4,7 @@
 import argparse
 import fcntl
 import hashlib
+import ipaddress
 import json
 import os
 from pathlib import Path
@@ -44,6 +45,7 @@ LIFECYCLE_ENV_KEYS = {
     "FM_AZURE_BUDGET_START_DATE", "FM_AZURE_DEPLOYMENT_GENERATION",
     "FM_AZURE_OWNER_TAG", "FM_AZURE_NAMING_PREFIX", "FM_AZURE_RESOURCE_GROUP",
     "FM_AZURE_STORAGE_NAME", "FM_AZURE_LOCATION", "FM_AZURE_WORKER_STATE_DIR",
+    "FM_AZURE_WORKER_IMAGE_ID", "FM_AZURE_OPERATOR_DATA_PLANE_IP",
     "FM_AZURE_WORKER_MAX", "FM_AZURE_WORKER_IDLE_COOLDOWN_SECONDS",
     "FM_AZURE_WORKER_IDLE_RELEASE_SECONDS", "FM_AZURE_WORKER_POLICY_PHASE",
     "FM_AZURE_WORKER_STEADY_TARGET_USD", "FM_AZURE_WORKER_COMMISSIONING_CEILING_USD",
@@ -115,6 +117,20 @@ def validate_config(value):
         not isinstance(item, str) or not item or "\x00" in item for item in env.values()
     ):
         raise WrapperError("lifecycle environment is incomplete or malformed", "config_invalid", False)
+    operator_ip = env.get("FM_AZURE_OPERATOR_DATA_PLANE_IP")
+    if operator_ip is not None:
+        try:
+            parsed_operator_ip = ipaddress.IPv4Address(operator_ip)
+        except ipaddress.AddressValueError:
+            raise WrapperError(
+                "operator data-plane route must be one canonical IPv4 address",
+                "config_invalid", False,
+            )
+        if str(parsed_operator_ip) != operator_ip:
+            raise WrapperError(
+                "operator data-plane route must be one canonical IPv4 address",
+                "config_invalid", False,
+            )
     for field, low, high in (
         ("assignment_timeout_seconds", 1, 7200),
         ("cleanup_timeout_seconds", 1, 7200),

docs/azure-no-mistakes-worker-config.example.json

@@ -18,7 +18,9 @@
     "FM_AZURE_DEPLOYMENT_GENERATION": "replace-with-deployment-generation",
     "FM_AZURE_OWNER_TAG": "replace-with-owner-tag",
     "FM_AZURE_NAMING_PREFIX": "replace-with-naming-prefix",
-    "FM_AZURE_STORAGE_NAME": "replace-with-storage-account"
+    "FM_AZURE_STORAGE_NAME": "replace-with-storage-account",
+    "FM_AZURE_OPERATOR_DATA_PLANE_IP": "replace-with-one-exact-operator-egress-ipv4",
+    "FM_AZURE_WORKER_IMAGE_ID": "replace-with-exact-compute-gallery-image-version-resource-id"
   },
   "assignment_timeout_seconds": 1800,
   "cleanup_timeout_seconds": 1800,

docs/azure-workers.md

@@ -363,6 +363,7 @@ The request and result echo the canonical `step` (`review` or `test`) separately
 They also echo the caller's lowercase SHA-256 `runtime_identity`, which binds the exact wrapper bytes, private wrapper-config bytes, and transport protocol into the job's content-addressed input; a changed runtime is a new job identity, never a replay under mutable code.
 The caller payload contains exactly `repo.bundle` and `brief.md`; the wrapper verifies both against the request, stages the configured digest-bound credential-free `runtime.tar.gz`, and submits the request's exact argv without a shell.
 The owner-private config uses `fm.no-mistakes-worker-wrapper-config/v1` and names the Firstmate home, canonical Pi account pool home, sealed runtime path and digest, lifecycle executable and exact clean Firstmate source commit, bounded assignment/cleanup/wall times, and the non-secret lifecycle environment.
+That environment may carry the exact immutable `FM_AZURE_WORKER_IMAGE_ID` and single reviewed `FM_AZURE_OPERATOR_DATA_PLANE_IP`; retaining them across incremental retries prevents an existing claimed VM from being replayed against a different image reference or disabling the controller's default-deny operator route mid-recovery.
 The wrapper rechecks that source commit and clean tracked lifecycle closure before every lifecycle call, and re-verifies the staged guest runtime against its configured digest after copying it, so neither an ordinary Firstmate update nor a path replacement can silently change an admitted job.
 
 `docs/azure-no-mistakes-worker-config.example.json` is the copy-and-fill wrapper config template.

tests/behavior-test-durations.tsv

@@ -76,8 +76,8 @@
 1686	tests/fm-macos-permissions.test.sh
 3428	tests/fm-nm-step-liveness.test.sh
 1560	tests/fm-no-mistakes-reattach.test.sh
-3500	tests/fm-no-mistakes-runtime.test.sh
-8500	tests/fm-no-mistakes-worker.test.sh
+2770	tests/fm-no-mistakes-runtime.test.sh
+7260	tests/fm-no-mistakes-worker.test.sh
 2000	tests/fm-pi-account-home.test.sh
 10	tests/fm-pi-primary-live-e2e.test.sh
 80	tests/fm-pi-primary-types.test.sh

tests/fm-azure-pilot.test.sh

@@ -593,6 +593,7 @@ run_bounded_mutation_deadline_checks() {
   state_dir=$(mktemp -d)
   write_sourceable_script "$sourceable"
   set +e
+  # shellcheck disable=SC2030,SC2031
   output=$(
     (
       set -- help "$state_dir"
@@ -614,7 +615,7 @@ run_bounded_mutation_deadline_checks() {
   status=$?
   set -e
   [ "$status" -ne 0 ] || fail "bounded Azure mutation accepted a hung CLI"
-  # shellcheck disable=SC2031
+  # shellcheck disable=SC2030,SC2031
   [ "$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["phase"])' "$state_dir/apply.json")" = retained ] || fail "timed-out mutation did not retain exact operation state"
   python3 - "$SCRIPT" <<'PY' || fail "foundation mutating Azure calls bypass bounded state owner"
 from pathlib import Path
@@ -750,6 +751,7 @@ run_worker_create_replay_quota_checks() {
   sourceable=$(mktemp)
   write_sourceable_script "$sourceable"
   set +e
+  # These exports intentionally reset names assigned in earlier isolated runtime-test subshells.
   # shellcheck disable=SC2030,SC2031
   output=$(
     (

tests/fm-credential-expiry.test.sh

@@ -280,10 +280,9 @@ home = work / "home"
 
 # The preflight runs before the FIFO lane wait, before runtime_config, and
 # again once the lane is held because the lane wait can outlast the
-# credential. A third check immediately before staging still precedes every
-# uploaded object and shared-host run. The snapshot wrapper deliberately
-# delegates these contracts to the two functions that own lane admission and
-# review execution.
+# credential. A third check inside the held lane still precedes every remote
+# staged object and reviewer-host use. The snapshot wrapper delegates these
+# contracts to the two functions that own lane admission and paid compute.
 lane_source = inspect.getsource(adapter._run_azure_review_after_snapshot)
 entry_source = inspect.getsource(adapter.run_azure_review)
 assert "_run_azure_review_after_snapshot" in entry_source
@@ -292,7 +291,7 @@ assert lane_source.index("preflight_reviewer_credential") < lane_source.index("r
 assert lane_source.rindex("preflight_reviewer_credential") > lane_source.index("acquire_review_lane")
 compute_source = inspect.getsource(adapter._run_azure_review_in_lane)
 assert compute_source.index("preflight_reviewer_credential") < compute_source.index("upload_blob")
-assert compute_source.index("preflight_reviewer_credential") < compute_source.index("submit_model_run")
+assert compute_source.index("preflight_reviewer_credential") < compute_source.index("ensure_model_host")
 
 
 def review(profile, harness):

tests/fm-no-mistakes-worker.test.sh

@@ -37,6 +37,11 @@ import subprocess
 import sys
 
 root = Path(__file__).parent
+expected_image = "/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/fixture/providers/Microsoft.Compute/galleries/fixture/images/worker/versions/1.0.0"
+if os.environ.get("FM_AZURE_WORKER_IMAGE_ID") != expected_image:
+    raise SystemExit("exact worker image identity was not forwarded to lifecycle")
+if os.environ.get("FM_AZURE_OPERATOR_DATA_PLANE_IP") != "203.0.113.10":
+    raise SystemExit("exact operator data-plane IP was not forwarded to lifecycle")
 state_path = root / "fake-state.json"
 complete = root / "complete"
 log = root / "calls.log"
@@ -286,7 +291,7 @@ pass "guest supervisor re-verifies the sealed runtime inventory and executable"
 
 RUNTIME_SHA=$(shasum -a 256 "$TMP_ROOT/runtime.tar.gz" | awk '{print $1}')
 cat > "$TMP_ROOT/config.json" <<JSON
-{"schema":"fm.no-mistakes-worker-wrapper-config/v1","fm_home":"$HOME_DIR","account_pool_home":"$HOME_DIR/accounts","runtime_bundle":"$TMP_ROOT/runtime.tar.gz","runtime_bundle_sha256":"$RUNTIME_SHA","lifecycle_path":"$FAKE","lifecycle_source_commit":"$LIFECYCLE_COMMIT","lifecycle_env":{"FM_AZURE_TENANT_ID":"22222222-2222-4222-8222-222222222222","FM_AZURE_SUBSCRIPTION_ID":"11111111-1111-4111-8111-111111111111","FM_AZURE_ADMIN_EMAIL":"fixture@example.invalid","FM_AZURE_ADMIN_USERNAME":"fixtureadmin","FM_AZURE_ADMIN_SSH_PUBLIC_KEY":"ssh-ed25519 AAAATEST fixture","FM_AZURE_RUNNER_OPERATOR_OBJECT_ID":"33333333-3333-4333-8333-333333333333","FM_AZURE_KEY_VAULT_NAME":"fixture-vault","FM_AZURE_BUDGET_START_DATE":"2026-08-01","FM_AZURE_DEPLOYMENT_GENERATION":"dep-fixture","FM_AZURE_OWNER_TAG":"owner","FM_AZURE_NAMING_PREFIX":"fixture","FM_AZURE_STORAGE_NAME":"fixturestorage"},"assignment_timeout_seconds":30,"cleanup_timeout_seconds":30,"poll_seconds":1,"wall_seconds":60}
+{"schema":"fm.no-mistakes-worker-wrapper-config/v1","fm_home":"$HOME_DIR","account_pool_home":"$HOME_DIR/accounts","runtime_bundle":"$TMP_ROOT/runtime.tar.gz","runtime_bundle_sha256":"$RUNTIME_SHA","lifecycle_path":"$FAKE","lifecycle_source_commit":"$LIFECYCLE_COMMIT","lifecycle_env":{"FM_AZURE_TENANT_ID":"22222222-2222-4222-8222-222222222222","FM_AZURE_SUBSCRIPTION_ID":"11111111-1111-4111-8111-111111111111","FM_AZURE_ADMIN_EMAIL":"fixture@example.invalid","FM_AZURE_ADMIN_USERNAME":"fixtureadmin","FM_AZURE_ADMIN_SSH_PUBLIC_KEY":"ssh-ed25519 AAAATEST fixture","FM_AZURE_RUNNER_OPERATOR_OBJECT_ID":"33333333-3333-4333-8333-333333333333","FM_AZURE_KEY_VAULT_NAME":"fixture-vault","FM_AZURE_BUDGET_START_DATE":"2026-08-01","FM_AZURE_DEPLOYMENT_GENERATION":"dep-fixture","FM_AZURE_OWNER_TAG":"owner","FM_AZURE_NAMING_PREFIX":"fixture","FM_AZURE_STORAGE_NAME":"fixturestorage","FM_AZURE_OPERATOR_DATA_PLANE_IP":"203.0.113.10","FM_AZURE_WORKER_IMAGE_ID":"/subscriptions/11111111-1111-4111-8111-111111111111/resourceGroups/fixture/providers/Microsoft.Compute/galleries/fixture/images/worker/versions/1.0.0"},"assignment_timeout_seconds":30,"cleanup_timeout_seconds":30,"poll_seconds":1,"wall_seconds":60}
 JSON
 chmod 600 "$TMP_ROOT/config.json"
 
@@ -343,7 +348,31 @@ PY
   [ ! -e "$TMP_ROOT/calls.log" ] \
     || fail "missing Azure foundation field reached lifecycle before config validation"
 done
-pass "wrapper requires the complete Azure foundation identity before lifecycle dispatch"
+
+python3 - "$TMP_ROOT/config.json" "$TMP_ROOT/config-broad-route.json" <<'PY'
+import json, pathlib, sys
+source, target = pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2])
+value = json.loads(source.read_text())
+value["lifecycle_env"]["FM_AZURE_OPERATOR_DATA_PLANE_IP"] = "0.0.0.0/0"
+target.write_text(json.dumps(value, separators=(",", ":")))
+PY
+chmod 600 "$TMP_ROOT/config-broad-route.json"
+write_request "$TMP_ROOT/request-broad-route.json" job-broad-route
+"$WRAPPER" --config "$TMP_ROOT/config-broad-route.json" execute \
+  --request "$TMP_ROOT/request-broad-route.json" --payload "$PAYLOAD" \
+  --result "$TMP_ROOT/result-broad-route.json" --outcome "$TMP_ROOT/outcome-broad-route.bundle" \
+  --step-outcome "$TMP_ROOT/step-outcome-broad-route.json"
+python3 - "$TMP_ROOT/result-broad-route.json" <<'PY' \
+  || fail "broad operator data-plane route did not return a closed config failure"
+import json, pathlib, sys
+result = json.loads(pathlib.Path(sys.argv[1]).read_text())
+assert result["outcome"] == "failed"
+assert result["error_category"] == "config_invalid"
+assert result["retryable"] is False
+PY
+[ ! -e "$TMP_ROOT/calls.log" ] \
+  || fail "broad operator data-plane route reached lifecycle before config validation"
+pass "wrapper requires exact Azure foundation identity and a single IPv4 operator route before dispatch"
 
 printf 'ok\n' > "$TMP_ROOT/mode"
 write_request "$TMP_ROOT/request.json" job-success

tests/fm-watch-triage.test.sh

@@ -1887,13 +1887,11 @@ SH
     . "$1"
     run_bounded 1 "$2"
   ' _ "$WATCH" "$stubborn" || rc=$?
-  # GNU timeout reports 137 when its KILL fallback fires; the Perl fallback
-  # reports the logical timeout status 124. Both prove the same bounded
-  # termination, and the process check below proves cleanup.
-  case "$rc" in
-    124|137) ;;
-    *) fail "watcher timeout wrapper returned unexpected status $rc" ;;
-  esac
+  # GNU timeout may return 137 after its --kill-after escalation sends KILL;
+  # both values prove the bounded timeout path, and process absence below is
+  # the independent cleanup proof.
+  [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ] \
+    || fail "watcher timeout wrapper returned $rc instead of 124 or 137"
   [ -s "$pid_file" ] || fail "watcher timeout fixture did not start"
   ! kill -0 "$(cat "$pid_file")" 2>/dev/null \
     || fail "watcher timeout wrapper left the TERM-resistant process alive"