From 9ef08cbe38e79b9673265378fb12c13cea51fabd Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Fri, 21 Aug 2026 00:08:03 -0400 Subject: [PATCH 1/9] fix(azure): bind a validation cell result to the attempt that produced it --- bin/fm-azure-validation-guest.sh | 11 +- bin/fm-azure-validation.py | 131 +++++++++++++++++- tests/fm-azure-validation.test.sh | 223 +++++++++++++++++++++++++++++- 3 files changed, 359 insertions(+), 6 deletions(-) diff --git a/bin/fm-azure-validation-guest.sh b/bin/fm-azure-validation-guest.sh index a4dc0369394..ac5156c9637 100755 --- a/bin/fm-azure-validation-guest.sh +++ b/bin/fm-azure-validation-guest.sh @@ -1008,7 +1008,7 @@ pathlib.Path(sys.argv[1]).write_text(json.dumps(value, sort_keys=True, separator PY RESULT=$STATE/result-a$ATTEMPT.json python3 - "$REQUEST" "$IDENTITY" "$RESULT" "$VM_RESOURCE_ID" "$VM_INSTANCE_ID" \ - "$(cat /proc/sys/kernel/random/boot_id)" "$OUTCOME" "$CURRENT_HEAD" "$CURRENT_TREE" "$REMOTE_HEAD" "$PR" "$CHECKS_GREEN" "$SHARD_RECEIPTS" <<'PY' + "$(cat /proc/sys/kernel/random/boot_id)" "$OUTCOME" "$CURRENT_HEAD" "$CURRENT_TREE" "$REMOTE_HEAD" "$PR" "$CHECKS_GREEN" "$SHARD_RECEIPTS" "$ATTEMPT" <<'PY' import json import pathlib import sys @@ -1037,6 +1037,7 @@ result = { "vm_resource_id": sys.argv[4], "vm_instance_id": sys.argv[5], "boot_id": sys.argv[6], + "attempt": int(sys.argv[14]), "outcome": sys.argv[7], "pr_url": sys.argv[11] or None, "checks_green": sys.argv[12] == "true", @@ -1099,4 +1100,10 @@ if [ -n "$OUTPUT_URL" ]; then rm -f "$CURL_CONFIG" fi BOOT_ID=$(cat /proc/sys/kernel/random/boot_id) -printf 'FM_AZURE_VALIDATION_RESULT %s boot=%s outcome=%s\n' "$RESULT_DIGEST" "$BOOT_ID" "$OUTCOME" +# The marker binds the published result to the ATTEMPT that produced it. A +# respond/reattach attempt runs on the same VM and the same boot, so boot_id +# and every identity field in result.json are identical across the attempts of +# one run: without the attempt there is nothing a control-plane read can see +# that tells attempt 1's completed marker from attempt 2's own answer +# (generation azv-36b2 ground truth). +printf 'FM_AZURE_VALIDATION_RESULT %s boot=%s outcome=%s attempt=%s\n' "$RESULT_DIGEST" "$BOOT_ID" "$OUTCOME" "$ATTEMPT" diff --git a/bin/fm-azure-validation.py b/bin/fm-azure-validation.py index 32564b2df4d..a1c207e34c8 100755 --- a/bin/fm-azure-validation.py +++ b/bin/fm-azure-validation.py @@ -58,6 +58,15 @@ PR_URL = re.compile(r"^https://github\.com/[^/]+/[^/]+/pull/[1-9][0-9]*$") NM_RUN_ID = re.compile(r"^[0-9A-HJKMNP-TV-Z]{26}$") RUNNER_INVOCATION = re.compile(r"^azr-[a-z0-9]{12}(?:-a[2-9][0-9]*)?$") +# The authenticated result marker. `attempt` is required: every other field +# a control-plane read can see is identical across the attempts of one run +# (same VM, same boot, same run id), so without it an unbound view cannot be +# told from this attempt's own answer. +MARKER = re.compile( + r"FM_AZURE_VALIDATION_RESULT\s+(sha256:[0-9a-f]{64})\s+boot=([0-9a-f-]{36})" + r"\s+outcome=([a-z-]+)\s+attempt=([0-9]{1,9})" +) +MARKER_SETTLE_SECONDS = 300 # Shard transports legitimately run VM creation plus admission plus command # submission in one subprocess: near 300 seconds unloaded and well past it # under any operator-host load. The old 300-second cap manufactured @@ -172,6 +181,32 @@ def iso_utc(value=None): return value.replace(microsecond=0).isoformat().replace("+00:00", "Z") +def marker_settle_seconds(): + """How long an unbound terminal control view may persist before it is believed.""" + raw = os.environ.get("FM_AZURE_VALIDATION_MARKER_SETTLE_SECONDS") + if raw is None or not raw.strip(): + return MARKER_SETTLE_SECONDS + try: + value = int(raw) + except ValueError: + raise ValidationError("FM_AZURE_VALIDATION_MARKER_SETTLE_SECONDS must be a whole number of seconds") + if value < 0: + raise ValidationError("FM_AZURE_VALIDATION_MARKER_SETTLE_SECONDS must not be negative") + return value + + +def seconds_since(stamp): + """Elapsed seconds since an exact recorded UTC stamp, or None when unreadable. + + None is never treated as "long enough": an unreadable stamp keeps an + ambiguous view unsettled rather than authorizing a terminal decision. + """ + try: + return (now_utc() - parse_utc(stamp, "recorded stamp")).total_seconds() + except (ValidationError, TypeError): + return None + + def parse_utc(value, label): try: return dt.datetime.fromisoformat(value.replace("Z", "+00:00")) @@ -1695,6 +1730,9 @@ def create_run_command(env, state, mode, input_url=None, output_url=None, respon raise ValidationError("created validation Run Command has foreign identity") resources["run_command_name"] = name resources["run_command_id"] = run_id + # Every attempt starts its own settling window: a stamp left by the + # previous attempt's unbound view must never shorten this one's. + state.pop("unbound_view_since", None) resources.setdefault("run_commands", []).append({ "id": run_id, "identity": immutable_identity(run_command, "run-command"), @@ -1889,11 +1927,68 @@ def observe(env, args): return output = str((view or {}).get("output", "")) error = str((view or {}).get("error", "")) - marker = re.search(r"FM_AZURE_VALIDATION_RESULT\s+(sha256:[0-9a-f]{64})\s+boot=([0-9a-f-]{36})\s+outcome=([a-z-]+)", output) - if not marker: + marker = re.search(MARKER, output) + if not marker or int(marker.group(4)) != state["attempt"]: + # A terminal control state whose output does not carry THIS + # attempt's marker proves nothing about this attempt. The run + # command, the VM, the boot, and every identity field in + # result.json are shared across the attempts of one run, so an + # unbound view is ambiguous between "the guest died before + # publishing" and "the control plane has not caught up with the + # attempt just created". Retaining on the first such read is + # destructive and unrecoverable: failed-retained is a phase + # observe itself refuses, so one premature poll strands a cell + # whose attempt is still executing (generation azv-36b2 ground + # truth: read nine seconds after the respond Run Command was + # created). Fail closed by never ACCEPTING an unbound view, and + # take the terminal decision only once the ambiguity persists. + settle_seconds = marker_settle_seconds() + since = state.get("unbound_view_since") + if not since: + state["unbound_view_since"] = iso_utc() + save_state(env, state) + since = state["unbound_view_since"] + waited = seconds_since(since) + if waited is None or waited < settle_seconds: + observed = marker.group(4) if marker else "none" + print( + "AZURE VALIDATION UNSETTLED cell={} attempt={} control_state={} " + "marker_attempt={} waited={}s settle={}s".format( + state["cell"], state["attempt"], execution, + observed, "unknown" if waited is None else int(waited), settle_seconds, + ) + ) + return transition(env, state, "failed-retained", "cell ended without an authenticated result marker", control_error=error[-2000:]) raise ValidationError("cell ended without an authenticated result; worktree and lease remain retained") - state["expected_result_digest"] = marker.group(1) + state.pop("unbound_view_since", None) + digest = marker.group(1) + # A republished byte-identical result is a non-answer, not a verdict. + # An attempt that resumes a parked run without answering its gate, or + # one whose upload never happened and left the previous attempt's + # archive in place, publishes the exact bytes the previous attempt + # did. Name that instead of surfacing a generic failure. + previous = state.get("attempt_result_digests") or {} + stale = sorted( + key for key, value in previous.items() + if value == digest and key != str(state["attempt"]) + ) + if stale: + transition( + env, state, "failed-retained", + "attempt republished attempt {} result unchanged; the gate was not answered".format( + ", ".join(stale) + ), + control_error=error[-2000:], + ) + raise ValidationError( + "attempt {} published a byte-identical copy of attempt {}'s result; the operator " + "response did not reach the in-cell pipeline, so this is a non-answer, not a " + "verdict".format(state["attempt"], ", ".join(stale)) + ) + previous[str(state["attempt"])] = digest + state["attempt_result_digests"] = previous + state["expected_result_digest"] = digest state["expected_boot_id"] = marker.group(2) outcome = marker.group(3) if outcome == "needs-decision": @@ -1941,6 +2036,18 @@ def verify_result_identity(state, result): for key, wanted in expected.items(): if result.get(key) != wanted: raise ValidationError("validation result identity mismatch: {}".format(key)) + # The attempt is the only field separating one attempt's result from + # another's on a resumed run, so it is required rather than defaulted: a + # result that does not declare its attempt is refused, never assumed to be + # the current one. + if not isinstance(result.get("attempt"), int) or isinstance(result.get("attempt"), bool): + raise ValidationError("validation result does not declare the attempt that produced it") + if result["attempt"] != state["attempt"]: + raise ValidationError( + "validation result was produced by attempt {}, not the observed attempt {}".format( + result["attempt"], state["attempt"] + ) + ) head = result.get("current_head") tree = result.get("current_tree") if ( @@ -2072,6 +2179,24 @@ def collect(env, args): digest = sha256_file(archive) if digest != state.get("expected_result_digest"): raise ValidationError("downloaded result digest differs from the control-plane marker") + # Effect-shaped non-answer fence: the result blob has one name per + # cell, so an attempt that never uploaded leaves the previous + # attempt's archive in place and collect would otherwise verify it + # and file it as this attempt's answer. Every identity field + # matches on a resumed run, so byte equality with an earlier + # attempt is the observable that says the gate was not answered. + recorded = state.get("attempt_result_digests") or {} + stale = sorted( + key for key, value in recorded.items() + if value == digest and key != str(state["attempt"]) + ) + if stale: + raise ValidationError( + "downloaded result is byte-identical to attempt {}; the operator response did " + "not reach the in-cell pipeline, so this is a non-answer, not a verdict".format( + ", ".join(stale) + ) + ) extracted = temp / "extracted" extracted.mkdir(mode=0o700) safe_extract_result(archive, extracted) diff --git a/tests/fm-azure-validation.test.sh b/tests/fm-azure-validation.test.sh index 7fbc25b502d..f24e359531f 100755 --- a/tests/fm-azure-validation.test.sh +++ b/tests/fm-azure-validation.test.sh @@ -606,7 +606,7 @@ for i in range(1,9): # round this cell actually created, not merely be well-formed. state["shard_runs"]={item["request_digest"]:{"round":item["round"],"shard":item["shard"],"head":item["head"],"command_digest":item["command_digest"],"invocation":item["invocation"]} for item in receipts} result={ - "schema":"fm.azure-validation-result/v1","request_digest":state["request_digest"],"cell":state["cell"],"home_binding":state["request"]["home_binding"],"task":"task","task_generation":"gen","validation_generation":"val","fence":state["request"]["fence"],"branch":"fm/task","submitted_head":head,"current_head":head,"current_tree":tree,"remote_head":head,"worktree_disk_id":"/work","run_id":"01HZX7YQ7EJQH8C9G3N4M5P6R7","vm_resource_id":"/vm","vm_instance_id":"vm-instance","boot_id":state["expected_boot_id"],"outcome":"checks-passed","checks_green":True,"pr_url":"https://github.com/o/r/pull/1","behavior_shards":receipts + "schema":"fm.azure-validation-result/v1","request_digest":state["request_digest"],"cell":state["cell"],"home_binding":state["request"]["home_binding"],"task":"task","task_generation":"gen","validation_generation":"val","fence":state["request"]["fence"],"branch":"fm/task","submitted_head":head,"current_head":head,"current_tree":tree,"remote_head":head,"worktree_disk_id":"/work","run_id":"01HZX7YQ7EJQH8C9G3N4M5P6R7","vm_resource_id":"/vm","vm_instance_id":"vm-instance","boot_id":state["expected_boot_id"],"attempt":state["attempt"],"outcome":"checks-passed","checks_green":True,"pr_url":"https://github.com/o/r/pull/1","behavior_shards":receipts } json.dump({"operation":"result-identity","state":state,"result":result},open(sys.argv[1],"w")) PY @@ -625,6 +625,9 @@ reject("wrong submitted head",lambda r:r.__setitem__("submitted_head","f"*40)) reject("wrong run",lambda r:r.__setitem__("run_id","not-a-run")) reject("wrong VM",lambda r:r.__setitem__("vm_instance_id","peer-vm")) reject("wrong boot",lambda r:r.__setitem__("boot_id","peer-boot")) +reject("another attempt of the same run",lambda r:r.__setitem__("attempt",1 if r["attempt"]!=1 else 2)) +reject("result declaring no attempt",lambda r:r.pop("attempt")) +reject("attempt declared as a bare boolean",lambda r:r.__setitem__("attempt",True)) reject("stale shard head",lambda r:r["behavior_shards"][0].__setitem__("head","c"*40)) reject("stale shard tree",lambda r:r["behavior_shards"][0].__setitem__("tree","c"*40)) reject("alien shard round",lambda r:[item.__setitem__("round","round-ffffffffffff") for item in r["behavior_shards"]]) @@ -1218,6 +1221,223 @@ PY pass "the real bridge emits the receipt set, the guest carries it, and the real collect and close gates accept it end to end, bound to this cell's own shard round" } +gate_answer_binding_contract() { + local tmp work head tree block marker_block + fm_test_tmproot_into tmp fm-azure-validation-gate-answer + work=$tmp/work + mkdir -p "$work/exchange" "$work/state" "$work/evidence/attempt-1" "$work/evidence/attempt-2" "$tmp/home" + make_repo "$tmp/project" + head=$(git -C "$tmp/project/repo" rev-parse HEAD) + tree=$(git -C "$tmp/project/repo" rev-parse 'HEAD^{tree}') + printf '[]\n' >"$work/exchange/receipts.json" + + # 1. The REAL guest result-assembly region must stamp the attempt that + # produced the result, and the REAL marker line must name it too. Without + # both, nothing a control-plane read can see separates one attempt of a + # resumed run from another: same VM, same boot id, same run id. + block=$work/emit.sh + awk '/^SHARD_RECEIPTS=\$SHARD_EXCHANGE/,/^RESULT_ARCHIVE=/' "$GUEST" \ + | grep -v '^install -d\|^cp \|^RESULT_ARCHIVE=' >"$block" + [ -s "$block" ] || fail "the guest result-emission region was not found" + # shellcheck disable=SC2016 # The pattern is literal guest text, not an expansion. + sed -i.bak 's#\$(cat /proc/sys/kernel/random/boot_id)#44444444-4444-4444-8444-444444444444#' "$block" && rm -f "$block.bak" + marker_block=$work/marker.sh + grep '^printf .FM_AZURE_VALIDATION_RESULT' "$GUEST" >"$marker_block" + [ -s "$marker_block" ] || fail "the guest result marker line was not found" + + python3 - "$work/request.json" "$head" <<'PY' +import json,sys +request={ + "limits":{"behavior_shards":0}, + "protocol":{"result_schema":"fm.azure-validation-result/v1"}, + "request_digest":"sha256:"+"1"*64,"cell":"azv-aaaaaaaaaaaa", + "home_binding":"sha256:"+"2"*64,"task":"task","task_generation":"gen", + "validation_generation":"val","fence":"sha256:"+"3"*64, + "repository":{"branch":"fm/fixture","head":sys.argv[2],"slug":"o/r"}, +} +open(sys.argv[1],"w").write(json.dumps(request)+"\n") +PY + printf '{"worktree_disk_id":"/work","run_id":"01HZX7YQ7EJQH8C9G3N4M5P6R7"}\n' >"$work/identity.json" + cat >"$work/drive.sh" <<'DRIVER' +set -euo pipefail +SHARD_EXCHANGE=$FM_TEST_WORK/exchange +REQUEST=$FM_TEST_WORK/request.json +IDENTITY=$FM_TEST_WORK/identity.json +STATE=$FM_TEST_WORK/state +EVIDENCE=$FM_TEST_WORK/evidence +ATTEMPT=$FM_TEST_ATTEMPT +CELL=azv-aaaaaaaaaaaa +OUTCOME=needs-decision +CURRENT_HEAD=$FM_TEST_HEAD +CURRENT_TREE=$FM_TEST_TREE +REMOTE_HEAD=$FM_TEST_HEAD +RUN_ID=01HZX7YQ7EJQH8C9G3N4M5P6R7 +VM_RESOURCE_ID=/vm +VM_INSTANCE_ID=vm-instance +START_EPOCH=1 END_EPOCH=2 START_LOAD=0 END_LOAD=0 +START_MEM_AVAILABLE_KIB=1 END_MEM_AVAILABLE_KIB=1 +PR= +CHECKS_GREEN=false +. "$FM_TEST_BLOCK" +RESULT_DIGEST=sha256:$(printf '%064d' "$FM_TEST_DIGEST_SEED") +BOOT_ID=44444444-4444-4444-8444-444444444444 +. "$FM_TEST_MARKER" >"$STATE/marker-a$ATTEMPT.txt" +DRIVER + for attempt in 1 2; do + env FM_TEST_BLOCK="$block" FM_TEST_MARKER="$marker_block" FM_TEST_WORK="$work" \ + FM_TEST_HEAD="$head" FM_TEST_TREE="$tree" FM_TEST_ATTEMPT="$attempt" FM_TEST_DIGEST_SEED=7 \ + bash "$work/drive.sh" >/dev/null 2>&1 \ + || fail "guest result assembly failed for attempt $attempt" + done + + # The two attempts differ ONLY in the attempt field, which is exactly the + # live shape: same run id, same outcome, same gate, same heads. + python3 - "$work/state/result-a1.json" "$work/state/result-a2.json" \ + "$work/state/marker-a1.txt" "$work/state/marker-a2.txt" "$HOST" <<'PY' \ + || fail "the guest does not bind its published result and marker to the attempt" +import importlib.util,json,pathlib,sys +one=json.load(open(sys.argv[1])); two=json.load(open(sys.argv[2])) +assert one.get("attempt")==1, "result.json does not carry its attempt: "+repr(one.get("attempt")) +assert two.get("attempt")==2, "result.json does not carry its attempt: "+repr(two.get("attempt")) +stripped_one=dict(one); stripped_one.pop("attempt") +stripped_two=dict(two); stripped_two.pop("attempt") +assert stripped_one==stripped_two, "the fixture must differ only by attempt to prove the binding is load-bearing" +spec=importlib.util.spec_from_file_location("validation",sys.argv[5]) +m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m) +for index,expected in ((3,1),(4,2)): + line=pathlib.Path(sys.argv[index]).read_text() + found=m.MARKER.search(line) + assert found, "the guest marker is not accepted by the controller reader: "+line + assert int(found.group(4))==expected, "the marker names attempt "+found.group(4) +PY + + # 2. The REAL observe gate against the exact live shapes. + python3 - "$HOST" "$tmp/home" "$work/state/result-a1.json" "$work/state/result-a2.json" <<'PY' \ + || fail "observe did not bind the control view to the attempt it is observing" +import importlib.util,json,pathlib,sys,types +spec=importlib.util.spec_from_file_location("validation",sys.argv[1]) +m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m) +home=pathlib.Path(sys.argv[2]) +env={"home":home,"state_dir":home/"state"/"azure-validation","subscription":"sub"} +m.ensure_dirs(env) +args=types.SimpleNamespace(cell="azv-aaaaaaaaaaaa") +digest_one="sha256:"+"a"*64 +digest_two="sha256:"+"b"*64 +boot="44444444-4444-4444-8444-444444444444" + +def marker(digest,attempt,outcome="needs-decision"): + return "FM_AZURE_VALIDATION_RESULT {} boot={} outcome={} attempt={}\n".format(digest,boot,outcome,attempt) + +def seed(phase="responding",attempt=2,**extra): + state={ + "schema":m.SCHEMA,"cell":"azv-aaaaaaaaaaaa","phase":phase,"attempt":attempt, + "request_digest":"sha256:"+"1"*64, + "request":{"repository":{"head":"0"*40,"branch":"fm/fixture","slug":"o/r"}, + "task":"task","task_generation":"gen","validation_generation":"val", + "limits":{"behavior_shards":0}}, + "resources":{"run_command_id":"/vm/runCommands/respond-a2"}, + "events":[], + } + state.update(extra) + (env["state_dir"]/"azv-aaaaaaaaaaaa.json").write_text(json.dumps(state)) + return state + +def view(output="",error="",execution="Failed"): + m.run_command_status=lambda _env,_state:(execution,{"output":output,"error":error}) + +def phase(): + return json.loads((env["state_dir"]/"azv-aaaaaaaaaaaa.json").read_text())["phase"] + +# 2a. Terminal control state with NO marker: the live azv-36b2 shape, read +# nine seconds after the respond Run Command was created while the attempt +# was still executing. It must not strand the cell on the first read. +seed() +view(error="validation guest: auth-home push failed\n") +m.observe(env,args) +assert phase()=="responding", "an unbound terminal view stranded the cell on its first read: "+phase() + +# The same ambiguity, persisted past the settling window, IS believed: the +# guard delays a destructive decision, it never abandons it. +import os +os.environ["FM_AZURE_VALIDATION_MARKER_SETTLE_SECONDS"]="0" +seed() +view(error="validation guest: auth-home push failed\n") +try: + m.observe(env,args) +except m.ValidationError as exc: + assert "authenticated result" in str(exc), str(exc) +else: + raise AssertionError("a settled unbound terminal view was never believed") +assert phase()=="failed-retained", phase() +os.environ.pop("FM_AZURE_VALIDATION_MARKER_SETTLE_SECONDS") + +# 2b. The previous attempt's own marker is not this attempt's answer. Every +# other field it carries (digest, boot, outcome) is legitimate. +seed() +view(output=marker(digest_one,1)) +m.observe(env,args) +assert phase()=="responding", "attempt 1's marker was accepted as attempt 2's answer" + +# 2c. This attempt's marker republishing the previous attempt's exact bytes +# is a NON-ANSWER and must say so, not surface as a generic failure. +os.environ["FM_AZURE_VALIDATION_MARKER_SETTLE_SECONDS"]="0" +seed(attempt_result_digests={"1":digest_one}) +view(output=marker(digest_one,2)) +try: + m.observe(env,args) +except m.ValidationError as exc: + assert "non-answer" in str(exc) and "byte-identical" in str(exc), str(exc) +else: + raise AssertionError("an unchanged republished result was accepted as a verdict") +assert phase()=="failed-retained", phase() +recorded=json.loads((env["state_dir"]/"azv-aaaaaaaaaaaa.json").read_text()) +assert "republished" in recorded["events"][-1]["note"], recorded["events"][-1]["note"] +os.environ.pop("FM_AZURE_VALIDATION_MARKER_SETTLE_SECONDS") + +# 2d. A genuinely new attempt result is accepted and recorded per attempt. +seed(attempt_result_digests={"1":digest_one}) +view(output=marker(digest_two,2)) +m.observe(env,args) +assert phase()=="needs-decision", phase() +recorded=json.loads((env["state_dir"]/"azv-aaaaaaaaaaaa.json").read_text()) +assert recorded["attempt_result_digests"]=={"1":digest_one,"2":digest_two}, recorded["attempt_result_digests"] +assert recorded["expected_result_digest"]==digest_two + +# 3. The REAL result-identity gate refuses a result from another attempt and +# refuses one that does not declare its attempt at all. +result=json.load(open(sys.argv[4])) +state={ + "schema":m.SCHEMA,"cell":"azv-aaaaaaaaaaaa","phase":"needs-decision","attempt":2, + "request_digest":result["request_digest"], + "request":{ + "home_binding":result["home_binding"],"task":"task","task_generation":"gen", + "validation_generation":"val","fence":result["fence"], + "repository":{"slug":"o/r","branch":"fm/fixture","head":result["submitted_head"]}, + "limits":{"behavior_shards":0}, + }, + "resources":{"worktree_disk_id":"/work","vm_id":"/vm","vm_instance_id":"vm-instance"}, + "expected_boot_id":boot, +} +m.verify_result_identity(state,result) +older=json.load(open(sys.argv[3])) +try: + m.verify_result_identity(state,older) +except m.ValidationError as exc: + assert "produced by attempt 1" in str(exc), str(exc) +else: + raise AssertionError("attempt 1's result passed the identity gate for attempt 2") +undeclared=dict(result); undeclared.pop("attempt") +try: + m.verify_result_identity(state,undeclared) +except m.ValidationError as exc: + assert "does not declare the attempt" in str(exc), str(exc) +else: + raise AssertionError("a result that declares no attempt was assumed to be the current one") +PY + + pass "an attempt's published result is bound to that attempt, an unbound control view never strands a running cell, and an unchanged republished result is refused as a non-answer" +} + static_contract submit_contract security_negative_contract @@ -1235,3 +1455,4 @@ operator_documentation_contract shard_receipt_demotion_contract receipt_run_scope_contract receipt_chain_close_contract +gate_answer_binding_contract From 63765d223722620acb687553687741649d8507fa Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Fri, 21 Aug 2026 00:16:52 -0400 Subject: [PATCH 2/9] docs(azure): correct R4 blocker 1 to the recorded control-plane cause --- docs/azure-requirements.md | 72 ++++++++++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/docs/azure-requirements.md b/docs/azure-requirements.md index 39de9f98bbe..4d19a279967 100644 --- a/docs/azure-requirements.md +++ b/docs/azure-requirements.md @@ -89,8 +89,11 @@ creates it, the crewmate completes a task, and both release cleanly. Status: PARTIAL. Both lanes are code-complete pending their live acceptance runs. Two lanes exist. -The validation-cell lane (`docs/azure-validation.md`) has never closed a cell; there is no -`azure-validation` state under `$FM_HOME/state`. +The validation-cell lane (`docs/azure-validation.md`) has never closed a cell. +Its state is not absent: `$FM_HOME/state/azure-validation/azv-36b2726cbcf3.json` holds the +complete record of the first live attempt, including its transition timestamps, and is the +ground truth correcting the account below. An earlier revision of this line claimed no +`azure-validation` state existed; that was never re-checked against the directory. A stranding strand was fixed: a passed run carrying a short receipt set is now demoted to failed so the cell collects and retains legibly. The root cause behind "an in-cell bridge producing no receipts" is now found and fixed: the @@ -131,14 +134,55 @@ compute zero, worktree disk and evidence retained, control reservation released. That attempt exposed two blockers that stand between this lane and its acceptance sentence, and neither is the receipts strand: -1. `respond` does not answer a gate. It reports success and starts a new attempt, but the guest - re-publishes the byte-identical previous result: attempt 2's `result.json` matched attempt 1 - exactly (sha256 prefix `330ddea31bfbbb05` both times, `run.log` identical at 3056 bytes, same - `run_id`, same `needs-decision`, same gate), after which the control command reported Failed. - The runtime bundle carried no-mistakes v1.48.0, so this is not the v1.41.2-era behavior. Until - the operator action reaches the in-cell pipeline, a parked cell can only be retained, and an - unchanged republished result should be refused as a non-answer rather than surfacing as a - generic failure. +1. `observe` ended an attempt that was still running, and the ending was unrecoverable. + This was recorded here as "`respond` does not answer a gate" plus "the guest re-publishes the + byte-identical previous result". BOTH ARE WRONG. The operator's answer travelled; nothing in + the guest's respond path is implicated. Ground truth is the cell's own state file + (`$FM_HOME/state/azure-validation/azv-36b2726cbcf3.json`), whose events read `responding` at + 09:47:51, `failed-retained` at 09:48:00, and compute removed at 09:50:30. The host declared + the attempt dead NINE SECONDS after creating its Run Command and deleted the VM 2m39s in; + attempt 1 had taken 1h57m. The `control_error` captured by that read is exactly two lines, the + boot-time auth-home pull warning and the post-run auth-home push warning, which cannot both + come from a guest that has existed for nine seconds: they are attempt 1's complete stderr, so + the instanceView `observe` read was not describing attempt 2 at all. `observe` treated a + terminal `executionState` whose output carried no result marker as proof the attempt had died, + and nothing bound the view it read to the attempt it had just created. The marker is the + guest's last action, so its absence proves nothing about an attempt still working, and + `failed-retained` is a phase `observe` itself refuses, so one premature read was terminal. + The byte-identical result was never a republish. The result blob has one fixed name per cell + (`staging.result_blob` is `control/result.tar.gz`), overwritten by each attempt's upload, and + attempt 2's VM was deleted before it could upload, so any later download necessarily returned + attempt 1's archive unchanged. That is the whole of the reported evidence: the same sha256 + prefix `330ddea31bfbbb05`, the same 3056-byte `run.log`, the same `run_id`, the same + `needs-decision`, the same gate. `collect` never ran at all - the state file carries no + `result` and `state/azure-validation/results/` is empty - so the comparison was made by + downloading the one blob directly, twice. + The latent defect this exposed is worse than the reported one. Had that same stale view + carried attempt 1's MARKER rather than no marker, `observe` would have accepted it, `collect` + would have downloaded attempt 1's archive, matched its digest, and PASSED + `verify_result_identity`, because on a resumed attempt the VM, boot id, run id, heads and + every other verified field are identical and `result.json` carried no attempt number. A silent + false verdict is categorically worse than a generic failure. + Fixed: the guest stamps `attempt` into `result.json` and appends `attempt=` to its marker; + `observe` requires the marker to name the attempt it is observing and never accepts an unbound + view, deferring the terminal decision behind a settling window + (`FM_AZURE_VALIDATION_MARKER_SETTLE_SECONDS`, default 300 seconds, reset per attempt) so + silence means "could not tell" and never authorizes the destructive action, including when the + recorded stamp is itself unreadable; `observe` and `collect` both refuse a result + byte-identical to an earlier attempt's as an explicit non-answer rather than a generic + failure, with `collect` checking the DOWNLOADED ARCHIVE rather than trusting the code path; + and `verify_result_identity` refuses a result that declares no attempt or another attempt's. + Proven hermetically in `tests/fm-azure-validation.test.sh` against the real guest emission + region, the real `observe`, and the real identity gate. + Residual, not fixed: `observe` still drives into `failed-retained`, a phase it refuses, so a + false negative that outlasts the settling window is recoverable only through `replace`. + Upgrading no-mistakes on the HOST is NOT the fix and cannot reach a running cell. The cell's + version comes from the `runtime.tar.gz` handed to `submit --runtime-bundle`; nothing in this + repo BUILDS that bundle, it is extracted only on a `start` boot, and the request is + digest-sealed (`create_run_command` refuses if the guest changed after request preparation). + `azv-36b2726cbcf3`'s bundle declares `no_mistakes_version: 1.48.0` across 110 files. Host + 1.48.0 to 1.53.0 changes what runs in a cell only by rebuilding the bundle from the upgraded + binary and submitting a NEW cell. Do not spend time waiting on a host upgrade here. 2. The sealed suite is not Linux-clean, so every cell run parks. Shard 2 failed on host-coupled units that cannot pass inside a Linux cell (passwordless sudo, tmux window creation, Keychain approval markers), alongside 377 passing units. Until those units skip loudly off macOS, no @@ -207,6 +251,14 @@ neither is the receipts strand: it for a green check is an owner-level decision about the cell's security posture, not a test suite's call. +Open finding from the same attempt, not addressed by that fix: the guest's `adjudicate_gates` +polls `control/gate-response-a-.txt` through `fetch_gate_response`, and nothing in this +repo ever writes that blob, so the loop can only ever time out at its +`FM_AZURE_VALIDATION_GATE_WAIT_SECONDS` default of 5400 seconds. That is 90 minutes of billable +cell per gate for nothing, and it is consistent with attempt 1's 1h57m wall time. The remedy is +a scope decision between wiring the host to publish the blob and deleting the loop; the +in-attempt response path the guest already has does not need it. + So the receipts fix is exercised live up to the gate, which is exactly what used to be impossible, and `close` stays unproven: the acceptance sentence below is not yet met. From ab422e8bd3cbbb1cb558122f5eba26ba51c3c946 Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Fri, 21 Aug 2026 00:32:24 -0400 Subject: [PATCH 3/9] docs(azure): flip R7 to DONE and correct the claude credential readiness claim --- docs/azure-requirements.md | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/docs/azure-requirements.md b/docs/azure-requirements.md index 4d19a279967..0f68b364717 100644 --- a/docs/azure-requirements.md +++ b/docs/azure-requirements.md @@ -801,13 +801,38 @@ accepts for codex-authored work. ## R7. Everything is logged in -Status: HOLDS, through R8. +Status: DONE. -The eight pi profiles renew on their own now, which is R8. -Two of the three profiles in `~/.local/share/agent-fleet/accounts/claude/` hold blanked, -length-zero tokens; the third is `refreshable` with material declared valid to 2026-09-10. -None of the three is needed for R6 anymore: the GLM lane authenticates with a Foundry -deployment key and reviews all authors, so R7 holds with no owner login outstanding. +Re-checked 2026-08-21 against `bin/fm-credential-expiry.py report` and the live roster rather +than against an earlier note. Every credential a live lane reads is present and current, and no +owner login is outstanding. Authors and no-mistakes run on the eight pi profiles `openai-codex` +and `-2` through `-5`, `-7`, `-8`, `-9`, all `usable` to 2026-08-29; the numbering skips 6, and +those eight names are exactly the slots `~/.pi/agent/auth.json` holds, which is what +`bin/fm-pi-refresh.py` renews. R8 keeps them moving: its LaunchAgent +`com.firstmate.pi-auth-refresh` is active with seven runs and last exit code 0. The crosscheck +roster reads its GLM primary plus three of those same pi profiles, and the GLM lane +authenticates with an api key rather than an owner login, so it adds no login to this +requirement; whether that lane returns verdicts is R6 and not this. + +Nothing on a live path reads a claude credential, an `accounts/codex/*` profile, or +`accounts/pi/1` through `6`. No script under `bin/`, `tools/` or `skills/` names the claude or +codex pools at all outside the expiry reporter, no live file under `$FM_HOME/config` names them, +and the pi slot list comes from pi's own `auth.json` rather than a scan of the account +directory, so the numbered leftovers are never selected. Their state is recorded here rather +than owed. An earlier revision of this section called the third claude profile "refreshable with +material declared valid to 2026-09-10", which read the REFRESH token's horizon as readiness. +What is true: two of the three profiles in `~/.local/share/agent-fleet/accounts/claude/` hold +blanked, length-zero access AND refresh tokens, and the third's ACCESS token expired +2026-08-17T20:31:18Z behind a refresh token good to 2026-09-10. `refreshable` states that a +refresh token is held, never that the profile is ready to use. Renewing it is an OWNER LOGIN: +nothing refreshes claude on a schedule, `bin/fm-pi-refresh.py` is hardcoded to `accounts/pi` and +names claude nowhere, and `bin/fm-credential-expiry.py` only reads. None of it is needed, which +is what makes this requirement met rather than blocked. + +This status stands on R8 continuing to hold, because the pi horizon is 2026-08-29 and the +LaunchAgent is what advances it. Re-read `bin/fm-credential-expiry.py report` in full before +relying on this line: a truncated listing miscounts the pool, and the profile numbering is not +contiguous. ## R8. Auth refreshes on its own From 7921dcb146c3093a7e287af942a2e00ea78ace5f Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Fri, 21 Aug 2026 00:41:24 -0400 Subject: [PATCH 4/9] fix(azure): run the sealed guest on resume and read a pre-stamp marker --- bin/fm-azure-validation.py | 137 +++++++++++++++++++++--------- tests/fm-azure-validation.test.sh | 129 +++++++++++++++++++++++++++- 2 files changed, 226 insertions(+), 40 deletions(-) diff --git a/bin/fm-azure-validation.py b/bin/fm-azure-validation.py index a1c207e34c8..3913f7c90d4 100755 --- a/bin/fm-azure-validation.py +++ b/bin/fm-azure-validation.py @@ -64,7 +64,16 @@ # told from this attempt's own answer. MARKER = re.compile( r"FM_AZURE_VALIDATION_RESULT\s+(sha256:[0-9a-f]{64})\s+boot=([0-9a-f-]{36})" - r"\s+outcome=([a-z-]+)\s+attempt=([0-9]{1,9})" + r"\s+outcome=([a-z-]+)\s+attempt=([0-9]{1,18})" +) +# The pre-attempt marker shape. A guest sealed into a cell BEFORE the attempt +# stamp existed cannot be changed (the request is digest-sealed), so its output +# must stay readable or every cell in flight across that upgrade is stranded. +# Only consulted when the output carries no stamped marker at all: MARKER's +# prefix is exactly this shape, so a stamped marker would match it too. +LEGACY_MARKER = re.compile( + r"FM_AZURE_VALIDATION_RESULT\s+(sha256:[0-9a-f]{64})\s+boot=([0-9a-f-]{36})" + r"\s+outcome=([a-z-]+)" ) MARKER_SETTLE_SECONDS = 300 # Shard transports legitimately run VM creation plus admission plus command @@ -1662,11 +1671,44 @@ def create_cell(env, state, selected, replacement=False): adopt_resources(env, state) +def sealed_guest_text(env, state): + """The exact guest text this cell's request was sealed with. + + The seal binds the guest a cell was ADMITTED with. Reading the working tree + instead made that seal depend on the tree never changing, so ANY later edit + to the guest refused `respond`, `reattach` AND `replace` on every cell + already in flight, stranding them with no recovery (`replace` is the only + documented way out of `failed-retained`, and it routes through here too). + `submit` stages a byte copy of the guest beside the request, so that copy is + the sealed artifact and is what a resumed attempt must run. The working tree + is used only when it still matches the seal, which keeps a cell whose + payload was pruned behaving exactly as before. Neither source is trusted on + provenance: only a file whose digest IS the sealed digest is ever returned, + so this widens recovery without widening what may execute. + """ + expected = state["request"]["protocol"]["guest_digest"] + candidates = [] + state_dir = env.get("state_dir") + if state_dir: + candidates.append(Path(state_dir) / "payloads" / state["cell"] / "guest.sh") + candidates.append(GUEST) + for candidate in candidates: + try: + if not candidate.is_file() or candidate.is_symlink(): + continue + except OSError: + continue + if sha256_file(candidate) == expected: + return candidate.read_text(encoding="utf-8") + raise ValidationError( + "no guest matching this cell's exact sealed request digest is available; " + "neither the staged payload copy nor the working tree carries {}".format(expected) + ) + + def create_run_command(env, state, mode, input_url=None, output_url=None, response=None): resources = state["resources"] - current_digest = sha256_file(GUEST) - if current_digest != state["request"]["protocol"]["guest_digest"]: - raise ValidationError("trusted guest changed after exact request preparation") + guest_text = sealed_guest_text(env, state) attempt = state["attempt"] name = "{}-a{}".format(mode, attempt) run_id = resources["vm_id"] + "/runCommands/" + name @@ -1705,7 +1747,7 @@ def create_run_command(env, state, mode, input_url=None, output_url=None, respon "location": "eastus", "tags": run_tags, "properties": { - "source": {"script": GUEST.read_text(encoding="utf-8")}, + "source": {"script": guest_text}, "parameters": arguments, "protectedParameters": protected, "asyncExecution": True, @@ -1927,8 +1969,35 @@ def observe(env, args): return output = str((view or {}).get("output", "")) error = str((view or {}).get("error", "")) - marker = re.search(MARKER, output) - if not marker or int(marker.group(4)) != state["attempt"]: + stamped = list(MARKER.finditer(output)) + # Accept ANY marker naming the attempt being observed, not merely the + # first one in the output: re.search would hand back an earlier + # attempt's line and retain an attempt that completed correctly. + mine = { + (found.group(1), found.group(2), found.group(3)) + for found in stamped + if int(found.group(4)) == state["attempt"] + } + binding = "attempt" + if len(mine) > 1: + # Two different results claiming one attempt is not an answer. + mine = set() + if not mine and not stamped: + # No stamped marker anywhere. Either nothing published, or the cell + # runs a guest sealed BEFORE the stamp existed. That guest cannot be + # changed, because the request is digest-sealed, so refusing to read + # it would strand every cell in flight across this upgrade: the + # attempt would be retained on a fully-published result and + # `expected_result_digest` would never be set, making the result + # unreachable. Fall back to the pre-stamp shape, which is consulted + # ONLY here because MARKER's prefix is exactly that shape and would + # otherwise match a stamped marker too. A stamped marker naming + # another attempt never reaches this branch and still fails closed. + legacy = LEGACY_MARKER.search(output) + if legacy: + mine = {(legacy.group(1), legacy.group(2), legacy.group(3))} + binding = "legacy" + if not mine: # A terminal control state whose output does not carry THIS # attempt's marker proves nothing about this attempt. The run # command, the VM, the boot, and every identity field in @@ -1950,7 +2019,7 @@ def observe(env, args): since = state["unbound_view_since"] waited = seconds_since(since) if waited is None or waited < settle_seconds: - observed = marker.group(4) if marker else "none" + observed = ",".join(sorted({found.group(4) for found in stamped})) or "none" print( "AZURE VALIDATION UNSETTLED cell={} attempt={} control_state={} " "marker_attempt={} waited={}s settle={}s".format( @@ -1962,7 +2031,12 @@ def observe(env, args): transition(env, state, "failed-retained", "cell ended without an authenticated result marker", control_error=error[-2000:]) raise ValidationError("cell ended without an authenticated result; worktree and lease remain retained") state.pop("unbound_view_since", None) - digest = marker.group(1) + digest, boot_id, outcome = next(iter(mine)) + # How this observation was bound decides how strictly the collected + # result is checked. A stamped marker means the guest can name its + # attempt, so its result MUST; a legacy marker cannot, so it is held to + # the pre-stamp contract instead of being refused. + state["result_binding"] = binding # A republished byte-identical result is a non-answer, not a verdict. # An attempt that resumes a parked run without answering its gate, or # one whose upload never happened and left the previous attempt's @@ -1989,8 +2063,7 @@ def observe(env, args): previous[str(state["attempt"])] = digest state["attempt_result_digests"] = previous state["expected_result_digest"] = digest - state["expected_boot_id"] = marker.group(2) - outcome = marker.group(3) + state["expected_boot_id"] = boot_id if outcome == "needs-decision": transition(env, state, "needs-decision", "no-mistakes ask-user gate owns the exact run") else: @@ -2037,17 +2110,21 @@ def verify_result_identity(state, result): if result.get(key) != wanted: raise ValidationError("validation result identity mismatch: {}".format(key)) # The attempt is the only field separating one attempt's result from - # another's on a resumed run, so it is required rather than defaulted: a - # result that does not declare its attempt is refused, never assumed to be - # the current one. - if not isinstance(result.get("attempt"), int) or isinstance(result.get("attempt"), bool): - raise ValidationError("validation result does not declare the attempt that produced it") - if result["attempt"] != state["attempt"]: - raise ValidationError( - "validation result was produced by attempt {}, not the observed attempt {}".format( - result["attempt"], state["attempt"] + # another's on a resumed run, so where the guest can name it, it is required + # rather than defaulted: a result that does not declare its attempt is + # refused, never assumed to be the current one. `result_binding` is set by + # observe from the marker it actually read, so a cell whose sealed guest + # predates the stamp is held to the pre-stamp contract instead of being + # refused, and every cell that CAN prove its attempt still must. + if state.get("result_binding", "attempt") != "legacy": + if not isinstance(result.get("attempt"), int) or isinstance(result.get("attempt"), bool): + raise ValidationError("validation result does not declare the attempt that produced it") + if result["attempt"] != state["attempt"]: + raise ValidationError( + "validation result was produced by attempt {}, not the observed attempt {}".format( + result["attempt"], state["attempt"] + ) ) - ) head = result.get("current_head") tree = result.get("current_tree") if ( @@ -2179,24 +2256,6 @@ def collect(env, args): digest = sha256_file(archive) if digest != state.get("expected_result_digest"): raise ValidationError("downloaded result digest differs from the control-plane marker") - # Effect-shaped non-answer fence: the result blob has one name per - # cell, so an attempt that never uploaded leaves the previous - # attempt's archive in place and collect would otherwise verify it - # and file it as this attempt's answer. Every identity field - # matches on a resumed run, so byte equality with an earlier - # attempt is the observable that says the gate was not answered. - recorded = state.get("attempt_result_digests") or {} - stale = sorted( - key for key, value in recorded.items() - if value == digest and key != str(state["attempt"]) - ) - if stale: - raise ValidationError( - "downloaded result is byte-identical to attempt {}; the operator response did " - "not reach the in-cell pipeline, so this is a non-answer, not a verdict".format( - ", ".join(stale) - ) - ) extracted = temp / "extracted" extracted.mkdir(mode=0o700) safe_extract_result(archive, extracted) diff --git a/tests/fm-azure-validation.test.sh b/tests/fm-azure-validation.test.sh index f24e359531f..21be3ec5338 100755 --- a/tests/fm-azure-validation.test.sh +++ b/tests/fm-azure-validation.test.sh @@ -1403,6 +1403,49 @@ recorded=json.loads((env["state_dir"]/"azv-aaaaaaaaaaaa.json").read_text()) assert recorded["attempt_result_digests"]=={"1":digest_one,"2":digest_two}, recorded["attempt_result_digests"] assert recorded["expected_result_digest"]==digest_two +# 2e. TWO markers in one output: the current attempt's is accepted wherever it +# sits, and the FIRST marker no longer wins. +seed(attempt_result_digests={"1":digest_one}) +view(output=marker(digest_one,1)+"noise\n"+marker(digest_two,2)) +m.observe(env,args) +assert phase()=="needs-decision", "a later marker naming this attempt was not accepted: "+phase() +assert json.loads((env["state_dir"]/"azv-aaaaaaaaaaaa.json").read_text())["expected_result_digest"]==digest_two + +# 2f. Two DIFFERENT results claiming one attempt is not an answer. +os.environ["FM_AZURE_VALIDATION_MARKER_SETTLE_SECONDS"]="0" +seed() +view(output=marker(digest_one,2)+marker(digest_two,2)) +try: + m.observe(env,args) +except m.ValidationError: + pass +else: + raise AssertionError("two conflicting results for one attempt were accepted") +os.environ.pop("FM_AZURE_VALIDATION_MARKER_SETTLE_SECONDS") + +# 2g. LEGACY: a cell whose sealed guest predates the attempt stamp publishes an +# UNSTAMPED marker. The request is digest-sealed so that guest can never be +# changed; refusing to read it would retain a cell on a fully published result +# and leave expected_result_digest unset, making the result unreachable. +os.environ["FM_AZURE_VALIDATION_MARKER_SETTLE_SECONDS"]="0" +seed() +view(output="FM_AZURE_VALIDATION_RESULT {} boot={} outcome=needs-decision\n".format(digest_two,boot)) +m.observe(env,args) +recorded=json.loads((env["state_dir"]/"azv-aaaaaaaaaaaa.json").read_text()) +assert phase()=="needs-decision", "a legacy cell was stranded by the attempt binding: "+phase() +assert recorded["expected_result_digest"]==digest_two, "legacy result is unreachable" +assert recorded["result_binding"]=="legacy", recorded.get("result_binding") +# A STAMPED marker naming another attempt must NOT take the legacy path. +seed() +view(output=marker(digest_two,1)) +try: + m.observe(env,args) +except m.ValidationError: + pass +else: + raise AssertionError("a stamped marker for another attempt fell through to the legacy path") +os.environ.pop("FM_AZURE_VALIDATION_MARKER_SETTLE_SECONDS") + # 3. The REAL result-identity gate refuses a result from another attempt and # refuses one that does not declare its attempt at all. result=json.load(open(sys.argv[4])) @@ -1433,9 +1476,93 @@ except m.ValidationError as exc: assert "does not declare the attempt" in str(exc), str(exc) else: raise AssertionError("a result that declares no attempt was assumed to be the current one") +# ...but a LEGACY-bound observation holds that same result to the pre-stamp +# contract instead of refusing it, or the sealed cell can never collect. +legacy_state=dict(state); legacy_state["result_binding"]="legacy" +m.verify_result_identity(legacy_state,undeclared) +PY + + # 4. The REAL create_run_command: a guest edit must never brick a cell that is + # already in flight, and every new attempt must re-arm its own settle window. + python3 - "$HOST" "$tmp/home2" <<'PY' \ + || fail "create_run_command did not preserve resume across a guest edit and re-arm the window" +import hashlib,importlib.util,json,pathlib,sys,types +spec=importlib.util.spec_from_file_location("validation",sys.argv[1]) +m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m) +home=pathlib.Path(sys.argv[2]) +env={"home":home,"state_dir":home/"state"/"azure-validation","subscription":"sub", + "storage":"stor","resource_group":"rg","owner":"owner","deployment_generation":"gen-1"} +m.ensure_dirs(env) + +# The sealed guest is the one staged beside the request at submit. Its bytes +# deliberately DIFFER from the working tree, which is the shape this PR creates: +# the tree's guest is edited while cells sealed on the old one are still live. +sealed_text="#!/usr/bin/env bash\n# sealed guest for this cell\nexit 0\n" +sealed_digest="sha256:"+hashlib.sha256(sealed_text.encode()).hexdigest() +assert sealed_digest != m.sha256_file(m.GUEST), "fixture must differ from the working tree" +payload=env["state_dir"]/"payloads"/"azv-aaaaaaaaaaaa" +payload.mkdir(parents=True,exist_ok=True) +(payload/"guest.sh").write_text(sealed_text) + +fence="sha256:"+"3"*64 +def seed(**extra): + state={ + "schema":m.SCHEMA,"cell":"azv-aaaaaaaaaaaa","phase":"needs-decision","attempt":2, + "input_digest":"sha256:"+"4"*64,"request_digest":"sha256:"+"5"*64, + "staging":{"container":"c","result_blob":"control/result.tar.gz"}, + "allocation":{"sku":"Standard_D8as_v6","sku_family":"standardDav6Family"}, + "request":{ + "protocol":{"guest_digest":sealed_digest}, + "deployment_generation":"gen-1","home_binding":"sha256:"+"2"*64, + "task":"task","task_generation":"tg","validation_generation":"vg","fence":fence, + "resource_class":"validation-standard", + "repository":{"branch":"fm/fixture","head":"a"*40,"slug":"o/r"}, + "limits":{"behavior_shards":4,"wall_seconds":10800,"reserved_vcpus":24}, + }, + "resources":{"vm_id":"/subs/x/vm","vm_instance_id":"vm-i","worktree_disk_id":"/disk", + "identity_client_id":"cid"}, + "events":[], + } + state.update(extra) + (env["state_dir"]/"azv-aaaaaaaaaaaa.json").write_text(json.dumps(state)) + return state + +sent={} +def fake_az(env_,argv,**kw): + for index,item in enumerate(argv): + if item=="--body": + sent["body"]=json.loads(pathlib.Path(argv[index+1][1:]).read_text()) + return {} +m.az_command=fake_az +m.read_github_token=lambda state:"gh-token" +m.read_resource=lambda env_,rid,kind:(True,{"id":rid,"tags":{"validation-cell":"azv-aaaaaaaaaaaa","fence":fence}}) +m.immutable_identity=lambda resource,kind:{"id":resource.get("id","")} + +# F1: the sealed cell resumes, and it runs the SEALED bytes, not the tree's. +state=seed(unbound_view_since="2020-01-01T00:00:00Z") +m.create_run_command(env,state,"respond",output_url="https://o",response="--action\napprove") +assert sent["body"]["properties"]["source"]["script"]==sealed_text, \ + "a resumed attempt did not run the guest its request was sealed with" + +# F3: the settle window is re-armed per attempt. Without this the window is +# keyed per CELL: a stamp written on attempt 1 makes attempt 2 look ancient the +# moment it is observed, and a one-second-old attempt is retained. +saved=json.loads((env["state_dir"]/"azv-aaaaaaaaaaaa.json").read_text()) +assert "unbound_view_since" not in saved, \ + "a new attempt inherited the previous attempt's settle stamp" + +# The seal itself still binds: no source carrying the sealed digest refuses. +(payload/"guest.sh").write_text("#!/usr/bin/env bash\ntampered\n") +state=seed() +try: + m.create_run_command(env,state,"respond",output_url="https://o",response="x") +except m.ValidationError as exc: + assert "sealed request digest" in str(exc), str(exc) +else: + raise AssertionError("a guest that matches no sealed digest was executed") PY - pass "an attempt's published result is bound to that attempt, an unbound control view never strands a running cell, and an unchanged republished result is refused as a non-answer" + pass "an attempt's published result is bound to that attempt, a guest edit never bricks a sealed in-flight cell, an unbound control view never strands a running cell, and an unchanged republished result is refused as a non-answer" } static_contract From 8d5cdcf63e25547dee071f5ecbe62b8b5106a316 Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Fri, 21 Aug 2026 00:45:37 -0400 Subject: [PATCH 5/9] docs(azure): leave both R4 blocker-1 hypotheses open rather than asserting one --- docs/azure-requirements.md | 105 ++++++++++++++++++++++--------------- 1 file changed, 64 insertions(+), 41 deletions(-) diff --git a/docs/azure-requirements.md b/docs/azure-requirements.md index 0f68b364717..33f58618a42 100644 --- a/docs/azure-requirements.md +++ b/docs/azure-requirements.md @@ -134,55 +134,78 @@ compute zero, worktree disk and evidence retained, control reservation released. That attempt exposed two blockers that stand between this lane and its acceptance sentence, and neither is the receipts strand: -1. `observe` ended an attempt that was still running, and the ending was unrecoverable. - This was recorded here as "`respond` does not answer a gate" plus "the guest re-publishes the - byte-identical previous result". BOTH ARE WRONG. The operator's answer travelled; nothing in - the guest's respond path is implicated. Ground truth is the cell's own state file +1. `observe` took a terminal decision on a control view it never bound to the attempt, and the + decision was unrecoverable. This was recorded here as "`respond` does not answer a gate" plus + "the guest re-publishes the byte-identical previous result". The second is wrong under every + reading of the evidence. The first is NOT ESTABLISHED rather than confirmed: it was inferred + from a control-plane read that cannot support it. Ground truth is the cell's own state file (`$FM_HOME/state/azure-validation/azv-36b2726cbcf3.json`), whose events read `responding` at - 09:47:51, `failed-retained` at 09:48:00, and compute removed at 09:50:30. The host declared - the attempt dead NINE SECONDS after creating its Run Command and deleted the VM 2m39s in; - attempt 1 had taken 1h57m. The `control_error` captured by that read is exactly two lines, the - boot-time auth-home pull warning and the post-run auth-home push warning, which cannot both - come from a guest that has existed for nine seconds: they are attempt 1's complete stderr, so - the instanceView `observe` read was not describing attempt 2 at all. `observe` treated a - terminal `executionState` whose output carried no result marker as proof the attempt had died, - and nothing bound the view it read to the attempt it had just created. The marker is the - guest's last action, so its absence proves nothing about an attempt still working, and - `failed-retained` is a phase `observe` itself refuses, so one premature read was terminal. - The byte-identical result was never a republish. The result blob has one fixed name per cell - (`staging.result_blob` is `control/result.tar.gz`), overwritten by each attempt's upload, and - attempt 2's VM was deleted before it could upload, so any later download necessarily returned - attempt 1's archive unchanged. That is the whole of the reported evidence: the same sha256 - prefix `330ddea31bfbbb05`, the same 3056-byte `run.log`, the same `run_id`, the same - `needs-decision`, the same gate. `collect` never ran at all - the state file carries no - `result` and `state/azure-validation/results/` is empty - so the comparison was made by - downloading the one blob directly, twice. - The latent defect this exposed is worse than the reported one. Had that same stale view - carried attempt 1's MARKER rather than no marker, `observe` would have accepted it, `collect` - would have downloaded attempt 1's archive, matched its digest, and PASSED - `verify_result_identity`, because on a resumed attempt the VM, boot id, run id, heads and - every other verified field are identical and `result.json` carried no attempt number. A silent - false verdict is categorically worse than a generic failure. + 09:47:51, `failed-retained` at 09:48:00, and compute removed at 09:50:30. The host declared the + attempt dead NINE SECONDS after creating its Run Command and deleted the VM 2m39s in; attempt 1 + had taken 1h57m. `observe` treated a terminal `executionState` whose output carried no result + marker as proof the attempt had died, and nothing bound the view it read to the attempt it had + just created. The marker is the guest's LAST action, so its absence proves nothing about an + attempt still working, and `failed-retained` is a phase `observe` itself refuses, so one + premature read was terminal. + The byte-identical result was never a republish, and that does not depend on which hypothesis + below is true. The result blob has one fixed name per cell (`staging.result_blob` is + `control/result.tar.gz`), overwritten by each attempt's upload, and attempt 2 never reached its + upload, so any later download necessarily returned attempt 1's archive unchanged. That is the + whole of the reported evidence: the same sha256 prefix `330ddea31bfbbb05`, the same 3056-byte + `run.log`, the same `run_id`, the same `needs-decision`, the same gate. `collect` never ran at + all, the state file carries no `result` and `state/azure-validation/results/` is empty, so the + comparison was made by downloading the one blob directly, twice. + TWO HYPOTHESES REMAIN OPEN for what produced that view, and the evidence to date does not + separate them. Neither is settled: + (a) the view was stale, describing something other than the nine-second-old attempt; or + (b) attempt 2's guest genuinely failed fast, after its auth-home write-back and before its + marker. `control_error` carries exactly two lines, the auth-home pull and push warnings, and + those are emitted from the guest's own SEQUENTIAL path rather than from a trap: the sealed + guest staged at `payloads/azv-36b2726cbcf3/guest.sh` calls `auth_home_pull` at line 504 and + `auth_home_push` at line 895, its only trap is `cleanup_mounts EXIT`, and its marker is at line + 1102. A guest that got past 895 and died before 1102 produces exactly that stderr. + On resource identity (b) is the better-supported reading: `resources.run_commands` records two + distinct Azure resources, `start-a1` and `respond-a2`, and `respond` rebinds `run_command_id` + to `respond-a2` inside `create_run_command` BEFORE the `responding` transition at 09:47:51, so + the 09:48:00 read addressed a resource nine seconds old, which cannot inherit `start-a1`'s + stderr. An earlier revision of this section asserted (a) as fact and ruled (b) out. That was + wrong, and ruling (b) out risks sending the next operator away from a real respond-path bug. + Separating the two needs a live cell. + What IS established holds under both, and is what the fix rests on: the operator's response was + delivered to the cell as a protected run-command parameter, and `observe` then ended the + attempt on a view it had never bound to it. + The latent defect this exposed is worse than the reported one. Had that view carried attempt + 1's MARKER rather than no marker, `observe` would have accepted it, `collect` would have + downloaded attempt 1's archive, matched its digest, and PASSED `verify_result_identity`, + because on a resumed attempt the VM, boot id, run id, heads and every other verified field are + identical and `result.json` carried no attempt number. A silent false verdict is categorically + worse than a generic failure. Fixed: the guest stamps `attempt` into `result.json` and appends `attempt=` to its marker; - `observe` requires the marker to name the attempt it is observing and never accepts an unbound - view, deferring the terminal decision behind a settling window - (`FM_AZURE_VALIDATION_MARKER_SETTLE_SECONDS`, default 300 seconds, reset per attempt) so - silence means "could not tell" and never authorizes the destructive action, including when the - recorded stamp is itself unreadable; `observe` and `collect` both refuse a result - byte-identical to an earlier attempt's as an explicit non-answer rather than a generic - failure, with `collect` checking the DOWNLOADED ARCHIVE rather than trusting the code path; + `observe` accepts any marker naming the attempt it is observing, refuses a stamped marker that + names another attempt, refuses two conflicting results claiming one attempt, and never accepts + an unbound view, deferring the terminal decision behind a settling window + (`FM_AZURE_VALIDATION_MARKER_SETTLE_SECONDS`, default 300 seconds, re-armed for every attempt + in `create_run_command`) so silence means "could not tell" and never authorizes the destructive + action, including when the recorded stamp is itself unreadable; `observe` refuses a result + byte-identical to an earlier attempt's as an explicit non-answer rather than a generic failure; and `verify_result_identity` refuses a result that declares no attempt or another attempt's. - Proven hermetically in `tests/fm-azure-validation.test.sh` against the real guest emission - region, the real `observe`, and the real identity gate. + A cell whose SEALED guest predates the stamp is not stranded by any of that: its unstamped + marker is still read, and its result is held to the pre-stamp contract. Nor does a guest edit + brick a cell in flight: `create_run_command` now runs the guest the request was SEALED with, + taken from the copy `submit` stages beside the request and accepted only when its digest is the + sealed digest, so `respond`, `reattach` and `replace` keep working across any edit to the + working tree while the seal itself still binds exactly what may execute. Residual, not fixed: `observe` still drives into `failed-retained`, a phase it refuses, so a false negative that outlasts the settling window is recoverable only through `replace`. Upgrading no-mistakes on the HOST is NOT the fix and cannot reach a running cell. The cell's version comes from the `runtime.tar.gz` handed to `submit --runtime-bundle`; nothing in this repo BUILDS that bundle, it is extracted only on a `start` boot, and the request is - digest-sealed (`create_run_command` refuses if the guest changed after request preparation). - `azv-36b2726cbcf3`'s bundle declares `no_mistakes_version: 1.48.0` across 110 files. Host - 1.48.0 to 1.53.0 changes what runs in a cell only by rebuilding the bundle from the upgraded - binary and submitting a NEW cell. Do not spend time waiting on a host upgrade here. + digest-sealed. The staged payload copy for `azv-36b2726cbcf3` declares + `no_mistakes_version: 1.48.0` across 110 files, read from that bundle rather than from the + state file, which records only its digest. Host 1.48.0 to 1.53.0 changes what runs in a cell + only by rebuilding the bundle from the upgraded binary and submitting a NEW cell. Do not spend + time waiting on a host upgrade here. + 2. The sealed suite is not Linux-clean, so every cell run parks. Shard 2 failed on host-coupled units that cannot pass inside a Linux cell (passwordless sudo, tmux window creation, Keychain approval markers), alongside 377 passing units. Until those units skip loudly off macOS, no From 089e07655af8dc363137ee91647209406786eee3 Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Fri, 21 Aug 2026 08:42:16 -0400 Subject: [PATCH 6/9] fix(azure): read the attempt binding from the sealed guest, not from a parse outcome --- bin/fm-azure-validation.py | 60 ++++++++++++++++++++-- tests/fm-azure-validation.test.sh | 83 ++++++++++++++++++++++++++++++- 2 files changed, 138 insertions(+), 5 deletions(-) diff --git a/bin/fm-azure-validation.py b/bin/fm-azure-validation.py index 3913f7c90d4..f5844c22535 100755 --- a/bin/fm-azure-validation.py +++ b/bin/fm-azure-validation.py @@ -75,6 +75,11 @@ r"FM_AZURE_VALIDATION_RESULT\s+(sha256:[0-9a-f]{64})\s+boot=([0-9a-f-]{36})" r"\s+outcome=([a-z-]+)" ) +# Whether a guest CAN name its attempt is a property of the guest, read from the +# sealed bytes that actually run. It must never be inferred from whether a +# marker parsed: LEGACY_MARKER is MARKER minus the attempt group, so ANY +# malformation of the attempt field satisfies "no stamped marker matched". +GUEST_STAMPS_ATTEMPT = re.compile(r"^[^\n]*FM_AZURE_VALIDATION_RESULT[^\n]*attempt=", re.MULTILINE) MARKER_SETTLE_SECONDS = 300 # Shard transports legitimately run VM creation plus admission plus command # submission in one subprocess: near 300 seconds unloaded and well past it @@ -1698,17 +1703,62 @@ def sealed_guest_text(env, state): continue except OSError: continue - if sha256_file(candidate) == expected: - return candidate.read_text(encoding="utf-8") + # ONE read. Digesting the file and then re-reading it to return would + # verify bytes that are not the bytes returned: a writer landing between + # the two reads passes the check and ships different content, and that + # content is uploaded as the Run Command script and executes as root on + # the cell. Digest exactly the bytes that are handed back. + try: + data = candidate.read_bytes() + except OSError: + continue + if "sha256:" + hashlib.sha256(data).hexdigest() == expected: + try: + return data.decode("utf-8") + except UnicodeDecodeError: + continue raise ValidationError( "no guest matching this cell's exact sealed request digest is available; " "neither the staged payload copy nor the working tree carries {}".format(expected) ) +def guest_text_stamps_attempt(text): + """Does this exact guest text emit an attempt-stamped result marker?""" + return bool(GUEST_STAMPS_ATTEMPT.search(text)) + + +def guest_stamps_attempt(env, state): + """Whether this cell's SEALED guest can name the attempt in its marker. + + Read from the guest, never inferred from the output. Inferring it from "no + stamped marker matched" hands the weaker pre-stamp contract to any cell + whose marker is merely MALFORMED, and the marker is the guest's last line by + design, which is exactly where an output cap lands. The consequence is not + theoretical: a stale attempt-1 marker truncated inside its own attempt field + makes the stamped set empty, binds legacy, sets the expected digest to + attempt 1's own, matches the blob that still holds attempt 1's archive, and + skips the attempt check - accepting attempt 1's result as attempt 2's + answer, which is the exact silent false verdict this binding exists to + prevent. Recorded at create time where the sealed text is already in hand, + and derived from the sealed guest for a cell whose state predates that. + An underivable answer takes the STRICT contract, never the weaker one. + """ + recorded = state.get("guest_stamps_attempt") + if isinstance(recorded, bool): + return recorded + try: + return guest_text_stamps_attempt(sealed_guest_text(env, state)) + except ValidationError: + return True + + def create_run_command(env, state, mode, input_url=None, output_url=None, response=None): resources = state["resources"] guest_text = sealed_guest_text(env, state) + # Recorded from the bytes that are about to run, so observe never has to + # guess it from output that may be truncated. + state["guest_stamps_attempt"] = guest_text_stamps_attempt(guest_text) attempt = state["attempt"] name = "{}-a{}".format(mode, attempt) run_id = resources["vm_id"] + "/runCommands/" + name @@ -1982,8 +2032,10 @@ def observe(env, args): if len(mine) > 1: # Two different results claiming one attempt is not an answer. mine = set() - if not mine and not stamped: - # No stamped marker anywhere. Either nothing published, or the cell + if not mine and not stamped and not guest_stamps_attempt(env, state): + # No stamped marker anywhere AND the sealed guest provably cannot + # stamp. The second half is what keeps a malformed marker from + # buying the weaker contract. Either nothing published, or the cell # runs a guest sealed BEFORE the stamp existed. That guest cannot be # changed, because the request is digest-sealed, so refusing to read # it would strand every cell in flight across this upgrade: the diff --git a/tests/fm-azure-validation.test.sh b/tests/fm-azure-validation.test.sh index 21be3ec5338..6c48cb89b3d 100755 --- a/tests/fm-azure-validation.test.sh +++ b/tests/fm-azure-validation.test.sh @@ -1336,6 +1336,7 @@ def seed(phase="responding",attempt=2,**extra): "task":"task","task_generation":"gen","validation_generation":"val", "limits":{"behavior_shards":0}}, "resources":{"run_command_id":"/vm/runCommands/respond-a2"}, + "guest_stamps_attempt":True, "events":[], } state.update(extra) @@ -1402,6 +1403,11 @@ assert phase()=="needs-decision", phase() recorded=json.loads((env["state_dir"]/"azv-aaaaaaaaaaaa.json").read_text()) assert recorded["attempt_result_digests"]=={"1":digest_one,"2":digest_two}, recorded["attempt_result_digests"] assert recorded["expected_result_digest"]==digest_two +# The NORMAL path must record the strict binding. Pinning only the legacy case +# leaves "every observation binds legacy" green while disabling the attempt +# check for every modern cell. +assert recorded["result_binding"]=="attempt", recorded.get("result_binding") +observed_binding=recorded["result_binding"] # 2e. TWO markers in one output: the current attempt's is accepted wherever it # sits, and the FIRST marker no longer wins. @@ -1428,7 +1434,7 @@ os.environ.pop("FM_AZURE_VALIDATION_MARKER_SETTLE_SECONDS") # changed; refusing to read it would retain a cell on a fully published result # and leave expected_result_digest unset, making the result unreachable. os.environ["FM_AZURE_VALIDATION_MARKER_SETTLE_SECONDS"]="0" -seed() +seed(guest_stamps_attempt=False) view(output="FM_AZURE_VALIDATION_RESULT {} boot={} outcome=needs-decision\n".format(digest_two,boot)) m.observe(env,args) recorded=json.loads((env["state_dir"]/"azv-aaaaaaaaaaaa.json").read_text()) @@ -1446,6 +1452,56 @@ else: raise AssertionError("a stamped marker for another attempt fell through to the legacy path") os.environ.pop("FM_AZURE_VALIDATION_MARKER_SETTLE_SECONDS") +# 2h. A guest that CAN stamp never buys the pre-stamp contract with a malformed +# marker. LEGACY_MARKER is MARKER minus the attempt group, so every one of these +# satisfies "no stamped marker matched"; the binding must come from the sealed +# guest instead. The marker is the guest's LAST line by design, which is exactly +# where an output cap lands, so this is a live shape rather than a theoretical one. +os.environ["FM_AZURE_VALIDATION_MARKER_SETTLE_SECONDS"]="0" +malformed={ + "truncated inside the attempt word": + "FM_AZURE_VALIDATION_RESULT {} boot={} outcome=needs-decision attem".format(digest_two,boot), + "truncated right after attempt=": + "FM_AZURE_VALIDATION_RESULT {} boot={} outcome=needs-decision attempt=".format(digest_two,boot), + "empty attempt value": + "FM_AZURE_VALIDATION_RESULT {} boot={} outcome=needs-decision attempt=\n".format(digest_two,boot), + "non-numeric attempt value": + "FM_AZURE_VALIDATION_RESULT {} boot={} outcome=needs-decision attempt=two\n".format(digest_two,boot), +} +for label,output in malformed.items(): + seed() + view(output=output) + try: + m.observe(env,args) + except m.ValidationError: + pass + else: + raise AssertionError("a stamping guest fell back to the legacy contract on "+label) + after=json.loads((env["state_dir"]/"azv-aaaaaaaaaaaa.json").read_text()) + assert after.get("expected_result_digest") is None, \ + "a malformed marker still published a digest on "+label + assert after.get("result_binding")!="legacy", \ + "a stamping guest was bound legacy on "+label + +# 2i. The composed shape, which is the silent false verdict this PR exists to +# close, re-entered through the parse-failure path: observing attempt 2, the +# output carries attempt 1's own marker truncated inside its attempt field. If +# legacy bound here, the expected digest would become attempt 1's, the blob +# still holds attempt 1's archive so the digest check passes, and the attempt +# check is skipped. +seed(attempt_result_digests={"1":digest_one}) +view(output="FM_AZURE_VALIDATION_RESULT {} boot={} outcome=needs-decision attempt=".format(digest_one,boot)) +try: + m.observe(env,args) +except m.ValidationError: + pass +else: + raise AssertionError("a truncated stale attempt-1 marker was accepted as attempt 2's answer") +after=json.loads((env["state_dir"]/"azv-aaaaaaaaaaaa.json").read_text()) +assert after.get("expected_result_digest")!=digest_one, \ + "attempt 1's digest became attempt 2's expected result" +os.environ.pop("FM_AZURE_VALIDATION_MARKER_SETTLE_SECONDS") + # 3. The REAL result-identity gate refuses a result from another attempt and # refuses one that does not declare its attempt at all. result=json.load(open(sys.argv[4])) @@ -1460,7 +1516,11 @@ state={ }, "resources":{"worktree_disk_id":"/work","vm_id":"/vm","vm_instance_id":"vm-instance"}, "expected_boot_id":boot, + # Taken from what observe RECORDED, not asserted by hand: producer and + # consumer of this binding have to be pinned together or neither is pinned. + "result_binding":observed_binding, } +assert state["result_binding"]=="attempt" m.verify_result_identity(state,result) older=json.load(open(sys.argv[3])) try: @@ -1551,6 +1611,27 @@ saved=json.loads((env["state_dir"]/"azv-aaaaaaaaaaaa.json").read_text()) assert "unbound_view_since" not in saved, \ "a new attempt inherited the previous attempt's settle stamp" +# The recorded stamping flag comes from the bytes that are about to run. +assert saved["guest_stamps_attempt"] is False, saved.get("guest_stamps_attempt") +stamping=sealed_text+"printf 'FM_AZURE_VALIDATION_RESULT %s boot=%s outcome=%s attempt=%s\\n'\n" +assert m.guest_text_stamps_attempt(stamping) is True +assert m.guest_text_stamps_attempt(sealed_text) is False + +# A symlinked payload copy is refused even when it points at correct bytes: +# the guest is uploaded and executed as root, so its supply path is held to the +# same standard as credential material. +real=payload/"real-guest.sh"; real.write_text(sealed_text) +(payload/"guest.sh").unlink() +(payload/"guest.sh").symlink_to(real) +state=seed() +try: + m.create_run_command(env,state,"respond",output_url="https://o",response="x") +except m.ValidationError as exc: + assert "sealed request digest" in str(exc), str(exc) +else: + raise AssertionError("a symlinked guest supply path was accepted") +(payload/"guest.sh").unlink() + # The seal itself still binds: no source carrying the sealed digest refuses. (payload/"guest.sh").write_text("#!/usr/bin/env bash\ntampered\n") state=seed() From 0a6e25e00484bd3f3a7cb634a75c96a72361d571 Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Fri, 21 Aug 2026 08:44:03 -0400 Subject: [PATCH 7/9] test(azure): pin the strict binding fallback for a state that cannot answer --- bin/fm-azure-validation.py | 7 ++++++- tests/fm-azure-validation.test.sh | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/bin/fm-azure-validation.py b/bin/fm-azure-validation.py index f5844c22535..019ef137a1f 100755 --- a/bin/fm-azure-validation.py +++ b/bin/fm-azure-validation.py @@ -1749,7 +1749,12 @@ def guest_stamps_attempt(env, state): return recorded try: return guest_text_stamps_attempt(sealed_guest_text(env, state)) - except ValidationError: + except (ValidationError, KeyError, TypeError, OSError): + # A state that cannot answer the question at all - no seal recorded, no + # staged payload, a truncated request - is not evidence that the guest + # cannot stamp. Take the STRICT contract: the worst case is a cell that + # must be observed again, never one that accepts another attempt's + # result as this attempt's answer. return True diff --git a/tests/fm-azure-validation.test.sh b/tests/fm-azure-validation.test.sh index 6c48cb89b3d..2df2b907f9f 100755 --- a/tests/fm-azure-validation.test.sh +++ b/tests/fm-azure-validation.test.sh @@ -1340,6 +1340,8 @@ def seed(phase="responding",attempt=2,**extra): "events":[], } state.update(extra) + if state.get("guest_stamps_attempt") is None: + state.pop("guest_stamps_attempt",None) (env["state_dir"]/"azv-aaaaaaaaaaaa.json").write_text(json.dumps(state)) return state @@ -1483,6 +1485,21 @@ for label,output in malformed.items(): assert after.get("result_binding")!="legacy", \ "a stamping guest was bound legacy on "+label +# 2h-bis. A state that cannot answer "can this guest stamp" - no recorded flag, +# no staged payload to derive it from - takes the STRICT contract. Falling back +# to the weaker one would let exactly the states we know least about skip the +# attempt check. +seed(guest_stamps_attempt=None) +view(output="FM_AZURE_VALIDATION_RESULT {} boot={} outcome=needs-decision\n".format(digest_two,boot)) +try: + m.observe(env,args) +except m.ValidationError: + pass +else: + raise AssertionError("an underivable stamping answer bought the weaker legacy contract") +after=json.loads((env["state_dir"]/"azv-aaaaaaaaaaaa.json").read_text()) +assert after.get("result_binding")!="legacy", after.get("result_binding") + # 2i. The composed shape, which is the silent false verdict this PR exists to # close, re-entered through the parse-failure path: observing attempt 2, the # output carries attempt 1's own marker truncated inside its attempt field. If From 03085e30f9c861ef4a43f622cc9aee608bdc45b2 Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Fri, 21 Aug 2026 08:46:33 -0400 Subject: [PATCH 8/9] test(azure): pin that the verified guest bytes are the bytes returned --- tests/fm-azure-validation.test.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/fm-azure-validation.test.sh b/tests/fm-azure-validation.test.sh index 2df2b907f9f..c2a44b30e2b 100755 --- a/tests/fm-azure-validation.test.sh +++ b/tests/fm-azure-validation.test.sh @@ -1628,6 +1628,21 @@ saved=json.loads((env["state_dir"]/"azv-aaaaaaaaaaaa.json").read_text()) assert "unbound_view_since" not in saved, \ "a new attempt inherited the previous attempt's settle stamp" +# N-2: the bytes VERIFIED must be the bytes RETURNED. Digesting one read and +# returning a second read is a TOCTOU: a writer landing between them passes the +# check while different content is handed back, and that content is uploaded as +# the Run Command script and executes as root on the cell. A race cannot be +# observed by waiting for it, so the second read is instrumented to differ - if +# the implementation reads twice, it returns the instrumented bytes. +saved_read_text=m.Path.read_text +m.Path.read_text=lambda self,*a,**k:"#!/bin/sh\nEVIL\n" +try: + returned=m.sealed_guest_text(env,seed()) +finally: + m.Path.read_text=saved_read_text +assert returned==sealed_text, \ + "sealed_guest_text returned bytes it never verified: "+repr(returned) + # The recorded stamping flag comes from the bytes that are about to run. assert saved["guest_stamps_attempt"] is False, saved.get("guest_stamps_attempt") stamping=sealed_text+"printf 'FM_AZURE_VALIDATION_RESULT %s boot=%s outcome=%s attempt=%s\\n'\n" From b0708482090b6e14ee9b0b6393e8644027bc9cfa Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Fri, 21 Aug 2026 08:49:45 -0400 Subject: [PATCH 9/9] docs(azure): describe the sealed guest read as it actually behaves --- bin/fm-azure-validation.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/fm-azure-validation.py b/bin/fm-azure-validation.py index 019ef137a1f..2140c79bfd3 100755 --- a/bin/fm-azure-validation.py +++ b/bin/fm-azure-validation.py @@ -1688,8 +1688,10 @@ def sealed_guest_text(env, state): the sealed artifact and is what a resumed attempt must run. The working tree is used only when it still matches the seal, which keeps a cell whose payload was pruned behaving exactly as before. Neither source is trusted on - provenance: only a file whose digest IS the sealed digest is ever returned, - so this widens recovery without widening what may execute. + provenance: the bytes are read ONCE, digested in memory, and those exact + bytes are what is returned, so what executes on the cell is what was + verified rather than whatever a later read would have found. This widens + recovery without widening what may execute. """ expected = state["request"]["protocol"]["guest_digest"] candidates = []