From a4828fa3f148d99544c01f701462c6060ad10a84 Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Wed, 19 Aug 2026 10:47:55 -0400 Subject: [PATCH 1/3] feat(worker): operator surrender lane for unrecoverable ordinary release authority An assigned worker whose task lost its ordinary authority (local teardown consumed state/.meta before any receipt existed) had no sanctioned exit: release requires a proof nothing can mint, withdraw only takes queued entries, and the remaining path was hand-editing controller state. All four pilot slots sit in exactly this shape. surrender is the refusal-first replacement for that hand edit: it runs the ordinary authority first and refuses when that succeeds, requires dark compute (deallocated or stopped VM), an operator --reason, --confirm-surrender and the exact subscription confirmation, refuses to replace an ordinary release proof, and refuses while a pending provider action exists. The minted bundle keeps the fm.worker-release/v2 shape the deallocate/delete-compute/reset machinery fences on, with every authority verdict 'surrendered' (release_receipt rejects that verdict, so the bundle cannot replay through ordinary release) plus a surrender block recording the reason and the authority's refusal verbatim. The wrapper removes the task's locally staged provider credential keyed off the FM-SURRENDERED receipt, mirroring the withdraw lane. --- bin/fm-worker-lifecycle.py | 157 +++++++++++++++++++++++ bin/fm-worker-lifecycle.sh | 23 ++++ docs/azure-workers.md | 7 ++ tests/fm-worker-lifecycle.test.sh | 203 ++++++++++++++++++++++++++++++ 4 files changed, 390 insertions(+) diff --git a/bin/fm-worker-lifecycle.py b/bin/fm-worker-lifecycle.py index 55f5dcd7181..a08188d3608 100755 --- a/bin/fm-worker-lifecycle.py +++ b/bin/fm-worker-lifecycle.py @@ -1667,6 +1667,17 @@ def parser(): withdraw_parser.add_argument("--task-generation", required=True) withdraw_parser.add_argument("--confirm-withdraw", action="store_true") withdraw_parser.add_argument("--confirm-subscription", required=True) + + surrender_parser = sub.add_parser( + "surrender", + help="release one exact assigned worker whose ordinary release authority is unrecoverable", + ) + surrender_parser.add_argument("--task", required=True) + surrender_parser.add_argument("--task-generation", required=True) + surrender_parser.add_argument("--reason", required=True) + surrender_parser.add_argument("--output", required=True) + surrender_parser.add_argument("--confirm-surrender", action="store_true") + surrender_parser.add_argument("--confirm-subscription", required=True) reconcile_parser = sub.add_parser("reconcile", help="plan or apply bounded convergence") reconcile_parser.add_argument("--apply", action="store_true") reconcile_parser.add_argument("--confirm-subscription") @@ -2449,6 +2460,150 @@ def command_withdraw(env, args): print("withdrew queued request {}".format(key)) +def ordinary_authority_attempt(env, args, worker): + """Run the ordinary release authority; None on success, its refusal text otherwise.""" + with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", delete=False) as handle: + json.dump(worker, handle, sort_keys=True, separators=(",", ":")) + worker_path = handle.name + output_path = worker_path + ".receipt" + try: + result = subprocess.run([ + "python3", str(ROOT / "bin" / "fm-worker-authority.py"), + "--home", str(env["home"]), "--task", args.task, + "--task-generation", args.task_generation, + "--assignment-generation", worker["assignment_generation"], + "--worker-state", worker_path, "--output", output_path, + ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=PROVIDER_TIMEOUT_SECONDS) + finally: + with contextlib.suppress(FileNotFoundError): + Path(worker_path).unlink() + with contextlib.suppress(FileNotFoundError): + Path(output_path).unlink() + if result.returncode == 0: + return None + return result.stderr.decode("utf-8", errors="replace").strip()[-1000:] + + +def command_surrender(env, args): + """Release an assigned worker whose ordinary release authority is unrecoverable. + + The ordinary lane is authority-receipt -> release: five receipts minted from + the live task metadata, endpoint oracle, landing graph, account directory, + and worktree. A task can lose that authority legitimately - local teardown + consumed state/.meta before any receipt existed - and the worker slot + is then stranded: release requires a proof nothing can mint, withdraw only + takes queued entries, and the only remaining path was hand-editing + controller state. + + Surrender is the durable, refusal-first replacement for that hand edit. It + is not a shortcut around release: it first runs the ordinary authority + itself and refuses when that succeeds, refuses live compute (the VM must be + deallocated or stopped), refuses to replace an ordinary release proof, and + demands an operator reason plus the same double confirmation as withdraw. + The minted bundle keeps the fm.worker-release/v2 shape the downstream + deallocate/delete-compute/reset machinery already fences on, but every + authority verdict is "surrendered" - release_receipt() rejects that verdict, + so a surrender bundle can never be replayed through the ordinary release + command - and a top-level surrender block records the reason and the + ordinary authority's refusal verbatim. + """ + require_id("task", args.task) + require_id("task generation", args.task_generation) + reason = (args.reason or "").strip() + if not reason or len(reason) > 1000: + raise LifecycleError("--reason must be 1..1000 characters of operator explanation") + if not args.confirm_surrender: + raise LifecycleError("--confirm-surrender is required") + if args.confirm_subscription != env["subscription"]: + raise LifecycleError("--confirm-subscription must exactly match FM_AZURE_SUBSCRIPTION_ID") + with controller_lock(env): + state = load_state(env) + if state.get("pending_action") is not None: + raise LifecycleError("a pending provider action exists; reconcile first") + key = request_key(args.task, args.task_generation) + item = state["queue"].get(key) + if item is None or item.get("status") not in ("assigned", "releasing"): + raise LifecycleError("surrender requires one exact assigned task generation") + worker = state["workers"].get(str(item.get("slot"))) + if worker is None or worker.get("queue_key") != key: + raise LifecycleError("surrender task has no exact durable worker owner") + existing = worker.get("release_proof") + if existing is not None: + if isinstance(existing.get("surrender"), dict): + write_surrender_output(args.output, existing) + print("surrender proof already recorded with exact identity") + print("FM-SURRENDERED {} {}".format(args.task, args.task_generation)) + return + raise LifecycleError("worker already has an ordinary release proof; reconcile releases it") + if item.get("status") != "assigned": + raise LifecycleError("surrender requires one exact assigned task generation") + refusal = ordinary_authority_attempt(env, args, worker) + if refusal is None: + raise LifecycleError( + "ordinary release authority succeeded; use authority-receipt and release" + ) + inventory = provider_call(env, "inventory")["inventory"] + cloud = inventory_by_slot(inventory).get(worker["slot"]) + classification, note = classify_worker(worker, cloud) + if classification != "assigned": + raise LifecycleError("surrender refuses a non-assigned or ambiguous worker: {}".format(note)) + power = str(((cloud.get("resources") or {}).get("vm") or {}).get("power_state", "")).lower() + if "deallocated" not in power and "stopped" not in power: + raise LifecycleError( + "surrender requires dark compute; the worker VM power state is {!r}".format(power or "unknown") + ) + surrendered_at = iso_utc() + surrender = { + "reason": reason, + "ordinary_refusal": refusal, + "surrendered_at": surrendered_at, + "power_state": power, + "last_execution_digest": worker.get("last_execution_digest"), + } + proof = { + "schema": RELEASE_SCHEMA, + "home_binding": worker["bindings"]["home_binding"], + "task": args.task, + "task_generation": args.task_generation, + "assignment_generation": worker["assignment_generation"], + "account_binding": worker["bindings"]["account_binding"], + "worktree_binding": worker["bindings"]["worktree_binding"], + "repository_binding": worker["bindings"]["repository_binding"], + "repository_generation": worker["bindings"]["repository_generation"], + "cloud_instance_id": worker["cloud_instance_id"], + "resources": worker["resources"], + "surrender": surrender, + "authorities": {}, + } + for name in ("endpoint", "report", "landing", "account", "worktree"): + receipt_value = { + "schema": AUTHORITY_SCHEMA, + "authority": name, + "task": args.task, + "task_generation": args.task_generation, + "assignment_generation": worker["assignment_generation"], + "verdict": "surrendered", + "evidence_digest": digest_value({"authority": name, "surrender": surrender}), + } + receipt_value["receipt_digest"] = digest_value(receipt_value) + proof["authorities"][name] = receipt_value + proof["proof_digest"] = digest_value(proof) + worker["release_proof"] = proof + worker["released_at"] = surrendered_at + worker["phase"] = "release-proved" + item["status"] = "releasing" + save_state(env, state) + write_surrender_output(args.output, proof) + print("FM-SURRENDERED {} {}".format(args.task, args.task_generation)) + print("surrendered release recorded; reconcile now owns deallocation, compute deletion, and reset") + + +def write_surrender_output(path, proof): + Path(path).write_text( + json.dumps(proof, sort_keys=True, separators=(",", ":")) + "\n", encoding="utf-8" + ) + + def command_resume(env, args): if not args.confirm_resume: raise LifecycleError("--confirm-resume is required") @@ -2586,6 +2741,8 @@ def main(argv=None): command_release(env, args) elif args.command == "withdraw": command_withdraw(env, args) + elif args.command == "surrender": + command_surrender(env, args) elif args.command == "resume": command_resume(env, args) elif args.command == "steer": diff --git a/bin/fm-worker-lifecycle.sh b/bin/fm-worker-lifecycle.sh index f84b1d140d9..4ddbc3b2a0e 100755 --- a/bin/fm-worker-lifecycle.sh +++ b/bin/fm-worker-lifecycle.sh @@ -35,6 +35,7 @@ # fm-worker-lifecycle.sh proof-template --task --task-generation # fm-worker-lifecycle.sh release --task --task-generation --proof-file # fm-worker-lifecycle.sh withdraw --task --task-generation --confirm-withdraw --confirm-subscription +# fm-worker-lifecycle.sh surrender --task --task-generation --reason --output --confirm-surrender --confirm-subscription # fm-worker-lifecycle.sh resume # fm-worker-lifecycle.sh steer # fm-worker-lifecycle.sh status [--live] [--json] @@ -83,6 +84,28 @@ case "${1:-}" in fi exit 0 ;; + surrender) + fm_refuse_if_gate_agent + # A surrendered task's local cloud state has the same no-owner problem as a + # withdrawn one: its endpoint, metadata and teardown are already gone (that + # is what made surrender necessary), so nothing else will ever remove the + # staged provider credential at $STATE/.cloud-account/auth.json. The + # cloud-side copies go later, through reconcile's fenced reset; the local + # staging is owned here, keyed off the FM-SURRENDERED receipt for exactly + # the reasons the withdraw lane documents above. + surrender_output=$(python3 "$SCRIPT_DIR/fm-worker-lifecycle.py" "$@") + printf '%s\n' "$surrender_output" + surrender_receipt=$(printf '%s\n' "$surrender_output" | awk '$1 == "FM-SURRENDERED" { print $2; exit }') + if [ -n "$surrender_receipt" ]; then + fm_cloud_state_remove "${FM_STATE_OVERRIDE:-${FM_HOME:?FM_HOME is required}/state}" "$surrender_receipt" + state_root="${FM_STATE_OVERRIDE:-$FM_HOME/state}" + if [ -e "$state_root/$surrender_receipt.cloud-account/auth.json" ]; then + echo "ELASTIC WORKER REFUSED: surrendered $surrender_receipt but its staged provider credential remains" >&2 + exit 4 + fi + fi + exit 0 + ;; reconcile) for argument in "$@"; do if [ "$argument" = --apply ]; then diff --git a/docs/azure-workers.md b/docs/azure-workers.md index 8007dd4ba67..43a4f310ed5 100644 --- a/docs/azure-workers.md +++ b/docs/azure-workers.md @@ -184,6 +184,13 @@ Reset deletion uses exact IDs, immutable identities, tags or metadata, detach re A replacement, unreadable relation, foreign tag, missing ETag, public NIC relation, or partial inventory records a bounded cleanup refusal and retains the resources. No age or cost override can convert retained-for-investigation into safe deletion. +### Operator surrender for unrecoverable ordinary authority + +`surrender` releases one exact ASSIGNED worker whose ordinary release authority can no longer be minted, for example when local teardown consumed the task metadata before any receipt existed. +It is refusal-first, not a shortcut: the command runs the ordinary authority itself and refuses when that succeeds, refuses live compute (the VM must be deallocated or stopped), refuses to replace an ordinary release proof, refuses while a pending provider action exists, and demands an operator `--reason` plus the same explicit confirmation pair as withdraw. +The minted bundle keeps the `fm.worker-release/v2` shape the deallocate/delete-compute/reset machinery fences on, but every authority verdict is `surrendered` - `release` rejects that verdict, so a surrender bundle can never replay through the ordinary release command - and a top-level `surrender` block records the operator reason and the ordinary authority's refusal verbatim. +After the proof is recorded, reconcile owns deallocation, compute deletion, and reset exactly as for an ordinary release, and the wrapper removes the task's locally staged provider credential keyed off the command's own `FM-SURRENDERED` receipt. + ## Recovery and reconciliation classes Every live read classifies each controller-owned slot into exactly one operator outcome: diff --git a/tests/fm-worker-lifecycle.test.sh b/tests/fm-worker-lifecycle.test.sh index 6da2a9e4132..9a208e5543d 100755 --- a/tests/fm-worker-lifecycle.test.sh +++ b/tests/fm-worker-lifecycle.test.sh @@ -2248,6 +2248,207 @@ PY pass "restart replays one exact idempotency key without duplicating assignment" } + +surrender_lane() { + local tmp provider fixture home envfile + fm_test_tmproot_into tmp fm-worker-surrender + provider="$tmp/provider.py" + fixture="$tmp/provider-state.json" + home="$tmp/home" + mkdir -p "$home" + write_fixture_provider "$provider" + envfile="$tmp/env" + cat >"$envfile" < Date: Wed, 19 Aug 2026 11:11:49 -0400 Subject: [PATCH 2/3] harden(worker): close the surrender review findings Adversarial review of the surrender lane found the refusal surface thinner than advertised. This commit closes every finding: - F1: the controller's own execution records outrank operator judgment. When any recorded execution for the exact worker shows outcome_present, outcome_uncommitted_changes, or outcome_commits > 0, surrender refuses and names the execution; --confirm-discard-unlanded overrides deliberately and the discard list is recorded in the durable surrender block. - F3: the authority-success gate fails closed. Only stderr carrying WORKER AUTHORITY REFUSED counts as a refusal; a broken tool (traceback, missing helper) raises instead of unlocking surrender. - F6: the idempotent path re-verifies the stored proof binds the exact task generation before re-issuing it, and the rerun's --output is asserted equal to the stored proof. - F4: a converged (complete) entry's refusal names the credential recovery (fm_cloud_state_remove) instead of a generic message; documented. - F5: docs no longer call withdraw the only other queue mutation, state that surrender mints the release rather than adding a second exit, and the static contract now pins command_surrender, the fail-closed marker, the discard confirmation, and the doc paragraph's gates line. - F2: a surrender_refusal_matrix unit pins every advertised gate at the command: malformed identity, pending provider action, converged-entry recovery, ordinary-proof non-replacement, foreign-generation stored proof, all three unlanded-evidence shapes, fail-closed tool breakage (subprocess boundary substituted, classification code real), and the ambiguous or missing-inventory worker. --- bin/fm-worker-lifecycle.py | 47 ++++++++- docs/azure-workers.md | 6 +- tests/fm-worker-lifecycle.test.sh | 167 +++++++++++++++++++++++++++++- 3 files changed, 216 insertions(+), 4 deletions(-) diff --git a/bin/fm-worker-lifecycle.py b/bin/fm-worker-lifecycle.py index a08188d3608..cbf5722e657 100755 --- a/bin/fm-worker-lifecycle.py +++ b/bin/fm-worker-lifecycle.py @@ -1677,6 +1677,10 @@ def parser(): surrender_parser.add_argument("--reason", required=True) surrender_parser.add_argument("--output", required=True) surrender_parser.add_argument("--confirm-surrender", action="store_true") + surrender_parser.add_argument( + "--confirm-discard-unlanded", action="store_true", + help="acknowledge that recorded execute outcomes never proven landed are discarded", + ) surrender_parser.add_argument("--confirm-subscription", required=True) reconcile_parser = sub.add_parser("reconcile", help="plan or apply bounded convergence") reconcile_parser.add_argument("--apply", action="store_true") @@ -2481,7 +2485,16 @@ def ordinary_authority_attempt(env, args, worker): Path(output_path).unlink() if result.returncode == 0: return None - return result.stderr.decode("utf-8", errors="replace").strip()[-1000:] + stderr_text = result.stderr.decode("utf-8", errors="replace").strip() + # Only a genuine refusal unlocks surrender. The authority also exits + # nonzero on tool or environment trouble (broken git, unreadable helper), + # and treating that as a refusal would make every dark worker + # surrenderable exactly when the machine is least trustworthy. + if "WORKER AUTHORITY REFUSED" not in stderr_text: + raise LifecycleError( + "ordinary release authority tool failed rather than refusing: {}".format(stderr_text[-500:]) + ) + return stderr_text[-1000:] def command_surrender(env, args): @@ -2522,6 +2535,12 @@ def command_surrender(env, args): raise LifecycleError("a pending provider action exists; reconcile first") key = request_key(args.task, args.task_generation) item = state["queue"].get(key) + if item is not None and item.get("status") == "complete": + raise LifecycleError( + "surrendered task generation already converged; if its staged credential " + "remains under state/, remove it with fm_cloud_state_remove from " + "bin/fm-cloud-state-lib.sh" + ) if item is None or item.get("status") not in ("assigned", "releasing"): raise LifecycleError("surrender requires one exact assigned task generation") worker = state["workers"].get(str(item.get("slot"))) @@ -2530,6 +2549,9 @@ def command_surrender(env, args): existing = worker.get("release_proof") if existing is not None: if isinstance(existing.get("surrender"), dict): + if (existing.get("task") != args.task + or existing.get("task_generation") != args.task_generation): + raise LifecycleError("stored surrender proof binds a different task generation") write_surrender_output(args.output, existing) print("surrender proof already recorded with exact identity") print("FM-SURRENDERED {} {}".format(args.task, args.task_generation)) @@ -2537,6 +2559,28 @@ def command_surrender(env, args): raise LifecycleError("worker already has an ordinary release proof; reconcile releases it") if item.get("status") != "assigned": raise LifecycleError("surrender requires one exact assigned task generation") + produced = [] + for request_digest, execution in sorted((state.get("executions") or {}).items()): + if not isinstance(execution, dict): + continue + if (execution.get("task") != args.task + or execution.get("task_generation") != args.task_generation + or execution.get("assignment_generation") != worker["assignment_generation"]): + continue + if (execution.get("outcome_present") is True + or execution.get("outcome_uncommitted_changes") is True + or (execution.get("outcome_commits") or 0) > 0): + produced.append(request_digest) + if produced and not args.confirm_discard_unlanded: + # The controller's own durable record says this worker produced + # repository work whose landing is unproven; surrendering it leads + # to a reset that deletes the task disk holding that work. The + # operator can override, but only by naming the discard. + raise LifecycleError( + "surrender refuses: execution(s) {} produced repository work whose landing " + "is unproven; inspect the task disk, then pass --confirm-discard-unlanded " + "to discard it deliberately".format(", ".join(produced)) + ) refusal = ordinary_authority_attempt(env, args, worker) if refusal is None: raise LifecycleError( @@ -2559,6 +2603,7 @@ def command_surrender(env, args): "surrendered_at": surrendered_at, "power_state": power, "last_execution_digest": worker.get("last_execution_digest"), + "discarded_unlanded_executions": produced, } proof = { "schema": RELEASE_SCHEMA, diff --git a/docs/azure-workers.md b/docs/azure-workers.md index 43a4f310ed5..cec7d01311f 100644 --- a/docs/azure-workers.md +++ b/docs/azure-workers.md @@ -38,8 +38,9 @@ The controller rejects duplicate active account or writable-worktree bindings. A general request has role `author`, is explicitly eligible, and is owned by either the primary or a secondmate. The same task generation and exact identity is idempotent, while a changed identity under the same task generation refuses. An assigned request stays in the queue until its ordinary release proof is accepted and every exact cloud resource is safely reset. -A request that never reached assignment leaves the queue by `withdraw`, which is the only other queue mutation: it accepts an entry still in `queued`, refuses anything a worker owns or a pending provider action names, requires `--confirm-withdraw` and `--confirm-subscription`, touches no capacity, and removes the per-task cloud state including the staged provider credential. +A request that never reached assignment leaves the queue by `withdraw`: it accepts an entry still in `queued`, refuses anything a worker owns or a pending provider action names, requires `--confirm-withdraw` and `--confirm-subscription`, touches no capacity, and removes the per-task cloud state including the staged provider credential. Release remains the only exit for work that ever held capacity. +Operator surrender is not a second exit: it mints that release proof for the one case where the ordinary authorities are unrecoverable, under its own refusal-first gates (below). Therefore a truly empty queue also means there is no active task worker and desired worker compute is zero. ```sh @@ -187,9 +188,10 @@ No age or cost override can convert retained-for-investigation into safe deletio ### Operator surrender for unrecoverable ordinary authority `surrender` releases one exact ASSIGNED worker whose ordinary release authority can no longer be minted, for example when local teardown consumed the task metadata before any receipt existed. -It is refusal-first, not a shortcut: the command runs the ordinary authority itself and refuses when that succeeds, refuses live compute (the VM must be deallocated or stopped), refuses to replace an ordinary release proof, refuses while a pending provider action exists, and demands an operator `--reason` plus the same explicit confirmation pair as withdraw. +Surrender is refusal-first, not a shortcut: the command runs the ordinary authority itself and refuses when that succeeds (and fails closed when the authority tool breaks rather than refuses), refuses live compute (the VM must be deallocated or stopped), refuses to replace an ordinary release proof, refuses while a pending provider action exists, refuses when the controller's own execution records show repository work whose landing is unproven unless the operator passes `--confirm-discard-unlanded`, and demands an operator `--reason` plus the same explicit confirmation pair as withdraw. The minted bundle keeps the `fm.worker-release/v2` shape the deallocate/delete-compute/reset machinery fences on, but every authority verdict is `surrendered` - `release` rejects that verdict, so a surrender bundle can never replay through the ordinary release command - and a top-level `surrender` block records the operator reason and the ordinary authority's refusal verbatim. After the proof is recorded, reconcile owns deallocation, compute deletion, and reset exactly as for an ordinary release, and the wrapper removes the task's locally staged provider credential keyed off the command's own `FM-SURRENDERED` receipt. +If the wrapper dies between the receipt and that removal and reconcile then converges the entry to `complete`, the rerun refusal names the recovery: remove the staged credential with `fm_cloud_state_remove` from `bin/fm-cloud-state-lib.sh`. ## Recovery and reconciliation classes diff --git a/tests/fm-worker-lifecycle.test.sh b/tests/fm-worker-lifecycle.test.sh index 9a208e5543d..66015937d49 100755 --- a/tests/fm-worker-lifecycle.test.sh +++ b/tests/fm-worker-lifecycle.test.sh @@ -44,13 +44,23 @@ assert any( "Release remains the only exit" in line for line in doc.splitlines() if not line.lstrip().startswith("