diff --git a/.pi/extensions/fm-primary-pi-watch.ts b/.pi/extensions/fm-primary-pi-watch.ts index 04b04f7b019..3b17426e153 100644 --- a/.pi/extensions/fm-primary-pi-watch.ts +++ b/.pi/extensions/fm-primary-pi-watch.ts @@ -1,10 +1,10 @@ // Firstmate primary watcher bridge for Pi. import { spawn, spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { mkdirSync, lstatSync, readFileSync, watch, writeFileSync, type FSWatcher } from "node:fs"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import type { ExtensionAPI, SessionEntry } from "@earendil-works/pi-coding-agent"; +import type { ExtensionAPI, ExtensionContext, SessionEntry } from "@earendil-works/pi-coding-agent"; import { Type } from "typebox"; type ArmResult = { @@ -43,6 +43,14 @@ type DirectExchangeState = { type LockOwnership = "owned" | "missing" | "other"; +type PendingWakeDelivery = { + deliveryId: string; + reason: string; + attempts: number; + requiresQueue: boolean; + handoffPending: boolean; +}; + const directExchangeEntryType = "firstmate-direct-exchange"; const directInputObservationEntryType = "firstmate-direct-input-observation"; const continuityMessageType = "firstmate-direct-exchange-continuity"; @@ -178,10 +186,34 @@ const state = process.env.FM_STATE_OVERRIDE || `${fmHome}/state`; const config = process.env.FM_CONFIG_OVERRIDE || `${fmHome}/config`; const armScript = `${fmRoot}/bin/fm-watch-arm.sh`; const marker = `${state}/.pi-watch-extension-loaded`; +const wakeQueue = `${state}/.wake-queue`; +const awayMarker = `${state}/.afk`; const extensionVersion = `sha256:${createHash("sha256").update(readFileSync(extensionFile)).digest("hex")}`; -let child: any = null; -let seq = 0; +function positiveInteger(value: string | undefined, fallback: number): number { + if (!value || !/^[0-9]+$/.test(value)) return fallback; + const parsed = Number(value); + return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback; +} + +function pathPresent(path: string): boolean { + try { + lstatSync(path); + return true; + } catch { + return false; + } +} + +function wakeQueuePending(): boolean { + try { + const queue = lstatSync(wakeQueue); + if (queue.isSymbolicLink() || !queue.isFile()) return true; + return queue.size > 0; + } catch (error) { + return (error as NodeJS.ErrnoException).code !== "ENOENT"; + } +} function parentPid(pid: string): string { const result = spawnSync("ps", ["-o", "ppid=", "-p", pid], { encoding: "utf8" }); @@ -245,43 +277,248 @@ function failureLine(stdout: string, stderr: string, code: number | null): strin } export default function (pi: ExtensionAPI) { + const deliveryRetryBaseMs = positiveInteger(process.env.FM_PI_WAKE_RETRY_BASE_MS, 250); + const deliveryRetryMaxMs = Math.max( + deliveryRetryBaseMs, + positiveInteger(process.env.FM_PI_WAKE_RETRY_MAX_MS, 5000), + ); + const armRestartBaseMs = positiveInteger(process.env.FM_PI_ARM_RESTART_BASE_MS, 250); + const armRestartMaxMs = Math.max( + armRestartBaseMs, + positiveInteger(process.env.FM_PI_ARM_RESTART_MAX_MS, 30000), + ); let exchangeSequence = 0; let agentRunSequence = 0; let activeAgentRunKey: string | undefined; let replyCohort = new Set(); let agentStartAwaitingHumanDelivery = false; + let runtimeContext: ExtensionContext | undefined; + let child: ReturnType | null = null; + let childSequence = 0; + let deliverySequence = 0; + let armRestartFailures = 0; + let armRestartTimer: ReturnType | undefined; + let deliveryRetryTimer: ReturnType | undefined; + let controlWatcher: FSWatcher | undefined; + let pendingWakeDelivery: PendingWakeDelivery | undefined; + let cycleActive = false; + let shuttingDown = false; function appendDirectExchange(event: DirectExchangeEvent): void { pi.appendEntry(directExchangeEntryType, event); } + function rememberContext(ctx: ExtensionContext): void { + runtimeContext = ctx; + } + + function runtimeIsIdle(): boolean | undefined { + try { + return runtimeContext?.isIdle(); + } catch { + return undefined; + } + } + + function retryDelay(base: number, maximum: number, attempt: number): number { + const exponent = Math.min(Math.max(attempt - 1, 0), 8); + return Math.min(maximum, base * (2 ** exponent)); + } + + function clearArmRestartTimer(): void { + if (armRestartTimer) clearTimeout(armRestartTimer); + armRestartTimer = undefined; + } + + function clearDeliveryRetryTimer(): void { + if (deliveryRetryTimer) clearTimeout(deliveryRetryTimer); + deliveryRetryTimer = undefined; + } + + function clearPendingWakeDelivery(): void { + clearDeliveryRetryTimer(); + pendingWakeDelivery = undefined; + } + function stopArm(): void { - if (child) child.kill("SIGTERM"); + const activeChild = child; child = null; + if (activeChild && !activeChild.killed) activeChild.kill("SIGTERM"); } - const cleanupOnProcessExit = () => { + function stopCycle(): void { + cycleActive = false; + clearArmRestartTimer(); + clearPendingWakeDelivery(); stopArm(); - }; - process.once("exit", cleanupOnProcessExit); + controlWatcher?.close(); + controlWatcher = undefined; + } - function sendWake(message: string): void { - pi.sendMessage( - { - customType: "firstmate-watcher-wake", - content: `FIRSTMATE WATCHER WAKE: ${message}\n\nRun bin/fm-wake-drain.sh first, handle the queued wake, then resume Pi supervision.`, - display: true, - details: { version: 1, source: "firstmate-extension", kind: "watcher-wake" }, - }, - { deliverAs: "followUp", triggerTurn: true }, - ); + function cycleCanRun(): boolean { + return cycleActive && !shuttingDown && !pathPresent(awayMarker) && sessionOwnsLock(); } - function startArm(): ArmResult { - if (!sessionOwnsLock()) return { ok: false, message: "watcher: read-only - session lock is held by another firstmate session" }; - markLoaded(); + function wakeDeliveryId(message: unknown): string | undefined { + if (!message || typeof message !== "object") return undefined; + const candidate = message as { role?: unknown; customType?: unknown; details?: unknown }; + if (candidate.role !== "custom" || candidate.customType !== "firstmate-watcher-wake") return undefined; + if (!candidate.details || typeof candidate.details !== "object") return undefined; + const deliveryId = (candidate.details as { deliveryId?: unknown }).deliveryId; + return typeof deliveryId === "string" ? deliveryId : undefined; + } + + function deliveryIsInSession(deliveryId: string): boolean { + try { + return runtimeContext?.sessionManager.getBranch().some((entry) => { + if (entry.type !== "custom_message" || entry.customType !== "firstmate-watcher-wake") return false; + if (!entry.details || typeof entry.details !== "object") return false; + return (entry.details as { deliveryId?: unknown }).deliveryId === deliveryId; + }) ?? false; + } catch { + return false; + } + } + + function observeWakeAdmission(message?: unknown): boolean { + const pending = pendingWakeDelivery; + if (!pending) return false; + const observedId = message ? wakeDeliveryId(message) : undefined; + if (observedId !== pending.deliveryId && !deliveryIsInSession(pending.deliveryId)) return false; + clearPendingWakeDelivery(); + return true; + } + + function scheduleWakeDeliveryRetry(): void { + const pending = pendingWakeDelivery; + if (!pending || pending.handoffPending || deliveryRetryTimer || !cycleCanRun()) return; + if (pending.requiresQueue && !wakeQueuePending()) { + clearPendingWakeDelivery(); + return; + } + if (!pending.requiresQueue && pending.attempts >= 3) { + clearPendingWakeDelivery(); + return; + } + const delay = retryDelay(deliveryRetryBaseMs, deliveryRetryMaxMs, pending.attempts); + deliveryRetryTimer = setTimeout(() => { + deliveryRetryTimer = undefined; + attemptWakeDelivery(); + }, delay); + deliveryRetryTimer.unref?.(); + } + + function attemptWakeDelivery(): void { + const pending = pendingWakeDelivery; + if (!pending || pending.handoffPending || !cycleCanRun()) return; + if (observeWakeAdmission()) return; + if (pending.requiresQueue && !wakeQueuePending()) { + clearPendingWakeDelivery(); + return; + } + if (pending.attempts > 0 && runtimeIsIdle() === false) { + return; + } + + pending.attempts += 1; + pending.handoffPending = true; + try { + pi.sendMessage( + { + customType: "firstmate-watcher-wake", + content: `FIRSTMATE WATCHER WAKE: ${pending.reason}\n\nRun bin/fm-wake-drain.sh first, handle the queued wake, then continue normal Pi supervision; the extension has already started the successor watcher.`, + display: true, + details: { + version: 1, + source: "firstmate-extension", + kind: "watcher-wake", + deliveryId: pending.deliveryId, + attempt: pending.attempts, + }, + }, + { deliverAs: "followUp", triggerTurn: true }, + ); + } catch { + if (pendingWakeDelivery?.deliveryId === pending.deliveryId) pending.handoffPending = false; + scheduleWakeDeliveryRetry(); + return; + } + + queueMicrotask(() => { + if (pendingWakeDelivery?.deliveryId !== pending.deliveryId) return; + if (observeWakeAdmission()) return; + if (pending.requiresQueue && !wakeQueuePending()) clearPendingWakeDelivery(); + }); + } + + function requestWakeDelivery(reason: string, requiresQueue: boolean): void { + if (!cycleCanRun()) return; + if (pendingWakeDelivery) { + if (!requiresQueue || pendingWakeDelivery.requiresQueue) return; + clearPendingWakeDelivery(); + } + pendingWakeDelivery = { + deliveryId: `${process.pid}:${Date.now()}:${++deliverySequence}`, + reason, + attempts: 0, + requiresQueue, + handoffPending: false, + }; + attemptWakeDelivery(); + } + + function scheduleArmRestart(): void { + if (armRestartTimer || !cycleCanRun()) return; + armRestartFailures += 1; + const delay = retryDelay(armRestartBaseMs, armRestartMaxMs, armRestartFailures); + armRestartTimer = setTimeout(() => { + armRestartTimer = undefined; + if (cycleCanRun() && !child) startArmChild(); + }, delay); + armRestartTimer.unref?.(); + } + + function handleArmExit( + armChild: ReturnType, + id: number, + stdout: string, + stderr: string, + code: number | null, + error?: Error, + ): void { + if (child !== armChild) return; + child = null; + if (!cycleCanRun()) { + if (cycleActive) stopCycle(); + return; + } + + const reason = actionableLine(`${stdout}\n${stderr}`); + const failure = reason + ? "" + : error + ? `watcher: FAILED - Pi extension arm child ${id} failed: ${error.message}` + : failureLine(stdout, stderr, code); + if (reason) { + armRestartFailures = 0; + startArmChild(); + requestWakeDelivery(reason, true); + return; + } + if (failure) requestWakeDelivery(failure, false); + scheduleArmRestart(); + } + + function startArmChild(): ArmResult { + if (!cycleCanRun()) { + stopCycle(); + return { ok: false, message: pathPresent(awayMarker) + ? "watcher: away - the away-mode daemon owns supervision" + : "watcher: read-only - session lock is held by another firstmate session" }; + } if (child) return { ok: true, message: "watcher: healthy - Pi extension already has an arm child" }; - const id = ++seq; + clearArmRestartTimer(); + const id = ++childSequence; const env = { ...process.env, FM_HOME: fmHome, @@ -289,50 +526,111 @@ export default function (pi: ExtensionAPI) { FM_CONFIG_OVERRIDE: config, FM_WATCH_ARM_SCRIPT: armScript, }; - child = spawn("bash", ["-lc", "config_dir=\"${FM_CONFIG_OVERRIDE:-$FM_HOME/config}\"; [ -f \"$config_dir/x-mode.env\" ] && . \"$config_dir/x-mode.env\"; exec \"$FM_WATCH_ARM_SCRIPT\" --restart"], { - cwd: fmRoot, - env, - stdio: ["ignore", "pipe", "pipe"], - }); + let armChild: ReturnType; + try { + armChild = spawn("bash", ["-lc", "config_dir=\"${FM_CONFIG_OVERRIDE:-$FM_HOME/config}\"; [ -f \"$config_dir/x-mode.env\" ] && . \"$config_dir/x-mode.env\"; exec \"$FM_WATCH_ARM_SCRIPT\" --restart"], { + cwd: fmRoot, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + } catch (error) { + requestWakeDelivery( + `watcher: FAILED - Pi extension could not start arm child ${id}: ${error instanceof Error ? error.message : String(error)}`, + false, + ); + scheduleArmRestart(); + return { ok: false, message: `watcher: FAILED - Pi extension could not start arm child ${id}` }; + } + child = armChild; let stdout = ""; let stderr = ""; - child.stdout.on("data", (chunk: Buffer) => { + let settled = false; + armChild.stdout?.on("data", (chunk: Buffer) => { stdout += chunk.toString(); }); - child.stderr.on("data", (chunk: Buffer) => { + armChild.stderr?.on("data", (chunk: Buffer) => { stderr += chunk.toString(); }); - child.on("close", async (code: number | null) => { - child = null; - const reason = actionableLine(`${stdout}\n${stderr}`); - const failure = reason ? "" : failureLine(stdout, stderr, code); - if (!reason && !failure) return; - try { - sendWake(reason || failure); - } catch { - // Pi owns delivery errors; fail open so the extension never wedges the session. - } - }); - child.on("error", async (error: Error) => { - child = null; - try { - sendWake(`watcher: FAILED - Pi extension arm child ${id} failed: ${error.message}`); - } catch { - // Fail open. - } - }); + const settle = (code: number | null, error?: Error) => { + if (settled) return; + settled = true; + handleArmExit(armChild, id, stdout, stderr, code, error); + }; + armChild.on("close", (code: number | null) => settle(code)); + armChild.on("error", (error: Error) => settle(null, error)); return { ok: true, message: `watcher: started Pi extension arm child ${id}` }; } - pi.on?.("session_start", () => { + function reconcileControlState(): void { + if (!cycleActive || shuttingDown) return; + if (pathPresent(awayMarker) || !sessionOwnsLock()) { + stopCycle(); + return; + } + if (pendingWakeDelivery?.requiresQueue && !wakeQueuePending()) clearPendingWakeDelivery(); + } + + function ensureControlWatcher(): void { + if (controlWatcher) return; + try { + controlWatcher = watch(state, { persistent: false }, (_event, filename) => { + const name = filename?.toString(); + if (name && name !== ".lock" && name !== ".afk" && name !== ".wake-queue") return; + reconcileControlState(); + }); + controlWatcher.on("error", () => { + controlWatcher?.close(); + controlWatcher = undefined; + reconcileControlState(); + }); + } catch { + reconcileControlState(); + } + } + + function startArm(): ArmResult { + if (pathPresent(awayMarker)) { + stopCycle(); + return { ok: false, message: "watcher: away - the away-mode daemon owns supervision" }; + } + if (!sessionOwnsLock()) { + stopCycle(); + return { ok: false, message: "watcher: read-only - session lock is held by another firstmate session" }; + } + cycleActive = true; + markLoaded(); + ensureControlWatcher(); + return startArmChild(); + } + + const cleanupOnProcessExit = () => { + shuttingDown = true; + stopCycle(); + }; + process.once("exit", cleanupOnProcessExit); + + pi.on?.("session_start", (_event, ctx) => { + rememberContext(ctx); markLoaded(); }); - pi.on("agent_start", () => { + pi.on("agent_start", (_event, ctx) => { + rememberContext(ctx); activeAgentRunKey = `${process.pid}:${++agentRunSequence}`; agentStartAwaitingHumanDelivery = true; }); + pi.on("agent_settled", (_event, ctx) => { + rememberContext(ctx); + if (!pendingWakeDelivery || observeWakeAdmission()) return; + pendingWakeDelivery.handoffPending = false; + if (pendingWakeDelivery.requiresQueue && !wakeQueuePending()) { + clearPendingWakeDelivery(); + return; + } + scheduleWakeDeliveryRetry(); + }); + function newExchangeId(at: number, content: unknown): string { return createHash("sha256") .update(`${at}\0${++exchangeSequence}\0${JSON.stringify(content)}`) @@ -366,7 +664,8 @@ export default function (pi: ExtensionAPI) { return exchangeId; } - pi.on("input", (event) => { + pi.on("input", (event, ctx) => { + rememberContext(ctx); if (event.source === "extension") return; const at = Date.now(); const inputContent = cloneJson([{ type: "text", text: event.text }, ...(event.images ?? [])]); @@ -383,6 +682,11 @@ export default function (pi: ExtensionAPI) { }); pi.on("message_end", (event, ctx) => { + rememberContext(ctx); + if (event.message.role === "custom") { + observeWakeAdmission(event.message); + return; + } if (event.message.role === "user") { const content = cloneJson(event.message.content); const exchangeId = createAdmittedExchange(content, "steer", event.message.timestamp); @@ -425,6 +729,7 @@ export default function (pi: ExtensionAPI) { }); pi.on("context", (event, ctx) => { + rememberContext(ctx); const continuity = renderDirectExchangeContinuity( ctx.sessionManager.getBranch(), event.messages, @@ -448,13 +753,15 @@ export default function (pi: ExtensionAPI) { return { messages: [...event.messages.slice(0, insertAt), message, ...event.messages.slice(insertAt)] }; }); pi.on?.("session_shutdown", () => { - stopArm(); + shuttingDown = true; + stopCycle(); process.off("exit", cleanupOnProcessExit); }); pi.registerCommand?.("fm-watch-arm-pi", { description: "Arm firstmate watcher supervision through the Pi extension instead of foreground bash.", handler: async (_args, ctx) => { + rememberContext(ctx); const result = startArm(); ctx.ui.notify(result.message, result.ok ? "info" : "warning"); }, @@ -469,7 +776,8 @@ export default function (pi: ExtensionAPI) { "For Pi watcher supervision, call fm_watch_arm_pi instead of running bin/fm-watch-arm.sh through bash.", ], parameters: Type.Object({}), - execute: async () => { + execute: async (_toolCallId, _params, _signal, _onUpdate, ctx) => { + rememberContext(ctx); const result = startArm(); return { content: [{ type: "text", text: result.message }], diff --git a/docs/architecture.md b/docs/architecture.md index ff6a9142698..b6afcdfd54a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -58,6 +58,7 @@ This Firstmate tooling change requires no web-app, API, or realtime deployment. At session start, `bin/fm-session-start.sh` emits exactly one primary-harness supervision block rendered by `bin/fm-supervision-instructions.sh` from `docs/supervision-protocols/`. That block owns the live wait shape for the running primary harness: Claude and Grok use background-notify cycles, Codex uses bounded foreground checkpoints, Pi uses its two tracked primary extensions, and OpenCode uses its TUI plugin. +The [Pi supervision protocol](supervision-protocols/pi.md) owns its persistent watcher cycle, delivery admission and retries, and stop/resume boundaries. Pi's supervision prompts remain custom-message context while exact direct exchanges and open reply obligations survive compaction through the contract in [`supervision-protocols/pi.md`](supervision-protocols/pi.md#input-provenance-and-compaction-continuity). `bin/fm-watch-arm.sh` remains the verified arm wrapper for protocols that call it; it forks the watcher as a tracked child, verifies it is genuinely alive with a fresh liveness beacon, and prints an honest initial status (`started` / `attached` / restart-only `healthy` / `FAILED`, the last exiting non-zero). On `attached` it stays live until that existing cycle ends so background-notify harnesses do not get an empty false wake from a healthy no-op exit. diff --git a/docs/supervision-protocols/pi.md b/docs/supervision-protocols/pi.md index 7d95c973494..32d37453b77 100644 --- a/docs/supervision-protocols/pi.md +++ b/docs/supervision-protocols/pi.md @@ -1,16 +1,27 @@ -Mode: Pi extension background wake with direct-exchange compaction continuity. +Mode: Pi extension persistent background-wake cycle with observable custom-message delivery and direct-exchange compaction continuity. When this session owns supervision and away mode is not active: 1. Drain first with `bin/fm-wake-drain.sh`. 2. Confirm the Pi primary auto-loaded both project extensions (plain `pi`, after approving project trust once per clone); if not, restart with `-e __FM_PI_TURNEND_EXT__ -e __FM_PI_EXT__` as a trust-free fallback. -3. Arm supervision with the `fm_watch_arm_pi` tool. +3. Arm supervision once with the `fm_watch_arm_pi` tool. Use `/fm-watch-arm-pi` only as a human-entered fallback. Never run `bin/fm-watch-arm.sh` through Pi's bash tool because that foreground arm can wedge the agent and bypasses extension-owned cleanup. -4. The extension starts `bin/fm-watch-arm.sh --restart`, keeps the child attached to the live Pi process, and sends a context-participating `firstmate-watcher-wake` custom follow-up when the child exits with an actionable watcher reason. -5. If the extension says the watcher is already healthy, do not start another cycle. -6. If the extension reports a watcher failure, drain queued wakes, inspect the failure text, and restart Pi with both extensions loaded if needed. -7. Never use shell `&` for watcher supervision. - The arm mechanism above is extension-owned, not a model tool call, but a manual recovery probe that backgrounds, pipes, or bundles the arm is denied automatically by the PreToolUse seatbelt (`bin/fm-arm-pretool-check.sh`, wired into the turn-end guard extension at `__FM_PI_TURNEND_EXT__`). +4. The extension starts `bin/fm-watch-arm.sh --restart`, keeps the child attached to the live Pi process, and owns the single-child cycle after that arm. +5. On every actionable child exit, the extension starts the successor immediately without waiting for a model or tool turn, then sends a visible context-participating `firstmate-watcher-wake` custom follow-up. +6. The wake carries a stable delivery identity. + Pi's `message_end` custom-message event or the matching `custom_message` session entry proves admission. + The extension marks each fire-and-forget handoff in flight before calling Pi and does not retry it while asynchronous admission remains possible. + Pi's `agent_settled` event is the concrete non-admission boundary: if the matching wake was not observed by then, capped exponential retry continues while the durable queue remains undrained. + A synchronous handoff failure also permits retry; observed admission or queue drainage clears the pending delivery and cancels its retry timer. +7. The follow-up delivery mode preserves Pi's steering priority for direct captain input, and one pending delivery plus one arm child prevents message floods and duplicate watchers. +8. After a watcher wake, drain and handle the queue, but do not call the arm tool again because the extension already owns the successor. +9. If the extension says the watcher is already healthy, do not start another cycle. +10. Session shutdown, process exit, session-lock loss, or away-mode entry stops the extension-owned child and retry state. + After lock reacquisition or away-mode exit, the same arm tool resumes the cycle. +11. If the extension reports a watcher failure, drain queued wakes, inspect the failure text, and restart Pi with both extensions loaded if needed. + Non-actionable exits and spawn failures schedule automatic child restarts with capped exponential spacing; failure notifications without a queue requirement stop after three unadmitted attempts. +12. Never use shell `&` for watcher supervision. + The arm mechanism above is extension-owned, not a model tool call, but a manual recovery probe that backgrounds, pipes, or bundles the arm is denied automatically by the PreToolUse seatbelt (`bin/fm-arm-pretool-check.sh`, wired into the turn-end guard extension at `__FM_PI_TURNEND_EXT__`). The turn-end guard extension lives at `__FM_PI_TURNEND_EXT__`. The watcher extension lives at `__FM_PI_EXT__`. @@ -20,6 +31,7 @@ Both are tracked, project-local `.pi/extensions/*.ts` files that Pi auto-discove This document owns the Pi-specific boundary between direct captain input, automated supervision prompts, pending input, and rebuilt post-compaction model context. The watcher and turn-end extensions use Pi's context-participating `sendMessage()` custom-message path with distinct `firstmate-watcher-wake` and `firstmate-turnend-guard` types, `deliverAs: "followUp"`, and `triggerTurn: true`. +The admission checks in the cycle above do not infer success from `sendMessage()` returning because the extension API intentionally returns `void` and reports asynchronous rejection separately. They never use `sendUserMessage()` for automation, so supervision remains visible to the model without becoming a human-authored `role: "user"` turn. The watcher extension observes Pi's `input` event and records `interactive` or `rpc` submissions as provisional non-context `firstmate-direct-input-observation` entries before Pi queues or delivers them. @@ -64,6 +76,20 @@ It also proves a consumed steer does not become continuity metadata while unrela A separate compaction fixture cuts an unanswered direct question behind a custom watcher turn and proves the exact question returns as `OPEN_REPLY_OBLIGATION` rather than being inferred from summary prose. `tests/fm-pi-retry-continuity.test.sh` exercises the installed-Pi proof that an error followed by a retry without another human delivery closes the retained exchange, while a new human delivery on the retry replaces the retained cohort so its answer cannot close the older failed exchange. +### 2026-08-28 persistent-cycle regression evidence + +The observed failure sequence was an armed Pi child, an actionable watcher exit with a durable queued wake, no persisted `firstmate-watcher-wake`, no successor child, and a stale beacon until later captain input caused a manual re-arm. +The cycle protocol above owns the recovery contract exercised by the following proofs. + +Deterministic command: `tests/fm-pi-watch-extension.test.sh`. +Observed output included `ok - Pi watcher avoids duplicate in-flight admission, retries after settlement, survives two wakes, and stops on ownership changes`. +The fixture reproduces arm, durable queue append, actionable exit, a delayed asynchronous admission that arrives after the former retry timer would have fired without a duplicate handoff, custom-message admission and triggered turn, successor with a fresh beacon before queue drain, a second actionable exit whose first handoff reaches settlement without admission, stable-identity retry and admission, another successor, away-mode stop and resume, lock-loss stop and resume, and session-shutdown cleanup. + +Live command: `FM_PI_LIVE_E2E=1 FM_PI_LIVE_AUTH_DIR='/Users/dongkeun/.pi/firstmate-local' tests/fm-pi-primary-live-e2e.test.sh`. +Observed output: `ok - Pi 0.84.2 live E2E persisted two custom wakes, self-rearmed both successors, and cleaned up on exit`. +The isolated smoke used installed Pi, a cloned project, a private tmux socket, isolated Pi and Firstmate homes, copied credential input, the real watcher and arm scripts, and two consecutive durable status wakes. +It proved each actionable exit gained a distinct live successor with a fresh beacon that remained live after model queue handling, each wake persisted as a visible custom message and triggered a model turn that drained the queue, no second arm-tool call occurred, and `/quit` left neither watcher nor arm child alive. + Live command: `FM_PI_COMPACTION_LIVE_E2E=1 FM_PI_LIVE_AUTH_DIR='/Users/dongkeun/.pi/firstmate-local' tests/fm-pi-primary-compaction-live-e2e.test.sh`. Observed output: `ok - Pi 0.84.2 live compaction rebuilt the exact answered captain exchange across a custom watcher turn (firstKeptType=message)`. The smoke used a cloned project, private tmux socket, isolated `PI_CODING_AGENT_DIR`, isolated `FM_HOME`, copied read-only credential input, synthetic isolated watcher arm, real Pi `/compact`, and a later model turn. diff --git a/docs/turnend-guard.md b/docs/turnend-guard.md index 28098c9af82..22e45459652 100644 --- a/docs/turnend-guard.md +++ b/docs/turnend-guard.md @@ -124,8 +124,8 @@ Only unmarked child worktrees fall through to the linked-worktree exemption, and "No turn ends blind" for a secondmate is delivered by the same two mechanisms the main primary relies on. Mechanism B, the turn-end backstop, is this guard; its secondmate-home behavior is covered by hermetic tests in `tests/fm-turnend-guard.test.sh` (`test_hook_blocks_in_secondmate_own_home`, `test_hook_blocks_in_treehouse_leased_secondmate_home`, `test_hook_silent_in_idle_secondmate_home`, `test_hook_secondmate_loop_guard_allows_retry`, `test_hook_secondmate_reinvoke_recovery_loop`, `test_hook_silent_in_secondmate_child_worktree`, and `test_hook_exempts_linked_worktree_with_stray_marker`). -Mechanism A, the autonomous wake, is a harness property: when a background watcher task exits, the harness re-invokes the model, which drains the wake, advances children, and re-arms a fresh watcher. -Mechanism A cannot be a hermetic CI assertion because it requires a live model session, so it is recorded here as a dated first-hand measurement while `test_hook_secondmate_reinvoke_recovery_loop` covers the guard's deterministic half of the same recovery loop. +Mechanism A, the autonomous wake, follows the active [harness supervision protocol](supervision-protocols/); Pi's extension-owned successor cycle is specified there rather than relying on model re-arming. +The Claude autonomous model re-invocation measured below requires a live model session, so it is recorded here as a dated first-hand measurement while `test_hook_secondmate_reinvoke_recovery_loop` covers the guard's deterministic half of the same recovery loop. Autonomous-re-invoke measurement, run first-hand on Claude Code 2.1.207 (Darwin 25.5.0) on 2026-07-12. Procedure: launch a detached `run_in_background` Bash task that models a one-shot watcher - it records a launch epoch, runs `sleep 25`, then records a completion epoch just before exit, writing only to the session scratchpad - then end the turn with no further tool calls and no pending question, a genuinely idle session with no human input. @@ -148,5 +148,5 @@ No Herdr command was issued and no fleet state was touched; the experiment wrote `tests/fm-turnend-guard.test.sh` covers the shared predicate, primary scoping (including a secondmate's own home being guarded like the main primary while its child worktrees stay exempt), `FM_HOME` and `FM_STATE_OVERRIDE` precedence, Pi logical-run latch behavior for no-tool and multi-tool runs, fail-open behavior without `jq`, tracked hook registration for all five harnesses, and the Grok adapter's forced-resume loop guard and permission-mode regression. The default behavior suite does not invoke live language-model harnesses. -`FM_PI_LIVE_E2E=1 tests/fm-pi-primary-live-e2e.test.sh` opts into the isolated interactive Pi regression recorded above. +[`supervision-protocols/pi.md`](supervision-protocols/pi.md#2026-08-28-persistent-cycle-regression-evidence) owns the current Pi cycle and delivery evidence, including the opt-in interactive regression command. `FM_PI_COMPACTION_LIVE_E2E=1 FM_PI_LIVE_AUTH_DIR= tests/fm-pi-primary-compaction-live-e2e.test.sh` opts into the isolated real-compaction regression owned and evidenced by [`supervision-protocols/pi.md`](supervision-protocols/pi.md#2026-08-25-regression-evidence). diff --git a/tests/fm-pi-primary-live-e2e.test.sh b/tests/fm-pi-primary-live-e2e.test.sh index 8983b9536ab..a86d8ec4b00 100755 --- a/tests/fm-pi-primary-live-e2e.test.sh +++ b/tests/fm-pi-primary-live-e2e.test.sh @@ -13,6 +13,13 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" command -v pi >/dev/null 2>&1 || { echo "skip: pi not found"; exit 0; } command -v tmux >/dev/null 2>&1 || { echo "skip: tmux not found"; exit 0; } +AUTH_DIR=${FM_PI_LIVE_AUTH_DIR:-${PI_CODING_AGENT_DIR:-$HOME/.pi/agent}} +AUTH_FILE="$AUTH_DIR/auth.json" +[ -f "$AUTH_FILE" ] || { + echo "not ok - live Pi auth is unavailable at $AUTH_FILE" >&2 + exit 1 +} + TMUX=$(command -v tmux) SOCKET="fm-pi-live-e2e-$$" SESSION=pi-live-e2e @@ -44,19 +51,6 @@ wait_for_text() { return 1 } -wait_for_exact_line() { - local expected=$1 attempts=${2:-120} i=0 - while [ "$i" -lt "$attempts" ]; do - if capture | grep -Fxq " $expected"; then - return 0 - fi - sleep 0.5 - i=$((i + 1)) - done - capture >&2 - return 1 -} - lab_pid_is_safe() { local pid=$1 command command=$(ps -p "$pid" -o command= 2>/dev/null || true) @@ -103,48 +97,177 @@ wait_pid_dead() { return 1 } +current_watcher_pid() { + cat "$HOME_DIR/state/.watch.lock/pid" 2>/dev/null || true +} + +wait_for_successor() { + local prior=$1 attempts=${2:-200} i=0 candidate + while [ "$i" -lt "$attempts" ]; do + candidate=$(current_watcher_pid) + if [ -n "$candidate" ] && [ "$candidate" != "$prior" ] && kill -0 "$candidate" 2>/dev/null; then + printf '%s\n' "$candidate" + return 0 + fi + sleep 0.1 + i=$((i + 1)) + done + printf 'successor diagnostic: prior=%s current=%s queue=%s\n' \ + "$prior" "$(current_watcher_pid)" "$(cat "$HOME_DIR/state/.wake-queue" 2>/dev/null || true)" >&2 + find "$HOME_DIR/state" -maxdepth 2 -type f -print -exec sh -c 'printf "%s\\n" "--- $1 ---"; tail -5 "$1" 2>/dev/null || true' _ {} \; >&2 + ps -p "$prior" -o pid=,ppid=,etime=,command= >&2 || true + capture >&2 + return 1 +} + +wait_for_queue_empty() { + local attempts=${1:-200} i=0 + while [ "$i" -lt "$attempts" ]; do + [ ! -s "$HOME_DIR/state/.wake-queue" ] && return 0 + sleep 0.1 + i=$((i + 1)) + done + return 1 +} + +watcher_beacon_is_fresh() { + local beacon="$HOME_DIR/state/.last-watcher-beat" mtime now + [ -f "$beacon" ] || return 1 + if [ "$(uname)" = Darwin ]; then + mtime=$(stat -f %m "$beacon" 2>/dev/null || true) + else + mtime=$(stat -c %Y "$beacon" 2>/dev/null || true) + fi + case "$mtime" in ''|*[!0-9]*) return 1 ;; esac + now=$(date +%s) + [ "$((now - mtime))" -lt 5 ] +} + +custom_wake_count() { + python3 - "$PI_DIR" <<'PY' +import json +import pathlib +import sys + +count = 0 +for session in pathlib.Path(sys.argv[1]).glob("sessions/**/*.jsonl"): + for line in session.read_text(encoding="utf-8").splitlines(): + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + if entry.get("type") == "custom_message" and entry.get("customType") == "firstmate-watcher-wake": + count += 1 +print(count) +PY +} + +answered_custom_wake_count() { + python3 - "$PI_DIR" <<'PY' +import json +import pathlib +import sys + +answered = 0 +for session in pathlib.Path(sys.argv[1]).glob("sessions/**/*.jsonl"): + entries = [] + for line in session.read_text(encoding="utf-8").splitlines(): + try: + entries.append(json.loads(line)) + except json.JSONDecodeError: + pass + wakes = [ + index for index, entry in enumerate(entries) + if entry.get("type") == "custom_message" and entry.get("customType") == "firstmate-watcher-wake" + ] + for offset, wake in enumerate(wakes): + end = wakes[offset + 1] if offset + 1 < len(wakes) else len(entries) + if any( + entry.get("type") == "message" + and entry.get("message", {}).get("role") == "assistant" + and entry["message"].get("stopReason") == "stop" + for entry in entries[wake + 1:end] + ): + answered += 1 +print(answered) +PY +} + +wait_for_answered_custom_wakes() { + local expected=$1 attempts=${2:-360} i=0 + while [ "$i" -lt "$attempts" ]; do + [ "$(answered_custom_wake_count)" -ge "$expected" ] && return 0 + sleep 0.5 + i=$((i + 1)) + done + capture >&2 + return 1 +} + mkdir -p "$LAB" git clone -q "$ROOT" "$PROJECT" cp "$ROOT/.pi/extensions/fm-primary-pi-watch.ts" "$PROJECT/.pi/extensions/fm-primary-pi-watch.ts" cp "$ROOT/.pi/extensions/fm-primary-turnend-guard.ts" "$PROJECT/.pi/extensions/fm-primary-turnend-guard.ts" cp "$ROOT/bin/fm-supervision-instructions.sh" "$PROJECT/bin/fm-supervision-instructions.sh" mkdir -p "$HOME_DIR/state" "$HOME_DIR/config" "$PI_DIR" +touch "$HOME_DIR/state/.last-report-retention-attempt" \ + "$HOME_DIR/state/.last-account-session-sync" \ + "$HOME_DIR/state/.last-check" \ + "$HOME_DIR/state/.last-heartbeat" +cp "$AUTH_FILE" "$PI_DIR/auth.json" +chmod 600 "$PI_DIR/auth.json" "$TMUX" -L "$SOCKET" new-session -d -s "$SESSION" -c "$PROJECT" \ - "env PI_CODING_AGENT_DIR='$PI_DIR' FM_HOME='$HOME_DIR' FM_ROOT_OVERRIDE='$PROJECT' FM_POLL=1 FM_SIGNAL_GRACE=0 FM_HEARTBEAT=600 PI_OFFLINE=1 bash -lc 'printf \"%s\\n\" \"\$\$\" > \"\$FM_HOME/state/.lock\"; pi; rc=\$?; printf \"PI_EXIT=%s\\n\" \"\$rc\"; sleep 300'" + "env PI_CODING_AGENT_DIR='$PI_DIR' FM_HOME='$HOME_DIR' FM_ROOT_OVERRIDE='$PROJECT' FM_POLL=1 FM_SIGNAL_GRACE=0 FM_HEARTBEAT=600 bash -lc 'printf \"%s\\n\" \"\$\$\" > \"\$FM_HOME/state/.lock\"; pi --approve --provider openai-codex --model gpt-5.6-sol --thinking low; rc=\$?; printf \"PI_EXIT=%s\\n\" \"\$rc\"; sleep 300'" -wait_for_text "Trust project folder?" 40 || fail "Pi trust prompt did not appear" -"$TMUX" -L "$SOCKET" send-keys -t "$SESSION" Enter -wait_for_text "fm-primary-turnend-guard.ts" 60 || fail "Pi primary extensions did not load" +wait_for_text "fm-primary-turnend-guard.ts" 120 || fail "Pi primary extensions did not load" send_prompt "Use the bash tool to run printf PI_E2E_BASH_ONE. Then reply exactly BASH-ONE." -wait_for_exact_line "BASH-ONE" || fail "first bash turn did not complete" +wait_for_text "BASH-ONE" || fail "first bash turn did not complete" send_prompt "Use the read tool to read the first five lines of README.md. Then reply exactly READ-ONE." -wait_for_exact_line "READ-ONE" || fail "read turn did not complete" +wait_for_text "READ-ONE" || fail "read turn did not complete" send_prompt "Use the bash tool to run printf PI_E2E_BASH_TWO. Then reply exactly BASH-TWO." -wait_for_exact_line "BASH-TWO" || fail "second bash turn did not complete" +wait_for_text "BASH-TWO" || fail "second bash turn did not complete" : > "$HOME_DIR/state/pi-e2e.meta" -send_prompt "Reply exactly GUARD-TRIGGER with no tools. When the guard follow-up arrives, use fm_watch_arm_pi and never use bash to arm supervision. After any FIRSTMATE WATCHER WAKE, run bin/fm-wake-drain.sh, read the signaled status, call fm_watch_arm_pi to re-arm, and finish exactly REARMED." +send_prompt "Reply exactly GUARD-TRIGGER with no tools. When the guard follow-up arrives, use fm_watch_arm_pi exactly once and never use bash to arm supervision. After each FIRSTMATE WATCHER WAKE, run bin/fm-wake-drain.sh and read the signaled status, but do not call fm_watch_arm_pi because the extension has already self-rearmed. Reply exactly WAKE-ONE after the status says fire one, and exactly WAKE-TWO after it says fire two." wait_for_text "watcher: started Pi extension arm child 1" || fail "guard follow-up did not render the Pi watcher tool result" +initial_watcher_pid=$(wait_for_successor 0) \ + || fail "initial watcher was not live after the Pi arm tool" + +printf 'done: pi live e2e watcher fire one\n' > "$HOME_DIR/state/pi-e2e.status" +first_successor_pid=$(wait_for_successor "$initial_watcher_pid") \ + || fail "extension did not start a live successor after the first actionable exit" +kill -0 "$first_successor_pid" 2>/dev/null || fail "first successor watcher was not live" +watcher_beacon_is_fresh || fail "first successor did not publish a fresh beacon" +wait_for_answered_custom_wakes 1 || fail "first custom watcher wake did not trigger and finish a model turn" +wait_for_queue_empty || fail "first model turn did not drain the durable wake queue" +kill -0 "$first_successor_pid" 2>/dev/null || fail "first successor did not remain live after queue drain" -printf 'done: pi live e2e watcher fire\n' > "$HOME_DIR/state/pi-e2e.status" -wait_for_text "watcher: started Pi extension arm child 2" 180 || fail "watcher wake did not drain and re-arm through the Pi tool" -wait_for_exact_line "REARMED" 120 || fail "Pi did not settle after re-arming watcher supervision" +printf 'done: pi live e2e watcher fire two\n' >> "$HOME_DIR/state/pi-e2e.status" +second_successor_pid=$(wait_for_successor "$first_successor_pid") \ + || fail "extension did not continue supervision through the second actionable exit" +kill -0 "$second_successor_pid" 2>/dev/null || fail "second successor watcher was not live" +watcher_beacon_is_fresh || fail "second successor did not publish a fresh beacon" +wait_for_answered_custom_wakes 2 || fail "second custom watcher wake did not trigger and finish a model turn" +wait_for_queue_empty || fail "second model turn did not drain the durable wake queue" +kill -0 "$second_successor_pid" 2>/dev/null || fail "second successor did not remain live after queue drain" +wake_count=$(custom_wake_count) +[ "$wake_count" -eq 2 ] || fail "expected two persisted custom watcher wakes, saw $wake_count" pane=$(capture) guard_count=$(printf '%s\n' "$pane" | grep -Fc "TURN WOULD END BLIND - supervision is off." || true) [ "$guard_count" -eq 1 ] || fail "expected one guard injection, saw $guard_count" +arm_tool_count=$(printf '%s\n' "$pane" | grep -Fc "watcher: started Pi extension arm child" || true) +[ "$arm_tool_count" -eq 1 ] || fail "model re-armed manually instead of relying on the persistent extension cycle ($arm_tool_count tool results)" foreground_arm='$ bin/fm-watch-arm.sh' if printf '%s\n' "$pane" | grep -Fq "$foreground_arm"; then fail "Pi used a foreground bash watcher arm" fi -pid_file=$(find "$HOME_DIR/state" -maxdepth 3 -type f -name pid | head -1) -[ -n "$pid_file" ] || fail "re-armed watcher pid was not recorded" -watcher_pid=$(sed -n '1p' "$pid_file") +watcher_pid=$second_successor_pid arm_pid=$(ps -p "$watcher_pid" -o ppid= | tr -d ' ') -[ -n "$arm_pid" ] || fail "re-armed watcher parent was not live" +[ -n "$arm_pid" ] || fail "second successor watcher parent was not live" "$TMUX" -L "$SOCKET" send-keys -t "$SESSION" -l '/quit' sleep 1 @@ -153,4 +276,4 @@ wait_for_text "PI_EXIT=0" 60 || fail "Pi did not exit cleanly" wait_pid_dead "$watcher_pid" || fail "watcher child survived clean Pi exit" wait_pid_dead "$arm_pid" || fail "arm child survived clean Pi exit" -printf 'ok - Pi %s live E2E rendered the tool, guarded once, woke, re-armed, and cleaned up on exit\n' "$PI_VERSION" +printf 'ok - Pi %s live E2E persisted two custom wakes, self-rearmed both successors, and cleaned up on exit\n' "$PI_VERSION" diff --git a/tests/fm-pi-watch-extension.test.sh b/tests/fm-pi-watch-extension.test.sh index 3593a853b1b..7f6043e1218 100755 --- a/tests/fm-pi-watch-extension.test.sh +++ b/tests/fm-pi-watch-extension.test.sh @@ -31,38 +31,104 @@ export const Type = { JS } -test_tracked_extension_present_and_self_hashing() { - local text expected_config_source - expected_config_source="config_dir=\\\"\${FM_CONFIG_OVERRIDE:-\$FM_HOME/config}\\\"" - assert_present "$EXT" "tracked Pi primary watcher extension is missing" - text=$(cat "$EXT") - assert_contains "$text" "fm_watch_arm_pi" "tracked extension missing tool name" - assert_contains "$text" "fm-watch-arm-pi" "tracked extension missing command name" - assert_contains "$text" "fm-watch-arm.sh" "tracked extension missing watcher arm" - assert_contains "$text" "deliverAs: \"followUp\"" "tracked extension missing followUp delivery" - assert_contains "$text" ".pi-watch-extension-loaded" "tracked extension missing loaded marker" - assert_contains "$text" 'createHash("sha256").update(readFileSync(extensionFile)).digest("hex")' "tracked extension does not self-hash its own content for extensionVersion" - assert_contains "$text" 'fileURLToPath(import.meta.url)' "tracked extension does not self-locate via import.meta.url" - assert_contains "$text" "sessionOwnsLock" "tracked extension missing session lock ownership check" - assert_contains "$text" 'type LockOwnership = "owned" | "missing" | "other"' "tracked extension does not distinguish missing lock from another owner" - assert_contains "$text" "readFileSync(\`\${state}/.lock\`" "tracked extension does not read the effective session lock" - assert_contains "$text" 'return pidAlive(lockPid) ? "other" : "missing"' "tracked extension does not allow a pre-lock load marker" - assert_contains "$text" 'if (lockOwnership() === "other") return' "tracked extension overwrites another live session marker" - assert_contains "$text" "if (!sessionOwnsLock()) return { ok: false" "tracked extension arms without the session lock" - assert_contains "$text" "writeFileSync(marker, \`\${extensionVersion}\\n\${process.pid}\\n\`)" "tracked extension does not write the content version and process marker" - assert_contains "$text" "const config = process.env.FM_CONFIG_OVERRIDE" "tracked extension missing effective config resolution" - assert_contains "$text" "FM_CONFIG_OVERRIDE: config" "tracked extension does not pass the effective config to the watcher arm" - assert_contains "$text" "FM_WATCH_ARM_SCRIPT: armScript" "tracked extension does not pass the effective watcher arm script" - assert_contains "$text" "$expected_config_source" "tracked extension does not source the effective x-mode config" - assert_contains "$text" "exec \\\"\$FM_WATCH_ARM_SCRIPT\\\" --restart" "tracked extension does not restart into a Pi-owned watcher child" - assert_contains "$text" 'label: "Arm firstmate watcher"' "tracked extension tool is missing its human-readable label" - assert_contains "$text" 'parameters: Type.Object({})' "tracked extension tool is not using Pi's canonical TypeBox schema" - assert_contains "$text" 'content: [{ type: "text", text: result.message }]' "tracked extension tool is missing Pi text content" - assert_contains "$text" 'details: result' "tracked extension tool is missing structured result details" - assert_contains "$text" 'ctx.ui.notify' "tracked extension command does not notify through Pi's UI" - assert_contains "$text" 'process.once("exit", cleanupOnProcessExit)' "tracked extension lacks clean-process-exit cleanup" - assert_not_contains "$text" "[ -f config/x-mode.env ]" "tracked extension kept a repo-relative x-mode config path" - pass "Pi primary watcher extension is tracked, self-hashing, and self-locating" +test_pi_extension_runtime_configuration_contract() { + local repo home config_dir plugin arm_log out status + fm_node_supports_ts_import || { pass "node lacks .ts import support, skipping Pi runtime configuration check"; return; } + repo="$TMP_ROOT/pi-runtime-contract-root" + home="$TMP_ROOT/pi-runtime-contract-home" + config_dir="$TMP_ROOT/pi-runtime-contract-config" + arm_log="$TMP_ROOT/pi-runtime-contract-arm.log" + mkdir -p "$repo/bin" "$home/state" "$home/config" "$config_dir" + install_pi_watch_extension_fixture "$repo" + plugin="$repo/.pi/extensions/fm-primary-pi-watch.ts" + printf 'export FM_POLL=17\n' > "$config_dir/x-mode.env" + cat > "$repo/bin/fm-watch-arm.sh" <<'SH' +#!/usr/bin/env bash +printf 'config=%s script=%s poll=%s args=%s\n' \ + "$FM_CONFIG_OVERRIDE" "$FM_WATCH_ARM_SCRIPT" "${FM_POLL:-}" "$*" > "$FM_ARM_LOG" +printf '%s\n' "$$" > "$FM_CHILD_PID_FILE" +exec sleep 600 +SH + chmod +x "$repo/bin/fm-watch-arm.sh" + out=$(NODE_OPTIONS=--disable-warning=ExperimentalWarning \ + PLUGIN="$plugin" FM_HOME="$home" FM_ROOT_OVERRIDE="$repo" FM_CONFIG_OVERRIDE="$config_dir" \ + FM_ARM_LOG="$arm_log" FM_CHILD_PID_FILE="$TMP_ROOT/pi-runtime-contract-child.pid" \ + node --input-type=module 2>&1 <<'EOF' +import { createHash } from "node:crypto"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; + +const handlers = new Map(); +let tool; +const ctx = { + isIdle: () => true, + sessionManager: { getBranch: () => [] }, + ui: { notify() {} }, +}; +const pi = { + on(event, handler) { + handlers.set(event, handler); + }, + appendEntry() {}, + registerCommand() {}, + registerTool(candidate) { + if (candidate.name === "fm_watch_arm_pi") tool = candidate; + }, + sendMessage() {}, +}; +const sleep = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); +async function waitFor(predicate, label) { + for (let i = 0; i < 100; i += 1) { + if (predicate()) return; + await sleep(20); + } + throw new Error(`timed out waiting for ${label}`); +} +function alive(pid) { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +writeFileSync(`${process.env.FM_HOME}/state/.lock`, "1\n"); +const mod = await import(pathToFileURL(process.env.PLUGIN).href); +mod.default(pi); +if (!tool) throw new Error("Pi watch tool was not registered"); +const refused = await tool.execute("arm-without-lock", {}, undefined, undefined, ctx); +if (refused.details?.ok !== false || !refused.content?.[0]?.text.includes("read-only")) { + throw new Error(`arm did not refuse another session lock: ${JSON.stringify(refused)}`); +} +if (existsSync(process.env.FM_ARM_LOG)) throw new Error("arm child started without session lock ownership"); + +writeFileSync(`${process.env.FM_HOME}/state/.lock`, `${process.pid}\n`); +await handlers.get("session_start")?.({ type: "session_start", reason: "startup" }, ctx); +const started = await tool.execute("arm-owned", {}, undefined, undefined, ctx); +if (started.details?.ok !== true || !started.content?.[0]?.text.includes("started Pi extension arm child")) { + throw new Error(`owned arm did not start: ${JSON.stringify(started)}`); +} +await waitFor(() => existsSync(process.env.FM_ARM_LOG), "effective arm configuration"); +const expectedArm = `config=${process.env.FM_CONFIG_OVERRIDE} script=${process.env.FM_ROOT_OVERRIDE}/bin/fm-watch-arm.sh poll=17 args=--restart`; +const armLog = readFileSync(process.env.FM_ARM_LOG, "utf8").trim(); +if (armLog !== expectedArm) throw new Error(`unexpected effective arm configuration: ${armLog}`); + +const marker = readFileSync(`${process.env.FM_HOME}/state/.pi-watch-extension-loaded`, "utf8").trim().split("\n"); +const expectedVersion = `sha256:${createHash("sha256").update(readFileSync(process.env.PLUGIN)).digest("hex")}`; +if (marker[0] !== expectedVersion || marker[1] !== String(process.pid)) { + throw new Error(`loaded marker did not bind extension content and process: ${JSON.stringify(marker)}`); +} +const childPid = Number(readFileSync(process.env.FM_CHILD_PID_FILE, "utf8").trim()); +await handlers.get("session_shutdown")?.({ type: "session_shutdown", reason: "quit" }, ctx); +await waitFor(() => !alive(childPid), "session shutdown cleanup"); +EOF +) + status=$? + [ "$status" -eq 0 ] || printf '%s\n' "$out" >&2 + expect_code 0 "$status" "Pi extension runtime must enforce lock, config, marker, arm, and cleanup contracts" + [ -z "$out" ] || fail "Pi runtime configuration test printed output: $out" + pass "Pi extension enforces its runtime lock, config, marker, arm, and cleanup contracts" } test_spawn_template_mentions_pi_watch_placeholder() { @@ -315,6 +381,250 @@ EOF pass "Pi process-exit cleanup stops the attached arm child" } +test_pi_durable_cycle_retries_delivery_and_rearms() { + local repo home plugin out status + fm_node_supports_ts_import || { pass "node lacks .ts import support, skipping Pi durable-cycle check"; return; } + repo="$TMP_ROOT/pi-durable-cycle-root" + home="$TMP_ROOT/pi-durable-cycle-home" + mkdir -p "$repo/bin" "$home/state" "$home/config" + install_pi_watch_extension_fixture "$repo" + plugin="$repo/.pi/extensions/fm-primary-pi-watch.ts" + cat > "$repo/bin/fm-watch-arm.sh" <<'SH' +#!/usr/bin/env bash +set -u +count_file="$FM_HOME/state/.fixture-arm-count" +count=$(( $(cat "$count_file" 2>/dev/null || echo 0) + 1 )) +printf '%s\n' "$count" > "$count_file" +printf '%s\n' "$$" > "$FM_HOME/state/.fixture-arm-$count.pid" +printf 'launch=%s pid=%s\n' "$count" "$$" > "$FM_HOME/state/.last-watcher-beat" +printf 'watcher: started pid=%s (beacon fresh)\n' "$$" +case "$count" in + 1|2) + while [ ! -e "$FM_HOME/state/.fixture-fire-$count" ]; do sleep 0.01; done + printf '%s\t%s\tsignal\tfixture-%s\tsignal: durable wake %s\n' \ + "$(date +%s)" "$count" "$count" "$count" >> "$FM_HOME/state/.wake-queue" + printf 'signal: durable wake %s\n' "$count" + ;; + *) exec sleep 600 ;; +esac +SH + chmod +x "$repo/bin/fm-watch-arm.sh" + out=$(NODE_OPTIONS=--disable-warning=ExperimentalWarning \ + PLUGIN="$plugin" FM_HOME="$home" FM_ROOT_OVERRIDE="$repo" \ + FM_PI_WAKE_RETRY_BASE_MS=10 FM_PI_WAKE_RETRY_MAX_MS=20 \ + FM_PI_ARM_RESTART_BASE_MS=10 FM_PI_ARM_RESTART_MAX_MS=20 \ + node --input-type=module 2>&1 <<'EOF' +import { readFileSync, writeFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; + +const handlers = new Map(); +const branch = []; +let tool; +let idle = true; +let sendAttempts = 0; +let admissions = 0; +let triggeredTurns = 0; +const deliveryAttempts = []; +const admittedMessages = []; +const ctx = { + sessionManager: { getBranch: () => branch }, + isIdle: () => idle, + ui: { notify() {} }, +}; +const pi = { + on(event, handler) { + handlers.set(event, handler); + }, + appendEntry() {}, + registerCommand() {}, + registerTool(candidate) { + if (candidate.name === "fm_watch_arm_pi") tool = candidate; + }, + sendMessage(message, options) { + sendAttempts += 1; + deliveryAttempts.push({ + deliveryId: message.details?.deliveryId, + attempt: message.details?.attempt, + }); + if (message.customType !== "firstmate-watcher-wake" || message.display !== true) { + throw new Error(`wake was not a visible custom message: ${JSON.stringify(message)}`); + } + if (options?.deliverAs !== "followUp" || options?.triggerTurn !== true) { + throw new Error(`wake did not use triggering follow-up delivery: ${JSON.stringify(options)}`); + } + if (sendAttempts === 1) { + setTimeout(() => void admitMessage(message), 80); + return; // Reproduce asynchronous admission after the old retry timer would have fired. + } + if (sendAttempts === 2) { + idle = false; + return; // Reproduce a handoff that reaches settlement without observable admission. + } + queueMicrotask(() => void admitMessage(message)); + }, +}; +async function admitMessage(message) { + idle = false; + triggeredTurns += 1; + await handlers.get("agent_start")?.({ type: "agent_start" }, ctx); + const customMessage = { role: "custom", ...message, timestamp: Date.now() }; + await handlers.get("message_end")?.({ type: "message_end", message: customMessage }, ctx); + branch.push({ + type: "custom_message", + id: `wake-${admissions + 1}`, + parentId: branch.at(-1)?.id ?? null, + timestamp: new Date().toISOString(), + customType: message.customType, + content: message.content, + display: message.display, + details: message.details, + }); + admittedMessages.push(customMessage); + admissions += 1; + idle = true; + await handlers.get("agent_settled")?.({ type: "agent_settled" }, ctx); +} +const sleep = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); +async function waitFor(predicate, label, attempts = 500) { + for (let i = 0; i < attempts; i += 1) { + if (predicate()) return; + await sleep(10); + } + throw new Error(`timed out waiting for ${label}`); +} +function launchCount() { + try { + return Number(readFileSync(`${process.env.FM_HOME}/state/.fixture-arm-count`, "utf8").trim()); + } catch { + return 0; + } +} +function armPid(number) { + try { + return Number(readFileSync(`${process.env.FM_HOME}/state/.fixture-arm-${number}.pid`, "utf8").trim()); + } catch { + return 0; + } +} +function alive(pid) { + if (!pid) return false; + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} +function queuePending() { + try { + return readFileSync(`${process.env.FM_HOME}/state/.wake-queue`, "utf8").length > 0; + } catch { + return false; + } +} +function assertSingleLiveArm(expected) { + const live = []; + for (let number = 1; number <= launchCount(); number += 1) { + if (alive(armPid(number))) live.push(number); + } + if (live.length !== 1 || live[0] !== expected) { + throw new Error(`expected only arm ${expected} live, saw ${JSON.stringify(live)}`); + } +} + +writeFileSync(`${process.env.FM_HOME}/state/.lock`, `${process.pid}\n`); +const mod = await import(pathToFileURL(process.env.PLUGIN).href); +mod.default(pi); +await handlers.get("session_start")?.({ type: "session_start", reason: "startup" }, ctx); +if (!tool) throw new Error("Pi watch tool was not registered"); +await tool.execute("arm-initial", {}, undefined, undefined, ctx); +await waitFor(() => launchCount() === 1 && alive(armPid(1)), "initial arm child"); +assertSingleLiveArm(1); + +writeFileSync(`${process.env.FM_HOME}/state/.fixture-fire-1`, "fire\n"); +await waitFor(() => queuePending(), "first durable queued wake"); +await waitFor(() => launchCount() >= 2 && alive(armPid(2)), "successor after first wake"); +if (!queuePending()) throw new Error("first queue drained before successor proof"); +assertSingleLiveArm(2); +const firstBeacon = readFileSync(`${process.env.FM_HOME}/state/.last-watcher-beat`, "utf8"); +if (!firstBeacon.includes("launch=2")) throw new Error(`successor beacon was not fresh: ${firstBeacon}`); +await waitFor(() => sendAttempts === 1, "first delayed delivery attempt"); +await sleep(50); +if (sendAttempts !== 1) throw new Error(`in-flight handoff was duplicated before admission: ${sendAttempts}`); +await waitFor(() => admissions >= 1 && triggeredTurns >= 1, "asynchronous admission of first wake"); +if (sendAttempts !== 1 || deliveryAttempts[0]?.attempt !== 1 || admittedMessages[0]?.details?.attempt !== 1) { + throw new Error(`first wake was not admitted exactly once: ${JSON.stringify({ deliveryAttempts, admittedMessages })}`); +} +if (!deliveryAttempts[0]?.deliveryId) throw new Error("first wake had no delivery identity"); +writeFileSync(`${process.env.FM_HOME}/state/.wake-queue`, ""); +await waitFor(() => !queuePending(), "first queue drain"); + +writeFileSync(`${process.env.FM_HOME}/state/.fixture-fire-2`, "fire\n"); +await waitFor(() => queuePending(), "second durable queued wake"); +await waitFor(() => launchCount() >= 3 && alive(armPid(3)), "successor after second wake"); +assertSingleLiveArm(3); +const secondBeacon = readFileSync(`${process.env.FM_HOME}/state/.last-watcher-beat`, "utf8"); +if (!secondBeacon.includes("launch=3")) throw new Error(`second successor beacon was not fresh: ${secondBeacon}`); +await waitFor(() => sendAttempts === 2, "second unadmitted delivery attempt"); +await sleep(60); +if (sendAttempts !== 2) throw new Error(`second wake retried before Pi settled: ${sendAttempts}`); +idle = true; +await handlers.get("agent_settled")?.({ type: "agent_settled" }, ctx); +await waitFor(() => sendAttempts >= 3 && admissions >= 2 && triggeredTurns >= 2, "settlement retry admitting second wake"); +if (sendAttempts !== 3) throw new Error(`second wake flooded delivery attempts: ${sendAttempts}`); +if ( + deliveryAttempts[1]?.attempt !== 1 || + deliveryAttempts[2]?.attempt !== 2 || + !deliveryAttempts[1]?.deliveryId || + deliveryAttempts[1].deliveryId !== deliveryAttempts[2]?.deliveryId || + deliveryAttempts[1].deliveryId === deliveryAttempts[0]?.deliveryId +) { + throw new Error(`second wake did not retain a distinct delivery identity across retry: ${JSON.stringify(deliveryAttempts)}`); +} +if (admittedMessages[1]?.customType !== "firstmate-watcher-wake" || admittedMessages[1]?.details?.attempt !== 2) { + throw new Error(`second wake was not admitted on retry as custom input: ${JSON.stringify(admittedMessages[1])}`); +} +writeFileSync(`${process.env.FM_HOME}/state/.wake-queue`, ""); +await waitFor(() => !queuePending(), "second queue drain"); + +writeFileSync(`${process.env.FM_HOME}/state/.afk`, "away\n"); +await waitFor(() => !alive(armPid(3)), "away-mode arm stop"); +const awayLaunchCount = launchCount(); +await sleep(80); +if (launchCount() !== awayLaunchCount) throw new Error("away mode restarted the Pi-owned watcher cycle"); +writeFileSync(`${process.env.FM_HOME}/state/.lock`, `${process.pid}\n`); +await import("node:fs/promises").then(({ unlink }) => unlink(`${process.env.FM_HOME}/state/.afk`)); +await tool.execute("arm-after-away", {}, undefined, undefined, ctx); +await waitFor(() => launchCount() === awayLaunchCount + 1 && alive(armPid(4)), "arm after away exit"); +assertSingleLiveArm(4); + +writeFileSync(`${process.env.FM_HOME}/state/.lock`, "999999\n"); +await waitFor(() => !alive(armPid(4)), "session-lock-loss arm stop"); +const lockLossCount = launchCount(); +await sleep(80); +if (launchCount() !== lockLossCount) throw new Error("lock loss restarted the Pi-owned watcher cycle"); +writeFileSync(`${process.env.FM_HOME}/state/.lock`, `${process.pid}\n`); +await tool.execute("arm-after-lock", {}, undefined, undefined, ctx); +await waitFor(() => launchCount() === lockLossCount + 1 && alive(armPid(5)), "arm after lock reacquisition"); +assertSingleLiveArm(5); + +await handlers.get("session_shutdown")?.({ type: "session_shutdown", reason: "quit" }, ctx); +await waitFor(() => !alive(armPid(5)), "session-shutdown arm cleanup"); +for (let number = 1; number <= launchCount(); number += 1) { + if (alive(armPid(number))) throw new Error(`arm child ${number} survived cleanup`); +} +if (branch.length !== 2 || new Set(branch.map((entry) => entry.details?.deliveryId)).size !== 2) { + throw new Error(`unexpected admitted wake entries: ${JSON.stringify(branch)}`); +} +EOF +) + status=$? + [ "$status" -eq 0 ] || printf '%s\n' "$out" >&2 + expect_code 0 "$status" "Pi durable watcher cycle must retry observable delivery and survive consecutive wakes" + [ -z "$out" ] || fail "Pi durable-cycle test printed output: $out" + pass "Pi watcher avoids duplicate in-flight admission, retries after settlement, survives two wakes, and stops on ownership changes" +} + test_pi_compaction_preserves_direct_exchange_and_pending_input() { local repo home plugin pi_bin pi_package_dir out status fm_node_supports_ts_import || { pass "node lacks .ts import support, skipping Pi compaction-continuity check"; return; } @@ -331,7 +641,15 @@ test_pi_compaction_preserves_direct_exchange_and_pending_input() { plugin="$repo/.pi/extensions/fm-primary-pi-watch.ts" cat > "$repo/bin/fm-watch-arm.sh" <<'SH' #!/usr/bin/env bash -printf 'signal: deterministic supervision prompt\n' +count_file="$FM_HOME/state/.fixture-arm-count" +count=$(( $(cat "$count_file" 2>/dev/null || echo 0) + 1 )) +printf '%s\n' "$count" > "$count_file" +if [ "$count" -eq 1 ]; then + printf '%s\t%s\t%s\t%s\t%s\n' "$(date +%s)" 1 signal deterministic 'signal: deterministic supervision prompt' >> "$FM_HOME/state/.wake-queue" + printf 'signal: deterministic supervision prompt\n' + exit 0 +fi +exec sleep 600 SH chmod +x "$repo/bin/fm-watch-arm.sh" out=$(NODE_OPTIONS=--disable-warning=ExperimentalWarning PLUGIN="$plugin" PI_PACKAGE_DIR="$pi_package_dir" REPO="$repo" FM_HOME="$home" FM_ROOT_OVERRIDE="$repo" node --input-type=module 2>&1 <<'EOF' @@ -843,6 +1161,7 @@ if (!replacementContinuity?.content.includes(`OPEN_REPLY_OBLIGATION\nHuman input if (!replacementContinuity.content.includes(`ANSWERED\nHuman input, exact JSON: ${JSON.stringify(replacementQuestion)}`)) { throw new Error(`replacement question was not answered: ${replacementContinuity.content}`); } +await handlers.get("session_shutdown")?.({ type: "session_shutdown", reason: "quit" }, ctx); EOF ) status=$? @@ -1427,12 +1746,13 @@ if [ "${FM_PI_RETRY_CONTINUITY_PROOF:-0}" = 1 ]; then exit 0 fi -test_tracked_extension_present_and_self_hashing +test_pi_extension_runtime_configuration_contract test_spawn_template_mentions_pi_watch_placeholder test_pi_extension_reports_external_healthy_watcher test_pi_tool_returns_agent_tool_result test_pi_process_exit_cleanup_listener_lifecycle test_pi_process_exit_cleanup_stops_arm_child +test_pi_durable_cycle_retries_delivery_and_rearms test_pi_compaction_preserves_direct_exchange_and_pending_input test_opencode_primary_watch_plugin_uses_effective_state_home test_opencode_primary_watch_plugin_sources_effective_config diff --git a/tests/fm-session-start.test.sh b/tests/fm-session-start.test.sh index b7f038e71c3..e2343bc2f50 100755 --- a/tests/fm-session-start.test.sh +++ b/tests/fm-session-start.test.sh @@ -679,7 +679,7 @@ EOF block_count=$(printf '%s\n' "$out" | grep -c '^SUPERVISION OPERATING INSTRUCTIONS - primary harness:') [ "$block_count" -eq 1 ] || fail "expected exactly one supervision block, got $block_count" assert_contains "$out" "SUPERVISION OPERATING INSTRUCTIONS - primary harness: pi" "pi supervision block missing" - assert_contains "$out" "Mode: Pi extension background wake with direct-exchange compaction continuity." "pi snippet missing from session start" + assert_contains "$out" "Mode: Pi extension persistent background-wake cycle with observable custom-message delivery and direct-exchange compaction continuity." "pi snippet missing from session start" assert_contains "$out" "PI_WATCH_EXTENSION: not loaded" "pi extension load diagnostic missing" assert_contains "$out" "restart plain pi so $root/.pi/extensions/fm-primary-turnend-guard.ts and $root/.pi/extensions/fm-primary-pi-watch.ts auto-load" "pi extension load diagnostic omits the turn-end guard extension"