From eadb32da289ef95d71c132c9165ff9f793317823 Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" <622227+FredKSchott@users.noreply.github.com> Date: Wed, 3 Jun 2026 11:16:13 -0500 Subject: [PATCH 01/17] fix(cloudflare): update agents routing baseline --- apps/docs/package.json | 2 +- apps/www/package.json | 2 +- examples/assistant/package.json | 4 +- examples/chat-sdk/package.json | 6 +- examples/cloudflare-websocket/package.json | 6 +- examples/cloudflare/package.json | 10 +- examples/hello-world/package.json | 4 +- examples/imported-skill/package.json | 4 +- packages/cli/package.json | 13 +- .../cli/src/lib/cloudflare-wrangler-merge.ts | 19 +- packages/runtime/package.json | 2 +- ...dflare-agent-extension.integration.test.ts | 42 +- .../test/cloudflare-run-registry.test.ts | 1 + pnpm-lock.yaml | 596 +++++++++--------- 14 files changed, 382 insertions(+), 329 deletions(-) diff --git a/apps/docs/package.json b/apps/docs/package.json index 24d18c6e..2ef9aa99 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -24,6 +24,6 @@ "pagefind": "^1.5.2", "typescript": "^6.0.3", "vite": "^7.3.3", - "wrangler": "^4.59.3" + "wrangler": "^4.97.0" } } diff --git a/apps/www/package.json b/apps/www/package.json index eb9d5508..e022caf5 100644 --- a/apps/www/package.json +++ b/apps/www/package.json @@ -19,6 +19,6 @@ "devDependencies": { "typescript": "^6.0.3", "vite": "^7.3.3", - "wrangler": "^4.59.3" + "wrangler": "^4.97.0" } } diff --git a/examples/assistant/package.json b/examples/assistant/package.json index 8f073b70..26c26dec 100644 --- a/examples/assistant/package.json +++ b/examples/assistant/package.json @@ -3,9 +3,9 @@ "private": true, "type": "module", "dependencies": { - "@cloudflare/sandbox": "*", + "@cloudflare/sandbox": "^0.11.0", "@flue/runtime": "workspace:*", - "agents": "*" + "agents": "^0.14.0" }, "devDependencies": { "@flue/cli": "workspace:*" diff --git a/examples/chat-sdk/package.json b/examples/chat-sdk/package.json index 4b3329c7..cdbaaadc 100644 --- a/examples/chat-sdk/package.json +++ b/examples/chat-sdk/package.json @@ -10,14 +10,14 @@ "@chat-adapter/state-memory": "^4.29.0", "@earendil-works/pi-ai": "*", "@flue/runtime": "workspace:*", - "agents": "*", + "agents": "^0.14.0", "chat": "^4.29.0", "hono": "^4.8.3", "just-bash": "^3.0.1" }, "devDependencies": { - "@cloudflare/workers-types": "^4.20260505.1", + "@cloudflare/workers-types": "^4.20260602.1", "@flue/cli": "workspace:*", - "wrangler": "^4.87.0" + "wrangler": "^4.97.0" } } diff --git a/examples/cloudflare-websocket/package.json b/examples/cloudflare-websocket/package.json index 95422907..7a394c5a 100644 --- a/examples/cloudflare-websocket/package.json +++ b/examples/cloudflare-websocket/package.json @@ -7,13 +7,13 @@ }, "dependencies": { "@flue/runtime": "workspace:*", - "agents": "*", + "agents": "^0.14.0", "hono": "^4.7.0" }, "devDependencies": { - "@cloudflare/workers-types": "^4.20260505.1", + "@cloudflare/workers-types": "^4.20260602.1", "@flue/cli": "workspace:*", "@flue/sdk": "workspace:*", - "wrangler": "^4.87.0" + "wrangler": "^4.97.0" } } diff --git a/examples/cloudflare/package.json b/examples/cloudflare/package.json index af430b04..8830c673 100644 --- a/examples/cloudflare/package.json +++ b/examples/cloudflare/package.json @@ -3,16 +3,16 @@ "private": true, "type": "module", "dependencies": { - "@cloudflare/codemode": "^0.3.4", - "@cloudflare/shell": "^0.3.2", + "@cloudflare/codemode": "^0.3.8", + "@cloudflare/shell": "^0.3.8", "@flue/runtime": "workspace:*", - "agents": "*", + "agents": "^0.14.0", "hono": "^4.7.0", "valibot": "^1.0.0" }, "devDependencies": { - "@cloudflare/workers-types": "^4.20260505.1", + "@cloudflare/workers-types": "^4.20260602.1", "@flue/cli": "workspace:*", - "wrangler": "^4.87.0" + "wrangler": "^4.97.0" } } diff --git a/examples/hello-world/package.json b/examples/hello-world/package.json index 3d59cde1..6b375a87 100644 --- a/examples/hello-world/package.json +++ b/examples/hello-world/package.json @@ -3,9 +3,9 @@ "private": true, "type": "module", "dependencies": { - "@flue/runtime": "workspace:*", "@daytona/sdk": "*", - "agents": "*", + "@flue/runtime": "workspace:*", + "agents": "^0.14.0", "hono": "^4.7.0", "just-bash": "^3.0.1", "valibot": "^1.0.0" diff --git a/examples/imported-skill/package.json b/examples/imported-skill/package.json index 377b4be9..d106ee6b 100644 --- a/examples/imported-skill/package.json +++ b/examples/imported-skill/package.json @@ -4,11 +4,11 @@ "type": "module", "dependencies": { "@flue/runtime": "workspace:*", - "agents": "*", + "agents": "^0.14.0", "just-bash": "^3.0.1" }, "devDependencies": { "@flue/cli": "workspace:*", - "wrangler": "^4.94.0" + "wrangler": "^4.97.0" } } diff --git a/packages/cli/package.json b/packages/cli/package.json index 9f37e2cd..d1121029 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -28,7 +28,7 @@ "prepublishOnly": "cp ../../README.md ." }, "dependencies": { - "@cloudflare/vite-plugin": "^1.38.0", + "@cloudflare/vite-plugin": "^1.39.2", "@flue/runtime": "workspace:*", "@vercel/detect-agent": "^1.2.3", "package-up": "^5.0.0", @@ -38,15 +38,6 @@ "devDependencies": { "tsdown": "^0.22.0", "tsx": "^4.19.2", - "typescript": "^6.0.3", - "wrangler": "^4.94.0" - }, - "peerDependencies": { - "wrangler": "^4.94.0" - }, - "peerDependenciesMeta": { - "wrangler": { - "optional": true - } + "typescript": "^6.0.3" } } diff --git a/packages/cli/src/lib/cloudflare-wrangler-merge.ts b/packages/cli/src/lib/cloudflare-wrangler-merge.ts index 723da19c..99ee9c1e 100644 --- a/packages/cli/src/lib/cloudflare-wrangler-merge.ts +++ b/packages/cli/src/lib/cloudflare-wrangler-merge.ts @@ -14,8 +14,8 @@ * entirely user-authored. */ import * as fs from 'node:fs'; +import { createRequire } from 'node:module'; import * as path from 'node:path'; -import type { Unstable_Config, Unstable_RawConfig } from 'wrangler'; // ─── Constants ────────────────────────────────────────────────────────────── @@ -74,20 +74,23 @@ export async function readUserWranglerConfig(root: string): Promise { headers: { 'content-type': 'application/json' }, body: JSON.stringify({ prompt: 'Hello' }), }); - expect(response.status).not.toBe(500); + expect(response.status, await response.text()).not.toBe(500); const heartbeat = await fetch(new URL('/heartbeat', server.url)); expect(heartbeat.status).toBe(200); } finally { @@ -64,6 +65,30 @@ describe('Cloudflare agent extension', () => { fs.rmSync(root, { recursive: true, force: true }); } }, 90000); + + it('bootstraps the inherited PartyServer name before accepting an agent WebSocket', async () => { + const root = await createGeneratedFixture(); + let server: Awaited> | undefined; + let socket: WebSocket | undefined; + try { + server = await startServer(root); + const url = new URL('/agents/assistant/socket-instance', server.url); + url.protocol = 'ws:'; + socket = new WebSocket(url.toString()); + const firstMessage = await waitForSocketMessage(socket); + expect(JSON.parse(firstMessage)).toMatchObject({ + version: 1, + type: 'ready', + target: 'agent', + name: 'assistant', + instanceId: 'socket-instance', + }); + } finally { + socket?.close(); + await server?.close(); + fs.rmSync(root, { recursive: true, force: true }); + } + }, 90000); }); async function createGeneratedFixture( @@ -136,6 +161,20 @@ async function startServer(root: string): Promise<{ url: string; close(): Promis } } +async function waitForSocketMessage(socket: WebSocket): Promise { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error('Timed out waiting for WebSocket message.')), 10_000); + socket.once('message', (data) => { + clearTimeout(timeout); + resolve(data.toString()); + }); + socket.once('error', (error) => { + clearTimeout(timeout); + reject(error); + }); + }); +} + async function waitFor( predicate: () => Promise<{ done: boolean; detail: unknown }>, ): Promise { @@ -156,6 +195,7 @@ const defaultAgentSource = `import { createAgent } from '@flue/runtime'; import { extend } from '@flue/runtime/cloudflare'; export default createAgent(() => ({ model: false })); export const route = async (_c, next) => next(); +export const websocket = async (_c, next) => next(); export const cloudflare = extend({ base: (Base) => class extends Base { async startHeartbeat() { return this.scheduleEvery(1, 'heartbeat'); } diff --git a/packages/runtime/test/cloudflare-run-registry.test.ts b/packages/runtime/test/cloudflare-run-registry.test.ts index e3e912fa..de177494 100644 --- a/packages/runtime/test/cloudflare-run-registry.test.ts +++ b/packages/runtime/test/cloudflare-run-registry.test.ts @@ -57,6 +57,7 @@ describe('createCloudflareRunRegistry()', () => { ); expect(fake.requests[0]?.method).toBe('POST'); expect(fake.requests[0]?.headers.get('content-type')).toBe('application/json'); + expect(fake.requests[0]?.headers.get('x-partykit-room')).toBeNull(); expect(await fake.requests[0]?.json()).toEqual({ owner: { kind: 'workflow', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0170e5f2..45adafca 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -71,8 +71,8 @@ importers: specifier: ^7.3.3 version: 7.3.3(@types/node@24.12.4)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.9.0) wrangler: - specifier: ^4.59.3 - version: 4.94.0(@cloudflare/workers-types@4.20260523.1) + specifier: ^4.97.0 + version: 4.97.0(@cloudflare/workers-types@4.20260602.1) apps/www: dependencies: @@ -96,20 +96,20 @@ importers: specifier: ^7.3.3 version: 7.3.3(@types/node@24.12.4)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.3)(yaml@2.9.0) wrangler: - specifier: ^4.59.3 - version: 4.94.0(@cloudflare/workers-types@4.20260523.1) + specifier: ^4.97.0 + version: 4.97.0(@cloudflare/workers-types@4.20260602.1) examples/assistant: dependencies: '@cloudflare/sandbox': - specifier: '*' - version: 0.10.2 + specifier: ^0.11.0 + version: 0.11.0 '@flue/runtime': specifier: workspace:* version: link:../../packages/runtime agents: - specifier: '*' - version: 0.13.2(@babel/core@7.29.0)(@babel/runtime@7.29.2)(@cloudflare/codemode@0.3.7(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260523.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) + specifier: ^0.14.0 + version: 0.14.0(@babel/core@7.29.0)(@babel/runtime@7.29.7)(@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260602.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(just-bash@3.0.1)(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) devDependencies: '@flue/cli': specifier: workspace:* @@ -146,8 +146,8 @@ importers: specifier: workspace:* version: link:../../packages/runtime agents: - specifier: '*' - version: 0.13.2(@babel/core@7.29.0)(@babel/runtime@7.29.2)(@cloudflare/codemode@0.3.7(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260523.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) + specifier: ^0.14.0 + version: 0.14.0(@babel/core@7.29.0)(@babel/runtime@7.29.7)(@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260602.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(just-bash@3.0.1)(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) chat: specifier: ^4.29.0 version: 4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3) @@ -159,29 +159,29 @@ importers: version: 3.0.1 devDependencies: '@cloudflare/workers-types': - specifier: ^4.20260505.1 - version: 4.20260523.1 + specifier: ^4.20260602.1 + version: 4.20260602.1 '@flue/cli': specifier: workspace:* version: link:../../packages/cli wrangler: - specifier: ^4.87.0 - version: 4.94.0(@cloudflare/workers-types@4.20260523.1) + specifier: ^4.97.0 + version: 4.97.0(@cloudflare/workers-types@4.20260602.1) examples/cloudflare: dependencies: '@cloudflare/codemode': - specifier: ^0.3.4 - version: 0.3.7(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3) + specifier: ^0.3.8 + version: 0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3) '@cloudflare/shell': - specifier: ^0.3.2 + specifier: ^0.3.8 version: 0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3) '@flue/runtime': specifier: workspace:* version: link:../../packages/runtime agents: - specifier: '*' - version: 0.13.2(@babel/core@7.29.0)(@babel/runtime@7.29.2)(@cloudflare/codemode@0.3.7(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260523.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) + specifier: ^0.14.0 + version: 0.14.0(@babel/core@7.29.0)(@babel/runtime@7.29.7)(@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260602.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(just-bash@3.0.1)(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) hono: specifier: ^4.7.0 version: 4.12.22 @@ -190,14 +190,14 @@ importers: version: 1.4.0(typescript@6.0.3) devDependencies: '@cloudflare/workers-types': - specifier: ^4.20260505.1 - version: 4.20260523.1 + specifier: ^4.20260602.1 + version: 4.20260602.1 '@flue/cli': specifier: workspace:* version: link:../../packages/cli wrangler: - specifier: ^4.87.0 - version: 4.94.0(@cloudflare/workers-types@4.20260523.1) + specifier: ^4.97.0 + version: 4.97.0(@cloudflare/workers-types@4.20260602.1) examples/cloudflare-websocket: dependencies: @@ -205,15 +205,15 @@ importers: specifier: workspace:* version: link:../../packages/runtime agents: - specifier: '*' - version: 0.13.2(@babel/core@7.29.0)(@babel/runtime@7.29.2)(@cloudflare/codemode@0.3.7(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260523.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) + specifier: ^0.14.0 + version: 0.14.0(@babel/core@7.29.0)(@babel/runtime@7.29.7)(@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260602.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(just-bash@3.0.1)(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) hono: specifier: ^4.7.0 version: 4.12.22 devDependencies: '@cloudflare/workers-types': - specifier: ^4.20260505.1 - version: 4.20260523.1 + specifier: ^4.20260602.1 + version: 4.20260602.1 '@flue/cli': specifier: workspace:* version: link:../../packages/cli @@ -221,8 +221,8 @@ importers: specifier: workspace:* version: link:../../packages/sdk wrangler: - specifier: ^4.87.0 - version: 4.94.0(@cloudflare/workers-types@4.20260523.1) + specifier: ^4.97.0 + version: 4.97.0(@cloudflare/workers-types@4.20260602.1) examples/hello-world: dependencies: @@ -233,8 +233,8 @@ importers: specifier: workspace:* version: link:../../packages/runtime agents: - specifier: '*' - version: 0.13.2(@babel/core@7.29.0)(@babel/runtime@7.29.2)(@cloudflare/codemode@0.3.7(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260523.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) + specifier: ^0.14.0 + version: 0.14.0(@babel/core@7.29.0)(@babel/runtime@7.29.7)(@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260602.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(just-bash@3.0.1)(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) hono: specifier: ^4.7.0 version: 4.12.22 @@ -255,8 +255,8 @@ importers: specifier: workspace:* version: link:../../packages/runtime agents: - specifier: '*' - version: 0.13.2(@babel/core@7.29.0)(@babel/runtime@7.29.2)(@cloudflare/codemode@0.3.7(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260523.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) + specifier: ^0.14.0 + version: 0.14.0(@babel/core@7.29.0)(@babel/runtime@7.29.7)(@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260602.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(just-bash@3.0.1)(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) just-bash: specifier: ^3.0.1 version: 3.0.1 @@ -265,8 +265,8 @@ importers: specifier: workspace:* version: link:../../packages/cli wrangler: - specifier: ^4.94.0 - version: 4.94.0(@cloudflare/workers-types@4.20260523.1) + specifier: ^4.97.0 + version: 4.97.0(@cloudflare/workers-types@4.20260602.1) examples/node-websocket: dependencies: @@ -300,8 +300,8 @@ importers: packages/cli: dependencies: '@cloudflare/vite-plugin': - specifier: ^1.38.0 - version: 1.38.0(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(workerd@1.20260521.1)(wrangler@4.94.0(@cloudflare/workers-types@4.20260523.1)) + specifier: ^1.39.2 + version: 1.39.2(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(workerd@1.20260601.1)(wrangler@4.97.0(@cloudflare/workers-types@4.20260602.1)) '@flue/runtime': specifier: workspace:* version: link:../runtime @@ -327,9 +327,6 @@ importers: typescript: specifier: ^6.0.3 version: 6.0.3 - wrangler: - specifier: ^4.94.0 - version: 4.94.0(@cloudflare/workers-types@4.20260523.1) packages/connectors: {} @@ -407,8 +404,8 @@ importers: version: 8.21.0 devDependencies: '@cloudflare/workers-types': - specifier: ^4.20250410.0 - version: 4.20260523.1 + specifier: ^4.20260602.1 + version: 4.20260602.1 '@types/js-yaml': specifier: ^4.0.9 version: 4.0.9 @@ -670,80 +667,84 @@ packages: resolution: {integrity: sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==} engines: {node: '>=18.0.0'} - '@babel/code-frame@7.29.0': - resolution: {integrity: sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==} + '@babel/code-frame@7.29.7': + resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==} engines: {node: '>=6.9.0'} - '@babel/compat-data@7.29.3': - resolution: {integrity: sha512-LIVqM46zQWZhj17qA8wb4nW/ixr2y1Nw+r1etiAWgRM6U1IqP+LNhL1yg440jYZR72jCWcWbLWzIosH+uP1fqg==} + '@babel/compat-data@7.29.7': + resolution: {integrity: sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==} engines: {node: '>=6.9.0'} '@babel/core@7.29.0': resolution: {integrity: sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==} engines: {node: '>=6.9.0'} - '@babel/generator@7.29.1': - resolution: {integrity: sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw==} + '@babel/generator@7.29.7': + resolution: {integrity: sha512-DkXD5OJQaAQIdZ1bt3UZdEnHAn9Imd3IVBdX03UFe+ony9Ojw5pzr9YVKGDY1jt+Gcn/FnGkNf8r+Vj5NOJWtQ==} engines: {node: '>=6.9.0'} '@babel/generator@8.0.0-rc.5': resolution: {integrity: sha512-nFZPWz3FHIS7y6rMIVoa/WBwjdutfIaRJIBQjzn+t3RnecZoRNlGmGcyR2wb0T/IgSd50Kz/6dG8/LvMCRunjg==} engines: {node: ^22.18.0 || >=24.11.0} - '@babel/helper-annotate-as-pure@7.27.3': - resolution: {integrity: sha512-fXSwMQqitTGeHLBC08Eq5yXz2m37E4pJX1qAU1+2cNedz/ifv/bVXft90VeSav5nFO61EcNgwr0aJxbyPaWBPg==} + '@babel/helper-annotate-as-pure@7.29.7': + resolution: {integrity: sha512-OoK6239jHPuSQOoS0kfTVKn0b/rVTk0seKq4Gd2UMLtmOVLjDC0ki3e+c90Trqv2gMfvJFqkiljrr568+qddiw==} engines: {node: '>=6.9.0'} - '@babel/helper-compilation-targets@7.28.6': - resolution: {integrity: sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA==} + '@babel/helper-compilation-targets@7.29.7': + resolution: {integrity: sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==} engines: {node: '>=6.9.0'} - '@babel/helper-create-class-features-plugin@7.29.3': - resolution: {integrity: sha512-RpLYy2sb51oNLjuu1iD3bwBqCBWUzjO0ocp+iaCP/lJtb2CPLcnC2Fftw+4sAzaMELGeWTgExSKADbdo0GFVzA==} + '@babel/helper-create-class-features-plugin@7.29.7': + resolution: {integrity: sha512-IY3ZD9Tmooqr3TUhc3DUWxiuo8xx1DWLhd5M7hQ+ZWJamqM2BbalrBJb2MisSLoYorOj75U03qULCxQTY9r3hg==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0 - '@babel/helper-globals@7.28.0': - resolution: {integrity: sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==} + '@babel/helper-globals@7.29.7': + resolution: {integrity: sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==} engines: {node: '>=6.9.0'} - '@babel/helper-member-expression-to-functions@7.28.5': - resolution: {integrity: sha512-cwM7SBRZcPCLgl8a7cY0soT1SptSzAlMH39vwiRpOQkJlh53r5hdHwLSCZpQdVLT39sZt+CRpNwYG4Y2v77atg==} + '@babel/helper-member-expression-to-functions@7.29.7': + resolution: {integrity: sha512-j+7JYmk1JYDtACIGj0QJqqWZjoUpMoEikQGADMaHgCMCSDqd2+P32rfcibUNrGOMWrlzK1WJBdxrB3JJQZwWtg==} engines: {node: '>=6.9.0'} - '@babel/helper-module-imports@7.28.6': - resolution: {integrity: sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw==} + '@babel/helper-module-imports@7.29.7': + resolution: {integrity: sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==} engines: {node: '>=6.9.0'} - '@babel/helper-module-transforms@7.28.6': - resolution: {integrity: sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA==} + '@babel/helper-module-transforms@7.29.7': + resolution: {integrity: sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0 - '@babel/helper-optimise-call-expression@7.27.1': - resolution: {integrity: sha512-URMGH08NzYFhubNSGJrpUEphGKQwMQYBySzat5cAByY1/YgIRkULnIy3tAMeszlL/so2HbeilYloUmSpd7GdVw==} + '@babel/helper-optimise-call-expression@7.29.7': + resolution: {integrity: sha512-+kmGVjcT9RGYzoDwdwEqEvGgKe3BYq+O1iGzjFubaNgZHwYHP6lsF2Yghf4kEuv9BV7tYDZ913aBW9am6YKong==} engines: {node: '>=6.9.0'} - '@babel/helper-plugin-utils@7.28.6': - resolution: {integrity: sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug==} + '@babel/helper-plugin-utils@7.29.7': + resolution: {integrity: sha512-G7sHYigPY17oO5SYWnfD/0MTBwVR781S/JI643e/JhUYgVgWE/61SoW3NH9KWUKyKq5LVh3npif99Wkt6j86Jw==} engines: {node: '>=6.9.0'} - '@babel/helper-replace-supers@7.28.6': - resolution: {integrity: sha512-mq8e+laIk94/yFec3DxSjCRD2Z0TAjhVbEJY3UQrlwVo15Lmt7C2wAUbK4bjnTs4APkwsYLTahXRraQXhb1WCg==} + '@babel/helper-replace-supers@7.29.7': + resolution: {integrity: sha512-atfGXWSeCiF4DnKZIfmJfQRkSw9b9gNNXR1kqKjbhG4pGYCOnkp8OcTB8E3NXjBu8NpheSnOeNKz8KT7UNFTmQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0 - '@babel/helper-skip-transparent-expression-wrappers@7.27.1': - resolution: {integrity: sha512-Tub4ZKEXqbPjXgWLl2+3JpQAYBJ8+ikpQ2Ocj/q/r0LwE3UhENh7EUabyHjz2kCEsrRY83ew2DQdHluuiDQFzg==} + '@babel/helper-skip-transparent-expression-wrappers@7.29.7': + resolution: {integrity: sha512-brcMGQaVzIeUb+6/bs1Av0f8YuNNjKY2JyvfRCsFuFsdKccEQ5Ges2y74D74NZ1Rz8lKJ9ksJkfqwQFJ/iNEyQ==} engines: {node: '>=6.9.0'} '@babel/helper-string-parser@7.27.1': resolution: {integrity: sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==} engines: {node: '>=6.9.0'} + '@babel/helper-string-parser@7.29.7': + resolution: {integrity: sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==} + engines: {node: '>=6.9.0'} + '@babel/helper-string-parser@8.0.0-rc.5': resolution: {integrity: sha512-sN7R8rBvDurfaziNfDEIjIntlazmlkCDGO4SNl2RJ3wRCn+QxspLV7hzYAE8WWVd2joVuT8sUxeePdLp2idI1A==} engines: {node: ^22.18.0 || >=24.11.0} @@ -752,16 +753,20 @@ packages: resolution: {integrity: sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==} engines: {node: '>=6.9.0'} + '@babel/helper-validator-identifier@7.29.7': + resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==} + engines: {node: '>=6.9.0'} + '@babel/helper-validator-identifier@8.0.0-rc.5': resolution: {integrity: sha512-ehJDxHvtbZ85RtX/L2fi0h9AGsBNqB5Euv1EB8RMAvGYvD+2X+QbpzzOpbklnNXO+WSZJNOaetw2BBj27xsWVg==} engines: {node: ^22.18.0 || >=24.11.0} - '@babel/helper-validator-option@7.27.1': - resolution: {integrity: sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==} + '@babel/helper-validator-option@7.29.7': + resolution: {integrity: sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==} engines: {node: '>=6.9.0'} - '@babel/helpers@7.29.2': - resolution: {integrity: sha512-HoGuUs4sCZNezVEKdVcwqmZN8GoHirLUcLaYVNBK2J0DadGtdcqgr3BCbvH8+XUo4NGjNl3VOtSjEKNzqfFgKw==} + '@babel/helpers@7.29.7': + resolution: {integrity: sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==} engines: {node: '>=6.9.0'} '@babel/parser@7.29.3': @@ -769,6 +774,11 @@ packages: engines: {node: '>=6.0.0'} hasBin: true + '@babel/parser@7.29.7': + resolution: {integrity: sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==} + engines: {node: '>=6.0.0'} + hasBin: true + '@babel/parser@8.0.0-rc.4': resolution: {integrity: sha512-0S/1yefMa15N4i2v3t8Fw9pgMHhf2gF6Lc1UEXI96Ls6FNAjqvHHZouZ2ZS/deqLhbMFtmfVeFac6iTsvFbLwA==} engines: {node: ^20.19.0 || >=22.12.0} @@ -779,38 +789,46 @@ packages: engines: {node: ^22.18.0 || >=24.11.0} hasBin: true - '@babel/plugin-proposal-decorators@7.29.0': - resolution: {integrity: sha512-CVBVv3VY/XRMxRYq5dwr2DS7/MvqPm23cOCjbwNnVrfOqcWlnefua1uUs0sjdKOGjvPUG633o07uWzJq4oI6dA==} + '@babel/plugin-proposal-decorators@7.29.7': + resolution: {integrity: sha512-EtU0Hi3GvrTqD56xKmZvV/uCXK2ZbwVNPNLAquVItcAZpUhkXwWlo3Fmj0c2LxgSf2I8IDULeAepwNP1OefLXg==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 - '@babel/plugin-syntax-decorators@7.28.6': - resolution: {integrity: sha512-71EYI0ONURHJBL4rSFXnITXqXrrY8q4P0q006DPfN+Rk+ASM+++IBXem/ruokgBZR8YNEWZ8R6B+rCb8VcUTqA==} + '@babel/plugin-syntax-decorators@7.29.7': + resolution: {integrity: sha512-9MTTLbF39X6sqM92JPEsoI7++26hjZvzkxKZy64aMhWLH2mPkJ/Q3AV4QLmls3R14FpSpkOwQQfUh962JGQxxg==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 - '@babel/runtime-corejs3@7.29.2': - resolution: {integrity: sha512-Lc94FOD5+0aXhdb0Tdg3RUtqT6yWbI/BbFWvlaSJ3gAb9Ks+99nHRDKADVqC37er4eCB0fHyWT+y+K3QOvJKbw==} + '@babel/runtime-corejs3@7.29.7': + resolution: {integrity: sha512-ppj9ouYku+RX0ljtgZd+KMO5mkM2bCqg8H2PYAFWnLsHEIKIdRojqbJ2i3eVHrisuxy7nOFCmngTDdWtUCdXUQ==} engines: {node: '>=6.9.0'} '@babel/runtime@7.29.2': resolution: {integrity: sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==} engines: {node: '>=6.9.0'} - '@babel/template@7.28.6': - resolution: {integrity: sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==} + '@babel/runtime@7.29.7': + resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==} + engines: {node: '>=6.9.0'} + + '@babel/template@7.29.7': + resolution: {integrity: sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==} engines: {node: '>=6.9.0'} - '@babel/traverse@7.29.0': - resolution: {integrity: sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA==} + '@babel/traverse@7.29.7': + resolution: {integrity: sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw==} engines: {node: '>=6.9.0'} '@babel/types@7.29.0': resolution: {integrity: sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==} engines: {node: '>=6.9.0'} + '@babel/types@7.29.7': + resolution: {integrity: sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==} + engines: {node: '>=6.9.0'} + '@babel/types@8.0.0-rc.5': resolution: {integrity: sha512-JeSVu/m8x/zpp4CLjYHVNXuhEyOkhPXuxM8YOXjh6L4LlvQNKuUNOTo5KdBuKAcTDHw8DquToTaEkhsBqPXOaA==} engines: {node: ^22.18.0 || >=24.11.0} @@ -902,8 +920,8 @@ packages: resolution: {integrity: sha512-S0My7XPGIgpRWMDG8uRqalbgT+a6FmCUdOW+HaIOVVpUPHOb7RrpvjTjiODadKp06fsrVDJZlIzc6yCTp4AnxA==} engines: {node: '>= 20.12.0'} - '@cloudflare/codemode@0.3.7': - resolution: {integrity: sha512-Tc2AcF39ZExXMcCkTYtYGRHgIbWLePdWcRErEAS+RrpNgQht5dG0A9TkQDUQ1kyE43AC+gl4leVwhoWY2V9sWw==} + '@cloudflare/codemode@0.3.8': + resolution: {integrity: sha512-PVe99dFf/dvf0JOh1SBTYL7YT0nXusmqaK0lRrrlHGIQdGAnKRSb4VtaE5atiDVgN/U9G16bSkej7Pn5lWhG1g==} peerDependencies: '@modelcontextprotocol/sdk': ^1.25.0 '@tanstack/ai': '>=0.8.0 <1.0.0' @@ -919,15 +937,15 @@ packages: zod: optional: true - '@cloudflare/containers@0.3.4': - resolution: {integrity: sha512-6kWodmgBSug/rr9fjcWrZcKhxmWTEc1BTKC6nVv7yvYaRRvV3rZPZjq9R17eN0l9cl2LNc03wp/Z6OV+0uDwXA==} + '@cloudflare/containers@0.3.6': + resolution: {integrity: sha512-8RrbK/Et165gjvXccui3pgkUuySVWysTC6bJRXfgqmbCA2vAmh8pm7cAKDh2nZFR/GSjW4BgxeKpffCTD8SJEg==} '@cloudflare/kv-asset-handler@0.5.0': resolution: {integrity: sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg==} engines: {node: '>=22.0.0'} - '@cloudflare/sandbox@0.10.2': - resolution: {integrity: sha512-gZXcUaVMaVc/aBYFdgvyJGyibx0K16j+wWH/XDET63nWRpQKBlWM0dijhZb17OMRw/FWb+7utOdpNGMKyNMyCg==} + '@cloudflare/sandbox@0.11.0': + resolution: {integrity: sha512-OdM26ucJlc5B4en8eZMI0C8/UE3gk9bi2sqmElsGyqZIk6nhAo1HqHQynmuLKIHfNo6qo4bt12tWjTgFzzzgBg==} peerDependencies: '@openai/agents': ^0.3.3 '@opencode-ai/sdk': ^1.1.40 @@ -952,44 +970,45 @@ packages: workerd: optional: true - '@cloudflare/vite-plugin@1.38.0': - resolution: {integrity: sha512-ndb83ywNF2fU5Eb/YSH1ik5cJAU0CSdkRacbV8UbnMw8DLyOgK9fC8ovjA7uT7xdHoPgZTyh5iR3pqFo/2UHrw==} + '@cloudflare/vite-plugin@1.39.2': + resolution: {integrity: sha512-QzD0NBT2t7BXP0yNcVe9iW+pLZO/gcrBtZagdAEbh6hjTLLLUSoLOByUMW+/WlV7pK5S1UPaZuJ02aRhUnD/2g==} + hasBin: true peerDependencies: vite: ^6.1.0 || ^7.0.0 || ^8.0.0 - wrangler: ^4.94.0 + wrangler: ^4.97.0 - '@cloudflare/workerd-darwin-64@1.20260521.1': - resolution: {integrity: sha512-aiNdXmxlhwGjTSajL3I7uQPpN4lAOcXjvg5ZOlJKIywnevr798n9XCS6lvuqgniM3KjurBNWRRypMJntg/eSLg==} + '@cloudflare/workerd-darwin-64@1.20260601.1': + resolution: {integrity: sha512-iXZBVuRbvuVqQ/63wul01hHCv/3R8G5S8zbkjfoHvyPZFynmlKTV59Hk+H8whyGwFAZuB71UJGLr+G5mJKfjWA==} engines: {node: '>=16'} cpu: [x64] os: [darwin] - '@cloudflare/workerd-darwin-arm64@1.20260521.1': - resolution: {integrity: sha512-ikN8aKSi4Ak28ndOkuSO5rq6lmV6wwDQu9F9Vu6J7EkwAOth74J/Hjn4j4EuFceW/npw2Ws0Y/muzA6WKHl4TA==} + '@cloudflare/workerd-darwin-arm64@1.20260601.1': + resolution: {integrity: sha512-veGpZQGBw07Twt+Y4z3oyo+/obKHt0iWUwvDV5GOiDAYjC/zW+YGstgVzg4SHq+k1sLH3ElqL2TXx20I5WBv3Q==} engines: {node: '>=16'} cpu: [arm64] os: [darwin] - '@cloudflare/workerd-linux-64@1.20260521.1': - resolution: {integrity: sha512-D/gUhvQcG0pJr5aJl6yUoi2JxbFpjVtDq9xUJHPjfkAjL28TUVgCR/e5r8YGirepv4I1DK7ihuii9LZ2GGMJbw==} + '@cloudflare/workerd-linux-64@1.20260601.1': + resolution: {integrity: sha512-n/9hDz7fPGpYF0J684+Xr5zgjcS2jdmY2Of5m6e+eQ/M9+RfR+UaU8Ee/tkA1dDC0LYQB13hfPafZG66Ff1CsA==} engines: {node: '>=16'} cpu: [x64] os: [linux] - '@cloudflare/workerd-linux-arm64@1.20260521.1': - resolution: {integrity: sha512-vhjWPIHenczegTakhRPwEmTeaavCpNqsuo3RlLCkUdU47HrwLvy/4QersGggs4+kF4Do+IE/EznCGyT40xYcLA==} + '@cloudflare/workerd-linux-arm64@1.20260601.1': + resolution: {integrity: sha512-VHRZZbexATS+n+1j3x/CZaYbIJEye0J3iIHgG0Wp+l+NrZCKQ8qi8Lq1uTV0dLJQ67FuZtJtWdQ95mm9F7Fc+A==} engines: {node: '>=16'} cpu: [arm64] os: [linux] - '@cloudflare/workerd-windows-64@1.20260521.1': - resolution: {integrity: sha512-wBolYC/+lnGIEbkkPdzFtjTOWip2uQH6maeAP1ZV0kyxi5SGpsa83+wD5rH5OOle+sHE5qJMdwCKjwRwj+FKJg==} + '@cloudflare/workerd-windows-64@1.20260601.1': + resolution: {integrity: sha512-ye0C7MFLkeH16iTo8Tcjv2KiFmp23+sZGvUzSQa4xhP0QMe6EoJ+H/4SqqvnZ5nfN54slqKvx2VnXceENWe2CQ==} engines: {node: '>=16'} cpu: [x64] os: [win32] - '@cloudflare/workers-types@4.20260523.1': - resolution: {integrity: sha512-Kr66Jip2K3t0srdoLLImzblTTx2T409Zk2J6AHANmlB950rlHr2henMQx7+cdQOLm2p1CttuBcYB1UVjdiFN8Q==} + '@cloudflare/workers-types@4.20260602.1': + resolution: {integrity: sha512-0VssYYXHUn4VR1BaV+GXfhFpI53P2f6AIi17qyA9lQFyTs/u5ZF6IDPda2enDTIPFz/02872RM8CYVlXvRKtUA==} '@colors/colors@1.5.0': resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==} @@ -3327,17 +3346,19 @@ packages: resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} engines: {node: '>= 14'} - agents@0.13.2: - resolution: {integrity: sha512-s4v/e+BHrDKowsfjoCbQVA9FGPZgWz+1QCX41rR7UA2CHh90ZkARej3qrHhT+YxzYUfuWwbR9yQGFzP7/8rLsQ==} + agents@0.14.0: + resolution: {integrity: sha512-4+H8Pm0kEx6pcIaRi8KCYueRCw5sM03hC+yZjJ7YYEGBLFcresxSMVi5Zd6IpyHQFDd3kp5yE/gWVmNz9jRrWQ==} hasBin: true peerDependencies: - '@cloudflare/ai-chat': '>=0.6.1 <1.0.0' - '@cloudflare/codemode': '>=0.3.4 <1.0.0' + '@cloudflare/ai-chat': '>=0.8.0 <1.0.0' + '@cloudflare/codemode': '>=0.3.8 <1.0.0' + '@cloudflare/worker-bundler': '>=0.2.0 <1.0.0' '@tanstack/ai': '>=0.10.2 <1.0.0' '@x402/core': ^2.0.0 '@x402/evm': ^2.0.0 ai: ^6.0.0 chat: ^4.29.0 + just-bash: ^3.0.0 react: ^19.0.0 vite: '>=6.0.0 <9.0.0' zod: ^4.0.0 @@ -3346,6 +3367,8 @@ packages: optional: true '@cloudflare/codemode': optional: true + '@cloudflare/worker-bundler': + optional: true '@tanstack/ai': optional: true '@x402/core': @@ -3354,6 +3377,8 @@ packages: optional: true chat: optional: true + just-bash: + optional: true vite: optional: true @@ -3471,8 +3496,8 @@ packages: base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} - baseline-browser-mapping@2.10.32: - resolution: {integrity: sha512-wbPvpyjJPC0zdfdKXxqEL3Ea+bOMD/87X4lftiJkkaBiuG6ALQy1SLmEd7BSmVCuwCQsBrCamgBoLyfFDD1EPg==} + baseline-browser-mapping@2.10.33: + resolution: {integrity: sha512-bA6+tcSLpz2tIEdDXZPpPTIuxBcC4+w6SieaYyfigIa4h8GlFxbA17v22Vx3JUtuZQj9SgOsnbK+aTBzyDyEuw==} engines: {node: '>=6.0.0'} hasBin: true @@ -3569,8 +3594,8 @@ packages: caniuse-lite@1.0.30001793: resolution: {integrity: sha512-iwSsYWaCOoh26cV8NwNRViHlrfUvYsHDfRVcbtmw0Kg6PJIZZXwMkj1442FYLBGkeUf1juAsU3DTfxW579mrPA==} - capnweb@0.6.1: - resolution: {integrity: sha512-fmhV26QPd1ewf5R74h55oVZnGwIcSaRMzbfLQUy8+zOBjuTmT3KXoT8wxHvnp1m9Ht9BoUUS5ZwNLoVLfQTyBg==} + capnweb@0.8.0: + resolution: {integrity: sha512-BK/TuXUiyfLSKsmjojn70yN7oYG/JJzoURZ3tckjg5Zj2KcygPm0A5jyOlswK7SYB4f0Gh9tt+RZ132b80iLfA==} ccount@2.0.1: resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} @@ -3878,8 +3903,8 @@ packages: ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} - electron-to-chromium@1.5.361: - resolution: {integrity: sha512-Q6Hts7N9FnJc5LeGRINFvLhCI9xZmNtTDe5ZbcVezQz7cU4a8Aua3GH1b8J2XY8Al9PF+OCwYqhgsOOheMdvkA==} + electron-to-chromium@1.5.365: + resolution: {integrity: sha512-xfip4u1QF1s+URFqpA6N+OeFpDGpN7VJz1f3MO3bVL0QYBjpGiZ5/Of7kugvM+o8TTqmanUlviHN3c8M9vYWCw==} emmet@2.4.11: resolution: {integrity: sha512-23QPJB3moh/U9sT4rQzGgeyyGIrcM+GH5uVYg2C6wZIxAIJq7Ng3QLT79tl8FUwDXhyq9SusfknOrofAKqvgyQ==} @@ -4025,6 +4050,10 @@ packages: resolution: {integrity: sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ==} engines: {node: '>=18.0.0'} + eventsource-parser@3.1.0: + resolution: {integrity: sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==} + engines: {node: '>=18.0.0'} + eventsource@3.0.7: resolution: {integrity: sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==} engines: {node: '>=18.0.0'} @@ -4349,6 +4378,10 @@ packages: resolution: {integrity: sha512-7fvVPbB92zNRsQke+uiRGwtTuef0tB2Dg4hWxYfFNvkQhIltWoyi0ONReM5LWA+jJWS3nfT5lTq+qbsIpX0IQw==} engines: {node: '>=16.9.0'} + hono@4.12.23: + resolution: {integrity: sha512-eIaZ9qDgu7XV0pxOCrg7/WhnQ6Ivm22UcxhXx/A3dcbqbbYgBEkc6e/J/s7j2tS96zoB0S9VBdLwQNCWwUo4LA==} + engines: {node: '>=16.9.0'} + hookable@6.1.1: resolution: {integrity: sha512-U9LYDy1CwhMCnprUfeAZWZGByVbhd54hwepegYTK7Pi5NvqEj63ifz5z+xukznehT7i6NIZRu89Ay1AZmRsLEQ==} @@ -4506,8 +4539,8 @@ packages: isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} - isomorphic-git@1.38.1: - resolution: {integrity: sha512-Vd2u5qDLa04fA/h5nUMU5UuffPXqg+3D3bJIV3n7Sno2qS3XMinUXRvNHrGPVy2kkC1ad5SPCC3WcpXjn0L9oQ==} + isomorphic-git@1.38.3: + resolution: {integrity: sha512-rOpt0yIW9HD1o6mA4w2f4XP5Lj42QnccbFbhzkby6L+t9c2klQxf9seqyrw5x8JcWWnbuPuN7v7YJ7yvHj9OPA==} engines: {node: '>=14.17'} hasBin: true @@ -4912,8 +4945,8 @@ packages: resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} engines: {node: '>=10'} - miniflare@4.20260521.0: - resolution: {integrity: sha512-roRfxPq49OkuSeQsc43hRjSB1+HdHtDNKRwDEVk2hCjCBuBWxb5Wvwq88b0ULj6QVEJLN/+ZqF19M+h4VYJ/zg==} + miniflare@4.20260601.0: + resolution: {integrity: sha512-56TFiulSEQu43cYxdXgCiA3U3i+Ls0NoXwJXd6DmpNsx8yl/1Il2T3DQ4CMXjR6yfE7CSvC5MuXaqcSAMREjgw==} engines: {node: '>=22.0.0'} hasBin: true @@ -5015,8 +5048,8 @@ packages: node-mock-http@1.0.4: resolution: {integrity: sha512-8DY+kFsDkNXy1sJglUfuODx1/opAGJGyrTuFqEoN90oRc2Vk0ZbD4K2qmKXBBEhZQzdKHIVfEJpDU8Ak2NJEvQ==} - node-releases@2.0.46: - resolution: {integrity: sha512-GYVXHE2KnrzAfsAjl4uP++evGFCrAU1jta4ubEjIG7YWt/64Gqv66a30yKwWczVjA6j3bM4nBwH7Pk1JmDHaxQ==} + node-releases@2.0.47: + resolution: {integrity: sha512-Uzmd6LXpouKo8EUK68IjH4+E01w/hXyV3R3g/geCJo+rXLNfh1xucB+LOzYEOQPSiUK3h/xZf0cQGcSsmyL2Og==} engines: {node: '>=18'} normalize-path@3.0.0: @@ -5473,26 +5506,6 @@ packages: engines: {node: '>=18.0.0', npm: '>=8.0.0'} hasBin: true - rosie-skills-darwin-arm64@0.6.4: - resolution: {integrity: sha512-rn1s5hqFKcxeiDEWWoFa1hdGPshR8TkwHLzy/cBavb9XJNAaUxbe3oQ78W9sQkRHAgRyzJYyk9tw68Qrdnizgg==} - cpu: [arm64] - os: [darwin] - - rosie-skills-freebsd-x64@0.6.4: - resolution: {integrity: sha512-SxCRduPBMtfjkQ+q56Yw9OLA3PyaqoALzt7kER7IDKuUVfM2O/1w8sa5xhTDiCvWkZJixnH5d5Ya6KT+/Mwcng==} - cpu: [x64] - os: [freebsd] - - rosie-skills-linux-x64@0.6.4: - resolution: {integrity: sha512-D9Y9mfu7goB0s0X59uU3hcFeUTef3VbpCIDwFMzyvJrAq3XhRACWBDMHQsHlyWdHxTXPX/ILyW65RXyrJlgqng==} - cpu: [x64] - os: [linux] - - rosie-skills@0.6.4: - resolution: {integrity: sha512-ojfhSiQRdZ2QyWbmKAHOSAUbaLYrTc5zIH7mS1jKoP8KCFSQddwVhMyFqldckTeybTfW3zNcsZzyOTzGTN1SBA==} - engines: {node: '>=18'} - hasBin: true - router@2.2.0: resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==} engines: {node: '>= 18'} @@ -6277,8 +6290,8 @@ packages: resolution: {integrity: sha512-n1brCuqClxfFfq/Rb0ICg9giSZqCS+pLtccdag6C2HyufBrh3fBOiy9nb6ggRMvWOVH5GrdJskj5iGTZNxd7SA==} engines: {node: '>=4'} - which-typed-array@1.1.20: - resolution: {integrity: sha512-LYfpUkmqwl0h9A2HL09Mms427Q1RZWuOHsukfVcKRq9q95iQxdw0ix1JQrqbcDR9PH1QDwf5Qo8OZb5lksZ8Xg==} + which-typed-array@1.1.21: + resolution: {integrity: sha512-zbRA8cVm6io/d5W8uIe2hblzN76/Wm3v/yiythQvr+dpBWeqhPSWIDNj4zOyHi4zKbMK6DN34Xsr9jPHJERAEw==} engines: {node: '>= 0.4'} which@2.0.2: @@ -6291,17 +6304,17 @@ packages: engines: {node: '>=8'} hasBin: true - workerd@1.20260521.1: - resolution: {integrity: sha512-HzIThcZ0ZVEuzVxpY2IYZ3yssSrTjtrWXAVfmOl5rVwyqcu7aeZXGMiwrEmi9MOcC3wjy+BNv+hFrMMY5OrjQQ==} + workerd@1.20260601.1: + resolution: {integrity: sha512-Bg4+HF3B8TW0urAv8chiz25HSQ/aJxMBjgheUzu/nB1NQa+CaKGrUPv+Z3bf0np/WxLHYW1kcseVEtzZVPbX4g==} engines: {node: '>=16'} hasBin: true - wrangler@4.94.0: - resolution: {integrity: sha512-GsNw0DomGFfeXFtKVTwn2X69UKcCxcTB0CXykjsMineJIxOeyrw7LovlHQ/3JU8KJHH7repLB+kOHvfTBA/Eew==} + wrangler@4.97.0: + resolution: {integrity: sha512-jzW/aNvjerV+4TmwbvwGY6lpcuBk7EFUTonMDNfci45wSmMTj2/OJN+83cc/CeepKdb+6ZjGJw9NRjmcQoxqRg==} engines: {node: '>=22.0.0'} hasBin: true peerDependencies: - '@cloudflare/workers-types': ^4.20260521.1 + '@cloudflare/workers-types': ^4.20260601.1 peerDependenciesMeta: '@cloudflare/workers-types': optional: true @@ -6427,7 +6440,7 @@ snapshots: dependencies: '@ai-sdk/provider': 3.0.10 '@standard-schema/spec': 1.1.0 - eventsource-parser: 3.0.8 + eventsource-parser: 3.1.0 zod: 4.4.3 '@ai-sdk/provider@3.0.10': @@ -6920,25 +6933,25 @@ snapshots: '@aws/lambda-invoke-store@0.2.4': {} - '@babel/code-frame@7.29.0': + '@babel/code-frame@7.29.7': dependencies: - '@babel/helper-validator-identifier': 7.28.5 + '@babel/helper-validator-identifier': 7.29.7 js-tokens: 4.0.0 picocolors: 1.1.1 - '@babel/compat-data@7.29.3': {} + '@babel/compat-data@7.29.7': {} '@babel/core@7.29.0': dependencies: - '@babel/code-frame': 7.29.0 - '@babel/generator': 7.29.1 - '@babel/helper-compilation-targets': 7.28.6 - '@babel/helper-module-transforms': 7.28.6(@babel/core@7.29.0) - '@babel/helpers': 7.29.2 - '@babel/parser': 7.29.3 - '@babel/template': 7.28.6 - '@babel/traverse': 7.29.0 - '@babel/types': 7.29.0 + '@babel/code-frame': 7.29.7 + '@babel/generator': 7.29.7 + '@babel/helper-compilation-targets': 7.29.7 + '@babel/helper-module-transforms': 7.29.7(@babel/core@7.29.0) + '@babel/helpers': 7.29.7 + '@babel/parser': 7.29.7 + '@babel/template': 7.29.7 + '@babel/traverse': 7.29.7 + '@babel/types': 7.29.7 '@jridgewell/remapping': 2.3.5 convert-source-map: 2.0.0 debug: 4.4.3 @@ -6948,10 +6961,10 @@ snapshots: transitivePeerDependencies: - supports-color - '@babel/generator@7.29.1': + '@babel/generator@7.29.7': dependencies: - '@babel/parser': 7.29.3 - '@babel/types': 7.29.0 + '@babel/parser': 7.29.7 + '@babel/types': 7.29.7 '@jridgewell/gen-mapping': 0.3.13 '@jridgewell/trace-mapping': 0.3.31 jsesc: 3.1.0 @@ -6965,97 +6978,105 @@ snapshots: '@types/jsesc': 2.5.1 jsesc: 3.1.0 - '@babel/helper-annotate-as-pure@7.27.3': + '@babel/helper-annotate-as-pure@7.29.7': dependencies: - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 - '@babel/helper-compilation-targets@7.28.6': + '@babel/helper-compilation-targets@7.29.7': dependencies: - '@babel/compat-data': 7.29.3 - '@babel/helper-validator-option': 7.27.1 + '@babel/compat-data': 7.29.7 + '@babel/helper-validator-option': 7.29.7 browserslist: 4.28.2 lru-cache: 5.1.1 semver: 6.3.1 - '@babel/helper-create-class-features-plugin@7.29.3(@babel/core@7.29.0)': + '@babel/helper-create-class-features-plugin@7.29.7(@babel/core@7.29.0)': dependencies: '@babel/core': 7.29.0 - '@babel/helper-annotate-as-pure': 7.27.3 - '@babel/helper-member-expression-to-functions': 7.28.5 - '@babel/helper-optimise-call-expression': 7.27.1 - '@babel/helper-replace-supers': 7.28.6(@babel/core@7.29.0) - '@babel/helper-skip-transparent-expression-wrappers': 7.27.1 - '@babel/traverse': 7.29.0 + '@babel/helper-annotate-as-pure': 7.29.7 + '@babel/helper-member-expression-to-functions': 7.29.7 + '@babel/helper-optimise-call-expression': 7.29.7 + '@babel/helper-replace-supers': 7.29.7(@babel/core@7.29.0) + '@babel/helper-skip-transparent-expression-wrappers': 7.29.7 + '@babel/traverse': 7.29.7 semver: 6.3.1 transitivePeerDependencies: - supports-color - '@babel/helper-globals@7.28.0': {} + '@babel/helper-globals@7.29.7': {} - '@babel/helper-member-expression-to-functions@7.28.5': + '@babel/helper-member-expression-to-functions@7.29.7': dependencies: - '@babel/traverse': 7.29.0 - '@babel/types': 7.29.0 + '@babel/traverse': 7.29.7 + '@babel/types': 7.29.7 transitivePeerDependencies: - supports-color - '@babel/helper-module-imports@7.28.6': + '@babel/helper-module-imports@7.29.7': dependencies: - '@babel/traverse': 7.29.0 - '@babel/types': 7.29.0 + '@babel/traverse': 7.29.7 + '@babel/types': 7.29.7 transitivePeerDependencies: - supports-color - '@babel/helper-module-transforms@7.28.6(@babel/core@7.29.0)': + '@babel/helper-module-transforms@7.29.7(@babel/core@7.29.0)': dependencies: '@babel/core': 7.29.0 - '@babel/helper-module-imports': 7.28.6 - '@babel/helper-validator-identifier': 7.28.5 - '@babel/traverse': 7.29.0 + '@babel/helper-module-imports': 7.29.7 + '@babel/helper-validator-identifier': 7.29.7 + '@babel/traverse': 7.29.7 transitivePeerDependencies: - supports-color - '@babel/helper-optimise-call-expression@7.27.1': + '@babel/helper-optimise-call-expression@7.29.7': dependencies: - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 - '@babel/helper-plugin-utils@7.28.6': {} + '@babel/helper-plugin-utils@7.29.7': {} - '@babel/helper-replace-supers@7.28.6(@babel/core@7.29.0)': + '@babel/helper-replace-supers@7.29.7(@babel/core@7.29.0)': dependencies: '@babel/core': 7.29.0 - '@babel/helper-member-expression-to-functions': 7.28.5 - '@babel/helper-optimise-call-expression': 7.27.1 - '@babel/traverse': 7.29.0 + '@babel/helper-member-expression-to-functions': 7.29.7 + '@babel/helper-optimise-call-expression': 7.29.7 + '@babel/traverse': 7.29.7 transitivePeerDependencies: - supports-color - '@babel/helper-skip-transparent-expression-wrappers@7.27.1': + '@babel/helper-skip-transparent-expression-wrappers@7.29.7': dependencies: - '@babel/traverse': 7.29.0 - '@babel/types': 7.29.0 + '@babel/traverse': 7.29.7 + '@babel/types': 7.29.7 transitivePeerDependencies: - supports-color '@babel/helper-string-parser@7.27.1': {} + '@babel/helper-string-parser@7.29.7': {} + '@babel/helper-string-parser@8.0.0-rc.5': {} '@babel/helper-validator-identifier@7.28.5': {} + '@babel/helper-validator-identifier@7.29.7': {} + '@babel/helper-validator-identifier@8.0.0-rc.5': {} - '@babel/helper-validator-option@7.27.1': {} + '@babel/helper-validator-option@7.29.7': {} - '@babel/helpers@7.29.2': + '@babel/helpers@7.29.7': dependencies: - '@babel/template': 7.28.6 - '@babel/types': 7.29.0 + '@babel/template': 7.29.7 + '@babel/types': 7.29.7 '@babel/parser@7.29.3': dependencies: '@babel/types': 7.29.0 + '@babel/parser@7.29.7': + dependencies: + '@babel/types': 7.29.7 + '@babel/parser@8.0.0-rc.4': dependencies: '@babel/types': 8.0.0-rc.5 @@ -7064,40 +7085,42 @@ snapshots: dependencies: '@babel/types': 8.0.0-rc.5 - '@babel/plugin-proposal-decorators@7.29.0(@babel/core@7.29.0)': + '@babel/plugin-proposal-decorators@7.29.7(@babel/core@7.29.0)': dependencies: '@babel/core': 7.29.0 - '@babel/helper-create-class-features-plugin': 7.29.3(@babel/core@7.29.0) - '@babel/helper-plugin-utils': 7.28.6 - '@babel/plugin-syntax-decorators': 7.28.6(@babel/core@7.29.0) + '@babel/helper-create-class-features-plugin': 7.29.7(@babel/core@7.29.0) + '@babel/helper-plugin-utils': 7.29.7 + '@babel/plugin-syntax-decorators': 7.29.7(@babel/core@7.29.0) transitivePeerDependencies: - supports-color - '@babel/plugin-syntax-decorators@7.28.6(@babel/core@7.29.0)': + '@babel/plugin-syntax-decorators@7.29.7(@babel/core@7.29.0)': dependencies: '@babel/core': 7.29.0 - '@babel/helper-plugin-utils': 7.28.6 + '@babel/helper-plugin-utils': 7.29.7 - '@babel/runtime-corejs3@7.29.2': + '@babel/runtime-corejs3@7.29.7': dependencies: core-js-pure: 3.49.0 '@babel/runtime@7.29.2': {} - '@babel/template@7.28.6': + '@babel/runtime@7.29.7': {} + + '@babel/template@7.29.7': dependencies: - '@babel/code-frame': 7.29.0 - '@babel/parser': 7.29.3 - '@babel/types': 7.29.0 + '@babel/code-frame': 7.29.7 + '@babel/parser': 7.29.7 + '@babel/types': 7.29.7 - '@babel/traverse@7.29.0': + '@babel/traverse@7.29.7': dependencies: - '@babel/code-frame': 7.29.0 - '@babel/generator': 7.29.1 - '@babel/helper-globals': 7.28.0 - '@babel/parser': 7.29.3 - '@babel/template': 7.28.6 - '@babel/types': 7.29.0 + '@babel/code-frame': 7.29.7 + '@babel/generator': 7.29.7 + '@babel/helper-globals': 7.29.7 + '@babel/parser': 7.29.7 + '@babel/template': 7.29.7 + '@babel/types': 7.29.7 debug: 4.4.3 transitivePeerDependencies: - supports-color @@ -7107,6 +7130,11 @@ snapshots: '@babel/helper-string-parser': 7.27.1 '@babel/helper-validator-identifier': 7.28.5 + '@babel/types@7.29.7': + dependencies: + '@babel/helper-string-parser': 7.29.7 + '@babel/helper-validator-identifier': 7.29.7 + '@babel/types@8.0.0-rc.5': dependencies: '@babel/helper-string-parser': 8.0.0-rc.5 @@ -7194,7 +7222,7 @@ snapshots: fast-wrap-ansi: 0.2.2 sisteransi: 1.0.5 - '@cloudflare/codemode@0.3.7(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3)': + '@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3)': dependencies: '@types/json-schema': 7.0.15 acorn: 8.16.0 @@ -7203,62 +7231,62 @@ snapshots: ai: 6.0.191(zod@4.4.3) zod: 4.4.3 - '@cloudflare/containers@0.3.4': {} + '@cloudflare/containers@0.3.6': {} '@cloudflare/kv-asset-handler@0.5.0': {} - '@cloudflare/sandbox@0.10.2': + '@cloudflare/sandbox@0.11.0': dependencies: - '@cloudflare/containers': 0.3.4 + '@cloudflare/containers': 0.3.6 aws4fetch: 1.0.20 - capnweb: 0.6.1 - hono: 4.12.22 + capnweb: 0.8.0 + hono: 4.12.23 '@cloudflare/shell@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3)': dependencies: - '@cloudflare/codemode': 0.3.7(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3) - isomorphic-git: 1.38.1 + '@cloudflare/codemode': 0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3) + isomorphic-git: 1.38.3 transitivePeerDependencies: - '@modelcontextprotocol/sdk' - '@tanstack/ai' - ai - zod - '@cloudflare/unenv-preset@2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260521.1)': + '@cloudflare/unenv-preset@2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260601.1)': dependencies: unenv: 2.0.0-rc.24 optionalDependencies: - workerd: 1.20260521.1 + workerd: 1.20260601.1 - '@cloudflare/vite-plugin@1.38.0(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(workerd@1.20260521.1)(wrangler@4.94.0(@cloudflare/workers-types@4.20260523.1))': + '@cloudflare/vite-plugin@1.39.2(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(workerd@1.20260601.1)(wrangler@4.97.0(@cloudflare/workers-types@4.20260602.1))': dependencies: - '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260521.1) - miniflare: 4.20260521.0 + '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260601.1) + miniflare: 4.20260601.0 unenv: 2.0.0-rc.24 vite: 8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0) - wrangler: 4.94.0(@cloudflare/workers-types@4.20260523.1) + wrangler: 4.97.0(@cloudflare/workers-types@4.20260602.1) ws: 8.20.1 transitivePeerDependencies: - bufferutil - utf-8-validate - workerd - '@cloudflare/workerd-darwin-64@1.20260521.1': + '@cloudflare/workerd-darwin-64@1.20260601.1': optional: true - '@cloudflare/workerd-darwin-arm64@1.20260521.1': + '@cloudflare/workerd-darwin-arm64@1.20260601.1': optional: true - '@cloudflare/workerd-linux-64@1.20260521.1': + '@cloudflare/workerd-linux-64@1.20260601.1': optional: true - '@cloudflare/workerd-linux-arm64@1.20260521.1': + '@cloudflare/workerd-linux-arm64@1.20260601.1': optional: true - '@cloudflare/workerd-windows-64@1.20260521.1': + '@cloudflare/workerd-windows-64@1.20260601.1': optional: true - '@cloudflare/workers-types@4.20260523.1': {} + '@cloudflare/workers-types@4.20260602.1': {} '@colors/colors@1.5.0': optional: true @@ -8753,13 +8781,13 @@ snapshots: '@rolldown/binding-win32-x64-msvc@1.0.2': optional: true - '@rolldown/plugin-babel@0.2.3(@babel/core@7.29.0)(@babel/runtime@7.29.2)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))': + '@rolldown/plugin-babel@0.2.3(@babel/core@7.29.0)(@babel/runtime@7.29.7)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))': dependencies: '@babel/core': 7.29.0 picomatch: 4.0.4 rolldown: 1.0.2 optionalDependencies: - '@babel/runtime': 7.29.2 + '@babel/runtime': 7.29.7 vite: 8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0) '@rolldown/pluginutils@1.0.1': {} @@ -9346,24 +9374,26 @@ snapshots: agent-base@7.1.4: {} - agents@0.13.2(@babel/core@7.29.0)(@babel/runtime@7.29.2)(@cloudflare/codemode@0.3.7(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260523.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3): + agents@0.14.0(@babel/core@7.29.0)(@babel/runtime@7.29.7)(@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260602.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(just-bash@3.0.1)(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3): dependencies: - '@babel/plugin-proposal-decorators': 7.29.0(@babel/core@7.29.0) + '@babel/plugin-proposal-decorators': 7.29.7(@babel/core@7.29.0) '@cfworker/json-schema': 4.1.1 '@modelcontextprotocol/sdk': 1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3) - '@rolldown/plugin-babel': 0.2.3(@babel/core@7.29.0)(@babel/runtime@7.29.2)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0)) + '@rolldown/plugin-babel': 0.2.3(@babel/core@7.29.0)(@babel/runtime@7.29.7)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0)) ai: 6.0.191(zod@4.4.3) cron-schedule: 6.0.0 mimetext: 3.0.28 nanoid: 5.1.11 - partyserver: 0.5.6(@cloudflare/workers-types@4.20260523.1) + partyserver: 0.5.6(@cloudflare/workers-types@4.20260602.1) partysocket: 1.1.19(react@19.2.6) react: 19.2.6 + yaml: 2.9.0 yargs: 18.0.0 zod: 4.4.3 optionalDependencies: - '@cloudflare/codemode': 0.3.7(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3) + '@cloudflare/codemode': 0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3) chat: 4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3) + just-bash: 3.0.1 vite: 8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0) transitivePeerDependencies: - '@babel/core' @@ -9557,7 +9587,7 @@ snapshots: base64-js@1.5.1: {} - baseline-browser-mapping@2.10.32: {} + baseline-browser-mapping@2.10.33: {} bcp-47-match@2.0.3: {} @@ -9646,10 +9676,10 @@ snapshots: browserslist@4.28.2: dependencies: - baseline-browser-mapping: 2.10.32 + baseline-browser-mapping: 2.10.33 caniuse-lite: 1.0.30001793 - electron-to-chromium: 1.5.361 - node-releases: 2.0.46 + electron-to-chromium: 1.5.365 + node-releases: 2.0.47 update-browserslist-db: 1.2.3(browserslist@4.28.2) buffer-equal-constant-time@1.0.1: {} @@ -9697,7 +9727,7 @@ snapshots: caniuse-lite@1.0.30001793: {} - capnweb@0.6.1: {} + capnweb@0.8.0: {} ccount@2.0.1: {} @@ -9949,7 +9979,7 @@ snapshots: ee-first@1.1.1: {} - electron-to-chromium@1.5.361: {} + electron-to-chromium@1.5.365: {} emmet@2.4.11: dependencies: @@ -10159,6 +10189,8 @@ snapshots: eventsource-parser@3.0.8: {} + eventsource-parser@3.1.0: {} + eventsource@3.0.7: dependencies: eventsource-parser: 3.0.8 @@ -10661,6 +10693,8 @@ snapshots: hono@4.12.22: {} + hono@4.12.23: {} + hookable@6.1.1: {} html-escaper@3.0.3: {} @@ -10782,7 +10816,7 @@ snapshots: is-typed-array@1.1.15: dependencies: - which-typed-array: 1.1.20 + which-typed-array: 1.1.21 is-wsl@3.1.1: dependencies: @@ -10792,7 +10826,7 @@ snapshots: isexe@2.0.0: {} - isomorphic-git@1.38.1: + isomorphic-git@1.38.3: dependencies: async-lock: 1.4.1 clean-git-ref: 2.0.1 @@ -11472,19 +11506,19 @@ snapshots: mimetext@3.0.28: dependencies: - '@babel/runtime': 7.29.2 - '@babel/runtime-corejs3': 7.29.2 + '@babel/runtime': 7.29.7 + '@babel/runtime-corejs3': 7.29.7 js-base64: 3.7.8 mime-types: 2.1.35 mimic-response@3.1.0: {} - miniflare@4.20260521.0: + miniflare@4.20260601.0: dependencies: '@cspotcode/source-map-support': 0.8.1 sharp: 0.34.5 undici: 7.24.8 - workerd: 1.20260521.1 + workerd: 1.20260601.1 ws: 8.20.1 youch: 4.1.0-beta.10 transitivePeerDependencies: @@ -11566,7 +11600,7 @@ snapshots: node-mock-http@1.0.4: {} - node-releases@2.0.46: {} + node-releases@2.0.47: {} normalize-path@3.0.0: {} @@ -11727,9 +11761,9 @@ snapshots: partial-json@0.1.7: {} - partyserver@0.5.6(@cloudflare/workers-types@4.20260523.1): + partyserver@0.5.6(@cloudflare/workers-types@4.20260602.1): dependencies: - '@cloudflare/workers-types': 4.20260523.1 + '@cloudflare/workers-types': 4.20260602.1 nanoid: 5.1.11 partysocket@1.1.19(react@19.2.6): @@ -12188,21 +12222,6 @@ snapshots: '@rollup/rollup-win32-x64-msvc': 4.60.4 fsevents: 2.3.3 - rosie-skills-darwin-arm64@0.6.4: - optional: true - - rosie-skills-freebsd-x64@0.6.4: - optional: true - - rosie-skills-linux-x64@0.6.4: - optional: true - - rosie-skills@0.6.4: - optionalDependencies: - rosie-skills-darwin-arm64: 0.6.4 - rosie-skills-freebsd-x64: 0.6.4 - rosie-skills-linux-x64: 0.6.4 - router@2.2.0: dependencies: debug: 4.4.3 @@ -12937,7 +12956,7 @@ snapshots: which-pm-runs@1.1.0: {} - which-typed-array@1.1.20: + which-typed-array@1.1.21: dependencies: available-typed-arrays: 1.0.7 call-bind: 1.0.9 @@ -12956,27 +12975,26 @@ snapshots: siginfo: 2.0.0 stackback: 0.0.2 - workerd@1.20260521.1: + workerd@1.20260601.1: optionalDependencies: - '@cloudflare/workerd-darwin-64': 1.20260521.1 - '@cloudflare/workerd-darwin-arm64': 1.20260521.1 - '@cloudflare/workerd-linux-64': 1.20260521.1 - '@cloudflare/workerd-linux-arm64': 1.20260521.1 - '@cloudflare/workerd-windows-64': 1.20260521.1 + '@cloudflare/workerd-darwin-64': 1.20260601.1 + '@cloudflare/workerd-darwin-arm64': 1.20260601.1 + '@cloudflare/workerd-linux-64': 1.20260601.1 + '@cloudflare/workerd-linux-arm64': 1.20260601.1 + '@cloudflare/workerd-windows-64': 1.20260601.1 - wrangler@4.94.0(@cloudflare/workers-types@4.20260523.1): + wrangler@4.97.0(@cloudflare/workers-types@4.20260602.1): dependencies: '@cloudflare/kv-asset-handler': 0.5.0 - '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260521.1) + '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260601.1) blake3-wasm: 2.1.5 esbuild: 0.27.3 - miniflare: 4.20260521.0 + miniflare: 4.20260601.0 path-to-regexp: 6.3.0 - rosie-skills: 0.6.4 unenv: 2.0.0-rc.24 - workerd: 1.20260521.1 + workerd: 1.20260601.1 optionalDependencies: - '@cloudflare/workers-types': 4.20260523.1 + '@cloudflare/workers-types': 4.20260602.1 fsevents: 2.3.3 transitivePeerDependencies: - bufferutil From fc689140ba87d52e17ac76abad81a24cc778d12c Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" <622227+FredKSchott@users.noreply.github.com> Date: Wed, 3 Jun 2026 11:53:33 -0500 Subject: [PATCH 02/17] test(cloudflare): cover named durable object routes --- ...dflare-agent-extension.integration.test.ts | 76 +++++++++++++++---- 1 file changed, 63 insertions(+), 13 deletions(-) diff --git a/packages/runtime/test/cloudflare-agent-extension.integration.test.ts b/packages/runtime/test/cloudflare-agent-extension.integration.test.ts index 24982c1b..8a931e13 100644 --- a/packages/runtime/test/cloudflare-agent-extension.integration.test.ts +++ b/packages/runtime/test/cloudflare-agent-extension.integration.test.ts @@ -40,7 +40,8 @@ describe('Cloudflare agent extension', () => { headers: { 'content-type': 'application/json' }, body: JSON.stringify({ prompt: 'Hello' }), }); - expect(response.status).not.toBe(404); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ error: { type: 'invalid_request' } }); } finally { await server?.close(); fs.rmSync(root, { recursive: true, force: true }); @@ -57,7 +58,8 @@ describe('Cloudflare agent extension', () => { headers: { 'content-type': 'application/json' }, body: JSON.stringify({ prompt: 'Hello' }), }); - expect(response.status, await response.text()).not.toBe(500); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ error: { type: 'invalid_request' } }); const heartbeat = await fetch(new URL('/heartbeat', server.url)); expect(heartbeat.status).toBe(200); } finally { @@ -66,25 +68,56 @@ describe('Cloudflare agent extension', () => { } }, 90000); - it('bootstraps the inherited PartyServer name before accepting an agent WebSocket', async () => { - const root = await createGeneratedFixture(); + it('routes named generated Durable Objects across dispatch, workflow, and WebSocket transports', async () => { + const root = await createGeneratedFixture(defaultAgentSource, '', true); let server: Awaited> | undefined; - let socket: WebSocket | undefined; + let agentSocket: WebSocket | undefined; + let workflowSocket: WebSocket | undefined; try { server = await startServer(root); - const url = new URL('/agents/assistant/socket-instance', server.url); - url.protocol = 'ws:'; - socket = new WebSocket(url.toString()); - const firstMessage = await waitForSocketMessage(socket); - expect(JSON.parse(firstMessage)).toMatchObject({ + const agentSocketUrl = new URL('/agents/assistant/socket-instance', server.url); + agentSocketUrl.protocol = 'ws:'; + agentSocket = new WebSocket(agentSocketUrl.toString()); + const agentFirstMessage = await waitForSocketMessage(agentSocket); + expect(JSON.parse(agentFirstMessage)).toMatchObject({ version: 1, type: 'ready', target: 'agent', name: 'assistant', instanceId: 'socket-instance', }); + + const dispatchResponse = await fetch(new URL('/dispatch', server.url)); + const dispatchBody = await dispatchResponse.text(); + expect(dispatchResponse.status, dispatchBody).toBe(200); + expect(JSON.parse(dispatchBody)).toEqual({ + dispatchId: expect.any(String), + acceptedAt: expect.any(String), + }); + + const workflowResponse = await fetch(new URL('/workflows/smoke?wait=result', server.url), { + method: 'POST', + }); + const workflowBody = await workflowResponse.text(); + expect(workflowResponse.status, workflowBody).toBe(200); + expect(JSON.parse(workflowBody)).toEqual({ + result: { ok: true }, + _meta: { runId: expect.any(String) }, + }); + + const workflowSocketUrl = new URL('/workflows/smoke', server.url); + workflowSocketUrl.protocol = 'ws:'; + workflowSocket = new WebSocket(workflowSocketUrl.toString()); + const workflowFirstMessage = await waitForSocketMessage(workflowSocket); + expect(JSON.parse(workflowFirstMessage)).toEqual({ + version: 1, + type: 'ready', + target: 'workflow', + name: 'smoke', + }); } finally { - socket?.close(); + agentSocket?.close(); + workflowSocket?.close(); await server?.close(); fs.rmSync(root, { recursive: true, force: true }); } @@ -94,6 +127,7 @@ describe('Cloudflare agent extension', () => { async function createGeneratedFixture( agentSource = defaultAgentSource, mount = '', + withWorkflow = false, ): Promise { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'flue-cloudflare-agent-extension-')); const output = path.join(root, 'generated'); @@ -111,19 +145,35 @@ async function createGeneratedFixture( 'dir', ); fs.mkdirSync(path.join(root, 'src', 'agents'), { recursive: true }); + if (withWorkflow) fs.mkdirSync(path.join(root, 'src', 'workflows'), { recursive: true }); fs.writeFileSync( path.join(root, 'wrangler.jsonc'), JSON.stringify({ name: 'cloudflare-agent-extension', compatibility_date: '2026-04-01', compatibility_flags: ['nodejs_compat'], - migrations: [{ tag: 'v1', new_sqlite_classes: ['FlueAssistantAgent', 'FlueRegistry'] }], + migrations: [ + { + tag: 'v1', + new_sqlite_classes: [ + 'FlueAssistantAgent', + ...(withWorkflow ? ['FlueSmokeWorkflow'] : []), + 'FlueRegistry', + ], + }, + ], }), ); fs.writeFileSync(path.join(root, 'src', 'agents', 'assistant.ts'), agentSource); + if (withWorkflow) { + fs.writeFileSync( + path.join(root, 'src', 'workflows', 'smoke.ts'), + `export const route = async (_c, next) => next();\nexport const websocket = async (_c, next) => next();\nexport async function run() { return { ok: true }; }\n`, + ); + } fs.writeFileSync( path.join(root, 'src', 'app.ts'), - `import { Hono } from 'hono';\nimport { getAgentByName } from 'agents';\nimport { flue } from '@flue/runtime/routing';\nlet started = false;\nconst app = new Hono();\napp.route('${mount}', flue());\napp.get('${mount}/heartbeat', async (c) => {\n const agent = await getAgentByName(c.env.FLUE_ASSISTANT_AGENT, 'scheduled');\n if (!started) { await agent.startHeartbeat(); started = true; }\n return c.json({ count: await agent.getHeartbeatCount() });\n});\nexport default app;\n`, + `import { Hono } from 'hono';\nimport { getAgentByName } from 'agents';\nimport { dispatch } from '@flue/runtime';\nimport { flue } from '@flue/runtime/routing';\nlet started = false;\nconst app = new Hono();\napp.route('${mount}', flue());\napp.get('${mount}/heartbeat', async (c) => {\n const agent = await getAgentByName(c.env.FLUE_ASSISTANT_AGENT, 'scheduled');\n if (!started) { await agent.startHeartbeat(); started = true; }\n return c.json({ count: await agent.getHeartbeatCount() });\n});\napp.get('${mount}/dispatch', async (c) => c.json(await dispatch({ agent: 'assistant', id: 'dispatched', input: { text: 'Hello' } })));\nexport default app;\n`, ); try { await build({ From 5a393ad9505871fee0ac4e12b3216f3f4bf8c4f3 Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" <622227+FredKSchott@users.noreply.github.com> Date: Wed, 3 Jun 2026 12:59:59 -0500 Subject: [PATCH 03/17] feat(cloudflare): add agent execution store boundary --- .../cli/src/lib/build-plugin-cloudflare.ts | 79 +++++------ .../src/cloudflare/agent-execution-store.ts | 112 +++++++++++++++ packages/runtime/src/internal.ts | 5 + .../test/build-plugin-cloudflare.test.ts | 29 ++++ .../cloudflare-agent-execution-store.test.ts | 127 ++++++++++++++++++ 5 files changed, 314 insertions(+), 38 deletions(-) create mode 100644 packages/runtime/src/cloudflare/agent-execution-store.ts create mode 100644 packages/runtime/test/cloudflare-agent-execution-store.test.ts diff --git a/packages/cli/src/lib/build-plugin-cloudflare.ts b/packages/cli/src/lib/build-plugin-cloudflare.ts index f8efbccc..1d6ef4bb 100644 --- a/packages/cli/src/lib/build-plugin-cloudflare.ts +++ b/packages/cli/src/lib/build-plugin-cloudflare.ts @@ -51,6 +51,12 @@ export class CloudflarePlugin implements BuildPlugin { index, ) => `const agentExtension${index} = resolveCloudflareAgentExtension(agentModules[${JSON.stringify(agent.name)}], ${JSON.stringify(agent.name)}); const ${agentClassName(agent.name)} = class ${agentClassName(agent.name)} extends agentExtension${index}.base(Agent) { + constructor(ctx, env) { + const executionStore = createSqlAgentExecutionStore(ctx.storage, ${JSON.stringify(agentClassName(agent.name))}); + super(ctx, env); + this[FLUE_AGENT_EXECUTION_STORE] = executionStore; + } + async onRequest(request) { return dispatchAgent(request, this, ${JSON.stringify(agent.name)}, directHandlers[${JSON.stringify(agent.name)}]); } @@ -187,6 +193,8 @@ import { InMemorySessionStore, InMemoryRunStore, createDurableRunStore, + createSqlAgentExecutionStore, + createSqlSessionStore, createRunSubscriberRegistry, bashFactoryToSessionEnv, resolveModel, @@ -329,9 +337,9 @@ function resolveSandbox(sandbox) { return null; } -// Fallback in-memory store (used if no DO storage is available). -const memoryStore = new InMemorySessionStore(); +const memoryWorkflowSessionStore = new InMemorySessionStore(); const memoryRunStore = new InMemoryRunStore(); +const FLUE_AGENT_EXECUTION_STORE = Symbol('flueAgentExecutionStore'); const INTERNAL_DISPATCH_PATH = '/__flue/internal/dispatch'; const dispatchQueue = { async enqueue(input) { @@ -351,37 +359,13 @@ const dispatchQueue = { // Module-scoped per-isolate registry; run ids isolate buckets across DOs. const runSubscribers = createRunSubscriberRegistry(); -// Create a DO-backed session store from the Durable Object's SQL storage. -function createDOStore(sql) { - // Ensure the table exists - sql.exec( - 'CREATE TABLE IF NOT EXISTS flue_sessions (id TEXT PRIMARY KEY, data TEXT NOT NULL, updated_at INTEGER NOT NULL)' - ); - return { - async save(id, data) { - const json = JSON.stringify(data); - sql.exec( - 'INSERT OR REPLACE INTO flue_sessions (id, data, updated_at) VALUES (?, ?, ?)', - id, json, Date.now() - ); - }, - async load(id) { - const rows = sql.exec('SELECT data FROM flue_sessions WHERE id = ?', id).toArray(); - if (rows.length === 0) return null; - return JSON.parse(rows[0].data); - }, - async delete(id) { - sql.exec('DELETE FROM flue_sessions WHERE id = ?', id); - }, - }; +function getAgentExecutionStore(doInstance) { + const store = doInstance[FLUE_AGENT_EXECUTION_STORE]; + if (!store) throw new Error('[flue] Generated Cloudflare agent execution store was not initialized.'); + return store; } -function createContextForRequest(id, runId, payload, doInstance, req, initialEventIndex, dispatchId) { - // Use DO SQLite storage by default, fall back to in-memory - const defaultStore = doInstance?.ctx?.storage?.sql - ? createDOStore(doInstance.ctx.storage.sql) - : memoryStore; - +function createContextForRequest(id, runId, payload, doInstance, req, defaultStore, initialEventIndex, dispatchId) { return createFlueContext({ id, runId, @@ -399,6 +383,25 @@ function createContextForRequest(id, runId, payload, doInstance, req, initialEve }); } +function createAgentContextForRequest(id, payload, doInstance, req, initialEventIndex, dispatchId) { + return createContextForRequest( + id, + undefined, + payload, + doInstance, + req, + getAgentExecutionStore(doInstance).sessions, + initialEventIndex, + dispatchId, + ); +} + +function createWorkflowContextForRequest(id, runId, payload, doInstance, req, initialEventIndex, dispatchId) { + const sql = doInstance?.ctx?.storage?.sql; + const defaultStore = sql ? createSqlSessionStore(sql) : memoryWorkflowSessionStore; + return createContextForRequest(id, runId, payload, doInstance, req, defaultStore, initialEventIndex, dispatchId); +} + function createRunStoreForRequest(doInstance) { return doInstance?.ctx?.storage?.sql ? createDurableRunStore(doInstance.ctx.storage.sql) @@ -469,7 +472,7 @@ async function handleFlueDirectRecovered(ctx, doInstance, agentName) { assertAgentsDurabilityApi(doInstance, 'runFiber'); await doInstance.runFiber('flue:direct', async (fiberCtx) => { fiberCtx.stash({ payload }); - const directCtx = createContextForRequest(doInstance.name, undefined, payload, doInstance, request); + const directCtx = createAgentContextForRequest(doInstance.name, payload, doInstance, request); return runWithInstanceContext(doInstance, identity, () => handler(directCtx)); }); console.info('[flue:direct-recovery]', { agentName, instanceId: doInstance.name, operation: 'retry', outcome: 'restart_completed' }); @@ -491,7 +494,7 @@ async function handleFlueWorkflowFiberRecovered(ctx, doInstance, workflowName) { runStore, runSubscribers, runRegistry: createRunRegistryForRequest(doInstance.env), - createContext: (id_, recoveredRunId, payload, req, initialEventIndex) => createContextForRequest(id_, recoveredRunId, payload, doInstance, req, initialEventIndex), + createContext: (id_, recoveredRunId, payload, req, initialEventIndex) => createWorkflowContextForRequest(id_, recoveredRunId, payload, doInstance, req, initialEventIndex), }); } @@ -524,7 +527,7 @@ async function processManagedAgentDispatch(input, doInstance, agentName, fiberId const releaseSessionLock = await reserveDispatchAgentSession(target, input); const request = new Request('https://flue.invalid' + INTERNAL_DISPATCH_PATH, { method: 'POST' }); try { - const ctx = createContextForRequest(doInstance.name, undefined, input, doInstance, request, undefined, input.dispatchId); + const ctx = createAgentContextForRequest(doInstance.name, input, doInstance, request, undefined, input.dispatchId); await runWithInstanceContext(doInstance, agentRuntimeIdentity(agentName), () => createDispatchAgentHandler(agent, input)(ctx)); } finally { releaseSessionLock?.(); @@ -573,7 +576,7 @@ async function dispatchWorkflow(request, doInstance, workflowName) { runStore: createRunStoreForRequest(doInstance), runSubscribers, runRegistry: createRunRegistryForRequest(doInstance.env), - createContext: (id_, runId, payload, req, initialEventIndex, dispatchId) => createContextForRequest(id_, runId, payload, doInstance, req, initialEventIndex, dispatchId), + createContext: (id_, runId, payload, req, initialEventIndex, dispatchId) => createWorkflowContextForRequest(id_, runId, payload, doInstance, req, initialEventIndex, dispatchId), startWorkflowAdmission: (runId, run) => { assertAgentsDurabilityApi(doInstance, 'runFiber'); return doInstance.runFiber('flue:workflow:' + runId, () => runWithInstanceContext(doInstance, identity, run)); @@ -612,7 +615,7 @@ async function dispatchAgent(request, doInstance, agentName, handler) { agentName, id, handler, - createContext: (id_, runId, payload, req, initialEventIndex, dispatchId) => createContextForRequest(id_, runId, payload, doInstance, req, initialEventIndex, dispatchId), + createContext: (id_, runId, payload, req, initialEventIndex, dispatchId) => createAgentContextForRequest(id_, payload, doInstance, req, initialEventIndex, dispatchId), runHandler: (ctx, h) => { assertAgentsDurabilityApi(doInstance, 'runFiber'); return doInstance.runFiber('flue:direct', (fiberCtx) => { @@ -673,7 +676,7 @@ async function messageAgentSocket(connection, message, doInstance, agentName) { request: socketRequest(connection), handler, beforePrompt: (session) => assertNoPendingDispatchForDirectSession(doInstance, agentName, session), - createContext: (id_, runId, payload, req) => createContextForRequest(id_, runId, payload, doInstance, req), + createContext: (id_, runId, payload, req) => createAgentContextForRequest(id_, payload, doInstance, req), runHandler: (ctx, h) => { assertAgentsDurabilityApi(doInstance, 'runFiber'); return doInstance.runFiber('flue:direct', (fiberCtx) => { @@ -696,7 +699,7 @@ async function messageWorkflowSocket(connection, message, doInstance, workflowNa runStore: createRunStoreForRequest(doInstance), runSubscribers, runRegistry: createRunRegistryForRequest(doInstance.env), - createContext: (id_, runId, payload, req) => createContextForRequest(id_, runId, payload, doInstance, req), + createContext: (id_, runId, payload, req) => createWorkflowContextForRequest(id_, runId, payload, doInstance, req), startWorkflowAdmission: (runId, run) => { assertAgentsDurabilityApi(doInstance, 'runFiber'); return doInstance.runFiber('flue:workflow:' + runId, () => runWithInstanceContext(doInstance, identity, run)); diff --git a/packages/runtime/src/cloudflare/agent-execution-store.ts b/packages/runtime/src/cloudflare/agent-execution-store.ts new file mode 100644 index 00000000..949b985e --- /dev/null +++ b/packages/runtime/src/cloudflare/agent-execution-store.ts @@ -0,0 +1,112 @@ +import type { SessionData, SessionStore } from '../types.ts'; + +interface SqlResult { + toArray(): SqlRow[]; +} + +type SqlRow = Record; + +interface SqlStorage { + exec(query: string, ...bindings: unknown[]): SqlResult; +} + +interface DurableObjectStorage { + readonly sql?: SqlStorage; +} + +export interface SqlAgentExecutionStore { + readonly sessions: SessionStore; +} + +export function createSqlSessionStore(sql: SqlStorage): SessionStore { + ensureSessionTable(sql); + return new SqlSessionStore(sql); +} + +export function createSqlAgentExecutionStore( + storage: DurableObjectStorage | undefined, + className: string, +): SqlAgentExecutionStore { + const sql = storage?.sql; + if (!sql || typeof sql.exec !== 'function') { + throw new Error( + `[flue] Cloudflare durable agent class "${className}" requires Durable Object SQLite. ` + + `Add "${className}" to a Wrangler migration's "new_sqlite_classes" list before its first deploy; ` + + `do not use legacy "new_classes". Existing KV-backed Durable Object classes cannot be converted ` + + `to SQLite in place.`, + ); + } + try { + const sessions = createSqlSessionStore(sql); + ensureSubmissionTable(sql); + return { sessions }; + } catch (cause) { + const detail = cause instanceof Error ? cause.message : String(cause); + throw new Error( + `[flue] Cloudflare durable agent class "${className}" could not initialize its SQLite execution store. ` + + `Underlying error: ${detail}`, + { cause }, + ); + } +} + +class SqlSessionStore implements SessionStore { + constructor(private sql: SqlStorage) {} + + async save(id: string, data: SessionData): Promise { + this.sql.exec( + 'INSERT OR REPLACE INTO flue_sessions (id, data, updated_at) VALUES (?, ?, ?)', + id, + JSON.stringify(data), + Date.now(), + ); + } + + async load(id: string): Promise { + const rows = this.sql.exec('SELECT data FROM flue_sessions WHERE id = ?', id).toArray(); + const row = rows[0]; + if (!row) return null; + if (typeof row.data !== 'string') throw new Error('[flue] Persisted session row is malformed.'); + return JSON.parse(row.data) as SessionData; + } + + async delete(id: string): Promise { + this.sql.exec('DELETE FROM flue_sessions WHERE id = ?', id); + } +} + +function ensureSessionTable(sql: SqlStorage): void { + sql.exec( + `CREATE TABLE IF NOT EXISTS flue_sessions ( + id TEXT PRIMARY KEY, + data TEXT NOT NULL, + updated_at INTEGER NOT NULL + )`, + ); +} + +function ensureSubmissionTable(sql: SqlStorage): void { + sql.exec( + `CREATE TABLE IF NOT EXISTS flue_agent_submissions ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + submission_id TEXT NOT NULL UNIQUE, + session TEXT NOT NULL, + session_key TEXT NOT NULL, + kind TEXT NOT NULL, + payload TEXT NOT NULL, + status TEXT NOT NULL, + accepted_at INTEGER NOT NULL, + attempt_id TEXT, + input_applied_at INTEGER, + started_at INTEGER, + completed_at INTEGER, + error TEXT + )`, + ); + sql.exec( + 'CREATE INDEX IF NOT EXISTS flue_agent_submissions_status_sequence_idx ON flue_agent_submissions (status, sequence ASC)', + ); + sql.exec( + 'CREATE INDEX IF NOT EXISTS flue_agent_submissions_session_status_sequence_idx ON flue_agent_submissions (session_key, status, sequence ASC)', + ); +} diff --git a/packages/runtime/src/internal.ts b/packages/runtime/src/internal.ts index 4858c754..3145873c 100644 --- a/packages/runtime/src/internal.ts +++ b/packages/runtime/src/internal.ts @@ -21,6 +21,11 @@ export { createFlueContext } from './client.ts'; // (registry client) live in the `@flue/runtime/cloudflare` subpath because // they pull in `cloudflare:workers`, a virtual module Node can't resolve. // The generated CF entry imports them from there directly. +export type { SqlAgentExecutionStore } from './cloudflare/agent-execution-store.ts'; +export { + createSqlAgentExecutionStore, + createSqlSessionStore, +} from './cloudflare/agent-execution-store.ts'; export { createDurableRunStore } from './cloudflare/run-store.ts'; export { InMemoryRunRegistry } from './node/run-registry.ts'; export { InMemoryRunStore } from './node/run-store.ts'; diff --git a/packages/runtime/test/build-plugin-cloudflare.test.ts b/packages/runtime/test/build-plugin-cloudflare.test.ts index e04b4c51..533e381a 100644 --- a/packages/runtime/test/build-plugin-cloudflare.test.ts +++ b/packages/runtime/test/build-plugin-cloudflare.test.ts @@ -17,6 +17,35 @@ describe('CloudflarePlugin', () => { expect(entry).toContain('bindingName: "FLUE_DRAFT_WORKFLOW"'); }); + it('initializes durable agent execution stores without changing workflow run-store behavior', async () => { + const entry = await new CloudflarePlugin().generateEntryPoint( + testBuildContext({ + agents: [{ name: 'assistant', filePath: '/fixture/agents/assistant.ts' }], + workflows: [{ name: 'draft', filePath: '/fixture/workflows/draft.ts' }], + }), + ); + + expect(entry).toContain('createSqlAgentExecutionStore'); + expect(entry).toContain('createSqlSessionStore'); + expect(entry).toContain( + `constructor(ctx, env) { + const executionStore = createSqlAgentExecutionStore(ctx.storage, "FlueAssistantAgent"); + super(ctx, env); + this[FLUE_AGENT_EXECUTION_STORE] = executionStore; + }`, + ); + expect(entry).not.toContain('const agentExecutionStores = new WeakMap();'); + expect(entry).toContain('const memoryWorkflowSessionStore = new InMemorySessionStore();'); + expect(entry).toContain( + 'const defaultStore = sql ? createSqlSessionStore(sql) : memoryWorkflowSessionStore;', + ); + expect(entry).toContain('createDurableRunStore(doInstance.ctx.storage.sql)'); + expect(entry).toContain(': memoryRunStore;'); + expect(entry).not.toContain('function createDOStore(sql)'); + expect(entry).not.toContain('const memoryStore = new InMemorySessionStore();'); + expect(entry).not.toContain('CREATE TABLE IF NOT EXISTS flue_sessions'); + }); + it('uses explicit Flue routing instead of the Agents SDK router', async () => { const entry = await new CloudflarePlugin().generateEntryPoint( testBuildContext({ diff --git a/packages/runtime/test/cloudflare-agent-execution-store.test.ts b/packages/runtime/test/cloudflare-agent-execution-store.test.ts new file mode 100644 index 00000000..2cded226 --- /dev/null +++ b/packages/runtime/test/cloudflare-agent-execution-store.test.ts @@ -0,0 +1,127 @@ +import { DatabaseSync } from 'node:sqlite'; +import { describe, expect, it } from 'vitest'; +import { + createSqlAgentExecutionStore, + createSqlSessionStore, +} from '../src/cloudflare/agent-execution-store.ts'; +import type { SessionData } from '../src/types.ts'; + +function makeFakeSql() { + const db = new DatabaseSync(':memory:'); + return { + db, + sql: { + exec(query: string, ...bindings: unknown[]) { + const stmt = db.prepare(query); + let rows: unknown[]; + try { + rows = stmt.all(...(bindings as never[])); + } catch { + stmt.run(...(bindings as never[])); + rows = []; + } + return { + toArray() { + return rows as Record[]; + }, + }; + }, + }, + }; +} + +function sessionData(): SessionData { + return { + version: 5, + affinityKey: 'affinity-1', + entries: [], + leafId: null, + metadata: {}, + createdAt: '2026-06-03T00:00:00.000Z', + updatedAt: '2026-06-03T00:00:00.000Z', + }; +} + +describe('createSqlAgentExecutionStore()', () => { + it('loads, saves, and deletes existing flue_sessions rows when SQLite snapshot persistence is initialized', async () => { + const { db, sql } = makeFakeSql(); + db.exec( + 'CREATE TABLE flue_sessions (id TEXT PRIMARY KEY, data TEXT NOT NULL, updated_at INTEGER NOT NULL)', + ); + db.prepare('INSERT INTO flue_sessions (id, data, updated_at) VALUES (?, ?, ?)').run( + 'existing', + JSON.stringify(sessionData()), + 1, + ); + + const store = createSqlAgentExecutionStore({ sql }, 'FlueAssistantAgent'); + + expect(await store.sessions.load('existing')).toEqual(sessionData()); + await store.sessions.save('saved', sessionData()); + expect(await store.sessions.load('saved')).toEqual(sessionData()); + await store.sessions.delete('existing'); + expect(await store.sessions.load('existing')).toBeNull(); + }); + + it('creates the initial flue_agent_submissions schema and ordering indexes when initialized', () => { + const { db, sql } = makeFakeSql(); + + createSqlAgentExecutionStore({ sql }, 'FlueAssistantAgent'); + + expect( + db.prepare("SELECT name FROM pragma_table_info('flue_agent_submissions') ORDER BY cid").all(), + ).toEqual([ + { name: 'sequence' }, + { name: 'submission_id' }, + { name: 'session' }, + { name: 'session_key' }, + { name: 'kind' }, + { name: 'payload' }, + { name: 'status' }, + { name: 'accepted_at' }, + { name: 'attempt_id' }, + { name: 'input_applied_at' }, + { name: 'started_at' }, + { name: 'completed_at' }, + { name: 'error' }, + ]); + expect( + db + .prepare( + "SELECT name FROM sqlite_schema WHERE type = 'index' AND tbl_name = 'flue_agent_submissions' ORDER BY name", + ) + .all(), + ).toEqual([ + { name: 'flue_agent_submissions_session_status_sequence_idx' }, + { name: 'flue_agent_submissions_status_sequence_idx' }, + { name: 'sqlite_autoindex_flue_agent_submissions_1' }, + ]); + }); + + it('rejects missing Durable Object SQLite with migration guidance', () => { + expect(() => createSqlAgentExecutionStore({}, 'FlueAssistantAgent')).toThrow( + 'Add "FlueAssistantAgent" to a Wrangler migration\'s "new_sqlite_classes" list before its first deploy; do not use legacy "new_classes". Existing KV-backed Durable Object classes cannot be converted to SQLite in place.', + ); + }); + + it('reports SQL initialization failures without misdiagnosing missing SQLite', () => { + const { sql } = makeFakeSql(); + sql.exec('CREATE TABLE flue_agent_submissions (sequence INTEGER PRIMARY KEY AUTOINCREMENT)'); + + expect(() => createSqlAgentExecutionStore({ sql }, 'FlueAssistantAgent')).toThrow( + '[flue] Cloudflare durable agent class "FlueAssistantAgent" could not initialize its SQLite execution store. Underlying error: no such column: status', + ); + }); +}); + +describe('createSqlSessionStore()', () => { + it('creates only flue_sessions when workflow-compatible snapshot persistence is initialized', () => { + const { db, sql } = makeFakeSql(); + + createSqlSessionStore(sql); + + expect( + db.prepare("SELECT name FROM sqlite_schema WHERE type = 'table' ORDER BY name").all(), + ).toEqual([{ name: 'flue_sessions' }]); + }); +}); From 6353f94eb23ed7abe5ce1265ed886578447f8c7f Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" <622227+FredKSchott@users.noreply.github.com> Date: Wed, 3 Jun 2026 14:50:42 -0500 Subject: [PATCH 04/17] feat(cloudflare): add durable dispatch admission bridge --- examples/assistant/package.json | 2 +- examples/chat-sdk/package.json | 2 +- examples/cloudflare-websocket/package.json | 2 +- examples/cloudflare/package.json | 2 +- examples/hello-world/package.json | 2 +- examples/imported-skill/package.json | 2 +- .../cli/src/lib/build-plugin-cloudflare.ts | 206 ++++++++++++--- packages/cli/src/lib/build.ts | 39 ++- .../src/cloudflare/agent-execution-store.ts | 245 +++++++++++++++++- packages/runtime/src/internal.ts | 7 +- .../test/build-plugin-cloudflare.test.ts | 27 ++ .../cloudflare-agent-execution-store.test.ts | 105 ++++++++ ...dflare-agent-extension.integration.test.ts | 49 +++- pnpm-lock.yaml | 30 +-- pnpm-workspace.yaml | 1 + 15 files changed, 665 insertions(+), 56 deletions(-) diff --git a/examples/assistant/package.json b/examples/assistant/package.json index 26c26dec..99cc9b26 100644 --- a/examples/assistant/package.json +++ b/examples/assistant/package.json @@ -5,7 +5,7 @@ "dependencies": { "@cloudflare/sandbox": "^0.11.0", "@flue/runtime": "workspace:*", - "agents": "^0.14.0" + "agents": "^0.14.1" }, "devDependencies": { "@flue/cli": "workspace:*" diff --git a/examples/chat-sdk/package.json b/examples/chat-sdk/package.json index cdbaaadc..306e63b5 100644 --- a/examples/chat-sdk/package.json +++ b/examples/chat-sdk/package.json @@ -10,7 +10,7 @@ "@chat-adapter/state-memory": "^4.29.0", "@earendil-works/pi-ai": "*", "@flue/runtime": "workspace:*", - "agents": "^0.14.0", + "agents": "^0.14.1", "chat": "^4.29.0", "hono": "^4.8.3", "just-bash": "^3.0.1" diff --git a/examples/cloudflare-websocket/package.json b/examples/cloudflare-websocket/package.json index 7a394c5a..36eb602e 100644 --- a/examples/cloudflare-websocket/package.json +++ b/examples/cloudflare-websocket/package.json @@ -7,7 +7,7 @@ }, "dependencies": { "@flue/runtime": "workspace:*", - "agents": "^0.14.0", + "agents": "^0.14.1", "hono": "^4.7.0" }, "devDependencies": { diff --git a/examples/cloudflare/package.json b/examples/cloudflare/package.json index 8830c673..86fe1a02 100644 --- a/examples/cloudflare/package.json +++ b/examples/cloudflare/package.json @@ -6,7 +6,7 @@ "@cloudflare/codemode": "^0.3.8", "@cloudflare/shell": "^0.3.8", "@flue/runtime": "workspace:*", - "agents": "^0.14.0", + "agents": "^0.14.1", "hono": "^4.7.0", "valibot": "^1.0.0" }, diff --git a/examples/hello-world/package.json b/examples/hello-world/package.json index 6b375a87..a8da8c70 100644 --- a/examples/hello-world/package.json +++ b/examples/hello-world/package.json @@ -5,7 +5,7 @@ "dependencies": { "@daytona/sdk": "*", "@flue/runtime": "workspace:*", - "agents": "^0.14.0", + "agents": "^0.14.1", "hono": "^4.7.0", "just-bash": "^3.0.1", "valibot": "^1.0.0" diff --git a/examples/imported-skill/package.json b/examples/imported-skill/package.json index d106ee6b..fb7bea73 100644 --- a/examples/imported-skill/package.json +++ b/examples/imported-skill/package.json @@ -4,7 +4,7 @@ "type": "module", "dependencies": { "@flue/runtime": "workspace:*", - "agents": "^0.14.0", + "agents": "^0.14.1", "just-bash": "^3.0.1" }, "devDependencies": { diff --git a/packages/cli/src/lib/build-plugin-cloudflare.ts b/packages/cli/src/lib/build-plugin-cloudflare.ts index 1d6ef4bb..c718c56c 100644 --- a/packages/cli/src/lib/build-plugin-cloudflare.ts +++ b/packages/cli/src/lib/build-plugin-cloudflare.ts @@ -57,6 +57,15 @@ const ${agentClassName(agent.name)} = class ${agentClassName(agent.name)} extend this[FLUE_AGENT_EXECUTION_STORE] = executionStore; } + async onStart(props) { + if (typeof super.onStart === 'function') await super.onStart(props); + await reconcileFlueAgentSubmissions(this, ${JSON.stringify(agent.name)}, { preserveSuccessor: true }); + } + + async __flueWakeAgentSubmissions(wake) { + await reconcileFlueAgentSubmissions(this, ${JSON.stringify(agent.name)}, { preserveSuccessor: true, executingWake: wake }); + } + async onRequest(request) { return dispatchAgent(request, this, ${JSON.stringify(agent.name)}, directHandlers[${JSON.stringify(agent.name)}]); } @@ -195,6 +204,7 @@ import { createDurableRunStore, createSqlAgentExecutionStore, createSqlSessionStore, + SqlAgentSubmissionConflictError, createRunSubscriberRegistry, bashFactoryToSessionEnv, resolveModel, @@ -340,6 +350,8 @@ function resolveSandbox(sandbox) { const memoryWorkflowSessionStore = new InMemorySessionStore(); const memoryRunStore = new InMemoryRunStore(); const FLUE_AGENT_EXECUTION_STORE = Symbol('flueAgentExecutionStore'); +const FLUE_AGENT_SUBMISSION_WAKE_CALLBACK = '__flueWakeAgentSubmissions'; +const FLUE_AGENT_SUBMISSION_RETRY_SECONDS = 30; const INTERNAL_DISPATCH_PATH = '/__flue/internal/dispatch'; const dispatchQueue = { async enqueue(input) { @@ -451,8 +463,11 @@ async function handleFlueDispatchRecovered(ctx, doInstance, agentName) { const input = ctx.metadata?.input; assertCurrentDispatchInput(input); if (!input || input.agent !== agentName || input.id !== doInstance.name) return { status: 'error', error: 'Dispatch recovery metadata is invalid.' }; + const submissions = getAgentExecutionStore(doInstance).submissions; + const submission = submissions.getDispatch(input.dispatchId); + if (submission && submission.status !== 'queued') return { status: submission.status === 'completed' ? 'completed' : 'error', error: submission.error }; try { - await processManagedAgentDispatch(input, doInstance, agentName, ctx.id); + await processManagedAgentDispatch(submission ?? { input }, doInstance, agentName, ctx.id); return { status: 'completed' }; } catch (error) { return { status: 'error', error: error instanceof Error ? error.message : String(error) }; @@ -500,52 +515,184 @@ async function handleFlueWorkflowFiberRecovered(ctx, doInstance, workflowName) { // ─── Per-DO Dispatch ─────────────────────────────────────────────────────── -async function waitForEarlierManagedDispatch(doInstance, input, fiberId) { - if (typeof doInstance.listFibers !== 'function') return; +function armFlueAgentSubmissionWake(doInstance, options = {}) { + assertAgentsDurabilityApi(doInstance, 'schedule'); + return doInstance.schedule( + options.delaySeconds ?? 0, + FLUE_AGENT_SUBMISSION_WAKE_CALLBACK, + { generation: options.generation ?? 0 }, + { idempotent: true }, + ); +} + +function armFlueAgentSubmissionAdmissionWakes(doInstance) { + return Promise.all([ + armFlueAgentSubmissionWake(doInstance, { generation: 0 }), + armFlueAgentSubmissionWake(doInstance, { generation: 1 }), + ]); +} + +async function reconcileFlueAgentSubmissions(doInstance, agentName, options = {}) { + const submissions = getAgentExecutionStore(doInstance).submissions; + if (!submissions.hasQueuedDispatches()) return; + if (options.preserveSuccessor) { + if (options.executingWake) { + await armFlueAgentSubmissionWake(doInstance, { + delaySeconds: FLUE_AGENT_SUBMISSION_RETRY_SECONDS, + generation: options.executingWake.generation === 0 ? 1 : 0, + }); + } else { + await Promise.all([ + armFlueAgentSubmissionWake(doInstance, { delaySeconds: FLUE_AGENT_SUBMISSION_RETRY_SECONDS, generation: 0 }), + armFlueAgentSubmissionWake(doInstance, { delaySeconds: FLUE_AGENT_SUBMISSION_RETRY_SECONDS, generation: 1 }), + ]); + } + } + try { + while (true) { + const runnable = submissions.listRunnableDispatches(); + let repairedTerminal = false; + for (const submission of runnable) { + if (await ensureManagedDispatchFiber(submission, doInstance, agentName)) repairedTerminal = true; + } + if (!repairedTerminal) break; + } + } catch (error) { + console.error('[flue:dispatch-reconciliation]', { agentName, instanceId: doInstance.name, operation: 'reconcile', outcome: 'deferred_to_scheduled_wake' }, error); + return; + } +} + +async function ensureManagedDispatchFiber(submission, doInstance, agentName) { + assertAgentsDurabilityApi(doInstance, 'inspectFiberByKey'); + assertAgentsDurabilityApi(doInstance, 'startFiber'); + const idempotencyKey = 'flue:dispatch:' + submission.submissionId; + const prior = await doInstance.inspectFiberByKey(idempotencyKey); + if (prior) { + try { + await validateAgentDispatchAdmission({ input: prior.metadata?.input }); + } catch { + getAgentExecutionStore(doInstance).submissions.failDispatch(submission.submissionId, new Error('[flue] Persisted dispatch Fiber metadata is malformed.')); + return true; + } + if (JSON.stringify(prior.metadata.input) !== JSON.stringify(submission.input)) { + getAgentExecutionStore(doInstance).submissions.failDispatch(submission.submissionId, new Error('[flue] Persisted dispatch Fiber conflicts with SQL submission.')); + return true; + } + } + if (prior?.status === 'completed') { + getAgentExecutionStore(doInstance).submissions.completeDispatch(submission.submissionId); + return true; + } + if (prior?.status === 'error' || prior?.status === 'aborted' || prior?.status === 'interrupted') { + getAgentExecutionStore(doInstance).submissions.failDispatch(submission.submissionId, prior.error ?? 'Managed dispatch Fiber did not complete.'); + return true; + } + if (prior) return false; + await doInstance.startFiber('flue:dispatch', async (fiberCtx) => processManagedAgentDispatch(submission, doInstance, agentName, fiberCtx.id), { + idempotencyKey, + metadata: { input: submission.input, submissionSequence: submission.sequence }, + }); + return false; +} + +function isPersistedDispatchInput(input) { + return !!input && typeof input === 'object' && !Array.isArray(input) && + typeof input.dispatchId === 'string' && input.dispatchId.trim() !== '' && + typeof input.agent === 'string' && input.agent.trim() !== '' && + typeof input.id === 'string' && input.id.trim() !== '' && + typeof input.session === 'string' && input.session.trim() !== '' && + input.input !== undefined && typeof input.acceptedAt === 'string' && input.acceptedAt.trim() !== ''; +} + +function listActiveLegacyManagedDispatches(doInstance) { + const rows = doInstance.ctx.storage.sql.exec( + 'SELECT fiber_id, metadata_json, created_at FROM cf_agents_fibers ' + + "WHERE name = 'flue:dispatch' AND (status IN ('pending', 'running') OR " + + "(status = 'interrupted' AND EXISTS (SELECT 1 FROM cf_agents_runs WHERE id = fiber_id))) " + + 'ORDER BY created_at ASC, fiber_id ASC', + ).toArray(); + return rows.flatMap((row) => { + if (typeof row.fiber_id !== 'string' || typeof row.created_at !== 'number') { + throw new Error('[flue] Persisted dispatch Fiber row is malformed.'); + } + let metadata; + try { + metadata = typeof row.metadata_json === 'string' ? JSON.parse(row.metadata_json) : null; + } catch { + throw new Error('[flue] Persisted dispatch Fiber metadata is malformed.'); + } + if (typeof metadata?.submissionSequence === 'number') return []; + const input = metadata?.input; + assertCurrentDispatchInput(input); + if (!isPersistedDispatchInput(input)) throw new Error('[flue] Persisted dispatch Fiber metadata is malformed.'); + return [{ fiberId: row.fiber_id, createdAt: row.created_at, input }]; + }); +} + +async function waitForEarlierLegacyManagedDispatch(doInstance, input, fiberId) { while (true) { - const fibers = await doInstance.listFibers({ name: 'flue:dispatch' }); - for (const fiber of fibers) assertCurrentDispatchInput(fiber.metadata?.input); - const current = fibers.find((fiber) => fiber.id === fiberId); - if (!current) return; + const fibers = listActiveLegacyManagedDispatches(doInstance); + const current = fibers.find((fiber) => fiber.fiberId === fiberId); const blocked = fibers.some((fiber) => { - if (fiber.id === fiberId || fiber.status === 'completed' || fiber.status === 'error' || fiber.status === 'aborted') return false; - const other = fiber.metadata?.input; - if (!other || other.agent !== input.agent || other.id !== input.id || other.session !== input.session) return false; - return fiber.createdAt < current.createdAt || (fiber.createdAt === current.createdAt && fiber.id < fiberId); + if (fiber.fiberId === fiberId) return false; + const other = fiber.input; + if (other.agent !== input.agent || other.id !== input.id || other.session !== input.session) return false; + if (!current) return true; + return fiber.createdAt < current.createdAt || (fiber.createdAt === current.createdAt && fiber.fiberId < current.fiberId); }); if (!blocked) return; await new Promise((resolve) => setTimeout(resolve, 0)); } } -async function processManagedAgentDispatch(input, doInstance, agentName, fiberId) { +async function processManagedAgentDispatch(submission, doInstance, agentName, fiberId) { + const input = submission.input; + const submissions = getAgentExecutionStore(doInstance).submissions; + const persisted = submissions.getDispatch(input.dispatchId); + if (persisted && persisted.status !== 'queued') return; const agent = createdAgents[agentName]; if (!agent) throw new Error('[flue] Dispatch target unavailable during durable processing.'); await validateAgentDispatchAdmission({ input }); const target = { agentName, instanceId: doInstance.name }; - await waitForEarlierManagedDispatch(doInstance, input, fiberId); + if (typeof submission.sequence === 'number') { + while (submissions.hasEarlierQueuedDispatch(submission)) await new Promise((resolve) => setTimeout(resolve, 0)); + } + await waitForEarlierLegacyManagedDispatch(doInstance, input, fiberId); const releaseSessionLock = await reserveDispatchAgentSession(target, input); const request = new Request('https://flue.invalid' + INTERNAL_DISPATCH_PATH, { method: 'POST' }); try { const ctx = createAgentContextForRequest(doInstance.name, input, doInstance, request, undefined, input.dispatchId); await runWithInstanceContext(doInstance, agentRuntimeIdentity(agentName), () => createDispatchAgentHandler(agent, input)(ctx)); + if (persisted) submissions.completeDispatch(input.dispatchId); + } catch (error) { + if (persisted) submissions.failDispatch(input.dispatchId, error); + throw error; } finally { releaseSessionLock?.(); + if (persisted) void reconcileFlueAgentSubmissions(doInstance, agentName, { preserveSuccessor: true }).catch((error) => { + console.error('[flue:dispatch-reconciliation]', { agentName, instanceId: doInstance.name, operation: 'settlement', outcome: 'reconcile_failed' }, error); + }); } } -async function assertNoPendingDispatchForDirectSession(doInstance, agentName, session) { - if (typeof doInstance.listFibers !== 'function') return; - const fibers = await doInstance.listFibers({ name: 'flue:dispatch' }); - for (const fiber of fibers) assertCurrentDispatchInput(fiber.metadata?.input); +async function assertNoLegacyPendingDispatchFiberForDirectSession(doInstance, agentName, session) { + const fibers = listActiveLegacyManagedDispatches(doInstance); if (fibers.some((fiber) => { - const input = fiber.metadata?.input; - return fiber.status !== 'completed' && fiber.status !== 'error' && fiber.status !== 'aborted' && input?.agent === agentName && input.id === doInstance.name && input.session === session; + const input = fiber.input; + return input.agent === agentName && input.id === doInstance.name && input.session === session; })) { throw new Error('[flue] This agent session has pending dispatched input and cannot accept direct input yet.'); } } +async function assertNoPendingDispatchForDirectSession(doInstance, agentName, session) { + if (getAgentExecutionStore(doInstance).submissions.hasQueuedDispatchForSession(doInstance.name, session)) { + throw new Error('[flue] This agent session has pending dispatched input and cannot accept direct input yet.'); + } + await assertNoLegacyPendingDispatchFiberForDirectSession(doInstance, agentName, session); +} + async function dispatchWorkflow(request, doInstance, workflowName) { // The DO room name is the workflow instance id. For workflows that // equals the run id (one run per instance), so callers reach this DO @@ -592,19 +739,16 @@ async function dispatchAgent(request, doInstance, agentName, handler) { if (input.agent !== agentName || input.id !== id) return new Response('Invalid internal dispatch target.', { status: 400 }); if (!createdAgents[agentName]) return new Response('Dispatch target unavailable.', { status: 404 }); await validateAgentDispatchAdmission({ input }); - assertAgentsDurabilityApi(doInstance, 'startFiber'); - assertAgentsDurabilityApi(doInstance, 'inspectFiberByKey'); - const idempotencyKey = 'flue:dispatch:' + input.dispatchId; - const prior = await doInstance.inspectFiberByKey(idempotencyKey); - assertCurrentDispatchInput(prior?.metadata?.input); - if (prior?.metadata?.input && JSON.stringify(prior.metadata.input) !== JSON.stringify(input)) { - return new Response('Conflicting internal dispatch replay.', { status: 409 }); + await armFlueAgentSubmissionAdmissionWakes(doInstance); + let submission; + try { + submission = getAgentExecutionStore(doInstance).submissions.admitDispatch(input); + } catch (error) { + if (error instanceof SqlAgentSubmissionConflictError) return new Response('Conflicting internal dispatch replay.', { status: 409 }); + throw error; } - await doInstance.startFiber('flue:dispatch', async (fiberCtx) => processManagedAgentDispatch(input, doInstance, agentName, fiberCtx.id), { - idempotencyKey, - metadata: { input }, - }); - return Response.json({ dispatchId: input.dispatchId, acceptedAt: input.acceptedAt }); + await reconcileFlueAgentSubmissions(doInstance, agentName); + return Response.json({ dispatchId: submission.submissionId, acceptedAt: submission.input.acceptedAt }); } const payload = await request.clone().json().catch(() => null); const session = typeof payload?.session === 'string' && payload.session.trim() !== '' ? payload.session : 'default'; diff --git a/packages/cli/src/lib/build.ts b/packages/cli/src/lib/build.ts index 958d5be5..d42ca4b5 100644 --- a/packages/cli/src/lib/build.ts +++ b/packages/cli/src/lib/build.ts @@ -40,7 +40,8 @@ export async function build(options: BuildOptions): Promise { async function buildApplication(options: BuildOptions): Promise { const root = path.resolve(options.root); const output = path.resolve(options.output ?? path.join(root, 'dist')); - const plugin = resolvePlugin(options); + const plugin: BuildPlugin = resolvePlugin(options); + if (!options.plugin && options.target === 'cloudflare') assertCloudflareAgentsSdkFloor(root); const sourceRoot = path.resolve(options.sourceRoot); @@ -429,6 +430,42 @@ function readRuntimeVersion(root: string): string { } } +function assertCloudflareAgentsSdkFloor(root: string): void { + const minimum = [0, 14, 1] as const; + let entry: string; + try { + entry = createRequire(path.join(root, '__flue_resolve__.cjs')).resolve('agents'); + } catch { + throw new Error( + '[flue] Cloudflare target requires the installed "agents" package to be at least 0.14.1. Install or upgrade "agents" in this project.', + ); + } + const pkgPath = packageUpSync({ cwd: path.dirname(entry) }); + if (!pkgPath) throw new Error('[flue] Could not inspect the installed "agents" package version.'); + let version: unknown; + try { + version = JSON.parse(fs.readFileSync(pkgPath, 'utf-8')).version; + } catch { + throw new Error('[flue] Could not inspect the installed "agents" package version.'); + } + const match = typeof version === 'string' ? /^(\d+)\.(\d+)\.(\d+)$/.exec(version) : null; + const current = match?.slice(1).map(Number); + if (!current || isVersionBelow(current, minimum)) { + throw new Error( + `[flue] Cloudflare target requires the installed "agents" package to be at least 0.14.1. Found ${String(version)}. Install or upgrade "agents" in this project.`, + ); + } +} + +function isVersionBelow(current: readonly number[], minimum: readonly number[]): boolean { + for (let index = 0; index < minimum.length; index++) { + const currentPart = current[index] ?? 0; + const minimumPart = minimum[index] ?? 0; + if (currentPart !== minimumPart) return currentPart < minimumPart; + } + return false; +} + function getCLIDir(): string { try { return path.dirname(new URL(import.meta.url).pathname); diff --git a/packages/runtime/src/cloudflare/agent-execution-store.ts b/packages/runtime/src/cloudflare/agent-execution-store.ts index 949b985e..fee462a4 100644 --- a/packages/runtime/src/cloudflare/agent-execution-store.ts +++ b/packages/runtime/src/cloudflare/agent-execution-store.ts @@ -1,3 +1,5 @@ +import type { DispatchInput } from '../runtime/dispatch-queue.ts'; +import { createSessionStorageKey } from '../session-identity.ts'; import type { SessionData, SessionStore } from '../types.ts'; interface SqlResult { @@ -14,10 +16,39 @@ interface DurableObjectStorage { readonly sql?: SqlStorage; } +type SqlAgentSubmissionStatus = 'queued' | 'completed' | 'error'; + +export interface SqlAgentDispatchSubmission { + readonly sequence: number; + readonly submissionId: string; + readonly session: string; + readonly sessionKey: string; + readonly input: DispatchInput; + readonly status: SqlAgentSubmissionStatus; + readonly acceptedAt: number; + readonly completedAt?: number; + readonly error?: string; +} + +export interface SqlAgentSubmissionStore { + getDispatch(submissionId: string): SqlAgentDispatchSubmission | null; + admitDispatch(input: DispatchInput): SqlAgentDispatchSubmission; + hasQueuedDispatches(): boolean; + listQueuedDispatches(): SqlAgentDispatchSubmission[]; + listRunnableDispatches(): SqlAgentDispatchSubmission[]; + hasQueuedDispatchForSession(instanceId: string, session: string): boolean; + hasEarlierQueuedDispatch(submission: SqlAgentDispatchSubmission): boolean; + completeDispatch(submissionId: string): void; + failDispatch(submissionId: string, error: unknown): void; +} + export interface SqlAgentExecutionStore { readonly sessions: SessionStore; + readonly submissions: SqlAgentSubmissionStore; } +export class SqlAgentSubmissionConflictError extends Error {} + export function createSqlSessionStore(sql: SqlStorage): SessionStore { ensureSessionTable(sql); return new SqlSessionStore(sql); @@ -39,7 +70,7 @@ export function createSqlAgentExecutionStore( try { const sessions = createSqlSessionStore(sql); ensureSubmissionTable(sql); - return { sessions }; + return { sessions, submissions: new SqlAgentSubmissionStoreImpl(sql) }; } catch (cause) { const detail = cause instanceof Error ? cause.message : String(cause); throw new Error( @@ -75,6 +106,218 @@ class SqlSessionStore implements SessionStore { } } +class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { + constructor(private sql: SqlStorage) {} + + getDispatch(submissionId: string): SqlAgentDispatchSubmission | null { + const row = this.readDispatchRow(submissionId); + return row ? parseDispatchSubmission(row) : null; + } + + admitDispatch(input: DispatchInput): SqlAgentDispatchSubmission { + const payload = JSON.stringify(input); + const acceptedAt = Date.parse(input.acceptedAt); + if (!Number.isFinite(acceptedAt)) { + throw new Error('[flue] Internal dispatch admission received an invalid acceptedAt timestamp.'); + } + this.sql.exec( + `INSERT OR IGNORE INTO flue_agent_submissions + (submission_id, session, session_key, kind, payload, status, accepted_at) + VALUES (?, ?, ?, 'dispatch', ?, 'queued', ?)`, + input.dispatchId, + input.session, + createSessionStorageKey(input.id, 'default', input.session), + payload, + acceptedAt, + ); + const row = this.readDispatchRow(input.dispatchId); + if (!row) throw new Error('[flue] Durable dispatch admission did not create a submission row.'); + if (row.payload !== payload) { + throw new SqlAgentSubmissionConflictError('[flue] Conflicting internal dispatch replay.'); + } + return parseDispatchSubmission(row); + } + + hasQueuedDispatches(): boolean { + return ( + this.sql + .exec( + `SELECT 1 + FROM flue_agent_submissions + WHERE kind = 'dispatch' AND status = 'queued' + LIMIT 1`, + ) + .toArray().length > 0 + ); + } + + listQueuedDispatches(): SqlAgentDispatchSubmission[] { + return this.parseQueuedRows( + this.sql + .exec( + `SELECT sequence, submission_id, session, session_key, kind, payload, status, + accepted_at, completed_at, error + FROM flue_agent_submissions + WHERE kind = 'dispatch' AND status = 'queued' + ORDER BY sequence ASC`, + ) + .toArray(), + ); + } + + listRunnableDispatches(): SqlAgentDispatchSubmission[] { + const rows = this.sql + .exec( + `SELECT current.sequence, current.submission_id, current.session, current.session_key, + current.kind, current.payload, current.status, current.accepted_at, + current.completed_at, current.error + FROM flue_agent_submissions AS current + WHERE current.kind = 'dispatch' AND current.status = 'queued' + AND NOT EXISTS ( + SELECT 1 + FROM flue_agent_submissions AS earlier + WHERE earlier.kind = 'dispatch' + AND earlier.session_key = current.session_key + AND earlier.status = 'queued' + AND earlier.sequence < current.sequence + ) + ORDER BY current.sequence ASC`, + ) + .toArray(); + return this.parseQueuedRows(rows); + } + + hasQueuedDispatchForSession(instanceId: string, session: string): boolean { + return ( + this.sql + .exec( + `SELECT 1 + FROM flue_agent_submissions + WHERE kind = 'dispatch' AND session_key = ? AND status = 'queued' + LIMIT 1`, + createSessionStorageKey(instanceId, 'default', session), + ) + .toArray().length > 0 + ); + } + + hasEarlierQueuedDispatch(submission: SqlAgentDispatchSubmission): boolean { + return ( + this.sql + .exec( + `SELECT 1 + FROM flue_agent_submissions + WHERE kind = 'dispatch' AND session_key = ? AND status = 'queued' AND sequence < ? + LIMIT 1`, + submission.sessionKey, + submission.sequence, + ) + .toArray().length > 0 + ); + } + + completeDispatch(submissionId: string): void { + this.sql.exec( + `UPDATE flue_agent_submissions + SET status = 'completed', completed_at = ?, error = NULL + WHERE submission_id = ? AND kind = 'dispatch' AND status = 'queued'`, + Date.now(), + submissionId, + ); + } + + failDispatch(submissionId: string, error: unknown): void { + this.sql.exec( + `UPDATE flue_agent_submissions + SET status = 'error', completed_at = ?, error = ? + WHERE submission_id = ? AND kind = 'dispatch' AND status = 'queued'`, + Date.now(), + error instanceof Error ? error.message : String(error), + submissionId, + ); + } + + private parseQueuedRows(rows: SqlRow[]): SqlAgentDispatchSubmission[] { + const submissions: SqlAgentDispatchSubmission[] = []; + for (const row of rows) { + try { + submissions.push(parseDispatchSubmission(row)); + } catch (error) { + if (typeof row.sequence !== 'number') throw error; + this.failDispatchSequence(row.sequence, error); + } + } + return submissions; + } + + private failDispatchSequence(sequence: number, error: unknown): void { + this.sql.exec( + `UPDATE flue_agent_submissions + SET status = 'error', completed_at = ?, error = ? + WHERE sequence = ? AND kind = 'dispatch' AND status = 'queued'`, + Date.now(), + error instanceof Error ? error.message : String(error), + sequence, + ); + } + + private readDispatchRow(submissionId: string): SqlRow | undefined { + return this.sql + .exec( + `SELECT sequence, submission_id, session, session_key, kind, payload, status, + accepted_at, completed_at, error + FROM flue_agent_submissions + WHERE submission_id = ? AND kind = 'dispatch' + LIMIT 1`, + submissionId, + ) + .toArray()[0]; + } +} + +function parseDispatchSubmission(row: SqlRow): SqlAgentDispatchSubmission { + if ( + typeof row.sequence !== 'number' || + typeof row.submission_id !== 'string' || + typeof row.session !== 'string' || + typeof row.session_key !== 'string' || + row.kind !== 'dispatch' || + typeof row.payload !== 'string' || + (row.status !== 'queued' && row.status !== 'completed' && row.status !== 'error') || + typeof row.accepted_at !== 'number' + ) { + throw new Error('[flue] Persisted dispatch submission row is malformed.'); + } + const input = JSON.parse(row.payload) as DispatchInput; + if ( + !input || + typeof input !== 'object' || + typeof input.dispatchId !== 'string' || + typeof input.agent !== 'string' || + typeof input.id !== 'string' || + typeof input.session !== 'string' || + input.input === undefined || + typeof input.acceptedAt !== 'string' || + input.dispatchId !== row.submission_id || + input.session !== row.session || + createSessionStorageKey(input.id, 'default', input.session) !== row.session_key || + Date.parse(input.acceptedAt) !== row.accepted_at + ) { + throw new Error('[flue] Persisted dispatch submission payload is malformed.'); + } + return { + sequence: row.sequence, + submissionId: row.submission_id, + session: row.session, + sessionKey: row.session_key, + input, + status: row.status, + acceptedAt: row.accepted_at, + ...(typeof row.completed_at === 'number' ? { completedAt: row.completed_at } : {}), + ...(typeof row.error === 'string' ? { error: row.error } : {}), + }; +} + function ensureSessionTable(sql: SqlStorage): void { sql.exec( `CREATE TABLE IF NOT EXISTS flue_sessions ( diff --git a/packages/runtime/src/internal.ts b/packages/runtime/src/internal.ts index 3145873c..44ebff9f 100644 --- a/packages/runtime/src/internal.ts +++ b/packages/runtime/src/internal.ts @@ -21,10 +21,15 @@ export { createFlueContext } from './client.ts'; // (registry client) live in the `@flue/runtime/cloudflare` subpath because // they pull in `cloudflare:workers`, a virtual module Node can't resolve. // The generated CF entry imports them from there directly. -export type { SqlAgentExecutionStore } from './cloudflare/agent-execution-store.ts'; +export type { + SqlAgentDispatchSubmission, + SqlAgentExecutionStore, + SqlAgentSubmissionStore, +} from './cloudflare/agent-execution-store.ts'; export { createSqlAgentExecutionStore, createSqlSessionStore, + SqlAgentSubmissionConflictError, } from './cloudflare/agent-execution-store.ts'; export { createDurableRunStore } from './cloudflare/run-store.ts'; export { InMemoryRunRegistry } from './node/run-registry.ts'; diff --git a/packages/runtime/test/build-plugin-cloudflare.test.ts b/packages/runtime/test/build-plugin-cloudflare.test.ts index 533e381a..fe93af37 100644 --- a/packages/runtime/test/build-plugin-cloudflare.test.ts +++ b/packages/runtime/test/build-plugin-cloudflare.test.ts @@ -46,6 +46,33 @@ describe('CloudflarePlugin', () => { expect(entry).not.toContain('CREATE TABLE IF NOT EXISTS flue_sessions'); }); + it('pre-arms SQL-backed dispatch admission and reconciles runnable rows through managed Fibers', async () => { + const entry = await new CloudflarePlugin().generateEntryPoint( + testBuildContext({ + agents: [{ name: 'assistant', filePath: '/fixture/agents/assistant.ts' }], + }), + ); + + expect(entry).toContain( + `async onStart(props) { + if (typeof super.onStart === 'function') await super.onStart(props); + await reconcileFlueAgentSubmissions(this, "assistant", { preserveSuccessor: true }); + } + + async __flueWakeAgentSubmissions(wake) { + await reconcileFlueAgentSubmissions(this, "assistant", { preserveSuccessor: true, executingWake: wake }); + }`, + ); + expect(entry).toContain("const FLUE_AGENT_SUBMISSION_WAKE_CALLBACK = '__flueWakeAgentSubmissions';"); + expect(entry).toContain("await armFlueAgentSubmissionAdmissionWakes(doInstance);\n let submission;"); + expect(entry).toContain('submission = getAgentExecutionStore(doInstance).submissions.admitDispatch(input);'); + expect(entry).toContain('const runnable = submissions.listRunnableDispatches();'); + expect(entry).toContain('metadata: { input: submission.input, submissionSequence: submission.sequence },'); + expect(entry).toContain('submissions.hasQueuedDispatchForSession(doInstance.name, session)'); + expect(entry).toContain('async function waitForEarlierLegacyManagedDispatch'); + expect(entry).not.toContain('ctx.storage.setAlarm'); + }); + it('uses explicit Flue routing instead of the Agents SDK router', async () => { const entry = await new CloudflarePlugin().generateEntryPoint( testBuildContext({ diff --git a/packages/runtime/test/cloudflare-agent-execution-store.test.ts b/packages/runtime/test/cloudflare-agent-execution-store.test.ts index 2cded226..ba66fd47 100644 --- a/packages/runtime/test/cloudflare-agent-execution-store.test.ts +++ b/packages/runtime/test/cloudflare-agent-execution-store.test.ts @@ -4,6 +4,7 @@ import { createSqlAgentExecutionStore, createSqlSessionStore, } from '../src/cloudflare/agent-execution-store.ts'; +import type { DispatchInput } from '../src/runtime/dispatch-queue.ts'; import type { SessionData } from '../src/types.ts'; function makeFakeSql() { @@ -30,6 +31,18 @@ function makeFakeSql() { }; } +function dispatchInput(overrides: Partial = {}): DispatchInput { + return { + dispatchId: 'dispatch-1', + agent: 'assistant', + id: 'agent-1', + session: 'default', + input: { text: 'Hello' }, + acceptedAt: '2026-06-03T00:00:00.000Z', + ...overrides, + }; +} + function sessionData(): SessionData { return { version: 5, @@ -98,6 +111,98 @@ describe('createSqlAgentExecutionStore()', () => { ]); }); + it('admits one queued dispatch row when the same submission is replayed', () => { + const { db, sql } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql }, 'FlueAssistantAgent'); + + const first = store.submissions.admitDispatch(dispatchInput()); + const replay = store.submissions.admitDispatch(dispatchInput()); + + expect(replay).toEqual(first); + expect(db.prepare('SELECT COUNT(*) AS count FROM flue_agent_submissions').get()).toEqual({ + count: 1, + }); + expect(first).toMatchObject({ + submissionId: 'dispatch-1', + session: 'default', + sessionKey: 'agent-session:["agent-1","default","default"]', + status: 'queued', + }); + }); + + it('rejects conflicting replay when one dispatch id is reused with another payload', () => { + const { sql } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql }, 'FlueAssistantAgent'); + store.submissions.admitDispatch(dispatchInput()); + + expect(() => + store.submissions.admitDispatch(dispatchInput({ input: { text: 'Different' } })), + ).toThrow('[flue] Conflicting internal dispatch replay.'); + }); + + it('lists queued dispatches in admission order and selects one runnable head per session', () => { + const { sql } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql }, 'FlueAssistantAgent'); + const first = store.submissions.admitDispatch(dispatchInput()); + const second = store.submissions.admitDispatch(dispatchInput({ dispatchId: 'dispatch-2' })); + const other = store.submissions.admitDispatch( + dispatchInput({ dispatchId: 'dispatch-3', session: 'other' }), + ); + + expect(store.submissions.listQueuedDispatches()).toEqual([first, second, other]); + expect(store.submissions.listRunnableDispatches()).toEqual([first, other]); + expect(store.submissions.hasEarlierQueuedDispatch(first)).toBe(false); + expect(store.submissions.hasEarlierQueuedDispatch(second)).toBe(true); + expect(store.submissions.hasEarlierQueuedDispatch(other)).toBe(false); + }); + + it('terminalizes malformed queued payloads while returning healthy runnable rows', () => { + const { db, sql } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql }, 'FlueAssistantAgent'); + store.submissions.admitDispatch(dispatchInput({ dispatchId: 'healthy' })); + db.prepare( + `INSERT INTO flue_agent_submissions + (submission_id, session, session_key, kind, payload, status, accepted_at) + VALUES (?, ?, ?, 'dispatch', ?, 'queued', ?)`, + ).run('malformed', 'other', 'agent-session:["agent-1","default","other"]', '{', 1); + + expect(store.submissions.listRunnableDispatches()).toEqual([ + expect.objectContaining({ submissionId: 'healthy' }), + ]); + expect( + db + .prepare('SELECT status, error FROM flue_agent_submissions WHERE submission_id = ?') + .get('malformed'), + ).toMatchObject({ status: 'error', error: expect.any(String) }); + }); + + it('reports queued session visibility until a dispatch completes', () => { + const { sql } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql }, 'FlueAssistantAgent'); + store.submissions.admitDispatch(dispatchInput({ session: 'case-1' })); + + expect(store.submissions.hasQueuedDispatchForSession('agent-1', 'case-1')).toBe(true); + expect(store.submissions.hasQueuedDispatchForSession('agent-1', 'case-2')).toBe(false); + store.submissions.completeDispatch('dispatch-1'); + expect(store.submissions.hasQueuedDispatchForSession('agent-1', 'case-1')).toBe(false); + expect(store.submissions.getDispatch('dispatch-1')).toMatchObject({ status: 'completed' }); + }); + + it('keeps the first terminal dispatch state when a later settlement races it', () => { + const { sql } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql }, 'FlueAssistantAgent'); + store.submissions.admitDispatch(dispatchInput()); + + store.submissions.failDispatch('dispatch-1', new Error('first failure')); + store.submissions.completeDispatch('dispatch-1'); + store.submissions.failDispatch('dispatch-1', new Error('later failure')); + + expect(store.submissions.getDispatch('dispatch-1')).toMatchObject({ + status: 'error', + error: 'first failure', + }); + }); + it('rejects missing Durable Object SQLite with migration guidance', () => { expect(() => createSqlAgentExecutionStore({}, 'FlueAssistantAgent')).toThrow( 'Add "FlueAssistantAgent" to a Wrangler migration\'s "new_sqlite_classes" list before its first deploy; do not use legacy "new_classes". Existing KV-backed Durable Object classes cannot be converted to SQLite in place.', diff --git a/packages/runtime/test/cloudflare-agent-extension.integration.test.ts b/packages/runtime/test/cloudflare-agent-extension.integration.test.ts index 8a931e13..5a914a99 100644 --- a/packages/runtime/test/cloudflare-agent-extension.integration.test.ts +++ b/packages/runtime/test/cloudflare-agent-extension.integration.test.ts @@ -12,6 +12,41 @@ import { } from '../../cli/src/lib/build.ts'; describe('Cloudflare agent extension', () => { + it('rejects Cloudflare builds below the audited Agents SDK durability floor', async () => { + const root = createAgentsFloorFixture('0.14.0'); + try { + await expect( + build({ root, sourceRoot: path.join(root, 'src'), target: 'cloudflare', mode: 'development' }), + ).rejects.toThrow( + '[flue] Cloudflare target requires the installed "agents" package to be at least 0.14.1. Found 0.14.0. Install or upgrade "agents" in this project.', + ); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it('accepts the audited Agents SDK durability floor for Cloudflare builds', async () => { + const root = createAgentsFloorFixture('0.14.1'); + try { + await expect( + build({ root, sourceRoot: path.join(root, 'src'), target: 'cloudflare', mode: 'development' }), + ).rejects.toThrow('[flue] No agent or workflow files found.'); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it('does not apply the Agents SDK durability floor to Node builds', async () => { + const root = createAgentsFloorFixture('0.14.0'); + try { + await expect( + build({ root, sourceRoot: path.join(root, 'src'), target: 'node' }), + ).rejects.toThrow('[flue] No agent or workflow files found.'); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + it('runs inherited scheduled callbacks when an agent module extends its base class', async () => { const root = await createGeneratedFixture(); let server: Awaited> | undefined; @@ -124,6 +159,18 @@ describe('Cloudflare agent extension', () => { }, 90000); }); +function createAgentsFloorFixture(version: string): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'flue-cloudflare-agents-floor-')); + fs.mkdirSync(path.join(root, 'node_modules', 'agents'), { recursive: true }); + fs.mkdirSync(path.join(root, 'src'), { recursive: true }); + fs.writeFileSync( + path.join(root, 'node_modules', 'agents', 'package.json'), + JSON.stringify({ name: 'agents', version, main: 'index.js' }), + ); + fs.writeFileSync(path.join(root, 'node_modules', 'agents', 'index.js'), 'module.exports = {};\n'); + return root; +} + async function createGeneratedFixture( agentSource = defaultAgentSource, mount = '', @@ -255,7 +302,7 @@ export const cloudflare = extend({ wrap: (Final) => new Proxy(Final, { construct(target, args) { if (target.name !== 'FlueAssistantAgent') throw new Error('wrapper did not receive stable agent class identity'); - for (const method of ['onRequest', 'fetch', 'webSocketMessage', 'webSocketClose', 'webSocketError', 'onFiberRecovered']) { + for (const method of ['onStart', '__flueWakeAgentSubmissions', 'onRequest', 'fetch', 'webSocketMessage', 'webSocketClose', 'webSocketError', 'onFiberRecovered']) { if (!Object.prototype.hasOwnProperty.call(target.prototype, method)) { throw new Error('wrapper did not receive generated Flue class'); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 45adafca..e69df14b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -108,8 +108,8 @@ importers: specifier: workspace:* version: link:../../packages/runtime agents: - specifier: ^0.14.0 - version: 0.14.0(@babel/core@7.29.0)(@babel/runtime@7.29.7)(@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260602.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(just-bash@3.0.1)(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) + specifier: ^0.14.1 + version: 0.14.1(@babel/core@7.29.0)(@babel/runtime@7.29.7)(@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260602.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(just-bash@3.0.1)(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) devDependencies: '@flue/cli': specifier: workspace:* @@ -146,8 +146,8 @@ importers: specifier: workspace:* version: link:../../packages/runtime agents: - specifier: ^0.14.0 - version: 0.14.0(@babel/core@7.29.0)(@babel/runtime@7.29.7)(@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260602.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(just-bash@3.0.1)(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) + specifier: ^0.14.1 + version: 0.14.1(@babel/core@7.29.0)(@babel/runtime@7.29.7)(@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260602.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(just-bash@3.0.1)(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) chat: specifier: ^4.29.0 version: 4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3) @@ -180,8 +180,8 @@ importers: specifier: workspace:* version: link:../../packages/runtime agents: - specifier: ^0.14.0 - version: 0.14.0(@babel/core@7.29.0)(@babel/runtime@7.29.7)(@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260602.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(just-bash@3.0.1)(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) + specifier: ^0.14.1 + version: 0.14.1(@babel/core@7.29.0)(@babel/runtime@7.29.7)(@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260602.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(just-bash@3.0.1)(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) hono: specifier: ^4.7.0 version: 4.12.22 @@ -205,8 +205,8 @@ importers: specifier: workspace:* version: link:../../packages/runtime agents: - specifier: ^0.14.0 - version: 0.14.0(@babel/core@7.29.0)(@babel/runtime@7.29.7)(@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260602.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(just-bash@3.0.1)(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) + specifier: ^0.14.1 + version: 0.14.1(@babel/core@7.29.0)(@babel/runtime@7.29.7)(@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260602.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(just-bash@3.0.1)(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) hono: specifier: ^4.7.0 version: 4.12.22 @@ -233,8 +233,8 @@ importers: specifier: workspace:* version: link:../../packages/runtime agents: - specifier: ^0.14.0 - version: 0.14.0(@babel/core@7.29.0)(@babel/runtime@7.29.7)(@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260602.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(just-bash@3.0.1)(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) + specifier: ^0.14.1 + version: 0.14.1(@babel/core@7.29.0)(@babel/runtime@7.29.7)(@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260602.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(just-bash@3.0.1)(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) hono: specifier: ^4.7.0 version: 4.12.22 @@ -255,8 +255,8 @@ importers: specifier: workspace:* version: link:../../packages/runtime agents: - specifier: ^0.14.0 - version: 0.14.0(@babel/core@7.29.0)(@babel/runtime@7.29.7)(@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260602.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(just-bash@3.0.1)(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) + specifier: ^0.14.1 + version: 0.14.1(@babel/core@7.29.0)(@babel/runtime@7.29.7)(@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260602.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(just-bash@3.0.1)(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3) just-bash: specifier: ^3.0.1 version: 3.0.1 @@ -3346,8 +3346,8 @@ packages: resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} engines: {node: '>= 14'} - agents@0.14.0: - resolution: {integrity: sha512-4+H8Pm0kEx6pcIaRi8KCYueRCw5sM03hC+yZjJ7YYEGBLFcresxSMVi5Zd6IpyHQFDd3kp5yE/gWVmNz9jRrWQ==} + agents@0.14.1: + resolution: {integrity: sha512-BZEntZYyAJRYwSqFA1/gcmTYOq72U+X7NPzYu8MMZX9CUfmS9lFYP325yK3SXDvzw+y2XLDzmW5wcvPPqHsHqw==} hasBin: true peerDependencies: '@cloudflare/ai-chat': '>=0.8.0 <1.0.0' @@ -9374,7 +9374,7 @@ snapshots: agent-base@7.1.4: {} - agents@0.14.0(@babel/core@7.29.0)(@babel/runtime@7.29.7)(@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260602.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(just-bash@3.0.1)(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3): + agents@0.14.1(@babel/core@7.29.0)(@babel/runtime@7.29.7)(@cloudflare/codemode@0.3.8(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3))(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(@cloudflare/workers-types@4.20260602.1)(ai@6.0.191(zod@4.4.3))(chat@4.29.0(ai@6.0.191(zod@4.4.3))(zod@4.4.3))(just-bash@3.0.1)(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)(yaml@2.9.0))(zod@4.4.3): dependencies: '@babel/plugin-proposal-decorators': 7.29.7(@babel/core@7.29.0) '@cfworker/json-schema': 4.1.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 8c1496f7..0b7a0366 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -12,6 +12,7 @@ allowBuilds: minimumReleaseAge: 1440 resolutionMode: highest minimumReleaseAgeExclude: + - agents - '@aws-sdk/*' - '@smithy/*' - protobufjs From 439376c169c348ac09de7edf54e50eb1f8557219 Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" <622227+FredKSchott@users.noreply.github.com> Date: Wed, 3 Jun 2026 16:22:13 -0500 Subject: [PATCH 05/17] feat(cloudflare): claim durable dispatch submissions --- .../cli/src/lib/build-plugin-cloudflare.ts | 244 +++++++++++------- .../src/cloudflare/agent-execution-store.ts | 210 ++++++++++++--- .../test/build-plugin-cloudflare.test.ts | 25 +- .../cloudflare-agent-execution-store.test.ts | 159 +++++++++--- 4 files changed, 476 insertions(+), 162 deletions(-) diff --git a/packages/cli/src/lib/build-plugin-cloudflare.ts b/packages/cli/src/lib/build-plugin-cloudflare.ts index c718c56c..e2cc6a71 100644 --- a/packages/cli/src/lib/build-plugin-cloudflare.ts +++ b/packages/cli/src/lib/build-plugin-cloudflare.ts @@ -59,6 +59,7 @@ const ${agentClassName(agent.name)} = class ${agentClassName(agent.name)} extend async onStart(props) { if (typeof super.onStart === 'function') await super.onStart(props); + await armFlueAgentSubmissionRetry(this); await reconcileFlueAgentSubmissions(this, ${JSON.stringify(agent.name)}, { preserveSuccessor: true }); } @@ -66,6 +67,12 @@ const ${agentClassName(agent.name)} = class ${agentClassName(agent.name)} extend await reconcileFlueAgentSubmissions(this, ${JSON.stringify(agent.name)}, { preserveSuccessor: true, executingWake: wake }); } + async __flueRetryAgentSubmissions(_payload, schedule) { + if (!(await reconcileFlueAgentSubmissions(this, ${JSON.stringify(agent.name)}))) { + await this.cancelSchedule(schedule.id); + } + } + async onRequest(request) { return dispatchAgent(request, this, ${JSON.stringify(agent.name)}, directHandlers[${JSON.stringify(agent.name)}]); } @@ -100,6 +107,9 @@ const ${agentClassName(agent.name)} = class ${agentClassName(agent.name)} extend if (ctx.name === 'flue:dispatch') { return handleFlueDispatchRecovered(ctx, this, ${JSON.stringify(agent.name)}); } + if (ctx.name === 'flue:dispatch-attempt') { + return handleFlueDispatchAttemptRecovered(ctx, this); + } if (ctx.name === 'flue:direct') { return handleFlueDirectRecovered(ctx, this, ${JSON.stringify(agent.name)}); } @@ -351,6 +361,7 @@ const memoryWorkflowSessionStore = new InMemorySessionStore(); const memoryRunStore = new InMemoryRunStore(); const FLUE_AGENT_EXECUTION_STORE = Symbol('flueAgentExecutionStore'); const FLUE_AGENT_SUBMISSION_WAKE_CALLBACK = '__flueWakeAgentSubmissions'; +const FLUE_AGENT_SUBMISSION_RETRY_CALLBACK = '__flueRetryAgentSubmissions'; const FLUE_AGENT_SUBMISSION_RETRY_SECONDS = 30; const INTERNAL_DISPATCH_PATH = '/__flue/internal/dispatch'; const dispatchQueue = { @@ -370,6 +381,7 @@ const dispatchQueue = { // Module-scoped per-isolate registry; run ids isolate buckets across DOs. const runSubscribers = createRunSubscriberRegistry(); +const activeFlueAgentDispatchAttempts = new Set(); function getAgentExecutionStore(doInstance) { const store = doInstance[FLUE_AGENT_EXECUTION_STORE]; @@ -463,17 +475,32 @@ async function handleFlueDispatchRecovered(ctx, doInstance, agentName) { const input = ctx.metadata?.input; assertCurrentDispatchInput(input); if (!input || input.agent !== agentName || input.id !== doInstance.name) return { status: 'error', error: 'Dispatch recovery metadata is invalid.' }; - const submissions = getAgentExecutionStore(doInstance).submissions; - const submission = submissions.getDispatch(input.dispatchId); - if (submission && submission.status !== 'queued') return { status: submission.status === 'completed' ? 'completed' : 'error', error: submission.error }; try { - await processManagedAgentDispatch(submission ?? { input }, doInstance, agentName, ctx.id); + await validateAgentDispatchAdmission({ input }); + await armFlueAgentSubmissionAdmissionWakes(doInstance); + const submissions = getAgentExecutionStore(doInstance).submissions; + const legacy = listActiveLegacyManagedDispatches(doInstance, agentName); + legacy.push({ fiberId: ctx.id, createdAt: ctx.createdAt, input }); + legacy.sort(compareLegacyManagedDispatches); + submissions.adoptLegacyDispatches(legacy.map((dispatch) => dispatch.input)); + await reconcileFlueAgentSubmissions(doInstance, agentName, { preserveSuccessor: true }); return { status: 'completed' }; } catch (error) { return { status: 'error', error: error instanceof Error ? error.message : String(error) }; } } +async function handleFlueDispatchAttemptRecovered(ctx, doInstance) { + const submissionId = ctx.snapshot?.submissionId; + const attemptId = ctx.snapshot?.attemptId; + if (typeof submissionId !== 'string' || typeof attemptId !== 'string') return; + const submissions = getAgentExecutionStore(doInstance).submissions; + const submission = submissions.getDispatch(submissionId); + if (submission?.status !== 'running' || submission.attemptId !== attemptId) return; + submissions.recoverDispatchAttempt(submissionId, attemptId, crypto.randomUUID()); + await armFlueAgentSubmissionAdmissionWakes(doInstance); +} + async function handleFlueDirectRecovered(ctx, doInstance, agentName) { const payload = ctx.snapshot?.payload; const handler = localAgentHandlers[agentName]; @@ -483,7 +510,10 @@ async function handleFlueDirectRecovered(ctx, doInstance, agentName) { } const identity = agentRuntimeIdentity(agentName); const request = new Request('https://flue.invalid/agents/' + encodeURIComponent(agentName) + '/' + encodeURIComponent(doInstance.name), { method: 'POST' }); + let releaseSessionLock; try { + await adoptLegacyManagedDispatches(doInstance, agentName); + releaseSessionLock = await reserveDispatchAgentSession({ agentName, instanceId: doInstance.name }, payload); assertAgentsDurabilityApi(doInstance, 'runFiber'); await doInstance.runFiber('flue:direct', async (fiberCtx) => { fiberCtx.stash({ payload }); @@ -493,6 +523,8 @@ async function handleFlueDirectRecovered(ctx, doInstance, agentName) { console.info('[flue:direct-recovery]', { agentName, instanceId: doInstance.name, operation: 'retry', outcome: 'restart_completed' }); } catch (error) { console.error('[flue:direct-recovery]', { agentName, instanceId: doInstance.name, operation: 'retry', outcome: 'restart_failed' }, error); + } finally { + releaseSessionLock?.(); } } @@ -529,12 +561,20 @@ function armFlueAgentSubmissionAdmissionWakes(doInstance) { return Promise.all([ armFlueAgentSubmissionWake(doInstance, { generation: 0 }), armFlueAgentSubmissionWake(doInstance, { generation: 1 }), + armFlueAgentSubmissionRetry(doInstance), ]); } +function armFlueAgentSubmissionRetry(doInstance) { + assertAgentsDurabilityApi(doInstance, 'scheduleEvery'); + return doInstance.scheduleEvery(FLUE_AGENT_SUBMISSION_RETRY_SECONDS, FLUE_AGENT_SUBMISSION_RETRY_CALLBACK); +} + async function reconcileFlueAgentSubmissions(doInstance, agentName, options = {}) { const submissions = getAgentExecutionStore(doInstance).submissions; - if (!submissions.hasQueuedDispatches()) return; + const legacySessions = await adoptLegacyManagedDispatches(doInstance, agentName); + if (!submissions.hasUnsettledDispatches()) return false; + await armFlueAgentSubmissionRetry(doInstance); if (options.preserveSuccessor) { if (options.executingWake) { await armFlueAgentSubmissionWake(doInstance, { @@ -549,70 +589,105 @@ async function reconcileFlueAgentSubmissions(doInstance, agentName, options = {} } } try { - while (true) { - const runnable = submissions.listRunnableDispatches(); - let repairedTerminal = false; - for (const submission of runnable) { - if (await ensureManagedDispatchFiber(submission, doInstance, agentName)) repairedTerminal = true; - } - if (!repairedTerminal) break; + const attemptMarkers = listActiveSqlAgentDispatchAttemptMarkers(doInstance); + const directMarkers = listActiveDirectAgentSessionMarkers(doInstance); + if (attemptMarkers.blockAll || directMarkers.blockAll) return true; + for (const submission of submissions.listRunningDispatches()) { + if (attemptMarkers.keys.has(dispatchAttemptMarkerKey(submission))) continue; + if (directMarkers.sessions.has(submission.session) || legacySessions.has(submission.session)) continue; + startSqlAgentDispatchAttempt(submission, doInstance, agentName); + } + for (const submission of submissions.listRunnableDispatches()) { + if (directMarkers.sessions.has(submission.session) || legacySessions.has(submission.session)) continue; + const claimed = submissions.claimDispatch(submission.submissionId, crypto.randomUUID()); + if (claimed) startSqlAgentDispatchAttempt(claimed, doInstance, agentName); } } catch (error) { console.error('[flue:dispatch-reconciliation]', { agentName, instanceId: doInstance.name, operation: 'reconcile', outcome: 'deferred_to_scheduled_wake' }, error); - return; + return true; } + return submissions.hasUnsettledDispatches(); } -async function ensureManagedDispatchFiber(submission, doInstance, agentName) { - assertAgentsDurabilityApi(doInstance, 'inspectFiberByKey'); - assertAgentsDurabilityApi(doInstance, 'startFiber'); - const idempotencyKey = 'flue:dispatch:' + submission.submissionId; - const prior = await doInstance.inspectFiberByKey(idempotencyKey); - if (prior) { +function startSqlAgentDispatchAttempt(submission, doInstance, agentName) { + if (submission.status !== 'running' || !submission.attemptId) return; + const attemptKey = doInstance.ctx.id.toString() + ':' + submission.attemptId; + if (activeFlueAgentDispatchAttempts.has(attemptKey)) return; + activeFlueAgentDispatchAttempts.add(attemptKey); + assertAgentsDurabilityApi(doInstance, 'runFiber'); + void doInstance.runFiber('flue:dispatch-attempt', async (fiberCtx) => { + fiberCtx.stash({ submissionId: submission.submissionId, attemptId: submission.attemptId }); + await processSqlAgentDispatch(submission, doInstance, agentName); + }).catch((error) => { + console.error('[flue:dispatch-processing]', { agentName, instanceId: doInstance.name, submissionId: submission.submissionId, operation: 'process', outcome: 'failed' }, error); + }).finally(() => { + activeFlueAgentDispatchAttempts.delete(attemptKey); + }); +} + +function dispatchAttemptMarkerKey(submission) { + return submission.submissionId + ':' + submission.attemptId; +} + +function listActiveSqlAgentDispatchAttemptMarkers(doInstance) { + const keys = new Set(); + let blockAll = false; + const rows = doInstance.ctx.storage.sql.exec( + "SELECT snapshot FROM cf_agents_runs WHERE name = 'flue:dispatch-attempt'", + ).toArray(); + for (const row of rows) { + let snapshot; try { - await validateAgentDispatchAdmission({ input: prior.metadata?.input }); + snapshot = typeof row.snapshot === 'string' ? JSON.parse(row.snapshot) : null; } catch { - getAgentExecutionStore(doInstance).submissions.failDispatch(submission.submissionId, new Error('[flue] Persisted dispatch Fiber metadata is malformed.')); - return true; + blockAll = true; + continue; } - if (JSON.stringify(prior.metadata.input) !== JSON.stringify(submission.input)) { - getAgentExecutionStore(doInstance).submissions.failDispatch(submission.submissionId, new Error('[flue] Persisted dispatch Fiber conflicts with SQL submission.')); - return true; + if (typeof snapshot?.submissionId !== 'string' || typeof snapshot?.attemptId !== 'string') { + blockAll = true; + continue; } + keys.add(snapshot.submissionId + ':' + snapshot.attemptId); } - if (prior?.status === 'completed') { - getAgentExecutionStore(doInstance).submissions.completeDispatch(submission.submissionId); - return true; - } - if (prior?.status === 'error' || prior?.status === 'aborted' || prior?.status === 'interrupted') { - getAgentExecutionStore(doInstance).submissions.failDispatch(submission.submissionId, prior.error ?? 'Managed dispatch Fiber did not complete.'); - return true; + return { blockAll, keys }; +} + +function listActiveDirectAgentSessionMarkers(doInstance) { + const sessions = new Set(); + let blockAll = false; + const rows = doInstance.ctx.storage.sql.exec( + "SELECT snapshot FROM cf_agents_runs WHERE name = 'flue:direct'", + ).toArray(); + for (const row of rows) { + let snapshot; + try { + snapshot = typeof row.snapshot === 'string' ? JSON.parse(row.snapshot) : null; + } catch { + blockAll = true; + continue; + } + const payload = snapshot?.payload; + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { + blockAll = true; + continue; + } + sessions.add(typeof payload.session === 'string' && payload.session.trim() !== '' ? payload.session : 'default'); } - if (prior) return false; - await doInstance.startFiber('flue:dispatch', async (fiberCtx) => processManagedAgentDispatch(submission, doInstance, agentName, fiberCtx.id), { - idempotencyKey, - metadata: { input: submission.input, submissionSequence: submission.sequence }, - }); - return false; + return { blockAll, sessions }; } -function isPersistedDispatchInput(input) { - return !!input && typeof input === 'object' && !Array.isArray(input) && - typeof input.dispatchId === 'string' && input.dispatchId.trim() !== '' && - typeof input.agent === 'string' && input.agent.trim() !== '' && - typeof input.id === 'string' && input.id.trim() !== '' && - typeof input.session === 'string' && input.session.trim() !== '' && - input.input !== undefined && typeof input.acceptedAt === 'string' && input.acceptedAt.trim() !== ''; +function compareLegacyManagedDispatches(a, b) { + return a.createdAt - b.createdAt || a.fiberId.localeCompare(b.fiberId); } -function listActiveLegacyManagedDispatches(doInstance) { +function listActiveLegacyManagedDispatches(doInstance, agentName) { const rows = doInstance.ctx.storage.sql.exec( 'SELECT fiber_id, metadata_json, created_at FROM cf_agents_fibers ' + "WHERE name = 'flue:dispatch' AND (status IN ('pending', 'running') OR " + "(status = 'interrupted' AND EXISTS (SELECT 1 FROM cf_agents_runs WHERE id = fiber_id))) " + 'ORDER BY created_at ASC, fiber_id ASC', ).toArray(); - return rows.flatMap((row) => { + return rows.map((row) => { if (typeof row.fiber_id !== 'string' || typeof row.created_at !== 'number') { throw new Error('[flue] Persisted dispatch Fiber row is malformed.'); } @@ -622,75 +697,60 @@ function listActiveLegacyManagedDispatches(doInstance) { } catch { throw new Error('[flue] Persisted dispatch Fiber metadata is malformed.'); } - if (typeof metadata?.submissionSequence === 'number') return []; const input = metadata?.input; assertCurrentDispatchInput(input); - if (!isPersistedDispatchInput(input)) throw new Error('[flue] Persisted dispatch Fiber metadata is malformed.'); - return [{ fiberId: row.fiber_id, createdAt: row.created_at, input }]; + if (!input || input.agent !== agentName || input.id !== doInstance.name) { + throw new Error('[flue] Persisted dispatch Fiber metadata is invalid.'); + } + return { fiberId: row.fiber_id, createdAt: row.created_at, input }; }); } -async function waitForEarlierLegacyManagedDispatch(doInstance, input, fiberId) { - while (true) { - const fibers = listActiveLegacyManagedDispatches(doInstance); - const current = fibers.find((fiber) => fiber.fiberId === fiberId); - const blocked = fibers.some((fiber) => { - if (fiber.fiberId === fiberId) return false; - const other = fiber.input; - if (other.agent !== input.agent || other.id !== input.id || other.session !== input.session) return false; - if (!current) return true; - return fiber.createdAt < current.createdAt || (fiber.createdAt === current.createdAt && fiber.fiberId < current.fiberId); - }); - if (!blocked) return; - await new Promise((resolve) => setTimeout(resolve, 0)); - } +async function adoptLegacyManagedDispatches(doInstance, agentName) { + const dispatches = listActiveLegacyManagedDispatches(doInstance, agentName); + if (dispatches.length === 0) return new Set(); + for (const { input } of dispatches) await validateAgentDispatchAdmission({ input }); + getAgentExecutionStore(doInstance).submissions.adoptLegacyDispatches(dispatches.map((dispatch) => dispatch.input)); + return new Set(dispatches.map((dispatch) => dispatch.input.session)); } -async function processManagedAgentDispatch(submission, doInstance, agentName, fiberId) { - const input = submission.input; +async function processSqlAgentDispatch(submission, doInstance, agentName) { + const { attemptId, input } = submission; + if (!attemptId) return; const submissions = getAgentExecutionStore(doInstance).submissions; const persisted = submissions.getDispatch(input.dispatchId); - if (persisted && persisted.status !== 'queued') return; - const agent = createdAgents[agentName]; - if (!agent) throw new Error('[flue] Dispatch target unavailable during durable processing.'); - await validateAgentDispatchAdmission({ input }); - const target = { agentName, instanceId: doInstance.name }; - if (typeof submission.sequence === 'number') { - while (submissions.hasEarlierQueuedDispatch(submission)) await new Promise((resolve) => setTimeout(resolve, 0)); - } - await waitForEarlierLegacyManagedDispatch(doInstance, input, fiberId); - const releaseSessionLock = await reserveDispatchAgentSession(target, input); - const request = new Request('https://flue.invalid' + INTERNAL_DISPATCH_PATH, { method: 'POST' }); + if (persisted?.status !== 'running' || persisted.attemptId !== attemptId) return; + let releaseSessionLock; try { + const agent = createdAgents[agentName]; + if (!agent) throw new Error('[flue] Dispatch target unavailable during durable processing.'); + await validateAgentDispatchAdmission({ input }); + const target = { agentName, instanceId: doInstance.name }; + releaseSessionLock = await reserveDispatchAgentSession(target, input); + const request = new Request('https://flue.invalid' + INTERNAL_DISPATCH_PATH, { method: 'POST' }); const ctx = createAgentContextForRequest(doInstance.name, input, doInstance, request, undefined, input.dispatchId); await runWithInstanceContext(doInstance, agentRuntimeIdentity(agentName), () => createDispatchAgentHandler(agent, input)(ctx)); - if (persisted) submissions.completeDispatch(input.dispatchId); + submissions.completeDispatch(input.dispatchId, attemptId); } catch (error) { - if (persisted) submissions.failDispatch(input.dispatchId, error); + submissions.failDispatch(input.dispatchId, attemptId, error); throw error; } finally { releaseSessionLock?.(); - if (persisted) void reconcileFlueAgentSubmissions(doInstance, agentName, { preserveSuccessor: true }).catch((error) => { + void reconcileFlueAgentSubmissions(doInstance, agentName, { preserveSuccessor: true }).catch((error) => { console.error('[flue:dispatch-reconciliation]', { agentName, instanceId: doInstance.name, operation: 'settlement', outcome: 'reconcile_failed' }, error); }); } } -async function assertNoLegacyPendingDispatchFiberForDirectSession(doInstance, agentName, session) { - const fibers = listActiveLegacyManagedDispatches(doInstance); - if (fibers.some((fiber) => { - const input = fiber.input; - return input.agent === agentName && input.id === doInstance.name && input.session === session; - })) { - throw new Error('[flue] This agent session has pending dispatched input and cannot accept direct input yet.'); - } -} - async function assertNoPendingDispatchForDirectSession(doInstance, agentName, session) { - if (getAgentExecutionStore(doInstance).submissions.hasQueuedDispatchForSession(doInstance.name, session)) { + await adoptLegacyManagedDispatches(doInstance, agentName); + const directMarkers = listActiveDirectAgentSessionMarkers(doInstance); + if (directMarkers.blockAll || directMarkers.sessions.has(session)) { + throw new Error('[flue] This agent session has an interrupted direct prompt and cannot accept direct input yet.'); + } + if (getAgentExecutionStore(doInstance).submissions.hasUnsettledDispatchForSession(doInstance.name, session)) { throw new Error('[flue] This agent session has pending dispatched input and cannot accept direct input yet.'); } - await assertNoLegacyPendingDispatchFiberForDirectSession(doInstance, agentName, session); } async function dispatchWorkflow(request, doInstance, workflowName) { diff --git a/packages/runtime/src/cloudflare/agent-execution-store.ts b/packages/runtime/src/cloudflare/agent-execution-store.ts index fee462a4..6388c29f 100644 --- a/packages/runtime/src/cloudflare/agent-execution-store.ts +++ b/packages/runtime/src/cloudflare/agent-execution-store.ts @@ -14,9 +14,10 @@ interface SqlStorage { interface DurableObjectStorage { readonly sql?: SqlStorage; + transactionSync?(closure: () => T): T; } -type SqlAgentSubmissionStatus = 'queued' | 'completed' | 'error'; +type SqlAgentSubmissionStatus = 'queued' | 'running' | 'completed' | 'error'; export interface SqlAgentDispatchSubmission { readonly sequence: number; @@ -26,6 +27,8 @@ export interface SqlAgentDispatchSubmission { readonly input: DispatchInput; readonly status: SqlAgentSubmissionStatus; readonly acceptedAt: number; + readonly attemptId?: string; + readonly startedAt?: number; readonly completedAt?: number; readonly error?: string; } @@ -33,13 +36,20 @@ export interface SqlAgentDispatchSubmission { export interface SqlAgentSubmissionStore { getDispatch(submissionId: string): SqlAgentDispatchSubmission | null; admitDispatch(input: DispatchInput): SqlAgentDispatchSubmission; - hasQueuedDispatches(): boolean; + adoptLegacyDispatches(inputs: readonly DispatchInput[]): SqlAgentDispatchSubmission[]; + hasUnsettledDispatches(): boolean; listQueuedDispatches(): SqlAgentDispatchSubmission[]; listRunnableDispatches(): SqlAgentDispatchSubmission[]; - hasQueuedDispatchForSession(instanceId: string, session: string): boolean; - hasEarlierQueuedDispatch(submission: SqlAgentDispatchSubmission): boolean; - completeDispatch(submissionId: string): void; - failDispatch(submissionId: string, error: unknown): void; + listRunningDispatches(): SqlAgentDispatchSubmission[]; + hasUnsettledDispatchForSession(instanceId: string, session: string): boolean; + claimDispatch(submissionId: string, attemptId: string): SqlAgentDispatchSubmission | null; + recoverDispatchAttempt( + submissionId: string, + expectedAttemptId: string, + nextAttemptId: string, + ): SqlAgentDispatchSubmission | null; + completeDispatch(submissionId: string, attemptId: string): void; + failDispatch(submissionId: string, attemptId: string, error: unknown): void; } export interface SqlAgentExecutionStore { @@ -59,7 +69,8 @@ export function createSqlAgentExecutionStore( className: string, ): SqlAgentExecutionStore { const sql = storage?.sql; - if (!sql || typeof sql.exec !== 'function') { + const transactionSync = storage?.transactionSync; + if (!sql || typeof sql.exec !== 'function' || typeof transactionSync !== 'function') { throw new Error( `[flue] Cloudflare durable agent class "${className}" requires Durable Object SQLite. ` + `Add "${className}" to a Wrangler migration's "new_sqlite_classes" list before its first deploy; ` + @@ -70,7 +81,11 @@ export function createSqlAgentExecutionStore( try { const sessions = createSqlSessionStore(sql); ensureSubmissionTable(sql); - return { sessions, submissions: new SqlAgentSubmissionStoreImpl(sql) }; + const runTransaction = (closure: () => T): T => transactionSync.call(storage, closure) as T; + return { + sessions, + submissions: new SqlAgentSubmissionStoreImpl(sql, runTransaction), + }; } catch (cause) { const detail = cause instanceof Error ? cause.message : String(cause); throw new Error( @@ -107,7 +122,10 @@ class SqlSessionStore implements SessionStore { } class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { - constructor(private sql: SqlStorage) {} + constructor( + private sql: SqlStorage, + private transactionSync: NonNullable, + ) {} getDispatch(submissionId: string): SqlAgentDispatchSubmission | null { const row = this.readDispatchRow(submissionId); @@ -138,13 +156,66 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { return parseDispatchSubmission(row); } - hasQueuedDispatches(): boolean { + adoptLegacyDispatches(inputs: readonly DispatchInput[]): SqlAgentDispatchSubmission[] { + const unique = new Map(); + for (const input of inputs) { + const payload = JSON.stringify(input); + const row = this.readDispatchRow(input.dispatchId); + if (row && row.payload !== payload) { + throw new SqlAgentSubmissionConflictError('[flue] Conflicting legacy dispatch adoption.'); + } + const prior = unique.get(input.dispatchId); + if (prior && JSON.stringify(prior) !== payload) { + throw new SqlAgentSubmissionConflictError('[flue] Conflicting legacy dispatch adoption.'); + } + if (!prior) unique.set(input.dispatchId, input); + } + const missing = [...unique.values()].filter((input) => !this.readDispatchRow(input.dispatchId)); + const adopt = () => { + const first = this.sql + .exec('SELECT MIN(sequence) AS sequence FROM flue_agent_submissions') + .toArray()[0]?.sequence; + let sequence = typeof first === 'number' ? first - missing.length : -missing.length; + for (let offset = 0; offset < missing.length; offset += 16) { + const batch = missing.slice(offset, offset + 16); + const values: unknown[] = []; + for (const input of batch) { + const acceptedAt = Date.parse(input.acceptedAt); + if (!Number.isFinite(acceptedAt)) { + throw new Error('[flue] Legacy dispatch adoption received an invalid acceptedAt timestamp.'); + } + values.push( + sequence++, + input.dispatchId, + input.session, + createSessionStorageKey(input.id, 'default', input.session), + JSON.stringify(input), + acceptedAt, + ); + } + this.sql.exec( + `INSERT INTO flue_agent_submissions + (sequence, submission_id, session, session_key, kind, payload, status, accepted_at) + VALUES ${batch.map(() => "(?, ?, ?, ?, 'dispatch', ?, 'queued', ?)").join(', ')}`, + ...values, + ); + } + }; + if (missing.length > 0) this.transactionSync(adopt); + return inputs.map((input) => { + const submission = this.getDispatch(input.dispatchId); + if (!submission) throw new Error('[flue] Legacy dispatch adoption did not create a submission row.'); + return submission; + }); + } + + hasUnsettledDispatches(): boolean { return ( this.sql .exec( `SELECT 1 FROM flue_agent_submissions - WHERE kind = 'dispatch' AND status = 'queued' + WHERE kind = 'dispatch' AND status IN ('queued', 'running') LIMIT 1`, ) .toArray().length > 0 @@ -156,7 +227,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { this.sql .exec( `SELECT sequence, submission_id, session, session_key, kind, payload, status, - accepted_at, completed_at, error + accepted_at, attempt_id, started_at, completed_at, error FROM flue_agent_submissions WHERE kind = 'dispatch' AND status = 'queued' ORDER BY sequence ASC`, @@ -170,7 +241,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { .exec( `SELECT current.sequence, current.submission_id, current.session, current.session_key, current.kind, current.payload, current.status, current.accepted_at, - current.completed_at, current.error + current.attempt_id, current.started_at, current.completed_at, current.error FROM flue_agent_submissions AS current WHERE current.kind = 'dispatch' AND current.status = 'queued' AND NOT EXISTS ( @@ -178,7 +249,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { FROM flue_agent_submissions AS earlier WHERE earlier.kind = 'dispatch' AND earlier.session_key = current.session_key - AND earlier.status = 'queued' + AND earlier.status IN ('queued', 'running') AND earlier.sequence < current.sequence ) ORDER BY current.sequence ASC`, @@ -187,77 +258,137 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { return this.parseQueuedRows(rows); } - hasQueuedDispatchForSession(instanceId: string, session: string): boolean { - return ( + listRunningDispatches(): SqlAgentDispatchSubmission[] { + return this.parseRunningRows( this.sql .exec( - `SELECT 1 + `SELECT sequence, submission_id, session, session_key, kind, payload, status, + accepted_at, attempt_id, started_at, completed_at, error FROM flue_agent_submissions - WHERE kind = 'dispatch' AND session_key = ? AND status = 'queued' - LIMIT 1`, - createSessionStorageKey(instanceId, 'default', session), + WHERE kind = 'dispatch' AND status = 'running' + ORDER BY sequence ASC`, ) - .toArray().length > 0 + .toArray(), ); } - hasEarlierQueuedDispatch(submission: SqlAgentDispatchSubmission): boolean { + hasUnsettledDispatchForSession(instanceId: string, session: string): boolean { return ( this.sql .exec( `SELECT 1 FROM flue_agent_submissions - WHERE kind = 'dispatch' AND session_key = ? AND status = 'queued' AND sequence < ? + WHERE kind = 'dispatch' AND session_key = ? AND status IN ('queued', 'running') LIMIT 1`, - submission.sessionKey, - submission.sequence, + createSessionStorageKey(instanceId, 'default', session), ) .toArray().length > 0 ); } - completeDispatch(submissionId: string): void { + claimDispatch(submissionId: string, attemptId: string): SqlAgentDispatchSubmission | null { + this.sql.exec( + `UPDATE flue_agent_submissions AS current + SET status = 'running', attempt_id = ?, started_at = ? + WHERE current.submission_id = ? AND current.kind = 'dispatch' AND current.status = 'queued' + AND NOT EXISTS ( + SELECT 1 + FROM flue_agent_submissions AS earlier + WHERE earlier.kind = 'dispatch' + AND earlier.session_key = current.session_key + AND earlier.status IN ('queued', 'running') + AND earlier.sequence < current.sequence + )`, + attemptId, + Date.now(), + submissionId, + ); + const submission = this.getDispatch(submissionId); + return submission?.status === 'running' && submission.attemptId === attemptId + ? submission + : null; + } + + recoverDispatchAttempt( + submissionId: string, + expectedAttemptId: string, + nextAttemptId: string, + ): SqlAgentDispatchSubmission | null { + this.sql.exec( + `UPDATE flue_agent_submissions + SET attempt_id = ?, started_at = ? + WHERE submission_id = ? AND kind = 'dispatch' AND status = 'running' AND attempt_id = ?`, + nextAttemptId, + Date.now(), + submissionId, + expectedAttemptId, + ); + const submission = this.getDispatch(submissionId); + return submission?.status === 'running' && submission.attemptId === nextAttemptId + ? submission + : null; + } + + completeDispatch(submissionId: string, attemptId: string): void { this.sql.exec( `UPDATE flue_agent_submissions SET status = 'completed', completed_at = ?, error = NULL - WHERE submission_id = ? AND kind = 'dispatch' AND status = 'queued'`, + WHERE submission_id = ? AND kind = 'dispatch' AND status = 'running' AND attempt_id = ?`, Date.now(), submissionId, + attemptId, ); } - failDispatch(submissionId: string, error: unknown): void { + failDispatch(submissionId: string, attemptId: string, error: unknown): void { this.sql.exec( `UPDATE flue_agent_submissions SET status = 'error', completed_at = ?, error = ? - WHERE submission_id = ? AND kind = 'dispatch' AND status = 'queued'`, + WHERE submission_id = ? AND kind = 'dispatch' AND status = 'running' AND attempt_id = ?`, Date.now(), error instanceof Error ? error.message : String(error), submissionId, + attemptId, ); } private parseQueuedRows(rows: SqlRow[]): SqlAgentDispatchSubmission[] { + return this.parseOperationalRows(rows, 'queued'); + } + + private parseRunningRows(rows: SqlRow[]): SqlAgentDispatchSubmission[] { + return this.parseOperationalRows(rows, 'running'); + } + + private parseOperationalRows( + rows: SqlRow[], + status: Extract, + ): SqlAgentDispatchSubmission[] { const submissions: SqlAgentDispatchSubmission[] = []; for (const row of rows) { try { submissions.push(parseDispatchSubmission(row)); } catch (error) { if (typeof row.sequence !== 'number') throw error; - this.failDispatchSequence(row.sequence, error); + this.failDispatchSequence(row.sequence, status, error); } } return submissions; } - private failDispatchSequence(sequence: number, error: unknown): void { + private failDispatchSequence( + sequence: number, + status: Extract, + error: unknown, + ): void { this.sql.exec( `UPDATE flue_agent_submissions SET status = 'error', completed_at = ?, error = ? - WHERE sequence = ? AND kind = 'dispatch' AND status = 'queued'`, + WHERE sequence = ? AND kind = 'dispatch' AND status = ?`, Date.now(), error instanceof Error ? error.message : String(error), sequence, + status, ); } @@ -265,7 +396,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { return this.sql .exec( `SELECT sequence, submission_id, session, session_key, kind, payload, status, - accepted_at, completed_at, error + accepted_at, attempt_id, started_at, completed_at, error FROM flue_agent_submissions WHERE submission_id = ? AND kind = 'dispatch' LIMIT 1`, @@ -283,8 +414,15 @@ function parseDispatchSubmission(row: SqlRow): SqlAgentDispatchSubmission { typeof row.session_key !== 'string' || row.kind !== 'dispatch' || typeof row.payload !== 'string' || - (row.status !== 'queued' && row.status !== 'completed' && row.status !== 'error') || - typeof row.accepted_at !== 'number' + (row.status !== 'queued' && + row.status !== 'running' && + row.status !== 'completed' && + row.status !== 'error') || + typeof row.accepted_at !== 'number' || + (row.attempt_id !== null && row.attempt_id !== undefined && typeof row.attempt_id !== 'string') || + (row.started_at !== null && row.started_at !== undefined && typeof row.started_at !== 'number') || + (row.status === 'running' && + (typeof row.attempt_id !== 'string' || typeof row.started_at !== 'number')) ) { throw new Error('[flue] Persisted dispatch submission row is malformed.'); } @@ -313,6 +451,8 @@ function parseDispatchSubmission(row: SqlRow): SqlAgentDispatchSubmission { input, status: row.status, acceptedAt: row.accepted_at, + ...(typeof row.attempt_id === 'string' ? { attemptId: row.attempt_id } : {}), + ...(typeof row.started_at === 'number' ? { startedAt: row.started_at } : {}), ...(typeof row.completed_at === 'number' ? { completedAt: row.completed_at } : {}), ...(typeof row.error === 'string' ? { error: row.error } : {}), }; diff --git a/packages/runtime/test/build-plugin-cloudflare.test.ts b/packages/runtime/test/build-plugin-cloudflare.test.ts index fe93af37..6657d1c4 100644 --- a/packages/runtime/test/build-plugin-cloudflare.test.ts +++ b/packages/runtime/test/build-plugin-cloudflare.test.ts @@ -46,7 +46,7 @@ describe('CloudflarePlugin', () => { expect(entry).not.toContain('CREATE TABLE IF NOT EXISTS flue_sessions'); }); - it('pre-arms SQL-backed dispatch admission and reconciles runnable rows through managed Fibers', async () => { + it('pre-arms SQL-backed dispatch admission and drains claimed rows without managed Fibers', async () => { const entry = await new CloudflarePlugin().generateEntryPoint( testBuildContext({ agents: [{ name: 'assistant', filePath: '/fixture/agents/assistant.ts' }], @@ -56,20 +56,35 @@ describe('CloudflarePlugin', () => { expect(entry).toContain( `async onStart(props) { if (typeof super.onStart === 'function') await super.onStart(props); + await armFlueAgentSubmissionRetry(this); await reconcileFlueAgentSubmissions(this, "assistant", { preserveSuccessor: true }); } async __flueWakeAgentSubmissions(wake) { await reconcileFlueAgentSubmissions(this, "assistant", { preserveSuccessor: true, executingWake: wake }); + } + + async __flueRetryAgentSubmissions(_payload, schedule) { + if (!(await reconcileFlueAgentSubmissions(this, "assistant"))) { + await this.cancelSchedule(schedule.id); + } }`, ); expect(entry).toContain("const FLUE_AGENT_SUBMISSION_WAKE_CALLBACK = '__flueWakeAgentSubmissions';"); + expect(entry).toContain("const FLUE_AGENT_SUBMISSION_RETRY_CALLBACK = '__flueRetryAgentSubmissions';"); + expect(entry).toContain("return doInstance.scheduleEvery(FLUE_AGENT_SUBMISSION_RETRY_SECONDS, FLUE_AGENT_SUBMISSION_RETRY_CALLBACK);"); expect(entry).toContain("await armFlueAgentSubmissionAdmissionWakes(doInstance);\n let submission;"); expect(entry).toContain('submission = getAgentExecutionStore(doInstance).submissions.admitDispatch(input);'); - expect(entry).toContain('const runnable = submissions.listRunnableDispatches();'); - expect(entry).toContain('metadata: { input: submission.input, submissionSequence: submission.sequence },'); - expect(entry).toContain('submissions.hasQueuedDispatchForSession(doInstance.name, session)'); - expect(entry).toContain('async function waitForEarlierLegacyManagedDispatch'); + expect(entry).toContain('for (const submission of submissions.listRunningDispatches()) {'); + expect(entry).toContain('const claimed = submissions.claimDispatch(submission.submissionId, crypto.randomUUID());'); + expect(entry).toContain("void doInstance.runFiber('flue:dispatch-attempt', async (fiberCtx) => {"); + expect(entry).toContain("fiberCtx.stash({ submissionId: submission.submissionId, attemptId: submission.attemptId });"); + expect(entry).toContain('submissions.hasUnsettledDispatchForSession(doInstance.name, session)'); + expect(entry).toContain('if (directMarkers.blockAll || directMarkers.sessions.has(session)) {'); + expect(entry).toContain('getAgentExecutionStore(doInstance).submissions.adoptLegacyDispatches(dispatches.map((dispatch) => dispatch.input));'); + expect(entry).toContain("return handleFlueDispatchAttemptRecovered(ctx, this);"); + expect(entry).not.toContain("startFiber('flue:dispatch'"); + expect(entry).not.toContain('inspectFiberByKey'); expect(entry).not.toContain('ctx.storage.setAlarm'); }); diff --git a/packages/runtime/test/cloudflare-agent-execution-store.test.ts b/packages/runtime/test/cloudflare-agent-execution-store.test.ts index ba66fd47..7feb5415 100644 --- a/packages/runtime/test/cloudflare-agent-execution-store.test.ts +++ b/packages/runtime/test/cloudflare-agent-execution-store.test.ts @@ -11,6 +11,17 @@ function makeFakeSql() { const db = new DatabaseSync(':memory:'); return { db, + transactionSync(closure: () => T): T { + db.exec('BEGIN'); + try { + const result = closure(); + db.exec('COMMIT'); + return result; + } catch (error) { + db.exec('ROLLBACK'); + throw error; + } + }, sql: { exec(query: string, ...bindings: unknown[]) { const stmt = db.prepare(query); @@ -57,7 +68,7 @@ function sessionData(): SessionData { describe('createSqlAgentExecutionStore()', () => { it('loads, saves, and deletes existing flue_sessions rows when SQLite snapshot persistence is initialized', async () => { - const { db, sql } = makeFakeSql(); + const { db, sql, transactionSync } = makeFakeSql(); db.exec( 'CREATE TABLE flue_sessions (id TEXT PRIMARY KEY, data TEXT NOT NULL, updated_at INTEGER NOT NULL)', ); @@ -67,7 +78,7 @@ describe('createSqlAgentExecutionStore()', () => { 1, ); - const store = createSqlAgentExecutionStore({ sql }, 'FlueAssistantAgent'); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); expect(await store.sessions.load('existing')).toEqual(sessionData()); await store.sessions.save('saved', sessionData()); @@ -77,9 +88,9 @@ describe('createSqlAgentExecutionStore()', () => { }); it('creates the initial flue_agent_submissions schema and ordering indexes when initialized', () => { - const { db, sql } = makeFakeSql(); + const { db, sql, transactionSync } = makeFakeSql(); - createSqlAgentExecutionStore({ sql }, 'FlueAssistantAgent'); + createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); expect( db.prepare("SELECT name FROM pragma_table_info('flue_agent_submissions') ORDER BY cid").all(), @@ -112,8 +123,8 @@ describe('createSqlAgentExecutionStore()', () => { }); it('admits one queued dispatch row when the same submission is replayed', () => { - const { db, sql } = makeFakeSql(); - const store = createSqlAgentExecutionStore({ sql }, 'FlueAssistantAgent'); + const { db, sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); const first = store.submissions.admitDispatch(dispatchInput()); const replay = store.submissions.admitDispatch(dispatchInput()); @@ -131,8 +142,8 @@ describe('createSqlAgentExecutionStore()', () => { }); it('rejects conflicting replay when one dispatch id is reused with another payload', () => { - const { sql } = makeFakeSql(); - const store = createSqlAgentExecutionStore({ sql }, 'FlueAssistantAgent'); + const { sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); store.submissions.admitDispatch(dispatchInput()); expect(() => @@ -141,8 +152,8 @@ describe('createSqlAgentExecutionStore()', () => { }); it('lists queued dispatches in admission order and selects one runnable head per session', () => { - const { sql } = makeFakeSql(); - const store = createSqlAgentExecutionStore({ sql }, 'FlueAssistantAgent'); + const { sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); const first = store.submissions.admitDispatch(dispatchInput()); const second = store.submissions.admitDispatch(dispatchInput({ dispatchId: 'dispatch-2' })); const other = store.submissions.admitDispatch( @@ -151,14 +162,38 @@ describe('createSqlAgentExecutionStore()', () => { expect(store.submissions.listQueuedDispatches()).toEqual([first, second, other]); expect(store.submissions.listRunnableDispatches()).toEqual([first, other]); - expect(store.submissions.hasEarlierQueuedDispatch(first)).toBe(false); - expect(store.submissions.hasEarlierQueuedDispatch(second)).toBe(true); - expect(store.submissions.hasEarlierQueuedDispatch(other)).toBe(false); + }); + + it('claims only runnable session heads while allowing separate sessions to claim independently', () => { + const { sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); + store.submissions.admitDispatch(dispatchInput()); + store.submissions.admitDispatch(dispatchInput({ dispatchId: 'dispatch-2' })); + store.submissions.admitDispatch(dispatchInput({ dispatchId: 'dispatch-3', session: 'other' })); + + const first = store.submissions.claimDispatch('dispatch-1', 'attempt-1'); + const blocked = store.submissions.claimDispatch('dispatch-2', 'attempt-2'); + const other = store.submissions.claimDispatch('dispatch-3', 'attempt-3'); + + expect(first).toMatchObject({ + submissionId: 'dispatch-1', + status: 'running', + attemptId: 'attempt-1', + startedAt: expect.any(Number), + }); + expect(blocked).toBeNull(); + expect(other).toMatchObject({ + submissionId: 'dispatch-3', + status: 'running', + attemptId: 'attempt-3', + }); + expect(store.submissions.listRunningDispatches()).toEqual([first, other]); + expect(store.submissions.listRunnableDispatches()).toEqual([]); }); it('terminalizes malformed queued payloads while returning healthy runnable rows', () => { - const { db, sql } = makeFakeSql(); - const store = createSqlAgentExecutionStore({ sql }, 'FlueAssistantAgent'); + const { db, sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); store.submissions.admitDispatch(dispatchInput({ dispatchId: 'healthy' })); db.prepare( `INSERT INTO flue_agent_submissions @@ -176,26 +211,82 @@ describe('createSqlAgentExecutionStore()', () => { ).toMatchObject({ status: 'error', error: expect.any(String) }); }); - it('reports queued session visibility until a dispatch completes', () => { - const { sql } = makeFakeSql(); - const store = createSqlAgentExecutionStore({ sql }, 'FlueAssistantAgent'); + it('adopts legacy dispatches ahead of existing SQL submissions in historical order', () => { + const { sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); + store.submissions.admitDispatch(dispatchInput({ dispatchId: 'current' })); + + const adopted = store.submissions.adoptLegacyDispatches([ + dispatchInput({ dispatchId: 'legacy-1' }), + dispatchInput({ dispatchId: 'legacy-2' }), + ]); + + expect(adopted.map((submission) => submission.submissionId)).toEqual(['legacy-1', 'legacy-2']); + expect(store.submissions.listQueuedDispatches().map((submission) => submission.submissionId)).toEqual([ + 'legacy-1', + 'legacy-2', + 'current', + ]); + }); + + it('adopts more than sixteen legacy dispatches without exceeding the Cloudflare SQL binding limit', () => { + const { sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); + store.submissions.admitDispatch(dispatchInput({ dispatchId: 'current' })); + const legacy = Array.from({ length: 17 }, (_, index) => + dispatchInput({ dispatchId: `legacy-${index + 1}` }), + ); + + store.submissions.adoptLegacyDispatches(legacy); + + expect(store.submissions.listQueuedDispatches().map((submission) => submission.submissionId)).toEqual([ + ...legacy.map((input) => input.dispatchId), + 'current', + ]); + }); + + it('rotates recovered attempt ownership without allowing a stale attempt to settle', () => { + const { sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); + store.submissions.admitDispatch(dispatchInput()); + store.submissions.claimDispatch('dispatch-1', 'attempt-1'); + + const recovered = store.submissions.recoverDispatchAttempt('dispatch-1', 'attempt-1', 'attempt-2'); + store.submissions.completeDispatch('dispatch-1', 'attempt-1'); + + expect(recovered).toMatchObject({ status: 'running', attemptId: 'attempt-2' }); + expect(store.submissions.getDispatch('dispatch-1')).toMatchObject({ + status: 'running', + attemptId: 'attempt-2', + }); + }); + + it('reports unsettled session visibility until a claimed dispatch completes', () => { + const { sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); store.submissions.admitDispatch(dispatchInput({ session: 'case-1' })); - expect(store.submissions.hasQueuedDispatchForSession('agent-1', 'case-1')).toBe(true); - expect(store.submissions.hasQueuedDispatchForSession('agent-1', 'case-2')).toBe(false); - store.submissions.completeDispatch('dispatch-1'); - expect(store.submissions.hasQueuedDispatchForSession('agent-1', 'case-1')).toBe(false); + expect(store.submissions.hasUnsettledDispatches()).toBe(true); + expect(store.submissions.hasUnsettledDispatchForSession('agent-1', 'case-1')).toBe(true); + expect(store.submissions.hasUnsettledDispatchForSession('agent-1', 'case-2')).toBe(false); + store.submissions.claimDispatch('dispatch-1', 'attempt-1'); + expect(store.submissions.hasUnsettledDispatchForSession('agent-1', 'case-1')).toBe(true); + store.submissions.completeDispatch('dispatch-1', 'attempt-1'); + expect(store.submissions.hasUnsettledDispatches()).toBe(false); + expect(store.submissions.hasUnsettledDispatchForSession('agent-1', 'case-1')).toBe(false); expect(store.submissions.getDispatch('dispatch-1')).toMatchObject({ status: 'completed' }); }); - it('keeps the first terminal dispatch state when a later settlement races it', () => { - const { sql } = makeFakeSql(); - const store = createSqlAgentExecutionStore({ sql }, 'FlueAssistantAgent'); + it('ignores stale-attempt settlement and keeps the first owning terminal dispatch state', () => { + const { sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); store.submissions.admitDispatch(dispatchInput()); + store.submissions.claimDispatch('dispatch-1', 'attempt-1'); - store.submissions.failDispatch('dispatch-1', new Error('first failure')); - store.submissions.completeDispatch('dispatch-1'); - store.submissions.failDispatch('dispatch-1', new Error('later failure')); + store.submissions.completeDispatch('dispatch-1', 'stale-attempt'); + store.submissions.failDispatch('dispatch-1', 'attempt-1', new Error('first failure')); + store.submissions.completeDispatch('dispatch-1', 'attempt-1'); + store.submissions.failDispatch('dispatch-1', 'attempt-1', new Error('later failure')); expect(store.submissions.getDispatch('dispatch-1')).toMatchObject({ status: 'error', @@ -209,11 +300,19 @@ describe('createSqlAgentExecutionStore()', () => { ); }); - it('reports SQL initialization failures without misdiagnosing missing SQLite', () => { + it('rejects SQLite-compatible storage without synchronous transaction support', () => { const { sql } = makeFakeSql(); - sql.exec('CREATE TABLE flue_agent_submissions (sequence INTEGER PRIMARY KEY AUTOINCREMENT)'); expect(() => createSqlAgentExecutionStore({ sql }, 'FlueAssistantAgent')).toThrow( + '[flue] Cloudflare durable agent class "FlueAssistantAgent" requires Durable Object SQLite.', + ); + }); + + it('reports SQL initialization failures without misdiagnosing missing SQLite', () => { + const { sql, transactionSync } = makeFakeSql(); + sql.exec('CREATE TABLE flue_agent_submissions (sequence INTEGER PRIMARY KEY AUTOINCREMENT)'); + + expect(() => createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent')).toThrow( '[flue] Cloudflare durable agent class "FlueAssistantAgent" could not initialize its SQLite execution store. Underlying error: no such column: status', ); }); From 657e1785eaf19b6542d9ac3cd310fb64eb887691 Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" <622227+FredKSchott@users.noreply.github.com> Date: Wed, 3 Jun 2026 17:42:07 -0500 Subject: [PATCH 06/17] feat(cloudflare): reconcile interrupted dispatch attempts --- .../cli/src/lib/build-plugin-cloudflare.ts | 68 ++++++- .../src/cloudflare/agent-execution-store.ts | 87 +++++++-- packages/runtime/src/internal.ts | 1 + .../runtime/src/runtime/dispatch-queue.ts | 6 + packages/runtime/src/runtime/handle-agent.ts | 41 ++++- packages/runtime/src/session.ts | 36 +++- .../test/build-plugin-cloudflare.test.ts | 9 + .../cloudflare-agent-execution-store.test.ts | 81 ++++++++- packages/runtime/test/dispatch.test.ts | 168 +++++++++++++++++- 9 files changed, 459 insertions(+), 38 deletions(-) diff --git a/packages/cli/src/lib/build-plugin-cloudflare.ts b/packages/cli/src/lib/build-plugin-cloudflare.ts index e2cc6a71..f151deda 100644 --- a/packages/cli/src/lib/build-plugin-cloudflare.ts +++ b/packages/cli/src/lib/build-plugin-cloudflare.ts @@ -224,6 +224,7 @@ import { validateAgentDispatchAdmission, assertCurrentDispatchInput, createDispatchAgentHandler, + createDispatchInputInspectionHandler, reserveDispatchAgentSession, failRecoveredRun, configureFlueRuntime, @@ -363,6 +364,7 @@ const FLUE_AGENT_EXECUTION_STORE = Symbol('flueAgentExecutionStore'); const FLUE_AGENT_SUBMISSION_WAKE_CALLBACK = '__flueWakeAgentSubmissions'; const FLUE_AGENT_SUBMISSION_RETRY_CALLBACK = '__flueRetryAgentSubmissions'; const FLUE_AGENT_SUBMISSION_RETRY_SECONDS = 30; +const FLUE_AGENT_SUBMISSION_ATTEMPT_STALE_MS = 15 * 60 * 1000; const INTERNAL_DISPATCH_PATH = '/__flue/internal/dispatch'; const dispatchQueue = { async enqueue(input) { @@ -495,9 +497,8 @@ async function handleFlueDispatchAttemptRecovered(ctx, doInstance) { const attemptId = ctx.snapshot?.attemptId; if (typeof submissionId !== 'string' || typeof attemptId !== 'string') return; const submissions = getAgentExecutionStore(doInstance).submissions; - const submission = submissions.getDispatch(submissionId); - if (submission?.status !== 'running' || submission.attemptId !== attemptId) return; - submissions.recoverDispatchAttempt(submissionId, attemptId, crypto.randomUUID()); + const submission = submissions.requestDispatchRecovery(submissionId, attemptId); + if (!submission) return; await armFlueAgentSubmissionAdmissionWakes(doInstance); } @@ -593,9 +594,10 @@ async function reconcileFlueAgentSubmissions(doInstance, agentName, options = {} const directMarkers = listActiveDirectAgentSessionMarkers(doInstance); if (attemptMarkers.blockAll || directMarkers.blockAll) return true; for (const submission of submissions.listRunningDispatches()) { - if (attemptMarkers.keys.has(dispatchAttemptMarkerKey(submission))) continue; + if (activeFlueAgentDispatchAttempts.has(dispatchAttemptLocalKey(doInstance, submission))) continue; + if (attemptMarkers.keys.has(dispatchAttemptMarkerKey(submission)) && submission.recoveryRequestedAt === undefined) continue; if (directMarkers.sessions.has(submission.session) || legacySessions.has(submission.session)) continue; - startSqlAgentDispatchAttempt(submission, doInstance, agentName); + await reconcileInterruptedSqlAgentDispatch(submission, doInstance, agentName); } for (const submission of submissions.listRunnableDispatches()) { if (directMarkers.sessions.has(submission.session) || legacySessions.has(submission.session)) continue; @@ -609,9 +611,42 @@ async function reconcileFlueAgentSubmissions(doInstance, agentName, options = {} return submissions.hasUnsettledDispatches(); } +async function reconcileInterruptedSqlAgentDispatch(submission, doInstance, agentName) { + const { attemptId, input } = submission; + if (!attemptId) return; + const submissions = getAgentExecutionStore(doInstance).submissions; + if (submission.inputAppliedAt === undefined) { + const agent = createdAgents[agentName]; + if (!agent) throw new Error('[flue] Dispatch target unavailable during durable reconciliation.'); + const request = new Request('https://flue.invalid' + INTERNAL_DISPATCH_PATH, { method: 'POST' }); + const ctx = createAgentContextForRequest(doInstance.name, input, doInstance, request, undefined, input.dispatchId); + const state = await runWithInstanceContext(doInstance, agentRuntimeIdentity(agentName), () => + createDispatchInputInspectionHandler(agent, input)(ctx), + ); + if (state === 'absent') { + submissions.requeueDispatchBeforeInputApplied(input.dispatchId, attemptId); + return; + } + submissions.failDispatch(input.dispatchId, attemptId, new Error('[flue] Dispatch attempt was interrupted after canonical input persistence but before the input-application marker was recorded. Provider replay was not attempted.')); + return; + } + const agent = createdAgents[agentName]; + if (!agent) throw new Error('[flue] Dispatch target unavailable during durable reconciliation.'); + const request = new Request('https://flue.invalid' + INTERNAL_DISPATCH_PATH, { method: 'POST' }); + const ctx = createAgentContextForRequest(doInstance.name, input, doInstance, request, undefined, input.dispatchId); + const state = await runWithInstanceContext(doInstance, agentRuntimeIdentity(agentName), () => + createDispatchInputInspectionHandler(agent, input)(ctx), + ); + if (state === 'completed') { + submissions.completeDispatch(input.dispatchId, attemptId); + return; + } + submissions.failDispatch(input.dispatchId, attemptId, new Error('[flue] Dispatch attempt was interrupted after input application without a completed canonical response. Provider replay was not attempted.')); +} + function startSqlAgentDispatchAttempt(submission, doInstance, agentName) { if (submission.status !== 'running' || !submission.attemptId) return; - const attemptKey = doInstance.ctx.id.toString() + ':' + submission.attemptId; + const attemptKey = dispatchAttemptLocalKey(doInstance, submission); if (activeFlueAgentDispatchAttempts.has(attemptKey)) return; activeFlueAgentDispatchAttempts.add(attemptKey); assertAgentsDurabilityApi(doInstance, 'runFiber'); @@ -625,6 +660,10 @@ function startSqlAgentDispatchAttempt(submission, doInstance, agentName) { }); } +function dispatchAttemptLocalKey(doInstance, submission) { + return doInstance.ctx.id.toString() + ':' + submission.attemptId; +} + function dispatchAttemptMarkerKey(submission) { return submission.submissionId + ':' + submission.attemptId; } @@ -633,9 +672,14 @@ function listActiveSqlAgentDispatchAttemptMarkers(doInstance) { const keys = new Set(); let blockAll = false; const rows = doInstance.ctx.storage.sql.exec( - "SELECT snapshot FROM cf_agents_runs WHERE name = 'flue:dispatch-attempt'", + "SELECT snapshot, created_at FROM cf_agents_runs WHERE name = 'flue:dispatch-attempt'", ).toArray(); for (const row of rows) { + if (typeof row.created_at !== 'number') { + blockAll = true; + continue; + } + if (Date.now() - row.created_at > FLUE_AGENT_SUBMISSION_ATTEMPT_STALE_MS) continue; let snapshot; try { snapshot = typeof row.snapshot === 'string' ? JSON.parse(row.snapshot) : null; @@ -729,7 +773,15 @@ async function processSqlAgentDispatch(submission, doInstance, agentName) { releaseSessionLock = await reserveDispatchAgentSession(target, input); const request = new Request('https://flue.invalid' + INTERNAL_DISPATCH_PATH, { method: 'POST' }); const ctx = createAgentContextForRequest(doInstance.name, input, doInstance, request, undefined, input.dispatchId); - await runWithInstanceContext(doInstance, agentRuntimeIdentity(agentName), () => createDispatchAgentHandler(agent, input)(ctx)); + await runWithInstanceContext(doInstance, agentRuntimeIdentity(agentName), () => + createDispatchAgentHandler(agent, input, { + onInputApplied: () => { + if (!submissions.markDispatchInputApplied(input.dispatchId, attemptId)) { + throw new Error('[flue] Dispatch attempt lost ownership before input application.'); + } + }, + })(ctx), + ); submissions.completeDispatch(input.dispatchId, attemptId); } catch (error) { submissions.failDispatch(input.dispatchId, attemptId, error); diff --git a/packages/runtime/src/cloudflare/agent-execution-store.ts b/packages/runtime/src/cloudflare/agent-execution-store.ts index 6388c29f..9aef232e 100644 --- a/packages/runtime/src/cloudflare/agent-execution-store.ts +++ b/packages/runtime/src/cloudflare/agent-execution-store.ts @@ -28,6 +28,8 @@ export interface SqlAgentDispatchSubmission { readonly status: SqlAgentSubmissionStatus; readonly acceptedAt: number; readonly attemptId?: string; + readonly inputAppliedAt?: number; + readonly recoveryRequestedAt?: number; readonly startedAt?: number; readonly completedAt?: number; readonly error?: string; @@ -43,10 +45,11 @@ export interface SqlAgentSubmissionStore { listRunningDispatches(): SqlAgentDispatchSubmission[]; hasUnsettledDispatchForSession(instanceId: string, session: string): boolean; claimDispatch(submissionId: string, attemptId: string): SqlAgentDispatchSubmission | null; - recoverDispatchAttempt( + markDispatchInputApplied(submissionId: string, attemptId: string): SqlAgentDispatchSubmission | null; + requestDispatchRecovery(submissionId: string, attemptId: string): SqlAgentDispatchSubmission | null; + requeueDispatchBeforeInputApplied( submissionId: string, - expectedAttemptId: string, - nextAttemptId: string, + attemptId: string, ): SqlAgentDispatchSubmission | null; completeDispatch(submissionId: string, attemptId: string): void; failDispatch(submissionId: string, attemptId: string, error: unknown): void; @@ -227,7 +230,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { this.sql .exec( `SELECT sequence, submission_id, session, session_key, kind, payload, status, - accepted_at, attempt_id, started_at, completed_at, error + accepted_at, attempt_id, input_applied_at, recovery_requested_at, started_at, completed_at, error FROM flue_agent_submissions WHERE kind = 'dispatch' AND status = 'queued' ORDER BY sequence ASC`, @@ -241,7 +244,8 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { .exec( `SELECT current.sequence, current.submission_id, current.session, current.session_key, current.kind, current.payload, current.status, current.accepted_at, - current.attempt_id, current.started_at, current.completed_at, current.error + current.attempt_id, current.input_applied_at, current.recovery_requested_at, + current.started_at, current.completed_at, current.error FROM flue_agent_submissions AS current WHERE current.kind = 'dispatch' AND current.status = 'queued' AND NOT EXISTS ( @@ -263,7 +267,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { this.sql .exec( `SELECT sequence, submission_id, session, session_key, kind, payload, status, - accepted_at, attempt_id, started_at, completed_at, error + accepted_at, attempt_id, input_applied_at, recovery_requested_at, started_at, completed_at, error FROM flue_agent_submissions WHERE kind = 'dispatch' AND status = 'running' ORDER BY sequence ASC`, @@ -309,26 +313,52 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { : null; } - recoverDispatchAttempt( - submissionId: string, - expectedAttemptId: string, - nextAttemptId: string, - ): SqlAgentDispatchSubmission | null { + markDispatchInputApplied(submissionId: string, attemptId: string): SqlAgentDispatchSubmission | null { + this.sql.exec( + `UPDATE flue_agent_submissions + SET input_applied_at = COALESCE(input_applied_at, ?) + WHERE submission_id = ? AND kind = 'dispatch' AND status = 'running' AND attempt_id = ?`, + Date.now(), + submissionId, + attemptId, + ); + const submission = this.getDispatch(submissionId); + return submission?.status === 'running' && submission.attemptId === attemptId + ? submission + : null; + } + + requestDispatchRecovery(submissionId: string, attemptId: string): SqlAgentDispatchSubmission | null { this.sql.exec( `UPDATE flue_agent_submissions - SET attempt_id = ?, started_at = ? + SET recovery_requested_at = COALESCE(recovery_requested_at, ?) WHERE submission_id = ? AND kind = 'dispatch' AND status = 'running' AND attempt_id = ?`, - nextAttemptId, Date.now(), submissionId, - expectedAttemptId, + attemptId, ); const submission = this.getDispatch(submissionId); - return submission?.status === 'running' && submission.attemptId === nextAttemptId + return submission?.status === 'running' && submission.attemptId === attemptId ? submission : null; } + requeueDispatchBeforeInputApplied( + submissionId: string, + attemptId: string, + ): SqlAgentDispatchSubmission | null { + this.sql.exec( + `UPDATE flue_agent_submissions + SET status = 'queued', attempt_id = NULL, recovery_requested_at = NULL, started_at = NULL + WHERE submission_id = ? AND kind = 'dispatch' AND status = 'running' + AND attempt_id = ? AND input_applied_at IS NULL`, + submissionId, + attemptId, + ); + const submission = this.getDispatch(submissionId); + return submission?.status === 'queued' ? submission : null; + } + completeDispatch(submissionId: string, attemptId: string): void { this.sql.exec( `UPDATE flue_agent_submissions @@ -396,7 +426,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { return this.sql .exec( `SELECT sequence, submission_id, session, session_key, kind, payload, status, - accepted_at, attempt_id, started_at, completed_at, error + accepted_at, attempt_id, input_applied_at, recovery_requested_at, started_at, completed_at, error FROM flue_agent_submissions WHERE submission_id = ? AND kind = 'dispatch' LIMIT 1`, @@ -420,7 +450,18 @@ function parseDispatchSubmission(row: SqlRow): SqlAgentDispatchSubmission { row.status !== 'error') || typeof row.accepted_at !== 'number' || (row.attempt_id !== null && row.attempt_id !== undefined && typeof row.attempt_id !== 'string') || + (row.input_applied_at !== null && + row.input_applied_at !== undefined && + typeof row.input_applied_at !== 'number') || + (row.recovery_requested_at !== null && + row.recovery_requested_at !== undefined && + typeof row.recovery_requested_at !== 'number') || (row.started_at !== null && row.started_at !== undefined && typeof row.started_at !== 'number') || + (row.status === 'queued' && + (row.attempt_id !== null || + row.input_applied_at !== null || + row.recovery_requested_at !== null || + row.started_at !== null)) || (row.status === 'running' && (typeof row.attempt_id !== 'string' || typeof row.started_at !== 'number')) ) { @@ -452,6 +493,10 @@ function parseDispatchSubmission(row: SqlRow): SqlAgentDispatchSubmission { status: row.status, acceptedAt: row.accepted_at, ...(typeof row.attempt_id === 'string' ? { attemptId: row.attempt_id } : {}), + ...(typeof row.input_applied_at === 'number' ? { inputAppliedAt: row.input_applied_at } : {}), + ...(typeof row.recovery_requested_at === 'number' + ? { recoveryRequestedAt: row.recovery_requested_at } + : {}), ...(typeof row.started_at === 'number' ? { startedAt: row.started_at } : {}), ...(typeof row.completed_at === 'number' ? { completedAt: row.completed_at } : {}), ...(typeof row.error === 'string' ? { error: row.error } : {}), @@ -481,11 +526,14 @@ function ensureSubmissionTable(sql: SqlStorage): void { accepted_at INTEGER NOT NULL, attempt_id TEXT, input_applied_at INTEGER, + recovery_requested_at INTEGER, started_at INTEGER, completed_at INTEGER, error TEXT )`, ); + ensureSubmissionColumn(sql, 'input_applied_at', 'INTEGER'); + ensureSubmissionColumn(sql, 'recovery_requested_at', 'INTEGER'); sql.exec( 'CREATE INDEX IF NOT EXISTS flue_agent_submissions_status_sequence_idx ON flue_agent_submissions (status, sequence ASC)', ); @@ -493,3 +541,10 @@ function ensureSubmissionTable(sql: SqlStorage): void { 'CREATE INDEX IF NOT EXISTS flue_agent_submissions_session_status_sequence_idx ON flue_agent_submissions (session_key, status, sequence ASC)', ); } + +function ensureSubmissionColumn(sql: SqlStorage, name: string, type: string): void { + const rows = sql.exec(`SELECT name FROM pragma_table_info('flue_agent_submissions')`).toArray(); + if (!rows.some((row) => row.name === name)) { + sql.exec(`ALTER TABLE flue_agent_submissions ADD COLUMN ${name} ${type}`); + } +} diff --git a/packages/runtime/src/internal.ts b/packages/runtime/src/internal.ts index 44ebff9f..3260efc6 100644 --- a/packages/runtime/src/internal.ts +++ b/packages/runtime/src/internal.ts @@ -82,6 +82,7 @@ export { createAgentDispatchProcessor, createDirectAgentHandler, createDispatchAgentHandler, + createDispatchInputInspectionHandler, failRecoveredRun, handleAgentRequest, handleWorkflowRequest, diff --git a/packages/runtime/src/runtime/dispatch-queue.ts b/packages/runtime/src/runtime/dispatch-queue.ts index 4b49df54..10d799f3 100644 --- a/packages/runtime/src/runtime/dispatch-queue.ts +++ b/packages/runtime/src/runtime/dispatch-queue.ts @@ -17,6 +17,12 @@ export function assertCurrentDispatchInput(value: unknown): asserts value is Dis } } +export type DispatchInputInspection = 'absent' | 'applied' | 'completed' | 'advanced'; + +export interface ProcessDispatchInputOptions { + onInputApplied?: () => Promise | void; +} + export interface DispatchProcessor { process(input: DispatchInput): Promise | void; } diff --git a/packages/runtime/src/runtime/handle-agent.ts b/packages/runtime/src/runtime/handle-agent.ts index f7befca9..7752e4f2 100644 --- a/packages/runtime/src/runtime/handle-agent.ts +++ b/packages/runtime/src/runtime/handle-agent.ts @@ -1,7 +1,6 @@ /** Shared per-agent HTTP dispatcher for the Node and Cloudflare targets. */ import type { FlueContextInternal } from '../client.ts'; -import { isTaskSessionName } from '../session-identity.ts'; import { InvalidRequestError, parseJsonBody, @@ -10,6 +9,7 @@ import { toHttpResponse, toPublicError, } from '../errors.ts'; +import { isTaskSessionName } from '../session-identity.ts'; import type { AttachedAgentEvent, AttachedAgentEventCallback, @@ -22,7 +22,9 @@ import type { import { assertCurrentDispatchInput, type DispatchInput, + type DispatchInputInspection, type DispatchProcessor, + type ProcessDispatchInputOptions, } from './dispatch-queue.ts'; import { streamActiveRunEvents } from './handle-run-routes.ts'; import { generateWorkflowRunId } from './ids.ts'; @@ -40,7 +42,8 @@ interface DirectRequestSession { } interface DispatchSession { - processDispatchInput(input: DispatchInput): PromiseLike; + inspectDispatchInput?(input: DispatchInput): DispatchInputInspection; + processDispatchInput(input: DispatchInput, options?: ProcessDispatchInputOptions): PromiseLike; } export interface AgentSessionTarget { @@ -101,8 +104,16 @@ export async function validateAgentDispatchAdmission( export function createDispatchAgentHandler( agent: CreatedAgentHandler, input: DispatchInput, + options?: ProcessDispatchInputOptions, ): AgentHandler { - return (ctx) => processAgentDispatch(ctx, agent, input); + return (ctx) => processAgentDispatch(ctx, agent, input, options); +} + +export function createDispatchInputInspectionHandler( + agent: CreatedAgentHandler, + input: DispatchInput, +): AgentHandler { + return (ctx) => inspectAgentDispatchInput(ctx, agent, input); } export async function reserveDispatchAgentSession( @@ -116,13 +127,35 @@ async function processAgentDispatch( ctx: FlueContextInternal, agent: CreatedAgentHandler, input: DispatchInput, + options?: ProcessDispatchInputOptions, ): Promise { + const session = await openAgentDispatchSession(ctx, agent, input); + return session.processDispatchInput(input, options); +} + +async function inspectAgentDispatchInput( + ctx: FlueContextInternal, + agent: CreatedAgentHandler, + input: DispatchInput, +): Promise { + const session = await openAgentDispatchSession(ctx, agent, input); + if (!session.inspectDispatchInput) { + throw new Error('[flue] Internal session does not support dispatch input inspection.'); + } + return session.inspectDispatchInput(input); +} + +async function openAgentDispatchSession( + ctx: FlueContextInternal, + agent: CreatedAgentHandler, + input: DispatchInput, +): Promise { const harness = await ctx.initializeCreatedAgent(agent, undefined); const session = await harness.session(input.session); if (!isDispatchSession(session)) { throw new Error('[flue] Internal session does not support dispatch input processing.'); } - return session.processDispatchInput(input); + return session; } function isDispatchInput(value: unknown): value is DispatchInput { diff --git a/packages/runtime/src/session.ts b/packages/runtime/src/session.ts index a784e0b4..192bb9a2 100644 --- a/packages/runtime/src/session.ts +++ b/packages/runtime/src/session.ts @@ -49,7 +49,11 @@ import { type ResultToolBundle, ResultUnavailableError, } from './result.ts'; -import type { DispatchInput } from './runtime/dispatch-queue.ts'; +import type { + DispatchInput, + DispatchInputInspection, + ProcessDispatchInputOptions, +} from './runtime/dispatch-queue.ts'; import { generateOperationId, generateTurnId } from './runtime/ids.ts'; import { getProviderConfiguration, getRegisteredApiKey } from './runtime/providers.ts'; import { createFlueFs } from './sandbox.ts'; @@ -548,6 +552,10 @@ function isRetryableModelError(message: AssistantMessage): boolean { ); } +function isCompletedAssistantResponse(message: AssistantMessage): boolean { + return message.stopReason === 'stop' || message.stopReason === 'length'; +} + function modelRetryDelayMs(attempt: number): number { const baseDelay = TRANSIENT_MODEL_RETRY_BASE_DELAY_MS * 2 ** (attempt - 1); return Math.round(baseDelay * (0.75 + Math.random() * 0.25)); @@ -879,9 +887,27 @@ export class Session implements FlueSession { ); } - processDispatchInput(input: DispatchInput): CallHandle { + inspectDispatchInput(input: DispatchInput): DispatchInputInspection { + const inputEntry = this.history.findDispatchInput(input.dispatchId); + if (!inputEntry) return 'absent'; + const following = this.history.getActivePathSince(inputEntry.id); + if (following.some((entry) => entry.type === 'message' && entry.message.role === 'user')) { + return 'advanced'; + } + const assistant = following.findLast( + (entry): entry is MessageEntry => entry.type === 'message' && entry.message.role === 'assistant', + )?.message as AssistantMessage | undefined; + return assistant && isCompletedAssistantResponse(assistant) ? 'completed' : 'applied'; + } + + processDispatchInput( + input: DispatchInput, + options?: ProcessDispatchInputOptions, + ): CallHandle { return createCallHandle(undefined, (signal) => - this.runOperation('prompt', signal, () => this.runPersistedDispatchInput(input, signal)), + this.runOperation('prompt', signal, () => + this.runPersistedDispatchInput(input, signal, options), + ), ); } @@ -2009,6 +2035,7 @@ export class Session implements FlueSession { private async runPersistedDispatchInput( input: DispatchInput, signal: AbortSignal, + options?: ProcessDispatchInputOptions, ): Promise { return this.runPersistedContextInput({ findInput: () => this.history.findDispatchInput(input.dispatchId), @@ -2023,6 +2050,7 @@ export class Session implements FlueSession { callSite: 'this dispatched input', persistenceError: '[flue] Failed to persist dispatched input.', recoveryError: '[flue] Cannot recover dispatched input after the session has advanced.', + onInputApplied: options?.onInputApplied, signal, }); } @@ -2035,6 +2063,7 @@ export class Session implements FlueSession { callSite: string; persistenceError: string; recoveryError: string; + onInputApplied?: () => Promise | void; signal: AbortSignal; }): Promise { return this.withCallOverrides( @@ -2053,6 +2082,7 @@ export class Session implements FlueSession { inputEntry = options.findInput(); } if (!inputEntry) throw new Error(options.persistenceError); + await options.onInputApplied?.(); const following = this.history.getActivePathSince(inputEntry.id); if (following.some((entry) => entry.type === 'message' && entry.message.role === 'user')) { throw new Error(options.recoveryError); diff --git a/packages/runtime/test/build-plugin-cloudflare.test.ts b/packages/runtime/test/build-plugin-cloudflare.test.ts index 6657d1c4..51334aa6 100644 --- a/packages/runtime/test/build-plugin-cloudflare.test.ts +++ b/packages/runtime/test/build-plugin-cloudflare.test.ts @@ -76,6 +76,15 @@ describe('CloudflarePlugin', () => { expect(entry).toContain("await armFlueAgentSubmissionAdmissionWakes(doInstance);\n let submission;"); expect(entry).toContain('submission = getAgentExecutionStore(doInstance).submissions.admitDispatch(input);'); expect(entry).toContain('for (const submission of submissions.listRunningDispatches()) {'); + expect(entry).toContain('if (activeFlueAgentDispatchAttempts.has(dispatchAttemptLocalKey(doInstance, submission))) continue;'); + expect(entry).toContain('if (attemptMarkers.keys.has(dispatchAttemptMarkerKey(submission)) && submission.recoveryRequestedAt === undefined) continue;'); + expect(entry).toContain('await reconcileInterruptedSqlAgentDispatch(submission, doInstance, agentName);'); + expect(entry).toContain('const submission = submissions.requestDispatchRecovery(submissionId, attemptId);'); + expect(entry).toContain("SELECT snapshot, created_at FROM cf_agents_runs WHERE name = 'flue:dispatch-attempt'"); + expect(entry).toContain('if (Date.now() - row.created_at > FLUE_AGENT_SUBMISSION_ATTEMPT_STALE_MS) continue;'); + expect(entry).toContain('submissions.requeueDispatchBeforeInputApplied(input.dispatchId, attemptId);'); + expect(entry).toContain('createDispatchInputInspectionHandler(agent, input)(ctx)'); + expect(entry).toContain('if (!submissions.markDispatchInputApplied(input.dispatchId, attemptId)) {'); expect(entry).toContain('const claimed = submissions.claimDispatch(submission.submissionId, crypto.randomUUID());'); expect(entry).toContain("void doInstance.runFiber('flue:dispatch-attempt', async (fiberCtx) => {"); expect(entry).toContain("fiberCtx.stash({ submissionId: submission.submissionId, attemptId: submission.attemptId });"); diff --git a/packages/runtime/test/cloudflare-agent-execution-store.test.ts b/packages/runtime/test/cloudflare-agent-execution-store.test.ts index 7feb5415..4d95f36a 100644 --- a/packages/runtime/test/cloudflare-agent-execution-store.test.ts +++ b/packages/runtime/test/cloudflare-agent-execution-store.test.ts @@ -105,6 +105,7 @@ describe('createSqlAgentExecutionStore()', () => { { name: 'accepted_at' }, { name: 'attempt_id' }, { name: 'input_applied_at' }, + { name: 'recovery_requested_at' }, { name: 'started_at' }, { name: 'completed_at' }, { name: 'error' }, @@ -122,6 +123,33 @@ describe('createSqlAgentExecutionStore()', () => { ]); }); + it('adds recovery columns when an existing submission table predates the current schema', () => { + const { db, sql, transactionSync } = makeFakeSql(); + db.exec(`CREATE TABLE flue_agent_submissions ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + submission_id TEXT NOT NULL UNIQUE, + session TEXT NOT NULL, + session_key TEXT NOT NULL, + kind TEXT NOT NULL, + payload TEXT NOT NULL, + status TEXT NOT NULL, + accepted_at INTEGER NOT NULL, + attempt_id TEXT, + started_at INTEGER, + completed_at INTEGER, + error TEXT + )`); + + createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); + + expect( + db.prepare("SELECT name FROM pragma_table_info('flue_agent_submissions') ORDER BY cid").all(), + ).toContainEqual({ name: 'input_applied_at' }); + expect( + db.prepare("SELECT name FROM pragma_table_info('flue_agent_submissions') ORDER BY cid").all(), + ).toContainEqual({ name: 'recovery_requested_at' }); + }); + it('admits one queued dispatch row when the same submission is replayed', () => { const { db, sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); @@ -211,6 +239,22 @@ describe('createSqlAgentExecutionStore()', () => { ).toMatchObject({ status: 'error', error: expect.any(String) }); }); + it('terminalizes impossible queued input markers instead of replaying them', () => { + const { db, sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); + store.submissions.admitDispatch(dispatchInput()); + db.prepare('UPDATE flue_agent_submissions SET input_applied_at = ? WHERE submission_id = ?').run( + 1, + 'dispatch-1', + ); + + expect(store.submissions.listRunnableDispatches()).toEqual([]); + expect(store.submissions.getDispatch('dispatch-1')).toMatchObject({ + status: 'error', + error: expect.any(String), + }); + }); + it('adopts legacy dispatches ahead of existing SQL submissions in historical order', () => { const { sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); @@ -245,20 +289,45 @@ describe('createSqlAgentExecutionStore()', () => { ]); }); - it('rotates recovered attempt ownership without allowing a stale attempt to settle', () => { + it('records input application and recovery requests only for the owning running attempt', () => { const { sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); store.submissions.admitDispatch(dispatchInput()); store.submissions.claimDispatch('dispatch-1', 'attempt-1'); - const recovered = store.submissions.recoverDispatchAttempt('dispatch-1', 'attempt-1', 'attempt-2'); - store.submissions.completeDispatch('dispatch-1', 'attempt-1'); + const applied = store.submissions.markDispatchInputApplied('dispatch-1', 'attempt-1'); + const replay = store.submissions.markDispatchInputApplied('dispatch-1', 'attempt-1'); + const staleApplied = store.submissions.markDispatchInputApplied('dispatch-1', 'stale-attempt'); + const recovery = store.submissions.requestDispatchRecovery('dispatch-1', 'attempt-1'); + const staleRecovery = store.submissions.requestDispatchRecovery('dispatch-1', 'stale-attempt'); - expect(recovered).toMatchObject({ status: 'running', attemptId: 'attempt-2' }); - expect(store.submissions.getDispatch('dispatch-1')).toMatchObject({ + expect(applied).toMatchObject({ status: 'running', - attemptId: 'attempt-2', + attemptId: 'attempt-1', + inputAppliedAt: expect.any(Number), }); + expect(replay?.inputAppliedAt).toBe(applied?.inputAppliedAt); + expect(staleApplied).toBeNull(); + expect(recovery).toMatchObject({ recoveryRequestedAt: expect.any(Number) }); + expect(staleRecovery).toBeNull(); + }); + + it('requeues interrupted attempts only before canonical input application', () => { + const { sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); + store.submissions.admitDispatch(dispatchInput({ dispatchId: 'requeue-safe' })); + store.submissions.admitDispatch(dispatchInput({ dispatchId: 'requeue-unsafe', session: 'other' })); + store.submissions.claimDispatch('requeue-safe', 'attempt-safe'); + store.submissions.claimDispatch('requeue-unsafe', 'attempt-unsafe'); + store.submissions.markDispatchInputApplied('requeue-unsafe', 'attempt-unsafe'); + + const safe = store.submissions.requeueDispatchBeforeInputApplied('requeue-safe', 'attempt-safe'); + const unsafe = store.submissions.requeueDispatchBeforeInputApplied('requeue-unsafe', 'attempt-unsafe'); + + expect(safe).toMatchObject({ status: 'queued' }); + expect(safe).not.toHaveProperty('attemptId'); + expect(unsafe).toBeNull(); + expect(store.submissions.getDispatch('requeue-unsafe')).toMatchObject({ status: 'running' }); }); it('reports unsettled session visibility until a claimed dispatch completes', () => { diff --git a/packages/runtime/test/dispatch.test.ts b/packages/runtime/test/dispatch.test.ts index ef62a9c7..676eceb2 100644 --- a/packages/runtime/test/dispatch.test.ts +++ b/packages/runtime/test/dispatch.test.ts @@ -9,12 +9,14 @@ import { dispatch, observe } from '../src/index.ts'; import { configureFlueRuntime, createAgentDispatchProcessor, + createDispatchAgentHandler, + createDispatchInputInspectionHandler, createFlueContext, type DispatchInput, InMemoryDispatchQueue, - validateAgentDispatchAdmission, InMemorySessionStore, resetFlueRuntimeForTests, + validateAgentDispatchAdmission, } from '../src/internal.ts'; import type { AgentConfig, FlueHarness, FlueSession } from '../src/types.ts'; import { createNoopSessionEnv } from './fixtures/session-env.ts'; @@ -508,6 +510,170 @@ describe('dispatched session processing', () => { }); }); + it('marks input application after configured persistence and before model processing begins', async () => { + const order: string[] = []; + const provider = createProvider(); + provider.setResponses([ + () => { + order.push('provider'); + return fauxAssistantMessage('processed after marker'); + }, + ]); + const store = new InMemorySessionStore(); + const originalSave = store.save.bind(store); + store.save = async (id, data) => { + if (data.entries.some((entry) => entry.type === 'message' && entry.dispatch?.dispatchId === 'dispatch:input-marker-order')) { + order.push('persist-input'); + } + await originalSave(id, data); + }; + const agent = createAgent(() => ({ + model: `${provider.getModel().provider}/${provider.getModel().id}`, + persist: store, + })); + const input: DispatchInput = { + dispatchId: 'dispatch:input-marker-order', + agent: 'moderator', + id: 'guild:input-marker-order', + session: 'case:input-marker-order', + input: { type: 'flagged', reportId: 'report:input-marker-order' }, + acceptedAt: '2026-06-01T00:00:00.000Z', + }; + const ctx = createFlueContext({ + id: input.id, + dispatchId: input.dispatchId, + payload: input, + env: {}, + req: new Request('http://flue.local/_dispatch', { method: 'POST' }), + agentConfig: { + systemPrompt: '', + skills: {}, + subagents: {}, + model: undefined, + resolveModel: () => provider.getModel(), + }, + createDefaultEnv: async () => createNoopSessionEnv({ cwd: '/' }), + defaultStore: new InMemorySessionStore(), + }); + + await createDispatchAgentHandler(agent, input, { + onInputApplied: () => { + order.push('input-applied'); + }, + })(ctx); + + expect(order.indexOf('persist-input')).toBeLessThan(order.indexOf('input-applied')); + expect(order.indexOf('input-applied')).toBeLessThan(order.indexOf('provider')); + }); + + it('classifies a completed canonical dispatch response without model replay', async () => { + const provider = createProvider(); + const store = new InMemorySessionStore(); + const input: DispatchInput = { + dispatchId: 'dispatch:inspect-completed', + agent: 'moderator', + id: 'guild:inspect-completed', + session: 'case:inspect-completed', + input: { type: 'flagged', reportId: 'report:inspect-completed' }, + acceptedAt: '2026-06-01T00:00:00.000Z', + }; + const timestamp = '2026-06-01T00:00:00.000Z'; + await store.save(`agent-session:${JSON.stringify([input.id, 'default', input.session])}`, { + version: 5, + affinityKey: 'aff_01KT3P3GZGFBCKHKMQ11A7H2HW', + entries: [ + { + type: 'message', + id: 'dispatch-input', + parentId: null, + timestamp, + message: { role: 'user', content: [{ type: 'text', text: 'persisted dispatch' }], timestamp: 0 }, + source: 'dispatch', + dispatch: input, + }, + { + type: 'message', + id: 'assistant-response', + parentId: 'dispatch-input', + timestamp, + message: fauxAssistantMessage('persisted response'), + source: 'dispatch', + }, + ], + leafId: 'assistant-response', + metadata: {}, + createdAt: timestamp, + updatedAt: timestamp, + }); + const agent = createAgent(() => ({ + model: `${provider.getModel().provider}/${provider.getModel().id}`, + persist: store, + })); + const ctx = createFlueContext({ + id: input.id, + dispatchId: input.dispatchId, + payload: input, + env: {}, + req: new Request('http://flue.local/_dispatch', { method: 'POST' }), + agentConfig: testAgentConfig(), + createDefaultEnv: async () => createNoopSessionEnv({ cwd: '/' }), + defaultStore: new InMemorySessionStore(), + }); + + await expect(createDispatchInputInspectionHandler(agent, input)(ctx)).resolves.toBe('completed'); + expect(provider.state.callCount).toBe(0); + }); + + it('classifies an incomplete canonical dispatch response without model replay', async () => { + const provider = createProvider(); + const store = new InMemorySessionStore(); + const input: DispatchInput = { + dispatchId: 'dispatch:inspect-applied', + agent: 'moderator', + id: 'guild:inspect-applied', + session: 'case:inspect-applied', + input: { type: 'flagged', reportId: 'report:inspect-applied' }, + acceptedAt: '2026-06-01T00:00:00.000Z', + }; + const timestamp = '2026-06-01T00:00:00.000Z'; + await store.save(`agent-session:${JSON.stringify([input.id, 'default', input.session])}`, { + version: 5, + affinityKey: 'aff_01KT3P3GZGFBCKHKMQ11A7H2HW', + entries: [ + { + type: 'message', + id: 'dispatch-input', + parentId: null, + timestamp, + message: { role: 'user', content: [{ type: 'text', text: 'persisted dispatch' }], timestamp: 0 }, + source: 'dispatch', + dispatch: input, + }, + ], + leafId: 'dispatch-input', + metadata: {}, + createdAt: timestamp, + updatedAt: timestamp, + }); + const agent = createAgent(() => ({ + model: `${provider.getModel().provider}/${provider.getModel().id}`, + persist: store, + })); + const ctx = createFlueContext({ + id: input.id, + dispatchId: input.dispatchId, + payload: input, + env: {}, + req: new Request('http://flue.local/_dispatch', { method: 'POST' }), + agentConfig: testAgentConfig(), + createDefaultEnv: async () => createNoopSessionEnv({ cwd: '/' }), + defaultStore: new InMemorySessionStore(), + }); + + await expect(createDispatchInputInspectionHandler(agent, input)(ctx)).resolves.toBe('applied'); + expect(provider.state.callCount).toBe(0); + }); + it('continues a persisted transient failure when the same dispatch id is replayed', async () => { vi.useFakeTimers(); try { From f8d58790775b4fd10476d604a794d88f94ec26ef Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" <622227+FredKSchott@users.noreply.github.com> Date: Wed, 3 Jun 2026 21:56:05 -0500 Subject: [PATCH 07/17] feat(cloudflare): unify durable agent submissions --- .../cli/src/lib/build-plugin-cloudflare.ts | 255 ++++++------- .../src/cloudflare/agent-execution-store.ts | 335 ++++++++++-------- packages/runtime/src/cloudflare/websocket.ts | 8 +- packages/runtime/src/internal.ts | 13 +- .../runtime/src/runtime/dispatch-queue.ts | 86 ++++- packages/runtime/src/runtime/handle-agent.ts | 76 ++++ packages/runtime/src/session.ts | 78 +++- packages/runtime/src/types.ts | 1 + .../test/build-plugin-cloudflare.test.ts | 31 +- .../cloudflare-agent-execution-store.test.ts | 107 +++--- .../runtime/test/cloudflare-websocket.test.ts | 42 ++- packages/runtime/test/dispatch.test.ts | 122 +++++++ 12 files changed, 785 insertions(+), 369 deletions(-) diff --git a/packages/cli/src/lib/build-plugin-cloudflare.ts b/packages/cli/src/lib/build-plugin-cloudflare.ts index f151deda..d75d8831 100644 --- a/packages/cli/src/lib/build-plugin-cloudflare.ts +++ b/packages/cli/src/lib/build-plugin-cloudflare.ts @@ -225,7 +225,9 @@ import { assertCurrentDispatchInput, createDispatchAgentHandler, createDispatchInputInspectionHandler, - reserveDispatchAgentSession, + createDirectSubmissionAgentHandler, + createDirectSubmissionInputInspectionHandler, + createAgentSubmissionObserverRegistry, failRecoveredRun, configureFlueRuntime, createDefaultFlueApp, @@ -383,7 +385,8 @@ const dispatchQueue = { // Module-scoped per-isolate registry; run ids isolate buckets across DOs. const runSubscribers = createRunSubscriberRegistry(); -const activeFlueAgentDispatchAttempts = new Set(); +const agentSubmissionObservers = createAgentSubmissionObserverRegistry(); +const activeFlueAgentSubmissionAttempts = new Set(); function getAgentExecutionStore(doInstance) { const store = doInstance[FLUE_AGENT_EXECUTION_STORE]; @@ -497,36 +500,13 @@ async function handleFlueDispatchAttemptRecovered(ctx, doInstance) { const attemptId = ctx.snapshot?.attemptId; if (typeof submissionId !== 'string' || typeof attemptId !== 'string') return; const submissions = getAgentExecutionStore(doInstance).submissions; - const submission = submissions.requestDispatchRecovery(submissionId, attemptId); + const submission = submissions.requestSubmissionRecovery(submissionId, attemptId); if (!submission) return; await armFlueAgentSubmissionAdmissionWakes(doInstance); } -async function handleFlueDirectRecovered(ctx, doInstance, agentName) { - const payload = ctx.snapshot?.payload; - const handler = localAgentHandlers[agentName]; - if (!handler || !payload || typeof payload !== 'object' || Array.isArray(payload) || typeof payload.message !== 'string') { - console.error('[flue:direct-recovery]', { agentName, instanceId: doInstance.name, operation: 'retry', outcome: 'restart_failed' }, new Error('Direct agent recovery input is unavailable; retry was not attempted.')); - return; - } - const identity = agentRuntimeIdentity(agentName); - const request = new Request('https://flue.invalid/agents/' + encodeURIComponent(agentName) + '/' + encodeURIComponent(doInstance.name), { method: 'POST' }); - let releaseSessionLock; - try { - await adoptLegacyManagedDispatches(doInstance, agentName); - releaseSessionLock = await reserveDispatchAgentSession({ agentName, instanceId: doInstance.name }, payload); - assertAgentsDurabilityApi(doInstance, 'runFiber'); - await doInstance.runFiber('flue:direct', async (fiberCtx) => { - fiberCtx.stash({ payload }); - const directCtx = createAgentContextForRequest(doInstance.name, payload, doInstance, request); - return runWithInstanceContext(doInstance, identity, () => handler(directCtx)); - }); - console.info('[flue:direct-recovery]', { agentName, instanceId: doInstance.name, operation: 'retry', outcome: 'restart_completed' }); - } catch (error) { - console.error('[flue:direct-recovery]', { agentName, instanceId: doInstance.name, operation: 'retry', outcome: 'restart_failed' }, error); - } finally { - releaseSessionLock?.(); - } +async function handleFlueDirectRecovered(_ctx, doInstance, agentName) { + console.error('[flue:direct-recovery]', { agentName, instanceId: doInstance.name, operation: 'legacy_handoff', outcome: 'terminal_uncertainty' }, new Error('A pre-upgrade direct prompt was interrupted. Provider replay was not attempted.')); } async function handleFlueWorkflowFiberRecovered(ctx, doInstance, workflowName) { @@ -574,7 +554,7 @@ function armFlueAgentSubmissionRetry(doInstance) { async function reconcileFlueAgentSubmissions(doInstance, agentName, options = {}) { const submissions = getAgentExecutionStore(doInstance).submissions; const legacySessions = await adoptLegacyManagedDispatches(doInstance, agentName); - if (!submissions.hasUnsettledDispatches()) return false; + if (!submissions.hasUnsettledSubmissions()) return false; await armFlueAgentSubmissionRetry(doInstance); if (options.preserveSuccessor) { if (options.executingWake) { @@ -590,85 +570,86 @@ async function reconcileFlueAgentSubmissions(doInstance, agentName, options = {} } } try { - const attemptMarkers = listActiveSqlAgentDispatchAttemptMarkers(doInstance); - const directMarkers = listActiveDirectAgentSessionMarkers(doInstance); - if (attemptMarkers.blockAll || directMarkers.blockAll) return true; - for (const submission of submissions.listRunningDispatches()) { - if (activeFlueAgentDispatchAttempts.has(dispatchAttemptLocalKey(doInstance, submission))) continue; - if (attemptMarkers.keys.has(dispatchAttemptMarkerKey(submission)) && submission.recoveryRequestedAt === undefined) continue; - if (directMarkers.sessions.has(submission.session) || legacySessions.has(submission.session)) continue; - await reconcileInterruptedSqlAgentDispatch(submission, doInstance, agentName); + const attemptMarkers = listActiveSqlAgentSubmissionAttemptMarkers(doInstance); + if (attemptMarkers.blockAll) return true; + for (const submission of submissions.listRunningSubmissions()) { + if (activeFlueAgentSubmissionAttempts.has(submissionAttemptLocalKey(doInstance, submission))) continue; + if (attemptMarkers.keys.has(submissionAttemptMarkerKey(submission)) && submission.recoveryRequestedAt === undefined) continue; + if (legacySessions.has(submission.session)) continue; + await reconcileInterruptedSqlAgentSubmission(submission, doInstance, agentName); } - for (const submission of submissions.listRunnableDispatches()) { - if (directMarkers.sessions.has(submission.session) || legacySessions.has(submission.session)) continue; - const claimed = submissions.claimDispatch(submission.submissionId, crypto.randomUUID()); - if (claimed) startSqlAgentDispatchAttempt(claimed, doInstance, agentName); + for (const submission of submissions.listRunnableSubmissions()) { + if (legacySessions.has(submission.session)) continue; + const claimed = submissions.claimSubmission(submission.submissionId, crypto.randomUUID()); + if (claimed) startSqlAgentSubmissionAttempt(claimed, doInstance, agentName); } } catch (error) { console.error('[flue:dispatch-reconciliation]', { agentName, instanceId: doInstance.name, operation: 'reconcile', outcome: 'deferred_to_scheduled_wake' }, error); return true; } - return submissions.hasUnsettledDispatches(); + return submissions.hasUnsettledSubmissions(); } -async function reconcileInterruptedSqlAgentDispatch(submission, doInstance, agentName) { +async function reconcileInterruptedSqlAgentSubmission(submission, doInstance, agentName) { const { attemptId, input } = submission; if (!attemptId) return; const submissions = getAgentExecutionStore(doInstance).submissions; + const agent = createdAgents[agentName]; + if (!agent) throw new Error('[flue] Agent target unavailable during durable reconciliation.'); + const request = new Request('https://flue.invalid' + INTERNAL_DISPATCH_PATH, { method: 'POST' }); + const ctx = createAgentContextForRequest(doInstance.name, input, doInstance, request, undefined, input.dispatchId); + const state = await runWithInstanceContext(doInstance, agentRuntimeIdentity(agentName), () => + submission.kind === 'dispatch' + ? createDispatchInputInspectionHandler(agent, input)(ctx) + : createDirectSubmissionInputInspectionHandler(agent, input)(ctx), + ); if (submission.inputAppliedAt === undefined) { - const agent = createdAgents[agentName]; - if (!agent) throw new Error('[flue] Dispatch target unavailable during durable reconciliation.'); - const request = new Request('https://flue.invalid' + INTERNAL_DISPATCH_PATH, { method: 'POST' }); - const ctx = createAgentContextForRequest(doInstance.name, input, doInstance, request, undefined, input.dispatchId); - const state = await runWithInstanceContext(doInstance, agentRuntimeIdentity(agentName), () => - createDispatchInputInspectionHandler(agent, input)(ctx), - ); if (state === 'absent') { - submissions.requeueDispatchBeforeInputApplied(input.dispatchId, attemptId); + submissions.requeueSubmissionBeforeInputApplied(submission.submissionId, attemptId); return; } - submissions.failDispatch(input.dispatchId, attemptId, new Error('[flue] Dispatch attempt was interrupted after canonical input persistence but before the input-application marker was recorded. Provider replay was not attempted.')); + submissions.failSubmission(submission.submissionId, attemptId, new Error('[flue] Agent submission attempt was interrupted after canonical input persistence but before the input-application marker was recorded. Provider replay was not attempted.')); return; } - const agent = createdAgents[agentName]; - if (!agent) throw new Error('[flue] Dispatch target unavailable during durable reconciliation.'); - const request = new Request('https://flue.invalid' + INTERNAL_DISPATCH_PATH, { method: 'POST' }); - const ctx = createAgentContextForRequest(doInstance.name, input, doInstance, request, undefined, input.dispatchId); - const state = await runWithInstanceContext(doInstance, agentRuntimeIdentity(agentName), () => - createDispatchInputInspectionHandler(agent, input)(ctx), - ); if (state === 'completed') { - submissions.completeDispatch(input.dispatchId, attemptId); + submissions.completeSubmission(submission.submissionId, attemptId); return; } - submissions.failDispatch(input.dispatchId, attemptId, new Error('[flue] Dispatch attempt was interrupted after input application without a completed canonical response. Provider replay was not attempted.')); + submissions.failSubmission(submission.submissionId, attemptId, new Error('[flue] Agent submission attempt was interrupted after input application without a completed canonical response. Provider replay was not attempted.')); } -function startSqlAgentDispatchAttempt(submission, doInstance, agentName) { +function startSqlAgentSubmissionAttempt(submission, doInstance, agentName) { if (submission.status !== 'running' || !submission.attemptId) return; - const attemptKey = dispatchAttemptLocalKey(doInstance, submission); - if (activeFlueAgentDispatchAttempts.has(attemptKey)) return; - activeFlueAgentDispatchAttempts.add(attemptKey); + const attemptKey = submissionAttemptLocalKey(doInstance, submission); + if (activeFlueAgentSubmissionAttempts.has(attemptKey)) return; assertAgentsDurabilityApi(doInstance, 'runFiber'); - void doInstance.runFiber('flue:dispatch-attempt', async (fiberCtx) => { - fiberCtx.stash({ submissionId: submission.submissionId, attemptId: submission.attemptId }); - await processSqlAgentDispatch(submission, doInstance, agentName); - }).catch((error) => { - console.error('[flue:dispatch-processing]', { agentName, instanceId: doInstance.name, submissionId: submission.submissionId, operation: 'process', outcome: 'failed' }, error); + activeFlueAgentSubmissionAttempts.add(attemptKey); + let running; + try { + running = doInstance.runFiber('flue:dispatch-attempt', async (fiberCtx) => { + fiberCtx.stash({ submissionId: submission.submissionId, attemptId: submission.attemptId }); + await processSqlAgentSubmission(submission, doInstance, agentName); + }); + } catch (error) { + activeFlueAgentSubmissionAttempts.delete(attemptKey); + throw error; + } + void running.catch((error) => { + console.error('[flue:submission-processing]', { agentName, instanceId: doInstance.name, submissionId: submission.submissionId, operation: 'process', outcome: 'failed' }, error); }).finally(() => { - activeFlueAgentDispatchAttempts.delete(attemptKey); + activeFlueAgentSubmissionAttempts.delete(attemptKey); }); } -function dispatchAttemptLocalKey(doInstance, submission) { +function submissionAttemptLocalKey(doInstance, submission) { return doInstance.ctx.id.toString() + ':' + submission.attemptId; } -function dispatchAttemptMarkerKey(submission) { +function submissionAttemptMarkerKey(submission) { return submission.submissionId + ':' + submission.attemptId; } -function listActiveSqlAgentDispatchAttemptMarkers(doInstance) { +function listActiveSqlAgentSubmissionAttemptMarkers(doInstance) { const keys = new Set(); let blockAll = false; const rows = doInstance.ctx.storage.sql.exec( @@ -696,30 +677,6 @@ function listActiveSqlAgentDispatchAttemptMarkers(doInstance) { return { blockAll, keys }; } -function listActiveDirectAgentSessionMarkers(doInstance) { - const sessions = new Set(); - let blockAll = false; - const rows = doInstance.ctx.storage.sql.exec( - "SELECT snapshot FROM cf_agents_runs WHERE name = 'flue:direct'", - ).toArray(); - for (const row of rows) { - let snapshot; - try { - snapshot = typeof row.snapshot === 'string' ? JSON.parse(row.snapshot) : null; - } catch { - blockAll = true; - continue; - } - const payload = snapshot?.payload; - if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { - blockAll = true; - continue; - } - sessions.add(typeof payload.session === 'string' && payload.session.trim() !== '' ? payload.session : 'default'); - } - return { blockAll, sessions }; -} - function compareLegacyManagedDispatches(a, b) { return a.createdAt - b.createdAt || a.fiberId.localeCompare(b.fiberId); } @@ -758,50 +715,76 @@ async function adoptLegacyManagedDispatches(doInstance, agentName) { return new Set(dispatches.map((dispatch) => dispatch.input.session)); } -async function processSqlAgentDispatch(submission, doInstance, agentName) { +async function processSqlAgentSubmission(submission, doInstance, agentName) { const { attemptId, input } = submission; if (!attemptId) return; const submissions = getAgentExecutionStore(doInstance).submissions; - const persisted = submissions.getDispatch(input.dispatchId); + const persisted = submissions.getSubmission(submission.submissionId); if (persisted?.status !== 'running' || persisted.attemptId !== attemptId) return; - let releaseSessionLock; + let ctx; try { const agent = createdAgents[agentName]; - if (!agent) throw new Error('[flue] Dispatch target unavailable during durable processing.'); - await validateAgentDispatchAdmission({ input }); - const target = { agentName, instanceId: doInstance.name }; - releaseSessionLock = await reserveDispatchAgentSession(target, input); - const request = new Request('https://flue.invalid' + INTERNAL_DISPATCH_PATH, { method: 'POST' }); - const ctx = createAgentContextForRequest(doInstance.name, input, doInstance, request, undefined, input.dispatchId); - await runWithInstanceContext(doInstance, agentRuntimeIdentity(agentName), () => - createDispatchAgentHandler(agent, input, { - onInputApplied: () => { - if (!submissions.markDispatchInputApplied(input.dispatchId, attemptId)) { - throw new Error('[flue] Dispatch attempt lost ownership before input application.'); - } - }, - })(ctx), + if (!agent) throw new Error('[flue] Agent target unavailable during durable processing.'); + if (submission.kind === 'dispatch') await validateAgentDispatchAdmission({ input }); + const request = submission.kind === 'direct' + ? new Request('https://flue.invalid/agents/' + encodeURIComponent(agentName) + '/' + encodeURIComponent(doInstance.name), { method: 'POST' }) + : new Request('https://flue.invalid' + INTERNAL_DISPATCH_PATH, { method: 'POST' }); + ctx = createAgentContextForRequest(doInstance.name, submission.kind === 'direct' ? input.payload : input, doInstance, request, undefined, input.dispatchId); + if (submission.kind === 'direct') { + ctx.setEventCallback((event) => { + if (event.type === 'run_start' || event.type === 'run_end') return; + const attachedEvent = { ...event, instanceId: doInstance.name }; + delete attachedEvent.runId; + return agentSubmissionObservers.publish(submission.submissionId, attachedEvent); + }); + } + const result = await runWithInstanceContext(doInstance, agentRuntimeIdentity(agentName), () => + submission.kind === 'dispatch' + ? createDispatchAgentHandler(agent, input, { + onInputApplied: () => markSubmissionInputApplied(submissions, submission, attemptId), + })(ctx) + : createDirectSubmissionAgentHandler(agent, input, { + onInputApplied: () => markSubmissionInputApplied(submissions, submission, attemptId), + })(ctx), ); - submissions.completeDispatch(input.dispatchId, attemptId); + const completed = submissions.completeSubmission(submission.submissionId, attemptId); + if (completed && submission.kind === 'direct') agentSubmissionObservers.complete(submission.submissionId, result); } catch (error) { - submissions.failDispatch(input.dispatchId, attemptId, error); + const failed = submissions.failSubmission(submission.submissionId, attemptId, error); + if (failed && submission.kind === 'direct') agentSubmissionObservers.fail(submission.submissionId, error); throw error; } finally { - releaseSessionLock?.(); + if (submission.kind === 'direct') ctx?.setEventCallback(undefined); void reconcileFlueAgentSubmissions(doInstance, agentName, { preserveSuccessor: true }).catch((error) => { - console.error('[flue:dispatch-reconciliation]', { agentName, instanceId: doInstance.name, operation: 'settlement', outcome: 'reconcile_failed' }, error); + console.error('[flue:submission-reconciliation]', { agentName, instanceId: doInstance.name, operation: 'settlement', outcome: 'reconcile_failed' }, error); }); } } -async function assertNoPendingDispatchForDirectSession(doInstance, agentName, session) { - await adoptLegacyManagedDispatches(doInstance, agentName); - const directMarkers = listActiveDirectAgentSessionMarkers(doInstance); - if (directMarkers.blockAll || directMarkers.sessions.has(session)) { - throw new Error('[flue] This agent session has an interrupted direct prompt and cannot accept direct input yet.'); +function markSubmissionInputApplied(submissions, submission, attemptId) { + if (!submissions.markSubmissionInputApplied(submission.submissionId, attemptId)) { + throw new Error('[flue] Agent submission attempt lost ownership before input application.'); } - if (getAgentExecutionStore(doInstance).submissions.hasUnsettledDispatchForSession(doInstance.name, session)) { - throw new Error('[flue] This agent session has pending dispatched input and cannot accept direct input yet.'); +} + +async function admitAttachedAgentSubmission(doInstance, agentName, payload, _request, onEvent) { + const submissionId = crypto.randomUUID(); + const input = { + submissionId, + agent: agentName, + id: doInstance.name, + session: typeof payload.session === 'string' && payload.session.trim() !== '' ? payload.session : 'default', + payload, + acceptedAt: new Date().toISOString(), + }; + const attachment = agentSubmissionObservers.attach(submissionId, { onEvent }); + try { + await armFlueAgentSubmissionAdmissionWakes(doInstance); + getAgentExecutionStore(doInstance).submissions.admitDirect(input); + await reconcileFlueAgentSubmissions(doInstance, agentName); + return await attachment.completion; + } finally { + attachment.detach(); } } @@ -862,9 +845,6 @@ async function dispatchAgent(request, doInstance, agentName, handler) { await reconcileFlueAgentSubmissions(doInstance, agentName); return Response.json({ dispatchId: submission.submissionId, acceptedAt: submission.input.acceptedAt }); } - const payload = await request.clone().json().catch(() => null); - const session = typeof payload?.session === 'string' && payload.session.trim() !== '' ? payload.session : 'default'; - await assertNoPendingDispatchForDirectSession(doInstance, agentName, session); const identity = agentRuntimeIdentity(agentName); return runWithInstanceContext(doInstance, identity, () => handleAgentRequest({ request, @@ -872,13 +852,7 @@ async function dispatchAgent(request, doInstance, agentName, handler) { id, handler, createContext: (id_, runId, payload, req, initialEventIndex, dispatchId) => createAgentContextForRequest(id_, payload, doInstance, req, initialEventIndex, dispatchId), - runHandler: (ctx, h) => { - assertAgentsDurabilityApi(doInstance, 'runFiber'); - return doInstance.runFiber('flue:direct', (fiberCtx) => { - fiberCtx.stash({ payload: ctx.payload }); - return h(ctx); - }); - }, + admitAttachedSubmission: (payload, req, onEvent) => admitAttachedAgentSubmission(doInstance, agentName, payload, req, onEvent), })); } @@ -931,15 +905,8 @@ async function messageAgentSocket(connection, message, doInstance, agentName) { id: doInstance.name, request: socketRequest(connection), handler, - beforePrompt: (session) => assertNoPendingDispatchForDirectSession(doInstance, agentName, session), createContext: (id_, runId, payload, req) => createAgentContextForRequest(id_, payload, doInstance, req), - runHandler: (ctx, h) => { - assertAgentsDurabilityApi(doInstance, 'runFiber'); - return doInstance.runFiber('flue:direct', (fiberCtx) => { - fiberCtx.stash({ payload: ctx.payload }); - return h(ctx); - }); - }, + admitAttachedSubmission: (payload, req, onEvent) => admitAttachedAgentSubmission(doInstance, agentName, payload, req, onEvent), })); } diff --git a/packages/runtime/src/cloudflare/agent-execution-store.ts b/packages/runtime/src/cloudflare/agent-execution-store.ts index 9aef232e..30168bfd 100644 --- a/packages/runtime/src/cloudflare/agent-execution-store.ts +++ b/packages/runtime/src/cloudflare/agent-execution-store.ts @@ -1,4 +1,9 @@ -import type { DispatchInput } from '../runtime/dispatch-queue.ts'; +import type { + AgentSubmissionInput, + DirectSubmissionInput, + DispatchInput, +} from '../runtime/dispatch-queue.ts'; +import { isDispatchSubmissionInput } from '../runtime/dispatch-queue.ts'; import { createSessionStorageKey } from '../session-identity.ts'; import type { SessionData, SessionStore } from '../types.ts'; @@ -19,12 +24,13 @@ interface DurableObjectStorage { type SqlAgentSubmissionStatus = 'queued' | 'running' | 'completed' | 'error'; -export interface SqlAgentDispatchSubmission { +export interface SqlAgentSubmission { readonly sequence: number; readonly submissionId: string; readonly session: string; readonly sessionKey: string; - readonly input: DispatchInput; + readonly kind: 'dispatch' | 'direct'; + readonly input: AgentSubmissionInput; readonly status: SqlAgentSubmissionStatus; readonly acceptedAt: number; readonly attemptId?: string; @@ -36,23 +42,23 @@ export interface SqlAgentDispatchSubmission { } export interface SqlAgentSubmissionStore { - getDispatch(submissionId: string): SqlAgentDispatchSubmission | null; - admitDispatch(input: DispatchInput): SqlAgentDispatchSubmission; - adoptLegacyDispatches(inputs: readonly DispatchInput[]): SqlAgentDispatchSubmission[]; - hasUnsettledDispatches(): boolean; - listQueuedDispatches(): SqlAgentDispatchSubmission[]; - listRunnableDispatches(): SqlAgentDispatchSubmission[]; - listRunningDispatches(): SqlAgentDispatchSubmission[]; - hasUnsettledDispatchForSession(instanceId: string, session: string): boolean; - claimDispatch(submissionId: string, attemptId: string): SqlAgentDispatchSubmission | null; - markDispatchInputApplied(submissionId: string, attemptId: string): SqlAgentDispatchSubmission | null; - requestDispatchRecovery(submissionId: string, attemptId: string): SqlAgentDispatchSubmission | null; - requeueDispatchBeforeInputApplied( + getSubmission(submissionId: string): SqlAgentSubmission | null; + admitDispatch(input: DispatchInput): SqlAgentSubmission; + admitDirect(input: DirectSubmissionInput): SqlAgentSubmission; + adoptLegacyDispatches(inputs: readonly DispatchInput[]): SqlAgentSubmission[]; + hasUnsettledSubmissions(): boolean; + listQueuedSubmissions(): SqlAgentSubmission[]; + listRunnableSubmissions(): SqlAgentSubmission[]; + listRunningSubmissions(): SqlAgentSubmission[]; + claimSubmission(submissionId: string, attemptId: string): SqlAgentSubmission | null; + markSubmissionInputApplied(submissionId: string, attemptId: string): SqlAgentSubmission | null; + requestSubmissionRecovery(submissionId: string, attemptId: string): SqlAgentSubmission | null; + requeueSubmissionBeforeInputApplied( submissionId: string, attemptId: string, - ): SqlAgentDispatchSubmission | null; - completeDispatch(submissionId: string, attemptId: string): void; - failDispatch(submissionId: string, attemptId: string, error: unknown): void; + ): SqlAgentSubmission | null; + completeSubmission(submissionId: string, attemptId: string): boolean; + failSubmission(submissionId: string, attemptId: string, error: unknown): boolean; } export interface SqlAgentExecutionStore { @@ -130,41 +136,25 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { private transactionSync: NonNullable, ) {} - getDispatch(submissionId: string): SqlAgentDispatchSubmission | null { - const row = this.readDispatchRow(submissionId); - return row ? parseDispatchSubmission(row) : null; + getSubmission(submissionId: string): SqlAgentSubmission | null { + const row = this.readSubmissionRow(submissionId); + return row ? parseSubmission(row) : null; } - admitDispatch(input: DispatchInput): SqlAgentDispatchSubmission { - const payload = JSON.stringify(input); - const acceptedAt = Date.parse(input.acceptedAt); - if (!Number.isFinite(acceptedAt)) { - throw new Error('[flue] Internal dispatch admission received an invalid acceptedAt timestamp.'); - } - this.sql.exec( - `INSERT OR IGNORE INTO flue_agent_submissions - (submission_id, session, session_key, kind, payload, status, accepted_at) - VALUES (?, ?, ?, 'dispatch', ?, 'queued', ?)`, - input.dispatchId, - input.session, - createSessionStorageKey(input.id, 'default', input.session), - payload, - acceptedAt, - ); - const row = this.readDispatchRow(input.dispatchId); - if (!row) throw new Error('[flue] Durable dispatch admission did not create a submission row.'); - if (row.payload !== payload) { - throw new SqlAgentSubmissionConflictError('[flue] Conflicting internal dispatch replay.'); - } - return parseDispatchSubmission(row); + admitDispatch(input: DispatchInput): SqlAgentSubmission { + return this.admitSubmission('dispatch', input.dispatchId, input); } - adoptLegacyDispatches(inputs: readonly DispatchInput[]): SqlAgentDispatchSubmission[] { + admitDirect(input: DirectSubmissionInput): SqlAgentSubmission { + return this.admitSubmission('direct', input.submissionId, input); + } + + adoptLegacyDispatches(inputs: readonly DispatchInput[]): SqlAgentSubmission[] { const unique = new Map(); for (const input of inputs) { const payload = JSON.stringify(input); - const row = this.readDispatchRow(input.dispatchId); - if (row && row.payload !== payload) { + const row = this.readSubmissionRow(input.dispatchId); + if (row && (row.kind !== 'dispatch' || row.payload !== payload)) { throw new SqlAgentSubmissionConflictError('[flue] Conflicting legacy dispatch adoption.'); } const prior = unique.get(input.dispatchId); @@ -173,7 +163,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { } if (!prior) unique.set(input.dispatchId, input); } - const missing = [...unique.values()].filter((input) => !this.readDispatchRow(input.dispatchId)); + const missing = [...unique.values()].filter((input) => !this.readSubmissionRow(input.dispatchId)); const adopt = () => { const first = this.sql .exec('SELECT MIN(sequence) AS sequence FROM flue_agent_submissions') @@ -183,10 +173,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { const batch = missing.slice(offset, offset + 16); const values: unknown[] = []; for (const input of batch) { - const acceptedAt = Date.parse(input.acceptedAt); - if (!Number.isFinite(acceptedAt)) { - throw new Error('[flue] Legacy dispatch adoption received an invalid acceptedAt timestamp.'); - } + const acceptedAt = parseAcceptedAt(input.acceptedAt, 'Legacy dispatch adoption'); values.push( sequence++, input.dispatchId, @@ -206,100 +193,83 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { }; if (missing.length > 0) this.transactionSync(adopt); return inputs.map((input) => { - const submission = this.getDispatch(input.dispatchId); - if (!submission) throw new Error('[flue] Legacy dispatch adoption did not create a submission row.'); + const submission = this.getSubmission(input.dispatchId); + if (!submission || submission.kind !== 'dispatch') { + throw new Error('[flue] Legacy dispatch adoption did not create a submission row.'); + } return submission; }); } - hasUnsettledDispatches(): boolean { + hasUnsettledSubmissions(): boolean { return ( this.sql .exec( `SELECT 1 FROM flue_agent_submissions - WHERE kind = 'dispatch' AND status IN ('queued', 'running') + WHERE status IN ('queued', 'running') LIMIT 1`, ) .toArray().length > 0 ); } - listQueuedDispatches(): SqlAgentDispatchSubmission[] { - return this.parseQueuedRows( + listQueuedSubmissions(): SqlAgentSubmission[] { + return this.parseOperationalRows( this.sql .exec( - `SELECT sequence, submission_id, session, session_key, kind, payload, status, - accepted_at, attempt_id, input_applied_at, recovery_requested_at, started_at, completed_at, error + `SELECT ${submissionColumns} FROM flue_agent_submissions - WHERE kind = 'dispatch' AND status = 'queued' + WHERE status = 'queued' ORDER BY sequence ASC`, ) .toArray(), + 'queued', ); } - listRunnableDispatches(): SqlAgentDispatchSubmission[] { + listRunnableSubmissions(): SqlAgentSubmission[] { const rows = this.sql .exec( - `SELECT current.sequence, current.submission_id, current.session, current.session_key, - current.kind, current.payload, current.status, current.accepted_at, - current.attempt_id, current.input_applied_at, current.recovery_requested_at, - current.started_at, current.completed_at, current.error + `SELECT ${submissionColumnsFor('current')} FROM flue_agent_submissions AS current - WHERE current.kind = 'dispatch' AND current.status = 'queued' + WHERE current.status = 'queued' AND NOT EXISTS ( SELECT 1 FROM flue_agent_submissions AS earlier - WHERE earlier.kind = 'dispatch' - AND earlier.session_key = current.session_key + WHERE earlier.session_key = current.session_key AND earlier.status IN ('queued', 'running') AND earlier.sequence < current.sequence ) ORDER BY current.sequence ASC`, ) .toArray(); - return this.parseQueuedRows(rows); + return this.parseOperationalRows(rows, 'queued'); } - listRunningDispatches(): SqlAgentDispatchSubmission[] { - return this.parseRunningRows( + listRunningSubmissions(): SqlAgentSubmission[] { + return this.parseOperationalRows( this.sql .exec( - `SELECT sequence, submission_id, session, session_key, kind, payload, status, - accepted_at, attempt_id, input_applied_at, recovery_requested_at, started_at, completed_at, error + `SELECT ${submissionColumns} FROM flue_agent_submissions - WHERE kind = 'dispatch' AND status = 'running' + WHERE status = 'running' ORDER BY sequence ASC`, ) .toArray(), + 'running', ); } - hasUnsettledDispatchForSession(instanceId: string, session: string): boolean { - return ( - this.sql - .exec( - `SELECT 1 - FROM flue_agent_submissions - WHERE kind = 'dispatch' AND session_key = ? AND status IN ('queued', 'running') - LIMIT 1`, - createSessionStorageKey(instanceId, 'default', session), - ) - .toArray().length > 0 - ); - } - - claimDispatch(submissionId: string, attemptId: string): SqlAgentDispatchSubmission | null { + claimSubmission(submissionId: string, attemptId: string): SqlAgentSubmission | null { this.sql.exec( `UPDATE flue_agent_submissions AS current SET status = 'running', attempt_id = ?, started_at = ? - WHERE current.submission_id = ? AND current.kind = 'dispatch' AND current.status = 'queued' + WHERE current.submission_id = ? AND current.status = 'queued' AND NOT EXISTS ( SELECT 1 FROM flue_agent_submissions AS earlier - WHERE earlier.kind = 'dispatch' - AND earlier.session_key = current.session_key + WHERE earlier.session_key = current.session_key AND earlier.status IN ('queued', 'running') AND earlier.sequence < current.sequence )`, @@ -307,106 +277,132 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { Date.now(), submissionId, ); - const submission = this.getDispatch(submissionId); + const submission = this.getSubmission(submissionId); return submission?.status === 'running' && submission.attemptId === attemptId ? submission : null; } - markDispatchInputApplied(submissionId: string, attemptId: string): SqlAgentDispatchSubmission | null { + markSubmissionInputApplied(submissionId: string, attemptId: string): SqlAgentSubmission | null { this.sql.exec( `UPDATE flue_agent_submissions SET input_applied_at = COALESCE(input_applied_at, ?) - WHERE submission_id = ? AND kind = 'dispatch' AND status = 'running' AND attempt_id = ?`, + WHERE submission_id = ? AND status = 'running' AND attempt_id = ?`, Date.now(), submissionId, attemptId, ); - const submission = this.getDispatch(submissionId); - return submission?.status === 'running' && submission.attemptId === attemptId - ? submission - : null; + return this.getOwnedRunningSubmission(submissionId, attemptId); } - requestDispatchRecovery(submissionId: string, attemptId: string): SqlAgentDispatchSubmission | null { + requestSubmissionRecovery(submissionId: string, attemptId: string): SqlAgentSubmission | null { this.sql.exec( `UPDATE flue_agent_submissions SET recovery_requested_at = COALESCE(recovery_requested_at, ?) - WHERE submission_id = ? AND kind = 'dispatch' AND status = 'running' AND attempt_id = ?`, + WHERE submission_id = ? AND status = 'running' AND attempt_id = ?`, Date.now(), submissionId, attemptId, ); - const submission = this.getDispatch(submissionId); - return submission?.status === 'running' && submission.attemptId === attemptId - ? submission - : null; + return this.getOwnedRunningSubmission(submissionId, attemptId); } - requeueDispatchBeforeInputApplied( + requeueSubmissionBeforeInputApplied( submissionId: string, attemptId: string, - ): SqlAgentDispatchSubmission | null { + ): SqlAgentSubmission | null { this.sql.exec( `UPDATE flue_agent_submissions SET status = 'queued', attempt_id = NULL, recovery_requested_at = NULL, started_at = NULL - WHERE submission_id = ? AND kind = 'dispatch' AND status = 'running' + WHERE submission_id = ? AND status = 'running' AND attempt_id = ? AND input_applied_at IS NULL`, submissionId, attemptId, ); - const submission = this.getDispatch(submissionId); + const submission = this.getSubmission(submissionId); return submission?.status === 'queued' ? submission : null; } - completeDispatch(submissionId: string, attemptId: string): void { + completeSubmission(submissionId: string, attemptId: string): boolean { this.sql.exec( `UPDATE flue_agent_submissions SET status = 'completed', completed_at = ?, error = NULL - WHERE submission_id = ? AND kind = 'dispatch' AND status = 'running' AND attempt_id = ?`, + WHERE submission_id = ? AND status = 'running' AND attempt_id = ?`, Date.now(), submissionId, attemptId, ); + const submission = this.getSubmission(submissionId); + return submission?.status === 'completed' && submission.attemptId === attemptId; } - failDispatch(submissionId: string, attemptId: string, error: unknown): void { + failSubmission(submissionId: string, attemptId: string, error: unknown): boolean { this.sql.exec( `UPDATE flue_agent_submissions SET status = 'error', completed_at = ?, error = ? - WHERE submission_id = ? AND kind = 'dispatch' AND status = 'running' AND attempt_id = ?`, + WHERE submission_id = ? AND status = 'running' AND attempt_id = ?`, Date.now(), error instanceof Error ? error.message : String(error), submissionId, attemptId, ); + const submission = this.getSubmission(submissionId); + return submission?.status === 'error' && submission.attemptId === attemptId; } - private parseQueuedRows(rows: SqlRow[]): SqlAgentDispatchSubmission[] { - return this.parseOperationalRows(rows, 'queued'); + private admitSubmission( + kind: SqlAgentSubmission['kind'], + submissionId: string, + input: AgentSubmissionInput, + ): SqlAgentSubmission { + const payload = JSON.stringify(input); + const acceptedAt = parseAcceptedAt(input.acceptedAt, `${kind} admission`); + this.sql.exec( + `INSERT OR IGNORE INTO flue_agent_submissions + (submission_id, session, session_key, kind, payload, status, accepted_at) + VALUES (?, ?, ?, ?, ?, 'queued', ?)`, + submissionId, + input.session, + createSessionStorageKey(input.id, 'default', input.session), + kind, + payload, + acceptedAt, + ); + const row = this.readSubmissionRow(submissionId); + if (!row) throw new Error(`[flue] Durable ${kind} admission did not create a submission row.`); + if (row.kind !== kind || row.payload !== payload) { + throw new SqlAgentSubmissionConflictError(`[flue] Conflicting internal ${kind} replay.`); + } + return parseSubmission(row); } - private parseRunningRows(rows: SqlRow[]): SqlAgentDispatchSubmission[] { - return this.parseOperationalRows(rows, 'running'); + private getOwnedRunningSubmission( + submissionId: string, + attemptId: string, + ): SqlAgentSubmission | null { + const submission = this.getSubmission(submissionId); + return submission?.status === 'running' && submission.attemptId === attemptId + ? submission + : null; } private parseOperationalRows( rows: SqlRow[], status: Extract, - ): SqlAgentDispatchSubmission[] { - const submissions: SqlAgentDispatchSubmission[] = []; + ): SqlAgentSubmission[] { + const submissions: SqlAgentSubmission[] = []; for (const row of rows) { try { - submissions.push(parseDispatchSubmission(row)); + submissions.push(parseSubmission(row)); } catch (error) { if (typeof row.sequence !== 'number') throw error; - this.failDispatchSequence(row.sequence, status, error); + this.failSubmissionSequence(row.sequence, status, error); } } return submissions; } - private failDispatchSequence( + private failSubmissionSequence( sequence: number, status: Extract, error: unknown, @@ -414,7 +410,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { this.sql.exec( `UPDATE flue_agent_submissions SET status = 'error', completed_at = ?, error = ? - WHERE sequence = ? AND kind = 'dispatch' AND status = ?`, + WHERE sequence = ? AND status = ?`, Date.now(), error instanceof Error ? error.message : String(error), sequence, @@ -422,13 +418,12 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { ); } - private readDispatchRow(submissionId: string): SqlRow | undefined { + private readSubmissionRow(submissionId: string): SqlRow | undefined { return this.sql .exec( - `SELECT sequence, submission_id, session, session_key, kind, payload, status, - accepted_at, attempt_id, input_applied_at, recovery_requested_at, started_at, completed_at, error + `SELECT ${submissionColumns} FROM flue_agent_submissions - WHERE submission_id = ? AND kind = 'dispatch' + WHERE submission_id = ? LIMIT 1`, submissionId, ) @@ -436,13 +431,23 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { } } -function parseDispatchSubmission(row: SqlRow): SqlAgentDispatchSubmission { +const submissionColumns = + 'sequence, submission_id, session, session_key, kind, payload, status, accepted_at, attempt_id, input_applied_at, recovery_requested_at, started_at, completed_at, error'; + +function submissionColumnsFor(table: string): string { + return submissionColumns + .split(', ') + .map((column) => `${table}.${column}`) + .join(', '); +} + +function parseSubmission(row: SqlRow): SqlAgentSubmission { if ( typeof row.sequence !== 'number' || typeof row.submission_id !== 'string' || typeof row.session !== 'string' || typeof row.session_key !== 'string' || - row.kind !== 'dispatch' || + (row.kind !== 'dispatch' && row.kind !== 'direct') || typeof row.payload !== 'string' || (row.status !== 'queued' && row.status !== 'running' && @@ -465,30 +470,18 @@ function parseDispatchSubmission(row: SqlRow): SqlAgentDispatchSubmission { (row.status === 'running' && (typeof row.attempt_id !== 'string' || typeof row.started_at !== 'number')) ) { - throw new Error('[flue] Persisted dispatch submission row is malformed.'); + throw new Error('[flue] Persisted agent submission row is malformed.'); } - const input = JSON.parse(row.payload) as DispatchInput; - if ( - !input || - typeof input !== 'object' || - typeof input.dispatchId !== 'string' || - typeof input.agent !== 'string' || - typeof input.id !== 'string' || - typeof input.session !== 'string' || - input.input === undefined || - typeof input.acceptedAt !== 'string' || - input.dispatchId !== row.submission_id || - input.session !== row.session || - createSessionStorageKey(input.id, 'default', input.session) !== row.session_key || - Date.parse(input.acceptedAt) !== row.accepted_at - ) { - throw new Error('[flue] Persisted dispatch submission payload is malformed.'); + const input = JSON.parse(row.payload) as unknown; + if (!isSubmissionPayload(input, row)) { + throw new Error('[flue] Persisted agent submission payload is malformed.'); } return { sequence: row.sequence, submissionId: row.submission_id, session: row.session, sessionKey: row.session_key, + kind: row.kind, input, status: row.status, acceptedAt: row.accepted_at, @@ -503,6 +496,56 @@ function parseDispatchSubmission(row: SqlRow): SqlAgentDispatchSubmission { }; } +function isSubmissionPayload(input: unknown, row: SqlRow): input is AgentSubmissionInput { + if (!input || typeof input !== 'object') return false; + const value = input as Partial; + if ( + typeof value.agent !== 'string' || + typeof value.id !== 'string' || + typeof value.session !== 'string' || + typeof value.acceptedAt !== 'string' || + value.session !== row.session || + createSessionStorageKey(value.id, 'default', value.session) !== row.session_key || + Date.parse(value.acceptedAt) !== row.accepted_at + ) { + return false; + } + if (row.kind === 'dispatch') { + return ( + 'dispatchId' in value && + typeof value.dispatchId === 'string' && + value.dispatchId === row.submission_id && + 'input' in value && + value.input !== undefined + ); + } + return ( + 'submissionId' in value && + typeof value.submissionId === 'string' && + value.submissionId === row.submission_id && + 'payload' in value && + isDirectPayload(value.payload) && + !isDispatchSubmissionInput(value as AgentSubmissionInput) + ); +} + +function isDirectPayload(value: unknown): boolean { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const payload = value as { message?: unknown; session?: unknown }; + return ( + typeof payload.message === 'string' && + (payload.session === undefined || typeof payload.session === 'string') + ); +} + +function parseAcceptedAt(value: string, label: string): number { + const acceptedAt = Date.parse(value); + if (!Number.isFinite(acceptedAt)) { + throw new Error(`[flue] Internal ${label} received an invalid acceptedAt timestamp.`); + } + return acceptedAt; +} + function ensureSessionTable(sql: SqlStorage): void { sql.exec( `CREATE TABLE IF NOT EXISTS flue_sessions ( diff --git a/packages/runtime/src/cloudflare/websocket.ts b/packages/runtime/src/cloudflare/websocket.ts index ae18fb42..d2da4b8c 100644 --- a/packages/runtime/src/cloudflare/websocket.ts +++ b/packages/runtime/src/cloudflare/websocket.ts @@ -1,8 +1,8 @@ import { InvalidRequestError } from '../errors.ts'; +import type { AttachedAgentSubmissionAdmission } from '../runtime/dispatch-queue.ts'; import type { AgentHandler, CreateContextFn, - RunHandlerFn, StartWorkflowAdmissionFn, WorkflowHandler, } from '../runtime/handle-agent.ts'; @@ -51,8 +51,7 @@ export interface CloudflareAgentWebSocketOptions extends CloudflareAttachedOptio name: string; id: string; handler: AgentHandler; - beforePrompt?: (session: string) => void | Promise; - runHandler?: RunHandlerFn; + admitAttachedSubmission?: AttachedAgentSubmissionAdmission; } export interface CloudflareWorkflowWebSocketOptions extends CloudflareAttachedOptions { @@ -186,7 +185,6 @@ async function invokeAgentPrompt( ): Promise { let didStart = false; try { - await options.beforePrompt?.(message.session ?? 'default'); const result = await invokeDirectAttached({ agentName: options.name, id: options.id, @@ -194,7 +192,7 @@ async function invokeAgentPrompt( request: options.request, handler: options.handler, createContext: options.createContext, - runHandler: options.runHandler, + admitAttachedSubmission: options.admitAttachedSubmission, onEvent: (event) => { if (!didStart) { didStart = true; diff --git a/packages/runtime/src/internal.ts b/packages/runtime/src/internal.ts index 3260efc6..21e895ff 100644 --- a/packages/runtime/src/internal.ts +++ b/packages/runtime/src/internal.ts @@ -22,8 +22,8 @@ export { createFlueContext } from './client.ts'; // they pull in `cloudflare:workers`, a virtual module Node can't resolve. // The generated CF entry imports them from there directly. export type { - SqlAgentDispatchSubmission, SqlAgentExecutionStore, + SqlAgentSubmission, SqlAgentSubmissionStore, } from './cloudflare/agent-execution-store.ts'; export { @@ -34,8 +34,13 @@ export { export { createDurableRunStore } from './cloudflare/run-store.ts'; export { InMemoryRunRegistry } from './node/run-registry.ts'; export { InMemoryRunStore } from './node/run-store.ts'; -export type { DispatchInput, DispatchProcessor, DispatchQueue } from './runtime/dispatch-queue.ts'; -export { assertCurrentDispatchInput, InMemoryDispatchQueue } from './runtime/dispatch-queue.ts'; +export type { + DirectSubmissionInput, + DispatchInput, + DispatchProcessor, + DispatchQueue, +} from './runtime/dispatch-queue.ts'; +export { assertCurrentDispatchInput, createAgentSubmissionObserverRegistry, InMemoryDispatchQueue } from './runtime/dispatch-queue.ts'; export type { ExposedTransport, FlueRuntime } from './runtime/flue-app.ts'; export { configureFlueRuntime, @@ -81,6 +86,8 @@ export type { export { createAgentDispatchProcessor, createDirectAgentHandler, + createDirectSubmissionAgentHandler, + createDirectSubmissionInputInspectionHandler, createDispatchAgentHandler, createDispatchInputInspectionHandler, failRecoveredRun, diff --git a/packages/runtime/src/runtime/dispatch-queue.ts b/packages/runtime/src/runtime/dispatch-queue.ts index 10d799f3..e000c03e 100644 --- a/packages/runtime/src/runtime/dispatch-queue.ts +++ b/packages/runtime/src/runtime/dispatch-queue.ts @@ -1,4 +1,4 @@ -import type { DispatchReceipt } from '../types.ts'; +import type { AttachedAgentEvent, DirectAgentPayload, DispatchReceipt } from '../types.ts'; export interface DispatchInput { dispatchId: string; @@ -9,6 +9,17 @@ export interface DispatchInput { acceptedAt: string; } +export interface DirectSubmissionInput { + submissionId: string; + agent: string; + id: string; + session: string; + payload: DirectAgentPayload; + acceptedAt: string; +} + +export type AgentSubmissionInput = DispatchInput | DirectSubmissionInput; + export function assertCurrentDispatchInput(value: unknown): asserts value is DispatchInput { if (value && typeof value === 'object' && 'targetAgent' in value) { throw new Error( @@ -17,12 +28,81 @@ export function assertCurrentDispatchInput(value: unknown): asserts value is Dis } } -export type DispatchInputInspection = 'absent' | 'applied' | 'completed' | 'advanced'; +export type AgentSubmissionInputInspection = 'absent' | 'applied' | 'completed' | 'advanced'; +export type DispatchInputInspection = AgentSubmissionInputInspection; -export interface ProcessDispatchInputOptions { +export interface ProcessAgentSubmissionInputOptions { onInputApplied?: () => Promise | void; } +export type ProcessDispatchInputOptions = ProcessAgentSubmissionInputOptions; + +export function isDispatchSubmissionInput(input: AgentSubmissionInput): input is DispatchInput { + return 'dispatchId' in input; +} + +interface AgentSubmissionObserver { + onEvent?: (event: AttachedAgentEvent) => Promise | void; +} + +interface AgentSubmissionAttachment { + readonly completion: Promise; + detach(): void; +} + +interface AgentSubmissionObserverRegistry { + attach(submissionId: string, observer: AgentSubmissionObserver): AgentSubmissionAttachment; + publish(submissionId: string, event: AttachedAgentEvent): Promise; + complete(submissionId: string, result: unknown): void; + fail(submissionId: string, error: unknown): void; +} + +export type AttachedAgentSubmissionAdmission = ( + payload: DirectAgentPayload, + request: Request, + onEvent?: (event: AttachedAgentEvent) => Promise | void, +) => Promise; + +export function createAgentSubmissionObserverRegistry(): AgentSubmissionObserverRegistry { + const observers = new Map>(); + return { + attach(submissionId, observer) { + let resolve!: (value: unknown) => void; + let reject!: (error: unknown) => void; + const completion = new Promise((resolve_, reject_) => { + resolve = resolve_; + reject = reject_; + }); + const attached = { ...observer, resolve, reject }; + const bucket = observers.get(submissionId) ?? new Set(); + bucket.add(attached); + observers.set(submissionId, bucket); + return { + completion, + detach() { + bucket.delete(attached); + if (bucket.size === 0) observers.delete(submissionId); + }, + }; + }, + async publish(submissionId, event) { + for (const observer of observers.get(submissionId) ?? []) { + try { + await observer.onEvent?.(event); + } catch {} + } + }, + complete(submissionId, result) { + for (const observer of observers.get(submissionId) ?? []) observer.resolve(result); + observers.delete(submissionId); + }, + fail(submissionId, error) { + for (const observer of observers.get(submissionId) ?? []) observer.reject(error); + observers.delete(submissionId); + }, + }; +} + export interface DispatchProcessor { process(input: DispatchInput): Promise | void; } diff --git a/packages/runtime/src/runtime/handle-agent.ts b/packages/runtime/src/runtime/handle-agent.ts index 7752e4f2..91bc8a02 100644 --- a/packages/runtime/src/runtime/handle-agent.ts +++ b/packages/runtime/src/runtime/handle-agent.ts @@ -20,10 +20,14 @@ import type { FlueEventCallback, } from '../types.ts'; import { + type AgentSubmissionInputInspection, + type AttachedAgentSubmissionAdmission, assertCurrentDispatchInput, + type DirectSubmissionInput, type DispatchInput, type DispatchInputInspection, type DispatchProcessor, + type ProcessAgentSubmissionInputOptions, type ProcessDispatchInputOptions, } from './dispatch-queue.ts'; import { streamActiveRunEvents } from './handle-run-routes.ts'; @@ -46,6 +50,14 @@ interface DispatchSession { processDispatchInput(input: DispatchInput, options?: ProcessDispatchInputOptions): PromiseLike; } +interface DirectSubmissionSession { + inspectDirectSubmissionInput?(input: DirectSubmissionInput): AgentSubmissionInputInspection; + processDirectSubmissionInput( + input: DirectSubmissionInput, + options?: ProcessAgentSubmissionInputOptions, + ): PromiseLike; +} + export interface AgentSessionTarget { agentName: string; instanceId: string; @@ -116,6 +128,21 @@ export function createDispatchInputInspectionHandler( return (ctx) => inspectAgentDispatchInput(ctx, agent, input); } +export function createDirectSubmissionAgentHandler( + agent: CreatedAgentHandler, + input: DirectSubmissionInput, + options?: ProcessAgentSubmissionInputOptions, +): AgentHandler { + return (ctx) => processAgentDirectSubmission(ctx, agent, input, options); +} + +export function createDirectSubmissionInputInspectionHandler( + agent: CreatedAgentHandler, + input: DirectSubmissionInput, +): AgentHandler { + return (ctx) => inspectAgentDirectSubmissionInput(ctx, agent, input); +} + export async function reserveDispatchAgentSession( target: AgentSessionTarget, payload: unknown, @@ -145,6 +172,28 @@ async function inspectAgentDispatchInput( return session.inspectDispatchInput(input); } +async function processAgentDirectSubmission( + ctx: FlueContextInternal, + agent: CreatedAgentHandler, + input: DirectSubmissionInput, + options?: ProcessAgentSubmissionInputOptions, +): Promise { + const session = await openAgentDirectSubmissionSession(ctx, agent, input); + return session.processDirectSubmissionInput(input, options); +} + +async function inspectAgentDirectSubmissionInput( + ctx: FlueContextInternal, + agent: CreatedAgentHandler, + input: DirectSubmissionInput, +): Promise { + const session = await openAgentDirectSubmissionSession(ctx, agent, input); + if (!session.inspectDirectSubmissionInput) { + throw new Error('[flue] Internal session does not support direct input inspection.'); + } + return session.inspectDirectSubmissionInput(input); +} + async function openAgentDispatchSession( ctx: FlueContextInternal, agent: CreatedAgentHandler, @@ -158,6 +207,19 @@ async function openAgentDispatchSession( return session; } +async function openAgentDirectSubmissionSession( + ctx: FlueContextInternal, + agent: CreatedAgentHandler, + input: DirectSubmissionInput, +): Promise { + const harness = await ctx.initializeCreatedAgent(agent, undefined); + const session = await harness.session(input.session); + if (!isDirectSubmissionSession(session)) { + throw new Error('[flue] Internal session does not support direct input processing.'); + } + return session; +} + function isDispatchInput(value: unknown): value is DispatchInput { if (!value || typeof value !== 'object') return false; const input = value as Partial; @@ -188,6 +250,14 @@ function isDispatchSession(value: unknown): value is DispatchSession { ); } +function isDirectSubmissionSession(value: unknown): value is DirectSubmissionSession { + return ( + !!value && + typeof value === 'object' && + typeof (value as DirectSubmissionSession).processDirectSubmissionInput === 'function' + ); +} + export function createDirectAgentHandler(agent: CreatedAgentHandler): AgentHandler { return async (ctx) => { const payload = parseDirectAgentPayload(ctx.payload); @@ -281,6 +351,7 @@ export interface HandleAgentOptions { handler: AgentHandler; createContext: CreateContextFn; runHandler?: RunHandlerFn; + admitAttachedSubmission?: AttachedAgentSubmissionAdmission; } export interface HandleWorkflowOptions { @@ -326,6 +397,7 @@ export async function handleAgentRequest(opts: HandleAgentOptions): Promise } export async function invokeDirectAttached(opts: DirectAttachedOptions): Promise { + if (opts.admitAttachedSubmission) { + return opts.admitAttachedSubmission(opts.payload, opts.request, opts.onEvent); + } const sessionLock = acquireDirectAgentSessionLock(opts.agentName, opts.id, opts.payload); try { const ctx = opts.createContext(opts.id, undefined, opts.payload, opts.request); diff --git a/packages/runtime/src/session.ts b/packages/runtime/src/session.ts index 192bb9a2..38528677 100644 --- a/packages/runtime/src/session.ts +++ b/packages/runtime/src/session.ts @@ -50,8 +50,11 @@ import { ResultUnavailableError, } from './result.ts'; import type { + AgentSubmissionInputInspection, + DirectSubmissionInput, DispatchInput, DispatchInputInspection, + ProcessAgentSubmissionInputOptions, ProcessDispatchInputOptions, } from './runtime/dispatch-queue.ts'; import { generateOperationId, generateTurnId } from './runtime/ids.ts'; @@ -370,7 +373,7 @@ export class SessionHistory { appendMessage( message: AgentMessage, source?: MessageSource, - dispatch?: DispatchMessageMetadata, + metadata?: { dispatch?: DispatchMessageMetadata; directSubmissionId?: string }, ): string { const entry: MessageEntry = { type: 'message', @@ -380,7 +383,8 @@ export class SessionHistory { message, source, }; - if (dispatch) entry.dispatch = dispatch; + if (metadata?.dispatch) entry.dispatch = metadata.dispatch; + if (metadata?.directSubmissionId) entry.directSubmissionId = metadata.directSubmissionId; this.appendEntry(entry); return entry.id; } @@ -392,6 +396,13 @@ export class SessionHistory { ); } + findDirectSubmissionInput(submissionId: string): MessageEntry | undefined { + return this.getActivePath().find( + (entry): entry is MessageEntry => + entry.type === 'message' && entry.directSubmissionId === submissionId, + ); + } + appendMessages(messages: AgentMessage[], source?: MessageSource): string[] { return messages.map((message) => this.appendMessage(message, source)); } @@ -888,16 +899,11 @@ export class Session implements FlueSession { } inspectDispatchInput(input: DispatchInput): DispatchInputInspection { - const inputEntry = this.history.findDispatchInput(input.dispatchId); - if (!inputEntry) return 'absent'; - const following = this.history.getActivePathSince(inputEntry.id); - if (following.some((entry) => entry.type === 'message' && entry.message.role === 'user')) { - return 'advanced'; - } - const assistant = following.findLast( - (entry): entry is MessageEntry => entry.type === 'message' && entry.message.role === 'assistant', - )?.message as AssistantMessage | undefined; - return assistant && isCompletedAssistantResponse(assistant) ? 'completed' : 'applied'; + return this.inspectPersistedInput(this.history.findDispatchInput(input.dispatchId)); + } + + inspectDirectSubmissionInput(input: DirectSubmissionInput): AgentSubmissionInputInspection { + return this.inspectPersistedInput(this.history.findDirectSubmissionInput(input.submissionId)); } processDispatchInput( @@ -911,6 +917,17 @@ export class Session implements FlueSession { ); } + processDirectSubmissionInput( + input: DirectSubmissionInput, + options?: ProcessAgentSubmissionInputOptions, + ): CallHandle { + return createCallHandle(undefined, (signal) => + this.runOperation('prompt', signal, () => + this.runPersistedDirectSubmissionInput(input, signal, options), + ), + ); + } + skill( skill: SkillReference | string, options: SkillOptions & { result: S }, @@ -2032,6 +2049,18 @@ export class Session implements FlueSession { return undefined; } + private inspectPersistedInput(inputEntry: MessageEntry | undefined): AgentSubmissionInputInspection { + if (!inputEntry) return 'absent'; + const following = this.history.getActivePathSince(inputEntry.id); + if (following.some((entry) => entry.type === 'message' && entry.message.role === 'user')) { + return 'advanced'; + } + const assistant = following.findLast( + (entry): entry is MessageEntry => entry.type === 'message' && entry.message.role === 'assistant', + )?.message as AssistantMessage | undefined; + return assistant && isCompletedAssistantResponse(assistant) ? 'completed' : 'applied'; + } + private async runPersistedDispatchInput( input: DispatchInput, signal: AbortSignal, @@ -2043,7 +2072,7 @@ export class Session implements FlueSession { this.history.appendMessage( createUserContextMessage(renderDispatchInput(input), new Date().toISOString()), 'dispatch', - dispatchMetadata(input), + { dispatch: dispatchMetadata(input) }, ), errorLabel: `dispatch(${input.dispatchId})`, outputSource: 'dispatch', @@ -2055,6 +2084,29 @@ export class Session implements FlueSession { }); } + private async runPersistedDirectSubmissionInput( + input: DirectSubmissionInput, + signal: AbortSignal, + options?: ProcessAgentSubmissionInputOptions, + ): Promise { + return this.runPersistedContextInput({ + findInput: () => this.history.findDirectSubmissionInput(input.submissionId), + persistInput: () => + this.history.appendMessage( + createUserContextMessage(input.payload.message, new Date().toISOString()), + 'prompt', + { directSubmissionId: input.submissionId }, + ), + errorLabel: `direct(${input.submissionId})`, + outputSource: 'prompt', + callSite: 'this direct input', + persistenceError: '[flue] Failed to persist direct input.', + recoveryError: '[flue] Cannot recover direct input after the session has advanced.', + onInputApplied: options?.onInputApplied, + signal, + }); + } + private async runPersistedContextInput(options: { findInput: () => MessageEntry | undefined; persistInput: () => string; diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts index 1c6d20db..3eaabc38 100644 --- a/packages/runtime/src/types.ts +++ b/packages/runtime/src/types.ts @@ -795,6 +795,7 @@ export interface MessageEntry extends SessionEntryBase { message: AgentMessage; source?: 'prompt' | 'skill' | 'shell' | 'task' | 'retry' | 'dispatch'; dispatch?: DispatchMessageMetadata; + directSubmissionId?: string; } export interface DispatchMessageMetadata { diff --git a/packages/runtime/test/build-plugin-cloudflare.test.ts b/packages/runtime/test/build-plugin-cloudflare.test.ts index 51334aa6..e093b330 100644 --- a/packages/runtime/test/build-plugin-cloudflare.test.ts +++ b/packages/runtime/test/build-plugin-cloudflare.test.ts @@ -75,22 +75,31 @@ describe('CloudflarePlugin', () => { expect(entry).toContain("return doInstance.scheduleEvery(FLUE_AGENT_SUBMISSION_RETRY_SECONDS, FLUE_AGENT_SUBMISSION_RETRY_CALLBACK);"); expect(entry).toContain("await armFlueAgentSubmissionAdmissionWakes(doInstance);\n let submission;"); expect(entry).toContain('submission = getAgentExecutionStore(doInstance).submissions.admitDispatch(input);'); - expect(entry).toContain('for (const submission of submissions.listRunningDispatches()) {'); - expect(entry).toContain('if (activeFlueAgentDispatchAttempts.has(dispatchAttemptLocalKey(doInstance, submission))) continue;'); - expect(entry).toContain('if (attemptMarkers.keys.has(dispatchAttemptMarkerKey(submission)) && submission.recoveryRequestedAt === undefined) continue;'); - expect(entry).toContain('await reconcileInterruptedSqlAgentDispatch(submission, doInstance, agentName);'); - expect(entry).toContain('const submission = submissions.requestDispatchRecovery(submissionId, attemptId);'); + expect(entry).toContain('for (const submission of submissions.listRunningSubmissions()) {'); + expect(entry).toContain('if (activeFlueAgentSubmissionAttempts.has(submissionAttemptLocalKey(doInstance, submission))) continue;'); + expect(entry).toContain('if (attemptMarkers.keys.has(submissionAttemptMarkerKey(submission)) && submission.recoveryRequestedAt === undefined) continue;'); + expect(entry).toContain('await reconcileInterruptedSqlAgentSubmission(submission, doInstance, agentName);'); + expect(entry).toContain('const submission = submissions.requestSubmissionRecovery(submissionId, attemptId);'); expect(entry).toContain("SELECT snapshot, created_at FROM cf_agents_runs WHERE name = 'flue:dispatch-attempt'"); expect(entry).toContain('if (Date.now() - row.created_at > FLUE_AGENT_SUBMISSION_ATTEMPT_STALE_MS) continue;'); - expect(entry).toContain('submissions.requeueDispatchBeforeInputApplied(input.dispatchId, attemptId);'); + expect(entry).toContain('submissions.requeueSubmissionBeforeInputApplied(submission.submissionId, attemptId);'); expect(entry).toContain('createDispatchInputInspectionHandler(agent, input)(ctx)'); - expect(entry).toContain('if (!submissions.markDispatchInputApplied(input.dispatchId, attemptId)) {'); - expect(entry).toContain('const claimed = submissions.claimDispatch(submission.submissionId, crypto.randomUUID());'); - expect(entry).toContain("void doInstance.runFiber('flue:dispatch-attempt', async (fiberCtx) => {"); + expect(entry).toContain('createDirectSubmissionInputInspectionHandler(agent, input)(ctx)'); + expect(entry).toContain('if (!submissions.markSubmissionInputApplied(submission.submissionId, attemptId)) {'); + expect(entry).toContain('const claimed = submissions.claimSubmission(submission.submissionId, crypto.randomUUID());'); + expect(entry).toContain("running = doInstance.runFiber('flue:dispatch-attempt', async (fiberCtx) => {"); expect(entry).toContain("fiberCtx.stash({ submissionId: submission.submissionId, attemptId: submission.attemptId });"); - expect(entry).toContain('submissions.hasUnsettledDispatchForSession(doInstance.name, session)'); - expect(entry).toContain('if (directMarkers.blockAll || directMarkers.sessions.has(session)) {'); + expect(entry).toContain("activeFlueAgentSubmissionAttempts.delete(attemptKey);\n throw error;"); + expect(entry).toContain('const completed = submissions.completeSubmission(submission.submissionId, attemptId);'); + expect(entry).toContain("if (completed && submission.kind === 'direct') agentSubmissionObservers.complete(submission.submissionId, result);"); + expect(entry).toContain("if (submission.kind === 'direct') ctx?.setEventCallback(undefined);"); + expect(entry).toContain('getAgentExecutionStore(doInstance).submissions.admitDirect(input);'); + expect(entry).toContain('admitAttachedSubmission: (payload, req, onEvent) => admitAttachedAgentSubmission(doInstance, agentName, payload, req, onEvent)'); expect(entry).toContain('getAgentExecutionStore(doInstance).submissions.adoptLegacyDispatches(dispatches.map((dispatch) => dispatch.input));'); + expect(entry).not.toContain('assertNoPendingDispatchForDirectSession'); + expect(entry).not.toContain("runFiber('flue:direct'"); + expect(entry).not.toContain('listActiveDirectAgentSessionMarkers'); + expect(entry).not.toContain('agentSubmissionObservers.takeRequest'); expect(entry).toContain("return handleFlueDispatchAttemptRecovered(ctx, this);"); expect(entry).not.toContain("startFiber('flue:dispatch'"); expect(entry).not.toContain('inspectFiberByKey'); diff --git a/packages/runtime/test/cloudflare-agent-execution-store.test.ts b/packages/runtime/test/cloudflare-agent-execution-store.test.ts index 4d95f36a..70663574 100644 --- a/packages/runtime/test/cloudflare-agent-execution-store.test.ts +++ b/packages/runtime/test/cloudflare-agent-execution-store.test.ts @@ -4,7 +4,7 @@ import { createSqlAgentExecutionStore, createSqlSessionStore, } from '../src/cloudflare/agent-execution-store.ts'; -import type { DispatchInput } from '../src/runtime/dispatch-queue.ts'; +import type { DirectSubmissionInput, DispatchInput } from '../src/runtime/dispatch-queue.ts'; import type { SessionData } from '../src/types.ts'; function makeFakeSql() { @@ -54,6 +54,18 @@ function dispatchInput(overrides: Partial = {}): DispatchInput { }; } +function directInput(overrides: Partial = {}): DirectSubmissionInput { + return { + submissionId: 'direct-1', + agent: 'assistant', + id: 'agent-1', + session: 'default', + payload: { message: 'Hello' }, + acceptedAt: '2026-06-03T00:00:00.000Z', + ...overrides, + }; +} + function sessionData(): SessionData { return { version: 5, @@ -179,6 +191,22 @@ describe('createSqlAgentExecutionStore()', () => { ).toThrow('[flue] Conflicting internal dispatch replay.'); }); + it('orders direct and dispatched submissions together within one session', () => { + const { sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); + const direct = store.submissions.admitDirect(directInput()); + const dispatch = store.submissions.admitDispatch(dispatchInput()); + const other = store.submissions.admitDirect(directInput({ submissionId: 'direct-2', session: 'other' })); + + expect(store.submissions.listQueuedSubmissions()).toEqual([direct, dispatch, other]); + expect(store.submissions.listRunnableSubmissions()).toEqual([direct, other]); + expect(store.submissions.claimSubmission('dispatch-1', 'attempt-blocked')).toBeNull(); + expect(store.submissions.claimSubmission('direct-1', 'attempt-direct')).toMatchObject({ + kind: 'direct', + status: 'running', + }); + }); + it('lists queued dispatches in admission order and selects one runnable head per session', () => { const { sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); @@ -188,8 +216,8 @@ describe('createSqlAgentExecutionStore()', () => { dispatchInput({ dispatchId: 'dispatch-3', session: 'other' }), ); - expect(store.submissions.listQueuedDispatches()).toEqual([first, second, other]); - expect(store.submissions.listRunnableDispatches()).toEqual([first, other]); + expect(store.submissions.listQueuedSubmissions()).toEqual([first, second, other]); + expect(store.submissions.listRunnableSubmissions()).toEqual([first, other]); }); it('claims only runnable session heads while allowing separate sessions to claim independently', () => { @@ -199,9 +227,9 @@ describe('createSqlAgentExecutionStore()', () => { store.submissions.admitDispatch(dispatchInput({ dispatchId: 'dispatch-2' })); store.submissions.admitDispatch(dispatchInput({ dispatchId: 'dispatch-3', session: 'other' })); - const first = store.submissions.claimDispatch('dispatch-1', 'attempt-1'); - const blocked = store.submissions.claimDispatch('dispatch-2', 'attempt-2'); - const other = store.submissions.claimDispatch('dispatch-3', 'attempt-3'); + const first = store.submissions.claimSubmission('dispatch-1', 'attempt-1'); + const blocked = store.submissions.claimSubmission('dispatch-2', 'attempt-2'); + const other = store.submissions.claimSubmission('dispatch-3', 'attempt-3'); expect(first).toMatchObject({ submissionId: 'dispatch-1', @@ -215,8 +243,8 @@ describe('createSqlAgentExecutionStore()', () => { status: 'running', attemptId: 'attempt-3', }); - expect(store.submissions.listRunningDispatches()).toEqual([first, other]); - expect(store.submissions.listRunnableDispatches()).toEqual([]); + expect(store.submissions.listRunningSubmissions()).toEqual([first, other]); + expect(store.submissions.listRunnableSubmissions()).toEqual([]); }); it('terminalizes malformed queued payloads while returning healthy runnable rows', () => { @@ -229,7 +257,7 @@ describe('createSqlAgentExecutionStore()', () => { VALUES (?, ?, ?, 'dispatch', ?, 'queued', ?)`, ).run('malformed', 'other', 'agent-session:["agent-1","default","other"]', '{', 1); - expect(store.submissions.listRunnableDispatches()).toEqual([ + expect(store.submissions.listRunnableSubmissions()).toEqual([ expect.objectContaining({ submissionId: 'healthy' }), ]); expect( @@ -248,8 +276,8 @@ describe('createSqlAgentExecutionStore()', () => { 'dispatch-1', ); - expect(store.submissions.listRunnableDispatches()).toEqual([]); - expect(store.submissions.getDispatch('dispatch-1')).toMatchObject({ + expect(store.submissions.listRunnableSubmissions()).toEqual([]); + expect(store.submissions.getSubmission('dispatch-1')).toMatchObject({ status: 'error', error: expect.any(String), }); @@ -266,7 +294,7 @@ describe('createSqlAgentExecutionStore()', () => { ]); expect(adopted.map((submission) => submission.submissionId)).toEqual(['legacy-1', 'legacy-2']); - expect(store.submissions.listQueuedDispatches().map((submission) => submission.submissionId)).toEqual([ + expect(store.submissions.listQueuedSubmissions().map((submission) => submission.submissionId)).toEqual([ 'legacy-1', 'legacy-2', 'current', @@ -283,7 +311,7 @@ describe('createSqlAgentExecutionStore()', () => { store.submissions.adoptLegacyDispatches(legacy); - expect(store.submissions.listQueuedDispatches().map((submission) => submission.submissionId)).toEqual([ + expect(store.submissions.listQueuedSubmissions().map((submission) => submission.submissionId)).toEqual([ ...legacy.map((input) => input.dispatchId), 'current', ]); @@ -293,13 +321,13 @@ describe('createSqlAgentExecutionStore()', () => { const { sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); store.submissions.admitDispatch(dispatchInput()); - store.submissions.claimDispatch('dispatch-1', 'attempt-1'); + store.submissions.claimSubmission('dispatch-1', 'attempt-1'); - const applied = store.submissions.markDispatchInputApplied('dispatch-1', 'attempt-1'); - const replay = store.submissions.markDispatchInputApplied('dispatch-1', 'attempt-1'); - const staleApplied = store.submissions.markDispatchInputApplied('dispatch-1', 'stale-attempt'); - const recovery = store.submissions.requestDispatchRecovery('dispatch-1', 'attempt-1'); - const staleRecovery = store.submissions.requestDispatchRecovery('dispatch-1', 'stale-attempt'); + const applied = store.submissions.markSubmissionInputApplied('dispatch-1', 'attempt-1'); + const replay = store.submissions.markSubmissionInputApplied('dispatch-1', 'attempt-1'); + const staleApplied = store.submissions.markSubmissionInputApplied('dispatch-1', 'stale-attempt'); + const recovery = store.submissions.requestSubmissionRecovery('dispatch-1', 'attempt-1'); + const staleRecovery = store.submissions.requestSubmissionRecovery('dispatch-1', 'stale-attempt'); expect(applied).toMatchObject({ status: 'running', @@ -317,17 +345,17 @@ describe('createSqlAgentExecutionStore()', () => { const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); store.submissions.admitDispatch(dispatchInput({ dispatchId: 'requeue-safe' })); store.submissions.admitDispatch(dispatchInput({ dispatchId: 'requeue-unsafe', session: 'other' })); - store.submissions.claimDispatch('requeue-safe', 'attempt-safe'); - store.submissions.claimDispatch('requeue-unsafe', 'attempt-unsafe'); - store.submissions.markDispatchInputApplied('requeue-unsafe', 'attempt-unsafe'); + store.submissions.claimSubmission('requeue-safe', 'attempt-safe'); + store.submissions.claimSubmission('requeue-unsafe', 'attempt-unsafe'); + store.submissions.markSubmissionInputApplied('requeue-unsafe', 'attempt-unsafe'); - const safe = store.submissions.requeueDispatchBeforeInputApplied('requeue-safe', 'attempt-safe'); - const unsafe = store.submissions.requeueDispatchBeforeInputApplied('requeue-unsafe', 'attempt-unsafe'); + const safe = store.submissions.requeueSubmissionBeforeInputApplied('requeue-safe', 'attempt-safe'); + const unsafe = store.submissions.requeueSubmissionBeforeInputApplied('requeue-unsafe', 'attempt-unsafe'); expect(safe).toMatchObject({ status: 'queued' }); expect(safe).not.toHaveProperty('attemptId'); expect(unsafe).toBeNull(); - expect(store.submissions.getDispatch('requeue-unsafe')).toMatchObject({ status: 'running' }); + expect(store.submissions.getSubmission('requeue-unsafe')).toMatchObject({ status: 'running' }); }); it('reports unsettled session visibility until a claimed dispatch completes', () => { @@ -335,29 +363,28 @@ describe('createSqlAgentExecutionStore()', () => { const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); store.submissions.admitDispatch(dispatchInput({ session: 'case-1' })); - expect(store.submissions.hasUnsettledDispatches()).toBe(true); - expect(store.submissions.hasUnsettledDispatchForSession('agent-1', 'case-1')).toBe(true); - expect(store.submissions.hasUnsettledDispatchForSession('agent-1', 'case-2')).toBe(false); - store.submissions.claimDispatch('dispatch-1', 'attempt-1'); - expect(store.submissions.hasUnsettledDispatchForSession('agent-1', 'case-1')).toBe(true); - store.submissions.completeDispatch('dispatch-1', 'attempt-1'); - expect(store.submissions.hasUnsettledDispatches()).toBe(false); - expect(store.submissions.hasUnsettledDispatchForSession('agent-1', 'case-1')).toBe(false); - expect(store.submissions.getDispatch('dispatch-1')).toMatchObject({ status: 'completed' }); + expect(store.submissions.hasUnsettledSubmissions()).toBe(true); + expect(store.submissions.listQueuedSubmissions()).toHaveLength(1); + store.submissions.claimSubmission('dispatch-1', 'attempt-1'); + expect(store.submissions.listRunningSubmissions()).toHaveLength(1); + store.submissions.completeSubmission('dispatch-1', 'attempt-1'); + expect(store.submissions.hasUnsettledSubmissions()).toBe(false); + expect(store.submissions.listRunningSubmissions()).toEqual([]); + expect(store.submissions.getSubmission('dispatch-1')).toMatchObject({ status: 'completed' }); }); it('ignores stale-attempt settlement and keeps the first owning terminal dispatch state', () => { const { sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); store.submissions.admitDispatch(dispatchInput()); - store.submissions.claimDispatch('dispatch-1', 'attempt-1'); + store.submissions.claimSubmission('dispatch-1', 'attempt-1'); - store.submissions.completeDispatch('dispatch-1', 'stale-attempt'); - store.submissions.failDispatch('dispatch-1', 'attempt-1', new Error('first failure')); - store.submissions.completeDispatch('dispatch-1', 'attempt-1'); - store.submissions.failDispatch('dispatch-1', 'attempt-1', new Error('later failure')); + store.submissions.completeSubmission('dispatch-1', 'stale-attempt'); + store.submissions.failSubmission('dispatch-1', 'attempt-1', new Error('first failure')); + store.submissions.completeSubmission('dispatch-1', 'attempt-1'); + store.submissions.failSubmission('dispatch-1', 'attempt-1', new Error('later failure')); - expect(store.submissions.getDispatch('dispatch-1')).toMatchObject({ + expect(store.submissions.getSubmission('dispatch-1')).toMatchObject({ status: 'error', error: 'first failure', }); diff --git a/packages/runtime/test/cloudflare-websocket.test.ts b/packages/runtime/test/cloudflare-websocket.test.ts index c69615f7..9d123476 100644 --- a/packages/runtime/test/cloudflare-websocket.test.ts +++ b/packages/runtime/test/cloudflare-websocket.test.ts @@ -88,9 +88,6 @@ describe('Cloudflare agent WebSockets', () => { name: 'assistant', id: 'agent-instance-1', request: new Request('https://example.com/flue/agents/assistant/agent-instance-1'), - beforePrompt: async (session) => { - calls.push(`restore:${session}`); - }, handler: async (ctx) => { calls.push('invoke'); return ctx.payload; @@ -99,7 +96,7 @@ describe('Cloudflare agent WebSockets', () => { }, ); - expect(calls).toEqual(['restore:support', 'invoke']); + expect(calls).toEqual(['invoke']); expect(connection.messages).toContainEqual({ version: 1, type: 'result', @@ -109,6 +106,43 @@ describe('Cloudflare agent WebSockets', () => { expect(connection.closed).toBeUndefined(); }); + it('uses attached durable submission admission when configured for a Cloudflare agent socket', async () => { + const connection = new TestConnection(); + const calls: string[] = []; + + await messageCloudflareAgentWebSocket( + connection, + JSON.stringify({ version: 1, type: 'prompt', requestId: 'prompt-1', message: 'Hello' }), + { + name: 'assistant', + id: 'agent-instance-1', + request: new Request('https://example.com/flue/agents/assistant/agent-instance-1'), + handler: async () => { + calls.push('handler'); + return null; + }, + createContext, + admitAttachedSubmission: async (payload, _request, onEvent) => { + calls.push(`admit:${payload.message}`); + await onEvent?.({ type: 'idle', instanceId: 'agent-instance-1' }); + return 'done'; + }, + }, + ); + + expect(calls).toEqual(['admit:Hello']); + expect(connection.messages).toEqual([ + { version: 1, type: 'started', requestId: 'prompt-1' }, + { + version: 1, + type: 'event', + requestId: 'prompt-1', + event: { type: 'idle', instanceId: 'agent-instance-1' }, + }, + { version: 1, type: 'result', requestId: 'prompt-1', result: 'done' }, + ]); + }); + it('rejects oversized messages when a Cloudflare agent socket exceeds the byte limit', async () => { const connection = new TestConnection(); let invocations = 0; diff --git a/packages/runtime/test/dispatch.test.ts b/packages/runtime/test/dispatch.test.ts index 676eceb2..1e6974c9 100644 --- a/packages/runtime/test/dispatch.test.ts +++ b/packages/runtime/test/dispatch.test.ts @@ -9,9 +9,12 @@ import { dispatch, observe } from '../src/index.ts'; import { configureFlueRuntime, createAgentDispatchProcessor, + createDirectSubmissionAgentHandler, + createDirectSubmissionInputInspectionHandler, createDispatchAgentHandler, createDispatchInputInspectionHandler, createFlueContext, + type DirectSubmissionInput, type DispatchInput, InMemoryDispatchQueue, InMemorySessionStore, @@ -566,6 +569,125 @@ describe('dispatched session processing', () => { expect(order.indexOf('input-applied')).toBeLessThan(order.indexOf('provider')); }); + it('persists plain direct submission input before marking input application and invoking the provider', async () => { + const order: string[] = []; + const provider = createProvider(); + provider.setResponses([ + () => { + order.push('provider'); + return fauxAssistantMessage('processed direct input'); + }, + ]); + const store = new InMemorySessionStore(); + const originalSave = store.save.bind(store); + store.save = async (id, data) => { + if (data.entries.some((entry) => entry.type === 'message' && entry.directSubmissionId === 'direct:input-marker-order')) { + order.push('persist-input'); + } + await originalSave(id, data); + }; + const agent = createAgent(() => ({ + model: `${provider.getModel().provider}/${provider.getModel().id}`, + persist: store, + })); + const input: DirectSubmissionInput = { + submissionId: 'direct:input-marker-order', + agent: 'moderator', + id: 'guild:direct-input-marker-order', + session: 'case:direct-input-marker-order', + payload: { message: 'Hello directly' }, + acceptedAt: '2026-06-01T00:00:00.000Z', + }; + const ctx = createFlueContext({ + id: input.id, + payload: input.payload, + env: {}, + req: new Request('http://flue.local/agents/moderator/guild:direct-input-marker-order', { method: 'POST' }), + agentConfig: { + systemPrompt: '', + skills: {}, + subagents: {}, + model: undefined, + resolveModel: () => provider.getModel(), + }, + createDefaultEnv: async () => createNoopSessionEnv({ cwd: '/' }), + defaultStore: new InMemorySessionStore(), + }); + + await createDirectSubmissionAgentHandler(agent, input, { + onInputApplied: () => { + order.push('input-applied'); + }, + })(ctx); + + const data = await store.load(`agent-session:${JSON.stringify([input.id, 'default', input.session])}`); + expect(order.indexOf('persist-input')).toBeLessThan(order.indexOf('input-applied')); + expect(order.indexOf('input-applied')).toBeLessThan(order.indexOf('provider')); + expect(data?.entries[0]).toMatchObject({ + source: 'prompt', + directSubmissionId: input.submissionId, + message: { role: 'user', content: [{ type: 'text', text: 'Hello directly' }] }, + }); + expect(data?.entries[0]).not.toHaveProperty('dispatch'); + }); + + it('classifies a completed canonical direct response without model replay', async () => { + const provider = createProvider(); + const store = new InMemorySessionStore(); + const input: DirectSubmissionInput = { + submissionId: 'direct:inspect-completed', + agent: 'moderator', + id: 'guild:direct-inspect-completed', + session: 'case:direct-inspect-completed', + payload: { message: 'Hello directly' }, + acceptedAt: '2026-06-01T00:00:00.000Z', + }; + const timestamp = '2026-06-01T00:00:00.000Z'; + await store.save(`agent-session:${JSON.stringify([input.id, 'default', input.session])}`, { + version: 5, + affinityKey: 'aff_01KT3P3GZGFBCKHKMQ11A7H2HW', + entries: [ + { + type: 'message', + id: 'direct-input', + parentId: null, + timestamp, + message: { role: 'user', content: [{ type: 'text', text: input.payload.message }], timestamp: 0 }, + source: 'prompt', + directSubmissionId: input.submissionId, + }, + { + type: 'message', + id: 'assistant-response', + parentId: 'direct-input', + timestamp, + message: fauxAssistantMessage('persisted response'), + source: 'prompt', + }, + ], + leafId: 'assistant-response', + metadata: {}, + createdAt: timestamp, + updatedAt: timestamp, + }); + const agent = createAgent(() => ({ + model: `${provider.getModel().provider}/${provider.getModel().id}`, + persist: store, + })); + const ctx = createFlueContext({ + id: input.id, + payload: input.payload, + env: {}, + req: new Request('http://flue.local/agents/moderator/guild:direct-inspect-completed', { method: 'POST' }), + agentConfig: testAgentConfig(), + createDefaultEnv: async () => createNoopSessionEnv({ cwd: '/' }), + defaultStore: new InMemorySessionStore(), + }); + + await expect(createDirectSubmissionInputInspectionHandler(agent, input)(ctx)).resolves.toBe('completed'); + expect(provider.state.callCount).toBe(0); + }); + it('classifies a completed canonical dispatch response without model replay', async () => { const provider = createProvider(); const store = new InMemorySessionStore(); From 88199743ada133a98077d76263ef6720a7f21e3f Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" <622227+FredKSchott@users.noreply.github.com> Date: Thu, 4 Jun 2026 08:40:17 -0500 Subject: [PATCH 08/17] hardening --- CHANGELOG.md | 4 + README.md | 6 +- apps/docs/src/content/docs/api/agent-api.md | 6 +- .../content/docs/api/data-persistence-api.md | 5 +- .../docs/ecosystem/deploy/cloudflare.md | 30 ++- .../docs/guide/message-driven-agents.md | 2 +- .../src/content/docs/guide/observability.md | 2 +- packages/cli/README.md | 6 +- .../cli/src/lib/build-plugin-cloudflare.ts | 203 ++++++++++----- packages/cli/src/lib/build.ts | 7 +- packages/runtime/README.md | 6 +- packages/runtime/src/client.ts | 7 + .../src/cloudflare/agent-execution-store.ts | 241 ++++++++++++++---- packages/runtime/src/harness.ts | 13 +- packages/runtime/src/internal.ts | 3 + .../runtime/src/runtime/dispatch-queue.ts | 7 + packages/runtime/src/runtime/handle-agent.ts | 47 ++++ packages/runtime/src/session.ts | 60 ++++- packages/runtime/src/types.ts | 22 +- .../test/build-plugin-cloudflare.test.ts | 50 ++-- .../cloudflare-agent-execution-store.test.ts | 165 ++++++++++-- ...dflare-agent-extension.integration.test.ts | 2 +- .../runtime/test/cloudflare-websocket.test.ts | 39 +++ packages/runtime/test/dispatch.test.ts | 127 +++++++++ packages/sdk/README.md | 6 +- pnpm-workspace.yaml | 1 - 26 files changed, 874 insertions(+), 193 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e90b71d..e5a9d945 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ - **Agents: Rename ordinary sessions beginning with `task:`.** Session names with that prefix are now reserved for framework-owned delegated-task history. Flue retains detached child history until parent deletion or application-owned retention cleanup, and no stored records are deleted automatically during upgrade. - **Agents: Clear or migrate persisted beta session state before upgrading.** Session records now persist one opaque `aff_` provider-affinity key instead of deriving affinity from agent instance, harness, and session names. This keeps prompt-cache and routing-affinity identifiers bounded and distinct for nested tasks. Existing version-4 beta session records are rejected; storage keys are unchanged. - **Cloudflare: Rename generated Durable Object identities.** Generated bindings now use `FLUE__AGENT`, `FLUE__WORKFLOW`, and `FLUE_REGISTRY`; generated classes use `FlueAgent`, `FlueWorkflow`, and `FlueRegistry`. Existing deployments must append authored Wrangler `renamed_classes` migrations for deployed agent and workflow classes and update authored direct binding access such as `env.Assistant` to `env.FLUE_ASSISTANT_AGENT`. +- **Cloudflare: Use SQLite-backed generated agent classes.** Generated agent Durable Objects now require SQLite for durable submission admission and ordering. Introduce classes through Wrangler `new_sqlite_classes`, not legacy `new_classes`. Already deployed KV-backed classes cannot be converted to SQLite in place; use a fresh class identity or an application-owned migration plan. +- **Cloudflare: Upgrade Cloudflare Agents SDK within the audited range.** Cloudflare builds require `agents >=0.14.1 <0.15.0`. +- **Cloudflare: Capture request metadata before durable direct admission.** Route middleware sees the original inbound request, but SQL-backed direct processing uses a deterministic internal `Request`. Do not rely on later operation-time `ctx.req` to preserve original headers, cookies, query parameters, URL, or body. ### Fixes & Other Changes @@ -19,6 +22,7 @@ - **Cloudflare: Preserve workflow-run history parity.** Cloudflare workflow storage now ignores events for unknown runs, resets same-ID event history when a run is initialized, preserves absent optional fields separately from explicit `null`, and retains explicit terminal `null` results during recovery. - **Harden persisted workflow-event identity.** Workflow history now treats `(runId, eventIndex)` as one immutable append-only event identity and SSE resume cursor across Node and Cloudflare. Malformed or duplicate persisted events fail instead of producing ambiguous history, and pre-event stream failures no longer fabricate cursor `0`. - **Cloudflare: Own Durable Object routing.** Flue now resolves generated agent and workflow bindings explicitly instead of deriving public routes from the Agents SDK environment scanner, then forwards requests through the Agents SDK custom-routing helper. Public routes remain independent from generated Durable Object identities. +- **Cloudflare: Queue direct and dispatched agent input through one durable lifecycle.** Direct HTTP, SSE, WebSocket, and `dispatch(...)` submissions now share SQLite-backed same-session ordering. Transport loss does not cancel accepted backend work. Interrupted attempts reconcile conservatively: Flue requeues only when safe, recognizes persisted completion, and otherwise records a visible session interruption instead of blindly replaying provider work. Terminal operational payload copies become eligible for bounded lazy cleanup after a seven-day internal inspection and duplicate-forwarding horizon, and session deletion rejects while durable submissions remain queued or running. ## 0.9.1 - 2026-06-02 diff --git a/README.md b/README.md index 24cef48a..b9bce5a8 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ export async function run({ init, payload }: FlueContext) { A support agent can also run on Cloudflare without a container by using Flue's default virtual sandbox. Populate its filesystem with the context the agent needs, then it can search that content with its built-in `grep`, `glob`, and `read` tools. -Because this agent is deployed to Cloudflare, message history and session state are automatically persisted for you. So you (or your customer) can revisit this support session days, weeks, or years later and pick up exactly where you left off. +Cloudflare Durable Objects persist stored conversation history across later requests. The default virtual sandbox remains in memory, so seed any files each time you need them or choose a durable sandbox connector for long-lived workspace state. ```ts // .flue/workflows/support.ts @@ -281,7 +281,7 @@ curl http://localhost:3583/agents/hello/session-xyz \ -d '{"message": "Hello from another conversation"}' ``` -Agent instances own sandbox state such as files written across prompt and dispatched-input interactions. Harnesses group related session state within an instance. Sessions persist message history and conversation metadata inside a harness. On Cloudflare, session data is backed by Durable Objects and survives across requests. On Node.js, sessions are stored in memory by default unless you provide a custom store. +Agent instances provide a stable identity boundary for prompt and dispatched-input interactions. Harnesses group related session state within an instance. Sessions persist message history and conversation metadata inside a harness. Sandbox-file durability depends on the configured sandbox or connector; the default virtual filesystem is in memory. On Cloudflare, session conversation data is backed by Durable Objects and survives across requests. On Node.js, sessions are stored in memory by default unless you provide a custom store. In production, generate a stable URL `` for the agent instance you want to preserve. Use `harness.session(threadName)` when you need multiple conversations inside the same harness. @@ -418,7 +418,7 @@ await job.ready; await job.invoke({ text: 'Summarize me' }); ``` -An exported `websocket` middleware can authenticate its own agent or workflow socket endpoint. Use a custom `app.ts` for centralized authentication or mounted prefixes, applying ordinary Hono middleware before `app.route('/api', flue())`; the same routing model works on Node and Cloudflare. Include the custom mount in `baseUrl: 'https://example.com/api'` and use `websocketUrl: (url) => { url.searchParams.set('token', socketToken); return url; }` for URL-carried or signed handshake authentication. HTTP `token` and `headers` options do not automatically apply to WebSocket upgrades; browsers should use cookies or application-designed URL authentication, while Node clients needing implementation-specific headers can provide a custom `websocket` factory. Avoid header-mutating middleware around WebSocket upgrade routes. `flue dev --target cloudflare` requires `wrangler` as a peer dependency in your project (`npm install --save-dev wrangler`). +An exported `websocket` middleware can authenticate its own agent or workflow socket endpoint. Use a custom `app.ts` for centralized authentication or mounted prefixes, applying ordinary Hono middleware before `app.route('/api', flue())`; the same routing model works on Node and Cloudflare. Include the custom mount in `baseUrl: 'https://example.com/api'` and use `websocketUrl: (url) => { url.searchParams.set('token', socketToken); return url; }` for URL-carried or signed handshake authentication. HTTP `token` and `headers` options do not automatically apply to WebSocket upgrades; browsers should use cookies or application-designed URL authentication, while Node clients needing implementation-specific headers can provide a custom `websocket` factory. Avoid header-mutating middleware around WebSocket upgrade routes. Install `wrangler` in Cloudflare projects that use Wrangler commands such as `wrangler deploy` or secret management; `flue dev --target cloudflare` no longer requires it as an `@flue/cli` peer dependency. #### Loading environment variables diff --git a/apps/docs/src/content/docs/api/agent-api.md b/apps/docs/src/content/docs/api/agent-api.md index 0e7245fd..4616801d 100644 --- a/apps/docs/src/content/docs/api/agent-api.md +++ b/apps/docs/src/content/docs/api/agent-api.md @@ -246,7 +246,7 @@ Accepts input for asynchronous delivery to a continuing agent session. The creat `await dispatch(...)` resolves when the current runtime accepts and queues the input. It does not wait for model processing, tool calls, or an agent reply. Dispatched activity belongs to the continuing agent session: it does not create workflow-run history and does not appear in `/runs` or `flue logs`. -Delivery durability depends on the generated target. Node uses a process-lifetime in-memory queue by default. Cloudflare durably admits delivery to the target agent Durable Object and may retry processing after an interruption. Design external side effects to be idempotent. See [Deploy Agents on Node.js](/docs/ecosystem/deploy/node/) and [Deploy Agents on Cloudflare](/docs/ecosystem/deploy/cloudflare/). +Delivery durability depends on the generated target. Node uses a process-lifetime in-memory queue by default. Cloudflare durably admits delivery to the target agent Durable Object, orders it with direct prompts for the same session, and reconciles interruptions conservatively. It retries only when replay safety is provable; external effects still require application-level idempotency. See [Deploy Agents on Node.js](/docs/ecosystem/deploy/node/) and [Deploy Agents on Cloudflare](/docs/ecosystem/deploy/cloudflare/). ## `init(...)` @@ -319,7 +319,7 @@ Creates a new session. Defaults to `'default'`. Throws if it already exists. delete(name?: string): Promise; ``` -Deletes a session's stored conversation state. Defaults to `'default'`. No-op when missing. Rejects if the open session has an active operation. Session-management requests for one name are applied in request order. +Deletes a session's stored conversation state. Defaults to `'default'`. No-op when missing. Rejects if the open session has an active operation. On Cloudflare, it also rejects while the session has accepted durable submissions queued or running. Session-management requests for one name are applied in request order. ### `harness.shell(...)` @@ -526,7 +526,7 @@ Triggers conversation compaction immediately. Resolves without work when there i delete(): Promise; ``` -Deletes this session's stored conversation state. Rejects while an operation is active. Once deletion starts, the session is unusable and concurrent calls share the same deletion work. +Deletes this session's stored conversation state. Rejects while an operation is active. On Cloudflare, it also rejects while accepted durable submissions are queued or running for the session. Once deletion starts, the session is unusable and concurrent calls share the same deletion work. #### `CallHandle` diff --git a/apps/docs/src/content/docs/api/data-persistence-api.md b/apps/docs/src/content/docs/api/data-persistence-api.md index cebe90fa..d8aa44d8 100644 --- a/apps/docs/src/content/docs/api/data-persistence-api.md +++ b/apps/docs/src/content/docs/api/data-persistence-api.md @@ -95,15 +95,16 @@ Treat `SessionData` as potentially sensitive. It can include model-visible text, | State category | Controlled by | | -------------------------------------------------------------- | ------------------------------------------------------ | | Agent session messages and compaction state | `SessionStore` / `persist` or the target default | +| Cloudflare agent submission admission, ordering, and terminal inspection rows | The owning agent Durable Object SQLite store | | Sandbox files, installed dependencies, and workspace artifacts | The configured sandbox or connector | | Workflow run records and persisted run events | Workflow-run runtime storage, not `SessionStore` alone | | Mutations performed through tools or external APIs | The external system and application idempotency policy | -A persisted conversation does not make sandbox files durable. A durable workspace does not retain conversation history unless session persistence does as well. +A persisted conversation does not make sandbox files durable. A durable workspace does not retain conversation history unless session persistence does as well. On Cloudflare, a created agent's `persist` override replaces canonical session snapshots only: Flue still stores operational submission rows locally in the owning Durable Object SQLite database. Those rows can contain submitted payloads while queued and running. Terminal rows become eligible for bounded lazy cleanup after seven days and are removed during later agent activity; an entirely idle Durable Object may retain eligible rows longer. The same eligibility horizon bounds duplicate-delivery protection for repeated forwarding of one `dispatchId`; it does not create a public submission lookup API. ## Identity and deletion -Session data is stored under keys derived from Flue identity boundaries: agent instance or workflow invocation ownership, harness name, and session name. The stored record contains a separate opaque provider-affinity key. Delegated `task(...)` calls use internal child sessions whose retained history remains parent-owned; names beginning with `task:` are reserved for those children and cannot be selected as ordinary sessions. Deleting a parent session removes its stored conversation data and retained child task-session tree; application-owned stores may apply broader retention separately. Deletion does not undo external effects or remove sandbox files. +Session data is stored under keys derived from Flue identity boundaries: agent instance or workflow invocation ownership, harness name, and session name. The stored record contains a separate opaque provider-affinity key. Delegated `task(...)` calls use internal child sessions whose retained history remains parent-owned; names beginning with `task:` are reserved for those children and cannot be selected as ordinary sessions. Deleting a parent session removes its stored conversation data and retained child task-session tree; application-owned stores may apply broader retention separately. On Cloudflare, deletion rejects while the session still has queued or running durable submissions and removes settled operational payload copies after snapshot deletion succeeds. Deletion does not undo external effects or remove sandbox files. ## Implementing a store diff --git a/apps/docs/src/content/docs/ecosystem/deploy/cloudflare.md b/apps/docs/src/content/docs/ecosystem/deploy/cloudflare.md index c7e24791..4c60bba6 100644 --- a/apps/docs/src/content/docs/ecosystem/deploy/cloudflare.md +++ b/apps/docs/src/content/docs/ecosystem/deploy/cloudflare.md @@ -22,11 +22,11 @@ The simplest agent — no container, no storage, just a prompt and a typed resul ```bash mkdir my-flue-worker && cd my-flue-worker npm init -y -npm install @flue/runtime valibot agents +npm install @flue/runtime valibot 'agents@^0.14.1' npm install -D @flue/cli wrangler ``` -`agents` is Cloudflare's Agents SDK — Flue uses its Durable Object base class and native lifecycle capabilities while retaining ownership of application routing. If you also need a remote sandbox, additionally install `@cloudflare/sandbox` (see [Connecting a remote sandbox](#connecting-a-remote-sandbox) below). +`agents` is Cloudflare's Agents SDK — Flue uses its Durable Object base class and native lifecycle capabilities while retaining ownership of application routing. Cloudflare builds currently require the audited `agents >=0.14.1 <0.15.0` range. If you also need a remote sandbox, additionally install `@cloudflare/sandbox` (see [Connecting a remote sandbox](#connecting-a-remote-sandbox) below). ### 2. Create your first agent @@ -82,7 +82,7 @@ Cloudflare requires an explicit migration whenever a Worker adds a Durable Objec Every Cloudflare target includes `FlueRegistry`. Flue-owned bindings use upper snake case and generated classes use PascalCase: `.flue/workflows/translate.ts` binds `FLUE_TRANSLATE_WORKFLOW` to `FlueTranslateWorkflow`, while `.flue/agents/support-chat.ts` binds `FLUE_SUPPORT_CHAT_AGENT` to `FlueSupportChatAgent`. -Keep deployed migration entries in order. When you add an agent or workflow later, append a uniquely tagged migration for its new class. Use Cloudflare's explicit rename or delete migrations when changing a deployed class lifecycle. When upgrading a deployment created before this naming scheme, append `renamed_classes` entries such as `{ "from": "SupportChat", "to": "FlueSupportChatAgent" }` and `{ "from": "TranslateWorkflow", "to": "FlueTranslateWorkflow" }`; do not rewrite deployed migration history. +Keep deployed migration entries in order. When you add an agent or workflow later, append a uniquely tagged migration for its new class. Generated Flue agent classes require Durable Object SQLite: introduce them through `new_sqlite_classes`, not legacy `new_classes`. An already deployed KV-backed Durable Object class cannot be converted to SQLite in place; use a fresh class identity or plan an application-owned state migration. Use Cloudflare's explicit rename or delete migrations when changing a deployed class lifecycle. When upgrading a deployment created before this naming scheme, append `renamed_classes` entries such as `{ "from": "SupportChat", "to": "FlueSupportChatAgent" }` and `{ "from": "TranslateWorkflow", "to": "FlueTranslateWorkflow" }`; do not rewrite deployed migration history. ### 4. Build and deploy @@ -152,7 +152,7 @@ chat.close(); An exported `websocket` middleware can authenticate its own agent or workflow socket endpoint. Custom `.flue/app.ts` applications provide centralized authentication and mounted prefixes: for example, apply `app.use('/api/agents/*', authenticate)` and `app.use('/api/workflows/*', authenticate)` before `app.route('/api', flue())` to cover both socket surfaces before Flue forwards accepted upgrades into their owning Durable Objects. SDK clients can connect through that mount with `baseUrl: 'https://example.com/api'` and attach query-token or signed handshake context with `websocketUrl: (url) => { url.searchParams.set('token', socketToken); return url; }`. HTTP `token` and `headers` options do not automatically apply to WebSocket upgrades; browsers should use cookies or application-designed URL authentication, while Node clients requiring implementation-specific headers can provide a custom `websocket` factory. -Ordinary initial HTTP forwarding from the Worker into a Durable Object preserves request headers. WebSocket hibernation is a narrower boundary: Cloudflare socket authentication is established during the handshake, and Flue does not restore original upgrade headers or query parameters into later operation-time request context. Avoid header-mutating middleware such as CORS wrapping WebSocket upgrade routes, because WebSocket upgrade responses may have immutable headers. +Route middleware sees the original inbound HTTP request before Flue forwards accepted work into its Durable Object. Durable direct-agent processing is a later boundary: after admission, Flue uses a deterministic internal request and does not persist or reconstruct the caller's original headers, cookies, query parameters, URL, or body as operation-time `ctx.req`. Authenticate before admission and carry any non-secret correlation you need later in application-owned input or storage. WebSocket hibernation has the same limitation for original upgrade metadata. Avoid header-mutating middleware such as CORS wrapping WebSocket upgrade routes, because WebSocket upgrade responses may have immutable headers. ### Extending an addressable Cloudflare agent @@ -464,27 +464,29 @@ WebSocket-exposed created agents use the same owning Durable Object scope. Flue' A deployment or code update can reset a Durable Object while an operation is running. Flue handles interrupted Cloudflare operations according to their execution model: -| Operation | After interruption | -| ---------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Direct attached agent HTTP/WebSocket prompt | Flue makes a best-effort retry of interrupted prompt work from its captured input and the latest saved session state. The attached response or socket stream may fail and missed output is not replayed. No public agent run exists. | -| Dispatched agent input | Durable delivery and deduplication are keyed by `dispatchId` and persisted session/delivery state, not by a run. | -| Flue workflow invocation (`202`, SSE, `?wait=result`, or workflow WebSocket) | Flue terminalizes the interrupted run as errored. An attached SSE, synchronous response, or WebSocket may fail. Flue does not automatically start a replacement run. | +| Operation | After interruption | +| ---------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Direct attached agent HTTP/SSE/WebSocket prompt | The accepted prompt remains queued independently of its transport. Flue requeues only when canonical input is provably absent, recognizes provably completed canonical output, and otherwise records a visible terminal interruption without blindly replaying provider work. No public agent run exists. | +| Dispatched agent input | Durable delivery and internal deduplication are keyed by `dispatchId` and persisted submission state, not by a run. Direct and dispatched inputs share one same-session order. Reconciliation uses the same conservative replay rules. | +| Flue workflow invocation (`202`, SSE, `?wait=result`, or workflow WebSocket) | Flue terminalizes the interrupted run as errored. An attached SSE, synchronous response, or WebSocket may fail. Flue does not automatically start a replacement run. | -Cloudflare direct HTTP and WebSocket prompt execution is wrapped in a Fiber that checkpoints the submitted prompt for best-effort retry. Session transcript persistence is unchanged: interrupted assistant/tool progress not yet saved at the normal idle boundary is regenerated from the latest saved session snapshot. There are narrow interruption windows before prompt checkpointing or after transcript save but before Fiber cleanup where retry can be unavailable or duplicate already-completed work. +Cloudflare direct prompts and dispatched inputs enter one SQLite-backed submission queue owned by the target agent Durable Object. The attached transport observes accepted backend work but does not own it: losing an HTTP response, SSE stream, or WebSocket does not cancel the accepted submission. Flue does not recreate a lost live subscription, expose direct-submission lookup, or replay missed stream events. + +Before provider processing starts, Flue persists canonical submitted input and records an operational input-application boundary. After interruption, Flue retries only when it can prove provider work did not cross that boundary. If replay safety is uncertain, it appends a framework interruption advisory to canonical session history and terminalizes the operational submission instead of risking duplicate model work or external effects. Later same-session prompts can see that factual advisory. All Cloudflare workflow invocation transports use the same Fiber-backed durable admission path. The transport controls only how the initiating caller observes the admitted run: immediate `202`, live SSE, a synchronous result, or workflow WebSocket events while the connection remains available. -Recovery is **at-least-once** where durable prompt retry or asynchronous processing applies. An interruption after an external action has begun can cause that action to execute again. For dispatched agent work, use `dispatchId` or an application-level idempotency key when coordinating external side effects. Direct attached prompts do not expose a run identifier or replay API. +External effects remain application-owned. An interruption can leave the outcome of already-started model or tool activity uncertain, and an explicit caller retry can repeat effects. For dispatched agent work, correlate effects with `dispatchId` or an application-level idempotency key. Direct attached prompts do not expose a public receipt or replay API. -Flue persists workflow invocation payloads with workflow run records before admitted work starts so operators can inspect the original input after an interruption. Flue does not automatically retry interrupted workflows. The caller or application should decide whether retry is appropriate and explicitly invoke the workflow again when needed. Use an application-level idempotency key when a repeated invocation may encounter external side effects from an earlier attempt. Dispatched agent inputs are persisted as delivery/session state correlated by `dispatchId`, not as agent runs. Direct prompt Fiber checkpoints capture submitted prompt input for retry but do not create agent runs. Treat persisted inputs as durable application data: do not submit secrets or sensitive values unless your application retention and access policy permits storing them. +Flue persists workflow invocation payloads with workflow run records before admitted work starts so operators can inspect the original input after an interruption. Flue does not automatically retry interrupted workflows. The caller or application should decide whether retry is appropriate and explicitly invoke the workflow again when needed. Use an application-level idempotency key when a repeated invocation may encounter external side effects from an earlier attempt. Agent submission payloads are likewise durable application data while queued and running. Terminal rows become eligible for bounded lazy cleanup after seven days and are removed during later agent activity; an entirely idle Durable Object may retain eligible rows longer. The same eligibility horizon bounds duplicate-delivery protection for repeated forwarding of one `dispatchId`; it is not a public lookup API. Treat persisted inputs as sensitive: do not submit secrets unless your application retention and access policy permits storing them. -When Flue terminalizes an admitted interrupted workflow run, it emits `run_resume` before `run_end`, including when the interruption happened before live observers received `run_start`. This is recovery of terminal handling, not resumed or retried workflow code. Flue does not automatically propagate a trace carrier with dispatched input or restore one when retrying an interrupted direct prompt. For trace interpretation and application-owned HTTP extraction, see [Observability](/docs/guide/observability/#attach-application-trace-context). +When Flue terminalizes an admitted interrupted workflow run, it emits `run_resume` before `run_end`, including when the interruption happened before live observers received `run_start`. This is recovery of terminal handling, not resumed or retried workflow code. Flue does not automatically propagate a trace carrier with dispatched input or preserve the original attached direct request after durable admission. For trace interpretation and application-owned HTTP extraction, see [Observability](/docs/guide/observability/#attach-application-trace-context). Flue workflows do not resume from checkpointed durable steps after Durable Object interruption. For jobs that require durable step-level continuation, implement those steps with [Cloudflare Workflows](https://developers.cloudflare.com/workflows/). ### Beta persisted-schema boundary -Flue supports the Cloudflare Durable Object SQL table shape created by `v0.8.0` or newer. Existing supported databases may retain unused historical columns; they do not require table rebuilds. Persisted agent session records still follow Flue's beta session-data version boundary. Clear or separately migrate records written by an older session-data schema before upgrading. +Flue supports the Cloudflare Durable Object SQL table shape created by `v0.8.0` or newer. Existing supported SQLite-backed databases receive additive execution-store tables such as `flue_agent_submissions` at runtime and may retain unused historical columns; they do not require table rebuilds. KV-backed Durable Object classes remain outside this boundary because Cloudflare cannot convert them to SQLite in place. Persisted agent session records still follow Flue's beta session-data version boundary. Clear or separately migrate records written by an older session-data schema before upgrading. ## Sandbox context diff --git a/apps/docs/src/content/docs/guide/message-driven-agents.md b/apps/docs/src/content/docs/guide/message-driven-agents.md index a734e75d..310298cd 100644 --- a/apps/docs/src/content/docs/guide/message-driven-agents.md +++ b/apps/docs/src/content/docs/guide/message-driven-agents.md @@ -167,7 +167,7 @@ Fields: `await dispatch(...)` means the input was accepted and queued for the target session according to the current runtime's guarantees. It does not mean the model finished processing, produced a reply, or completed tool calls. The returned `dispatchId` identifies asynchronous delivery and any delivery recovery or idempotency behavior; it is not a run ID. -Flue preserves the structured input in session storage and renders it deterministically into model-visible context. Dispatched inputs emit agent lifecycle events correlated by instance, session, and `dispatchId`; they do not enter `/runs` or `flue logs`. +On Cloudflare, direct prompts and dispatched input enter the same durable per-session order. Flue preserves structured dispatch input in session storage and renders it deterministically into model-visible context. Dispatched inputs emit agent lifecycle events correlated by instance, session, and `dispatchId`; they do not enter `/runs` or `flue logs`. Design external effects to be idempotent: interruption reconciliation avoids blind replay, but an explicit caller retry can still repeat effects whose prior outcome is unknown. ## Lifecycle and correlation diff --git a/apps/docs/src/content/docs/guide/observability.md b/apps/docs/src/content/docs/guide/observability.md index 658e5c0f..5d2b1907 100644 --- a/apps/docs/src/content/docs/guide/observability.md +++ b/apps/docs/src/content/docs/guide/observability.md @@ -125,7 +125,7 @@ observe( ); ``` -This is an application-owned extraction policy, not automatic Flue propagation. On Cloudflare, the initial Worker request keeps its ordinary HTTP headers when Flue forwards it into a Durable Object. Flue does not automatically propagate a trace carrier with dispatched input or restore one when retrying an interrupted direct prompt. Resolve dispatched parents from application-owned correlation state when needed. See [Deploy Agents on Cloudflare](/docs/ecosystem/deploy/cloudflare/#interruption-and-recovery-semantics) for the platform-specific transport boundaries. +This is an application-owned extraction policy, not automatic Flue propagation. On Cloudflare, route middleware sees the original inbound request before durable admission. Later SQL-backed direct-agent processing uses a synthetic internal request, and dispatched work does not carry an HTTP trace carrier automatically. Capture correlation before admission and resolve later parents from application-owned state when needed. See [Deploy Agents on Cloudflare](/docs/ecosystem/deploy/cloudflare/#interruption-and-recovery-semantics) for the platform-specific transport boundaries. Flue spans describe semantic work such as workflows, operations, turns, and tools. The adapter does not activate OpenTelemetry context around provider SDK calls, so spans created by separate provider auto-instrumentation may require application-owned instrumentation or composition to appear beneath the intended Flue span. diff --git a/packages/cli/README.md b/packages/cli/README.md index 24cef48a..b9bce5a8 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -67,7 +67,7 @@ export async function run({ init, payload }: FlueContext) { A support agent can also run on Cloudflare without a container by using Flue's default virtual sandbox. Populate its filesystem with the context the agent needs, then it can search that content with its built-in `grep`, `glob`, and `read` tools. -Because this agent is deployed to Cloudflare, message history and session state are automatically persisted for you. So you (or your customer) can revisit this support session days, weeks, or years later and pick up exactly where you left off. +Cloudflare Durable Objects persist stored conversation history across later requests. The default virtual sandbox remains in memory, so seed any files each time you need them or choose a durable sandbox connector for long-lived workspace state. ```ts // .flue/workflows/support.ts @@ -281,7 +281,7 @@ curl http://localhost:3583/agents/hello/session-xyz \ -d '{"message": "Hello from another conversation"}' ``` -Agent instances own sandbox state such as files written across prompt and dispatched-input interactions. Harnesses group related session state within an instance. Sessions persist message history and conversation metadata inside a harness. On Cloudflare, session data is backed by Durable Objects and survives across requests. On Node.js, sessions are stored in memory by default unless you provide a custom store. +Agent instances provide a stable identity boundary for prompt and dispatched-input interactions. Harnesses group related session state within an instance. Sessions persist message history and conversation metadata inside a harness. Sandbox-file durability depends on the configured sandbox or connector; the default virtual filesystem is in memory. On Cloudflare, session conversation data is backed by Durable Objects and survives across requests. On Node.js, sessions are stored in memory by default unless you provide a custom store. In production, generate a stable URL `` for the agent instance you want to preserve. Use `harness.session(threadName)` when you need multiple conversations inside the same harness. @@ -418,7 +418,7 @@ await job.ready; await job.invoke({ text: 'Summarize me' }); ``` -An exported `websocket` middleware can authenticate its own agent or workflow socket endpoint. Use a custom `app.ts` for centralized authentication or mounted prefixes, applying ordinary Hono middleware before `app.route('/api', flue())`; the same routing model works on Node and Cloudflare. Include the custom mount in `baseUrl: 'https://example.com/api'` and use `websocketUrl: (url) => { url.searchParams.set('token', socketToken); return url; }` for URL-carried or signed handshake authentication. HTTP `token` and `headers` options do not automatically apply to WebSocket upgrades; browsers should use cookies or application-designed URL authentication, while Node clients needing implementation-specific headers can provide a custom `websocket` factory. Avoid header-mutating middleware around WebSocket upgrade routes. `flue dev --target cloudflare` requires `wrangler` as a peer dependency in your project (`npm install --save-dev wrangler`). +An exported `websocket` middleware can authenticate its own agent or workflow socket endpoint. Use a custom `app.ts` for centralized authentication or mounted prefixes, applying ordinary Hono middleware before `app.route('/api', flue())`; the same routing model works on Node and Cloudflare. Include the custom mount in `baseUrl: 'https://example.com/api'` and use `websocketUrl: (url) => { url.searchParams.set('token', socketToken); return url; }` for URL-carried or signed handshake authentication. HTTP `token` and `headers` options do not automatically apply to WebSocket upgrades; browsers should use cookies or application-designed URL authentication, while Node clients needing implementation-specific headers can provide a custom `websocket` factory. Avoid header-mutating middleware around WebSocket upgrade routes. Install `wrangler` in Cloudflare projects that use Wrangler commands such as `wrangler deploy` or secret management; `flue dev --target cloudflare` no longer requires it as an `@flue/cli` peer dependency. #### Loading environment variables diff --git a/packages/cli/src/lib/build-plugin-cloudflare.ts b/packages/cli/src/lib/build-plugin-cloudflare.ts index d75d8831..457ac35e 100644 --- a/packages/cli/src/lib/build-plugin-cloudflare.ts +++ b/packages/cli/src/lib/build-plugin-cloudflare.ts @@ -58,19 +58,16 @@ const ${agentClassName(agent.name)} = class ${agentClassName(agent.name)} extend } async onStart(props) { + await restoreFlueAgentSubmissionWake(this); if (typeof super.onStart === 'function') await super.onStart(props); - await armFlueAgentSubmissionRetry(this); - await reconcileFlueAgentSubmissions(this, ${JSON.stringify(agent.name)}, { preserveSuccessor: true }); + await reconcileFlueAgentSubmissions(this, ${JSON.stringify(agent.name)}, { driverAlreadyArmed: true }); } - async __flueWakeAgentSubmissions(wake) { - await reconcileFlueAgentSubmissions(this, ${JSON.stringify(agent.name)}, { preserveSuccessor: true, executingWake: wake }); - } - - async __flueRetryAgentSubmissions(_payload, schedule) { - if (!(await reconcileFlueAgentSubmissions(this, ${JSON.stringify(agent.name)}))) { - await this.cancelSchedule(schedule.id); - } + async __flueWakeAgentSubmissions() { + const submissions = getAgentExecutionStore(this).submissions; + if (!submissions.hasUnsettledSubmissions()) return; + await armFlueAgentSubmissionWake(this, { idempotent: false }); + await reconcileFlueAgentSubmissions(this, ${JSON.stringify(agent.name)}, { driverAlreadyArmed: true }); } async onRequest(request) { @@ -107,7 +104,7 @@ const ${agentClassName(agent.name)} = class ${agentClassName(agent.name)} extend if (ctx.name === 'flue:dispatch') { return handleFlueDispatchRecovered(ctx, this, ${JSON.stringify(agent.name)}); } - if (ctx.name === 'flue:dispatch-attempt') { + if (ctx.name === 'flue:submission-attempt') { return handleFlueDispatchAttemptRecovered(ctx, this); } if (ctx.name === 'flue:direct') { @@ -214,6 +211,7 @@ import { createDurableRunStore, createSqlAgentExecutionStore, createSqlSessionStore, + SqlAgentDispatchReceiptRetainedError, SqlAgentSubmissionConflictError, createRunSubscriberRegistry, bashFactoryToSessionEnv, @@ -225,8 +223,10 @@ import { assertCurrentDispatchInput, createDispatchAgentHandler, createDispatchInputInspectionHandler, + createLegacyDirectSubmissionTerminalHandler, createDirectSubmissionAgentHandler, createDirectSubmissionInputInspectionHandler, + createSubmissionTerminalHandler, createAgentSubmissionObserverRegistry, failRecoveredRun, configureFlueRuntime, @@ -284,7 +284,7 @@ const workflowModules = { ${workflowModuleEntries} }; const normalized = normalizeBuiltModules(agentModules, workflowModules); -const { manifest, directHandlers, localAgentHandlers, createdAgents, dispatchAgentNames, websocketAgentHandlers, workflowHandlers, websocketWorkflowHandlers, agentRouteMiddleware, agentWebSocketMiddleware, workflowRouteMiddleware, workflowWebSocketMiddleware } = normalized; +const { manifest, directHandlers, createdAgents, dispatchAgentNames, websocketAgentHandlers, workflowHandlers, websocketWorkflowHandlers, agentRouteMiddleware, agentWebSocketMiddleware, workflowRouteMiddleware, workflowWebSocketMiddleware } = normalized; const agentIdentities = { ${agentIdentityEntries} }; @@ -364,9 +364,9 @@ const memoryWorkflowSessionStore = new InMemorySessionStore(); const memoryRunStore = new InMemoryRunStore(); const FLUE_AGENT_EXECUTION_STORE = Symbol('flueAgentExecutionStore'); const FLUE_AGENT_SUBMISSION_WAKE_CALLBACK = '__flueWakeAgentSubmissions'; -const FLUE_AGENT_SUBMISSION_RETRY_CALLBACK = '__flueRetryAgentSubmissions'; -const FLUE_AGENT_SUBMISSION_RETRY_SECONDS = 30; +const FLUE_AGENT_SUBMISSION_WAKE_SECONDS = 30; const FLUE_AGENT_SUBMISSION_ATTEMPT_STALE_MS = 15 * 60 * 1000; +const FLUE_AGENT_SUBMISSION_TERMINAL_RETENTION_MS = 7 * 24 * 60 * 60 * 1000; const INTERNAL_DISPATCH_PATH = '/__flue/internal/dispatch'; const dispatchQueue = { async enqueue(input) { @@ -413,16 +413,25 @@ function createContextForRequest(id, runId, payload, doInstance, req, defaultSto } function createAgentContextForRequest(id, payload, doInstance, req, initialEventIndex, dispatchId) { - return createContextForRequest( + const executionStore = getAgentExecutionStore(doInstance); + return createFlueContext({ id, - undefined, payload, - doInstance, + env: doInstance?.env ?? {}, req, - getAgentExecutionStore(doInstance).sessions, initialEventIndex, dispatchId, - ); + agentConfig: { + systemPrompt, skills, packagedSkills, model: undefined, resolveModel, + }, + createDefaultEnv, + defaultStore: executionStore.sessions, + resolveSandbox, + sessionDeletionCoordinator: { + begin: (sessionKey) => executionStore.submissions.beginSessionDeletion(sessionKey), + finish: (sessionKey) => executionStore.submissions.finishSessionDeletion(sessionKey), + }, + }); } function createWorkflowContextForRequest(id, runId, payload, doInstance, req, initialEventIndex, dispatchId) { @@ -482,13 +491,13 @@ async function handleFlueDispatchRecovered(ctx, doInstance, agentName) { if (!input || input.agent !== agentName || input.id !== doInstance.name) return { status: 'error', error: 'Dispatch recovery metadata is invalid.' }; try { await validateAgentDispatchAdmission({ input }); - await armFlueAgentSubmissionAdmissionWakes(doInstance); + await armFlueAgentSubmissionAdmissionWake(doInstance); const submissions = getAgentExecutionStore(doInstance).submissions; const legacy = listActiveLegacyManagedDispatches(doInstance, agentName); legacy.push({ fiberId: ctx.id, createdAt: ctx.createdAt, input }); legacy.sort(compareLegacyManagedDispatches); submissions.adoptLegacyDispatches(legacy.map((dispatch) => dispatch.input)); - await reconcileFlueAgentSubmissions(doInstance, agentName, { preserveSuccessor: true }); + await reconcileFlueAgentSubmissions(doInstance, agentName, { driverAlreadyArmed: true }); return { status: 'completed' }; } catch (error) { return { status: 'error', error: error instanceof Error ? error.message : String(error) }; @@ -499,14 +508,38 @@ async function handleFlueDispatchAttemptRecovered(ctx, doInstance) { const submissionId = ctx.snapshot?.submissionId; const attemptId = ctx.snapshot?.attemptId; if (typeof submissionId !== 'string' || typeof attemptId !== 'string') return; + await restoreFlueAgentSubmissionWake(doInstance); const submissions = getAgentExecutionStore(doInstance).submissions; - const submission = submissions.requestSubmissionRecovery(submissionId, attemptId); - if (!submission) return; - await armFlueAgentSubmissionAdmissionWakes(doInstance); + submissions.requestSubmissionRecovery(submissionId, attemptId); } -async function handleFlueDirectRecovered(_ctx, doInstance, agentName) { - console.error('[flue:direct-recovery]', { agentName, instanceId: doInstance.name, operation: 'legacy_handoff', outcome: 'terminal_uncertainty' }, new Error('A pre-upgrade direct prompt was interrupted. Provider replay was not attempted.')); +async function handleFlueDirectRecovered(ctx, doInstance, agentName) { + const payload = ctx.snapshot?.payload; + const message = 'A pre-upgrade direct prompt was interrupted. Provider replay was not attempted.'; + if (!payload || typeof payload !== 'object' || Array.isArray(payload) || typeof payload.message !== 'string' || (payload.session !== undefined && (typeof payload.session !== 'string' || payload.session.trim() === '' || payload.session.startsWith('task:')))) { + console.error('[flue:direct-recovery]', { agentName, instanceId: doInstance.name, operation: 'legacy_handoff', outcome: 'terminal_uncertainty' }, new Error(message)); + return; + } + const agent = createdAgents[agentName]; + if (!agent) throw new Error('[flue] Agent target unavailable during legacy direct terminalization.'); + const input = { + submissionId: 'legacy-direct:' + ctx.id, + agent: agentName, + id: doInstance.name, + session: typeof payload.session === 'string' && payload.session.trim() !== '' ? payload.session : 'default', + payload, + acceptedAt: new Date(ctx.createdAt).toISOString(), + }; + const request = new Request('https://flue.invalid/agents/' + encodeURIComponent(agentName) + '/' + encodeURIComponent(doInstance.name), { method: 'POST' }); + const directCtx = createAgentContextForRequest(doInstance.name, payload, doInstance, request); + await runWithInstanceContext(doInstance, agentRuntimeIdentity(agentName), () => + createLegacyDirectSubmissionTerminalHandler(agent, input, { + submissionId: input.submissionId, + kind: 'direct', + reason: 'interrupted_after_input_application', + message, + })(directCtx), + ); } async function handleFlueWorkflowFiberRecovered(ctx, doInstance, workflowName) { @@ -531,50 +564,43 @@ async function handleFlueWorkflowFiberRecovered(ctx, doInstance, workflowName) { function armFlueAgentSubmissionWake(doInstance, options = {}) { assertAgentsDurabilityApi(doInstance, 'schedule'); return doInstance.schedule( - options.delaySeconds ?? 0, + options.delaySeconds ?? FLUE_AGENT_SUBMISSION_WAKE_SECONDS, FLUE_AGENT_SUBMISSION_WAKE_CALLBACK, - { generation: options.generation ?? 0 }, - { idempotent: true }, + undefined, + { idempotent: options.idempotent ?? true }, ); } -function armFlueAgentSubmissionAdmissionWakes(doInstance) { - return Promise.all([ - armFlueAgentSubmissionWake(doInstance, { generation: 0 }), - armFlueAgentSubmissionWake(doInstance, { generation: 1 }), - armFlueAgentSubmissionRetry(doInstance), - ]); +function armFlueAgentSubmissionAdmissionWake(doInstance) { + return armFlueAgentSubmissionWake(doInstance); } -function armFlueAgentSubmissionRetry(doInstance) { - assertAgentsDurabilityApi(doInstance, 'scheduleEvery'); - return doInstance.scheduleEvery(FLUE_AGENT_SUBMISSION_RETRY_SECONDS, FLUE_AGENT_SUBMISSION_RETRY_CALLBACK); +function cleanupFlueAgentSubmissionTerminalState(doInstance) { + return getAgentExecutionStore(doInstance).submissions.cleanupTerminalSubmissions( + Date.now() - FLUE_AGENT_SUBMISSION_TERMINAL_RETENTION_MS, + ); +} + +async function restoreFlueAgentSubmissionWake(doInstance) { + const submissions = getAgentExecutionStore(doInstance).submissions; + if (!submissions.hasUnsettledSubmissions()) return false; + await armFlueAgentSubmissionWake(doInstance); + return true; } async function reconcileFlueAgentSubmissions(doInstance, agentName, options = {}) { const submissions = getAgentExecutionStore(doInstance).submissions; + cleanupFlueAgentSubmissionTerminalState(doInstance); + if (!submissions.hasUnsettledSubmissions()) return false; + if (!options.driverAlreadyArmed) await restoreFlueAgentSubmissionWake(doInstance); const legacySessions = await adoptLegacyManagedDispatches(doInstance, agentName); if (!submissions.hasUnsettledSubmissions()) return false; - await armFlueAgentSubmissionRetry(doInstance); - if (options.preserveSuccessor) { - if (options.executingWake) { - await armFlueAgentSubmissionWake(doInstance, { - delaySeconds: FLUE_AGENT_SUBMISSION_RETRY_SECONDS, - generation: options.executingWake.generation === 0 ? 1 : 0, - }); - } else { - await Promise.all([ - armFlueAgentSubmissionWake(doInstance, { delaySeconds: FLUE_AGENT_SUBMISSION_RETRY_SECONDS, generation: 0 }), - armFlueAgentSubmissionWake(doInstance, { delaySeconds: FLUE_AGENT_SUBMISSION_RETRY_SECONDS, generation: 1 }), - ]); - } - } try { const attemptMarkers = listActiveSqlAgentSubmissionAttemptMarkers(doInstance); if (attemptMarkers.blockAll) return true; for (const submission of submissions.listRunningSubmissions()) { if (activeFlueAgentSubmissionAttempts.has(submissionAttemptLocalKey(doInstance, submission))) continue; - if (attemptMarkers.keys.has(submissionAttemptMarkerKey(submission)) && submission.recoveryRequestedAt === undefined) continue; + if (submission.status !== 'terminalizing' && attemptMarkers.keys.has(submissionAttemptMarkerKey(submission)) && submission.recoveryRequestedAt === undefined) continue; if (legacySessions.has(submission.session)) continue; await reconcileInterruptedSqlAgentSubmission(submission, doInstance, agentName); } @@ -594,6 +620,18 @@ async function reconcileInterruptedSqlAgentSubmission(submission, doInstance, ag const { attemptId, input } = submission; if (!attemptId) return; const submissions = getAgentExecutionStore(doInstance).submissions; + if (submission.status === 'terminalizing') { + await failInterruptedSqlAgentSubmission( + submission, + doInstance, + agentName, + submission.inputAppliedAt === undefined ? 'interrupted_before_input_marker' : 'interrupted_after_input_application', + new Error(submission.inputAppliedAt === undefined + ? '[flue] Agent submission attempt was interrupted after canonical input persistence but before the input-application marker was recorded. Provider replay was not attempted.' + : '[flue] Agent submission attempt was interrupted after input application without a completed canonical response. Provider replay was not attempted.'), + ); + return; + } const agent = createdAgents[agentName]; if (!agent) throw new Error('[flue] Agent target unavailable during durable reconciliation.'); const request = new Request('https://flue.invalid' + INTERNAL_DISPATCH_PATH, { method: 'POST' }); @@ -608,14 +646,47 @@ async function reconcileInterruptedSqlAgentSubmission(submission, doInstance, ag submissions.requeueSubmissionBeforeInputApplied(submission.submissionId, attemptId); return; } - submissions.failSubmission(submission.submissionId, attemptId, new Error('[flue] Agent submission attempt was interrupted after canonical input persistence but before the input-application marker was recorded. Provider replay was not attempted.')); + await failInterruptedSqlAgentSubmission( + submission, + doInstance, + agentName, + 'interrupted_before_input_marker', + new Error('[flue] Agent submission attempt was interrupted after canonical input persistence but before the input-application marker was recorded. Provider replay was not attempted.'), + ); return; } if (state === 'completed') { submissions.completeSubmission(submission.submissionId, attemptId); return; } - submissions.failSubmission(submission.submissionId, attemptId, new Error('[flue] Agent submission attempt was interrupted after input application without a completed canonical response. Provider replay was not attempted.')); + await failInterruptedSqlAgentSubmission( + submission, + doInstance, + agentName, + 'interrupted_after_input_application', + new Error('[flue] Agent submission attempt was interrupted after input application without a completed canonical response. Provider replay was not attempted.'), + ); +} + +async function failInterruptedSqlAgentSubmission(submission, doInstance, agentName, reason, error) { + const { attemptId, input } = submission; + if (!attemptId) return; + const submissions = getAgentExecutionStore(doInstance).submissions; + if (submission.status !== 'terminalizing' && !submissions.beginSubmissionTerminalization(submission.submissionId, attemptId)) return; + const agent = createdAgents[agentName]; + if (!agent) throw new Error('[flue] Agent target unavailable during durable terminalization.'); + const request = new Request('https://flue.invalid' + INTERNAL_DISPATCH_PATH, { method: 'POST' }); + const ctx = createAgentContextForRequest(doInstance.name, submission.kind === 'direct' ? input.payload : input, doInstance, request, undefined, input.dispatchId); + await runWithInstanceContext(doInstance, agentRuntimeIdentity(agentName), () => + createSubmissionTerminalHandler(agent, input, { + submissionId: submission.submissionId, + kind: submission.kind, + reason, + message: error.message, + })(ctx), + ); + const failed = submissions.finalizeSubmissionTerminalization(submission.submissionId, attemptId, error); + if (failed && submission.kind === 'direct') agentSubmissionObservers.fail(submission.submissionId, error); } function startSqlAgentSubmissionAttempt(submission, doInstance, agentName) { @@ -626,7 +697,7 @@ function startSqlAgentSubmissionAttempt(submission, doInstance, agentName) { activeFlueAgentSubmissionAttempts.add(attemptKey); let running; try { - running = doInstance.runFiber('flue:dispatch-attempt', async (fiberCtx) => { + running = doInstance.runFiber('flue:submission-attempt', async (fiberCtx) => { fiberCtx.stash({ submissionId: submission.submissionId, attemptId: submission.attemptId }); await processSqlAgentSubmission(submission, doInstance, agentName); }); @@ -653,7 +724,7 @@ function listActiveSqlAgentSubmissionAttemptMarkers(doInstance) { const keys = new Set(); let blockAll = false; const rows = doInstance.ctx.storage.sql.exec( - "SELECT snapshot, created_at FROM cf_agents_runs WHERE name = 'flue:dispatch-attempt'", + "SELECT snapshot, created_at FROM cf_agents_runs WHERE name = 'flue:submission-attempt'", ).toArray(); for (const row of rows) { if (typeof row.created_at !== 'number') { @@ -755,7 +826,7 @@ async function processSqlAgentSubmission(submission, doInstance, agentName) { throw error; } finally { if (submission.kind === 'direct') ctx?.setEventCallback(undefined); - void reconcileFlueAgentSubmissions(doInstance, agentName, { preserveSuccessor: true }).catch((error) => { + void reconcileFlueAgentSubmissions(doInstance, agentName).catch((error) => { console.error('[flue:submission-reconciliation]', { agentName, instanceId: doInstance.name, operation: 'settlement', outcome: 'reconcile_failed' }, error); }); } @@ -779,9 +850,9 @@ async function admitAttachedAgentSubmission(doInstance, agentName, payload, _req }; const attachment = agentSubmissionObservers.attach(submissionId, { onEvent }); try { - await armFlueAgentSubmissionAdmissionWakes(doInstance); + await armFlueAgentSubmissionAdmissionWake(doInstance); getAgentExecutionStore(doInstance).submissions.admitDirect(input); - await reconcileFlueAgentSubmissions(doInstance, agentName); + await reconcileFlueAgentSubmissions(doInstance, agentName, { driverAlreadyArmed: true }); return await attachment.completion; } finally { attachment.detach(); @@ -834,15 +905,21 @@ async function dispatchAgent(request, doInstance, agentName, handler) { if (input.agent !== agentName || input.id !== id) return new Response('Invalid internal dispatch target.', { status: 400 }); if (!createdAgents[agentName]) return new Response('Dispatch target unavailable.', { status: 404 }); await validateAgentDispatchAdmission({ input }); - await armFlueAgentSubmissionAdmissionWakes(doInstance); + const submissions = getAgentExecutionStore(doInstance).submissions; + cleanupFlueAgentSubmissionTerminalState(doInstance); + submissions.cleanupDispatchReceipt(input.dispatchId, Date.now() - FLUE_AGENT_SUBMISSION_TERMINAL_RETENTION_MS); + const priorReceipt = submissions.getDispatchReceipt(input.dispatchId); + if (priorReceipt) return Response.json({ dispatchId: priorReceipt.submissionId, acceptedAt: new Date(priorReceipt.acceptedAt).toISOString() }); + await armFlueAgentSubmissionAdmissionWake(doInstance); let submission; try { - submission = getAgentExecutionStore(doInstance).submissions.admitDispatch(input); + submission = submissions.admitDispatch(input); } catch (error) { + if (error instanceof SqlAgentDispatchReceiptRetainedError) return Response.json({ dispatchId: error.receipt.submissionId, acceptedAt: new Date(error.receipt.acceptedAt).toISOString() }); if (error instanceof SqlAgentSubmissionConflictError) return new Response('Conflicting internal dispatch replay.', { status: 409 }); throw error; } - await reconcileFlueAgentSubmissions(doInstance, agentName); + await reconcileFlueAgentSubmissions(doInstance, agentName, { driverAlreadyArmed: true }); return Response.json({ dispatchId: submission.submissionId, acceptedAt: submission.input.acceptedAt }); } const identity = agentRuntimeIdentity(agentName); diff --git a/packages/cli/src/lib/build.ts b/packages/cli/src/lib/build.ts index d42ca4b5..496b9263 100644 --- a/packages/cli/src/lib/build.ts +++ b/packages/cli/src/lib/build.ts @@ -432,12 +432,13 @@ function readRuntimeVersion(root: string): string { function assertCloudflareAgentsSdkFloor(root: string): void { const minimum = [0, 14, 1] as const; + const nextBreaking = [0, 15, 0] as const; let entry: string; try { entry = createRequire(path.join(root, '__flue_resolve__.cjs')).resolve('agents'); } catch { throw new Error( - '[flue] Cloudflare target requires the installed "agents" package to be at least 0.14.1. Install or upgrade "agents" in this project.', + '[flue] Cloudflare target requires the installed "agents" package to satisfy >=0.14.1 <0.15.0. Install a compatible "agents" version in this project.', ); } const pkgPath = packageUpSync({ cwd: path.dirname(entry) }); @@ -450,9 +451,9 @@ function assertCloudflareAgentsSdkFloor(root: string): void { } const match = typeof version === 'string' ? /^(\d+)\.(\d+)\.(\d+)$/.exec(version) : null; const current = match?.slice(1).map(Number); - if (!current || isVersionBelow(current, minimum)) { + if (!current || isVersionBelow(current, minimum) || !isVersionBelow(current, nextBreaking)) { throw new Error( - `[flue] Cloudflare target requires the installed "agents" package to be at least 0.14.1. Found ${String(version)}. Install or upgrade "agents" in this project.`, + `[flue] Cloudflare target requires the installed "agents" package to satisfy >=0.14.1 <0.15.0. Found ${String(version)}. Install a compatible "agents" version in this project.`, ); } } diff --git a/packages/runtime/README.md b/packages/runtime/README.md index 24cef48a..b9bce5a8 100644 --- a/packages/runtime/README.md +++ b/packages/runtime/README.md @@ -67,7 +67,7 @@ export async function run({ init, payload }: FlueContext) { A support agent can also run on Cloudflare without a container by using Flue's default virtual sandbox. Populate its filesystem with the context the agent needs, then it can search that content with its built-in `grep`, `glob`, and `read` tools. -Because this agent is deployed to Cloudflare, message history and session state are automatically persisted for you. So you (or your customer) can revisit this support session days, weeks, or years later and pick up exactly where you left off. +Cloudflare Durable Objects persist stored conversation history across later requests. The default virtual sandbox remains in memory, so seed any files each time you need them or choose a durable sandbox connector for long-lived workspace state. ```ts // .flue/workflows/support.ts @@ -281,7 +281,7 @@ curl http://localhost:3583/agents/hello/session-xyz \ -d '{"message": "Hello from another conversation"}' ``` -Agent instances own sandbox state such as files written across prompt and dispatched-input interactions. Harnesses group related session state within an instance. Sessions persist message history and conversation metadata inside a harness. On Cloudflare, session data is backed by Durable Objects and survives across requests. On Node.js, sessions are stored in memory by default unless you provide a custom store. +Agent instances provide a stable identity boundary for prompt and dispatched-input interactions. Harnesses group related session state within an instance. Sessions persist message history and conversation metadata inside a harness. Sandbox-file durability depends on the configured sandbox or connector; the default virtual filesystem is in memory. On Cloudflare, session conversation data is backed by Durable Objects and survives across requests. On Node.js, sessions are stored in memory by default unless you provide a custom store. In production, generate a stable URL `` for the agent instance you want to preserve. Use `harness.session(threadName)` when you need multiple conversations inside the same harness. @@ -418,7 +418,7 @@ await job.ready; await job.invoke({ text: 'Summarize me' }); ``` -An exported `websocket` middleware can authenticate its own agent or workflow socket endpoint. Use a custom `app.ts` for centralized authentication or mounted prefixes, applying ordinary Hono middleware before `app.route('/api', flue())`; the same routing model works on Node and Cloudflare. Include the custom mount in `baseUrl: 'https://example.com/api'` and use `websocketUrl: (url) => { url.searchParams.set('token', socketToken); return url; }` for URL-carried or signed handshake authentication. HTTP `token` and `headers` options do not automatically apply to WebSocket upgrades; browsers should use cookies or application-designed URL authentication, while Node clients needing implementation-specific headers can provide a custom `websocket` factory. Avoid header-mutating middleware around WebSocket upgrade routes. `flue dev --target cloudflare` requires `wrangler` as a peer dependency in your project (`npm install --save-dev wrangler`). +An exported `websocket` middleware can authenticate its own agent or workflow socket endpoint. Use a custom `app.ts` for centralized authentication or mounted prefixes, applying ordinary Hono middleware before `app.route('/api', flue())`; the same routing model works on Node and Cloudflare. Include the custom mount in `baseUrl: 'https://example.com/api'` and use `websocketUrl: (url) => { url.searchParams.set('token', socketToken); return url; }` for URL-carried or signed handshake authentication. HTTP `token` and `headers` options do not automatically apply to WebSocket upgrades; browsers should use cookies or application-designed URL authentication, while Node clients needing implementation-specific headers can provide a custom `websocket` factory. Avoid header-mutating middleware around WebSocket upgrade routes. Install `wrangler` in Cloudflare projects that use Wrangler commands such as `wrangler deploy` or secret management; `flue dev --target cloudflare` no longer requires it as an `@flue/cli` peer dependency. #### Loading environment variables diff --git a/packages/runtime/src/client.ts b/packages/runtime/src/client.ts index 330feb8d..026f2d26 100644 --- a/packages/runtime/src/client.ts +++ b/packages/runtime/src/client.ts @@ -45,6 +45,12 @@ export interface FlueContextConfig { */ req?: Request; initialEventIndex?: number; + sessionDeletionCoordinator?: SessionDeletionCoordinator; +} + +export interface SessionDeletionCoordinator { + begin(storageKey: string): void; + finish(storageKey: string): void; } /** Extends FlueContext with server-only methods. Agent handlers only see FlueContext. */ @@ -226,6 +232,7 @@ export function createFlueContext(config: FlueContextConfig): FlueContextInterna }, definition.tools, toolFactory, + config.sessionDeletionCoordinator, ); } catch (error) { initializedHarnessNames.delete(name); diff --git a/packages/runtime/src/cloudflare/agent-execution-store.ts b/packages/runtime/src/cloudflare/agent-execution-store.ts index 30168bfd..b5f29076 100644 --- a/packages/runtime/src/cloudflare/agent-execution-store.ts +++ b/packages/runtime/src/cloudflare/agent-execution-store.ts @@ -22,7 +22,12 @@ interface DurableObjectStorage { transactionSync?(closure: () => T): T; } -type SqlAgentSubmissionStatus = 'queued' | 'running' | 'completed' | 'error'; +type SqlAgentSubmissionStatus = 'queued' | 'running' | 'terminalizing' | 'completed' | 'error'; + +export interface SqlAgentDispatchReceipt { + readonly submissionId: string; + readonly acceptedAt: number; +} export interface SqlAgentSubmission { readonly sequence: number; @@ -43,13 +48,17 @@ export interface SqlAgentSubmission { export interface SqlAgentSubmissionStore { getSubmission(submissionId: string): SqlAgentSubmission | null; + getDispatchReceipt(submissionId: string): SqlAgentDispatchReceipt | null; admitDispatch(input: DispatchInput): SqlAgentSubmission; admitDirect(input: DirectSubmissionInput): SqlAgentSubmission; adoptLegacyDispatches(inputs: readonly DispatchInput[]): SqlAgentSubmission[]; hasUnsettledSubmissions(): boolean; - listQueuedSubmissions(): SqlAgentSubmission[]; listRunnableSubmissions(): SqlAgentSubmission[]; listRunningSubmissions(): SqlAgentSubmission[]; + beginSessionDeletion(sessionKey: string): void; + finishSessionDeletion(sessionKey: string): void; + cleanupTerminalSubmissions(completedBefore: number, limit?: number): number; + cleanupDispatchReceipt(submissionId: string, settledBefore: number): void; claimSubmission(submissionId: string, attemptId: string): SqlAgentSubmission | null; markSubmissionInputApplied(submissionId: string, attemptId: string): SqlAgentSubmission | null; requestSubmissionRecovery(submissionId: string, attemptId: string): SqlAgentSubmission | null; @@ -57,8 +66,10 @@ export interface SqlAgentSubmissionStore { submissionId: string, attemptId: string, ): SqlAgentSubmission | null; + beginSubmissionTerminalization(submissionId: string, attemptId: string): SqlAgentSubmission | null; completeSubmission(submissionId: string, attemptId: string): boolean; failSubmission(submissionId: string, attemptId: string, error: unknown): boolean; + finalizeSubmissionTerminalization(submissionId: string, attemptId: string, error: unknown): boolean; } export interface SqlAgentExecutionStore { @@ -68,6 +79,12 @@ export interface SqlAgentExecutionStore { export class SqlAgentSubmissionConflictError extends Error {} +export class SqlAgentDispatchReceiptRetainedError extends Error { + constructor(readonly receipt: SqlAgentDispatchReceipt) { + super('[flue] Internal dispatch replay is already settled.'); + } +} + export function createSqlSessionStore(sql: SqlStorage): SessionStore { ensureSessionTable(sql); return new SqlSessionStore(sql); @@ -141,6 +158,20 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { return row ? parseSubmission(row) : null; } + getDispatchReceipt(submissionId: string): SqlAgentDispatchReceipt | null { + const row = this.sql + .exec( + 'SELECT dispatch_id, accepted_at FROM flue_agent_dispatch_receipts WHERE dispatch_id = ? LIMIT 1', + submissionId, + ) + .toArray()[0]; + if (!row) return null; + if (typeof row.dispatch_id !== 'string' || typeof row.accepted_at !== 'number') { + throw new Error('[flue] Persisted dispatch receipt row is malformed.'); + } + return { submissionId: row.dispatch_id, acceptedAt: row.accepted_at }; + } + admitDispatch(input: DispatchInput): SqlAgentSubmission { return this.admitSubmission('dispatch', input.dispatchId, input); } @@ -207,27 +238,13 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { .exec( `SELECT 1 FROM flue_agent_submissions - WHERE status IN ('queued', 'running') + WHERE status IN ('queued', 'running', 'terminalizing') LIMIT 1`, ) .toArray().length > 0 ); } - listQueuedSubmissions(): SqlAgentSubmission[] { - return this.parseOperationalRows( - this.sql - .exec( - `SELECT ${submissionColumns} - FROM flue_agent_submissions - WHERE status = 'queued' - ORDER BY sequence ASC`, - ) - .toArray(), - 'queued', - ); - } - listRunnableSubmissions(): SqlAgentSubmission[] { const rows = this.sql .exec( @@ -238,7 +255,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { SELECT 1 FROM flue_agent_submissions AS earlier WHERE earlier.session_key = current.session_key - AND earlier.status IN ('queued', 'running') + AND earlier.status IN ('queued', 'running', 'terminalizing') AND earlier.sequence < current.sequence ) ORDER BY current.sequence ASC`, @@ -253,11 +270,91 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { .exec( `SELECT ${submissionColumns} FROM flue_agent_submissions - WHERE status = 'running' + WHERE status IN ('running', 'terminalizing') ORDER BY sequence ASC`, ) .toArray(), - 'running', + 'active', + ); + } + + beginSessionDeletion(sessionKey: string): void { + this.transactionSync(() => { + const active = this.sql + .exec( + `SELECT 1 + FROM flue_agent_submissions + WHERE session_key = ? AND status IN ('queued', 'running', 'terminalizing') + LIMIT 1`, + sessionKey, + ) + .toArray(); + if (active.length > 0) { + throw new Error( + '[flue] Session cannot be deleted while durable agent submissions are queued or running. Wait for accepted work to settle, then retry deletion.', + ); + } + this.sql.exec( + 'INSERT OR IGNORE INTO flue_agent_session_deletions (session_key, started_at) VALUES (?, ?)', + sessionKey, + Date.now(), + ); + }); + } + + finishSessionDeletion(sessionKey: string): void { + this.transactionSync(() => { + this.sql.exec( + `INSERT OR IGNORE INTO flue_agent_dispatch_receipts (dispatch_id, accepted_at, settled_at) + SELECT submission_id, accepted_at, completed_at + FROM flue_agent_submissions + WHERE session_key = ? AND kind = 'dispatch' AND status IN ('completed', 'error')`, + sessionKey, + ); + this.sql.exec( + `DELETE FROM flue_agent_submissions + WHERE session_key = ? AND status IN ('completed', 'error')`, + sessionKey, + ); + this.sql.exec('DELETE FROM flue_agent_session_deletions WHERE session_key = ?', sessionKey); + }); + } + + cleanupTerminalSubmissions(completedBefore: number, limit = 100): number { + if (!Number.isInteger(limit) || limit <= 0) { + throw new Error('[flue] Terminal submission cleanup limit must be a positive integer.'); + } + const rows = this.sql + .exec( + `SELECT sequence + FROM flue_agent_submissions + WHERE status IN ('completed', 'error') AND completed_at < ? + ORDER BY completed_at ASC, sequence ASC + LIMIT ?`, + completedBefore, + limit, + ) + .toArray(); + for (const row of rows) { + if (typeof row.sequence !== 'number') { + throw new Error('[flue] Persisted terminal submission row is malformed.'); + } + this.sql.exec( + `DELETE FROM flue_agent_submissions + WHERE sequence = ? AND status IN ('completed', 'error') AND completed_at < ?`, + row.sequence, + completedBefore, + ); + } + this.sql.exec('DELETE FROM flue_agent_dispatch_receipts WHERE settled_at < ?', completedBefore); + return rows.length; + } + + cleanupDispatchReceipt(submissionId: string, settledBefore: number): void { + this.sql.exec( + 'DELETE FROM flue_agent_dispatch_receipts WHERE dispatch_id = ? AND settled_at < ?', + submissionId, + settledBefore, ); } @@ -270,7 +367,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { SELECT 1 FROM flue_agent_submissions AS earlier WHERE earlier.session_key = current.session_key - AND earlier.status IN ('queued', 'running') + AND earlier.status IN ('queued', 'running', 'terminalizing') AND earlier.sequence < current.sequence )`, attemptId, @@ -323,6 +420,20 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { return submission?.status === 'queued' ? submission : null; } + beginSubmissionTerminalization(submissionId: string, attemptId: string): SqlAgentSubmission | null { + this.sql.exec( + `UPDATE flue_agent_submissions + SET status = 'terminalizing' + WHERE submission_id = ? AND status = 'running' AND attempt_id = ?`, + submissionId, + attemptId, + ); + const submission = this.getSubmission(submissionId); + return submission?.status === 'terminalizing' && submission.attemptId === attemptId + ? submission + : null; + } + completeSubmission(submissionId: string, attemptId: string): boolean { this.sql.exec( `UPDATE flue_agent_submissions @@ -350,6 +461,20 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { return submission?.status === 'error' && submission.attemptId === attemptId; } + finalizeSubmissionTerminalization(submissionId: string, attemptId: string, error: unknown): boolean { + this.sql.exec( + `UPDATE flue_agent_submissions + SET status = 'error', completed_at = ?, error = ? + WHERE submission_id = ? AND status = 'terminalizing' AND attempt_id = ?`, + Date.now(), + error instanceof Error ? error.message : String(error), + submissionId, + attemptId, + ); + const submission = this.getSubmission(submissionId); + return submission?.status === 'error' && submission.attemptId === attemptId; + } + private admitSubmission( kind: SqlAgentSubmission['kind'], submissionId: string, @@ -357,23 +482,36 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { ): SqlAgentSubmission { const payload = JSON.stringify(input); const acceptedAt = parseAcceptedAt(input.acceptedAt, `${kind} admission`); - this.sql.exec( - `INSERT OR IGNORE INTO flue_agent_submissions - (submission_id, session, session_key, kind, payload, status, accepted_at) - VALUES (?, ?, ?, ?, ?, 'queued', ?)`, - submissionId, - input.session, - createSessionStorageKey(input.id, 'default', input.session), - kind, - payload, - acceptedAt, - ); - const row = this.readSubmissionRow(submissionId); - if (!row) throw new Error(`[flue] Durable ${kind} admission did not create a submission row.`); - if (row.kind !== kind || row.payload !== payload) { - throw new SqlAgentSubmissionConflictError(`[flue] Conflicting internal ${kind} replay.`); - } - return parseSubmission(row); + const sessionKey = createSessionStorageKey(input.id, 'default', input.session); + return this.transactionSync(() => { + if (kind === 'dispatch') { + const receipt = this.getDispatchReceipt(submissionId); + if (receipt) throw new SqlAgentDispatchReceiptRetainedError(receipt); + } + const deleting = this.sql + .exec('SELECT 1 FROM flue_agent_session_deletions WHERE session_key = ? LIMIT 1', sessionKey) + .toArray(); + if (deleting.length > 0) { + throw new Error('[flue] Durable agent submission admission is unavailable while this session is being deleted. Retry after deletion completes.'); + } + this.sql.exec( + `INSERT OR IGNORE INTO flue_agent_submissions + (submission_id, session, session_key, kind, payload, status, accepted_at) + VALUES (?, ?, ?, ?, ?, 'queued', ?)`, + submissionId, + input.session, + sessionKey, + kind, + payload, + acceptedAt, + ); + const row = this.readSubmissionRow(submissionId); + if (!row) throw new Error(`[flue] Durable ${kind} admission did not create a submission row.`); + if (row.kind !== kind || row.payload !== payload) { + throw new SqlAgentSubmissionConflictError(`[flue] Conflicting internal ${kind} replay.`); + } + return parseSubmission(row); + }); } private getOwnedRunningSubmission( @@ -388,7 +526,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { private parseOperationalRows( rows: SqlRow[], - status: Extract, + status: 'queued' | 'active', ): SqlAgentSubmission[] { const submissions: SqlAgentSubmission[] = []; for (const row of rows) { @@ -402,19 +540,14 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { return submissions; } - private failSubmissionSequence( - sequence: number, - status: Extract, - error: unknown, - ): void { + private failSubmissionSequence(sequence: number, status: 'queued' | 'active', error: unknown): void { this.sql.exec( `UPDATE flue_agent_submissions SET status = 'error', completed_at = ?, error = ? - WHERE sequence = ? AND status = ?`, + WHERE sequence = ? AND ${status === 'queued' ? "status = 'queued'" : "status IN ('running', 'terminalizing')"}`, Date.now(), error instanceof Error ? error.message : String(error), sequence, - status, ); } @@ -451,6 +584,7 @@ function parseSubmission(row: SqlRow): SqlAgentSubmission { typeof row.payload !== 'string' || (row.status !== 'queued' && row.status !== 'running' && + row.status !== 'terminalizing' && row.status !== 'completed' && row.status !== 'error') || typeof row.accepted_at !== 'number' || @@ -467,7 +601,7 @@ function parseSubmission(row: SqlRow): SqlAgentSubmission { row.input_applied_at !== null || row.recovery_requested_at !== null || row.started_at !== null)) || - (row.status === 'running' && + ((row.status === 'running' || row.status === 'terminalizing') && (typeof row.attempt_id !== 'string' || typeof row.started_at !== 'number')) ) { throw new Error('[flue] Persisted agent submission row is malformed.'); @@ -577,6 +711,19 @@ function ensureSubmissionTable(sql: SqlStorage): void { ); ensureSubmissionColumn(sql, 'input_applied_at', 'INTEGER'); ensureSubmissionColumn(sql, 'recovery_requested_at', 'INTEGER'); + sql.exec( + `CREATE TABLE IF NOT EXISTS flue_agent_session_deletions ( + session_key TEXT PRIMARY KEY, + started_at INTEGER NOT NULL + )`, + ); + sql.exec( + `CREATE TABLE IF NOT EXISTS flue_agent_dispatch_receipts ( + dispatch_id TEXT PRIMARY KEY, + accepted_at INTEGER NOT NULL, + settled_at INTEGER NOT NULL + )`, + ); sql.exec( 'CREATE INDEX IF NOT EXISTS flue_agent_submissions_status_sequence_idx ON flue_agent_submissions (status, sequence ASC)', ); diff --git a/packages/runtime/src/harness.ts b/packages/runtime/src/harness.ts index cd5ad3d7..300f0bc9 100644 --- a/packages/runtime/src/harness.ts +++ b/packages/runtime/src/harness.ts @@ -1,15 +1,16 @@ import type { AgentToolResult } from '@earendil-works/pi-agent-core'; import { createCallHandle } from './abort.ts'; import { formatBashResult } from './agent.ts'; +import type { SessionDeletionCoordinator } from './client.ts'; import { discoverSessionContext } from './context.ts'; import { generateSessionAffinityKey } from './runtime/ids.ts'; import { createCwdSessionEnv, createFlueFs } from './sandbox.ts'; +import { type CreateTaskSessionOptions, deleteSessionTree, Session } from './session.ts'; import { assertPublicSessionName, createSessionStorageKey, createTaskSessionName, } from './session-identity.ts'; -import { type CreateTaskSessionOptions, deleteSessionTree, Session } from './session.ts'; import type { AgentConfig, AgentProfile, @@ -54,6 +55,7 @@ export class Harness implements FlueHarness { private eventCallback?: FlueEventCallback, private agentTools: ToolDefinition[] = [], private toolFactory?: SessionToolFactory, + private sessionDeletionCoordinator?: SessionDeletionCoordinator, ) { this.fs = createFlueFs(env); } @@ -170,6 +172,7 @@ export class Harness implements FlueHarness { taskDepth: 0, createTaskSession: (taskOptions) => this.createTaskSession(taskOptions), onDelete: () => this.openSessions.delete(sessionName), + sessionDeletionCoordinator: this.sessionDeletionCoordinator, }); this.openSessions.set(sessionName, session); return session; @@ -184,10 +187,10 @@ export class Harness implements FlueHarness { await open.delete(); return; } - await deleteSessionTree( - this.store, - createSessionStorageKey(this.instanceId, this.name, sessionName), - ); + const storageKey = createSessionStorageKey(this.instanceId, this.name, sessionName); + this.sessionDeletionCoordinator?.begin(storageKey); + await deleteSessionTree(this.store, storageKey); + this.sessionDeletionCoordinator?.finish(storageKey); }); } diff --git a/packages/runtime/src/internal.ts b/packages/runtime/src/internal.ts index 21e895ff..50e87eb1 100644 --- a/packages/runtime/src/internal.ts +++ b/packages/runtime/src/internal.ts @@ -29,6 +29,7 @@ export type { export { createSqlAgentExecutionStore, createSqlSessionStore, + SqlAgentDispatchReceiptRetainedError, SqlAgentSubmissionConflictError, } from './cloudflare/agent-execution-store.ts'; export { createDurableRunStore } from './cloudflare/run-store.ts'; @@ -90,6 +91,8 @@ export { createDirectSubmissionInputInspectionHandler, createDispatchAgentHandler, createDispatchInputInspectionHandler, + createLegacyDirectSubmissionTerminalHandler, + createSubmissionTerminalHandler, failRecoveredRun, handleAgentRequest, handleWorkflowRequest, diff --git a/packages/runtime/src/runtime/dispatch-queue.ts b/packages/runtime/src/runtime/dispatch-queue.ts index e000c03e..92106cc1 100644 --- a/packages/runtime/src/runtime/dispatch-queue.ts +++ b/packages/runtime/src/runtime/dispatch-queue.ts @@ -20,6 +20,13 @@ export interface DirectSubmissionInput { export type AgentSubmissionInput = DispatchInput | DirectSubmissionInput; +export interface AgentSubmissionTerminalInput { + submissionId: string; + kind: 'dispatch' | 'direct'; + reason: 'interrupted_before_input_marker' | 'interrupted_after_input_application'; + message: string; +} + export function assertCurrentDispatchInput(value: unknown): asserts value is DispatchInput { if (value && typeof value === 'object' && 'targetAgent' in value) { throw new Error( diff --git a/packages/runtime/src/runtime/handle-agent.ts b/packages/runtime/src/runtime/handle-agent.ts index 91bc8a02..4c74e75a 100644 --- a/packages/runtime/src/runtime/handle-agent.ts +++ b/packages/runtime/src/runtime/handle-agent.ts @@ -21,6 +21,7 @@ import type { } from '../types.ts'; import { type AgentSubmissionInputInspection, + type AgentSubmissionTerminalInput, type AttachedAgentSubmissionAdmission, assertCurrentDispatchInput, type DirectSubmissionInput, @@ -58,6 +59,14 @@ interface DirectSubmissionSession { ): PromiseLike; } +interface SubmissionTerminalSession { + recordSubmissionTerminal(input: AgentSubmissionTerminalInput): Promise; + recordLegacyDirectSubmissionTerminal( + input: DirectSubmissionInput, + terminal: AgentSubmissionTerminalInput, + ): Promise; +} + export interface AgentSessionTarget { agentName: string; instanceId: string; @@ -143,6 +152,36 @@ export function createDirectSubmissionInputInspectionHandler( return (ctx) => inspectAgentDirectSubmissionInput(ctx, agent, input); } +export function createSubmissionTerminalHandler( + agent: CreatedAgentHandler, + input: DispatchInput | DirectSubmissionInput, + terminal: AgentSubmissionTerminalInput, +): AgentHandler { + return async (ctx) => { + const harness = await ctx.initializeCreatedAgent(agent, undefined); + const session = await harness.session(input.session); + if (!isSubmissionTerminalSession(session)) { + throw new Error('[flue] Internal session does not support submission terminal persistence.'); + } + await session.recordSubmissionTerminal(terminal); + }; +} + +export function createLegacyDirectSubmissionTerminalHandler( + agent: CreatedAgentHandler, + input: DirectSubmissionInput, + terminal: AgentSubmissionTerminalInput, +): AgentHandler { + return async (ctx) => { + const harness = await ctx.initializeCreatedAgent(agent, undefined); + const session = await harness.session(input.session); + if (!isSubmissionTerminalSession(session)) { + throw new Error('[flue] Internal session does not support submission terminal persistence.'); + } + await session.recordLegacyDirectSubmissionTerminal(input, terminal); + }; +} + export async function reserveDispatchAgentSession( target: AgentSessionTarget, payload: unknown, @@ -258,6 +297,14 @@ function isDirectSubmissionSession(value: unknown): value is DirectSubmissionSes ); } +function isSubmissionTerminalSession(value: unknown): value is SubmissionTerminalSession { + return ( + !!value && + typeof value === 'object' && + typeof (value as SubmissionTerminalSession).recordSubmissionTerminal === 'function' + ); +} + export function createDirectAgentHandler(agent: CreatedAgentHandler): AgentHandler { return async (ctx) => { const payload = parseDirectAgentPayload(ctx.payload); diff --git a/packages/runtime/src/session.ts b/packages/runtime/src/session.ts index 38528677..9fabde12 100644 --- a/packages/runtime/src/session.ts +++ b/packages/runtime/src/session.ts @@ -27,6 +27,7 @@ import { type TaskToolParams, type TaskToolResultDetails, } from './agent.ts'; +import type { SessionDeletionCoordinator } from './client.ts'; import { type CompactionSettings, type CompactionTurnHandle, @@ -51,6 +52,7 @@ import { } from './result.ts'; import type { AgentSubmissionInputInspection, + AgentSubmissionTerminalInput, DirectSubmissionInput, DispatchInput, DispatchInputInspection, @@ -192,6 +194,7 @@ interface SessionInitOptions { taskDepth?: number; createTaskSession?: CreateTaskSession; onDelete?: () => void; + sessionDeletionCoordinator?: SessionDeletionCoordinator; } interface CallOverrides { @@ -373,7 +376,11 @@ export class SessionHistory { appendMessage( message: AgentMessage, source?: MessageSource, - metadata?: { dispatch?: DispatchMessageMetadata; directSubmissionId?: string }, + metadata?: { + dispatch?: DispatchMessageMetadata; + directSubmissionId?: string; + submissionTerminal?: MessageEntry['submissionTerminal']; + }, ): string { const entry: MessageEntry = { type: 'message', @@ -385,6 +392,7 @@ export class SessionHistory { }; if (metadata?.dispatch) entry.dispatch = metadata.dispatch; if (metadata?.directSubmissionId) entry.directSubmissionId = metadata.directSubmissionId; + if (metadata?.submissionTerminal) entry.submissionTerminal = metadata.submissionTerminal; this.appendEntry(entry); return entry.id; } @@ -403,6 +411,13 @@ export class SessionHistory { ); } + findSubmissionTerminal(submissionId: string): MessageEntry | undefined { + return this.getActivePath().find( + (entry): entry is MessageEntry => + entry.type === 'message' && entry.submissionTerminal?.submissionId === submissionId, + ); + } + appendMessages(messages: AgentMessage[], source?: MessageSource): string[] { return messages.map((message) => this.appendMessage(message, source)); } @@ -628,6 +643,7 @@ export class Session implements FlueSession { private taskDepth: number; private createTaskSession: CreateTaskSession | undefined; private onDelete: (() => void) | undefined; + private sessionDeletionCoordinator: SessionDeletionCoordinator | undefined; private pendingSave: Promise = Promise.resolve(); private emitTurnRequestAndStream: StreamFn = (model, context, options) => { @@ -684,6 +700,7 @@ export class Session implements FlueSession { this.taskDepth = options.taskDepth ?? 0; this.createTaskSession = options.createTaskSession; this.onDelete = options.onDelete; + this.sessionDeletionCoordinator = options.sessionDeletionCoordinator; this.metadata = options.existingData?.metadata ?? {}; this.createdAt = options.existingData?.createdAt; @@ -928,6 +945,40 @@ export class Session implements FlueSession { ); } + async recordLegacyDirectSubmissionTerminal( + input: DirectSubmissionInput, + terminal: AgentSubmissionTerminalInput, + ): Promise { + if (!this.history.findDirectSubmissionInput(input.submissionId)) { + this.history.appendMessage( + createUserContextMessage(input.payload.message, new Date().toISOString()), + 'prompt', + { directSubmissionId: input.submissionId }, + ); + } + await this.recordSubmissionTerminal(terminal); + } + + async recordSubmissionTerminal(input: AgentSubmissionTerminalInput): Promise { + if (this.history.findSubmissionTerminal(input.submissionId)) return; + this.history.appendMessage( + createUserContextMessage( + `[Flue Submission Interrupted]\n\n${input.message}`, + new Date().toISOString(), + ), + undefined, + { + submissionTerminal: { + submissionId: input.submissionId, + kind: input.kind, + reason: input.reason, + }, + }, + ); + this.harness.state.messages = this.history.buildContext(); + await this.save(); + } + skill( skill: SkillReference | string, options: SkillOptions & { result: S }, @@ -1100,9 +1151,16 @@ export class Session implements FlueSession { } this.deleted = true; this.deletionPromise = Promise.resolve() + .then(() => this.sessionDeletionCoordinator?.begin(this.storageKey)) .then(() => deleteSessionTree(this.store, this.storageKey)) .then(() => { + this.sessionDeletionCoordinator?.finish(this.storageKey); this.onDelete?.(); + }) + .catch((error) => { + this.deleted = false; + this.deletionPromise = undefined; + throw error; }); return this.deletionPromise; } diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts index 3eaabc38..910dcbf6 100644 --- a/packages/runtime/src/types.ts +++ b/packages/runtime/src/types.ts @@ -552,10 +552,11 @@ export interface FlueContext> { * body-reading method, calling another will throw. Use `req.clone()` if * you need to read it more than once. * - * Undefined when the agent is invoked outside an HTTP context (e.g. future - * cron / queue triggers). Today every trigger is HTTP, so in practice this - * is always defined — the optional type lets the contract hold when other - * trigger types ship. + * Undefined when the agent is invoked outside an HTTP context. Durable or + * recovered processing may receive a synthetic internal request instead of + * the original caller request. Authenticate and capture required transport + * metadata before durable admission; do not assume later processing retains + * original headers, cookies, query parameters, URL, or body. * * For client IP, parse the platform header yourself, e.g. * `req.headers.get('cf-connecting-ip')` on Cloudflare, or @@ -615,7 +616,8 @@ export interface FlueSessions { create(name?: string): Promise; /** * Delete a session's stored conversation state. Defaults to `'default'`. - * No-op when missing. Rejects if the open session has an active operation. + * No-op when missing. Rejects if the open session has an active operation or + * the target runtime still has accepted durable submissions for that session. * Session-management requests for one name are applied in request order. */ delete(name?: string): Promise; @@ -701,7 +703,8 @@ export interface FlueSession { /** * Delete this session's stored conversation state. Rejects while an - * operation is active. Once deletion starts, the session is unusable and + * operation or accepted durable submission is active. Once deletion starts, + * the session is unusable and * concurrent calls share the same deletion work. */ delete(): Promise; @@ -796,6 +799,13 @@ export interface MessageEntry extends SessionEntryBase { source?: 'prompt' | 'skill' | 'shell' | 'task' | 'retry' | 'dispatch'; dispatch?: DispatchMessageMetadata; directSubmissionId?: string; + submissionTerminal?: SubmissionTerminalMetadata; +} + +interface SubmissionTerminalMetadata { + submissionId: string; + kind: 'dispatch' | 'direct'; + reason: 'interrupted_before_input_marker' | 'interrupted_after_input_application'; } export interface DispatchMessageMetadata { diff --git a/packages/runtime/test/build-plugin-cloudflare.test.ts b/packages/runtime/test/build-plugin-cloudflare.test.ts index e093b330..8b4ec70a 100644 --- a/packages/runtime/test/build-plugin-cloudflare.test.ts +++ b/packages/runtime/test/build-plugin-cloudflare.test.ts @@ -55,39 +55,39 @@ describe('CloudflarePlugin', () => { expect(entry).toContain( `async onStart(props) { + await restoreFlueAgentSubmissionWake(this); if (typeof super.onStart === 'function') await super.onStart(props); - await armFlueAgentSubmissionRetry(this); - await reconcileFlueAgentSubmissions(this, "assistant", { preserveSuccessor: true }); + await reconcileFlueAgentSubmissions(this, "assistant", { driverAlreadyArmed: true }); } - async __flueWakeAgentSubmissions(wake) { - await reconcileFlueAgentSubmissions(this, "assistant", { preserveSuccessor: true, executingWake: wake }); - } - - async __flueRetryAgentSubmissions(_payload, schedule) { - if (!(await reconcileFlueAgentSubmissions(this, "assistant"))) { - await this.cancelSchedule(schedule.id); - } + async __flueWakeAgentSubmissions() { + const submissions = getAgentExecutionStore(this).submissions; + if (!submissions.hasUnsettledSubmissions()) return; + await armFlueAgentSubmissionWake(this, { idempotent: false }); + await reconcileFlueAgentSubmissions(this, "assistant", { driverAlreadyArmed: true }); }`, ); expect(entry).toContain("const FLUE_AGENT_SUBMISSION_WAKE_CALLBACK = '__flueWakeAgentSubmissions';"); - expect(entry).toContain("const FLUE_AGENT_SUBMISSION_RETRY_CALLBACK = '__flueRetryAgentSubmissions';"); - expect(entry).toContain("return doInstance.scheduleEvery(FLUE_AGENT_SUBMISSION_RETRY_SECONDS, FLUE_AGENT_SUBMISSION_RETRY_CALLBACK);"); - expect(entry).toContain("await armFlueAgentSubmissionAdmissionWakes(doInstance);\n let submission;"); - expect(entry).toContain('submission = getAgentExecutionStore(doInstance).submissions.admitDispatch(input);'); + expect(entry).not.toContain('scheduleEvery'); + expect(entry).toContain("await armFlueAgentSubmissionAdmissionWake(doInstance);\n let submission;"); + expect(entry).toContain('cleanupFlueAgentSubmissionTerminalState(doInstance);'); + expect(entry).toContain('submissions.cleanupDispatchReceipt(input.dispatchId, Date.now() - FLUE_AGENT_SUBMISSION_TERMINAL_RETENTION_MS);'); + expect(entry).toContain('const priorReceipt = submissions.getDispatchReceipt(input.dispatchId);'); + expect(entry).toContain('submission = submissions.admitDispatch(input);'); + expect(entry).toContain('if (error instanceof SqlAgentDispatchReceiptRetainedError) return Response.json({ dispatchId: error.receipt.submissionId, acceptedAt: new Date(error.receipt.acceptedAt).toISOString() });'); expect(entry).toContain('for (const submission of submissions.listRunningSubmissions()) {'); expect(entry).toContain('if (activeFlueAgentSubmissionAttempts.has(submissionAttemptLocalKey(doInstance, submission))) continue;'); - expect(entry).toContain('if (attemptMarkers.keys.has(submissionAttemptMarkerKey(submission)) && submission.recoveryRequestedAt === undefined) continue;'); + expect(entry).toContain("if (submission.status !== 'terminalizing' && attemptMarkers.keys.has(submissionAttemptMarkerKey(submission)) && submission.recoveryRequestedAt === undefined) continue;"); expect(entry).toContain('await reconcileInterruptedSqlAgentSubmission(submission, doInstance, agentName);'); - expect(entry).toContain('const submission = submissions.requestSubmissionRecovery(submissionId, attemptId);'); - expect(entry).toContain("SELECT snapshot, created_at FROM cf_agents_runs WHERE name = 'flue:dispatch-attempt'"); + expect(entry).toContain('await restoreFlueAgentSubmissionWake(doInstance);\n const submissions = getAgentExecutionStore(doInstance).submissions;\n submissions.requestSubmissionRecovery(submissionId, attemptId);'); + expect(entry).toContain("SELECT snapshot, created_at FROM cf_agents_runs WHERE name = 'flue:submission-attempt'"); expect(entry).toContain('if (Date.now() - row.created_at > FLUE_AGENT_SUBMISSION_ATTEMPT_STALE_MS) continue;'); expect(entry).toContain('submissions.requeueSubmissionBeforeInputApplied(submission.submissionId, attemptId);'); expect(entry).toContain('createDispatchInputInspectionHandler(agent, input)(ctx)'); expect(entry).toContain('createDirectSubmissionInputInspectionHandler(agent, input)(ctx)'); expect(entry).toContain('if (!submissions.markSubmissionInputApplied(submission.submissionId, attemptId)) {'); expect(entry).toContain('const claimed = submissions.claimSubmission(submission.submissionId, crypto.randomUUID());'); - expect(entry).toContain("running = doInstance.runFiber('flue:dispatch-attempt', async (fiberCtx) => {"); + expect(entry).toContain("running = doInstance.runFiber('flue:submission-attempt', async (fiberCtx) => {"); expect(entry).toContain("fiberCtx.stash({ submissionId: submission.submissionId, attemptId: submission.attemptId });"); expect(entry).toContain("activeFlueAgentSubmissionAttempts.delete(attemptKey);\n throw error;"); expect(entry).toContain('const completed = submissions.completeSubmission(submission.submissionId, attemptId);'); @@ -96,11 +96,25 @@ describe('CloudflarePlugin', () => { expect(entry).toContain('getAgentExecutionStore(doInstance).submissions.admitDirect(input);'); expect(entry).toContain('admitAttachedSubmission: (payload, req, onEvent) => admitAttachedAgentSubmission(doInstance, agentName, payload, req, onEvent)'); expect(entry).toContain('getAgentExecutionStore(doInstance).submissions.adoptLegacyDispatches(dispatches.map((dispatch) => dispatch.input));'); + expect(entry).toContain("idempotent: options.idempotent ?? true"); + expect(entry).toContain("idempotent: false"); + expect(entry).not.toContain('generation:'); + expect(entry).not.toContain('beginFlueAgentSubmissionAdmission'); + expect(entry).not.toContain('cancelSchedule(schedule.id)'); + expect(entry).toContain('getAgentExecutionStore(doInstance).submissions.cleanupTerminalSubmissions(\n Date.now() - FLUE_AGENT_SUBMISSION_TERMINAL_RETENTION_MS,'); + expect(entry).toContain('begin: (sessionKey) => executionStore.submissions.beginSessionDeletion(sessionKey)'); + expect(entry).toContain('if (submission.status !== \'terminalizing\' && !submissions.beginSubmissionTerminalization(submission.submissionId, attemptId)) return;'); + expect(entry).toContain('createSubmissionTerminalHandler(agent, input, {'); + expect(entry).toContain('submissions.finalizeSubmissionTerminalization(submission.submissionId, attemptId, error)'); + expect(entry).toContain("if (failed && submission.kind === 'direct') agentSubmissionObservers.fail(submission.submissionId, error);"); + expect(entry).not.toContain('const { manifest, directHandlers, localAgentHandlers, createdAgents'); expect(entry).not.toContain('assertNoPendingDispatchForDirectSession'); expect(entry).not.toContain("runFiber('flue:direct'"); expect(entry).not.toContain('listActiveDirectAgentSessionMarkers'); expect(entry).not.toContain('agentSubmissionObservers.takeRequest'); expect(entry).toContain("return handleFlueDispatchAttemptRecovered(ctx, this);"); + expect(entry).toContain("submissionId: 'legacy-direct:' + ctx.id"); + expect(entry).toContain('createLegacyDirectSubmissionTerminalHandler(agent, input, {'); expect(entry).not.toContain("startFiber('flue:dispatch'"); expect(entry).not.toContain('inspectFiberByKey'); expect(entry).not.toContain('ctx.storage.setAlarm'); diff --git a/packages/runtime/test/cloudflare-agent-execution-store.test.ts b/packages/runtime/test/cloudflare-agent-execution-store.test.ts index 70663574..5a5ddcba 100644 --- a/packages/runtime/test/cloudflare-agent-execution-store.test.ts +++ b/packages/runtime/test/cloudflare-agent-execution-store.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from 'vitest'; import { createSqlAgentExecutionStore, createSqlSessionStore, + SqlAgentDispatchReceiptRetainedError, } from '../src/cloudflare/agent-execution-store.ts'; import type { DirectSubmissionInput, DispatchInput } from '../src/runtime/dispatch-queue.ts'; import type { SessionData } from '../src/types.ts'; @@ -122,6 +123,15 @@ describe('createSqlAgentExecutionStore()', () => { { name: 'completed_at' }, { name: 'error' }, ]); + expect( + db.prepare("SELECT name FROM sqlite_schema WHERE type = 'table' ORDER BY name").all(), + ).toEqual([ + { name: 'flue_agent_dispatch_receipts' }, + { name: 'flue_agent_session_deletions' }, + { name: 'flue_agent_submissions' }, + { name: 'flue_sessions' }, + { name: 'sqlite_sequence' }, + ]); expect( db .prepare( @@ -195,10 +205,9 @@ describe('createSqlAgentExecutionStore()', () => { const { sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); const direct = store.submissions.admitDirect(directInput()); - const dispatch = store.submissions.admitDispatch(dispatchInput()); + store.submissions.admitDispatch(dispatchInput()); const other = store.submissions.admitDirect(directInput({ submissionId: 'direct-2', session: 'other' })); - expect(store.submissions.listQueuedSubmissions()).toEqual([direct, dispatch, other]); expect(store.submissions.listRunnableSubmissions()).toEqual([direct, other]); expect(store.submissions.claimSubmission('dispatch-1', 'attempt-blocked')).toBeNull(); expect(store.submissions.claimSubmission('direct-1', 'attempt-direct')).toMatchObject({ @@ -211,12 +220,11 @@ describe('createSqlAgentExecutionStore()', () => { const { sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); const first = store.submissions.admitDispatch(dispatchInput()); - const second = store.submissions.admitDispatch(dispatchInput({ dispatchId: 'dispatch-2' })); + store.submissions.admitDispatch(dispatchInput({ dispatchId: 'dispatch-2' })); const other = store.submissions.admitDispatch( dispatchInput({ dispatchId: 'dispatch-3', session: 'other' }), ); - expect(store.submissions.listQueuedSubmissions()).toEqual([first, second, other]); expect(store.submissions.listRunnableSubmissions()).toEqual([first, other]); }); @@ -284,7 +292,7 @@ describe('createSqlAgentExecutionStore()', () => { }); it('adopts legacy dispatches ahead of existing SQL submissions in historical order', () => { - const { sql, transactionSync } = makeFakeSql(); + const { db, sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); store.submissions.admitDispatch(dispatchInput({ dispatchId: 'current' })); @@ -294,15 +302,17 @@ describe('createSqlAgentExecutionStore()', () => { ]); expect(adopted.map((submission) => submission.submissionId)).toEqual(['legacy-1', 'legacy-2']); - expect(store.submissions.listQueuedSubmissions().map((submission) => submission.submissionId)).toEqual([ - 'legacy-1', - 'legacy-2', - 'current', + expect( + db.prepare('SELECT submission_id FROM flue_agent_submissions ORDER BY sequence ASC').all(), + ).toEqual([ + { submission_id: 'legacy-1' }, + { submission_id: 'legacy-2' }, + { submission_id: 'current' }, ]); }); it('adopts more than sixteen legacy dispatches without exceeding the Cloudflare SQL binding limit', () => { - const { sql, transactionSync } = makeFakeSql(); + const { db, sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); store.submissions.admitDispatch(dispatchInput({ dispatchId: 'current' })); const legacy = Array.from({ length: 17 }, (_, index) => @@ -311,10 +321,9 @@ describe('createSqlAgentExecutionStore()', () => { store.submissions.adoptLegacyDispatches(legacy); - expect(store.submissions.listQueuedSubmissions().map((submission) => submission.submissionId)).toEqual([ - ...legacy.map((input) => input.dispatchId), - 'current', - ]); + expect( + db.prepare('SELECT submission_id FROM flue_agent_submissions ORDER BY sequence ASC').all(), + ).toEqual([...legacy.map((input) => ({ submission_id: input.dispatchId })), { submission_id: 'current' }]); }); it('records input application and recovery requests only for the owning running attempt', () => { @@ -364,7 +373,7 @@ describe('createSqlAgentExecutionStore()', () => { store.submissions.admitDispatch(dispatchInput({ session: 'case-1' })); expect(store.submissions.hasUnsettledSubmissions()).toBe(true); - expect(store.submissions.listQueuedSubmissions()).toHaveLength(1); + expect(store.submissions.listRunnableSubmissions()).toHaveLength(1); store.submissions.claimSubmission('dispatch-1', 'attempt-1'); expect(store.submissions.listRunningSubmissions()).toHaveLength(1); store.submissions.completeSubmission('dispatch-1', 'attempt-1'); @@ -390,6 +399,132 @@ describe('createSqlAgentExecutionStore()', () => { }); }); + it('fences ordinary completion after interrupted terminalization begins', () => { + const { sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); + store.submissions.admitDispatch(dispatchInput()); + store.submissions.claimSubmission('dispatch-1', 'attempt-1'); + + expect(store.submissions.beginSubmissionTerminalization('dispatch-1', 'attempt-1')).toMatchObject({ + status: 'terminalizing', + }); + expect(store.submissions.completeSubmission('dispatch-1', 'attempt-1')).toBe(false); + expect( + store.submissions.finalizeSubmissionTerminalization('dispatch-1', 'attempt-1', new Error('interrupted')), + ).toBe(true); + expect(store.submissions.getSubmission('dispatch-1')).toMatchObject({ + status: 'error', + error: 'interrupted', + }); + }); + + it('rejects session deletion while durable submissions are queued or running', () => { + const { sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); + store.submissions.admitDispatch(dispatchInput()); + + expect(() => + store.submissions.beginSessionDeletion('agent-session:["agent-1","default","default"]'), + ).toThrow('Session cannot be deleted while durable agent submissions are queued or running.'); + + store.submissions.claimSubmission('dispatch-1', 'attempt-1'); + expect(() => + store.submissions.beginSessionDeletion('agent-session:["agent-1","default","default"]'), + ).toThrow('Session cannot be deleted while durable agent submissions are queued or running.'); + }); + + it('blocks new submissions until session deletion completes', () => { + const { sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); + const sessionKey = 'agent-session:["agent-1","default","default"]'; + + store.submissions.beginSessionDeletion(sessionKey); + expect(() => store.submissions.admitDispatch(dispatchInput())).toThrow( + 'Durable agent submission admission is unavailable while this session is being deleted.', + ); + store.submissions.finishSessionDeletion(sessionKey); + expect(store.submissions.admitDispatch(dispatchInput())).toMatchObject({ status: 'queued' }); + }); + + it('clears terminal rows when a settled session is deleted', () => { + const { sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); + const sessionKey = 'agent-session:["agent-1","default","default"]'; + store.submissions.admitDispatch(dispatchInput()); + store.submissions.claimSubmission('dispatch-1', 'attempt-1'); + store.submissions.completeSubmission('dispatch-1', 'attempt-1'); + + store.submissions.beginSessionDeletion(sessionKey); + store.submissions.finishSessionDeletion(sessionKey); + + expect(store.submissions.getSubmission('dispatch-1')).toBeNull(); + expect(store.submissions.getDispatchReceipt('dispatch-1')).toEqual({ + submissionId: 'dispatch-1', + acceptedAt: Date.parse('2026-06-03T00:00:00.000Z'), + }); + }); + + it('rejects replay admission transactionally when deletion retained the dispatch receipt', () => { + const { sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); + const sessionKey = 'agent-session:["agent-1","default","default"]'; + store.submissions.admitDispatch(dispatchInput()); + store.submissions.claimSubmission('dispatch-1', 'attempt-1'); + store.submissions.completeSubmission('dispatch-1', 'attempt-1'); + store.submissions.beginSessionDeletion(sessionKey); + store.submissions.finishSessionDeletion(sessionKey); + + try { + store.submissions.admitDispatch(dispatchInput()); + throw new Error('Expected dispatch replay to be retained.'); + } catch (error) { + expect(error).toBeInstanceOf(SqlAgentDispatchReceiptRetainedError); + expect((error as SqlAgentDispatchReceiptRetainedError).receipt).toEqual({ + submissionId: 'dispatch-1', + acceptedAt: Date.parse('2026-06-03T00:00:00.000Z'), + }); + } + expect(store.submissions.getSubmission('dispatch-1')).toBeNull(); + }); + + it('expires payload-free dispatch receipts lazily after the replay horizon', () => { + const { db, sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); + const sessionKey = 'agent-session:["agent-1","default","default"]'; + store.submissions.admitDispatch(dispatchInput()); + store.submissions.claimSubmission('dispatch-1', 'attempt-1'); + store.submissions.completeSubmission('dispatch-1', 'attempt-1'); + store.submissions.beginSessionDeletion(sessionKey); + store.submissions.finishSessionDeletion(sessionKey); + db.prepare('UPDATE flue_agent_dispatch_receipts SET settled_at = 1 WHERE dispatch_id = ?').run( + 'dispatch-1', + ); + + store.submissions.cleanupDispatchReceipt('dispatch-1', 2); + + expect(store.submissions.getDispatchReceipt('dispatch-1')).toBeNull(); + }); + + it('sweeps only expired terminal submissions in bounded batches', () => { + const { db, sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); + store.submissions.admitDispatch(dispatchInput({ dispatchId: 'expired-1' })); + store.submissions.admitDispatch(dispatchInput({ dispatchId: 'expired-2', session: 'other' })); + store.submissions.admitDispatch(dispatchInput({ dispatchId: 'active', session: 'active' })); + store.submissions.claimSubmission('expired-1', 'attempt-1'); + store.submissions.claimSubmission('expired-2', 'attempt-2'); + store.submissions.completeSubmission('expired-1', 'attempt-1'); + store.submissions.failSubmission('expired-2', 'attempt-2', new Error('failed')); + db.prepare("UPDATE flue_agent_submissions SET completed_at = 1 WHERE status IN ('completed', 'error')").run(); + + expect(store.submissions.cleanupTerminalSubmissions(2, 1)).toBe(1); + expect(store.submissions.cleanupTerminalSubmissions(2, 1)).toBe(1); + expect(store.submissions.cleanupTerminalSubmissions(2, 1)).toBe(0); + expect(store.submissions.getDispatchReceipt('expired-1')).toBeNull(); + expect(store.submissions.getDispatchReceipt('expired-2')).toBeNull(); + expect(store.submissions.getSubmission('active')).toMatchObject({ status: 'queued' }); + }); + it('rejects missing Durable Object SQLite with migration guidance', () => { expect(() => createSqlAgentExecutionStore({}, 'FlueAssistantAgent')).toThrow( 'Add "FlueAssistantAgent" to a Wrangler migration\'s "new_sqlite_classes" list before its first deploy; do not use legacy "new_classes". Existing KV-backed Durable Object classes cannot be converted to SQLite in place.', diff --git a/packages/runtime/test/cloudflare-agent-extension.integration.test.ts b/packages/runtime/test/cloudflare-agent-extension.integration.test.ts index 5a914a99..dae666fb 100644 --- a/packages/runtime/test/cloudflare-agent-extension.integration.test.ts +++ b/packages/runtime/test/cloudflare-agent-extension.integration.test.ts @@ -18,7 +18,7 @@ describe('Cloudflare agent extension', () => { await expect( build({ root, sourceRoot: path.join(root, 'src'), target: 'cloudflare', mode: 'development' }), ).rejects.toThrow( - '[flue] Cloudflare target requires the installed "agents" package to be at least 0.14.1. Found 0.14.0. Install or upgrade "agents" in this project.', + '[flue] Cloudflare target requires the installed "agents" package to satisfy >=0.14.1 <0.15.0. Found 0.14.0. Install a compatible "agents" version in this project.', ); } finally { fs.rmSync(root, { recursive: true, force: true }); diff --git a/packages/runtime/test/cloudflare-websocket.test.ts b/packages/runtime/test/cloudflare-websocket.test.ts index 9d123476..545062c5 100644 --- a/packages/runtime/test/cloudflare-websocket.test.ts +++ b/packages/runtime/test/cloudflare-websocket.test.ts @@ -143,6 +143,31 @@ describe('Cloudflare agent WebSockets', () => { ]); }); + it('allows durable attached admission to complete when the Cloudflare agent socket disconnects', async () => { + const connection = new ThrowingConnection(); + const calls: string[] = []; + + await messageCloudflareAgentWebSocket( + connection, + JSON.stringify({ version: 1, type: 'prompt', requestId: 'prompt-disconnected', message: 'Hello' }), + { + name: 'assistant', + id: 'agent-instance-1', + request: new Request('https://example.com/flue/agents/assistant/agent-instance-1'), + handler: async () => null, + createContext, + admitAttachedSubmission: async (_payload, _request, onEvent) => { + calls.push('admitted'); + await onEvent?.({ type: 'idle', instanceId: 'agent-instance-1' }); + calls.push('completed'); + return 'done'; + }, + }, + ); + + expect(calls).toEqual(['admitted', 'completed']); + }); + it('rejects oversized messages when a Cloudflare agent socket exceeds the byte limit', async () => { const connection = new TestConnection(); let invocations = 0; @@ -527,6 +552,20 @@ describe('Cloudflare workflow WebSockets', () => { }); }); +class ThrowingConnection implements CloudflareWebSocketConnection { + serializeAttachment(): void {} + + deserializeAttachment(): CloudflareWebSocketAttachment | null { + return null; + } + + send(): void { + throw new Error('Socket disconnected'); + } + + close(): void {} +} + class TestConnection implements CloudflareWebSocketConnection { attachment: CloudflareWebSocketAttachment | null = null; messages: WebSocketServerMessage[] = []; diff --git a/packages/runtime/test/dispatch.test.ts b/packages/runtime/test/dispatch.test.ts index 1e6974c9..6ac3cdff 100644 --- a/packages/runtime/test/dispatch.test.ts +++ b/packages/runtime/test/dispatch.test.ts @@ -9,11 +9,14 @@ import { dispatch, observe } from '../src/index.ts'; import { configureFlueRuntime, createAgentDispatchProcessor, + createAgentSubmissionObserverRegistry, createDirectSubmissionAgentHandler, createDirectSubmissionInputInspectionHandler, createDispatchAgentHandler, createDispatchInputInspectionHandler, createFlueContext, + createLegacyDirectSubmissionTerminalHandler, + createSubmissionTerminalHandler, type DirectSubmissionInput, type DispatchInput, InMemoryDispatchQueue, @@ -31,6 +34,24 @@ afterEach(() => { for (const provider of providers.splice(0)) provider.unregister(); }); +describe('createAgentSubmissionObserverRegistry()', () => { + it('settles attached observers when event callbacks fail', async () => { + const registry = createAgentSubmissionObserverRegistry(); + const attachment = registry.attach('direct:observer-failure', { + onEvent: async () => { + throw new Error('Socket disconnected'); + }, + }); + + await expect( + registry.publish('direct:observer-failure', { type: 'idle', instanceId: 'agent-1' }), + ).resolves.toBeUndefined(); + registry.complete('direct:observer-failure', 'done'); + + await expect(attachment.completion).resolves.toBe('done'); + }); +}); + describe('dispatch()', () => { it('rejects calls when the runtime has not been configured', async () => { await expect( @@ -631,6 +652,112 @@ describe('dispatched session processing', () => { expect(data?.entries[0]).not.toHaveProperty('dispatch'); }); + it('persists one provider-visible terminal advisory when an interrupted submission cannot replay safely', async () => { + const provider = createProvider(); + const store = new InMemorySessionStore(); + const input: DirectSubmissionInput = { + submissionId: 'direct:terminal-advisory', + agent: 'moderator', + id: 'guild:terminal-advisory', + session: 'case:terminal-advisory', + payload: { message: 'Hello directly' }, + acceptedAt: '2026-06-01T00:00:00.000Z', + }; + const agent = createAgent(() => ({ + model: `${provider.getModel().provider}/${provider.getModel().id}`, + persist: store, + })); + const createContext = () => + createFlueContext({ + id: input.id, + payload: input.payload, + env: {}, + req: new Request('http://flue.local/agents/moderator/guild:terminal-advisory', { method: 'POST' }), + agentConfig: testAgentConfig(), + createDefaultEnv: async () => createNoopSessionEnv({ cwd: '/' }), + defaultStore: new InMemorySessionStore(), + }); + const terminal = { + submissionId: input.submissionId, + kind: 'direct' as const, + reason: 'interrupted_after_input_application' as const, + message: 'Provider replay was not attempted because prior execution could not be proven safe.', + }; + + await createSubmissionTerminalHandler(agent, input, terminal)(createContext()); + await createSubmissionTerminalHandler(agent, input, terminal)(createContext()); + + const data = await store.load(`agent-session:${JSON.stringify([input.id, 'default', input.session])}`); + expect(data?.entries).toHaveLength(1); + expect(data?.entries[0]).toMatchObject({ + submissionTerminal: { + submissionId: input.submissionId, + kind: 'direct', + reason: 'interrupted_after_input_application', + }, + message: { + role: 'user', + content: [ + { + type: 'text', + text: '[Flue Submission Interrupted]\n\nProvider replay was not attempted because prior execution could not be proven safe.', + }, + ], + }, + }); + }); + + it('persists captured input and one terminal advisory for an interrupted legacy direct prompt', async () => { + const provider = createProvider(); + const store = new InMemorySessionStore(); + const input: DirectSubmissionInput = { + submissionId: 'legacy-direct:fiber-1', + agent: 'moderator', + id: 'guild:legacy-terminal', + session: 'case:legacy-terminal', + payload: { message: 'Captured legacy prompt' }, + acceptedAt: '2026-06-01T00:00:00.000Z', + }; + const agent = createAgent(() => ({ + model: `${provider.getModel().provider}/${provider.getModel().id}`, + persist: store, + })); + const createContext = () => + createFlueContext({ + id: input.id, + payload: input.payload, + env: {}, + req: new Request('http://flue.local/agents/moderator/guild:legacy-terminal', { method: 'POST' }), + agentConfig: testAgentConfig(), + createDefaultEnv: async () => createNoopSessionEnv({ cwd: '/' }), + defaultStore: new InMemorySessionStore(), + }); + const terminal = { + submissionId: input.submissionId, + kind: 'direct' as const, + reason: 'interrupted_after_input_application' as const, + message: 'A pre-upgrade direct prompt was interrupted. Provider replay was not attempted.', + }; + + await createLegacyDirectSubmissionTerminalHandler(agent, input, terminal)(createContext()); + await createLegacyDirectSubmissionTerminalHandler(agent, input, terminal)(createContext()); + + const data = await store.load(`agent-session:${JSON.stringify([input.id, 'default', input.session])}`); + expect(data?.entries).toHaveLength(2); + expect(data?.entries[0]).toMatchObject({ + directSubmissionId: input.submissionId, + message: { role: 'user', content: [{ type: 'text', text: 'Captured legacy prompt' }] }, + }); + expect(data?.entries[1]).toMatchObject({ + submissionTerminal: { + submissionId: input.submissionId, + kind: 'direct', + reason: 'interrupted_after_input_application', + }, + }); + expect(provider.state.callCount).toBe(0); + }); + it('classifies a completed canonical direct response without model replay', async () => { const provider = createProvider(); const store = new InMemorySessionStore(); diff --git a/packages/sdk/README.md b/packages/sdk/README.md index 24cef48a..b9bce5a8 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -67,7 +67,7 @@ export async function run({ init, payload }: FlueContext) { A support agent can also run on Cloudflare without a container by using Flue's default virtual sandbox. Populate its filesystem with the context the agent needs, then it can search that content with its built-in `grep`, `glob`, and `read` tools. -Because this agent is deployed to Cloudflare, message history and session state are automatically persisted for you. So you (or your customer) can revisit this support session days, weeks, or years later and pick up exactly where you left off. +Cloudflare Durable Objects persist stored conversation history across later requests. The default virtual sandbox remains in memory, so seed any files each time you need them or choose a durable sandbox connector for long-lived workspace state. ```ts // .flue/workflows/support.ts @@ -281,7 +281,7 @@ curl http://localhost:3583/agents/hello/session-xyz \ -d '{"message": "Hello from another conversation"}' ``` -Agent instances own sandbox state such as files written across prompt and dispatched-input interactions. Harnesses group related session state within an instance. Sessions persist message history and conversation metadata inside a harness. On Cloudflare, session data is backed by Durable Objects and survives across requests. On Node.js, sessions are stored in memory by default unless you provide a custom store. +Agent instances provide a stable identity boundary for prompt and dispatched-input interactions. Harnesses group related session state within an instance. Sessions persist message history and conversation metadata inside a harness. Sandbox-file durability depends on the configured sandbox or connector; the default virtual filesystem is in memory. On Cloudflare, session conversation data is backed by Durable Objects and survives across requests. On Node.js, sessions are stored in memory by default unless you provide a custom store. In production, generate a stable URL `` for the agent instance you want to preserve. Use `harness.session(threadName)` when you need multiple conversations inside the same harness. @@ -418,7 +418,7 @@ await job.ready; await job.invoke({ text: 'Summarize me' }); ``` -An exported `websocket` middleware can authenticate its own agent or workflow socket endpoint. Use a custom `app.ts` for centralized authentication or mounted prefixes, applying ordinary Hono middleware before `app.route('/api', flue())`; the same routing model works on Node and Cloudflare. Include the custom mount in `baseUrl: 'https://example.com/api'` and use `websocketUrl: (url) => { url.searchParams.set('token', socketToken); return url; }` for URL-carried or signed handshake authentication. HTTP `token` and `headers` options do not automatically apply to WebSocket upgrades; browsers should use cookies or application-designed URL authentication, while Node clients needing implementation-specific headers can provide a custom `websocket` factory. Avoid header-mutating middleware around WebSocket upgrade routes. `flue dev --target cloudflare` requires `wrangler` as a peer dependency in your project (`npm install --save-dev wrangler`). +An exported `websocket` middleware can authenticate its own agent or workflow socket endpoint. Use a custom `app.ts` for centralized authentication or mounted prefixes, applying ordinary Hono middleware before `app.route('/api', flue())`; the same routing model works on Node and Cloudflare. Include the custom mount in `baseUrl: 'https://example.com/api'` and use `websocketUrl: (url) => { url.searchParams.set('token', socketToken); return url; }` for URL-carried or signed handshake authentication. HTTP `token` and `headers` options do not automatically apply to WebSocket upgrades; browsers should use cookies or application-designed URL authentication, while Node clients needing implementation-specific headers can provide a custom `websocket` factory. Avoid header-mutating middleware around WebSocket upgrade routes. Install `wrangler` in Cloudflare projects that use Wrangler commands such as `wrangler deploy` or secret management; `flue dev --target cloudflare` no longer requires it as an `@flue/cli` peer dependency. #### Loading environment variables diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 0b7a0366..8c1496f7 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -12,7 +12,6 @@ allowBuilds: minimumReleaseAge: 1440 resolutionMode: highest minimumReleaseAgeExclude: - - agents - '@aws-sdk/*' - '@smithy/*' - protobufjs From 3a36fe63db5692ca245eb27ef74593285d7988b2 Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" <622227+FredKSchott@users.noreply.github.com> Date: Thu, 4 Jun 2026 10:25:43 -0500 Subject: [PATCH 09/17] refactor(cloudflare): remove legacy submission bridges --- CHANGELOG.md | 2 +- .../docs/ecosystem/deploy/cloudflare.md | 2 +- .../cli/src/lib/build-plugin-cloudflare.ts | 118 ++---------------- .../src/cloudflare/agent-execution-store.ts | 74 +---------- packages/runtime/src/internal.ts | 10 +- .../runtime/src/runtime/dispatch-queue.ts | 8 -- packages/runtime/src/runtime/handle-agent.ts | 46 ++----- packages/runtime/src/session.ts | 14 --- .../test/build-plugin-cloudflare.test.ts | 17 ++- .../cloudflare-agent-execution-store.test.ts | 95 ++------------ packages/runtime/test/dispatch.test.ts | 52 -------- 11 files changed, 47 insertions(+), 391 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5a9d945..e444e1f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ - **Agents: Rename ordinary sessions beginning with `task:`.** Session names with that prefix are now reserved for framework-owned delegated-task history. Flue retains detached child history until parent deletion or application-owned retention cleanup, and no stored records are deleted automatically during upgrade. - **Agents: Clear or migrate persisted beta session state before upgrading.** Session records now persist one opaque `aff_` provider-affinity key instead of deriving affinity from agent instance, harness, and session names. This keeps prompt-cache and routing-affinity identifiers bounded and distinct for nested tasks. Existing version-4 beta session records are rejected; storage keys are unchanged. - **Cloudflare: Rename generated Durable Object identities.** Generated bindings now use `FLUE__AGENT`, `FLUE__WORKFLOW`, and `FLUE_REGISTRY`; generated classes use `FlueAgent`, `FlueWorkflow`, and `FlueRegistry`. Existing deployments must append authored Wrangler `renamed_classes` migrations for deployed agent and workflow classes and update authored direct binding access such as `env.Assistant` to `env.FLUE_ASSISTANT_AGENT`. -- **Cloudflare: Use SQLite-backed generated agent classes.** Generated agent Durable Objects now require SQLite for durable submission admission and ordering. Introduce classes through Wrangler `new_sqlite_classes`, not legacy `new_classes`. Already deployed KV-backed classes cannot be converted to SQLite in place; use a fresh class identity or an application-owned migration plan. +- **Cloudflare: Use fresh SQLite-backed generated agent classes.** Generated agent Durable Objects now require SQLite for durable submission admission and ordering. Introduce classes through Wrangler `new_sqlite_classes`, not legacy `new_classes`. Already deployed KV-backed classes cannot be converted to SQLite in place. This pre-1.0 upgrade does not adopt in-flight direct prompts or dispatched inputs from earlier generated-agent identities; use a fresh class identity and treat the deployment as a hard execution-state boundary. - **Cloudflare: Upgrade Cloudflare Agents SDK within the audited range.** Cloudflare builds require `agents >=0.14.1 <0.15.0`. - **Cloudflare: Capture request metadata before durable direct admission.** Route middleware sees the original inbound request, but SQL-backed direct processing uses a deterministic internal `Request`. Do not rely on later operation-time `ctx.req` to preserve original headers, cookies, query parameters, URL, or body. diff --git a/apps/docs/src/content/docs/ecosystem/deploy/cloudflare.md b/apps/docs/src/content/docs/ecosystem/deploy/cloudflare.md index 4c60bba6..f59d7a82 100644 --- a/apps/docs/src/content/docs/ecosystem/deploy/cloudflare.md +++ b/apps/docs/src/content/docs/ecosystem/deploy/cloudflare.md @@ -82,7 +82,7 @@ Cloudflare requires an explicit migration whenever a Worker adds a Durable Objec Every Cloudflare target includes `FlueRegistry`. Flue-owned bindings use upper snake case and generated classes use PascalCase: `.flue/workflows/translate.ts` binds `FLUE_TRANSLATE_WORKFLOW` to `FlueTranslateWorkflow`, while `.flue/agents/support-chat.ts` binds `FLUE_SUPPORT_CHAT_AGENT` to `FlueSupportChatAgent`. -Keep deployed migration entries in order. When you add an agent or workflow later, append a uniquely tagged migration for its new class. Generated Flue agent classes require Durable Object SQLite: introduce them through `new_sqlite_classes`, not legacy `new_classes`. An already deployed KV-backed Durable Object class cannot be converted to SQLite in place; use a fresh class identity or plan an application-owned state migration. Use Cloudflare's explicit rename or delete migrations when changing a deployed class lifecycle. When upgrading a deployment created before this naming scheme, append `renamed_classes` entries such as `{ "from": "SupportChat", "to": "FlueSupportChatAgent" }` and `{ "from": "TranslateWorkflow", "to": "FlueTranslateWorkflow" }`; do not rewrite deployed migration history. +Keep deployed migration entries in order. When you add an agent or workflow later, append a uniquely tagged migration for its new class. Generated Flue agent classes require Durable Object SQLite: introduce them through `new_sqlite_classes`, not legacy `new_classes`. An already deployed KV-backed Durable Object class cannot be converted to SQLite in place. For this pre-1.0 agent durability upgrade, use a fresh generated agent class identity and treat deployment as a hard execution-state boundary: in-flight direct prompts and dispatched inputs from earlier generated-agent identities are not adopted. Use Cloudflare's explicit rename or delete migrations when changing a deployed class lifecycle. When upgrading a workflow deployment created before this naming scheme, append `renamed_classes` entries such as `{ "from": "TranslateWorkflow", "to": "FlueTranslateWorkflow" }`; do not rewrite deployed migration history. ### 4. Build and deploy diff --git a/packages/cli/src/lib/build-plugin-cloudflare.ts b/packages/cli/src/lib/build-plugin-cloudflare.ts index 457ac35e..e05c7be5 100644 --- a/packages/cli/src/lib/build-plugin-cloudflare.ts +++ b/packages/cli/src/lib/build-plugin-cloudflare.ts @@ -101,14 +101,8 @@ const ${agentClassName(agent.name)} = class ${agentClassName(agent.name)} extend } async onFiberRecovered(ctx) { - if (ctx.name === 'flue:dispatch') { - return handleFlueDispatchRecovered(ctx, this, ${JSON.stringify(agent.name)}); - } if (ctx.name === 'flue:submission-attempt') { - return handleFlueDispatchAttemptRecovered(ctx, this); - } - if (ctx.name === 'flue:direct') { - return handleFlueDirectRecovered(ctx, this, ${JSON.stringify(agent.name)}); + return handleFlueAgentSubmissionAttemptRecovered(ctx, this); } if (typeof super.onFiberRecovered === 'function') { return super.onFiberRecovered(ctx); @@ -220,10 +214,8 @@ import { handleWorkflowRequest, handleRunRouteRequest, validateAgentDispatchAdmission, - assertCurrentDispatchInput, createDispatchAgentHandler, createDispatchInputInspectionHandler, - createLegacyDirectSubmissionTerminalHandler, createDirectSubmissionAgentHandler, createDirectSubmissionInputInspectionHandler, createSubmissionTerminalHandler, @@ -485,26 +477,7 @@ function assertAgentsDurabilityApi(doInstance, method) { } } -async function handleFlueDispatchRecovered(ctx, doInstance, agentName) { - const input = ctx.metadata?.input; - assertCurrentDispatchInput(input); - if (!input || input.agent !== agentName || input.id !== doInstance.name) return { status: 'error', error: 'Dispatch recovery metadata is invalid.' }; - try { - await validateAgentDispatchAdmission({ input }); - await armFlueAgentSubmissionAdmissionWake(doInstance); - const submissions = getAgentExecutionStore(doInstance).submissions; - const legacy = listActiveLegacyManagedDispatches(doInstance, agentName); - legacy.push({ fiberId: ctx.id, createdAt: ctx.createdAt, input }); - legacy.sort(compareLegacyManagedDispatches); - submissions.adoptLegacyDispatches(legacy.map((dispatch) => dispatch.input)); - await reconcileFlueAgentSubmissions(doInstance, agentName, { driverAlreadyArmed: true }); - return { status: 'completed' }; - } catch (error) { - return { status: 'error', error: error instanceof Error ? error.message : String(error) }; - } -} - -async function handleFlueDispatchAttemptRecovered(ctx, doInstance) { +async function handleFlueAgentSubmissionAttemptRecovered(ctx, doInstance) { const submissionId = ctx.snapshot?.submissionId; const attemptId = ctx.snapshot?.attemptId; if (typeof submissionId !== 'string' || typeof attemptId !== 'string') return; @@ -513,35 +486,6 @@ async function handleFlueDispatchAttemptRecovered(ctx, doInstance) { submissions.requestSubmissionRecovery(submissionId, attemptId); } -async function handleFlueDirectRecovered(ctx, doInstance, agentName) { - const payload = ctx.snapshot?.payload; - const message = 'A pre-upgrade direct prompt was interrupted. Provider replay was not attempted.'; - if (!payload || typeof payload !== 'object' || Array.isArray(payload) || typeof payload.message !== 'string' || (payload.session !== undefined && (typeof payload.session !== 'string' || payload.session.trim() === '' || payload.session.startsWith('task:')))) { - console.error('[flue:direct-recovery]', { agentName, instanceId: doInstance.name, operation: 'legacy_handoff', outcome: 'terminal_uncertainty' }, new Error(message)); - return; - } - const agent = createdAgents[agentName]; - if (!agent) throw new Error('[flue] Agent target unavailable during legacy direct terminalization.'); - const input = { - submissionId: 'legacy-direct:' + ctx.id, - agent: agentName, - id: doInstance.name, - session: typeof payload.session === 'string' && payload.session.trim() !== '' ? payload.session : 'default', - payload, - acceptedAt: new Date(ctx.createdAt).toISOString(), - }; - const request = new Request('https://flue.invalid/agents/' + encodeURIComponent(agentName) + '/' + encodeURIComponent(doInstance.name), { method: 'POST' }); - const directCtx = createAgentContextForRequest(doInstance.name, payload, doInstance, request); - await runWithInstanceContext(doInstance, agentRuntimeIdentity(agentName), () => - createLegacyDirectSubmissionTerminalHandler(agent, input, { - submissionId: input.submissionId, - kind: 'direct', - reason: 'interrupted_after_input_application', - message, - })(directCtx), - ); -} - async function handleFlueWorkflowFiberRecovered(ctx, doInstance, workflowName) { if (!ctx.name || ctx.name !== 'flue:workflow:' + doInstance.name) return; const interruptedRunId = doInstance.name; @@ -559,7 +503,7 @@ async function handleFlueWorkflowFiberRecovered(ctx, doInstance, workflowName) { }); } -// ─── Per-DO Dispatch ─────────────────────────────────────────────────────── +// ─── Per-DO Agent Submissions ───────────────────────────────────────────── function armFlueAgentSubmissionWake(doInstance, options = {}) { assertAgentsDurabilityApi(doInstance, 'schedule'); @@ -593,7 +537,6 @@ async function reconcileFlueAgentSubmissions(doInstance, agentName, options = {} cleanupFlueAgentSubmissionTerminalState(doInstance); if (!submissions.hasUnsettledSubmissions()) return false; if (!options.driverAlreadyArmed) await restoreFlueAgentSubmissionWake(doInstance); - const legacySessions = await adoptLegacyManagedDispatches(doInstance, agentName); if (!submissions.hasUnsettledSubmissions()) return false; try { const attemptMarkers = listActiveSqlAgentSubmissionAttemptMarkers(doInstance); @@ -601,16 +544,14 @@ async function reconcileFlueAgentSubmissions(doInstance, agentName, options = {} for (const submission of submissions.listRunningSubmissions()) { if (activeFlueAgentSubmissionAttempts.has(submissionAttemptLocalKey(doInstance, submission))) continue; if (submission.status !== 'terminalizing' && attemptMarkers.keys.has(submissionAttemptMarkerKey(submission)) && submission.recoveryRequestedAt === undefined) continue; - if (legacySessions.has(submission.session)) continue; await reconcileInterruptedSqlAgentSubmission(submission, doInstance, agentName); } for (const submission of submissions.listRunnableSubmissions()) { - if (legacySessions.has(submission.session)) continue; const claimed = submissions.claimSubmission(submission.submissionId, crypto.randomUUID()); if (claimed) startSqlAgentSubmissionAttempt(claimed, doInstance, agentName); } } catch (error) { - console.error('[flue:dispatch-reconciliation]', { agentName, instanceId: doInstance.name, operation: 'reconcile', outcome: 'deferred_to_scheduled_wake' }, error); + console.error('[flue:submission-reconciliation]', { agentName, instanceId: doInstance.name, operation: 'reconcile', outcome: 'deferred_to_scheduled_wake' }, error); return true; } return submissions.hasUnsettledSubmissions(); @@ -732,9 +673,14 @@ function listActiveSqlAgentSubmissionAttemptMarkers(doInstance) { continue; } if (Date.now() - row.created_at > FLUE_AGENT_SUBMISSION_ATTEMPT_STALE_MS) continue; + if (row.snapshot === null) continue; + if (typeof row.snapshot !== 'string') { + blockAll = true; + continue; + } let snapshot; try { - snapshot = typeof row.snapshot === 'string' ? JSON.parse(row.snapshot) : null; + snapshot = JSON.parse(row.snapshot); } catch { blockAll = true; continue; @@ -748,44 +694,6 @@ function listActiveSqlAgentSubmissionAttemptMarkers(doInstance) { return { blockAll, keys }; } -function compareLegacyManagedDispatches(a, b) { - return a.createdAt - b.createdAt || a.fiberId.localeCompare(b.fiberId); -} - -function listActiveLegacyManagedDispatches(doInstance, agentName) { - const rows = doInstance.ctx.storage.sql.exec( - 'SELECT fiber_id, metadata_json, created_at FROM cf_agents_fibers ' + - "WHERE name = 'flue:dispatch' AND (status IN ('pending', 'running') OR " + - "(status = 'interrupted' AND EXISTS (SELECT 1 FROM cf_agents_runs WHERE id = fiber_id))) " + - 'ORDER BY created_at ASC, fiber_id ASC', - ).toArray(); - return rows.map((row) => { - if (typeof row.fiber_id !== 'string' || typeof row.created_at !== 'number') { - throw new Error('[flue] Persisted dispatch Fiber row is malformed.'); - } - let metadata; - try { - metadata = typeof row.metadata_json === 'string' ? JSON.parse(row.metadata_json) : null; - } catch { - throw new Error('[flue] Persisted dispatch Fiber metadata is malformed.'); - } - const input = metadata?.input; - assertCurrentDispatchInput(input); - if (!input || input.agent !== agentName || input.id !== doInstance.name) { - throw new Error('[flue] Persisted dispatch Fiber metadata is invalid.'); - } - return { fiberId: row.fiber_id, createdAt: row.created_at, input }; - }); -} - -async function adoptLegacyManagedDispatches(doInstance, agentName) { - const dispatches = listActiveLegacyManagedDispatches(doInstance, agentName); - if (dispatches.length === 0) return new Set(); - for (const { input } of dispatches) await validateAgentDispatchAdmission({ input }); - getAgentExecutionStore(doInstance).submissions.adoptLegacyDispatches(dispatches.map((dispatch) => dispatch.input)); - return new Set(dispatches.map((dispatch) => dispatch.input.session)); -} - async function processSqlAgentSubmission(submission, doInstance, agentName) { const { attemptId, input } = submission; if (!attemptId) return; @@ -901,15 +809,11 @@ async function dispatchAgent(request, doInstance, agentName, handler) { const id = doInstance.name; if (isInternalDispatchRequest(request)) { const input = await request.json(); - assertCurrentDispatchInput(input); + await validateAgentDispatchAdmission({ input }); if (input.agent !== agentName || input.id !== id) return new Response('Invalid internal dispatch target.', { status: 400 }); if (!createdAgents[agentName]) return new Response('Dispatch target unavailable.', { status: 404 }); - await validateAgentDispatchAdmission({ input }); const submissions = getAgentExecutionStore(doInstance).submissions; cleanupFlueAgentSubmissionTerminalState(doInstance); - submissions.cleanupDispatchReceipt(input.dispatchId, Date.now() - FLUE_AGENT_SUBMISSION_TERMINAL_RETENTION_MS); - const priorReceipt = submissions.getDispatchReceipt(input.dispatchId); - if (priorReceipt) return Response.json({ dispatchId: priorReceipt.submissionId, acceptedAt: new Date(priorReceipt.acceptedAt).toISOString() }); await armFlueAgentSubmissionAdmissionWake(doInstance); let submission; try { diff --git a/packages/runtime/src/cloudflare/agent-execution-store.ts b/packages/runtime/src/cloudflare/agent-execution-store.ts index b5f29076..2c094b50 100644 --- a/packages/runtime/src/cloudflare/agent-execution-store.ts +++ b/packages/runtime/src/cloudflare/agent-execution-store.ts @@ -48,17 +48,14 @@ export interface SqlAgentSubmission { export interface SqlAgentSubmissionStore { getSubmission(submissionId: string): SqlAgentSubmission | null; - getDispatchReceipt(submissionId: string): SqlAgentDispatchReceipt | null; admitDispatch(input: DispatchInput): SqlAgentSubmission; admitDirect(input: DirectSubmissionInput): SqlAgentSubmission; - adoptLegacyDispatches(inputs: readonly DispatchInput[]): SqlAgentSubmission[]; hasUnsettledSubmissions(): boolean; listRunnableSubmissions(): SqlAgentSubmission[]; listRunningSubmissions(): SqlAgentSubmission[]; beginSessionDeletion(sessionKey: string): void; finishSessionDeletion(sessionKey: string): void; cleanupTerminalSubmissions(completedBefore: number, limit?: number): number; - cleanupDispatchReceipt(submissionId: string, settledBefore: number): void; claimSubmission(submissionId: string, attemptId: string): SqlAgentSubmission | null; markSubmissionInputApplied(submissionId: string, attemptId: string): SqlAgentSubmission | null; requestSubmissionRecovery(submissionId: string, attemptId: string): SqlAgentSubmission | null; @@ -158,7 +155,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { return row ? parseSubmission(row) : null; } - getDispatchReceipt(submissionId: string): SqlAgentDispatchReceipt | null { + private getDispatchReceipt(submissionId: string): SqlAgentDispatchReceipt | null { const row = this.sql .exec( 'SELECT dispatch_id, accepted_at FROM flue_agent_dispatch_receipts WHERE dispatch_id = ? LIMIT 1', @@ -180,58 +177,6 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { return this.admitSubmission('direct', input.submissionId, input); } - adoptLegacyDispatches(inputs: readonly DispatchInput[]): SqlAgentSubmission[] { - const unique = new Map(); - for (const input of inputs) { - const payload = JSON.stringify(input); - const row = this.readSubmissionRow(input.dispatchId); - if (row && (row.kind !== 'dispatch' || row.payload !== payload)) { - throw new SqlAgentSubmissionConflictError('[flue] Conflicting legacy dispatch adoption.'); - } - const prior = unique.get(input.dispatchId); - if (prior && JSON.stringify(prior) !== payload) { - throw new SqlAgentSubmissionConflictError('[flue] Conflicting legacy dispatch adoption.'); - } - if (!prior) unique.set(input.dispatchId, input); - } - const missing = [...unique.values()].filter((input) => !this.readSubmissionRow(input.dispatchId)); - const adopt = () => { - const first = this.sql - .exec('SELECT MIN(sequence) AS sequence FROM flue_agent_submissions') - .toArray()[0]?.sequence; - let sequence = typeof first === 'number' ? first - missing.length : -missing.length; - for (let offset = 0; offset < missing.length; offset += 16) { - const batch = missing.slice(offset, offset + 16); - const values: unknown[] = []; - for (const input of batch) { - const acceptedAt = parseAcceptedAt(input.acceptedAt, 'Legacy dispatch adoption'); - values.push( - sequence++, - input.dispatchId, - input.session, - createSessionStorageKey(input.id, 'default', input.session), - JSON.stringify(input), - acceptedAt, - ); - } - this.sql.exec( - `INSERT INTO flue_agent_submissions - (sequence, submission_id, session, session_key, kind, payload, status, accepted_at) - VALUES ${batch.map(() => "(?, ?, ?, ?, 'dispatch', ?, 'queued', ?)").join(', ')}`, - ...values, - ); - } - }; - if (missing.length > 0) this.transactionSync(adopt); - return inputs.map((input) => { - const submission = this.getSubmission(input.dispatchId); - if (!submission || submission.kind !== 'dispatch') { - throw new Error('[flue] Legacy dispatch adoption did not create a submission row.'); - } - return submission; - }); - } - hasUnsettledSubmissions(): boolean { return ( this.sql @@ -350,14 +295,6 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { return rows.length; } - cleanupDispatchReceipt(submissionId: string, settledBefore: number): void { - this.sql.exec( - 'DELETE FROM flue_agent_dispatch_receipts WHERE dispatch_id = ? AND settled_at < ?', - submissionId, - settledBefore, - ); - } - claimSubmission(submissionId: string, attemptId: string): SqlAgentSubmission | null { this.sql.exec( `UPDATE flue_agent_submissions AS current @@ -709,8 +646,6 @@ function ensureSubmissionTable(sql: SqlStorage): void { error TEXT )`, ); - ensureSubmissionColumn(sql, 'input_applied_at', 'INTEGER'); - ensureSubmissionColumn(sql, 'recovery_requested_at', 'INTEGER'); sql.exec( `CREATE TABLE IF NOT EXISTS flue_agent_session_deletions ( session_key TEXT PRIMARY KEY, @@ -731,10 +666,3 @@ function ensureSubmissionTable(sql: SqlStorage): void { 'CREATE INDEX IF NOT EXISTS flue_agent_submissions_session_status_sequence_idx ON flue_agent_submissions (session_key, status, sequence ASC)', ); } - -function ensureSubmissionColumn(sql: SqlStorage, name: string, type: string): void { - const rows = sql.exec(`SELECT name FROM pragma_table_info('flue_agent_submissions')`).toArray(); - if (!rows.some((row) => row.name === name)) { - sql.exec(`ALTER TABLE flue_agent_submissions ADD COLUMN ${name} ${type}`); - } -} diff --git a/packages/runtime/src/internal.ts b/packages/runtime/src/internal.ts index 50e87eb1..bf6323de 100644 --- a/packages/runtime/src/internal.ts +++ b/packages/runtime/src/internal.ts @@ -21,11 +21,6 @@ export { createFlueContext } from './client.ts'; // (registry client) live in the `@flue/runtime/cloudflare` subpath because // they pull in `cloudflare:workers`, a virtual module Node can't resolve. // The generated CF entry imports them from there directly. -export type { - SqlAgentExecutionStore, - SqlAgentSubmission, - SqlAgentSubmissionStore, -} from './cloudflare/agent-execution-store.ts'; export { createSqlAgentExecutionStore, createSqlSessionStore, @@ -41,7 +36,7 @@ export type { DispatchProcessor, DispatchQueue, } from './runtime/dispatch-queue.ts'; -export { assertCurrentDispatchInput, createAgentSubmissionObserverRegistry, InMemoryDispatchQueue } from './runtime/dispatch-queue.ts'; +export { createAgentSubmissionObserverRegistry, InMemoryDispatchQueue } from './runtime/dispatch-queue.ts'; export type { ExposedTransport, FlueRuntime } from './runtime/flue-app.ts'; export { configureFlueRuntime, @@ -53,7 +48,6 @@ export { } from './runtime/flue-app.ts'; export type { AgentHandler, - AgentSessionTarget, CreateContextFn, CreatedAgentHandler, DirectAttachedOptions, @@ -91,14 +85,12 @@ export { createDirectSubmissionInputInspectionHandler, createDispatchAgentHandler, createDispatchInputInspectionHandler, - createLegacyDirectSubmissionTerminalHandler, createSubmissionTerminalHandler, failRecoveredRun, handleAgentRequest, handleWorkflowRequest, invokeDirectAttached, invokeWorkflowAttached, - reserveDispatchAgentSession, validateAgentDispatchAdmission, } from './runtime/handle-agent.ts'; export type { HandleRunRouteOptions } from './runtime/handle-run-routes.ts'; diff --git a/packages/runtime/src/runtime/dispatch-queue.ts b/packages/runtime/src/runtime/dispatch-queue.ts index 92106cc1..918f0d6f 100644 --- a/packages/runtime/src/runtime/dispatch-queue.ts +++ b/packages/runtime/src/runtime/dispatch-queue.ts @@ -27,14 +27,6 @@ export interface AgentSubmissionTerminalInput { message: string; } -export function assertCurrentDispatchInput(value: unknown): asserts value is DispatchInput { - if (value && typeof value === 'object' && 'targetAgent' in value) { - throw new Error( - '[flue] Legacy dispatch metadata is unsupported. Clear persisted dispatch state created by an earlier Flue beta.', - ); - } -} - export type AgentSubmissionInputInspection = 'absent' | 'applied' | 'completed' | 'advanced'; export type DispatchInputInspection = AgentSubmissionInputInspection; diff --git a/packages/runtime/src/runtime/handle-agent.ts b/packages/runtime/src/runtime/handle-agent.ts index 4c74e75a..c6915e72 100644 --- a/packages/runtime/src/runtime/handle-agent.ts +++ b/packages/runtime/src/runtime/handle-agent.ts @@ -19,17 +19,16 @@ import type { FlueEvent, FlueEventCallback, } from '../types.ts'; -import { - type AgentSubmissionInputInspection, - type AgentSubmissionTerminalInput, - type AttachedAgentSubmissionAdmission, - assertCurrentDispatchInput, - type DirectSubmissionInput, - type DispatchInput, - type DispatchInputInspection, - type DispatchProcessor, - type ProcessAgentSubmissionInputOptions, - type ProcessDispatchInputOptions, +import type { + AgentSubmissionInputInspection, + AgentSubmissionTerminalInput, + AttachedAgentSubmissionAdmission, + DirectSubmissionInput, + DispatchInput, + DispatchInputInspection, + DispatchProcessor, + ProcessAgentSubmissionInputOptions, + ProcessDispatchInputOptions, } from './dispatch-queue.ts'; import { streamActiveRunEvents } from './handle-run-routes.ts'; import { generateWorkflowRunId } from './ids.ts'; @@ -61,13 +60,9 @@ interface DirectSubmissionSession { interface SubmissionTerminalSession { recordSubmissionTerminal(input: AgentSubmissionTerminalInput): Promise; - recordLegacyDirectSubmissionTerminal( - input: DirectSubmissionInput, - terminal: AgentSubmissionTerminalInput, - ): Promise; } -export interface AgentSessionTarget { +interface AgentSessionTarget { agentName: string; instanceId: string; } @@ -78,7 +73,6 @@ export function createAgentDispatchProcessor(options: { }): DispatchProcessor { return { async process(input) { - assertCurrentDispatchInput(input); const agent = options.agents[input.agent]; if (!agent) throw new Error(`[flue] dispatch target agent "${input.agent}" has no created agent.`); @@ -111,7 +105,6 @@ export async function validateAgentDispatchAdmission( options: ValidateAgentDispatchAdmissionOptions, ): Promise { const { input } = options; - assertCurrentDispatchInput(input); if (!isDispatchInput(input)) throw new Error('[flue] Internal dispatch admission received an invalid payload.'); if (isTaskSessionName(input.session)) { @@ -167,22 +160,7 @@ export function createSubmissionTerminalHandler( }; } -export function createLegacyDirectSubmissionTerminalHandler( - agent: CreatedAgentHandler, - input: DirectSubmissionInput, - terminal: AgentSubmissionTerminalInput, -): AgentHandler { - return async (ctx) => { - const harness = await ctx.initializeCreatedAgent(agent, undefined); - const session = await harness.session(input.session); - if (!isSubmissionTerminalSession(session)) { - throw new Error('[flue] Internal session does not support submission terminal persistence.'); - } - await session.recordLegacyDirectSubmissionTerminal(input, terminal); - }; -} - -export async function reserveDispatchAgentSession( +async function reserveDispatchAgentSession( target: AgentSessionTarget, payload: unknown, ): Promise<() => void> { diff --git a/packages/runtime/src/session.ts b/packages/runtime/src/session.ts index 9fabde12..843b0a01 100644 --- a/packages/runtime/src/session.ts +++ b/packages/runtime/src/session.ts @@ -945,20 +945,6 @@ export class Session implements FlueSession { ); } - async recordLegacyDirectSubmissionTerminal( - input: DirectSubmissionInput, - terminal: AgentSubmissionTerminalInput, - ): Promise { - if (!this.history.findDirectSubmissionInput(input.submissionId)) { - this.history.appendMessage( - createUserContextMessage(input.payload.message, new Date().toISOString()), - 'prompt', - { directSubmissionId: input.submissionId }, - ); - } - await this.recordSubmissionTerminal(terminal); - } - async recordSubmissionTerminal(input: AgentSubmissionTerminalInput): Promise { if (this.history.findSubmissionTerminal(input.submissionId)) return; this.history.appendMessage( diff --git a/packages/runtime/test/build-plugin-cloudflare.test.ts b/packages/runtime/test/build-plugin-cloudflare.test.ts index 8b4ec70a..8074a0fc 100644 --- a/packages/runtime/test/build-plugin-cloudflare.test.ts +++ b/packages/runtime/test/build-plugin-cloudflare.test.ts @@ -71,8 +71,8 @@ describe('CloudflarePlugin', () => { expect(entry).not.toContain('scheduleEvery'); expect(entry).toContain("await armFlueAgentSubmissionAdmissionWake(doInstance);\n let submission;"); expect(entry).toContain('cleanupFlueAgentSubmissionTerminalState(doInstance);'); - expect(entry).toContain('submissions.cleanupDispatchReceipt(input.dispatchId, Date.now() - FLUE_AGENT_SUBMISSION_TERMINAL_RETENTION_MS);'); - expect(entry).toContain('const priorReceipt = submissions.getDispatchReceipt(input.dispatchId);'); + expect(entry).not.toContain('cleanupDispatchReceipt'); + expect(entry).not.toContain('getDispatchReceipt'); expect(entry).toContain('submission = submissions.admitDispatch(input);'); expect(entry).toContain('if (error instanceof SqlAgentDispatchReceiptRetainedError) return Response.json({ dispatchId: error.receipt.submissionId, acceptedAt: new Date(error.receipt.acceptedAt).toISOString() });'); expect(entry).toContain('for (const submission of submissions.listRunningSubmissions()) {'); @@ -80,8 +80,11 @@ describe('CloudflarePlugin', () => { expect(entry).toContain("if (submission.status !== 'terminalizing' && attemptMarkers.keys.has(submissionAttemptMarkerKey(submission)) && submission.recoveryRequestedAt === undefined) continue;"); expect(entry).toContain('await reconcileInterruptedSqlAgentSubmission(submission, doInstance, agentName);'); expect(entry).toContain('await restoreFlueAgentSubmissionWake(doInstance);\n const submissions = getAgentExecutionStore(doInstance).submissions;\n submissions.requestSubmissionRecovery(submissionId, attemptId);'); + expect(entry).toContain('return handleFlueAgentSubmissionAttemptRecovered(ctx, this);'); expect(entry).toContain("SELECT snapshot, created_at FROM cf_agents_runs WHERE name = 'flue:submission-attempt'"); expect(entry).toContain('if (Date.now() - row.created_at > FLUE_AGENT_SUBMISSION_ATTEMPT_STALE_MS) continue;'); + expect(entry).toContain('if (row.snapshot === null) continue;'); + expect(entry).toContain("if (typeof row.snapshot !== 'string') {"); expect(entry).toContain('submissions.requeueSubmissionBeforeInputApplied(submission.submissionId, attemptId);'); expect(entry).toContain('createDispatchInputInspectionHandler(agent, input)(ctx)'); expect(entry).toContain('createDirectSubmissionInputInspectionHandler(agent, input)(ctx)'); @@ -95,7 +98,8 @@ describe('CloudflarePlugin', () => { expect(entry).toContain("if (submission.kind === 'direct') ctx?.setEventCallback(undefined);"); expect(entry).toContain('getAgentExecutionStore(doInstance).submissions.admitDirect(input);'); expect(entry).toContain('admitAttachedSubmission: (payload, req, onEvent) => admitAttachedAgentSubmission(doInstance, agentName, payload, req, onEvent)'); - expect(entry).toContain('getAgentExecutionStore(doInstance).submissions.adoptLegacyDispatches(dispatches.map((dispatch) => dispatch.input));'); + expect(entry).not.toContain('adoptLegacyDispatches'); + expect(entry).not.toContain('cf_agents_fibers'); expect(entry).toContain("idempotent: options.idempotent ?? true"); expect(entry).toContain("idempotent: false"); expect(entry).not.toContain('generation:'); @@ -112,9 +116,10 @@ describe('CloudflarePlugin', () => { expect(entry).not.toContain("runFiber('flue:direct'"); expect(entry).not.toContain('listActiveDirectAgentSessionMarkers'); expect(entry).not.toContain('agentSubmissionObservers.takeRequest'); - expect(entry).toContain("return handleFlueDispatchAttemptRecovered(ctx, this);"); - expect(entry).toContain("submissionId: 'legacy-direct:' + ctx.id"); - expect(entry).toContain('createLegacyDirectSubmissionTerminalHandler(agent, input, {'); + expect(entry).not.toContain('handleFlueDispatchAttemptRecovered'); + expect(entry).not.toContain("ctx.name === 'flue:dispatch'"); + expect(entry).not.toContain("ctx.name === 'flue:direct'"); + expect(entry).not.toContain('createLegacyDirectSubmissionTerminalHandler'); expect(entry).not.toContain("startFiber('flue:dispatch'"); expect(entry).not.toContain('inspectFiberByKey'); expect(entry).not.toContain('ctx.storage.setAlarm'); diff --git a/packages/runtime/test/cloudflare-agent-execution-store.test.ts b/packages/runtime/test/cloudflare-agent-execution-store.test.ts index 5a5ddcba..e292685a 100644 --- a/packages/runtime/test/cloudflare-agent-execution-store.test.ts +++ b/packages/runtime/test/cloudflare-agent-execution-store.test.ts @@ -145,33 +145,6 @@ describe('createSqlAgentExecutionStore()', () => { ]); }); - it('adds recovery columns when an existing submission table predates the current schema', () => { - const { db, sql, transactionSync } = makeFakeSql(); - db.exec(`CREATE TABLE flue_agent_submissions ( - sequence INTEGER PRIMARY KEY AUTOINCREMENT, - submission_id TEXT NOT NULL UNIQUE, - session TEXT NOT NULL, - session_key TEXT NOT NULL, - kind TEXT NOT NULL, - payload TEXT NOT NULL, - status TEXT NOT NULL, - accepted_at INTEGER NOT NULL, - attempt_id TEXT, - started_at INTEGER, - completed_at INTEGER, - error TEXT - )`); - - createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); - - expect( - db.prepare("SELECT name FROM pragma_table_info('flue_agent_submissions') ORDER BY cid").all(), - ).toContainEqual({ name: 'input_applied_at' }); - expect( - db.prepare("SELECT name FROM pragma_table_info('flue_agent_submissions') ORDER BY cid").all(), - ).toContainEqual({ name: 'recovery_requested_at' }); - }); - it('admits one queued dispatch row when the same submission is replayed', () => { const { db, sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); @@ -291,41 +264,6 @@ describe('createSqlAgentExecutionStore()', () => { }); }); - it('adopts legacy dispatches ahead of existing SQL submissions in historical order', () => { - const { db, sql, transactionSync } = makeFakeSql(); - const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); - store.submissions.admitDispatch(dispatchInput({ dispatchId: 'current' })); - - const adopted = store.submissions.adoptLegacyDispatches([ - dispatchInput({ dispatchId: 'legacy-1' }), - dispatchInput({ dispatchId: 'legacy-2' }), - ]); - - expect(adopted.map((submission) => submission.submissionId)).toEqual(['legacy-1', 'legacy-2']); - expect( - db.prepare('SELECT submission_id FROM flue_agent_submissions ORDER BY sequence ASC').all(), - ).toEqual([ - { submission_id: 'legacy-1' }, - { submission_id: 'legacy-2' }, - { submission_id: 'current' }, - ]); - }); - - it('adopts more than sixteen legacy dispatches without exceeding the Cloudflare SQL binding limit', () => { - const { db, sql, transactionSync } = makeFakeSql(); - const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); - store.submissions.admitDispatch(dispatchInput({ dispatchId: 'current' })); - const legacy = Array.from({ length: 17 }, (_, index) => - dispatchInput({ dispatchId: `legacy-${index + 1}` }), - ); - - store.submissions.adoptLegacyDispatches(legacy); - - expect( - db.prepare('SELECT submission_id FROM flue_agent_submissions ORDER BY sequence ASC').all(), - ).toEqual([...legacy.map((input) => ({ submission_id: input.dispatchId })), { submission_id: 'current' }]); - }); - it('records input application and recovery requests only for the owning running attempt', () => { const { sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); @@ -447,7 +385,7 @@ describe('createSqlAgentExecutionStore()', () => { }); it('clears terminal rows when a settled session is deleted', () => { - const { sql, transactionSync } = makeFakeSql(); + const { db, sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); const sessionKey = 'agent-session:["agent-1","default","default"]'; store.submissions.admitDispatch(dispatchInput()); @@ -458,9 +396,13 @@ describe('createSqlAgentExecutionStore()', () => { store.submissions.finishSessionDeletion(sessionKey); expect(store.submissions.getSubmission('dispatch-1')).toBeNull(); - expect(store.submissions.getDispatchReceipt('dispatch-1')).toEqual({ - submissionId: 'dispatch-1', - acceptedAt: Date.parse('2026-06-03T00:00:00.000Z'), + expect( + db.prepare('SELECT dispatch_id, accepted_at FROM flue_agent_dispatch_receipts WHERE dispatch_id = ?').get( + 'dispatch-1', + ), + ).toEqual({ + dispatch_id: 'dispatch-1', + accepted_at: Date.parse('2026-06-03T00:00:00.000Z'), }); }); @@ -487,24 +429,6 @@ describe('createSqlAgentExecutionStore()', () => { expect(store.submissions.getSubmission('dispatch-1')).toBeNull(); }); - it('expires payload-free dispatch receipts lazily after the replay horizon', () => { - const { db, sql, transactionSync } = makeFakeSql(); - const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); - const sessionKey = 'agent-session:["agent-1","default","default"]'; - store.submissions.admitDispatch(dispatchInput()); - store.submissions.claimSubmission('dispatch-1', 'attempt-1'); - store.submissions.completeSubmission('dispatch-1', 'attempt-1'); - store.submissions.beginSessionDeletion(sessionKey); - store.submissions.finishSessionDeletion(sessionKey); - db.prepare('UPDATE flue_agent_dispatch_receipts SET settled_at = 1 WHERE dispatch_id = ?').run( - 'dispatch-1', - ); - - store.submissions.cleanupDispatchReceipt('dispatch-1', 2); - - expect(store.submissions.getDispatchReceipt('dispatch-1')).toBeNull(); - }); - it('sweeps only expired terminal submissions in bounded batches', () => { const { db, sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); @@ -520,8 +444,7 @@ describe('createSqlAgentExecutionStore()', () => { expect(store.submissions.cleanupTerminalSubmissions(2, 1)).toBe(1); expect(store.submissions.cleanupTerminalSubmissions(2, 1)).toBe(1); expect(store.submissions.cleanupTerminalSubmissions(2, 1)).toBe(0); - expect(store.submissions.getDispatchReceipt('expired-1')).toBeNull(); - expect(store.submissions.getDispatchReceipt('expired-2')).toBeNull(); + expect(db.prepare('SELECT dispatch_id FROM flue_agent_dispatch_receipts').all()).toEqual([]); expect(store.submissions.getSubmission('active')).toMatchObject({ status: 'queued' }); }); diff --git a/packages/runtime/test/dispatch.test.ts b/packages/runtime/test/dispatch.test.ts index 6ac3cdff..15bd0844 100644 --- a/packages/runtime/test/dispatch.test.ts +++ b/packages/runtime/test/dispatch.test.ts @@ -15,7 +15,6 @@ import { createDispatchAgentHandler, createDispatchInputInspectionHandler, createFlueContext, - createLegacyDirectSubmissionTerminalHandler, createSubmissionTerminalHandler, type DirectSubmissionInput, type DispatchInput, @@ -707,57 +706,6 @@ describe('dispatched session processing', () => { }); }); - it('persists captured input and one terminal advisory for an interrupted legacy direct prompt', async () => { - const provider = createProvider(); - const store = new InMemorySessionStore(); - const input: DirectSubmissionInput = { - submissionId: 'legacy-direct:fiber-1', - agent: 'moderator', - id: 'guild:legacy-terminal', - session: 'case:legacy-terminal', - payload: { message: 'Captured legacy prompt' }, - acceptedAt: '2026-06-01T00:00:00.000Z', - }; - const agent = createAgent(() => ({ - model: `${provider.getModel().provider}/${provider.getModel().id}`, - persist: store, - })); - const createContext = () => - createFlueContext({ - id: input.id, - payload: input.payload, - env: {}, - req: new Request('http://flue.local/agents/moderator/guild:legacy-terminal', { method: 'POST' }), - agentConfig: testAgentConfig(), - createDefaultEnv: async () => createNoopSessionEnv({ cwd: '/' }), - defaultStore: new InMemorySessionStore(), - }); - const terminal = { - submissionId: input.submissionId, - kind: 'direct' as const, - reason: 'interrupted_after_input_application' as const, - message: 'A pre-upgrade direct prompt was interrupted. Provider replay was not attempted.', - }; - - await createLegacyDirectSubmissionTerminalHandler(agent, input, terminal)(createContext()); - await createLegacyDirectSubmissionTerminalHandler(agent, input, terminal)(createContext()); - - const data = await store.load(`agent-session:${JSON.stringify([input.id, 'default', input.session])}`); - expect(data?.entries).toHaveLength(2); - expect(data?.entries[0]).toMatchObject({ - directSubmissionId: input.submissionId, - message: { role: 'user', content: [{ type: 'text', text: 'Captured legacy prompt' }] }, - }); - expect(data?.entries[1]).toMatchObject({ - submissionTerminal: { - submissionId: input.submissionId, - kind: 'direct', - reason: 'interrupted_after_input_application', - }, - }); - expect(provider.state.callCount).toBe(0); - }); - it('classifies a completed canonical direct response without model replay', async () => { const provider = createProvider(); const store = new InMemorySessionStore(); From 1370d88ad9a19d547a7b22ad1398ec1209a1e49f Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" <622227+FredKSchott@users.noreply.github.com> Date: Thu, 4 Jun 2026 10:54:59 -0500 Subject: [PATCH 10/17] refactor(cloudflare): extract durable agent coordinator --- .../cli/src/lib/build-plugin-cloudflare.ts | 431 +--------- .../src/cloudflare/agent-coordinator.ts | 770 ++++++++++++++++++ packages/runtime/src/internal.ts | 1 + .../test/build-plugin-cloudflare.test.ts | 88 +- .../test/cloudflare-agent-coordinator.test.ts | 382 +++++++++ 5 files changed, 1205 insertions(+), 467 deletions(-) create mode 100644 packages/runtime/src/cloudflare/agent-coordinator.ts create mode 100644 packages/runtime/test/cloudflare-agent-coordinator.test.ts diff --git a/packages/cli/src/lib/build-plugin-cloudflare.ts b/packages/cli/src/lib/build-plugin-cloudflare.ts index e05c7be5..99c0dc7e 100644 --- a/packages/cli/src/lib/build-plugin-cloudflare.ts +++ b/packages/cli/src/lib/build-plugin-cloudflare.ts @@ -52,61 +52,41 @@ export class CloudflarePlugin implements BuildPlugin { ) => `const agentExtension${index} = resolveCloudflareAgentExtension(agentModules[${JSON.stringify(agent.name)}], ${JSON.stringify(agent.name)}); const ${agentClassName(agent.name)} = class ${agentClassName(agent.name)} extends agentExtension${index}.base(Agent) { constructor(ctx, env) { - const executionStore = createSqlAgentExecutionStore(ctx.storage, ${JSON.stringify(agentClassName(agent.name))}); + const prepared = cloudflareAgents.prepare({ storage: ctx.storage, className: ${JSON.stringify(agentClassName(agent.name))}, agentName: ${JSON.stringify(agent.name)} }); super(ctx, env); - this[FLUE_AGENT_EXECUTION_STORE] = executionStore; + cloudflareAgents.attach(this, prepared); } - async onStart(props) { - await restoreFlueAgentSubmissionWake(this); - if (typeof super.onStart === 'function') await super.onStart(props); - await reconcileFlueAgentSubmissions(this, ${JSON.stringify(agent.name)}, { driverAlreadyArmed: true }); + onStart(props) { + return cloudflareAgents.onStart(this, () => typeof super.onStart === 'function' ? super.onStart(props) : undefined); } - async __flueWakeAgentSubmissions() { - const submissions = getAgentExecutionStore(this).submissions; - if (!submissions.hasUnsettledSubmissions()) return; - await armFlueAgentSubmissionWake(this, { idempotent: false }); - await reconcileFlueAgentSubmissions(this, ${JSON.stringify(agent.name)}, { driverAlreadyArmed: true }); + __flueWakeAgentSubmissions() { + return cloudflareAgents.wakeSubmissions(this); } - async onRequest(request) { - return dispatchAgent(request, this, ${JSON.stringify(agent.name)}, directHandlers[${JSON.stringify(agent.name)}]); + onRequest(request) { + return cloudflareAgents.onRequest(this, request); } - async fetch(request) { - if (isWebSocketUpgrade(request)) { - await this.__unsafe_ensureInitialized(); - return acceptAgentSocket(request, this, ${JSON.stringify(agent.name)}); - } - return super.fetch(request); + fetch(request) { + return cloudflareAgents.fetch(this, request, () => super.fetch(request)); } - async webSocketMessage(socket, message) { - if (isFlueSocket(socket, 'agent', ${JSON.stringify(agent.name)})) { - await this.__unsafe_ensureInitialized(); - return messageAgentSocket(socket, message, this, ${JSON.stringify(agent.name)}); - } - return super.webSocketMessage(socket, message); + webSocketMessage(socket, message) { + return cloudflareAgents.webSocketMessage(this, socket, message, () => super.webSocketMessage(socket, message)); } - async webSocketClose(socket, code, reason, wasClean) { - if (isFlueSocket(socket, 'agent', ${JSON.stringify(agent.name)})) return closeFlueSocket(socket, code, reason); - return super.webSocketClose(socket, code, reason, wasClean); + webSocketClose(socket, code, reason, wasClean) { + return cloudflareAgents.webSocketClose(this, socket, code, reason, () => super.webSocketClose(socket, code, reason, wasClean)); } - async webSocketError(socket, error) { - if (isFlueSocket(socket, 'agent', ${JSON.stringify(agent.name)})) return closeFlueSocket(socket, 1011, 'WebSocket error'); - return super.webSocketError(socket, error); + webSocketError(socket, error) { + return cloudflareAgents.webSocketError(this, socket, () => super.webSocketError(socket, error)); } - async onFiberRecovered(ctx) { - if (ctx.name === 'flue:submission-attempt') { - return handleFlueAgentSubmissionAttemptRecovered(ctx, this); - } - if (typeof super.onFiberRecovered === 'function') { - return super.onFiberRecovered(ctx); - } + onFiberRecovered(ctx) { + return cloudflareAgents.onFiberRecovered(this, ctx, () => typeof super.onFiberRecovered === 'function' ? super.onFiberRecovered(ctx) : undefined); } }; const Wrapped${agentClassName(agent.name)} = agentExtension${index}.wrap(${agentClassName(agent.name)}); @@ -203,23 +183,15 @@ import { InMemorySessionStore, InMemoryRunStore, createDurableRunStore, - createSqlAgentExecutionStore, + CLOUDFLARE_AGENT_INTERNAL_DISPATCH_PATH, + createCloudflareAgentRuntime, createSqlSessionStore, - SqlAgentDispatchReceiptRetainedError, - SqlAgentSubmissionConflictError, createRunSubscriberRegistry, bashFactoryToSessionEnv, resolveModel, handleAgentRequest, handleWorkflowRequest, handleRunRouteRequest, - validateAgentDispatchAdmission, - createDispatchAgentHandler, - createDispatchInputInspectionHandler, - createDirectSubmissionAgentHandler, - createDirectSubmissionInputInspectionHandler, - createSubmissionTerminalHandler, - createAgentSubmissionObserverRegistry, failRecoveredRun, configureFlueRuntime, createDefaultFlueApp, @@ -232,9 +204,7 @@ import { getCloudflareAIBindingApiProvider, FlueRegistry, createCloudflareRunRegistry, - connectCloudflareAgentWebSocket, connectCloudflareWorkflowWebSocket, - messageCloudflareAgentWebSocket, messageCloudflareWorkflowWebSocket, resolveCloudflareAgentExtension, } from '@flue/runtime/cloudflare'; @@ -354,12 +324,7 @@ function resolveSandbox(sandbox) { const memoryWorkflowSessionStore = new InMemorySessionStore(); const memoryRunStore = new InMemoryRunStore(); -const FLUE_AGENT_EXECUTION_STORE = Symbol('flueAgentExecutionStore'); -const FLUE_AGENT_SUBMISSION_WAKE_CALLBACK = '__flueWakeAgentSubmissions'; -const FLUE_AGENT_SUBMISSION_WAKE_SECONDS = 30; -const FLUE_AGENT_SUBMISSION_ATTEMPT_STALE_MS = 15 * 60 * 1000; -const FLUE_AGENT_SUBMISSION_TERMINAL_RETENTION_MS = 7 * 24 * 60 * 60 * 1000; -const INTERNAL_DISPATCH_PATH = '/__flue/internal/dispatch'; +const INTERNAL_DISPATCH_PATH = CLOUDFLARE_AGENT_INTERNAL_DISPATCH_PATH; const dispatchQueue = { async enqueue(input) { const identity = agentIdentities[input.agent]; @@ -377,14 +342,6 @@ const dispatchQueue = { // Module-scoped per-isolate registry; run ids isolate buckets across DOs. const runSubscribers = createRunSubscriberRegistry(); -const agentSubmissionObservers = createAgentSubmissionObserverRegistry(); -const activeFlueAgentSubmissionAttempts = new Set(); - -function getAgentExecutionStore(doInstance) { - const store = doInstance[FLUE_AGENT_EXECUTION_STORE]; - if (!store) throw new Error('[flue] Generated Cloudflare agent execution store was not initialized.'); - return store; -} function createContextForRequest(id, runId, payload, doInstance, req, defaultStore, initialEventIndex, dispatchId) { return createFlueContext({ @@ -404,8 +361,7 @@ function createContextForRequest(id, runId, payload, doInstance, req, defaultSto }); } -function createAgentContextForRequest(id, payload, doInstance, req, initialEventIndex, dispatchId) { - const executionStore = getAgentExecutionStore(doInstance); +function createAgentContextForRequest(executionStore, id, payload, doInstance, req, initialEventIndex, dispatchId) { return createFlueContext({ id, payload, @@ -467,6 +423,19 @@ function createDurableObjectIdentity(doInstance, identity) { }; } +const cloudflareAgents = createCloudflareAgentRuntime({ + createdAgents, + directHandlers, + websocketAgentHandlers, + createContext: ({ executionStore, instance, payload, request, initialEventIndex, dispatchId }) => + createAgentContextForRequest(executionStore, instance.name, payload, instance, request, initialEventIndex, dispatchId), + runWithInstanceContext: (instance, agentName, fn) => runWithInstanceContext(instance, agentRuntimeIdentity(agentName), fn), + createWebSocketPair: () => { + const pair = new WebSocketPair(); + return { client: pair[0], server: pair[1] }; + }, +}); + function assertAgentsDurabilityApi(doInstance, method) { if (typeof doInstance[method] !== 'function') { throw new Error( @@ -477,15 +446,6 @@ function assertAgentsDurabilityApi(doInstance, method) { } } -async function handleFlueAgentSubmissionAttemptRecovered(ctx, doInstance) { - const submissionId = ctx.snapshot?.submissionId; - const attemptId = ctx.snapshot?.attemptId; - if (typeof submissionId !== 'string' || typeof attemptId !== 'string') return; - await restoreFlueAgentSubmissionWake(doInstance); - const submissions = getAgentExecutionStore(doInstance).submissions; - submissions.requestSubmissionRecovery(submissionId, attemptId); -} - async function handleFlueWorkflowFiberRecovered(ctx, doInstance, workflowName) { if (!ctx.name || ctx.name !== 'flue:workflow:' + doInstance.name) return; const interruptedRunId = doInstance.name; @@ -503,270 +463,6 @@ async function handleFlueWorkflowFiberRecovered(ctx, doInstance, workflowName) { }); } -// ─── Per-DO Agent Submissions ───────────────────────────────────────────── - -function armFlueAgentSubmissionWake(doInstance, options = {}) { - assertAgentsDurabilityApi(doInstance, 'schedule'); - return doInstance.schedule( - options.delaySeconds ?? FLUE_AGENT_SUBMISSION_WAKE_SECONDS, - FLUE_AGENT_SUBMISSION_WAKE_CALLBACK, - undefined, - { idempotent: options.idempotent ?? true }, - ); -} - -function armFlueAgentSubmissionAdmissionWake(doInstance) { - return armFlueAgentSubmissionWake(doInstance); -} - -function cleanupFlueAgentSubmissionTerminalState(doInstance) { - return getAgentExecutionStore(doInstance).submissions.cleanupTerminalSubmissions( - Date.now() - FLUE_AGENT_SUBMISSION_TERMINAL_RETENTION_MS, - ); -} - -async function restoreFlueAgentSubmissionWake(doInstance) { - const submissions = getAgentExecutionStore(doInstance).submissions; - if (!submissions.hasUnsettledSubmissions()) return false; - await armFlueAgentSubmissionWake(doInstance); - return true; -} - -async function reconcileFlueAgentSubmissions(doInstance, agentName, options = {}) { - const submissions = getAgentExecutionStore(doInstance).submissions; - cleanupFlueAgentSubmissionTerminalState(doInstance); - if (!submissions.hasUnsettledSubmissions()) return false; - if (!options.driverAlreadyArmed) await restoreFlueAgentSubmissionWake(doInstance); - if (!submissions.hasUnsettledSubmissions()) return false; - try { - const attemptMarkers = listActiveSqlAgentSubmissionAttemptMarkers(doInstance); - if (attemptMarkers.blockAll) return true; - for (const submission of submissions.listRunningSubmissions()) { - if (activeFlueAgentSubmissionAttempts.has(submissionAttemptLocalKey(doInstance, submission))) continue; - if (submission.status !== 'terminalizing' && attemptMarkers.keys.has(submissionAttemptMarkerKey(submission)) && submission.recoveryRequestedAt === undefined) continue; - await reconcileInterruptedSqlAgentSubmission(submission, doInstance, agentName); - } - for (const submission of submissions.listRunnableSubmissions()) { - const claimed = submissions.claimSubmission(submission.submissionId, crypto.randomUUID()); - if (claimed) startSqlAgentSubmissionAttempt(claimed, doInstance, agentName); - } - } catch (error) { - console.error('[flue:submission-reconciliation]', { agentName, instanceId: doInstance.name, operation: 'reconcile', outcome: 'deferred_to_scheduled_wake' }, error); - return true; - } - return submissions.hasUnsettledSubmissions(); -} - -async function reconcileInterruptedSqlAgentSubmission(submission, doInstance, agentName) { - const { attemptId, input } = submission; - if (!attemptId) return; - const submissions = getAgentExecutionStore(doInstance).submissions; - if (submission.status === 'terminalizing') { - await failInterruptedSqlAgentSubmission( - submission, - doInstance, - agentName, - submission.inputAppliedAt === undefined ? 'interrupted_before_input_marker' : 'interrupted_after_input_application', - new Error(submission.inputAppliedAt === undefined - ? '[flue] Agent submission attempt was interrupted after canonical input persistence but before the input-application marker was recorded. Provider replay was not attempted.' - : '[flue] Agent submission attempt was interrupted after input application without a completed canonical response. Provider replay was not attempted.'), - ); - return; - } - const agent = createdAgents[agentName]; - if (!agent) throw new Error('[flue] Agent target unavailable during durable reconciliation.'); - const request = new Request('https://flue.invalid' + INTERNAL_DISPATCH_PATH, { method: 'POST' }); - const ctx = createAgentContextForRequest(doInstance.name, input, doInstance, request, undefined, input.dispatchId); - const state = await runWithInstanceContext(doInstance, agentRuntimeIdentity(agentName), () => - submission.kind === 'dispatch' - ? createDispatchInputInspectionHandler(agent, input)(ctx) - : createDirectSubmissionInputInspectionHandler(agent, input)(ctx), - ); - if (submission.inputAppliedAt === undefined) { - if (state === 'absent') { - submissions.requeueSubmissionBeforeInputApplied(submission.submissionId, attemptId); - return; - } - await failInterruptedSqlAgentSubmission( - submission, - doInstance, - agentName, - 'interrupted_before_input_marker', - new Error('[flue] Agent submission attempt was interrupted after canonical input persistence but before the input-application marker was recorded. Provider replay was not attempted.'), - ); - return; - } - if (state === 'completed') { - submissions.completeSubmission(submission.submissionId, attemptId); - return; - } - await failInterruptedSqlAgentSubmission( - submission, - doInstance, - agentName, - 'interrupted_after_input_application', - new Error('[flue] Agent submission attempt was interrupted after input application without a completed canonical response. Provider replay was not attempted.'), - ); -} - -async function failInterruptedSqlAgentSubmission(submission, doInstance, agentName, reason, error) { - const { attemptId, input } = submission; - if (!attemptId) return; - const submissions = getAgentExecutionStore(doInstance).submissions; - if (submission.status !== 'terminalizing' && !submissions.beginSubmissionTerminalization(submission.submissionId, attemptId)) return; - const agent = createdAgents[agentName]; - if (!agent) throw new Error('[flue] Agent target unavailable during durable terminalization.'); - const request = new Request('https://flue.invalid' + INTERNAL_DISPATCH_PATH, { method: 'POST' }); - const ctx = createAgentContextForRequest(doInstance.name, submission.kind === 'direct' ? input.payload : input, doInstance, request, undefined, input.dispatchId); - await runWithInstanceContext(doInstance, agentRuntimeIdentity(agentName), () => - createSubmissionTerminalHandler(agent, input, { - submissionId: submission.submissionId, - kind: submission.kind, - reason, - message: error.message, - })(ctx), - ); - const failed = submissions.finalizeSubmissionTerminalization(submission.submissionId, attemptId, error); - if (failed && submission.kind === 'direct') agentSubmissionObservers.fail(submission.submissionId, error); -} - -function startSqlAgentSubmissionAttempt(submission, doInstance, agentName) { - if (submission.status !== 'running' || !submission.attemptId) return; - const attemptKey = submissionAttemptLocalKey(doInstance, submission); - if (activeFlueAgentSubmissionAttempts.has(attemptKey)) return; - assertAgentsDurabilityApi(doInstance, 'runFiber'); - activeFlueAgentSubmissionAttempts.add(attemptKey); - let running; - try { - running = doInstance.runFiber('flue:submission-attempt', async (fiberCtx) => { - fiberCtx.stash({ submissionId: submission.submissionId, attemptId: submission.attemptId }); - await processSqlAgentSubmission(submission, doInstance, agentName); - }); - } catch (error) { - activeFlueAgentSubmissionAttempts.delete(attemptKey); - throw error; - } - void running.catch((error) => { - console.error('[flue:submission-processing]', { agentName, instanceId: doInstance.name, submissionId: submission.submissionId, operation: 'process', outcome: 'failed' }, error); - }).finally(() => { - activeFlueAgentSubmissionAttempts.delete(attemptKey); - }); -} - -function submissionAttemptLocalKey(doInstance, submission) { - return doInstance.ctx.id.toString() + ':' + submission.attemptId; -} - -function submissionAttemptMarkerKey(submission) { - return submission.submissionId + ':' + submission.attemptId; -} - -function listActiveSqlAgentSubmissionAttemptMarkers(doInstance) { - const keys = new Set(); - let blockAll = false; - const rows = doInstance.ctx.storage.sql.exec( - "SELECT snapshot, created_at FROM cf_agents_runs WHERE name = 'flue:submission-attempt'", - ).toArray(); - for (const row of rows) { - if (typeof row.created_at !== 'number') { - blockAll = true; - continue; - } - if (Date.now() - row.created_at > FLUE_AGENT_SUBMISSION_ATTEMPT_STALE_MS) continue; - if (row.snapshot === null) continue; - if (typeof row.snapshot !== 'string') { - blockAll = true; - continue; - } - let snapshot; - try { - snapshot = JSON.parse(row.snapshot); - } catch { - blockAll = true; - continue; - } - if (typeof snapshot?.submissionId !== 'string' || typeof snapshot?.attemptId !== 'string') { - blockAll = true; - continue; - } - keys.add(snapshot.submissionId + ':' + snapshot.attemptId); - } - return { blockAll, keys }; -} - -async function processSqlAgentSubmission(submission, doInstance, agentName) { - const { attemptId, input } = submission; - if (!attemptId) return; - const submissions = getAgentExecutionStore(doInstance).submissions; - const persisted = submissions.getSubmission(submission.submissionId); - if (persisted?.status !== 'running' || persisted.attemptId !== attemptId) return; - let ctx; - try { - const agent = createdAgents[agentName]; - if (!agent) throw new Error('[flue] Agent target unavailable during durable processing.'); - if (submission.kind === 'dispatch') await validateAgentDispatchAdmission({ input }); - const request = submission.kind === 'direct' - ? new Request('https://flue.invalid/agents/' + encodeURIComponent(agentName) + '/' + encodeURIComponent(doInstance.name), { method: 'POST' }) - : new Request('https://flue.invalid' + INTERNAL_DISPATCH_PATH, { method: 'POST' }); - ctx = createAgentContextForRequest(doInstance.name, submission.kind === 'direct' ? input.payload : input, doInstance, request, undefined, input.dispatchId); - if (submission.kind === 'direct') { - ctx.setEventCallback((event) => { - if (event.type === 'run_start' || event.type === 'run_end') return; - const attachedEvent = { ...event, instanceId: doInstance.name }; - delete attachedEvent.runId; - return agentSubmissionObservers.publish(submission.submissionId, attachedEvent); - }); - } - const result = await runWithInstanceContext(doInstance, agentRuntimeIdentity(agentName), () => - submission.kind === 'dispatch' - ? createDispatchAgentHandler(agent, input, { - onInputApplied: () => markSubmissionInputApplied(submissions, submission, attemptId), - })(ctx) - : createDirectSubmissionAgentHandler(agent, input, { - onInputApplied: () => markSubmissionInputApplied(submissions, submission, attemptId), - })(ctx), - ); - const completed = submissions.completeSubmission(submission.submissionId, attemptId); - if (completed && submission.kind === 'direct') agentSubmissionObservers.complete(submission.submissionId, result); - } catch (error) { - const failed = submissions.failSubmission(submission.submissionId, attemptId, error); - if (failed && submission.kind === 'direct') agentSubmissionObservers.fail(submission.submissionId, error); - throw error; - } finally { - if (submission.kind === 'direct') ctx?.setEventCallback(undefined); - void reconcileFlueAgentSubmissions(doInstance, agentName).catch((error) => { - console.error('[flue:submission-reconciliation]', { agentName, instanceId: doInstance.name, operation: 'settlement', outcome: 'reconcile_failed' }, error); - }); - } -} - -function markSubmissionInputApplied(submissions, submission, attemptId) { - if (!submissions.markSubmissionInputApplied(submission.submissionId, attemptId)) { - throw new Error('[flue] Agent submission attempt lost ownership before input application.'); - } -} - -async function admitAttachedAgentSubmission(doInstance, agentName, payload, _request, onEvent) { - const submissionId = crypto.randomUUID(); - const input = { - submissionId, - agent: agentName, - id: doInstance.name, - session: typeof payload.session === 'string' && payload.session.trim() !== '' ? payload.session : 'default', - payload, - acceptedAt: new Date().toISOString(), - }; - const attachment = agentSubmissionObservers.attach(submissionId, { onEvent }); - try { - await armFlueAgentSubmissionAdmissionWake(doInstance); - getAgentExecutionStore(doInstance).submissions.admitDirect(input); - await reconcileFlueAgentSubmissions(doInstance, agentName, { driverAlreadyArmed: true }); - return await attachment.completion; - } finally { - attachment.detach(); - } -} - async function dispatchWorkflow(request, doInstance, workflowName) { // The DO room name is the workflow instance id. For workflows that // equals the run id (one run per instance), so callers reach this DO @@ -805,38 +501,6 @@ async function dispatchWorkflow(request, doInstance, workflowName) { })); } -async function dispatchAgent(request, doInstance, agentName, handler) { - const id = doInstance.name; - if (isInternalDispatchRequest(request)) { - const input = await request.json(); - await validateAgentDispatchAdmission({ input }); - if (input.agent !== agentName || input.id !== id) return new Response('Invalid internal dispatch target.', { status: 400 }); - if (!createdAgents[agentName]) return new Response('Dispatch target unavailable.', { status: 404 }); - const submissions = getAgentExecutionStore(doInstance).submissions; - cleanupFlueAgentSubmissionTerminalState(doInstance); - await armFlueAgentSubmissionAdmissionWake(doInstance); - let submission; - try { - submission = submissions.admitDispatch(input); - } catch (error) { - if (error instanceof SqlAgentDispatchReceiptRetainedError) return Response.json({ dispatchId: error.receipt.submissionId, acceptedAt: new Date(error.receipt.acceptedAt).toISOString() }); - if (error instanceof SqlAgentSubmissionConflictError) return new Response('Conflicting internal dispatch replay.', { status: 409 }); - throw error; - } - await reconcileFlueAgentSubmissions(doInstance, agentName, { driverAlreadyArmed: true }); - return Response.json({ dispatchId: submission.submissionId, acceptedAt: submission.input.acceptedAt }); - } - const identity = agentRuntimeIdentity(agentName); - return runWithInstanceContext(doInstance, identity, () => handleAgentRequest({ - request, - agentName, - id, - handler, - createContext: (id_, runId, payload, req, initialEventIndex, dispatchId) => createAgentContextForRequest(id_, payload, doInstance, req, initialEventIndex, dispatchId), - admitAttachedSubmission: (payload, req, onEvent) => admitAttachedAgentSubmission(doInstance, agentName, payload, req, onEvent), - })); -} - function isWebSocketUpgrade(request) { return request.method === 'GET' && request.headers.get('upgrade')?.toLowerCase() === 'websocket'; } @@ -855,17 +519,6 @@ function closeFlueSocket(socket, code, reason) { } } -function acceptAgentSocket(request, doInstance, agentName) { - const handler = websocketAgentHandlers[agentName]; - if (!handler) return new Response(null, { status: 404 }); - const pair = new WebSocketPair(); - const client = pair[0]; - const server = pair[1]; - doInstance.ctx.acceptWebSocket(server); - connectCloudflareAgentWebSocket(server, { name: agentName, id: doInstance.name, requestUrl: socketRequestUrl(request) }); - return new Response(null, { status: 101, webSocket: client }); -} - function acceptWorkflowSocket(request, doInstance, workflowName) { const handler = websocketWorkflowHandlers[workflowName]; if (!handler) return new Response(null, { status: 404 }); @@ -877,20 +530,6 @@ function acceptWorkflowSocket(request, doInstance, workflowName) { return new Response(null, { status: 101, webSocket: client }); } -async function messageAgentSocket(connection, message, doInstance, agentName) { - const handler = websocketAgentHandlers[agentName]; - if (!handler) return; - const identity = agentRuntimeIdentity(agentName); - return runWithInstanceContext(doInstance, identity, () => messageCloudflareAgentWebSocket(connection, message, { - name: agentName, - id: doInstance.name, - request: socketRequest(connection), - handler, - createContext: (id_, runId, payload, req) => createAgentContextForRequest(id_, payload, doInstance, req), - admitAttachedSubmission: (payload, req, onEvent) => admitAttachedAgentSubmission(doInstance, agentName, payload, req, onEvent), - })); -} - async function messageWorkflowSocket(connection, message, doInstance, workflowName) { const handler = websocketWorkflowHandlers[workflowName]; if (!handler) return; diff --git a/packages/runtime/src/cloudflare/agent-coordinator.ts b/packages/runtime/src/cloudflare/agent-coordinator.ts new file mode 100644 index 00000000..1bde8eb1 --- /dev/null +++ b/packages/runtime/src/cloudflare/agent-coordinator.ts @@ -0,0 +1,770 @@ +import type { FlueContextInternal } from '../client.ts'; +import { + createAgentSubmissionObserverRegistry, + type DirectSubmissionInput, + type DispatchInput, +} from '../runtime/dispatch-queue.ts'; +import { + type AgentHandler, + createDirectSubmissionAgentHandler, + createDirectSubmissionInputInspectionHandler, + createDispatchAgentHandler, + createDispatchInputInspectionHandler, + createSubmissionTerminalHandler, + handleAgentRequest, + validateAgentDispatchAdmission, +} from '../runtime/handle-agent.ts'; +import type { AttachedAgentEvent, DirectAgentPayload } from '../types.ts'; +import { + createSqlAgentExecutionStore, + SqlAgentDispatchReceiptRetainedError, + type SqlAgentExecutionStore, + type SqlAgentSubmission, + SqlAgentSubmissionConflictError, + type SqlAgentSubmissionStore, +} from './agent-execution-store.ts'; +import { + type CloudflareWebSocketConnection, + connectCloudflareAgentWebSocket, + messageCloudflareAgentWebSocket, +} from './websocket.ts'; + +export const CLOUDFLARE_AGENT_INTERNAL_DISPATCH_PATH = '/__flue/internal/dispatch'; + +const FLUE_AGENT_SUBMISSION_WAKE_CALLBACK = '__flueWakeAgentSubmissions'; +const FLUE_AGENT_SUBMISSION_WAKE_SECONDS = 30; +const FLUE_AGENT_SUBMISSION_ATTEMPT_STALE_MS = 15 * 60 * 1000; +const FLUE_AGENT_SUBMISSION_TERMINAL_RETENTION_MS = 7 * 24 * 60 * 60 * 1000; +const FLUE_AGENT_SUBMISSION_ATTEMPT_FIBER = 'flue:submission-attempt'; + +interface SqlResult { + toArray(): Array>; +} + +interface CloudflareAgentStorage { + sql?: { + exec(query: string, ...bindings: unknown[]): SqlResult; + }; + transactionSync?(closure: () => T): T; +} + +interface CloudflareAgentInstance { + readonly name: string; + readonly env: Record; + readonly ctx: { + readonly id: { toString(): string }; + readonly storage: CloudflareAgentStorage; + acceptWebSocket(connection: CloudflareWebSocketConnection): void; + }; + __unsafe_ensureInitialized(): Promise; + schedule( + delaySeconds: number, + callback: string, + payload: undefined, + options: { idempotent: boolean }, + ): Promise; + runFiber( + name: string, + callback: (ctx: { stash(snapshot: unknown): void }) => Promise, + ): Promise; +} + +interface CloudflareAgentRecoveredFiberContext { + readonly name?: string; + readonly snapshot?: Record; +} + +interface CloudflareAgentPreparedCoordinator { + readonly agentName: string; + readonly executionStore: SqlAgentExecutionStore; +} + +interface CloudflareAgentRuntimeOptions { + readonly createdAgents: Record[0]>; + readonly directHandlers: Record; + readonly websocketAgentHandlers: Record; + readonly createContext: (options: { + readonly executionStore: SqlAgentExecutionStore; + readonly instance: CloudflareAgentInstance; + readonly payload: unknown; + readonly request: Request; + readonly initialEventIndex?: number; + readonly dispatchId?: string; + }) => FlueContextInternal; + readonly runWithInstanceContext: ( + instance: CloudflareAgentInstance, + agentName: string, + callback: () => T, + ) => T; + readonly createWebSocketPair: () => { + readonly client: unknown; + readonly server: CloudflareWebSocketConnection; + }; +} + +export interface CloudflareAgentRuntime { + prepare(options: { + readonly storage: CloudflareAgentStorage; + readonly className: string; + readonly agentName: string; + }): CloudflareAgentPreparedCoordinator; + attach(instance: CloudflareAgentInstance, prepared: CloudflareAgentPreparedCoordinator): void; + onStart( + instance: CloudflareAgentInstance, + inherited: () => Promise | unknown, + ): Promise; + wakeSubmissions(instance: CloudflareAgentInstance): Promise; + onRequest(instance: CloudflareAgentInstance, request: Request): Promise; + fetch( + instance: CloudflareAgentInstance, + request: Request, + inherited: () => Promise | Response, + ): Promise; + webSocketMessage( + instance: CloudflareAgentInstance, + connection: CloudflareWebSocketConnection, + message: string | ArrayBuffer | ArrayBufferView, + inherited: () => Promise | unknown, + ): Promise; + webSocketClose( + instance: CloudflareAgentInstance, + connection: CloudflareWebSocketConnection, + code: number, + reason: string, + inherited: () => Promise | unknown, + ): Promise | unknown; + webSocketError( + instance: CloudflareAgentInstance, + connection: CloudflareWebSocketConnection, + inherited: () => Promise | unknown, + ): Promise | unknown; + onFiberRecovered( + instance: CloudflareAgentInstance, + ctx: CloudflareAgentRecoveredFiberContext, + inherited: () => Promise | unknown, + ): Promise; +} + +export function createCloudflareAgentRuntime(options: CloudflareAgentRuntimeOptions): CloudflareAgentRuntime { + const coordinators = new WeakMap(); + const observers = createAgentSubmissionObserverRegistry(); + const activeAttempts = new Set(); + + const getCoordinator = (instance: CloudflareAgentInstance): CloudflareAgentCoordinator => { + const coordinator = coordinators.get(instance); + if (!coordinator) { + throw new Error('[flue] Generated Cloudflare agent coordinator was not initialized.'); + } + return coordinator; + }; + + return { + prepare({ storage, className, agentName }) { + return { + agentName, + executionStore: createSqlAgentExecutionStore(storage, className), + }; + }, + attach(instance, prepared) { + coordinators.set( + instance, + new CloudflareAgentCoordinator(instance, prepared, options, observers, activeAttempts), + ); + }, + onStart(instance, inherited) { + return getCoordinator(instance).onStart(inherited); + }, + wakeSubmissions(instance) { + return getCoordinator(instance).wakeSubmissions(); + }, + onRequest(instance, request) { + return getCoordinator(instance).onRequest(request); + }, + fetch(instance, request, inherited) { + return getCoordinator(instance).fetch(request, inherited); + }, + webSocketMessage(instance, connection, message, inherited) { + return getCoordinator(instance).webSocketMessage(connection, message, inherited); + }, + webSocketClose(instance, connection, code, reason, inherited) { + return getCoordinator(instance).webSocketClose(connection, code, reason, inherited); + }, + webSocketError(instance, connection, inherited) { + return getCoordinator(instance).webSocketError(connection, inherited); + }, + onFiberRecovered(instance, ctx, inherited) { + return getCoordinator(instance).onFiberRecovered(ctx, inherited); + }, + }; +} + +class CloudflareAgentCoordinator { + constructor( + private readonly instance: CloudflareAgentInstance, + private readonly prepared: CloudflareAgentPreparedCoordinator, + private readonly options: CloudflareAgentRuntimeOptions, + private readonly observers: ReturnType, + private readonly activeAttempts: Set, + ) {} + + async onStart(inherited: () => Promise | unknown): Promise { + await this.restoreSubmissionWake(); + await inherited(); + await this.reconcileSubmissions({ driverAlreadyArmed: true }); + } + + async wakeSubmissions(): Promise { + if (!this.submissions.hasUnsettledSubmissions()) return; + await this.armSubmissionWake({ idempotent: false }); + await this.reconcileSubmissions({ driverAlreadyArmed: true }); + } + + async onRequest(request: Request): Promise { + if (isInternalDispatchRequest(request)) return this.admitDispatch(request); + const handler = this.options.directHandlers[this.agentName]; + if (!handler) throw new Error('[flue] Agent direct handler is unavailable.'); + return this.runWithInstanceContext(() => + handleAgentRequest({ + request, + agentName: this.agentName, + id: this.instance.name, + handler, + createContext: (_id, _runId, payload, req, initialEventIndex, dispatchId) => + this.createContext(payload, req, initialEventIndex, dispatchId), + admitAttachedSubmission: (payload, _req, onEvent) => + this.admitAttachedSubmission(payload, onEvent), + }), + ); + } + + async fetch(request: Request, inherited: () => Promise | Response): Promise { + if (!isWebSocketUpgrade(request)) return inherited(); + await this.instance.__unsafe_ensureInitialized(); + return this.acceptSocket(request); + } + + async webSocketMessage( + connection: CloudflareWebSocketConnection, + message: string | ArrayBuffer | ArrayBufferView, + inherited: () => Promise | unknown, + ): Promise { + if (!isFlueAgentSocket(connection, this.agentName)) return inherited(); + await this.instance.__unsafe_ensureInitialized(); + const handler = this.options.websocketAgentHandlers[this.agentName]; + if (!handler) return; + return this.runWithInstanceContext(() => + messageCloudflareAgentWebSocket(connection, message, { + name: this.agentName, + id: this.instance.name, + request: socketRequest(connection), + handler, + createContext: (_id, _runId, payload, req) => this.createContext(payload, req), + admitAttachedSubmission: (payload, _req, onEvent) => + this.admitAttachedSubmission(payload, onEvent), + }), + ); + } + + webSocketClose( + connection: CloudflareWebSocketConnection, + code: number, + reason: string, + inherited: () => Promise | unknown, + ): Promise | unknown { + if (!isFlueAgentSocket(connection, this.agentName)) return inherited(); + return closeSocket(connection, code, reason); + } + + webSocketError( + connection: CloudflareWebSocketConnection, + inherited: () => Promise | unknown, + ): Promise | unknown { + if (!isFlueAgentSocket(connection, this.agentName)) return inherited(); + return closeSocket(connection, 1011, 'WebSocket error'); + } + + async onFiberRecovered( + ctx: CloudflareAgentRecoveredFiberContext, + inherited: () => Promise | unknown, + ): Promise { + if (ctx.name !== FLUE_AGENT_SUBMISSION_ATTEMPT_FIBER) return inherited(); + const submissionId = ctx.snapshot?.submissionId; + const attemptId = ctx.snapshot?.attemptId; + if (typeof submissionId !== 'string' || typeof attemptId !== 'string') return; + await this.restoreSubmissionWake(); + this.submissions.requestSubmissionRecovery(submissionId, attemptId); + } + + private get agentName(): string { + return this.prepared.agentName; + } + + private get executionStore(): SqlAgentExecutionStore { + return this.prepared.executionStore; + } + + private get submissions(): SqlAgentSubmissionStore { + return this.executionStore.submissions; + } + + private runWithInstanceContext(callback: () => T): T { + return this.options.runWithInstanceContext(this.instance, this.agentName, callback); + } + + private createContext( + payload: unknown, + request: Request, + initialEventIndex?: number, + dispatchId?: string, + ): FlueContextInternal { + return this.options.createContext({ + executionStore: this.executionStore, + instance: this.instance, + payload, + request, + initialEventIndex, + dispatchId, + }); + } + + private assertAgentsDurabilityApi(method: 'runFiber' | 'schedule'): void { + if (typeof this.instance[method] !== 'function') { + throw new Error( + `[flue] The installed "agents" package does not provide the required Cloudflare Agents SDK method "${method}". Install or upgrade the "agents" package in your project.`, + ); + } + } + + private armSubmissionWake(options: { delaySeconds?: number; idempotent?: boolean } = {}): Promise { + this.assertAgentsDurabilityApi('schedule'); + return this.instance.schedule( + options.delaySeconds ?? FLUE_AGENT_SUBMISSION_WAKE_SECONDS, + FLUE_AGENT_SUBMISSION_WAKE_CALLBACK, + undefined, + { idempotent: options.idempotent ?? true }, + ); + } + + private cleanupTerminalState(): number { + return this.submissions.cleanupTerminalSubmissions( + Date.now() - FLUE_AGENT_SUBMISSION_TERMINAL_RETENTION_MS, + ); + } + + private async restoreSubmissionWake(): Promise { + if (!this.submissions.hasUnsettledSubmissions()) return false; + await this.armSubmissionWake(); + return true; + } + + private async reconcileSubmissions(options: { driverAlreadyArmed?: boolean } = {}): Promise { + this.cleanupTerminalState(); + if (!this.submissions.hasUnsettledSubmissions()) return false; + if (!options.driverAlreadyArmed) await this.restoreSubmissionWake(); + if (!this.submissions.hasUnsettledSubmissions()) return false; + try { + const attemptMarkers = this.listActiveAttemptMarkers(); + if (attemptMarkers.blockAll) return true; + for (const submission of this.submissions.listRunningSubmissions()) { + if (this.activeAttempts.has(this.submissionAttemptLocalKey(submission))) continue; + if ( + submission.status !== 'terminalizing' && + attemptMarkers.keys.has(submissionAttemptMarkerKey(submission)) && + submission.recoveryRequestedAt === undefined + ) + continue; + await this.reconcileInterruptedSubmission(submission); + } + for (const submission of this.submissions.listRunnableSubmissions()) { + const claimed = this.submissions.claimSubmission(submission.submissionId, crypto.randomUUID()); + if (claimed) this.startSubmissionAttempt(claimed); + } + } catch (error) { + console.error( + '[flue:submission-reconciliation]', + { + agentName: this.agentName, + instanceId: this.instance.name, + operation: 'reconcile', + outcome: 'deferred_to_scheduled_wake', + }, + error, + ); + return true; + } + return this.submissions.hasUnsettledSubmissions(); + } + + private async reconcileInterruptedSubmission(submission: SqlAgentSubmission): Promise { + const { attemptId, input } = submission; + if (!attemptId) return; + if (submission.status === 'terminalizing') { + await this.failInterruptedSubmission( + submission, + submission.inputAppliedAt === undefined + ? 'interrupted_before_input_marker' + : 'interrupted_after_input_application', + new Error( + submission.inputAppliedAt === undefined + ? '[flue] Agent submission attempt was interrupted after canonical input persistence but before the input-application marker was recorded. Provider replay was not attempted.' + : '[flue] Agent submission attempt was interrupted after input application without a completed canonical response. Provider replay was not attempted.', + ), + ); + return; + } + const agent = this.options.createdAgents[this.agentName]; + if (!agent) throw new Error('[flue] Agent target unavailable during durable reconciliation.'); + const request = new Request(`https://flue.invalid${CLOUDFLARE_AGENT_INTERNAL_DISPATCH_PATH}`, { + method: 'POST', + }); + const ctx = this.createContext(input, request, undefined, dispatchIdFor(input)); + const state = await this.runWithInstanceContext(() => + submission.kind === 'dispatch' + ? createDispatchInputInspectionHandler(agent, input as DispatchInput)(ctx) + : createDirectSubmissionInputInspectionHandler(agent, input as DirectSubmissionInput)(ctx), + ); + if (submission.inputAppliedAt === undefined) { + if (state === 'absent') { + this.submissions.requeueSubmissionBeforeInputApplied(submission.submissionId, attemptId); + return; + } + await this.failInterruptedSubmission( + submission, + 'interrupted_before_input_marker', + new Error( + '[flue] Agent submission attempt was interrupted after canonical input persistence but before the input-application marker was recorded. Provider replay was not attempted.', + ), + ); + return; + } + if (state === 'completed') { + this.submissions.completeSubmission(submission.submissionId, attemptId); + return; + } + await this.failInterruptedSubmission( + submission, + 'interrupted_after_input_application', + new Error( + '[flue] Agent submission attempt was interrupted after input application without a completed canonical response. Provider replay was not attempted.', + ), + ); + } + + private async failInterruptedSubmission( + submission: SqlAgentSubmission, + reason: 'interrupted_before_input_marker' | 'interrupted_after_input_application', + error: Error, + ): Promise { + const { attemptId, input } = submission; + if (!attemptId) return; + if ( + submission.status !== 'terminalizing' && + !this.submissions.beginSubmissionTerminalization(submission.submissionId, attemptId) + ) + return; + const agent = this.options.createdAgents[this.agentName]; + if (!agent) throw new Error('[flue] Agent target unavailable during durable terminalization.'); + const request = new Request(`https://flue.invalid${CLOUDFLARE_AGENT_INTERNAL_DISPATCH_PATH}`, { + method: 'POST', + }); + const ctx = this.createContext( + submission.kind === 'direct' ? (input as DirectSubmissionInput).payload : input, + request, + undefined, + dispatchIdFor(input), + ); + await this.runWithInstanceContext(() => + createSubmissionTerminalHandler(agent, input, { + submissionId: submission.submissionId, + kind: submission.kind, + reason, + message: error.message, + })(ctx), + ); + const failed = this.submissions.finalizeSubmissionTerminalization( + submission.submissionId, + attemptId, + error, + ); + if (failed && submission.kind === 'direct') this.observers.fail(submission.submissionId, error); + } + + private startSubmissionAttempt(submission: SqlAgentSubmission): void { + if (submission.status !== 'running' || !submission.attemptId) return; + const attemptKey = this.submissionAttemptLocalKey(submission); + if (this.activeAttempts.has(attemptKey)) return; + this.assertAgentsDurabilityApi('runFiber'); + this.activeAttempts.add(attemptKey); + let running: Promise; + try { + running = this.instance.runFiber(FLUE_AGENT_SUBMISSION_ATTEMPT_FIBER, async (fiberCtx) => { + fiberCtx.stash({ submissionId: submission.submissionId, attemptId: submission.attemptId }); + await this.processSubmission(submission); + }); + } catch (error) { + this.activeAttempts.delete(attemptKey); + throw error; + } + void running + .catch((error) => { + console.error( + '[flue:submission-processing]', + { + agentName: this.agentName, + instanceId: this.instance.name, + submissionId: submission.submissionId, + operation: 'process', + outcome: 'failed', + }, + error, + ); + }) + .finally(() => { + this.activeAttempts.delete(attemptKey); + }); + } + + private submissionAttemptLocalKey(submission: SqlAgentSubmission): string { + return `${this.instance.ctx.id.toString()}:${submission.attemptId}`; + } + + private listActiveAttemptMarkers(): { blockAll: boolean; keys: Set } { + const keys = new Set(); + let blockAll = false; + const rows = this.instance.ctx.storage.sql + ?.exec( + `SELECT snapshot, created_at FROM cf_agents_runs WHERE name = '${FLUE_AGENT_SUBMISSION_ATTEMPT_FIBER}'`, + ) + .toArray(); + if (!rows) throw new Error('[flue] Cloudflare durable agent SQL storage is unavailable.'); + for (const row of rows) { + if (typeof row.created_at !== 'number') { + blockAll = true; + continue; + } + if (Date.now() - row.created_at > FLUE_AGENT_SUBMISSION_ATTEMPT_STALE_MS) continue; + if (row.snapshot === null) continue; + if (typeof row.snapshot !== 'string') { + blockAll = true; + continue; + } + let snapshot: unknown; + try { + snapshot = JSON.parse(row.snapshot); + } catch { + blockAll = true; + continue; + } + if (!isAttemptMarkerSnapshot(snapshot)) { + blockAll = true; + continue; + } + keys.add(`${snapshot.submissionId}:${snapshot.attemptId}`); + } + return { blockAll, keys }; + } + + private async processSubmission(submission: SqlAgentSubmission): Promise { + const { attemptId, input } = submission; + if (!attemptId) return; + const persisted = this.submissions.getSubmission(submission.submissionId); + if (persisted?.status !== 'running' || persisted.attemptId !== attemptId) return; + let ctx: FlueContextInternal | undefined; + try { + const agent = this.options.createdAgents[this.agentName]; + if (!agent) throw new Error('[flue] Agent target unavailable during durable processing.'); + if (submission.kind === 'dispatch') await validateAgentDispatchAdmission({ input: input as DispatchInput }); + const request = + submission.kind === 'direct' + ? new Request( + `https://flue.invalid/agents/${encodeURIComponent(this.agentName)}/${encodeURIComponent(this.instance.name)}`, + { method: 'POST' }, + ) + : new Request(`https://flue.invalid${CLOUDFLARE_AGENT_INTERNAL_DISPATCH_PATH}`, { + method: 'POST', + }); + ctx = this.createContext( + submission.kind === 'direct' ? (input as DirectSubmissionInput).payload : input, + request, + undefined, + dispatchIdFor(input), + ); + const operationCtx = ctx; + if (submission.kind === 'direct') { + operationCtx.setEventCallback((event) => { + if (event.type === 'run_start' || event.type === 'run_end') return; + const attachedEvent = { ...event, instanceId: this.instance.name } as AttachedAgentEvent & { + runId?: string; + }; + delete attachedEvent.runId; + return this.observers.publish(submission.submissionId, attachedEvent); + }); + } + const result = await this.runWithInstanceContext(() => + submission.kind === 'dispatch' + ? createDispatchAgentHandler(agent, input as DispatchInput, { + onInputApplied: () => this.markInputApplied(submission, attemptId), + })(operationCtx) + : createDirectSubmissionAgentHandler(agent, input as DirectSubmissionInput, { + onInputApplied: () => this.markInputApplied(submission, attemptId), + })(operationCtx), + ); + const completed = this.submissions.completeSubmission(submission.submissionId, attemptId); + if (completed && submission.kind === 'direct') this.observers.complete(submission.submissionId, result); + } catch (error) { + const failed = this.submissions.failSubmission(submission.submissionId, attemptId, error); + if (failed && submission.kind === 'direct') this.observers.fail(submission.submissionId, error); + throw error; + } finally { + if (submission.kind === 'direct') ctx?.setEventCallback(undefined); + void this.reconcileSubmissions().catch((error) => { + console.error( + '[flue:submission-reconciliation]', + { + agentName: this.agentName, + instanceId: this.instance.name, + operation: 'settlement', + outcome: 'reconcile_failed', + }, + error, + ); + }); + } + } + + private markInputApplied(submission: SqlAgentSubmission, attemptId: string): void { + if (!this.submissions.markSubmissionInputApplied(submission.submissionId, attemptId)) { + throw new Error('[flue] Agent submission attempt lost ownership before input application.'); + } + } + + private async admitAttachedSubmission( + payload: DirectAgentPayload, + onEvent?: (event: AttachedAgentEvent) => Promise | void, + ): Promise { + const submissionId = crypto.randomUUID(); + const input: DirectSubmissionInput = { + submissionId, + agent: this.agentName, + id: this.instance.name, + session: typeof payload.session === 'string' && payload.session.trim() !== '' ? payload.session : 'default', + payload, + acceptedAt: new Date().toISOString(), + }; + const attachment = this.observers.attach(submissionId, { onEvent }); + try { + await this.armSubmissionWake(); + this.submissions.admitDirect(input); + await this.reconcileSubmissions({ driverAlreadyArmed: true }); + return await attachment.completion; + } finally { + attachment.detach(); + } + } + + private async admitDispatch(request: Request): Promise { + const input: unknown = await request.json(); + await validateAgentDispatchAdmission({ input: input as DispatchInput }); + if (!isDispatchInput(input)) { + throw new Error('[flue] Internal dispatch admission received an invalid payload.'); + } + if (input.agent !== this.agentName || input.id !== this.instance.name) { + return new Response('Invalid internal dispatch target.', { status: 400 }); + } + if (!this.options.createdAgents[this.agentName]) { + return new Response('Dispatch target unavailable.', { status: 404 }); + } + this.cleanupTerminalState(); + await this.armSubmissionWake(); + let submission: SqlAgentSubmission; + try { + submission = this.submissions.admitDispatch(input); + } catch (error) { + if (error instanceof SqlAgentDispatchReceiptRetainedError) { + return Response.json({ + dispatchId: error.receipt.submissionId, + acceptedAt: new Date(error.receipt.acceptedAt).toISOString(), + }); + } + if (error instanceof SqlAgentSubmissionConflictError) { + return new Response('Conflicting internal dispatch replay.', { status: 409 }); + } + throw error; + } + await this.reconcileSubmissions({ driverAlreadyArmed: true }); + return Response.json({ dispatchId: submission.submissionId, acceptedAt: submission.input.acceptedAt }); + } + + private acceptSocket(request: Request): Response { + const handler = this.options.websocketAgentHandlers[this.agentName]; + if (!handler) return new Response(null, { status: 404 }); + const { client, server } = this.options.createWebSocketPair(); + this.instance.ctx.acceptWebSocket(server); + connectCloudflareAgentWebSocket(server, { + name: this.agentName, + id: this.instance.name, + requestUrl: socketRequestUrl(request), + }); + return new Response(null, { status: 101, webSocket: client } as ResponseInit); + } +} + +function dispatchIdFor(input: DispatchInput | DirectSubmissionInput): string | undefined { + return 'dispatchId' in input ? input.dispatchId : undefined; +} + +function isAttemptMarkerSnapshot(value: unknown): value is { submissionId: string; attemptId: string } { + if (!value || typeof value !== 'object') return false; + const snapshot = value as Record; + return typeof snapshot.submissionId === 'string' && typeof snapshot.attemptId === 'string'; +} + +function isDispatchInput(value: unknown): value is DispatchInput { + if (!value || typeof value !== 'object') return false; + const input = value as Partial; + return ( + typeof input.dispatchId === 'string' && + typeof input.agent === 'string' && + typeof input.id === 'string' && + typeof input.session === 'string' && + typeof input.acceptedAt === 'string' + ); +} + +function submissionAttemptMarkerKey(submission: SqlAgentSubmission): string { + return `${submission.submissionId}:${submission.attemptId}`; +} + +function isInternalDispatchRequest(request: Request): boolean { + return request.method === 'POST' && new URL(request.url).pathname === CLOUDFLARE_AGENT_INTERNAL_DISPATCH_PATH; +} + +function isWebSocketUpgrade(request: Request): boolean { + return request.method === 'GET' && request.headers.get('upgrade')?.toLowerCase() === 'websocket'; +} + +function isFlueAgentSocket(connection: CloudflareWebSocketConnection, agentName: string): boolean { + const attachment = connection.deserializeAttachment?.(); + return attachment?.version === 1 && attachment.target === 'agent' && attachment.name === agentName; +} + +function closeSocket(connection: CloudflareWebSocketConnection, code: number, reason: string): void { + if (code === 1005 || code === 1006 || code === 1015) return; + try { + connection.close(code, reason); + } catch { + return; + } +} + +function socketRequest(connection: CloudflareWebSocketConnection): Request { + const attachment = connection.deserializeAttachment?.(); + return new Request(attachment?.requestUrl || 'https://flue.invalid/'); +} + +function socketRequestUrl(request: Request): string { + const url = new URL(request.url); + url.search = ''; + url.hash = ''; + return url.toString(); +} diff --git a/packages/runtime/src/internal.ts b/packages/runtime/src/internal.ts index bf6323de..9e4fc2d8 100644 --- a/packages/runtime/src/internal.ts +++ b/packages/runtime/src/internal.ts @@ -21,6 +21,7 @@ export { createFlueContext } from './client.ts'; // (registry client) live in the `@flue/runtime/cloudflare` subpath because // they pull in `cloudflare:workers`, a virtual module Node can't resolve. // The generated CF entry imports them from there directly. +export { CLOUDFLARE_AGENT_INTERNAL_DISPATCH_PATH, createCloudflareAgentRuntime } from './cloudflare/agent-coordinator.ts'; export { createSqlAgentExecutionStore, createSqlSessionStore, diff --git a/packages/runtime/test/build-plugin-cloudflare.test.ts b/packages/runtime/test/build-plugin-cloudflare.test.ts index 8074a0fc..f664a60f 100644 --- a/packages/runtime/test/build-plugin-cloudflare.test.ts +++ b/packages/runtime/test/build-plugin-cloudflare.test.ts @@ -25,16 +25,16 @@ describe('CloudflarePlugin', () => { }), ); - expect(entry).toContain('createSqlAgentExecutionStore'); + expect(entry).toContain('createCloudflareAgentRuntime'); expect(entry).toContain('createSqlSessionStore'); expect(entry).toContain( `constructor(ctx, env) { - const executionStore = createSqlAgentExecutionStore(ctx.storage, "FlueAssistantAgent"); + const prepared = cloudflareAgents.prepare({ storage: ctx.storage, className: "FlueAssistantAgent", agentName: "assistant" }); super(ctx, env); - this[FLUE_AGENT_EXECUTION_STORE] = executionStore; + cloudflareAgents.attach(this, prepared); }`, ); - expect(entry).not.toContain('const agentExecutionStores = new WeakMap();'); + expect(entry).not.toContain('createSqlAgentExecutionStore'); expect(entry).toContain('const memoryWorkflowSessionStore = new InMemorySessionStore();'); expect(entry).toContain( 'const defaultStore = sql ? createSqlSessionStore(sql) : memoryWorkflowSessionStore;', @@ -46,82 +46,28 @@ describe('CloudflarePlugin', () => { expect(entry).not.toContain('CREATE TABLE IF NOT EXISTS flue_sessions'); }); - it('pre-arms SQL-backed dispatch admission and drains claimed rows without managed Fibers', async () => { + it('delegates durable agent execution to the typed Cloudflare coordinator', async () => { const entry = await new CloudflarePlugin().generateEntryPoint( testBuildContext({ agents: [{ name: 'assistant', filePath: '/fixture/agents/assistant.ts' }], }), ); - expect(entry).toContain( - `async onStart(props) { - await restoreFlueAgentSubmissionWake(this); - if (typeof super.onStart === 'function') await super.onStart(props); - await reconcileFlueAgentSubmissions(this, "assistant", { driverAlreadyArmed: true }); - } - - async __flueWakeAgentSubmissions() { - const submissions = getAgentExecutionStore(this).submissions; - if (!submissions.hasUnsettledSubmissions()) return; - await armFlueAgentSubmissionWake(this, { idempotent: false }); - await reconcileFlueAgentSubmissions(this, "assistant", { driverAlreadyArmed: true }); - }`, - ); - expect(entry).toContain("const FLUE_AGENT_SUBMISSION_WAKE_CALLBACK = '__flueWakeAgentSubmissions';"); - expect(entry).not.toContain('scheduleEvery'); - expect(entry).toContain("await armFlueAgentSubmissionAdmissionWake(doInstance);\n let submission;"); - expect(entry).toContain('cleanupFlueAgentSubmissionTerminalState(doInstance);'); - expect(entry).not.toContain('cleanupDispatchReceipt'); - expect(entry).not.toContain('getDispatchReceipt'); - expect(entry).toContain('submission = submissions.admitDispatch(input);'); - expect(entry).toContain('if (error instanceof SqlAgentDispatchReceiptRetainedError) return Response.json({ dispatchId: error.receipt.submissionId, acceptedAt: new Date(error.receipt.acceptedAt).toISOString() });'); - expect(entry).toContain('for (const submission of submissions.listRunningSubmissions()) {'); - expect(entry).toContain('if (activeFlueAgentSubmissionAttempts.has(submissionAttemptLocalKey(doInstance, submission))) continue;'); - expect(entry).toContain("if (submission.status !== 'terminalizing' && attemptMarkers.keys.has(submissionAttemptMarkerKey(submission)) && submission.recoveryRequestedAt === undefined) continue;"); - expect(entry).toContain('await reconcileInterruptedSqlAgentSubmission(submission, doInstance, agentName);'); - expect(entry).toContain('await restoreFlueAgentSubmissionWake(doInstance);\n const submissions = getAgentExecutionStore(doInstance).submissions;\n submissions.requestSubmissionRecovery(submissionId, attemptId);'); - expect(entry).toContain('return handleFlueAgentSubmissionAttemptRecovered(ctx, this);'); - expect(entry).toContain("SELECT snapshot, created_at FROM cf_agents_runs WHERE name = 'flue:submission-attempt'"); - expect(entry).toContain('if (Date.now() - row.created_at > FLUE_AGENT_SUBMISSION_ATTEMPT_STALE_MS) continue;'); - expect(entry).toContain('if (row.snapshot === null) continue;'); - expect(entry).toContain("if (typeof row.snapshot !== 'string') {"); - expect(entry).toContain('submissions.requeueSubmissionBeforeInputApplied(submission.submissionId, attemptId);'); - expect(entry).toContain('createDispatchInputInspectionHandler(agent, input)(ctx)'); - expect(entry).toContain('createDirectSubmissionInputInspectionHandler(agent, input)(ctx)'); - expect(entry).toContain('if (!submissions.markSubmissionInputApplied(submission.submissionId, attemptId)) {'); - expect(entry).toContain('const claimed = submissions.claimSubmission(submission.submissionId, crypto.randomUUID());'); - expect(entry).toContain("running = doInstance.runFiber('flue:submission-attempt', async (fiberCtx) => {"); - expect(entry).toContain("fiberCtx.stash({ submissionId: submission.submissionId, attemptId: submission.attemptId });"); - expect(entry).toContain("activeFlueAgentSubmissionAttempts.delete(attemptKey);\n throw error;"); - expect(entry).toContain('const completed = submissions.completeSubmission(submission.submissionId, attemptId);'); - expect(entry).toContain("if (completed && submission.kind === 'direct') agentSubmissionObservers.complete(submission.submissionId, result);"); - expect(entry).toContain("if (submission.kind === 'direct') ctx?.setEventCallback(undefined);"); - expect(entry).toContain('getAgentExecutionStore(doInstance).submissions.admitDirect(input);'); - expect(entry).toContain('admitAttachedSubmission: (payload, req, onEvent) => admitAttachedAgentSubmission(doInstance, agentName, payload, req, onEvent)'); - expect(entry).not.toContain('adoptLegacyDispatches'); + expect(entry).toContain('const cloudflareAgents = createCloudflareAgentRuntime({'); + expect(entry).toContain('const prepared = cloudflareAgents.prepare({ storage: ctx.storage, className: "FlueAssistantAgent", agentName: "assistant" });'); + expect(entry).toContain('cloudflareAgents.attach(this, prepared);'); + expect(entry).toContain("return cloudflareAgents.onStart(this, () => typeof super.onStart === 'function' ? super.onStart(props) : undefined);"); + expect(entry).toContain('return cloudflareAgents.wakeSubmissions(this);'); + expect(entry).toContain('return cloudflareAgents.onRequest(this, request);'); + expect(entry).toContain('return cloudflareAgents.fetch(this, request, () => super.fetch(request));'); + expect(entry).toContain('return cloudflareAgents.webSocketMessage(this, socket, message, () => super.webSocketMessage(socket, message));'); + expect(entry).toContain('return cloudflareAgents.onFiberRecovered(this, ctx, () => typeof super.onFiberRecovered === \'function\' ? super.onFiberRecovered(ctx) : undefined);'); + expect(entry).not.toContain('reconcileFlueAgentSubmissions'); + expect(entry).not.toContain('cf_agents_runs'); expect(entry).not.toContain('cf_agents_fibers'); - expect(entry).toContain("idempotent: options.idempotent ?? true"); - expect(entry).toContain("idempotent: false"); - expect(entry).not.toContain('generation:'); - expect(entry).not.toContain('beginFlueAgentSubmissionAdmission'); - expect(entry).not.toContain('cancelSchedule(schedule.id)'); - expect(entry).toContain('getAgentExecutionStore(doInstance).submissions.cleanupTerminalSubmissions(\n Date.now() - FLUE_AGENT_SUBMISSION_TERMINAL_RETENTION_MS,'); - expect(entry).toContain('begin: (sessionKey) => executionStore.submissions.beginSessionDeletion(sessionKey)'); - expect(entry).toContain('if (submission.status !== \'terminalizing\' && !submissions.beginSubmissionTerminalization(submission.submissionId, attemptId)) return;'); - expect(entry).toContain('createSubmissionTerminalHandler(agent, input, {'); - expect(entry).toContain('submissions.finalizeSubmissionTerminalization(submission.submissionId, attemptId, error)'); - expect(entry).toContain("if (failed && submission.kind === 'direct') agentSubmissionObservers.fail(submission.submissionId, error);"); - expect(entry).not.toContain('const { manifest, directHandlers, localAgentHandlers, createdAgents'); - expect(entry).not.toContain('assertNoPendingDispatchForDirectSession'); + expect(entry).not.toContain('scheduleEvery'); expect(entry).not.toContain("runFiber('flue:direct'"); - expect(entry).not.toContain('listActiveDirectAgentSessionMarkers'); - expect(entry).not.toContain('agentSubmissionObservers.takeRequest'); - expect(entry).not.toContain('handleFlueDispatchAttemptRecovered'); - expect(entry).not.toContain("ctx.name === 'flue:dispatch'"); - expect(entry).not.toContain("ctx.name === 'flue:direct'"); - expect(entry).not.toContain('createLegacyDirectSubmissionTerminalHandler'); expect(entry).not.toContain("startFiber('flue:dispatch'"); - expect(entry).not.toContain('inspectFiberByKey'); expect(entry).not.toContain('ctx.storage.setAlarm'); }); diff --git a/packages/runtime/test/cloudflare-agent-coordinator.test.ts b/packages/runtime/test/cloudflare-agent-coordinator.test.ts new file mode 100644 index 00000000..1acba929 --- /dev/null +++ b/packages/runtime/test/cloudflare-agent-coordinator.test.ts @@ -0,0 +1,382 @@ +import { DatabaseSync } from 'node:sqlite'; +import { describe, expect, it } from 'vitest'; +import type { FlueContextInternal } from '../src/client.ts'; +import { createCloudflareAgentRuntime } from '../src/cloudflare/agent-coordinator.ts'; +import type { SqlAgentExecutionStore } from '../src/cloudflare/agent-execution-store.ts'; +import type { AgentSubmissionInputInspection, AgentSubmissionTerminalInput } from '../src/runtime/dispatch-queue.ts'; + +function makeFakeSql(events: string[] = []) { + const db = new DatabaseSync(':memory:'); + db.exec('CREATE TABLE cf_agents_runs (name TEXT NOT NULL, snapshot TEXT, created_at INTEGER NOT NULL)'); + return { + db, + storage: { + sql: { + exec(query: string, ...bindings: unknown[]) { + if (query.includes('SET recovery_requested_at')) events.push('request-recovery'); + if (query.includes("SET status = 'queued'")) events.push('requeue'); + if (query.includes("SET status = 'terminalizing'")) events.push('begin-terminalization'); + if (query.includes("SET status = 'error', completed_at")) events.push('finish-terminalization'); + const stmt = db.prepare(query); + let rows: unknown[]; + try { + rows = stmt.all(...(bindings as never[])); + } catch { + stmt.run(...(bindings as never[])); + rows = []; + } + return { + toArray() { + return rows as Record[]; + }, + }; + }, + }, + transactionSync(closure: () => T): T { + db.exec('BEGIN'); + try { + const result = closure(); + db.exec('COMMIT'); + return result; + } catch (error) { + db.exec('ROLLBACK'); + throw error; + } + }, + }, + }; +} + +function makeRuntime(options: { + createdAgent?: Parameters[0]['createdAgents'][string]; + createContext?: Parameters[0]['createContext']; +} = {}) { + return createCloudflareAgentRuntime({ + createdAgents: options.createdAgent ? { assistant: options.createdAgent } : {}, + directHandlers: {}, + websocketAgentHandlers: {}, + createContext: options.createContext ?? (() => { + throw new Error('Unexpected context creation.'); + }), + runWithInstanceContext(_instance, _agentName, callback) { + return callback(); + }, + createWebSocketPair() { + throw new Error('Unexpected WebSocket pair creation.'); + }, + }); +} + +function makeInstance( + storage: ReturnType['storage'], + events: string[] = [], +) { + return { + name: 'agent-1', + env: {}, + ctx: { + id: { toString: () => 'do-1' }, + storage, + acceptWebSocket() {}, + }, + async __unsafe_ensureInitialized() {}, + async schedule(_delaySeconds: number, _callback: string, _payload: undefined, options: { idempotent: boolean }) { + events.push(options.idempotent ? 'schedule-idempotent' : 'schedule-successor'); + }, + async runFiber() {}, + }; +} + +function makeRecoveryContext(options: { + inspection?: AgentSubmissionInputInspection; + events?: string[]; +}) { + const terminalRecords: AgentSubmissionTerminalInput[] = []; + const session = { + processDispatchInput() { + throw new Error('Unexpected dispatch processing.'); + }, + processDirectSubmissionInput() { + throw new Error('Unexpected direct processing.'); + }, + inspectDispatchInput() { + return options.inspection ?? 'applied'; + }, + inspectDirectSubmissionInput() { + return options.inspection ?? 'applied'; + }, + async recordSubmissionTerminal(input: AgentSubmissionTerminalInput) { + options.events?.push('record-terminal'); + terminalRecords.push(input); + }, + }; + const ctx = { + async initializeCreatedAgent() { + return { + async session() { + return session; + }, + }; + }, + } as unknown as FlueContextInternal; + return { ctx, terminalRecords }; +} + +function directInput() { + return { + submissionId: 'direct-1', + agent: 'assistant', + id: 'agent-1', + session: 'default', + payload: { message: 'Hello' }, + acceptedAt: '2026-06-03T00:00:00.000Z', + }; +} + +function dispatchInput() { + return { + dispatchId: 'dispatch-1', + agent: 'assistant', + id: 'agent-1', + session: 'default', + input: { message: 'Hello' }, + acceptedAt: '2026-06-03T00:00:00.000Z', + }; +} + +function prepare( + runtime: ReturnType, + instance: ReturnType, +): SqlAgentExecutionStore { + const prepared = runtime.prepare({ + storage: instance.ctx.storage, + className: 'FlueAssistantAgent', + agentName: 'assistant', + }); + runtime.attach(instance, prepared); + return prepared.executionStore; +} + +describe('createCloudflareAgentRuntime()', () => { + it('initializes SQLite during preparation before instance attachment', () => { + const runtime = makeRuntime(); + + expect(() => + runtime.prepare({ storage: {}, className: 'FlueAssistantAgent', agentName: 'assistant' }), + ).toThrow('Cloudflare durable agent class "FlueAssistantAgent" requires Durable Object SQLite.'); + }); + + it('restores a pending wake before inherited startup when unsettled work exists', async () => { + const events: string[] = []; + const { storage } = makeFakeSql(); + const runtime = makeRuntime(); + const instance = makeInstance(storage, events); + const executionStore = prepare(runtime, instance); + executionStore.submissions.admitDirect({ + submissionId: 'direct-1', + agent: 'assistant', + id: 'agent-1', + session: 'default', + payload: { message: 'Hello' }, + acceptedAt: '2026-06-03T00:00:00.000Z', + }); + + await runtime.onStart(instance, () => { + events.push('inherited-start'); + }); + + expect(events.slice(0, 2)).toEqual(['schedule-idempotent', 'inherited-start']); + }); + + it('arms a fresh non-idempotent successor before scheduled reconciliation', async () => { + const events: string[] = []; + const { storage } = makeFakeSql(); + const runtime = makeRuntime(); + const instance = makeInstance(storage, events); + const executionStore = prepare(runtime, instance); + executionStore.submissions.admitDirect({ + submissionId: 'direct-1', + agent: 'assistant', + id: 'agent-1', + session: 'default', + payload: { message: 'Hello' }, + acceptedAt: '2026-06-03T00:00:00.000Z', + }); + + await runtime.wakeSubmissions(instance); + + expect(events[0]).toBe('schedule-successor'); + }); + + it('restores a wake before recording recovered raw Fiber ownership', async () => { + const events: string[] = []; + const { storage } = makeFakeSql(events); + const runtime = makeRuntime(); + const instance = makeInstance(storage, events); + const executionStore = prepare(runtime, instance); + executionStore.submissions.admitDirect({ + submissionId: 'direct-1', + agent: 'assistant', + id: 'agent-1', + session: 'default', + payload: { message: 'Hello' }, + acceptedAt: '2026-06-03T00:00:00.000Z', + }); + executionStore.submissions.claimSubmission('direct-1', 'attempt-1'); + + await runtime.onFiberRecovered( + instance, + { name: 'flue:submission-attempt', snapshot: { submissionId: 'direct-1', attemptId: 'attempt-1' } }, + () => {}, + ); + + expect(events).toEqual(['schedule-idempotent', 'request-recovery']); + }); + + it('ignores SQL NULL pre-stash markers so queued submissions remain claimable', async () => { + const { db, storage } = makeFakeSql(); + const runtime = makeRuntime(); + const instance = makeInstance(storage); + const executionStore = prepare(runtime, instance); + db.prepare('INSERT INTO cf_agents_runs (name, snapshot, created_at) VALUES (?, ?, ?)').run( + 'flue:submission-attempt', + null, + Date.now(), + ); + executionStore.submissions.admitDirect({ + submissionId: 'direct-1', + agent: 'assistant', + id: 'agent-1', + session: 'default', + payload: { message: 'Hello' }, + acceptedAt: '2026-06-03T00:00:00.000Z', + }); + + await runtime.onStart(instance, () => {}); + + expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ status: 'running' }); + }); + + it('blocks claims when an active raw Fiber marker has malformed non-NULL evidence', async () => { + const { db, storage } = makeFakeSql(); + const runtime = makeRuntime(); + const instance = makeInstance(storage); + const executionStore = prepare(runtime, instance); + db.prepare('INSERT INTO cf_agents_runs (name, snapshot, created_at) VALUES (?, ?, ?)').run( + 'flue:submission-attempt', + 'null', + Date.now(), + ); + executionStore.submissions.admitDirect(directInput()); + + await runtime.onStart(instance, () => {}); + + expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ status: 'queued' }); + }); + + it('requeues interrupted attempts when canonical input is absent', async () => { + const events: string[] = []; + const { storage } = makeFakeSql(events); + const recovery = makeRecoveryContext({ inspection: 'absent' }); + const runtime = makeRuntime({ + createdAgent: {} as never, + createContext: () => recovery.ctx, + }); + const instance = makeInstance(storage); + const executionStore = prepare(runtime, instance); + executionStore.submissions.admitDirect(directInput()); + executionStore.submissions.claimSubmission('direct-1', 'attempt-1'); + + await runtime.onStart(instance, () => {}); + + expect(events).toContain('requeue'); + expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ status: 'running' }); + }); + + it('records interruption before settling applied incomplete canonical input as error', async () => { + const events: string[] = []; + const { storage } = makeFakeSql(events); + const recovery = makeRecoveryContext({ inspection: 'applied', events }); + const payloads: unknown[] = []; + const runtime = makeRuntime({ + createdAgent: {} as never, + createContext: ({ payload }) => { + payloads.push(payload); + return recovery.ctx; + }, + }); + const instance = makeInstance(storage); + const executionStore = prepare(runtime, instance); + executionStore.submissions.admitDirect(directInput()); + executionStore.submissions.claimSubmission('direct-1', 'attempt-1'); + executionStore.submissions.markSubmissionInputApplied('direct-1', 'attempt-1'); + + await runtime.onStart(instance, () => {}); + + expect(events).toEqual(['begin-terminalization', 'record-terminal', 'finish-terminalization']); + expect(payloads).toEqual([directInput(), directInput().payload]); + expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ status: 'error' }); + }); + + it('resumes terminalizing rows by recording interruption before final SQL settlement', async () => { + const events: string[] = []; + const { storage } = makeFakeSql(events); + const recovery = makeRecoveryContext({ events }); + const runtime = makeRuntime({ + createdAgent: {} as never, + createContext: () => recovery.ctx, + }); + const instance = makeInstance(storage); + const executionStore = prepare(runtime, instance); + executionStore.submissions.admitDirect(directInput()); + executionStore.submissions.claimSubmission('direct-1', 'attempt-1'); + executionStore.submissions.beginSubmissionTerminalization('direct-1', 'attempt-1'); + events.splice(0); + + await runtime.onStart(instance, () => {}); + + expect(events).toEqual(['record-terminal', 'finish-terminalization']); + expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ status: 'error' }); + }); + + it('settles interrupted attempts when canonical completion is already persisted', async () => { + const { storage } = makeFakeSql(); + const recovery = makeRecoveryContext({ inspection: 'completed' }); + const runtime = makeRuntime({ + createdAgent: {} as never, + createContext: () => recovery.ctx, + }); + const instance = makeInstance(storage); + const executionStore = prepare(runtime, instance); + executionStore.submissions.admitDirect(directInput()); + executionStore.submissions.claimSubmission('direct-1', 'attempt-1'); + executionStore.submissions.markSubmissionInputApplied('direct-1', 'attempt-1'); + + await runtime.onStart(instance, () => {}); + + expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ status: 'completed' }); + }); + + it('uses the full dispatch input when constructing detached recovery context', async () => { + const { storage } = makeFakeSql(); + const recovery = makeRecoveryContext({ inspection: 'completed' }); + const payloads: unknown[] = []; + const runtime = makeRuntime({ + createdAgent: {} as never, + createContext: ({ payload }) => { + payloads.push(payload); + return recovery.ctx; + }, + }); + const instance = makeInstance(storage); + const executionStore = prepare(runtime, instance); + executionStore.submissions.admitDispatch(dispatchInput()); + executionStore.submissions.claimSubmission('dispatch-1', 'attempt-1'); + executionStore.submissions.markSubmissionInputApplied('dispatch-1', 'attempt-1'); + + await runtime.onStart(instance, () => {}); + + expect(payloads).toEqual([dispatchInput()]); + expect(executionStore.submissions.getSubmission('dispatch-1')).toMatchObject({ status: 'completed' }); + }); +}); From 75ff7be382b85e05a37eb32d98c8b9ac8989f92d Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" <622227+FredKSchott@users.noreply.github.com> Date: Thu, 4 Jun 2026 11:10:56 -0500 Subject: [PATCH 11/17] refactor(runtime): unify durable agent submissions --- .../src/cloudflare/agent-coordinator.ts | 69 +++---- .../src/cloudflare/agent-execution-store.ts | 74 ++++--- packages/runtime/src/cloudflare/websocket.ts | 2 +- packages/runtime/src/internal.ts | 14 +- .../runtime/src/runtime/agent-submissions.ts | 188 ++++++++++++++++++ .../runtime/src/runtime/dispatch-queue.ts | 95 +-------- packages/runtime/src/runtime/handle-agent.ts | 172 +--------------- packages/runtime/src/session.ts | 57 +++--- .../test/cloudflare-agent-coordinator.test.ts | 99 ++++----- .../cloudflare-agent-execution-store.test.ts | 6 +- packages/runtime/test/dispatch.test.ts | 52 ++--- 11 files changed, 371 insertions(+), 457 deletions(-) create mode 100644 packages/runtime/src/runtime/agent-submissions.ts diff --git a/packages/runtime/src/cloudflare/agent-coordinator.ts b/packages/runtime/src/cloudflare/agent-coordinator.ts index 1bde8eb1..a6104ced 100644 --- a/packages/runtime/src/cloudflare/agent-coordinator.ts +++ b/packages/runtime/src/cloudflare/agent-coordinator.ts @@ -1,19 +1,16 @@ import type { FlueContextInternal } from '../client.ts'; import { + agentSubmissionContextPayload, + agentSubmissionDispatchId, + agentSubmissionInspectionContextPayload, + createAgentSubmissionHandler, + createAgentSubmissionInspectionHandler, createAgentSubmissionObserverRegistry, - type DirectSubmissionInput, - type DispatchInput, -} from '../runtime/dispatch-queue.ts'; -import { - type AgentHandler, - createDirectSubmissionAgentHandler, - createDirectSubmissionInputInspectionHandler, - createDispatchAgentHandler, - createDispatchInputInspectionHandler, - createSubmissionTerminalHandler, - handleAgentRequest, - validateAgentDispatchAdmission, -} from '../runtime/handle-agent.ts'; + createAgentSubmissionTerminalHandler, + type DirectAgentSubmissionInput, +} from '../runtime/agent-submissions.ts'; +import type { DispatchInput } from '../runtime/dispatch-queue.ts'; +import { type AgentHandler, handleAgentRequest, validateAgentDispatchAdmission } from '../runtime/handle-agent.ts'; import type { AttachedAgentEvent, DirectAgentPayload } from '../types.ts'; import { createSqlAgentExecutionStore, @@ -80,7 +77,7 @@ interface CloudflareAgentPreparedCoordinator { } interface CloudflareAgentRuntimeOptions { - readonly createdAgents: Record[0]>; + readonly createdAgents: Record[0]>; readonly directHandlers: Record; readonly websocketAgentHandlers: Record; readonly createContext: (options: { @@ -417,11 +414,14 @@ class CloudflareAgentCoordinator { const request = new Request(`https://flue.invalid${CLOUDFLARE_AGENT_INTERNAL_DISPATCH_PATH}`, { method: 'POST', }); - const ctx = this.createContext(input, request, undefined, dispatchIdFor(input)); + const ctx = this.createContext( + agentSubmissionInspectionContextPayload(input), + request, + undefined, + agentSubmissionDispatchId(input), + ); const state = await this.runWithInstanceContext(() => - submission.kind === 'dispatch' - ? createDispatchInputInspectionHandler(agent, input as DispatchInput)(ctx) - : createDirectSubmissionInputInspectionHandler(agent, input as DirectSubmissionInput)(ctx), + createAgentSubmissionInspectionHandler(agent, input)(ctx), ); if (submission.inputAppliedAt === undefined) { if (state === 'absent') { @@ -468,13 +468,13 @@ class CloudflareAgentCoordinator { method: 'POST', }); const ctx = this.createContext( - submission.kind === 'direct' ? (input as DirectSubmissionInput).payload : input, + agentSubmissionContextPayload(input), request, undefined, - dispatchIdFor(input), + agentSubmissionDispatchId(input), ); await this.runWithInstanceContext(() => - createSubmissionTerminalHandler(agent, input, { + createAgentSubmissionTerminalHandler(agent, input, { submissionId: submission.submissionId, kind: submission.kind, reason, @@ -573,9 +573,9 @@ class CloudflareAgentCoordinator { try { const agent = this.options.createdAgents[this.agentName]; if (!agent) throw new Error('[flue] Agent target unavailable during durable processing.'); - if (submission.kind === 'dispatch') await validateAgentDispatchAdmission({ input: input as DispatchInput }); + if (input.kind === 'dispatch') await validateAgentDispatchAdmission({ input }); const request = - submission.kind === 'direct' + input.kind === 'direct' ? new Request( `https://flue.invalid/agents/${encodeURIComponent(this.agentName)}/${encodeURIComponent(this.instance.name)}`, { method: 'POST' }, @@ -584,10 +584,10 @@ class CloudflareAgentCoordinator { method: 'POST', }); ctx = this.createContext( - submission.kind === 'direct' ? (input as DirectSubmissionInput).payload : input, + agentSubmissionContextPayload(input), request, undefined, - dispatchIdFor(input), + agentSubmissionDispatchId(input), ); const operationCtx = ctx; if (submission.kind === 'direct') { @@ -601,13 +601,9 @@ class CloudflareAgentCoordinator { }); } const result = await this.runWithInstanceContext(() => - submission.kind === 'dispatch' - ? createDispatchAgentHandler(agent, input as DispatchInput, { - onInputApplied: () => this.markInputApplied(submission, attemptId), - })(operationCtx) - : createDirectSubmissionAgentHandler(agent, input as DirectSubmissionInput, { - onInputApplied: () => this.markInputApplied(submission, attemptId), - })(operationCtx), + createAgentSubmissionHandler(agent, input, { + onInputApplied: () => this.markInputApplied(submission, attemptId), + })(operationCtx), ); const completed = this.submissions.completeSubmission(submission.submissionId, attemptId); if (completed && submission.kind === 'direct') this.observers.complete(submission.submissionId, result); @@ -643,7 +639,8 @@ class CloudflareAgentCoordinator { onEvent?: (event: AttachedAgentEvent) => Promise | void, ): Promise { const submissionId = crypto.randomUUID(); - const input: DirectSubmissionInput = { + const input: DirectAgentSubmissionInput = { + kind: 'direct', submissionId, agent: this.agentName, id: this.instance.name, @@ -692,7 +689,7 @@ class CloudflareAgentCoordinator { throw error; } await this.reconcileSubmissions({ driverAlreadyArmed: true }); - return Response.json({ dispatchId: submission.submissionId, acceptedAt: submission.input.acceptedAt }); + return Response.json({ dispatchId: submission.submissionId, acceptedAt: input.acceptedAt }); } private acceptSocket(request: Request): Response { @@ -709,10 +706,6 @@ class CloudflareAgentCoordinator { } } -function dispatchIdFor(input: DispatchInput | DirectSubmissionInput): string | undefined { - return 'dispatchId' in input ? input.dispatchId : undefined; -} - function isAttemptMarkerSnapshot(value: unknown): value is { submissionId: string; attemptId: string } { if (!value || typeof value !== 'object') return false; const snapshot = value as Record; diff --git a/packages/runtime/src/cloudflare/agent-execution-store.ts b/packages/runtime/src/cloudflare/agent-execution-store.ts index 2c094b50..c2932b3a 100644 --- a/packages/runtime/src/cloudflare/agent-execution-store.ts +++ b/packages/runtime/src/cloudflare/agent-execution-store.ts @@ -1,9 +1,10 @@ -import type { - AgentSubmissionInput, - DirectSubmissionInput, - DispatchInput, -} from '../runtime/dispatch-queue.ts'; -import { isDispatchSubmissionInput } from '../runtime/dispatch-queue.ts'; +import { + type AgentSubmissionInput, + createDispatchAgentSubmissionInput, + type DirectAgentSubmissionInput, + type DispatchAgentSubmissionInput, +} from '../runtime/agent-submissions.ts'; +import type { DispatchInput } from '../runtime/dispatch-queue.ts'; import { createSessionStorageKey } from '../session-identity.ts'; import type { SessionData, SessionStore } from '../types.ts'; @@ -49,7 +50,7 @@ export interface SqlAgentSubmission { export interface SqlAgentSubmissionStore { getSubmission(submissionId: string): SqlAgentSubmission | null; admitDispatch(input: DispatchInput): SqlAgentSubmission; - admitDirect(input: DirectSubmissionInput): SqlAgentSubmission; + admitDirect(input: DirectAgentSubmissionInput): SqlAgentSubmission; hasUnsettledSubmissions(): boolean; listRunnableSubmissions(): SqlAgentSubmission[]; listRunningSubmissions(): SqlAgentSubmission[]; @@ -170,11 +171,11 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { } admitDispatch(input: DispatchInput): SqlAgentSubmission { - return this.admitSubmission('dispatch', input.dispatchId, input); + return this.admitSubmission(createDispatchAgentSubmissionInput(input)); } - admitDirect(input: DirectSubmissionInput): SqlAgentSubmission { - return this.admitSubmission('direct', input.submissionId, input); + admitDirect(input: DirectAgentSubmissionInput): SqlAgentSubmission { + return this.admitSubmission(input); } hasUnsettledSubmissions(): boolean { @@ -412,11 +413,8 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { return submission?.status === 'error' && submission.attemptId === attemptId; } - private admitSubmission( - kind: SqlAgentSubmission['kind'], - submissionId: string, - input: AgentSubmissionInput, - ): SqlAgentSubmission { + private admitSubmission(input: AgentSubmissionInput): SqlAgentSubmission { + const { kind, submissionId } = input; const payload = JSON.stringify(input); const acceptedAt = parseAcceptedAt(input.acceptedAt, `${kind} admission`); const sessionKey = createSessionStorageKey(input.id, 'default', input.session); @@ -570,33 +568,33 @@ function parseSubmission(row: SqlRow): SqlAgentSubmission { function isSubmissionPayload(input: unknown, row: SqlRow): input is AgentSubmissionInput { if (!input || typeof input !== 'object') return false; const value = input as Partial; - if ( - typeof value.agent !== 'string' || - typeof value.id !== 'string' || - typeof value.session !== 'string' || - typeof value.acceptedAt !== 'string' || - value.session !== row.session || - createSessionStorageKey(value.id, 'default', value.session) !== row.session_key || - Date.parse(value.acceptedAt) !== row.accepted_at - ) { - return false; - } - if (row.kind === 'dispatch') { + if (value.kind !== row.kind || value.submissionId !== row.submission_id) return false; + if (value.kind === 'dispatch') { + const dispatch = value as Partial; return ( - 'dispatchId' in value && - typeof value.dispatchId === 'string' && - value.dispatchId === row.submission_id && - 'input' in value && - value.input !== undefined + typeof dispatch.dispatchId === 'string' && + dispatch.dispatchId === value.submissionId && + typeof dispatch.agent === 'string' && + typeof dispatch.id === 'string' && + typeof dispatch.session === 'string' && + dispatch.session === row.session && + createSessionStorageKey(dispatch.id, 'default', dispatch.session) === row.session_key && + typeof dispatch.acceptedAt === 'string' && + Date.parse(dispatch.acceptedAt) === row.accepted_at && + 'input' in dispatch && + dispatch.input !== undefined ); } + const direct = value as Partial; return ( - 'submissionId' in value && - typeof value.submissionId === 'string' && - value.submissionId === row.submission_id && - 'payload' in value && - isDirectPayload(value.payload) && - !isDispatchSubmissionInput(value as AgentSubmissionInput) + typeof direct.agent === 'string' && + typeof direct.id === 'string' && + typeof direct.session === 'string' && + direct.session === row.session && + createSessionStorageKey(direct.id, 'default', direct.session) === row.session_key && + typeof direct.acceptedAt === 'string' && + Date.parse(direct.acceptedAt) === row.accepted_at && + isDirectPayload(direct.payload) ); } diff --git a/packages/runtime/src/cloudflare/websocket.ts b/packages/runtime/src/cloudflare/websocket.ts index d2da4b8c..f29697b6 100644 --- a/packages/runtime/src/cloudflare/websocket.ts +++ b/packages/runtime/src/cloudflare/websocket.ts @@ -1,5 +1,5 @@ import { InvalidRequestError } from '../errors.ts'; -import type { AttachedAgentSubmissionAdmission } from '../runtime/dispatch-queue.ts'; +import type { AttachedAgentSubmissionAdmission } from '../runtime/agent-submissions.ts'; import type { AgentHandler, CreateContextFn, diff --git a/packages/runtime/src/internal.ts b/packages/runtime/src/internal.ts index 9e4fc2d8..14f05510 100644 --- a/packages/runtime/src/internal.ts +++ b/packages/runtime/src/internal.ts @@ -31,13 +31,8 @@ export { export { createDurableRunStore } from './cloudflare/run-store.ts'; export { InMemoryRunRegistry } from './node/run-registry.ts'; export { InMemoryRunStore } from './node/run-store.ts'; -export type { - DirectSubmissionInput, - DispatchInput, - DispatchProcessor, - DispatchQueue, -} from './runtime/dispatch-queue.ts'; -export { createAgentSubmissionObserverRegistry, InMemoryDispatchQueue } from './runtime/dispatch-queue.ts'; +export type { DispatchInput, DispatchProcessor, DispatchQueue } from './runtime/dispatch-queue.ts'; +export { InMemoryDispatchQueue } from './runtime/dispatch-queue.ts'; export type { ExposedTransport, FlueRuntime } from './runtime/flue-app.ts'; export { configureFlueRuntime, @@ -82,11 +77,6 @@ export type { export { createAgentDispatchProcessor, createDirectAgentHandler, - createDirectSubmissionAgentHandler, - createDirectSubmissionInputInspectionHandler, - createDispatchAgentHandler, - createDispatchInputInspectionHandler, - createSubmissionTerminalHandler, failRecoveredRun, handleAgentRequest, handleWorkflowRequest, diff --git a/packages/runtime/src/runtime/agent-submissions.ts b/packages/runtime/src/runtime/agent-submissions.ts new file mode 100644 index 00000000..ec3c3de1 --- /dev/null +++ b/packages/runtime/src/runtime/agent-submissions.ts @@ -0,0 +1,188 @@ +import type { FlueContextInternal } from '../client.ts'; +import type { + AttachedAgentEvent, + CreatedAgent, + DirectAgentPayload, +} from '../types.ts'; +import type { DispatchInput } from './dispatch-queue.ts'; + +export interface DispatchAgentSubmissionInput extends DispatchInput { + readonly kind: 'dispatch'; + readonly submissionId: string; +} + +export interface DirectAgentSubmissionInput { + readonly kind: 'direct'; + readonly submissionId: string; + readonly agent: string; + readonly id: string; + readonly session: string; + readonly payload: DirectAgentPayload; + readonly acceptedAt: string; +} + +export type AgentSubmissionInput = DispatchAgentSubmissionInput | DirectAgentSubmissionInput; + +export interface AgentSubmissionInterruption { + readonly submissionId: string; + readonly kind: AgentSubmissionInput['kind']; + readonly reason: 'interrupted_before_input_marker' | 'interrupted_after_input_application'; + readonly message: string; +} + +export type AgentSubmissionInspection = 'absent' | 'applied' | 'completed' | 'advanced'; + +export interface ProcessAgentSubmissionOptions { + onInputApplied?: () => Promise | void; +} + +interface AgentSubmissionSession { + inspectSubmissionInput?(input: AgentSubmissionInput): AgentSubmissionInspection; + processSubmissionInput?( + input: AgentSubmissionInput, + options?: ProcessAgentSubmissionOptions, + ): PromiseLike; + recordSubmissionTerminal?(input: AgentSubmissionInterruption): Promise; +} + +type AgentSubmissionHandler = (ctx: FlueContextInternal) => unknown | Promise; + +interface AgentSubmissionObserver { + onEvent?: (event: AttachedAgentEvent) => Promise | void; +} + +interface AgentSubmissionAttachment { + readonly completion: Promise; + detach(): void; +} + +interface AgentSubmissionObserverRegistry { + attach(submissionId: string, observer: AgentSubmissionObserver): AgentSubmissionAttachment; + publish(submissionId: string, event: AttachedAgentEvent): Promise; + complete(submissionId: string, result: unknown): void; + fail(submissionId: string, error: unknown): void; +} + +export type AttachedAgentSubmissionAdmission = ( + payload: DirectAgentPayload, + request: Request, + onEvent?: (event: AttachedAgentEvent) => Promise | void, +) => Promise; + +export function createDispatchAgentSubmissionInput(input: DispatchInput): DispatchAgentSubmissionInput { + return { ...input, kind: 'dispatch', submissionId: input.dispatchId }; +} + +export function createAgentSubmissionHandler( + agent: CreatedAgent, + input: AgentSubmissionInput, + options?: ProcessAgentSubmissionOptions, +): AgentSubmissionHandler { + return async (ctx) => { + const session = await openAgentSubmissionSession(ctx, agent, input); + if (typeof session.processSubmissionInput !== 'function') { + throw new Error('[flue] Internal session does not support submission input processing.'); + } + return session.processSubmissionInput(input, options); + }; +} + +export function createAgentSubmissionInspectionHandler( + agent: CreatedAgent, + input: AgentSubmissionInput, +): AgentSubmissionHandler { + return async (ctx) => { + const session = await openAgentSubmissionSession(ctx, agent, input); + if (typeof session.inspectSubmissionInput !== 'function') { + throw new Error('[flue] Internal session does not support submission input inspection.'); + } + return session.inspectSubmissionInput(input); + }; +} + +export function createAgentSubmissionTerminalHandler( + agent: CreatedAgent, + input: AgentSubmissionInput, + terminal: AgentSubmissionInterruption, +): AgentSubmissionHandler { + return async (ctx) => { + const session = await openAgentSubmissionSession(ctx, agent, input); + if (typeof session.recordSubmissionTerminal !== 'function') { + throw new Error('[flue] Internal session does not support submission terminal persistence.'); + } + await session.recordSubmissionTerminal(terminal); + }; +} + +export function agentSubmissionContextPayload(input: AgentSubmissionInput): unknown { + return input.kind === 'dispatch' ? agentSubmissionDispatchInput(input) : input.payload; +} + +export function agentSubmissionInspectionContextPayload(input: AgentSubmissionInput): unknown { + return input.kind === 'dispatch' ? agentSubmissionDispatchInput(input) : input; +} + +export function agentSubmissionDispatchId(input: AgentSubmissionInput): string | undefined { + return input.kind === 'dispatch' ? input.dispatchId : undefined; +} + +export function agentSubmissionDispatchInput(input: DispatchAgentSubmissionInput): DispatchInput { + const { kind: _kind, submissionId: _submissionId, ...dispatch } = input; + return dispatch; +} + +export function createAgentSubmissionObserverRegistry(): AgentSubmissionObserverRegistry { + const observers = new Map< + string, + Set + >(); + return { + attach(submissionId, observer) { + let resolve!: (value: unknown) => void; + let reject!: (error: unknown) => void; + const completion = new Promise((resolve_, reject_) => { + resolve = resolve_; + reject = reject_; + }); + const attached = { ...observer, resolve, reject }; + const bucket = observers.get(submissionId) ?? new Set(); + bucket.add(attached); + observers.set(submissionId, bucket); + return { + completion, + detach() { + bucket.delete(attached); + if (bucket.size === 0) observers.delete(submissionId); + }, + }; + }, + async publish(submissionId, event) { + for (const observer of observers.get(submissionId) ?? []) { + try { + await observer.onEvent?.(event); + } catch {} + } + }, + complete(submissionId, result) { + for (const observer of observers.get(submissionId) ?? []) observer.resolve(result); + observers.delete(submissionId); + }, + fail(submissionId, error) { + for (const observer of observers.get(submissionId) ?? []) observer.reject(error); + observers.delete(submissionId); + }, + }; +} + +async function openAgentSubmissionSession( + ctx: FlueContextInternal, + agent: CreatedAgent, + input: AgentSubmissionInput, +): Promise { + const harness = await ctx.initializeCreatedAgent(agent, undefined); + const session = await harness.session(input.session); + if (!session || typeof session !== 'object') { + throw new Error('[flue] Internal session is unavailable for submission processing.'); + } + return session as AgentSubmissionSession; +} diff --git a/packages/runtime/src/runtime/dispatch-queue.ts b/packages/runtime/src/runtime/dispatch-queue.ts index 918f0d6f..c8ae73c9 100644 --- a/packages/runtime/src/runtime/dispatch-queue.ts +++ b/packages/runtime/src/runtime/dispatch-queue.ts @@ -1,4 +1,4 @@ -import type { AttachedAgentEvent, DirectAgentPayload, DispatchReceipt } from '../types.ts'; +import type { DispatchReceipt } from '../types.ts'; export interface DispatchInput { dispatchId: string; @@ -9,99 +9,6 @@ export interface DispatchInput { acceptedAt: string; } -export interface DirectSubmissionInput { - submissionId: string; - agent: string; - id: string; - session: string; - payload: DirectAgentPayload; - acceptedAt: string; -} - -export type AgentSubmissionInput = DispatchInput | DirectSubmissionInput; - -export interface AgentSubmissionTerminalInput { - submissionId: string; - kind: 'dispatch' | 'direct'; - reason: 'interrupted_before_input_marker' | 'interrupted_after_input_application'; - message: string; -} - -export type AgentSubmissionInputInspection = 'absent' | 'applied' | 'completed' | 'advanced'; -export type DispatchInputInspection = AgentSubmissionInputInspection; - -export interface ProcessAgentSubmissionInputOptions { - onInputApplied?: () => Promise | void; -} - -export type ProcessDispatchInputOptions = ProcessAgentSubmissionInputOptions; - -export function isDispatchSubmissionInput(input: AgentSubmissionInput): input is DispatchInput { - return 'dispatchId' in input; -} - -interface AgentSubmissionObserver { - onEvent?: (event: AttachedAgentEvent) => Promise | void; -} - -interface AgentSubmissionAttachment { - readonly completion: Promise; - detach(): void; -} - -interface AgentSubmissionObserverRegistry { - attach(submissionId: string, observer: AgentSubmissionObserver): AgentSubmissionAttachment; - publish(submissionId: string, event: AttachedAgentEvent): Promise; - complete(submissionId: string, result: unknown): void; - fail(submissionId: string, error: unknown): void; -} - -export type AttachedAgentSubmissionAdmission = ( - payload: DirectAgentPayload, - request: Request, - onEvent?: (event: AttachedAgentEvent) => Promise | void, -) => Promise; - -export function createAgentSubmissionObserverRegistry(): AgentSubmissionObserverRegistry { - const observers = new Map>(); - return { - attach(submissionId, observer) { - let resolve!: (value: unknown) => void; - let reject!: (error: unknown) => void; - const completion = new Promise((resolve_, reject_) => { - resolve = resolve_; - reject = reject_; - }); - const attached = { ...observer, resolve, reject }; - const bucket = observers.get(submissionId) ?? new Set(); - bucket.add(attached); - observers.set(submissionId, bucket); - return { - completion, - detach() { - bucket.delete(attached); - if (bucket.size === 0) observers.delete(submissionId); - }, - }; - }, - async publish(submissionId, event) { - for (const observer of observers.get(submissionId) ?? []) { - try { - await observer.onEvent?.(event); - } catch {} - } - }, - complete(submissionId, result) { - for (const observer of observers.get(submissionId) ?? []) observer.resolve(result); - observers.delete(submissionId); - }, - fail(submissionId, error) { - for (const observer of observers.get(submissionId) ?? []) observer.reject(error); - observers.delete(submissionId); - }, - }; -} - export interface DispatchProcessor { process(input: DispatchInput): Promise | void; } diff --git a/packages/runtime/src/runtime/handle-agent.ts b/packages/runtime/src/runtime/handle-agent.ts index c6915e72..e5e0455f 100644 --- a/packages/runtime/src/runtime/handle-agent.ts +++ b/packages/runtime/src/runtime/handle-agent.ts @@ -19,17 +19,9 @@ import type { FlueEvent, FlueEventCallback, } from '../types.ts'; -import type { - AgentSubmissionInputInspection, - AgentSubmissionTerminalInput, - AttachedAgentSubmissionAdmission, - DirectSubmissionInput, - DispatchInput, - DispatchInputInspection, - DispatchProcessor, - ProcessAgentSubmissionInputOptions, - ProcessDispatchInputOptions, -} from './dispatch-queue.ts'; +import type { AttachedAgentSubmissionAdmission } from './agent-submissions.ts'; +import { createAgentSubmissionHandler, createDispatchAgentSubmissionInput } from './agent-submissions.ts'; +import type { DispatchInput, DispatchProcessor } from './dispatch-queue.ts'; import { streamActiveRunEvents } from './handle-run-routes.ts'; import { generateWorkflowRunId } from './ids.ts'; import type { RunOwner, RunRegistry } from './run-registry.ts'; @@ -45,23 +37,6 @@ interface DirectRequestSession { processDirectInput(input: { message: string }): PromiseLike; } -interface DispatchSession { - inspectDispatchInput?(input: DispatchInput): DispatchInputInspection; - processDispatchInput(input: DispatchInput, options?: ProcessDispatchInputOptions): PromiseLike; -} - -interface DirectSubmissionSession { - inspectDirectSubmissionInput?(input: DirectSubmissionInput): AgentSubmissionInputInspection; - processDirectSubmissionInput( - input: DirectSubmissionInput, - options?: ProcessAgentSubmissionInputOptions, - ): PromiseLike; -} - -interface SubmissionTerminalSession { - recordSubmissionTerminal(input: AgentSubmissionTerminalInput): Promise; -} - interface AgentSessionTarget { agentName: string; instanceId: string; @@ -89,7 +64,7 @@ export function createAgentDispatchProcessor(options: { undefined, input.dispatchId, ); - await createDispatchAgentHandler(agent, input)(ctx); + await createAgentSubmissionHandler(agent, createDispatchAgentSubmissionInput(input))(ctx); } finally { releaseSessionLock(); } @@ -115,51 +90,6 @@ export async function validateAgentDispatchAdmission( return { dispatchId: input.dispatchId, acceptedAt: input.acceptedAt }; } -export function createDispatchAgentHandler( - agent: CreatedAgentHandler, - input: DispatchInput, - options?: ProcessDispatchInputOptions, -): AgentHandler { - return (ctx) => processAgentDispatch(ctx, agent, input, options); -} - -export function createDispatchInputInspectionHandler( - agent: CreatedAgentHandler, - input: DispatchInput, -): AgentHandler { - return (ctx) => inspectAgentDispatchInput(ctx, agent, input); -} - -export function createDirectSubmissionAgentHandler( - agent: CreatedAgentHandler, - input: DirectSubmissionInput, - options?: ProcessAgentSubmissionInputOptions, -): AgentHandler { - return (ctx) => processAgentDirectSubmission(ctx, agent, input, options); -} - -export function createDirectSubmissionInputInspectionHandler( - agent: CreatedAgentHandler, - input: DirectSubmissionInput, -): AgentHandler { - return (ctx) => inspectAgentDirectSubmissionInput(ctx, agent, input); -} - -export function createSubmissionTerminalHandler( - agent: CreatedAgentHandler, - input: DispatchInput | DirectSubmissionInput, - terminal: AgentSubmissionTerminalInput, -): AgentHandler { - return async (ctx) => { - const harness = await ctx.initializeCreatedAgent(agent, undefined); - const session = await harness.session(input.session); - if (!isSubmissionTerminalSession(session)) { - throw new Error('[flue] Internal session does not support submission terminal persistence.'); - } - await session.recordSubmissionTerminal(terminal); - }; -} - async function reserveDispatchAgentSession( target: AgentSessionTarget, payload: unknown, @@ -167,76 +97,6 @@ async function reserveDispatchAgentSession( return waitForAgentSessionLock(target, payload); } -async function processAgentDispatch( - ctx: FlueContextInternal, - agent: CreatedAgentHandler, - input: DispatchInput, - options?: ProcessDispatchInputOptions, -): Promise { - const session = await openAgentDispatchSession(ctx, agent, input); - return session.processDispatchInput(input, options); -} - -async function inspectAgentDispatchInput( - ctx: FlueContextInternal, - agent: CreatedAgentHandler, - input: DispatchInput, -): Promise { - const session = await openAgentDispatchSession(ctx, agent, input); - if (!session.inspectDispatchInput) { - throw new Error('[flue] Internal session does not support dispatch input inspection.'); - } - return session.inspectDispatchInput(input); -} - -async function processAgentDirectSubmission( - ctx: FlueContextInternal, - agent: CreatedAgentHandler, - input: DirectSubmissionInput, - options?: ProcessAgentSubmissionInputOptions, -): Promise { - const session = await openAgentDirectSubmissionSession(ctx, agent, input); - return session.processDirectSubmissionInput(input, options); -} - -async function inspectAgentDirectSubmissionInput( - ctx: FlueContextInternal, - agent: CreatedAgentHandler, - input: DirectSubmissionInput, -): Promise { - const session = await openAgentDirectSubmissionSession(ctx, agent, input); - if (!session.inspectDirectSubmissionInput) { - throw new Error('[flue] Internal session does not support direct input inspection.'); - } - return session.inspectDirectSubmissionInput(input); -} - -async function openAgentDispatchSession( - ctx: FlueContextInternal, - agent: CreatedAgentHandler, - input: DispatchInput, -): Promise { - const harness = await ctx.initializeCreatedAgent(agent, undefined); - const session = await harness.session(input.session); - if (!isDispatchSession(session)) { - throw new Error('[flue] Internal session does not support dispatch input processing.'); - } - return session; -} - -async function openAgentDirectSubmissionSession( - ctx: FlueContextInternal, - agent: CreatedAgentHandler, - input: DirectSubmissionInput, -): Promise { - const harness = await ctx.initializeCreatedAgent(agent, undefined); - const session = await harness.session(input.session); - if (!isDirectSubmissionSession(session)) { - throw new Error('[flue] Internal session does not support direct input processing.'); - } - return session; -} - function isDispatchInput(value: unknown): value is DispatchInput { if (!value || typeof value !== 'object') return false; const input = value as Partial; @@ -259,30 +119,6 @@ function dispatchRequest(): Request { return new Request('http://flue.local/_dispatch', { method: 'POST' }); } -function isDispatchSession(value: unknown): value is DispatchSession { - return ( - !!value && - typeof value === 'object' && - typeof (value as DispatchSession).processDispatchInput === 'function' - ); -} - -function isDirectSubmissionSession(value: unknown): value is DirectSubmissionSession { - return ( - !!value && - typeof value === 'object' && - typeof (value as DirectSubmissionSession).processDirectSubmissionInput === 'function' - ); -} - -function isSubmissionTerminalSession(value: unknown): value is SubmissionTerminalSession { - return ( - !!value && - typeof value === 'object' && - typeof (value as SubmissionTerminalSession).recordSubmissionTerminal === 'function' - ); -} - export function createDirectAgentHandler(agent: CreatedAgentHandler): AgentHandler { return async (ctx) => { const payload = parseDirectAgentPayload(ctx.payload); diff --git a/packages/runtime/src/session.ts b/packages/runtime/src/session.ts index 843b0a01..56c2a9ea 100644 --- a/packages/runtime/src/session.ts +++ b/packages/runtime/src/session.ts @@ -51,14 +51,14 @@ import { ResultUnavailableError, } from './result.ts'; import type { - AgentSubmissionInputInspection, - AgentSubmissionTerminalInput, - DirectSubmissionInput, - DispatchInput, - DispatchInputInspection, - ProcessAgentSubmissionInputOptions, - ProcessDispatchInputOptions, -} from './runtime/dispatch-queue.ts'; + AgentSubmissionInput, + AgentSubmissionInspection, + AgentSubmissionInterruption, + DirectAgentSubmissionInput, + ProcessAgentSubmissionOptions, +} from './runtime/agent-submissions.ts'; +import { agentSubmissionDispatchInput } from './runtime/agent-submissions.ts'; +import type { DispatchInput } from './runtime/dispatch-queue.ts'; import { generateOperationId, generateTurnId } from './runtime/ids.ts'; import { getProviderConfiguration, getRegisteredApiKey } from './runtime/providers.ts'; import { createFlueFs } from './sandbox.ts'; @@ -915,37 +915,28 @@ export class Session implements FlueSession { ); } - inspectDispatchInput(input: DispatchInput): DispatchInputInspection { - return this.inspectPersistedInput(this.history.findDispatchInput(input.dispatchId)); - } - - inspectDirectSubmissionInput(input: DirectSubmissionInput): AgentSubmissionInputInspection { - return this.inspectPersistedInput(this.history.findDirectSubmissionInput(input.submissionId)); - } - - processDispatchInput( - input: DispatchInput, - options?: ProcessDispatchInputOptions, - ): CallHandle { - return createCallHandle(undefined, (signal) => - this.runOperation('prompt', signal, () => - this.runPersistedDispatchInput(input, signal, options), - ), + inspectSubmissionInput(input: AgentSubmissionInput): AgentSubmissionInspection { + return this.inspectPersistedInput( + input.kind === 'dispatch' + ? this.history.findDispatchInput(input.dispatchId) + : this.history.findDirectSubmissionInput(input.submissionId), ); } - processDirectSubmissionInput( - input: DirectSubmissionInput, - options?: ProcessAgentSubmissionInputOptions, + processSubmissionInput( + input: AgentSubmissionInput, + options?: ProcessAgentSubmissionOptions, ): CallHandle { return createCallHandle(undefined, (signal) => this.runOperation('prompt', signal, () => - this.runPersistedDirectSubmissionInput(input, signal, options), + input.kind === 'dispatch' + ? this.runPersistedDispatchInput(agentSubmissionDispatchInput(input), signal, options) + : this.runPersistedDirectSubmissionInput(input, signal, options), ), ); } - async recordSubmissionTerminal(input: AgentSubmissionTerminalInput): Promise { + async recordSubmissionTerminal(input: AgentSubmissionInterruption): Promise { if (this.history.findSubmissionTerminal(input.submissionId)) return; this.history.appendMessage( createUserContextMessage( @@ -2093,7 +2084,7 @@ export class Session implements FlueSession { return undefined; } - private inspectPersistedInput(inputEntry: MessageEntry | undefined): AgentSubmissionInputInspection { + private inspectPersistedInput(inputEntry: MessageEntry | undefined): AgentSubmissionInspection { if (!inputEntry) return 'absent'; const following = this.history.getActivePathSince(inputEntry.id); if (following.some((entry) => entry.type === 'message' && entry.message.role === 'user')) { @@ -2108,7 +2099,7 @@ export class Session implements FlueSession { private async runPersistedDispatchInput( input: DispatchInput, signal: AbortSignal, - options?: ProcessDispatchInputOptions, + options?: ProcessAgentSubmissionOptions, ): Promise { return this.runPersistedContextInput({ findInput: () => this.history.findDispatchInput(input.dispatchId), @@ -2129,9 +2120,9 @@ export class Session implements FlueSession { } private async runPersistedDirectSubmissionInput( - input: DirectSubmissionInput, + input: DirectAgentSubmissionInput, signal: AbortSignal, - options?: ProcessAgentSubmissionInputOptions, + options?: ProcessAgentSubmissionOptions, ): Promise { return this.runPersistedContextInput({ findInput: () => this.history.findDirectSubmissionInput(input.submissionId), diff --git a/packages/runtime/test/cloudflare-agent-coordinator.test.ts b/packages/runtime/test/cloudflare-agent-coordinator.test.ts index 1acba929..06210718 100644 --- a/packages/runtime/test/cloudflare-agent-coordinator.test.ts +++ b/packages/runtime/test/cloudflare-agent-coordinator.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from 'vitest'; import type { FlueContextInternal } from '../src/client.ts'; import { createCloudflareAgentRuntime } from '../src/cloudflare/agent-coordinator.ts'; import type { SqlAgentExecutionStore } from '../src/cloudflare/agent-execution-store.ts'; -import type { AgentSubmissionInputInspection, AgentSubmissionTerminalInput } from '../src/runtime/dispatch-queue.ts'; +import type { AgentSubmissionInspection, AgentSubmissionInterruption } from '../src/runtime/agent-submissions.ts'; function makeFakeSql(events: string[] = []) { const db = new DatabaseSync(':memory:'); @@ -83,29 +83,23 @@ function makeInstance( async schedule(_delaySeconds: number, _callback: string, _payload: undefined, options: { idempotent: boolean }) { events.push(options.idempotent ? 'schedule-idempotent' : 'schedule-successor'); }, - async runFiber() {}, + async runFiber(_name: string, _callback: (ctx: { stash(snapshot: unknown): void }) => Promise) {}, }; } function makeRecoveryContext(options: { - inspection?: AgentSubmissionInputInspection; + inspection?: AgentSubmissionInspection; events?: string[]; }) { - const terminalRecords: AgentSubmissionTerminalInput[] = []; + const terminalRecords: AgentSubmissionInterruption[] = []; const session = { - processDispatchInput() { - throw new Error('Unexpected dispatch processing.'); + processSubmissionInput() { + throw new Error('Unexpected submission processing.'); }, - processDirectSubmissionInput() { - throw new Error('Unexpected direct processing.'); - }, - inspectDispatchInput() { - return options.inspection ?? 'applied'; - }, - inspectDirectSubmissionInput() { + inspectSubmissionInput() { return options.inspection ?? 'applied'; }, - async recordSubmissionTerminal(input: AgentSubmissionTerminalInput) { + async recordSubmissionTerminal(input: AgentSubmissionInterruption) { options.events?.push('record-terminal'); terminalRecords.push(input); }, @@ -124,6 +118,7 @@ function makeRecoveryContext(options: { function directInput() { return { + kind: 'direct' as const, submissionId: 'direct-1', agent: 'assistant', id: 'agent-1', @@ -172,14 +167,7 @@ describe('createCloudflareAgentRuntime()', () => { const runtime = makeRuntime(); const instance = makeInstance(storage, events); const executionStore = prepare(runtime, instance); - executionStore.submissions.admitDirect({ - submissionId: 'direct-1', - agent: 'assistant', - id: 'agent-1', - session: 'default', - payload: { message: 'Hello' }, - acceptedAt: '2026-06-03T00:00:00.000Z', - }); + executionStore.submissions.admitDirect(directInput()); await runtime.onStart(instance, () => { events.push('inherited-start'); @@ -194,14 +182,7 @@ describe('createCloudflareAgentRuntime()', () => { const runtime = makeRuntime(); const instance = makeInstance(storage, events); const executionStore = prepare(runtime, instance); - executionStore.submissions.admitDirect({ - submissionId: 'direct-1', - agent: 'assistant', - id: 'agent-1', - session: 'default', - payload: { message: 'Hello' }, - acceptedAt: '2026-06-03T00:00:00.000Z', - }); + executionStore.submissions.admitDirect(directInput()); await runtime.wakeSubmissions(instance); @@ -214,14 +195,7 @@ describe('createCloudflareAgentRuntime()', () => { const runtime = makeRuntime(); const instance = makeInstance(storage, events); const executionStore = prepare(runtime, instance); - executionStore.submissions.admitDirect({ - submissionId: 'direct-1', - agent: 'assistant', - id: 'agent-1', - session: 'default', - payload: { message: 'Hello' }, - acceptedAt: '2026-06-03T00:00:00.000Z', - }); + executionStore.submissions.admitDirect(directInput()); executionStore.submissions.claimSubmission('direct-1', 'attempt-1'); await runtime.onFiberRecovered( @@ -243,14 +217,7 @@ describe('createCloudflareAgentRuntime()', () => { null, Date.now(), ); - executionStore.submissions.admitDirect({ - submissionId: 'direct-1', - agent: 'assistant', - id: 'agent-1', - session: 'default', - payload: { message: 'Hello' }, - acceptedAt: '2026-06-03T00:00:00.000Z', - }); + executionStore.submissions.admitDirect(directInput()); await runtime.onStart(instance, () => {}); @@ -357,6 +324,46 @@ describe('createCloudflareAgentRuntime()', () => { expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ status: 'completed' }); }); + it('uses the public dispatch input as processing context payload without internal envelope fields', async () => { + const { storage } = makeFakeSql(); + const payloads: unknown[] = []; + let resolveProcessed!: () => void; + const processed = new Promise((resolve) => { + resolveProcessed = resolve; + }); + const session = { + async processSubmissionInput() { + resolveProcessed(); + }, + async recordSubmissionTerminal() {}, + }; + const runtime = makeRuntime({ + createdAgent: {} as never, + createContext: ({ payload }) => { + payloads.push(payload); + return { + async initializeCreatedAgent() { + return { + async session() { + return session; + }, + }; + }, + setEventCallback() {}, + } as unknown as FlueContextInternal; + }, + }); + const instance = makeInstance(storage); + instance.runFiber = async (_name, callback) => callback({ stash() {} }); + const executionStore = prepare(runtime, instance); + executionStore.submissions.admitDispatch(dispatchInput()); + + await runtime.onStart(instance, () => {}); + await processed; + + expect(payloads).toEqual([dispatchInput()]); + }); + it('uses the full dispatch input when constructing detached recovery context', async () => { const { storage } = makeFakeSql(); const recovery = makeRecoveryContext({ inspection: 'completed' }); diff --git a/packages/runtime/test/cloudflare-agent-execution-store.test.ts b/packages/runtime/test/cloudflare-agent-execution-store.test.ts index e292685a..389d24cc 100644 --- a/packages/runtime/test/cloudflare-agent-execution-store.test.ts +++ b/packages/runtime/test/cloudflare-agent-execution-store.test.ts @@ -5,7 +5,8 @@ import { createSqlSessionStore, SqlAgentDispatchReceiptRetainedError, } from '../src/cloudflare/agent-execution-store.ts'; -import type { DirectSubmissionInput, DispatchInput } from '../src/runtime/dispatch-queue.ts'; +import type { DirectAgentSubmissionInput } from '../src/runtime/agent-submissions.ts'; +import type { DispatchInput } from '../src/runtime/dispatch-queue.ts'; import type { SessionData } from '../src/types.ts'; function makeFakeSql() { @@ -55,8 +56,9 @@ function dispatchInput(overrides: Partial = {}): DispatchInput { }; } -function directInput(overrides: Partial = {}): DirectSubmissionInput { +function directInput(overrides: Partial = {}): DirectAgentSubmissionInput { return { + kind: 'direct', submissionId: 'direct-1', agent: 'assistant', id: 'agent-1', diff --git a/packages/runtime/test/dispatch.test.ts b/packages/runtime/test/dispatch.test.ts index 15bd0844..959e4eb2 100644 --- a/packages/runtime/test/dispatch.test.ts +++ b/packages/runtime/test/dispatch.test.ts @@ -9,20 +9,21 @@ import { dispatch, observe } from '../src/index.ts'; import { configureFlueRuntime, createAgentDispatchProcessor, - createAgentSubmissionObserverRegistry, - createDirectSubmissionAgentHandler, - createDirectSubmissionInputInspectionHandler, - createDispatchAgentHandler, - createDispatchInputInspectionHandler, createFlueContext, - createSubmissionTerminalHandler, - type DirectSubmissionInput, type DispatchInput, InMemoryDispatchQueue, InMemorySessionStore, resetFlueRuntimeForTests, validateAgentDispatchAdmission, } from '../src/internal.ts'; +import { + createAgentSubmissionHandler, + createAgentSubmissionInspectionHandler, + createAgentSubmissionObserverRegistry, + createAgentSubmissionTerminalHandler, + createDispatchAgentSubmissionInput, + type DirectAgentSubmissionInput, +} from '../src/runtime/agent-submissions.ts'; import type { AgentConfig, FlueHarness, FlueSession } from '../src/types.ts'; import { createNoopSessionEnv } from './fixtures/session-env.ts'; @@ -394,12 +395,11 @@ describe('dispatched session processing', () => { session: async (name?: string) => ({ name: name ?? 'default', - processDispatchInput: async () => { + processSubmissionInput: async () => { ctx.emitEvent({ type: 'idle' }); }, - }) as unknown as FlueSession & { - processDispatchInput(input: DispatchInput): Promise; - }, + recordSubmissionTerminal: async () => {}, + }) as unknown as FlueSession, sessions: {} as never, shell: (() => Promise.resolve({ stdout: '', stderr: '', exitCode: 0 })) as never, fs: {} as never, @@ -448,10 +448,9 @@ describe('dispatched session processing', () => { session: async (name?: string) => ({ name: name ?? 'default', - processDispatchInput: async () => {}, - }) as unknown as FlueSession & { - processDispatchInput(input: DispatchInput): Promise; - }, + processSubmissionInput: async () => {}, + recordSubmissionTerminal: async () => {}, + }) as unknown as FlueSession, sessions: {} as never, shell: (() => Promise.resolve({ stdout: '', stderr: '', exitCode: 0 })) as never, fs: {} as never, @@ -579,7 +578,7 @@ describe('dispatched session processing', () => { defaultStore: new InMemorySessionStore(), }); - await createDispatchAgentHandler(agent, input, { + await createAgentSubmissionHandler(agent, createDispatchAgentSubmissionInput(input), { onInputApplied: () => { order.push('input-applied'); }, @@ -610,7 +609,8 @@ describe('dispatched session processing', () => { model: `${provider.getModel().provider}/${provider.getModel().id}`, persist: store, })); - const input: DirectSubmissionInput = { + const input: DirectAgentSubmissionInput = { + kind: 'direct', submissionId: 'direct:input-marker-order', agent: 'moderator', id: 'guild:direct-input-marker-order', @@ -634,7 +634,7 @@ describe('dispatched session processing', () => { defaultStore: new InMemorySessionStore(), }); - await createDirectSubmissionAgentHandler(agent, input, { + await createAgentSubmissionHandler(agent, input, { onInputApplied: () => { order.push('input-applied'); }, @@ -654,7 +654,8 @@ describe('dispatched session processing', () => { it('persists one provider-visible terminal advisory when an interrupted submission cannot replay safely', async () => { const provider = createProvider(); const store = new InMemorySessionStore(); - const input: DirectSubmissionInput = { + const input: DirectAgentSubmissionInput = { + kind: 'direct', submissionId: 'direct:terminal-advisory', agent: 'moderator', id: 'guild:terminal-advisory', @@ -683,8 +684,8 @@ describe('dispatched session processing', () => { message: 'Provider replay was not attempted because prior execution could not be proven safe.', }; - await createSubmissionTerminalHandler(agent, input, terminal)(createContext()); - await createSubmissionTerminalHandler(agent, input, terminal)(createContext()); + await createAgentSubmissionTerminalHandler(agent, input, terminal)(createContext()); + await createAgentSubmissionTerminalHandler(agent, input, terminal)(createContext()); const data = await store.load(`agent-session:${JSON.stringify([input.id, 'default', input.session])}`); expect(data?.entries).toHaveLength(1); @@ -709,7 +710,8 @@ describe('dispatched session processing', () => { it('classifies a completed canonical direct response without model replay', async () => { const provider = createProvider(); const store = new InMemorySessionStore(); - const input: DirectSubmissionInput = { + const input: DirectAgentSubmissionInput = { + kind: 'direct', submissionId: 'direct:inspect-completed', agent: 'moderator', id: 'guild:direct-inspect-completed', @@ -759,7 +761,7 @@ describe('dispatched session processing', () => { defaultStore: new InMemorySessionStore(), }); - await expect(createDirectSubmissionInputInspectionHandler(agent, input)(ctx)).resolves.toBe('completed'); + await expect(createAgentSubmissionInspectionHandler(agent, input)(ctx)).resolves.toBe('completed'); expect(provider.state.callCount).toBe(0); }); @@ -817,7 +819,7 @@ describe('dispatched session processing', () => { defaultStore: new InMemorySessionStore(), }); - await expect(createDispatchInputInspectionHandler(agent, input)(ctx)).resolves.toBe('completed'); + await expect(createAgentSubmissionInspectionHandler(agent, createDispatchAgentSubmissionInput(input))(ctx)).resolves.toBe('completed'); expect(provider.state.callCount).toBe(0); }); @@ -867,7 +869,7 @@ describe('dispatched session processing', () => { defaultStore: new InMemorySessionStore(), }); - await expect(createDispatchInputInspectionHandler(agent, input)(ctx)).resolves.toBe('applied'); + await expect(createAgentSubmissionInspectionHandler(agent, createDispatchAgentSubmissionInput(input))(ctx)).resolves.toBe('applied'); expect(provider.state.callCount).toBe(0); }); From 6e3a73133ac66449b48df7e76d4d81dd5e99ce88 Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" <622227+FredKSchott@users.noreply.github.com> Date: Thu, 4 Jun 2026 12:17:34 -0500 Subject: [PATCH 12/17] refactor(cloudflare): collapse session deletion coordination --- .../cli/src/lib/build-plugin-cloudflare.ts | 6 +- packages/runtime/src/client.ts | 8 +- .../src/cloudflare/agent-execution-store.ts | 29 +++++-- packages/runtime/src/harness.ts | 5 +- packages/runtime/src/session.ts | 7 +- .../test/build-plugin-cloudflare.test.ts | 5 ++ .../cloudflare-agent-execution-store.test.ts | 80 +++++++++++++++---- packages/runtime/test/harness-session.test.ts | 50 +++++++++++- 8 files changed, 152 insertions(+), 38 deletions(-) diff --git a/packages/cli/src/lib/build-plugin-cloudflare.ts b/packages/cli/src/lib/build-plugin-cloudflare.ts index 99c0dc7e..183b4cbe 100644 --- a/packages/cli/src/lib/build-plugin-cloudflare.ts +++ b/packages/cli/src/lib/build-plugin-cloudflare.ts @@ -375,10 +375,8 @@ function createAgentContextForRequest(executionStore, id, payload, doInstance, r createDefaultEnv, defaultStore: executionStore.sessions, resolveSandbox, - sessionDeletionCoordinator: { - begin: (sessionKey) => executionStore.submissions.beginSessionDeletion(sessionKey), - finish: (sessionKey) => executionStore.submissions.finishSessionDeletion(sessionKey), - }, + sessionDeletionCoordinator: (sessionKey, deleteSessionTree) => + executionStore.submissions.deleteSession(sessionKey, deleteSessionTree), }); } diff --git a/packages/runtime/src/client.ts b/packages/runtime/src/client.ts index 026f2d26..6ed8ca1a 100644 --- a/packages/runtime/src/client.ts +++ b/packages/runtime/src/client.ts @@ -48,10 +48,10 @@ export interface FlueContextConfig { sessionDeletionCoordinator?: SessionDeletionCoordinator; } -export interface SessionDeletionCoordinator { - begin(storageKey: string): void; - finish(storageKey: string): void; -} +export type SessionDeletionCoordinator = ( + storageKey: string, + deleteSessionTree: () => Promise, +) => Promise; /** Extends FlueContext with server-only methods. Agent handlers only see FlueContext. */ export interface FlueContextInternal extends FlueContext { diff --git a/packages/runtime/src/cloudflare/agent-execution-store.ts b/packages/runtime/src/cloudflare/agent-execution-store.ts index c2932b3a..f91ed07d 100644 --- a/packages/runtime/src/cloudflare/agent-execution-store.ts +++ b/packages/runtime/src/cloudflare/agent-execution-store.ts @@ -54,8 +54,7 @@ export interface SqlAgentSubmissionStore { hasUnsettledSubmissions(): boolean; listRunnableSubmissions(): SqlAgentSubmission[]; listRunningSubmissions(): SqlAgentSubmission[]; - beginSessionDeletion(sessionKey: string): void; - finishSessionDeletion(sessionKey: string): void; + deleteSession(sessionKey: string, deleteSessionTree: () => Promise): Promise; cleanupTerminalSubmissions(completedBefore: number, limit?: number): number; claimSubmission(submissionId: string, attemptId: string): SqlAgentSubmission | null; markSubmissionInputApplied(submissionId: string, attemptId: string): SqlAgentSubmission | null; @@ -146,6 +145,8 @@ class SqlSessionStore implements SessionStore { } class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { + private pendingSessionDeletions = new Map>(); + constructor( private sql: SqlStorage, private transactionSync: NonNullable, @@ -224,7 +225,19 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { ); } - beginSessionDeletion(sessionKey: string): void { + deleteSession(sessionKey: string, deleteSessionTree: () => Promise): Promise { + const pending = this.pendingSessionDeletions.get(sessionKey); + if (pending) return pending; + const deletion = this.runSessionDeletion(sessionKey, deleteSessionTree); + this.pendingSessionDeletions.set(sessionKey, deletion); + void deletion.then( + () => this.clearPendingSessionDeletion(sessionKey, deletion), + () => this.clearPendingSessionDeletion(sessionKey, deletion), + ); + return deletion; + } + + private async runSessionDeletion(sessionKey: string, deleteSessionTree: () => Promise): Promise { this.transactionSync(() => { const active = this.sql .exec( @@ -246,9 +259,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { Date.now(), ); }); - } - - finishSessionDeletion(sessionKey: string): void { + await deleteSessionTree(); this.transactionSync(() => { this.sql.exec( `INSERT OR IGNORE INTO flue_agent_dispatch_receipts (dispatch_id, accepted_at, settled_at) @@ -266,6 +277,12 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { }); } + private clearPendingSessionDeletion(sessionKey: string, deletion: Promise): void { + if (this.pendingSessionDeletions.get(sessionKey) === deletion) { + this.pendingSessionDeletions.delete(sessionKey); + } + } + cleanupTerminalSubmissions(completedBefore: number, limit = 100): number { if (!Number.isInteger(limit) || limit <= 0) { throw new Error('[flue] Terminal submission cleanup limit must be a positive integer.'); diff --git a/packages/runtime/src/harness.ts b/packages/runtime/src/harness.ts index 300f0bc9..3e758325 100644 --- a/packages/runtime/src/harness.ts +++ b/packages/runtime/src/harness.ts @@ -188,9 +188,8 @@ export class Harness implements FlueHarness { return; } const storageKey = createSessionStorageKey(this.instanceId, this.name, sessionName); - this.sessionDeletionCoordinator?.begin(storageKey); - await deleteSessionTree(this.store, storageKey); - this.sessionDeletionCoordinator?.finish(storageKey); + const deleteTree = () => deleteSessionTree(this.store, storageKey); + await (this.sessionDeletionCoordinator?.(storageKey, deleteTree) ?? deleteTree()); }); } diff --git a/packages/runtime/src/session.ts b/packages/runtime/src/session.ts index 56c2a9ea..cf7a9107 100644 --- a/packages/runtime/src/session.ts +++ b/packages/runtime/src/session.ts @@ -1128,10 +1128,11 @@ export class Session implements FlueSession { } this.deleted = true; this.deletionPromise = Promise.resolve() - .then(() => this.sessionDeletionCoordinator?.begin(this.storageKey)) - .then(() => deleteSessionTree(this.store, this.storageKey)) .then(() => { - this.sessionDeletionCoordinator?.finish(this.storageKey); + const deleteTree = () => deleteSessionTree(this.store, this.storageKey); + return this.sessionDeletionCoordinator?.(this.storageKey, deleteTree) ?? deleteTree(); + }) + .then(() => { this.onDelete?.(); }) .catch((error) => { diff --git a/packages/runtime/test/build-plugin-cloudflare.test.ts b/packages/runtime/test/build-plugin-cloudflare.test.ts index f664a60f..cac2212e 100644 --- a/packages/runtime/test/build-plugin-cloudflare.test.ts +++ b/packages/runtime/test/build-plugin-cloudflare.test.ts @@ -35,6 +35,11 @@ describe('CloudflarePlugin', () => { }`, ); expect(entry).not.toContain('createSqlAgentExecutionStore'); + expect(entry).toContain( + 'executionStore.submissions.deleteSession(sessionKey, deleteSessionTree)', + ); + expect(entry).not.toContain('beginSessionDeletion'); + expect(entry).not.toContain('finishSessionDeletion'); expect(entry).toContain('const memoryWorkflowSessionStore = new InMemorySessionStore();'); expect(entry).toContain( 'const defaultStore = sql ? createSqlSessionStore(sql) : memoryWorkflowSessionStore;', diff --git a/packages/runtime/test/cloudflare-agent-execution-store.test.ts b/packages/runtime/test/cloudflare-agent-execution-store.test.ts index 389d24cc..04c2930f 100644 --- a/packages/runtime/test/cloudflare-agent-execution-store.test.ts +++ b/packages/runtime/test/cloudflare-agent-execution-store.test.ts @@ -358,35 +358,85 @@ describe('createSqlAgentExecutionStore()', () => { }); }); - it('rejects session deletion while durable submissions are queued or running', () => { + it('rejects session deletion while durable submissions are queued or running', async () => { const { sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); store.submissions.admitDispatch(dispatchInput()); - expect(() => - store.submissions.beginSessionDeletion('agent-session:["agent-1","default","default"]'), - ).toThrow('Session cannot be deleted while durable agent submissions are queued or running.'); + await expect( + store.submissions.deleteSession('agent-session:["agent-1","default","default"]', async () => {}), + ).rejects.toThrow('Session cannot be deleted while durable agent submissions are queued or running.'); store.submissions.claimSubmission('dispatch-1', 'attempt-1'); - expect(() => - store.submissions.beginSessionDeletion('agent-session:["agent-1","default","default"]'), - ).toThrow('Session cannot be deleted while durable agent submissions are queued or running.'); + await expect( + store.submissions.deleteSession('agent-session:["agent-1","default","default"]', async () => {}), + ).rejects.toThrow('Session cannot be deleted while durable agent submissions are queued or running.'); + }); + + it('blocks new submissions until session deletion completes', async () => { + const { sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); + const sessionKey = 'agent-session:["agent-1","default","default"]'; + let releaseDeletion: () => void = () => {}; + const deletionReleased = new Promise((resolve) => { + releaseDeletion = resolve; + }); + + const deletion = store.submissions.deleteSession(sessionKey, () => deletionReleased); + expect(() => store.submissions.admitDispatch(dispatchInput())).toThrow( + 'Durable agent submission admission is unavailable while this session is being deleted.', + ); + releaseDeletion(); + await deletion; + expect(store.submissions.admitDispatch(dispatchInput())).toMatchObject({ status: 'queued' }); + }); + + it('shares session deletion work while snapshot deletion is in progress', async () => { + const { sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); + const sessionKey = 'agent-session:["agent-1","default","default"]'; + let releaseDeletion: () => void = () => {}; + const deletionReleased = new Promise((resolve) => { + releaseDeletion = resolve; + }); + let deletionCalls = 0; + + const first = store.submissions.deleteSession(sessionKey, async () => { + deletionCalls += 1; + await deletionReleased; + }); + const second = store.submissions.deleteSession(sessionKey, async () => { + deletionCalls += 1; + }); + + expect(second).toBe(first); + expect(deletionCalls).toBe(1); + expect(() => store.submissions.admitDispatch(dispatchInput())).toThrow( + 'Durable agent submission admission is unavailable while this session is being deleted.', + ); + releaseDeletion(); + await Promise.all([first, second]); + expect(store.submissions.admitDispatch(dispatchInput())).toMatchObject({ status: 'queued' }); }); - it('blocks new submissions until session deletion completes', () => { + it('keeps new submissions blocked when session snapshot deletion fails', async () => { const { sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); const sessionKey = 'agent-session:["agent-1","default","default"]'; - store.submissions.beginSessionDeletion(sessionKey); + await expect( + store.submissions.deleteSession(sessionKey, async () => { + throw new Error('snapshot deletion failed'); + }), + ).rejects.toThrow('snapshot deletion failed'); expect(() => store.submissions.admitDispatch(dispatchInput())).toThrow( 'Durable agent submission admission is unavailable while this session is being deleted.', ); - store.submissions.finishSessionDeletion(sessionKey); + await expect(store.submissions.deleteSession(sessionKey, async () => {})).resolves.toBeUndefined(); expect(store.submissions.admitDispatch(dispatchInput())).toMatchObject({ status: 'queued' }); }); - it('clears terminal rows when a settled session is deleted', () => { + it('clears terminal rows when a settled session is deleted', async () => { const { db, sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); const sessionKey = 'agent-session:["agent-1","default","default"]'; @@ -394,8 +444,7 @@ describe('createSqlAgentExecutionStore()', () => { store.submissions.claimSubmission('dispatch-1', 'attempt-1'); store.submissions.completeSubmission('dispatch-1', 'attempt-1'); - store.submissions.beginSessionDeletion(sessionKey); - store.submissions.finishSessionDeletion(sessionKey); + await store.submissions.deleteSession(sessionKey, async () => {}); expect(store.submissions.getSubmission('dispatch-1')).toBeNull(); expect( @@ -408,15 +457,14 @@ describe('createSqlAgentExecutionStore()', () => { }); }); - it('rejects replay admission transactionally when deletion retained the dispatch receipt', () => { + it('rejects replay admission transactionally when deletion retained the dispatch receipt', async () => { const { sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); const sessionKey = 'agent-session:["agent-1","default","default"]'; store.submissions.admitDispatch(dispatchInput()); store.submissions.claimSubmission('dispatch-1', 'attempt-1'); store.submissions.completeSubmission('dispatch-1', 'attempt-1'); - store.submissions.beginSessionDeletion(sessionKey); - store.submissions.finishSessionDeletion(sessionKey); + await store.submissions.deleteSession(sessionKey, async () => {}); try { store.submissions.admitDispatch(dispatchInput()); diff --git a/packages/runtime/test/harness-session.test.ts b/packages/runtime/test/harness-session.test.ts index ae0d462f..d1f30acf 100644 --- a/packages/runtime/test/harness-session.test.ts +++ b/packages/runtime/test/harness-session.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; import { createAgent } from '../src/index.ts'; -import { createFlueContext } from '../src/internal.ts'; +import { createFlueContext, type FlueContextConfig } from '../src/internal.ts'; import type { FlueEvent, SessionData, SessionEnv, SessionStore } from '../src/types.ts'; describe('FlueHarness', () => { @@ -291,6 +291,28 @@ describe('FlueHarness', () => { ]); }); + it('wraps selected-store deletion in one coordinated operation when delete() targets an unopened session', async () => { + const defaultStore = new TrackingSessionStore(); + const authoredStore = new TrackingSessionStore(); + const calls: string[] = []; + const harness = await createContext(createEnv(), defaultStore, { + sessionDeletionCoordinator: async (storageKey, deleteSessionTree) => { + calls.push(`begin:${storageKey}`); + await deleteSessionTree(); + calls.push(`finish:${storageKey}`); + }, + }).init(createAgent(() => ({ model: false, persist: authoredStore }))); + + await harness.sessions.delete('review'); + + expect(calls).toEqual([ + 'begin:agent-session:["agent-instance","default","review"]', + 'finish:agent-session:["agent-instance","default","review"]', + ]); + expect(defaultStore.deleteCalls).toEqual([]); + expect(authoredStore.deleteCalls).toEqual(['agent-session:["agent-instance","default","review"]']); + }); + it('applies session management requests in order when concurrent requests target the same name', async () => { const store = new TrackingSessionStore(); store.blockLoads(); @@ -497,6 +519,29 @@ describe('FlueSession', () => { ); }); + it('wraps selected-store deletion in one coordinated operation when an opened session is deleted', async () => { + const defaultStore = new TrackingSessionStore(); + const authoredStore = new TrackingSessionStore(); + const calls: string[] = []; + const harness = await createContext(createEnv(), defaultStore, { + sessionDeletionCoordinator: async (storageKey, deleteSessionTree) => { + calls.push(`begin:${storageKey}`); + await deleteSessionTree(); + calls.push(`finish:${storageKey}`); + }, + }).init(createAgent(() => ({ model: false, persist: authoredStore }))); + const session = await harness.session('review'); + + await session.delete(); + + expect(calls).toEqual([ + 'begin:agent-session:["agent-instance","default","review"]', + 'finish:agent-session:["agent-instance","default","review"]', + ]); + expect(defaultStore.deleteCalls).toEqual([]); + expect(authoredStore.deleteCalls).toEqual(['agent-session:["agent-instance","default","review"]']); + }); + it('shares deletion work when delete() is called concurrently', async () => { const store = new TrackingSessionStore(); const harness = await createContext(createEnv(), store).init( @@ -532,7 +577,7 @@ describe('FlueSession', () => { }); }); -function createContext(env: SessionEnv, store: SessionStore) { +function createContext(env: SessionEnv, store: SessionStore, overrides: Partial = {}) { return createFlueContext({ id: 'agent-instance', payload: undefined, @@ -545,6 +590,7 @@ function createContext(env: SessionEnv, store: SessionStore) { }, createDefaultEnv: async () => env, defaultStore: store, + ...overrides, }); } From 887739bd09c96ef84993019794b8b48dab6609e4 Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" <622227+FredKSchott@users.noreply.github.com> Date: Thu, 4 Jun 2026 12:31:35 -0500 Subject: [PATCH 13/17] refactor(cloudflare): tighten submission transitions --- .../src/cloudflare/agent-coordinator.ts | 75 +++---- .../src/cloudflare/agent-execution-store.ts | 191 +++++++++--------- packages/runtime/src/internal.ts | 1 - .../test/cloudflare-agent-coordinator.test.ts | 37 ++-- .../cloudflare-agent-execution-store.test.ts | 158 ++++++++------- 5 files changed, 237 insertions(+), 225 deletions(-) diff --git a/packages/runtime/src/cloudflare/agent-coordinator.ts b/packages/runtime/src/cloudflare/agent-coordinator.ts index a6104ced..298b70f7 100644 --- a/packages/runtime/src/cloudflare/agent-coordinator.ts +++ b/packages/runtime/src/cloudflare/agent-coordinator.ts @@ -14,11 +14,11 @@ import { type AgentHandler, handleAgentRequest, validateAgentDispatchAdmission } import type { AttachedAgentEvent, DirectAgentPayload } from '../types.ts'; import { createSqlAgentExecutionStore, - SqlAgentDispatchReceiptRetainedError, type SqlAgentExecutionStore, type SqlAgentSubmission, SqlAgentSubmissionConflictError, type SqlAgentSubmissionStore, + type SubmissionAttemptRef, } from './agent-execution-store.ts'; import { type CloudflareWebSocketConnection, @@ -289,7 +289,7 @@ class CloudflareAgentCoordinator { const attemptId = ctx.snapshot?.attemptId; if (typeof submissionId !== 'string' || typeof attemptId !== 'string') return; await this.restoreSubmissionWake(); - this.submissions.requestSubmissionRecovery(submissionId, attemptId); + this.submissions.requestSubmissionRecovery({ submissionId, attemptId }); } private get agentName(): string { @@ -365,7 +365,7 @@ class CloudflareAgentCoordinator { for (const submission of this.submissions.listRunningSubmissions()) { if (this.activeAttempts.has(this.submissionAttemptLocalKey(submission))) continue; if ( - submission.status !== 'terminalizing' && + submission.status !== 'recording_interruption' && attemptMarkers.keys.has(submissionAttemptMarkerKey(submission)) && submission.recoveryRequestedAt === undefined ) @@ -373,7 +373,10 @@ class CloudflareAgentCoordinator { await this.reconcileInterruptedSubmission(submission); } for (const submission of this.submissions.listRunnableSubmissions()) { - const claimed = this.submissions.claimSubmission(submission.submissionId, crypto.randomUUID()); + const claimed = this.submissions.claimSubmission({ + submissionId: submission.submissionId, + attemptId: crypto.randomUUID(), + }); if (claimed) this.startSubmissionAttempt(claimed); } } catch (error) { @@ -393,9 +396,10 @@ class CloudflareAgentCoordinator { } private async reconcileInterruptedSubmission(submission: SqlAgentSubmission): Promise { - const { attemptId, input } = submission; - if (!attemptId) return; - if (submission.status === 'terminalizing') { + const { input } = submission; + const attempt = submissionAttemptRef(submission); + if (!attempt) return; + if (submission.status === 'recording_interruption') { await this.failInterruptedSubmission( submission, submission.inputAppliedAt === undefined @@ -425,7 +429,7 @@ class CloudflareAgentCoordinator { ); if (submission.inputAppliedAt === undefined) { if (state === 'absent') { - this.submissions.requeueSubmissionBeforeInputApplied(submission.submissionId, attemptId); + this.submissions.requeueSubmissionBeforeInputApplied(attempt); return; } await this.failInterruptedSubmission( @@ -438,7 +442,7 @@ class CloudflareAgentCoordinator { return; } if (state === 'completed') { - this.submissions.completeSubmission(submission.submissionId, attemptId); + this.submissions.completeSubmission(attempt); return; } await this.failInterruptedSubmission( @@ -455,11 +459,12 @@ class CloudflareAgentCoordinator { reason: 'interrupted_before_input_marker' | 'interrupted_after_input_application', error: Error, ): Promise { - const { attemptId, input } = submission; - if (!attemptId) return; + const { input } = submission; + const attempt = submissionAttemptRef(submission); + if (!attempt) return; if ( - submission.status !== 'terminalizing' && - !this.submissions.beginSubmissionTerminalization(submission.submissionId, attemptId) + submission.status !== 'recording_interruption' && + !this.submissions.beginSubmissionInterruptionRecording(attempt) ) return; const agent = this.options.createdAgents[this.agentName]; @@ -481,11 +486,7 @@ class CloudflareAgentCoordinator { message: error.message, })(ctx), ); - const failed = this.submissions.finalizeSubmissionTerminalization( - submission.submissionId, - attemptId, - error, - ); + const failed = this.submissions.finishSubmissionInterruptionRecording(attempt, error); if (failed && submission.kind === 'direct') this.observers.fail(submission.submissionId, error); } @@ -565,10 +566,11 @@ class CloudflareAgentCoordinator { } private async processSubmission(submission: SqlAgentSubmission): Promise { - const { attemptId, input } = submission; - if (!attemptId) return; + const { input } = submission; + const attempt = submissionAttemptRef(submission); + if (!attempt) return; const persisted = this.submissions.getSubmission(submission.submissionId); - if (persisted?.status !== 'running' || persisted.attemptId !== attemptId) return; + if (persisted?.status !== 'running' || persisted.attemptId !== attempt.attemptId) return; let ctx: FlueContextInternal | undefined; try { const agent = this.options.createdAgents[this.agentName]; @@ -602,13 +604,13 @@ class CloudflareAgentCoordinator { } const result = await this.runWithInstanceContext(() => createAgentSubmissionHandler(agent, input, { - onInputApplied: () => this.markInputApplied(submission, attemptId), + onInputApplied: () => this.markInputApplied(attempt), })(operationCtx), ); - const completed = this.submissions.completeSubmission(submission.submissionId, attemptId); + const completed = this.submissions.completeSubmission(attempt); if (completed && submission.kind === 'direct') this.observers.complete(submission.submissionId, result); } catch (error) { - const failed = this.submissions.failSubmission(submission.submissionId, attemptId, error); + const failed = this.submissions.failSubmission(attempt, error); if (failed && submission.kind === 'direct') this.observers.fail(submission.submissionId, error); throw error; } finally { @@ -628,8 +630,8 @@ class CloudflareAgentCoordinator { } } - private markInputApplied(submission: SqlAgentSubmission, attemptId: string): void { - if (!this.submissions.markSubmissionInputApplied(submission.submissionId, attemptId)) { + private markInputApplied(attempt: SubmissionAttemptRef): void { + if (!this.submissions.markSubmissionInputApplied(attempt)) { throw new Error('[flue] Agent submission attempt lost ownership before input application.'); } } @@ -673,23 +675,22 @@ class CloudflareAgentCoordinator { } this.cleanupTerminalState(); await this.armSubmissionWake(); - let submission: SqlAgentSubmission; try { - submission = this.submissions.admitDispatch(input); - } catch (error) { - if (error instanceof SqlAgentDispatchReceiptRetainedError) { + const admission = this.submissions.admitDispatch(input); + if (admission.kind === 'retained_receipt') { return Response.json({ - dispatchId: error.receipt.submissionId, - acceptedAt: new Date(error.receipt.acceptedAt).toISOString(), + dispatchId: admission.receipt.submissionId, + acceptedAt: new Date(admission.receipt.acceptedAt).toISOString(), }); } + await this.reconcileSubmissions({ driverAlreadyArmed: true }); + return Response.json({ dispatchId: admission.submission.submissionId, acceptedAt: input.acceptedAt }); + } catch (error) { if (error instanceof SqlAgentSubmissionConflictError) { return new Response('Conflicting internal dispatch replay.', { status: 409 }); } throw error; } - await this.reconcileSubmissions({ driverAlreadyArmed: true }); - return Response.json({ dispatchId: submission.submissionId, acceptedAt: input.acceptedAt }); } private acceptSocket(request: Request): Response { @@ -724,6 +725,12 @@ function isDispatchInput(value: unknown): value is DispatchInput { ); } +function submissionAttemptRef(submission: SqlAgentSubmission): SubmissionAttemptRef | undefined { + return submission.attemptId + ? { submissionId: submission.submissionId, attemptId: submission.attemptId } + : undefined; +} + function submissionAttemptMarkerKey(submission: SqlAgentSubmission): string { return `${submission.submissionId}:${submission.attemptId}`; } diff --git a/packages/runtime/src/cloudflare/agent-execution-store.ts b/packages/runtime/src/cloudflare/agent-execution-store.ts index f91ed07d..302a107d 100644 --- a/packages/runtime/src/cloudflare/agent-execution-store.ts +++ b/packages/runtime/src/cloudflare/agent-execution-store.ts @@ -23,13 +23,22 @@ interface DurableObjectStorage { transactionSync?(closure: () => T): T; } -type SqlAgentSubmissionStatus = 'queued' | 'running' | 'terminalizing' | 'completed' | 'error'; +type SqlAgentSubmissionStatus = 'queued' | 'running' | 'recording_interruption' | 'completed' | 'failed'; -export interface SqlAgentDispatchReceipt { +interface SqlAgentDispatchReceipt { readonly submissionId: string; readonly acceptedAt: number; } +type SqlAgentDispatchAdmission = + | { readonly kind: 'submission'; readonly submission: SqlAgentSubmission } + | { readonly kind: 'retained_receipt'; readonly receipt: SqlAgentDispatchReceipt }; + +export interface SubmissionAttemptRef { + readonly submissionId: string; + readonly attemptId: string; +} + export interface SqlAgentSubmission { readonly sequence: number; readonly submissionId: string; @@ -49,24 +58,21 @@ export interface SqlAgentSubmission { export interface SqlAgentSubmissionStore { getSubmission(submissionId: string): SqlAgentSubmission | null; - admitDispatch(input: DispatchInput): SqlAgentSubmission; + admitDispatch(input: DispatchInput): SqlAgentDispatchAdmission; admitDirect(input: DirectAgentSubmissionInput): SqlAgentSubmission; hasUnsettledSubmissions(): boolean; listRunnableSubmissions(): SqlAgentSubmission[]; listRunningSubmissions(): SqlAgentSubmission[]; deleteSession(sessionKey: string, deleteSessionTree: () => Promise): Promise; cleanupTerminalSubmissions(completedBefore: number, limit?: number): number; - claimSubmission(submissionId: string, attemptId: string): SqlAgentSubmission | null; - markSubmissionInputApplied(submissionId: string, attemptId: string): SqlAgentSubmission | null; - requestSubmissionRecovery(submissionId: string, attemptId: string): SqlAgentSubmission | null; - requeueSubmissionBeforeInputApplied( - submissionId: string, - attemptId: string, - ): SqlAgentSubmission | null; - beginSubmissionTerminalization(submissionId: string, attemptId: string): SqlAgentSubmission | null; - completeSubmission(submissionId: string, attemptId: string): boolean; - failSubmission(submissionId: string, attemptId: string, error: unknown): boolean; - finalizeSubmissionTerminalization(submissionId: string, attemptId: string, error: unknown): boolean; + claimSubmission(attempt: SubmissionAttemptRef): SqlAgentSubmission | null; + markSubmissionInputApplied(attempt: SubmissionAttemptRef): boolean; + requestSubmissionRecovery(attempt: SubmissionAttemptRef): boolean; + requeueSubmissionBeforeInputApplied(attempt: SubmissionAttemptRef): boolean; + beginSubmissionInterruptionRecording(attempt: SubmissionAttemptRef): boolean; + completeSubmission(attempt: SubmissionAttemptRef): boolean; + failSubmission(attempt: SubmissionAttemptRef, error: unknown): boolean; + finishSubmissionInterruptionRecording(attempt: SubmissionAttemptRef, error: unknown): boolean; } export interface SqlAgentExecutionStore { @@ -76,12 +82,6 @@ export interface SqlAgentExecutionStore { export class SqlAgentSubmissionConflictError extends Error {} -export class SqlAgentDispatchReceiptRetainedError extends Error { - constructor(readonly receipt: SqlAgentDispatchReceipt) { - super('[flue] Internal dispatch replay is already settled.'); - } -} - export function createSqlSessionStore(sql: SqlStorage): SessionStore { ensureSessionTable(sql); return new SqlSessionStore(sql); @@ -171,12 +171,16 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { return { submissionId: row.dispatch_id, acceptedAt: row.accepted_at }; } - admitDispatch(input: DispatchInput): SqlAgentSubmission { + admitDispatch(input: DispatchInput): SqlAgentDispatchAdmission { return this.admitSubmission(createDispatchAgentSubmissionInput(input)); } admitDirect(input: DirectAgentSubmissionInput): SqlAgentSubmission { - return this.admitSubmission(input); + const admission = this.admitSubmission(input); + if (admission.kind !== 'submission') { + throw new Error('[flue] Internal direct admission returned an unexpected retained receipt.'); + } + return admission.submission; } hasUnsettledSubmissions(): boolean { @@ -185,7 +189,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { .exec( `SELECT 1 FROM flue_agent_submissions - WHERE status IN ('queued', 'running', 'terminalizing') + WHERE status IN ('queued', 'running', 'recording_interruption') LIMIT 1`, ) .toArray().length > 0 @@ -202,7 +206,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { SELECT 1 FROM flue_agent_submissions AS earlier WHERE earlier.session_key = current.session_key - AND earlier.status IN ('queued', 'running', 'terminalizing') + AND earlier.status IN ('queued', 'running', 'recording_interruption') AND earlier.sequence < current.sequence ) ORDER BY current.sequence ASC`, @@ -217,7 +221,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { .exec( `SELECT ${submissionColumns} FROM flue_agent_submissions - WHERE status IN ('running', 'terminalizing') + WHERE status IN ('running', 'recording_interruption') ORDER BY sequence ASC`, ) .toArray(), @@ -243,7 +247,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { .exec( `SELECT 1 FROM flue_agent_submissions - WHERE session_key = ? AND status IN ('queued', 'running', 'terminalizing') + WHERE session_key = ? AND status IN ('queued', 'running', 'recording_interruption') LIMIT 1`, sessionKey, ) @@ -265,12 +269,12 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { `INSERT OR IGNORE INTO flue_agent_dispatch_receipts (dispatch_id, accepted_at, settled_at) SELECT submission_id, accepted_at, completed_at FROM flue_agent_submissions - WHERE session_key = ? AND kind = 'dispatch' AND status IN ('completed', 'error')`, + WHERE session_key = ? AND kind = 'dispatch' AND status IN ('completed', 'failed')`, sessionKey, ); this.sql.exec( `DELETE FROM flue_agent_submissions - WHERE session_key = ? AND status IN ('completed', 'error')`, + WHERE session_key = ? AND status IN ('completed', 'failed')`, sessionKey, ); this.sql.exec('DELETE FROM flue_agent_session_deletions WHERE session_key = ?', sessionKey); @@ -291,7 +295,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { .exec( `SELECT sequence FROM flue_agent_submissions - WHERE status IN ('completed', 'error') AND completed_at < ? + WHERE status IN ('completed', 'failed') AND completed_at < ? ORDER BY completed_at ASC, sequence ASC LIMIT ?`, completedBefore, @@ -304,7 +308,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { } this.sql.exec( `DELETE FROM flue_agent_submissions - WHERE sequence = ? AND status IN ('completed', 'error') AND completed_at < ?`, + WHERE sequence = ? AND status IN ('completed', 'failed') AND completed_at < ?`, row.sequence, completedBefore, ); @@ -313,7 +317,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { return rows.length; } - claimSubmission(submissionId: string, attemptId: string): SqlAgentSubmission | null { + claimSubmission(attempt: SubmissionAttemptRef): SqlAgentSubmission | null { this.sql.exec( `UPDATE flue_agent_submissions AS current SET status = 'running', attempt_id = ?, started_at = ? @@ -322,115 +326,109 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { SELECT 1 FROM flue_agent_submissions AS earlier WHERE earlier.session_key = current.session_key - AND earlier.status IN ('queued', 'running', 'terminalizing') + AND earlier.status IN ('queued', 'running', 'recording_interruption') AND earlier.sequence < current.sequence )`, - attemptId, + attempt.attemptId, Date.now(), - submissionId, + attempt.submissionId, ); - const submission = this.getSubmission(submissionId); - return submission?.status === 'running' && submission.attemptId === attemptId + const submission = this.getSubmission(attempt.submissionId); + return submission?.status === 'running' && submission.attemptId === attempt.attemptId ? submission : null; } - markSubmissionInputApplied(submissionId: string, attemptId: string): SqlAgentSubmission | null { + markSubmissionInputApplied(attempt: SubmissionAttemptRef): boolean { this.sql.exec( `UPDATE flue_agent_submissions SET input_applied_at = COALESCE(input_applied_at, ?) WHERE submission_id = ? AND status = 'running' AND attempt_id = ?`, Date.now(), - submissionId, - attemptId, + attempt.submissionId, + attempt.attemptId, ); - return this.getOwnedRunningSubmission(submissionId, attemptId); + return this.hasOwnedSubmission(attempt, 'running'); } - requestSubmissionRecovery(submissionId: string, attemptId: string): SqlAgentSubmission | null { + requestSubmissionRecovery(attempt: SubmissionAttemptRef): boolean { this.sql.exec( `UPDATE flue_agent_submissions SET recovery_requested_at = COALESCE(recovery_requested_at, ?) WHERE submission_id = ? AND status = 'running' AND attempt_id = ?`, Date.now(), - submissionId, - attemptId, + attempt.submissionId, + attempt.attemptId, ); - return this.getOwnedRunningSubmission(submissionId, attemptId); + return this.hasOwnedSubmission(attempt, 'running'); } - requeueSubmissionBeforeInputApplied( - submissionId: string, - attemptId: string, - ): SqlAgentSubmission | null { - this.sql.exec( - `UPDATE flue_agent_submissions - SET status = 'queued', attempt_id = NULL, recovery_requested_at = NULL, started_at = NULL - WHERE submission_id = ? AND status = 'running' - AND attempt_id = ? AND input_applied_at IS NULL`, - submissionId, - attemptId, + requeueSubmissionBeforeInputApplied(attempt: SubmissionAttemptRef): boolean { + return ( + this.sql + .exec( + `UPDATE flue_agent_submissions + SET status = 'queued', attempt_id = NULL, recovery_requested_at = NULL, started_at = NULL + WHERE submission_id = ? AND status = 'running' + AND attempt_id = ? AND input_applied_at IS NULL + RETURNING submission_id`, + attempt.submissionId, + attempt.attemptId, + ) + .toArray().length > 0 ); - const submission = this.getSubmission(submissionId); - return submission?.status === 'queued' ? submission : null; } - beginSubmissionTerminalization(submissionId: string, attemptId: string): SqlAgentSubmission | null { + beginSubmissionInterruptionRecording(attempt: SubmissionAttemptRef): boolean { this.sql.exec( `UPDATE flue_agent_submissions - SET status = 'terminalizing' + SET status = 'recording_interruption' WHERE submission_id = ? AND status = 'running' AND attempt_id = ?`, - submissionId, - attemptId, + attempt.submissionId, + attempt.attemptId, ); - const submission = this.getSubmission(submissionId); - return submission?.status === 'terminalizing' && submission.attemptId === attemptId - ? submission - : null; + return this.hasOwnedSubmission(attempt, 'recording_interruption'); } - completeSubmission(submissionId: string, attemptId: string): boolean { + completeSubmission(attempt: SubmissionAttemptRef): boolean { this.sql.exec( `UPDATE flue_agent_submissions SET status = 'completed', completed_at = ?, error = NULL WHERE submission_id = ? AND status = 'running' AND attempt_id = ?`, Date.now(), - submissionId, - attemptId, + attempt.submissionId, + attempt.attemptId, ); - const submission = this.getSubmission(submissionId); - return submission?.status === 'completed' && submission.attemptId === attemptId; + return this.hasOwnedSubmission(attempt, 'completed'); } - failSubmission(submissionId: string, attemptId: string, error: unknown): boolean { + failSubmission(attempt: SubmissionAttemptRef, error: unknown): boolean { this.sql.exec( `UPDATE flue_agent_submissions - SET status = 'error', completed_at = ?, error = ? + SET status = 'failed', completed_at = ?, error = ? WHERE submission_id = ? AND status = 'running' AND attempt_id = ?`, Date.now(), error instanceof Error ? error.message : String(error), - submissionId, - attemptId, + attempt.submissionId, + attempt.attemptId, ); - const submission = this.getSubmission(submissionId); - return submission?.status === 'error' && submission.attemptId === attemptId; + return this.hasOwnedSubmission(attempt, 'failed'); } - finalizeSubmissionTerminalization(submissionId: string, attemptId: string, error: unknown): boolean { + finishSubmissionInterruptionRecording(attempt: SubmissionAttemptRef, error: unknown): boolean { this.sql.exec( `UPDATE flue_agent_submissions - SET status = 'error', completed_at = ?, error = ? - WHERE submission_id = ? AND status = 'terminalizing' AND attempt_id = ?`, + SET status = 'failed', completed_at = ?, error = ? + WHERE submission_id = ? AND status = 'recording_interruption' AND attempt_id = ?`, Date.now(), error instanceof Error ? error.message : String(error), - submissionId, - attemptId, + attempt.submissionId, + attempt.attemptId, ); - const submission = this.getSubmission(submissionId); - return submission?.status === 'error' && submission.attemptId === attemptId; + return this.hasOwnedSubmission(attempt, 'failed'); } - private admitSubmission(input: AgentSubmissionInput): SqlAgentSubmission { + private admitSubmission(input: AgentSubmissionInput): SqlAgentDispatchAdmission { const { kind, submissionId } = input; const payload = JSON.stringify(input); const acceptedAt = parseAcceptedAt(input.acceptedAt, `${kind} admission`); @@ -438,7 +436,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { return this.transactionSync(() => { if (kind === 'dispatch') { const receipt = this.getDispatchReceipt(submissionId); - if (receipt) throw new SqlAgentDispatchReceiptRetainedError(receipt); + if (receipt) return { kind: 'retained_receipt', receipt }; } const deleting = this.sql .exec('SELECT 1 FROM flue_agent_session_deletions WHERE session_key = ? LIMIT 1', sessionKey) @@ -462,18 +460,13 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { if (row.kind !== kind || row.payload !== payload) { throw new SqlAgentSubmissionConflictError(`[flue] Conflicting internal ${kind} replay.`); } - return parseSubmission(row); + return { kind: 'submission', submission: parseSubmission(row) }; }); } - private getOwnedRunningSubmission( - submissionId: string, - attemptId: string, - ): SqlAgentSubmission | null { - const submission = this.getSubmission(submissionId); - return submission?.status === 'running' && submission.attemptId === attemptId - ? submission - : null; + private hasOwnedSubmission(attempt: SubmissionAttemptRef, status: SqlAgentSubmissionStatus): boolean { + const submission = this.getSubmission(attempt.submissionId); + return submission?.status === status && submission.attemptId === attempt.attemptId; } private parseOperationalRows( @@ -495,8 +488,8 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { private failSubmissionSequence(sequence: number, status: 'queued' | 'active', error: unknown): void { this.sql.exec( `UPDATE flue_agent_submissions - SET status = 'error', completed_at = ?, error = ? - WHERE sequence = ? AND ${status === 'queued' ? "status = 'queued'" : "status IN ('running', 'terminalizing')"}`, + SET status = 'failed', completed_at = ?, error = ? + WHERE sequence = ? AND ${status === 'queued' ? "status = 'queued'" : "status IN ('running', 'recording_interruption')"}`, Date.now(), error instanceof Error ? error.message : String(error), sequence, @@ -536,9 +529,9 @@ function parseSubmission(row: SqlRow): SqlAgentSubmission { typeof row.payload !== 'string' || (row.status !== 'queued' && row.status !== 'running' && - row.status !== 'terminalizing' && + row.status !== 'recording_interruption' && row.status !== 'completed' && - row.status !== 'error') || + row.status !== 'failed') || typeof row.accepted_at !== 'number' || (row.attempt_id !== null && row.attempt_id !== undefined && typeof row.attempt_id !== 'string') || (row.input_applied_at !== null && @@ -553,7 +546,7 @@ function parseSubmission(row: SqlRow): SqlAgentSubmission { row.input_applied_at !== null || row.recovery_requested_at !== null || row.started_at !== null)) || - ((row.status === 'running' || row.status === 'terminalizing') && + ((row.status === 'running' || row.status === 'recording_interruption') && (typeof row.attempt_id !== 'string' || typeof row.started_at !== 'number')) ) { throw new Error('[flue] Persisted agent submission row is malformed.'); diff --git a/packages/runtime/src/internal.ts b/packages/runtime/src/internal.ts index 14f05510..70034c0a 100644 --- a/packages/runtime/src/internal.ts +++ b/packages/runtime/src/internal.ts @@ -25,7 +25,6 @@ export { CLOUDFLARE_AGENT_INTERNAL_DISPATCH_PATH, createCloudflareAgentRuntime } export { createSqlAgentExecutionStore, createSqlSessionStore, - SqlAgentDispatchReceiptRetainedError, SqlAgentSubmissionConflictError, } from './cloudflare/agent-execution-store.ts'; export { createDurableRunStore } from './cloudflare/run-store.ts'; diff --git a/packages/runtime/test/cloudflare-agent-coordinator.test.ts b/packages/runtime/test/cloudflare-agent-coordinator.test.ts index 06210718..349bb992 100644 --- a/packages/runtime/test/cloudflare-agent-coordinator.test.ts +++ b/packages/runtime/test/cloudflare-agent-coordinator.test.ts @@ -15,8 +15,8 @@ function makeFakeSql(events: string[] = []) { exec(query: string, ...bindings: unknown[]) { if (query.includes('SET recovery_requested_at')) events.push('request-recovery'); if (query.includes("SET status = 'queued'")) events.push('requeue'); - if (query.includes("SET status = 'terminalizing'")) events.push('begin-terminalization'); - if (query.includes("SET status = 'error', completed_at")) events.push('finish-terminalization'); + if (query.includes("SET status = 'recording_interruption'")) events.push('begin-interruption-recording'); + if (query.includes("SET status = 'failed', completed_at")) events.push('finish-interruption-recording'); const stmt = db.prepare(query); let rows: unknown[]; try { @@ -196,7 +196,7 @@ describe('createCloudflareAgentRuntime()', () => { const instance = makeInstance(storage, events); const executionStore = prepare(runtime, instance); executionStore.submissions.admitDirect(directInput()); - executionStore.submissions.claimSubmission('direct-1', 'attempt-1'); + executionStore.submissions.claimSubmission({ submissionId: 'direct-1', attemptId: 'attempt-1' }); await runtime.onFiberRecovered( instance, @@ -252,7 +252,7 @@ describe('createCloudflareAgentRuntime()', () => { const instance = makeInstance(storage); const executionStore = prepare(runtime, instance); executionStore.submissions.admitDirect(directInput()); - executionStore.submissions.claimSubmission('direct-1', 'attempt-1'); + executionStore.submissions.claimSubmission({ submissionId: 'direct-1', attemptId: 'attempt-1' }); await runtime.onStart(instance, () => {}); @@ -275,17 +275,17 @@ describe('createCloudflareAgentRuntime()', () => { const instance = makeInstance(storage); const executionStore = prepare(runtime, instance); executionStore.submissions.admitDirect(directInput()); - executionStore.submissions.claimSubmission('direct-1', 'attempt-1'); - executionStore.submissions.markSubmissionInputApplied('direct-1', 'attempt-1'); + executionStore.submissions.claimSubmission({ submissionId: 'direct-1', attemptId: 'attempt-1' }); + executionStore.submissions.markSubmissionInputApplied({ submissionId: 'direct-1', attemptId: 'attempt-1' }); await runtime.onStart(instance, () => {}); - expect(events).toEqual(['begin-terminalization', 'record-terminal', 'finish-terminalization']); + expect(events).toEqual(['begin-interruption-recording', 'record-terminal', 'finish-interruption-recording']); expect(payloads).toEqual([directInput(), directInput().payload]); - expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ status: 'error' }); + expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ status: 'failed' }); }); - it('resumes terminalizing rows by recording interruption before final SQL settlement', async () => { + it('resumes recording interruption rows by recording interruption before final SQL settlement', async () => { const events: string[] = []; const { storage } = makeFakeSql(events); const recovery = makeRecoveryContext({ events }); @@ -296,14 +296,17 @@ describe('createCloudflareAgentRuntime()', () => { const instance = makeInstance(storage); const executionStore = prepare(runtime, instance); executionStore.submissions.admitDirect(directInput()); - executionStore.submissions.claimSubmission('direct-1', 'attempt-1'); - executionStore.submissions.beginSubmissionTerminalization('direct-1', 'attempt-1'); + executionStore.submissions.claimSubmission({ submissionId: 'direct-1', attemptId: 'attempt-1' }); + executionStore.submissions.beginSubmissionInterruptionRecording({ + submissionId: 'direct-1', + attemptId: 'attempt-1', + }); events.splice(0); await runtime.onStart(instance, () => {}); - expect(events).toEqual(['record-terminal', 'finish-terminalization']); - expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ status: 'error' }); + expect(events).toEqual(['record-terminal', 'finish-interruption-recording']); + expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ status: 'failed' }); }); it('settles interrupted attempts when canonical completion is already persisted', async () => { @@ -316,8 +319,8 @@ describe('createCloudflareAgentRuntime()', () => { const instance = makeInstance(storage); const executionStore = prepare(runtime, instance); executionStore.submissions.admitDirect(directInput()); - executionStore.submissions.claimSubmission('direct-1', 'attempt-1'); - executionStore.submissions.markSubmissionInputApplied('direct-1', 'attempt-1'); + executionStore.submissions.claimSubmission({ submissionId: 'direct-1', attemptId: 'attempt-1' }); + executionStore.submissions.markSubmissionInputApplied({ submissionId: 'direct-1', attemptId: 'attempt-1' }); await runtime.onStart(instance, () => {}); @@ -378,8 +381,8 @@ describe('createCloudflareAgentRuntime()', () => { const instance = makeInstance(storage); const executionStore = prepare(runtime, instance); executionStore.submissions.admitDispatch(dispatchInput()); - executionStore.submissions.claimSubmission('dispatch-1', 'attempt-1'); - executionStore.submissions.markSubmissionInputApplied('dispatch-1', 'attempt-1'); + executionStore.submissions.claimSubmission({ submissionId: 'dispatch-1', attemptId: 'attempt-1' }); + executionStore.submissions.markSubmissionInputApplied({ submissionId: 'dispatch-1', attemptId: 'attempt-1' }); await runtime.onStart(instance, () => {}); diff --git a/packages/runtime/test/cloudflare-agent-execution-store.test.ts b/packages/runtime/test/cloudflare-agent-execution-store.test.ts index 04c2930f..5289a857 100644 --- a/packages/runtime/test/cloudflare-agent-execution-store.test.ts +++ b/packages/runtime/test/cloudflare-agent-execution-store.test.ts @@ -3,7 +3,6 @@ import { describe, expect, it } from 'vitest'; import { createSqlAgentExecutionStore, createSqlSessionStore, - SqlAgentDispatchReceiptRetainedError, } from '../src/cloudflare/agent-execution-store.ts'; import type { DirectAgentSubmissionInput } from '../src/runtime/agent-submissions.ts'; import type { DispatchInput } from '../src/runtime/dispatch-queue.ts'; @@ -69,6 +68,10 @@ function directInput(overrides: Partial = {}): Direc }; } +function attempt(submissionId: string, attemptId: string) { + return { submissionId, attemptId }; +} + function sessionData(): SessionData { return { version: 5, @@ -159,10 +162,13 @@ describe('createSqlAgentExecutionStore()', () => { count: 1, }); expect(first).toMatchObject({ - submissionId: 'dispatch-1', - session: 'default', - sessionKey: 'agent-session:["agent-1","default","default"]', - status: 'queued', + kind: 'submission', + submission: { + submissionId: 'dispatch-1', + session: 'default', + sessionKey: 'agent-session:["agent-1","default","default"]', + status: 'queued', + }, }); }); @@ -184,8 +190,8 @@ describe('createSqlAgentExecutionStore()', () => { const other = store.submissions.admitDirect(directInput({ submissionId: 'direct-2', session: 'other' })); expect(store.submissions.listRunnableSubmissions()).toEqual([direct, other]); - expect(store.submissions.claimSubmission('dispatch-1', 'attempt-blocked')).toBeNull(); - expect(store.submissions.claimSubmission('direct-1', 'attempt-direct')).toMatchObject({ + expect(store.submissions.claimSubmission(attempt('dispatch-1', 'attempt-blocked'))).toBeNull(); + expect(store.submissions.claimSubmission(attempt('direct-1', 'attempt-direct'))).toMatchObject({ kind: 'direct', status: 'running', }); @@ -194,13 +200,14 @@ describe('createSqlAgentExecutionStore()', () => { it('lists queued dispatches in admission order and selects one runnable head per session', () => { const { sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); - const first = store.submissions.admitDispatch(dispatchInput()); + store.submissions.admitDispatch(dispatchInput()); store.submissions.admitDispatch(dispatchInput({ dispatchId: 'dispatch-2' })); - const other = store.submissions.admitDispatch( - dispatchInput({ dispatchId: 'dispatch-3', session: 'other' }), - ); + store.submissions.admitDispatch(dispatchInput({ dispatchId: 'dispatch-3', session: 'other' })); - expect(store.submissions.listRunnableSubmissions()).toEqual([first, other]); + expect(store.submissions.listRunnableSubmissions()).toEqual([ + expect.objectContaining({ submissionId: 'dispatch-1' }), + expect.objectContaining({ submissionId: 'dispatch-3' }), + ]); }); it('claims only runnable session heads while allowing separate sessions to claim independently', () => { @@ -210,9 +217,9 @@ describe('createSqlAgentExecutionStore()', () => { store.submissions.admitDispatch(dispatchInput({ dispatchId: 'dispatch-2' })); store.submissions.admitDispatch(dispatchInput({ dispatchId: 'dispatch-3', session: 'other' })); - const first = store.submissions.claimSubmission('dispatch-1', 'attempt-1'); - const blocked = store.submissions.claimSubmission('dispatch-2', 'attempt-2'); - const other = store.submissions.claimSubmission('dispatch-3', 'attempt-3'); + const first = store.submissions.claimSubmission(attempt('dispatch-1', 'attempt-1')); + const blocked = store.submissions.claimSubmission(attempt('dispatch-2', 'attempt-2')); + const other = store.submissions.claimSubmission(attempt('dispatch-3', 'attempt-3')); expect(first).toMatchObject({ submissionId: 'dispatch-1', @@ -247,7 +254,7 @@ describe('createSqlAgentExecutionStore()', () => { db .prepare('SELECT status, error FROM flue_agent_submissions WHERE submission_id = ?') .get('malformed'), - ).toMatchObject({ status: 'error', error: expect.any(String) }); + ).toMatchObject({ status: 'failed', error: expect.any(String) }); }); it('terminalizes impossible queued input markers instead of replaying them', () => { @@ -261,7 +268,7 @@ describe('createSqlAgentExecutionStore()', () => { expect(store.submissions.listRunnableSubmissions()).toEqual([]); expect(store.submissions.getSubmission('dispatch-1')).toMatchObject({ - status: 'error', + status: 'failed', error: expect.any(String), }); }); @@ -270,23 +277,22 @@ describe('createSqlAgentExecutionStore()', () => { const { sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); store.submissions.admitDispatch(dispatchInput()); - store.submissions.claimSubmission('dispatch-1', 'attempt-1'); + store.submissions.claimSubmission(attempt('dispatch-1', 'attempt-1')); - const applied = store.submissions.markSubmissionInputApplied('dispatch-1', 'attempt-1'); - const replay = store.submissions.markSubmissionInputApplied('dispatch-1', 'attempt-1'); - const staleApplied = store.submissions.markSubmissionInputApplied('dispatch-1', 'stale-attempt'); - const recovery = store.submissions.requestSubmissionRecovery('dispatch-1', 'attempt-1'); - const staleRecovery = store.submissions.requestSubmissionRecovery('dispatch-1', 'stale-attempt'); + expect(store.submissions.markSubmissionInputApplied(attempt('dispatch-1', 'attempt-1'))).toBe(true); + const appliedAt = store.submissions.getSubmission('dispatch-1')?.inputAppliedAt; + expect(store.submissions.markSubmissionInputApplied(attempt('dispatch-1', 'attempt-1'))).toBe(true); + expect(store.submissions.markSubmissionInputApplied(attempt('dispatch-1', 'stale-attempt'))).toBe(false); + expect(store.submissions.requestSubmissionRecovery(attempt('dispatch-1', 'attempt-1'))).toBe(true); + expect(store.submissions.requestSubmissionRecovery(attempt('dispatch-1', 'stale-attempt'))).toBe(false); - expect(applied).toMatchObject({ + expect(appliedAt).toEqual(expect.any(Number)); + expect(store.submissions.getSubmission('dispatch-1')).toMatchObject({ status: 'running', attemptId: 'attempt-1', - inputAppliedAt: expect.any(Number), + inputAppliedAt: appliedAt, + recoveryRequestedAt: expect.any(Number), }); - expect(replay?.inputAppliedAt).toBe(applied?.inputAppliedAt); - expect(staleApplied).toBeNull(); - expect(recovery).toMatchObject({ recoveryRequestedAt: expect.any(Number) }); - expect(staleRecovery).toBeNull(); }); it('requeues interrupted attempts only before canonical input application', () => { @@ -294,16 +300,17 @@ describe('createSqlAgentExecutionStore()', () => { const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); store.submissions.admitDispatch(dispatchInput({ dispatchId: 'requeue-safe' })); store.submissions.admitDispatch(dispatchInput({ dispatchId: 'requeue-unsafe', session: 'other' })); - store.submissions.claimSubmission('requeue-safe', 'attempt-safe'); - store.submissions.claimSubmission('requeue-unsafe', 'attempt-unsafe'); - store.submissions.markSubmissionInputApplied('requeue-unsafe', 'attempt-unsafe'); + store.submissions.claimSubmission(attempt('requeue-safe', 'attempt-safe')); + store.submissions.claimSubmission(attempt('requeue-unsafe', 'attempt-unsafe')); + store.submissions.markSubmissionInputApplied(attempt('requeue-unsafe', 'attempt-unsafe')); - const safe = store.submissions.requeueSubmissionBeforeInputApplied('requeue-safe', 'attempt-safe'); - const unsafe = store.submissions.requeueSubmissionBeforeInputApplied('requeue-unsafe', 'attempt-unsafe'); + const safe = store.submissions.requeueSubmissionBeforeInputApplied(attempt('requeue-safe', 'attempt-safe')); + const unsafe = store.submissions.requeueSubmissionBeforeInputApplied(attempt('requeue-unsafe', 'attempt-unsafe')); - expect(safe).toMatchObject({ status: 'queued' }); - expect(safe).not.toHaveProperty('attemptId'); - expect(unsafe).toBeNull(); + expect(safe).toBe(true); + expect(unsafe).toBe(false); + expect(store.submissions.getSubmission('requeue-safe')).toMatchObject({ status: 'queued' }); + expect(store.submissions.getSubmission('requeue-safe')).not.toHaveProperty('attemptId'); expect(store.submissions.getSubmission('requeue-unsafe')).toMatchObject({ status: 'running' }); }); @@ -314,9 +321,9 @@ describe('createSqlAgentExecutionStore()', () => { expect(store.submissions.hasUnsettledSubmissions()).toBe(true); expect(store.submissions.listRunnableSubmissions()).toHaveLength(1); - store.submissions.claimSubmission('dispatch-1', 'attempt-1'); + store.submissions.claimSubmission(attempt('dispatch-1', 'attempt-1')); expect(store.submissions.listRunningSubmissions()).toHaveLength(1); - store.submissions.completeSubmission('dispatch-1', 'attempt-1'); + store.submissions.completeSubmission(attempt('dispatch-1', 'attempt-1')); expect(store.submissions.hasUnsettledSubmissions()).toBe(false); expect(store.submissions.listRunningSubmissions()).toEqual([]); expect(store.submissions.getSubmission('dispatch-1')).toMatchObject({ status: 'completed' }); @@ -326,34 +333,40 @@ describe('createSqlAgentExecutionStore()', () => { const { sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); store.submissions.admitDispatch(dispatchInput()); - store.submissions.claimSubmission('dispatch-1', 'attempt-1'); + store.submissions.claimSubmission(attempt('dispatch-1', 'attempt-1')); - store.submissions.completeSubmission('dispatch-1', 'stale-attempt'); - store.submissions.failSubmission('dispatch-1', 'attempt-1', new Error('first failure')); - store.submissions.completeSubmission('dispatch-1', 'attempt-1'); - store.submissions.failSubmission('dispatch-1', 'attempt-1', new Error('later failure')); + store.submissions.completeSubmission(attempt('dispatch-1', 'stale-attempt')); + store.submissions.failSubmission(attempt('dispatch-1', 'attempt-1'), new Error('first failure')); + store.submissions.completeSubmission(attempt('dispatch-1', 'attempt-1')); + store.submissions.failSubmission(attempt('dispatch-1', 'attempt-1'), new Error('later failure')); expect(store.submissions.getSubmission('dispatch-1')).toMatchObject({ - status: 'error', + status: 'failed', error: 'first failure', }); }); - it('fences ordinary completion after interrupted terminalization begins', () => { + it('fences ordinary completion while an interrupted submission records its advisory', () => { const { sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); store.submissions.admitDispatch(dispatchInput()); - store.submissions.claimSubmission('dispatch-1', 'attempt-1'); + store.submissions.claimSubmission(attempt('dispatch-1', 'attempt-1')); - expect(store.submissions.beginSubmissionTerminalization('dispatch-1', 'attempt-1')).toMatchObject({ - status: 'terminalizing', + expect(store.submissions.beginSubmissionInterruptionRecording(attempt('dispatch-1', 'attempt-1'))).toBe( + true, + ); + expect(store.submissions.getSubmission('dispatch-1')).toMatchObject({ + status: 'recording_interruption', }); - expect(store.submissions.completeSubmission('dispatch-1', 'attempt-1')).toBe(false); + expect(store.submissions.completeSubmission(attempt('dispatch-1', 'attempt-1'))).toBe(false); expect( - store.submissions.finalizeSubmissionTerminalization('dispatch-1', 'attempt-1', new Error('interrupted')), + store.submissions.finishSubmissionInterruptionRecording( + attempt('dispatch-1', 'attempt-1'), + new Error('interrupted'), + ), ).toBe(true); expect(store.submissions.getSubmission('dispatch-1')).toMatchObject({ - status: 'error', + status: 'failed', error: 'interrupted', }); }); @@ -367,7 +380,7 @@ describe('createSqlAgentExecutionStore()', () => { store.submissions.deleteSession('agent-session:["agent-1","default","default"]', async () => {}), ).rejects.toThrow('Session cannot be deleted while durable agent submissions are queued or running.'); - store.submissions.claimSubmission('dispatch-1', 'attempt-1'); + store.submissions.claimSubmission(attempt('dispatch-1', 'attempt-1')); await expect( store.submissions.deleteSession('agent-session:["agent-1","default","default"]', async () => {}), ).rejects.toThrow('Session cannot be deleted while durable agent submissions are queued or running.'); @@ -388,7 +401,7 @@ describe('createSqlAgentExecutionStore()', () => { ); releaseDeletion(); await deletion; - expect(store.submissions.admitDispatch(dispatchInput())).toMatchObject({ status: 'queued' }); + expect(store.submissions.admitDispatch(dispatchInput())).toMatchObject({ kind: 'submission', submission: { status: 'queued' } }); }); it('shares session deletion work while snapshot deletion is in progress', async () => { @@ -416,7 +429,7 @@ describe('createSqlAgentExecutionStore()', () => { ); releaseDeletion(); await Promise.all([first, second]); - expect(store.submissions.admitDispatch(dispatchInput())).toMatchObject({ status: 'queued' }); + expect(store.submissions.admitDispatch(dispatchInput())).toMatchObject({ kind: 'submission', submission: { status: 'queued' } }); }); it('keeps new submissions blocked when session snapshot deletion fails', async () => { @@ -433,7 +446,7 @@ describe('createSqlAgentExecutionStore()', () => { 'Durable agent submission admission is unavailable while this session is being deleted.', ); await expect(store.submissions.deleteSession(sessionKey, async () => {})).resolves.toBeUndefined(); - expect(store.submissions.admitDispatch(dispatchInput())).toMatchObject({ status: 'queued' }); + expect(store.submissions.admitDispatch(dispatchInput())).toMatchObject({ kind: 'submission', submission: { status: 'queued' } }); }); it('clears terminal rows when a settled session is deleted', async () => { @@ -441,8 +454,8 @@ describe('createSqlAgentExecutionStore()', () => { const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); const sessionKey = 'agent-session:["agent-1","default","default"]'; store.submissions.admitDispatch(dispatchInput()); - store.submissions.claimSubmission('dispatch-1', 'attempt-1'); - store.submissions.completeSubmission('dispatch-1', 'attempt-1'); + store.submissions.claimSubmission(attempt('dispatch-1', 'attempt-1')); + store.submissions.completeSubmission(attempt('dispatch-1', 'attempt-1')); await store.submissions.deleteSession(sessionKey, async () => {}); @@ -457,25 +470,22 @@ describe('createSqlAgentExecutionStore()', () => { }); }); - it('rejects replay admission transactionally when deletion retained the dispatch receipt', async () => { + it('returns retained receipt admission transactionally when deletion removed the settled dispatch row', async () => { const { sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); const sessionKey = 'agent-session:["agent-1","default","default"]'; store.submissions.admitDispatch(dispatchInput()); - store.submissions.claimSubmission('dispatch-1', 'attempt-1'); - store.submissions.completeSubmission('dispatch-1', 'attempt-1'); + store.submissions.claimSubmission(attempt('dispatch-1', 'attempt-1')); + store.submissions.completeSubmission(attempt('dispatch-1', 'attempt-1')); await store.submissions.deleteSession(sessionKey, async () => {}); - try { - store.submissions.admitDispatch(dispatchInput()); - throw new Error('Expected dispatch replay to be retained.'); - } catch (error) { - expect(error).toBeInstanceOf(SqlAgentDispatchReceiptRetainedError); - expect((error as SqlAgentDispatchReceiptRetainedError).receipt).toEqual({ + expect(store.submissions.admitDispatch(dispatchInput())).toEqual({ + kind: 'retained_receipt', + receipt: { submissionId: 'dispatch-1', acceptedAt: Date.parse('2026-06-03T00:00:00.000Z'), - }); - } + }, + }); expect(store.submissions.getSubmission('dispatch-1')).toBeNull(); }); @@ -485,11 +495,11 @@ describe('createSqlAgentExecutionStore()', () => { store.submissions.admitDispatch(dispatchInput({ dispatchId: 'expired-1' })); store.submissions.admitDispatch(dispatchInput({ dispatchId: 'expired-2', session: 'other' })); store.submissions.admitDispatch(dispatchInput({ dispatchId: 'active', session: 'active' })); - store.submissions.claimSubmission('expired-1', 'attempt-1'); - store.submissions.claimSubmission('expired-2', 'attempt-2'); - store.submissions.completeSubmission('expired-1', 'attempt-1'); - store.submissions.failSubmission('expired-2', 'attempt-2', new Error('failed')); - db.prepare("UPDATE flue_agent_submissions SET completed_at = 1 WHERE status IN ('completed', 'error')").run(); + store.submissions.claimSubmission(attempt('expired-1', 'attempt-1')); + store.submissions.claimSubmission(attempt('expired-2', 'attempt-2')); + store.submissions.completeSubmission(attempt('expired-1', 'attempt-1')); + store.submissions.failSubmission(attempt('expired-2', 'attempt-2'), new Error('failed')); + db.prepare("UPDATE flue_agent_submissions SET completed_at = 1 WHERE status IN ('completed', 'failed')").run(); expect(store.submissions.cleanupTerminalSubmissions(2, 1)).toBe(1); expect(store.submissions.cleanupTerminalSubmissions(2, 1)).toBe(1); From 894cb105ecf53bcd2d277212c65694caa303fe31 Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" <622227+FredKSchott@users.noreply.github.com> Date: Thu, 4 Jun 2026 12:40:43 -0500 Subject: [PATCH 14/17] refactor(runtime): narrow attached submission observers --- .../src/cloudflare/agent-coordinator.ts | 6 ++--- .../runtime/src/runtime/agent-submissions.ts | 25 ++++++++----------- packages/runtime/src/runtime/handle-agent.ts | 2 +- .../runtime/test/cloudflare-websocket.test.ts | 4 +-- 4 files changed, 16 insertions(+), 21 deletions(-) diff --git a/packages/runtime/src/cloudflare/agent-coordinator.ts b/packages/runtime/src/cloudflare/agent-coordinator.ts index 298b70f7..fde3b0d8 100644 --- a/packages/runtime/src/cloudflare/agent-coordinator.ts +++ b/packages/runtime/src/cloudflare/agent-coordinator.ts @@ -228,8 +228,7 @@ class CloudflareAgentCoordinator { handler, createContext: (_id, _runId, payload, req, initialEventIndex, dispatchId) => this.createContext(payload, req, initialEventIndex, dispatchId), - admitAttachedSubmission: (payload, _req, onEvent) => - this.admitAttachedSubmission(payload, onEvent), + admitAttachedSubmission: (payload, onEvent) => this.admitAttachedSubmission(payload, onEvent), }), ); } @@ -256,8 +255,7 @@ class CloudflareAgentCoordinator { request: socketRequest(connection), handler, createContext: (_id, _runId, payload, req) => this.createContext(payload, req), - admitAttachedSubmission: (payload, _req, onEvent) => - this.admitAttachedSubmission(payload, onEvent), + admitAttachedSubmission: (payload, onEvent) => this.admitAttachedSubmission(payload, onEvent), }), ); } diff --git a/packages/runtime/src/runtime/agent-submissions.ts b/packages/runtime/src/runtime/agent-submissions.ts index ec3c3de1..d1733585 100644 --- a/packages/runtime/src/runtime/agent-submissions.ts +++ b/packages/runtime/src/runtime/agent-submissions.ts @@ -65,7 +65,6 @@ interface AgentSubmissionObserverRegistry { export type AttachedAgentSubmissionAdmission = ( payload: DirectAgentPayload, - request: Request, onEvent?: (event: AttachedAgentEvent) => Promise | void, ) => Promise; @@ -134,10 +133,13 @@ export function agentSubmissionDispatchInput(input: DispatchAgentSubmissionInput export function createAgentSubmissionObserverRegistry(): AgentSubmissionObserverRegistry { const observers = new Map< string, - Set + AgentSubmissionObserver & { resolve(value: unknown): void; reject(error: unknown): void } >(); return { attach(submissionId, observer) { + if (observers.has(submissionId)) { + throw new Error('[flue] Internal agent submission observer is already attached.'); + } let resolve!: (value: unknown) => void; let reject!: (error: unknown) => void; const completion = new Promise((resolve_, reject_) => { @@ -145,30 +147,25 @@ export function createAgentSubmissionObserverRegistry(): AgentSubmissionObserver reject = reject_; }); const attached = { ...observer, resolve, reject }; - const bucket = observers.get(submissionId) ?? new Set(); - bucket.add(attached); - observers.set(submissionId, bucket); + observers.set(submissionId, attached); return { completion, detach() { - bucket.delete(attached); - if (bucket.size === 0) observers.delete(submissionId); + if (observers.get(submissionId) === attached) observers.delete(submissionId); }, }; }, async publish(submissionId, event) { - for (const observer of observers.get(submissionId) ?? []) { - try { - await observer.onEvent?.(event); - } catch {} - } + try { + await observers.get(submissionId)?.onEvent?.(event); + } catch {} }, complete(submissionId, result) { - for (const observer of observers.get(submissionId) ?? []) observer.resolve(result); + observers.get(submissionId)?.resolve(result); observers.delete(submissionId); }, fail(submissionId, error) { - for (const observer of observers.get(submissionId) ?? []) observer.reject(error); + observers.get(submissionId)?.reject(error); observers.delete(submissionId); }, }; diff --git a/packages/runtime/src/runtime/handle-agent.ts b/packages/runtime/src/runtime/handle-agent.ts index e5e0455f..b2aef45b 100644 --- a/packages/runtime/src/runtime/handle-agent.ts +++ b/packages/runtime/src/runtime/handle-agent.ts @@ -685,7 +685,7 @@ async function runDirectSyncMode(opts: DirectAttachedOptions): Promise export async function invokeDirectAttached(opts: DirectAttachedOptions): Promise { if (opts.admitAttachedSubmission) { - return opts.admitAttachedSubmission(opts.payload, opts.request, opts.onEvent); + return opts.admitAttachedSubmission(opts.payload, opts.onEvent); } const sessionLock = acquireDirectAgentSessionLock(opts.agentName, opts.id, opts.payload); try { diff --git a/packages/runtime/test/cloudflare-websocket.test.ts b/packages/runtime/test/cloudflare-websocket.test.ts index 545062c5..cabd3c9e 100644 --- a/packages/runtime/test/cloudflare-websocket.test.ts +++ b/packages/runtime/test/cloudflare-websocket.test.ts @@ -122,7 +122,7 @@ describe('Cloudflare agent WebSockets', () => { return null; }, createContext, - admitAttachedSubmission: async (payload, _request, onEvent) => { + admitAttachedSubmission: async (payload, onEvent) => { calls.push(`admit:${payload.message}`); await onEvent?.({ type: 'idle', instanceId: 'agent-instance-1' }); return 'done'; @@ -156,7 +156,7 @@ describe('Cloudflare agent WebSockets', () => { request: new Request('https://example.com/flue/agents/assistant/agent-instance-1'), handler: async () => null, createContext, - admitAttachedSubmission: async (_payload, _request, onEvent) => { + admitAttachedSubmission: async (_payload, onEvent) => { calls.push('admitted'); await onEvent?.({ type: 'idle', instanceId: 'agent-instance-1' }); calls.push('completed'); From 0c2529ab9437e527b110e03d51b2465de2977c30 Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" <622227+FredKSchott@users.noreply.github.com> Date: Thu, 4 Jun 2026 16:54:30 -0500 Subject: [PATCH 15/17] fix(cloudflare): isolate submission reconciliation failures --- .../cli/src/lib/build-plugin-cloudflare.ts | 5 - .../src/cloudflare/agent-coordinator.ts | 33 ++++- .../src/cloudflare/agent-execution-store.ts | 16 ++- .../test/build-plugin-cloudflare.test.ts | 2 + .../test/cloudflare-agent-coordinator.test.ts | 134 +++++++++++++++++- .../cloudflare-agent-execution-store.test.ts | 44 ++++++ 6 files changed, 222 insertions(+), 12 deletions(-) diff --git a/packages/cli/src/lib/build-plugin-cloudflare.ts b/packages/cli/src/lib/build-plugin-cloudflare.ts index 183b4cbe..0b3658d0 100644 --- a/packages/cli/src/lib/build-plugin-cloudflare.ts +++ b/packages/cli/src/lib/build-plugin-cloudflare.ts @@ -189,7 +189,6 @@ import { createRunSubscriberRegistry, bashFactoryToSessionEnv, resolveModel, - handleAgentRequest, handleWorkflowRequest, handleRunRouteRequest, failRecoveredRun, @@ -568,10 +567,6 @@ function agentRuntimeIdentity(agentName) { return agentIdentities[agentName]; } -function isInternalDispatchRequest(request) { - return request.method === 'POST' && new URL(request.url).pathname === INTERNAL_DISPATCH_PATH; -} - function parseWorkflowStart(request, workflowName) { if (request.method !== 'POST') return false; const url = new URL(request.url); diff --git a/packages/runtime/src/cloudflare/agent-coordinator.ts b/packages/runtime/src/cloudflare/agent-coordinator.ts index fde3b0d8..178072bc 100644 --- a/packages/runtime/src/cloudflare/agent-coordinator.ts +++ b/packages/runtime/src/cloudflare/agent-coordinator.ts @@ -368,14 +368,23 @@ class CloudflareAgentCoordinator { submission.recoveryRequestedAt === undefined ) continue; - await this.reconcileInterruptedSubmission(submission); + try { + await this.reconcileInterruptedSubmission(submission); + } catch (error) { + this.logSubmissionReconciliationFailure(submission, 'reconcile_submission', error); + } } for (const submission of this.submissions.listRunnableSubmissions()) { const claimed = this.submissions.claimSubmission({ submissionId: submission.submissionId, attemptId: crypto.randomUUID(), }); - if (claimed) this.startSubmissionAttempt(claimed); + if (!claimed) continue; + try { + this.startSubmissionAttempt(claimed); + } catch (error) { + this.logSubmissionReconciliationFailure(claimed, 'start_submission', error); + } } } catch (error) { console.error( @@ -393,6 +402,26 @@ class CloudflareAgentCoordinator { return this.submissions.hasUnsettledSubmissions(); } + private logSubmissionReconciliationFailure( + submission: SqlAgentSubmission, + operation: 'reconcile_submission' | 'start_submission', + error: unknown, + ): void { + console.error( + '[flue:submission-reconciliation]', + { + agentName: this.agentName, + instanceId: this.instance.name, + submissionId: submission.submissionId, + sessionKey: submission.sessionKey, + attemptId: submission.attemptId, + operation, + outcome: 'deferred_to_scheduled_wake', + }, + error, + ); + } + private async reconcileInterruptedSubmission(submission: SqlAgentSubmission): Promise { const { input } = submission; const attempt = submissionAttemptRef(submission); diff --git a/packages/runtime/src/cloudflare/agent-execution-store.ts b/packages/runtime/src/cloudflare/agent-execution-store.ts index 302a107d..9abb2376 100644 --- a/packages/runtime/src/cloudflare/agent-execution-store.ts +++ b/packages/runtime/src/cloudflare/agent-execution-store.ts @@ -313,7 +313,18 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { completedBefore, ); } - this.sql.exec('DELETE FROM flue_agent_dispatch_receipts WHERE settled_at < ?', completedBefore); + this.sql.exec( + `DELETE FROM flue_agent_dispatch_receipts + WHERE dispatch_id IN ( + SELECT dispatch_id + FROM flue_agent_dispatch_receipts + WHERE settled_at < ? + ORDER BY settled_at ASC, dispatch_id ASC + LIMIT ? + )`, + completedBefore, + limit, + ); return rows.length; } @@ -673,4 +684,7 @@ function ensureSubmissionTable(sql: SqlStorage): void { sql.exec( 'CREATE INDEX IF NOT EXISTS flue_agent_submissions_session_status_sequence_idx ON flue_agent_submissions (session_key, status, sequence ASC)', ); + sql.exec( + 'CREATE INDEX IF NOT EXISTS flue_agent_dispatch_receipts_settled_at_dispatch_id_idx ON flue_agent_dispatch_receipts (settled_at ASC, dispatch_id ASC)', + ); } diff --git a/packages/runtime/test/build-plugin-cloudflare.test.ts b/packages/runtime/test/build-plugin-cloudflare.test.ts index cac2212e..69d1a67b 100644 --- a/packages/runtime/test/build-plugin-cloudflare.test.ts +++ b/packages/runtime/test/build-plugin-cloudflare.test.ts @@ -89,6 +89,8 @@ describe('CloudflarePlugin', () => { expect(entry).toContain('return fetchAgent(binding, target.instanceId, request)'); expect(entry).toContain('(await getAgentByName(binding, instanceId)).fetch(request)'); expect(entry).not.toContain("routeAgentRequest } from 'agents'"); + expect(entry).not.toContain(' handleAgentRequest,'); + expect(entry).not.toContain('function isInternalDispatchRequest(request)'); }); }); diff --git a/packages/runtime/test/cloudflare-agent-coordinator.test.ts b/packages/runtime/test/cloudflare-agent-coordinator.test.ts index 349bb992..8b79c7c5 100644 --- a/packages/runtime/test/cloudflare-agent-coordinator.test.ts +++ b/packages/runtime/test/cloudflare-agent-coordinator.test.ts @@ -1,9 +1,17 @@ import { DatabaseSync } from 'node:sqlite'; -import { describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import type { FlueContextInternal } from '../src/client.ts'; import { createCloudflareAgentRuntime } from '../src/cloudflare/agent-coordinator.ts'; import type { SqlAgentExecutionStore } from '../src/cloudflare/agent-execution-store.ts'; -import type { AgentSubmissionInspection, AgentSubmissionInterruption } from '../src/runtime/agent-submissions.ts'; +import type { + AgentSubmissionInspection, + AgentSubmissionInterruption, + DirectAgentSubmissionInput, +} from '../src/runtime/agent-submissions.ts'; + +afterEach(() => { + vi.restoreAllMocks(); +}); function makeFakeSql(events: string[] = []) { const db = new DatabaseSync(':memory:'); @@ -116,15 +124,16 @@ function makeRecoveryContext(options: { return { ctx, terminalRecords }; } -function directInput() { +function directInput(overrides: Partial = {}): DirectAgentSubmissionInput { return { - kind: 'direct' as const, + kind: 'direct', submissionId: 'direct-1', agent: 'assistant', id: 'agent-1', session: 'default', payload: { message: 'Hello' }, acceptedAt: '2026-06-03T00:00:00.000Z', + ...overrides, }; } @@ -327,6 +336,123 @@ describe('createCloudflareAgentRuntime()', () => { expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ status: 'completed' }); }); + it('starts unrelated queued sessions when interrupted-session inspection fails', async () => { + const { storage } = makeFakeSql(); + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + let resolveProcessed!: () => void; + const processed = new Promise((resolve) => { + resolveProcessed = resolve; + }); + const runtime = makeRuntime({ + createdAgent: {} as never, + createContext: () => { + return { + async initializeCreatedAgent() { + return { + async session() { + return { + inspectSubmissionInput() { + throw new Error('snapshot inspection failed'); + }, + async processSubmissionInput(input: DirectAgentSubmissionInput) { + if (input.session === 'healthy') resolveProcessed(); + }, + }; + }, + }; + }, + setEventCallback() {}, + } as unknown as FlueContextInternal; + }, + }); + const instance = makeInstance(storage); + instance.runFiber = async (_name, callback) => callback({ stash() {} }); + const executionStore = prepare(runtime, instance); + executionStore.submissions.admitDirect(directInput()); + executionStore.submissions.claimSubmission({ submissionId: 'direct-1', attemptId: 'attempt-1' }); + executionStore.submissions.admitDirect(directInput({ submissionId: 'direct-2', session: 'healthy' })); + + await runtime.onStart(instance, () => {}); + await processed; + + expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ status: 'running' }); + await vi.waitFor(() => { + expect(executionStore.submissions.getSubmission('direct-2')).toMatchObject({ status: 'completed' }); + }); + expect(consoleError).toHaveBeenCalledWith( + '[flue:submission-reconciliation]', + expect.objectContaining({ + submissionId: 'direct-1', + operation: 'reconcile_submission', + outcome: 'deferred_to_scheduled_wake', + }), + expect.any(Error), + ); + }); + + it('claims unrelated queued sessions when another attempt fails to start synchronously', async () => { + const { storage } = makeFakeSql(); + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + let startCalls = 0; + const runtime = makeRuntime(); + const instance = makeInstance(storage); + instance.runFiber = (_name, _callback) => { + startCalls += 1; + if (startCalls === 1) throw new Error('Fiber startup failed'); + return new Promise(() => {}); + }; + const executionStore = prepare(runtime, instance); + executionStore.submissions.admitDirect(directInput()); + executionStore.submissions.admitDirect(directInput({ submissionId: 'direct-2', session: 'healthy' })); + + await runtime.onStart(instance, () => {}); + + expect(startCalls).toBe(2); + expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ status: 'running' }); + expect(executionStore.submissions.getSubmission('direct-2')).toMatchObject({ status: 'running' }); + expect(consoleError).toHaveBeenCalledWith( + '[flue:submission-reconciliation]', + expect.objectContaining({ + submissionId: 'direct-1', + operation: 'start_submission', + outcome: 'deferred_to_scheduled_wake', + }), + expect.any(Error), + ); + }); + + it('retries a synchronously failed attempt on a later wake when canonical input is absent', async () => { + const events: string[] = []; + const { storage } = makeFakeSql(events); + vi.spyOn(console, 'error').mockImplementation(() => {}); + const recovery = makeRecoveryContext({ inspection: 'absent' }); + let startCalls = 0; + const runtime = makeRuntime({ + createdAgent: {} as never, + createContext: () => recovery.ctx, + }); + const instance = makeInstance(storage); + instance.runFiber = (_name, _callback) => { + startCalls += 1; + if (startCalls === 1) throw new Error('Fiber startup failed'); + return new Promise(() => {}); + }; + const executionStore = prepare(runtime, instance); + executionStore.submissions.admitDirect(directInput()); + + await runtime.onStart(instance, () => {}); + const failedAttempt = executionStore.submissions.getSubmission('direct-1')?.attemptId; + await runtime.wakeSubmissions(instance); + + expect(startCalls).toBe(2); + expect(events).toContain('requeue'); + expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ + status: 'running', + attemptId: expect.any(String), + }); + expect(executionStore.submissions.getSubmission('direct-1')?.attemptId).not.toBe(failedAttempt); + }); + it('uses the public dispatch input as processing context payload without internal envelope fields', async () => { const { storage } = makeFakeSql(); const payloads: unknown[] = []; diff --git a/packages/runtime/test/cloudflare-agent-execution-store.test.ts b/packages/runtime/test/cloudflare-agent-execution-store.test.ts index 5289a857..d1375e21 100644 --- a/packages/runtime/test/cloudflare-agent-execution-store.test.ts +++ b/packages/runtime/test/cloudflare-agent-execution-store.test.ts @@ -148,6 +148,21 @@ describe('createSqlAgentExecutionStore()', () => { { name: 'flue_agent_submissions_status_sequence_idx' }, { name: 'sqlite_autoindex_flue_agent_submissions_1' }, ]); + expect( + db + .prepare( + "SELECT name FROM sqlite_schema WHERE type = 'index' AND tbl_name = 'flue_agent_dispatch_receipts' ORDER BY name", + ) + .all(), + ).toEqual([ + { name: 'flue_agent_dispatch_receipts_settled_at_dispatch_id_idx' }, + { name: 'sqlite_autoindex_flue_agent_dispatch_receipts_1' }, + ]); + expect( + db + .prepare("SELECT name FROM pragma_index_info('flue_agent_dispatch_receipts_settled_at_dispatch_id_idx') ORDER BY seqno") + .all(), + ).toEqual([{ name: 'settled_at' }, { name: 'dispatch_id' }]); }); it('admits one queued dispatch row when the same submission is replayed', () => { @@ -508,6 +523,35 @@ describe('createSqlAgentExecutionStore()', () => { expect(store.submissions.getSubmission('active')).toMatchObject({ status: 'queued' }); }); + it('sweeps expired dispatch receipts in bounded batches', async () => { + const { db, sql, transactionSync } = makeFakeSql(); + const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); + store.submissions.admitDispatch(dispatchInput({ dispatchId: 'expired-1' })); + store.submissions.claimSubmission(attempt('expired-1', 'attempt-1')); + store.submissions.completeSubmission(attempt('expired-1', 'attempt-1')); + await store.submissions.deleteSession('agent-session:["agent-1","default","default"]', async () => {}); + store.submissions.admitDispatch(dispatchInput({ dispatchId: 'expired-2', session: 'other' })); + store.submissions.claimSubmission(attempt('expired-2', 'attempt-2')); + store.submissions.completeSubmission(attempt('expired-2', 'attempt-2')); + await store.submissions.deleteSession('agent-session:["agent-1","default","other"]', async () => {}); + store.submissions.admitDispatch(dispatchInput({ dispatchId: 'retained', session: 'retained' })); + store.submissions.claimSubmission(attempt('retained', 'attempt-3')); + store.submissions.completeSubmission(attempt('retained', 'attempt-3')); + await store.submissions.deleteSession('agent-session:["agent-1","default","retained"]', async () => {}); + db.prepare('UPDATE flue_agent_dispatch_receipts SET settled_at = ? WHERE dispatch_id != ?').run(1, 'retained'); + + store.submissions.cleanupTerminalSubmissions(2, 1); + + expect(db.prepare('SELECT dispatch_id FROM flue_agent_dispatch_receipts ORDER BY dispatch_id').all()).toEqual([ + { dispatch_id: 'expired-2' }, + { dispatch_id: 'retained' }, + ]); + store.submissions.cleanupTerminalSubmissions(2, 1); + expect(db.prepare('SELECT dispatch_id FROM flue_agent_dispatch_receipts ORDER BY dispatch_id').all()).toEqual([ + { dispatch_id: 'retained' }, + ]); + }); + it('rejects missing Durable Object SQLite with migration guidance', () => { expect(() => createSqlAgentExecutionStore({}, 'FlueAssistantAgent')).toThrow( 'Add "FlueAssistantAgent" to a Wrangler migration\'s "new_sqlite_classes" list before its first deploy; do not use legacy "new_classes". Existing KV-backed Durable Object classes cannot be converted to SQLite in place.', From 5d53970d843bc126c7c3778c223c4b912eb01cfa Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" <622227+FredKSchott@users.noreply.github.com> Date: Thu, 4 Jun 2026 17:29:46 -0500 Subject: [PATCH 16/17] refactor(cloudflare): finalize durable submission boundary --- CHANGELOG.md | 3 +- packages/cli/src/lib/build.ts | 38 ----- .../src/cloudflare/agent-coordinator.ts | 49 ++---- .../src/cloudflare/agent-execution-store.ts | 140 ++++++++---------- packages/runtime/src/cloudflare/index.ts | 8 +- packages/runtime/src/internal.ts | 13 +- .../runtime/src/runtime/agent-submissions.ts | 2 +- packages/runtime/src/runtime/handle-agent.ts | 11 +- packages/runtime/src/session.ts | 4 +- .../test/cloudflare-agent-coordinator.test.ts | 16 +- .../cloudflare-agent-execution-store.test.ts | 30 ++-- ...dflare-agent-extension.integration.test.ts | 47 ------ packages/runtime/test/dispatch.test.ts | 26 ++-- .../runtime/test/package-entrypoints.test.ts | 2 - 14 files changed, 119 insertions(+), 270 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e444e1f1..30dbf41b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,9 @@ - **Agents: Clear or migrate persisted beta session state before upgrading.** Session records now persist one opaque `aff_` provider-affinity key instead of deriving affinity from agent instance, harness, and session names. This keeps prompt-cache and routing-affinity identifiers bounded and distinct for nested tasks. Existing version-4 beta session records are rejected; storage keys are unchanged. - **Cloudflare: Rename generated Durable Object identities.** Generated bindings now use `FLUE__AGENT`, `FLUE__WORKFLOW`, and `FLUE_REGISTRY`; generated classes use `FlueAgent`, `FlueWorkflow`, and `FlueRegistry`. Existing deployments must append authored Wrangler `renamed_classes` migrations for deployed agent and workflow classes and update authored direct binding access such as `env.Assistant` to `env.FLUE_ASSISTANT_AGENT`. - **Cloudflare: Use fresh SQLite-backed generated agent classes.** Generated agent Durable Objects now require SQLite for durable submission admission and ordering. Introduce classes through Wrangler `new_sqlite_classes`, not legacy `new_classes`. Already deployed KV-backed classes cannot be converted to SQLite in place. This pre-1.0 upgrade does not adopt in-flight direct prompts or dispatched inputs from earlier generated-agent identities; use a fresh class identity and treat the deployment as a hard execution-state boundary. -- **Cloudflare: Upgrade Cloudflare Agents SDK within the audited range.** Cloudflare builds require `agents >=0.14.1 <0.15.0`. +- **Cloudflare: Upgrade Cloudflare Agents SDK within the audited range.** Install `agents >=0.14.1 <0.15.0` for the published scheduling and Fiber behavior audited by this release. - **Cloudflare: Capture request metadata before durable direct admission.** Route middleware sees the original inbound request, but SQL-backed direct processing uses a deterministic internal `Request`. Do not rely on later operation-time `ctx.req` to preserve original headers, cookies, query parameters, URL, or body. +- **Cloudflare: Remove public agent WebSocket adapters.** `CloudflareAgentWebSocketOptions`, `connectCloudflareAgentWebSocket(...)`, and `messageCloudflareAgentWebSocket(...)` are now internal generated-agent coordinator details. Use generated Flue agent routing instead of importing these helpers from `@flue/runtime/cloudflare`. ### Fixes & Other Changes diff --git a/packages/cli/src/lib/build.ts b/packages/cli/src/lib/build.ts index 496b9263..4bc859d3 100644 --- a/packages/cli/src/lib/build.ts +++ b/packages/cli/src/lib/build.ts @@ -41,7 +41,6 @@ async function buildApplication(options: BuildOptions): Promise { const root = path.resolve(options.root); const output = path.resolve(options.output ?? path.join(root, 'dist')); const plugin: BuildPlugin = resolvePlugin(options); - if (!options.plugin && options.target === 'cloudflare') assertCloudflareAgentsSdkFloor(root); const sourceRoot = path.resolve(options.sourceRoot); @@ -430,43 +429,6 @@ function readRuntimeVersion(root: string): string { } } -function assertCloudflareAgentsSdkFloor(root: string): void { - const minimum = [0, 14, 1] as const; - const nextBreaking = [0, 15, 0] as const; - let entry: string; - try { - entry = createRequire(path.join(root, '__flue_resolve__.cjs')).resolve('agents'); - } catch { - throw new Error( - '[flue] Cloudflare target requires the installed "agents" package to satisfy >=0.14.1 <0.15.0. Install a compatible "agents" version in this project.', - ); - } - const pkgPath = packageUpSync({ cwd: path.dirname(entry) }); - if (!pkgPath) throw new Error('[flue] Could not inspect the installed "agents" package version.'); - let version: unknown; - try { - version = JSON.parse(fs.readFileSync(pkgPath, 'utf-8')).version; - } catch { - throw new Error('[flue] Could not inspect the installed "agents" package version.'); - } - const match = typeof version === 'string' ? /^(\d+)\.(\d+)\.(\d+)$/.exec(version) : null; - const current = match?.slice(1).map(Number); - if (!current || isVersionBelow(current, minimum) || !isVersionBelow(current, nextBreaking)) { - throw new Error( - `[flue] Cloudflare target requires the installed "agents" package to satisfy >=0.14.1 <0.15.0. Found ${String(version)}. Install a compatible "agents" version in this project.`, - ); - } -} - -function isVersionBelow(current: readonly number[], minimum: readonly number[]): boolean { - for (let index = 0; index < minimum.length; index++) { - const currentPart = current[index] ?? 0; - const minimumPart = minimum[index] ?? 0; - if (currentPart !== minimumPart) return currentPart < minimumPart; - } - return false; -} - function getCLIDir(): string { try { return path.dirname(new URL(import.meta.url).pathname); diff --git a/packages/runtime/src/cloudflare/agent-coordinator.ts b/packages/runtime/src/cloudflare/agent-coordinator.ts index 178072bc..a785b2ea 100644 --- a/packages/runtime/src/cloudflare/agent-coordinator.ts +++ b/packages/runtime/src/cloudflare/agent-coordinator.ts @@ -9,14 +9,12 @@ import { createAgentSubmissionTerminalHandler, type DirectAgentSubmissionInput, } from '../runtime/agent-submissions.ts'; -import type { DispatchInput } from '../runtime/dispatch-queue.ts'; -import { type AgentHandler, handleAgentRequest, validateAgentDispatchAdmission } from '../runtime/handle-agent.ts'; +import { type AgentHandler, assertAgentDispatchAdmissionInput, handleAgentRequest } from '../runtime/handle-agent.ts'; import type { AttachedAgentEvent, DirectAgentPayload } from '../types.ts'; import { createSqlAgentExecutionStore, type SqlAgentExecutionStore, type SqlAgentSubmission, - SqlAgentSubmissionConflictError, type SqlAgentSubmissionStore, type SubmissionAttemptRef, } from './agent-execution-store.ts'; @@ -602,7 +600,7 @@ class CloudflareAgentCoordinator { try { const agent = this.options.createdAgents[this.agentName]; if (!agent) throw new Error('[flue] Agent target unavailable during durable processing.'); - if (input.kind === 'dispatch') await validateAgentDispatchAdmission({ input }); + if (input.kind === 'dispatch') assertAgentDispatchAdmissionInput(input); const request = input.kind === 'direct' ? new Request( @@ -690,10 +688,7 @@ class CloudflareAgentCoordinator { private async admitDispatch(request: Request): Promise { const input: unknown = await request.json(); - await validateAgentDispatchAdmission({ input: input as DispatchInput }); - if (!isDispatchInput(input)) { - throw new Error('[flue] Internal dispatch admission received an invalid payload.'); - } + assertAgentDispatchAdmissionInput(input); if (input.agent !== this.agentName || input.id !== this.instance.name) { return new Response('Invalid internal dispatch target.', { status: 400 }); } @@ -702,22 +697,18 @@ class CloudflareAgentCoordinator { } this.cleanupTerminalState(); await this.armSubmissionWake(); - try { - const admission = this.submissions.admitDispatch(input); - if (admission.kind === 'retained_receipt') { - return Response.json({ - dispatchId: admission.receipt.submissionId, - acceptedAt: new Date(admission.receipt.acceptedAt).toISOString(), - }); - } - await this.reconcileSubmissions({ driverAlreadyArmed: true }); - return Response.json({ dispatchId: admission.submission.submissionId, acceptedAt: input.acceptedAt }); - } catch (error) { - if (error instanceof SqlAgentSubmissionConflictError) { - return new Response('Conflicting internal dispatch replay.', { status: 409 }); - } - throw error; + const admission = this.submissions.admitDispatch(input); + if (admission.kind === 'retained_receipt') { + return Response.json({ + dispatchId: admission.receipt.submissionId, + acceptedAt: new Date(admission.receipt.acceptedAt).toISOString(), + }); } + if (admission.kind === 'conflict') { + return new Response('Conflicting internal dispatch replay.', { status: 409 }); + } + await this.reconcileSubmissions({ driverAlreadyArmed: true }); + return Response.json({ dispatchId: admission.submission.submissionId, acceptedAt: input.acceptedAt }); } private acceptSocket(request: Request): Response { @@ -740,18 +731,6 @@ function isAttemptMarkerSnapshot(value: unknown): value is { submissionId: strin return typeof snapshot.submissionId === 'string' && typeof snapshot.attemptId === 'string'; } -function isDispatchInput(value: unknown): value is DispatchInput { - if (!value || typeof value !== 'object') return false; - const input = value as Partial; - return ( - typeof input.dispatchId === 'string' && - typeof input.agent === 'string' && - typeof input.id === 'string' && - typeof input.session === 'string' && - typeof input.acceptedAt === 'string' - ); -} - function submissionAttemptRef(submission: SqlAgentSubmission): SubmissionAttemptRef | undefined { return submission.attemptId ? { submissionId: submission.submissionId, attemptId: submission.attemptId } diff --git a/packages/runtime/src/cloudflare/agent-execution-store.ts b/packages/runtime/src/cloudflare/agent-execution-store.ts index 9abb2376..95044451 100644 --- a/packages/runtime/src/cloudflare/agent-execution-store.ts +++ b/packages/runtime/src/cloudflare/agent-execution-store.ts @@ -23,7 +23,7 @@ interface DurableObjectStorage { transactionSync?(closure: () => T): T; } -type SqlAgentSubmissionStatus = 'queued' | 'running' | 'recording_interruption' | 'completed' | 'failed'; +type SqlAgentSubmissionStatus = 'queued' | 'running' | 'recording_interruption' | 'settled'; interface SqlAgentDispatchReceipt { readonly submissionId: string; @@ -32,7 +32,8 @@ interface SqlAgentDispatchReceipt { type SqlAgentDispatchAdmission = | { readonly kind: 'submission'; readonly submission: SqlAgentSubmission } - | { readonly kind: 'retained_receipt'; readonly receipt: SqlAgentDispatchReceipt }; + | { readonly kind: 'retained_receipt'; readonly receipt: SqlAgentDispatchReceipt } + | { readonly kind: 'conflict' }; export interface SubmissionAttemptRef { readonly submissionId: string; @@ -42,7 +43,6 @@ export interface SubmissionAttemptRef { export interface SqlAgentSubmission { readonly sequence: number; readonly submissionId: string; - readonly session: string; readonly sessionKey: string; readonly kind: 'dispatch' | 'direct'; readonly input: AgentSubmissionInput; @@ -52,7 +52,6 @@ export interface SqlAgentSubmission { readonly inputAppliedAt?: number; readonly recoveryRequestedAt?: number; readonly startedAt?: number; - readonly completedAt?: number; readonly error?: string; } @@ -80,8 +79,6 @@ export interface SqlAgentExecutionStore { readonly submissions: SqlAgentSubmissionStore; } -export class SqlAgentSubmissionConflictError extends Error {} - export function createSqlSessionStore(sql: SqlStorage): SessionStore { ensureSessionTable(sql); return new SqlSessionStore(sql); @@ -178,7 +175,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { admitDirect(input: DirectAgentSubmissionInput): SqlAgentSubmission { const admission = this.admitSubmission(input); if (admission.kind !== 'submission') { - throw new Error('[flue] Internal direct admission returned an unexpected retained receipt.'); + throw new Error('[flue] Internal direct admission returned an unexpected result.'); } return admission.submission; } @@ -267,14 +264,14 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { this.transactionSync(() => { this.sql.exec( `INSERT OR IGNORE INTO flue_agent_dispatch_receipts (dispatch_id, accepted_at, settled_at) - SELECT submission_id, accepted_at, completed_at + SELECT submission_id, accepted_at, settled_at FROM flue_agent_submissions - WHERE session_key = ? AND kind = 'dispatch' AND status IN ('completed', 'failed')`, + WHERE session_key = ? AND kind = 'dispatch' AND status = 'settled'`, sessionKey, ); this.sql.exec( `DELETE FROM flue_agent_submissions - WHERE session_key = ? AND status IN ('completed', 'failed')`, + WHERE session_key = ? AND status = 'settled'`, sessionKey, ); this.sql.exec('DELETE FROM flue_agent_session_deletions WHERE session_key = ?', sessionKey); @@ -287,7 +284,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { } } - cleanupTerminalSubmissions(completedBefore: number, limit = 100): number { + cleanupTerminalSubmissions(settledBefore: number, limit = 100): number { if (!Number.isInteger(limit) || limit <= 0) { throw new Error('[flue] Terminal submission cleanup limit must be a positive integer.'); } @@ -295,10 +292,10 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { .exec( `SELECT sequence FROM flue_agent_submissions - WHERE status IN ('completed', 'failed') AND completed_at < ? - ORDER BY completed_at ASC, sequence ASC + WHERE status = 'settled' AND settled_at < ? + ORDER BY settled_at ASC, sequence ASC LIMIT ?`, - completedBefore, + settledBefore, limit, ) .toArray(); @@ -308,9 +305,9 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { } this.sql.exec( `DELETE FROM flue_agent_submissions - WHERE sequence = ? AND status IN ('completed', 'failed') AND completed_at < ?`, + WHERE sequence = ? AND status = 'settled' AND settled_at < ?`, row.sequence, - completedBefore, + settledBefore, ); } this.sql.exec( @@ -322,56 +319,56 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { ORDER BY settled_at ASC, dispatch_id ASC LIMIT ? )`, - completedBefore, + settledBefore, limit, ); return rows.length; } claimSubmission(attempt: SubmissionAttemptRef): SqlAgentSubmission | null { - this.sql.exec( - `UPDATE flue_agent_submissions AS current - SET status = 'running', attempt_id = ?, started_at = ? - WHERE current.submission_id = ? AND current.status = 'queued' - AND NOT EXISTS ( - SELECT 1 - FROM flue_agent_submissions AS earlier - WHERE earlier.session_key = current.session_key - AND earlier.status IN ('queued', 'running', 'recording_interruption') - AND earlier.sequence < current.sequence - )`, - attempt.attemptId, - Date.now(), - attempt.submissionId, - ); - const submission = this.getSubmission(attempt.submissionId); - return submission?.status === 'running' && submission.attemptId === attempt.attemptId - ? submission - : null; + const row = this.sql + .exec( + `UPDATE flue_agent_submissions AS current + SET status = 'running', attempt_id = ?, started_at = ? + WHERE current.submission_id = ? AND current.status = 'queued' + AND NOT EXISTS ( + SELECT 1 + FROM flue_agent_submissions AS earlier + WHERE earlier.session_key = current.session_key + AND earlier.status IN ('queued', 'running', 'recording_interruption') + AND earlier.sequence < current.sequence + ) + RETURNING ${submissionColumns}`, + attempt.attemptId, + Date.now(), + attempt.submissionId, + ) + .toArray()[0]; + return row ? parseSubmission(row) : null; } markSubmissionInputApplied(attempt: SubmissionAttemptRef): boolean { - this.sql.exec( + return this.updateOwnedSubmission( `UPDATE flue_agent_submissions SET input_applied_at = COALESCE(input_applied_at, ?) - WHERE submission_id = ? AND status = 'running' AND attempt_id = ?`, + WHERE submission_id = ? AND status = 'running' AND attempt_id = ? + RETURNING submission_id`, Date.now(), attempt.submissionId, attempt.attemptId, ); - return this.hasOwnedSubmission(attempt, 'running'); } requestSubmissionRecovery(attempt: SubmissionAttemptRef): boolean { - this.sql.exec( + return this.updateOwnedSubmission( `UPDATE flue_agent_submissions SET recovery_requested_at = COALESCE(recovery_requested_at, ?) - WHERE submission_id = ? AND status = 'running' AND attempt_id = ?`, + WHERE submission_id = ? AND status = 'running' AND attempt_id = ? + RETURNING submission_id`, Date.now(), attempt.submissionId, attempt.attemptId, ); - return this.hasOwnedSubmission(attempt, 'running'); } requeueSubmissionBeforeInputApplied(attempt: SubmissionAttemptRef): boolean { @@ -391,52 +388,52 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { } beginSubmissionInterruptionRecording(attempt: SubmissionAttemptRef): boolean { - this.sql.exec( + return this.updateOwnedSubmission( `UPDATE flue_agent_submissions SET status = 'recording_interruption' - WHERE submission_id = ? AND status = 'running' AND attempt_id = ?`, + WHERE submission_id = ? AND status = 'running' AND attempt_id = ? + RETURNING submission_id`, attempt.submissionId, attempt.attemptId, ); - return this.hasOwnedSubmission(attempt, 'recording_interruption'); } completeSubmission(attempt: SubmissionAttemptRef): boolean { - this.sql.exec( + return this.updateOwnedSubmission( `UPDATE flue_agent_submissions - SET status = 'completed', completed_at = ?, error = NULL - WHERE submission_id = ? AND status = 'running' AND attempt_id = ?`, + SET status = 'settled', settled_at = ?, error = NULL + WHERE submission_id = ? AND status = 'running' AND attempt_id = ? + RETURNING submission_id`, Date.now(), attempt.submissionId, attempt.attemptId, ); - return this.hasOwnedSubmission(attempt, 'completed'); } failSubmission(attempt: SubmissionAttemptRef, error: unknown): boolean { - this.sql.exec( + return this.updateOwnedSubmission( `UPDATE flue_agent_submissions - SET status = 'failed', completed_at = ?, error = ? - WHERE submission_id = ? AND status = 'running' AND attempt_id = ?`, + SET status = 'settled', settled_at = ?, error = ? + WHERE submission_id = ? AND status = 'running' AND attempt_id = ? + RETURNING submission_id`, Date.now(), error instanceof Error ? error.message : String(error), attempt.submissionId, attempt.attemptId, ); - return this.hasOwnedSubmission(attempt, 'failed'); } finishSubmissionInterruptionRecording(attempt: SubmissionAttemptRef, error: unknown): boolean { - this.sql.exec( + return this.updateOwnedSubmission( `UPDATE flue_agent_submissions - SET status = 'failed', completed_at = ?, error = ? - WHERE submission_id = ? AND status = 'recording_interruption' AND attempt_id = ?`, + SET status = 'settled', settled_at = ?, error = ? + WHERE submission_id = ? AND status = 'recording_interruption' AND attempt_id = ? + RETURNING submission_id`, Date.now(), error instanceof Error ? error.message : String(error), attempt.submissionId, attempt.attemptId, ); - return this.hasOwnedSubmission(attempt, 'failed'); } private admitSubmission(input: AgentSubmissionInput): SqlAgentDispatchAdmission { @@ -457,10 +454,9 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { } this.sql.exec( `INSERT OR IGNORE INTO flue_agent_submissions - (submission_id, session, session_key, kind, payload, status, accepted_at) - VALUES (?, ?, ?, ?, ?, 'queued', ?)`, + (submission_id, session_key, kind, payload, status, accepted_at) + VALUES (?, ?, ?, ?, 'queued', ?)`, submissionId, - input.session, sessionKey, kind, payload, @@ -468,16 +464,13 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { ); const row = this.readSubmissionRow(submissionId); if (!row) throw new Error(`[flue] Durable ${kind} admission did not create a submission row.`); - if (row.kind !== kind || row.payload !== payload) { - throw new SqlAgentSubmissionConflictError(`[flue] Conflicting internal ${kind} replay.`); - } + if (row.kind !== kind || row.payload !== payload) return { kind: 'conflict' }; return { kind: 'submission', submission: parseSubmission(row) }; }); } - private hasOwnedSubmission(attempt: SubmissionAttemptRef, status: SqlAgentSubmissionStatus): boolean { - const submission = this.getSubmission(attempt.submissionId); - return submission?.status === status && submission.attemptId === attempt.attemptId; + private updateOwnedSubmission(query: string, ...bindings: unknown[]): boolean { + return this.sql.exec(query, ...bindings).toArray().length > 0; } private parseOperationalRows( @@ -499,7 +492,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { private failSubmissionSequence(sequence: number, status: 'queued' | 'active', error: unknown): void { this.sql.exec( `UPDATE flue_agent_submissions - SET status = 'failed', completed_at = ?, error = ? + SET status = 'settled', settled_at = ?, error = ? WHERE sequence = ? AND ${status === 'queued' ? "status = 'queued'" : "status IN ('running', 'recording_interruption')"}`, Date.now(), error instanceof Error ? error.message : String(error), @@ -521,7 +514,7 @@ class SqlAgentSubmissionStoreImpl implements SqlAgentSubmissionStore { } const submissionColumns = - 'sequence, submission_id, session, session_key, kind, payload, status, accepted_at, attempt_id, input_applied_at, recovery_requested_at, started_at, completed_at, error'; + 'sequence, submission_id, session_key, kind, payload, status, accepted_at, attempt_id, input_applied_at, recovery_requested_at, started_at, error'; function submissionColumnsFor(table: string): string { return submissionColumns @@ -534,15 +527,13 @@ function parseSubmission(row: SqlRow): SqlAgentSubmission { if ( typeof row.sequence !== 'number' || typeof row.submission_id !== 'string' || - typeof row.session !== 'string' || typeof row.session_key !== 'string' || (row.kind !== 'dispatch' && row.kind !== 'direct') || typeof row.payload !== 'string' || (row.status !== 'queued' && row.status !== 'running' && row.status !== 'recording_interruption' && - row.status !== 'completed' && - row.status !== 'failed') || + row.status !== 'settled') || typeof row.accepted_at !== 'number' || (row.attempt_id !== null && row.attempt_id !== undefined && typeof row.attempt_id !== 'string') || (row.input_applied_at !== null && @@ -569,7 +560,6 @@ function parseSubmission(row: SqlRow): SqlAgentSubmission { return { sequence: row.sequence, submissionId: row.submission_id, - session: row.session, sessionKey: row.session_key, kind: row.kind, input, @@ -581,7 +571,6 @@ function parseSubmission(row: SqlRow): SqlAgentSubmission { ? { recoveryRequestedAt: row.recovery_requested_at } : {}), ...(typeof row.started_at === 'number' ? { startedAt: row.started_at } : {}), - ...(typeof row.completed_at === 'number' ? { completedAt: row.completed_at } : {}), ...(typeof row.error === 'string' ? { error: row.error } : {}), }; } @@ -598,7 +587,6 @@ function isSubmissionPayload(input: unknown, row: SqlRow): input is AgentSubmiss typeof dispatch.agent === 'string' && typeof dispatch.id === 'string' && typeof dispatch.session === 'string' && - dispatch.session === row.session && createSessionStorageKey(dispatch.id, 'default', dispatch.session) === row.session_key && typeof dispatch.acceptedAt === 'string' && Date.parse(dispatch.acceptedAt) === row.accepted_at && @@ -611,7 +599,6 @@ function isSubmissionPayload(input: unknown, row: SqlRow): input is AgentSubmiss typeof direct.agent === 'string' && typeof direct.id === 'string' && typeof direct.session === 'string' && - direct.session === row.session && createSessionStorageKey(direct.id, 'default', direct.session) === row.session_key && typeof direct.acceptedAt === 'string' && Date.parse(direct.acceptedAt) === row.accepted_at && @@ -651,7 +638,6 @@ function ensureSubmissionTable(sql: SqlStorage): void { `CREATE TABLE IF NOT EXISTS flue_agent_submissions ( sequence INTEGER PRIMARY KEY AUTOINCREMENT, submission_id TEXT NOT NULL UNIQUE, - session TEXT NOT NULL, session_key TEXT NOT NULL, kind TEXT NOT NULL, payload TEXT NOT NULL, @@ -661,7 +647,7 @@ function ensureSubmissionTable(sql: SqlStorage): void { input_applied_at INTEGER, recovery_requested_at INTEGER, started_at INTEGER, - completed_at INTEGER, + settled_at INTEGER, error TEXT )`, ); diff --git a/packages/runtime/src/cloudflare/index.ts b/packages/runtime/src/cloudflare/index.ts index 3b8bbf02..4d7a34d1 100644 --- a/packages/runtime/src/cloudflare/index.ts +++ b/packages/runtime/src/cloudflare/index.ts @@ -41,16 +41,10 @@ export { FlueRegistry } from './registry-do.ts'; export { createCloudflareRunRegistry } from './run-registry.ts'; export { store } from './session-store.ts'; export type { - CloudflareAgentWebSocketOptions, CloudflareWebSocketAttachment, CloudflareWebSocketConnection, CloudflareWorkflowWebSocketOptions, } from './websocket.ts'; -export { - connectCloudflareAgentWebSocket, - connectCloudflareWorkflowWebSocket, - messageCloudflareAgentWebSocket, - messageCloudflareWorkflowWebSocket, -} from './websocket.ts'; +export { connectCloudflareWorkflowWebSocket, messageCloudflareWorkflowWebSocket } from './websocket.ts'; export { getCloudflareAIBindingApiProvider } from './workers-ai-provider.ts'; diff --git a/packages/runtime/src/internal.ts b/packages/runtime/src/internal.ts index 70034c0a..ea2dadc2 100644 --- a/packages/runtime/src/internal.ts +++ b/packages/runtime/src/internal.ts @@ -22,11 +22,7 @@ export { createFlueContext } from './client.ts'; // they pull in `cloudflare:workers`, a virtual module Node can't resolve. // The generated CF entry imports them from there directly. export { CLOUDFLARE_AGENT_INTERNAL_DISPATCH_PATH, createCloudflareAgentRuntime } from './cloudflare/agent-coordinator.ts'; -export { - createSqlAgentExecutionStore, - createSqlSessionStore, - SqlAgentSubmissionConflictError, -} from './cloudflare/agent-execution-store.ts'; +export { createSqlSessionStore } from './cloudflare/agent-execution-store.ts'; export { createDurableRunStore } from './cloudflare/run-store.ts'; export { InMemoryRunRegistry } from './node/run-registry.ts'; export { InMemoryRunStore } from './node/run-store.ts'; @@ -57,11 +53,6 @@ export type { } from './runtime/handle-agent.ts'; // Runtime modules consumed by the generated server entries. // -// - `handleAgentRequest` handles attached per-agent HTTP prompts (SSE / -// sync). Used directly by the Cloudflare entry's `dispatchAgent` -// wrapper to layer in DO-specific keepalive handling. The -// Node target reaches the same dispatcher through `flue()`. -// // - `configureFlueRuntime` seeds the module-scoped config that // `flue()` reads at request time. Called once per generated entry, // before the listener (Node) or `default.fetch` (Cloudflare) takes @@ -77,11 +68,9 @@ export { createAgentDispatchProcessor, createDirectAgentHandler, failRecoveredRun, - handleAgentRequest, handleWorkflowRequest, invokeDirectAttached, invokeWorkflowAttached, - validateAgentDispatchAdmission, } from './runtime/handle-agent.ts'; export type { HandleRunRouteOptions } from './runtime/handle-run-routes.ts'; export { handleRunRouteRequest } from './runtime/handle-run-routes.ts'; diff --git a/packages/runtime/src/runtime/agent-submissions.ts b/packages/runtime/src/runtime/agent-submissions.ts index d1733585..5234e17f 100644 --- a/packages/runtime/src/runtime/agent-submissions.ts +++ b/packages/runtime/src/runtime/agent-submissions.ts @@ -30,7 +30,7 @@ export interface AgentSubmissionInterruption { readonly message: string; } -export type AgentSubmissionInspection = 'absent' | 'applied' | 'completed' | 'advanced'; +export type AgentSubmissionInspection = 'absent' | 'completed' | 'uncertain'; export interface ProcessAgentSubmissionOptions { onInputApplied?: () => Promise | void; diff --git a/packages/runtime/src/runtime/handle-agent.ts b/packages/runtime/src/runtime/handle-agent.ts index b2aef45b..94f72627 100644 --- a/packages/runtime/src/runtime/handle-agent.ts +++ b/packages/runtime/src/runtime/handle-agent.ts @@ -15,7 +15,6 @@ import type { AttachedAgentEventCallback, CreatedAgent, DirectAgentPayload, - DispatchReceipt, FlueEvent, FlueEventCallback, } from '../types.ts'; @@ -72,14 +71,7 @@ export function createAgentDispatchProcessor(options: { }; } -interface ValidateAgentDispatchAdmissionOptions { - input: DispatchInput; -} - -export async function validateAgentDispatchAdmission( - options: ValidateAgentDispatchAdmissionOptions, -): Promise { - const { input } = options; +export function assertAgentDispatchAdmissionInput(input: unknown): asserts input is DispatchInput { if (!isDispatchInput(input)) throw new Error('[flue] Internal dispatch admission received an invalid payload.'); if (isTaskSessionName(input.session)) { @@ -87,7 +79,6 @@ export async function validateAgentDispatchAdmission( '[flue] Internal dispatch admission session names beginning with "task:" are reserved for delegated tasks.', ); } - return { dispatchId: input.dispatchId, acceptedAt: input.acceptedAt }; } async function reserveDispatchAgentSession( diff --git a/packages/runtime/src/session.ts b/packages/runtime/src/session.ts index cf7a9107..ae561508 100644 --- a/packages/runtime/src/session.ts +++ b/packages/runtime/src/session.ts @@ -2089,12 +2089,12 @@ export class Session implements FlueSession { if (!inputEntry) return 'absent'; const following = this.history.getActivePathSince(inputEntry.id); if (following.some((entry) => entry.type === 'message' && entry.message.role === 'user')) { - return 'advanced'; + return 'uncertain'; } const assistant = following.findLast( (entry): entry is MessageEntry => entry.type === 'message' && entry.message.role === 'assistant', )?.message as AssistantMessage | undefined; - return assistant && isCompletedAssistantResponse(assistant) ? 'completed' : 'applied'; + return assistant && isCompletedAssistantResponse(assistant) ? 'completed' : 'uncertain'; } private async runPersistedDispatchInput( diff --git a/packages/runtime/test/cloudflare-agent-coordinator.test.ts b/packages/runtime/test/cloudflare-agent-coordinator.test.ts index 8b79c7c5..a8d2541f 100644 --- a/packages/runtime/test/cloudflare-agent-coordinator.test.ts +++ b/packages/runtime/test/cloudflare-agent-coordinator.test.ts @@ -24,7 +24,7 @@ function makeFakeSql(events: string[] = []) { if (query.includes('SET recovery_requested_at')) events.push('request-recovery'); if (query.includes("SET status = 'queued'")) events.push('requeue'); if (query.includes("SET status = 'recording_interruption'")) events.push('begin-interruption-recording'); - if (query.includes("SET status = 'failed', completed_at")) events.push('finish-interruption-recording'); + if (query.includes("SET status = 'settled', settled_at")) events.push('finish-interruption-recording'); const stmt = db.prepare(query); let rows: unknown[]; try { @@ -105,7 +105,7 @@ function makeRecoveryContext(options: { throw new Error('Unexpected submission processing.'); }, inspectSubmissionInput() { - return options.inspection ?? 'applied'; + return options.inspection ?? 'uncertain'; }, async recordSubmissionTerminal(input: AgentSubmissionInterruption) { options.events?.push('record-terminal'); @@ -272,7 +272,7 @@ describe('createCloudflareAgentRuntime()', () => { it('records interruption before settling applied incomplete canonical input as error', async () => { const events: string[] = []; const { storage } = makeFakeSql(events); - const recovery = makeRecoveryContext({ inspection: 'applied', events }); + const recovery = makeRecoveryContext({ inspection: 'uncertain', events }); const payloads: unknown[] = []; const runtime = makeRuntime({ createdAgent: {} as never, @@ -291,7 +291,7 @@ describe('createCloudflareAgentRuntime()', () => { expect(events).toEqual(['begin-interruption-recording', 'record-terminal', 'finish-interruption-recording']); expect(payloads).toEqual([directInput(), directInput().payload]); - expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ status: 'failed' }); + expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ status: 'settled' }); }); it('resumes recording interruption rows by recording interruption before final SQL settlement', async () => { @@ -315,7 +315,7 @@ describe('createCloudflareAgentRuntime()', () => { await runtime.onStart(instance, () => {}); expect(events).toEqual(['record-terminal', 'finish-interruption-recording']); - expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ status: 'failed' }); + expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ status: 'settled' }); }); it('settles interrupted attempts when canonical completion is already persisted', async () => { @@ -333,7 +333,7 @@ describe('createCloudflareAgentRuntime()', () => { await runtime.onStart(instance, () => {}); - expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ status: 'completed' }); + expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ status: 'settled' }); }); it('starts unrelated queued sessions when interrupted-session inspection fails', async () => { @@ -377,7 +377,7 @@ describe('createCloudflareAgentRuntime()', () => { expect(executionStore.submissions.getSubmission('direct-1')).toMatchObject({ status: 'running' }); await vi.waitFor(() => { - expect(executionStore.submissions.getSubmission('direct-2')).toMatchObject({ status: 'completed' }); + expect(executionStore.submissions.getSubmission('direct-2')).toMatchObject({ status: 'settled' }); }); expect(consoleError).toHaveBeenCalledWith( '[flue:submission-reconciliation]', @@ -513,6 +513,6 @@ describe('createCloudflareAgentRuntime()', () => { await runtime.onStart(instance, () => {}); expect(payloads).toEqual([dispatchInput()]); - expect(executionStore.submissions.getSubmission('dispatch-1')).toMatchObject({ status: 'completed' }); + expect(executionStore.submissions.getSubmission('dispatch-1')).toMatchObject({ status: 'settled' }); }); }); diff --git a/packages/runtime/test/cloudflare-agent-execution-store.test.ts b/packages/runtime/test/cloudflare-agent-execution-store.test.ts index d1375e21..ff517790 100644 --- a/packages/runtime/test/cloudflare-agent-execution-store.test.ts +++ b/packages/runtime/test/cloudflare-agent-execution-store.test.ts @@ -115,7 +115,6 @@ describe('createSqlAgentExecutionStore()', () => { ).toEqual([ { name: 'sequence' }, { name: 'submission_id' }, - { name: 'session' }, { name: 'session_key' }, { name: 'kind' }, { name: 'payload' }, @@ -125,7 +124,7 @@ describe('createSqlAgentExecutionStore()', () => { { name: 'input_applied_at' }, { name: 'recovery_requested_at' }, { name: 'started_at' }, - { name: 'completed_at' }, + { name: 'settled_at' }, { name: 'error' }, ]); expect( @@ -180,21 +179,20 @@ describe('createSqlAgentExecutionStore()', () => { kind: 'submission', submission: { submissionId: 'dispatch-1', - session: 'default', sessionKey: 'agent-session:["agent-1","default","default"]', status: 'queued', }, }); }); - it('rejects conflicting replay when one dispatch id is reused with another payload', () => { + it('returns conflict when one dispatch id is reused with another payload', () => { const { sql, transactionSync } = makeFakeSql(); const store = createSqlAgentExecutionStore({ sql, transactionSync }, 'FlueAssistantAgent'); store.submissions.admitDispatch(dispatchInput()); - expect(() => - store.submissions.admitDispatch(dispatchInput({ input: { text: 'Different' } })), - ).toThrow('[flue] Conflicting internal dispatch replay.'); + expect(store.submissions.admitDispatch(dispatchInput({ input: { text: 'Different' } }))).toEqual({ + kind: 'conflict', + }); }); it('orders direct and dispatched submissions together within one session', () => { @@ -258,9 +256,9 @@ describe('createSqlAgentExecutionStore()', () => { store.submissions.admitDispatch(dispatchInput({ dispatchId: 'healthy' })); db.prepare( `INSERT INTO flue_agent_submissions - (submission_id, session, session_key, kind, payload, status, accepted_at) - VALUES (?, ?, ?, 'dispatch', ?, 'queued', ?)`, - ).run('malformed', 'other', 'agent-session:["agent-1","default","other"]', '{', 1); + (submission_id, session_key, kind, payload, status, accepted_at) + VALUES (?, ?, 'dispatch', ?, 'queued', ?)`, + ).run('malformed', 'agent-session:["agent-1","default","other"]', '{', 1); expect(store.submissions.listRunnableSubmissions()).toEqual([ expect.objectContaining({ submissionId: 'healthy' }), @@ -269,7 +267,7 @@ describe('createSqlAgentExecutionStore()', () => { db .prepare('SELECT status, error FROM flue_agent_submissions WHERE submission_id = ?') .get('malformed'), - ).toMatchObject({ status: 'failed', error: expect.any(String) }); + ).toMatchObject({ status: 'settled', error: expect.any(String) }); }); it('terminalizes impossible queued input markers instead of replaying them', () => { @@ -283,7 +281,7 @@ describe('createSqlAgentExecutionStore()', () => { expect(store.submissions.listRunnableSubmissions()).toEqual([]); expect(store.submissions.getSubmission('dispatch-1')).toMatchObject({ - status: 'failed', + status: 'settled', error: expect.any(String), }); }); @@ -341,7 +339,7 @@ describe('createSqlAgentExecutionStore()', () => { store.submissions.completeSubmission(attempt('dispatch-1', 'attempt-1')); expect(store.submissions.hasUnsettledSubmissions()).toBe(false); expect(store.submissions.listRunningSubmissions()).toEqual([]); - expect(store.submissions.getSubmission('dispatch-1')).toMatchObject({ status: 'completed' }); + expect(store.submissions.getSubmission('dispatch-1')).toMatchObject({ status: 'settled' }); }); it('ignores stale-attempt settlement and keeps the first owning terminal dispatch state', () => { @@ -356,7 +354,7 @@ describe('createSqlAgentExecutionStore()', () => { store.submissions.failSubmission(attempt('dispatch-1', 'attempt-1'), new Error('later failure')); expect(store.submissions.getSubmission('dispatch-1')).toMatchObject({ - status: 'failed', + status: 'settled', error: 'first failure', }); }); @@ -381,7 +379,7 @@ describe('createSqlAgentExecutionStore()', () => { ), ).toBe(true); expect(store.submissions.getSubmission('dispatch-1')).toMatchObject({ - status: 'failed', + status: 'settled', error: 'interrupted', }); }); @@ -514,7 +512,7 @@ describe('createSqlAgentExecutionStore()', () => { store.submissions.claimSubmission(attempt('expired-2', 'attempt-2')); store.submissions.completeSubmission(attempt('expired-1', 'attempt-1')); store.submissions.failSubmission(attempt('expired-2', 'attempt-2'), new Error('failed')); - db.prepare("UPDATE flue_agent_submissions SET completed_at = 1 WHERE status IN ('completed', 'failed')").run(); + db.prepare("UPDATE flue_agent_submissions SET settled_at = 1 WHERE status = 'settled'").run(); expect(store.submissions.cleanupTerminalSubmissions(2, 1)).toBe(1); expect(store.submissions.cleanupTerminalSubmissions(2, 1)).toBe(1); diff --git a/packages/runtime/test/cloudflare-agent-extension.integration.test.ts b/packages/runtime/test/cloudflare-agent-extension.integration.test.ts index dae666fb..f6761b68 100644 --- a/packages/runtime/test/cloudflare-agent-extension.integration.test.ts +++ b/packages/runtime/test/cloudflare-agent-extension.integration.test.ts @@ -12,41 +12,6 @@ import { } from '../../cli/src/lib/build.ts'; describe('Cloudflare agent extension', () => { - it('rejects Cloudflare builds below the audited Agents SDK durability floor', async () => { - const root = createAgentsFloorFixture('0.14.0'); - try { - await expect( - build({ root, sourceRoot: path.join(root, 'src'), target: 'cloudflare', mode: 'development' }), - ).rejects.toThrow( - '[flue] Cloudflare target requires the installed "agents" package to satisfy >=0.14.1 <0.15.0. Found 0.14.0. Install a compatible "agents" version in this project.', - ); - } finally { - fs.rmSync(root, { recursive: true, force: true }); - } - }); - - it('accepts the audited Agents SDK durability floor for Cloudflare builds', async () => { - const root = createAgentsFloorFixture('0.14.1'); - try { - await expect( - build({ root, sourceRoot: path.join(root, 'src'), target: 'cloudflare', mode: 'development' }), - ).rejects.toThrow('[flue] No agent or workflow files found.'); - } finally { - fs.rmSync(root, { recursive: true, force: true }); - } - }); - - it('does not apply the Agents SDK durability floor to Node builds', async () => { - const root = createAgentsFloorFixture('0.14.0'); - try { - await expect( - build({ root, sourceRoot: path.join(root, 'src'), target: 'node' }), - ).rejects.toThrow('[flue] No agent or workflow files found.'); - } finally { - fs.rmSync(root, { recursive: true, force: true }); - } - }); - it('runs inherited scheduled callbacks when an agent module extends its base class', async () => { const root = await createGeneratedFixture(); let server: Awaited> | undefined; @@ -159,18 +124,6 @@ describe('Cloudflare agent extension', () => { }, 90000); }); -function createAgentsFloorFixture(version: string): string { - const root = fs.mkdtempSync(path.join(os.tmpdir(), 'flue-cloudflare-agents-floor-')); - fs.mkdirSync(path.join(root, 'node_modules', 'agents'), { recursive: true }); - fs.mkdirSync(path.join(root, 'src'), { recursive: true }); - fs.writeFileSync( - path.join(root, 'node_modules', 'agents', 'package.json'), - JSON.stringify({ name: 'agents', version, main: 'index.js' }), - ); - fs.writeFileSync(path.join(root, 'node_modules', 'agents', 'index.js'), 'module.exports = {};\n'); - return root; -} - async function createGeneratedFixture( agentSource = defaultAgentSource, mount = '', diff --git a/packages/runtime/test/dispatch.test.ts b/packages/runtime/test/dispatch.test.ts index 959e4eb2..646a4609 100644 --- a/packages/runtime/test/dispatch.test.ts +++ b/packages/runtime/test/dispatch.test.ts @@ -14,7 +14,6 @@ import { InMemoryDispatchQueue, InMemorySessionStore, resetFlueRuntimeForTests, - validateAgentDispatchAdmission, } from '../src/internal.ts'; import { createAgentSubmissionHandler, @@ -24,6 +23,7 @@ import { createDispatchAgentSubmissionInput, type DirectAgentSubmissionInput, } from '../src/runtime/agent-submissions.ts'; +import { assertAgentDispatchAdmissionInput } from '../src/runtime/handle-agent.ts'; import type { AgentConfig, FlueHarness, FlueSession } from '../src/types.ts'; import { createNoopSessionEnv } from './fixtures/session-env.ts'; @@ -310,19 +310,17 @@ describe('dispatch()', () => { ).rejects.toThrow('session names beginning with "task:" are reserved for delegated tasks'); }); - it('rejects a reserved task session name when durable dispatch admission receives internal input', async () => { - await expect( - validateAgentDispatchAdmission({ - input: { - dispatchId: 'dispatch:task-session', - agent: 'moderator', - id: 'guild:task-session', - session: 'task:default:child', - input: null, - acceptedAt: '2026-06-02T00:00:00.000Z', - }, + it('rejects a reserved task session name when durable dispatch admission receives internal input', () => { + expect(() => + assertAgentDispatchAdmissionInput({ + dispatchId: 'dispatch:task-session', + agent: 'moderator', + id: 'guild:task-session', + session: 'task:default:child', + input: null, + acceptedAt: '2026-06-02T00:00:00.000Z', }), - ).rejects.toThrow('session names beginning with "task:" are reserved for delegated tasks'); + ).toThrow('session names beginning with "task:" are reserved for delegated tasks'); }); it('rejects calls when the runtime has no dispatch queue', async () => { @@ -869,7 +867,7 @@ describe('dispatched session processing', () => { defaultStore: new InMemorySessionStore(), }); - await expect(createAgentSubmissionInspectionHandler(agent, createDispatchAgentSubmissionInput(input))(ctx)).resolves.toBe('applied'); + await expect(createAgentSubmissionInspectionHandler(agent, createDispatchAgentSubmissionInput(input))(ctx)).resolves.toBe('uncertain'); expect(provider.state.callCount).toBe(0); }); diff --git a/packages/runtime/test/package-entrypoints.test.ts b/packages/runtime/test/package-entrypoints.test.ts index fa153422..905ae9bf 100644 --- a/packages/runtime/test/package-entrypoints.test.ts +++ b/packages/runtime/test/package-entrypoints.test.ts @@ -65,13 +65,11 @@ describe('package entrypoints', () => { expect(cloudflare).toMatchObject({ cfSandboxToSessionEnv: expect.any(Function), - connectCloudflareAgentWebSocket: expect.any(Function), connectCloudflareWorkflowWebSocket: expect.any(Function), createCloudflareRunRegistry: expect.any(Function), extend: expect.any(Function), FlueRegistry: expect.any(Function), getCloudflareAIBindingApiProvider: expect.any(Function), - messageCloudflareAgentWebSocket: expect.any(Function), messageCloudflareWorkflowWebSocket: expect.any(Function), runWithCloudflareContext: expect.any(Function), }); From f247fe853f4510d5d4761ddb571e6e03f6fcd2e4 Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" <622227+FredKSchott@users.noreply.github.com> Date: Thu, 4 Jun 2026 17:34:59 -0500 Subject: [PATCH 17/17] docs: add durable execution guide --- .../content/docs/guide/durable-execution.md | 83 +++++++++++++++++++ apps/docs/src/lib/docs-navigation.ts | 1 + 2 files changed, 84 insertions(+) create mode 100644 apps/docs/src/content/docs/guide/durable-execution.md diff --git a/apps/docs/src/content/docs/guide/durable-execution.md b/apps/docs/src/content/docs/guide/durable-execution.md new file mode 100644 index 00000000..77d41ec5 --- /dev/null +++ b/apps/docs/src/content/docs/guide/durable-execution.md @@ -0,0 +1,83 @@ +--- +title: Durable Execution +description: Understand how Flue agents and workflows handle server restarts, interrupted connections, and other disruptions. +--- + +Durable execution is about recovering safely when running work is disrupted by a server restart, deployment, lost connection, or unexpected failure. Flue handles that recovery differently for continuing agents and finite workflows. + +## Durable Agents + +Agents are continuing, stateful contexts. An agent instance can own named sessions, and each session records conversation history so later operations can continue from where earlier work ended. The next message may arrive immediately or months later. + +Direct prompts and asynchronous `dispatch(...)` inputs are operations inside these continuing sessions. They are not workflow runs. When you need to send application-owned events such as webhooks or chat messages to an agent, see [Message-Driven Agents](/docs/guide/message-driven-agents/). + +```txt +agent input → stored session history → operation completes + ↓ +later input → reopens the same session → continues with earlier context +``` + +### Persist session history + +A stored session includes messages and compacted context needed to reopen the conversation later. This makes the session history the durable record that lets an agent continue working after an earlier operation has finished. + +Return a custom store through the `persist` option when conversation history belongs in an application-controlled database. See the [Data Persistence API](/docs/api/data-persistence-api/) for the public storage contract. + +### Durable Agents on Cloudflare + +On Cloudflare, generated Durable Object-backed agents store session history in SQLite by default. They also protect accepted agent input while it is being processed. Direct HTTP, SSE, and WebSocket prompts and asynchronous `dispatch(...)` inputs enter the same durable queue for their session. Inputs for one session keep their accepted order, while separate sessions can progress independently. + +```txt +direct HTTP, SSE, or WebSocket prompt ─┐ + ├→ durable per-session queue → stored session history +dispatch(...) input ────────────────────┘ +``` + +The connection that submitted a prompt observes the work but does not own it. If an HTTP response, SSE stream, or WebSocket closes after Cloudflare accepts the prompt, the backend work can continue. Flue does not reconstruct the lost connection or replay missed direct-agent stream events. + +When the Cloudflare runtime is interrupted, Flue checks the stored input and session history before deciding what to do next. It starts work again only when it can prove that the input was not applied. If a completed response was already stored, Flue recognizes that completion. When the outcome is uncertain, Flue records a visible interruption message in the session instead of blindly repeating model or tool activity. + +This recovery is intentionally conservative. Once model or tool activity may have started, repeating it could duplicate external effects such as creating a ticket, posting a reply, or sending a payment request. Use application-owned idempotency keys where repeated effects would be harmful. For dispatched input, use `dispatchId` to correlate one accepted delivery with application records. + +See [Deploy Agents on Cloudflare](/docs/ecosystem/deploy/cloudflare/) for Durable Object configuration, migrations, and platform-specific recovery details. + +### Durable Agents on Node.js + +On Node.js, sessions live in process memory by default and are lost when the process restarts. Return a custom store through the `persist` option when conversation history must survive restarts or multiple application replicas. + +Persisting session history does not make accepted agent input durable while it is being processed. The generated Node.js target keeps its `dispatch(...)` queue in process memory, and direct prompts remain attached to their connection. If your application needs stronger guarantees, provide them through application-owned infrastructure appropriate to your deployment. + +See [Deploy Agents on Node.js](/docs/ecosystem/deploy/node/) for session persistence setup and deployment guidance. + +### Keep workspace state separate + +Persisting a conversation does not make sandbox files durable. The default virtual sandbox is an in-memory workspace, even when the session history is stored in a database. Likewise, a durable remote workspace does not preserve conversation history by itself. + +Use the [Sandboxes](/docs/guide/sandboxes/) guide to choose a workspace lifecycle separately from session persistence. Keep durable application data, such as customer records or ticket state, in your own data layer. + +## Durable Workflows + +Workflows are finite function invocations. Each invocation runs your authored `run(...)` function once and receives its own `runId`. A workflow may load data, call external services, initialize agents, and return a result or error. + +Flue workflows are not resumable. If a workflow is interrupted, Flue does not checkpoint arbitrary TypeScript execution and continue the function from the last completed line or step. Your application decides whether starting the workflow again is appropriate. + +### Retry workflows explicitly + +Design workflows so they can be invoked again when retry is appropriate, much like CI jobs. Make repeated steps safe where practical, and use application-owned idempotency keys around external effects whose earlier outcome may be unknown. + +Starting a workflow again creates a new invocation. It does not continue the previous function call. + +```txt +workflow invocation → run(...) → result or error + +interrupted invocation + └→ start a new invocation when retry is appropriate +``` + +If a job requires checkpointed steps that resume automatically after disruption, use a durable orchestration system appropriate to your deployment. + +### Inspect workflow runs + +Use a workflow's `runId` to inspect its recorded outcome and events independently of the connection that started it. This is useful for debugging, live progress, and operational tooling. + +Agent prompts and dispatched agent input do not create workflow runs. Use agent operation observation for continuing agents, and reserve workflow history and `flue logs` for workflow invocations. See [Workflows](/docs/guide/workflows/) for authoring and run inspection, and [Observability](/docs/guide/observability/) for runtime events and telemetry. diff --git a/apps/docs/src/lib/docs-navigation.ts b/apps/docs/src/lib/docs-navigation.ts index 4584b780..a276d379 100644 --- a/apps/docs/src/lib/docs-navigation.ts +++ b/apps/docs/src/lib/docs-navigation.ts @@ -45,6 +45,7 @@ export const docsSections: DocsSection[] = [ { title: 'Agents', slug: 'guide/building-agents' }, { title: 'Message-Driven Agents', slug: 'guide/message-driven-agents' }, { title: 'Workflows', slug: 'guide/workflows' }, + { title: 'Durable Execution', slug: 'guide/durable-execution' }, { title: 'Skills', slug: 'guide/skills' }, { title: 'Tools', slug: 'guide/tools' }, { title: 'Subagents', slug: 'guide/subagents' },