diff --git a/CHANGELOG.md b/CHANGELOG.md index df5d075e0..45e3a37a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ `output-error` (#162, #441) - Store a step's reasoning once, in content, so long reasoning fits the document size limit (#418, #423) +- `agent.addToolOutput` saves a client-side tool's result next to its call, and + `createTool` accepts a tool without `execute` (#231) ## 0.7.9 diff --git a/docs/tool-approval.mdx b/docs/tool-approval.mdx index aa6a0ea4f..ada4c92f7 100644 --- a/docs/tool-approval.mdx +++ b/docs/tool-approval.mdx @@ -218,6 +218,65 @@ The `ToolUIPart` states relevant to approval are: | `output-denied` | Tool was denied | | `output-error` | Tool execution failed | +## Client-side tools + +A tool without `execute` runs in the browser. Generation stops at its call, and +the tool part waits in the `input-available` state until the client answers it +with `addToolOutput`. Like an approval, the output is saved next to its call, +and a second answer is rejected. Naming the tool keeps a client from answering a +call to any other tool. A client tool cannot set `needsApproval`: ask the user +in the client before answering. + +```ts +const getLocation = createTool({ + description: "Get the user's location from their browser", + inputSchema: z.object({ precise: z.boolean() }), +}); + +export const submitToolOutput = mutation({ + args: { threadId: v.string(), toolCallId: v.string(), output: v.any() }, + handler: async (ctx, { threadId, toolCallId, output }) => { + // Check the user can access the thread. + const { messageId } = await agent.addToolOutput(ctx, { + threadId, + tool: "getLocation", + toolCallId, + output, + }); + // If no call in this step is still unanswered, schedule + // generateResponse with promptMessageId: messageId, as submitApproval + // does in the example files below. + }, +}); +``` + +The output is sent to the model as text (a string) or JSON; a tool's +`toModelOutput` is not applied. Pass `state: "output-error"` and `errorText` +instead of `output` to report a failure to the model. + +On the client, answer each call once its message is saved. A call seen only in +stream deltas is not saved yet, and `addToolOutput` rejects it: + +```tsx +const submitToolOutput = useMutation(api.chat.submitToolOutput); +const answered = useRef(new Set()); +useEffect(() => { + for (const message of messages) { + if (message.status !== "success") continue; + for (const part of message.parts) { + if (part.type !== "tool-getLocation") continue; + if (part.state !== "input-available") continue; + const { toolCallId } = part; + if (answered.current.has(toolCallId)) continue; + answered.current.add(toolCallId); + readLocation(part.input) + .then((output) => submitToolOutput({ threadId, toolCallId, output })) + .catch(() => answered.current.delete(toolCallId)); + } + } +}, [messages, threadId, submitToolOutput]); +``` + ## Example files For a complete working example, see: diff --git a/src/approvals.ts b/src/approvals.ts index 635d316a9..90b391a61 100644 --- a/src/approvals.ts +++ b/src/approvals.ts @@ -1,4 +1,5 @@ -import type { Message, MessageDoc } from "./validators.js"; +import type { Infer } from "convex/values"; +import type { Message, MessageDoc, vToolResultOutput } from "./validators.js"; export type ToolCallApprovalDecision = { approvalId: string; @@ -6,61 +7,127 @@ export type ToolCallApprovalDecision = { reason?: string; }; -// The window a batch is resolved against. Approvals are near the end of a -// thread, so one bounded page finds them. +export type ToolCallOutput = { + toolCallId: string; + toolName: string; + output: Infer; +}; + +// The window a batch is resolved against. Approvals and client tool calls are +// near the end of a thread, so one bounded page finds them. export const APPROVAL_LOOKUP_MESSAGES = 100; -export function planToolCallApprovals( +export function planToolCallResponses( messages: MessageDoc[], - decisions: ToolCallApprovalDecision[], + { + decisions, + outputs, + }: { decisions: ToolCallApprovalDecision[]; outputs: ToolCallOutput[] }, threadId: string, ): { requestMessageId: string; message: Extract } { - if (decisions.length === 0) { + if (decisions.length + outputs.length === 0) { throw new Error("An approval batch must contain at least one decision"); } - const pending = new Set(decisions.map(({ approvalId }) => approvalId)); - if (pending.size !== decisions.length) { + const pendingApprovals = new Set( + decisions.map(({ approvalId }) => approvalId), + ); + if (pendingApprovals.size !== decisions.length) { throw new Error("Duplicate approval ids in batch"); } + const pendingCalls = new Set(outputs.map(({ toolCallId }) => toolCallId)); + if (pendingCalls.size !== outputs.length) { + throw new Error("Duplicate tool call ids in batch"); + } let requestMessageId: string | undefined; + const claim = (doc: MessageDoc) => { + if (requestMessageId && requestMessageId !== doc._id) { + throw new Error( + "Approval decisions in one batch must belong to the same request message", + ); + } + requestMessageId = doc._id; + }; const providerExecuted = new Set(); + const answeredApprovals = new Set(); + const answeredCalls = new Set(); + // Newest first, so a call's answers are seen before the call itself. for (const doc of messages) { if (!Array.isArray(doc.message?.content)) continue; - const providerCalls = new Set( - doc.message.content.flatMap((part) => - part.type === "tool-call" && part.providerExecuted - ? [part.toolCallId] - : [], - ), - ); - for (const part of doc.message.content) { - if (!("approvalId" in part) || !pending.has(part.approvalId)) continue; + const content = doc.message.content; + for (const part of content) { if (part.type === "tool-approval-response") { - throw new Error(`Approval ${part.approvalId} was already handled`); + answeredApprovals.add(part.approvalId); + } else if (part.type === "tool-result") { + answeredCalls.add(part.toolCallId); } + } + if (doc.message.role !== "assistant") continue; + const providerCalls = new Set(); + const approvalRequests = new Map(); + for (const part of content) { + if (part.type === "tool-call" && part.providerExecuted) { + providerCalls.add(part.toolCallId); + } else if (part.type === "tool-approval-request") { + approvalRequests.set(part.toolCallId, part.approvalId); + } + } + for (const part of content) { if ( part.type === "tool-approval-request" && - doc.message.role === "assistant" + pendingApprovals.has(part.approvalId) ) { - if (requestMessageId && requestMessageId !== doc._id) { - throw new Error( - "Approval decisions in one batch must belong to the same request message", - ); + if ( + answeredApprovals.has(part.approvalId) || + answeredCalls.has(part.toolCallId) + ) { + throw new Error(`Approval ${part.approvalId} was already handled`); } - requestMessageId = doc._id; + claim(doc); if (providerCalls.has(part.toolCallId)) { providerExecuted.add(part.approvalId); } - pending.delete(part.approvalId); + pendingApprovals.delete(part.approvalId); + } else if ( + part.type === "tool-call" && + pendingCalls.has(part.toolCallId) + ) { + const approvalId = approvalRequests.get(part.toolCallId); + if ( + answeredCalls.has(part.toolCallId) || + (approvalId && answeredApprovals.has(approvalId)) + ) { + throw new Error(`Tool call ${part.toolCallId} was already handled`); + } + if (approvalId) { + throw new Error(`Tool call ${part.toolCallId} is awaiting approval`); + } + if (part.providerExecuted) { + throw new Error(`Tool call ${part.toolCallId} is provider-executed`); + } + const { toolName } = outputs.find( + ({ toolCallId }) => toolCallId === part.toolCallId, + )!; + if (part.toolName !== toolName) { + throw new Error( + `Tool call ${part.toolCallId} is a call to ${part.toolName}, not ${toolName}`, + ); + } + claim(doc); + pendingCalls.delete(part.toolCallId); } } - if (pending.size === 0) break; + if (pendingApprovals.size === 0 && pendingCalls.size === 0) break; } - if (pending.size > 0) { + if (pendingApprovals.size > 0) { + throw new Error( + `Approval request ${pendingApprovals.values().next().value} was not found in the last ${APPROVAL_LOOKUP_MESSAGES} messages of thread ${threadId}`, + ); + } + if (pendingCalls.size > 0) { throw new Error( - `Approval request ${pending.values().next().value} was not found in the last ${APPROVAL_LOOKUP_MESSAGES} messages of thread ${threadId}`, + `Tool call ${pendingCalls.values().next().value} was not found in the last ${APPROVAL_LOOKUP_MESSAGES} messages of thread ${threadId}`, ); } @@ -68,13 +135,23 @@ export function planToolCallApprovals( requestMessageId: requestMessageId!, message: { role: "tool", - content: decisions.map(({ approvalId, approved, reason }) => ({ - type: "tool-approval-response", - approvalId, - approved, - reason, - ...(providerExecuted.has(approvalId) ? { providerExecuted: true } : {}), - })), + content: [ + ...decisions.map(({ approvalId, approved, reason }) => ({ + type: "tool-approval-response" as const, + approvalId, + approved, + reason, + ...(providerExecuted.has(approvalId) + ? { providerExecuted: true } + : {}), + })), + ...outputs.map(({ toolCallId, toolName, output }) => ({ + type: "tool-result" as const, + toolCallId, + toolName, + output, + })), + ], }, }; } diff --git a/src/component/_generated/component.ts b/src/component/_generated/component.ts index dcd59237c..8f720e1bd 100644 --- a/src/component/_generated/component.ts +++ b/src/component/_generated/component.ts @@ -2954,6 +2954,116 @@ export type ComponentApi = approved: boolean; reason?: string; }>; + outputs?: Array<{ + output: + | { + providerOptions?: Record>; + type: "text"; + value: string; + } + | { + providerOptions?: Record>; + type: "json"; + value: any; + } + | { + providerOptions?: Record>; + type: "error-text"; + value: string; + } + | { + providerOptions?: Record>; + type: "error-json"; + value: any; + } + | { + providerOptions?: Record>; + reason?: string; + type: "execution-denied"; + } + | { + type: "content"; + value: Array< + | { + providerOptions?: Record>; + text: string; + type: "text"; + } + | { + data: + | { type: "url"; url: string } + | { + reference: Record; + type: "reference"; + } + | { + data: string | ArrayBuffer; + type: "data"; + } + | { text: string; type: "text" }; + filename?: string; + mediaType: string; + providerOptions?: Record>; + type: "file"; + } + | { + data: string; + mediaType: string; + type: "media"; + } + | { + data: string; + filename?: string; + mediaType: string; + providerOptions?: Record>; + type: "file-data"; + } + | { + mediaType?: string; + providerOptions?: Record>; + type: "file-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record>; + type: "file-id"; + } + | { + providerOptions?: Record>; + providerReference: Record; + type: "file-reference"; + } + | { + data: string; + mediaType: string; + providerOptions?: Record>; + type: "image-data"; + } + | { + providerOptions?: Record>; + providerReference: Record; + type: "image-file-reference"; + } + | { + providerOptions?: Record>; + type: "image-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record>; + type: "image-file-id"; + } + | { + providerOptions?: Record>; + type: "custom"; + } + >; + }; + toolCallId: string; + toolName: string; + }>; threadId: string; }, { messageId: string }, diff --git a/src/component/messages.ts b/src/component/messages.ts index 3339b5ff8..ccb109eb6 100644 --- a/src/component/messages.ts +++ b/src/component/messages.ts @@ -19,6 +19,7 @@ import { vMessageStatus, vMessageWithMetadataInternal, vPaginationResult, + vToolResultOutput, type MessageDoc, } from "../validators.js"; import { api, internal } from "./_generated/api.js"; @@ -50,7 +51,7 @@ import { import { partial } from "convex-helpers/validators"; import { APPROVAL_LOOKUP_MESSAGES, - planToolCallApprovals, + planToolCallResponses, } from "../approvals.js"; function publicMessage(message: Doc<"messages">): MessageDoc { @@ -612,6 +613,15 @@ export const respondToToolCallApprovals = mutation({ reason: v.optional(v.string()), }), ), + outputs: v.optional( + v.array( + v.object({ + toolCallId: v.string(), + toolName: v.string(), + output: vToolResultOutput, + }), + ), + ), }, returns: v.object({ messageId: v.id("messages") }), handler: async (ctx, args) => { @@ -620,9 +630,9 @@ export const respondToToolCallApprovals = mutation({ order: "desc", paginationOpts: { cursor: null, numItems: APPROVAL_LOOKUP_MESSAGES }, }); - const plan = planToolCallApprovals( + const plan = planToolCallResponses( page.page, - args.decisions, + { decisions: args.decisions, outputs: args.outputs ?? [] }, args.threadId, ); const saved = await addMessagesHandler(ctx, { diff --git a/src/vercel/client/clientTools.test.ts b/src/vercel/client/clientTools.test.ts new file mode 100644 index 000000000..3f5ec5dbd --- /dev/null +++ b/src/vercel/client/clientTools.test.ts @@ -0,0 +1,529 @@ +import { describe, expect, test } from "vitest"; +import { v } from "convex/values"; +import { stepCountIs } from "ai"; +import { z } from "zod/v4"; +import { + Agent, + createTool, + listMessages, + listUIMessages, + saveMessage, +} from "../index.js"; +import { app, components } from "./setup.test.js"; +import { mockModel, type MockLanguageModel } from "./mockModel.js"; + +const getLocation = createTool({ + description: "Get the user's location from their browser", + inputSchema: z.object({ precise: z.boolean() }), +}); + +const executed: string[] = []; +const deleteFile = createTool({ + inputSchema: z.object({ filename: z.string() }), + needsApproval: true, + execute: async (_ctx, { filename }) => { + executed.push(filename); + return `Deleted ${filename}`; + }, +}); + +const tools = { getLocation, deleteFile }; +const callIds: Record = { + getLocation: "tc-client", + deleteFile: "tc-delete", +}; + +const models = new Map(); +function agentFor(name: string, toolNames = ["getLocation"]) { + let model = models.get(name); + if (!model) { + model = mockModel({ + contentSteps: [ + toolNames.map((toolName) => ({ + type: "tool-call" as const, + toolCallId: callIds[toolName], + toolName, + input: JSON.stringify( + toolName === "deleteFile" + ? { filename: "a.txt" } + : { precise: true }, + ), + })), + [{ type: "text", text: "You are in Paris." }], + ], + }) as MockLanguageModel; + models.set(name, model); + } + return new Agent(components.agent, { + name, + tools, + languageModel: model, + stopWhen: stepCountIs(5), + }); +} + +const start = app.action({ + args: { + name: v.string(), + stream: v.boolean(), + toolNames: v.optional(v.array(v.string())), + }, + handler: async (ctx, { name, stream, toolNames }) => { + const agent = agentFor(name, toolNames); + const { threadId } = await agent.createThread(ctx, {}); + if (stream) { + const result = await agent.streamText( + ctx, + { threadId }, + { prompt: "Where am I?" }, + { saveStreamDeltas: { chunking: "word", throttleMs: 0 } }, + ); + await result.consumeStream(); + } else { + await agent.generateText(ctx, { threadId }, { prompt: "Where am I?" }); + } + return threadId; + }, +}); + +const cont = app.action({ + args: { + name: v.string(), + threadId: v.string(), + promptMessageId: v.string(), + stream: v.boolean(), + }, + handler: async (ctx, { name, threadId, promptMessageId, stream }) => { + const agent = agentFor(name); + let text: string; + if (stream) { + const result = await agent.streamText( + ctx, + { threadId }, + { promptMessageId }, + { saveStreamDeltas: { chunking: "word", throttleMs: 0 } }, + ); + await result.consumeStream(); + text = await result.text; + } else { + text = (await agent.generateText(ctx, { threadId }, { promptMessageId })) + .text; + } + const model = models.get(name)!; + const calls = stream ? model.doStreamCalls : model.doGenerateCalls; + return { text, prompt: JSON.parse(JSON.stringify(calls.at(-1)!.prompt)) }; + }, +}); + +const addOutput = app.mutation({ + args: { + threadId: v.string(), + tool: v.optional(v.string()), + toolCallId: v.string(), + output: v.optional(v.any()), + errorText: v.optional(v.string()), + }, + handler: async (ctx, args) => { + const { threadId, toolCallId, output, errorText } = args; + const tool = args.tool ?? "getLocation"; + const agent = agentFor("mutations"); + return errorText === undefined + ? agent.addToolOutput(ctx, { threadId, tool, toolCallId, output }) + : agent.addToolOutput(ctx, { + threadId, + tool, + toolCallId, + state: "output-error", + errorText, + }); + }, +}); + +const decide = app.mutation({ + args: { threadId: v.string(), approved: v.boolean() }, + handler: async (ctx, { threadId, approved }) => { + const agent = agentFor("mutations"); + const { page } = await listMessages(ctx, components.agent, { + threadId, + paginationOpts: { cursor: null, numItems: 10 }, + }); + const request = page + .flatMap((m) => + m.message?.role === "assistant" && Array.isArray(m.message.content) + ? m.message.content + : [], + ) + .find((p) => p.type === "tool-approval-request"); + const approvalId = (request as { approvalId: string }).approvalId; + return approved + ? agent.approveToolCall(ctx, { threadId, approvalId }) + : agent.denyToolCall(ctx, { threadId, approvalId }); + }, +}); + +const saveProviderCall = app.mutation({ + args: { threadId: v.string() }, + handler: async (ctx, { threadId }) => + saveMessage(ctx, components.agent, { + threadId, + message: { + role: "assistant", + content: [ + { + type: "tool-call", + toolCallId: "tc-provider", + toolName: "web_search", + input: {}, + providerExecuted: true, + }, + ], + }, + }), +}); + +const saveDeleteResult = app.mutation({ + args: { threadId: v.string() }, + handler: async (ctx, { threadId }) => + saveMessage(ctx, components.agent, { + threadId, + message: { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "tc-delete", + toolName: "deleteFile", + output: { type: "text", value: "Deleted a.txt" }, + }, + ], + }, + }), +}); + +const saveUserMessage = app.mutation({ + args: { threadId: v.string(), prompt: v.string() }, + handler: async (ctx, { threadId, prompt }) => + saveMessage(ctx, components.agent, { threadId, prompt }), +}); + +const messages = app.query({ + args: { threadId: v.string() }, + handler: async (ctx, { threadId }) => { + const { page } = await listMessages(ctx, components.agent, { + threadId, + paginationOpts: { cursor: null, numItems: 50 }, + }); + const ui = await listUIMessages(ctx, components.agent, { + threadId, + paginationOpts: { cursor: null, numItems: 50 }, + }); + return { + docs: page.map((m) => ({ _id: m._id, order: m.order })), + toolParts: ui.page + .flatMap((m) => m.parts) + .filter((p) => p.type === "tool-getLocation"), + }; + }, +}); + +const { api, createTest } = app.defineModules({ + clientTools: { + start, + cont, + addOutput, + decide, + saveProviderCall, + saveDeleteResult, + saveUserMessage, + messages, + }, +}); + +function promptRoles(prompt: any[]) { + return prompt + .filter((m) => m.role !== "system") + .map((m) => + [ + m.role, + ...(Array.isArray(m.content) ? m.content.map((p: any) => p.type) : []), + ].join(":"), + ); +} + +describe("addToolOutput", () => { + test("createTool without execute or outputSchema defines a client tool", () => { + expect(getLocation.execute).toBeUndefined(); + }); + + test("createTool rejects a client tool that needs approval", () => { + expect(() => + createTool({ + inputSchema: z.object({ precise: z.boolean() }), + needsApproval: true, + }), + ).toThrow(/cannot need approval/); + }); + + for (const stream of [false, true]) { + const mode = stream ? "streamText" : "generateText"; + + test(`${mode}: the output is saved next to its call and the continuation sees call then result`, async () => { + const t = createTest(); + const name = `order-${mode}`; + const threadId = await t.action(api.clientTools.start, { name, stream }); + const before = await t.query(api.clientTools.messages, { threadId }); + expect(before.toolParts).toEqual([ + expect.objectContaining({ + toolCallId: "tc-client", + state: "input-available", + input: { precise: true }, + }), + ]); + const callOrder = before.docs[0].order; + await t.mutation(api.clientTools.saveUserMessage, { + threadId, + prompt: "in between", + }); + + const { messageId } = await t.mutation(api.clientTools.addOutput, { + threadId, + toolCallId: "tc-client", + output: { city: "Paris" }, + }); + const after = await t.query(api.clientTools.messages, { threadId }); + expect(after.docs.find((m) => m._id === messageId)?.order).toBe( + callOrder, + ); + expect(after.toolParts).toEqual([ + expect.objectContaining({ + state: "output-available", + output: { city: "Paris" }, + }), + ]); + + const next = await t.action(api.clientTools.cont, { + name, + threadId, + promptMessageId: messageId, + stream, + }); + expect(next.text).toBe("You are in Paris."); + expect(promptRoles(next.prompt)).toEqual([ + "user:text", + "assistant:tool-call", + "tool:tool-result", + ]); + expect(next.prompt.at(-1).content[0]).toMatchObject({ + toolCallId: "tc-client", + toolName: "getLocation", + output: { type: "json", value: { city: "Paris" } }, + }); + }); + } + + test("errorText saves an error result the model sees", async () => { + const t = createTest(); + const name = "error-text"; + const threadId = await t.action(api.clientTools.start, { + name, + stream: false, + }); + const { messageId } = await t.mutation(api.clientTools.addOutput, { + threadId, + toolCallId: "tc-client", + errorText: "Permission denied", + }); + const after = await t.query(api.clientTools.messages, { threadId }); + expect(after.toolParts).toEqual([ + expect.objectContaining({ + state: "output-error", + errorText: "Permission denied", + }), + ]); + const next = await t.action(api.clientTools.cont, { + name, + threadId, + promptMessageId: messageId, + stream: false, + }); + expect(next.prompt.at(-1).content[0]).toMatchObject({ + toolCallId: "tc-client", + output: { type: "error-text", value: "Permission denied" }, + }); + }); + + test("a second output for the same call is rejected", async () => { + const t = createTest(); + const threadId = await t.action(api.clientTools.start, { + name: "twice", + stream: false, + }); + await t.mutation(api.clientTools.addOutput, { + threadId, + toolCallId: "tc-client", + output: { city: "Paris" }, + }); + await expect( + t.mutation(api.clientTools.addOutput, { + threadId, + toolCallId: "tc-client", + output: { city: "Rome" }, + }), + ).rejects.toThrow(/Tool call tc-client was already handled/); + }); + + for (const approved of [true, false]) { + test(`an output after ${approved ? "an approval" : "a denial"} is rejected`, async () => { + const t = createTest(); + const threadId = await t.action(api.clientTools.start, { + name: `after-${approved}`, + stream: false, + toolNames: ["deleteFile"], + }); + await t.mutation(api.clientTools.decide, { threadId, approved }); + await expect( + t.mutation(api.clientTools.addOutput, { + threadId, + tool: "deleteFile", + toolCallId: "tc-delete", + output: "done", + }), + ).rejects.toThrow(/Tool call tc-delete was already handled/); + }); + } + + for (const approved of [true, false]) { + test(`${approved ? "an approval" : "a denial"} after a result is rejected`, async () => { + const t = createTest(); + const threadId = await t.action(api.clientTools.start, { + name: `result-then-${approved}`, + stream: false, + toolNames: ["deleteFile"], + }); + await t.mutation(api.clientTools.saveDeleteResult, { threadId }); + await expect( + t.mutation(api.clientTools.decide, { threadId, approved }), + ).rejects.toThrow(/already handled/); + }); + } + + test("an output naming another tool is rejected", async () => { + const t = createTest(); + const threadId = await t.action(api.clientTools.start, { + name: "other-tool", + stream: false, + }); + await expect( + t.mutation(api.clientTools.addOutput, { + threadId, + tool: "deleteFile", + toolCallId: "tc-client", + output: "done", + }), + ).rejects.toThrow( + /Tool call tc-client is a call to getLocation, not deleteFile/, + ); + }); + + for (const approved of [true, false]) { + for (const outputFirst of [false, true]) { + test(`a step with a client call and ${approved ? "an approved" : "a denied"} call continues from the ${outputFirst ? "decision" : "output"}`, async () => { + const t = createTest(); + const name = `mixed-${approved}-${outputFirst}`; + const threadId = await t.action(api.clientTools.start, { + name, + stream: false, + toolNames: ["getLocation", "deleteFile"], + }); + const output = () => + t.mutation(api.clientTools.addOutput, { + threadId, + toolCallId: "tc-client", + output: { city: "Paris" }, + }); + const decision = () => + t.mutation(api.clientTools.decide, { threadId, approved }); + const { messageId } = outputFirst + ? (await output(), await decision()) + : (await decision(), await output()); + executed.length = 0; + + const next = await t.action(api.clientTools.cont, { + name, + threadId, + promptMessageId: messageId, + stream: false, + }); + expect(next.text).toBe("You are in Paris."); + expect(executed).toEqual(approved ? ["a.txt"] : []); + const results = next.prompt + .filter((m: any) => m.role === "tool") + .flatMap((m: any) => m.content) + .filter((p: any) => p.type === "tool-result"); + expect(results).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + toolCallId: "tc-client", + output: { type: "json", value: { city: "Paris" } }, + }), + expect.objectContaining({ + toolCallId: "tc-delete", + output: approved + ? { type: "text", value: "Deleted a.txt" } + : expect.objectContaining({ type: "execution-denied" }), + }), + ]), + ); + }); + } + } + + test("an output for a call awaiting approval is rejected", async () => { + const t = createTest(); + const threadId = await t.action(api.clientTools.start, { + name: "awaiting", + stream: false, + toolNames: ["deleteFile"], + }); + await expect( + t.mutation(api.clientTools.addOutput, { + threadId, + tool: "deleteFile", + toolCallId: "tc-delete", + output: "done", + }), + ).rejects.toThrow(/Tool call tc-delete is awaiting approval/); + }); + + test("an output for a provider-executed call is rejected", async () => { + const t = createTest(); + const threadId = await t.action(api.clientTools.start, { + name: "provider", + stream: false, + }); + await t.mutation(api.clientTools.saveProviderCall, { threadId }); + await expect( + t.mutation(api.clientTools.addOutput, { + threadId, + tool: "web_search", + toolCallId: "tc-provider", + output: {}, + }), + ).rejects.toThrow(/Tool call tc-provider is provider-executed/); + }); + + test("an unknown toolCallId is rejected", async () => { + const t = createTest(); + const threadId = await t.action(api.clientTools.start, { + name: "unknown", + stream: false, + }); + await expect( + t.mutation(api.clientTools.addOutput, { + threadId, + toolCallId: "nope", + output: {}, + }), + ).rejects.toThrow(/Tool call nope was not found in the last 100 messages/); + }); +}); diff --git a/src/vercel/client/createTool.ts b/src/vercel/client/createTool.ts index 6ef4a74ef..40eb00309 100644 --- a/src/vercel/client/createTool.ts +++ b/src/vercel/client/createTool.ts @@ -280,11 +280,12 @@ export function createTool< } const executeHandler = def.execute ?? (def as any).handler; - if (!executeHandler && !def.outputSchema) + if (!executeHandler && def.needsApproval) { throw new Error( - "To use a Convex tool, you must either provide an execute" + - " handler function, define an outputSchema, or both", + "A client tool (one without execute) cannot need approval." + + " Ask the user in the client before calling addToolOutput.", ); + } const t = tool({ type: "function", diff --git a/src/vercel/client/search.ts b/src/vercel/client/search.ts index fbbe4059a..e27865472 100644 --- a/src/vercel/client/search.ts +++ b/src/vercel/client/search.ts @@ -755,31 +755,36 @@ function approvalContinuation( searchMessages: MessageDoc[], recentMessages: MessageDoc[], ): { approvalIds: Set; storedContext: ModelMessage[] } | undefined { - if ( - promptMessage?.message?.role !== "tool" || - !promptMessage.message.content.some( - (part) => part.type === "tool-approval-response", - ) - ) { - return undefined; - } + if (promptMessage?.message?.role !== "tool") return undefined; const responseIds = new Set( promptMessage.message.content.flatMap((part) => part.type === "tool-approval-response" ? [part.approvalId] : [], ), ); + const resultCallIds = new Set( + promptMessage.message.content.flatMap((part) => + part.type === "tool-result" ? [part.toolCallId] : [], + ), + ); + // A client tool's result continues its step like a decision does when that + // step also requested approvals. const requestContents = recentMessages.flatMap((doc) => doc.order === promptMessage.order && doc.message?.role === "assistant" && Array.isArray(doc.message.content) && + doc.message.content.some((part) => part.type === "tool-approval-request") && doc.message.content.some( (part) => - part.type === "tool-approval-request" && - responseIds.has(part.approvalId), + (part.type === "tool-approval-request" && + responseIds.has(part.approvalId)) || + (part.type === "tool-call" && resultCallIds.has(part.toolCallId)), ) ? [doc.message.content] : [], ); + if (responseIds.size === 0 && requestContents.length === 0) { + return undefined; + } if (requestContents.length !== 1) { throw new Error( "Approval continuation requires one complete request message", diff --git a/src/vercel/index.ts b/src/vercel/index.ts index e0e74115d..d22bdc23d 100644 --- a/src/vercel/index.ts +++ b/src/vercel/index.ts @@ -1204,6 +1204,54 @@ export class Agent< }); } + /** + * Save the output of a client-side tool call (a tool without `execute`). + * Saves a `tool-result` message on the call's generation order, so it stays + * next to its call even if newer thread messages exist. Continue generation + * with `promptMessageId` set to the returned `messageId`. + * + * Rejects a call to another tool, a call that is not found or awaits + * approval, and a call that was already handled. + * + * @param ctx A ctx object from a mutation. + * @param args.threadId The thread containing the tool call. + * @param args.tool The name of the client tool that was called. + * @param args.toolCallId The ID of the tool call to answer. + * @param args.output The tool's output. Or pass `state: "output-error"` and + * `errorText` to save an error result. + * @returns The messageId of the saved tool result message. + */ + async addToolOutput( + ctx: MutationCtx, + { + threadId, + tool, + toolCallId, + ...result + }: { threadId: string; tool: string; toolCallId: string } & ( + | { state?: "output-available"; output: unknown; errorText?: never } + | { state: "output-error"; errorText: string; output?: never } + ), + ): Promise<{ messageId: string }> { + return ctx.runMutation(this.component.messages.respondToToolCallApprovals, { + threadId, + agentName: this.options.name, + decisions: [], + outputs: [ + { + toolCallId, + toolName: tool, + output: + result.state === "output-error" + ? { type: "error-text", value: result.errorText } + : typeof result.output === "string" + ? { type: "text", value: result.output } + : { type: "json", value: result.output ?? null }, + }, + ], + }); + } + /** * Explicitly save a "step" created by the AI SDK. For multi-step generation * loops, each SDK 7 step already owns its response messages.