diff --git a/src/api/providers/__tests__/anthropic-vertex.spec.ts b/src/api/providers/__tests__/anthropic-vertex.spec.ts index 3e98f3ec5b..32af2394eb 100644 --- a/src/api/providers/__tests__/anthropic-vertex.spec.ts +++ b/src/api/providers/__tests__/anthropic-vertex.spec.ts @@ -1248,7 +1248,7 @@ describe("VertexHandler", () => { expect(mockCreate).toHaveBeenCalledWith( expect.objectContaining({ - thinking: { type: "adaptive" }, + thinking: { type: "adaptive", display: "summarized" }, }), undefined, ) @@ -1279,7 +1279,7 @@ describe("VertexHandler", () => { expect(mockCreate).toHaveBeenCalledWith( expect.objectContaining({ - thinking: { type: "adaptive" }, + thinking: { type: "adaptive", display: "summarized" }, }), undefined, ) @@ -1312,7 +1312,7 @@ describe("VertexHandler", () => { expect(mockCreate).toHaveBeenCalledWith( expect.objectContaining({ - thinking: { type: "adaptive" }, + thinking: { type: "adaptive", display: "summarized" }, }), undefined, ) @@ -1343,7 +1343,7 @@ describe("VertexHandler", () => { expect(mockCreate).toHaveBeenCalledWith( expect.objectContaining({ - thinking: { type: "adaptive" }, + thinking: { type: "adaptive", display: "summarized" }, }), undefined, ) @@ -1352,6 +1352,112 @@ describe("VertexHandler", () => { expect(request.thinking).not.toHaveProperty("budget_tokens") expect(request.temperature).toBeUndefined() }) + + it("should surface thinking_tokens from output_tokens_details in usage chunks", async () => { + const thinkingTokensHandler = new AnthropicVertexHandler({ + apiModelId: "claude-opus-4-8", + vertexProjectId: "test-project", + vertexRegion: "us-central1", + enableReasoningEffort: true, + }) + + const mockCreate = vitest.fn().mockImplementation(async () => + asyncStreamFrom([ + { + type: "message_start", + message: { + usage: { + input_tokens: 100, + output_tokens: 10, + output_tokens_details: { + thinking_tokens: 5, + }, + }, + }, + }, + { + type: "message_delta", + usage: { + output_tokens: 200, + output_tokens_details: { + thinking_tokens: 150, + }, + }, + }, + ]), + ) + // Object.assign avoids the SDK's streaming-overload cast that a direct property assignment would require + Object.assign(thinkingTokensHandler["client"].messages, { create: mockCreate }) + + const stream = thinkingTokensHandler.createMessage("You are a helpful assistant", [ + { role: "user", content: "Hello" }, + ]) + const chunks = await collectStream(stream) + + expect(chunks).toEqual([ + { + type: "usage", + inputTokens: 100, + outputTokens: 10, + reasoningTokens: 5, + }, + { + type: "usage", + inputTokens: 0, + outputTokens: 200, + reasoningTokens: 150, + }, + ]) + }) + + it("should omit reasoningTokens when output_tokens_details.thinking_tokens is unset", async () => { + const noThinkingDetailsHandler = new AnthropicVertexHandler({ + apiModelId: "claude-opus-4-8", + vertexProjectId: "test-project", + vertexRegion: "us-central1", + enableReasoningEffort: true, + }) + + const mockCreate = vitest.fn().mockImplementation(async () => + asyncStreamFrom([ + { + type: "message_start", + message: { + usage: { + input_tokens: 100, + output_tokens: 10, + }, + }, + }, + { + type: "message_delta", + usage: { + output_tokens: 200, + }, + }, + ]), + ) + // Object.assign avoids the SDK's streaming-overload cast that a direct property assignment would require + Object.assign(noThinkingDetailsHandler["client"].messages, { create: mockCreate }) + + const stream = noThinkingDetailsHandler.createMessage("You are a helpful assistant", [ + { role: "user", content: "Hello" }, + ]) + const chunks = await collectStream(stream) + + expect(chunks).toEqual([ + { + type: "usage", + inputTokens: 100, + outputTokens: 10, + }, + { + type: "usage", + inputTokens: 0, + outputTokens: 200, + }, + ]) + }) }) describe("native tool calling", () => { diff --git a/src/api/providers/__tests__/anthropic.spec.ts b/src/api/providers/__tests__/anthropic.spec.ts index 21d2816ec7..1896fcb9f8 100644 --- a/src/api/providers/__tests__/anthropic.spec.ts +++ b/src/api/providers/__tests__/anthropic.spec.ts @@ -3,6 +3,7 @@ import { AnthropicHandler } from "../anthropic" import { ApiHandlerOptions } from "../../../shared/api" import { asyncStreamFrom, collectStream } from "../../../test-utils/stream" +import type { ApiStreamChunk } from "../../../api/transform/stream" import { clearAllMocks } from "../../../test-utils/reset" // Mock TelemetryService @@ -247,7 +248,7 @@ describe("AnthropicHandler", () => { await collectStream(stream) const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] - expect(requestBody?.thinking).toEqual({ type: "adaptive" }) + expect(requestBody?.thinking).toEqual({ type: "adaptive", display: "summarized" }) expect(requestBody?.max_tokens).toBe(16384) }) @@ -290,7 +291,7 @@ describe("AnthropicHandler", () => { await collectStream(stream) const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] - expect(requestBody?.thinking).toEqual({ type: "adaptive" }) + expect(requestBody?.thinking).toEqual({ type: "adaptive", display: "summarized" }) expect(requestBody?.max_tokens).toBe(32768) }) @@ -334,7 +335,7 @@ describe("AnthropicHandler", () => { await collectStream(stream) const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] - expect(requestBody?.thinking).toEqual({ type: "adaptive" }) + expect(requestBody?.thinking).toEqual({ type: "adaptive", display: "summarized" }) expect(requestBody?.max_tokens).toBe(16384) }) @@ -377,7 +378,7 @@ describe("AnthropicHandler", () => { await collectStream(stream) const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] - expect(requestBody?.thinking).toEqual({ type: "adaptive" }) + expect(requestBody?.thinking).toEqual({ type: "adaptive", display: "summarized" }) expect(requestBody?.max_tokens).toBe(32768) }) @@ -400,7 +401,7 @@ describe("AnthropicHandler", () => { const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] const requestOptions = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[1] - expect(requestBody?.thinking).toEqual({ type: "adaptive" }) + expect(requestBody?.thinking).toEqual({ type: "adaptive", display: "summarized" }) expect(requestBody?.temperature).toBeUndefined() expect(requestBody?.max_tokens).toBe(32768) expect(requestOptions?.headers?.["anthropic-beta"]).toContain("prompt-caching-2024-07-31") @@ -425,7 +426,7 @@ describe("AnthropicHandler", () => { const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] const requestOptions = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[1] - expect(requestBody?.thinking).toEqual({ type: "adaptive" }) + expect(requestBody?.thinking).toEqual({ type: "adaptive", display: "summarized" }) expect(requestBody?.temperature).toBeUndefined() expect(requestBody?.max_tokens).toBe(32768) expect(requestOptions?.headers?.["anthropic-beta"]).toContain("prompt-caching-2024-07-31") @@ -450,7 +451,7 @@ describe("AnthropicHandler", () => { const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] const requestOptions = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[1] - expect(requestBody?.thinking).toEqual({ type: "adaptive" }) + expect(requestBody?.thinking).toEqual({ type: "adaptive", display: "summarized" }) expect(requestBody?.temperature).toBeUndefined() expect(requestBody?.max_tokens).toBe(32768) expect(requestOptions?.headers?.["anthropic-beta"]).toContain("prompt-caching-2024-07-31") @@ -474,7 +475,163 @@ describe("AnthropicHandler", () => { const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] expect(requestBody?.model).toBe("claude-sonnet-5-bf") - expect(requestBody?.thinking).toEqual({ type: "adaptive" }) + expect(requestBody?.thinking).toEqual({ type: "adaptive", display: "summarized" }) + }) + + it("should surface thinking_tokens from output_tokens_details in usage chunks", async () => { + // Adaptive models report reasoning tokens inside + // `output_tokens_details.thinking_tokens` (message_start snapshot is + // typically absent; message_delta carries the final decomposition). + mockCreate.mockImplementationOnce(async () => + asyncStreamFrom([ + { + type: "message_start", + message: { + usage: { + input_tokens: 100, + output_tokens: 50, + }, + }, + }, + { + type: "content_block_start", + index: 0, + content_block: { + type: "text", + text: "Hello", + }, + }, + { + type: "content_block_delta", + index: 0, + delta: { + type: "text_delta", + text: " world", + }, + }, + { + type: "message_delta", + usage: { + output_tokens: 200, + output_tokens_details: { + thinking_tokens: 150, + }, + }, + delta: { + stop_reason: "end_turn", + stop_sequence: null, + }, + }, + ]), + ) + + const adaptiveHandler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-opus-4-7", + enableReasoningEffort: true, + }) + + const stream = adaptiveHandler.createMessage("prompt", [ + { + role: "user", + content: [{ type: "text", text: "Hi" }], + }, + ]) + + const chunks: ApiStreamChunk[] = await collectStream(stream) + + // message_start snapshot carries no thinking decomposition + const usageChunks = chunks.filter( + (chunk): chunk is Extract => chunk.type === "usage", + ) + const startUsage = usageChunks.find((chunk) => chunk.inputTokens > 0) + expect(startUsage).toBeDefined() + expect(startUsage?.reasoningTokens).toBeUndefined() + + // message_delta surfaces the final reasoning token count + const deltaUsage = usageChunks.find((chunk) => chunk.inputTokens === 0) + expect(deltaUsage).toBeDefined() + expect(deltaUsage?.outputTokens).toBe(200) + expect(deltaUsage?.reasoningTokens).toBe(150) + }) + + it("should surface numeric thinking_tokens from the message_start usage snapshot", async () => { + // Some surfaces report the thinking decomposition as early as the + // message_start snapshot; assert the snapshot usage chunk surfaces + // reasoningTokens, and that a message_delta without + // output_tokens_details omits it. + mockCreate.mockImplementationOnce(async () => + asyncStreamFrom([ + { + type: "message_start", + message: { + usage: { + input_tokens: 100, + output_tokens: 10, + output_tokens_details: { + thinking_tokens: 5, + }, + }, + }, + }, + { + type: "content_block_start", + index: 0, + content_block: { + type: "text", + text: "Hello", + }, + }, + { + type: "content_block_delta", + index: 0, + delta: { + type: "text_delta", + text: " world", + }, + }, + { + type: "message_delta", + usage: { + output_tokens: 200, + }, + delta: { + stop_reason: "end_turn", + stop_sequence: null, + }, + }, + ]), + ) + + const adaptiveHandler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-opus-4-7", + enableReasoningEffort: true, + }) + + const stream = adaptiveHandler.createMessage("prompt", [ + { + role: "user", + content: [{ type: "text", text: "Hi" }], + }, + ]) + + const chunks: ApiStreamChunk[] = await collectStream(stream) + + const usageChunks = chunks.filter( + (chunk): chunk is Extract => chunk.type === "usage", + ) + + // message_start snapshot surfaces the reported thinking decomposition + const startUsage = usageChunks.find((chunk) => chunk.inputTokens > 0) + expect(startUsage).toBeDefined() + expect(startUsage?.reasoningTokens).toBe(5) + + // message_delta without output_tokens_details omits reasoningTokens + const deltaUsage = usageChunks.find((chunk) => chunk.inputTokens === 0) + expect(deltaUsage).toBeDefined() + expect(deltaUsage?.outputTokens).toBe(200) + expect(deltaUsage?.reasoningTokens).toBeUndefined() }) }) diff --git a/src/api/providers/__tests__/requesty.spec.ts b/src/api/providers/__tests__/requesty.spec.ts index c685da0ed2..e806e3e0fa 100644 --- a/src/api/providers/__tests__/requesty.spec.ts +++ b/src/api/providers/__tests__/requesty.spec.ts @@ -271,7 +271,7 @@ describe("RequestyHandler", () => { expect.objectContaining({ model: "anthropic/claude-fable-5", max_tokens: 32768, - thinking: { type: "adaptive" }, + thinking: { type: "adaptive", display: "summarized" }, temperature: undefined, }), ) @@ -304,7 +304,7 @@ describe("RequestyHandler", () => { expect.objectContaining({ model: "anthropic/claude-sonnet-5", max_tokens: 32768, - thinking: { type: "adaptive" }, + thinking: { type: "adaptive", display: "summarized" }, temperature: undefined, }), ) @@ -337,7 +337,7 @@ describe("RequestyHandler", () => { expect.objectContaining({ model: "anthropic/claude-opus-5", max_tokens: 32768, - thinking: { type: "adaptive" }, + thinking: { type: "adaptive", display: "summarized" }, temperature: undefined, }), ) diff --git a/src/api/providers/anthropic-vertex.ts b/src/api/providers/anthropic-vertex.ts index 7b72b1100b..5d21ac3ffd 100644 --- a/src/api/providers/anthropic-vertex.ts +++ b/src/api/providers/anthropic-vertex.ts @@ -123,21 +123,30 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple case "message_start": { const usage = chunk.message!.usage + // `output_tokens_details.thinking_tokens` decomposes the billed + // output into internal reasoning tokens; surfaced for per-turn + // thinking telemetry (billing already includes it in output_tokens). yield { type: "usage", inputTokens: usage.input_tokens || 0, outputTokens: usage.output_tokens || 0, cacheWriteTokens: usage.cache_creation_input_tokens || undefined, cacheReadTokens: usage.cache_read_input_tokens || undefined, + ...(typeof usage.output_tokens_details?.thinking_tokens === "number" + ? { reasoningTokens: usage.output_tokens_details!.thinking_tokens } + : {}), } break } case "message_delta": { + const deltaThinkingTokens = chunk.usage!.output_tokens_details?.thinking_tokens + yield { type: "usage", inputTokens: 0, outputTokens: chunk.usage!.output_tokens || 0, + ...(typeof deltaThinkingTokens === "number" ? { reasoningTokens: deltaThinkingTokens } : {}), } break diff --git a/src/api/providers/anthropic.ts b/src/api/providers/anthropic.ts index b55c8b3089..71662f2b17 100644 --- a/src/api/providers/anthropic.ts +++ b/src/api/providers/anthropic.ts @@ -255,12 +255,19 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa cache_read_input_tokens, } = chunk.message.usage + // `output_tokens_details.thinking_tokens` decomposes the billed + // output into internal reasoning tokens (0 ⇒ the model skipped + // thinking this turn). `output_tokens` already includes them for + // billing; this field exists for per-turn thinking telemetry. + const startThinkingTokens = chunk.message.usage.output_tokens_details?.thinking_tokens + yield { type: "usage", inputTokens: input_tokens, outputTokens: output_tokens, cacheWriteTokens: cache_creation_input_tokens || undefined, cacheReadTokens: cache_read_input_tokens || undefined, + ...(typeof startThinkingTokens === "number" ? { reasoningTokens: startThinkingTokens } : {}), } inputTokens += input_tokens @@ -270,16 +277,22 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa break } - case "message_delta": + case "message_delta": { // Tells us stop_reason, stop_sequence, and output tokens // along the way and at the end of the message. + // Carries the final `thinking_tokens` decomposition for the + // whole message (the `message_start` snapshot typically has none). + const deltaThinkingTokens = chunk.usage.output_tokens_details?.thinking_tokens + yield { type: "usage", inputTokens: 0, outputTokens: chunk.usage.output_tokens || 0, + ...(typeof deltaThinkingTokens === "number" ? { reasoningTokens: deltaThinkingTokens } : {}), } break + } case "message_stop": // No usage data, just an indicator that the message is done. break diff --git a/src/api/transform/__tests__/reasoning.spec.ts b/src/api/transform/__tests__/reasoning.spec.ts index 004df4e60c..af96896ec0 100644 --- a/src/api/transform/__tests__/reasoning.spec.ts +++ b/src/api/transform/__tests__/reasoning.spec.ts @@ -477,7 +477,7 @@ describe("reasoning.ts", () => { reasoningBudget: undefined, }) - expect(result).toEqual({ type: "adaptive" }) + expect(result).toEqual({ type: "adaptive", display: "summarized" }) }) it("should omit thinking for Claude Opus 4.7 when reasoning is disabled", () => { @@ -1241,7 +1241,7 @@ describe("reasoning.ts", () => { reasoningBudget: undefined, }) - expect(result).toEqual({ type: "adaptive" }) + expect(result).toEqual({ type: "adaptive", display: "summarized" }) }) it("should return correct types for OpenAI reasoning params", () => { diff --git a/src/api/transform/reasoning.ts b/src/api/transform/reasoning.ts index c51111125a..da2bdd6f30 100644 --- a/src/api/transform/reasoning.ts +++ b/src/api/transform/reasoning.ts @@ -18,7 +18,19 @@ export type RooReasoningParams = { } export type AnthropicReasoningParams = BetaThinkingConfigParam -export type AnthropicProviderReasoningParams = AnthropicReasoningParams | { type: "adaptive" } +/** + * Adaptive thinking models (the `supportsReasoningBinary` class) reject + * `budget_tokens` payloads and use `thinking: { type: "adaptive" }`. + * + * `display` is carried explicitly because the API default for this model + * class is `"omitted"` — thinking blocks then arrive as empty text (still + * billed, but invisible to the user). `"summarized"` returns the + * server-side summarized thinking text with no extra cost (billing is on + * full thinking tokens, not the summary). + */ +export type AnthropicProviderReasoningParams = + | AnthropicReasoningParams + | { type: "adaptive"; display?: "summarized" | "omitted" } export type OpenAiReasoningParams = { reasoning_effort: OpenAI.Chat.ChatCompletionCreateParams["reasoning_effort"] } @@ -118,7 +130,7 @@ export const getAnthropicProviderReasoning = ({ settings, }: GetModelReasoningOptions): AnthropicProviderReasoningParams | undefined => { if (model.supportsReasoningBinary && settings.enableReasoningEffort) { - return { type: "adaptive" } + return { type: "adaptive", display: "summarized" } } return getAnthropicReasoning({ model, reasoningBudget, settings })