Skip to content

fix(api): surface adaptive thinking display and thinking_tokens for Anthropic models #1327

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
easonLiangWorldedtech wants to merge 3 commits into
base: main
Choose a base branch
from
Open

fix(api): surface adaptive thinking display and thinking_tokens for Anthropic models #1327

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a02b0da
fix(api): send display summarized for adaptive thinking and parse thi…
easonliang28 Aug 21, 2026
0b5b245
test(requesty): update adaptive thinking assertions to include displa…
easonliang28 Aug 21, 2026
b2e6883
test(anthropic): add thinking_tokens message_start coverage per CodeR…
easonliang28 Aug 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 110 additions & 4 deletions src/api/providers/__tests__/anthropic-vertex.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1248,7 +1248,7 @@ describe("VertexHandler", () => {

expect(mockCreate).toHaveBeenCalledWith(
expect.objectContaining({
thinking: { type: "adaptive" },
thinking: { type: "adaptive", display: "summarized" },
}),
undefined,
)
Expand Down Expand Up @@ -1279,7 +1279,7 @@ describe("VertexHandler", () => {

expect(mockCreate).toHaveBeenCalledWith(
expect.objectContaining({
thinking: { type: "adaptive" },
thinking: { type: "adaptive", display: "summarized" },
}),
undefined,
)
Expand Down Expand Up @@ -1312,7 +1312,7 @@ describe("VertexHandler", () => {

expect(mockCreate).toHaveBeenCalledWith(
expect.objectContaining({
thinking: { type: "adaptive" },
thinking: { type: "adaptive", display: "summarized" },
}),
undefined,
)
Expand Down Expand Up @@ -1343,7 +1343,7 @@ describe("VertexHandler", () => {

expect(mockCreate).toHaveBeenCalledWith(
expect.objectContaining({
thinking: { type: "adaptive" },
thinking: { type: "adaptive", display: "summarized" },
}),
undefined,
)
Expand All @@ -1352,6 +1352,112 @@ describe("VertexHandler", () => {
expect(request.thinking).not.toHaveProperty("budget_tokens")
expect(request.temperature).toBeUndefined()
})

it("should surface thinking_tokens from output_tokens_details in usage chunks", async () => {
const thinkingTokensHandler = new AnthropicVertexHandler({
apiModelId: "claude-opus-4-8",
vertexProjectId: "test-project",
vertexRegion: "us-central1",
enableReasoningEffort: true,
})

const mockCreate = vitest.fn().mockImplementation(async () =>
asyncStreamFrom([
{
type: "message_start",
message: {
usage: {
input_tokens: 100,
output_tokens: 10,
output_tokens_details: {
thinking_tokens: 5,
},
},
},
},
{
type: "message_delta",
usage: {
output_tokens: 200,
output_tokens_details: {
thinking_tokens: 150,
},
},
},
]),
)
// Object.assign avoids the SDK's streaming-overload cast that a direct property assignment would require
Object.assign(thinkingTokensHandler["client"].messages, { create: mockCreate })

const stream = thinkingTokensHandler.createMessage("You are a helpful assistant", [
{ role: "user", content: "Hello" },
])
const chunks = await collectStream(stream)

expect(chunks).toEqual([
{
type: "usage",
inputTokens: 100,
outputTokens: 10,
reasoningTokens: 5,
},
{
type: "usage",
inputTokens: 0,
outputTokens: 200,
reasoningTokens: 150,
},
])
})

it("should omit reasoningTokens when output_tokens_details.thinking_tokens is unset", async () => {
const noThinkingDetailsHandler = new AnthropicVertexHandler({
apiModelId: "claude-opus-4-8",
vertexProjectId: "test-project",
vertexRegion: "us-central1",
enableReasoningEffort: true,
})

const mockCreate = vitest.fn().mockImplementation(async () =>
asyncStreamFrom([
{
type: "message_start",
message: {
usage: {
input_tokens: 100,
output_tokens: 10,
},
},
},
{
type: "message_delta",
usage: {
output_tokens: 200,
},
},
]),
)
// Object.assign avoids the SDK's streaming-overload cast that a direct property assignment would require
Object.assign(noThinkingDetailsHandler["client"].messages, { create: mockCreate })

const stream = noThinkingDetailsHandler.createMessage("You are a helpful assistant", [
{ role: "user", content: "Hello" },
])
const chunks = await collectStream(stream)

expect(chunks).toEqual([
{
type: "usage",
inputTokens: 100,
outputTokens: 10,
},
{
type: "usage",
inputTokens: 0,
outputTokens: 200,
},
])
})
})

describe("native tool calling", () => {
Expand Down
173 changes: 165 additions & 8 deletions src/api/providers/__tests__/anthropic.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import { AnthropicHandler } from "../anthropic"
import { ApiHandlerOptions } from "../../../shared/api"
import { asyncStreamFrom, collectStream } from "../../../test-utils/stream"
import type { ApiStreamChunk } from "../../../api/transform/stream"
import { clearAllMocks } from "../../../test-utils/reset"

// Mock TelemetryService
Expand Down Expand Up @@ -247,7 +248,7 @@ describe("AnthropicHandler", () => {
await collectStream(stream)

const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0]
expect(requestBody?.thinking).toEqual({ type: "adaptive" })
expect(requestBody?.thinking).toEqual({ type: "adaptive", display: "summarized" })
expect(requestBody?.max_tokens).toBe(16384)
})

Expand Down Expand Up @@ -290,7 +291,7 @@ describe("AnthropicHandler", () => {
await collectStream(stream)

const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0]
expect(requestBody?.thinking).toEqual({ type: "adaptive" })
expect(requestBody?.thinking).toEqual({ type: "adaptive", display: "summarized" })
expect(requestBody?.max_tokens).toBe(32768)
})

Expand Down Expand Up @@ -334,7 +335,7 @@ describe("AnthropicHandler", () => {
await collectStream(stream)

const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0]
expect(requestBody?.thinking).toEqual({ type: "adaptive" })
expect(requestBody?.thinking).toEqual({ type: "adaptive", display: "summarized" })
expect(requestBody?.max_tokens).toBe(16384)
})

Expand Down Expand Up @@ -377,7 +378,7 @@ describe("AnthropicHandler", () => {
await collectStream(stream)

const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0]
expect(requestBody?.thinking).toEqual({ type: "adaptive" })
expect(requestBody?.thinking).toEqual({ type: "adaptive", display: "summarized" })
expect(requestBody?.max_tokens).toBe(32768)
})

Expand All @@ -400,7 +401,7 @@ describe("AnthropicHandler", () => {

const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0]
const requestOptions = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[1]
expect(requestBody?.thinking).toEqual({ type: "adaptive" })
expect(requestBody?.thinking).toEqual({ type: "adaptive", display: "summarized" })
expect(requestBody?.temperature).toBeUndefined()
expect(requestBody?.max_tokens).toBe(32768)
expect(requestOptions?.headers?.["anthropic-beta"]).toContain("prompt-caching-2024-07-31")
Expand All @@ -425,7 +426,7 @@ describe("AnthropicHandler", () => {

const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0]
const requestOptions = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[1]
expect(requestBody?.thinking).toEqual({ type: "adaptive" })
expect(requestBody?.thinking).toEqual({ type: "adaptive", display: "summarized" })
expect(requestBody?.temperature).toBeUndefined()
expect(requestBody?.max_tokens).toBe(32768)
expect(requestOptions?.headers?.["anthropic-beta"]).toContain("prompt-caching-2024-07-31")
Expand All @@ -450,7 +451,7 @@ describe("AnthropicHandler", () => {

const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0]
const requestOptions = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[1]
expect(requestBody?.thinking).toEqual({ type: "adaptive" })
expect(requestBody?.thinking).toEqual({ type: "adaptive", display: "summarized" })
expect(requestBody?.temperature).toBeUndefined()
expect(requestBody?.max_tokens).toBe(32768)
expect(requestOptions?.headers?.["anthropic-beta"]).toContain("prompt-caching-2024-07-31")
Expand All @@ -474,7 +475,163 @@ describe("AnthropicHandler", () => {

const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0]
expect(requestBody?.model).toBe("claude-sonnet-5-bf")
expect(requestBody?.thinking).toEqual({ type: "adaptive" })
expect(requestBody?.thinking).toEqual({ type: "adaptive", display: "summarized" })
})

it("should surface thinking_tokens from output_tokens_details in usage chunks", async () => {
// Adaptive models report reasoning tokens inside
// `output_tokens_details.thinking_tokens` (message_start snapshot is
// typically absent; message_delta carries the final decomposition).
mockCreate.mockImplementationOnce(async () =>
asyncStreamFrom([
{
type: "message_start",
message: {
usage: {
input_tokens: 100,
output_tokens: 50,
},
},
},
{
type: "content_block_start",
index: 0,
content_block: {
type: "text",
text: "Hello",
},
},
{
type: "content_block_delta",
index: 0,
delta: {
type: "text_delta",
text: " world",
},
},
{
type: "message_delta",
usage: {
output_tokens: 200,
output_tokens_details: {
thinking_tokens: 150,
},
},
delta: {
stop_reason: "end_turn",
stop_sequence: null,
},
},
]),
)

const adaptiveHandler = new AnthropicHandler({
apiKey: "test-api-key",
apiModelId: "claude-opus-4-7",
enableReasoningEffort: true,
})

const stream = adaptiveHandler.createMessage("prompt", [
{
role: "user",
content: [{ type: "text", text: "Hi" }],
},
])

const chunks: ApiStreamChunk[] = await collectStream(stream)

// message_start snapshot carries no thinking decomposition
const usageChunks = chunks.filter(
(chunk): chunk is Extract<ApiStreamChunk, { type: "usage" }> => chunk.type === "usage",
)
const startUsage = usageChunks.find((chunk) => chunk.inputTokens > 0)
expect(startUsage).toBeDefined()
expect(startUsage?.reasoningTokens).toBeUndefined()

// message_delta surfaces the final reasoning token count
const deltaUsage = usageChunks.find((chunk) => chunk.inputTokens === 0)
expect(deltaUsage).toBeDefined()
expect(deltaUsage?.outputTokens).toBe(200)
expect(deltaUsage?.reasoningTokens).toBe(150)
})

it("should surface numeric thinking_tokens from the message_start usage snapshot", async () => {
// Some surfaces report the thinking decomposition as early as the
// message_start snapshot; assert the snapshot usage chunk surfaces
// reasoningTokens, and that a message_delta without
// output_tokens_details omits it.
mockCreate.mockImplementationOnce(async () =>
asyncStreamFrom([
{
type: "message_start",
message: {
usage: {
input_tokens: 100,
output_tokens: 10,
output_tokens_details: {
thinking_tokens: 5,
},
},
},
},
{
type: "content_block_start",
index: 0,
content_block: {
type: "text",
text: "Hello",
},
},
{
type: "content_block_delta",
index: 0,
delta: {
type: "text_delta",
text: " world",
},
},
{
type: "message_delta",
usage: {
output_tokens: 200,
},
delta: {
stop_reason: "end_turn",
stop_sequence: null,
},
},
]),
)

const adaptiveHandler = new AnthropicHandler({
apiKey: "test-api-key",
apiModelId: "claude-opus-4-7",
enableReasoningEffort: true,
})

const stream = adaptiveHandler.createMessage("prompt", [
{
role: "user",
content: [{ type: "text", text: "Hi" }],
},
])

const chunks: ApiStreamChunk[] = await collectStream(stream)

const usageChunks = chunks.filter(
(chunk): chunk is Extract<ApiStreamChunk, { type: "usage" }> => chunk.type === "usage",
)

// message_start snapshot surfaces the reported thinking decomposition
const startUsage = usageChunks.find((chunk) => chunk.inputTokens > 0)
expect(startUsage).toBeDefined()
expect(startUsage?.reasoningTokens).toBe(5)

// message_delta without output_tokens_details omits reasoningTokens
const deltaUsage = usageChunks.find((chunk) => chunk.inputTokens === 0)
expect(deltaUsage).toBeDefined()
expect(deltaUsage?.outputTokens).toBe(200)
expect(deltaUsage?.reasoningTokens).toBeUndefined()
})
})

Expand Down
Loading
Loading