test(e2e): set_thinking_effort mid-task workflow — e2e coverage (DTE series 3/5 addendum)

apps/vscode-e2e/src/fixtures/thinking-effort.ts

@@ -0,0 +1,159 @@
+import { LLMock } from "@copilotkit/aimock"
+import type { ChatCompletionRequest } from "@copilotkit/aimock"
+
+// DTE e2e fixtures for the thinking-effort-tool and thinking-effort-switching
+// suites. Replaces the former JSON fixtures (fixtures/thinking-effort-*.json):
+// post-tool requests end with a role:user message (fresh <environment_details>
+// is appended after the tool result), so aimock's toolCallId matcher — which
+// requires the LAST message to be role:tool — can never bind those continuation
+// turns, and JSON fixtures cannot carry predicates. Each turn is instead scoped
+// by a predicate that searches the whole request for its own flow identifiers:
+// the unique prompt marker for the baseline turn and the previous turn's unique
+// tool call id for the continuations (same pattern as deepseek-v4.ts). No other
+// suite can serve these responses and these suites cannot match unrelated turns.
+
+const SWITCH_MODEL = "openai/gpt-5.1"
+const APPLY_MODEL = "openai/gpt-5"
+const SWITCH_MARKER = "DTE_E2E_SWITCH"
+const APPLY_MARKER = "DTE_E2E_EFFORT_APPLY"
+const SWITCH_DONE = "DTE_E2E_SWITCH_DONE"
+
+// Post-tool requests carry the tool result of the PREVIOUS turn and nothing
+// appends another tool result before the next API call, so the LAST role:tool
+// message is exactly the call whose result this request follows.
+const lastToolCallId = (req: ChatCompletionRequest): string | undefined => {
+	const messages = Array.isArray(req?.messages) ? req.messages : []
+	return messages.filter((message) => message?.role === "tool").at(-1)?.tool_call_id
+}
+
+// aimock's userMessage matcher only inspects the LAST user message and joins
+// only the type:"text" content parts (getTextContent in aimock's router) — the
+// predicate replicates that semantics for the baseline turns.
+const lastUserMessageContains = (req: ChatCompletionRequest, text: string): boolean => {
+	const userMessages = req.messages?.filter((message) => message.role === "user") ?? []
+	const last = userMessages.at(-1)
+	if (!last) return false
+	const content =
+		typeof last.content === "string"
+			? last.content
+			: (last.content ?? [])
+					.filter((part): part is { type: "text"; text: string } => part?.type === "text")
+					.map((part) => part.text)
+					.join("")
+	return content.includes(text)
+}
+
+export function addThinkingEffortFixtures(mock: InstanceType<typeof LLMock>) {
+	// --- thinking-effort-switching suite (openai/gpt-5.1) ---
+	// Baseline turn: bound to this suite's unique prompt marker.
+	mock.addFixture({
+		match: {
+			model: SWITCH_MODEL,
+			predicate: (req: ChatCompletionRequest) => lastUserMessageContains(req, SWITCH_MARKER),
+		},
+		response: {
+			toolCalls: [
+				{
+					name: "set_thinking_effort",
+					arguments: JSON.stringify({ effort: "medium", reason: "start at medium" }),
+					id: "call_dte_sw_001",
+				},
+			],
+		},
+	})
+
+	// Continuations: each turn binds to the previous turn's unique tool call id,
+	// so a future flow on the same model cannot serve these responses.
+	const switchingContinuations: Array<{
+		afterCallId: string
+		effort: string
+		reason: string
+		responseCallId: string
+	}> = [
+		{
+			afterCallId: "call_dte_sw_001",
+			effort: "medium",
+			reason: "confirm current level",
+			responseCallId: "call_dte_sw_002",
+		},
+		{ afterCallId: "call_dte_sw_002", effort: "high", reason: "raise to high", responseCallId: "call_dte_sw_003" },
+		{
+			afterCallId: "call_dte_sw_003",
+			effort: "medium",
+			reason: "try returning to medium",
+			responseCallId: "call_dte_sw_004",
+		},
+	]
+	for (const continuation of switchingContinuations) {
+		mock.addFixture({
+			match: {
+				model: SWITCH_MODEL,
+				predicate: (req: ChatCompletionRequest) => lastToolCallId(req) === continuation.afterCallId,
+			},
+			response: {
+				toolCalls: [
+					{
+						name: "set_thinking_effort",
+						arguments: JSON.stringify({
+							effort: continuation.effort,
+							reason: continuation.reason,
+						}),
+						id: continuation.responseCallId,
+					},
+				],
+			},
+		})
+	}
+
+	// Final turn: after the refused oscillation call, the task completes.
+	mock.addFixture({
+		match: {
+			model: SWITCH_MODEL,
+			predicate: (req: ChatCompletionRequest) => lastToolCallId(req) === "call_dte_sw_004",
+		},
+		response: {
+			toolCalls: [
+				{
+					name: "attempt_completion",
+					arguments: JSON.stringify({ result: SWITCH_DONE }),
+					id: "call_dte_sw_005",
+				},
+			],
+		},
+	})
+
+	// --- thinking-effort-tool suite (openai/gpt-5) ---
+	// Baseline turn: bound to this suite's unique prompt marker.
+	mock.addFixture({
+		match: {
+			model: APPLY_MODEL,
+			predicate: (req: ChatCompletionRequest) => lastUserMessageContains(req, APPLY_MARKER),
+		},
+		response: {
+			toolCalls: [
+				{
+					name: "set_thinking_effort",
+					arguments: JSON.stringify({ effort: "high", reason: "multi-step math" }),
+					id: "call_dte_e2e_001",
+				},
+			],
+		},
+	})
+
+	// Continuation: binds to the set_thinking_effort call's unique tool call id.
+	mock.addFixture({
+		match: {
+			model: APPLY_MODEL,
+			predicate: (req: ChatCompletionRequest) => lastToolCallId(req) === "call_dte_e2e_001",
+		},
+		response: {
+			toolCalls: [
+				{
+					name: "attempt_completion",
+					arguments: JSON.stringify({ result: "42" }),
+					id: "call_dte_e2e_002",
+				},
+			],
+		},
+	})
+}

apps/vscode-e2e/src/runTest.ts

@@ -19,6 +19,7 @@ import { addListFilesResultFixtures } from "./fixtures/list-files"
 import { addReadFileResultFixtures } from "./fixtures/read-file"
 import { addSearchFilesResultFixtures } from "./fixtures/search-files"
 import { addSubtaskFixtures } from "./fixtures/subtasks"
+import { addThinkingEffortFixtures } from "./fixtures/thinking-effort"
 import { addUseMcpToolResultFixtures } from "./fixtures/use-mcp-tool"
 import { addWriteToFileResultFixtures } from "./fixtures/write-to-file"
 import { createScenarioWorkspace, removeScenarioWorkspace } from "./restart/scenarioWorkspace"
@@ -140,6 +141,7 @@ async function main() {
 				addReadFileResultFixtures(mock)
 				addSearchFilesResultFixtures(mock)
 				addSubtaskFixtures(mock)
+				addThinkingEffortFixtures(mock)
 				addUseMcpToolResultFixtures(mock)
 				addWriteToFileResultFixtures(mock)
 				addDeepSeekV4Fixtures(mock)

apps/vscode-e2e/src/suite/thinking-effort-proxy.ts

@@ -0,0 +1,212 @@
+import { createServer, type IncomingMessage, type ServerResponse } from "http"
+
+/**
+ * Shared loopback capture proxy for the DTE e2e suites
+ * (thinking-effort-tool / thinking-effort-switching).
+ *
+ * Pattern from anthropic-opus-4-7.test.ts: it intercepts the
+ * OpenRouter-compatible chat/completions POST so request shapes can be
+ * asserted (model, reasoning envelope, message content), then forwards the
+ * request unchanged to the upstream — aimock in replay/record mode — which
+ * answers with the fixture-driven SSE.
+ */
+
+export type DteReasoningEnvelope = {
+	effort?: string
+	max_tokens?: number
+	exclude?: boolean
+}
+
+export type CapturedDteRequest = {
+	model?: string
+	reasoning: DteReasoningEnvelope | undefined
+	/** Raw JSON body, so assertions can inspect any part of the wire request (e.g. tool result text). */
+	bodyText: string
+	lastUserMessage: string
+}
+
+type OpenRouterChatCompletionBody = {
+	model?: string
+	reasoning?: DteReasoningEnvelope
+	messages?: Array<{ role?: string; content?: unknown }>
+}
+
+const ALLOWED_PROXY_HOSTS = new Set(["127.0.0.1", "localhost"])
+const CHAT_COMPLETIONS_PATH = "/v1/chat/completions"
+const HOP_BY_HOP = new Set([
+	"connection",
+	"keep-alive",
+	"transfer-encoding",
+	"te",
+	"trailer",
+	"upgrade",
+	"proxy-connection",
+	"proxy-authenticate",
+	"proxy-authorization",
+	"host",
+	"content-length",
+])
+
+/**
+ * Whether a raw URL targets the OpenRouter-compatible chat/completions endpoint.
+ */
+function isChatCompletionsUrl(rawUrl: string): boolean {
+	try {
+		return new URL(rawUrl).pathname.endsWith(CHAT_COMPLETIONS_PATH)
+	} catch {
+		return false
+	}
+}
+
+/**
+ * Collects the full request body as a UTF-8 string.
+ */
+function readRequestBody(req: IncomingMessage): Promise<string> {
+	return new Promise((resolve, reject) => {
+		const chunks: Buffer[] = []
+		req.on("data", (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)))
+		req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")))
+		req.on("error", reject)
+	})
+}
+
+/**
+ * Mirrors the upstream response headers onto the proxy response, dropping the
+ * headers that would break fetch()-decoded streaming (content-encoding / length).
+ */
+function writeResponseHeaders(target: ServerResponse, source: Response) {
+	const headers: Record<string, string> = {}
+	source.headers.forEach((value, key) => {
+		const lower = key.toLowerCase()
+		// fetch() automatically decompresses the body, so strip content-encoding to
+		// prevent the SDK from attempting a second decompression. Also strip
+		// content-length since the decoded body length differs from the compressed one.
+		if (lower !== "content-length" && lower !== "content-encoding") {
+			headers[key] = value
+		}
+	})
+	target.writeHead(source.status, headers)
+}
+
+/**
+ * Streams the upstream (already-decoded) fetch body through to the proxy
+ * response, ending the response when the body completes.
+ */
+async function pipeFetchResponse(target: ServerResponse, source: Response) {
+	writeResponseHeaders(target, source)
+
+	if (!source.body) {
+		target.end()
+		return
+	}
+
+	const reader = source.body.getReader()
+	while (true) {
+		const { done, value } = await reader.read()
+		if (done) {
+			break
+		}
+		target.write(value)
+	}
+
+	target.end()
+}
+
+/**
+ * Resolves the upstream chat/completions URL, rejecting any target that is not
+ * a loopback HTTP origin (the proxy must never forward to a real endpoint).
+ */
+function resolveAllowedUpstreamUrl(baseUrl: string): URL {
+	const upstreamBase = new URL(baseUrl)
+
+	if (!ALLOWED_PROXY_HOSTS.has(upstreamBase.hostname) || upstreamBase.protocol !== "http:") {
+		throw new Error("Unexpected OpenRouter proxy target: " + upstreamBase.origin)
+	}
+
+	return new URL(CHAT_COMPLETIONS_PATH, upstreamBase)
+}
+
+/**
+ * Serves a loopback capture proxy for the OpenRouter-compatible
+ * chat/completions endpoint: captures each request body for assertions and
+ * forwards it unchanged to the upstream (aimock in replay/record mode).
+ */
+export async function withOpenRouterCaptureProxy<T>(
+	upstreamUrl: string,
+	run: (args: { proxyUrl: string; requests: CapturedDteRequest[] }) => Promise<T>,
+): Promise<T> {
+	const requests: CapturedDteRequest[] = []
+	const upstreamTarget = resolveAllowedUpstreamUrl(upstreamUrl)
+	let proxyError: Error | undefined
+
+	const server = createServer(async (req, res) => {
+		try {
+			const requestUrl = req.url ?? "/"
+
+			if (!isChatCompletionsUrl("http://127.0.0.1" + requestUrl)) {
+				res.writeHead(404)
+				res.end("Not found")
+				return
+			}
+
+			const bodyText = await readRequestBody(req)
+			const body = JSON.parse(bodyText) as OpenRouterChatCompletionBody
+			const lastUser = [...(body.messages ?? [])].reverse().find((message) => message.role === "user")
+			const lastUserMessage =
+				typeof lastUser?.content === "string" ? lastUser.content : JSON.stringify(lastUser?.content ?? "")
+
+			requests.push({
+				model: body.model,
+				reasoning: body.reasoning,
+				bodyText,
+				lastUserMessage,
+			})
+
+			const forwardHeaders: Record<string, string> = {}
+			for (const [key, value] of Object.entries(req.headers)) {
+				if (!HOP_BY_HOP.has(key.toLowerCase()) && value !== undefined) {
+					forwardHeaders[key] = Array.isArray(value) ? value.join(", ") : value
+				}
+			}
+
+			const upstream = await fetch(upstreamTarget, {
+				method: req.method,
+				headers: forwardHeaders,
+				body: bodyText,
+			})
+
+			await pipeFetchResponse(res, upstream)
+		} catch (error) {
+			proxyError = error instanceof Error ? error : new Error(String(error))
+			console.error("OpenRouter proxy request failed:", proxyError)
+			if (!res.headersSent) {
+				res.writeHead(502)
+				res.end("Capture proxy error")
+			} else if (!res.writableEnded) {
+				res.destroy()
+			}
+		}
+	})
+
+	await new Promise<void>((resolve) => {
+		server.listen(0, "127.0.0.1", () => resolve())
+	})
+
+	const address = server.address()
+	if (address === null || typeof address === "string") {
+		server.close()
+		throw new Error("Capture proxy failed to bind a loopback port")
+	}
+
+	const proxyUrl = "http://127.0.0.1:" + address.port
+
+	try {
+		const result = await run({ proxyUrl, requests })
+		if (proxyError) {
+			throw proxyError
+		}
+		return result
+	} finally {
+		await new Promise<void>((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())))
+	}
+}