feat(caching): Anthropic prompt caching with cache-aware cost reporting

.changeset/anthropic-prompt-caching.md

@@ -0,0 +1,5 @@
+---
+"@nanocollective/nanocoder": minor
+---
+
+Added Anthropic prompt caching. The system prompt, tool schemas, and conversation history are now marked with cache breakpoints, so multi-turn sessions read the stable prefix back out of cache instead of paying full price for it every turn. Cost reporting is cache-aware throughout: `/usage` and the per-response indicator price cache reads and writes at their own rates and surface the cached token count. Opt out with `"promptCaching": false` on the provider config. Closes #888.

source/ai-sdk-client/chat/chat-handler.spec.ts

@@ -361,3 +361,121 @@ test('privacy: scrubs outgoing prompts and rehydrates the response at the histor
 	t.is(content, 'Saved real@example.com');
 	t.false(content.includes('«'));
 });
+
+function capturingModel(captured: {prompt?: unknown}): LanguageModel {
+	return {
+		specificationVersion: 'v3',
+		provider: 'anthropic',
+		modelId: 'claude-sonnet-4-5',
+		doStream: async (options: {prompt: unknown}) => {
+			captured.prompt = options.prompt;
+			return {
+				stream: new ReadableStream({
+					start(controller) {
+						controller.enqueue({type: 'text-start', id: '0'});
+						controller.enqueue({type: 'text-delta', id: '0', delta: 'ok'});
+						controller.enqueue({type: 'text-end', id: '0'});
+						controller.enqueue({
+							type: 'finish',
+							finishReason: 'stop',
+							usage: {inputTokens: 5000, outputTokens: 10, totalTokens: 5010},
+						});
+						controller.close();
+					},
+				}),
+			};
+		},
+	} as unknown as LanguageModel;
+}
+
+const anthropicConfig: AIProviderConfig = {
+	name: 'Anthropic',
+	type: 'anthropic',
+	sdkProvider: 'anthropic',
+	models: ['claude-sonnet-4-5'],
+	config: {apiKey: 'test-key'},
+};
+
+const LONG_SYSTEM = 'SYSTEM '.repeat(1000);
+
+test('handleChat sends the system prompt as a cache-marked message on anthropic', async t => {
+	const captured: {prompt?: unknown} = {};
+	await handleChat({
+		model: capturingModel(captured),
+		currentModel: 'claude-sonnet-4-5',
+		providerConfig: anthropicConfig,
+		messages: [
+			{role: 'system', content: LONG_SYSTEM},
+			{role: 'user', content: 'hello'},
+		],
+		tools: {},
+		callbacks: {},
+		maxRetries: 0,
+	});
+
+	const prompt = captured.prompt as Array<{
+		role: string;
+		providerOptions?: Record<string, unknown>;
+	}>;
+	t.is(prompt[0]?.role, 'system');
+	t.deepEqual(prompt[0]?.providerOptions, {
+		anthropic: {cacheControl: {type: 'ephemeral'}},
+	});
+	t.deepEqual(prompt[prompt.length - 1]?.providerOptions, {
+		anthropic: {cacheControl: {type: 'ephemeral'}},
+	});
+});
+
+test('handleChat does not emit the AI SDK system-in-messages warning', async t => {
+	const warnings: string[] = [];
+	const originalWarn = console.warn;
+	console.warn = (...args: unknown[]) => {
+		warnings.push(args.join(' '));
+	};
+	try {
+		await handleChat({
+			model: capturingModel({}),
+			currentModel: 'claude-sonnet-4-5',
+			providerConfig: anthropicConfig,
+			messages: [
+				{role: 'system', content: LONG_SYSTEM},
+				{role: 'user', content: 'hello'},
+			],
+			tools: {},
+			callbacks: {},
+			maxRetries: 0,
+		});
+	} finally {
+		console.warn = originalWarn;
+	}
+	t.false(warnings.some(w => w.includes('System messages in the prompt')));
+});
+
+test('handleChat keeps the system string for non-anthropic providers', async t => {
+	const captured: {prompt?: unknown} = {};
+	await handleChat({
+		model: capturingModel(captured),
+		currentModel: 'test-model',
+		providerConfig: {
+			name: 'TestProvider',
+			type: 'openai',
+			models: ['test-model'],
+			config: {baseURL: 'https://api.test.com'},
+		},
+		messages: [
+			{role: 'system', content: LONG_SYSTEM},
+			{role: 'user', content: 'hello'},
+		],
+		tools: {},
+		callbacks: {},
+		maxRetries: 0,
+	});
+
+	const prompt = captured.prompt as Array<{
+		role: string;
+		providerOptions?: Record<string, unknown>;
+	}>;
+	t.is(prompt[0]?.role, 'system');
+	t.is(prompt[0]?.providerOptions, undefined);
+	t.is(prompt[prompt.length - 1]?.providerOptions, undefined);
+});

source/ai-sdk-client/chat/chat-handler.ts

@@ -28,12 +28,18 @@ import {
 	startMetrics,
 } from '@/utils/logging/performance.js';
 import {getSafeMemory} from '@/utils/logging/safe-process.js';
-import {convertToModelMessages} from '../converters/message-converter.js';
+import {
+	convertToModelMessages,
+	withCacheBreakpoints,
+} from '../converters/message-converter.js';
 import {convertAISDKToolCalls} from '../converters/tool-converter.js';
 import {extractRootError} from '../error-handling/error-extractor.js';
 import {parseAPIError} from '../error-handling/error-parser.js';
 import {isToolSupportError} from '../error-handling/tool-error-detector.js';
-import {buildProviderOptions} from './provider-options.js';
+import {
+	buildProviderOptions,
+	isPromptCachingEnabled,
+} from './provider-options.js';
 import {
 	createOnStepFinishHandler,
 	createPrepareStepHandler,
@@ -187,7 +193,11 @@ export async function handleChat(
 			}
 
 			// Convert messages to AI SDK v5 ModelMessage format
-			const modelMessages = convertToModelMessages(finalNonSystemMessages);
+			const promptCaching = isPromptCachingEnabled(providerConfig);
+			const convertedMessages = convertToModelMessages(finalNonSystemMessages);
+			const modelMessages = promptCaching
+				? withCacheBreakpoints(convertedMessages, finalSystemContent)
+				: convertedMessages;
 
 			logger.debug('AI SDK request prepared', {
 				messageCount: modelMessages.length,
@@ -213,7 +223,10 @@ export async function handleChat(
 
 			const result = streamText({
 				model,
-				...(finalSystemContent ? {system: finalSystemContent} : {}),
+				...(finalSystemContent && !promptCaching
+					? {system: finalSystemContent}
+					: {}),
+				...(promptCaching ? {allowSystemInMessages: true} : {}),
 				messages: modelMessages,
 				tools: aiTools,
 				abortSignal: signal,
@@ -478,6 +491,8 @@ export async function handleChat(
 					inputTokens: usage.inputTokens,
 					outputTokens: usage.outputTokens,
 					totalTokens: usage.totalTokens,
+					cacheReadTokens: usage.inputTokenDetails?.cacheReadTokens,
+					cacheWriteTokens: usage.inputTokenDetails?.cacheWriteTokens,
 				},
 			};
 		} catch (error) {

source/ai-sdk-client/chat/provider-options.spec.ts

@@ -1,6 +1,9 @@
 import test from 'ava';
 import type {AIProviderConfig, OpenRouterParameters} from '@/types/index';
-import {buildProviderOptions} from './provider-options.js';
+import {
+	buildProviderOptions,
+	isPromptCachingEnabled,
+} from './provider-options.js';
 
 function makeProvider(
 	overrides: Partial<AIProviderConfig> = {},
@@ -276,3 +279,43 @@ test('OpenRouter extraBody alone is enough to emit providerOptions', t => {
 	const result = buildProviderOptions(provider, '', undefined);
 	t.deepEqual(result, {openrouter: {experimental_flag: true}});
 });
+
+test('isPromptCachingEnabled is on by default for the anthropic SDK', t => {
+	t.true(
+		isPromptCachingEnabled(makeProvider({sdkProvider: 'anthropic'})),
+	);
+});
+
+test('isPromptCachingEnabled honours an explicit opt-out', t => {
+	t.false(
+		isPromptCachingEnabled(
+			makeProvider({sdkProvider: 'anthropic', promptCaching: false}),
+		),
+	);
+});
+
+test('isPromptCachingEnabled honours an explicit opt-in', t => {
+	t.true(
+		isPromptCachingEnabled(
+			makeProvider({sdkProvider: 'anthropic', promptCaching: true}),
+		),
+	);
+});
+
+test('isPromptCachingEnabled is off for every non-anthropic SDK', t => {
+	t.false(isPromptCachingEnabled(makeProvider()));
+	t.false(
+		isPromptCachingEnabled(makeProvider({sdkProvider: 'openai-compatible'})),
+	);
+	t.false(isPromptCachingEnabled(makeProvider({sdkProvider: 'google'})));
+	t.false(isPromptCachingEnabled(makeProvider({sdkProvider: 'chatgpt-codex'})));
+	t.false(isPromptCachingEnabled(makeProvider({sdkProvider: 'github-copilot'})));
+});
+
+test('isPromptCachingEnabled stays off for non-anthropic providers that opt in', t => {
+	t.false(
+		isPromptCachingEnabled(
+			makeProvider({sdkProvider: 'openai-compatible', promptCaching: true}),
+		),
+	);
+});

source/ai-sdk-client/chat/provider-options.ts

@@ -14,6 +14,15 @@ import {isOpenRouterProvider} from '../providers/openrouter.js';
  */
 export type ProviderOptions = Record<string, Record<string, unknown>>;
 
+export function isPromptCachingEnabled(
+	providerConfig: AIProviderConfig,
+): boolean {
+	return (
+		providerConfig.sdkProvider === 'anthropic' &&
+		providerConfig.promptCaching !== false
+	);
+}
+
 /**
  * Build the `providerOptions` value for a streamText/generateText call.
  *

source/ai-sdk-client/converters/message-converter.spec.ts

@@ -1,10 +1,12 @@
+import type {ModelMessage} from 'ai';
 import test from 'ava';
 import {MAX_TOOL_RESULT_CHARS} from '@/constants';
 import type {Message} from '@/types/index';
 import {
 	convertToModelMessages,
 	dropOrphanedToolResults,
 	isEmptyAssistantMessage,
+	withCacheBreakpoints,
 } from './message-converter.js';
 import type {TestableMessage} from '../types.js';
 
@@ -398,3 +400,106 @@ test('dropOrphanedToolResults drops a tool result lacking a tool_call_id', t =>
 	t.is(result.length, 1);
 	t.is(result[0].role, 'user');
 });
+
+const BIG = 'S'.repeat(5000);
+const BREAKPOINT = {anthropic: {cacheControl: {type: 'ephemeral'}}};
+
+test('withCacheBreakpoints folds the system prompt in as the first message', t => {
+	const result = withCacheBreakpoints(
+		[{role: 'user', content: 'hi'}],
+		'system text',
+	);
+	t.is(result.length, 2);
+	t.is(result[0]?.role, 'system');
+	t.is(result[0]?.content, 'system text');
+});
+
+test('withCacheBreakpoints marks the system prompt and the last message', t => {
+	const result = withCacheBreakpoints(
+		[
+			{role: 'user', content: BIG},
+			{role: 'assistant', content: 'a'},
+			{role: 'user', content: 'b'},
+		],
+		'system text',
+	);
+	t.deepEqual(result[0]?.providerOptions, BREAKPOINT);
+	t.is(result[1]?.providerOptions, undefined);
+	t.is(result[2]?.providerOptions, undefined);
+	t.deepEqual(result[3]?.providerOptions, BREAKPOINT);
+});
+
+test('withCacheBreakpoints emits at most two breakpoints', t => {
+	const messages = Array.from({length: 12}, (_, i) => ({
+		role: 'user' as const,
+		content: `${BIG}${i}`,
+	}));
+	const marked = withCacheBreakpoints(messages, BIG).filter(
+		m => m.providerOptions !== undefined,
+	);
+	t.is(marked.length, 2);
+});
+
+test('withCacheBreakpoints skips breakpoints below the cacheable minimum', t => {
+	const result = withCacheBreakpoints(
+		[{role: 'user', content: 'hi'}],
+		'short system',
+	);
+	t.is(result.length, 2);
+	t.is(result[0]?.providerOptions, undefined);
+	t.is(result[1]?.providerOptions, undefined);
+});
+
+test('withCacheBreakpoints marks only the last message when there is no system prompt', t => {
+	const result = withCacheBreakpoints(
+		[
+			{role: 'user', content: BIG},
+			{role: 'assistant', content: 'tail'},
+		],
+		'',
+	);
+	t.is(result.length, 2);
+	t.is(result[0]?.providerOptions, undefined);
+	t.deepEqual(result[1]?.providerOptions, BREAKPOINT);
+});
+
+test('withCacheBreakpoints marks only the system prompt when there are no messages', t => {
+	const result = withCacheBreakpoints([], BIG);
+	t.is(result.length, 1);
+	t.deepEqual(result[0]?.providerOptions, BREAKPOINT);
+});
+
+test('withCacheBreakpoints returns an empty array for empty input', t => {
+	t.deepEqual(withCacheBreakpoints([], ''), []);
+});
+
+test('withCacheBreakpoints counts array content toward the threshold', t => {
+	const result = withCacheBreakpoints(
+		[
+			{
+				role: 'tool',
+				content: [
+					{
+						type: 'tool-result',
+						toolCallId: '1',
+						toolName: 'read_file',
+						output: {type: 'text', value: BIG},
+					},
+				],
+			},
+		],
+		'',
+	);
+	t.deepEqual(result[0]?.providerOptions, BREAKPOINT);
+});
+
+test('withCacheBreakpoints does not mutate its inputs', t => {
+	const messages: ModelMessage[] = [
+		{role: 'user', content: BIG},
+		{role: 'assistant', content: 'tail'},
+	];
+	withCacheBreakpoints(messages, BIG);
+	t.is(messages.length, 2);
+	t.is(messages[0]?.providerOptions, undefined);
+	t.is(messages[1]?.providerOptions, undefined);
+});