diff --git a/src/core/history.ts b/src/core/history.ts index 8e76935e1..7257ae376 100644 --- a/src/core/history.ts +++ b/src/core/history.ts @@ -13,7 +13,7 @@ import type { CacheControl, ContentBlock, ImageBlock, Message, TextBlock, ToolUseBlock, ToolResultBlock } from './types.js'; import type { RenderedImage } from './render.js'; -import { DENSE_CONTENT_CHARS_PER_IMAGE, DENSE_CONTENT_COLS, DENSE_RENDER_STYLE, MAX_HEIGHT_PX, neutralizeSentinel, reflow, renderTextToPngsWithCharLimit, roleSlotSegment, SLOT_MARK_ASSISTANT, SLOT_MARK_USER, type RenderStyle } from './render.js'; +import { DENSE_CONTENT_CHARS_PER_IMAGE, DENSE_CONTENT_COLS, maxCharsPerImage, DENSE_RENDER_STYLE, MAX_HEIGHT_PX, neutralizeSentinel, reflow, renderTextToPngsWithCharLimit, roleSlotSegment, SLOT_MARK_ASSISTANT, SLOT_MARK_USER, type RenderStyle } from './render.js'; import { factSheetText } from './factsheet.js'; import { bytesToBase64 } from './png.js'; @@ -76,8 +76,41 @@ export interface HistoryCollapseOptions { style: RenderStyle; /** Model-profile page-height cap. */ maxHeightPx: number; + /** Chars one rendered page holds. Only an ESTIMATOR input (the renderer paginates + * on its own); it decides how many pages a candidate chunk grid will produce. + * Default {@link DENSE_CONTENT_CHARS_PER_IMAGE}. */ + pageChars: number; + /** Hard cap on image blocks this collapse may emit. Anthropic rejects requests + * with more than 100 images (opaque 500), and a 10-message freeze grid emits + * ≈1 page per chunk regardless of how little text the chunk holds — a 3000-turn + * session hit 317 history images at 43% page fill (#161). When the grid would + * exceed the budget the freeze step is DOUBLED (chunks merge, pages fill) until + * the estimate fits; if even a single chunk cannot fit, the collapse range is + * trimmed from the tail and the remainder stays live text. 0 = unlimited. */ + imageBudget: number; + /** Fill-optimal repack. When true the freeze step is raised until the grid costs + * at most one page more than a perfectly packed render — trading the append-only + * cache freeze for ~2× fewer image tokens. Only correct when the upstream prefix + * cache is dead anyway (cold session, see node.ts session store); on a warm + * session it would re-key every frozen chunk. Default false. */ + packFill: boolean; + /** Sticky lower bound for the freeze step, in messages. Once a session has been + * repacked at a coarser grid, every later turn must keep that grid or the + * re-render re-keys the whole history. Rounded UP to a power-of-two multiple of + * `freezeChunk` so chunk boundaries stay a subset of the base grid. Default 0. */ + minFreezeStep: number; } +/** Images Anthropic accepts per request. Exceeding it fails the WHOLE request with + * an opaque `500` (observed 2026-07-31 at 387 images), not a typed 400 — so the + * cap has to be enforced on our side, before the wire. */ +export const ANTHROPIC_MAX_IMAGES = 100; + +/** Default history-image budget: the hard cap minus headroom for the slab, tool-doc + * and tool_result images that share the same request. transform.ts narrows this + * further with the count it has already emitted for this very request. */ +export const ANTHROPIC_HISTORY_IMAGE_BUDGET = 80; + export const HISTORY_DEFAULTS: HistoryCollapseOptions = { keepTail: 4, minCollapsePrefix: 10, @@ -88,6 +121,12 @@ export const HISTORY_DEFAULTS: HistoryCollapseOptions = { reflow: true, style: DENSE_RENDER_STYLE, maxHeightPx: MAX_HEIGHT_PX, + // MUST agree with `cols` above: pageChars is what the budget arithmetic thinks + // one image holds, and DENSE_CONTENT_CHARS_PER_IMAGE is only true at 312 cols. + pageChars: maxCharsPerImage(100), + imageBudget: ANTHROPIC_HISTORY_IMAGE_BUDGET, + packFill: false, + minFreezeStep: 0, }; /** Per-request telemetry surfaced back to TransformInfo. */ @@ -120,7 +159,16 @@ export interface HistoryCollapseInfo { | 'prefix_too_short' | 'no_closed_prefix' | 'not_profitable' - | 'render_empty'; + | 'render_empty' + | 'over_budget'; + /** Freeze step actually used (messages per chunk). Larger than `o.freezeChunk` + * when the image budget or fill-repack forced chunks to merge. The caller pins + * it per session (`minFreezeStep`) so the coarser grid never falls back — a + * fallback would re-key every frozen chunk it already paid to cache. */ + freezeStep?: number; + /** True when the collapse range had to be shortened to stay inside the image + * budget; the dropped tail stays as live text. */ + budgetTrimmed?: boolean; /** Dropped codepoints from the history render, merged into the * transform-wide map by the caller. */ droppedChars: number; @@ -571,6 +619,15 @@ async function userTurnBlocks( ): Promise { const out: ContentBlock[] = []; let pending: string[] = []; + // Over-cap prompts are collected and rendered TOGETHER at the end of the chunk. + // One image per pasted document would put a floor of ≥1 image on every such turn + // that no grid coarsening can lift: a session with 175 pasted logs rendered 175 + // near-empty images and blew the 100-image cap (#161), which upstream answers with + // a 500 rather than a usable error. Batching packs them at ~28k chars/image and + // makes the count a function of BYTES, which the freeze grid can actually control. + // Within a chunk the order is fixed and the chunk is frozen once closed, so the + // grouped bytes are as stable as the transcript image next to them. + const imaged: { idx: number; typed: string }[] = []; const flush = () => { if (pending.length === 0) return; out.push({ @@ -588,19 +645,33 @@ async function userTurnBlocks( pending.push(`${typed}`); continue; } - // Over the cap: this one prompt becomes its own image, kept separate from the - // history transcript image so it stays independently readable and attributable. - flush(); + // Past the cap this is a pasted document, not an instruction: it is rendered + // verbatim below instead of bloating the text. + imaged.push({ idx: i, typed }); + } + flush(); + if (imaged.length > 0) { + // Every batched turn is NAMED (attribution is the point), but only the newest + // few carry a preview: 60 pasted docs × a 300-char preview is a wall of text + // that buys nothing the images below don't already say, verbatim. + const PREVIEW_LIMIT = 8; + const previews = imaged + .map((u, k) => + k >= imaged.length - PREVIEW_LIMIT + ? ` (${u.typed.length} chars) Preview: ${compactPreview(u.typed)}` + : ` (${u.typed.length} chars)`, + ) + .join('\n'); + out.push({ + type: 'text', + text: `[${imaged.length} user turn(s) from this session were too long to carry as text; they are rendered verbatim, in turn order, in the image(s) immediately below, separate from the history transcript. Each begins with its own tag. PRIOR context, not the current request.\n${previews}]`, + }); const imgs = await renderTextToPngsWithCharLimit( - `\n${typed}\n`, + imaged.map((u) => `\n${u.typed}\n`).join('\n\n'), DENSE_CONTENT_COLS, DENSE_CONTENT_CHARS_PER_IMAGE, DENSE_RENDER_STYLE, ); - out.push({ - type: 'text', - text: `[ was too long to carry as text (${typed.length} chars); it is rendered verbatim in the image(s) immediately below, separate from the history transcript. PRIOR context, not the current request. Preview: ${compactPreview(typed)}]`, - }); for (const img of imgs) { out.push({ type: 'image', @@ -613,7 +684,6 @@ async function userTurnBlocks( onImage(img); } } - flush(); return out; } @@ -704,11 +774,60 @@ export async function collapseHistory( } // Need at least minCollapsePrefix turns in [protectedPrefix..boundary] — collapsing // 2-3 turns is net cost (cache-amortization math doesn't work at small scale). - const collapseLen = boundary + 1; + let collapseLen = boundary + 1; if (collapseLen - protectedPrefix < o.minCollapsePrefix) { info.reason = 'prefix_too_short'; return { messages, info }; } + + // ---- Image budget --------------------------------------------------------- + // Anthropic rejects the WHOLE request past ANTHROPIC_MAX_IMAGES with an opaque + // 500, so the budget is a hard constraint we must enforce before the wire. Price + // candidate grids off per-message serialized lengths: one pass here replaces + // re-serializing the transcript once per candidate step. + const budget = o.imageBudget > 0 ? o.imageBudget : Infinity; + const pageChars = Math.max(1, o.pageChars); + const msgLen: number[] = []; + // Over-cap user prompts are imaged too (userTurnBlocks), batched per chunk. They + // are NOT part of the transcript segments, so they must be priced separately or + // the estimate silently under-counts and the wire limit is what finds out. + const userImgLen: number[] = []; + for (let i = protectedPrefix; i < collapseLen; i++) { + const seg = messagesToHistorySegments(messages, i + 1, i).text; + msgLen.push(seg.length === 0 ? 0 : seg.length + 2); // +2 = the "\n\n" joiner + const m = messages[i]!; + const typed = m.role === 'user' ? typedUserText(m.content) : ''; + userImgLen.push(typed && typed.length > USER_TEXT_MAX_CHARS ? typed.length + 20 : 0); + } + const sumOf = (arr: number[], from: number, to: number): number => { + let n = 0; + for (let i = from; i < to; i++) n += arr[i]!; + return n; + }; + const sumLen = (from: number, to: number): number => sumOf(msgLen, from, to); + const perfectPages = + Math.ceil(sumLen(0, msgLen.length) / pageChars) + + Math.ceil(sumOf(userImgLen, 0, userImgLen.length) / pageChars); + if (perfectPages > budget) { + // Even a perfectly packed render of the full range overflows. Keep the OLDEST + // messages collapsed (the frozen prefix must stay anchored at protectedPrefix + // or every cached chunk re-keys) and leave the tail as live text. + let acc = 0; + let k = 0; + while (k < msgLen.length && acc + msgLen[k]! + userImgLen[k]! <= budget * pageChars) { + acc += msgLen[k]! + userImgLen[k]!; + k++; + } + const trimmedLen = findClosedPrefixBoundary(messages, protectedPrefix + k) + 1; + if (trimmedLen - protectedPrefix < o.minCollapsePrefix) { + info.reason = 'over_budget'; + return { messages, info }; + } + collapseLen = trimmedLen; + msgLen.length = collapseLen - protectedPrefix; + info.budgetTrimmed = true; + } + // Exclude slab messages (protectedPrefix) from serialization. const text = messagesToHistoryText(messages, collapseLen, protectedPrefix); if (!text || text.length === 0) { @@ -738,7 +857,43 @@ export async function collapseHistory( // cacheable image boundary instead of being silently flattened (count conserved, // never added). Each chunk is reflowed and rendered on its own, which is what // makes the bytes a pure function of the chunk's messages. - const step = o.freezeChunk > 0 ? o.freezeChunk : collapseLen - protectedPrefix; + // + // The grid step is ADAPTIVE. A fixed 10-message step emits ≥1 page per chunk no + // matter how little text the chunk holds: a long session of short turns rendered + // 317 pages at 43% fill and 500'd the request (#161). Doubling the step merges + // neighbouring chunks — pages fill up, count drops ~2× per doubling — while + // keeping chunk boundaries a SUBSET of the base grid, so a chunk frozen at the + // coarse step spans whole base chunks and stays byte-identical as long as the + // step never shrinks again (the caller pins it via minFreezeStep). + const baseStep = o.freezeChunk > 0 ? o.freezeChunk : collapseLen - protectedPrefix; + const rangeLen = collapseLen - protectedPrefix; + const pagesFor = (s: number): number => { + let pages = 0; + for (let a = 0; a < rangeLen; a += s) { + const b = Math.min(a + s, rangeLen); + const chars = sumLen(a, b); + if (chars > 0) pages += Math.ceil(chars / pageChars); + // Over-cap user prompts in this chunk are batched into their own image(s). + const uchars = sumOf(userImgLen, a, b); + if (uchars > 0) pages += Math.ceil(uchars / pageChars); + } + return pages; + }; + // Caller cache_control marks force extra splits below; charge one page each so a + // marked request can't slip past the budget the estimate just cleared. + let markSplits = 0; + for (let i = protectedPrefix; i < collapseLen; i++) { + if (messageCacheControl(messages[i]!) !== undefined) markSplits++; + } + let step = baseStep; + // Sticky floor first: a session already repacked coarse must STAY coarse. + while (step < o.minFreezeStep && step < rangeLen) step *= 2; + // packFill trades the append-only freeze for ~2× fewer image tokens and is only + // set when the upstream cache is dead anyway (cold session / after a 500). + const packedPages = Math.max(1, Math.ceil(sumLen(0, rangeLen) / pageChars)); + const goal = Math.min(budget, o.packFill ? packedPages + 1 : Infinity); + while (step < rangeLen && pagesFor(step) + markSplits > goal) step *= 2; + info.freezeStep = step; const ends = new Set(); for (let e = protectedPrefix + step; e < collapseLen; e += step) ends.add(e); const markerByEnd = new Map(); diff --git a/src/core/proxy.ts b/src/core/proxy.ts index 278a85ee9..be7416326 100644 --- a/src/core/proxy.ts +++ b/src/core/proxy.ts @@ -3,6 +3,7 @@ * Adapted by src/node.ts and src/worker.ts; uses only Request/Response/URL/fetch. */ +import { markCacheDead, noteCacheOutcome, responseLeftNoCache } from './session-state.js'; import { transformRequest, type TransformOptions, type TransformInfo } from './transform.js'; import { isClaudeModel, transformOpenAIChatCompletions, transformOpenAIResponses } from './openai.js'; import { isAnthropicMessagesPath, isPxpipeSupportedGptModel, isPxpipeSupportedModel } from './applicability.js'; @@ -1558,6 +1559,21 @@ export function createProxy(config: ProxyConfig = {}) { measurementPromise.catch(() => undefined), stopReasonPromise.catch(() => undefined), ]).then(([usage, errorBody, measurement, stopReason]) => { + // A rejected request never populated a prefix cache, so the append-only + // freeze this session was protecting protects nothing: let the next turn + // re-cut the grid for density instead of preserving dead bytes. + if (responseLeftNoCache(upstreamRes.status, errorBody)) { + markCacheDead(info?.firstUserSha8); + } + // Feed the provider's own cache accounting back into the session store. A + // read proves the prefix was live; a create proves one was just written. + // Either way the next turn must not re-cut the grid — which the wall clock + // alone could not tell, and got wrong on most gaps that mattered. + noteCacheOutcome( + info?.firstUserSha8, + usage?.cache_read_input_tokens, + usage?.cache_creation_input_tokens, + ); fire( upstreamRes.status, info, diff --git a/src/core/render.ts b/src/core/render.ts index 924a0ead9..90fb21316 100644 --- a/src/core/render.ts +++ b/src/core/render.ts @@ -257,6 +257,23 @@ export const DEFAULT_CELL_H_BONUS = 0; export const CELL_W = ATLAS_CELL_W + DEFAULT_CELL_W_BONUS; export const CELL_H = ATLAS_CELL_H + DEFAULT_CELL_H_BONUS; +/** Visual rows per image: `floor((MAX_HEIGHT_PX − 2·PAD_Y) / CELL_H)`. Derived + * from the cell geometry above so break-even math auto-tracks it. */ +export const LINES_PER_IMAGE = Math.max(1, Math.floor((MAX_HEIGHT_PX - 2 * PAD_Y) / CELL_H)); + +/** Real char capacity of one page AT A GIVEN COLUMN WIDTH. + * + * Lives here, next to the geometry it is derived from, because every caller that + * budgets images must price pages at the width it actually renders at. The + * DENSE_CONTENT_CHARS_PER_IMAGE constant is only correct for DENSE_CONTENT_COLS + * (312×90); using it while rendering at, say, COLS=100 overstates capacity 3.1×, + * so an image budget clears a plan that then emits 3× the images — the request + * blows the API's per-request limit and comes back 500. Always pass the cols the + * renderer will actually use. */ +export function maxCharsPerImage(cols: number): number { + return Math.min(Math.max(1, cols) * LINES_PER_IMAGE, READABLE_CHARS_PER_IMAGE); +} + export interface RenderedImage { png: Uint8Array; width: number; diff --git a/src/core/session-state.ts b/src/core/session-state.ts new file mode 100644 index 000000000..0d23ceb15 --- /dev/null +++ b/src/core/session-state.ts @@ -0,0 +1,263 @@ +/** + * Per-session cache-liveness state for the history collapse. + * + * ## Why this exists + * + * The history grid is append-only: chunk N's pixels are a pure function of its + * message range, so old chunks stay byte-identical as the conversation grows and + * ride Anthropic's prompt cache as `cache_read` forever. That freeze is worth a + * lot — but only while a cache actually exists. Where there is none, the freeze + * protects nothing and the grid is free to be re-cut for *density* instead: + * {@link HistoryCollapseOptions.packFill} raises the freeze step until the pages + * are nearly full, which roughly halves image tokens on long sessions of short + * turns (#161: 317 images at 43% fill). + * + * ## How we know whether a cache exists + * + * The provider tells us, and that beats inferring it. {@link noteCacheOutcome} + * feeds each response's accounting back in: a `cache_read` proves the prefix was + * live, a `cache_create` proves one was just written. Only when both are zero is + * there nothing to lose by re-cutting. + * + * This module used to decide from the wall clock alone, treating any gap past the + * ephemeral 5-minute TTL as cold. That was wrong most of the times it fired — + * measured over 143 gaps on a production host, the cache was still warm in 66% of + * gaps past 5.5 minutes, 40% past 15 minutes, 13% past an hour. Claude Code marks + * some blocks with the 1-hour TTL, so the short constant never described this + * traffic. The worst case was a session repacked three times in one hour on + * ~10-minute gaps, each repack re-keying the whole prefix — and each preceded by a + * turn that had just *written* a cache, which the clock could not see. + * + * The clock survives only as a backstop for responses whose accounting never + * arrived: {@link COLD_HORIZON_MS}. And a rejected request still marks the session + * dead outright ({@link markCacheDead}) — but only a 413 or a too-long 400, not + * a transient 5xx: see {@link responseLeftNoCache}. + * + * ## Why the step is sticky + * + * Once a session has been repacked coarse, every later turn must keep at least + * that step. Falling back to the fine grid would re-cut the same messages into + * different chunks — every chunk's bytes change, and the whole history re-keys as + * `cache_create`. {@link recordFreezeStep} pins the floor; the collapse only ever + * doubles it. + * + * ## Failure mode we deliberately accept + * + * State is in-memory and per proxy process. After a restart a live session looks + * *unknown*, and unknown is treated as WARM (no repack) — the conservative + * choice: at worst we keep paying the old image count, we never nuke a live cache + * on a guess. The state re-arms itself on the first idle gap after the restart. + */ + +/** Sessions tracked before the oldest is evicted. One small record each. */ +const SESSIONS_MAX = 512; + +/** + * Grace added to the provider TTL before we call a cache dead. Our clock is the + * request-arrival time, the provider's is its own; a request that lands one + * second inside the window can still miss. Only gaps clearly past the TTL flip + * the session cold, so a borderline case keeps the (cheap, correct) warm path. + */ +const COLD_GRACE_MS = 30_000; + +/** + * Idle gap past which we treat an unobserved cache as gone. + * + * Not the ephemeral-tier TTL (`CACHE_TTL_SEC`, 5 minutes). Using that here declared sessions cold while their caches were + * demonstrably alive: 66% of gaps past 5.5 minutes still cache-read, 40% past + * 15 minutes, 13% past an hour (143 gaps, one production host). Claude Code marks + * some blocks with the 1-hour TTL, so the short constant never described this + * traffic. + * + * An hour plus the grace is where the evidence turns: past it, warmth is the + * exception. This is only a backstop anyway — {@link noteCacheOutcome} answers + * from the provider's own accounting whenever a response has been seen. + */ +const COLD_HORIZON_MS = 3_600_000 + COLD_GRACE_MS; + +interface SessionRecord { + /** Wall-clock ms of the last request we saw for this session. */ + lastSeenMs: number; + /** Coarsest freeze step this session has been rendered at, in messages. */ + freezeStep: number; + /** Set when a request for this session failed in a way that leaves no cache. */ + cacheDead: boolean; + /** + * Did the provider's own accounting show a cache for this session on the last + * response — either read from it, or written to it? `undefined` = not observed + * yet. This is the server's answer to a question the clock can only guess at. + */ + lastCacheAlive?: boolean; + /** + * Has this session EVER shown a cache? Absence of caching is not the same fact + * as a cache that died: a request carrying no `cache_control` marker reports + * both counters zero forever, and repacking such a session every turn would + * re-cut the grid over and over for a cache that never existed. Only a session + * that once had one, and then lost it, has provably free room to re-cut. + */ + everCacheAlive?: boolean; +} + +const sessions = new Map(); + +function touch(key: string): SessionRecord { + const existing = sessions.get(key); + if (existing) { + sessions.delete(key); // refresh LRU position + sessions.set(key, existing); + return existing; + } + const fresh: SessionRecord = { lastSeenMs: 0, freezeStep: 0, cacheDead: false }; + sessions.set(key, fresh); + while (sessions.size > SESSIONS_MAX) { + const oldest = sessions.keys().next().value; + if (oldest === undefined) break; + sessions.delete(oldest); + } + return fresh; +} + +export interface HistorySessionState { + /** The upstream prefix cache is provably gone — re-cutting the grid is free. */ + cold: boolean; + /** Floor for the freeze step, in messages. 0 = no constraint. */ + minFreezeStep: number; +} + +/** Neutral answer for callers without a session identity (no fingerprint yet). */ +const UNKNOWN_STATE: HistorySessionState = { cold: false, minFreezeStep: 0 }; + +/** + * Record a request for `sessionKey` and report what the history collapse may + * assume about the upstream cache. Call once per transformed request, BEFORE the + * collapse runs; it advances the session's last-seen clock. + * + * A session we have never seen counts as warm (see module docs) — unknown must + * never authorize a repack. + */ +export function noteHistoryRequest( + sessionKey: string | undefined, + nowMs: number = Date.now(), +): HistorySessionState { + if (!sessionKey) return UNKNOWN_STATE; + const rec = touch(sessionKey); + const known = rec.lastSeenMs > 0; + const idleMs = nowMs - rec.lastSeenMs; + + // Server truth beats the clock. If the provider's accounting showed a cache on + // the last response — read from OR written to — one exists now, and re-cutting + // the grid would throw it away. + // + // The clock alone was wrong most of the time it mattered. Measured over 143 + // gaps on a production host: past a 5.5-minute gap the cache was still warm in + // 66% of cases, past 15 minutes in 40%, past an hour in 13%. One session was + // repacked three times in an hour on ~10-minute gaps, each time re-keying the + // whole prefix — and each of those turns had just WRITTEN a cache + // (create 60-98k, read 0), which the clock could not see and this can. + const serverSaysAlive = rec.lastCacheAlive === true; + // "Gone" requires that one existed. A session whose counters were always zero + // is not a dead cache, it is a session that never had one — repacking it every + // turn would churn the grid forever for nothing to reclaim. + const serverSaysGone = rec.lastCacheAlive === false && rec.everCacheAlive === true; + + // Backstop for the case the server cannot answer: no observation yet, or an + // observation old enough that warmth is empirically rare (13% past an hour). + const beyondHorizon = known && idleMs > COLD_HORIZON_MS; + + const cold = rec.cacheDead || (!serverSaysAlive && (serverSaysGone || beyondHorizon)); + rec.lastSeenMs = nowMs; + rec.cacheDead = false; // consumed: this request gets the repack + return { cold, minFreezeStep: rec.freezeStep }; +} + +/** + * Feed the provider's cache accounting back in, once per response. + * + * `read > 0` proves the prefix was live. `create > 0` proves one was just + * written, which is the case the wall clock got wrong: a turn that paid to build + * a cache looks identical to a turn that found none, and repacking on top of it + * discards the thing just paid for. + * + * Both zero means nothing is cached for this session, so a repack costs nothing + * — that is the only situation where coarsening the grid is free. + */ +export function noteCacheOutcome( + sessionKey: string | undefined, + cacheReadTokens: number | undefined, + cacheCreateTokens: number | undefined, +): void { + if (!sessionKey) return; + const rec = sessions.get(sessionKey); + if (!rec) return; // never transformed under this key — nothing to attribute + const alive = (cacheReadTokens ?? 0) > 0 || (cacheCreateTokens ?? 0) > 0; + rec.lastCacheAlive = alive; + if (alive) rec.everCacheAlive = true; +} + +/** + * Pin the grid this session was last rendered at. Monotonic: the floor only ever + * rises, because a later, finer render would re-key every chunk it re-cuts. + */ +export function recordFreezeStep( + sessionKey: string | undefined, + step: number | undefined, +): void { + if (!sessionKey || !step || !Number.isFinite(step) || step <= 0) return; + const rec = touch(sessionKey); + if (step > rec.freezeStep) rec.freezeStep = step; +} + +/** + * Mark this session's upstream cache as gone: the last request was rejected, so + * nothing was cached and the next one may re-cut the grid for density. Call on + * the failure paths that leave no cache entry (oversized request → opaque 500). + */ +export function markCacheDead(sessionKey: string | undefined): void { + if (!sessionKey) return; + touch(sessionKey).cacheDead = true; +} + +/** + * Did this response leave the upstream prefix cache unpopulated? + * + * A cache entry is written by a request the provider actually *accepted*. Three + * outcomes mean it never got that far, so the frozen grid we were protecting + * protects nothing and the next turn may re-cut for density: + * + * - `413` — the payload was rejected outright; + * - `400` whose body says the prompt is too long (Anthropic's wording varies: + * `prompt is too long`, `prompt_too_long`, `request_too_large`). + * + * NOT any 5xx, which is what this used to say. Production disagreed: of 20871 + * requests on one host the 5xx population was 177 × `529 overloaded`, 2 × `500` + * and 1 × `503` — and 129 of 250 repacks fired directly after one of them. A 529 + * means the provider declined to process the request; the prefix cache it never + * touched is still there, and re-cutting the grid threw it away for nothing. + * + * Nor any other 4xx. A bad key or a rate limit says nothing about the cache. + * + * A cache that genuinely died needs no error to be noticed: {@link + * noteCacheOutcome} sees the next response report neither a read nor a write, and + * that is both accurate and free. + */ +export function responseLeftNoCache(status: number, errorBody?: string): boolean { + if (status === 413) return true; + if (status === 400 && errorBody) { + return /prompt[\s_-]*(is\s*)?too[\s_-]*long|request[\s_-]*too[\s_-]*large|too many (images|tokens)/i + .test(errorBody); + } + return false; +} + +/** Test seam: drop all session state. */ +export function resetSessionState(): void { + sessions.clear(); +} + +/** Test/telemetry seam: inspect a session without mutating its clock. */ +export function peekSessionState( + sessionKey: string, +): { lastSeenMs: number; freezeStep: number; cacheDead: boolean } | undefined { + const rec = sessions.get(sessionKey); + return rec ? { ...rec } : undefined; +} diff --git a/src/core/tracker.ts b/src/core/tracker.ts index 4c59b7754..9c69d0cac 100644 --- a/src/core/tracker.ts +++ b/src/core/tracker.ts @@ -27,6 +27,18 @@ export interface TrackEvent { * Compare with image_count: textTokens(n/4) vs imageTokens(n×2500). */ compressed_chars?: number; image_count?: number; + /** Images the CLIENT already put on the wire (pasted screenshots, tool-returned + * pictures). They spend from the same provider cap as ours, so this is the + * number that explains an otherwise-surprising image_budget passthrough. */ + native_images?: number; + /** Imaging paths that degraded to text because the wire cap was full. Nonzero + * here plus a fat request means the client's own images crowded us out — the + * request stayed valid, but the token win was skipped. */ + image_budget_skips?: number; + /** Image blocks really on the wire. Present only when it differs from + * image_count + native_images — i.e. when the history collapse absorbed + * messages that already carried images, so we rendered more than we sent. */ + wire_images?: number; image_bytes?: number; /** Total pixel area across all rendered images; pairs with cache_create_tokens for px/token regression. */ image_pixels?: number; @@ -69,6 +81,16 @@ export interface TrackEvent { collapsed_images?: number; /** Why history collapse didn't run (or did). Diagnostic. */ history_reason?: string; + /** Messages packed per history image. Rises when the grid is re-cut coarser; + * never falls within a session, because a finer re-cut re-keys every chunk. */ + history_freeze_step?: number; + /** Set when the grid was coarsened purely to fit the image budget — the turn + * paid legibility for a request that would otherwise have been rejected. */ + history_budget_trimmed?: boolean; + /** Set when the session's upstream cache was provably dead and the collapse + * was therefore allowed to repack for density. Pair with cache_read_tokens: + * a repack that lands on a live cache would show as a cache_create spike. */ + history_pack_fill?: boolean; /** Codepoints not in the glyph atlas. A spike means users type glyphs we don't ship — widen ATLAS_PROFILE. */ dropped_chars?: number; /** Top-20 dropped codepoints (U+HHHH keys) by frequency. Only present when dropped_chars > 0. */ @@ -98,6 +120,19 @@ export interface TrackEvent { cache_prefix_sha8?: string; /** Approx chars in that pinned prefix (growth vs pure-invalidation split). */ cache_prefix_bytes?: number; + /** Per-layer digests of the same pinned prefix. Whichever one moves between + * two turns of a session IS the cache-bust cause: tools (client loaded a + * deferred tool), system (volatile text inside the pinned span), head + * (collapse boundary / marker placement moved). */ + cache_prefix_tools_sha8?: string; + cache_prefix_system_sha8?: string; + cache_prefix_head_sha8?: string; + /** The span Anthropic really caches (through the last cache_control marker), + * its size, and the marker's position. Unstable marked digest ⇒ pxpipe-side + * bust; stable digest with cache_read 0 ⇒ look upstream, not at the rewrite. */ + cache_prefix_marked_sha8?: string; + cache_prefix_marked_bytes?: number; + cache_prefix_marker_pos?: string; // From TransformInfo.env: cwd?: string; @@ -209,6 +244,16 @@ export function toTrackEvent(ev: ProxyEvent): TrackEvent { out.compressed_chars = info.compressedChars; } if (info.imageCount !== undefined) out.image_count = info.imageCount; + // Only when nonzero: a wire without client images is the common case and + // should not pay a key per event. + if ((info.nativeImages ?? 0) > 0) out.native_images = info.nativeImages; + if ((info.imageBudgetSkips ?? 0) > 0) out.image_budget_skips = info.imageBudgetSkips; + // Emit only when it disagrees with the render counter: equality is the common + // case and a per-event key for "nothing to see" is noise. + if (info.wireImages !== undefined + && info.wireImages !== (info.imageCount ?? 0) + (info.nativeImages ?? 0)) { + out.wire_images = info.wireImages; + } if (info.imageBytes !== undefined) out.image_bytes = info.imageBytes; if (info.imagePixels !== undefined && info.imagePixels > 0) { out.image_pixels = info.imagePixels; @@ -254,6 +299,9 @@ export function toTrackEvent(ev: ProxyEvent): TrackEvent { if (info.historyReason !== undefined) { out.history_reason = info.historyReason; } + if (info.historyFreezeStep !== undefined) out.history_freeze_step = info.historyFreezeStep; + if (info.historyBudgetTrimmed) out.history_budget_trimmed = true; + if (info.historyPackFill) out.history_pack_fill = true; if (info.droppedChars !== undefined && info.droppedChars > 0) { out.dropped_chars = info.droppedChars; } @@ -262,7 +310,7 @@ export function toTrackEvent(ev: ProxyEvent): TrackEvent { } if (info.passthroughReasons) { const pr = info.passthroughReasons; - if ((pr.below_threshold ?? 0) > 0 || (pr.not_profitable ?? 0) > 0) { + if (Object.values(pr).some((n) => (n ?? 0) > 0)) { out.passthrough_reasons = pr; } } @@ -278,6 +326,13 @@ export function toTrackEvent(ev: ProxyEvent): TrackEvent { } if (info.cachePrefixSha8) out.cache_prefix_sha8 = info.cachePrefixSha8; if (info.cachePrefixBytes !== undefined) out.cache_prefix_bytes = info.cachePrefixBytes; + if (info.cachePrefixToolsSha8) out.cache_prefix_tools_sha8 = info.cachePrefixToolsSha8; + if (info.cachePrefixSystemSha8) out.cache_prefix_system_sha8 = info.cachePrefixSystemSha8; + if (info.cachePrefixHeadSha8) out.cache_prefix_head_sha8 = info.cachePrefixHeadSha8; + if (info.cachePrefixMarkedSha8) out.cache_prefix_marked_sha8 = info.cachePrefixMarkedSha8; + if (info.cachePrefixMarkedBytes !== undefined) + out.cache_prefix_marked_bytes = info.cachePrefixMarkedBytes; + if (info.cachePrefixMarkerPos) out.cache_prefix_marker_pos = info.cachePrefixMarkerPos; if (info.unknownStaticTags && info.unknownStaticTags.length > 0) out.unknown_static_tags = info.unknownStaticTags; if (info.churningStaticTags && info.churningStaticTags.length > 0) diff --git a/src/core/transform.ts b/src/core/transform.ts index 6d78b0bf5..c3d027abc 100644 --- a/src/core/transform.ts +++ b/src/core/transform.ts @@ -34,6 +34,8 @@ import { renderTextToPngsWithCharLimit, renderCellHeight, renderCellWidth, + LINES_PER_IMAGE, + maxCharsPerImage, type RenderStyle, } from './render.js'; import { @@ -43,7 +45,13 @@ import { import { factSheetText } from './factsheet.js'; import { stripSchemaDescriptions, schemaHasStructure } from './schema-strip.js'; import { bytesToBase64 } from './png.js'; -import { collapseHistory, HISTORY_SYNTHETIC_INTRO } from './history.js'; +import { + collapseHistory, + HISTORY_SYNTHETIC_INTRO, + ANTHROPIC_MAX_IMAGES, + ANTHROPIC_HISTORY_IMAGE_BUDGET, +} from './history.js'; +import { noteHistoryRequest, recordFreezeStep } from './session-state.js'; import type { GptHistoryOptions } from './openai-history.js'; import { CACHE_CREATE_RATE, CACHE_READ_RATE } from './baseline.js'; import { visionTokens, type VisionPricing } from './vision-cost.js'; @@ -303,10 +311,17 @@ function imageTokensCost( /** Gate geometry for dense tool-result, reminder, and history pages. */ function denseGateGeometry(o?: Required): GateGeometry { const profile = o?.model ? resolveGptProfile(o.model) : undefined; + const cols = o?.cols ?? profile?.stripCols ?? DENSE_CONTENT_COLS; return { - cols: o?.cols ?? profile?.stripCols ?? DENSE_CONTENT_COLS, + cols, maxHeightPx: profile?.maxHeightPx ?? MAX_HEIGHT_PX, - maxChars: DENSE_CONTENT_CHARS_PER_IMAGE, + // Price a page at the width we actually render at, NOT at the 312-col constant. + // These are the same number in the default Anthropic geometry, but a narrower + // COLS (env override, GPT strip profile) holds proportionally fewer chars: the + // old constant overstated capacity up to 3.1× at COLS=100, so the history image + // budget cleared a plan that then emitted 3× the images and the oversized + // request came back 500. Capacity must track cols or the budget is fiction. + maxChars: maxCharsPerImage(cols), style: profile?.style ?? DENSE_RENDER_STYLE, // No model on the request (Anthropic slab path): price at the Claude // profile, which is what is actually serving it. @@ -314,13 +329,10 @@ function denseGateGeometry(o?: Required): GateGeometry { }; } -/** Visual rows per image: `floor((MAX_HEIGHT_PX − 2·PAD_Y) / CELL_H)`. Derived - * from render.ts constants so break-even math auto-tracks cell geometry changes. */ -export const LINES_PER_IMAGE = Math.max(1, Math.floor((MAX_HEIGHT_PX - 2 * PAD_Y) / CELL_H)); - -export function maxCharsPerImage(cols: number): number { - return Math.min(cols * LINES_PER_IMAGE, READABLE_CHARS_PER_IMAGE); -} +/** Re-exported from render.ts, which owns the cell geometry these derive from. + * Kept exported here because the eval harnesses and gpt paths import them from + * transform. Single implementation, so the page-capacity math can't fork. */ +export { LINES_PER_IMAGE, maxCharsPerImage }; /** Lossless pre-render whitespace compactor (each `\n` costs ≥1 visual row): * 1. Strip trailing whitespace per line (preserves leading indent). @@ -479,7 +491,7 @@ export function isCompressionProfitableAmortized( /** Increment a passthrough-reason counter on `info`. Lazily allocates `passthroughReasons`. */ function bumpPassthrough( info: TransformInfo, - reason: 'below_threshold' | 'not_profitable' | 'kept_sharp', + reason: 'below_threshold' | 'not_profitable' | 'kept_sharp' | 'image_budget', ): void { if (!info.passthroughReasons) info.passthroughReasons = {}; info.passthroughReasons[reason] = (info.passthroughReasons[reason] ?? 0) + 1; @@ -635,6 +647,18 @@ export interface TransformInfo { * render may repeat its section source. Dashboard-only; not persisted. */ imageSourceTexts?: Array; toolResultImgs?: number; + /** Image blocks the CLIENT already sent (screenshots, pasted images, prior + * tool_result images). They count against the provider's hard image cap just + * like ours do, so every pxpipe imaging path must price them in — a request + * whose own images already fill the cap must not get a single one from us. + * Counted once, before any rewrite. See {@link imageHeadroom}. */ + nativeImages?: number; + /** Imaging steps skipped because the cap was exhausted (telemetry for tuning). */ + imageBudgetSkips?: number; + /** Image blocks actually present in the outgoing body — ours AND the client's. + * This is the only number the provider counts. It is <= imageCount + nativeImages + * because the history collapse can absorb messages that already carried images. */ + wireImages?: number; /** Chars of tool docs moved to the system-text Tool Reference (not imaged). */ toolDocsChars?: number; /** Codepoints missing from the atlas (rendered as blank cells). Telemetry for atlas tuning. */ @@ -642,7 +666,7 @@ export interface TransformInfo { /** Top dropped codepoints by frequency (`U+HHHH` → count), at most 20 entries. */ droppedCodepointsTop?: Record; /** Why blocks passed through without compression. Only present when count > 0. */ - passthroughReasons?: { below_threshold?: number; not_profitable?: number; kept_sharp?: number }; + passthroughReasons?: { below_threshold?: number; not_profitable?: number; kept_sharp?: number; image_budget?: number }; /** Slab gate diagnostics — imageTokens, textTokens, burn terms, and verdict. * Lets hosts measure flap-prevention efficacy and tune amortization horizon. */ gateEval?: { @@ -674,6 +698,15 @@ export interface TransformInfo { * proves Anthropic's prompt cache can `cache_read` (0.1×) instead of `cache_create`. * A changing hash means cache-key drift is back. Only set when collapse produced images. */ historyImageSha?: string; + /** Freeze-grid step the history collapse actually used, in messages. Rises when the + * adaptive packer merges chunks to fit the image budget; must never fall within a + * session (a finer re-cut re-keys every chunk). */ + historyFreezeStep?: number; + /** The collapse re-cut the grid for page fill instead of cache freeze — only set when + * the session's upstream cache was provably dead (idle past TTL, or after a reject). */ + historyPackFill?: boolean; + /** The image budget could not hold the whole closed prefix; the tail stayed live text. */ + historyBudgetTrimmed?: boolean; /** sha8 of the ACTUAL cacheable prefix sent this turn (tools + system + * message blocks through the imaged history/slab boundary; the live tail is * excluded). Read-only measurement. A change turn-over-turn within a session @@ -684,6 +717,23 @@ export interface TransformInfo { /** Approx size (chars) of that cached prefix — pairs with cachePrefixSha8 so a * bust reads as growth (size up) vs pure invalidation (size unchanged). */ cachePrefixBytes?: number; + /** Per-layer digests of that same pinned prefix, in wire order: tool + * definitions, system blocks, and the imaged head (messages up to and + * including the history/slab boundary). Exactly one of these moving names + * the cache-bust culprit; the aggregate cachePrefixSha8 alone cannot. */ + cachePrefixToolsSha8?: string; + cachePrefixSystemSha8?: string; + cachePrefixHeadSha8?: string; + /** Digest of the span Anthropic actually caches: everything up to and + * including the LAST cache_control marker. After a collapse this is a strict + * subset of the boundary-scoped prefix — the newest freeze chunk re-renders + * every turn by design and sits after the marker — so THIS is the digest that + * must stay stable turn over turn, and the boundary one is context. */ + cachePrefixMarkedSha8?: string; + cachePrefixMarkedBytes?: number; + /** Where that last marker sits, as `m.b`. A marker that + * roams between turns re-cuts the cached span and busts it on its own. */ + cachePrefixMarkerPos?: string; /** Why the history collapse didn't run (or did). Diagnostic only. */ historyReason?: | 'no_history' @@ -694,6 +744,7 @@ export interface TransformInfo { | 'not_profitable' | 'too_many_images' | 'render_empty' + | 'over_budget' | 'collapsed'; /** Token count of the pre-compression body from /v1/messages/count_tokens (free). * Absent when probe failed — event excluded from savings rollup. */ @@ -1016,13 +1067,27 @@ async function recordRecoverable( }); } -/** Hash the concatenated base64 of every image block on `messages[0]` (the synthetic - * history message). Stable across the quantized collapse window → proves Anthropic - * can cache_read the history prefix. Returns undefined if no images on messages[0]. */ +/** Hash the concatenated base64 of every image block on the synthetic history + * message. Stable across the quantized collapse window → proves Anthropic can + * cache_read the history prefix. Returns undefined if there is no such message. + * + * The synthetic message is NOT `messages[0]` whenever a slab anchor exists: + * collapseHistory returns `[...head, syntheticUser, ...tail]` and transform.ts + * passes `protectedPrefix = slabAnchorIdx + 1`, so on a Claude Code request + * `messages[0]` is the protected slab message. Hashing it reported SLAB image + * stability under the `history_image_sha8` name — the one field meant to prove + * history-image byte-identity was blind to the history images, so a drifting + * collapse boundary still looked stable in telemetry (#11 attribution). Locate + * the message by its banner instead, exactly like cachePrefixDigest does. */ async function historyImageSha8( messages: Message[], ): Promise { - const synthetic = messages[0]; + const synthetic = messages.find( + (m) => + Array.isArray(m.content) && + (m.content[0] as TextBlock | undefined)?.type === 'text' && + (m.content[0] as TextBlock).text === HISTORY_SYNTHETIC_INTRO, + ); if (!synthetic || !Array.isArray(synthetic.content)) return undefined; let concat = ''; for (const blk of synthetic.content) { @@ -1110,7 +1175,19 @@ function relocateAnchorToHistoryImage(messages: Message[] | undefined, anchorOrd */ async function cachePrefixDigest( req: { tools?: unknown; system?: unknown; messages?: unknown }, -): Promise<{ sha8: string; bytes: number } | undefined> { +): Promise< + | { + sha8: string; + bytes: number; + toolsSha8: string; + systemSha8: string; + headSha8: string; + markedSha8: string; + markedBytes: number; + markerPos: string; + } + | undefined +> { const msgs = Array.isArray(req.messages) ? (req.messages as Message[]) : []; // Boundary = latest message carrying pxpipe's imaged prefix: the history image // (banner) when collapse ran, else the slab message ('[End of rendered @@ -1127,19 +1204,70 @@ async function cachePrefixDigest( if (isHistory || hasSlab) boundary = i; } if (boundary < 0) return undefined; // not an imaged-prefix shape — nothing pinned - const parts: string[] = []; - if (Array.isArray(req.tools)) for (const t of req.tools) parts.push(JSON.stringify(t)); + // Component digests, in wire order. A whole-prefix hash proves THAT the cache + // busted but never WHICH layer moved, and the layers fail for different + // reasons: tools drift when the client loads a deferred tool, system drifts + // when volatile text (env/git status) rides inside the pinned span, and the + // imaged head drifts when a collapse boundary or marker placement moves. + // Hashing each separately turns "prefix changed" into a one-line diagnosis. + const toolParts: string[] = []; + if (Array.isArray(req.tools)) for (const t of req.tools) toolParts.push(JSON.stringify(t)); + const sysParts: string[] = []; const sys = req.system; - if (typeof sys === 'string') parts.push(sys); - else if (Array.isArray(sys)) for (const b of sys) parts.push(JSON.stringify(b)); + if (typeof sys === 'string') sysParts.push(sys); + else if (Array.isArray(sys)) for (const b of sys) sysParts.push(JSON.stringify(b)); + const headParts: string[] = []; for (let i = 0; i <= boundary; i++) { const content = msgs[i]?.content; - if (typeof content === 'string') parts.push(content); + if (typeof content === 'string') headParts.push(content); else if (Array.isArray(content)) - for (const b of content) parts.push(typeof b === 'string' ? b : JSON.stringify(b)); + for (const b of content) headParts.push(typeof b === 'string' ? b : JSON.stringify(b)); + } + // The span Anthropic actually caches ends at the LAST cache_control marker — + // not at the message boundary above. Those differ by construction after a + // collapse: the synthetic message's newest freeze chunk re-renders every turn + // BY DESIGN and sits AFTER the pinned marker, so the boundary-scoped digest + // reports a bust on every single turn even when the cached span is perfectly + // stable. Digest the marker-scoped span too, and record where the marker sits: + // a moving marker is itself a bust cause, and telemetry could not see it. + let markPos = ''; + const markedParts: string[] = [...toolParts, ...sysParts]; + const markedUpTo: string[] = []; + outer: for (let i = msgs.length - 1; i >= 0; i--) { + const content = msgs[i]?.content; + if (!Array.isArray(content)) continue; + for (let k = content.length - 1; k >= 0; k--) { + const b = content[k] as { cache_control?: unknown } | undefined; + if (b && typeof b === 'object' && b.cache_control !== undefined) { + markPos = `m${i}.b${k}`; + for (let mi = 0; mi <= i; mi++) { + const c = msgs[mi]?.content; + if (typeof c === 'string') markedUpTo.push(c); + else if (Array.isArray(c)) + for (let bk = 0; bk < c.length; bk++) { + if (mi === i && bk > k) break; + const blk = c[bk]; + markedUpTo.push(typeof blk === 'string' ? blk : JSON.stringify(blk)); + } + } + break outer; + } + } } + markedParts.push(...markedUpTo); + const marked = markedParts.join('\x00'); + const parts = [...toolParts, ...sysParts, ...headParts]; const prefix = parts.join('\x00'); - return { sha8: await sha8(prefix), bytes: prefix.length }; + return { + sha8: await sha8(prefix), + bytes: prefix.length, + toolsSha8: await sha8(toolParts.join('\x00')), + systemSha8: await sha8(sysParts.join('\x00')), + headSha8: await sha8(headParts.join('\x00')), + markedSha8: await sha8(marked), + markedBytes: marked.length, + markerPos: markPos, + }; } // Removed: extractClaudeMdSlab(). It scanned the static system text for @@ -1624,6 +1752,71 @@ function applyPins(req: MessagesRequest, info: TransformInfo, pins: Pin[]): void * Called from both the main path AND early-exit paths (below_min_chars, * not_profitable) — history collapse must run even when the slab skips. * Tolerant to missing/short message arrays (collapseHistory short-circuits). */ +/** Slack between our page estimate and the provider's hard image cap. The renderer + * paginates on its own, so a candidate grid can come out one page heavier than the + * estimator predicted; the request must still land under {@link ANTHROPIC_MAX_IMAGES}. */ +const HISTORY_IMAGE_SAFETY_MARGIN = 5; + +/** + * Image blocks this request may still add before the provider's hard cap. + * + * The cap counts EVERY image on the wire: the client's own (`nativeImages`) and + * ours (`imageCount`). Pricing only ours is how a request with 103 client images + * still got imaged further and came back 400 — the cap is a wire property, not a + * pxpipe property. Never negative; callers treat 0 as "keep it as text". + */ +export function imageHeadroom(info: TransformInfo): number { + return Math.max( + 0, + ANTHROPIC_MAX_IMAGES - + HISTORY_IMAGE_SAFETY_MARGIN - + info.imageCount - + (info.nativeImages ?? 0), + ); +} + +/** Count image blocks already present in the caller's messages. Runs BEFORE any + * rewrite, so it sees the client's images only — ours do not exist yet. */ +export function countNativeImages(messages: readonly Message[] | undefined): number { + let n = 0; + for (const m of messages ?? []) { + if (!Array.isArray(m.content)) continue; + for (const blk of m.content) { + const t = (blk as { type?: string } | null)?.type; + if (t === 'image') n++; + else if (t === 'tool_result') { + const inner = (blk as ToolResultBlock).content; + if (Array.isArray(inner)) { + for (const ib of inner) if ((ib as { type?: string } | null)?.type === 'image') n++; + } + } + } + } + return n; +} + +/** + * Per-request history-grid tuning: how many images the collapse may still spend, + * and whether it is allowed to re-cut the grid for density. + * + * `info.imageCount` already holds every image this request emitted before the + * collapse (slab, tool docs, tool_results), so the remaining headroom is the hard + * cap minus those, minus a margin for estimator drift — and never more than the + * standing cost guard {@link ANTHROPIC_HISTORY_IMAGE_BUDGET}. + * + * `packFill`/`minFreezeStep` come from the session store: repack only when the + * upstream cache is provably dead, and never below a grid this session already + * froze at. See {@link noteHistoryRequest}. + */ +function historyGridTuning( + info: TransformInfo, +): { imageBudget: number; packFill: boolean; minFreezeStep: number } { + const headroom = imageHeadroom(info); + const imageBudget = Math.max(1, Math.min(ANTHROPIC_HISTORY_IMAGE_BUDGET, headroom)); + const session = noteHistoryRequest(info.firstUserSha8); + return { imageBudget, packFill: session.cold, minFreezeStep: session.minFreezeStep }; +} + async function runHistoryCollapseAndFinalize( req: MessagesRequest, info: TransformInfo, @@ -1633,7 +1826,15 @@ async function runHistoryCollapseAndFinalize( pins: Pin[], ): Promise<{ body: Uint8Array; info: TransformInfo; collapsed: boolean }> { let collapsedFlag = false; - if (Array.isArray(req.messages) && req.messages.length > 0) { + // Same wire cap as every other imaging path: the collapse buys tokens with + // image blocks, and a request whose client images already fill the cap has + // none left to spend. `historyGridTuning` floors the budget at 1, so without + // this guard a full wire would still emit exactly one image — and 400. + if (imageHeadroom(info) <= 0) { + bumpPassthrough(info, 'image_budget'); + info.imageBudgetSkips = (info.imageBudgetSkips ?? 0) + 1; + info.historyReason ??= 'too_many_images'; + } else if (Array.isArray(req.messages) && req.messages.length > 0) { const historyCpt = opts.charsPerToken !== undefined ? o.charsPerToken : HISTORY_CHARS_PER_TOKEN; @@ -1661,6 +1862,7 @@ async function runHistoryCollapseAndFinalize( // message carrying them is protected from collapse the same way the // non-collapse path already keeps as text below. const protectedPrefix = firstMessageHasSystemReminder(req.messages) ? 1 : 0; + const tuning = historyGridTuning(info); const { messages: newMessages, info: histInfo } = await collapseHistory( req.messages, historyProfitable, @@ -1670,8 +1872,16 @@ async function runHistoryCollapseAndFinalize( reflow: o.reflow, style: historyGeometry.style, maxHeightPx: historyGeometry.maxHeightPx, + pageChars: historyGeometry.maxChars, + imageBudget: tuning.imageBudget, + packFill: tuning.packFill, + minFreezeStep: tuning.minFreezeStep, }, ); + recordFreezeStep(info.firstUserSha8, histInfo.freezeStep); + if (histInfo.freezeStep !== undefined) info.historyFreezeStep = histInfo.freezeStep; + if (histInfo.budgetTrimmed) info.historyBudgetTrimmed = true; + if (tuning.packFill) info.historyPackFill = true; if (histInfo.collapsedTurns > 0) { req.messages = newMessages; info.collapsedTurns = histInfo.collapsedTurns; @@ -1700,6 +1910,13 @@ async function runHistoryCollapseAndFinalize( } applyPins(req, info, pins); info.outgoingTextChars = countOutgoingTextChars(req); + // Ground truth, counted from the bytes we are about to send. `imageCount` is + // what we RENDERED, and the two differ: the collapse replaces whole messages, + // so any tool_result image inside the collapsed range never reaches the wire. + // Measured on a tool-heavy shape: 95 rendered, 27 on the wire. The provider + // only ever sees this number, so telemetry and any future headroom math must + // read it and not the render counter. + info.wireImages = countNativeImages(req.messages); const outBody = new TextEncoder().encode(JSON.stringify(req)); return { body: outBody, info, collapsed: collapsedFlag }; } @@ -1764,6 +1981,11 @@ export async function transformRequest( return { body, info }; } + // Price the caller's OWN images before we rewrite anything: they occupy the + // same wire cap we are about to spend from. Counted once, here, because every + // later path (slab, tool_results, history) rewrites content in place. + info.nativeImages = countNativeImages(req.messages); + // 0. User-pinned instructions. Fold the transcript's pin commands, then remove // them from the outbound copy — the client's own transcript is untouched, so // the next request still carries every command and re-derives the same state. @@ -1965,6 +2187,18 @@ export async function transformRequest( return { body: pinsRewrote ? finalized.body : body, info }; } + // The wire cap guards even the slab. Imaging it is our biggest single win, but + // a request whose client images already fill the provider's cap has no image + // left to spend: emitting one anyway turns a large-but-valid request into a + // hard 400. Degrade to plain text and say why. + if (imageHeadroom(info) <= 0) { + info.reason = 'image_budget'; + bumpPassthrough(info, 'image_budget'); + info.imageBudgetSkips = (info.imageBudgetSkips ?? 0) + 1; + const finalized = await runHistoryCollapseAndFinalize(req, info, o, opts, droppedCodepoints, pins); + return { body: pinsRewrote || finalized.collapsed ? finalized.body : body, info }; + } + // Break-even check guards even the slab (rare edge: tiny tool docs + tiny slab < 10k chars). const denseGeo = denseGateGeometry(o); // Use slab cpt (2.0) unless host pinned charsPerToken explicitly. @@ -2050,6 +2284,23 @@ export async function transformRequest( denseGeo.style, denseGeo.maxHeightPx, ); + // Quantitative cap, not just "is there room": the slab is many pages, and a + // boolean "headroom > 0" check happily emitted 15 of them into a single slot + // (measured: 94 client images + 400k slab -> 109 on the wire -> 400). All or + // nothing, because the slab is part of the cache prefix — imaging half of it + // would re-key that prefix on every turn whose client-image count moved. + if (images.length > imageHeadroom(info)) { + info.reason = `image_budget (slab needs ${images.length}, headroom ${imageHeadroom(info)})`; + bumpPassthrough(info, 'image_budget'); + info.imageBudgetSkips = (info.imageBudgetSkips ?? 0) + 1; + const finalized = await runHistoryCollapseAndFinalize(req, info, o, opts, droppedCodepoints, pins); + if (finalized.collapsed) { + info.compressed = true; + return { body: finalized.body, info }; + } + return { body: pinsRewrote ? finalized.body : body, info }; + } + const imageBlocks: ImageBlock[] = []; for (let i = 0; i < images.length; i++) { const img = images[i]!; @@ -2154,6 +2405,15 @@ export async function transformRequest( rewritten.push(blk); continue; } + // Hard wire cap: the client's own images plus ours must stay + // under the provider's limit, else the WHOLE request is rejected. + // Out of headroom → keep the text sharp; a big body beats a 400. + if (imageHeadroom(info) <= 0) { + bumpPassthrough(info, 'image_budget'); + info.imageBudgetSkips = (info.imageBudgetSkips ?? 0) + 1; + rewritten.push(blk); + continue; + } const inner = compactSlabWhitespace(innerRaw); // classifyContent sees pre-reflow `inner` so shape bucketing reflects real structure. const innerR = maybeReflow(inner, o.reflow); @@ -2165,13 +2425,17 @@ export async function transformRequest( rewritten.push(blk); } else { // Paging: truncate before render if it would blow the image cap. + // The per-result cap is the SMALLER of the configured max and what + // is actually left on the wire — the headroom shrinks as earlier + // results spend it, so each result sees the live number. + const resultImageCap = Math.min(o.maxImagesPerToolResult, imageHeadroom(info)); const linesPerImage = Math.max( 1, Math.floor((denseGeo.maxHeightPx - 2 * PAD_Y) / renderCellHeight(denseGeo.style)), ); const paged = truncateForBudget( innerR, - o.maxImagesPerToolResult, + resultImageCap, denseGeo.cols, denseGeo.maxChars, linesPerImage, @@ -2188,6 +2452,16 @@ export async function transformRequest( denseGeo.style, denseGeo.maxHeightPx, ); + // Paging is budgeted at denseGeo.cols but rendering happens at + // o.cols; when they differ the real page count can exceed the plan. + // Verify against the actual render and drop OUR images rather than + // ship a request the provider rejects outright. + if (imgs.length > imageHeadroom(info)) { + bumpPassthrough(info, 'image_budget'); + info.imageBudgetSkips = (info.imageBudgetSkips ?? 0) + 1; + rewritten.push(blk); + continue; + } (info.imagePngs ??= []).push(...rawPngs); (info.imageDims ??= []).push(...rawDims); for (const img of imgs) info.imageBytes += approxBlockBytes(img); @@ -2233,6 +2507,13 @@ export async function transformRequest( newInner.push(ib as TextBlock | ImageBlock); continue; } + // Hard wire cap — see the string-content path above. + if (imageHeadroom(info) <= 0) { + bumpPassthrough(info, 'image_budget'); + info.imageBudgetSkips = (info.imageBudgetSkips ?? 0) + 1; + newInner.push(ib as TextBlock | ImageBlock); + continue; + } // Lossless whitespace compaction before gate + render. const innerText = compactSlabWhitespace(innerTextRaw); // R3: gate/page/render on reflowed text; classify pre-reflow. @@ -2251,9 +2532,10 @@ export async function transformRequest( 1, Math.floor((denseGeo.maxHeightPx - 2 * PAD_Y) / renderCellHeight(denseGeo.style)), ); + const resultImageCap = Math.min(o.maxImagesPerToolResult, imageHeadroom(info)); const paged = truncateForBudget( innerTextR, - o.maxImagesPerToolResult, + resultImageCap, denseGeo.cols, denseGeo.maxChars, linesPerImage, @@ -2270,6 +2552,16 @@ export async function transformRequest( denseGeo.style, denseGeo.maxHeightPx, ); + // Paging is budgeted at denseGeo.cols but rendering happens at + // o.cols; when they differ the real page count can exceed the plan. + // Verify against the actual render and drop OUR images rather than + // ship a request the provider rejects outright. + if (imgs.length > imageHeadroom(info)) { + bumpPassthrough(info, 'image_budget'); + info.imageBudgetSkips = (info.imageBudgetSkips ?? 0) + 1; + newInner.push(ib as TextBlock | ImageBlock); + continue; + } (info.imagePngs ??= []).push(...rawPngs); (info.imageDims ??= []).push(...rawDims); const srcCacheControl = demoteRelocatedCacheControl((ib as { cache_control?: unknown }).cache_control); @@ -2339,6 +2631,7 @@ export async function transformRequest( ); }; const slabAnchorIdx = (req.messages ?? []).findIndex((m) => m.role === 'user'); + const tuning = historyGridTuning(info); const { messages: newMessages, info: histInfo } = await collapseHistory( req.messages, historyProfitable, @@ -2348,8 +2641,16 @@ export async function transformRequest( reflow: o.reflow, style: historyGeometry.style, maxHeightPx: historyGeometry.maxHeightPx, + pageChars: historyGeometry.maxChars, + imageBudget: tuning.imageBudget, + packFill: tuning.packFill, + minFreezeStep: tuning.minFreezeStep, }, ); + recordFreezeStep(info.firstUserSha8, histInfo.freezeStep); + if (histInfo.freezeStep !== undefined) info.historyFreezeStep = histInfo.freezeStep; + if (histInfo.budgetTrimmed) info.historyBudgetTrimmed = true; + if (tuning.packFill) info.historyPackFill = true; if (histInfo.collapsedTurns > 0) { req.messages = newMessages; info.collapsedTurns = histInfo.collapsedTurns; @@ -2398,6 +2699,12 @@ export async function transformRequest( if (pfx) { info.cachePrefixSha8 = pfx.sha8; info.cachePrefixBytes = pfx.bytes; + info.cachePrefixToolsSha8 = pfx.toolsSha8; + info.cachePrefixSystemSha8 = pfx.systemSha8; + info.cachePrefixHeadSha8 = pfx.headSha8; + info.cachePrefixMarkedSha8 = pfx.markedSha8; + info.cachePrefixMarkedBytes = pfx.markedBytes; + if (pfx.markerPos) info.cachePrefixMarkerPos = pfx.markerPos; } } // Top dropped codepoints, capped at 20 entries to bound JSONL row size. @@ -2415,6 +2722,8 @@ export async function transformRequest( } applyPins(req, info, pins); info.outgoingTextChars = countOutgoingTextChars(req); + // Ground truth for the main path too — see the note at the other call site. + info.wireImages = countNativeImages(req.messages); const outBody = new TextEncoder().encode(JSON.stringify(req)); return { body: outBody, info }; } diff --git a/src/dashboard.ts b/src/dashboard.ts index f6a21e7c5..3e66faff9 100644 --- a/src/dashboard.ts +++ b/src/dashboard.ts @@ -29,6 +29,7 @@ */ import * as fs from 'node:fs'; +import * as os from 'node:os'; import * as readline from 'node:readline'; import type { ProxyEvent } from './core/proxy.js'; import type { TrackEvent } from './core/tracker.js'; @@ -855,6 +856,9 @@ export class DashboardState { warm: warmForRow, output: out, imageCount: info.imageCount ?? 0, + nativeImages: info.nativeImages, + wireImages: info.wireImages, + imageBudgetSkips: info.imageBudgetSkips, baselineImagedTokens: info.baselineImagedTokens, buckets: { ...(info.bucketChars ?? {}) }, imageIds: [...imgIds], @@ -1199,6 +1203,9 @@ export class DashboardState { warm: warmForRow, output: out, imageCount, + nativeImages: (t as { native_images?: number }).native_images, + wireImages: (t as { wire_images?: number }).wire_images, + imageBudgetSkips: (t as { image_budget_skips?: number }).image_budget_skips, baselineImagedTokens: (t as { baseline_imaged_tokens?: number }).baseline_imaged_tokens, buckets: { ...((t as { bucket_chars?: Record }).bucket_chars ?? {}) }, imageIds: [], // PNG ring is in-memory; not restorable across restart @@ -1497,7 +1504,7 @@ export class DashboardState { } serveHtml(port: number): Response { - return htmlResponse(renderPage(port)); + return htmlResponse(renderPage(port, dashboardHostLabel())); } /** GET /fragments/ — server-rendered htmx fragments. Each one reuses @@ -1716,6 +1723,22 @@ export function dashboardPath(pathname: string): DashboardRoute | null { return null; } +/** Name of the machine serving this dashboard, shown in the title and topbar. + * PXPIPE_DASH_LABEL overrides it for hosts whose system hostname says nothing + * useful (containers, "localhost"); an explicitly empty label opts out and + * renders the unlabelled page. */ +export function dashboardHostLabel(): string { + const override = process.env.PXPIPE_DASH_LABEL; + if (override !== undefined) return override.trim(); + try { + const h = os.hostname().trim(); + // Keep it short: FQDNs push the chip past the wordmark for no added meaning. + return h.split('.')[0] || ''; + } catch { + return ''; + } +} + function htmlResponse(body: string): Response { return new Response(body, { headers: { 'content-type': 'text/html; charset=utf-8' }, diff --git a/src/dashboard/fragments.ts b/src/dashboard/fragments.ts index f875e9067..cc0d77fcb 100644 --- a/src/dashboard/fragments.ts +++ b/src/dashboard/fragments.ts @@ -410,6 +410,14 @@ export interface ContextMapData { // on the same cache state as the image path; no wall-clock-only inference. output: number; imageCount: number; + /** Image blocks the CLIENT sent. They spend from the provider's cap exactly + * like ours, so they explain a turn that compressed less than usual. */ + nativeImages?: number; + /** Image blocks really on the wire. Lower than imageCount when the history + * collapse absorbed messages that already carried our images. */ + wireImages?: number; + /** Imaging steps that degraded to text because the cap was full. */ + imageBudgetSkips?: number; baselineImagedTokens?: number; buckets: Partial>; // bucket → chars rendered to PNG imageIds: number[]; // image-ring ids for the gallery @@ -555,6 +563,28 @@ export function renderContextMapFragment( : `Billed = after cache discounts (reads at 0.1×), same basis as the Saved column. ${rawPhrase}`; const title = isLatest ? 'Latest request' : 'Selected request'; + // The provider caps a request at 100 image blocks and counts the CLIENT's + // images against the same limit. Three facts are worth showing, and only when + // they are true — a quiet turn should stay quiet: + // - the client brought its own images (they shrank our room), + // - we rendered more pages than we shipped (the collapse ate some), + // - we gave up on imaging something because the cap was full. + const capBits: string[] = []; + if ((c.nativeImages ?? 0) > 0) { + capBits.push(`${c.nativeImages} image${c.nativeImages === 1 ? '' : 's'} came from your side and count against the same 100-image request cap`); + } + if (c.wireImages !== undefined && c.wireImages < c.imageCount + (c.nativeImages ?? 0)) { + const absorbed = c.imageCount + (c.nativeImages ?? 0) - c.wireImages; + capBits.push(`${absorbed} rendered page${absorbed === 1 ? '' : 's'} never went out — the history collapse absorbed those messages (${c.wireImages} on the wire)`); + } + if ((c.imageBudgetSkips ?? 0) > 0) { + capBits.push(`${c.imageBudgetSkips} block${c.imageBudgetSkips === 1 ? '' : 's'} stayed as text because the image cap was full`); + } + const capNote = capBits.length + ? `
${capBits.map(escapeHtml).join(' · ')}
` + : ''; + + return ( `
` + `
${title} ${headline}
` + @@ -564,6 +594,7 @@ export function renderContextMapFragment( `
` + `
Compressed into images ${kFmt(totalImagedChars)} chars · ${c.imageCount} page${c.imageCount === 1 ? '' : 's'}
` + (imgRows || `
nothing imaged this request
`) + + capNote + `
pxpipe can misread exact values inside images — treat these as gist, not byte-exact.
` + `
` + `
` + @@ -766,7 +797,7 @@ export function renderStatsTableFragment(p: FullStatsPayload): string { const totalIn = (s.inputTokensTotal || 0) + (s.cacheCreateTokensTotal || 0) + (s.cacheReadTokensTotal || 0); const hitRateTok = totalIn > 0 ? ((s.cacheReadTokensTotal / totalIn) * 100).toFixed(1) + '%' : '-'; const hitRateEv = - s.eventsWithBaseline > 0 ? ((s.cacheHitEvents / s.eventsWithBaseline) * 100).toFixed(1) + '%' : '-'; + s.eventsWithUsage > 0 ? ((s.cacheHitEvents / s.eventsWithUsage) * 100).toFixed(1) + '%' : '-'; const charRatio = s.origCharsTotal > 0 ? ((s.imageBytesTotal / s.origCharsTotal) * 100).toFixed(3) + 'x' : '-'; @@ -864,6 +895,10 @@ const CSS = ` background: radial-gradient(circle at 35% 30%, #ffd0a8, var(--flame) 55%, var(--flame-strong)); box-shadow: 0 0 0 4px var(--flame-tint); flex: none; } .wordmark { font-size: 22px; font-weight: 800; color: var(--ink); letter-spacing: -0.02em; } + .wordmark-row { display: flex; align-items: baseline; gap: 8px; flex-wrap: wrap; } + /* Which machine is this? Two dashboards from two hosts look identical otherwise. */ + .hostchip { font-size: 11.5px; font-weight: 600; color: var(--muted); padding: 1px 7px; + border: 1px solid var(--line); border-radius: 999px; white-space: nowrap; } .tagline { font-size: 12.5px; color: var(--muted); margin-top: 1px; max-width: 460px; } .controls { display: flex; flex-direction: column; align-items: flex-end; gap: 6px; } @@ -1041,6 +1076,10 @@ const CSS = ` .ctx-lbl { color: var(--ink-2); } .ctx-val { color: var(--ink); font-variant-numeric: tabular-nums; white-space: nowrap; } .muted-row { color: var(--muted); font-style: italic; } .split-note { font-size: 10.5px; color: var(--muted); margin-top: 7px; } + /* Cap notes explain a turn that compressed less than the user expects, so they + must read as a reason, not as fine print. Warm tint, not an error colour — + nothing here is broken. */ + .cap-note { color: var(--ink); border-left: 2px solid var(--flame); padding-left: 7px; } .pages-title { font-size: 11px; color: var(--ink-2); margin: 12px 0 6px; } .pages { display: flex; flex-wrap: wrap; gap: 6px; max-height: 320px; overflow: auto; background: var(--surface-2); padding: 6px; border: 1px solid var(--border); border-radius: 8px; } @@ -1188,14 +1227,19 @@ const THEME_JS = ` })(); `; -export function renderPage(port: number): string { +/** `hostLabel` names the machine this proxy runs on. The dashboard is otherwise + * byte-identical across hosts, so a tab opened against a remote host through + * the tailnet front is indistinguishable from the local one - which is how a + * session gets read on the wrong box. Empty label = render as before. */ +export function renderPage(port: number, hostLabel = ''): string { + const host = escapeHtml(hostLabel.trim()); // hx-trigger="load, every Ns": paint on load then poll (2s live, 5s aggregates). return ` -pxpipe — live dashboard +${host ? `${host} · pxpipe dashboard` : 'pxpipe — live dashboard'}