diff --git a/.changeset/tiny-ants-share.md b/.changeset/tiny-ants-share.md new file mode 100644 index 00000000..c6593b56 --- /dev/null +++ b/.changeset/tiny-ants-share.md @@ -0,0 +1,5 @@ +--- +"@martian-engineering/lossless-claw": minor +--- + +Add runtime-assisted transcript GC for summarized externalized tool results so active session transcripts can shrink after oversized tool output has been condensed and preserved in `large_files`. diff --git a/.pebbles/events.jsonl b/.pebbles/events.jsonl index 8994a7dc..3efb1248 100644 --- a/.pebbles/events.jsonl +++ b/.pebbles/events.jsonl @@ -157,3 +157,18 @@ {"type":"create","timestamp":"2026-03-19T22:55:53.112027Z","issue_id":"lossless-claw-744","payload":{"description":"Observed on 2026-03-19 against live OpenClaw using ~/.openclaw/lcm.db with largeFileThresholdTokens=8000.\n\nRepro:\n1. Ask the agent to print the first 200000 characters of extensions/diffs/assets/viewer-runtime.js.\n2. LCM stores the payload inline in messages/message_parts for conversation 642.\n3. No new large_files row is created and no [LCM Tool Output: file_...] placeholder appears in messages.content.\n\nEvidence from live DB:\n- messages 118156 and 118158 are role=tool in conversation 642\n- message_parts rows for those messages are part_type=text with metadata.originalRole=toolResult, toolName=exec\n- total placeholder count in messages is 0\n- large_files has no rows created in the last day\n\nLikely cause:\ninterceptLargeToolResults() only handles params.message.role === \"toolResult\" with array content items of type tool_result/toolResult/function_call_output. Live exec output is arriving as stored role=tool with plain text parts, so the interceptor never rewrites it.\n\nExpected:\nOversized plain-text tool outputs from live OpenClaw exec/tool calls should externalize into large_files and leave an [LCM Tool Output: file_...] placeholder.","priority":"1","title":"Large tool output externalization misses plain-text tool parts from live OpenClaw exec results","type":"bug"}} {"type":"status_update","timestamp":"2026-03-19T23:24:13.028377Z","issue_id":"lossless-claw-744","payload":{"status":"in_progress"}} {"type":"close","timestamp":"2026-03-19T23:27:39.59025Z","issue_id":"lossless-claw-744","payload":{}} +{"type":"create","timestamp":"2026-03-20T23:33:25.840447Z","issue_id":"lossless-claw-71a","payload":{"description":"Add the LCM-side query/helper described in the spec to identify transcript GC candidates. Candidates must be tool-result messages, outside the protected fresh tail, already covered by summaries via summary_messages, and not currently protected in context_items. The helper should return enough metadata to build compact replacements in oldest-first batches. Suggested file touch map: src/store/summary-store.ts, src/engine.ts, test/*summary-store*, test/*engine*.","priority":"1","title":"Select transcript-GC candidates from summarized tool results","type":"task"}} +{"type":"create","timestamp":"2026-03-20T23:33:25.840444Z","issue_id":"lossless-claw-3ea","payload":{"description":"Implement Phase 2 from specs/tool-result-externalization-and-incremental-bootstrap.md now that OpenClaw has merged context-engine transcript maintenance support. Scope: conservative transcript GC for summarized oversized tool results via ContextEngine.maintain(), runtimeContext.rewriteTranscriptEntries(), candidate selection from LCM state, and integration tests. Goal: shrink active session JSONL after content is safely condensed while preserving large_files-backed recall and crash-safe transcript correctness. Reference: specs/tool-result-externalization-and-incremental-bootstrap.md and OpenClaw PR #51191.","priority":"1","title":"Phase 2: runtime-assisted transcript GC","type":"epic"}} +{"type":"create","timestamp":"2026-03-20T23:33:25.840453Z","issue_id":"lossless-claw-b6e","payload":{"description":"Implement LosslessClawEngine.maintain() using the merged OpenClaw maintenance API. Build replacement toolResult messages from existing large_files-backed placeholders, align candidates to transcript entry ids conservatively, call runtimeContext.rewriteTranscriptEntries(), and add tests proving rewrites run only for eligible summarized tool outputs on bootstrap/turn/compaction paths. Suggested file touch map: src/engine.ts, test/*engine*, test/*integration*.","priority":"1","title":"Implement maintain() transcript rewrites and tests","type":"task"}} +{"type":"rename","timestamp":"2026-03-20T23:33:41.143578Z","issue_id":"lossless-claw-71a","payload":{"new_id":"lossless-claw-3ea.1"}} +{"type":"dep_add","timestamp":"2026-03-20T23:33:41.143578Z","issue_id":"lossless-claw-3ea.1","payload":{"dep_type":"parent-child","depends_on":"lossless-claw-3ea"}} +{"type":"rename","timestamp":"2026-03-20T23:33:41.205701Z","issue_id":"lossless-claw-b6e","payload":{"new_id":"lossless-claw-3ea.2"}} +{"type":"dep_add","timestamp":"2026-03-20T23:33:41.205701Z","issue_id":"lossless-claw-3ea.2","payload":{"dep_type":"parent-child","depends_on":"lossless-claw-3ea"}} +{"type":"dep_add","timestamp":"2026-03-20T23:33:41.266466Z","issue_id":"lossless-claw-3ea","payload":{"dep_type":"blocks","depends_on":"lossless-claw-3ea.1"}} +{"type":"dep_add","timestamp":"2026-03-20T23:33:41.370143Z","issue_id":"lossless-claw-3ea","payload":{"dep_type":"blocks","depends_on":"lossless-claw-3ea.2"}} +{"type":"dep_add","timestamp":"2026-03-20T23:33:41.445513Z","issue_id":"lossless-claw-3ea.2","payload":{"dep_type":"blocks","depends_on":"lossless-claw-3ea.1"}} +{"type":"status_update","timestamp":"2026-03-20T23:33:41.51239Z","issue_id":"lossless-claw-3ea.1","payload":{"status":"in_progress"}} +{"type":"status_update","timestamp":"2026-03-20T23:41:13.844213Z","issue_id":"lossless-claw-3ea.2","payload":{"status":"in_progress"}} +{"type":"close","timestamp":"2026-03-20T23:41:13.915384Z","issue_id":"lossless-claw-3ea.1","payload":{}} +{"type":"close","timestamp":"2026-03-20T23:41:13.977321Z","issue_id":"lossless-claw-3ea.2","payload":{}} +{"type":"close","timestamp":"2026-03-20T23:41:14.053332Z","issue_id":"lossless-claw-3ea","payload":{}} diff --git a/specs/tool-result-externalization-and-incremental-bootstrap.md b/specs/tool-result-externalization-and-incremental-bootstrap.md new file mode 100644 index 00000000..8c27d27f --- /dev/null +++ b/specs/tool-result-externalization-and-incremental-bootstrap.md @@ -0,0 +1,192 @@ +# Tool Result Externalization, Transcript GC, and Incremental Bootstrap + +**Status:** In progress +**Date:** 2026-03-20 +**Scope:** `lossless-claw` plugin with small OpenClaw runtime/API support +**Priority:** High + +## Problem + +`lossless-claw` bounds model context growth, but long-lived tool-heavy sessions can still grow their active session JSONL without bound. + +Without transcript maintenance: + +- large `toolResult` payloads remain inline in the active transcript +- restart/bootstrap cost grows with transcript size +- crashes force the same oversized history to be replayed +- LCM compaction helps the model context, but not the hot transcript on disk + +The design here addresses three related concerns: + +1. externalize oversized tool output into `large_files` +2. GC old transcript entries once their content is safely condensed +3. make bootstrap proportional to transcript deltas instead of full history size + +## Current Implementation Status + +### Implemented in `lossless-claw` + +#### Phase 1: Incremental bootstrap and ingest-time externalization + +These pieces are implemented on `main`: + +- `large_files` storage with retrieval-friendly `file_...` references +- ingest-time externalization of oversized tool-result payloads +- compact `[LCM Tool Output: ...]` placeholders in stored message content +- `message_parts.metadata` linkage for `externalizedFileId`, `originalByteSize`, and `toolOutputExternalized` +- `conversation_bootstrap_state` persistence +- unchanged-file bootstrap fast path +- append-only tail-import bootstrap fast path +- streaming fallback bootstrap parsing +- constrained FTS indexing for externalized placeholders + +Relevant code: + +- [engine.ts](/Users/phaedrus/Projects/lossless-claw/src/engine.ts) +- [large-files.ts](/Users/phaedrus/Projects/lossless-claw/src/large-files.ts) +- [summary-store.ts](/Users/phaedrus/Projects/lossless-claw/src/store/summary-store.ts) +- [conversation-store.ts](/Users/phaedrus/Projects/lossless-claw/src/store/conversation-store.ts) + +#### Phase 2: Runtime-assisted transcript GC, first pass + +This branch adds the first transcript-GC pass: + +- `SummaryStore.listTranscriptGcCandidates()` returns summarized tool-result messages that are: + - already externalized into `large_files` + - covered by `summary_messages` + - no longer present as raw `context_items` +- `LcmContextEngine.maintain()` rebuilds compact replacement `toolResult` messages from stored `message_parts` +- transcript rewrite requests are sent through OpenClaw's runtime-owned `rewriteTranscriptEntries()` hook +- alignment is conservative and only proceeds when a candidate can be matched to a unique active transcript entry by `toolCallId` + +This intentionally skips ambiguous cases instead of attempting unsafe transcript surgery. + +Relevant code: + +- [engine.ts](/Users/phaedrus/Projects/lossless-claw/src/engine.ts) +- [assembler.ts](/Users/phaedrus/Projects/lossless-claw/src/assembler.ts) +- [summary-store.ts](/Users/phaedrus/Projects/lossless-claw/src/store/summary-store.ts) + +### Implemented in OpenClaw + +OpenClaw now provides the runtime support this design needed: + +- `ContextEngine.maintain()` +- `runtimeContext.rewriteTranscriptEntries()` +- safe branch-and-reappend transcript rewrites owned by the runtime +- maintenance call sites after bootstrap, successful turns, and compaction + +That runtime support landed upstream via OpenClaw PR `#51191`. + +## Design + +### Proposal A: Tool-result externalization + +Oversized tool outputs should live in `large_files`, not inline in ordinary message storage. + +Current behavior: + +- tool outputs above the configured threshold are stored out-of-line +- LCM persists a compact tool-output placeholder instead of the raw blob +- retrieval remains possible via `file_...` references + +### Proposal B: Transcript GC + +Once old tool-result content has been safely condensed, the active transcript should no longer retain the giant inline blob. + +The first pass uses this eligibility rule: + +1. message is a tool-result row in LCM +2. content was already externalized during ingest +3. message is linked through `summary_messages` +4. message is no longer a raw `context_items` entry +5. the active transcript contains a unique matching tool-result entry for the same `toolCallId` + +When all of those are true, `maintain()` asks the runtime to replace the active transcript entry with the compact placeholder-backed `toolResult`. + +### Proposal C: Incremental bootstrap + +Bootstrap should skip or tail-import when the transcript is unchanged or append-only. + +Current behavior: + +- unchanged transcript: skip bootstrap work +- append-only transcript: ingest only the tail +- suspicious rewrite/truncation: fall back to full streaming reconciliation + +## Why This Matters + +This work addresses an operational problem, not just a model-context problem. + +Benefits: + +- active session transcripts stop accumulating unbounded large tool blobs +- restarts become cheaper over time +- crash recovery avoids repeatedly paying for the same oversized raw history +- recall remains intact through `large_files` + +## Remaining Work + +The implementation is useful now, but it is not the full end state. + +### 1. Handle legacy inline oversized tool results + +The current transcript-GC pass only rewrites tool results that were already externalized during ingest. + +Still needed: + +- nominate old oversized inline tool results that predate externalization +- externalize their raw content during maintenance if needed +- then rewrite those transcript entries + +### 2. Improve transcript-entry alignment + +The current pass aligns transcript entries by unique `toolCallId`. + +That is safe, but conservative. It skips cases where: + +- the same `toolCallId` appears ambiguously +- the active transcript shape cannot be matched with confidence + +Still needed: + +- a more robust mapping strategy, or +- additive persistence of stable transcript entry ids + +### 3. Tighten eligibility and fresh-tail protection + +Today the effective protection rule is "summarized and not still a raw context item". + +Still needed: + +- an explicit fresh-tail policy +- optional size/noise thresholds for GC +- bounded batch tuning and observability for maintenance passes + +### 4. Add end-to-end runtime integration coverage + +Focused unit coverage exists for candidate selection and rewrite request generation. + +Still needed: + +- integration coverage against the real merged OpenClaw maintenance lifecycle +- verification of bootstrap/turn/compaction-triggered rewrites in realistic session files + +### 5. Phase 3 preventive hygiene + +The current model is still mostly reactive. + +Still needed: + +- write-time transcript paths that avoid landing giant inline tool blobs in the first place where possible +- optional normalization of repeated low-value progress spam + +## Recommendation + +Keep the current first pass narrow and safe, and continue Phase 2 with: + +1. legacy inline tool-result cleanup +2. stronger transcript-entry identity/alignment +3. end-to-end integration coverage + +That sequence preserves correctness while moving steadily toward bounded transcript growth in real long-lived sessions. diff --git a/src/assembler.ts b/src/assembler.ts index efcd6fd9..511112a8 100644 --- a/src/assembler.ts +++ b/src/assembler.ts @@ -420,7 +420,8 @@ export function blockFromPart(part: MessagePartRecord): unknown { return { type: "text", text: "" }; } -function contentFromParts( +/** @internal Exported for transcript-maintenance reconstruction. */ +export function contentFromParts( parts: MessagePartRecord[], role: "user" | "assistant" | "toolResult", fallbackContent: string, @@ -449,7 +450,8 @@ function contentFromParts( return blocks; } -function pickToolCallId(parts: MessagePartRecord[]): string | undefined { +/** @internal Exported for transcript-maintenance reconstruction. */ +export function pickToolCallId(parts: MessagePartRecord[]): string | undefined { for (const part of parts) { if (typeof part.toolCallId === "string" && part.toolCallId.length > 0) { return part.toolCallId; @@ -478,7 +480,8 @@ function pickToolCallId(parts: MessagePartRecord[]): string | undefined { return undefined; } -function pickToolName(parts: MessagePartRecord[]): string | undefined { +/** @internal Exported for transcript-maintenance reconstruction. */ +export function pickToolName(parts: MessagePartRecord[]): string | undefined { for (const part of parts) { if (typeof part.toolName === "string" && part.toolName.length > 0) { return part.toolName; @@ -507,7 +510,8 @@ function pickToolName(parts: MessagePartRecord[]): string | undefined { return undefined; } -function pickToolIsError(parts: MessagePartRecord[]): boolean | undefined { +/** @internal Exported for transcript-maintenance reconstruction. */ +export function pickToolIsError(parts: MessagePartRecord[]): boolean | undefined { for (const part of parts) { const decoded = parseJson(part.metadata); if (!decoded || typeof decoded !== "object") { diff --git a/src/engine.ts b/src/engine.ts index 73623e5d..03bc6b54 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -5,6 +5,7 @@ import { homedir } from "node:os"; import { join } from "node:path"; import type { DatabaseSync } from "node:sqlite"; import { createInterface } from "node:readline"; +import { SessionManager } from "@mariozechner/pi-coding-agent"; import type { ContextEngine, ContextEngineInfo, @@ -16,7 +17,14 @@ import type { SubagentEndReason, SubagentSpawnPreparation, } from "openclaw/plugin-sdk"; -import { blockFromPart, ContextAssembler } from "./assembler.js"; +import { + blockFromPart, + contentFromParts, + ContextAssembler, + pickToolCallId, + pickToolIsError, + pickToolName, +} from "./assembler.js"; import { CompactionEngine, type CompactionConfig } from "./compaction.js"; import type { LcmConfig } from "./db/config.js"; import { getLcmDbFeatures } from "./db/features.js"; @@ -50,6 +58,26 @@ import type { LcmDependencies } from "./types.js"; type AgentMessage = Parameters[0]["message"]; type AssembleResultWithSystemPrompt = AssembleResult & { systemPromptAddition?: string }; +type TranscriptRewriteReplacement = { + entryId: string; + message: AgentMessage; +}; +type TranscriptRewriteRequest = { + replacements: TranscriptRewriteReplacement[]; +}; +type ContextEngineMaintenanceResult = { + changed: boolean; + bytesFreed: number; + rewrittenEntries: number; + reason?: string; +}; +type ContextEngineMaintenanceRuntimeContext = Record & { + rewriteTranscriptEntries?: ( + request: TranscriptRewriteRequest, + ) => Promise; +}; + +const TRANSCRIPT_GC_BATCH_SIZE = 12; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -77,6 +105,71 @@ function safeBoolean(value: unknown): boolean | undefined { return typeof value === "boolean" ? value : undefined; } +function extractTranscriptToolCallId(message: AgentMessage): string | undefined { + const topLevel = message as Record; + const direct = + safeString(topLevel.toolCallId) ?? + safeString(topLevel.tool_call_id) ?? + safeString(topLevel.toolUseId) ?? + safeString(topLevel.tool_use_id) ?? + safeString(topLevel.call_id) ?? + safeString(topLevel.id); + if (direct) { + return direct; + } + + if (!Array.isArray(topLevel.content)) { + return undefined; + } + + for (const item of topLevel.content) { + const record = asRecord(item); + if (!record) { + continue; + } + const nested = + safeString(record.toolCallId) ?? + safeString(record.tool_call_id) ?? + safeString(record.toolUseId) ?? + safeString(record.tool_use_id) ?? + safeString(record.call_id) ?? + safeString(record.id); + if (nested) { + return nested; + } + } + + return undefined; +} + +function listTranscriptToolResultEntryIdsByCallId(sessionFile: string): Map { + const sessionManager = SessionManager.open(sessionFile); + const branch = sessionManager.getBranch(); + const entryIdsByCallId = new Map(); + const duplicateCallIds = new Set(); + + for (const entry of branch) { + if (entry.type !== "message" || entry.message.role !== "toolResult") { + continue; + } + const toolCallId = extractTranscriptToolCallId(entry.message as AgentMessage); + if (!toolCallId) { + continue; + } + if (entryIdsByCallId.has(toolCallId)) { + duplicateCallIds.add(toolCallId); + continue; + } + entryIdsByCallId.set(toolCallId, entry.id); + } + + for (const duplicateCallId of duplicateCallIds) { + entryIdsByCallId.delete(duplicateCallId); + } + + return entryIdsByCallId; +} + function appendTextValue(value: unknown, out: string[]): void { if (typeof value === "string") { out.push(value); @@ -1925,6 +2018,146 @@ export class LcmContextEngine implements ContextEngine { return result; } + /** + * Rebuild a compact tool-result message from stored message parts. + * + * The first transcript-GC pass only rewrites tool results that were already + * externalized into large_files during ingest, so the stored placeholder is + * the canonical replacement content. + */ + private async buildTranscriptGcReplacementMessage( + messageId: number, + ): Promise { + const message = await this.conversationStore.getMessageById(messageId); + if (!message) { + return null; + } + + const parts = await this.conversationStore.getMessageParts(messageId); + const toolCallId = pickToolCallId(parts); + if (!toolCallId) { + return null; + } + + const content = contentFromParts(parts, "toolResult", message.content); + const toolName = pickToolName(parts) ?? "unknown"; + const isError = pickToolIsError(parts); + + return { + role: "toolResult", + toolCallId, + toolName, + content, + ...(isError !== undefined ? { isError } : {}), + } as AgentMessage; + } + + /** + * Run transcript GC for summarized tool-result messages that already have a + * large_files-backed placeholder stored in LCM. + */ + async maintain(params: { + sessionId: string; + sessionFile: string; + sessionKey?: string; + runtimeContext?: ContextEngineMaintenanceRuntimeContext; + }): Promise { + if (this.shouldIgnoreSession({ sessionId: params.sessionId, sessionKey: params.sessionKey })) { + return { + changed: false, + bytesFreed: 0, + rewrittenEntries: 0, + reason: "session excluded by pattern", + }; + } + if (this.isStatelessSession(params.sessionKey)) { + return { + changed: false, + bytesFreed: 0, + rewrittenEntries: 0, + reason: "stateless session", + }; + } + if (typeof params.runtimeContext?.rewriteTranscriptEntries !== "function") { + return { + changed: false, + bytesFreed: 0, + rewrittenEntries: 0, + reason: "runtime rewrite helper unavailable", + }; + } + + return this.withSessionQueue( + this.resolveSessionQueueKey(params.sessionId, params.sessionKey), + async () => { + const conversation = await this.conversationStore.getConversationForSession({ + sessionId: params.sessionId, + sessionKey: params.sessionKey, + }); + if (!conversation) { + return { + changed: false, + bytesFreed: 0, + rewrittenEntries: 0, + reason: "conversation not found", + }; + } + + const candidates = await this.summaryStore.listTranscriptGcCandidates( + conversation.conversationId, + { limit: TRANSCRIPT_GC_BATCH_SIZE }, + ); + if (candidates.length === 0) { + return { + changed: false, + bytesFreed: 0, + rewrittenEntries: 0, + reason: "no transcript GC candidates", + }; + } + + const transcriptEntryIdsByCallId = listTranscriptToolResultEntryIdsByCallId( + params.sessionFile, + ); + const replacements: TranscriptRewriteReplacement[] = []; + const seenEntryIds = new Set(); + + for (const candidate of candidates) { + const entryId = transcriptEntryIdsByCallId.get(candidate.toolCallId); + if (!entryId || seenEntryIds.has(entryId)) { + continue; + } + + const replacementMessage = await this.buildTranscriptGcReplacementMessage( + candidate.messageId, + ); + if (!replacementMessage) { + continue; + } + + seenEntryIds.add(entryId); + replacements.push({ + entryId, + message: replacementMessage, + }); + } + + if (replacements.length === 0) { + return { + changed: false, + bytesFreed: 0, + rewrittenEntries: 0, + reason: "no matching transcript entries", + }; + } + + return params.runtimeContext.rewriteTranscriptEntries({ + replacements, + }); + }, + ); + } + private async ingestSingle(params: { sessionId: string; sessionKey?: string; diff --git a/src/store/summary-store.ts b/src/store/summary-store.ts index 31801621..9de5791f 100644 --- a/src/store/summary-store.ts +++ b/src/store/summary-store.ts @@ -110,6 +110,16 @@ export type ConversationBootstrapStateRecord = { updatedAt: Date; }; +export type TranscriptGcCandidateRecord = { + messageId: number; + conversationId: number; + seq: number; + toolCallId: string; + toolName: string | null; + externalizedFileId: string | null; + originalByteSize: number | null; +}; + // ── DB row shapes (snake_case) ──────────────────────────────────────────────── interface SummaryRow { @@ -190,6 +200,15 @@ interface ConversationBootstrapStateRow { updated_at: string; } +interface TranscriptGcCandidateRow { + message_id: number; + conversation_id: number; + seq: number; + tool_call_id: string | null; + tool_name: string | null; + metadata: string | null; +} + // ── Row mappers ─────────────────────────────────────────────────────────────── function toSummaryRecord(row: SummaryRow): SummaryRecord { @@ -280,6 +299,42 @@ function toConversationBootstrapStateRecord( }; } +function toTranscriptGcCandidateRecord( + row: TranscriptGcCandidateRow, +): TranscriptGcCandidateRecord | null { + if (typeof row.tool_call_id !== "string" || row.tool_call_id.length === 0) { + return null; + } + + let metadata: Record | null = null; + try { + metadata = + typeof row.metadata === "string" && row.metadata.length > 0 + ? (JSON.parse(row.metadata) as Record) + : null; + } catch { + metadata = null; + } + + if (!metadata || metadata.toolOutputExternalized !== true) { + return null; + } + + return { + messageId: row.message_id, + conversationId: row.conversation_id, + seq: row.seq, + toolCallId: row.tool_call_id, + toolName: row.tool_name, + externalizedFileId: + typeof metadata.externalizedFileId === "string" ? metadata.externalizedFileId : null, + originalByteSize: + typeof metadata.originalByteSize === "number" && Number.isFinite(metadata.originalByteSize) + ? Math.max(0, Math.floor(metadata.originalByteSize)) + : null, + }; +} + // ── SummaryStore ────────────────────────────────────────────────────────────── export class SummaryStore { @@ -454,6 +509,72 @@ export class SummaryStore { return rows.map((r) => r.message_id); } + /** + * Return summarized tool-result messages that are safe candidates for + * transcript GC because they are no longer present as raw context items. + */ + async listTranscriptGcCandidates( + conversationId: number, + options?: { limit?: number }, + ): Promise { + const limit = + typeof options?.limit === "number" && Number.isFinite(options.limit) && options.limit > 0 + ? Math.max(1, Math.floor(options.limit)) + : 25; + + const rows = this.db + .prepare( + `SELECT + m.message_id, + m.conversation_id, + m.seq, + mp.tool_call_id, + mp.tool_name, + mp.metadata + FROM messages m + JOIN message_parts mp + ON mp.message_id = m.message_id + WHERE m.conversation_id = ? + AND m.role = 'tool' + AND mp.part_type = 'tool' + AND mp.tool_call_id IS NOT NULL + AND mp.tool_call_id != '' + AND EXISTS ( + SELECT 1 + FROM summary_messages sm + WHERE sm.message_id = m.message_id + ) + AND NOT EXISTS ( + SELECT 1 + FROM context_items ci + WHERE ci.conversation_id = m.conversation_id + AND ci.item_type = 'message' + AND ci.message_id = m.message_id + ) + ORDER BY m.seq ASC, mp.ordinal ASC`, + ) + .all(conversationId) as unknown as TranscriptGcCandidateRow[]; + + const seenMessageIds = new Set(); + const candidates: TranscriptGcCandidateRecord[] = []; + for (const row of rows) { + if (seenMessageIds.has(row.message_id)) { + continue; + } + const candidate = toTranscriptGcCandidateRecord(row); + if (!candidate) { + continue; + } + seenMessageIds.add(candidate.messageId); + candidates.push(candidate); + if (candidates.length >= limit) { + break; + } + } + + return candidates; + } + async getSummaryChildren(parentSummaryId: string): Promise { const rows = this.db .prepare( diff --git a/test/engine.test.ts b/test/engine.test.ts index 2042aeb1..6e4780ef 100644 --- a/test/engine.test.ts +++ b/test/engine.test.ts @@ -1160,6 +1160,240 @@ describe("LcmContextEngine.ingest content extraction", () => { }); }); + it("lists summarized externalized tool results as transcript GC candidates", async () => { + await withTempHome(async () => { + const engine = createEngineWithConfig({ largeFileTokenThreshold: 20 }); + const sessionId = randomUUID(); + const toolOutput = `${"tool output line\n".repeat(160)}done`; + + await engine.ingest({ + sessionId, + message: { + role: "assistant", + content: [ + { + type: "toolCall", + id: "call_gc_candidate", + name: "exec", + input: { cmd: "pwd" }, + }, + ], + } as AgentMessage, + }); + + await engine.ingest({ + sessionId, + message: { + role: "toolResult", + toolCallId: "call_gc_candidate", + toolName: "exec", + content: [ + { + type: "tool_result", + tool_use_id: "call_gc_candidate", + name: "exec", + content: [{ type: "text", text: toolOutput }], + }, + ], + } as AgentMessage, + }); + + const conversation = await engine + .getConversationStore() + .getConversationBySessionId(sessionId); + expect(conversation).not.toBeNull(); + + const storedMessages = await engine + .getConversationStore() + .getMessages(conversation!.conversationId); + const toolMessage = storedMessages[1]; + expect(toolMessage?.role).toBe("tool"); + + const summaryId = `sum_${randomUUID().replace(/-/g, "").slice(0, 12)}`; + await engine.getSummaryStore().insertSummary({ + summaryId, + conversationId: conversation!.conversationId, + kind: "leaf", + content: "summarized tool output", + tokenCount: 16, + }); + await engine.getSummaryStore().linkSummaryToMessages(summaryId, [toolMessage.messageId]); + await engine.getSummaryStore().replaceContextRangeWithSummary({ + conversationId: conversation!.conversationId, + startOrdinal: 1, + endOrdinal: 1, + summaryId, + }); + + const candidates = await engine + .getSummaryStore() + .listTranscriptGcCandidates(conversation!.conversationId); + + expect(candidates).toHaveLength(1); + expect(candidates[0]).toMatchObject({ + messageId: toolMessage.messageId, + conversationId: conversation!.conversationId, + toolCallId: "call_gc_candidate", + toolName: "exec", + }); + expect(candidates[0]?.externalizedFileId).toMatch(/^file_[a-f0-9]{16}$/); + expect(candidates[0]?.originalByteSize).toBe(Buffer.byteLength(toolOutput, "utf8")); + }); + }); + + it("maintain() requests transcript rewrites for summarized externalized tool results", async () => { + await withTempHome(async () => { + const engine = createEngineWithConfig({ largeFileTokenThreshold: 20 }); + const sessionId = randomUUID(); + const sessionFile = createSessionFilePath("transcript-gc-maintain"); + const toolOutput = `${"tool output line\n".repeat(160)}done`; + + const sm = SessionManager.open(sessionFile); + sm.appendMessage({ + role: "assistant", + content: [ + { + type: "toolCall", + id: "call_gc_rewrite", + name: "exec", + arguments: { cmd: "pwd" }, + }, + ], + } as AgentMessage); + const toolResultEntryId = sm.appendMessage({ + role: "toolResult", + toolCallId: "call_gc_rewrite", + toolName: "exec", + content: [ + { + type: "tool_result", + tool_use_id: "call_gc_rewrite", + name: "exec", + content: [{ type: "text", text: toolOutput }], + }, + ], + } as AgentMessage); + sm.appendMessage({ + role: "assistant", + content: [{ type: "text", text: "done" }], + } as AgentMessage); + + await engine.ingest({ + sessionId, + message: { + role: "assistant", + content: [ + { + type: "toolCall", + id: "call_gc_rewrite", + name: "exec", + input: { cmd: "pwd" }, + }, + ], + } as AgentMessage, + }); + + await engine.ingest({ + sessionId, + message: { + role: "toolResult", + toolCallId: "call_gc_rewrite", + toolName: "exec", + content: [ + { + type: "tool_result", + tool_use_id: "call_gc_rewrite", + name: "exec", + content: [{ type: "text", text: toolOutput }], + }, + ], + } as AgentMessage, + }); + + await engine.ingest({ + sessionId, + message: { + role: "assistant", + content: [{ type: "text", text: "done" }], + } as AgentMessage, + }); + + const conversation = await engine + .getConversationStore() + .getConversationBySessionId(sessionId); + expect(conversation).not.toBeNull(); + + const storedMessages = await engine + .getConversationStore() + .getMessages(conversation!.conversationId); + const toolMessage = storedMessages[1]; + expect(toolMessage?.content).toContain("[LCM Tool Output: file_"); + + const summaryId = `sum_${randomUUID().replace(/-/g, "").slice(0, 12)}`; + await engine.getSummaryStore().insertSummary({ + summaryId, + conversationId: conversation!.conversationId, + kind: "leaf", + content: "summarized tool output", + tokenCount: 16, + }); + await engine.getSummaryStore().linkSummaryToMessages(summaryId, [toolMessage.messageId]); + await engine.getSummaryStore().replaceContextRangeWithSummary({ + conversationId: conversation!.conversationId, + startOrdinal: 1, + endOrdinal: 1, + summaryId, + }); + + const rewriteTranscriptEntries = vi.fn(async (request: { replacements: unknown[] }) => ({ + changed: true, + bytesFreed: 123, + rewrittenEntries: request.replacements.length, + })); + + const result = await engine.maintain({ + sessionId, + sessionFile, + runtimeContext: { + rewriteTranscriptEntries, + }, + }); + + expect(result).toEqual({ + changed: true, + bytesFreed: 123, + rewrittenEntries: 1, + }); + expect(rewriteTranscriptEntries).toHaveBeenCalledTimes(1); + expect(rewriteTranscriptEntries).toHaveBeenCalledWith({ + replacements: [ + { + entryId: toolResultEntryId, + message: expect.objectContaining({ + role: "toolResult", + toolCallId: "call_gc_rewrite", + toolName: "exec", + }), + }, + ], + }); + + const replacement = ( + rewriteTranscriptEntries.mock.calls[0]?.[0] as { + replacements?: Array<{ message?: { content?: unknown } }>; + } + )?.replacements?.[0]?.message; + expect(replacement?.content).toEqual([ + expect.objectContaining({ + type: "tool_result", + tool_use_id: "call_gc_rewrite", + name: "exec", + output: expect.stringContaining("[LCM Tool Output: file_"), + }), + ]); + }); + }); + it("serializes recycled session writes by stable sessionKey", async () => { const engine = createEngine(); const sessionKey = "agent:main:main";