diff --git a/.gitignore b/.gitignore index 9aacea9237bbd..4b3cf605bbe83 100644 --- a/.gitignore +++ b/.gitignore @@ -47,7 +47,7 @@ apps/macos/.build-local/ apps/macos/.swiftpm/ apps/shared/MoltbotKit/.swiftpm/ apps/shared/OpenClawKit/.swiftpm/ -Core/ +/Core/ apps/ios/*.xcodeproj/ apps/ios/*.xcworkspace/ apps/ios/.swiftpm/ diff --git a/extensions/openbmb-clawxmemory/src/core/index.ts b/extensions/openbmb-clawxmemory/src/core/index.ts new file mode 100644 index 0000000000000..12058d41ab6a0 --- /dev/null +++ b/extensions/openbmb-clawxmemory/src/core/index.ts @@ -0,0 +1,14 @@ +export * from "./types.js"; +export * from "./utils/id.js"; +export * from "./utils/text.js"; +export * from "./skills/types.js"; +export * from "./skills/defaults.js"; +export * from "./skills/loader.js"; +export * from "./skills/llm-extraction.js"; +export * from "./skills/intent-skill.js"; +export * from "./indexers/l1-extractor.js"; +export * from "./indexers/l2-builder.js"; +export * from "./review/dream-review.js"; +export * from "./pipeline/heartbeat.js"; +export * from "./retrieval/reasoning-loop.js"; +export * from "./storage/sqlite.js"; diff --git a/extensions/openbmb-clawxmemory/src/core/indexers/l1-extractor.ts b/extensions/openbmb-clawxmemory/src/core/indexers/l1-extractor.ts new file mode 100644 index 0000000000000..982de9312d0d0 --- /dev/null +++ b/extensions/openbmb-clawxmemory/src/core/indexers/l1-extractor.ts @@ -0,0 +1,110 @@ +import { LlmMemoryExtractor } from "../skills/llm-extraction.js"; +import type { L0SessionRecord, L1WindowRecord, MemoryMessage } from "../types.js"; +import { buildL1IndexId, nowIso } from "../utils/id.js"; + +function sameMessage(left: MemoryMessage | undefined, right: MemoryMessage | undefined): boolean { + if (!left || !right) return false; + return left.role === right.role && left.content === right.content; +} + +function startsWithMessages(list: MemoryMessage[], prefix: MemoryMessage[]): boolean { + if (prefix.length > list.length) return false; + for (let index = 0; index < prefix.length; index += 1) { + if (!sameMessage(list[index], prefix[index])) return false; + } + return true; +} + +function findOverlap(existing: MemoryMessage[], incoming: MemoryMessage[]): number { + const max = Math.min(existing.length, incoming.length); + for (let size = max; size > 0; size -= 1) { + let matched = true; + for (let index = 0; index < size; index += 1) { + if (!sameMessage(existing[existing.length - size + index], incoming[index])) { + matched = false; + break; + } + } + if (matched) return size; + } + return 0; +} + +function dedupeAdjacentMessages(messages: MemoryMessage[]): MemoryMessage[] { + const merged: MemoryMessage[] = []; + for (const message of messages) { + const previous = merged[merged.length - 1]; + if (sameMessage(previous, message)) continue; + merged.push(message); + } + return merged; +} + +function mergeMessageStream(existing: MemoryMessage[], incoming: MemoryMessage[]): MemoryMessage[] { + if (incoming.length === 0) return existing; + if (existing.length === 0) return dedupeAdjacentMessages(incoming); + if (startsWithMessages(incoming, existing)) return dedupeAdjacentMessages(incoming); + if (startsWithMessages(existing, incoming)) return dedupeAdjacentMessages(existing); + const overlap = findOverlap(existing, incoming); + return dedupeAdjacentMessages([...existing, ...incoming.slice(overlap)]); +} + +function mergeWindowMessages(records: L0SessionRecord[]): MemoryMessage[] { + return records.reduce( + (combined, record) => mergeMessageStream(combined, record.messages), + [], + ); +} + +function buildTimePeriod(startedAt: string, endedAt: string): string { + const start = new Date(startedAt); + const end = new Date(endedAt); + if (Number.isNaN(start.getTime()) || Number.isNaN(end.getTime())) return "unknown"; + const yyyyMmDd = (date: Date): string => date.toISOString().slice(0, 10); + const hhMm = (date: Date): string => + `${String(date.getHours()).padStart(2, "0")}:${String(date.getMinutes()).padStart(2, "0")}`; + if (yyyyMmDd(start) === yyyyMmDd(end)) { + return `${yyyyMmDd(start)} ${hhMm(start)}-${hhMm(end)}`; + } + return `${yyyyMmDd(start)} ${hhMm(start)} -> ${yyyyMmDd(end)} ${hhMm(end)}`; +} + +export async function extractL1FromWindow( + records: L0SessionRecord[], + extractor: LlmMemoryExtractor, +): Promise { + if (records.length === 0) { + throw new Error("Cannot build L1 window from empty L0 records"); + } + const ordered = [...records].sort((left, right) => left.timestamp.localeCompare(right.timestamp)); + const startedAt = ordered[0]!.timestamp; + const endedAt = ordered[ordered.length - 1]!.timestamp; + const mergedMessages = mergeWindowMessages(ordered); + const extracted = await extractor.extract({ + timestamp: endedAt, + messages: mergedMessages, + }); + const l0Source = ordered.map((record) => record.l0IndexId); + const l1IndexId = buildL1IndexId(startedAt, l0Source); + return { + l1IndexId, + sessionKey: ordered[0]!.sessionKey, + timePeriod: buildTimePeriod(startedAt, endedAt), + startedAt, + endedAt, + summary: extracted.summary, + facts: extracted.facts, + situationTimeInfo: extracted.situationTimeInfo, + projectTags: extracted.projectDetails.map((item) => item.name), + projectDetails: extracted.projectDetails, + l0Source, + createdAt: nowIso(), + }; +} + +export async function extractL1FromL0( + record: L0SessionRecord, + extractor: LlmMemoryExtractor, +): Promise { + return extractL1FromWindow([record], extractor); +} diff --git a/extensions/openbmb-clawxmemory/src/core/indexers/l2-builder.ts b/extensions/openbmb-clawxmemory/src/core/indexers/l2-builder.ts new file mode 100644 index 0000000000000..adeec4250c72a --- /dev/null +++ b/extensions/openbmb-clawxmemory/src/core/indexers/l2-builder.ts @@ -0,0 +1,51 @@ +import type { + L1WindowRecord, + L2ProjectIndexRecord, + L2TimeIndexRecord, + ProjectDetail, +} from "../types.js"; +import { buildL2ProjectIndexId, buildL2TimeIndexId, nowIso } from "../utils/id.js"; + +function formatLocalDayKey(value: string): string { + const date = new Date(value); + if (Number.isNaN(date.getTime())) return value.slice(0, 10) || "unknown-day"; + const year = String(date.getFullYear()); + const month = String(date.getMonth() + 1).padStart(2, "0"); + const day = String(date.getDate()).padStart(2, "0"); + return `${year}-${month}-${day}`; +} + +export function buildL2TimeFromL1(l1: L1WindowRecord, summary: string): L2TimeIndexRecord { + const dateKey = formatLocalDayKey(l1.startedAt || l1.endedAt || l1.createdAt); + const now = nowIso(); + return { + l2IndexId: buildL2TimeIndexId(dateKey), + dateKey, + summary, + l1Source: [l1.l1IndexId], + createdAt: now, + updatedAt: now, + }; +} + +export function buildL2ProjectFromDetail( + project: ProjectDetail, + l1IndexId: string, +): L2ProjectIndexRecord { + const now = nowIso(); + return { + l2IndexId: buildL2ProjectIndexId(project.key), + projectKey: project.key, + projectName: project.name, + summary: project.summary, + currentStatus: project.status, + latestProgress: project.latestProgress, + l1Source: [l1IndexId], + createdAt: now, + updatedAt: now, + }; +} + +export function buildL2ProjectsFromL1(l1: L1WindowRecord): L2ProjectIndexRecord[] { + return l1.projectDetails.map((project) => buildL2ProjectFromDetail(project, l1.l1IndexId)); +} diff --git a/extensions/openbmb-clawxmemory/src/core/pipeline/heartbeat.ts b/extensions/openbmb-clawxmemory/src/core/pipeline/heartbeat.ts new file mode 100644 index 0000000000000..91c5a282bfe55 --- /dev/null +++ b/extensions/openbmb-clawxmemory/src/core/pipeline/heartbeat.ts @@ -0,0 +1,769 @@ +import { extractL1FromWindow } from "../indexers/l1-extractor.js"; +import { buildL2ProjectFromDetail, buildL2TimeFromL1 } from "../indexers/l2-builder.js"; +import { LlmMemoryExtractor } from "../skills/llm-extraction.js"; +import { MemoryRepository } from "../storage/sqlite.js"; +import type { + ActiveTopicBufferRecord, + IndexingSettings, + L0SessionRecord, + L1WindowRecord, + L2ProjectIndexRecord, + MemoryMessage, + ProjectDetail, +} from "../types.js"; +import { buildL0IndexId, nowIso } from "../utils/id.js"; + +export interface HeartbeatOptions { + batchSize?: number; + source?: string; + settings: IndexingSettings; + logger?: { + info?: (...args: unknown[]) => void; + warn?: (...args: unknown[]) => void; + }; +} + +export interface HeartbeatRunOptions { + batchSize?: number; + sessionKeys?: string[]; + reason?: string; +} + +export interface HeartbeatStats { + l0Captured: number; + l1Created: number; + l2TimeUpdated: number; + l2ProjectUpdated: number; + profileUpdated: number; + failed: number; +} + +function sameMessage(left: MemoryMessage | undefined, right: MemoryMessage | undefined): boolean { + if (!left || !right) return false; + return left.role === right.role && left.content === right.content; +} + +function hasNewContent(previous: MemoryMessage[], incoming: MemoryMessage[]): boolean { + if (incoming.length === 0) return false; + if (previous.length === 0) return true; + if (incoming.length > previous.length) return true; + for (let index = 0; index < incoming.length; index += 1) { + if (!sameMessage(previous[index], incoming[index])) return true; + } + return false; +} + +function normalizeTurn(value: string): string { + return value.replace(/\s+/g, " ").trim(); +} + +function userTurnsFromRecord(record: L0SessionRecord): string[] { + return record.messages + .filter((message) => message.role === "user") + .map((message) => normalizeTurn(message.content)) + .filter(Boolean); +} + +function findTurnOverlap(existing: string[], incoming: string[]): number { + const max = Math.min(existing.length, incoming.length); + for (let size = max; size > 0; size -= 1) { + let matched = true; + for (let index = 0; index < size; index += 1) { + if (existing[existing.length - size + index] !== incoming[index]) { + matched = false; + break; + } + } + if (matched) return size; + } + return 0; +} + +function extractIncomingUserTurns( + record: L0SessionRecord, + buffer?: ActiveTopicBufferRecord, +): string[] { + const turns = userTurnsFromRecord(record); + if (!buffer || buffer.userTurns.length === 0) return turns; + const overlap = findTurnOverlap(buffer.userTurns, turns); + return turns.slice(overlap); +} + +function summarizeTopicSeed(turns: string[]): string { + const raw = normalizeTurn(turns[turns.length - 1] ?? turns[0] ?? "当前话题"); + return raw.length <= 120 ? raw : raw.slice(0, 120).trim(); +} + +function mergeUniqueStrings(existing: string[], incoming: string[]): string[] { + const next = [...existing]; + for (const item of incoming) { + if (!next.includes(item)) next.push(item); + } + return next; +} + +function buildLocalDateKey(timestamp: string): string { + const date = new Date(timestamp); + if (Number.isNaN(date.getTime())) return timestamp.slice(0, 10) || "unknown"; + const year = String(date.getFullYear()); + const month = String(date.getMonth() + 1).padStart(2, "0"); + const day = String(date.getDate()).padStart(2, "0"); + return `${year}-${month}-${day}`; +} + +const PROJECT_STATUS_RANK: Record = { + done: 3, + in_progress: 2, + planned: 1, +}; + +const GENERIC_PROJECT_SUMMARY_PATTERNS = [ + /^(用户|我).{0,12}(正在|目前|继续|开始|还在)/, + /^(正在|目前).{0,16}(推进|处理|做|写|准备)/, + /(进展顺利|进展还可以|还可以|持续推进|正在推进|正在处理|目前顺利|目前正常)/, +]; + +type ProjectRewriteContext = { + incomingProject: ProjectDetail; + existingProject: L2ProjectIndexRecord | null; + recentWindows: L1WindowRecord[]; +}; + +function truncateProjectText(value: string, maxLength: number): string { + const normalized = value.trim(); + if (normalized.length <= maxLength) return normalized; + return normalized.slice(0, maxLength).trim(); +} + +function normalizeComparableProjectText(value: string): string { + return value + .toLowerCase() + .replace(/\s+/g, "") + .replace(/[，。；;、,:：!?！？"'`~\-_/\\()[\]{}]/g, ""); +} + +function preferProjectStatus( + existing: ProjectDetail["status"], + incoming: ProjectDetail["status"], +): ProjectDetail["status"] { + return PROJECT_STATUS_RANK[incoming] >= PROJECT_STATUS_RANK[existing] ? incoming : existing; +} + +function chooseProjectName(existingName: string, incomingName: string): string { + return incomingName.length >= existingName.length ? incomingName : existingName; +} + +function isWeakProjectSummary( + summary: string, + projectName: string, + latestProgress: string, +): boolean { + const normalized = normalizeComparableProjectText(summary); + if (!normalized) return true; + + const normalizedName = normalizeComparableProjectText(projectName); + const normalizedLatest = normalizeComparableProjectText(latestProgress); + if (normalized.length <= Math.max(10, normalizedName.length + 4)) return true; + if ( + normalizedLatest && + (normalized === normalizedLatest || normalized.endsWith(normalizedLatest)) + ) + return true; + if ( + GENERIC_PROJECT_SUMMARY_PATTERNS.some((pattern) => pattern.test(summary)) && + !summary.includes(projectName) + ) + return true; + return false; +} + +function chooseRicherProjectSummary( + existingSummary: string, + incomingSummary: string, + projectName: string, + latestProgress: string, +): string { + const normalizedExisting = truncateProjectText(existingSummary, 360); + const normalizedIncoming = truncateProjectText(incomingSummary, 360); + const existingWeak = isWeakProjectSummary(normalizedExisting, projectName, latestProgress); + const incomingWeak = isWeakProjectSummary(normalizedIncoming, projectName, latestProgress); + + if ( + normalizedExisting && + !existingWeak && + (incomingWeak || normalizedExisting.length >= normalizedIncoming.length) + ) { + return normalizedExisting; + } + if (normalizedIncoming && !incomingWeak) return normalizedIncoming; + return normalizedIncoming || normalizedExisting; +} + +function mergeDistinctProjectText(base: string, addition: string, maxLength = 360): string { + const normalizedBase = truncateProjectText(base, maxLength); + const normalizedAddition = truncateProjectText(addition, maxLength); + if (!normalizedAddition) return normalizedBase; + if (!normalizedBase) return normalizedAddition; + + const comparableBase = normalizeComparableProjectText(normalizedBase); + const comparableAddition = normalizeComparableProjectText(normalizedAddition); + if ( + !comparableAddition || + comparableBase.includes(comparableAddition) || + comparableAddition.includes(comparableBase) + ) { + return normalizedBase; + } + + const separator = /[。！？.!?]$/.test(normalizedBase) ? " " : "；"; + return truncateProjectText(`${normalizedBase}${separator}${normalizedAddition}`, maxLength); +} + +function chooseLatestProgress( + existingProgress: string, + incomingProgress: string, + l1: L1WindowRecord, +): string { + return truncateProjectText( + incomingProgress || existingProgress || l1.situationTimeInfo || l1.summary, + 220, + ); +} + +function fallbackRewriteProjectDetail( + context: ProjectRewriteContext, + l1: L1WindowRecord, + rewritten?: ProjectDetail, +): ProjectDetail { + const { incomingProject, existingProject } = context; + const latestProgress = chooseLatestProgress( + existingProject?.latestProgress ?? "", + rewritten?.latestProgress || incomingProject.latestProgress, + l1, + ); + let summary = chooseRicherProjectSummary( + existingProject?.summary ?? "", + rewritten?.summary || incomingProject.summary, + incomingProject.name, + latestProgress, + ); + summary = mergeDistinctProjectText(summary, incomingProject.summary, 360); + summary = mergeDistinctProjectText(summary, latestProgress, 360); + if (isWeakProjectSummary(summary, incomingProject.name, latestProgress)) { + summary = mergeDistinctProjectText(summary, l1.summary, 360); + } + summary = truncateProjectText( + summary || incomingProject.summary || existingProject?.summary || incomingProject.name, + 360, + ); + + return { + ...incomingProject, + name: chooseProjectName( + existingProject?.projectName ?? "", + rewritten?.name || incomingProject.name, + ), + status: + rewritten?.status ?? incomingProject.status ?? existingProject?.currentStatus ?? "planned", + summary, + latestProgress, + confidence: Math.max(incomingProject.confidence, rewritten?.confidence ?? 0), + }; +} + +function mergeProjectDetail(existing: ProjectDetail, incoming: ProjectDetail): ProjectDetail { + const preferredStatus = preferProjectStatus(existing.status, incoming.status); + return { + ...existing, + name: chooseProjectName(existing.name, incoming.name), + status: preferredStatus, + summary: chooseRicherProjectSummary( + existing.summary, + incoming.summary, + incoming.name, + incoming.latestProgress, + ), + latestProgress: incoming.latestProgress || existing.latestProgress, + confidence: Math.max(existing.confidence, incoming.confidence), + }; +} + +async function canonicalizeL1Projects( + projects: Awaited>["projectDetails"], + repository: MemoryRepository, + extractor: LlmMemoryExtractor, +): Promise>["projectDetails"]> { + if (projects.length === 0) return projects; + const catalog = new Map< + string, + { + projectKey: string; + projectName: string; + summary: string; + currentStatus: ProjectDetail["status"]; + latestProgress: string; + } + >(); + for (const item of repository.listRecentL2Projects(60)) { + catalog.set(item.projectKey, { + projectKey: item.projectKey, + projectName: item.projectName, + summary: item.summary, + currentStatus: item.currentStatus, + latestProgress: item.latestProgress, + }); + } + + const existingProjects = Array.from(catalog.values()).map((item) => ({ + l2IndexId: `catalog:${item.projectKey}`, + projectKey: item.projectKey, + projectName: item.projectName, + summary: item.summary, + currentStatus: item.currentStatus, + latestProgress: item.latestProgress, + l1Source: [], + createdAt: "", + updatedAt: "", + })); + const normalizedProjects = await extractor.resolveProjectIdentities({ + projects, + existingProjects, + }); + const resolved = new Map(); + for (const normalized of normalizedProjects) { + const merged = resolved.has(normalized.key) + ? mergeProjectDetail(resolved.get(normalized.key)!, normalized) + : normalized; + resolved.set(merged.key, merged); + catalog.set(merged.key, { + projectKey: merged.key, + projectName: merged.name, + summary: merged.summary, + currentStatus: merged.status, + latestProgress: merged.latestProgress, + }); + } + + return Array.from(resolved.values()); +} + +async function rewriteRollingProjectMemories( + projects: ProjectDetail[], + l1: L1WindowRecord, + repository: MemoryRepository, + extractor: LlmMemoryExtractor, +): Promise { + if (projects.length === 0) return projects; + + const contexts: ProjectRewriteContext[] = projects.map((incomingProject) => { + const existingProject = repository.getL2ProjectByKey(incomingProject.key) ?? null; + const recentWindowIds = existingProject?.l1Source.slice(-4) ?? []; + const recentWindows = + recentWindowIds.length > 0 ? repository.getL1ByIds(recentWindowIds).slice(0, 4) : []; + return { + incomingProject, + existingProject, + recentWindows, + }; + }); + + try { + const rewrittenProjects = await extractor.rewriteProjectMemories({ + l1, + projects: contexts, + }); + const rewrittenByKey = new Map(rewrittenProjects.map((project) => [project.key, project])); + return contexts.map((context) => { + const rewritten = rewrittenByKey.get(context.incomingProject.key); + if (!rewritten) return fallbackRewriteProjectDetail(context, l1); + const merged = fallbackRewriteProjectDetail(context, l1, rewritten); + if (isWeakProjectSummary(merged.summary, merged.name, merged.latestProgress)) { + return fallbackRewriteProjectDetail(context, l1); + } + return merged; + }); + } catch { + return contexts.map((context) => fallbackRewriteProjectDetail(context, l1)); + } +} + +export class HeartbeatIndexer { + private readonly batchSize: number; + private readonly source: string; + private readonly logger: HeartbeatOptions["logger"]; + private settings: IndexingSettings; + + constructor( + private readonly repository: MemoryRepository, + private readonly extractor: LlmMemoryExtractor, + options: HeartbeatOptions, + ) { + this.batchSize = options.batchSize ?? 30; + this.source = options.source ?? "openclaw"; + this.settings = options.settings; + this.logger = options.logger; + } + + getSettings(): IndexingSettings { + return { ...this.settings }; + } + + setSettings(settings: IndexingSettings): void { + this.settings = { ...settings }; + } + + captureL0Session(input: { + sessionKey: string; + timestamp?: string; + messages: MemoryMessage[]; + source?: string; + }): L0SessionRecord | undefined { + const timestamp = input.timestamp ?? nowIso(); + const recent = this.repository.listRecentL0(1)[0]; + if ( + recent?.sessionKey === input.sessionKey && + !hasNewContent(recent.messages, input.messages) + ) { + this.logger?.info?.( + `[clawxmemory] skip duplicate l0 capture for session=${input.sessionKey}`, + ); + return undefined; + } + const payload = JSON.stringify(input.messages); + const l0IndexId = buildL0IndexId(input.sessionKey, timestamp, payload); + const record: L0SessionRecord = { + l0IndexId, + sessionKey: input.sessionKey, + timestamp, + messages: input.messages, + source: input.source ?? this.source, + indexed: false, + createdAt: nowIso(), + }; + this.repository.insertL0Session(record); + return record; + } + + private createTopicBuffer( + record: L0SessionRecord, + incomingUserTurns: string[], + topicSummary?: string, + ): ActiveTopicBufferRecord { + const seedTurns = + incomingUserTurns.length > 0 ? incomingUserTurns : userTurnsFromRecord(record); + const now = nowIso(); + return { + sessionKey: record.sessionKey, + startedAt: record.timestamp, + updatedAt: record.timestamp, + topicSummary: topicSummary?.trim() || summarizeTopicSeed(seedTurns), + userTurns: seedTurns, + l0Ids: [record.l0IndexId], + lastL0Id: record.l0IndexId, + createdAt: now, + }; + } + + private createTopicBufferFromBatch( + records: L0SessionRecord[], + incomingUserTurns: string[], + topicSummary?: string, + ): ActiveTopicBufferRecord { + const first = records[0]!; + const last = records[records.length - 1]!; + const seedTurns = + incomingUserTurns.length > 0 + ? incomingUserTurns + : records.flatMap((record) => userTurnsFromRecord(record)); + return { + sessionKey: first.sessionKey, + startedAt: first.timestamp, + updatedAt: last.timestamp, + topicSummary: topicSummary?.trim() || summarizeTopicSeed(seedTurns), + userTurns: seedTurns, + l0Ids: records.map((record) => record.l0IndexId), + lastL0Id: last.l0IndexId, + createdAt: nowIso(), + }; + } + + private extendTopicBuffer( + buffer: ActiveTopicBufferRecord, + record: L0SessionRecord, + incomingUserTurns: string[], + topicSummary?: string, + ): ActiveTopicBufferRecord { + return { + ...buffer, + updatedAt: record.timestamp, + topicSummary: + topicSummary?.trim() || buffer.topicSummary || summarizeTopicSeed(buffer.userTurns), + userTurns: mergeUniqueStrings(buffer.userTurns, incomingUserTurns), + l0Ids: mergeUniqueStrings(buffer.l0Ids, [record.l0IndexId]), + lastL0Id: record.l0IndexId, + }; + } + + private async closeTopicBuffer( + sessionKey: string, + stats: HeartbeatStats, + reason: string, + ): Promise { + const buffer = this.repository.getActiveTopicBuffer(sessionKey); + if (!buffer || buffer.l0Ids.length === 0) { + if (buffer) this.repository.deleteActiveTopicBuffer(sessionKey); + return; + } + + const records = this.repository.getL0ByIds(buffer.l0Ids); + if (records.length === 0) { + this.repository.deleteActiveTopicBuffer(sessionKey); + return; + } + + const extracted = await extractL1FromWindow(records, this.extractor); + const canonicalProjects = await canonicalizeL1Projects( + extracted.projectDetails, + this.repository, + this.extractor, + ); + const l1 = { + ...extracted, + projectDetails: canonicalProjects, + projectTags: canonicalProjects.map((project) => project.name), + }; + this.repository.insertL1Window(l1); + for (const l0 of records) { + this.repository.insertLink("l1", l1.l1IndexId, "l0", l0.l0IndexId); + } + stats.l1Created += 1; + + const dateKey = buildLocalDateKey(l1.endedAt); + const existingDay = this.repository.getL2TimeByDate(dateKey); + const daySummary = await this.extractor.rewriteDailyTimeSummary({ + dateKey, + existingSummary: existingDay?.summary ?? "", + l1, + }); + const l2Time = buildL2TimeFromL1(l1, daySummary); + this.repository.upsertL2TimeIndex(l2Time); + this.repository.insertLink("l2", l2Time.l2IndexId, "l1", l1.l1IndexId); + stats.l2TimeUpdated += 1; + + const rollingProjects = await rewriteRollingProjectMemories( + l1.projectDetails, + l1, + this.repository, + this.extractor, + ); + const projectIndexes = rollingProjects.map((project) => + buildL2ProjectFromDetail(project, l1.l1IndexId), + ); + for (const l2Project of projectIndexes) { + this.repository.upsertL2ProjectIndex(l2Project); + this.repository.insertLink("l2", l2Project.l2IndexId, "l1", l1.l1IndexId); + stats.l2ProjectUpdated += 1; + } + + const currentProfile = this.repository.getGlobalProfileRecord(); + const nextProfileText = await this.extractor.rewriteGlobalProfile({ + existingProfile: currentProfile.profileText, + l1, + }); + this.repository.upsertGlobalProfile(nextProfileText, [l1.l1IndexId]); + stats.profileUpdated += 1; + + this.repository.deleteActiveTopicBuffer(sessionKey); + this.logger?.info?.( + `[clawxmemory] closed topic session=${sessionKey} reason=${reason} l1=${l1.l1IndexId} l0=${records.length}`, + ); + } + + private async closeOtherSessionBuffers( + currentSessionKey: string, + stats: HeartbeatStats, + reason: string, + ): Promise { + const openBuffers = this.repository.listActiveTopicBuffers(); + for (const buffer of openBuffers) { + if (buffer.sessionKey === currentSessionKey) continue; + await this.closeTopicBuffer(buffer.sessionKey, stats, `${reason}:session_boundary`); + } + } + + private async processPendingRecord( + record: L0SessionRecord, + stats: HeartbeatStats, + reason: string, + ): Promise { + await this.closeOtherSessionBuffers(record.sessionKey, stats, reason); + + const buffer = this.repository.getActiveTopicBuffer(record.sessionKey); + const incomingUserTurns = extractIncomingUserTurns(record, buffer); + if (!buffer) { + this.repository.upsertActiveTopicBuffer(this.createTopicBuffer(record, incomingUserTurns)); + return; + } + + if (incomingUserTurns.length === 0) { + this.repository.upsertActiveTopicBuffer( + this.extendTopicBuffer(buffer, record, incomingUserTurns), + ); + return; + } + + const decision = await this.extractor.judgeTopicShift({ + currentTopicSummary: buffer.topicSummary, + recentUserTurns: buffer.userTurns.slice(-8), + incomingUserTurns, + }); + + if (decision.topicChanged) { + await this.closeTopicBuffer(record.sessionKey, stats, `${reason}:topic_shift`); + this.repository.upsertActiveTopicBuffer( + this.createTopicBuffer(record, incomingUserTurns, decision.topicSummary), + ); + return; + } + + this.repository.upsertActiveTopicBuffer( + this.extendTopicBuffer(buffer, record, incomingUserTurns, decision.topicSummary), + ); + } + + private async processPendingSession( + records: L0SessionRecord[], + stats: HeartbeatStats, + reason: string, + ): Promise { + if (records.length === 0) return; + const sessionKey = records[0]!.sessionKey; + await this.closeOtherSessionBuffers(sessionKey, stats, reason); + + const buffer = this.repository.getActiveTopicBuffer(sessionKey); + if (!buffer) { + const mergedTurns = records.flatMap((record) => userTurnsFromRecord(record)); + this.repository.upsertActiveTopicBuffer( + this.createTopicBufferFromBatch(records, mergedTurns), + ); + return; + } + + let scratch = buffer; + let mergedIncomingTurns: string[] = []; + for (const record of records) { + const incomingUserTurns = extractIncomingUserTurns(record, scratch); + if (incomingUserTurns.length > 0) { + mergedIncomingTurns = mergeUniqueStrings(mergedIncomingTurns, incomingUserTurns); + } + scratch = this.extendTopicBuffer(scratch, record, incomingUserTurns); + } + + if (mergedIncomingTurns.length === 0) { + this.repository.upsertActiveTopicBuffer(scratch); + return; + } + + const decision = await this.extractor.judgeTopicShift({ + currentTopicSummary: buffer.topicSummary, + recentUserTurns: buffer.userTurns.slice(-8), + incomingUserTurns: mergedIncomingTurns, + }); + + if (decision.topicChanged) { + await this.closeTopicBuffer(sessionKey, stats, `${reason}:topic_shift`); + this.repository.upsertActiveTopicBuffer( + this.createTopicBufferFromBatch(records, mergedIncomingTurns, decision.topicSummary), + ); + return; + } + + this.repository.upsertActiveTopicBuffer({ + ...scratch, + topicSummary: decision.topicSummary?.trim() || scratch.topicSummary, + }); + } + + async runHeartbeat(options: HeartbeatRunOptions = {}): Promise { + const stats: HeartbeatStats = { + l0Captured: 0, + l1Created: 0, + l2TimeUpdated: 0, + l2ProjectUpdated: 0, + profileUpdated: 0, + failed: 0, + }; + + const batchSize = options.batchSize ?? this.batchSize; + const sessionKeys = + Array.isArray(options.sessionKeys) && options.sessionKeys.length > 0 + ? Array.from(new Set(options.sessionKeys)) + : undefined; + const reason = options.reason ?? "heartbeat"; + + while (true) { + const pending = this.repository.listUnindexedL0Sessions(batchSize, sessionKeys); + if (pending.length === 0) break; + stats.l0Captured += pending.length; + + const indexedIds: string[] = []; + const grouped = new Map(); + for (const record of pending) { + const list = grouped.get(record.sessionKey) ?? []; + list.push(record); + grouped.set(record.sessionKey, list); + } + for (const records of grouped.values()) { + try { + await this.processPendingSession(records, stats, reason); + indexedIds.push(...records.map((record) => record.l0IndexId)); + } catch (error) { + stats.failed += 1; + this.logger?.warn?.( + `[clawxmemory] heartbeat failed reason=${reason} session=${records[0]?.sessionKey ?? "unknown"} l0=${records[0]?.l0IndexId ?? "unknown"}: ${String(error)}`, + ); + } + } + + this.repository.markL0Indexed(indexedIds); + if (indexedIds.length === 0) break; + if (pending.length < batchSize) break; + } + + if (reason === "session_boundary" && sessionKeys && sessionKeys.length > 0) { + for (const sessionKey of sessionKeys) { + try { + await this.closeTopicBuffer(sessionKey, stats, reason); + } catch (error) { + stats.failed += 1; + this.logger?.warn?.( + `[clawxmemory] close topic failed reason=${reason} session=${sessionKey}: ${String(error)}`, + ); + } + } + } + + if (reason === "manual") { + for (const buffer of this.repository.listActiveTopicBuffers()) { + try { + await this.closeTopicBuffer(buffer.sessionKey, stats, reason); + } catch (error) { + stats.failed += 1; + this.logger?.warn?.( + `[clawxmemory] close topic failed reason=${reason} session=${buffer.sessionKey}: ${String(error)}`, + ); + } + } + } + + if ( + stats.l1Created > 0 || + stats.l2TimeUpdated > 0 || + stats.l2ProjectUpdated > 0 || + stats.profileUpdated > 0 + ) { + this.repository.setPipelineState("lastIndexedAt", nowIso()); + } + return stats; + } +} diff --git a/extensions/openbmb-clawxmemory/src/core/retrieval/reasoning-loop.ts b/extensions/openbmb-clawxmemory/src/core/retrieval/reasoning-loop.ts new file mode 100644 index 0000000000000..1079fb97ef0ae --- /dev/null +++ b/extensions/openbmb-clawxmemory/src/core/retrieval/reasoning-loop.ts @@ -0,0 +1,1717 @@ +import { + LlmMemoryExtractor, + type L2CatalogEntry, + type LookupQuerySpec, +} from "../skills/llm-extraction.js"; +import type { SkillsRuntime } from "../skills/types.js"; +import { MemoryRepository } from "../storage/sqlite.js"; +import type { + GlobalProfileRecord, + IndexingSettings, + L0SearchResult, + L0SessionRecord, + L1SearchResult, + L1WindowRecord, + L2SearchResult, + MemoryMessage, + RetrievalPromptDebug, + RetrievalTraceDetail, + RetrievalTrace, + RetrievalTraceStep, + RetrievalResult, + RecallMode, +} from "../types.js"; +import { hashText, nowIso } from "../utils/id.js"; +import { truncate } from "../utils/text.js"; + +const RECALL_CACHE_TTL_MS = 30_000; +const DEFAULT_RECALL_TOP_K = 10; + +export interface RetrievalOptions { + l2Limit?: number; + l1Limit?: number; + l0Limit?: number; + includeFacts?: boolean; + retrievalMode?: "auto" | "explicit"; + recentMessages?: MemoryMessage[]; +} + +export interface RetrievalRuntimeOptions { + getSettings?: () => IndexingSettings; + isBackgroundBusy?: () => boolean; +} + +export interface RetrievalRuntimeStats { + lastRecallMs: number; + recallTimeouts: number; + lastRecallMode: RecallMode; + lastRecallPath: "auto" | "explicit" | "shadow"; + lastRecallBudgetLimited: boolean; + lastShadowDeepQueued: boolean; + lastRecallInjected: boolean; + lastRecallEnoughAt: RetrievalResult["enoughAt"]; + lastRecallCacheHit: boolean; +} + +interface RecallCacheEntry { + expiresAt: number; + result: RetrievalResult; +} + +interface RetrieveExecutionOptions { + retrievalMode: "auto" | "explicit"; + updateRuntimeStats: boolean; + savePrimaryCache: boolean; +} + +interface LocalFallbackCandidates { + profile: GlobalProfileRecord | null; + l2: L2SearchResult[]; + truncated: boolean; +} + +interface PackedL2Catalog { + entries: L2CatalogEntry[]; + byId: Map; + truncated: boolean; +} + +interface RecallLimits { + l2: number; + l1: number; + l0: number; +} + +function renderProfile(profile: GlobalProfileRecord | null): string { + if (!profile?.profileText.trim()) return ""; + return ["## Global Profile", profile.profileText.trim()].join("\n"); +} + +function renderEvidenceNote(note: string): string { + if (!note.trim()) return ""; + return ["## Evidence Note", note.trim()].join("\n"); +} + +function renderL2(results: L2SearchResult[]): string { + if (results.length === 0) return ""; + const lines: string[] = ["## L2 Indexes"]; + for (const hit of results) { + if (hit.level === "l2_time") { + lines.push(`- [time:${hit.item.dateKey}] ${truncate(hit.item.summary, 180)}`); + continue; + } + lines.push( + `- [project:${hit.item.projectName}] status=${hit.item.currentStatus} | ${truncate(hit.item.latestProgress || hit.item.summary, 140)}`, + ); + } + return lines.join("\n"); +} + +function renderL1(results: L1SearchResult[]): string { + if (results.length === 0) return ""; + const lines: string[] = ["## L1 Windows"]; + for (const hit of results) { + lines.push(`- [${hit.item.timePeriod}] ${truncate(hit.item.summary, 180)}`); + } + return lines.join("\n"); +} + +function renderL0(results: L0SearchResult[]): string { + if (results.length === 0) return ""; + const lines: string[] = ["## L0 Raw Sessions"]; + for (const hit of results) { + lines.push(`- [${hit.item.timestamp}]`); + for (const message of hit.item.messages.slice(-4)) { + lines.push(` ${message.role}: ${truncate(message.content, 260)}`); + } + } + return lines.join("\n"); +} + +function renderContextTemplate( + template: string, + input: { + intent: RetrievalResult["intent"]; + enoughAt: RetrievalResult["enoughAt"]; + profileBlock: string; + evidenceNoteBlock: string; + l2Block: string; + l1Block: string; + l0Block: string; + }, +): string { + let content = template; + content = content.replaceAll("{{intent}}", input.intent); + content = content.replaceAll("{{enoughAt}}", input.enoughAt); + content = content.replaceAll("{{profileBlock}}", input.profileBlock); + content = content.replaceAll("{{evidenceNoteBlock}}", input.evidenceNoteBlock); + content = content.replaceAll("{{l2Block}}", input.l2Block); + content = content.replaceAll("{{l1Block}}", input.l1Block); + content = content.replaceAll("{{l0Block}}", input.l0Block); + return content.trim(); +} + +function toRankScore(index: number): number { + return Math.max(0.1, 1 - index * 0.12); +} + +function normalizeQueryKey(query: string): string { + return query.toLowerCase().replace(/\s+/g, " ").trim(); +} + +function isTimeoutError(error: unknown): boolean { + return error instanceof Error && (error.name === "AbortError" || /timeout/i.test(error.message)); +} + +function withDebug(result: RetrievalResult, debug: RetrievalResult["debug"]): RetrievalResult { + return debug ? { ...result, debug } : result; +} + +function buildTraceId(prefix: string, seed: string): string { + return `${prefix}_${hashText(`${seed}:${Date.now()}:${Math.random().toString(36).slice(2, 10)}`)}`; +} + +function previewText(value: string, max = 220): string { + return truncate(value.trim(), max); +} + +function asDisplayText(value: unknown): string { + if (typeof value === "string") return value.trim(); + if (typeof value === "number" || typeof value === "boolean") return String(value); + if (value == null) return ""; + try { + return JSON.stringify(value); + } catch { + return String(value); + } +} + +function textDetail(key: string, label: string, text: string): RetrievalTraceDetail { + return { key, label, kind: "text", text }; +} + +function noteDetail(key: string, label: string, text: string): RetrievalTraceDetail { + return { key, label, kind: "note", text }; +} + +function listDetail(key: string, label: string, items: string[]): RetrievalTraceDetail { + return { key, label, kind: "list", items }; +} + +function kvDetail( + key: string, + label: string, + entries: Array<{ label: string; value: unknown }>, +): RetrievalTraceDetail { + return { + key, + label, + kind: "kv", + entries: entries + .map((entry) => ({ label: entry.label, value: asDisplayText(entry.value) })) + .filter((entry) => entry.value), + }; +} + +function jsonDetail(key: string, label: string, json: unknown): RetrievalTraceDetail { + return { key, label, kind: "json", json }; +} + +function describeL2Result(hit: L2SearchResult): string { + return hit.level === "l2_time" + ? `${hit.item.l2IndexId} · ${hit.item.dateKey} · ${previewText(hit.item.summary, 180)}` + : `${hit.item.l2IndexId} · ${hit.item.projectName} · status=${hit.item.currentStatus} · ${previewText(hit.item.latestProgress || hit.item.summary, 180)}`; +} + +function describeL1Result(item: L1WindowRecord): string { + const projects = item.projectDetails + .map((project) => project.name) + .filter(Boolean) + .slice(0, 4) + .join(", "); + return `${item.l1IndexId} · ${item.timePeriod} · ${previewText(item.summary, 140)}${projects ? ` · projects=${projects}` : ""}`; +} + +function describeL0Result(item: L0SessionRecord): string { + const preview = item.messages + .slice(-2) + .map((message) => `${message.role}: ${previewText(message.content, 120)}`) + .join(" | "); + return `${item.l0IndexId} · ${item.timestamp}${preview ? ` · ${preview}` : ""}`; +} + +function buildLookupQueryItems(lookupQueries: LookupQuerySpec[]): string[] { + return lookupQueries.map((entry) => { + const route = entry.targetTypes.join("+") || "time+project"; + const range = entry.timeRange + ? ` [${entry.timeRange.startDate}..${entry.timeRange.endDate}]` + : ""; + return `${route} · ${entry.lookupQuery || "(same as query)"}${range}`; + }); +} + +function summarizeLookupQueries(lookupQueries: LookupQuerySpec[]): string { + if (lookupQueries.length === 0) return "No structured lookup queries."; + return lookupQueries + .map((entry) => { + const route = entry.targetTypes.join("+") || "time+project"; + const range = entry.timeRange + ? ` [${entry.timeRange.startDate}..${entry.timeRange.endDate}]` + : ""; + return `${route}: ${entry.lookupQuery || "(same as query)"}${range}`; + }) + .join(" | "); +} + +function summarizeL2Results(results: L2SearchResult[]): string { + if (results.length === 0) return "No L2 candidates."; + return results + .map((hit) => + hit.level === "l2_time" + ? `${hit.item.dateKey}` + : `${hit.item.projectName} (${hit.item.currentStatus})`, + ) + .join(" | "); +} + +function summarizeL1Windows(results: L1WindowRecord[]): string { + if (results.length === 0) return "No L1 candidates."; + return results.map((item) => `${item.l1IndexId} · ${item.timePeriod}`).join(" | "); +} + +function summarizeL0Sessions(results: L0SessionRecord[]): string { + if (results.length === 0) return "No L0 candidates."; + return results.map((item) => `${item.l0IndexId} · ${item.timestamp}`).join(" | "); +} + +function createRetrievalTrace(query: string, mode: "auto" | "explicit"): RetrievalTrace { + const startedAt = nowIso(); + return { + traceId: buildTraceId("trace", `${mode}:${query}`), + query, + mode, + startedAt, + finishedAt: startedAt, + steps: [], + }; +} + +function appendTraceStep(trace: RetrievalTrace, step: Omit): void { + trace.steps.push({ + ...step, + stepId: `${trace.traceId}:step:${trace.steps.length + 1}`, + }); +} + +function finishTrace(trace: RetrievalTrace): RetrievalTrace { + return { + ...trace, + finishedAt: nowIso(), + steps: trace.steps.map((step) => ({ + ...step, + ...(step.refs ? { refs: { ...step.refs } } : {}), + ...(step.metrics ? { metrics: { ...step.metrics } } : {}), + ...(step.details ? { details: structuredClone(step.details) } : {}), + ...(step.promptDebug ? { promptDebug: structuredClone(step.promptDebug) } : {}), + })), + }; +} + +function attachTrace(result: RetrievalResult, trace: RetrievalTrace): RetrievalResult { + return { + ...result, + trace: finishTrace(trace), + }; +} + +function summarizeCorrections(corrections: string[] | undefined): string { + if (!corrections || corrections.length === 0) return "No corrections."; + return corrections.join(" | "); +} + +function appendContextRenderedStep(trace: RetrievalTrace, result: RetrievalResult): void { + const injected = Boolean(result.context.trim()); + appendTraceStep(trace, { + kind: "context_rendered", + title: "Context Rendered", + status: injected ? "success" : "skipped", + inputSummary: `intent=${result.intent} · enoughAt=${result.enoughAt}`, + outputSummary: injected + ? `Injected ${result.context.length} chars. ${previewText(result.context, 260)}` + : "No context injected.", + refs: { + intent: result.intent, + enoughAt: result.enoughAt, + injected, + }, + metrics: { + contextChars: result.context.length, + l2Count: result.l2Results.length, + l1Count: result.l1Results.length, + l0Count: result.l0Results.length, + }, + details: [ + kvDetail("final-state", "Final State", [ + { label: "intent", value: result.intent }, + { label: "enoughAt", value: result.enoughAt }, + { label: "injected", value: injected ? "true" : "false" }, + ]), + ...(result.evidenceNote.trim() + ? [noteDetail("final-note", "Final Evidence Note", result.evidenceNote)] + : []), + ...(injected + ? [textDetail("context-preview", "Injected Context Preview", result.context)] + : []), + kvDetail("level-counts", "Rendered Evidence Counts", [ + { label: "L2", value: result.l2Results.length }, + { label: "L1", value: result.l1Results.length }, + { label: "L0", value: result.l0Results.length }, + ]), + ], + }); +} + +function appendRecallSkippedStep( + trace: RetrievalTrace, + reason: string, + outputSummary: string, + refs?: Record, +): void { + appendTraceStep(trace, { + kind: "recall_skipped", + title: "Recall Skipped", + status: "skipped", + inputSummary: `reason=${reason}`, + outputSummary, + ...(refs ? { refs } : {}), + details: [ + kvDetail("skip-reason", "Skip Reason", [{ label: "reason", value: reason }]), + ...(refs ? [jsonDetail("skip-refs", "Skip Metadata", refs)] : []), + ], + }); +} + +function appendFallbackStep( + trace: RetrievalTrace, + corrections: string[], + result: RetrievalResult, +): void { + appendTraceStep(trace, { + kind: "fallback_applied", + title: "Fallback Applied", + status: "warning", + inputSummary: summarizeCorrections(corrections), + outputSummary: previewText( + result.evidenceNote || result.context || "No fallback evidence.", + 260, + ), + refs: { + enoughAt: result.enoughAt, + corrections, + }, + metrics: { + l2Count: result.l2Results.length, + l1Count: result.l1Results.length, + l0Count: result.l0Results.length, + }, + details: [ + listDetail("fallback-corrections", "Fallback Reasons", corrections), + ...(result.evidenceNote.trim() + ? [noteDetail("fallback-note", "Fallback Evidence Note", result.evidenceNote)] + : []), + kvDetail("fallback-levels", "Fallback Result", [ + { label: "enoughAt", value: result.enoughAt }, + { label: "L2", value: result.l2Results.length }, + { label: "L1", value: result.l1Results.length }, + { label: "L0", value: result.l0Results.length }, + ]), + ], + }); +} + +function uniqueById(items: T[], getId: (item: T) => string): T[] { + const seen = new Set(); + const next: T[] = []; + for (const item of items) { + const id = getId(item); + if (!id || seen.has(id)) continue; + seen.add(id); + next.push(item); + } + return next; +} + +function pad2(value: number): string { + return String(value).padStart(2, "0"); +} + +function formatLocalDateKey(date: Date): string { + return `${date.getFullYear()}-${pad2(date.getMonth() + 1)}-${pad2(date.getDate())}`; +} + +function buildLocalDateKey(timestamp: string): string { + const date = new Date(timestamp); + if (Number.isNaN(date.getTime())) return timestamp.slice(0, 10) || "unknown"; + return formatLocalDateKey(date); +} + +function parseDateKey(dateKey: string): Date | null { + const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(dateKey.trim()); + if (!match) return null; + const date = new Date(Number(match[1]), Number(match[2]) - 1, Number(match[3])); + return Number.isNaN(date.getTime()) ? null : date; +} + +function enumerateRecentDateKeys( + startDate: string, + endDate: string, + maxDays: number, +): { dateKeys: string[]; truncated: boolean } { + const start = parseDateKey(startDate); + const end = parseDateKey(endDate); + if (!start || !end || maxDays <= 0) return { dateKeys: [], truncated: false }; + + const cursor = new Date(end.getFullYear(), end.getMonth(), end.getDate()); + const floor = new Date(start.getFullYear(), start.getMonth(), start.getDate()); + const dateKeys: string[] = []; + let truncated = false; + + while (cursor.getTime() >= floor.getTime()) { + dateKeys.push(formatLocalDateKey(cursor)); + if (dateKeys.length >= maxDays) { + truncated = cursor.getTime() > floor.getTime(); + break; + } + cursor.setDate(cursor.getDate() - 1); + } + + return { dateKeys, truncated }; +} + +function compareIsoDesc(left: string, right: string): number { + return right.localeCompare(left); +} + +function coerceEnoughAt( + enoughAt: RetrievalResult["enoughAt"], + input: { l2: number; l1: number; l0: number }, +): RetrievalResult["enoughAt"] { + if (enoughAt === "profile" && input.l2 === 0 && input.l1 === 0 && input.l0 === 0) + return "profile"; + if (enoughAt === "l0" && input.l0 > 0) return "l0"; + if (enoughAt === "l1" && input.l1 > 0) return "l1"; + if (enoughAt === "l2" && input.l2 > 0) return "l2"; + if (input.l0 > 0) return "l0"; + if (input.l1 > 0) return "l1"; + if (input.l2 > 0) return "l2"; + return "none"; +} + +export class ReasoningRetriever { + private readonly cache = new Map(); + private runtimeStats: RetrievalRuntimeStats = { + lastRecallMs: 0, + recallTimeouts: 0, + lastRecallMode: "none", + lastRecallPath: "explicit", + lastRecallBudgetLimited: false, + lastShadowDeepQueued: false, + lastRecallInjected: false, + lastRecallEnoughAt: "none", + lastRecallCacheHit: false, + }; + + constructor( + private readonly repository: MemoryRepository, + private readonly skills: SkillsRuntime, + private readonly extractor: LlmMemoryExtractor, + private readonly runtime: RetrievalRuntimeOptions = {}, + ) {} + + getRuntimeStats(): RetrievalRuntimeStats { + return { ...this.runtimeStats }; + } + + resetTransientState(): void { + this.cache.clear(); + this.runtimeStats = { + lastRecallMs: 0, + recallTimeouts: 0, + lastRecallMode: "none", + lastRecallPath: "explicit", + lastRecallBudgetLimited: false, + lastShadowDeepQueued: false, + lastRecallInjected: false, + lastRecallEnoughAt: "none", + lastRecallCacheHit: false, + }; + } + + private currentSettings(): IndexingSettings { + return ( + this.runtime.getSettings?.() ?? { + reasoningMode: "answer_first", + recallTopK: DEFAULT_RECALL_TOP_K, + autoIndexIntervalMinutes: 60, + autoDreamIntervalMinutes: 360, + autoDreamMinNewL1: 10, + } + ); + } + + private buildCacheKey( + query: string, + settings: IndexingSettings, + retrievalMode: "auto" | "explicit", + ): string { + return JSON.stringify({ + query: normalizeQueryKey(query), + snapshot: this.repository.getSnapshotVersion(), + retrievalMode, + settings: { + reasoningMode: settings.reasoningMode, + recallTopK: settings.recallTopK, + }, + }); + } + + private getCachedResult(cacheKey: string): RetrievalResult | null { + const cached = this.cache.get(cacheKey); + if (!cached) return null; + if (cached.expiresAt <= Date.now()) { + this.cache.delete(cacheKey); + return null; + } + return cached.result; + } + + private saveCachedResult(cacheKey: string, result: RetrievalResult): void { + this.cache.set(cacheKey, { + expiresAt: Date.now() + RECALL_CACHE_TTL_MS, + result, + }); + if (this.cache.size > 80) { + const oldestKey = this.cache.keys().next().value; + if (oldestKey) this.cache.delete(oldestKey); + } + } + + private getBaseProfile(includeFacts: boolean | undefined): GlobalProfileRecord | null { + if (includeFacts === false) return null; + const profile = this.repository.getGlobalProfileRecord(); + return profile.profileText.trim() ? profile : null; + } + + private buildContext( + intent: RetrievalResult["intent"], + enoughAt: RetrievalResult["enoughAt"], + profile: GlobalProfileRecord | null, + evidenceNote: string, + l2Results: L2SearchResult[], + l1Results: L1SearchResult[], + l0Results: L0SearchResult[], + ): string { + const hasEvidence = + Boolean(profile?.profileText.trim()) || + Boolean(evidenceNote.trim()) || + l2Results.length > 0 || + l1Results.length > 0 || + l0Results.length > 0; + if (!hasEvidence) return ""; + return renderContextTemplate(this.skills.contextTemplate, { + intent, + enoughAt, + profileBlock: renderProfile(profile), + evidenceNoteBlock: renderEvidenceNote(evidenceNote), + l2Block: renderL2(l2Results), + l1Block: renderL1(l1Results), + l0Block: renderL0(l0Results), + }); + } + + private updateRuntimeStats(result: RetrievalResult, timedOut = false): void { + const mode = result.debug?.mode ?? "none"; + const elapsedMs = result.debug?.elapsedMs ?? 0; + this.runtimeStats.lastRecallMs = elapsedMs; + this.runtimeStats.lastRecallMode = mode; + this.runtimeStats.lastRecallPath = result.debug?.path ?? "explicit"; + this.runtimeStats.lastRecallBudgetLimited = Boolean(result.debug?.budgetLimited); + this.runtimeStats.lastShadowDeepQueued = Boolean(result.debug?.shadowDeepQueued); + this.runtimeStats.lastRecallInjected = Boolean(result.context?.trim()); + this.runtimeStats.lastRecallEnoughAt = result.enoughAt; + this.runtimeStats.lastRecallCacheHit = Boolean(result.debug?.cacheHit); + if (timedOut) this.runtimeStats.recallTimeouts += 1; + } + + private buildL2CatalogHit(item: L2SearchResult["item"]): L2SearchResult { + if ("dateKey" in item) return { level: "l2_time", score: 1, item }; + return { level: "l2_project", score: 1, item }; + } + + private buildL2CatalogEntry(hit: L2SearchResult): L2CatalogEntry { + if (hit.level === "l2_time") { + return { + id: hit.item.l2IndexId, + type: "time", + label: hit.item.dateKey, + lookupKeys: [hit.item.dateKey], + compressedContent: truncate(hit.item.summary, 180), + }; + } + return { + id: hit.item.l2IndexId, + type: "project", + label: hit.item.projectName, + lookupKeys: [hit.item.projectKey, hit.item.projectName].filter(Boolean), + compressedContent: truncate( + [hit.item.summary, `status=${hit.item.currentStatus}`, hit.item.latestProgress] + .filter(Boolean) + .join(" | "), + 220, + ), + }; + } + + private getRequestedLookupTypes( + lookupQueries: LookupQuerySpec[], + ): Set { + const requestedTypes = new Set(); + for (const spec of lookupQueries) { + for (const type of spec.targetTypes) requestedTypes.add(type); + } + return requestedTypes; + } + + private buildLookupSpecs(query: string, lookupQueries: LookupQuerySpec[]): LookupQuerySpec[] { + const specs = + lookupQueries.length > 0 + ? lookupQueries + : [ + { + targetTypes: ["time", "project"] as const, + lookupQuery: query, + timeRange: null, + }, + ]; + + const normalized: LookupQuerySpec[] = []; + const seen = new Set(); + for (const spec of specs) { + const lookupQuery = spec.lookupQuery.trim() || query.trim(); + const targetTypes: LookupQuerySpec["targetTypes"] = + spec.targetTypes.length > 0 ? [...spec.targetTypes] : ["time", "project"]; + const timeRange = spec.timeRange ?? null; + const key = JSON.stringify({ + lookupQuery: normalizeQueryKey(lookupQuery), + targetTypes, + timeRange, + }); + if (seen.has(key)) continue; + seen.add(key); + normalized.push({ targetTypes, lookupQuery, timeRange }); + } + return normalized; + } + + private buildTimeCandidates( + query: string, + specs: LookupQuerySpec[], + limit: number, + ): { hits: L2SearchResult[]; truncated: boolean } { + if (limit <= 0) return { hits: [], truncated: false }; + const hits: L2SearchResult[] = []; + const seen = new Set(); + let truncated = false; + + for (const spec of specs.filter((item) => item.targetTypes.includes("time"))) { + if (!spec.timeRange) continue; + const expanded = enumerateRecentDateKeys( + spec.timeRange.startDate, + spec.timeRange.endDate, + limit, + ); + truncated = truncated || expanded.truncated; + for (const dateKey of expanded.dateKeys) { + const item = this.repository.getL2TimeByDate(dateKey); + if (!item || seen.has(item.l2IndexId)) continue; + seen.add(item.l2IndexId); + hits.push({ level: "l2_time", score: toRankScore(hits.length), item }); + if (hits.length >= limit) { + return { hits, truncated: true }; + } + } + } + + if (hits.length === 0) { + for (const spec of specs.filter((item) => item.targetTypes.includes("time"))) { + const queryHits = this.repository.searchL2TimeIndexes( + spec.lookupQuery || query, + Math.max(limit, 4), + ); + for (const hit of queryHits) { + if (seen.has(hit.item.l2IndexId)) continue; + seen.add(hit.item.l2IndexId); + hits.push({ ...hit, score: hit.score }); + } + } + } + + const ordered = hits + .filter( + (hit): hit is Extract => hit.level === "l2_time", + ) + .sort((left, right) => compareIsoDesc(left.item.dateKey, right.item.dateKey)) + .slice(0, limit); + if (hits.length > ordered.length) truncated = true; + return { hits: ordered, truncated }; + } + + private buildProjectCandidates( + query: string, + specs: LookupQuerySpec[], + limit: number, + ): { hits: L2SearchResult[]; truncated: boolean } { + if (limit <= 0) return { hits: [], truncated: false }; + const merged = new Map(); + + for (const spec of specs.filter((item) => item.targetTypes.includes("project"))) { + const queryHits = this.repository.searchL2ProjectIndexes( + spec.lookupQuery || query, + Math.max(limit, 4), + ); + for (const hit of queryHits) { + const previous = merged.get(hit.item.l2IndexId); + if (!previous || hit.score > previous.score) { + merged.set(hit.item.l2IndexId, hit); + } + } + } + + const ordered = Array.from(merged.values()) + .sort((left, right) => { + if (right.score !== left.score) return right.score - left.score; + return compareIsoDesc( + left.level === "l2_project" ? left.item.updatedAt : "", + right.level === "l2_project" ? right.item.updatedAt : "", + ); + }) + .slice(0, limit); + + return { + hits: ordered, + truncated: merged.size > ordered.length, + }; + } + + private buildL2Catalog( + query: string, + lookupQueries: LookupQuerySpec[], + limit: number, + ): PackedL2Catalog { + const specs = this.buildLookupSpecs(query, lookupQueries); + const requestedTypes = this.getRequestedLookupTypes(specs); + const includeTime = requestedTypes.size === 0 || requestedTypes.has("time"); + const includeProject = requestedTypes.size === 0 || requestedTypes.has("project"); + const timeLimit = + includeTime && includeProject ? Math.ceil(limit / 2) : includeTime ? limit : 0; + const projectLimit = + includeTime && includeProject ? Math.floor(limit / 2) : includeProject ? limit : 0; + const timeCandidates = includeTime + ? this.buildTimeCandidates(query, specs, timeLimit) + : { hits: [], truncated: false }; + const projectCandidates = includeProject + ? this.buildProjectCandidates(query, specs, projectLimit) + : { hits: [], truncated: false }; + const orderedHits = [...timeCandidates.hits, ...projectCandidates.hits].slice( + 0, + Math.max(1, limit), + ); + const byId = new Map( + orderedHits.map((hit) => [hit.item.l2IndexId, hit]), + ); + + return { + entries: orderedHits.map((hit) => this.buildL2CatalogEntry(hit)), + byId, + truncated: + timeCandidates.truncated || + projectCandidates.truncated || + orderedHits.length < timeCandidates.hits.length + projectCandidates.hits.length, + }; + } + + private buildL1CandidatesFromL2(l2Results: L2SearchResult[], limit: number): L1WindowRecord[] { + const l1Ids = uniqueById( + l2Results.flatMap((result) => result.item.l1Source.map((id) => ({ id }))), + (item) => item.id, + ).map((item) => item.id); + if (l1Ids.length === 0) return []; + return this.repository + .getL1ByIds(l1Ids) + .sort((left, right) => { + const endedCompare = compareIsoDesc(left.endedAt, right.endedAt); + return endedCompare !== 0 ? endedCompare : compareIsoDesc(left.createdAt, right.createdAt); + }) + .slice(0, limit); + } + + private buildL0CandidatesFromL1(l1Windows: L1WindowRecord[], limit: number): L0SessionRecord[] { + const l0Ids = uniqueById( + l1Windows.flatMap((item) => item.l0Source.map((id) => ({ id }))), + (item) => item.id, + ).map((item) => item.id); + if (l0Ids.length === 0) return []; + return this.repository + .getL0ByIds(l0Ids) + .sort((left, right) => compareIsoDesc(left.timestamp, right.timestamp)) + .slice(0, limit); + } + + private buildFallbackEvidenceNote(l2Results: L2SearchResult[], seed = ""): string { + const note = l2Results + .map((hit) => + hit.level === "l2_time" + ? `${hit.item.dateKey}: ${hit.item.summary}` + : `${hit.item.projectName}: ${hit.item.latestProgress || hit.item.summary}`, + ) + .join("\n"); + return truncate(note || seed.trim(), 800); + } + + private buildLocalFallbackCandidates( + query: string, + l2Limit: number, + profile: GlobalProfileRecord | null, + lookupQueries: LookupQuerySpec[] = [], + ): LocalFallbackCandidates { + const catalog = this.buildL2Catalog(query, lookupQueries, Math.max(1, l2Limit)); + return { + profile, + l2: catalog.entries + .map((entry, index) => { + const hit = catalog.byId.get(entry.id); + return hit ? { ...hit, score: toRankScore(index) } : undefined; + }) + .filter((hit): hit is L2SearchResult => Boolean(hit)), + truncated: catalog.truncated, + }; + } + + private buildLocalFallback( + resultQuery: string, + fallbackQuery: string, + candidates: LocalFallbackCandidates, + execution: RetrieveExecutionOptions, + elapsedMs: number, + cacheHit: boolean, + corrections: string[] = ["fallback"], + ): RetrievalResult { + const profile = candidates.profile; + const l2Results = candidates.l2.slice(0, Math.max(1, Math.min(4, candidates.l2.length || 1))); + const evidenceNote = this.buildFallbackEvidenceNote(l2Results, fallbackQuery); + const intent = + l2Results[0]?.level === "l2_project" + ? "project" + : l2Results[0]?.level === "l2_time" + ? "time" + : profile + ? "fact" + : "general"; + const enoughAt = + l2Results.length > 0 + ? coerceEnoughAt("l2", { l2: l2Results.length, l1: 0, l0: 0 }) + : profile + ? "profile" + : "none"; + return withDebug( + { + query: resultQuery, + intent, + enoughAt, + profile, + evidenceNote, + l2Results, + l1Results: [], + l0Results: [], + context: this.buildContext(intent, enoughAt, profile, evidenceNote, l2Results, [], []), + }, + { + mode: l2Results.length > 0 || profile ? "local_fallback" : "none", + elapsedMs, + cacheHit, + path: execution.retrievalMode, + catalogTruncated: candidates.truncated, + corrections, + }, + ); + } + + private resolveRecallLimits(settings: IndexingSettings, options: RetrievalOptions): RecallLimits { + const baseLimit = Math.max(1, Math.min(50, settings.recallTopK || DEFAULT_RECALL_TOP_K)); + return { + l2: Math.max(1, Math.min(50, options.l2Limit ?? baseLimit)), + l1: Math.max(1, Math.min(50, options.l1Limit ?? baseLimit)), + l0: Math.max(1, Math.min(50, options.l0Limit ?? baseLimit)), + }; + } + + private resolveBaseIntent(profile: GlobalProfileRecord | null): RetrievalResult["intent"] { + return profile ? "fact" : "general"; + } + + private resolveBaseEnoughAt(profile: GlobalProfileRecord | null): RetrievalResult["enoughAt"] { + return profile ? "profile" : "none"; + } + + private finalizeResult( + result: RetrievalResult, + execution: RetrieveExecutionOptions, + cacheKey: string, + timedOut = false, + ): RetrievalResult { + if (execution.savePrimaryCache) this.saveCachedResult(cacheKey, result); + if (execution.updateRuntimeStats) this.updateRuntimeStats(result, timedOut); + return result; + } + + private shouldStayShallow( + settings: IndexingSettings, + retrievalMode: "auto" | "explicit", + ): boolean { + return retrievalMode === "auto" && settings.reasoningMode === "answer_first"; + } + + private buildL1Results(candidates: L1WindowRecord[]): L1SearchResult[] { + return candidates.map((item, index) => ({ item, score: toRankScore(index) })); + } + + private buildL0Results(candidates: L0SessionRecord[]): L0SearchResult[] { + return candidates.map((item, index) => ({ item, score: toRankScore(index) })); + } + + private async runRetrieve( + query: string, + options: RetrievalOptions, + execution: RetrieveExecutionOptions, + ): Promise { + const startedAt = Date.now(); + const settings = this.currentSettings(); + const limits = this.resolveRecallLimits(settings, options); + const trace = createRetrievalTrace(query, execution.retrievalMode); + let hop1PromptDebug: RetrievalPromptDebug | undefined; + let hop2PromptDebug: RetrievalPromptDebug | undefined; + let hop3PromptDebug: RetrievalPromptDebug | undefined; + let hop4PromptDebug: RetrievalPromptDebug | undefined; + appendTraceStep(trace, { + kind: "recall_start", + title: "Recall Started", + status: "info", + inputSummary: previewText(query, 220), + outputSummary: `mode=${execution.retrievalMode} · reasoning=${settings.reasoningMode} · limits=${limits.l2}/${limits.l1}/${limits.l0}`, + refs: { + retrievalMode: execution.retrievalMode, + reasoningMode: settings.reasoningMode, + }, + metrics: { + l2Limit: limits.l2, + l1Limit: limits.l1, + l0Limit: limits.l0, + }, + details: [ + kvDetail("recall-config", "Recall Configuration", [ + { label: "query", value: query }, + { label: "retrievalMode", value: execution.retrievalMode }, + { label: "reasoningMode", value: settings.reasoningMode }, + { label: "l2Limit", value: limits.l2 }, + { label: "l1Limit", value: limits.l1 }, + { label: "l0Limit", value: limits.l0 }, + ]), + ], + }); + const cacheKey = this.buildCacheKey(query, settings, execution.retrievalMode); + const cached = this.getCachedResult(cacheKey); + if (cached) { + const result = withDebug(cached, { + ...(cached.debug ?? {}), + mode: cached.debug?.mode ?? "llm", + elapsedMs: Date.now() - startedAt, + cacheHit: true, + path: execution.retrievalMode, + }); + appendTraceStep(trace, { + kind: "cache_hit", + title: "Cache Hit", + status: "success", + inputSummary: "Used cached retrieval result for this query snapshot.", + outputSummary: `intent=${result.intent} · enoughAt=${result.enoughAt} · mode=${result.debug?.mode ?? "llm"}`, + refs: { + enoughAt: result.enoughAt, + intent: result.intent, + }, + metrics: { + cacheHit: 1, + elapsedMs: Date.now() - startedAt, + }, + details: [ + kvDetail("cache-summary", "Cached Result", [ + { label: "intent", value: result.intent }, + { label: "enoughAt", value: result.enoughAt }, + { label: "mode", value: result.debug?.mode ?? "llm" }, + ]), + ...(result.evidenceNote.trim() + ? [noteDetail("cache-note", "Cached Evidence Note", result.evidenceNote)] + : []), + ], + }); + appendContextRenderedStep(trace, result); + return this.finalizeResult(attachTrace(result, trace), execution, cacheKey, false); + } + + const baseProfile = this.getBaseProfile(options.includeFacts); + const recentMessages = Array.isArray(options.recentMessages) + ? options.recentMessages.slice(0, 4) + : []; + + if (execution.retrievalMode === "auto" && this.runtime.isBackgroundBusy?.()) { + const fallbackCandidates = this.buildLocalFallbackCandidates(query, limits.l2, baseProfile); + const fallback = this.buildLocalFallback( + query, + query, + fallbackCandidates, + execution, + Date.now() - startedAt, + false, + ["background_busy", "fallback"], + ); + appendFallbackStep(trace, ["background_busy", "fallback"], fallback); + appendContextRenderedStep(trace, fallback); + return this.finalizeResult(attachTrace(fallback, trace), execution, cacheKey, false); + } + + let workingQuery = query; + let workingLookupQueries: LookupQuerySpec[] = []; + try { + const hop1 = await this.extractor.decideMemoryLookup({ + query, + profile: baseProfile, + recentMessages, + debugTrace: (debug) => { + hop1PromptDebug = debug; + }, + }); + const hop1QueryScope = hop1.queryScope === "continuation" ? "continuation" : "standalone"; + workingQuery = + typeof hop1.effectiveQuery === "string" && hop1.effectiveQuery.trim() + ? hop1.effectiveQuery.trim() + : query; + workingLookupQueries = hop1.lookupQueries; + const routedFallbackCandidates = this.buildLocalFallbackCandidates( + workingQuery, + limits.l2, + baseProfile, + hop1.lookupQueries, + ); + appendTraceStep(trace, { + kind: "hop1_decision", + title: "Hop 1 Decision", + status: "success", + inputSummary: baseProfile?.profileText.trim() + ? `profile_available=1 · recent_messages=${recentMessages.length} · ${previewText(query, 160)}` + : `recent_messages=${recentMessages.length} · ${previewText(query, 160)}`, + outputSummary: [ + `queryScope=${hop1QueryScope}`, + `memoryRelevant=${hop1.memoryRelevant ? "yes" : "no"}`, + `baseOnly=${hop1.baseOnly ? "yes" : "no"}`, + summarizeLookupQueries(hop1.lookupQueries), + ].join(" · "), + refs: { + queryScope: hop1QueryScope, + effectiveQuery: workingQuery, + recentMessagesCount: recentMessages.length, + memoryRelevant: hop1.memoryRelevant, + baseOnly: hop1.baseOnly, + lookupQueries: hop1.lookupQueries, + }, + details: [ + kvDetail("hop1-decision", "Hop 1 Decision", [ + { label: "queryScope", value: hop1QueryScope }, + { label: "effectiveQuery", value: workingQuery }, + { label: "recentMessagesCount", value: recentMessages.length }, + { label: "memoryRelevant", value: hop1.memoryRelevant ? "true" : "false" }, + { label: "baseOnly", value: hop1.baseOnly ? "true" : "false" }, + ]), + ...(hop1.lookupQueries.length > 0 + ? [ + listDetail( + "hop1-queries", + "Lookup Queries", + buildLookupQueryItems(hop1.lookupQueries), + ), + ] + : []), + jsonDetail("hop1-json", "Hop 1 Structured Result", { + queryScope: hop1QueryScope, + effectiveQuery: workingQuery, + memoryRelevant: hop1.memoryRelevant, + baseOnly: hop1.baseOnly, + lookupQueries: hop1.lookupQueries, + }), + ], + ...(hop1PromptDebug ? { promptDebug: hop1PromptDebug } : {}), + }); + + if (!hop1.memoryRelevant) { + const result = withDebug( + { + query, + intent: "general", + enoughAt: "none", + profile: null, + evidenceNote: "", + l2Results: [], + l1Results: [], + l0Results: [], + context: "", + }, + { + mode: "none", + elapsedMs: Date.now() - startedAt, + cacheHit: false, + path: execution.retrievalMode, + hop1QueryScope, + hop1EffectiveQuery: workingQuery, + hop1BaseOnly: hop1.baseOnly, + hop1LookupQueries: hop1.lookupQueries.map((entry) => ({ + targetTypes: entry.targetTypes, + lookupQuery: entry.lookupQuery, + })), + }, + ); + appendRecallSkippedStep( + trace, + "memory_not_relevant", + "Hop 1 judged that memory recall is unnecessary for this query.", + { + hop1QueryScope, + hop1EffectiveQuery: workingQuery, + hop1BaseOnly: hop1.baseOnly, + }, + ); + appendContextRenderedStep(trace, result); + return this.finalizeResult(attachTrace(result, trace), execution, cacheKey, false); + } + + if (hop1.baseOnly) { + const intent = this.resolveBaseIntent(baseProfile); + const enoughAt = this.resolveBaseEnoughAt(baseProfile); + const result = withDebug( + { + query, + intent, + enoughAt, + profile: baseProfile, + evidenceNote: "", + l2Results: [], + l1Results: [], + l0Results: [], + context: this.buildContext(intent, enoughAt, baseProfile, "", [], [], []), + }, + { + mode: "llm", + elapsedMs: Date.now() - startedAt, + cacheHit: false, + path: execution.retrievalMode, + hop1QueryScope, + hop1EffectiveQuery: workingQuery, + hop1BaseOnly: hop1.baseOnly, + hop1LookupQueries: hop1.lookupQueries.map((entry) => ({ + targetTypes: entry.targetTypes, + lookupQuery: entry.lookupQuery, + })), + }, + ); + appendContextRenderedStep(trace, result); + return this.finalizeResult(attachTrace(result, trace), execution, cacheKey, false); + } + + const catalog = this.buildL2Catalog(workingQuery, hop1.lookupQueries, limits.l2); + const l2Results = catalog.entries + .map((entry, index) => { + const hit = catalog.byId.get(entry.id); + return hit ? { ...hit, score: toRankScore(index) } : undefined; + }) + .filter((hit): hit is L2SearchResult => Boolean(hit)); + appendTraceStep(trace, { + kind: "l2_candidates", + title: "L2 Candidates", + status: l2Results.length > 0 ? "success" : "warning", + inputSummary: summarizeLookupQueries(hop1.lookupQueries), + outputSummary: summarizeL2Results(l2Results), + refs: { + l2Ids: l2Results.map((item) => item.item.l2IndexId), + catalogTruncated: catalog.truncated, + }, + metrics: { + count: l2Results.length, + truncated: catalog.truncated ? 1 : 0, + }, + details: [ + kvDetail("l2-summary", "L2 Candidate Summary", [ + { label: "count", value: l2Results.length }, + { label: "catalogTruncated", value: catalog.truncated ? "true" : "false" }, + ]), + ...(l2Results.length > 0 + ? [listDetail("l2-items", "L2 Candidates", l2Results.map(describeL2Result))] + : []), + ], + }); + + if (l2Results.length === 0) { + const fallback = this.buildLocalFallback( + query, + workingQuery, + routedFallbackCandidates, + execution, + Date.now() - startedAt, + false, + ["catalog_empty", "fallback"], + ); + appendFallbackStep(trace, ["catalog_empty", "fallback"], fallback); + appendContextRenderedStep(trace, fallback); + return this.finalizeResult(attachTrace(fallback, trace), execution, cacheKey, false); + } + + const hop2 = await this.extractor.selectL2FromCatalog({ + query: workingQuery, + profile: baseProfile, + lookupQueries: hop1.lookupQueries, + l2Entries: catalog.entries, + catalogTruncated: catalog.truncated, + debugTrace: (debug) => { + hop2PromptDebug = debug; + }, + }); + const hop2Note = + hop2.evidenceNote.trim() || this.buildFallbackEvidenceNote(l2Results, workingQuery); + const shallowMode = this.shouldStayShallow(settings, execution.retrievalMode); + const hop1DebugQueries = hop1.lookupQueries.map((entry) => ({ + targetTypes: entry.targetTypes, + lookupQuery: entry.lookupQuery, + })); + const hop2SelectedL2Ids = l2Results.map((item) => item.item.l2IndexId); + appendTraceStep(trace, { + kind: "hop2_decision", + title: "Hop 2 Decision", + status: hop2.enoughAt === "none" ? "warning" : "success", + inputSummary: summarizeL2Results(l2Results), + outputSummary: `intent=${hop2.intent} · enoughAt=${hop2.enoughAt} · ${previewText(hop2Note, 220)}`, + refs: { + enoughAt: hop2.enoughAt, + intent: hop2.intent, + selectedL2Ids: hop2SelectedL2Ids, + }, + details: [ + kvDetail("hop2-result", "Hop 2 Result", [ + { label: "intent", value: hop2.intent }, + { label: "enoughAt", value: hop2.enoughAt }, + ]), + noteDetail("hop2-note-before", "Evidence Note Before Hop 2", "(empty)"), + noteDetail("hop2-note-after", "Evidence Note After Hop 2", hop2Note), + listDetail("hop2-selected-l2", "Selected L2 IDs", hop2SelectedL2Ids), + ], + ...(hop2PromptDebug ? { promptDebug: hop2PromptDebug } : {}), + }); + + if (shallowMode) { + const enoughAt = coerceEnoughAt("l2", { l2: l2Results.length, l1: 0, l0: 0 }); + const corrections = hop2.enoughAt === "l2" ? [] : ["hop2_unverified_shallow_stop"]; + const result = withDebug( + { + query, + intent: hop2.intent, + enoughAt, + profile: null, + evidenceNote: hop2Note, + l2Results, + l1Results: [], + l0Results: [], + context: this.buildContext(hop2.intent, enoughAt, null, hop2Note, l2Results, [], []), + }, + { + mode: "llm", + elapsedMs: Date.now() - startedAt, + cacheHit: false, + path: execution.retrievalMode, + hop1QueryScope, + hop1EffectiveQuery: workingQuery, + hop1BaseOnly: hop1.baseOnly, + hop1LookupQueries: hop1DebugQueries, + hop2EnoughAt: hop2.enoughAt, + hop2SelectedL2Ids, + catalogTruncated: catalog.truncated, + ...(corrections.length > 0 ? { corrections } : {}), + }, + ); + if (corrections.length > 0) { + appendFallbackStep(trace, corrections, result); + } + appendContextRenderedStep(trace, result); + return this.finalizeResult(attachTrace(result, trace), execution, cacheKey, false); + } + + if (hop2.enoughAt === "l2") { + const enoughAt = coerceEnoughAt("l2", { l2: l2Results.length, l1: 0, l0: 0 }); + const result = withDebug( + { + query, + intent: hop2.intent, + enoughAt, + profile: null, + evidenceNote: hop2Note, + l2Results, + l1Results: [], + l0Results: [], + context: this.buildContext(hop2.intent, enoughAt, null, hop2Note, l2Results, [], []), + }, + { + mode: "llm", + elapsedMs: Date.now() - startedAt, + cacheHit: false, + path: execution.retrievalMode, + hop1QueryScope, + hop1EffectiveQuery: workingQuery, + hop1BaseOnly: hop1.baseOnly, + hop1LookupQueries: hop1DebugQueries, + hop2EnoughAt: hop2.enoughAt, + hop2SelectedL2Ids, + catalogTruncated: catalog.truncated, + }, + ); + appendContextRenderedStep(trace, result); + return this.finalizeResult(attachTrace(result, trace), execution, cacheKey, false); + } + + const l1Candidates = this.buildL1CandidatesFromL2(l2Results, limits.l1); + appendTraceStep(trace, { + kind: "l1_candidates", + title: "L1 Candidates", + status: l1Candidates.length > 0 ? "success" : "warning", + inputSummary: hop2SelectedL2Ids.join(" | ") || "No selected L2 ids.", + outputSummary: summarizeL1Windows(l1Candidates), + refs: { + l1Ids: l1Candidates.map((item) => item.l1IndexId), + }, + metrics: { + count: l1Candidates.length, + }, + details: [ + kvDetail("l1-summary", "L1 Candidate Summary", [ + { label: "count", value: l1Candidates.length }, + ]), + ...(l1Candidates.length > 0 + ? [listDetail("l1-items", "L1 Candidates", l1Candidates.map(describeL1Result))] + : []), + ], + }); + if (l1Candidates.length === 0) { + const enoughAt = coerceEnoughAt("l2", { l2: l2Results.length, l1: 0, l0: 0 }); + const result = withDebug( + { + query, + intent: hop2.intent, + enoughAt, + profile: null, + evidenceNote: hop2Note, + l2Results, + l1Results: [], + l0Results: [], + context: this.buildContext(hop2.intent, enoughAt, null, hop2Note, l2Results, [], []), + }, + { + mode: "llm", + elapsedMs: Date.now() - startedAt, + cacheHit: false, + path: execution.retrievalMode, + hop1QueryScope, + hop1EffectiveQuery: workingQuery, + hop1BaseOnly: hop1.baseOnly, + hop1LookupQueries: hop1DebugQueries, + hop2EnoughAt: hop2.enoughAt, + hop2SelectedL2Ids, + catalogTruncated: catalog.truncated, + corrections: ["missing_l1_candidates"], + }, + ); + appendFallbackStep(trace, ["missing_l1_candidates"], result); + appendContextRenderedStep(trace, result); + return this.finalizeResult(attachTrace(result, trace), execution, cacheKey, false); + } + + const hop3 = await this.extractor.selectL1FromEvidence({ + query: workingQuery, + evidenceNote: hop2Note, + selectedL2Entries: catalog.entries, + l1Windows: l1Candidates, + debugTrace: (debug) => { + hop3PromptDebug = debug; + }, + }); + const l1Results = this.buildL1Results(l1Candidates); + const hop3Note = hop3.evidenceNote.trim() || hop2Note; + appendTraceStep(trace, { + kind: "hop3_decision", + title: "Hop 3 Decision", + status: hop3.enoughAt === "none" ? "warning" : "success", + inputSummary: previewText(hop2Note, 220), + outputSummary: `enoughAt=${hop3.enoughAt} · ${previewText(hop3Note, 220)}`, + refs: { + enoughAt: hop3.enoughAt, + selectedL1Ids: l1Results.map((item) => item.item.l1IndexId), + }, + details: [ + kvDetail("hop3-result", "Hop 3 Result", [{ label: "enoughAt", value: hop3.enoughAt }]), + noteDetail("hop3-note-before", "Evidence Note Before Hop 3", hop2Note), + noteDetail("hop3-note-after", "Evidence Note After Hop 3", hop3Note), + listDetail( + "hop3-selected-l1", + "Selected L1 IDs", + l1Results.map((item) => item.item.l1IndexId), + ), + ], + ...(hop3PromptDebug ? { promptDebug: hop3PromptDebug } : {}), + }); + + if (hop3.enoughAt === "l1") { + const enoughAt = coerceEnoughAt("l1", { + l2: l2Results.length, + l1: l1Results.length, + l0: 0, + }); + const result = withDebug( + { + query, + intent: hop2.intent, + enoughAt, + profile: null, + evidenceNote: hop3Note, + l2Results, + l1Results, + l0Results: [], + context: this.buildContext( + hop2.intent, + enoughAt, + null, + hop3Note, + l2Results, + l1Results, + [], + ), + }, + { + mode: "llm", + elapsedMs: Date.now() - startedAt, + cacheHit: false, + path: execution.retrievalMode, + hop1QueryScope, + hop1EffectiveQuery: workingQuery, + hop1BaseOnly: hop1.baseOnly, + hop1LookupQueries: hop1DebugQueries, + hop2EnoughAt: hop2.enoughAt, + hop2SelectedL2Ids, + hop3EnoughAt: hop3.enoughAt, + hop3SelectedL1Ids: l1Results.map((item) => item.item.l1IndexId), + catalogTruncated: catalog.truncated, + }, + ); + appendContextRenderedStep(trace, result); + return this.finalizeResult(attachTrace(result, trace), execution, cacheKey, false); + } + + const l0Candidates = this.buildL0CandidatesFromL1(l1Candidates, limits.l0); + appendTraceStep(trace, { + kind: "l0_candidates", + title: "L0 Candidates", + status: l0Candidates.length > 0 ? "success" : "warning", + inputSummary: + l1Results.map((item) => item.item.l1IndexId).join(" | ") || "No selected L1 ids.", + outputSummary: summarizeL0Sessions(l0Candidates), + refs: { + l0Ids: l0Candidates.map((item) => item.l0IndexId), + }, + metrics: { + count: l0Candidates.length, + }, + details: [ + kvDetail("l0-summary", "L0 Candidate Summary", [ + { label: "count", value: l0Candidates.length }, + ]), + ...(l0Candidates.length > 0 + ? [listDetail("l0-items", "L0 Candidates", l0Candidates.map(describeL0Result))] + : []), + ], + }); + if (l0Candidates.length === 0) { + const enoughAt = coerceEnoughAt("l1", { + l2: l2Results.length, + l1: l1Results.length, + l0: 0, + }); + const result = withDebug( + { + query, + intent: hop2.intent, + enoughAt, + profile: null, + evidenceNote: hop3Note, + l2Results, + l1Results, + l0Results: [], + context: this.buildContext( + hop2.intent, + enoughAt, + null, + hop3Note, + l2Results, + l1Results, + [], + ), + }, + { + mode: "llm", + elapsedMs: Date.now() - startedAt, + cacheHit: false, + path: execution.retrievalMode, + hop1QueryScope, + hop1EffectiveQuery: workingQuery, + hop1BaseOnly: hop1.baseOnly, + hop1LookupQueries: hop1DebugQueries, + hop2EnoughAt: hop2.enoughAt, + hop2SelectedL2Ids, + hop3EnoughAt: hop3.enoughAt, + hop3SelectedL1Ids: l1Results.map((item) => item.item.l1IndexId), + catalogTruncated: catalog.truncated, + corrections: ["missing_l0_candidates"], + }, + ); + appendFallbackStep(trace, ["missing_l0_candidates"], result); + appendContextRenderedStep(trace, result); + return this.finalizeResult(attachTrace(result, trace), execution, cacheKey, false); + } + + const hop4 = await this.extractor.selectL0FromEvidence({ + query: workingQuery, + evidenceNote: hop3Note, + selectedL2Entries: catalog.entries, + selectedL1Windows: l1Candidates, + l0Sessions: l0Candidates, + debugTrace: (debug) => { + hop4PromptDebug = debug; + }, + }); + const l0Results = this.buildL0Results(l0Candidates); + const finalNote = hop4.evidenceNote.trim() || hop3Note; + const useL0Results = hop4.enoughAt === "l0"; + const finalEnoughAt = useL0Results + ? coerceEnoughAt("l0", { l2: l2Results.length, l1: l1Results.length, l0: l0Results.length }) + : coerceEnoughAt("l1", { l2: l2Results.length, l1: l1Results.length, l0: 0 }); + appendTraceStep(trace, { + kind: "hop4_decision", + title: "Hop 4 Decision", + status: hop4.enoughAt === "none" ? "warning" : "success", + inputSummary: previewText(hop3Note, 220), + outputSummary: `enoughAt=${hop4.enoughAt} · ${previewText(finalNote, 220)}`, + refs: { + enoughAt: hop4.enoughAt, + selectedL0Ids: useL0Results ? l0Results.map((item) => item.item.l0IndexId) : [], + }, + details: [ + kvDetail("hop4-result", "Hop 4 Result", [{ label: "enoughAt", value: hop4.enoughAt }]), + noteDetail("hop4-note-before", "Evidence Note Before Hop 4", hop3Note), + noteDetail("hop4-note-after", "Evidence Note After Hop 4", finalNote), + ...(useL0Results + ? [ + listDetail( + "hop4-selected-l0", + "Selected L0 IDs", + l0Results.map((item) => item.item.l0IndexId), + ), + ] + : []), + ], + ...(hop4PromptDebug ? { promptDebug: hop4PromptDebug } : {}), + }); + const result = withDebug( + { + query, + intent: hop2.intent, + enoughAt: finalEnoughAt, + profile: null, + evidenceNote: finalNote, + l2Results, + l1Results, + l0Results: useL0Results ? l0Results : [], + context: this.buildContext( + hop2.intent, + finalEnoughAt, + null, + finalNote, + l2Results, + l1Results, + useL0Results ? l0Results : [], + ), + }, + { + mode: "llm", + elapsedMs: Date.now() - startedAt, + cacheHit: false, + path: execution.retrievalMode, + hop1QueryScope, + hop1EffectiveQuery: workingQuery, + hop1BaseOnly: hop1.baseOnly, + hop1LookupQueries: hop1DebugQueries, + hop2EnoughAt: hop2.enoughAt, + hop2SelectedL2Ids, + hop3EnoughAt: hop3.enoughAt, + hop3SelectedL1Ids: l1Results.map((item) => item.item.l1IndexId), + hop4SelectedL0Ids: useL0Results ? l0Results.map((item) => item.item.l0IndexId) : [], + catalogTruncated: catalog.truncated, + }, + ); + appendContextRenderedStep(trace, result); + return this.finalizeResult(attachTrace(result, trace), execution, cacheKey, false); + } catch (error) { + const timedOut = isTimeoutError(error); + const fallbackCandidates = this.buildLocalFallbackCandidates( + workingQuery, + limits.l2, + baseProfile, + workingLookupQueries, + ); + const fallback = this.buildLocalFallback( + query, + workingQuery, + fallbackCandidates, + execution, + Date.now() - startedAt, + false, + ["error", "fallback"], + ); + appendFallbackStep(trace, ["error", "fallback"], fallback); + appendContextRenderedStep(trace, fallback); + return this.finalizeResult(attachTrace(fallback, trace), execution, cacheKey, timedOut); + } + } + + async retrieve(query: string, options: RetrievalOptions = {}): Promise { + const retrievalMode = options.retrievalMode ?? "explicit"; + return this.runRetrieve( + query, + { ...options, retrievalMode }, + { + retrievalMode, + updateRuntimeStats: true, + savePrimaryCache: true, + }, + ); + } +} diff --git a/extensions/openbmb-clawxmemory/src/core/review/dream-review.ts b/extensions/openbmb-clawxmemory/src/core/review/dream-review.ts new file mode 100644 index 0000000000000..8b40555d9a5bb --- /dev/null +++ b/extensions/openbmb-clawxmemory/src/core/review/dream-review.ts @@ -0,0 +1,869 @@ +import type { HeartbeatStats } from "../pipeline/heartbeat.js"; +import { LlmMemoryExtractor } from "../skills/llm-extraction.js"; +import type { + DreamEvidenceRef, + DreamReviewFinding, + DreamReviewFocus, + DreamReviewResult, + GlobalProfileRecord, + L0SessionRecord, + L1WindowRecord, + L2ProjectIndexRecord, + ProjectStatus, +} from "../types.js"; +import { buildL2ProjectIndexId, nowIso } from "../utils/id.js"; + +type LoggerLike = { + info?: (...args: unknown[]) => void; + warn?: (...args: unknown[]) => void; + error?: (...args: unknown[]) => void; +}; + +interface DreamReviewRunnerOptions { + logger?: LoggerLike; +} + +interface DreamEvidencePack { + focus: DreamReviewFocus; + profile: GlobalProfileRecord | null; + l2Projects: L2ProjectIndexRecord[]; + l1Windows: L1WindowRecord[]; + l0Previews: L0SessionRecord[]; + timeLayerNotes: DreamReviewFinding[]; + evidenceRefs: DreamEvidenceRef[]; +} + +interface DreamProjectCandidate { + l1IndexId: string; + endedAt: string; + projectKey: string; + projectName: string; + status: ProjectStatus; + summary: string; + latestProgress: string; + confidence: number; +} + +interface DreamProjectCluster { + clusterId: string; + label: string; + candidateKeys: string[]; + candidateNames: string[]; + currentProjectKeys: string[]; + l1Ids: string[]; + statuses: ProjectStatus[]; + summaries: string[]; + latestProgresses: string[]; + issueHints: DreamL1Issue["issueType"][]; + representativeWindows: Array<{ + l1IndexId: string; + endedAt: string; + summary: string; + }>; +} + +interface DreamRewriteEvidence { + currentProjects: L2ProjectIndexRecord[]; + currentProfile: GlobalProfileRecord; + allL1Windows: L1WindowRecord[]; + l0Previews: L0SessionRecord[]; + projectClusters: DreamProjectCluster[]; +} + +export interface DreamL1Issue { + issueType: "duplicate" | "conflict" | "isolated"; + title: string; + l1Ids: string[]; + relatedProjectKeys: string[]; +} + +export interface DreamProjectRebuildItem { + projectKey: string; + projectName: string; + currentStatus: ProjectStatus; + summary: string; + latestProgress: string; + retainedL1Ids: string[]; +} + +export interface DreamProjectRebuildPlan { + summary: string; + duplicateTopicCount: number; + conflictTopicCount: number; + projects: DreamProjectRebuildItem[]; + deletedProjectKeys: string[]; + l1Issues: DreamL1Issue[]; +} + +export interface DreamGlobalProfileRewrite { + profileText: string; + sourceL1Ids: string[]; + conflictWithExisting: boolean; +} + +export interface DreamRewriteOutcome { + reviewedL1: number; + rewrittenProjects: number; + deletedProjects: number; + profileUpdated: boolean; + duplicateTopicCount: number; + conflictTopicCount: number; + prunedProjectL1Refs: number; + prunedProfileL1Refs: number; + summary: string; +} + +export interface DreamRunResult extends DreamRewriteOutcome { + prepFlush: HeartbeatStats; + trigger?: "manual" | "scheduled"; + status?: "success" | "skipped"; + skipReason?: string; +} + +const DREAM_RECENT_L1_LIMIT = 12; +const DREAM_MAX_PROJECTS = 12; +const DREAM_MAX_L0_SPOTCHECK = 4; +const DREAM_MAX_TIME_NOTES = 6; +const DREAM_MAX_REWRITE_L0_SPOTCHECK = 6; + +function truncate(value: string, maxLength: number): string { + if (value.length <= maxLength) return value; + return `${value.slice(0, Math.max(0, maxLength - 3)).trimEnd()}...`; +} + +function normalizeWhitespace(value: string): string { + return value.replace(/\s+/g, " ").trim(); +} + +function buildRefId(level: DreamEvidenceRef["level"], id: string): string { + return `${level}:${id}`; +} + +function buildL0Preview(record: L0SessionRecord): string { + const preview = record.messages + .slice(-4) + .map((message) => `${message.role}: ${normalizeWhitespace(message.content)}`) + .filter((value) => value !== ":") + .join(" | "); + return truncate(preview, 220); +} + +function formatDateKey(value: string): string { + const date = new Date(value); + if (Number.isNaN(date.getTime())) return value.slice(0, 10) || "unknown-day"; + const year = String(date.getFullYear()); + const month = String(date.getMonth() + 1).padStart(2, "0"); + const day = String(date.getDate()).padStart(2, "0"); + return `${year}-${month}-${day}`; +} + +function clampConfidence(value: number): number { + if (!Number.isFinite(value)) return 0.5; + return Math.max(0, Math.min(1, value)); +} + +function emptyDreamReview( + summary: string, + evidenceRefs: DreamEvidenceRef[] = [], + timeLayerNotes: DreamReviewFinding[] = [], +): DreamReviewResult { + return { + summary, + projectRebuild: [], + profileSuggestions: [], + cleanup: [], + ambiguous: [], + noAction: [], + timeLayerNotes, + evidenceRefs, + }; +} + +function normalizeProjectIdentity(value: string): string { + return value + .toLowerCase() + .replace(/[^a-z0-9\u4e00-\u9fff]+/g, " ") + .trim(); +} + +function tokenizeForSimilarity(value: string): string[] { + const normalized = normalizeWhitespace(value).toLowerCase(); + const tokens = normalized.match(/[a-z0-9]{2,}|[\u4e00-\u9fff]/g) ?? []; + return Array.from(new Set(tokens)); +} + +function similarityScore(left: string, right: string): number { + const leftTokens = tokenizeForSimilarity(left); + const rightTokens = tokenizeForSimilarity(right); + if (leftTokens.length === 0 || rightTokens.length === 0) return 0; + const rightSet = new Set(rightTokens); + let overlap = 0; + for (const token of leftTokens) { + if (rightSet.has(token)) overlap += 1; + } + return overlap / new Set([...leftTokens, ...rightTokens]).size; +} + +function resolveProjectStatus(statuses: ProjectStatus[]): ProjectStatus { + const values = new Set(statuses); + if (values.has("in_progress")) return "in_progress"; + if (values.size === 1 && values.has("done")) return "done"; + if (values.has("done") && values.size > 1) return "in_progress"; + return "planned"; +} + +function countProjectSourceRefs(projects: readonly L2ProjectIndexRecord[]): number { + return projects.reduce((total, project) => total + project.l1Source.length, 0); +} + +function sortL1IdsByEndedAt( + ids: string[], + windowsById: ReadonlyMap, +): string[] { + return Array.from(new Set(ids)) + .filter((id) => windowsById.has(id)) + .sort((left, right) => { + const leftWindow = windowsById.get(left)!; + const rightWindow = windowsById.get(right)!; + return rightWindow.endedAt.localeCompare(leftWindow.endedAt); + }); +} + +function arraysEqual(left: readonly string[], right: readonly string[]): boolean { + return left.length === right.length && left.every((value, index) => value === right[index]); +} + +export class DreamReviewRunner { + constructor( + private readonly repository: { + listRecentL1(limit?: number, offset?: number): L1WindowRecord[]; + getL2ProjectByKey(projectKey: string): L2ProjectIndexRecord | undefined; + getGlobalProfileRecord(): GlobalProfileRecord; + getL0ByL1Ids(l1Ids: string[], limit?: number): L0SessionRecord[]; + getL2TimeByDate( + dateKey: string, + ): { l2IndexId: string; dateKey: string; summary: string; l1Source: string[] } | undefined; + }, + private readonly extractor: LlmMemoryExtractor, + private readonly options: DreamReviewRunnerOptions = {}, + ) {} + + async review(focus: DreamReviewFocus): Promise { + const evidence = this.buildEvidencePack(focus); + if ( + evidence.l1Windows.length === 0 && + evidence.l2Projects.length === 0 && + !evidence.profile?.profileText.trim() && + evidence.timeLayerNotes.length === 0 + ) { + return emptyDreamReview( + "Not enough indexed memory evidence to run Dream review yet.", + evidence.evidenceRefs, + ); + } + + const llmResult = await this.extractor.reviewDream({ + focus, + profile: evidence.profile, + l2Projects: evidence.l2Projects, + l1Windows: evidence.l1Windows, + l0Sessions: evidence.l0Previews, + evidenceRefs: evidence.evidenceRefs, + timeLayerNotes: evidence.timeLayerNotes, + }); + + return { + ...llmResult, + timeLayerNotes: evidence.timeLayerNotes, + evidenceRefs: evidence.evidenceRefs, + }; + } + + private buildEvidencePack(focus: DreamReviewFocus): DreamEvidencePack { + const rawRecentL1 = this.repository.listRecentL1(DREAM_RECENT_L1_LIMIT, 0); + const l1Windows = + focus === "profile" + ? rawRecentL1 + : (() => { + const projectWindows = rawRecentL1.filter((window) => window.projectDetails.length > 0); + return projectWindows.length > 0 ? projectWindows : rawRecentL1; + })(); + + const profileRecord = this.repository.getGlobalProfileRecord(); + const profile = + focus === "projects" ? null : profileRecord.profileText.trim() ? profileRecord : null; + + const projectKeys = Array.from( + new Set( + l1Windows + .flatMap((window) => window.projectDetails.map((project) => project.key)) + .filter(Boolean), + ), + ).slice(0, DREAM_MAX_PROJECTS); + const l2Projects = projectKeys + .map((projectKey) => this.repository.getL2ProjectByKey(projectKey)) + .filter((item): item is L2ProjectIndexRecord => Boolean(item)); + + const suspiciousL1Ids = l1Windows + .filter( + (window) => + window.projectDetails.length > 1 || + window.projectDetails.some( + (project) => + project.confidence < 0.6 || !project.summary.trim() || !project.latestProgress.trim(), + ), + ) + .map((window) => window.l1IndexId) + .slice(0, DREAM_MAX_L0_SPOTCHECK); + const l0Previews = + suspiciousL1Ids.length > 0 + ? this.repository.getL0ByL1Ids(suspiciousL1Ids, DREAM_MAX_L0_SPOTCHECK) + : []; + + const evidenceRefs: DreamEvidenceRef[] = []; + const addRef = (ref: DreamEvidenceRef): void => { + if (evidenceRefs.some((existing) => existing.refId === ref.refId)) return; + evidenceRefs.push(ref); + }; + + if (profile) { + addRef({ + refId: buildRefId("profile", profile.recordId), + level: "profile", + id: profile.recordId, + label: "Global Profile", + summary: truncate(profile.profileText, 220), + }); + } + + for (const project of l2Projects) { + addRef({ + refId: buildRefId("l2_project", project.l2IndexId), + level: "l2_project", + id: project.l2IndexId, + label: project.projectName || project.projectKey, + summary: truncate(`${project.summary} | latest: ${project.latestProgress}`.trim(), 220), + }); + } + + for (const window of l1Windows) { + const projectSummary = window.projectDetails + .map((project) => `${project.name}(${project.status})`) + .slice(0, 3) + .join(", "); + addRef({ + refId: buildRefId("l1", window.l1IndexId), + level: "l1", + id: window.l1IndexId, + label: window.timePeriod || window.endedAt, + summary: truncate([window.summary, projectSummary].filter(Boolean).join(" | "), 220), + }); + } + + for (const session of l0Previews) { + addRef({ + refId: buildRefId("l0", session.l0IndexId), + level: "l0", + id: session.l0IndexId, + label: `${session.sessionKey} @ ${session.timestamp}`, + summary: buildL0Preview(session), + }); + } + + const timeLayerNotes = this.buildTimeLayerNotes(l1Windows, addRef); + + return { + focus, + profile, + l2Projects, + l1Windows, + l0Previews, + timeLayerNotes, + evidenceRefs, + }; + } + + private buildTimeLayerNotes( + l1Windows: L1WindowRecord[], + addRef: (ref: DreamEvidenceRef) => void, + ): DreamReviewFinding[] { + const notes: DreamReviewFinding[] = []; + const byDate = new Map(); + for (const window of l1Windows) { + const dateKey = formatDateKey(window.startedAt || window.endedAt || window.createdAt); + const current = byDate.get(dateKey) ?? []; + current.push(window.l1IndexId); + byDate.set(dateKey, current); + } + + for (const [dateKey, l1Ids] of Array.from(byDate.entries()).slice(0, DREAM_MAX_TIME_NOTES)) { + const timeIndex = this.repository.getL2TimeByDate(dateKey); + if (timeIndex) { + addRef({ + refId: buildRefId("l2_time", timeIndex.l2IndexId), + level: "l2_time", + id: timeIndex.l2IndexId, + label: timeIndex.dateKey, + summary: truncate(timeIndex.summary, 220), + }); + } + + const evidenceRefs = [ + ...(timeIndex ? [buildRefId("l2_time", timeIndex.l2IndexId)] : []), + ...l1Ids.map((id) => buildRefId("l1", id)), + ]; + + if (!timeIndex) { + notes.push({ + title: `Missing L2Time summary for ${dateKey}`, + rationale: `Recent L1 windows exist for ${dateKey}, but no daily L2Time summary was found. Dream should only note this gap; it should not rebuild the time layer semantically.`, + confidence: 0.85, + target: "time_note", + evidenceRefs, + }); + continue; + } + + if (!timeIndex.summary.trim()) { + notes.push({ + title: `Empty L2Time summary for ${dateKey}`, + rationale: `The day bucket exists but the summary is empty. This is a time-layer integrity issue, not a Dream rewrite target.`, + confidence: 0.88, + target: "time_note", + evidenceRefs, + }); + } else if (timeIndex.l1Source.length === 0) { + notes.push({ + title: `Unlinked L2Time summary for ${dateKey}`, + rationale: `The daily summary has no linked L1 sources. Keep the time layer read-only for Dream, but flag the missing linkage.`, + confidence: 0.82, + target: "time_note", + evidenceRefs, + }); + } else { + const missingL1 = l1Ids.filter((id) => !timeIndex.l1Source.includes(id)); + if (missingL1.length > 0) { + notes.push({ + title: `L2Time source coverage looks incomplete for ${dateKey}`, + rationale: `Some recent L1 windows for this day are not linked from the L2Time record. Record the integrity gap, but do not treat it as a Dream semantic rewrite.`, + confidence: 0.7, + target: "time_note", + evidenceRefs, + }); + } + } + } + + return notes.slice(0, DREAM_MAX_TIME_NOTES).map((note) => ({ + ...note, + confidence: clampConfidence(note.confidence), + })); + } +} + +export class DreamRewriteRunner { + constructor( + private readonly repository: { + listAllL1(): L1WindowRecord[]; + listAllL2Projects(): L2ProjectIndexRecord[]; + getGlobalProfileRecord(): GlobalProfileRecord; + getL0ByL1Ids(l1Ids: string[], limit?: number): L0SessionRecord[]; + applyDreamRewrite(input: { + projects: L2ProjectIndexRecord[]; + profileText: string; + profileSourceL1Ids: string[]; + }): void; + }, + private readonly extractor: LlmMemoryExtractor, + private readonly options: DreamReviewRunnerOptions = {}, + ) {} + + async run(): Promise { + const evidence = this.buildRewriteEvidence(); + if (evidence.allL1Windows.length === 0) { + return { + reviewedL1: 0, + rewrittenProjects: 0, + deletedProjects: 0, + profileUpdated: false, + duplicateTopicCount: 0, + conflictTopicCount: 0, + prunedProjectL1Refs: 0, + prunedProfileL1Refs: 0, + summary: "No L1 windows are available for Dream reconstruction.", + }; + } + + const plan = await this.buildProjectPlan(evidence); + const profileRewrite = await this.buildProfileRewrite(evidence, plan); + + const currentProjectsByKey = new Map( + evidence.currentProjects.map((project) => [project.projectKey, project]), + ); + const finalProjects = plan.projects.map((item) => { + const existing = currentProjectsByKey.get(item.projectKey); + const timestamp = nowIso(); + return { + l2IndexId: existing?.l2IndexId ?? buildL2ProjectIndexId(item.projectKey), + projectKey: item.projectKey, + projectName: item.projectName, + summary: item.summary, + currentStatus: item.currentStatus, + latestProgress: item.latestProgress, + l1Source: sortL1IdsByEndedAt( + item.retainedL1Ids, + new Map(evidence.allL1Windows.map((window) => [window.l1IndexId, window])), + ), + createdAt: existing?.createdAt ?? timestamp, + updatedAt: timestamp, + } satisfies L2ProjectIndexRecord; + }); + + const currentProjectSourceRefs = countProjectSourceRefs(evidence.currentProjects); + const nextProjectSourceRefs = countProjectSourceRefs(finalProjects); + const prunedProjectL1Refs = Math.max(0, currentProjectSourceRefs - nextProjectSourceRefs); + const profileUpdated = + profileRewrite.profileText !== evidence.currentProfile.profileText || + !arraysEqual(profileRewrite.sourceL1Ids, evidence.currentProfile.sourceL1Ids); + const prunedProfileL1Refs = Math.max( + 0, + evidence.currentProfile.sourceL1Ids.length - profileRewrite.sourceL1Ids.length, + ); + const deletedProjectKeys = Array.from( + new Set([ + ...plan.deletedProjectKeys, + ...evidence.currentProjects + .map((project) => project.projectKey) + .filter((projectKey) => !finalProjects.some((item) => item.projectKey === projectKey)), + ]), + ); + + this.repository.applyDreamRewrite({ + projects: finalProjects, + profileText: profileRewrite.profileText, + profileSourceL1Ids: profileRewrite.sourceL1Ids, + }); + + return { + reviewedL1: evidence.allL1Windows.length, + rewrittenProjects: finalProjects.length, + deletedProjects: deletedProjectKeys.length, + profileUpdated, + duplicateTopicCount: plan.duplicateTopicCount, + conflictTopicCount: plan.conflictTopicCount, + prunedProjectL1Refs, + prunedProfileL1Refs, + summary: truncate(plan.summary || "Dream reconstruction completed.", 280), + }; + } + + private buildRewriteEvidence(): DreamRewriteEvidence { + const allL1Windows = this.repository.listAllL1(); + const currentProjects = this.repository.listAllL2Projects(); + const currentProfile = this.repository.getGlobalProfileRecord(); + const clusters = this.clusterProjects(allL1Windows, currentProjects); + const suspiciousL1Ids = clusters + .filter((cluster) => cluster.issueHints.length > 0 || cluster.currentProjectKeys.length !== 1) + .flatMap((cluster) => cluster.l1Ids) + .slice(0, DREAM_MAX_REWRITE_L0_SPOTCHECK); + const l0Previews = + suspiciousL1Ids.length > 0 + ? this.repository.getL0ByL1Ids(suspiciousL1Ids, DREAM_MAX_REWRITE_L0_SPOTCHECK) + : []; + return { + currentProjects, + currentProfile, + allL1Windows, + l0Previews, + projectClusters: clusters, + }; + } + + private clusterProjects( + l1Windows: L1WindowRecord[], + currentProjects: L2ProjectIndexRecord[], + ): DreamProjectCluster[] { + const clusters: Array = []; + const currentByKey = new Map(currentProjects.map((project) => [project.projectKey, project])); + + const candidates: DreamProjectCandidate[] = l1Windows.flatMap((window) => + window.projectDetails.map((project) => ({ + l1IndexId: window.l1IndexId, + endedAt: window.endedAt, + projectKey: project.key, + projectName: project.name, + status: project.status, + summary: project.summary, + latestProgress: project.latestProgress, + confidence: project.confidence, + })), + ); + + for (const candidate of candidates) { + const identity = normalizeProjectIdentity(candidate.projectKey || candidate.projectName); + const text = `${candidate.projectName} ${candidate.summary} ${candidate.latestProgress}`; + const cluster = clusters.find((item) => { + const sameIdentity = + item.candidateKeys.includes(candidate.projectKey) || + item.candidateNames.some( + (name) => + normalizeProjectIdentity(name) === normalizeProjectIdentity(candidate.projectName), + ) || + identity === normalizeProjectIdentity(item.label); + if (sameIdentity) return true; + return similarityScore(item.anchorText, text) >= 0.45; + }); + + if (cluster) { + cluster.candidateKeys.push(candidate.projectKey); + cluster.candidateNames.push(candidate.projectName); + cluster.l1Ids.push(candidate.l1IndexId); + cluster.statuses.push(candidate.status); + cluster.summaries.push(candidate.summary); + cluster.latestProgresses.push(candidate.latestProgress); + cluster.representativeWindows.push({ + l1IndexId: candidate.l1IndexId, + endedAt: candidate.endedAt, + summary: truncate(candidate.summary || candidate.latestProgress, 180), + }); + if (candidate.summary.trim()) { + cluster.anchorText = `${cluster.anchorText} ${candidate.summary}`.trim(); + } + continue; + } + + clusters.push({ + clusterId: `cluster-${clusters.length + 1}`, + label: candidate.projectName || candidate.projectKey, + candidateKeys: [candidate.projectKey], + candidateNames: [candidate.projectName], + currentProjectKeys: currentByKey.has(candidate.projectKey) ? [candidate.projectKey] : [], + l1Ids: [candidate.l1IndexId], + statuses: [candidate.status], + summaries: [candidate.summary], + latestProgresses: [candidate.latestProgress], + issueHints: [], + representativeWindows: [ + { + l1IndexId: candidate.l1IndexId, + endedAt: candidate.endedAt, + summary: truncate(candidate.summary || candidate.latestProgress, 180), + }, + ], + anchorText: text, + }); + } + + return clusters.map((cluster) => { + const candidateKeySet = Array.from(new Set(cluster.candidateKeys.filter(Boolean))); + const candidateNameSet = Array.from(new Set(cluster.candidateNames.filter(Boolean))); + const statusSet = Array.from(new Set(cluster.statuses)); + const currentProjectKeys = Array.from( + new Set([ + ...cluster.currentProjectKeys, + ...currentProjects + .filter( + (project) => + candidateKeySet.includes(project.projectKey) || + candidateNameSet.some( + (name) => + normalizeProjectIdentity(name) === + normalizeProjectIdentity(project.projectName), + ) || + project.l1Source.some((id) => cluster.l1Ids.includes(id)), + ) + .map((project) => project.projectKey), + ]), + ); + const issueHints: DreamL1Issue["issueType"][] = []; + if ( + currentProjectKeys.length > 1 || + candidateKeySet.length > 1 || + candidateNameSet.length > 1 + ) { + issueHints.push("duplicate"); + } + if (statusSet.includes("done") && statusSet.includes("in_progress")) { + issueHints.push("conflict"); + } + if (cluster.l1Ids.length === 1 && currentProjectKeys.length === 0) { + issueHints.push("isolated"); + } + return { + clusterId: cluster.clusterId, + label: cluster.label, + candidateKeys: candidateKeySet, + candidateNames: candidateNameSet, + currentProjectKeys, + l1Ids: Array.from(new Set(cluster.l1Ids)), + statuses: cluster.statuses, + summaries: cluster.summaries.map((summary) => truncate(summary, 180)).filter(Boolean), + latestProgresses: cluster.latestProgresses + .map((value) => truncate(value, 140)) + .filter(Boolean), + issueHints, + representativeWindows: cluster.representativeWindows + .sort((left, right) => right.endedAt.localeCompare(left.endedAt)) + .slice(0, 4), + }; + }); + } + + private buildFallbackProjectPlan(evidence: DreamRewriteEvidence): DreamProjectRebuildPlan { + const windowsById = new Map(evidence.allL1Windows.map((window) => [window.l1IndexId, window])); + const fallbackProjects = evidence.projectClusters + .filter((cluster) => cluster.l1Ids.length > 0) + .map((cluster) => { + const retainedL1Ids = sortL1IdsByEndedAt(cluster.l1Ids, windowsById); + const mostRecentWindow = + retainedL1Ids.length > 0 ? windowsById.get(retainedL1Ids[0]!) : undefined; + const mostRecentProject = mostRecentWindow?.projectDetails.find( + (project) => + cluster.candidateKeys.includes(project.key) || + cluster.candidateNames.includes(project.name), + ); + return { + projectKey: + (cluster.currentProjectKeys[0] ?? + cluster.candidateKeys[0] ?? + normalizeProjectIdentity(cluster.candidateNames[0] ?? cluster.label).replace( + /\s+/g, + "-", + )) || + `dream-project-${cluster.clusterId}`, + projectName: cluster.currentProjectKeys[0] + ? (evidence.currentProjects.find( + (project) => project.projectKey === cluster.currentProjectKeys[0], + )?.projectName ?? cluster.label) + : (cluster.candidateNames[0] ?? cluster.label), + currentStatus: mostRecentProject?.status ?? resolveProjectStatus(cluster.statuses), + summary: + mostRecentProject?.summary || + cluster.summaries.find(Boolean) || + mostRecentWindow?.summary || + cluster.label, + latestProgress: + mostRecentProject?.latestProgress || + cluster.latestProgresses.find(Boolean) || + mostRecentWindow?.situationTimeInfo || + cluster.label, + retainedL1Ids, + } satisfies DreamProjectRebuildItem; + }); + + const existingKeys = new Set(evidence.currentProjects.map((project) => project.projectKey)); + const finalKeys = new Set(fallbackProjects.map((project) => project.projectKey)); + return { + summary: "Dream fallback rebuilt project memory from current L1 clusters.", + duplicateTopicCount: evidence.projectClusters.filter((cluster) => + cluster.issueHints.includes("duplicate"), + ).length, + conflictTopicCount: evidence.projectClusters.filter((cluster) => + cluster.issueHints.includes("conflict"), + ).length, + projects: fallbackProjects, + deletedProjectKeys: Array.from(existingKeys).filter((key) => !finalKeys.has(key)), + l1Issues: evidence.projectClusters.flatMap((cluster) => + cluster.issueHints.map((issueType) => ({ + issueType, + title: `${cluster.label} ${issueType}`, + l1Ids: cluster.l1Ids, + relatedProjectKeys: + cluster.currentProjectKeys.length > 0 + ? cluster.currentProjectKeys + : cluster.candidateKeys, + })), + ), + }; + } + + private async buildProjectPlan(evidence: DreamRewriteEvidence): Promise { + const planned = await this.extractor.planDreamProjectRebuild({ + currentProjects: evidence.currentProjects, + profile: evidence.currentProfile, + l1Windows: evidence.allL1Windows, + l0Sessions: evidence.l0Previews, + clusters: evidence.projectClusters, + }); + if (planned.projects.length === 0) { + throw new Error("Dream project rebuild returned no valid projects."); + } + const explainedCurrentKeys = new Set([ + ...planned.projects.map((project) => project.projectKey), + ...planned.deletedProjectKeys, + ]); + const unexplainedCurrentKeys = evidence.currentProjects + .map((project) => project.projectKey) + .filter((projectKey) => !explainedCurrentKeys.has(projectKey)); + if (unexplainedCurrentKeys.length > 0) { + throw new Error( + `Dream project rebuild did not explain current projects: ${unexplainedCurrentKeys.join(", ")}`, + ); + } + return planned; + } + + private async buildProfileRewrite( + evidence: DreamRewriteEvidence, + plan: DreamProjectRebuildPlan, + ): Promise { + const rewritten = await this.extractor.rewriteDreamGlobalProfile({ + existingProfile: evidence.currentProfile, + l1Windows: evidence.allL1Windows, + currentProjects: evidence.currentProjects, + plannedProjects: plan.projects, + l1Issues: plan.l1Issues, + }); + const normalizedIds = sortL1IdsByEndedAt( + rewritten.sourceL1Ids, + new Map(evidence.allL1Windows.map((window) => [window.l1IndexId, window])), + ); + if ( + !(normalizedIds.length >= 2 || (rewritten.conflictWithExisting && normalizedIds.length >= 1)) + ) { + throw new Error("Dream global profile rewrite did not satisfy the source support gate."); + } + return { + profileText: rewritten.profileText.trim() || evidence.currentProfile.profileText, + sourceL1Ids: normalizedIds, + conflictWithExisting: rewritten.conflictWithExisting, + }; + } + + private buildFallbackProfileRewrite(evidence: DreamRewriteEvidence): DreamGlobalProfileRewrite { + const groupedFacts = new Map(); + for (const window of evidence.allL1Windows) { + for (const fact of window.facts) { + const existing = groupedFacts.get(fact.factKey); + if (existing) { + existing.l1Ids.push(window.l1IndexId); + continue; + } + groupedFacts.set(fact.factKey, { + value: fact.factValue, + l1Ids: [window.l1IndexId], + }); + } + } + const stableFacts = Array.from(groupedFacts.values()) + .filter((entry) => new Set(entry.l1Ids).size >= 2) + .sort((left, right) => new Set(right.l1Ids).size - new Set(left.l1Ids).size) + .slice(0, 8); + if (stableFacts.length === 0) { + return { + profileText: evidence.currentProfile.profileText, + sourceL1Ids: evidence.currentProfile.sourceL1Ids, + conflictWithExisting: false, + }; + } + const sourceL1Ids = sortL1IdsByEndedAt( + stableFacts.flatMap((entry) => entry.l1Ids), + new Map(evidence.allL1Windows.map((window) => [window.l1IndexId, window])), + ); + return { + profileText: truncate(stableFacts.map((entry) => entry.value).join("；"), 420), + sourceL1Ids, + conflictWithExisting: false, + }; + } +} diff --git a/extensions/openbmb-clawxmemory/src/core/skills/defaults.ts b/extensions/openbmb-clawxmemory/src/core/skills/defaults.ts new file mode 100644 index 0000000000000..f92f51cfa5be5 --- /dev/null +++ b/extensions/openbmb-clawxmemory/src/core/skills/defaults.ts @@ -0,0 +1,91 @@ +import type { ExtractionRulesFile, IntentRulesFile, ProjectStatusRulesFile } from "./types.js"; + +export const DEFAULT_INTENT_RULES: IntentRulesFile = { + timeKeywords: ["今天", "昨天", "最近", "本周", "时间", "日期", "timeline", "when", "day"], + projectKeywords: ["项目", "进展", "里程碑", "roadmap", "project", "status", "ultrarag"], + factKeywords: ["偏好", "事实", "画像", "profile", "fact", "习惯", "喜欢", "不喜欢"], +}; + +export const DEFAULT_EXTRACTION_RULES: ExtractionRulesFile = { + projectPatterns: [ + { pattern: "(?:项目|project)\\s*[:：]?\\s*([A-Za-z][A-Za-z0-9_-]{1,48})", flags: "gi" }, + { pattern: "\\b([A-Z][A-Za-z0-9]+(?:[A-Z][A-Za-z0-9]+)+)\\b", flags: "g" }, + ], + factRules: [ + { + name: "techStack", + pattern: "(?:我在用|我使用|使用的是|技术栈是)\\s*([A-Za-z0-9.+#_-]{2,40})", + flags: "gi", + keyPrefix: "tech", + confidence: 0.82, + maxLength: 120, + }, + { + name: "activity", + pattern: "(?:我正在|我在)\\s*([^，。,.!?]{2,60})", + flags: "gi", + keyPrefix: "activity", + confidence: 0.68, + maxLength: 120, + }, + { + name: "preference", + pattern: "(?:喜欢|偏好)\\s*([^，。,.!?]{2,40})", + flags: "gi", + keyPrefix: "preference", + confidence: 0.72, + maxLength: 120, + }, + { + name: "plan", + pattern: "(?:计划|准备)\\s*([^，。,.!?]{2,40})", + flags: "gi", + keyPrefix: "plan", + confidence: 0.65, + maxLength: 120, + }, + ], + maxProjectTags: 8, + maxFacts: 16, + projectTagMinLength: 2, + projectTagMaxLength: 50, + summaryLimits: { + head: 80, + tail: 80, + assistant: 80, + }, +}; + +export const DEFAULT_PROJECT_STATUS_RULES: ProjectStatusRulesFile = { + defaultStatus: "in_progress", + rules: [ + { + status: "done", + keywords: ["完成", "done", "已上线"], + }, + { + status: "planned", + keywords: ["计划", "准备"], + }, + { + status: "in_progress", + keywords: ["推进", "进行中", "跟进"], + }, + ], +}; + +export const DEFAULT_CONTEXT_TEMPLATE = `You are using multi-level memory indexes for this turn. +intent={{intent}} +enoughAt={{enoughAt}} + +{{profileBlock}} + +{{evidenceNoteBlock}} + +{{l2Block}} + +{{l1Block}} + +{{l0Block}} + +Only use the above as supporting context; prioritize the user's latest request.`; diff --git a/extensions/openbmb-clawxmemory/src/core/skills/extraction-skill.ts b/extensions/openbmb-clawxmemory/src/core/skills/extraction-skill.ts new file mode 100644 index 0000000000000..6fcda35ed6967 --- /dev/null +++ b/extensions/openbmb-clawxmemory/src/core/skills/extraction-skill.ts @@ -0,0 +1,97 @@ +import type { FactCandidate, MemoryMessage } from "../types.js"; +import { normalizeText, truncate } from "../utils/text.js"; +import type { SkillsRuntime } from "./types.js"; + +function extractFromPattern(text: string, pattern: RegExp): string[] { + const results: string[] = []; + for (const match of text.matchAll(pattern)) { + const value = normalizeText(match[1] ?? ""); + if (value) results.push(value); + } + return results; +} + +export function extractProjectTags(messages: MemoryMessage[], skills: SkillsRuntime): string[] { + const tags = new Set(); + const userText = messages + .filter((msg) => msg.role === "user") + .map((msg) => msg.content) + .join("\n"); + + for (const pattern of skills.extractionRules.projectPatterns) { + for (const value of extractFromPattern(userText, pattern)) { + const cleaned = value.replace(/[^\w.-]/g, ""); + if ( + cleaned.length >= skills.extractionRules.projectTagMinLength && + cleaned.length <= skills.extractionRules.projectTagMaxLength + ) { + tags.add(cleaned); + } + } + } + return Array.from(tags).slice(0, skills.extractionRules.maxProjectTags); +} + +export function extractFactCandidates( + messages: MemoryMessage[], + skills: SkillsRuntime, +): FactCandidate[] { + const facts = new Map(); + const userText = messages + .filter((msg) => msg.role === "user") + .map((msg) => msg.content) + .join("\n"); + + for (const rule of skills.extractionRules.factRules) { + for (const value of extractFromPattern(userText, rule.regex)) { + const text = truncate(value, rule.maxLength); + const key = `${rule.keyPrefix}:${text.toLowerCase()}`; + facts.set(key, { + factKey: key, + factValue: text, + confidence: rule.confidence, + }); + } + } + + for (const projectName of extractProjectTags(messages, skills)) { + facts.set(`project:${projectName.toLowerCase()}`, { + factKey: `project:${projectName.toLowerCase()}`, + factValue: projectName, + confidence: 0.78, + }); + } + + return Array.from(facts.values()).slice(0, skills.extractionRules.maxFacts); +} + +export function buildSessionSummary(messages: MemoryMessage[], skills: SkillsRuntime): string { + const userMessages = messages + .filter((msg) => msg.role === "user") + .map((msg) => normalizeText(msg.content)); + const assistantMessages = messages + .filter((msg) => msg.role === "assistant") + .map((msg) => normalizeText(msg.content)); + + const userHead = userMessages[0] ?? ""; + const userTail = userMessages[userMessages.length - 1] ?? ""; + const assistantTail = assistantMessages[assistantMessages.length - 1] ?? ""; + + const limits = skills.extractionRules.summaryLimits; + const parts = [ + userHead ? `用户提到：${truncate(userHead, limits.head)}` : "", + userTail && userTail !== userHead ? `后续重点：${truncate(userTail, limits.tail)}` : "", + assistantTail ? `助手响应：${truncate(assistantTail, limits.assistant)}` : "", + ].filter(Boolean); + + if (parts.length === 0) return "该窗口没有可用文本，跳过结构化摘要。"; + return parts.join("；"); +} + +export function buildSituationTimeInfo(timestamp: string, summary: string): string { + const date = new Date(timestamp); + if (Number.isNaN(date.getTime())) return `未知时间场景：${truncate(summary, 120)}`; + const hour = String(date.getHours()).padStart(2, "0"); + const minute = String(date.getMinutes()).padStart(2, "0"); + return `${date.toISOString().slice(0, 10)} ${hour}:${minute} 用户正在推进：${truncate(summary, 120)}`; +} diff --git a/extensions/openbmb-clawxmemory/src/core/skills/intent-skill.ts b/extensions/openbmb-clawxmemory/src/core/skills/intent-skill.ts new file mode 100644 index 0000000000000..650122d478ec7 --- /dev/null +++ b/extensions/openbmb-clawxmemory/src/core/skills/intent-skill.ts @@ -0,0 +1,21 @@ +import type { IntentType } from "../types.js"; +import type { SkillsRuntime } from "./types.js"; + +export function classifyIntent(query: string, skills: SkillsRuntime): IntentType { + const normalized = query.toLowerCase(); + const score = { + time: skills.intentRules.timeKeywords.filter((word) => normalized.includes(word.toLowerCase())) + .length, + project: skills.intentRules.projectKeywords.filter((word) => + normalized.includes(word.toLowerCase()), + ).length, + fact: skills.intentRules.factKeywords.filter((word) => normalized.includes(word.toLowerCase())) + .length, + }; + + if (score.project > 0 && score.project >= score.time && score.project >= score.fact) + return "project"; + if (score.time > 0 && score.time >= score.fact) return "time"; + if (score.fact > 0) return "fact"; + return "general"; +} diff --git a/extensions/openbmb-clawxmemory/src/core/skills/llm-extraction.ts b/extensions/openbmb-clawxmemory/src/core/skills/llm-extraction.ts new file mode 100644 index 0000000000000..f2845d3782080 --- /dev/null +++ b/extensions/openbmb-clawxmemory/src/core/skills/llm-extraction.ts @@ -0,0 +1,3092 @@ +import type { + DreamEvidenceRef, + DreamReviewFinding, + DreamReviewFocus, + DreamReviewResult, + FactCandidate, + GlobalProfileRecord, + IntentType, + L0SessionRecord, + L1WindowRecord, + L2ProjectIndexRecord, + L2TimeIndexRecord, + MemoryMessage, + ProjectDetail, + ProjectStatus, + RetrievalResult, + RetrievalPromptDebug, +} from "../types.js"; + +type LoggerLike = { + info?: (...args: unknown[]) => void; + warn?: (...args: unknown[]) => void; + error?: (...args: unknown[]) => void; +}; + +type ProviderHeaders = Record | undefined; +type PromptDebugSink = (debug: RetrievalPromptDebug) => void; + +function isTimeoutError(error: unknown): boolean { + return error instanceof Error && (error.name === "AbortError" || /timeout/i.test(error.message)); +} + +function resolveRequestTimeoutMs(timeoutMs: number | undefined): number | null { + if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs)) return 15_000; + if (timeoutMs <= 0) return null; + return timeoutMs; +} + +interface ModelSelection { + provider: string; + model: string; + api: string; + baseUrl?: string; + headers?: ProviderHeaders; +} + +interface RawFactItem { + category?: unknown; + subject?: unknown; + value?: unknown; + confidence?: unknown; +} + +interface RawProjectItem { + key?: unknown; + name?: unknown; + status?: unknown; + summary?: unknown; + latest_progress?: unknown; + confidence?: unknown; +} + +interface RawExtractionPayload { + summary?: unknown; + situation_time_info?: unknown; + facts?: unknown; + projects?: unknown; +} + +interface RawProjectResolutionPayload { + matched_project_key?: unknown; + canonical_key?: unknown; + canonical_name?: unknown; +} + +interface RawProjectBatchResolutionPayload { + projects?: unknown; +} + +interface RawTopicShiftPayload { + topic_changed?: unknown; + topic_summary?: unknown; +} + +interface RawDailySummaryPayload { + summary?: unknown; +} + +interface RawProfilePayload { + profile_text?: unknown; +} + +interface RawDreamFindingPayload { + title?: unknown; + rationale?: unknown; + confidence?: unknown; + target?: unknown; + evidence_refs?: unknown; +} + +interface RawDreamReviewPayload { + summary?: unknown; + project_rebuild?: unknown; + profile_suggestions?: unknown; + cleanup?: unknown; + ambiguous?: unknown; + no_action?: unknown; +} + +interface RawDreamProjectPlanItemPayload { + project_key?: unknown; + project_name?: unknown; + current_status?: unknown; + summary?: unknown; + latest_progress?: unknown; + retained_l1_ids?: unknown; +} + +interface RawDreamL1IssuePayload { + issue_type?: unknown; + title?: unknown; + l1_ids?: unknown; + related_project_keys?: unknown; +} + +interface RawDreamProjectPlanPayload { + summary?: unknown; + duplicate_topic_count?: unknown; + conflict_topic_count?: unknown; + projects?: unknown; + deleted_project_keys?: unknown; + l1_issues?: unknown; +} + +interface RawDreamGlobalProfileRewritePayload { + profile_text?: unknown; + source_l1_ids?: unknown; + conflict_with_existing?: unknown; +} + +interface RawReasoningPayload { + intent?: unknown; + enough_at?: unknown; + use_profile?: unknown; + l2_ids?: unknown; + l1_ids?: unknown; + l0_ids?: unknown; +} + +interface RawHop1RoutePayload { + query_scope?: unknown; + effective_query?: unknown; + memory_relevant?: unknown; + base_only?: unknown; + lookup_queries?: unknown; +} + +interface RawHop2L2Payload { + intent?: unknown; + evidence_note?: unknown; + enough_at?: unknown; +} + +interface RawHop3L1Payload { + evidence_note?: unknown; + enough_at?: unknown; +} + +interface RawHop4L0Payload { + evidence_note?: unknown; + enough_at?: unknown; +} + +interface RawLookupQueryPayload { + target_types?: unknown; + lookup_query?: unknown; + time_range?: unknown; +} + +interface RawTimeRangePayload { + start_date?: unknown; + end_date?: unknown; +} + +export interface SessionExtractionResult { + summary: string; + situationTimeInfo: string; + facts: FactCandidate[]; + projectDetails: ProjectDetail[]; +} + +export interface LlmProjectResolutionInput { + project: ProjectDetail; + existingProjects: L2ProjectIndexRecord[]; + agentId?: string; +} + +export interface LlmTopicShiftInput { + currentTopicSummary: string; + recentUserTurns: string[]; + incomingUserTurns: string[]; + agentId?: string; +} + +export interface LlmTopicShiftDecision { + topicChanged: boolean; + topicSummary: string; +} + +export interface LlmDailyTimeSummaryInput { + dateKey: string; + existingSummary: string; + l1: L1WindowRecord; + agentId?: string; +} + +export interface LlmGlobalProfileInput { + existingProfile: string; + l1: L1WindowRecord; + agentId?: string; +} + +export interface LlmDreamReviewInput { + focus: DreamReviewFocus; + profile: GlobalProfileRecord | null; + l2Projects: L2ProjectIndexRecord[]; + l1Windows: L1WindowRecord[]; + l0Sessions: L0SessionRecord[]; + evidenceRefs: DreamEvidenceRef[]; + timeLayerNotes: DreamReviewFinding[]; + agentId?: string; +} + +export type LlmDreamReviewResult = Omit; + +export interface LlmDreamProjectClusterInput { + clusterId: string; + label: string; + candidateKeys: string[]; + candidateNames: string[]; + currentProjectKeys: string[]; + l1Ids: string[]; + statuses: ProjectStatus[]; + summaries: string[]; + latestProgresses: string[]; + issueHints: Array<"duplicate" | "conflict" | "isolated">; + representativeWindows: Array<{ + l1IndexId: string; + endedAt: string; + summary: string; + }>; +} + +export interface LlmDreamProjectRebuildInput { + currentProjects: L2ProjectIndexRecord[]; + profile: GlobalProfileRecord | null; + l1Windows: L1WindowRecord[]; + l0Sessions: L0SessionRecord[]; + clusters: LlmDreamProjectClusterInput[]; + agentId?: string; +} + +export interface LlmDreamL1Issue { + issueType: "duplicate" | "conflict" | "isolated"; + title: string; + l1Ids: string[]; + relatedProjectKeys: string[]; +} + +export interface LlmDreamProjectRebuildOutput { + summary: string; + duplicateTopicCount: number; + conflictTopicCount: number; + projects: Array<{ + projectKey: string; + projectName: string; + currentStatus: ProjectStatus; + summary: string; + latestProgress: string; + retainedL1Ids: string[]; + }>; + deletedProjectKeys: string[]; + l1Issues: LlmDreamL1Issue[]; +} + +export interface LlmDreamGlobalProfileRewriteInput { + existingProfile: GlobalProfileRecord | null; + l1Windows: L1WindowRecord[]; + currentProjects: L2ProjectIndexRecord[]; + plannedProjects: Array<{ + projectKey: string; + projectName: string; + currentStatus: ProjectStatus; + summary: string; + latestProgress: string; + retainedL1Ids: string[]; + }>; + l1Issues: LlmDreamL1Issue[]; + agentId?: string; +} + +export interface LlmDreamGlobalProfileRewriteOutput { + profileText: string; + sourceL1Ids: string[]; + conflictWithExisting: boolean; +} + +export interface LlmReasoningInput { + query: string; + profile: GlobalProfileRecord | null; + l2Time: L2TimeIndexRecord[]; + l2Projects: L2ProjectIndexRecord[]; + l1Windows: L1WindowRecord[]; + l0Sessions: L0SessionRecord[]; + limits: { + l2: number; + l1: number; + l0: number; + }; + timeoutMs?: number; + agentId?: string; +} + +export interface LlmReasoningSelection { + intent: IntentType; + enoughAt: RetrievalResult["enoughAt"]; + useProfile: boolean; + l2Ids: string[]; + l1Ids: string[]; + l0Ids: string[]; +} + +export interface LlmProjectBatchResolutionInput { + projects: ProjectDetail[]; + existingProjects: L2ProjectIndexRecord[]; + agentId?: string; +} + +export interface LlmProjectMemoryRewriteItem { + incomingProject: ProjectDetail; + existingProject: L2ProjectIndexRecord | null; + recentWindows: L1WindowRecord[]; +} + +export interface LlmProjectMemoryRewriteInput { + l1: L1WindowRecord; + projects: LlmProjectMemoryRewriteItem[]; + agentId?: string; +} + +export type LookupTargetType = "time" | "project"; + +export interface LlmMemoryRouteInput { + query: string; + profile: GlobalProfileRecord | null; + recentMessages: MemoryMessage[]; + timeoutMs?: number; + agentId?: string; + debugTrace?: PromptDebugSink; +} + +export interface LookupQuerySpec { + targetTypes: LookupTargetType[]; + lookupQuery: string; + timeRange?: { + startDate: string; + endDate: string; + } | null; +} + +export interface Hop1LookupDecision { + queryScope: "standalone" | "continuation"; + effectiveQuery: string; + memoryRelevant: boolean; + baseOnly: boolean; + lookupQueries: LookupQuerySpec[]; +} + +export interface L2CatalogEntry { + id: string; + type: LookupTargetType; + label: string; + lookupKeys: string[]; + compressedContent: string; +} + +export interface LlmHop2L2Input { + query: string; + profile: GlobalProfileRecord | null; + lookupQueries: LookupQuerySpec[]; + l2Entries: L2CatalogEntry[]; + catalogTruncated?: boolean; + timeoutMs?: number; + agentId?: string; + debugTrace?: PromptDebugSink; +} + +export interface Hop2L2Decision { + intent: IntentType; + evidenceNote: string; + enoughAt: "l2" | "descend_l1" | "none"; +} + +export interface L0HeaderCandidate { + l0IndexId: string; + sessionKey: string; + timestamp: string; + lastUserMessage: string; + lastAssistantMessage: string; +} + +export interface LlmHop3L1Input { + query: string; + evidenceNote: string; + selectedL2Entries: L2CatalogEntry[]; + l1Windows: L1WindowRecord[]; + timeoutMs?: number; + agentId?: string; + debugTrace?: PromptDebugSink; +} + +export interface Hop3L1Decision { + evidenceNote: string; + enoughAt: "l1" | "descend_l0" | "none"; +} + +export interface LlmHop4L0Input { + query: string; + evidenceNote: string; + selectedL2Entries: L2CatalogEntry[]; + selectedL1Windows: L1WindowRecord[]; + l0Sessions: L0SessionRecord[]; + timeoutMs?: number; + agentId?: string; + debugTrace?: PromptDebugSink; +} + +export interface Hop4L0Decision { + evidenceNote: string; + enoughAt: "l0" | "none"; +} + +const EXTRACTION_SYSTEM_PROMPT = ` +You are a memory indexing engine for a conversational assistant. + +Your job is to convert a visible user/assistant conversation into durable memory indexes. + +Rules: +- Only use information explicitly present in the conversation. +- Ignore system prompts, tool scaffolding, hidden reasoning, formatting artifacts, and operational chatter. +- Be conservative. If something is ambiguous, omit it. +- Track projects only when they look like a real ongoing effort, task stream, research topic, implementation effort, or recurring problem worth revisiting later. +- "Project" here is broad: it can be a workstream, submission, research effort, health/problem thread, or other ongoing topic the user is likely to revisit. +- If the conversation contains multiple independent ongoing threads, return multiple project items instead of collapsing them into one. +- Repeated caregiving, illness handling, symptom tracking, recovery follow-up, or other ongoing real-world problem-solving threads should be treated as projects when the user is actively managing them. +- Example: "friend has diarrhea / user buys medicine / later reports recovery" is a project-like thread. +- Example: "preparing an EMNLP submission" is another independent project-like thread. +- Do not treat casual one-off mentions as projects. +- Extract facts only when they are likely to matter in future conversations: preferences, constraints, goals, identity, long-lived context, stable relationships, or durable project context. +- The facts are intermediate material for a later global profile rewrite, so prefer stable facts over temporary situation notes. +- Natural-language output fields must use the dominant language of the user messages. If user messages are mixed, prefer the most recent user language. Keys and enums must stay in English. +- Each project summary must be a compact 1-2 sentence project memory, not a generic status line. +- A good project summary should preserve: what the project is, what stage it is in now, and the next step / blocker / missing info when available. +- Do not output vague summaries like "the user is working on this project", "progress is going well", "things are okay", or "handling something" unless the project-specific context is also included. +- latest_progress must stay short and only capture the newest meaningful update, newest blocker, or newest confirmation state. +- Return valid JSON only. No markdown fences, no commentary. + +Use this exact JSON shape: +{ + "summary": "short session summary", + "situation_time_info": "short time-aware progress line", + "facts": [ + { + "category": "preference | profile | goal | constraint | relationship | project | context | other", + "subject": "stable english key fragment", + "value": "durable fact text", + "confidence": 0.0 + } + ], + "projects": [ + { + "key": "stable english identifier, lower-kebab-case", + "name": "project name as the user would recognize it", + "status": "planned | in_progress | done", + "summary": "rolling 1-2 sentence summary: what this project is + current phase + next step/blocker when known", + "latest_progress": "short latest meaningful progress or blocker, without repeating the full project background", + "confidence": 0.0 + } + ] +} +`.trim(); + +const PROJECT_RESOLUTION_SYSTEM_PROMPT = ` +You resolve whether an incoming project memory should merge into an existing project memory. + +Rules: +- Prefer merging duplicates caused by wording differences, synonyms, or different granularity of the same effort. +- Match only when the underlying ongoing effort is clearly the same. +- Reuse an existing project when possible. +- If multiple labels refer to the same EMNLP submission, the same health follow-up, or the same long-running effort, merge them. +- Return JSON only. + +Use this exact JSON shape: +{ + "matched_project_key": "existing project key or null", + "canonical_key": "stable lower-kebab-case key", + "canonical_name": "project name users would recognize" +} +`.trim(); + +const PROJECT_BATCH_RESOLUTION_SYSTEM_PROMPT = ` +You resolve whether each incoming project memory should merge into an existing project memory. + +Rules: +- Process all incoming projects together so duplicates inside the same batch can be merged. +- Prefer merging duplicates caused by wording differences, synonyms, or different granularity of the same effort. +- Reuse an existing project when possible. +- Only create a new canonical project when none of the existing projects match. +- Return JSON only. + +Use this exact JSON shape: +{ + "projects": [ + { + "incoming_key": "original incoming project key", + "matched_project_key": "existing project key or null", + "canonical_key": "stable lower-kebab-case key", + "canonical_name": "project name users would recognize" + } + ] +} +`.trim(); + +const PROJECT_COMPLETION_SYSTEM_PROMPT = ` +You review an extracted project list and complete any missing ongoing threads from the conversation. + +Rules: +- Return the full corrected project list, not just additions. +- Include all independent ongoing threads that are likely to matter in future conversation. +- Health/caregiving/problem-management threads count as projects when the user is actively managing them. +- Resolved but substantial threads from the current window may still be kept with status "done" if they are a meaningful thread the user may refer back to. +- Example pair of separate projects in one window: "friend's stomach illness and medicine follow-up" plus "EMNLP submission preparation". +- Merge duplicates caused by wording differences. +- For each project summary, write a compact 1-2 sentence project memory that explains what the project is, what phase it is in, and the next step / blocker / missing info when available. +- Do not flatten summaries into generic text like "the user is working on something", "progress is okay", or "making progress". +- latest_progress should stay short and only describe the newest concrete update. +- Return JSON only. + +Use this exact JSON shape: +{ + "projects": [ + { + "key": "stable english identifier, lower-kebab-case", + "name": "project name as the user would recognize it", + "status": "planned | in_progress | done", + "summary": "rolling 1-2 sentence summary: what this project is + current phase + next step/blocker when known", + "latest_progress": "short latest meaningful progress or blocker, without repeating the full project background", + "confidence": 0.0 + } + ] +} +`.trim(); + +const TOPIC_BOUNDARY_SYSTEM_PROMPT = ` +You judge whether new user messages continue the current topic or start a new topic. + +Rules: +- Use only semantic meaning, not keyword overlap. +- Treat a topic as the same if the user is still talking about the same underlying problem, project, situation, or intent. +- Treat it as changed only when the new user messages clearly pivot to a different underlying topic. +- You are given only user messages. Do not assume any assistant content. +- Return JSON only. + +Use this exact JSON shape: +{ + "topic_changed": true, + "topic_summary": "short topic summary in the user's language" +} +`.trim(); + +const DAILY_TIME_SUMMARY_SYSTEM_PROMPT = ` +You maintain a single daily episodic memory summary for a user. + +Rules: +- Focus on what happened during that day, what the user was dealing with, and the day's situation. +- Do not turn the summary into a long-term profile. +- Do not over-focus on project metadata; describe the day's lived context. +- Merge the existing daily summary with the new L1 window into one concise updated daily summary. +- Natural-language output must follow the language used by the user in the new L1 window. +- Return JSON only. + +Use this exact JSON shape: +{ + "summary": "updated daily summary" +} +`.trim(); + +const PROJECT_MEMORY_REWRITE_SYSTEM_PROMPT = ` +You maintain rolling L2 project memories for a conversational memory system. + +Rules: +- Rewrite the full project memory for each incoming project using the existing L2 memory, recent linked L1 windows, and the new L1 window. +- Preserve earlier project background and major stage transitions whenever they are still useful. +- The new summary must not overwrite older context with only the newest update. +- summary must be a compact 1-2 sentence rolling project memory that preserves: + 1. what the project is, + 2. important stage progression or milestones so far, + 3. the current phase, + 4. the next step / blocker / missing info when present. +- latest_progress must stay short and only describe the newest meaningful update, blocker, or confirmation state. +- Do not output generic summaries like "the user is working on this project", "progress is going well", "things are okay", or "handling something" unless the project-specific context is explicitly preserved. +- Keep each project's incoming key stable. +- Natural-language output must follow the language used by the user in the new L1 window. +- Return JSON only. + +Use this exact JSON shape: +{ + "projects": [ + { + "key": "same stable english identifier as the incoming project", + "name": "project name as the user would recognize it", + "status": "planned | in_progress | done", + "summary": "rolling 1-2 sentence project memory with background + stage progression + current phase + next step/blocker when known", + "latest_progress": "short latest meaningful progress or blocker", + "confidence": 0.0 + } + ] +} +`.trim(); + +const GLOBAL_PROFILE_SYSTEM_PROMPT = ` +You maintain a single global user profile summary. + +Rules: +- Rewrite the whole profile as one concise paragraph. +- Keep only stable user traits, identity, long-term preferences, constraints, relationships, communication style, and long-range goals. +- Do not include temporary daily events, short-lived situations, or project progress updates. +- Use the existing profile plus the new L1 facts as evidence, then rewrite the full profile. +- Natural-language output must follow the user's dominant language in the new L1 window. +- Return JSON only. + +Use this exact JSON shape: +{ + "profile_text": "updated stable user profile paragraph" +} +`.trim(); + +const DREAM_REVIEW_SYSTEM_PROMPT = ` +You are a read-only Dream review engine for a layered conversational memory system. + +Your job is to review recent memory quality and emit governance findings without modifying memory. + +Rules: +- Treat recent L1 windows as the primary source of truth. +- Review whether current L2 project memories still match the recent L1 evidence. +- Review whether stable cross-window signals should be promoted into the global profile, or whether existing profile content now conflicts with recent evidence. +- L2Time is a diary-like time layer. Do not treat it as a semantic rewrite target. Use time-layer integrity notes only as context. +- Use L0 previews only when they help explain why an L1 window may be suspicious or ambiguous. +- Only output findings supported by the provided evidence_refs. +- Each finding must cite only ref ids that appear in the provided evidence set. +- Keep findings concise, concrete, and implementer-friendly. +- If a finding is mainly about rebuilding or merging project memories, use target="l2_project". +- If a finding is mainly about stable profile promotion or profile conflict, use target="global_profile". +- If a finding is about L1 extraction/grouping quality and should not directly rewrite higher layers yet, use target="l1_only". +- Natural-language output should follow the dominant language already used in the supplied evidence. +- Return valid JSON only. + +Use this exact JSON shape: +{ + "summary": "short review summary", + "project_rebuild": [ + { + "title": "short finding title", + "rationale": "why this should be reviewed or rebuilt", + "confidence": 0.0, + "target": "l2_project | global_profile | l1_only", + "evidence_refs": ["ref:id"] + } + ], + "profile_suggestions": [ + { + "title": "short finding title", + "rationale": "why this stable signal should be promoted or why profile conflicts", + "confidence": 0.0, + "target": "global_profile | l1_only", + "evidence_refs": ["ref:id"] + } + ], + "cleanup": [ + { + "title": "short finding title", + "rationale": "duplicate, stale, or noisy memory issue", + "confidence": 0.0, + "target": "l2_project | global_profile | l1_only", + "evidence_refs": ["ref:id"] + } + ], + "ambiguous": [ + { + "title": "short finding title", + "rationale": "what remains ambiguous and why more verification is needed", + "confidence": 0.0, + "target": "l1_only | l2_project | global_profile", + "evidence_refs": ["ref:id"] + } + ], + "no_action": [ + { + "title": "short finding title", + "rationale": "why this memory looks healthy and should remain as-is", + "confidence": 0.0, + "target": "l2_project | global_profile | l1_only", + "evidence_refs": ["ref:id"] + } + ] +} +`.trim(); + +const DREAM_PROJECT_REBUILD_SYSTEM_PROMPT = ` +You are the Dream project reconstruction planner for a layered conversational memory system. + +Your job is to inspect all supplied L1 windows, current L2 project memories, and local topic clusters, then output the final L2 project set that should be written back. + +Rules: +- L1 windows are the primary evidence source. +- Rebuild project memories from L1. Do not preserve an old L2 project just because it already exists. +- Merge duplicate or overlapping projects when they clearly describe the same ongoing effort. +- If a current L2 project is stale, duplicated, or no longer supported by the supplied L1 evidence, include its key in deleted_project_keys. +- Do not modify, delete, or rewrite L1. Only decide how L1 should map into rebuilt L2 projects. +- Prefer one main project owner for each L1 window. Reuse the same L1 in multiple rebuilt projects only when the overlap is genuinely necessary. +- Each rebuilt project must provide a stable project_key, human-recognizable project_name, current_status, rolling summary, latest_progress, and the retained_l1_ids that justify it. +- retained_l1_ids must only contain ids that appear in the supplied l1_windows. +- Use l1_issues only for duplicate/conflict/isolated L1 topic notes that help explain the rebuild. +- Natural-language output should follow the dominant language already present in the supplied evidence. +- Return valid JSON only. + +Use this exact JSON shape: +{ + "summary": "short rebuild summary", + "duplicate_topic_count": 0, + "conflict_topic_count": 0, + "projects": [ + { + "project_key": "stable lower-kebab-case key", + "project_name": "project name users would recognize", + "current_status": "planned | in_progress | done", + "summary": "compact rolling project memory", + "latest_progress": "latest meaningful update or blocker", + "retained_l1_ids": ["l1-1", "l1-2"] + } + ], + "deleted_project_keys": ["old-project-key"], + "l1_issues": [ + { + "issue_type": "duplicate | conflict | isolated", + "title": "short issue title", + "l1_ids": ["l1-1", "l1-2"], + "related_project_keys": ["project-key"] + } + ] +} +`.trim(); + +const DREAM_GLOBAL_PROFILE_REWRITE_SYSTEM_PROMPT = ` +You are the Dream global profile rewrite engine for a layered conversational memory system. + +Your job is to rewrite the global profile using only stable signals supported by the supplied L1 windows and the planned Dream project rebuild. + +Rules: +- The global profile stores stable user preferences, identity, habits, constraints, working style, and long-lived relationships. +- Do not include daily events, diary-like time details, or temporary project updates. +- Use only supplied L1 windows as evidence. Do not invent new traits. +- source_l1_ids must be an exact supporting set chosen from the supplied L1 ids. +- Only include signals that are stable across multiple windows, or clearly override an older profile statement with stronger recent evidence. +- If the existing profile is still mostly valid, rewrite it conservatively instead of discarding everything. +- Natural-language output should follow the dominant language already present in the supplied evidence. +- Return valid JSON only. + +Use this exact JSON shape: +{ + "profile_text": "rewritten global profile paragraph", + "source_l1_ids": ["l1-1", "l1-2"], + "conflict_with_existing": false +} +`.trim(); + +const REASONING_SYSTEM_PROMPT = ` +You are a semantic memory retrieval reasoner. + +Your job is to decide which memory records are relevant to the user's query. + +Rules: +- Use semantic meaning, not keyword overlap. +- Use high recall for obvious paraphrases and near-synonyms. +- Temporal summary questions asking what the user did today, what happened today, or what they were recently working on should usually select L2 time indexes. +- If there is a current-day or recent-day L2 time summary and the user asks about today/recent activity, prefer that L2 time record even if wording differs. +- For project queries, prefer L2 project indexes when they already capture enough. +- For time queries, prefer L2 time indexes when they already capture enough. +- For profile/fact queries about the user's identity, preferences, habits, or stable traits, set use_profile=true when the global profile is useful. +- Select the smallest set of records needed to answer the query well. +- enough_at only refers to L2/L1/L0 structured memory. The profile is an additional supporting source. +- If L2 already captures enough, set enough_at to "l2". +- If L2 is insufficient but L1 is enough, set enough_at to "l1". +- If detailed raw conversation is needed, set enough_at to "l0". +- Return JSON only. + +Use this exact JSON shape: +{ + "intent": "time | project | fact | general", + "enough_at": "l2 | l1 | l0 | none", + "use_profile": true, + "l2_ids": ["l2 index id"], + "l1_ids": ["l1 index id"], + "l0_ids": ["l0 index id"] +} +`.trim(); + +const HOP1_LOOKUP_SYSTEM_PROMPT = ` +You are the first-hop planner for a memory retrieval system. + +Your job is not to choose concrete record ids. Your job is to decide: +1. whether the current query is standalone or a continuation of the recent conversation, +2. what the effective self-contained query should be, +3. whether this question needs dynamic memory, +4. which index types the next step should search, +5. which lookup query terms should be used for that search. + +Rules: +- Use semantic meaning, not surface keyword matching. +- current_local_date is the current local date in YYYY-MM-DD format. +- global_profile is the top-level stable profile. +- recent_messages are the most recent cleaned user/assistant turns from the short-term session context. +- recent_messages are only supporting context for understanding the current query. They are not durable memory evidence. +- First decide query_scope: + - use "continuation" only when the current query clearly depends on recent_messages to resolve omitted topic, entity, or time anchor. + - use "standalone" when the current query is already self-contained, or when it clearly starts a new topic and should ignore old context. +- If query_scope="standalone", effective_query should restate the current query faithfully and should not inherit irrelevant topic details from recent_messages. +- If query_scope="continuation", effective_query must rewrite the current query into a short self-contained query using only the needed context from recent_messages. +- effective_query must stay close to the user's intent. Do not add new goals or assumptions. +- If the question can be answered from global_profile alone, you must set base_only=true. +- Typical base_only questions include: + - user identity, preferences, habits, and long-term traits + - for example: "What language do I prefer to use?" + - for example: "What food do I usually like?" + - for example: "Introduce me." + - for example: "Who is Liangzi?" + - for example: "Do you still remember me?" +- If the question asks what happened on a certain day, what happened recently, what the user was busy with today, how a project has progressed recently, or what was recommended earlier, base_only must be false. +- If base_only=false, you must output at least one lookup_queries entry. +- If base_only=true, lookup_queries must be an empty array. +- A mixed question can involve both time and project retrieval, so target_types may be ["time","project"]. +- lookup_query should be a short search phrase usable by the retrieval step, not a restatement of the full rule. +- Only output time_range when the question is truly about a time range. +- time_range must be normalized to a local date range in this exact format: + { "start_date": "YYYY-MM-DD", "end_date": "YYYY-MM-DD" } +- Expressions like "today", "yesterday", "the last week", "last month", or "from March 16 to March 18" should be normalized into explicit date ranges whenever possible. +- If the question is project-related and also time-bounded, a single lookup_query may use target_types=["time","project"] and include time_range. +- Do not choose concrete record ids at this hop. +- Return JSON only. Do not explain. + +Examples: +- Query: "What language do I prefer to use?" + -> memory_relevant=true, base_only=true, lookup_queries=[] +- Query: "Introduce me." + -> memory_relevant=true, base_only=true, lookup_queries=[] +- Query: "Who is Liangzi?" + -> memory_relevant=true, base_only=true, lookup_queries=[] +- Query: "What was I busy with today?" + -> memory_relevant=true, base_only=false, lookup_queries=[{"target_types":["time"],"lookup_query":"what I did today","time_range":{"start_date":"YYYY-MM-DD","end_date":"YYYY-MM-DD"}}] +- Query: "How is my paper progressing today?" + -> query_scope="standalone", effective_query="How is my paper progressing today?", memory_relevant=true, base_only=false, lookup_queries=[{"target_types":["time","project"],"lookup_query":"today EMNLP paper progress","time_range":{"start_date":"YYYY-MM-DD","end_date":"YYYY-MM-DD"}}] +- Query: "Which Beijing barbecue place did you recommend before?" + -> query_scope="standalone", effective_query="Which Beijing barbecue place did you recommend before?", memory_relevant=true, base_only=false, lookup_queries=[{"target_types":["project"],"lookup_query":"Beijing barbecue recommendation"}] +- recent_messages include: + user: "我在西北旺都做了什么" + assistant: "你主要在西北旺处理了几个工作点。" + Query: "不够详细" + -> query_scope="continuation", effective_query="更详细地回忆我在西北旺都做了什么", memory_relevant=true, base_only=false, lookup_queries=[{"target_types":["time","project"],"lookup_query":"西北旺 做了什么 详细回忆"}] +- recent_messages include: + user: "最近在改 retrieval 路由" + assistant: "主要在改 Hop1 和 L2 候选构建。" + Query: "帮我查一下上海天气" + -> query_scope="standalone", effective_query="帮我查一下上海天气", memory_relevant=false, base_only=false, lookup_queries=[] + +Use this exact JSON shape: +{ + "query_scope": "standalone | continuation", + "effective_query": "self-contained query", + "memory_relevant": true, + "base_only": false, + "lookup_queries": [ + { + "target_types": ["time", "project"], + "lookup_query": "short lookup query", + "time_range": { + "start_date": "YYYY-MM-DD", + "end_date": "YYYY-MM-DD" + } + } + ] +} +`.trim(); + +const HOP2_L2_SYSTEM_PROMPT = ` +You are the second-hop planner for a memory retrieval system. + +You have already received the real L2 entries selected by code-side retrieval. Your job is to: +1. read the L2 evidence, +2. write an evidence_note directly relevant to the current question, +3. decide whether stopping at L2 is sufficient, or whether the system should descend to L1. + +Rules: +- l2_entries are not catalog names. They already contain the compressed real L2 content. +- Use semantic meaning, not surface keyword matching. +- evidence_note must be a compact knowledge note that keeps only information relevant to answering the current query. Do not restate every entry. +- If L2 already answers the query, set enough_at="l2". +- If L2 is relevant but still insufficient and linked L1 windows should be read, set enough_at="descend_l1". +- Only set enough_at="none" when L2 genuinely does not help. +- If the query is about stable profile information, such as language preference, long-term identity, or communication style, and global_profile is already sufficient, leave evidence_note empty and set enough_at="none". +- A mixed question may include both time L2 and project L2 evidence. +- If an exact answer already appears in a project L2 latest_progress or summary, you may stop at L2 instead of forcing a descent. +- catalog_truncated=true only means older entries were omitted for prompt budget reasons; it does not make the current entries unreliable. +- Return JSON only. Do not explain. + +Examples: +- Query: "What language do I prefer to use?" + -> evidence_note="", enough_at="none" +- Query: "What was I busy with today?" + -> derive an evidence_note from today's time L2 and set enough_at="l2" +- Query: "How is my paper progressing today?" + -> merge today's time L2 and the related project L2 into one evidence_note +- Query: "Which Beijing barbecue place did you recommend before?" + -> if the exact venue name already appears in project L2 latest_progress, set enough_at="l2"; otherwise set enough_at="descend_l1" + +Use this exact JSON shape: +{ + "intent": "time | project | fact | general", + "evidence_note": "condensed note from L2 evidence", + "enough_at": "l2 | descend_l1 | none" +} +`.trim(); + +const HOP3_L1_SYSTEM_PROMPT = ` +You are the L1 evidence-note updater for a memory retrieval system. + +Your job is to read the current evidence note, selected L2 evidence, plus linked L1 windows, then update the note and decide whether L1 is enough. + +Rules: +- current_evidence_note is the knowledge note produced from previous hops. Refine it instead of discarding it. +- Read the selected L2 entries as higher-level context. +- Read the candidate L1 windows as the next level of evidence. +- Do not choose L0 here. +- evidence_note should preserve only information relevant to the user's query. +- If selected L1 windows already answer the query, set enough_at="l1". +- If lower raw conversation detail is still needed, set enough_at="descend_l0". +- If neither L1 nor lower levels help, set enough_at="none". +- Return JSON only. + +Use this exact JSON shape: +{ + "evidence_note": "updated note from L1 evidence", + "enough_at": "l1 | descend_l0 | none" +} +`.trim(); + +const HOP4_L0_SYSTEM_PROMPT = ` +You are the raw-conversation evidence-note updater for a memory retrieval system. + +Your job is to read the current evidence note, selected L2 evidence, selected L1 windows, and linked raw L0 conversations, then update the note and choose whether raw L0 detail is enough. + +Rules: +- current_evidence_note is the note produced by earlier hops. Refine it with exact conversation details when useful. +- Use raw L0 only when exact prior wording, exact recommendation, exact names, or other conversation-level detail is needed. +- evidence_note should be the best final note after incorporating L0 detail. +- If one or more selected L0 sessions contain the needed detail, set enough_at="l0". +- Otherwise set enough_at="none". +- Return JSON only. + +Use this exact JSON shape: +{ + "evidence_note": "final note from L0 evidence", + "enough_at": "l0 | none" +} +`.trim(); + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function truncate(value: string, maxLength: number): string { + if (value.length <= maxLength) return value; + return value.slice(0, maxLength).trim(); +} + +function normalizeWhitespace(value: string): string { + return value.replace(/\s+/g, " ").trim(); +} + +function stripTrailingSlash(value: string): string { + return value.replace(/\/+$/, ""); +} + +function sanitizeHeaders(headers: unknown): ProviderHeaders { + if (!isRecord(headers)) return undefined; + const next: Record = {}; + for (const [key, value] of Object.entries(headers)) { + if (typeof value === "string" && value.trim()) next[key] = value; + } + return Object.keys(next).length > 0 ? next : undefined; +} + +function parseModelRef( + modelRef: string | undefined, + config: Record, +): { provider: string; model: string } | undefined { + if (typeof modelRef === "string" && modelRef.includes("/")) { + const [provider, ...rest] = modelRef.split("/"); + const model = rest.join("/").trim(); + if (provider?.trim() && model) { + return { provider: provider.trim(), model }; + } + } + + const modelsConfig = isRecord(config.models) ? config.models : undefined; + const providers = + modelsConfig && isRecord(modelsConfig.providers) ? modelsConfig.providers : undefined; + if (!providers) return undefined; + + if (typeof modelRef === "string" && modelRef.trim()) { + const providerEntries = Object.entries(providers); + if (providerEntries.length === 1) { + return { provider: providerEntries[0]![0], model: modelRef.trim() }; + } + } + + for (const [provider, providerConfig] of Object.entries(providers)) { + if (!isRecord(providerConfig)) continue; + const models = Array.isArray(providerConfig.models) ? providerConfig.models : []; + const firstModel = models.find( + (entry) => isRecord(entry) && typeof entry.id === "string" && entry.id.trim(), + ); + if (firstModel && isRecord(firstModel)) { + return { provider, model: String(firstModel.id).trim() }; + } + } + return undefined; +} + +function resolveAgentPrimaryModel( + config: Record, + agentId?: string, +): string | undefined { + const agents = isRecord(config.agents) ? config.agents : undefined; + const defaults = agents && isRecord(agents.defaults) ? agents.defaults : undefined; + const defaultsModel = defaults && isRecord(defaults.model) ? defaults.model : undefined; + + if (agentId && agents && isRecord(agents[agentId])) { + const agentConfig = agents[agentId] as Record; + const agentModel = isRecord(agentConfig.model) ? agentConfig.model : undefined; + if (typeof agentModel?.primary === "string" && agentModel.primary.trim()) { + return agentModel.primary.trim(); + } + } + + if (typeof defaultsModel?.primary === "string" && defaultsModel.primary.trim()) { + return defaultsModel.primary.trim(); + } + + return undefined; +} + +function detectPreferredOutputLanguage(messages: MemoryMessage[]): string | undefined { + const userText = messages + .filter((message) => message.role === "user") + .map((message) => message.content) + .join("\n"); + if (/[\u4e00-\u9fff]/.test(userText)) return "Simplified Chinese"; + return undefined; +} + +function buildPrompt( + timestamp: string, + messages: MemoryMessage[], + extraInstruction?: string, +): string { + const conversation = messages.map((message, index) => ({ + index, + role: message.role, + content: message.content, + })); + const preferredLanguage = detectPreferredOutputLanguage(messages); + + const sections = [ + "Conversation timestamp:", + timestamp, + "", + "Visible conversation messages:", + JSON.stringify(conversation, null, 2), + "", + "Remember:", + "- summary should describe the session at a glance.", + "- situation_time_info should read like a short progress update anchored to this conversation moment.", + "- facts should be durable and future-useful, not turn-specific noise.", + "- projects should only include trackable ongoing efforts.", + "- if there are two or more unrelated ongoing threads, list them as separate project entries.", + "- health/caregiving/problem-management threads count as projects when they are ongoing across turns.", + "- each project summary should explain what the project is, what phase it is in, and the next step or blocker when available.", + "- avoid generic project summaries that only say progress is fine or ongoing.", + ]; + if (preferredLanguage) { + sections.push(`- Write all natural-language output fields in ${preferredLanguage}.`); + } + if (extraInstruction) { + sections.push("", "Additional requirement:", extraInstruction); + } + return sections.join("\n"); +} + +function buildProjectCompletionPrompt(input: { + timestamp: string; + messages: MemoryMessage[]; + summary: string; + facts: FactCandidate[]; + projectDetails: ProjectDetail[]; +}): string { + return JSON.stringify( + { + timestamp: input.timestamp, + messages: input.messages.map((message, index) => ({ + index, + role: message.role, + content: truncateForPrompt(message.content, 220), + })), + current_summary: input.summary, + current_facts: input.facts, + current_projects: input.projectDetails, + completion_goal: + "Keep all meaningful ongoing projects. Each summary should preserve project background, current phase, and next step or blocker when available.", + }, + null, + 2, + ); +} + +function buildTopicShiftPrompt(input: LlmTopicShiftInput): string { + return JSON.stringify( + { + current_topic_summary: truncateForPrompt(input.currentTopicSummary, 160), + recent_user_turns: input.recentUserTurns + .map((value) => truncateForPrompt(value, 180)) + .slice(-8), + incoming_user_turns: input.incomingUserTurns + .map((value) => truncateForPrompt(value, 180)) + .slice(-6), + }, + null, + 2, + ); +} + +function buildDailyTimeSummaryPrompt(input: LlmDailyTimeSummaryInput): string { + return JSON.stringify( + { + date_key: input.dateKey, + existing_daily_summary: truncateForPrompt(input.existingSummary, 320), + new_l1: { + summary: truncateForPrompt(input.l1.summary, 220), + situation_time_info: truncateForPrompt(input.l1.situationTimeInfo, 220), + projects: input.l1.projectDetails.map((project) => ({ + name: project.name, + status: project.status, + summary: truncateForPrompt(project.summary, 160), + latest_progress: truncateForPrompt(project.latestProgress, 160), + })), + facts: input.l1.facts + .map((fact) => ({ + key: fact.factKey, + value: truncateForPrompt(fact.factValue, 120), + })) + .slice(0, 10), + }, + }, + null, + 2, + ); +} + +function buildProjectMemoryRewritePrompt(input: LlmProjectMemoryRewriteInput): string { + return JSON.stringify( + { + current_l1: { + id: input.l1.l1IndexId, + time_period: input.l1.timePeriod, + summary: truncateForPrompt(input.l1.summary, 220), + situation_time_info: truncateForPrompt(input.l1.situationTimeInfo, 220), + projects: input.l1.projectDetails.map((project) => ({ + key: project.key, + name: project.name, + status: project.status, + summary: truncateForPrompt(project.summary, 220), + latest_progress: truncateForPrompt(project.latestProgress, 180), + })), + }, + incoming_projects: input.projects.map((item) => ({ + incoming_project: { + key: item.incomingProject.key, + name: item.incomingProject.name, + status: item.incomingProject.status, + summary: truncateForPrompt(item.incomingProject.summary, 240), + latest_progress: truncateForPrompt(item.incomingProject.latestProgress, 180), + confidence: item.incomingProject.confidence, + }, + existing_project_memory: item.existingProject + ? { + project_key: item.existingProject.projectKey, + project_name: item.existingProject.projectName, + status: item.existingProject.currentStatus, + summary: truncateForPrompt(item.existingProject.summary, 320), + latest_progress: truncateForPrompt(item.existingProject.latestProgress, 180), + } + : null, + recent_stage_windows: item.recentWindows.slice(0, 5).map((window) => ({ + id: window.l1IndexId, + time_period: window.timePeriod, + summary: truncateForPrompt(window.summary, 180), + situation_time_info: truncateForPrompt(window.situationTimeInfo, 180), + matching_project_details: window.projectDetails + .filter( + (project) => + project.key === item.incomingProject.key || + project.name === item.incomingProject.name, + ) + .slice(0, 2) + .map((project) => ({ + key: project.key, + name: project.name, + status: project.status, + summary: truncateForPrompt(project.summary, 180), + latest_progress: truncateForPrompt(project.latestProgress, 160), + })), + })), + })), + }, + null, + 2, + ); +} + +function buildGlobalProfilePrompt(input: LlmGlobalProfileInput): string { + return JSON.stringify( + { + existing_profile: truncateForPrompt(input.existingProfile, 320), + new_l1: { + summary: truncateForPrompt(input.l1.summary, 220), + situation_time_info: truncateForPrompt(input.l1.situationTimeInfo, 160), + facts: input.l1.facts + .map((fact) => ({ + key: fact.factKey, + value: truncateForPrompt(fact.factValue, 140), + confidence: fact.confidence, + })) + .slice(0, 16), + projects: input.l1.projectDetails + .map((project) => ({ + name: project.name, + status: project.status, + summary: truncateForPrompt(project.summary, 140), + })) + .slice(0, 8), + }, + }, + null, + 2, + ); +} + +function buildDreamReviewPrompt(input: LlmDreamReviewInput): string { + return JSON.stringify( + { + review_focus: input.focus, + governance_scope: { + source_of_truth: "recent_l1_windows", + primary_targets: ["l2_project", "global_profile"], + time_layer_policy: "integrity_notes_only_do_not_rewrite", + read_only: true, + }, + current_profile: input.profile + ? { + id: input.profile.recordId, + text: truncateForPrompt(input.profile.profileText, 360), + source_l1_ids: input.profile.sourceL1Ids.slice(-12), + } + : null, + current_l2_projects: input.l2Projects.map((project) => ({ + id: project.l2IndexId, + project_key: project.projectKey, + project_name: project.projectName, + summary: truncateForPrompt(project.summary, 240), + current_status: project.currentStatus, + latest_progress: truncateForPrompt(project.latestProgress, 180), + l1_source: project.l1Source.slice(-8), + updated_at: project.updatedAt, + })), + recent_l1_windows: input.l1Windows.map((window) => ({ + id: window.l1IndexId, + session_key: window.sessionKey, + time_period: window.timePeriod, + started_at: window.startedAt, + ended_at: window.endedAt, + summary: truncateForPrompt(window.summary, 220), + situation_time_info: truncateForPrompt(window.situationTimeInfo, 180), + project_details: window.projectDetails.map((project) => ({ + key: project.key, + name: project.name, + status: project.status, + summary: truncateForPrompt(project.summary, 180), + latest_progress: truncateForPrompt(project.latestProgress, 140), + confidence: project.confidence, + })), + facts: window.facts + .map((fact) => ({ + key: fact.factKey, + value: truncateForPrompt(fact.factValue, 140), + confidence: fact.confidence, + })) + .slice(0, 10), + })), + suspicious_l0_previews: input.l0Sessions.map((session) => ({ + id: session.l0IndexId, + session_key: session.sessionKey, + timestamp: session.timestamp, + preview_messages: session.messages.slice(-4).map((message) => ({ + role: message.role, + content: truncateForPrompt(message.content, 180), + })), + })), + time_layer_integrity_notes: input.timeLayerNotes.map((note) => ({ + title: note.title, + rationale: truncateForPrompt(note.rationale, 220), + confidence: note.confidence, + evidence_refs: note.evidenceRefs, + })), + evidence_refs: input.evidenceRefs.map((ref) => ({ + ref_id: ref.refId, + level: ref.level, + id: ref.id, + label: truncateForPrompt(ref.label, 120), + summary: truncateForPrompt(ref.summary, 220), + })), + }, + null, + 2, + ); +} + +function buildDreamProjectRebuildPrompt(input: LlmDreamProjectRebuildInput): string { + return JSON.stringify( + { + governance_scope: { + mode: "manual_dream_rebuild", + primary_truth: "all_l1_windows", + writable_targets: ["l2_project"], + read_only_targets: ["l1", "l2_time"], + profile_context_included: Boolean(input.profile), + }, + current_profile: input.profile + ? { + id: input.profile.recordId, + text: truncateForPrompt(input.profile.profileText, 360), + source_l1_ids: input.profile.sourceL1Ids.slice(-16), + } + : null, + current_l2_projects: input.currentProjects.map((project) => ({ + id: project.l2IndexId, + project_key: project.projectKey, + project_name: project.projectName, + current_status: project.currentStatus, + summary: truncateForPrompt(project.summary, 260), + latest_progress: truncateForPrompt(project.latestProgress, 180), + l1_source: project.l1Source.slice(-12), + updated_at: project.updatedAt, + })), + l1_windows: input.l1Windows.map((window) => ({ + id: window.l1IndexId, + time_period: window.timePeriod, + started_at: window.startedAt, + ended_at: window.endedAt, + summary: truncateForPrompt(window.summary, 220), + situation_time_info: truncateForPrompt(window.situationTimeInfo, 180), + facts: window.facts + .map((fact) => ({ + key: fact.factKey, + value: truncateForPrompt(fact.factValue, 120), + confidence: fact.confidence, + })) + .slice(0, 8), + project_details: window.projectDetails + .map((project) => ({ + key: project.key, + name: project.name, + status: project.status, + summary: truncateForPrompt(project.summary, 160), + latest_progress: truncateForPrompt(project.latestProgress, 120), + confidence: project.confidence, + })) + .slice(0, 6), + })), + suspicious_l0_previews: input.l0Sessions.map((session) => ({ + id: session.l0IndexId, + session_key: session.sessionKey, + timestamp: session.timestamp, + preview_messages: session.messages.slice(-4).map((message) => ({ + role: message.role, + content: truncateForPrompt(message.content, 180), + })), + })), + local_clusters: input.clusters.map((cluster) => ({ + cluster_id: cluster.clusterId, + label: truncateForPrompt(cluster.label, 120), + candidate_keys: cluster.candidateKeys.slice(0, 8), + candidate_names: cluster.candidateNames + .map((value) => truncateForPrompt(value, 80)) + .slice(0, 8), + current_project_keys: cluster.currentProjectKeys.slice(0, 8), + l1_ids: cluster.l1Ids.slice(0, 16), + statuses: cluster.statuses, + summaries: cluster.summaries.map((value) => truncateForPrompt(value, 140)).slice(0, 6), + latest_progresses: cluster.latestProgresses + .map((value) => truncateForPrompt(value, 120)) + .slice(0, 6), + issue_hints: cluster.issueHints, + representative_windows: cluster.representativeWindows.slice(0, 4).map((window) => ({ + l1_index_id: window.l1IndexId, + ended_at: window.endedAt, + summary: truncateForPrompt(window.summary, 140), + })), + })), + }, + null, + 2, + ); +} + +function buildDreamGlobalProfileRewritePrompt(input: LlmDreamGlobalProfileRewriteInput): string { + return JSON.stringify( + { + governance_scope: { + mode: "manual_dream_profile_rewrite", + stable_only: true, + exact_source_pruning: true, + }, + existing_profile: input.existingProfile + ? { + id: input.existingProfile.recordId, + text: truncateForPrompt(input.existingProfile.profileText, 420), + source_l1_ids: input.existingProfile.sourceL1Ids.slice(-20), + } + : null, + planned_projects: input.plannedProjects.map((project) => ({ + project_key: project.projectKey, + project_name: project.projectName, + current_status: project.currentStatus, + summary: truncateForPrompt(project.summary, 200), + latest_progress: truncateForPrompt(project.latestProgress, 140), + retained_l1_ids: project.retainedL1Ids.slice(0, 16), + })), + current_projects: input.currentProjects.map((project) => ({ + project_key: project.projectKey, + project_name: project.projectName, + current_status: project.currentStatus, + summary: truncateForPrompt(project.summary, 180), + l1_source: project.l1Source.slice(-12), + })), + l1_windows: input.l1Windows.map((window) => ({ + id: window.l1IndexId, + ended_at: window.endedAt, + summary: truncateForPrompt(window.summary, 220), + situation_time_info: truncateForPrompt(window.situationTimeInfo, 160), + facts: window.facts + .map((fact) => ({ + key: fact.factKey, + value: truncateForPrompt(fact.factValue, 140), + confidence: fact.confidence, + })) + .slice(0, 12), + project_details: window.projectDetails + .map((project) => ({ + key: project.key, + name: project.name, + status: project.status, + })) + .slice(0, 6), + })), + l1_issues: input.l1Issues.map((issue) => ({ + issue_type: issue.issueType, + title: truncateForPrompt(issue.title, 120), + l1_ids: issue.l1Ids.slice(0, 12), + related_project_keys: issue.relatedProjectKeys.slice(0, 8), + })), + }, + null, + 2, + ); +} + +function buildHop1RoutePrompt(input: LlmMemoryRouteInput): string { + const currentLocalDate = new Date().toLocaleDateString("en-CA"); + return JSON.stringify( + { + query: input.query, + current_local_date: currentLocalDate, + global_profile: input.profile + ? { + id: input.profile.recordId, + text: truncateForPrompt(input.profile.profileText, 140), + } + : null, + recent_messages: input.recentMessages.map((message) => ({ + role: message.role, + content: truncateForPrompt(message.content, 160), + })), + }, + null, + 2, + ); +} + +function buildHop2L2Prompt(input: LlmHop2L2Input): string { + return JSON.stringify( + { + query: input.query, + global_profile: input.profile + ? { + id: input.profile.recordId, + text: truncateForPrompt(input.profile.profileText, 220), + } + : null, + lookup_queries: input.lookupQueries.map((entry) => ({ + target_types: entry.targetTypes, + lookup_query: truncateForPrompt(entry.lookupQuery, 120), + time_range: entry.timeRange + ? { + start_date: entry.timeRange.startDate, + end_date: entry.timeRange.endDate, + } + : null, + })), + catalog_truncated: Boolean(input.catalogTruncated), + l2_entries: input.l2Entries.map((item) => ({ + id: item.id, + type: item.type, + label: item.label, + lookup_keys: item.lookupKeys.map((value) => truncateForPrompt(value, 80)).slice(0, 6), + compressed_content: truncateForPrompt(item.compressedContent, 140), + })), + }, + null, + 2, + ); +} + +function buildHop3L1Prompt(input: LlmHop3L1Input): string { + return JSON.stringify( + { + query: input.query, + current_evidence_note: truncateForPrompt(input.evidenceNote, 320), + selected_l2_entries: input.selectedL2Entries.map((item) => ({ + id: item.id, + type: item.type, + label: item.label, + lookup_keys: item.lookupKeys.map((value) => truncateForPrompt(value, 80)).slice(0, 6), + compressed_content: truncateForPrompt(item.compressedContent, 220), + })), + l1_windows: input.l1Windows.map((item) => ({ + id: item.l1IndexId, + session_key: item.sessionKey, + time_period: item.timePeriod, + summary: truncateForPrompt(item.summary, 180), + situation: truncateForPrompt(item.situationTimeInfo, 160), + projects: item.projectDetails.map((project) => project.name).slice(0, 6), + })), + }, + null, + 2, + ); +} + +function buildHop4L0Prompt(input: LlmHop4L0Input): string { + return JSON.stringify( + { + query: input.query, + current_evidence_note: truncateForPrompt(input.evidenceNote, 360), + selected_l2_entries: input.selectedL2Entries.map((item) => ({ + id: item.id, + type: item.type, + label: item.label, + lookup_keys: item.lookupKeys.map((value) => truncateForPrompt(value, 80)).slice(0, 6), + compressed_content: truncateForPrompt(item.compressedContent, 220), + })), + selected_l1_windows: input.selectedL1Windows.map((item) => ({ + id: item.l1IndexId, + session_key: item.sessionKey, + time_period: item.timePeriod, + summary: truncateForPrompt(item.summary, 180), + situation: truncateForPrompt(item.situationTimeInfo, 160), + projects: item.projectDetails.map((project) => project.name).slice(0, 6), + })), + l0_sessions: input.l0Sessions.map((item) => ({ + id: item.l0IndexId, + session_key: item.sessionKey, + timestamp: item.timestamp, + messages: item.messages.slice(-8).map((message) => ({ + role: message.role, + content: truncateForPrompt(message.content, 220), + })), + })), + }, + null, + 2, + ); +} + +function extractFirstJsonObject(raw: string): string { + const trimmed = raw.trim(); + if (!trimmed) throw new Error("Empty extraction response"); + if (trimmed.startsWith("{") && trimmed.endsWith("}")) return trimmed; + + const start = trimmed.indexOf("{"); + if (start < 0) throw new Error("No JSON object found in extraction response"); + + let depth = 0; + let inString = false; + let escaped = false; + for (let index = start; index < trimmed.length; index += 1) { + const char = trimmed[index]!; + if (inString) { + if (escaped) { + escaped = false; + } else if (char === "\\") { + escaped = true; + } else if (char === '"') { + inString = false; + } + continue; + } + + if (char === '"') { + inString = true; + continue; + } + if (char === "{") depth += 1; + if (char === "}") { + depth -= 1; + if (depth === 0) return trimmed.slice(start, index + 1); + } + } + + throw new Error("Incomplete JSON object in extraction response"); +} + +function slugifyKeyPart(value: string): string { + const normalized = value + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, ""); + return normalized || "item"; +} + +function clampConfidence(value: unknown, fallback: number): number { + if (typeof value !== "number" || Number.isNaN(value)) return fallback; + return Math.max(0, Math.min(1, value)); +} + +function normalizeDreamTarget( + value: unknown, + fallback: DreamReviewFinding["target"], +): DreamReviewFinding["target"] { + if ( + value === "l2_project" || + value === "global_profile" || + value === "l1_only" || + value === "time_note" + ) { + return value; + } + return fallback; +} + +function normalizeDreamEvidenceRefs(items: unknown, allowedRefs: ReadonlySet): string[] { + if (!Array.isArray(items)) return []; + return Array.from( + new Set( + items + .filter((item): item is string => typeof item === "string") + .map((item) => item.trim()) + .filter((item) => item && allowedRefs.has(item)), + ), + ).slice(0, 8); +} + +function normalizeDreamFinding( + item: unknown, + allowedRefs: ReadonlySet, + fallbackTarget: DreamReviewFinding["target"], +): DreamReviewFinding | null { + if (!isRecord(item)) return null; + const title = typeof item.title === "string" ? normalizeWhitespace(item.title) : ""; + const rationale = typeof item.rationale === "string" ? normalizeWhitespace(item.rationale) : ""; + if (!title || !rationale) return null; + return { + title: truncate(title, 120), + rationale: truncate(rationale, 320), + confidence: clampConfidence(item.confidence, 0.65), + target: normalizeDreamTarget(item.target, fallbackTarget), + evidenceRefs: normalizeDreamEvidenceRefs(item.evidence_refs, allowedRefs), + }; +} + +function normalizeDreamFindings( + items: unknown, + allowedRefs: ReadonlySet, + fallbackTarget: DreamReviewFinding["target"], +): DreamReviewFinding[] { + if (!Array.isArray(items)) return []; + const findings: DreamReviewFinding[] = []; + const seen = new Set(); + for (const item of items) { + const normalized = normalizeDreamFinding(item, allowedRefs, fallbackTarget); + if (!normalized) continue; + const dedupeKey = `${normalized.target}:${normalized.title}:${normalized.rationale}`; + if (seen.has(dedupeKey)) continue; + seen.add(dedupeKey); + findings.push(normalized); + if (findings.length >= 8) break; + } + return findings; +} + +function passesDreamProfileGate( + finding: DreamReviewFinding, + evidenceRefs: ReadonlyMap, +): boolean { + const l1Count = finding.evidenceRefs.reduce((count, refId) => { + const ref = evidenceRefs.get(refId); + return ref?.level === "l1" ? count + 1 : count; + }, 0); + if (l1Count >= 2) return true; + const hasProfileConflictContext = finding.evidenceRefs.some( + (refId) => evidenceRefs.get(refId)?.level === "profile", + ); + return hasProfileConflictContext && l1Count >= 1; +} + +function normalizeDreamProjectKey(value: unknown, fallback: string): string { + const candidate = typeof value === "string" ? normalizeWhitespace(value) : ""; + if (!candidate) return fallback; + return slugifyKeyPart(candidate); +} + +function normalizeDreamProjectName(value: unknown, fallback: string): string { + const candidate = typeof value === "string" ? normalizeWhitespace(value) : ""; + return truncate(candidate || fallback || "Dream Project", 120); +} + +function normalizeDreamL1Ids(items: unknown, allowedIds: ReadonlySet): string[] { + if (!Array.isArray(items)) return []; + return Array.from( + new Set( + items + .filter((item): item is string => typeof item === "string") + .map((item) => item.trim()) + .filter((item) => item && allowedIds.has(item)), + ), + ).slice(0, 32); +} + +function normalizeDreamProjectKeys(items: unknown): string[] { + if (!Array.isArray(items)) return []; + return Array.from( + new Set( + items + .filter((item): item is string => typeof item === "string") + .map((item) => slugifyKeyPart(item)) + .filter(Boolean), + ), + ).slice(0, 24); +} + +function normalizeDreamL1IssueType(value: unknown): LlmDreamL1Issue["issueType"] { + if (value === "duplicate" || value === "conflict" || value === "isolated") return value; + return "isolated"; +} + +function normalizeDreamProjectPlanItem( + item: unknown, + allowedL1Ids: ReadonlySet, +): LlmDreamProjectRebuildOutput["projects"][number] | null { + if (!isRecord(item)) return null; + const retainedL1Ids = normalizeDreamL1Ids(item.retained_l1_ids, allowedL1Ids); + if (retainedL1Ids.length === 0) return null; + const projectKey = normalizeDreamProjectKey( + item.project_key, + retainedL1Ids[0] ?? "dream-project", + ); + const projectName = normalizeDreamProjectName(item.project_name, projectKey); + const summary = + typeof item.summary === "string" ? truncate(normalizeWhitespace(item.summary), 320) : ""; + const latestProgress = + typeof item.latest_progress === "string" + ? truncate(normalizeWhitespace(item.latest_progress), 220) + : ""; + return { + projectKey, + projectName, + currentStatus: normalizeStatus(item.current_status), + summary: summary || projectName, + latestProgress: latestProgress || summary || projectName, + retainedL1Ids, + }; +} + +function normalizeDreamL1Issue( + item: unknown, + allowedL1Ids: ReadonlySet, +): LlmDreamL1Issue | null { + if (!isRecord(item)) return null; + const l1Ids = normalizeDreamL1Ids(item.l1_ids, allowedL1Ids); + if (l1Ids.length === 0) return null; + const title = + typeof item.title === "string" ? truncate(normalizeWhitespace(item.title), 160) : ""; + return { + issueType: normalizeDreamL1IssueType(item.issue_type), + title: title || `Dream issue for ${l1Ids[0]}`, + l1Ids, + relatedProjectKeys: normalizeDreamProjectKeys(item.related_project_keys), + }; +} + +function normalizeStatus(value: unknown): ProjectStatus { + if (typeof value !== "string") return "planned"; + const normalized = value.trim().toLowerCase(); + if (normalized === "planned") return "planned"; + if (normalized === "in_progress" || normalized === "in progress") return "in_progress"; + if (normalized === "blocked") return "in_progress"; + if (normalized === "on_hold" || normalized === "on hold") return "in_progress"; + if (normalized === "unknown") return "planned"; + if (normalized === "done" || normalized === "completed" || normalized === "complete") + return "done"; + return "planned"; +} + +function buildFallbackSituationTimeInfo(timestamp: string, summary: string): string { + const date = new Date(timestamp); + if (Number.isNaN(date.getTime())) return summary; + const yyyyMmDd = date.toISOString().slice(0, 10); + const hour = String(date.getHours()).padStart(2, "0"); + const minute = String(date.getMinutes()).padStart(2, "0"); + return `${yyyyMmDd} ${hour}:${minute} ${summary}`.trim(); +} + +function normalizeFacts(items: unknown): FactCandidate[] { + if (!Array.isArray(items)) return []; + const facts = new Map(); + + for (const item of items) { + const raw = item as RawFactItem; + const category = typeof raw.category === "string" ? slugifyKeyPart(raw.category) : "context"; + const subject = + typeof raw.subject === "string" && raw.subject.trim() + ? slugifyKeyPart(raw.subject) + : slugifyKeyPart(typeof raw.value === "string" ? raw.value : "item"); + const value = typeof raw.value === "string" ? normalizeWhitespace(raw.value) : ""; + if (!value) continue; + const factKey = `${category}:${subject}`; + facts.set(factKey, { + factKey, + factValue: truncate(value, 180), + confidence: clampConfidence(raw.confidence, 0.65), + }); + } + + return Array.from(facts.values()).slice(0, 12); +} + +function normalizeProjectDetails(items: unknown): ProjectDetail[] { + if (!Array.isArray(items)) return []; + const projects = new Map(); + + for (const item of items) { + const raw = item as RawProjectItem; + const key = typeof raw.key === "string" && raw.key.trim() ? slugifyKeyPart(raw.key) : ""; + const name = typeof raw.name === "string" ? normalizeWhitespace(raw.name) : ""; + if (!name) continue; + const stableKey = key || slugifyKeyPart(name); + if (projects.has(stableKey)) continue; + projects.set(stableKey, { + key: stableKey, + name: truncate(name, 80), + status: normalizeStatus(raw.status), + summary: truncate( + typeof raw.summary === "string" ? normalizeWhitespace(raw.summary) : "", + 360, + ), + latestProgress: truncate( + typeof raw.latest_progress === "string" ? normalizeWhitespace(raw.latest_progress) : "", + 220, + ), + confidence: clampConfidence(raw.confidence, 0.7), + }); + } + + return Array.from(projects.values()).slice(0, 8); +} + +function truncateForPrompt(value: string, maxLength: number): string { + return truncate(normalizeWhitespace(value), maxLength); +} + +function normalizeDateKey(value: unknown): string | null { + if (typeof value !== "string") return null; + const trimmed = value.trim(); + return /^\d{4}-\d{2}-\d{2}$/.test(trimmed) ? trimmed : null; +} + +function normalizeTimeRange(value: unknown): { startDate: string; endDate: string } | null { + if (!isRecord(value)) return null; + const startDate = normalizeDateKey(value.start_date); + const endDate = normalizeDateKey(value.end_date); + if (!startDate || !endDate) return null; + return startDate <= endDate ? { startDate, endDate } : { startDate: endDate, endDate: startDate }; +} + +function normalizeStringArray(items: unknown, maxItems: number): string[] { + if (!Array.isArray(items)) return []; + return items + .filter((item): item is string => typeof item === "string") + .map((item) => item.trim()) + .filter(Boolean) + .slice(0, maxItems); +} + +function normalizeIntent(value: unknown): IntentType { + if (value === "time" || value === "project" || value === "fact" || value === "general") + return value; + return "general"; +} + +function normalizeEnoughAt(value: unknown): RetrievalResult["enoughAt"] { + if (value === "l2" || value === "l1" || value === "l0" || value === "none") return value; + return "none"; +} + +function normalizeBoolean(value: unknown, fallback = false): boolean { + if (typeof value === "boolean") return value; + if (typeof value === "string") { + const normalized = value.trim().toLowerCase(); + if (normalized === "true") return true; + if (normalized === "false") return false; + } + return fallback; +} + +function normalizeQueryScope(value: unknown): "standalone" | "continuation" { + return value === "continuation" ? "continuation" : "standalone"; +} + +function normalizeEffectiveQuery(value: unknown, fallback: string): string { + if (typeof value === "string") { + const normalized = truncateForPrompt(value, 180); + if (normalized) return normalized; + } + return fallback; +} + +function normalizeLookupTargetTypes(value: unknown): LookupTargetType[] { + if (!Array.isArray(value)) return []; + return uniqueById( + value + .filter((item): item is string => typeof item === "string") + .map((item) => item.trim()) + .filter((item): item is LookupTargetType => item === "time" || item === "project"), + (item) => item, + ); +} + +function normalizeLookupQueries( + value: unknown, + defaultQuery: string, + maxItems = 4, +): LookupQuerySpec[] { + if (!Array.isArray(value)) { + return [ + { + targetTypes: ["time", "project"], + lookupQuery: defaultQuery, + timeRange: null, + }, + ]; + } + const normalized = value + .filter(isRecord) + .map((item): LookupQuerySpec | undefined => { + const targetTypes = normalizeLookupTargetTypes(item.target_types); + const lookupQuery = + typeof item.lookup_query === "string" ? truncateForPrompt(item.lookup_query, 120) : ""; + if (targetTypes.length === 0 || !lookupQuery) return undefined; + return { + targetTypes, + lookupQuery, + timeRange: normalizeTimeRange(item.time_range), + }; + }) + .filter((item): item is LookupQuerySpec => Boolean(item)); + if (normalized.length > 0) return normalized.slice(0, maxItems); + return [ + { + targetTypes: ["time", "project"], + lookupQuery: defaultQuery, + timeRange: null, + }, + ]; +} + +function uniqueById(items: T[], getId: (item: T) => string): T[] { + const seen = new Set(); + const next: T[] = []; + for (const item of items) { + const id = getId(item); + if (!id || seen.has(id)) continue; + seen.add(id); + next.push(item); + } + return next; +} + +function fallbackEvidenceNote(lines: string[], fallback = ""): string { + const normalized = lines + .map((line) => normalizeWhitespace(line)) + .filter(Boolean) + .slice(0, 8); + const joined = normalized.join("\n"); + return truncate(joined || normalizeWhitespace(fallback), 800); +} + +function extractChatCompletionsText(payload: unknown): string { + if (!isRecord(payload) || !Array.isArray(payload.choices)) { + throw new Error("Invalid chat completions payload"); + } + const firstChoice = payload.choices[0]; + if (!isRecord(firstChoice) || !isRecord(firstChoice.message)) { + throw new Error("Missing chat completion message"); + } + const content = firstChoice.message.content; + if (typeof content === "string") return content; + if (Array.isArray(content)) { + return content + .map((item) => (isRecord(item) && typeof item.text === "string" ? item.text : "")) + .filter(Boolean) + .join("\n"); + } + throw new Error("Unsupported chat completion content shape"); +} + +function extractResponsesText(payload: unknown): string { + if (!isRecord(payload)) throw new Error("Invalid responses payload"); + if (typeof payload.output_text === "string" && payload.output_text.trim()) + return payload.output_text; + if (!Array.isArray(payload.output)) throw new Error("Responses payload missing output"); + + const chunks: string[] = []; + for (const item of payload.output) { + if (!isRecord(item) || !Array.isArray(item.content)) continue; + for (const part of item.content) { + if (isRecord(part) && typeof part.text === "string") chunks.push(part.text); + } + } + const text = chunks.join("\n").trim(); + if (!text) throw new Error("Responses payload did not contain text"); + return text; +} + +function looksLikeEnvVarName(value: string): boolean { + return /^[A-Z0-9_]+$/.test(value); +} + +export class LlmMemoryExtractor { + constructor( + private readonly config: Record, + private readonly runtime: Record | undefined, + private readonly logger?: LoggerLike, + ) {} + + private resolveSelection(agentId?: string): ModelSelection { + const modelRef = resolveAgentPrimaryModel(this.config, agentId); + const parsed = parseModelRef(modelRef, this.config); + if (!parsed) throw new Error("Could not resolve an OpenClaw model for memory extraction"); + + const modelsConfig = isRecord(this.config.models) ? this.config.models : undefined; + const providers = + modelsConfig && isRecord(modelsConfig.providers) ? modelsConfig.providers : undefined; + const providerConfig = + providers && isRecord(providers[parsed.provider]) + ? (providers[parsed.provider] as Record) + : undefined; + const configuredModel = Array.isArray(providerConfig?.models) + ? providerConfig.models.find((item) => isRecord(item) && item.id === parsed.model) + : undefined; + const modelConfig = isRecord(configuredModel) ? configuredModel : undefined; + + const api = + typeof modelConfig?.api === "string" + ? modelConfig.api + : typeof providerConfig?.api === "string" + ? providerConfig.api + : "openai-completions"; + const baseUrl = + typeof modelConfig?.baseUrl === "string" + ? modelConfig.baseUrl + : typeof providerConfig?.baseUrl === "string" + ? providerConfig.baseUrl + : undefined; + const headers = { + ...sanitizeHeaders(providerConfig?.headers), + ...sanitizeHeaders(modelConfig?.headers), + }; + + const selection: ModelSelection = { + provider: parsed.provider, + model: parsed.model, + api, + }; + if (baseUrl?.trim()) selection.baseUrl = stripTrailingSlash(baseUrl.trim()); + if (Object.keys(headers).length > 0) selection.headers = headers; + return selection; + } + + private async resolveApiKey(provider: string): Promise { + const modelsConfig = isRecord(this.config.models) ? this.config.models : undefined; + const providers = + modelsConfig && isRecord(modelsConfig.providers) ? modelsConfig.providers : undefined; + const providerConfig = + providers && isRecord(providers[provider]) + ? (providers[provider] as Record) + : undefined; + const configured = + typeof providerConfig?.apiKey === "string" ? providerConfig.apiKey.trim() : ""; + if (configured) { + if ( + looksLikeEnvVarName(configured) && + typeof process.env[configured] === "string" && + process.env[configured]?.trim() + ) { + return process.env[configured]!.trim(); + } + return configured; + } + + const modelAuth = + this.runtime && isRecord(this.runtime.modelAuth) + ? (this.runtime.modelAuth as Record) + : undefined; + const resolver = + typeof modelAuth?.resolveApiKeyForProvider === "function" + ? (modelAuth.resolveApiKeyForProvider as (params: { + provider: string; + cfg?: Record; + }) => Promise<{ apiKey?: string }>) + : undefined; + if (resolver) { + const auth = await resolver({ provider, cfg: this.config }); + if (auth?.apiKey && String(auth.apiKey).trim()) { + return String(auth.apiKey).trim(); + } + } + + throw new Error(`No API key resolved for extraction provider "${provider}"`); + } + + private async callStructuredJson(input: { + systemPrompt: string; + userPrompt: string; + agentId?: string; + requestLabel: string; + timeoutMs?: number; + }): Promise { + const selection = this.resolveSelection(input.agentId); + if (!selection.baseUrl) { + throw new Error( + `${input.requestLabel} provider "${selection.provider}" does not have a baseUrl`, + ); + } + const apiKey = await this.resolveApiKey(selection.provider); + const headers = new Headers(selection.headers); + if (!headers.has("content-type")) headers.set("content-type", "application/json"); + if (!headers.has("authorization")) headers.set("authorization", `Bearer ${apiKey}`); + const apiType = selection.api.trim().toLowerCase(); + let url = ""; + let body: Record; + + if (apiType === "openai-responses" || apiType === "responses") { + url = `${selection.baseUrl}/responses`; + body = { + model: selection.model, + temperature: 0, + input: [ + { role: "system", content: input.systemPrompt }, + { role: "user", content: input.userPrompt }, + ], + }; + } else { + url = `${selection.baseUrl}/chat/completions`; + body = { + model: selection.model, + temperature: 0, + stream: false, + response_format: { type: "json_object" }, + messages: [ + { role: "system", content: input.systemPrompt }, + { role: "user", content: input.userPrompt }, + ], + }; + } + + const execute = async (payloadBody: Record): Promise => { + const controller = new AbortController(); + const timeoutMs = resolveRequestTimeoutMs(input.timeoutMs); + const timeoutId = timeoutMs === null ? null : setTimeout(() => controller.abort(), timeoutMs); + try { + return await fetch(url, { + method: "POST", + headers, + body: JSON.stringify(payloadBody), + signal: controller.signal, + }); + } catch (error) { + if (timeoutMs !== null && error instanceof Error && error.name === "AbortError") { + throw new Error(`${input.requestLabel} request timed out after ${timeoutMs}ms`); + } + throw error; + } finally { + if (timeoutId) clearTimeout(timeoutId); + } + }; + + let response = await execute(body); + if (!response.ok && "response_format" in body) { + const fallbackBody = { ...body }; + delete fallbackBody.response_format; + response = await execute(fallbackBody); + } + if (!response.ok) { + const errorText = await response.text(); + throw new Error( + `${input.requestLabel} request failed (${response.status}): ${truncate(errorText, 300)}`, + ); + } + + const payload = await response.json(); + return apiType === "openai-responses" || apiType === "responses" + ? extractResponsesText(payload) + : extractChatCompletionsText(payload); + } + + private async callStructuredJsonWithDebug(input: { + systemPrompt: string; + userPrompt: string; + agentId?: string; + requestLabel: string; + timeoutMs?: number; + debugTrace?: PromptDebugSink; + parse: (raw: string) => T; + }): Promise { + let rawResponse = ""; + try { + rawResponse = await this.callStructuredJson(input); + const parsedResult = input.parse(rawResponse); + input.debugTrace?.({ + requestLabel: input.requestLabel, + systemPrompt: input.systemPrompt, + userPrompt: input.userPrompt, + rawResponse, + parsedResult, + }); + return parsedResult; + } catch (error) { + input.debugTrace?.({ + requestLabel: input.requestLabel, + systemPrompt: input.systemPrompt, + userPrompt: input.userPrompt, + rawResponse, + errored: true, + timedOut: + isTimeoutError(error) || (error instanceof Error && /timed out/i.test(error.message)), + errorMessage: error instanceof Error ? error.message : String(error), + }); + throw error; + } + } + + async extract(input: { + timestamp: string; + messages: MemoryMessage[]; + agentId?: string; + }): Promise { + let parsed: RawExtractionPayload | undefined; + let lastError: unknown; + for (const extraInstruction of [ + undefined, + "Return one complete JSON object only. Do not use ellipses, placeholders, comments, markdown fences, or trailing commas.", + ]) { + try { + const rawText = await this.callStructuredJson({ + systemPrompt: EXTRACTION_SYSTEM_PROMPT, + userPrompt: buildPrompt(input.timestamp, input.messages, extraInstruction), + requestLabel: "Extraction", + timeoutMs: 20_000, + ...(input.agentId ? { agentId: input.agentId } : {}), + }); + parsed = JSON.parse(extractFirstJsonObject(rawText)) as RawExtractionPayload; + break; + } catch (error) { + lastError = error; + } + } + if (!parsed) throw lastError; + const summary = truncate( + typeof parsed.summary === "string" ? normalizeWhitespace(parsed.summary) : "", + 280, + ); + if (!summary) { + throw new Error("Extraction payload did not include a usable summary"); + } + + let projectDetails = normalizeProjectDetails(parsed.projects); + const facts = normalizeFacts(parsed.facts); + projectDetails = await this.completeProjectDetails({ + timestamp: input.timestamp, + messages: input.messages, + summary, + facts, + projectDetails, + ...(input.agentId ? { agentId: input.agentId } : {}), + }); + const situationTimeInfoRaw = + typeof parsed.situation_time_info === "string" + ? normalizeWhitespace(parsed.situation_time_info) + : ""; + const situationTimeInfo = truncate( + situationTimeInfoRaw || buildFallbackSituationTimeInfo(input.timestamp, summary), + 220, + ); + + this.logger?.info?.( + `[clawxmemory] llm extraction complete summary=${summary.slice(0, 60)} projects=${projectDetails.length} facts=${facts.length}`, + ); + + return { + summary, + situationTimeInfo, + facts, + projectDetails, + }; + } + + private async completeProjectDetails(input: { + timestamp: string; + messages: MemoryMessage[]; + summary: string; + facts: FactCandidate[]; + projectDetails: ProjectDetail[]; + agentId?: string; + }): Promise { + try { + const raw = await this.callStructuredJson({ + systemPrompt: PROJECT_COMPLETION_SYSTEM_PROMPT, + userPrompt: buildProjectCompletionPrompt(input), + requestLabel: "Project completion", + timeoutMs: 20_000, + ...(input.agentId ? { agentId: input.agentId } : {}), + }); + const parsed = JSON.parse(extractFirstJsonObject(raw)) as RawExtractionPayload; + const completed = normalizeProjectDetails(parsed.projects); + return completed.length > 0 ? completed : input.projectDetails; + } catch (error) { + this.logger?.warn?.(`[clawxmemory] project completion fallback: ${String(error)}`); + return input.projectDetails; + } + } + + async judgeTopicShift(input: LlmTopicShiftInput): Promise { + const fallbackSummary = truncate( + normalizeWhitespace( + input.currentTopicSummary || + input.incomingUserTurns[input.incomingUserTurns.length - 1] || + input.recentUserTurns[input.recentUserTurns.length - 1] || + "current topic", + ), + 120, + ); + if (input.incomingUserTurns.length === 0) { + return { topicChanged: false, topicSummary: fallbackSummary }; + } + if (!input.currentTopicSummary.trim() && input.recentUserTurns.length === 0) { + return { + topicChanged: false, + topicSummary: + truncate( + input.incomingUserTurns.map((item) => normalizeWhitespace(item)).join(" / "), + 120, + ) || fallbackSummary, + }; + } + + try { + const raw = await this.callStructuredJson({ + systemPrompt: TOPIC_BOUNDARY_SYSTEM_PROMPT, + userPrompt: buildTopicShiftPrompt(input), + requestLabel: "Topic shift", + timeoutMs: 8_000, + ...(input.agentId ? { agentId: input.agentId } : {}), + }); + const parsed = JSON.parse(extractFirstJsonObject(raw)) as RawTopicShiftPayload; + return { + topicChanged: normalizeBoolean(parsed.topic_changed, false), + topicSummary: truncate( + typeof parsed.topic_summary === "string" && parsed.topic_summary.trim() + ? normalizeWhitespace(parsed.topic_summary) + : fallbackSummary, + 120, + ), + }; + } catch (error) { + this.logger?.warn?.(`[clawxmemory] topic shift fallback: ${String(error)}`); + return { topicChanged: false, topicSummary: fallbackSummary }; + } + } + + async resolveProjectIdentity(input: LlmProjectResolutionInput): Promise { + if (input.existingProjects.length === 0) return input.project; + const candidates = input.existingProjects.slice(0, 24).map((project) => ({ + project_key: project.projectKey, + project_name: project.projectName, + summary: truncateForPrompt(project.summary, 160), + latest_progress: truncateForPrompt(project.latestProgress, 160), + status: project.currentStatus, + })); + try { + const raw = await this.callStructuredJson({ + systemPrompt: PROJECT_RESOLUTION_SYSTEM_PROMPT, + userPrompt: JSON.stringify( + { + incoming_project: { + key: input.project.key, + name: input.project.name, + summary: input.project.summary, + latest_progress: input.project.latestProgress, + status: input.project.status, + }, + existing_projects: candidates, + }, + null, + 2, + ), + requestLabel: "Project resolution", + timeoutMs: 15_000, + ...(input.agentId ? { agentId: input.agentId } : {}), + }); + const parsed = JSON.parse(extractFirstJsonObject(raw)) as RawProjectResolutionPayload; + const matchedProjectKey = + typeof parsed.matched_project_key === "string" ? parsed.matched_project_key.trim() : ""; + const matched = matchedProjectKey + ? input.existingProjects.find((project) => project.projectKey === matchedProjectKey) + : undefined; + return { + ...input.project, + key: + matched?.projectKey ?? + (typeof parsed.canonical_key === "string" && parsed.canonical_key.trim() + ? slugifyKeyPart(parsed.canonical_key) + : input.project.key), + name: + matched?.projectName ?? + (typeof parsed.canonical_name === "string" && parsed.canonical_name.trim() + ? truncateForPrompt(parsed.canonical_name, 80) + : input.project.name), + }; + } catch (error) { + this.logger?.warn?.( + `[clawxmemory] project resolution fallback for ${input.project.key}: ${String(error)}`, + ); + return input.project; + } + } + + async resolveProjectIdentities(input: LlmProjectBatchResolutionInput): Promise { + if (input.projects.length === 0 || input.existingProjects.length === 0) return input.projects; + const candidates = input.existingProjects.slice(0, 40).map((project) => ({ + project_key: project.projectKey, + project_name: project.projectName, + summary: truncateForPrompt(project.summary, 140), + latest_progress: truncateForPrompt(project.latestProgress, 140), + status: project.currentStatus, + })); + try { + const raw = await this.callStructuredJson({ + systemPrompt: PROJECT_BATCH_RESOLUTION_SYSTEM_PROMPT, + userPrompt: JSON.stringify( + { + incoming_projects: input.projects.map((project) => ({ + incoming_key: project.key, + key: project.key, + name: project.name, + summary: truncateForPrompt(project.summary, 160), + latest_progress: truncateForPrompt(project.latestProgress, 160), + status: project.status, + })), + existing_projects: candidates, + }, + null, + 2, + ), + requestLabel: "Project batch resolution", + timeoutMs: 15_000, + ...(input.agentId ? { agentId: input.agentId } : {}), + }); + const parsed = JSON.parse(extractFirstJsonObject(raw)) as RawProjectBatchResolutionPayload; + const resolutions = Array.isArray(parsed.projects) ? parsed.projects : []; + const byIncomingKey = new Map< + string, + { matched?: string; canonicalKey?: string; canonicalName?: string } + >(); + for (const item of resolutions) { + if (!isRecord(item) || typeof item.incoming_key !== "string") continue; + const normalized: { matched?: string; canonicalKey?: string; canonicalName?: string } = {}; + if (typeof item.matched_project_key === "string" && item.matched_project_key.trim()) { + normalized.matched = item.matched_project_key.trim(); + } + if (typeof item.canonical_key === "string" && item.canonical_key.trim()) { + normalized.canonicalKey = item.canonical_key.trim(); + } + if (typeof item.canonical_name === "string" && item.canonical_name.trim()) { + normalized.canonicalName = item.canonical_name.trim(); + } + byIncomingKey.set(item.incoming_key.trim(), normalized); + } + + return input.projects.map((project) => { + const resolution = byIncomingKey.get(project.key); + const matched = resolution?.matched + ? input.existingProjects.find((existing) => existing.projectKey === resolution.matched) + : undefined; + return { + ...project, + key: + matched?.projectKey ?? + (resolution?.canonicalKey ? slugifyKeyPart(resolution.canonicalKey) : project.key), + name: + matched?.projectName ?? + (resolution?.canonicalName + ? truncateForPrompt(resolution.canonicalName, 80) + : project.name), + }; + }); + } catch (error) { + this.logger?.warn?.(`[clawxmemory] project batch resolution fallback: ${String(error)}`); + return input.projects; + } + } + + async rewriteDailyTimeSummary(input: LlmDailyTimeSummaryInput): Promise { + try { + const raw = await this.callStructuredJson({ + systemPrompt: DAILY_TIME_SUMMARY_SYSTEM_PROMPT, + userPrompt: buildDailyTimeSummaryPrompt(input), + requestLabel: "Daily summary", + timeoutMs: 15_000, + ...(input.agentId ? { agentId: input.agentId } : {}), + }); + const parsed = JSON.parse(extractFirstJsonObject(raw)) as RawDailySummaryPayload; + const summary = typeof parsed.summary === "string" ? normalizeWhitespace(parsed.summary) : ""; + if (summary) return truncate(summary, 280); + } catch (error) { + this.logger?.warn?.(`[clawxmemory] daily summary fallback: ${String(error)}`); + } + return truncate(input.l1.situationTimeInfo || input.l1.summary || input.existingSummary, 280); + } + + async rewriteProjectMemories(input: LlmProjectMemoryRewriteInput): Promise { + if (input.projects.length === 0) return []; + + const fallbackProjects = input.projects.map((item) => item.incomingProject); + try { + const raw = await this.callStructuredJson({ + systemPrompt: PROJECT_MEMORY_REWRITE_SYSTEM_PROMPT, + userPrompt: buildProjectMemoryRewritePrompt(input), + requestLabel: "Project memory rewrite", + timeoutMs: 20_000, + ...(input.agentId ? { agentId: input.agentId } : {}), + }); + const parsed = JSON.parse(extractFirstJsonObject(raw)) as RawExtractionPayload; + const rewritten = normalizeProjectDetails(parsed.projects); + if (rewritten.length === 0) throw new Error("Project memory rewrite returned no projects"); + + const rewrittenByKey = new Map(rewritten.map((project) => [project.key, project])); + return fallbackProjects.map((project) => { + const next = rewrittenByKey.get(project.key); + if (!next) return project; + return { + ...project, + name: next.name || project.name, + status: next.status, + summary: next.summary || project.summary, + latestProgress: next.latestProgress || project.latestProgress, + confidence: Math.max(project.confidence, next.confidence), + }; + }); + } catch (error) { + this.logger?.warn?.(`[clawxmemory] project memory rewrite fallback: ${String(error)}`); + throw error; + } + } + + async rewriteGlobalProfile(input: LlmGlobalProfileInput): Promise { + try { + const raw = await this.callStructuredJson({ + systemPrompt: GLOBAL_PROFILE_SYSTEM_PROMPT, + userPrompt: buildGlobalProfilePrompt(input), + requestLabel: "Global profile", + timeoutMs: 15_000, + ...(input.agentId ? { agentId: input.agentId } : {}), + }); + const parsed = JSON.parse(extractFirstJsonObject(raw)) as RawProfilePayload; + const profileText = + typeof parsed.profile_text === "string" ? normalizeWhitespace(parsed.profile_text) : ""; + if (profileText) return truncate(profileText, 420); + } catch (error) { + this.logger?.warn?.(`[clawxmemory] global profile fallback: ${String(error)}`); + } + + const fallbackFacts = input.l1.facts + .map((fact) => fact.factValue) + .filter(Boolean) + .slice(0, 8) + .join("；"); + return truncate(input.existingProfile || fallbackFacts || input.l1.summary, 420); + } + + async reviewDream(input: LlmDreamReviewInput): Promise { + const emptySummary = + input.evidenceRefs.length === 0 + ? "Not enough indexed memory evidence to run Dream review yet." + : "No reliable Dream findings were produced from the selected evidence."; + const emptyResult = (): LlmDreamReviewResult => ({ + summary: emptySummary, + projectRebuild: [], + profileSuggestions: [], + cleanup: [], + ambiguous: [], + noAction: [], + }); + + if (input.evidenceRefs.length === 0) return emptyResult(); + + const allowedRefs = new Set(input.evidenceRefs.map((ref) => ref.refId)); + const evidenceRefsById = new Map(input.evidenceRefs.map((ref) => [ref.refId, ref] as const)); + try { + const raw = await this.callStructuredJson({ + systemPrompt: DREAM_REVIEW_SYSTEM_PROMPT, + userPrompt: buildDreamReviewPrompt(input), + requestLabel: "Dream review", + timeoutMs: 20_000, + ...(input.agentId ? { agentId: input.agentId } : {}), + }); + const parsed = JSON.parse(extractFirstJsonObject(raw)) as RawDreamReviewPayload; + const summary = + typeof parsed.summary === "string" + ? truncate(normalizeWhitespace(parsed.summary), 280) + : emptySummary; + const profileSuggestions = normalizeDreamFindings( + parsed.profile_suggestions, + allowedRefs, + "global_profile", + ).filter((finding) => passesDreamProfileGate(finding, evidenceRefsById)); + return { + summary: summary || emptySummary, + projectRebuild: normalizeDreamFindings(parsed.project_rebuild, allowedRefs, "l2_project"), + profileSuggestions, + cleanup: normalizeDreamFindings(parsed.cleanup, allowedRefs, "l1_only"), + ambiguous: normalizeDreamFindings(parsed.ambiguous, allowedRefs, "l1_only"), + noAction: normalizeDreamFindings(parsed.no_action, allowedRefs, "l1_only"), + }; + } catch (error) { + this.logger?.warn?.(`[clawxmemory] dream review fallback: ${String(error)}`); + return emptyResult(); + } + } + + async planDreamProjectRebuild( + input: LlmDreamProjectRebuildInput, + ): Promise { + if (input.l1Windows.length === 0) { + throw new Error("No L1 windows are available for Dream reconstruction."); + } + + const allowedL1Ids = new Set(input.l1Windows.map((window) => window.l1IndexId)); + const currentProjectKeys = new Set(input.currentProjects.map((project) => project.projectKey)); + const clusterProjectKeys = new Set( + input.clusters + .flatMap((cluster) => [...cluster.candidateKeys, ...cluster.currentProjectKeys]) + .filter(Boolean), + ); + const raw = await this.callStructuredJson({ + systemPrompt: DREAM_PROJECT_REBUILD_SYSTEM_PROMPT, + userPrompt: buildDreamProjectRebuildPrompt(input), + requestLabel: "Dream project rebuild", + timeoutMs: 30_000, + ...(input.agentId ? { agentId: input.agentId } : {}), + }); + const parsed = JSON.parse(extractFirstJsonObject(raw)) as RawDreamProjectPlanPayload; + const normalizedProjects = Array.isArray(parsed.projects) + ? parsed.projects + .map((item) => normalizeDreamProjectPlanItem(item, allowedL1Ids)) + .filter((item): item is LlmDreamProjectRebuildOutput["projects"][number] => Boolean(item)) + : []; + + const dedupedProjects: LlmDreamProjectRebuildOutput["projects"] = []; + const seenProjectKeys = new Set(); + for (const item of normalizedProjects) { + if (seenProjectKeys.has(item.projectKey)) continue; + seenProjectKeys.add(item.projectKey); + dedupedProjects.push(item); + if (dedupedProjects.length >= 20) break; + } + if (dedupedProjects.length === 0) { + throw new Error("Dream project rebuild returned no valid projects."); + } + + const deletedProjectKeys = Array.from( + new Set( + normalizeDreamProjectKeys(parsed.deleted_project_keys).filter( + (key) => currentProjectKeys.has(key) || clusterProjectKeys.has(key), + ), + ), + ); + + const l1Issues = Array.isArray(parsed.l1_issues) + ? parsed.l1_issues + .map((item) => normalizeDreamL1Issue(item, allowedL1Ids)) + .filter((item): item is LlmDreamL1Issue => Boolean(item)) + .slice(0, 20) + : []; + + return { + summary: + typeof parsed.summary === "string" + ? truncate(normalizeWhitespace(parsed.summary), 320) + : "Dream project rebuild completed.", + duplicateTopicCount: Math.max( + 0, + Math.floor( + typeof parsed.duplicate_topic_count === "number" + ? parsed.duplicate_topic_count + : l1Issues.filter((item) => item.issueType === "duplicate").length, + ), + ), + conflictTopicCount: Math.max( + 0, + Math.floor( + typeof parsed.conflict_topic_count === "number" + ? parsed.conflict_topic_count + : l1Issues.filter((item) => item.issueType === "conflict").length, + ), + ), + projects: dedupedProjects, + deletedProjectKeys, + l1Issues, + }; + } + + async rewriteDreamGlobalProfile( + input: LlmDreamGlobalProfileRewriteInput, + ): Promise { + if (input.l1Windows.length === 0) { + throw new Error("No L1 windows are available for Dream profile rewrite."); + } + + const allowedL1Ids = new Set(input.l1Windows.map((window) => window.l1IndexId)); + const raw = await this.callStructuredJson({ + systemPrompt: DREAM_GLOBAL_PROFILE_REWRITE_SYSTEM_PROMPT, + userPrompt: buildDreamGlobalProfileRewritePrompt(input), + requestLabel: "Dream global profile rewrite", + timeoutMs: 20_000, + ...(input.agentId ? { agentId: input.agentId } : {}), + }); + const parsed = JSON.parse(extractFirstJsonObject(raw)) as RawDreamGlobalProfileRewritePayload; + const profileText = + typeof parsed.profile_text === "string" + ? truncate(normalizeWhitespace(parsed.profile_text), 420) + : ""; + if (!profileText) { + throw new Error("Dream global profile rewrite returned an empty profile."); + } + const sourceL1Ids = normalizeDreamL1Ids(parsed.source_l1_ids, allowedL1Ids); + return { + profileText, + sourceL1Ids, + conflictWithExisting: normalizeBoolean(parsed.conflict_with_existing, false), + }; + } + + async decideMemoryLookup(input: LlmMemoryRouteInput): Promise { + const defaultQuery = truncateForPrompt(input.query, 120); + const systemPrompt = HOP1_LOOKUP_SYSTEM_PROMPT; + const userPrompt = buildHop1RoutePrompt(input); + try { + const parsed = await this.callStructuredJsonWithDebug({ + systemPrompt, + userPrompt, + requestLabel: "Hop1 lookup", + timeoutMs: input.timeoutMs ?? 4_000, + ...(input.agentId ? { agentId: input.agentId } : {}), + ...(input.debugTrace ? { debugTrace: input.debugTrace } : {}), + parse: (raw) => JSON.parse(extractFirstJsonObject(raw)) as RawHop1RoutePayload, + }); + const baseOnly = normalizeBoolean(parsed.base_only, false); + const queryScope = normalizeQueryScope(parsed.query_scope); + const effectiveQuery = normalizeEffectiveQuery(parsed.effective_query, defaultQuery); + return { + queryScope, + effectiveQuery, + memoryRelevant: normalizeBoolean(parsed.memory_relevant, true), + baseOnly, + lookupQueries: baseOnly + ? [] + : normalizeLookupQueries(parsed.lookup_queries, effectiveQuery), + }; + } catch (error) { + this.logger?.warn?.(`[clawxmemory] hop1 lookup fallback: ${String(error)}`); + return { + queryScope: "standalone", + effectiveQuery: defaultQuery, + memoryRelevant: true, + baseOnly: false, + lookupQueries: [ + { + targetTypes: ["time", "project"], + lookupQuery: defaultQuery, + timeRange: null, + }, + ], + }; + } + } + + private async runL2SelectionOnce(input: LlmHop2L2Input): Promise { + try { + const parsed = await this.callStructuredJsonWithDebug({ + systemPrompt: HOP2_L2_SYSTEM_PROMPT, + userPrompt: buildHop2L2Prompt(input), + requestLabel: "Hop2 L2 selection", + timeoutMs: input.timeoutMs ?? 5_000, + ...(input.agentId ? { agentId: input.agentId } : {}), + ...(input.debugTrace ? { debugTrace: input.debugTrace } : {}), + parse: (raw) => JSON.parse(extractFirstJsonObject(raw)) as RawHop2L2Payload, + }); + const enoughAt = + parsed.enough_at === "l2" || + parsed.enough_at === "descend_l1" || + parsed.enough_at === "none" + ? parsed.enough_at + : "none"; + return { + intent: normalizeIntent(parsed.intent), + evidenceNote: + typeof parsed.evidence_note === "string" + ? truncate(normalizeWhitespace(parsed.evidence_note), 800) + : "", + enoughAt, + }; + } catch (error) { + throw error; + } + } + + async selectL2FromCatalog(input: LlmHop2L2Input): Promise { + if (input.l2Entries.length === 0) { + return { + intent: input.profile ? "fact" : "general", + evidenceNote: "", + enoughAt: "none", + }; + } + try { + return await this.runL2SelectionOnce(input); + } catch (error) { + this.logger?.warn?.(`[clawxmemory] hop2 l2 fallback: ${String(error)}`); + const hasTime = input.l2Entries.some((entry) => entry.type === "time"); + const hasProject = input.l2Entries.some((entry) => entry.type === "project"); + const intent = + hasTime && hasProject + ? "general" + : hasTime + ? "time" + : hasProject + ? "project" + : input.profile + ? "fact" + : "general"; + return { + intent, + evidenceNote: fallbackEvidenceNote( + input.l2Entries.map((entry) => `${entry.label}: ${entry.compressedContent}`), + input.query, + ), + enoughAt: "none", + }; + } + } + + async selectL1FromEvidence(input: LlmHop3L1Input): Promise { + if (input.l1Windows.length === 0) { + return { + evidenceNote: input.evidenceNote, + enoughAt: "none", + }; + } + try { + const parsed = await this.callStructuredJsonWithDebug({ + systemPrompt: HOP3_L1_SYSTEM_PROMPT, + userPrompt: buildHop3L1Prompt(input), + requestLabel: "Hop3 L1 selection", + timeoutMs: input.timeoutMs ?? 5_000, + ...(input.agentId ? { agentId: input.agentId } : {}), + ...(input.debugTrace ? { debugTrace: input.debugTrace } : {}), + parse: (raw) => JSON.parse(extractFirstJsonObject(raw)) as RawHop3L1Payload, + }); + const enoughAt = + parsed.enough_at === "l1" || + parsed.enough_at === "descend_l0" || + parsed.enough_at === "none" + ? parsed.enough_at + : "none"; + return { + evidenceNote: + typeof parsed.evidence_note === "string" + ? truncate(normalizeWhitespace(parsed.evidence_note), 800) + : input.evidenceNote, + enoughAt, + }; + } catch (error) { + this.logger?.warn?.(`[clawxmemory] hop3 l1 fallback: ${String(error)}`); + return { + evidenceNote: fallbackEvidenceNote( + [ + input.evidenceNote, + ...input.l1Windows.map( + (item) => `${item.timePeriod}: ${item.summary} ${item.situationTimeInfo}`, + ), + ], + input.query, + ), + enoughAt: "none", + }; + } + } + + async selectL0FromEvidence(input: LlmHop4L0Input): Promise { + if (input.l0Sessions.length === 0) { + return { + evidenceNote: input.evidenceNote, + enoughAt: "none", + }; + } + try { + const parsed = await this.callStructuredJsonWithDebug({ + systemPrompt: HOP4_L0_SYSTEM_PROMPT, + userPrompt: buildHop4L0Prompt(input), + requestLabel: "Hop4 L0 selection", + timeoutMs: input.timeoutMs ?? 5_000, + ...(input.agentId ? { agentId: input.agentId } : {}), + ...(input.debugTrace ? { debugTrace: input.debugTrace } : {}), + parse: (raw) => JSON.parse(extractFirstJsonObject(raw)) as RawHop4L0Payload, + }); + const enoughAt = + parsed.enough_at === "l0" || parsed.enough_at === "none" ? parsed.enough_at : "none"; + return { + evidenceNote: + typeof parsed.evidence_note === "string" + ? truncate(normalizeWhitespace(parsed.evidence_note), 800) + : input.evidenceNote, + enoughAt, + }; + } catch (error) { + this.logger?.warn?.(`[clawxmemory] hop4 l0 fallback: ${String(error)}`); + return { + evidenceNote: fallbackEvidenceNote( + [ + input.evidenceNote, + ...input.l0Sessions.map((item) => { + const preview = item.messages + .slice(-3) + .map((message) => `${message.role}: ${message.content}`) + .join(" | "); + return `${item.timestamp}: ${preview}`; + }), + ], + input.query, + ), + enoughAt: "none", + }; + } + } + + async reasonOverMemory(input: LlmReasoningInput): Promise { + if ( + !input.profile && + input.l2Time.length === 0 && + input.l2Projects.length === 0 && + input.l1Windows.length === 0 && + input.l0Sessions.length === 0 + ) { + return { + intent: "general", + enoughAt: "none", + useProfile: false, + l2Ids: [], + l1Ids: [], + l0Ids: [], + }; + } + + const raw = await this.callStructuredJson({ + systemPrompt: REASONING_SYSTEM_PROMPT, + userPrompt: JSON.stringify( + { + query: input.query, + profile: input.profile + ? { + id: input.profile.recordId, + text: truncateForPrompt(input.profile.profileText, 260), + } + : null, + l2_time: input.l2Time.map((item) => ({ + id: item.l2IndexId, + date_key: item.dateKey, + summary: truncateForPrompt(item.summary, 180), + })), + l2_project: input.l2Projects.map((item) => ({ + id: item.l2IndexId, + project_key: item.projectKey, + project_name: item.projectName, + summary: truncateForPrompt(item.summary, 180), + latest_progress: truncateForPrompt(item.latestProgress, 180), + status: item.currentStatus, + })), + l1_windows: input.l1Windows.map((item) => ({ + id: item.l1IndexId, + session_key: item.sessionKey, + time_period: item.timePeriod, + summary: truncateForPrompt(item.summary, 180), + situation: truncateForPrompt(item.situationTimeInfo, 160), + projects: item.projectDetails.map((project) => project.name), + })), + l0_sessions: input.l0Sessions.map((item) => ({ + id: item.l0IndexId, + session_key: item.sessionKey, + timestamp: item.timestamp, + messages: item.messages + .filter((message) => message.role === "user") + .slice(-2) + .map((message) => truncateForPrompt(message.content, 160)), + })), + limits: input.limits, + }, + null, + 2, + ), + requestLabel: "Reasoning", + timeoutMs: input.timeoutMs ?? 8_000, + ...(input.agentId ? { agentId: input.agentId } : {}), + }); + const parsed = JSON.parse(extractFirstJsonObject(raw)) as RawReasoningPayload; + return { + intent: normalizeIntent(parsed.intent), + enoughAt: normalizeEnoughAt(parsed.enough_at), + useProfile: normalizeBoolean(parsed.use_profile, false), + l2Ids: normalizeStringArray(parsed.l2_ids, input.limits.l2), + l1Ids: normalizeStringArray(parsed.l1_ids, input.limits.l1), + l0Ids: normalizeStringArray(parsed.l0_ids, input.limits.l0), + }; + } +} diff --git a/extensions/openbmb-clawxmemory/src/core/skills/loader.ts b/extensions/openbmb-clawxmemory/src/core/skills/loader.ts new file mode 100644 index 0000000000000..1f07060863fbe --- /dev/null +++ b/extensions/openbmb-clawxmemory/src/core/skills/loader.ts @@ -0,0 +1,226 @@ +import { existsSync, readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + DEFAULT_CONTEXT_TEMPLATE, + DEFAULT_EXTRACTION_RULES, + DEFAULT_INTENT_RULES, + DEFAULT_PROJECT_STATUS_RULES, +} from "./defaults.js"; +import type { + ExtractionPatternFile, + ExtractionRulesFile, + IntentRulesFile, + ProjectStatusRulesFile, + SkillLoaderLogger, + SkillsRuntime, +} from "./types.js"; + +function safeJsonParse(raw: string): T | undefined { + try { + return JSON.parse(raw) as T; + } catch { + return undefined; + } +} + +function resolveDefaultSkillsDir(): string { + return fileURLToPath(new URL("../../../skills/", import.meta.url)); +} + +function readJsonWithFallback(path: string, fallback: T, errors: string[]): T { + if (!existsSync(path)) { + errors.push(`missing file: ${path}`); + return fallback; + } + const raw = readFileSync(path, "utf-8"); + const parsed = safeJsonParse(raw); + if (!parsed) { + errors.push(`invalid json: ${path}`); + return fallback; + } + return parsed; +} + +function ensureKeywords(values: unknown, fallback: string[]): string[] { + if (!Array.isArray(values)) return fallback; + const cleaned = values + .filter((v): v is string => typeof v === "string") + .map((v) => v.trim()) + .filter(Boolean); + return cleaned.length > 0 ? cleaned : fallback; +} + +function normalizeIntentRules(input: IntentRulesFile): IntentRulesFile { + return { + timeKeywords: ensureKeywords(input.timeKeywords, DEFAULT_INTENT_RULES.timeKeywords), + projectKeywords: ensureKeywords(input.projectKeywords, DEFAULT_INTENT_RULES.projectKeywords), + factKeywords: ensureKeywords(input.factKeywords, DEFAULT_INTENT_RULES.factKeywords), + }; +} + +function toRegExp(pattern: string, flags: string | undefined, fallback: RegExp): RegExp { + try { + return new RegExp(pattern, flags); + } catch { + return fallback; + } +} + +function normalizePattern(item: ExtractionPatternFile, fallback: ExtractionPatternFile): RegExp { + return toRegExp(item.pattern, item.flags, toRegExp(fallback.pattern, fallback.flags, /(?:)/g)); +} + +function normalizeExtractionRules(input: ExtractionRulesFile): SkillsRuntime["extractionRules"] { + const projectPatterns = ( + Array.isArray(input.projectPatterns) && input.projectPatterns.length > 0 + ? input.projectPatterns + : DEFAULT_EXTRACTION_RULES.projectPatterns + ).map((item, index) => { + const fallback = + DEFAULT_EXTRACTION_RULES.projectPatterns[index] ?? + DEFAULT_EXTRACTION_RULES.projectPatterns[0]!; + return normalizePattern(item, fallback); + }); + + const factRulesRaw = + Array.isArray(input.factRules) && input.factRules.length > 0 + ? input.factRules + : DEFAULT_EXTRACTION_RULES.factRules; + + const factRules = factRulesRaw.map((item, index) => { + const fallback = + DEFAULT_EXTRACTION_RULES.factRules[index] ?? DEFAULT_EXTRACTION_RULES.factRules[0]!; + return { + name: item.name || fallback.name || `rule_${index}`, + regex: toRegExp( + item.pattern, + item.flags, + toRegExp(fallback.pattern, fallback.flags, /(?:)/g), + ), + keyPrefix: item.keyPrefix || fallback.keyPrefix, + confidence: Number.isFinite(item.confidence) ? item.confidence : fallback.confidence, + maxLength: Number.isFinite(item.maxLength) ? item.maxLength! : (fallback.maxLength ?? 120), + }; + }); + + const summaryLimits = input.summaryLimits ?? DEFAULT_EXTRACTION_RULES.summaryLimits!; + + return { + projectPatterns, + factRules, + maxProjectTags: Number.isFinite(input.maxProjectTags) + ? input.maxProjectTags! + : (DEFAULT_EXTRACTION_RULES.maxProjectTags ?? 8), + maxFacts: Number.isFinite(input.maxFacts) + ? input.maxFacts! + : (DEFAULT_EXTRACTION_RULES.maxFacts ?? 16), + projectTagMinLength: Number.isFinite(input.projectTagMinLength) + ? input.projectTagMinLength! + : (DEFAULT_EXTRACTION_RULES.projectTagMinLength ?? 2), + projectTagMaxLength: Number.isFinite(input.projectTagMaxLength) + ? input.projectTagMaxLength! + : (DEFAULT_EXTRACTION_RULES.projectTagMaxLength ?? 50), + summaryLimits: { + head: Number.isFinite(summaryLimits.head) ? summaryLimits.head : 80, + tail: Number.isFinite(summaryLimits.tail) ? summaryLimits.tail : 80, + assistant: Number.isFinite(summaryLimits.assistant) ? summaryLimits.assistant : 80, + }, + }; +} + +function normalizeProjectStatusRules(input: ProjectStatusRulesFile): ProjectStatusRulesFile { + const rules = Array.isArray(input.rules) ? input.rules : DEFAULT_PROJECT_STATUS_RULES.rules; + const normalizedRules = rules + .map((rule, index) => { + const fallback = + DEFAULT_PROJECT_STATUS_RULES.rules[index] ?? DEFAULT_PROJECT_STATUS_RULES.rules[0]!; + const keywords = ensureKeywords(rule.keywords, fallback.keywords); + return { + status: rule.status || fallback.status, + keywords, + }; + }) + .filter((rule) => rule.status && rule.keywords.length > 0); + return { + defaultStatus: input.defaultStatus || DEFAULT_PROJECT_STATUS_RULES.defaultStatus, + rules: normalizedRules.length > 0 ? normalizedRules : DEFAULT_PROJECT_STATUS_RULES.rules, + }; +} + +export interface LoadSkillsOptions { + skillsDir?: string; + logger?: SkillLoaderLogger; +} + +function tryLoadSkillsFromDir(skillsDir: string): SkillsRuntime { + const errors: string[] = []; + + const intentPath = join(skillsDir, "intent-rules.json"); + const extractionPath = join(skillsDir, "extraction-rules.json"); + const projectStatusPath = join(skillsDir, "project-status-rules.json"); + const contextPath = join(skillsDir, "context-template.md"); + + const intentRaw = readJsonWithFallback(intentPath, DEFAULT_INTENT_RULES, errors); + const extractionRaw = readJsonWithFallback(extractionPath, DEFAULT_EXTRACTION_RULES, errors); + const projectStatusRaw = readJsonWithFallback( + projectStatusPath, + DEFAULT_PROJECT_STATUS_RULES, + errors, + ); + + let contextTemplate = DEFAULT_CONTEXT_TEMPLATE; + if (!existsSync(contextPath)) { + errors.push(`missing file: ${contextPath}`); + } else { + const raw = readFileSync(contextPath, "utf-8").trim(); + contextTemplate = raw || DEFAULT_CONTEXT_TEMPLATE; + } + + const runtime: SkillsRuntime = { + intentRules: normalizeIntentRules(intentRaw), + extractionRules: normalizeExtractionRules(extractionRaw), + projectStatusRules: normalizeProjectStatusRules(projectStatusRaw), + contextTemplate, + metadata: { + source: errors.length > 0 ? "fallback" : "files", + skillsDir, + errors, + }, + }; + return runtime; +} + +export function loadSkillsRuntime(options: LoadSkillsOptions = {}): SkillsRuntime { + const logger = options.logger ?? console; + const defaultSkillsDir = resolveDefaultSkillsDir(); + const candidateDirs = options.skillsDir + ? [resolve(options.skillsDir), defaultSkillsDir] + : [defaultSkillsDir]; + + let runtime: SkillsRuntime | undefined; + for (const skillsDir of candidateDirs) { + const loaded = tryLoadSkillsFromDir(skillsDir); + if (loaded.metadata.source === "files") { + logger.info?.(`[clawxmemory] skills loaded from ${skillsDir}`); + return loaded; + } + runtime = loaded; + } + + const fallback = runtime ?? tryLoadSkillsFromDir(defaultSkillsDir); + if (options.skillsDir && fallback.metadata.skillsDir !== defaultSkillsDir) { + const builtIn = tryLoadSkillsFromDir(defaultSkillsDir); + if (builtIn.metadata.source === "files") { + logger.warn?.( + `[clawxmemory] custom skillsDir unavailable (${resolve(options.skillsDir)}); falling back to built-in skills at ${defaultSkillsDir}`, + ); + return builtIn; + } + } + + logger.warn?.( + `[clawxmemory] skills loaded with fallback. errors=${fallback.metadata.errors.join(" | ")}`, + ); + return fallback; +} diff --git a/extensions/openbmb-clawxmemory/src/core/skills/types.ts b/extensions/openbmb-clawxmemory/src/core/skills/types.ts new file mode 100644 index 0000000000000..8d6e1b9d7b5de --- /dev/null +++ b/extensions/openbmb-clawxmemory/src/core/skills/types.ts @@ -0,0 +1,78 @@ +export interface IntentRulesFile { + timeKeywords: string[]; + projectKeywords: string[]; + factKeywords: string[]; +} + +export interface ExtractionPatternFile { + pattern: string; + flags?: string; +} + +export interface ExtractionFactRuleFile { + name?: string; + pattern: string; + flags?: string; + keyPrefix: string; + confidence: number; + maxLength?: number; +} + +export interface ExtractionRulesFile { + projectPatterns: ExtractionPatternFile[]; + factRules: ExtractionFactRuleFile[]; + maxProjectTags?: number; + maxFacts?: number; + projectTagMinLength?: number; + projectTagMaxLength?: number; + summaryLimits?: { + head: number; + tail: number; + assistant: number; + }; +} + +export interface ProjectStatusRulesFile { + defaultStatus: string; + rules: Array<{ + status: string; + keywords: string[]; + }>; +} + +export interface RuntimeFactRule { + name: string; + regex: RegExp; + keyPrefix: string; + confidence: number; + maxLength: number; +} + +export interface SkillsRuntime { + intentRules: IntentRulesFile; + extractionRules: { + projectPatterns: RegExp[]; + factRules: RuntimeFactRule[]; + maxProjectTags: number; + maxFacts: number; + projectTagMinLength: number; + projectTagMaxLength: number; + summaryLimits: { + head: number; + tail: number; + assistant: number; + }; + }; + projectStatusRules: ProjectStatusRulesFile; + contextTemplate: string; + metadata: { + source: "files" | "fallback"; + skillsDir: string; + errors: string[]; + }; +} + +export interface SkillLoaderLogger { + info?: (...args: unknown[]) => void; + warn?: (...args: unknown[]) => void; +} diff --git a/extensions/openbmb-clawxmemory/src/core/storage/sqlite.ts b/extensions/openbmb-clawxmemory/src/core/storage/sqlite.ts new file mode 100644 index 0000000000000..841971f6f05b5 --- /dev/null +++ b/extensions/openbmb-clawxmemory/src/core/storage/sqlite.ts @@ -0,0 +1,1946 @@ +import { mkdirSync } from "node:fs"; +import { dirname } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import type { + ActiveTopicBufferRecord, + CaseTraceRecord, + DashboardOverview, + FactCandidate, + GlobalProfileRecord, + IndexingSettings, + IndexLinkRecord, + L0SessionRecord, + L1SearchResult, + L1WindowRecord, + L2ProjectIndexRecord, + L2SearchResult, + L2TimeIndexRecord, + MemoryMessage, + MemoryExportBundle, + MemoryImportResult, + MemoryTransferCounts, + MemoryUiSnapshot, + ProjectStatus, +} from "../types.js"; +import { MEMORY_EXPORT_FORMAT_VERSION } from "../types.js"; +import { buildLinkId, nowIso } from "../utils/id.js"; +import { safeJsonParse, scoreMatch } from "../utils/text.js"; + +type DbRow = Record; +type SearchIdHit = { id: string; score: number }; + +const GLOBAL_PROFILE_RECORD_ID = "global_profile_record" as const; +const INDEXING_SETTINGS_STATE_KEY = "indexingSettings" as const; +const LAST_INDEXED_AT_STATE_KEY = "lastIndexedAt" as const; +const LAST_DREAM_AT_STATE_KEY = "lastDreamAt" as const; +const LAST_DREAM_STATUS_STATE_KEY = "lastDreamStatus" as const; +const LAST_DREAM_SUMMARY_STATE_KEY = "lastDreamSummary" as const; +const LAST_DREAM_L1_ENDED_AT_STATE_KEY = "lastDreamL1EndedAt" as const; +const RECENT_CASE_TRACES_STATE_KEY = "recentCaseTraces" as const; + +export class MemoryBundleValidationError extends Error { + constructor(message: string) { + super(message); + this.name = "MemoryBundleValidationError"; + } +} + +export interface ClearMemoryResult { + cleared: { + l0: number; + l1: number; + l2Time: number; + l2Project: number; + profile: number; + activeTopics: number; + links: number; + pipelineState: number; + }; + clearedAt: string; +} + +export interface RepairMemoryResult { + inspected: number; + updated: number; + removed: number; + rebuilt: boolean; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function parseCaseTraceRecord(value: unknown): CaseTraceRecord | null { + if (!isRecord(value)) return null; + if (typeof value.caseId !== "string" || !value.caseId.trim()) return null; + if (typeof value.sessionKey !== "string") return null; + if (typeof value.query !== "string") return null; + if (typeof value.startedAt !== "string" || !value.startedAt.trim()) return null; + const status = ( + typeof value.status === "string" ? value.status : "running" + ) as CaseTraceRecord["status"]; + if (!["running", "completed", "interrupted", "error"].includes(status)) return null; + let retrieval: CaseTraceRecord["retrieval"]; + if (isRecord(value.retrieval)) { + const next: NonNullable = { + injected: Boolean(value.retrieval.injected), + contextPreview: + typeof value.retrieval.contextPreview === "string" ? value.retrieval.contextPreview : "", + evidenceNotePreview: + typeof value.retrieval.evidenceNotePreview === "string" + ? value.retrieval.evidenceNotePreview + : "", + pathSummary: + typeof value.retrieval.pathSummary === "string" ? value.retrieval.pathSummary : "", + trace: + value.retrieval.trace && typeof value.retrieval.trace === "object" + ? (value.retrieval.trace as NonNullable["trace"]) + : null, + }; + if (typeof value.retrieval.intent === "string") { + next.intent = value.retrieval.intent as "time" | "project" | "fact" | "general"; + } + if (typeof value.retrieval.enoughAt === "string") { + next.enoughAt = value.retrieval.enoughAt as "profile" | "l2" | "l1" | "l0" | "none"; + } + retrieval = next; + } + return { + caseId: value.caseId, + sessionKey: value.sessionKey, + query: value.query, + startedAt: value.startedAt, + ...(typeof value.finishedAt === "string" && value.finishedAt.trim() + ? { finishedAt: value.finishedAt } + : {}), + status, + ...(retrieval ? { retrieval } : {}), + toolEvents: Array.isArray(value.toolEvents) + ? (value.toolEvents as CaseTraceRecord["toolEvents"]) + : [], + assistantReply: typeof value.assistantReply === "string" ? value.assistantReply : "", + }; +} + +function requireString(value: unknown, field: string): string { + if (typeof value !== "string" || !value.trim()) { + throw new MemoryBundleValidationError(`Invalid ${field}`); + } + return value; +} + +function readString(value: unknown, field: string): string { + if (typeof value !== "string") { + throw new MemoryBundleValidationError(`Invalid ${field}`); + } + return value; +} + +function normalizeStringArray(value: unknown, field: string): string[] { + if (!Array.isArray(value) || !value.every((item) => typeof item === "string")) { + throw new MemoryBundleValidationError(`Invalid ${field}`); + } + return value.map((item) => item.trim()).filter(Boolean); +} + +function normalizeMessages(value: unknown, field: string): MemoryMessage[] { + if (!Array.isArray(value)) throw new MemoryBundleValidationError(`Invalid ${field}`); + return value.map((item, index) => { + if (!isRecord(item)) throw new MemoryBundleValidationError(`Invalid ${field}[${index}]`); + return { + ...(typeof item.msgId === "string" && item.msgId.trim() ? { msgId: item.msgId.trim() } : {}), + role: requireString(item.role, `${field}[${index}].role`), + content: requireString(item.content, `${field}[${index}].content`), + }; + }); +} + +function normalizeL0Record(value: unknown, index: number): L0SessionRecord { + if (!isRecord(value)) throw new MemoryBundleValidationError(`Invalid l0Sessions[${index}]`); + return { + l0IndexId: requireString(value.l0IndexId, `l0Sessions[${index}].l0IndexId`), + sessionKey: requireString(value.sessionKey, `l0Sessions[${index}].sessionKey`), + timestamp: requireString(value.timestamp, `l0Sessions[${index}].timestamp`), + messages: normalizeMessages(value.messages, `l0Sessions[${index}].messages`), + source: requireString(value.source, `l0Sessions[${index}].source`), + indexed: Boolean(value.indexed), + createdAt: requireString(value.createdAt, `l0Sessions[${index}].createdAt`), + }; +} + +function normalizeFactCandidate(value: unknown, field: string): FactCandidate { + if (!isRecord(value)) throw new MemoryBundleValidationError(`Invalid ${field}`); + const confidence = + typeof value.confidence === "number" && Number.isFinite(value.confidence) + ? value.confidence + : 0; + return { + factKey: requireString(value.factKey, `${field}.factKey`), + factValue: readString(value.factValue, `${field}.factValue`), + confidence, + }; +} + +function normalizeStoredProjectStatus(value: unknown): ProjectStatus { + if (typeof value !== "string") return "planned"; + const normalized = value.trim().toLowerCase(); + if (normalized === "planned") return "planned"; + if (normalized === "in_progress" || normalized === "in progress") return "in_progress"; + if (normalized === "blocked" || normalized === "on_hold" || normalized === "on hold") + return "in_progress"; + if (normalized === "unknown") return "planned"; + if (normalized === "done" || normalized === "completed" || normalized === "complete") + return "done"; + return "planned"; +} + +function normalizeProjectDetail( + value: unknown, + field: string, +): L1WindowRecord["projectDetails"][number] { + if (!isRecord(value)) throw new MemoryBundleValidationError(`Invalid ${field}`); + const confidence = + typeof value.confidence === "number" && Number.isFinite(value.confidence) + ? value.confidence + : 0; + return { + key: requireString(value.key, `${field}.key`), + name: readString(value.name, `${field}.name`), + status: normalizeStoredProjectStatus(requireString(value.status, `${field}.status`)), + summary: readString(value.summary, `${field}.summary`), + latestProgress: readString(value.latestProgress, `${field}.latestProgress`), + confidence, + }; +} + +function normalizeL1Record(value: unknown, index: number): L1WindowRecord { + if (!isRecord(value)) throw new MemoryBundleValidationError(`Invalid l1Windows[${index}]`); + return { + l1IndexId: requireString(value.l1IndexId, `l1Windows[${index}].l1IndexId`), + sessionKey: readString(value.sessionKey, `l1Windows[${index}].sessionKey`), + timePeriod: requireString(value.timePeriod, `l1Windows[${index}].timePeriod`), + startedAt: requireString(value.startedAt, `l1Windows[${index}].startedAt`), + endedAt: requireString(value.endedAt, `l1Windows[${index}].endedAt`), + summary: readString(value.summary, `l1Windows[${index}].summary`), + facts: Array.isArray(value.facts) + ? value.facts.map((item, factIndex) => + normalizeFactCandidate(item, `l1Windows[${index}].facts[${factIndex}]`), + ) + : (() => { + throw new MemoryBundleValidationError(`Invalid l1Windows[${index}].facts`); + })(), + situationTimeInfo: readString(value.situationTimeInfo, `l1Windows[${index}].situationTimeInfo`), + projectTags: normalizeStringArray(value.projectTags, `l1Windows[${index}].projectTags`), + projectDetails: Array.isArray(value.projectDetails) + ? value.projectDetails.map((item, projectIndex) => + normalizeProjectDetail(item, `l1Windows[${index}].projectDetails[${projectIndex}]`), + ) + : (() => { + throw new MemoryBundleValidationError(`Invalid l1Windows[${index}].projectDetails`); + })(), + l0Source: normalizeStringArray(value.l0Source, `l1Windows[${index}].l0Source`), + createdAt: requireString(value.createdAt, `l1Windows[${index}].createdAt`), + }; +} + +function normalizeL2TimeRecord(value: unknown, index: number): L2TimeIndexRecord { + if (!isRecord(value)) throw new MemoryBundleValidationError(`Invalid l2TimeIndexes[${index}]`); + return { + l2IndexId: requireString(value.l2IndexId, `l2TimeIndexes[${index}].l2IndexId`), + dateKey: requireString(value.dateKey, `l2TimeIndexes[${index}].dateKey`), + summary: readString(value.summary, `l2TimeIndexes[${index}].summary`), + l1Source: normalizeStringArray(value.l1Source, `l2TimeIndexes[${index}].l1Source`), + createdAt: requireString(value.createdAt, `l2TimeIndexes[${index}].createdAt`), + updatedAt: requireString(value.updatedAt, `l2TimeIndexes[${index}].updatedAt`), + }; +} + +function normalizeL2ProjectRecord(value: unknown, index: number): L2ProjectIndexRecord { + if (!isRecord(value)) throw new MemoryBundleValidationError(`Invalid l2ProjectIndexes[${index}]`); + return { + l2IndexId: requireString(value.l2IndexId, `l2ProjectIndexes[${index}].l2IndexId`), + projectKey: requireString(value.projectKey, `l2ProjectIndexes[${index}].projectKey`), + projectName: readString(value.projectName, `l2ProjectIndexes[${index}].projectName`), + summary: readString(value.summary, `l2ProjectIndexes[${index}].summary`), + currentStatus: normalizeStoredProjectStatus( + requireString(value.currentStatus, `l2ProjectIndexes[${index}].currentStatus`), + ), + latestProgress: readString(value.latestProgress, `l2ProjectIndexes[${index}].latestProgress`), + l1Source: normalizeStringArray(value.l1Source, `l2ProjectIndexes[${index}].l1Source`), + createdAt: requireString(value.createdAt, `l2ProjectIndexes[${index}].createdAt`), + updatedAt: requireString(value.updatedAt, `l2ProjectIndexes[${index}].updatedAt`), + }; +} + +function normalizeGlobalProfile(value: unknown): GlobalProfileRecord { + if (!isRecord(value)) throw new MemoryBundleValidationError("Invalid globalProfile"); + const recordId = requireString(value.recordId, "globalProfile.recordId"); + if (recordId !== GLOBAL_PROFILE_RECORD_ID) { + throw new MemoryBundleValidationError("Invalid globalProfile.recordId"); + } + return { + recordId: GLOBAL_PROFILE_RECORD_ID, + profileText: readString(value.profileText ?? "", "globalProfile.profileText"), + sourceL1Ids: normalizeStringArray(value.sourceL1Ids, "globalProfile.sourceL1Ids"), + createdAt: requireString(value.createdAt, "globalProfile.createdAt"), + updatedAt: requireString(value.updatedAt, "globalProfile.updatedAt"), + }; +} + +function normalizeIndexLink(value: unknown, index: number): IndexLinkRecord { + if (!isRecord(value)) throw new MemoryBundleValidationError(`Invalid indexLinks[${index}]`); + return { + linkId: requireString(value.linkId, `indexLinks[${index}].linkId`), + fromLevel: requireString( + value.fromLevel, + `indexLinks[${index}].fromLevel`, + ) as IndexLinkRecord["fromLevel"], + fromId: requireString(value.fromId, `indexLinks[${index}].fromId`), + toLevel: requireString( + value.toLevel, + `indexLinks[${index}].toLevel`, + ) as IndexLinkRecord["toLevel"], + toId: requireString(value.toId, `indexLinks[${index}].toId`), + createdAt: requireString(value.createdAt, `indexLinks[${index}].createdAt`), + }; +} + +function normalizeMemoryExportBundle(value: unknown): MemoryExportBundle { + if (!isRecord(value)) throw new MemoryBundleValidationError("Invalid memory bundle"); + if (value.formatVersion !== MEMORY_EXPORT_FORMAT_VERSION) { + throw new MemoryBundleValidationError("Unsupported memory bundle formatVersion"); + } + if ( + !Array.isArray(value.l0Sessions) || + !Array.isArray(value.l1Windows) || + !Array.isArray(value.l2TimeIndexes) || + !Array.isArray(value.l2ProjectIndexes) || + !Array.isArray(value.indexLinks) + ) { + throw new MemoryBundleValidationError("Invalid memory bundle collections"); + } + return { + formatVersion: MEMORY_EXPORT_FORMAT_VERSION, + exportedAt: requireString(value.exportedAt, "exportedAt"), + ...(typeof value.lastIndexedAt === "string" && value.lastIndexedAt.trim() + ? { lastIndexedAt: value.lastIndexedAt } + : {}), + l0Sessions: value.l0Sessions.map((item, index) => normalizeL0Record(item, index)), + l1Windows: value.l1Windows.map((item, index) => normalizeL1Record(item, index)), + l2TimeIndexes: value.l2TimeIndexes.map((item, index) => normalizeL2TimeRecord(item, index)), + l2ProjectIndexes: value.l2ProjectIndexes.map((item, index) => + normalizeL2ProjectRecord(item, index), + ), + globalProfile: normalizeGlobalProfile(value.globalProfile), + indexLinks: value.indexLinks.map((item, index) => normalizeIndexLink(item, index)), + }; +} + +function parseL0Row(row: DbRow): L0SessionRecord { + return { + l0IndexId: String(row.l0_index_id), + sessionKey: String(row.session_key), + timestamp: String(row.timestamp), + messages: safeJsonParse(String(row.messages_json ?? "[]"), []), + source: String(row.source ?? "openclaw"), + indexed: Number(row.indexed ?? 0) === 1, + createdAt: String(row.created_at), + }; +} + +function parseActiveTopicBufferRow(row: DbRow): ActiveTopicBufferRecord { + return { + sessionKey: String(row.session_key), + startedAt: String(row.started_at), + updatedAt: String(row.updated_at), + topicSummary: String(row.topic_summary ?? ""), + userTurns: safeJsonParse(String(row.user_turns_json ?? "[]"), []), + l0Ids: safeJsonParse(String(row.l0_ids_json ?? "[]"), []), + lastL0Id: String(row.last_l0_id ?? ""), + createdAt: String(row.created_at), + }; +} + +function parseL1Row(row: DbRow): L1WindowRecord { + const rawProjectDetails = safeJsonParse(String(row.project_details_json ?? "[]"), []); + return { + l1IndexId: String(row.l1_index_id), + sessionKey: String(row.session_key ?? ""), + timePeriod: String(row.time_period), + startedAt: String(row.started_at ?? row.created_at), + endedAt: String(row.ended_at ?? row.created_at), + summary: String(row.summary), + facts: safeJsonParse(String(row.facts_json ?? "[]"), []), + situationTimeInfo: String(row.situation_time_info ?? ""), + projectTags: safeJsonParse(String(row.project_tags_json ?? "[]"), []), + projectDetails: Array.isArray(rawProjectDetails) + ? rawProjectDetails.map((item, index) => + normalizeProjectDetail(item, `l1.projectDetails[${index}]`), + ) + : [], + l0Source: safeJsonParse(String(row.l0_source_json ?? "[]"), []), + createdAt: String(row.created_at), + }; +} + +function parseL2TimeRow(row: DbRow): L2TimeIndexRecord { + return { + l2IndexId: String(row.l2_index_id), + dateKey: String(row.date_key), + summary: String(row.summary), + l1Source: safeJsonParse(String(row.l1_source_json ?? "[]"), []), + createdAt: String(row.created_at), + updatedAt: String(row.updated_at), + }; +} + +function parseL2ProjectRow(row: DbRow): L2ProjectIndexRecord { + return { + l2IndexId: String(row.l2_index_id), + projectKey: String(row.project_key ?? row.project_name), + projectName: String(row.project_name), + summary: String(row.summary), + currentStatus: normalizeStoredProjectStatus(row.current_status), + latestProgress: String(row.latest_progress), + l1Source: safeJsonParse(String(row.l1_source_json ?? "[]"), []), + createdAt: String(row.created_at), + updatedAt: String(row.updated_at), + }; +} + +function parseGlobalProfileRow(row: DbRow): GlobalProfileRecord { + return { + recordId: GLOBAL_PROFILE_RECORD_ID, + profileText: String(row.profile_text ?? ""), + sourceL1Ids: safeJsonParse(String(row.source_l1_ids_json ?? "[]"), []), + createdAt: String(row.created_at), + updatedAt: String(row.updated_at), + }; +} + +function parseIndexLinkRow(row: DbRow): IndexLinkRecord { + return { + linkId: String(row.link_id), + fromLevel: String(row.from_level) as IndexLinkRecord["fromLevel"], + fromId: String(row.from_id), + toLevel: String(row.to_level) as IndexLinkRecord["toLevel"], + toId: String(row.to_id), + createdAt: String(row.created_at), + }; +} + +function mergeSourceIds(existing: string[], incoming: string[]): string[] { + return Array.from(new Set([...existing, ...incoming])); +} + +function tokenizeQuery(query: string): string[] { + const trimmed = query.trim(); + if (!trimmed) return []; + const tokens = new Set(); + tokens.add(trimmed); + for (const token of trimmed.split(/[\s,.;:!?，。！？、]+/g)) { + const cleaned = token.trim(); + if (cleaned.length >= 2) tokens.add(cleaned); + } + return Array.from(tokens); +} + +function computeTokenScore(query: string, candidates: string[]): number { + const tokens = tokenizeQuery(query); + if (tokens.length === 0) return 1; + let best = 0; + for (const text of candidates) { + for (const token of tokens) { + best = Math.max(best, scoreMatch(token, text)); + } + } + return best; +} + +function buildSearchableMessageText(messages: MemoryMessage[]): string { + return messages.map((message) => `${message.role}: ${message.content}`).join("\n"); +} + +function normalizeIndexingSettings( + input: Partial | undefined, + defaults: IndexingSettings, +): IndexingSettings { + const legacy = input as Record | undefined; + const reasoningMode = + input?.reasoningMode === "accuracy_first" ? "accuracy_first" : "answer_first"; + const rawTopK = + typeof input?.recallTopK === "number" && Number.isFinite(input.recallTopK) + ? input.recallTopK + : typeof legacy?.recallTopK === "number" && Number.isFinite(legacy.recallTopK) + ? legacy.recallTopK + : typeof legacy?.recallTopK === "string" && legacy.recallTopK.trim() + ? Number.parseInt(legacy.recallTopK, 10) + : typeof legacy?.maxAutoReplyLatencyMs === "number" && + Number.isFinite(legacy.maxAutoReplyLatencyMs) + ? Math.max(1, Math.min(50, Math.round(legacy.maxAutoReplyLatencyMs / 180))) + : typeof legacy?.recallBudgetMs === "number" && Number.isFinite(legacy.recallBudgetMs) + ? Math.max(1, Math.min(50, Math.round(legacy.recallBudgetMs / 180))) + : defaults.recallTopK; + const rawAutoIndexIntervalMinutes = + typeof input?.autoIndexIntervalMinutes === "number" && + Number.isFinite(input.autoIndexIntervalMinutes) + ? input.autoIndexIntervalMinutes + : typeof legacy?.autoIndexIntervalMinutes === "number" && + Number.isFinite(legacy.autoIndexIntervalMinutes) + ? legacy.autoIndexIntervalMinutes + : typeof legacy?.autoIndexIntervalMinutes === "string" && + legacy.autoIndexIntervalMinutes.trim() + ? Number.parseInt(legacy.autoIndexIntervalMinutes, 10) + : defaults.autoIndexIntervalMinutes; + const rawAutoDreamIntervalMinutes = + typeof input?.autoDreamIntervalMinutes === "number" && + Number.isFinite(input.autoDreamIntervalMinutes) + ? input.autoDreamIntervalMinutes + : typeof legacy?.autoDreamIntervalMinutes === "number" && + Number.isFinite(legacy.autoDreamIntervalMinutes) + ? legacy.autoDreamIntervalMinutes + : typeof legacy?.autoDreamIntervalMinutes === "string" && + legacy.autoDreamIntervalMinutes.trim() + ? Number.parseInt(legacy.autoDreamIntervalMinutes, 10) + : defaults.autoDreamIntervalMinutes; + const rawAutoDreamMinNewL1 = + typeof input?.autoDreamMinNewL1 === "number" && Number.isFinite(input.autoDreamMinNewL1) + ? input.autoDreamMinNewL1 + : typeof legacy?.autoDreamMinNewL1 === "number" && Number.isFinite(legacy.autoDreamMinNewL1) + ? legacy.autoDreamMinNewL1 + : typeof legacy?.autoDreamMinNewL1 === "string" && legacy.autoDreamMinNewL1.trim() + ? Number.parseInt(legacy.autoDreamMinNewL1, 10) + : defaults.autoDreamMinNewL1; + return { + reasoningMode, + recallTopK: Math.max(1, Math.min(50, Math.floor(rawTopK))), + autoIndexIntervalMinutes: Math.max(0, Math.floor(rawAutoIndexIntervalMinutes)), + autoDreamIntervalMinutes: Math.max(0, Math.floor(rawAutoDreamIntervalMinutes)), + autoDreamMinNewL1: Math.max(0, Math.floor(rawAutoDreamMinNewL1)), + }; +} + +export class MemoryRepository { + private readonly db: DatabaseSync; + private ftsEnabled = false; + + constructor(private readonly dbPath: string) { + mkdirSync(dirname(dbPath), { recursive: true }); + this.db = new DatabaseSync(dbPath); + this.db.exec("PRAGMA journal_mode = WAL;"); + this.db.exec("PRAGMA synchronous = NORMAL;"); + this.db.exec("PRAGMA temp_store = MEMORY;"); + this.migrate(); + } + + close(): void { + this.db.close(); + } + + private hasColumn(tableName: string, columnName: string): boolean { + const stmt = this.db.prepare(`PRAGMA table_info(${tableName})`); + const rows = stmt.all() as Array<{ name?: string }>; + return rows.some((row) => row.name === columnName); + } + + private ensureColumn(tableName: string, columnName: string, definition: string): void { + if (this.hasColumn(tableName, columnName)) return; + this.db.exec(`ALTER TABLE ${tableName} ADD COLUMN ${columnName} ${definition};`); + } + + private ensureGlobalProfileRecord(): void { + const now = nowIso(); + const stmt = this.db.prepare(` + INSERT INTO global_profile_record ( + record_id, profile_text, source_l1_ids_json, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(record_id) DO NOTHING + `); + stmt.run(GLOBAL_PROFILE_RECORD_ID, "", "[]", now, now); + } + + private saveGlobalProfileRecord(record: GlobalProfileRecord): void { + const stmt = this.db.prepare(` + INSERT INTO global_profile_record ( + record_id, profile_text, source_l1_ids_json, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(record_id) DO UPDATE SET + profile_text = excluded.profile_text, + source_l1_ids_json = excluded.source_l1_ids_json, + created_at = excluded.created_at, + updated_at = excluded.updated_at + `); + stmt.run( + record.recordId, + record.profileText, + JSON.stringify(record.sourceL1Ids), + record.createdAt, + record.updatedAt, + ); + this.syncProfileFts(record); + } + + private initFts(): void { + try { + this.db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS global_profile_fts USING fts5(record_id UNINDEXED, content); + CREATE VIRTUAL TABLE IF NOT EXISTS l2_time_fts USING fts5(l2_index_id UNINDEXED, content); + CREATE VIRTUAL TABLE IF NOT EXISTS l2_project_fts USING fts5(l2_index_id UNINDEXED, content); + CREATE VIRTUAL TABLE IF NOT EXISTS l1_window_fts USING fts5(l1_index_id UNINDEXED, content); + `); + this.ftsEnabled = true; + } catch { + this.ftsEnabled = false; + } + } + + private upsertFtsDocument( + tableName: string, + idColumn: string, + id: string, + content: string, + ): void { + if (!this.ftsEnabled || !id.trim()) return; + const deleteStmt = this.db.prepare(`DELETE FROM ${tableName} WHERE ${idColumn} = ?`); + const insertStmt = this.db.prepare( + `INSERT INTO ${tableName} (${idColumn}, content) VALUES (?, ?)`, + ); + deleteStmt.run(id); + insertStmt.run(id, content.trim()); + } + + private deleteFtsDocument(tableName: string, idColumn: string, id: string): void { + if (!this.ftsEnabled || !id.trim()) return; + const stmt = this.db.prepare(`DELETE FROM ${tableName} WHERE ${idColumn} = ?`); + stmt.run(id); + } + + private buildFtsQuery(query: string): string { + const tokens = tokenizeQuery(query).slice(0, 8); + if (tokens.length === 0) return ""; + return tokens.map((token) => `"${token.replace(/"/g, '""')}"`).join(" OR "); + } + + private searchFts( + tableName: string, + idColumn: string, + query: string, + limit: number, + ): SearchIdHit[] { + if (!this.ftsEnabled) return []; + const ftsQuery = this.buildFtsQuery(query); + if (!ftsQuery) return []; + try { + const stmt = this.db.prepare(` + SELECT ${idColumn} AS id, bm25(${tableName}) AS rank + FROM ${tableName} + WHERE ${tableName} MATCH ? + ORDER BY rank ASC + LIMIT ? + `); + const rows = stmt.all(ftsQuery, limit) as Array<{ id?: string; rank?: number }>; + return rows + .filter((row) => typeof row.id === "string" && row.id.trim()) + .map((row, index) => ({ + id: String(row.id), + score: Math.max(0.2, 1 - Math.min(6, index) * 0.12), + })); + } catch { + return []; + } + } + + private compareL2SearchHits(left: L2SearchResult, right: L2SearchResult): number { + if (right.score !== left.score) return right.score - left.score; + if (left.level === right.level) { + if (left.level === "l2_time" && right.level === "l2_time") { + return right.item.dateKey.localeCompare(left.item.dateKey); + } + if (left.level === "l2_project" && right.level === "l2_project") { + return right.item.updatedAt.localeCompare(left.item.updatedAt); + } + } + const leftRecency = left.level === "l2_time" ? left.item.dateKey : left.item.updatedAt; + const rightRecency = right.level === "l2_time" ? right.item.dateKey : right.item.updatedAt; + return rightRecency.localeCompare(leftRecency); + } + + private searchRankedL2TimeIndexes( + query: string, + limit: number, + ): Array> { + if (limit <= 0) return []; + const recent = this.listRecentL2Time(Math.max(50, limit * 8)); + const recentById = new Map(recent.map((item) => [item.l2IndexId, item])); + const ftsHits = this.searchFts("l2_time_fts", "l2_index_id", query, Math.max(limit * 2, 8)); + const missingFtsIds = ftsHits.map((hit) => hit.id).filter((id) => !recentById.has(id)); + for (const item of this.getL2TimeByIds(missingFtsIds)) { + recentById.set(item.l2IndexId, item); + } + + const ordered: Array> = []; + const seen = new Set(); + for (const hit of ftsHits) { + const item = recentById.get(hit.id); + if (!item || seen.has(item.l2IndexId)) continue; + seen.add(item.l2IndexId); + ordered.push({ level: "l2_time", score: hit.score, item }); + if (ordered.length >= limit) return ordered; + } + + const fallback = recent + .filter((item) => !seen.has(item.l2IndexId)) + .map((item) => ({ + item, + score: computeTokenScore(query, [item.dateKey, item.summary]), + })) + .filter((hit) => hit.score > 0.12) + .sort((left, right) => { + if (right.score !== left.score) return right.score - left.score; + return right.item.dateKey.localeCompare(left.item.dateKey); + }) + .slice(0, Math.max(0, limit - ordered.length)) + .map((hit) => ({ + level: "l2_time" as const, + score: ordered.length > 0 ? Math.min(0.19, hit.score) : hit.score, + item: hit.item, + })); + + return [...ordered, ...fallback]; + } + + private searchRankedL2ProjectIndexes( + query: string, + limit: number, + ): Array> { + if (limit <= 0) return []; + const recent = this.listRecentL2Projects(Math.max(50, limit * 8)); + const recentById = new Map(recent.map((item) => [item.l2IndexId, item])); + const ftsHits = this.searchFts("l2_project_fts", "l2_index_id", query, Math.max(limit * 2, 8)); + const missingFtsIds = ftsHits.map((hit) => hit.id).filter((id) => !recentById.has(id)); + for (const item of this.getL2ProjectByIds(missingFtsIds)) { + recentById.set(item.l2IndexId, item); + } + + const ordered: Array> = []; + const seen = new Set(); + for (const hit of ftsHits) { + const item = recentById.get(hit.id); + if (!item || seen.has(item.l2IndexId)) continue; + seen.add(item.l2IndexId); + ordered.push({ level: "l2_project", score: hit.score, item }); + if (ordered.length >= limit) return ordered; + } + + const fallback = recent + .filter((item) => !seen.has(item.l2IndexId)) + .map((item) => ({ + item, + score: computeTokenScore(query, [ + item.projectKey, + item.projectName, + item.summary, + item.currentStatus, + item.latestProgress, + ]), + })) + .filter((hit) => hit.score > 0.12) + .sort((left, right) => { + if (right.score !== left.score) return right.score - left.score; + return right.item.updatedAt.localeCompare(left.item.updatedAt); + }) + .slice(0, Math.max(0, limit - ordered.length)) + .map((hit) => ({ + level: "l2_project" as const, + score: ordered.length > 0 ? Math.min(0.19, hit.score) : hit.score, + item: hit.item, + })); + + return [...ordered, ...fallback]; + } + + private syncProfileFts(profile: GlobalProfileRecord): void { + this.upsertFtsDocument( + "global_profile_fts", + "record_id", + profile.recordId, + [profile.profileText, profile.sourceL1Ids.join(" ")].filter(Boolean).join("\n"), + ); + } + + private syncL1Fts(window: L1WindowRecord): void { + this.upsertFtsDocument( + "l1_window_fts", + "l1_index_id", + window.l1IndexId, + [ + window.sessionKey, + window.timePeriod, + window.summary, + window.situationTimeInfo, + window.projectTags.join(" "), + window.projectDetails + .map((project) => `${project.name} ${project.summary} ${project.latestProgress}`) + .join(" "), + window.facts.map((fact) => `${fact.factKey} ${fact.factValue}`).join(" "), + ] + .filter(Boolean) + .join("\n"), + ); + } + + private syncL2TimeFts(index: L2TimeIndexRecord): void { + this.upsertFtsDocument( + "l2_time_fts", + "l2_index_id", + index.l2IndexId, + [index.dateKey, index.summary, index.l1Source.join(" ")].filter(Boolean).join("\n"), + ); + } + + private syncL2ProjectFts(index: L2ProjectIndexRecord): void { + this.upsertFtsDocument( + "l2_project_fts", + "l2_index_id", + index.l2IndexId, + [ + index.projectKey, + index.projectName, + index.summary, + index.latestProgress, + index.currentStatus, + index.l1Source.join(" "), + ] + .filter(Boolean) + .join("\n"), + ); + } + + migrate(): void { + this.db.exec(` + CREATE TABLE IF NOT EXISTS l0_sessions ( + l0_index_id TEXT PRIMARY KEY, + session_key TEXT NOT NULL, + timestamp TEXT NOT NULL, + messages_json TEXT NOT NULL, + source TEXT NOT NULL, + indexed INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS active_topic_buffers ( + session_key TEXT PRIMARY KEY, + started_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + topic_summary TEXT NOT NULL DEFAULT '', + user_turns_json TEXT NOT NULL DEFAULT '[]', + l0_ids_json TEXT NOT NULL DEFAULT '[]', + last_l0_id TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS l1_windows ( + l1_index_id TEXT PRIMARY KEY, + session_key TEXT NOT NULL DEFAULT '', + time_period TEXT NOT NULL, + started_at TEXT NOT NULL DEFAULT '', + ended_at TEXT NOT NULL DEFAULT '', + summary TEXT NOT NULL, + facts_json TEXT NOT NULL, + situation_time_info TEXT NOT NULL, + project_tags_json TEXT NOT NULL, + project_details_json TEXT NOT NULL DEFAULT '[]', + l0_source_json TEXT NOT NULL, + created_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS l2_time_indexes ( + l2_index_id TEXT PRIMARY KEY, + date_key TEXT NOT NULL UNIQUE, + summary TEXT NOT NULL, + l1_source_json TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS l2_project_indexes ( + l2_index_id TEXT PRIMARY KEY, + project_key TEXT NOT NULL DEFAULT '', + project_name TEXT NOT NULL UNIQUE, + summary TEXT NOT NULL, + current_status TEXT NOT NULL, + latest_progress TEXT NOT NULL, + l1_source_json TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS global_profile_record ( + record_id TEXT PRIMARY KEY, + profile_text TEXT NOT NULL, + source_l1_ids_json TEXT NOT NULL DEFAULT '[]', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS index_links ( + link_id TEXT PRIMARY KEY, + from_level TEXT NOT NULL, + from_id TEXT NOT NULL, + to_level TEXT NOT NULL, + to_id TEXT NOT NULL, + created_at TEXT NOT NULL, + UNIQUE(from_level, from_id, to_level, to_id) + ); + + CREATE TABLE IF NOT EXISTS pipeline_state ( + state_key TEXT PRIMARY KEY, + state_value TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_l0_session_time ON l0_sessions(session_key, timestamp); + CREATE INDEX IF NOT EXISTS idx_l0_indexed ON l0_sessions(indexed, timestamp); + CREATE INDEX IF NOT EXISTS idx_l1_time_period ON l1_windows(time_period); + CREATE INDEX IF NOT EXISTS idx_l2_time_date ON l2_time_indexes(date_key); + CREATE INDEX IF NOT EXISTS idx_l2_project_name ON l2_project_indexes(project_name); + CREATE INDEX IF NOT EXISTS idx_l2_project_key ON l2_project_indexes(project_key); + CREATE INDEX IF NOT EXISTS idx_active_topic_updated ON active_topic_buffers(updated_at); + `); + + this.ensureColumn("l1_windows", "session_key", "TEXT NOT NULL DEFAULT ''"); + this.ensureColumn("l1_windows", "started_at", "TEXT NOT NULL DEFAULT ''"); + this.ensureColumn("l1_windows", "ended_at", "TEXT NOT NULL DEFAULT ''"); + this.ensureColumn("l1_windows", "project_details_json", "TEXT NOT NULL DEFAULT '[]'"); + this.ensureColumn("l2_project_indexes", "project_key", "TEXT NOT NULL DEFAULT ''"); + this.ensureGlobalProfileRecord(); + this.initFts(); + this.rebuildSearchIndexes(); + } + + private rebuildSearchIndexes(): void { + if (!this.ftsEnabled) return; + this.db.exec(` + DELETE FROM global_profile_fts; + DELETE FROM l2_time_fts; + DELETE FROM l2_project_fts; + DELETE FROM l1_window_fts; + `); + this.syncProfileFts(this.getGlobalProfileRecord()); + for (const item of this.listAllL2Time()) this.syncL2TimeFts(item); + for (const item of this.listAllL2Projects()) this.syncL2ProjectFts(item); + for (const item of this.listAllL1()) this.syncL1Fts(item); + } + + insertL0Session(record: Omit & { createdAt?: string }): void { + const createdAt = record.createdAt ?? nowIso(); + const stmt = this.db.prepare(` + INSERT OR IGNORE INTO l0_sessions ( + l0_index_id, session_key, timestamp, messages_json, source, indexed, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?) + `); + stmt.run( + record.l0IndexId, + record.sessionKey, + record.timestamp, + JSON.stringify(record.messages), + record.source, + record.indexed ? 1 : 0, + createdAt, + ); + } + + listUnindexedL0Sessions(limit = 20, sessionKeys?: string[]): L0SessionRecord[] { + const keys = Array.isArray(sessionKeys) ? sessionKeys.filter(Boolean) : []; + const whereParts = ["indexed = 0"]; + const params: Array = []; + if (keys.length > 0) { + whereParts.push(`session_key IN (${keys.map(() => "?").join(", ")})`); + params.push(...keys); + } + const limitSql = Number.isFinite(limit) ? "LIMIT ?" : ""; + if (Number.isFinite(limit)) params.push(limit); + const stmt = this.db.prepare(` + SELECT * FROM l0_sessions + WHERE ${whereParts.join(" AND ")} + ORDER BY timestamp ASC + ${limitSql} + `); + const rows = stmt.all(...params) as DbRow[]; + return rows.map(parseL0Row); + } + + markL0Indexed(ids: string[]): void { + if (ids.length === 0) return; + const placeholders = ids.map(() => "?").join(", "); + const stmt = this.db.prepare( + `UPDATE l0_sessions SET indexed = 1 WHERE l0_index_id IN (${placeholders})`, + ); + stmt.run(...ids); + } + + getL0ByIds(ids: string[]): L0SessionRecord[] { + if (ids.length === 0) return []; + const placeholders = ids.map(() => "?").join(", "); + const stmt = this.db.prepare( + `SELECT * FROM l0_sessions WHERE l0_index_id IN (${placeholders}) ORDER BY timestamp ASC`, + ); + const rows = stmt.all(...ids) as DbRow[]; + return rows.map(parseL0Row); + } + + searchL0(query: string, limit = 8): L0SessionRecord[] { + const rows = this.listRecentL0(Math.max(50, limit * 10)); + const scored = rows.map((item) => ({ + item, + score: computeTokenScore(query, [item.sessionKey, buildSearchableMessageText(item.messages)]), + })); + return scored + .filter((hit) => hit.score > 0.2) + .sort((a, b) => b.score - a.score) + .slice(0, limit) + .map((hit) => hit.item); + } + + getL0ByL1Ids(l1Ids: string[], limit = 4): L0SessionRecord[] { + if (l1Ids.length === 0) return []; + const l1Rows = this.getL1ByIds(l1Ids); + const l0Ids = Array.from(new Set(l1Rows.flatMap((item) => item.l0Source))).slice(0, limit * 3); + return this.getL0ByIds(l0Ids).slice(0, limit); + } + + listRecentL0(limit = 20, offset = 0): L0SessionRecord[] { + const stmt = this.db.prepare( + "SELECT * FROM l0_sessions ORDER BY timestamp DESC LIMIT ? OFFSET ?", + ); + const rows = stmt.all(limit, offset) as DbRow[]; + return rows.map(parseL0Row); + } + + listAllL0(): L0SessionRecord[] { + const stmt = this.db.prepare("SELECT * FROM l0_sessions ORDER BY timestamp ASC"); + const rows = stmt.all() as DbRow[]; + return rows.map(parseL0Row); + } + + getActiveTopicBuffer(sessionKey: string): ActiveTopicBufferRecord | undefined { + const stmt = this.db.prepare("SELECT * FROM active_topic_buffers WHERE session_key = ?"); + const row = stmt.get(sessionKey) as DbRow | undefined; + return row ? parseActiveTopicBufferRow(row) : undefined; + } + + listActiveTopicBuffers(sessionKeys?: string[]): ActiveTopicBufferRecord[] { + const keys = Array.isArray(sessionKeys) ? sessionKeys.filter(Boolean) : []; + if (keys.length === 0) { + const stmt = this.db.prepare("SELECT * FROM active_topic_buffers ORDER BY updated_at DESC"); + return (stmt.all() as DbRow[]).map(parseActiveTopicBufferRow); + } + const placeholders = keys.map(() => "?").join(", "); + const stmt = this.db.prepare(` + SELECT * FROM active_topic_buffers + WHERE session_key IN (${placeholders}) + ORDER BY updated_at DESC + `); + return (stmt.all(...keys) as DbRow[]).map(parseActiveTopicBufferRow); + } + + upsertActiveTopicBuffer(buffer: ActiveTopicBufferRecord): void { + const stmt = this.db.prepare(` + INSERT INTO active_topic_buffers ( + session_key, started_at, updated_at, topic_summary, user_turns_json, l0_ids_json, last_l0_id, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(session_key) DO UPDATE SET + started_at = excluded.started_at, + updated_at = excluded.updated_at, + topic_summary = excluded.topic_summary, + user_turns_json = excluded.user_turns_json, + l0_ids_json = excluded.l0_ids_json, + last_l0_id = excluded.last_l0_id, + created_at = excluded.created_at + `); + stmt.run( + buffer.sessionKey, + buffer.startedAt, + buffer.updatedAt, + buffer.topicSummary, + JSON.stringify(buffer.userTurns), + JSON.stringify(buffer.l0Ids), + buffer.lastL0Id, + buffer.createdAt, + ); + } + + deleteActiveTopicBuffer(sessionKey: string): void { + const stmt = this.db.prepare("DELETE FROM active_topic_buffers WHERE session_key = ?"); + stmt.run(sessionKey); + } + + insertL1Window(window: L1WindowRecord): void { + const stmt = this.db.prepare(` + INSERT OR IGNORE INTO l1_windows ( + l1_index_id, session_key, time_period, started_at, ended_at, summary, facts_json, situation_time_info, project_tags_json, project_details_json, l0_source_json, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + stmt.run( + window.l1IndexId, + window.sessionKey, + window.timePeriod, + window.startedAt, + window.endedAt, + window.summary, + JSON.stringify(window.facts), + window.situationTimeInfo, + JSON.stringify(window.projectTags), + JSON.stringify(window.projectDetails), + JSON.stringify(window.l0Source), + window.createdAt, + ); + this.syncL1Fts(window); + } + + getL1ByIds(ids: string[]): L1WindowRecord[] { + if (ids.length === 0) return []; + const placeholders = ids.map(() => "?").join(", "); + const stmt = this.db.prepare( + `SELECT * FROM l1_windows WHERE l1_index_id IN (${placeholders}) ORDER BY created_at DESC`, + ); + const rows = stmt.all(...ids) as DbRow[]; + return rows.map(parseL1Row); + } + + searchL1(query: string, limit = 10): L1WindowRecord[] { + return this.searchL1Hits(query, limit).map((hit) => hit.item); + } + + listRecentL1(limit = 20, offset = 0): L1WindowRecord[] { + const stmt = this.db.prepare( + "SELECT * FROM l1_windows ORDER BY ended_at DESC, created_at DESC LIMIT ? OFFSET ?", + ); + const rows = stmt.all(limit, offset) as DbRow[]; + return rows.map(parseL1Row); + } + + listAllL1(): L1WindowRecord[] { + const stmt = this.db.prepare("SELECT * FROM l1_windows ORDER BY ended_at ASC, created_at ASC"); + const rows = stmt.all() as DbRow[]; + return rows.map(parseL1Row); + } + + searchL1Hits(query: string, limit = 10): L1SearchResult[] { + const recent = this.listRecentL1(Math.max(60, limit * 10)); + const recentById = new Map(recent.map((item) => [item.l1IndexId, item])); + const ftsHits = this.searchFts("l1_window_fts", "l1_index_id", query, Math.max(limit * 2, 8)); + const missingFtsIds = ftsHits.map((hit) => hit.id).filter((id) => !recentById.has(id)); + for (const item of this.getL1ByIds(missingFtsIds)) { + recentById.set(item.l1IndexId, item); + } + + const ordered: L1SearchResult[] = []; + const seen = new Set(); + for (const hit of ftsHits) { + const item = recentById.get(hit.id); + if (!item || seen.has(item.l1IndexId)) continue; + seen.add(item.l1IndexId); + ordered.push({ item, score: hit.score }); + if (ordered.length >= limit) return ordered; + } + + const fallback = recent + .filter((item) => !seen.has(item.l1IndexId)) + .map((item) => ({ + item, + score: computeTokenScore(query, [ + item.sessionKey, + item.timePeriod, + item.summary, + item.situationTimeInfo, + item.projectTags.join(" "), + item.projectDetails + .map((project) => `${project.name} ${project.summary} ${project.latestProgress}`) + .join(" "), + JSON.stringify(item.facts), + ]), + })) + .filter((hit) => hit.score > 0.15) + .sort((left, right) => { + if (right.score !== left.score) return right.score - left.score; + const endedCompare = right.item.endedAt.localeCompare(left.item.endedAt); + return endedCompare !== 0 + ? endedCompare + : right.item.createdAt.localeCompare(left.item.createdAt); + }) + .slice(0, Math.max(0, limit - ordered.length)) + .map((hit) => ({ + item: hit.item, + score: ordered.length > 0 ? Math.min(0.19, hit.score) : hit.score, + })); + + return [...ordered, ...fallback]; + } + + getL2TimeByDate(dateKey: string): L2TimeIndexRecord | undefined { + const stmt = this.db.prepare("SELECT * FROM l2_time_indexes WHERE date_key = ?"); + const row = stmt.get(dateKey) as DbRow | undefined; + return row ? parseL2TimeRow(row) : undefined; + } + + getL2TimeByIds(ids: string[]): L2TimeIndexRecord[] { + if (ids.length === 0) return []; + const placeholders = ids.map(() => "?").join(", "); + const stmt = this.db.prepare( + `SELECT * FROM l2_time_indexes WHERE l2_index_id IN (${placeholders}) ORDER BY updated_at DESC`, + ); + const rows = stmt.all(...ids) as DbRow[]; + return rows.map(parseL2TimeRow); + } + + upsertL2TimeIndex(index: L2TimeIndexRecord): void { + const previous = this.getL2TimeByDate(index.dateKey); + const now = nowIso(); + const mergedSources = mergeSourceIds(previous?.l1Source ?? [], index.l1Source); + if (previous) { + const updateStmt = this.db.prepare(` + UPDATE l2_time_indexes + SET summary = ?, l1_source_json = ?, updated_at = ? + WHERE l2_index_id = ? + `); + updateStmt.run(index.summary, JSON.stringify(mergedSources), now, previous.l2IndexId); + this.syncL2TimeFts({ + ...previous, + summary: index.summary, + l1Source: mergedSources, + updatedAt: now, + }); + return; + } + + const insertStmt = this.db.prepare(` + INSERT INTO l2_time_indexes ( + l2_index_id, date_key, summary, l1_source_json, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?) + `); + insertStmt.run( + index.l2IndexId, + index.dateKey, + index.summary, + JSON.stringify(mergedSources), + index.createdAt, + now, + ); + this.syncL2TimeFts({ + ...index, + l1Source: mergedSources, + updatedAt: now, + }); + } + + searchL2TimeIndexes(query: string, limit = 10): L2SearchResult[] { + return this.searchRankedL2TimeIndexes(query, limit); + } + + listRecentL2Time(limit = 20, offset = 0): L2TimeIndexRecord[] { + const stmt = this.db.prepare( + "SELECT * FROM l2_time_indexes ORDER BY updated_at DESC LIMIT ? OFFSET ?", + ); + const rows = stmt.all(limit, offset) as DbRow[]; + return rows.map(parseL2TimeRow); + } + + listAllL2Time(): L2TimeIndexRecord[] { + const stmt = this.db.prepare( + "SELECT * FROM l2_time_indexes ORDER BY date_key ASC, created_at ASC", + ); + const rows = stmt.all() as DbRow[]; + return rows.map(parseL2TimeRow); + } + + getL2ProjectByKey(projectKey: string): L2ProjectIndexRecord | undefined { + const stmt = this.db.prepare("SELECT * FROM l2_project_indexes WHERE project_key = ?"); + const row = stmt.get(projectKey) as DbRow | undefined; + return row ? parseL2ProjectRow(row) : undefined; + } + + getL2ProjectByIds(ids: string[]): L2ProjectIndexRecord[] { + if (ids.length === 0) return []; + const placeholders = ids.map(() => "?").join(", "); + const stmt = this.db.prepare( + `SELECT * FROM l2_project_indexes WHERE l2_index_id IN (${placeholders}) ORDER BY updated_at DESC`, + ); + const rows = stmt.all(...ids) as DbRow[]; + return rows.map(parseL2ProjectRow); + } + + upsertL2ProjectIndex(index: L2ProjectIndexRecord): void { + const previous = this.getL2ProjectByKey(index.projectKey); + const now = nowIso(); + const mergedSources = mergeSourceIds(previous?.l1Source ?? [], index.l1Source); + if (previous) { + const updateStmt = this.db.prepare(` + UPDATE l2_project_indexes + SET project_name = ?, summary = ?, current_status = ?, latest_progress = ?, l1_source_json = ?, updated_at = ? + WHERE l2_index_id = ? + `); + updateStmt.run( + index.projectName, + index.summary, + index.currentStatus, + index.latestProgress, + JSON.stringify(mergedSources), + now, + previous.l2IndexId, + ); + this.syncL2ProjectFts({ + ...previous, + projectName: index.projectName, + summary: index.summary, + currentStatus: index.currentStatus, + latestProgress: index.latestProgress, + l1Source: mergedSources, + updatedAt: now, + }); + return; + } + + const insertStmt = this.db.prepare(` + INSERT INTO l2_project_indexes ( + l2_index_id, project_key, project_name, summary, current_status, latest_progress, l1_source_json, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + insertStmt.run( + index.l2IndexId, + index.projectKey, + index.projectName, + index.summary, + index.currentStatus, + index.latestProgress, + JSON.stringify(mergedSources), + index.createdAt, + now, + ); + this.syncL2ProjectFts({ + ...index, + l1Source: mergedSources, + updatedAt: now, + }); + } + + searchL2ProjectIndexes(query: string, limit = 10): L2SearchResult[] { + return this.searchRankedL2ProjectIndexes(query, limit); + } + + listRecentL2Projects(limit = 20, offset = 0): L2ProjectIndexRecord[] { + const stmt = this.db.prepare( + "SELECT * FROM l2_project_indexes ORDER BY updated_at DESC LIMIT ? OFFSET ?", + ); + const rows = stmt.all(limit, offset) as DbRow[]; + return rows.map(parseL2ProjectRow); + } + + listAllL2Projects(): L2ProjectIndexRecord[] { + const stmt = this.db.prepare( + "SELECT * FROM l2_project_indexes ORDER BY updated_at ASC, created_at ASC", + ); + const rows = stmt.all() as DbRow[]; + return rows.map(parseL2ProjectRow); + } + + searchL2Hits(query: string, limit = 10): L2SearchResult[] { + const timeHits = this.searchRankedL2TimeIndexes(query, limit); + const projectHits = this.searchRankedL2ProjectIndexes(query, limit); + return [...timeHits, ...projectHits] + .sort((left, right) => this.compareL2SearchHits(left, right)) + .slice(0, limit); + } + + getGlobalProfileRecord(): GlobalProfileRecord { + this.ensureGlobalProfileRecord(); + const stmt = this.db.prepare("SELECT * FROM global_profile_record WHERE record_id = ?"); + const row = stmt.get(GLOBAL_PROFILE_RECORD_ID) as DbRow | undefined; + if (row) return parseGlobalProfileRow(row); + const now = nowIso(); + return { + recordId: GLOBAL_PROFILE_RECORD_ID, + profileText: "", + sourceL1Ids: [], + createdAt: now, + updatedAt: now, + }; + } + + upsertGlobalProfile(profileText: string, sourceL1Ids: string[]): GlobalProfileRecord { + const current = this.getGlobalProfileRecord(); + const now = nowIso(); + const next: GlobalProfileRecord = { + recordId: GLOBAL_PROFILE_RECORD_ID, + profileText: profileText.trim(), + sourceL1Ids: mergeSourceIds(current.sourceL1Ids, sourceL1Ids), + createdAt: current.createdAt, + updatedAt: now, + }; + this.saveGlobalProfileRecord(next); + this.syncProfileFts(next); + return next; + } + + applyDreamRewrite(input: { + projects: L2ProjectIndexRecord[]; + profileText: string; + profileSourceL1Ids: string[]; + }): void { + const currentProfile = this.getGlobalProfileRecord(); + const currentProjects = this.listAllL2Projects(); + const currentProjectIds = currentProjects.map((project) => project.l2IndexId).filter(Boolean); + const deleteProjectLinksStmt = + currentProjectIds.length > 0 + ? this.db.prepare(` + DELETE FROM index_links + WHERE from_level = 'l2' + AND from_id IN (${currentProjectIds.map(() => "?").join(", ")}) + `) + : null; + const deleteProjectRowsStmt = this.db.prepare("DELETE FROM l2_project_indexes"); + const insertProjectStmt = this.db.prepare(` + INSERT INTO l2_project_indexes ( + l2_index_id, project_key, project_name, summary, current_status, latest_progress, l1_source_json, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + const insertLinkStmt = this.db.prepare(` + INSERT OR IGNORE INTO index_links (link_id, from_level, from_id, to_level, to_id, created_at) + VALUES (?, ?, ?, ?, ?, ?) + `); + + this.db.exec("BEGIN"); + try { + if (deleteProjectLinksStmt) deleteProjectLinksStmt.run(...currentProjectIds); + deleteProjectRowsStmt.run(); + + for (const project of input.projects) { + insertProjectStmt.run( + project.l2IndexId, + project.projectKey, + project.projectName, + project.summary, + project.currentStatus, + project.latestProgress, + JSON.stringify(project.l1Source), + project.createdAt, + project.updatedAt, + ); + for (const l1Id of project.l1Source) { + insertLinkStmt.run( + buildLinkId("l2", project.l2IndexId, "l1", l1Id), + "l2", + project.l2IndexId, + "l1", + l1Id, + project.updatedAt || nowIso(), + ); + } + } + + this.saveGlobalProfileRecord({ + recordId: GLOBAL_PROFILE_RECORD_ID, + profileText: input.profileText.trim(), + sourceL1Ids: Array.from(new Set(input.profileSourceL1Ids.filter(Boolean))), + createdAt: currentProfile.createdAt, + updatedAt: nowIso(), + }); + + this.db.exec("COMMIT"); + this.rebuildSearchIndexes(); + } catch (error) { + this.db.exec("ROLLBACK"); + throw error; + } + } + + appendToGlobalProfile(content: string): GlobalProfileRecord { + const current = this.getGlobalProfileRecord(); + const nextText = [current.profileText, content.trim()].filter(Boolean).join("\n"); + return this.upsertGlobalProfile(nextText, []); + } + + searchGlobalProfile(query: string, limit = 1): GlobalProfileRecord[] { + const profile = this.getGlobalProfileRecord(); + if (!profile.profileText.trim()) return []; + if (!query.trim()) return [profile].slice(0, limit); + const score = computeTokenScore(query, [profile.profileText, profile.sourceL1Ids.join(" ")]); + return score > 0.15 ? [profile].slice(0, limit) : []; + } + + shortlistGlobalProfile(query: string): { item: GlobalProfileRecord; score: number } | null { + const profile = this.getGlobalProfileRecord(); + if (!profile.profileText.trim()) return null; + const ftsScore = this.searchFts("global_profile_fts", "record_id", query, 1)[0]?.score ?? 0; + const tokenScore = computeTokenScore(query, [ + profile.profileText, + profile.sourceL1Ids.join(" "), + ]); + const score = ftsScore > 0 ? ftsScore : tokenScore; + if (query.trim() && score <= 0.1) return null; + return { item: profile, score: Math.max(score, query.trim() ? score : 0.2) }; + } + + getSnapshotVersion(): string { + const overview = this.getOverview(); + return JSON.stringify({ + lastIndexedAt: overview.lastIndexedAt ?? "", + totalL1: overview.totalL1, + totalL2Time: overview.totalL2Time, + totalL2Project: overview.totalL2Project, + totalProfiles: overview.totalProfiles, + }); + } + + insertLink( + fromLevel: "l2" | "l1" | "l0", + fromId: string, + toLevel: "l2" | "l1" | "l0", + toId: string, + ): void { + const linkId = buildLinkId(fromLevel, fromId, toLevel, toId); + const stmt = this.db.prepare(` + INSERT OR IGNORE INTO index_links (link_id, from_level, from_id, to_level, to_id, created_at) + VALUES (?, ?, ?, ?, ?, ?) + `); + stmt.run(linkId, fromLevel, fromId, toLevel, toId, nowIso()); + } + + listAllIndexLinks(): IndexLinkRecord[] { + const stmt = this.db.prepare("SELECT * FROM index_links ORDER BY created_at ASC"); + const rows = stmt.all() as DbRow[]; + return rows.map(parseIndexLinkRow); + } + + getOverview(): DashboardOverview { + const count = (tableName: string): number => { + const stmt = this.db.prepare(`SELECT COUNT(1) AS total FROM ${tableName}`); + const row = stmt.get() as { total?: number } | undefined; + return Number(row?.total ?? 0); + }; + const stateStmt = this.db.prepare("SELECT state_value FROM pipeline_state WHERE state_key = ?"); + const readState = (key: string): string | undefined => { + const row = stateStmt.get(key) as { state_value?: string } | undefined; + return typeof row?.state_value === "string" && row.state_value.trim() + ? row.state_value + : undefined; + }; + const profile = this.getGlobalProfileRecord(); + const overview: DashboardOverview = { + totalL0: count("l0_sessions"), + pendingL0: (() => { + const stmt = this.db.prepare("SELECT COUNT(1) AS total FROM l0_sessions WHERE indexed = 0"); + const row = stmt.get() as { total?: number } | undefined; + return Number(row?.total ?? 0); + })(), + openTopics: count("active_topic_buffers"), + totalL1: count("l1_windows"), + totalL2Time: count("l2_time_indexes"), + totalL2Project: count("l2_project_indexes"), + totalProfiles: profile.profileText.trim() ? 1 : 0, + queuedSessions: 0, + lastRecallMs: 0, + recallTimeouts: 0, + lastRecallMode: "none", + }; + const lastIndexedAt = readState(LAST_INDEXED_AT_STATE_KEY); + const lastDreamAt = readState(LAST_DREAM_AT_STATE_KEY); + const lastDreamStatus = readState(LAST_DREAM_STATUS_STATE_KEY); + const lastDreamSummary = readState(LAST_DREAM_SUMMARY_STATE_KEY); + const lastDreamL1EndedAt = readState(LAST_DREAM_L1_ENDED_AT_STATE_KEY); + if (lastIndexedAt) overview.lastIndexedAt = lastIndexedAt; + if (lastDreamAt) overview.lastDreamAt = lastDreamAt; + if (lastDreamStatus) + overview.lastDreamStatus = lastDreamStatus as NonNullable< + DashboardOverview["lastDreamStatus"] + >; + if (lastDreamSummary) overview.lastDreamSummary = lastDreamSummary; + if (lastDreamL1EndedAt) overview.lastDreamL1EndedAt = lastDreamL1EndedAt; + return overview; + } + + setPipelineState(key: string, value: string): void { + const stmt = this.db.prepare(` + INSERT INTO pipeline_state (state_key, state_value, updated_at) + VALUES (?, ?, ?) + ON CONFLICT(state_key) DO UPDATE SET + state_value = excluded.state_value, + updated_at = excluded.updated_at + `); + const now = nowIso(); + stmt.run(key, value, now); + } + + getPipelineState(key: string): string | undefined { + const stmt = this.db.prepare("SELECT state_value FROM pipeline_state WHERE state_key = ?"); + const row = stmt.get(key) as { state_value?: string } | undefined; + return row?.state_value; + } + + deletePipelineState(key: string): void { + const stmt = this.db.prepare("DELETE FROM pipeline_state WHERE state_key = ?"); + stmt.run(key); + } + + listRecentCaseTraces(limit: number): CaseTraceRecord[] { + const raw = this.getPipelineState(RECENT_CASE_TRACES_STATE_KEY); + if (!raw) return []; + const parsed = safeJsonParse(raw, []); + if (!Array.isArray(parsed)) return []; + return parsed + .map(parseCaseTraceRecord) + .filter((record): record is CaseTraceRecord => Boolean(record)) + .slice(0, Math.max(1, Math.min(200, limit))); + } + + getCaseTrace(caseId: string): CaseTraceRecord | undefined { + if (!caseId.trim()) return undefined; + return this.listRecentCaseTraces(200).find((record) => record.caseId === caseId.trim()); + } + + saveCaseTrace(record: CaseTraceRecord, maxRecords = 30): void { + const normalized = parseCaseTraceRecord(record); + if (!normalized) return; + const next = this.listRecentCaseTraces(Math.max(1, Math.min(200, maxRecords + 20))).filter( + (item) => item.caseId !== normalized.caseId, + ); + next.unshift(normalized); + this.setPipelineState( + RECENT_CASE_TRACES_STATE_KEY, + JSON.stringify(next.slice(0, Math.max(1, Math.min(200, maxRecords)))), + ); + } + + getIndexingSettings(defaults: IndexingSettings): IndexingSettings { + const raw = this.getPipelineState(INDEXING_SETTINGS_STATE_KEY); + if (!raw) return normalizeIndexingSettings(undefined, defaults); + const parsed = safeJsonParse>(raw, {}); + return normalizeIndexingSettings(parsed, defaults); + } + + saveIndexingSettings( + input: Partial, + defaults: IndexingSettings, + ): IndexingSettings { + const next = normalizeIndexingSettings(input, defaults); + this.setPipelineState(INDEXING_SETTINGS_STATE_KEY, JSON.stringify(next)); + return next; + } + + exportMemoryBundle(): MemoryExportBundle { + const lastIndexedAt = this.getPipelineState(LAST_INDEXED_AT_STATE_KEY); + return { + formatVersion: MEMORY_EXPORT_FORMAT_VERSION, + exportedAt: nowIso(), + ...(lastIndexedAt ? { lastIndexedAt } : {}), + l0Sessions: this.listAllL0(), + l1Windows: this.listAllL1(), + l2TimeIndexes: this.listAllL2Time(), + l2ProjectIndexes: this.listAllL2Projects(), + globalProfile: this.getGlobalProfileRecord(), + indexLinks: this.listAllIndexLinks(), + }; + } + + importMemoryBundle(bundleLike: unknown): MemoryImportResult { + const bundle = normalizeMemoryExportBundle(bundleLike); + const importedAt = nowIso(); + const imported: MemoryTransferCounts = { + l0: bundle.l0Sessions.length, + l1: bundle.l1Windows.length, + l2Time: bundle.l2TimeIndexes.length, + l2Project: bundle.l2ProjectIndexes.length, + profile: bundle.globalProfile.profileText.trim() ? 1 : 0, + links: bundle.indexLinks.length, + }; + + this.db.exec("BEGIN"); + try { + this.db.exec(` + DELETE FROM active_topic_buffers; + DELETE FROM index_links; + DELETE FROM l2_project_indexes; + DELETE FROM l2_time_indexes; + DELETE FROM l1_windows; + DELETE FROM l0_sessions; + DELETE FROM global_profile_record; + `); + + const insertL0Stmt = this.db.prepare(` + INSERT INTO l0_sessions ( + l0_index_id, session_key, timestamp, messages_json, source, indexed, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?) + `); + const insertL1Stmt = this.db.prepare(` + INSERT INTO l1_windows ( + l1_index_id, session_key, time_period, started_at, ended_at, summary, facts_json, situation_time_info, project_tags_json, project_details_json, l0_source_json, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + const insertL2TimeStmt = this.db.prepare(` + INSERT INTO l2_time_indexes ( + l2_index_id, date_key, summary, l1_source_json, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?) + `); + const insertL2ProjectStmt = this.db.prepare(` + INSERT INTO l2_project_indexes ( + l2_index_id, project_key, project_name, summary, current_status, latest_progress, l1_source_json, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + const insertLinkStmt = this.db.prepare(` + INSERT INTO index_links (link_id, from_level, from_id, to_level, to_id, created_at) + VALUES (?, ?, ?, ?, ?, ?) + `); + + for (const session of bundle.l0Sessions) { + insertL0Stmt.run( + session.l0IndexId, + session.sessionKey, + session.timestamp, + JSON.stringify(session.messages), + session.source, + session.indexed ? 1 : 0, + session.createdAt, + ); + } + + for (const window of bundle.l1Windows) { + insertL1Stmt.run( + window.l1IndexId, + window.sessionKey, + window.timePeriod, + window.startedAt, + window.endedAt, + window.summary, + JSON.stringify(window.facts), + window.situationTimeInfo, + JSON.stringify(window.projectTags), + JSON.stringify(window.projectDetails), + JSON.stringify(window.l0Source), + window.createdAt, + ); + } + + for (const timeIndex of bundle.l2TimeIndexes) { + insertL2TimeStmt.run( + timeIndex.l2IndexId, + timeIndex.dateKey, + timeIndex.summary, + JSON.stringify(timeIndex.l1Source), + timeIndex.createdAt, + timeIndex.updatedAt, + ); + } + + for (const projectIndex of bundle.l2ProjectIndexes) { + insertL2ProjectStmt.run( + projectIndex.l2IndexId, + projectIndex.projectKey, + projectIndex.projectName, + projectIndex.summary, + projectIndex.currentStatus, + projectIndex.latestProgress, + JSON.stringify(projectIndex.l1Source), + projectIndex.createdAt, + projectIndex.updatedAt, + ); + } + + this.saveGlobalProfileRecord(bundle.globalProfile); + + for (const link of bundle.indexLinks) { + insertLinkStmt.run( + link.linkId, + link.fromLevel, + link.fromId, + link.toLevel, + link.toId, + link.createdAt, + ); + } + + if (bundle.lastIndexedAt) { + this.setPipelineState(LAST_INDEXED_AT_STATE_KEY, bundle.lastIndexedAt); + } else { + this.deletePipelineState(LAST_INDEXED_AT_STATE_KEY); + } + + this.db.exec("COMMIT"); + this.rebuildSearchIndexes(); + return { + formatVersion: MEMORY_EXPORT_FORMAT_VERSION, + imported, + importedAt, + ...(bundle.lastIndexedAt ? { lastIndexedAt: bundle.lastIndexedAt } : {}), + }; + } catch (error) { + this.db.exec("ROLLBACK"); + throw error; + } + } + + resetDerivedIndexes(): void { + const currentProfile = this.getGlobalProfileRecord(); + this.db.exec("BEGIN"); + try { + this.db.exec(` + DELETE FROM active_topic_buffers; + DELETE FROM index_links; + DELETE FROM l2_project_indexes; + DELETE FROM l2_time_indexes; + DELETE FROM l1_windows; + UPDATE l0_sessions SET indexed = 0; + `); + const clearStateStmt = this.db.prepare(`DELETE FROM pipeline_state WHERE state_key = ?`); + clearStateStmt.run(LAST_INDEXED_AT_STATE_KEY); + clearStateStmt.run(LAST_DREAM_AT_STATE_KEY); + clearStateStmt.run(LAST_DREAM_STATUS_STATE_KEY); + clearStateStmt.run(LAST_DREAM_SUMMARY_STATE_KEY); + clearStateStmt.run(LAST_DREAM_L1_ENDED_AT_STATE_KEY); + this.saveGlobalProfileRecord({ + recordId: GLOBAL_PROFILE_RECORD_ID, + profileText: "", + sourceL1Ids: [], + createdAt: currentProfile.createdAt, + updatedAt: nowIso(), + }); + this.db.exec("COMMIT"); + this.rebuildSearchIndexes(); + } catch (error) { + this.db.exec("ROLLBACK"); + throw error; + } + } + + repairL0Sessions(cleaner: (record: L0SessionRecord) => MemoryMessage[]): RepairMemoryResult { + const rows = this.listAllL0(); + const stats: RepairMemoryResult = { + inspected: rows.length, + updated: 0, + removed: 0, + rebuilt: false, + }; + if (rows.length === 0) return stats; + + const updateStmt = this.db.prepare(` + UPDATE l0_sessions + SET messages_json = ?, indexed = 0 + WHERE l0_index_id = ? + `); + const deleteStmt = this.db.prepare(`DELETE FROM l0_sessions WHERE l0_index_id = ?`); + + this.db.exec("BEGIN"); + try { + for (const row of rows) { + const cleaned = cleaner(row); + if (cleaned.length === 0) { + deleteStmt.run(row.l0IndexId); + stats.removed += 1; + continue; + } + + const previousJson = JSON.stringify(row.messages); + const nextJson = JSON.stringify(cleaned); + if (previousJson !== nextJson) { + updateStmt.run(nextJson, row.l0IndexId); + stats.updated += 1; + } + } + + if (stats.updated > 0 || stats.removed > 0) { + this.db.exec(` + DELETE FROM active_topic_buffers; + DELETE FROM index_links; + DELETE FROM l2_project_indexes; + DELETE FROM l2_time_indexes; + DELETE FROM l1_windows; + UPDATE l0_sessions SET indexed = 0; + `); + const clearStateStmt = this.db.prepare(`DELETE FROM pipeline_state WHERE state_key = ?`); + clearStateStmt.run(LAST_INDEXED_AT_STATE_KEY); + clearStateStmt.run(LAST_DREAM_AT_STATE_KEY); + clearStateStmt.run(LAST_DREAM_STATUS_STATE_KEY); + clearStateStmt.run(LAST_DREAM_SUMMARY_STATE_KEY); + clearStateStmt.run(LAST_DREAM_L1_ENDED_AT_STATE_KEY); + const currentProfile = this.getGlobalProfileRecord(); + this.saveGlobalProfileRecord({ + recordId: GLOBAL_PROFILE_RECORD_ID, + profileText: "", + sourceL1Ids: [], + createdAt: currentProfile.createdAt, + updatedAt: nowIso(), + }); + stats.rebuilt = true; + } + + this.db.exec("COMMIT"); + if (stats.rebuilt) this.rebuildSearchIndexes(); + return stats; + } catch (error) { + this.db.exec("ROLLBACK"); + throw error; + } + } + + clearAllMemoryData(): ClearMemoryResult { + const runDelete = (table: string): number => { + const stmt = this.db.prepare(`DELETE FROM ${table}`); + const result = stmt.run() as { changes?: number }; + return Number(result.changes ?? 0); + }; + + const profileCount = this.getGlobalProfileRecord().profileText.trim() ? 1 : 0; + const indexingSettings = this.getPipelineState(INDEXING_SETTINGS_STATE_KEY); + this.db.exec("BEGIN"); + try { + const cleared = { + activeTopics: runDelete("active_topic_buffers"), + links: runDelete("index_links"), + l2Project: runDelete("l2_project_indexes"), + l2Time: runDelete("l2_time_indexes"), + l1: runDelete("l1_windows"), + l0: runDelete("l0_sessions"), + profile: profileCount, + pipelineState: runDelete("pipeline_state"), + }; + runDelete("global_profile_record"); + const resetAt = nowIso(); + this.saveGlobalProfileRecord({ + recordId: GLOBAL_PROFILE_RECORD_ID, + profileText: "", + sourceL1Ids: [], + createdAt: resetAt, + updatedAt: resetAt, + }); + if (indexingSettings) { + this.setPipelineState(INDEXING_SETTINGS_STATE_KEY, indexingSettings); + } + this.db.exec("COMMIT"); + this.rebuildSearchIndexes(); + return { + cleared, + clearedAt: resetAt, + }; + } catch (error) { + this.db.exec("ROLLBACK"); + throw error; + } + } + + getUiSnapshot(limit = 20): MemoryUiSnapshot { + return { + overview: this.getOverview(), + settings: this.getIndexingSettings({ + reasoningMode: "answer_first", + recallTopK: 10, + autoIndexIntervalMinutes: 60, + autoDreamIntervalMinutes: 360, + autoDreamMinNewL1: 10, + }), + recentTimeIndexes: this.listRecentL2Time(limit), + recentProjectIndexes: this.listRecentL2Projects(limit), + recentL1Windows: this.listRecentL1(limit), + recentSessions: this.listRecentL0(limit), + globalProfile: this.getGlobalProfileRecord(), + }; + } +} diff --git a/extensions/openbmb-clawxmemory/src/core/types.ts b/extensions/openbmb-clawxmemory/src/core/types.ts new file mode 100644 index 0000000000000..e382a6db27259 --- /dev/null +++ b/extensions/openbmb-clawxmemory/src/core/types.ts @@ -0,0 +1,385 @@ +export type ChatRole = "user" | "assistant" | "system" | string; + +export interface MemoryMessage { + msgId?: string; + role: ChatRole; + content: string; +} + +export interface L0SessionRecord { + l0IndexId: string; + sessionKey: string; + timestamp: string; + messages: MemoryMessage[]; + source: string; + indexed: boolean; + createdAt: string; +} + +export interface FactCandidate { + factKey: string; + factValue: string; + confidence: number; +} + +export type ProjectStatus = "planned" | "in_progress" | "done"; +export type ReasoningMode = "answer_first" | "accuracy_first"; +export type DreamPipelineStatus = "running" | "success" | "skipped" | "failed"; + +export interface IndexingSettings { + reasoningMode: ReasoningMode; + recallTopK: number; + autoIndexIntervalMinutes: number; + autoDreamIntervalMinutes: number; + autoDreamMinNewL1: number; +} + +export interface ActiveTopicBufferRecord { + sessionKey: string; + startedAt: string; + updatedAt: string; + topicSummary: string; + userTurns: string[]; + l0Ids: string[]; + lastL0Id: string; + createdAt: string; +} + +export interface ProjectDetail { + key: string; + name: string; + status: ProjectStatus; + summary: string; + latestProgress: string; + confidence: number; +} + +export interface L1WindowRecord { + l1IndexId: string; + sessionKey: string; + timePeriod: string; + startedAt: string; + endedAt: string; + summary: string; + facts: FactCandidate[]; + situationTimeInfo: string; + projectTags: string[]; + projectDetails: ProjectDetail[]; + l0Source: string[]; + createdAt: string; +} + +export interface L2TimeIndexRecord { + l2IndexId: string; + dateKey: string; + summary: string; + l1Source: string[]; + createdAt: string; + updatedAt: string; +} + +export interface L2ProjectIndexRecord { + l2IndexId: string; + projectKey: string; + projectName: string; + summary: string; + currentStatus: ProjectStatus; + latestProgress: string; + l1Source: string[]; + createdAt: string; + updatedAt: string; +} + +export interface GlobalProfileRecord { + recordId: "global_profile_record"; + profileText: string; + sourceL1Ids: string[]; + createdAt: string; + updatedAt: string; +} + +export interface IndexLinkRecord { + linkId: string; + fromLevel: "l2" | "l1" | "l0"; + fromId: string; + toLevel: "l2" | "l1" | "l0"; + toId: string; + createdAt: string; +} + +export const MEMORY_EXPORT_FORMAT_VERSION = "clawxmemory-memory-bundle.v1" as const; + +export interface MemoryExportBundle { + formatVersion: typeof MEMORY_EXPORT_FORMAT_VERSION; + exportedAt: string; + lastIndexedAt?: string; + l0Sessions: L0SessionRecord[]; + l1Windows: L1WindowRecord[]; + l2TimeIndexes: L2TimeIndexRecord[]; + l2ProjectIndexes: L2ProjectIndexRecord[]; + globalProfile: GlobalProfileRecord; + indexLinks: IndexLinkRecord[]; +} + +export interface MemoryTransferCounts { + l0: number; + l1: number; + l2Time: number; + l2Project: number; + profile: number; + links: number; +} + +export interface MemoryImportResult { + formatVersion: typeof MEMORY_EXPORT_FORMAT_VERSION; + imported: MemoryTransferCounts; + importedAt: string; + lastIndexedAt?: string; +} + +export type IntentType = "time" | "project" | "fact" | "general"; + +export type L2SearchResult = + | { + score: number; + level: "l2_time"; + item: L2TimeIndexRecord; + } + | { + score: number; + level: "l2_project"; + item: L2ProjectIndexRecord; + }; + +export interface L1SearchResult { + score: number; + item: L1WindowRecord; +} + +export interface L0SearchResult { + score: number; + item: L0SessionRecord; +} + +export interface RetrievalTraceKvEntry { + label: string; + value: string; +} + +export type RetrievalTraceDetail = + | { + key: string; + label: string; + kind: "text" | "note"; + text: string; + } + | { + key: string; + label: string; + kind: "list"; + items: string[]; + } + | { + key: string; + label: string; + kind: "kv"; + entries: RetrievalTraceKvEntry[]; + } + | { + key: string; + label: string; + kind: "json"; + json: unknown; + }; + +export interface RetrievalPromptDebug { + requestLabel: string; + systemPrompt: string; + userPrompt: string; + rawResponse: string; + parsedResult?: unknown; + timedOut?: boolean; + errored?: boolean; + errorMessage?: string; +} + +export type RetrievalTraceStepKind = + | "recall_start" + | "cache_hit" + | "hop1_decision" + | "l2_candidates" + | "hop2_decision" + | "l1_candidates" + | "hop3_decision" + | "l0_candidates" + | "hop4_decision" + | "context_rendered" + | "fallback_applied" + | "recall_skipped"; + +export interface RetrievalTraceStep { + stepId: string; + kind: RetrievalTraceStepKind; + title: string; + status: "info" | "success" | "warning" | "error" | "skipped"; + inputSummary: string; + outputSummary: string; + refs?: Record; + metrics?: Record; + details?: RetrievalTraceDetail[]; + promptDebug?: RetrievalPromptDebug; +} + +export interface RetrievalTrace { + traceId: string; + query: string; + mode: "auto" | "explicit"; + startedAt: string; + finishedAt: string; + steps: RetrievalTraceStep[]; +} + +export interface CaseToolEvent { + eventId: string; + phase: "start" | "result"; + toolName: string; + toolCallId?: string; + occurredAt: string; + status: "running" | "success" | "error"; + summary: string; + paramsPreview?: string; + resultPreview?: string; + durationMs?: number; +} + +export interface CaseTraceRecord { + caseId: string; + sessionKey: string; + query: string; + startedAt: string; + finishedAt?: string; + status: "running" | "completed" | "interrupted" | "error"; + retrieval?: { + intent?: IntentType; + enoughAt?: RetrievalResult["enoughAt"]; + injected: boolean; + contextPreview: string; + evidenceNotePreview: string; + pathSummary: string; + trace: RetrievalTrace | null; + }; + toolEvents: CaseToolEvent[]; + assistantReply: string; +} + +export interface RetrievalResult { + query: string; + intent: IntentType; + enoughAt: "profile" | "l2" | "l1" | "l0" | "none"; + profile: GlobalProfileRecord | null; + evidenceNote: string; + l2Results: L2SearchResult[]; + l1Results: L1SearchResult[]; + l0Results: L0SearchResult[]; + context: string; + trace?: RetrievalTrace; + debug?: { + mode: "llm" | "local_fallback" | "none"; + elapsedMs: number; + cacheHit: boolean; + path?: "auto" | "explicit" | "shadow"; + budgetLimited?: boolean; + shadowDeepQueued?: boolean; + hop1QueryScope?: "standalone" | "continuation"; + hop1EffectiveQuery?: string; + hop1BaseOnly?: boolean; + hop1LookupQueries?: Array<{ + targetTypes: Array<"time" | "project">; + lookupQuery: string; + }>; + hop2EnoughAt?: "l2" | "descend_l1" | "none"; + hop2SelectedL2Ids?: string[]; + hop3EnoughAt?: "l1" | "descend_l0" | "none"; + hop3SelectedL1Ids?: string[]; + hop4SelectedL0Ids?: string[]; + catalogTruncated?: boolean; + corrections?: string[]; + }; +} + +export type RecallMode = "llm" | "local_fallback" | "none"; +export type StartupRepairStatus = "idle" | "running" | "failed"; + +export interface DashboardOverview { + totalL0: number; + pendingL0: number; + openTopics: number; + totalL1: number; + totalL2Time: number; + totalL2Project: number; + totalProfiles: number; + queuedSessions: number; + lastRecallMs: number; + recallTimeouts: number; + lastRecallMode: RecallMode; + currentReasoningMode?: ReasoningMode; + lastRecallPath?: "auto" | "explicit" | "shadow"; + lastRecallBudgetLimited?: boolean; + lastShadowDeepQueued?: boolean; + lastRecallInjected?: boolean; + lastRecallEnoughAt?: RetrievalResult["enoughAt"]; + lastRecallCacheHit?: boolean; + slotOwner?: string; + dynamicMemoryRuntime?: string; + workspaceBootstrapPresent?: boolean; + memoryRuntimeHealthy?: boolean; + runtimeIssues?: string[]; + lastIndexedAt?: string; + lastDreamAt?: string; + lastDreamStatus?: DreamPipelineStatus; + lastDreamSummary?: string; + lastDreamL1EndedAt?: string; + startupRepairStatus?: StartupRepairStatus; + startupRepairMessage?: string; +} + +export interface MemoryUiSnapshot { + overview: DashboardOverview; + settings: IndexingSettings; + recentTimeIndexes: L2TimeIndexRecord[]; + recentProjectIndexes: L2ProjectIndexRecord[]; + recentL1Windows: L1WindowRecord[]; + recentSessions: L0SessionRecord[]; + globalProfile: GlobalProfileRecord; +} + +export type DreamReviewFocus = "all" | "projects" | "profile"; + +export type DreamReviewTarget = "l2_project" | "global_profile" | "l1_only" | "time_note"; + +export interface DreamEvidenceRef { + refId: string; + level: "profile" | "l2_project" | "l2_time" | "l1" | "l0"; + id: string; + label: string; + summary: string; +} + +export interface DreamReviewFinding { + title: string; + rationale: string; + confidence: number; + target: DreamReviewTarget; + evidenceRefs: string[]; +} + +export interface DreamReviewResult { + summary: string; + projectRebuild: DreamReviewFinding[]; + profileSuggestions: DreamReviewFinding[]; + cleanup: DreamReviewFinding[]; + ambiguous: DreamReviewFinding[]; + noAction: DreamReviewFinding[]; + timeLayerNotes: DreamReviewFinding[]; + evidenceRefs: DreamEvidenceRef[]; +} diff --git a/extensions/openbmb-clawxmemory/src/core/utils/id.ts b/extensions/openbmb-clawxmemory/src/core/utils/id.ts new file mode 100644 index 0000000000000..d52dc303a5f68 --- /dev/null +++ b/extensions/openbmb-clawxmemory/src/core/utils/id.ts @@ -0,0 +1,39 @@ +import { createHash } from "node:crypto"; + +export function hashText(input: string): string { + return createHash("sha1").update(input).digest("hex").slice(0, 10); +} + +export function nowIso(): string { + return new Date().toISOString(); +} + +export function buildL0IndexId(sessionKey: string, timestamp: string, payload: string): string { + const key = sessionKey || "session"; + return `${key}_${hashText(`${timestamp}:${payload}`)}_raw`; +} + +export function buildL1IndexId(timestamp: string, sourceIds: string[]): string { + return `l1_${hashText(`${timestamp}:${sourceIds.sort().join(",")}`)}`; +} + +export function buildL2TimeIndexId(dateKey: string): string { + return `time_${hashText(dateKey)}`; +} + +export function buildL2ProjectIndexId(projectKey: string): string { + return `project_${hashText(projectKey.toLowerCase())}`; +} + +export function buildFactId(factKey: string): string { + return `fact_${hashText(factKey.toLowerCase())}`; +} + +export function buildLinkId( + fromLevel: string, + fromId: string, + toLevel: string, + toId: string, +): string { + return `link_${hashText(`${fromLevel}:${fromId}->${toLevel}:${toId}`)}`; +} diff --git a/extensions/openbmb-clawxmemory/src/core/utils/text.ts b/extensions/openbmb-clawxmemory/src/core/utils/text.ts new file mode 100644 index 0000000000000..94e2acc5fe5c4 --- /dev/null +++ b/extensions/openbmb-clawxmemory/src/core/utils/text.ts @@ -0,0 +1,48 @@ +export function truncate(text: string, maxLength: number): string { + if (!text) return ""; + if (maxLength <= 0 || text.length <= maxLength) return text; + return `${text.slice(0, maxLength)}...`; +} + +export function normalizeText(text: string): string { + return text.replace(/\s+/g, " ").trim(); +} + +export function scoreMatch(query: string, text: string): number { + const q = normalizeText(query).toLowerCase(); + const t = normalizeText(text).toLowerCase(); + if (!q || !t) return 0; + if (t === q) return 1; + if (t.startsWith(q)) return 0.92; + if (t.includes(q)) return 0.82; + + const qWords = q.split(" ").filter(Boolean); + if (qWords.length === 0) return 0; + let hits = 0; + for (const word of qWords) { + if (t.includes(word)) hits += 1; + } + const wordScore = (hits / qWords.length) * 0.7; + + const qCompact = q.replace(/\s+/g, ""); + const tCompact = t.replace(/\s+/g, ""); + if (qCompact.length < 2 || tCompact.length < 2) return wordScore; + + let gramHits = 0; + let grams = 0; + for (let i = 0; i < qCompact.length - 1; i += 1) { + const gram = qCompact.slice(i, i + 2); + grams += 1; + if (tCompact.includes(gram)) gramHits += 1; + } + const gramScore = grams > 0 ? (gramHits / grams) * 0.75 : 0; + return Math.max(wordScore, gramScore); +} + +export function safeJsonParse(raw: string, fallback: T): T { + try { + return JSON.parse(raw) as T; + } catch { + return fallback; + } +}