diff --git a/index.ts b/index.ts index c878a1e0..60345cc9 100644 --- a/index.ts +++ b/index.ts @@ -12,7 +12,9 @@ import { LcmContextEngine } from "./src/engine.js"; import { createLcmDescribeTool } from "./src/tools/lcm-describe-tool.js"; import { createLcmExpandQueryTool } from "./src/tools/lcm-expand-query-tool.js"; import { createLcmExpandTool } from "./src/tools/lcm-expand-tool.js"; +import { createLcmExportTool } from "./src/tools/lcm-export-tool.js"; import { createLcmGrepTool } from "./src/tools/lcm-grep-tool.js"; +import { createLcmUpdatePeerTool } from "./src/tools/lcm-update-peer-tool.js"; import type { LcmDependencies } from "./src/types.js"; /** Parse `agent::` session keys. */ @@ -1315,6 +1317,62 @@ const lcmPlugin = { requesterSessionKey: ctx.sessionKey, }), ); + api.registerTool((ctx) => + createLcmExportTool({ + deps, + }), + ); + api.registerTool((ctx) => + createLcmUpdatePeerTool({ + deps, + lcm, + sessionId: ctx.sessionId, + sessionKey: ctx.sessionKey, + }), + ); + + // Register hook to auto-extract peer info from inbound messages + api.registerHook(["message_received"], async (event) => { + const meta = event.context?.inboundMeta; + if (!meta) return; + + const sessionId = event.sessionId; + if (!sessionId) return; + + // Extract peer info from inbound metadata + const chatId = meta.chat_id; + const channel = meta.channel; + const chatType = meta.chat_type; + + // For DMs, the peer is the sender + // For groups, the peer is the chat/group + let peerId: string | undefined; + let peerName: string | undefined; + + if (chatType === "dm" && meta.sender_id) { + peerId = `user:${meta.sender_id}`; + peerName = meta.sender_name || meta.sender; + } else if (chatType === "group" && chatId) { + peerId = chatId; + peerName = meta.chat_name || meta.group_name; + } + + if (!peerId) return; + + // Update conversation with peer info + try { + await lcm.updateConversationPeer({ + sessionId, + peerId, + peerName, + channel, + chatType, + }); + deps.log.info(`[lcm] Auto-extracted peer: ${peerId} (${peerName || "unknown"}) for session ${sessionId}`); + } catch (error) { + deps.log.warn(`[lcm] Failed to update peer info: ${error}`); + } + }); api.logger.info( `[lcm] Plugin loaded (enabled=${deps.config.enabled}, db=${deps.config.databasePath}, threshold=${deps.config.contextThreshold})`, diff --git a/package-lock.json b/package-lock.json index c5d8487d..b5b68e7a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@martian-engineering/lossless-claw", - "version": "0.2.8", + "version": "0.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@martian-engineering/lossless-claw", - "version": "0.2.8", + "version": "0.3.0", "license": "MIT", "dependencies": { "@mariozechner/pi-agent-core": "*", diff --git a/src/db/migration.ts b/src/db/migration.ts index 832acdf9..2c358ca4 100644 --- a/src/db/migration.ts +++ b/src/db/migration.ts @@ -360,9 +360,17 @@ export function runLcmMigrations( options?: { fts5Available?: boolean }, ): void { db.exec(` + CREATE TABLE IF NOT EXISTS contacts ( + peer_id TEXT PRIMARY KEY, + peer_name TEXT, + chat_type TEXT CHECK (chat_type IN ('dm', 'group')) + ); + CREATE TABLE IF NOT EXISTS conversations ( conversation_id INTEGER PRIMARY KEY AUTOINCREMENT, session_id TEXT NOT NULL, + peer_id TEXT REFERENCES contacts(peer_id) ON DELETE SET NULL, + channel TEXT, title TEXT, bootstrapped_at TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')), @@ -491,6 +499,30 @@ export function runLcmMigrations( db.exec(`ALTER TABLE conversations ADD COLUMN bootstrapped_at TEXT`); } + // Add peer_id and channel columns to conversations for existing DBs + const hasPeerId = conversationColumns.some((col) => col.name === "peer_id"); + const hasChannel = conversationColumns.some((col) => col.name === "channel"); + if (!hasPeerId) { + db.exec(`ALTER TABLE conversations ADD COLUMN peer_id TEXT REFERENCES contacts(peer_id) ON DELETE SET NULL`); + } + if (!hasChannel) { + db.exec(`ALTER TABLE conversations ADD COLUMN channel TEXT`); + } + + // Ensure contacts table exists for existing DBs + const contactsTable = db + .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='contacts'") + .get(); + if (!contactsTable) { + db.exec(` + CREATE TABLE contacts ( + peer_id TEXT PRIMARY KEY, + peer_name TEXT, + chat_type TEXT CHECK (chat_type IN ('dm', 'group')) + ) + `); + } + ensureSummaryDepthColumn(db); ensureSummaryMetadataColumns(db); backfillSummaryDepths(db); diff --git a/src/engine.ts b/src/engine.ts index 08e6aa00..9e7964bd 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -38,6 +38,7 @@ import { type MessagePartType, } from "./store/conversation-store.js"; import { SummaryStore } from "./store/summary-store.js"; +import { ContactStore, type CreateContactInput } from "./store/contact-store.js"; import { createLcmSummarizeFromLegacyParams } from "./summarize.js"; import type { LcmDependencies } from "./types.js"; @@ -588,6 +589,7 @@ export class LcmContextEngine implements ContextEngine { private conversationStore: ConversationStore; private summaryStore: SummaryStore; + private contactStore: ContactStore; private assembler: ContextAssembler; private compaction: CompactionEngine; private retrieval: RetrievalEngine; @@ -607,6 +609,7 @@ export class LcmContextEngine implements ContextEngine { this.conversationStore = new ConversationStore(db, { fts5Available: this.fts5Available }); this.summaryStore = new SummaryStore(db, { fts5Available: this.fts5Available }); + this.contactStore = new ContactStore(db); if (!this.fts5Available) { this.deps.log.warn( @@ -1219,6 +1222,57 @@ export class LcmContextEngine implements ContextEngine { }); } + /** + * Update conversation with peer information. + * Creates or updates the contact and associates it with the conversation. + * + * @param sessionId - The session ID + * @param peerInfo - Peer information (peerId, peerName, channel, chatType) + */ + async updateConversationPeer(params: { + sessionId: string; + peerId: string; + peerName?: string; + channel?: string; + chatType?: "dm" | "group"; + }): Promise { + this.ensureMigrated(); + + const { sessionId, peerId, peerName, channel, chatType } = params; + + // Create or update contact + this.contactStore.create({ + peerId, + peerName, + chatType, + }); + + // Update conversation with peer info + const conversation = await this.conversationStore.getConversationBySessionId(sessionId); + if (conversation) { + // Only update if not already set + if (!conversation.peerId || !conversation.channel) { + const db = getLcmConnection(this.config.databasePath); + const updates: string[] = []; + const values: (string | null)[] = []; + + if (!conversation.peerId) { + updates.push("peer_id = ?"); + values.push(peerId); + } + if (!conversation.channel && channel) { + updates.push("channel = ?"); + values.push(channel); + } + + if (updates.length > 0) { + values.push(String(conversation.conversationId)); + db.prepare(`UPDATE conversations SET ${updates.join(", ")} WHERE conversation_id = ?`).run(...values); + } + } + } + } + async afterTurn(params: { sessionId: string; sessionFile: string; diff --git a/src/export.ts b/src/export.ts new file mode 100644 index 00000000..97ea13a5 --- /dev/null +++ b/src/export.ts @@ -0,0 +1,191 @@ +import type { DatabaseSync } from "node:sqlite"; +import { mkdirSync, writeFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; + +export type ExportOptions = { + outputDir: string; + peerId?: string; + chatType?: "dm" | "group"; + from?: Date; + to?: Date; +}; + +export type ExportResult = { + filesWritten: number; + messagesExported: number; + outputDir: string; +}; + +interface MessageExportRow { + message_id: number; + conversation_id: number; + peer_id: string | null; + peer_name: string | null; + chat_type: string | null; + channel: string | null; + role: string; + content: string; + created_at: string; +} + +/** + * Format a timestamp for the export line + */ +function formatTime(isoString: string): string { + const date = new Date(isoString); + const hours = date.getHours().toString().padStart(2, "0"); + const minutes = date.getMinutes().toString().padStart(2, "0"); + return `${hours}:${minutes}`; +} + +/** + * Format a date for the filename (YYYY-MM-DD) + */ +function formatDate(isoString: string): string { + const date = new Date(isoString); + const year = date.getFullYear(); + const month = (date.getMonth() + 1).toString().padStart(2, "0"); + const day = date.getDate().toString().padStart(2, "0"); + return `${year}-${month}-${day}`; +} + +/** + * Format a date for the folder name (YYYY-MM) + */ +function formatYearMonth(isoString: string): string { + const date = new Date(isoString); + const year = date.getFullYear(); + const month = (date.getMonth() + 1).toString().padStart(2, "0"); + return `${year}-${month}`; +} + +/** + * Get the peer name for folder structure + */ +function getPeerFolder(peerName: string | null, peerId: string | null, chatType: string | null): string { + if (chatType === "group") { + return peerName || peerId || "unknown-group"; + } + return peerName || peerId || "unknown"; +} + +/** + * Format a single message line + */ +function formatMessageLine(row: MessageExportRow): string { + const channel = row.channel || "unknown"; + const time = formatTime(row.created_at); + const name = row.role === "user" ? (row.peer_name || "User") : "Assistant"; + const content = row.content.replace(/\n/g, " ").trim(); + return `[${channel}] ${time} ${name}: ${content}`; +} + +/** + * Export conversations to markdown files organized by peer and date + */ +export function exportConversations(db: DatabaseSync, options: ExportOptions): ExportResult { + // Build query with filters + let sql = ` + SELECT + m.message_id, + m.conversation_id, + c.peer_id, + ct.peer_name, + ct.chat_type, + c.channel, + m.role, + m.content, + m.created_at + FROM messages m + JOIN conversations c ON c.conversation_id = m.conversation_id + LEFT JOIN contacts ct ON ct.peer_id = c.peer_id + WHERE 1=1 + `; + + const params: (string | number)[] = []; + + if (options.peerId) { + sql += ` AND c.peer_id = ?`; + params.push(options.peerId); + } + + if (options.chatType) { + sql += ` AND ct.chat_type = ?`; + params.push(options.chatType); + } + + if (options.from) { + sql += ` AND m.created_at >= ?`; + params.push(options.from.toISOString()); + } + + if (options.to) { + sql += ` AND m.created_at <= ?`; + params.push(options.to.toISOString()); + } + + sql += ` ORDER BY m.created_at ASC`; + + const stmt = db.prepare(sql); + const rows = stmt.all(...params) as MessageExportRow[]; + + if (rows.length === 0) { + return { + filesWritten: 0, + messagesExported: 0, + outputDir: options.outputDir, + }; + } + + // Group messages by peer and date + const grouped = new Map>(); + + for (const row of rows) { + const chatType = row.chat_type || "dm"; + const peerFolder = getPeerFolder(row.peer_name, row.peer_id, row.chat_type); + const dateFolder = formatYearMonth(row.created_at); + const dateFile = formatDate(row.created_at); + + const key = `${chatType}/${peerFolder}/${dateFolder}`; + if (!grouped.has(key)) { + grouped.set(key, new Map()); + } + const dateMap = grouped.get(key)!; + if (!dateMap.has(dateFile)) { + dateMap.set(dateFile, []); + } + dateMap.get(dateFile)!.push(row); + } + + // Write files + let filesWritten = 0; + let messagesExported = 0; + + for (const [key, dateMap] of grouped) { + const [chatType, peerFolder, dateFolder] = key.split("/"); + const dirPath = join(options.outputDir, chatType, peerFolder, dateFolder); + + for (const [dateFile, messages] of dateMap) { + // Ensure directory exists + if (!existsSync(dirPath)) { + mkdirSync(dirPath, { recursive: true }); + } + + // Build file content + const lines = messages.map(formatMessageLine); + const content = lines.join("\n") + "\n"; + + // Write file + const filePath = join(dirPath, `${dateFile}.md`); + writeFileSync(filePath, content, "utf-8"); + filesWritten++; + messagesExported += messages.length; + } + } + + return { + filesWritten, + messagesExported, + outputDir: options.outputDir, + }; +} diff --git a/src/retrieval.ts b/src/retrieval.ts index 867b0d2c..d0024269 100644 --- a/src/retrieval.ts +++ b/src/retrieval.ts @@ -65,6 +65,9 @@ export interface GrepInput { mode: "regex" | "full_text"; scope: "messages" | "summaries" | "both"; conversationId?: number; + peerId?: string; + channel?: string; + chatType?: "dm" | "group"; since?: Date; before?: Date; limit?: number; @@ -222,9 +225,9 @@ export class RetrievalEngine { * Depending on `scope`, searches messages, summaries, or both (in parallel). */ async grep(input: GrepInput): Promise { - const { query, mode, scope, conversationId, since, before, limit } = input; + const { query, mode, scope, conversationId, peerId, channel, chatType, since, before, limit } = input; - const searchInput = { query, mode, conversationId, since, before, limit }; + const searchInput = { query, mode, conversationId, peerId, channel, chatType, since, before, limit }; let messages: MessageSearchResult[] = []; let summaries: SummarySearchResult[] = []; diff --git a/src/store/contact-store.ts b/src/store/contact-store.ts new file mode 100644 index 00000000..e4991fd5 --- /dev/null +++ b/src/store/contact-store.ts @@ -0,0 +1,91 @@ +import type { DatabaseSync } from "node:sqlite"; + +export type ContactRecord = { + peerId: string; + peerName: string | null; + chatType: "dm" | "group" | null; +}; + +export type CreateContactInput = { + peerId: string; + peerName?: string; + chatType?: "dm" | "group"; +}; + +interface ContactRow { + peer_id: string; + peer_name: string | null; + chat_type: string | null; +} + +function rowToRecord(row: ContactRow): ContactRecord { + return { + peerId: row.peer_id, + peerName: row.peer_name, + chatType: row.chat_type as "dm" | "group" | null, + }; +} + +export class ContactStore { + constructor(private db: DatabaseSync) {} + + create(input: CreateContactInput): ContactRecord { + const stmt = this.db.prepare(` + INSERT INTO contacts (peer_id, peer_name, chat_type) + VALUES (?, ?, ?) + ON CONFLICT(peer_id) DO UPDATE SET + peer_name = COALESCE(excluded.peer_name, contacts.peer_name), + chat_type = COALESCE(excluded.chat_type, contacts.chat_type) + RETURNING peer_id, peer_name, chat_type + `); + const row = stmt.get(input.peerId, input.peerName ?? null, input.chatType ?? null) as ContactRow; + return rowToRecord(row); + } + + get(peerId: string): ContactRecord | null { + const stmt = this.db.prepare(` + SELECT peer_id, peer_name, chat_type FROM contacts WHERE peer_id = ? + `); + const row = stmt.get(peerId) as ContactRow | undefined; + return row ? rowToRecord(row) : null; + } + + getAll(): ContactRecord[] { + const stmt = this.db.prepare(` + SELECT peer_id, peer_name, chat_type FROM contacts ORDER BY peer_name, peer_id + `); + const rows = stmt.all() as ContactRow[]; + return rows.map(rowToRecord); + } + + update(peerId: string, updates: { peerName?: string; chatType?: "dm" | "group" }): ContactRecord | null { + const fields: string[] = []; + const values: (string | null)[] = []; + + if (updates.peerName !== undefined) { + fields.push("peer_name = ?"); + values.push(updates.peerName); + } + if (updates.chatType !== undefined) { + fields.push("chat_type = ?"); + values.push(updates.chatType); + } + + if (fields.length === 0) { + return this.get(peerId); + } + + values.push(peerId); + const stmt = this.db.prepare(` + UPDATE contacts SET ${fields.join(", ")} WHERE peer_id = ? + `); + stmt.run(...values); + return this.get(peerId); + } + + delete(peerId: string): boolean { + const stmt = this.db.prepare(`DELETE FROM contacts WHERE peer_id = ?`); + const result = stmt.run(peerId); + return result.changes > 0; + } +} diff --git a/src/store/conversation-store.ts b/src/store/conversation-store.ts index 0f5b83b4..9a4ba978 100644 --- a/src/store/conversation-store.ts +++ b/src/store/conversation-store.ts @@ -67,12 +67,16 @@ export type MessagePartRecord = { export type CreateConversationInput = { sessionId: string; + peerId?: string; + channel?: string; title?: string; }; export type ConversationRecord = { conversationId: ConversationId; sessionId: string; + peerId: string | null; + channel: string | null; title: string | null; bootstrappedAt: Date | null; createdAt: Date; @@ -83,6 +87,9 @@ export type MessageSearchInput = { conversationId?: ConversationId; query: string; mode: "regex" | "full_text"; + peerId?: string; + channel?: string; + chatType?: "dm" | "group"; since?: Date; before?: Date; limit?: number; @@ -102,6 +109,8 @@ export type MessageSearchResult = { interface ConversationRow { conversation_id: number; session_id: string; + peer_id: string | null; + channel: string | null; title: string | null; bootstrapped_at: string | null; created_at: string; @@ -155,6 +164,8 @@ function toConversationRecord(row: ConversationRow): ConversationRecord { return { conversationId: row.conversation_id, sessionId: row.session_id, + peerId: row.peer_id, + channel: row.channel, title: row.title, bootstrappedAt: row.bootstrapped_at ? new Date(row.bootstrapped_at) : null, createdAt: new Date(row.created_at), @@ -231,12 +242,12 @@ export class ConversationStore { async createConversation(input: CreateConversationInput): Promise { const result = this.db - .prepare(`INSERT INTO conversations (session_id, title) VALUES (?, ?)`) - .run(input.sessionId, input.title ?? null); + .prepare(`INSERT INTO conversations (session_id, peer_id, channel, title) VALUES (?, ?, ?, ?)`) + .run(input.sessionId, input.peerId ?? null, input.channel ?? null, input.title ?? null); const row = this.db .prepare( - `SELECT conversation_id, session_id, title, bootstrapped_at, created_at, updated_at + `SELECT conversation_id, session_id, peer_id, channel, title, bootstrapped_at, created_at, updated_at FROM conversations WHERE conversation_id = ?`, ) .get(Number(result.lastInsertRowid)) as unknown as ConversationRow; @@ -247,7 +258,7 @@ export class ConversationStore { async getConversation(conversationId: ConversationId): Promise { const row = this.db .prepare( - `SELECT conversation_id, session_id, title, bootstrapped_at, created_at, updated_at + `SELECT conversation_id, session_id, peer_id, channel, title, bootstrapped_at, created_at, updated_at FROM conversations WHERE conversation_id = ?`, ) .get(conversationId) as unknown as ConversationRow | undefined; @@ -258,7 +269,7 @@ export class ConversationStore { async getConversationBySessionId(sessionId: string): Promise { const row = this.db .prepare( - `SELECT conversation_id, session_id, title, bootstrapped_at, created_at, updated_at + `SELECT conversation_id, session_id, peer_id, channel, title, bootstrapped_at, created_at, updated_at FROM conversations WHERE session_id = ? ORDER BY created_at DESC @@ -269,12 +280,12 @@ export class ConversationStore { return row ? toConversationRecord(row) : null; } - async getOrCreateConversation(sessionId: string, title?: string): Promise { + async getOrCreateConversation(sessionId: string, title?: string, peerId?: string, channel?: string): Promise { const existing = await this.getConversationBySessionId(sessionId); if (existing) { return existing; } - return this.createConversation({ sessionId, title }); + return this.createConversation({ sessionId, peerId, channel, title }); } async markConversationBootstrapped(conversationId: ConversationId): Promise { @@ -562,6 +573,9 @@ export class ConversationStore { input.query, limit, input.conversationId, + input.peerId, + input.channel, + input.chatType, input.since, input.before, ); @@ -570,14 +584,17 @@ export class ConversationStore { input.query, limit, input.conversationId, + input.peerId, + input.channel, + input.chatType, input.since, input.before, ); } } - return this.searchLike(input.query, limit, input.conversationId, input.since, input.before); + return this.searchLike(input.query, limit, input.conversationId, input.peerId, input.channel, input.chatType, input.since, input.before); } - return this.searchRegex(input.query, limit, input.conversationId, input.since, input.before); + return this.searchRegex(input.query, limit, input.conversationId, input.peerId, input.channel, input.chatType, input.since, input.before); } private indexMessageForFullText(messageId: MessageId, content: string): void { @@ -608,6 +625,9 @@ export class ConversationStore { query: string, limit: number, conversationId?: ConversationId, + peerId?: string, + channel?: string, + chatType?: "dm" | "group", since?: Date, before?: Date, ): MessageSearchResult[] { @@ -617,6 +637,18 @@ export class ConversationStore { where.push("m.conversation_id = ?"); args.push(conversationId); } + if (peerId != null) { + where.push("c.peer_id = ?"); + args.push(peerId); + } + if (channel != null) { + where.push("c.channel = ?"); + args.push(channel); + } + if (chatType != null) { + where.push("ct.chat_type = ?"); + args.push(chatType); + } if (since) { where.push("julianday(m.created_at) >= julianday(?)"); args.push(since.toISOString()); @@ -636,6 +668,8 @@ export class ConversationStore { m.created_at FROM messages_fts JOIN messages m ON m.message_id = messages_fts.rowid + JOIN conversations c ON c.conversation_id = m.conversation_id + LEFT JOIN contacts ct ON ct.peer_id = c.peer_id WHERE ${where.join(" AND ")} ORDER BY m.created_at DESC LIMIT ?`; @@ -647,6 +681,9 @@ export class ConversationStore { query: string, limit: number, conversationId?: ConversationId, + peerId?: string, + channel?: string, + chatType?: "dm" | "group", since?: Date, before?: Date, ): MessageSearchResult[] { @@ -658,15 +695,27 @@ export class ConversationStore { const where: string[] = [...plan.where]; const args: Array = [...plan.args]; if (conversationId != null) { - where.push("conversation_id = ?"); + where.push("m.conversation_id = ?"); args.push(conversationId); } + if (peerId != null) { + where.push("c.peer_id = ?"); + args.push(peerId); + } + if (channel != null) { + where.push("c.channel = ?"); + args.push(channel); + } + if (chatType != null) { + where.push("ct.chat_type = ?"); + args.push(chatType); + } if (since) { - where.push("julianday(created_at) >= julianday(?)"); + where.push("julianday(m.created_at) >= julianday(?)"); args.push(since.toISOString()); } if (before) { - where.push("julianday(created_at) < julianday(?)"); + where.push("julianday(m.created_at) < julianday(?)"); args.push(before.toISOString()); } args.push(limit); @@ -674,10 +723,12 @@ export class ConversationStore { const whereClause = where.length > 0 ? `WHERE ${where.join(" AND ")}` : ""; const rows = this.db .prepare( - `SELECT message_id, conversation_id, seq, role, content, token_count, created_at - FROM messages + `SELECT m.message_id, m.conversation_id, m.seq, m.role, m.content, m.token_count, m.created_at + FROM messages m + JOIN conversations c ON c.conversation_id = m.conversation_id + LEFT JOIN contacts ct ON ct.peer_id = c.peer_id ${whereClause} - ORDER BY created_at DESC + ORDER BY m.created_at DESC LIMIT ?`, ) .all(...args) as unknown as MessageRow[]; @@ -696,6 +747,9 @@ export class ConversationStore { pattern: string, limit: number, conversationId?: ConversationId, + peerId?: string, + channel?: string, + chatType?: "dm" | "group", since?: Date, before?: Date, ): MessageSearchResult[] { @@ -705,24 +759,38 @@ export class ConversationStore { const where: string[] = []; const args: Array = []; if (conversationId != null) { - where.push("conversation_id = ?"); + where.push("m.conversation_id = ?"); args.push(conversationId); } + if (peerId != null) { + where.push("c.peer_id = ?"); + args.push(peerId); + } + if (channel != null) { + where.push("c.channel = ?"); + args.push(channel); + } + if (chatType != null) { + where.push("ct.chat_type = ?"); + args.push(chatType); + } if (since) { - where.push("julianday(created_at) >= julianday(?)"); + where.push("julianday(m.created_at) >= julianday(?)"); args.push(since.toISOString()); } if (before) { - where.push("julianday(created_at) < julianday(?)"); + where.push("julianday(m.created_at) < julianday(?)"); args.push(before.toISOString()); } const whereClause = where.length > 0 ? `WHERE ${where.join(" AND ")}` : ""; const rows = this.db .prepare( - `SELECT message_id, conversation_id, seq, role, content, token_count, created_at - FROM messages + `SELECT m.message_id, m.conversation_id, m.seq, m.role, m.content, m.token_count, m.created_at + FROM messages m + JOIN conversations c ON c.conversation_id = m.conversation_id + LEFT JOIN contacts ct ON ct.peer_id = c.peer_id ${whereClause} - ORDER BY created_at DESC`, + ORDER BY m.created_at DESC`, ) .all(...args) as unknown as MessageRow[]; diff --git a/src/store/index.ts b/src/store/index.ts index 9601e496..84bacac3 100644 --- a/src/store/index.ts +++ b/src/store/index.ts @@ -27,3 +27,9 @@ export type { CreateLargeFileInput, LargeFileRecord, } from "./summary-store.js"; + +export { ContactStore } from "./contact-store.js"; +export type { + ContactRecord, + CreateContactInput, +} from "./contact-store.js"; diff --git a/src/tools/lcm-export-tool.ts b/src/tools/lcm-export-tool.ts new file mode 100644 index 00000000..92a732a5 --- /dev/null +++ b/src/tools/lcm-export-tool.ts @@ -0,0 +1,74 @@ +import { Type } from "@sinclair/typebox"; +import type { LcmDependencies } from "../types.js"; +import type { AnyAgentTool } from "./common.js"; +import { jsonResult } from "./common.js"; +import { exportConversations, type ExportOptions } from "../export.js"; + +const LcmExportSchema = Type.Object({ + outputDir: Type.String({ + description: "Output directory for exported files (default: ./exports)", + default: "./exports", + }), + peerId: Type.Optional( + Type.String({ + description: "Filter by peer ID (e.g., user:ou_xxx, chat:xxx)", + }) + ), + chatType: Type.Optional( + Type.String({ + description: "Filter by chat type: dm or group", + enum: ["dm", "group"], + }) + ), + from: Type.Optional( + Type.String({ + description: "Export messages from this date (ISO format: YYYY-MM-DD)", + }) + ), + to: Type.Optional( + Type.String({ + description: "Export messages until this date (ISO format: YYYY-MM-DD)", + }) + ), +}); + +export function createLcmExportTool(input: { deps: LcmDependencies }): AnyAgentTool { + return { + name: "lcm_export", + label: "LCM Export", + description: + "Export conversations to markdown files organized by peer and date. " + + "Creates files in dm/{peer}/YYYY-MM/YYYY-MM-DD.md and group/{name}/YYYY-MM/YYYY-MM-DD.md format. " + + "Each message line format: [channel] time name: message", + parameters: LcmExportSchema, + async execute(args: Record): Promise { + const db = input.deps.db; + if (!db) { + return jsonResult({ success: false, error: "LCM database not available" }); + } + + const options: ExportOptions = { + outputDir: String(args.outputDir || "./exports"), + peerId: args.peerId ? String(args.peerId) : undefined, + chatType: args.chatType as "dm" | "group" | undefined, + from: args.from ? new Date(String(args.from)) : undefined, + to: args.to ? new Date(String(args.to)) : undefined, + }; + + try { + const result = exportConversations(db, options); + return jsonResult({ + success: true, + messagesExported: result.messagesExported, + filesWritten: result.filesWritten, + outputDir: result.outputDir, + }); + } catch (error) { + return jsonResult({ + success: false, + error: error instanceof Error ? error.message : String(error), + }); + } + }, + }; +} diff --git a/src/tools/lcm-grep-tool.ts b/src/tools/lcm-grep-tool.ts index f93b5281..ba2fc089 100644 --- a/src/tools/lcm-grep-tool.ts +++ b/src/tools/lcm-grep-tool.ts @@ -39,6 +39,22 @@ const LcmGrepSchema = Type.Object({ "Set true to explicitly search across all conversations. Ignored when conversationId is provided.", }), ), + peerId: Type.Optional( + Type.String({ + description: "Filter by peer ID (e.g., user:ou_xxx, chat:xxx).", + }) + ), + channel: Type.Optional( + Type.String({ + description: "Filter by channel (e.g., feishu, telegram).", + }) + ), + chatType: Type.Optional( + Type.String({ + description: "Filter by chat type: dm or group.", + enum: ["dm", "group"], + }) + ), since: Type.Optional( Type.String({ description: "Only return matches created at or after this ISO timestamp.", @@ -91,6 +107,9 @@ export function createLcmGrepTool(input: { const mode = (p.mode as "regex" | "full_text") ?? "regex"; const scope = (p.scope as "messages" | "summaries" | "both") ?? "both"; const limit = typeof p.limit === "number" ? Math.trunc(p.limit) : 50; + const peerId = p.peerId as string | undefined; + const channel = p.channel as string | undefined; + const chatType = p.chatType as "dm" | "group" | undefined; let since: Date | undefined; let before: Date | undefined; try { @@ -125,6 +144,9 @@ export function createLcmGrepTool(input: { mode, scope, conversationId: conversationScope.conversationId, + peerId, + channel, + chatType, limit, since, before, @@ -139,6 +161,13 @@ export function createLcmGrepTool(input: { } else if (conversationScope.conversationId != null) { lines.push(`**Conversation scope:** ${conversationScope.conversationId}`); } + if (peerId || channel || chatType) { + const filters: string[] = []; + if (peerId) filters.push(`peer: ${peerId}`); + if (channel) filters.push(`channel: ${channel}`); + if (chatType) filters.push(`type: ${chatType}`); + lines.push(`**Peer filter:** ${filters.join(" | ")}`); + } if (since || before) { lines.push( `**Time filter:** ${since ? `since ${formatTimestamp(since, timezone)}` : "since -∞"} | ${ diff --git a/src/tools/lcm-update-peer-tool.ts b/src/tools/lcm-update-peer-tool.ts new file mode 100644 index 00000000..6f0a6d1a --- /dev/null +++ b/src/tools/lcm-update-peer-tool.ts @@ -0,0 +1,72 @@ +import { Type } from "@sinclair/typebox"; +import type { LcmContextEngine } from "../engine.js"; +import type { LcmDependencies } from "../types.js"; +import type { AnyAgentTool } from "./common.js"; +import { jsonResult } from "./common.js"; + +const LcmUpdatePeerSchema = Type.Object({ + peerId: Type.String({ + description: "Peer ID (e.g., user:ou_xxx, chat:xxx)", + }), + peerName: Type.Optional( + Type.String({ + description: "Human-readable peer name", + }) + ), + channel: Type.Optional( + Type.String({ + description: "Channel name (e.g., feishu, telegram)", + }) + ), + chatType: Type.Optional( + Type.String({ + description: "Chat type: dm or group", + enum: ["dm", "group"], + }) + ), +}); + +export function createLcmUpdatePeerTool(input: { + deps: LcmDependencies; + lcm: LcmContextEngine; + sessionId?: string; + sessionKey?: string; +}): AnyAgentTool { + return { + name: "lcm_update_peer", + label: "LCM Update Peer", + description: + "Update the current conversation with peer information. " + + "Creates or updates the contact record and associates it with the conversation. " + + "Called automatically when inbound message metadata is available.", + parameters: LcmUpdatePeerSchema, + async execute(_toolCallId, params) { + const sessionId = input.sessionId; + if (!sessionId) { + return jsonResult({ success: false, error: "No session ID available" }); + } + + const p = params as Record; + const peerId = p.peerId as string; + const peerName = p.peerName as string | undefined; + const channel = p.channel as string | undefined; + const chatType = p.chatType as "dm" | "group" | undefined; + + try { + await input.lcm.updateConversationPeer({ + sessionId, + peerId, + peerName, + channel, + chatType, + }); + return jsonResult({ success: true, peerId, peerName, channel, chatType }); + } catch (error) { + return jsonResult({ + success: false, + error: error instanceof Error ? error.message : String(error), + }); + } + }, + }; +} diff --git a/test/export.test.ts b/test/export.test.ts new file mode 100644 index 00000000..05fdb2a3 --- /dev/null +++ b/test/export.test.ts @@ -0,0 +1,210 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { DatabaseSync } from "node:sqlite"; +import { mkdirSync, rmSync, existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { runLcmMigrations } from "../src/db/migration.js"; +import { exportConversations } from "../src/export.js"; +import { ConversationStore } from "../src/store/conversation-store.js"; +import { ContactStore } from "../src/store/contact-store.js"; + +describe("LCM Export", () => { + let db: DatabaseSync; + let tempDir: string; + let outputDir: string; + let conversationStore: ConversationStore; + let contactStore: ContactStore; + + beforeEach(() => { + // Create temp directory + tempDir = join(tmpdir(), `lcm-export-test-${Date.now()}`); + mkdirSync(tempDir, { recursive: true }); + outputDir = join(tempDir, "exports"); + + // Create in-memory database + db = new DatabaseSync(":memory:"); + runLcmMigrations(db); + + conversationStore = new ConversationStore(db); + contactStore = new ContactStore(db); + }); + + afterEach(() => { + db?.close(); + if (existsSync(tempDir)) { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("exports DM conversation to correct directory structure", async () => { + // Create contact + contactStore.create({ + peerId: "user:alice123", + peerName: "Alice", + chatType: "dm", + }); + + // Create conversation + const conv = await conversationStore.createConversation({ + sessionId: "test-session-1", + peerId: "user:alice123", + channel: "feishu", + title: "Chat with Alice", + }); + + // Add messages + await conversationStore.createMessage({ + conversationId: conv.conversationId, + seq: 1, + role: "user", + content: "Hello, how are you?", + tokenCount: 10, + }); + + await conversationStore.createMessage({ + conversationId: conv.conversationId, + seq: 2, + role: "assistant", + content: "I'm doing well, thanks for asking!", + tokenCount: 15, + }); + + // Export + const result = exportConversations(db, { outputDir }); + + expect(result.messagesExported).toBe(2); + expect(result.filesWritten).toBe(1); + + // Verify file exists + const files = findMarkdownFiles(outputDir); + expect(files.length).toBe(1); + expect(files[0]).toContain("dm/Alice/"); + expect(files[0]).toMatch(/\d{4}-\d{2}-\d{2}\.md$/); + + // Verify content format + const content = readFileSync(files[0], "utf-8"); + expect(content).toContain("[feishu]"); + expect(content).toContain("Alice: Hello, how are you?"); + expect(content).toContain("Assistant: I'm doing well"); + }); + + it("exports group conversation to group directory", async () => { + // Create contact + contactStore.create({ + peerId: "chat:project-alpha", + peerName: "Project Alpha", + chatType: "group", + }); + + // Create conversation + const conv = await conversationStore.createConversation({ + sessionId: "test-session-2", + peerId: "chat:project-alpha", + channel: "telegram", + title: "Project Discussion", + }); + + // Add messages + await conversationStore.createMessage({ + conversationId: conv.conversationId, + seq: 1, + role: "user", + content: "Team meeting at 3pm", + tokenCount: 8, + }); + + // Export + const result = exportConversations(db, { outputDir, chatType: "group" }); + + expect(result.messagesExported).toBe(1); + + // Verify file is in group directory + const files = findMarkdownFiles(outputDir); + expect(files[0]).toContain("group/Project Alpha"); + }); + + it("filters by peerId", async () => { + // Create two contacts + contactStore.create({ peerId: "user:alice", peerName: "Alice", chatType: "dm" }); + contactStore.create({ peerId: "user:bob", peerName: "Bob", chatType: "dm" }); + + // Create two conversations + const conv1 = await conversationStore.createConversation({ + sessionId: "session-1", + peerId: "user:alice", + channel: "feishu", + }); + const conv2 = await conversationStore.createConversation({ + sessionId: "session-2", + peerId: "user:bob", + channel: "feishu", + }); + + // Add messages + await conversationStore.createMessage({ + conversationId: conv1.conversationId, + seq: 1, + role: "user", + content: "Message to Alice", + tokenCount: 5, + }); + await conversationStore.createMessage({ + conversationId: conv2.conversationId, + seq: 1, + role: "user", + content: "Message to Bob", + tokenCount: 5, + }); + + // Export only Alice + const result = exportConversations(db, { outputDir, peerId: "user:alice" }); + + expect(result.messagesExported).toBe(1); + const content = readFileSync(findMarkdownFiles(outputDir)[0], "utf-8"); + expect(content).toContain("Message to Alice"); + expect(content).not.toContain("Message to Bob"); + }); + + it("handles missing peer info gracefully", async () => { + // Create conversation without peer + const conv = await conversationStore.createConversation({ + sessionId: "session-no-peer", + channel: "feishu", + }); + + await conversationStore.createMessage({ + conversationId: conv.conversationId, + seq: 1, + role: "user", + content: "Anonymous message", + tokenCount: 5, + }); + + // Export + const result = exportConversations(db, { outputDir }); + + expect(result.messagesExported).toBe(1); + const files = findMarkdownFiles(outputDir); + expect(files[0]).toContain("/unknown/"); + }); +}); + +function findMarkdownFiles(dir: string): string[] { + const files: string[] = []; + + function walk(d: string) { + if (!existsSync(d)) return; + const entries = require("fs").readdirSync(d, { withFileTypes: true }); + for (const entry of entries) { + const path = join(d, entry.name); + if (entry.isDirectory()) { + walk(path); + } else if (entry.name.endsWith(".md")) { + files.push(path); + } + } + } + + walk(dir); + return files; +}