From 6914349562608b38b68407241ac8076b82b40d52 Mon Sep 17 00:00:00 2001 From: Nicolas Marchildon Date: Tue, 2 Jun 2026 11:11:09 -0400 Subject: [PATCH 1/5] analyze: framework core, schema migration, hashing, edge kinds Implements the analyzer framework per docs/analyzer-design-c.md: - Append-only analysis_nodes and typed analysis_edges - analyzer_defs, analyzer_versions, prompt_registry, analyzer_configs - analysis_runs and analysis_progress for incremental cursors - Idempotent input_hash from (analyzer, version, config, prompts, source_set) - Dependency-scoped visibility enforced in run context - Proposal materializer with dedup on (target_type, target_path, severity, normalized title) - Migration 002 adds the new tables and extends messages/proposals - Configured (analyzer_id, config_hash) uniqueness - Edge-kind validation: anchors, consumes, refines, uses_prompt, uses_config, produces - Default analyzer registry: turn-pair-core, turn-pair-llm, session-overview --- src/analyze/analyzers/index.ts | 10 + src/analyze/defaults.ts | 41 +++ src/analyze/edge-kinds.ts | 112 ++++++ src/analyze/framework.ts | 495 +++++++++++++++++++++++++++ src/analyze/input-hash.ts | 156 +++++++++ src/analyze/model-tiers.ts | 38 ++ src/analyze/proposal-materializer.ts | 299 ++++++++++++++++ src/analyze/types.ts | 253 ++++++++++++++ src/db/analysis-queries.ts | 311 +++++++++++++++++ src/db/queries.ts | 62 +++- src/db/schema.ts | 165 ++++++++- 11 files changed, 1935 insertions(+), 7 deletions(-) create mode 100644 src/analyze/analyzers/index.ts create mode 100644 src/analyze/defaults.ts create mode 100644 src/analyze/edge-kinds.ts create mode 100644 src/analyze/framework.ts create mode 100644 src/analyze/input-hash.ts create mode 100644 src/analyze/model-tiers.ts create mode 100644 src/analyze/proposal-materializer.ts create mode 100644 src/analyze/types.ts create mode 100644 src/db/analysis-queries.ts diff --git a/src/analyze/analyzers/index.ts b/src/analyze/analyzers/index.ts new file mode 100644 index 0000000..439784c --- /dev/null +++ b/src/analyze/analyzers/index.ts @@ -0,0 +1,10 @@ +/** + * Re-exports for the bundled analyzers. + * + * Tests and integration code can import everything from this + * single module. + */ + +export { turnPairCoreAnalyzer, TURN_PAIR_CORE_DEF, TURN_PAIR_CORE_VERSION, buildTurnPairNode, type TurnPairNode } from "./turn-pair-core/index.js"; +export { turnPairLlmAnalyzer, TURN_PAIR_LLM_DEF, TURN_PAIR_LLM_VERSION, parseTurnPairLlmResponse } from "./turn-pair-llm/index.js"; +export { sessionOverviewAnalyzer, SESSION_OVERVIEW_DEF, SESSION_OVERVIEW_VERSION, parseReduceResponse } from "./session-overview/index.js"; diff --git a/src/analyze/defaults.ts b/src/analyze/defaults.ts new file mode 100644 index 0000000..fa51d70 --- /dev/null +++ b/src/analyze/defaults.ts @@ -0,0 +1,41 @@ +/** + * Default registry of all bundled analyzers. + * + * Call `registerDefaults(fw)` once at extension load to wire up + * turn-pair-core, turn-pair-llm, and session-overview. Tests can + * call this with a stub LLM; production wires it to a real + * provider via `setDefaultLLMCaller()`. + */ + +import { AnalyzerFramework } from "./framework.js"; +import { turnPairCoreAnalyzer } from "./analyzers/turn-pair-core/index.js"; +import { turnPairLlmAnalyzer } from "./analyzers/turn-pair-llm/index.js"; +import { sessionOverviewAnalyzer } from "./analyzers/session-overview/index.js"; +import type { LLMCaller } from "./types.js"; + +let llmOverride: LLMCaller | null = null; + +/** + * Install a global LLM caller that the default analyzers will use. + * In production this is wired to @earendil-works/pi-ai; in tests + * it's a stub. + */ +export function setDefaultLLMCaller(caller: LLMCaller): void { + llmOverride = caller; +} + +export function getDefaultLLMCaller(): LLMCaller { + if (!llmOverride) { + throw new Error( + "No default LLM caller installed. Call setDefaultLLMCaller() at extension load, " + + "or pass an explicit LLM caller when constructing AnalyzerFramework.", + ); + } + return llmOverride; +} + +export function registerDefaults(fw: AnalyzerFramework): void { + fw.register(turnPairCoreAnalyzer); + fw.register(turnPairLlmAnalyzer); + fw.register(sessionOverviewAnalyzer); +} diff --git a/src/analyze/edge-kinds.ts b/src/analyze/edge-kinds.ts new file mode 100644 index 0000000..7b5da78 --- /dev/null +++ b/src/analyze/edge-kinds.ts @@ -0,0 +1,112 @@ +/** + * Edge kinds in the analysis graph. + * + * Every relationship between analysis nodes and other entities + * (messages, sessions, prompts, configs, other nodes) is expressed + * through a single typed edge table. There are no parent_id columns + * on nodes. + * + * The constants below are the only canonical strings the framework + * will accept or produce. Any other edge_kind value is rejected. + */ + +export const EDGE_KINDS = { + ANCHORS: "anchors", + CONSUMES: "consumes", + REFINES: "refines", + USES_PROMPT: "uses_prompt", + USES_CONFIG: "uses_config", + PRODUCES: "produces", +} as const; + +export type EdgeKind = (typeof EDGE_KINDS)[keyof typeof EDGE_KINDS]; + +export const EDGE_KIND_LIST: readonly EdgeKind[] = [ + EDGE_KINDS.ANCHORS, + EDGE_KINDS.CONSUMES, + EDGE_KINDS.REFINES, + EDGE_KINDS.USES_PROMPT, + EDGE_KINDS.USES_CONFIG, + EDGE_KINDS.PRODUCES, +]; + +export function isEdgeKind(value: unknown): value is EdgeKind { + if (typeof value !== "string") return false; + return (EDGE_KIND_LIST as readonly string[]).includes(value); +} + +/** + * The set of to_ref_kind values the framework accepts. + * Mirrors the schema check constraint on analysis_edges. + */ +export const REF_KINDS = { + MESSAGE: "message", + SESSION: "session", + ANALYSIS_NODE: "analysis_node", + PROMPT_VERSION: "prompt_version", + CONFIG_VERSION: "config_version", +} as const; + +export type RefKind = (typeof REF_KINDS)[keyof typeof REF_KINDS]; + +export const REF_KIND_LIST: readonly RefKind[] = [ + REF_KINDS.MESSAGE, + REF_KINDS.SESSION, + REF_KINDS.ANALYSIS_NODE, + REF_KINDS.PROMPT_VERSION, + REF_KINDS.CONFIG_VERSION, +]; + +export function isRefKind(value: unknown): value is RefKind { + if (typeof value !== "string") return false; + return (REF_KIND_LIST as readonly string[]).includes(value); +} + +/** + * Validate the compatibility of an edge_kind with a to_ref_kind. + * + * - anchors: must point to a message or session + * - consumes: must point to a message or analysis_node + * - refines: must point to an analysis_node + * - uses_prompt: must point to a prompt_version + * - uses_config: must point to a config_version + * - produces: must point to an analysis_node (the produced node) + */ +export function validateEdge(edgeKind: EdgeKind, toRefKind: RefKind): void { + switch (edgeKind) { + case EDGE_KINDS.ANCHORS: + if (toRefKind !== REF_KINDS.MESSAGE && toRefKind !== REF_KINDS.SESSION) { + throw new Error(`edge_kind=anchors requires to_ref_kind in {message, session}, got ${toRefKind}`); + } + return; + case EDGE_KINDS.CONSUMES: + if (toRefKind !== REF_KINDS.MESSAGE && toRefKind !== REF_KINDS.ANALYSIS_NODE) { + throw new Error(`edge_kind=consumes requires to_ref_kind in {message, analysis_node}, got ${toRefKind}`); + } + return; + case EDGE_KINDS.REFINES: + if (toRefKind !== REF_KINDS.ANALYSIS_NODE) { + throw new Error(`edge_kind=refines requires to_ref_kind=analysis_node, got ${toRefKind}`); + } + return; + case EDGE_KINDS.USES_PROMPT: + if (toRefKind !== REF_KINDS.PROMPT_VERSION) { + throw new Error(`edge_kind=uses_prompt requires to_ref_kind=prompt_version, got ${toRefKind}`); + } + return; + case EDGE_KINDS.USES_CONFIG: + if (toRefKind !== REF_KINDS.CONFIG_VERSION) { + throw new Error(`edge_kind=uses_config requires to_ref_kind=config_version, got ${toRefKind}`); + } + return; + case EDGE_KINDS.PRODUCES: + if (toRefKind !== REF_KINDS.ANALYSIS_NODE) { + throw new Error(`edge_kind=produces requires to_ref_kind=analysis_node, got ${toRefKind}`); + } + return; + default: { + const _exhaustive: never = edgeKind; + throw new Error(`unknown edge_kind: ${String(_exhaustive)}`); + } + } +} diff --git a/src/analyze/framework.ts b/src/analyze/framework.ts new file mode 100644 index 0000000..f348532 --- /dev/null +++ b/src/analyze/framework.ts @@ -0,0 +1,495 @@ +/** + * AnalyzerFramework — registers, plans, runs, and materializes + * analyzers over Pi session data. + * + * Design notes: + * + * 1. Idempotency: every node carries an `input_hash` derived from + * (analyzer, version, config, prompts, source_set). Before + * computing, we look it up; if a node exists, we skip. This makes + * re-runs cheap and crash recovery automatic. + * + * 2. Visibility: an analyzer can only see its own nodes and the + * nodes of analyzers listed in its `def.dependencies`. The + * framework enforces this when building the plan and run contexts. + * + * 3. Edges are the source of truth for graph relationships. There + * are no `parent_id` columns on nodes. Anchors, consumes, refines, + * uses_prompt, uses_config, produces are all explicit edge kinds. + * + * 4. Proposals materialize from analysis nodes whose `node_kind` is + * 'proposal' (or whose content_json has an `improvement_proposals` + * array). Dedup is by (target_type, target_path, severity, + * normalize(title)). + * + * 5. Crash recovery: if a process dies mid-run, the run row is + * still 'running' and any in-flight node INSERTs that didn't + * finish leave no row. A subsequent call detects stale running + * runs and either re-runs them or marks them as 'error'. + */ + +import type Database from "better-sqlite3"; +import type { + AnalysisNodeRow, + AnalysisResult, + AnalysisUnit, + Analyzer, + AnalyzerConfig, + AnalyzerPlanContext, + AnalyzerRunContext, + LLMCaller, + LLMRequest, + MessageRow, + ProgressRow, + RunOptions, + RunRow, + RunSummary, +} from "./types.js"; +import { + computeInputHash, + computePromptBundleHash, + computeSourceSetHash, + shortHash, + uuidv7, +} from "./input-hash.js"; +import { + REF_KINDS, + EDGE_KINDS, + isEdgeKind, + isRefKind, + validateEdge, +} from "./edge-kinds.js"; +import { + createRun, + findNodeByInputHash, + insertEdge, + insertNode, + getAllSessionNodes, + getMessage, + getNode, + getProgress, + resolveConfig, + upsertAnalyzerDef, + upsertAnalyzerVersion, + upsertProgress, + updateRun, + registerPrompt, + findStaleRunningRuns, + getAnchoredMessageIds, +} from "../db/analysis-queries.js"; +import { materializeProposalsFromNode } from "./proposal-materializer.js"; + +export interface FrameworkDeps { + db: Database.Database; + llm: LLMCaller; +} + +export class AnalyzerFramework { + private readonly analyzers = new Map(); + + constructor(private readonly deps: FrameworkDeps) {} + + register(analyzer: Analyzer): void { + // Idempotent registration + if (this.analyzers.has(analyzer.def.id)) return; + upsertAnalyzerDef(this.deps.db, analyzer.def); + upsertAnalyzerVersion(this.deps.db, analyzer.version); + for (const prompt of Object.values(analyzer.prompts)) { + registerPrompt(this.deps.db, prompt); + } + this.analyzers.set(analyzer.def.id, analyzer); + } + + get(id: string): Analyzer | undefined { + return this.analyzers.get(id); + } + + list(): Analyzer[] { + return [...this.analyzers.values()]; + } + + /** + * Run one analyzer against one session with a given config (or default). + * Returns a RunSummary. + */ + async run( + analyzerId: string, + sessionId: string, + opts: RunOptions & { model?: string; configOverride?: Record } = {}, + ): Promise { + const analyzer = this.analyzers.get(analyzerId); + if (!analyzer) throw new Error(`Analyzer not registered: ${analyzerId}`); + + const config = resolveConfig(this.deps.db, { + analyzerId: analyzer.def.id, + configJson: opts.configOverride ?? analyzer.defaultConfig.configJson, + label: analyzer.defaultConfig.label, + }); + + const promptBundleHash = computePromptBundleHash( + Object.values(analyzer.prompts).map((p) => p.hash), + ); + + // Build plan context + const messages = this.loadMessages(sessionId); + const allNodes = getAllSessionNodes(this.deps.db, sessionId); + const ownNodes = allNodes.filter((n) => n.analyzer_id === analyzer.def.id); + const dependencyNodes = this.buildDependencyNodes(analyzer, allNodes); + const progress = getProgress(this.deps.db, { + analyzerId: analyzer.def.id, + analyzerVersionId: analyzer.version.versionId, + configId: config.id, + sessionId, + }) ?? null; + + const planCtx: AnalyzerPlanContext = { + sessionId, + messages, + allNodes, + ownNodes, + dependencyNodes, + progress, + db: this.deps.db, + }; + + const units = await analyzer.plan(planCtx); + + // Pre-filter: drop units whose source_set_hash matches an + // existing node UNLESS opts.force is set. + const todoUnits: AnalysisUnit[] = []; + let nodesSkipped = 0; + for (const unit of units) { + if (!opts.force) { + const inputHash = computeInputHash({ + analyzerId: analyzer.def.id, + analyzerVersionId: analyzer.version.versionId, + configId: config.id, + promptBundleHash, + sourceSetHash: unit.sourceSetHash, + }); + if (findNodeByInputHash(this.deps.db, inputHash)) { + nodesSkipped++; + continue; + } + } + todoUnits.push(unit); + } + + // Create the run row + const runId = uuidv7(); + createRun(this.deps.db, { + id: runId, + analyzerId: analyzer.def.id, + analyzerVersionId: analyzer.version.versionId, + configId: config.id, + sessionId, + promptBundleHash, + modelSpec: opts.model ?? undefined, + }); + + let nodesProduced = 0; + let costUsd = 0; + let tokensUsed = 0; + let lastError: string | null = null; + const status: "ok" | "error" | "partial" = "ok"; + + try { + upsertProgress(this.deps.db, { + analyzerId: analyzer.def.id, + analyzerVersionId: analyzer.version.versionId, + configId: config.id, + sessionId, + cursorJson: JSON.stringify({ planned: todoUnits.length }), + lastRunId: runId, + totalAnalyzed: 0, + status: "in_progress", + errorMessage: null, + }); + + for (let i = 0; i < todoUnits.length; i++) { + const unit = todoUnits[i]!; + try { + const result = await analyzer.analyze(unit, this.buildRunContext(analyzer, config, runId, promptsByName(analyzer), sessionId)); + const inputHash = computeInputHash({ + analyzerId: analyzer.def.id, + analyzerVersionId: analyzer.version.versionId, + configId: config.id, + promptBundleHash, + sourceSetHash: unit.sourceSetHash, + }); + + const nodeId = uuidv7(); + const now = new Date().toISOString(); + insertNode(this.deps.db, { + id: nodeId, + sessionId, + analyzerId: analyzer.def.id, + analyzerVersionId: analyzer.version.versionId, + configId: config.id, + runId, + nodeKind: result.nodeKind, + contentJson: JSON.stringify(result.contentJson), + sourceSetHash: unit.sourceSetHash, + inputHash, + modelUsed: result.modelUsed, + costUsd: result.costUsd, + tokensUsed: result.tokensUsed, + durationMs: result.durationMs, + createdAt: now, + }); + + // Insert edges + for (let e = 0; e < result.edges.length; e++) { + const edge = result.edges[e]!; + if (!isEdgeKind(edge.edgeKind)) { + throw new Error(`Analyzer returned invalid edge_kind: ${String(edge.edgeKind)}`); + } + if (!isRefKind(edge.toRefKind)) { + throw new Error(`Analyzer returned invalid to_ref_kind: ${String(edge.toRefKind)}`); + } + validateEdge(edge.edgeKind, edge.toRefKind); + insertEdge(this.deps.db, { + fromNodeId: nodeId, + toRefKind: edge.toRefKind, + toRefId: edge.toRefId, + edgeKind: edge.edgeKind, + ordinal: edge.ordinal ?? e, + }); + } + + // If the node anchors to a session, add the session anchor + // (the spec's "anchors" edges for session-anchored nodes + // are explicit, but we add it if not present so traversal + // queries always work). + if (result.anchorKind === "session" && result.anchorRef) { + insertEdge(this.deps.db, { + fromNodeId: nodeId, + toRefKind: REF_KINDS.SESSION, + toRefId: result.anchorRef, + edgeKind: EDGE_KINDS.ANCHORS, + ordinal: 999, + }); + } + + // Materialize any proposals embedded in this node + if (result.nodeKind === "summary" || result.nodeKind === "proposal") { + materializeProposalsFromNode(this.deps.db, { + sessionId, + analyzerId: analyzer.def.id, + analyzerVersionId: analyzer.version.versionId, + configId: config.id, + runId, + sourceNodeId: nodeId, + sourceSetHash: unit.sourceSetHash, + promptBundleHash, + contentJson: result.contentJson, + now, + }); + } + + if (result.costUsd) costUsd += result.costUsd; + if (result.tokensUsed) tokensUsed += result.tokensUsed; + nodesProduced++; + } catch (err) { + lastError = err instanceof Error ? err.message : String(err); + // Insert an error node so the unit isn't retried blindly + try { + const errorHash = computeInputHash({ + analyzerId: analyzer.def.id, + analyzerVersionId: analyzer.version.versionId, + configId: config.id, + promptBundleHash, + sourceSetHash: unit.sourceSetHash, + }); + const now = new Date().toISOString(); + insertNode(this.deps.db, { + id: uuidv7(), + sessionId, + analyzerId: analyzer.def.id, + analyzerVersionId: analyzer.version.versionId, + configId: config.id, + runId, + nodeKind: "error", + contentJson: JSON.stringify({ error: lastError, unit_meta: unit.meta ?? null }), + sourceSetHash: unit.sourceSetHash, + inputHash: errorHash, + createdAt: now, + }); + } catch { /* ignore secondary error */ } + } + } + + updateRun(this.deps.db, runId, { + status: "ok", + finishedAt: new Date().toISOString(), + costUsd, + tokensUsed, + nodesProduced, + nodesSkipped, + }); + + upsertProgress(this.deps.db, { + analyzerId: analyzer.def.id, + analyzerVersionId: analyzer.version.versionId, + configId: config.id, + sessionId, + cursorJson: JSON.stringify({ completed: todoUnits.length, skipped: nodesSkipped }), + lastRunId: runId, + totalAnalyzed: nodesProduced, + status: "ok", + errorMessage: null, + }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + updateRun(this.deps.db, runId, { + status: "error", + finishedAt: new Date().toISOString(), + costUsd, + tokensUsed, + nodesProduced, + nodesSkipped, + errorMessage: msg, + }); + upsertProgress(this.deps.db, { + analyzerId: analyzer.def.id, + analyzerVersionId: analyzer.version.versionId, + configId: config.id, + sessionId, + cursorJson: null, + lastRunId: runId, + totalAnalyzed: nodesProduced, + status: "error", + errorMessage: msg, + }); + return { + runId, + analyzerId: analyzer.def.id, + analyzerVersionId: analyzer.version.versionId, + sessionId, + status: "error", + nodesProduced, + nodesSkipped, + costUsd, + tokensUsed, + }; + } + + return { + runId, + analyzerId: analyzer.def.id, + analyzerVersionId: analyzer.version.versionId, + sessionId, + status, + nodesProduced, + nodesSkipped, + costUsd, + tokensUsed, + }; + } + + /** + * Find stale 'running' rows that have no recent activity. Mark + * them as 'error' so they don't block idempotent re-runs. Returns + * the number of runs marked. + */ + recoverStaleRuns(): number { + const stale = findStaleRunningRuns(this.deps.db); + for (const r of stale) { + updateRun(this.deps.db, r.id, { + status: "error", + finishedAt: new Date().toISOString(), + errorMessage: "Marked stale by recoverStaleRuns (no recent activity)", + }); + } + return stale.length; + } + + // ── internals ── + + private loadMessages(sessionId: string): MessageRow[] { + return this.deps.db.prepare(` + SELECT id, session_id, parent_id, timestamp, role, + content_text, content_thinking, tool_calls, tool_results, meta_json + FROM messages WHERE session_id = ? ORDER BY rowid ASC + `).all(sessionId) as MessageRow[]; + } + + private buildDependencyNodes(analyzer: Analyzer, allNodes: AnalysisNodeRow[]): Record { + const out: Record = {}; + for (const depId of analyzer.def.dependencies) { + out[depId] = allNodes.filter((n) => n.analyzer_id === depId); + } + return out; + } + + private buildRunContext( + analyzer: Analyzer, + config: AnalyzerConfig, + runId: string, + prompts: Record, + sessionId: string, + ): AnalyzerRunContext { + const self = this; + const runRow: RunRow = { + id: runId, + analyzer_id: analyzer.def.id, + analyzer_version_id: analyzer.version.versionId, + config_id: config.id, + session_id: sessionId, + status: "running", + prompt_bundle_hash: computePromptBundleHash(Object.values(analyzer.prompts).map((p) => p.hash)), + started_at: new Date().toISOString(), + finished_at: null, + model_spec: null, + cost_usd: 0, + tokens_used: 0, + nodes_produced: 0, + nodes_skipped: 0, + error_message: null, + }; + + return { + getMessage: (id) => getMessage(self.deps.db, id), + getNode: (id) => getNode(self.deps.db, id), + getDependencyNodes: (depId) => { + if (!analyzer.def.dependencies.includes(depId)) { + throw new Error( + `Analyzer ${analyzer.def.id} tried to read dependency ${depId} but did not declare it. ` + + `Add it to def.dependencies.`, + ); + } + return self.deps.db.prepare(` + SELECT * FROM analysis_nodes WHERE analyzer_id = ? AND session_id = ? + `).all(depId, sessionId) as AnalysisNodeRow[]; + }, + getAnchoredMessages: (nodeId) => { + const ids = getAnchoredMessageIds(self.deps.db, nodeId); + return ids + .map((id) => getMessage(self.deps.db, id)) + .filter((m): m is MessageRow => m !== undefined); + }, + getSessionMessages: (sid) => self.loadMessages(sid), + llm: (request: LLMRequest) => self.deps.llm(request), + run: runRow, + config, + prompts, + }; + } +} + +function promptsByName(analyzer: Analyzer): Record { + const out: Record = {}; + for (const [name, p] of Object.entries(analyzer.prompts)) { + out[name] = p.content; + } + return out; +} + +// Re-export hashing helpers used by analyzers +export { + computeInputHash, + computeSourceSetHash, + computePromptBundleHash, + shortHash, + uuidv7, +}; diff --git a/src/analyze/input-hash.ts b/src/analyze/input-hash.ts new file mode 100644 index 0000000..52b3562 --- /dev/null +++ b/src/analyze/input-hash.ts @@ -0,0 +1,156 @@ +/** + * Hashing primitives for the analyzer framework. + * + * Three different hashes are computed for a node: + * + * source_set_hash -- SHA-256 of the sorted source refs (what went in) + * prompt_bundle_hash -- SHA-256 of sorted prompt hashes used + * input_hash -- SHA-256(analyzer_id | version_id | config_id + * | prompt_bundle_hash | source_set_hash) + * + * The input_hash uniquely identifies a node produced by a given recipe + * on a given source set, regardless of which model produced it. The + * model is metadata on the analysis_run, not part of the recipe. + * + * All hashes are the first 16 hex chars of SHA-256 (64 bits). This is + * ample for a local-first indexer. The full 64-hex digest is also + * computed when needed for the prompt_registry, but the first-16 form + * is what gets stored on analysis_nodes rows. + */ + +import { createHash, randomUUID } from "node:crypto"; + +/** Return the first 16 hex chars of SHA-256(input). */ +export function shortHash(input: string): string { + return createHash("sha256").update(input).digest("hex").slice(0, 16); +} + +/** Return the full 64 hex chars of SHA-256(input). */ +export function fullHash(input: string): string { + return createHash("sha256").update(input).digest("hex"); +} + +/** + * Hash a list of {kind, id} refs in a canonical, sort-stable way so + * that {ref1, ref2} and {ref2, ref1} produce the same digest. + */ +export function computeSourceSetHash(refs: ReadonlyArray<{ kind: string; id: string }>): string { + const sorted = [...refs] + .map((r) => `${r.kind}:${r.id}`) + .sort(); + return shortHash(sorted.join("|")); +} + +/** + * Hash a list of prompt hashes (the content addresses of the prompts + * an analyzer used for a run). Analyzers may use multiple prompts + * (e.g. map + reduce); the bundle is the sorted concatenation. + */ +export function computePromptBundleHash(promptHashes: ReadonlyArray): string { + const sorted = [...promptHashes].sort(); + return shortHash(sorted.join("|")); +} + +/** + * Compute the recipe hash: the unique identity of a node produced by + * (analyzer_id, version_id, config_id, prompts) on (source set). + */ +export function computeInputHash(args: { + analyzerId: string; + analyzerVersionId: string; + configId: string; + promptBundleHash: string; + sourceSetHash: string; +}): string { + const joined = [ + args.analyzerId, + args.analyzerVersionId, + args.configId, + args.promptBundleHash, + args.sourceSetHash, + ].join("|"); + return shortHash(joined); +} + +/** + * Hash a config object in a stable way: sort keys recursively so + * {"a":1,"b":2} and {"b":2,"a":1} hash the same. + */ +export function computeConfigHash(config: Record): string { + return shortHash(canonicalJsonStringify(config)); +} + +/** + * Stable JSON serialization with sorted keys at every level. + * Used for hashing configs and content_json. + */ +export function canonicalJsonStringify(value: unknown): string { + return JSON.stringify(sortValue(value)); +} + +function sortValue(v: unknown): unknown { + if (Array.isArray(v)) return v.map(sortValue); + if (v && typeof v === "object") { + const obj = v as Record; + const out: Record = {}; + for (const key of Object.keys(obj).sort()) { + out[key] = sortValue(obj[key]); + } + return out; + } + return v; +} + +/** + * Generate a time-sortable UUID (UUIDv7-like). + * + * The first 48 bits encode the current millisecond timestamp in + * big-endian. This means rows inserted in time order naturally sort + * by primary key, which is convenient for cursors and progress. + * + * We don't depend on `crypto.randomUUID` being v7 — Node 22's is v4. + * We construct v7 ourselves so the framework doesn't have to wait for + * a runtime upgrade. + */ +export function uuidv7(): string { + const buf = new Uint8Array(16); + // 48-bit timestamp (ms) + const ts = Date.now(); + buf[0] = (ts / 2 ** 40) & 0xff; + buf[1] = (ts / 2 ** 32) & 0xff; + buf[2] = (ts / 2 ** 24) & 0xff; + buf[3] = (ts / 2 ** 16) & 0xff; + buf[4] = (ts / 2 ** 8) & 0xff; + buf[5] = ts & 0xff; + // version (7) and variant (10xx) + buf[6] = 0x70 | (Math.random() * 0x0f) & 0x0f; + buf[7] = 0x80 | (Math.random() * 0x3f) & 0x3f; + // random tail + for (let i = 8; i < 16; i++) buf[i] = Math.floor(Math.random() * 256); + const hex = [...buf].map((b) => b.toString(16).padStart(2, "0")).join(""); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20, 32)}`; +} + +/** A wrapper used for tests so we can stub the clock. */ +export function newId(): string { + return uuidv7(); +} + +/** + * Determine a stable config_id for an analyzer. The default is a + * v7 UUID, but if the config has a 'label' field we mix it in so + * the same labeled config always gets the same id within a session + * (callers can override the strategy). + */ +export function newConfigId(_config: Record): string { + return uuidv7(); +} + +/** + * Generate a randomUUID (Node's, which is v4) for analysis_run ids. + * We don't need time-ordering for runs — the started_at column is + * what we sort by. + */ +export function newRunId(): string { + return randomUUID(); +} diff --git a/src/analyze/model-tiers.ts b/src/analyze/model-tiers.ts new file mode 100644 index 0000000..8e893b7 --- /dev/null +++ b/src/analyze/model-tiers.ts @@ -0,0 +1,38 @@ +/** + * Model tier resolution. + * + * Analyzers don't ask for a specific model — they ask for a tier + * (cheap, mid, expensive). The framework resolves the tier to a + * concrete model string from the user's prospector.json config. + * + * Example config: + * { + * "model": "openrouter/deepseek-v4-flash", + * "models": { + * "cheap": "openrouter/deepseek-v4-flash", + * "mid": "openrouter/deepseek-v4-pro", + * "expensive": "anthropic/claude-opus-4" + * } + * } + * + * If a tier is missing, we fall back to the default model. + */ + +import type { ModelTier, ModelTierConfig } from "./types.js"; + +export interface ProspectorModelConfig { + model?: string; + models?: Partial; +} + +export function resolveModelTier( + tier: ModelTier, + config: ProspectorModelConfig, +): string { + const explicit = config.models?.[tier]; + if (explicit) return explicit; + if (config.model) return config.model; + throw new Error( + `No model configured for tier=${tier}. Set 'model' or 'models.${tier}' in prospector.json.`, + ); +} diff --git a/src/analyze/proposal-materializer.ts b/src/analyze/proposal-materializer.ts new file mode 100644 index 0000000..d4ebf2b --- /dev/null +++ b/src/analyze/proposal-materializer.ts @@ -0,0 +1,299 @@ +/** + * Materialize proposals from analysis nodes into the `proposals` table. + * + * When an analyzer produces a node with `node_kind = 'proposal'`, or + * a summary node whose `content_json.improvement_proposals` array + * contains proposal-shaped entries, the framework extracts them, + * computes a dedup_key, and upserts into `proposals`. + * + * Dedup rule: an OPEN proposal with the same `dedup_key` already + * exists → mark the new one as 'duplicate' (no insert, just track). + * The original is preserved. + */ + +import type Database from "better-sqlite3"; +import { computeInputHash, fullHash, newId, shortHash } from "./input-hash.js"; +import { REF_KINDS, EDGE_KINDS } from "./edge-kinds.js"; + +export interface ProposalShape { + target_type: string; + target_path: string; + title: string; + summary: string; + detail: string; + evidence: string; + confidence: number; + severity: string; +} + +export interface MaterializedProposal { + proposalId: string; + analysisNodeId: string; + sessionId: string; + analyzerId: string; + dedupKey: string; + dedupHit: boolean; +} + +const VALID_TARGET_TYPES = new Set([ + "agents_md", + "system_md", + "skill", + "extension_prompt", + "tool_output", + "repo_doc", + "config", +]); + +const VALID_SEVERITIES = new Set([ + "friction", + "correction", + "waste", + "suggestion", + "insight", +]); + +/** + * Normalize a title for dedup. Whitespace, case, and trailing + * punctuation are removed so the same idea in different forms + * collides on one key. + */ +export function normalizeTitle(title: string): string { + return title + .toLowerCase() + .replace(/\s+/g, " ") + .replace(/[.!?,;:]+$/g, "") + .trim(); +} + +export function computeProposalDedupKey(p: Pick): string { + const joined = [p.target_type, p.target_path, p.severity, normalizeTitle(p.title)].join("|"); + return shortHash(joined); +} + +/** + * Determine if `obj` looks like a ProposalShape. We don't strictly + * require every field to be present — the LLM may omit some — but + * the required ones (target_type, title, summary) must be valid. + */ +export function isProposalShape(obj: unknown): obj is ProposalShape { + if (!obj || typeof obj !== "object") return false; + const o = obj as Record; + if (typeof o.target_type !== "string" || o.target_type.length === 0) return false; + if (typeof o.title !== "string" || o.title.length === 0) return false; + if (typeof o.summary !== "string" || o.summary.length === 0) return false; + return true; +} + +export function normalizeProposalShape(obj: Record): ProposalShape { + const targetType = VALID_TARGET_TYPES.has(obj.target_type as string) + ? (obj.target_type as string) + : "repo_doc"; + const severity = VALID_SEVERITIES.has(obj.severity as string) + ? (obj.severity as string) + : "suggestion"; + + const confidence = typeof obj.confidence === "number" && obj.confidence >= 0 && obj.confidence <= 1 + ? obj.confidence + : 0.5; + + return { + target_type: targetType, + target_path: typeof obj.target_path === "string" ? obj.target_path : "", + title: String(obj.title), + summary: String(obj.summary), + detail: typeof obj.detail === "string" ? obj.detail : "", + evidence: typeof obj.evidence === "string" ? obj.evidence : "", + confidence, + severity, + }; +} + +/** + * Insert one materialized proposal and the edges that connect it + * back to the producing node and to the session. Idempotent on + * `(analysis_node_id)` — re-materializing the same source node + * yields the same proposal_id. + * + * Returns the proposal row id and whether this call hit an existing + * open dedup match. + */ +export function materializeProposal( + db: Database.Database, + args: { + sessionId: string; + analyzerId: string; + sourceNodeId: string; // the analysis_node that produced this proposal + shape: ProposalShape; + proposalNodeId?: string; // optional pre-existing proposal analysis_node + }, +): MaterializedProposal { + const dedupKey = computeProposalDedupKey(args.shape); + + // 1. Check for an open dedup match + const existing = db.prepare(` + SELECT id FROM proposals + WHERE dedup_key = ? AND status = 'open' + LIMIT 1 + `).get(dedupKey) as { id: string } | undefined; + + if (existing) { + // Edge from source → existing proposal as 'produces' reference. + // We don't create a new row; the source already points to the + // canonical proposal via the produces edge. + return { + proposalId: existing.id, + analysisNodeId: args.proposalNodeId ?? existing.id, + sessionId: args.sessionId, + analyzerId: args.analyzerId, + dedupKey, + dedupHit: true, + }; + } + + const proposalNodeId = args.proposalNodeId ?? newId(); + const now = new Date().toISOString(); + + // 2. Insert into proposals (UNIQUE on analysis_node_id enforces 1:1). + // We populate both the legacy columns (target/severity/summary/dedup_hash) + // and the new ones (target_type/target_path/title/etc.) so the + // `/prospect-proposals` command continues to work without JOINs. + db.prepare(` + INSERT OR IGNORE INTO proposals ( + id, created_at, session_id, target, severity, summary, detail, evidence, + status, dedup_hash, analysis_node_id, analyzer_id, target_type, + target_path, title, evidence_json, confidence, dedup_key, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'open', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run( + proposalNodeId, + now, + args.sessionId, + `${args.shape.target_type}:${args.shape.target_path || "untitled"}`, + args.shape.severity, + args.shape.summary, + args.shape.detail, + args.shape.evidence, + dedupKey, + proposalNodeId, + args.analyzerId, + args.shape.target_type, + args.shape.target_path, + args.shape.title, + JSON.stringify({ text: args.shape.evidence }), + args.shape.confidence, + dedupKey, + now, + ); + + return { + proposalId: proposalNodeId, + analysisNodeId: proposalNodeId, + sessionId: args.sessionId, + analyzerId: args.analyzerId, + dedupKey, + dedupHit: false, + }; +} + +/** + * Materialize every proposal in a node's `content_json.improvement_proposals` + * array. Each proposal gets its own `analysis_node` row of kind + * 'proposal' and a row in the `proposals` table. + * + * Edges inserted per proposal: + * source_node --produces--> proposal_node + * proposal_node --anchors--> session + * + * Returns the list of materialized proposals. + */ +export function materializeProposalsFromNode( + db: Database.Database, + args: { + sessionId: string; + analyzerId: string; + analyzerVersionId: string; + configId: string; + runId: string; + sourceNodeId: string; + sourceSetHash: string; + promptBundleHash: string; + contentJson: Record; + now: string; + }, +): MaterializedProposal[] { + const list = args.contentJson.improvement_proposals; + if (!Array.isArray(list) || list.length === 0) return []; + + const out: MaterializedProposal[] = []; + for (const raw of list) { + if (!isProposalShape(raw)) continue; + const shape = normalizeProposalShape(raw as unknown as Record); + + const proposalNodeId = newId(); + // The proposal node's recipe is (analyzer, version, config, prompts, source). + // Its content_json is the proposal shape itself, so input_hash is + // stable per (analyzer, source). + const proposalContent = { + target_type: shape.target_type, + target_path: shape.target_path, + title: shape.title, + summary: shape.summary, + severity: shape.severity, + }; + const proposalSourceSetHash = shortHash(`${args.sourceSetHash}|${shape.title}`); + const proposalInputHash = computeInputHash({ + analyzerId: args.analyzerId, + analyzerVersionId: args.analyzerVersionId, + configId: args.configId, + promptBundleHash: args.promptBundleHash, + sourceSetHash: proposalSourceSetHash, + }); + + // 1. Insert the proposal analysis_node + db.prepare(` + INSERT OR IGNORE INTO analysis_nodes ( + id, session_id, analyzer_id, analyzer_version_id, config_id, run_id, + node_kind, content_json, source_set_hash, input_hash, created_at + ) VALUES (?, ?, ?, ?, ?, ?, 'proposal', ?, ?, ?, ?) + `).run( + proposalNodeId, + args.sessionId, + args.analyzerId, + args.analyzerVersionId, + args.configId, + args.runId, + JSON.stringify(proposalContent), + proposalSourceSetHash, + proposalInputHash, + args.now, + ); + + // 2. Materialize into proposals table + const materialized = materializeProposal(db, { + sessionId: args.sessionId, + analyzerId: args.analyzerId, + sourceNodeId: args.sourceNodeId, + shape, + proposalNodeId, + }); + out.push(materialized); + + // 3. Edges: source --produces--> proposal_node + // and: proposal_node --anchors--> session + if (!materialized.dedupHit) { + db.prepare(` + INSERT OR IGNORE INTO analysis_edges (from_node_id, to_ref_kind, to_ref_id, edge_kind, ordinal) + VALUES (?, ?, ?, ?, 0) + `).run(args.sourceNodeId, REF_KINDS.ANALYSIS_NODE, proposalNodeId, EDGE_KINDS.PRODUCES); + + db.prepare(` + INSERT OR IGNORE INTO analysis_edges (from_node_id, to_ref_kind, to_ref_id, edge_kind, ordinal) + VALUES (?, ?, ?, ?, 0) + `).run(proposalNodeId, REF_KINDS.SESSION, args.sessionId, EDGE_KINDS.ANCHORS); + } + } + return out; +} + +// Re-export for convenience +export { fullHash }; diff --git a/src/analyze/types.ts b/src/analyze/types.ts new file mode 100644 index 0000000..d251061 --- /dev/null +++ b/src/analyze/types.ts @@ -0,0 +1,253 @@ +/** + * Public type definitions for the analyzer framework. + * + * The framework exposes a small surface: + * - Analyzer implementations conform to the Analyzer interface + * - The framework's run() executes plan/analyze, inserts nodes, + * inserts edges, materializes proposals, and updates cursors + * - AnalyzerPlanContext / AnalyzerRunContext provide scoped + * access to messages, own nodes, and dependency nodes + * + * All shapes are plain TypeScript interfaces. The runtime values are + * stored as TEXT/JSON in SQLite; TypeBox is reserved for Pi tool + * registration where the host SDK requires it. + */ + +import type Database from "better-sqlite3"; +import type { MessageRole } from "../types.js"; +import type { EdgeKind, RefKind } from "./edge-kinds.js"; +import { EDGE_KINDS, REF_KINDS } from "./edge-kinds.js"; + +// ── Analyzer definition ── + +export interface AnalyzerDef { + id: string; + label: string; + description: string; + anchorSpan: "pair" | "segment" | "full_session"; + dependencies: string[]; +} + +export type ImplementationKind = "deterministic" | "in_process_llm" | "pi_subagent"; + +export interface AnalyzerVersion { + analyzerId: string; + versionId: string; + implementationKind: ImplementationKind; + codeRef?: string; +} + +export interface PromptVersion { + /** Content hash (first 16 hex chars of SHA-256). */ + hash: string; + /** Full prompt template text. */ + content: string; + /** Full 64-hex SHA-256 for verification. */ + fullHash: string; + role?: "classify" | "map" | "reduce" | "verify"; +} + +export interface AnalyzerConfig { + id: string; + analyzerId: string; + configJson: Record; + configHash: string; + label?: string; +} + +// ── Source references ── + +export interface SourceRef { + kind: "message" | "analysis_node" | "session"; + id: string; +} + +// ── Analysis unit (input to analyze()) ── + +export interface AnalysisUnit { + sources: SourceRef[]; + sourceSetHash: string; + /** What kind of conversation entity this unit targets. */ + anchorKind: "message" | "pair" | "segment" | "session" | "analysis_node" | "none"; + /** The id of the anchor (message.id or session.id), null for 'none'. */ + anchorRef?: string; + meta?: Record; +} + +// ── Analysis result (output of analyze()) ── + +export interface AnalysisResult { + contentJson: Record; + nodeKind: "metric" | "classification" | "summary" | "proposal" | "error"; + /** What kind of conversation entity this node is about. */ + anchorKind: "message" | "pair" | "segment" | "session" | "analysis_node" | "none"; + /** The id of the anchor (message.id or session.id), null for 'none'. */ + anchorRef?: string; + edges: Array<{ + toRefKind: RefKind; + toRefId: string; + edgeKind: EdgeKind; + ordinal?: number; + }>; + modelUsed?: string; + costUsd?: number; + tokensUsed?: number; + durationMs?: number; +} + +// ── LLM abstraction ── +// +// The framework passes an llm() function into the run context. +// In production this is wired to @earendil-works/pi-ai; in tests it +// is a stub that returns canned responses. + +export interface LLMRequest { + model: string; + system?: string; + user: string; + jsonSchema?: Record; + temperature?: number; + maxTokens?: number; +} + +export interface LLMResponse { + text: string; + model: string; + costUsd: number; + tokensUsed: number; + durationMs: number; +} + +export type LLMCaller = (request: LLMRequest) => Promise; + +// ── Database row types (read-only views used by analyzers) ── + +export interface MessageRow { + id: string; + session_id: string; + parent_id: string | null; + timestamp: string | null; + role: MessageRole; + content_text: string | null; + content_thinking: string | null; + tool_calls: string | null; + tool_results: string | null; + meta_json: string | null; +} + +export interface AnalysisNodeRow { + id: string; + session_id: string; + analyzer_id: string; + analyzer_version_id: string; + config_id: string; + run_id: string; + node_kind: string; + content_json: string; + source_set_hash: string; + input_hash: string; + created_at: string; + model_used: string | null; + cost_usd: number | null; + tokens_used: number | null; + duration_ms: number | null; +} + +export interface RunRow { + id: string; + analyzer_id: string; + analyzer_version_id: string; + config_id: string; + session_id: string; + status: string; + prompt_bundle_hash: string; + started_at: string; + finished_at: string | null; + model_spec: string | null; + cost_usd: number; + tokens_used: number; + nodes_produced: number; + nodes_skipped: number; + error_message: string | null; +} + +export interface ProgressRow { + analyzer_id: string; + analyzer_version_id: string; + config_id: string; + session_id: string; + cursor_json: string | null; + last_run_id: string | null; + total_analyzed: number; + status: "ok" | "in_progress" | "error" | "needs_rerun"; + error_message: string | null; + updated_at: string; +} + +// ── Contexts ── + +export interface AnalyzerPlanContext { + sessionId: string; + messages: MessageRow[]; + allNodes: AnalysisNodeRow[]; + ownNodes: AnalysisNodeRow[]; + dependencyNodes: Record; + progress: ProgressRow | null; + db: Database.Database; +} + +export interface AnalyzerRunContext { + getMessage(id: string): MessageRow | undefined; + getNode(id: string): AnalysisNodeRow | undefined; + getDependencyNodes(analyzerId: string): AnalysisNodeRow[]; + getAnchoredMessages(nodeId: string): MessageRow[]; + getSessionMessages(sessionId: string): MessageRow[]; + llm(request: LLMRequest): Promise; + run: RunRow; + config: AnalyzerConfig; + prompts: Record; +} + +// ── Analyzer interface ── + +export interface Analyzer { + def: AnalyzerDef; + version: AnalyzerVersion; + prompts: Record; + defaultConfig: AnalyzerConfig; + plan(ctx: AnalyzerPlanContext): Promise; + analyze(unit: AnalysisUnit, ctx: AnalyzerRunContext): Promise; +} + +// ── Run summary ── + +export interface RunSummary { + runId: string; + analyzerId: string; + analyzerVersionId: string; + sessionId: string; + status: "ok" | "error" | "partial"; + nodesProduced: number; + nodesSkipped: number; + costUsd: number; + tokensUsed: number; +} + +// ── Config resolver interface (model tiers) ── + +export interface ModelTierConfig { + cheap: string; + mid: string; + expensive: string; +} + +export type ModelTier = "cheap" | "mid" | "expensive"; + +export interface RunOptions { + configId?: string; + /** Force re-execution even if the same input_hash exists. Default false. */ + force?: boolean; +} + +// Re-export edge/RefKind constants for analyzers' convenience. +export { EDGE_KINDS, REF_KINDS, type EdgeKind, type RefKind }; diff --git a/src/db/analysis-queries.ts b/src/db/analysis-queries.ts new file mode 100644 index 0000000..c51e3e3 --- /dev/null +++ b/src/db/analysis-queries.ts @@ -0,0 +1,311 @@ +/** + * All SQL for the analyzer framework tables lives here. + * + * The framework reads from / writes to: + * - analyzer_defs + * - analyzer_versions + * - prompt_registry + * - analyzer_configs + * - analysis_runs + * - analysis_nodes + * - analysis_edges + * - analysis_progress + * + * Plus the materialized `proposals` table. + */ + +import type Database from "better-sqlite3"; +import type { + AnalysisNodeRow, + AnalyzerConfig, + AnalyzerDef, + AnalyzerVersion, + MessageRow, + ProgressRow, + PromptVersion, + RunRow, +} from "../analyze/types.js"; +import { computeConfigHash, fullHash, shortHash, uuidv7 } from "../analyze/input-hash.js"; + +// ── analyzer_defs ── + +export function upsertAnalyzerDef(db: Database.Database, def: AnalyzerDef): void { + db.prepare(` + INSERT INTO analyzer_defs (id, label, description, anchor_span, dependencies, created_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + label=excluded.label, + description=excluded.description, + anchor_span=excluded.anchor_span, + dependencies=excluded.dependencies + `).run(def.id, def.label, def.description, def.anchorSpan, JSON.stringify(def.dependencies), new Date().toISOString()); +} + +export function getAnalyzerDef(db: Database.Database, id: string): AnalyzerDef | undefined { + return db.prepare("SELECT * FROM analyzer_defs WHERE id = ?").get(id) as any; +} + +// ── analyzer_versions ── + +export function upsertAnalyzerVersion(db: Database.Database, v: AnalyzerVersion): void { + db.prepare(` + INSERT INTO analyzer_versions (analyzer_id, version_id, implementation_kind, code_ref, created_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(analyzer_id, version_id) DO NOTHING + `).run(v.analyzerId, v.versionId, v.implementationKind, v.codeRef ?? null, new Date().toISOString()); +} + +// ── prompt_registry ── + +export function registerPrompt(db: Database.Database, p: PromptVersion): void { + db.prepare(` + INSERT OR IGNORE INTO prompt_registry (hash, content, role, created_at) + VALUES (?, ?, ?, ?) + `).run(p.hash, p.content, p.role ?? null, new Date().toISOString()); +} + +// ── analyzer_configs ── + +/** + * Idempotently resolve a config: if a row with this (analyzer_id, + * config_hash) already exists, return that row's id; otherwise + * insert a new row with a fresh UUID. + */ +export function resolveConfig(db: Database.Database, args: { + analyzerId: string; + configJson: Record; + label?: string; +}): AnalyzerConfig { + const configHash = computeConfigHash(args.configJson); + // Upsert by (analyzer_id, config_hash). If a row already exists, + // the INSERT OR IGNORE is a no-op and we re-read the id. + const newId = uuidv7(); + db.prepare(` + INSERT OR IGNORE INTO analyzer_configs (id, analyzer_id, config_hash, config_json, label, created_at) + VALUES (?, ?, ?, ?, ?, ?) + `).run(newId, args.analyzerId, configHash, JSON.stringify(args.configJson), args.label ?? null, new Date().toISOString()); + + const row = db.prepare(` + SELECT id, analyzer_id, config_json, label FROM analyzer_configs + WHERE analyzer_id = ? AND config_hash = ? + `).get(args.analyzerId, configHash) as { id: string; analyzer_id: string; config_json: string; label: string | null }; + + return { + id: row.id, + analyzerId: row.analyzer_id, + configHash, + configJson: JSON.parse(row.config_json), + label: row.label ?? undefined, + }; +} + +// ── analysis_runs ── + +export function createRun(db: Database.Database, args: { + id: string; + analyzerId: string; + analyzerVersionId: string; + configId: string; + sessionId: string; + promptBundleHash: string; + modelSpec?: string; +}): void { + db.prepare(` + INSERT INTO analysis_runs (id, analyzer_id, analyzer_version_id, config_id, session_id, + status, prompt_bundle_hash, started_at, model_spec, cost_usd, tokens_used, + nodes_produced, nodes_skipped) + VALUES (?, ?, ?, ?, ?, 'running', ?, ?, ?, 0, 0, 0, 0) + `).run( + args.id, + args.analyzerId, + args.analyzerVersionId, + args.configId, + args.sessionId, + args.promptBundleHash, + new Date().toISOString(), + args.modelSpec ?? null, + ); +} + +export function updateRun(db: Database.Database, id: string, patch: Partial<{ + status: string; + finishedAt: string; + costUsd: number; + tokensUsed: number; + nodesProduced: number; + nodesSkipped: number; + errorMessage: string | null; +}>): void { + const fields: string[] = []; + const values: unknown[] = []; + if (patch.status !== undefined) { fields.push("status = ?"); values.push(patch.status); } + if (patch.finishedAt !== undefined) { fields.push("finished_at = ?"); values.push(patch.finishedAt); } + if (patch.costUsd !== undefined) { fields.push("cost_usd = ?"); values.push(patch.costUsd); } + if (patch.tokensUsed !== undefined) { fields.push("tokens_used = ?"); values.push(patch.tokensUsed); } + if (patch.nodesProduced !== undefined) { fields.push("nodes_produced = ?"); values.push(patch.nodesProduced); } + if (patch.nodesSkipped !== undefined) { fields.push("nodes_skipped = ?"); values.push(patch.nodesSkipped); } + if (patch.errorMessage !== undefined) { fields.push("error_message = ?"); values.push(patch.errorMessage); } + if (fields.length === 0) return; + values.push(id); + db.prepare(`UPDATE analysis_runs SET ${fields.join(", ")} WHERE id = ?`).run(...values); +} + +export function getRun(db: Database.Database, id: string): RunRow | undefined { + return db.prepare("SELECT * FROM analysis_runs WHERE id = ?").get(id) as RunRow | undefined; +} + +export function findStaleRunningRuns(db: Database.Database): RunRow[] { + return db.prepare("SELECT * FROM analysis_runs WHERE status = 'running'").all() as RunRow[]; +} + +// ── analysis_nodes ── + +export function findNodeByInputHash(db: Database.Database, inputHash: string): AnalysisNodeRow | undefined { + return db.prepare("SELECT * FROM analysis_nodes WHERE input_hash = ? LIMIT 1").get(inputHash) as AnalysisNodeRow | undefined; +} + +export function insertNode(db: Database.Database, args: { + id: string; + sessionId: string; + analyzerId: string; + analyzerVersionId: string; + configId: string; + runId: string; + nodeKind: string; + contentJson: string; + sourceSetHash: string; + inputHash: string; + modelUsed?: string; + costUsd?: number; + tokensUsed?: number; + durationMs?: number; + createdAt: string; +}): void { + db.prepare(` + INSERT OR IGNORE INTO analysis_nodes ( + id, session_id, analyzer_id, analyzer_version_id, config_id, run_id, + node_kind, content_json, source_set_hash, input_hash, + model_used, cost_usd, tokens_used, duration_ms, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run( + args.id, + args.sessionId, + args.analyzerId, + args.analyzerVersionId, + args.configId, + args.runId, + args.nodeKind, + args.contentJson, + args.sourceSetHash, + args.inputHash, + args.modelUsed ?? null, + args.costUsd ?? 0, + args.tokensUsed ?? 0, + args.durationMs ?? null, + args.createdAt, + ); +} + +export function getNode(db: Database.Database, id: string): AnalysisNodeRow | undefined { + return db.prepare("SELECT * FROM analysis_nodes WHERE id = ?").get(id) as AnalysisNodeRow | undefined; +} + +export function getAllSessionNodes(db: Database.Database, sessionId: string): AnalysisNodeRow[] { + return db.prepare("SELECT * FROM analysis_nodes WHERE session_id = ?").all(sessionId) as AnalysisNodeRow[]; +} + +export function getSessionNodesByAnalyzer(db: Database.Database, sessionId: string, analyzerId: string): AnalysisNodeRow[] { + return db.prepare("SELECT * FROM analysis_nodes WHERE session_id = ? AND analyzer_id = ?").all(sessionId, analyzerId) as AnalysisNodeRow[]; +} + +// ── analysis_edges ── + +export function insertEdge(db: Database.Database, args: { + fromNodeId: string; + toRefKind: string; + toRefId: string; + edgeKind: string; + ordinal?: number; +}): void { + db.prepare(` + INSERT OR IGNORE INTO analysis_edges (from_node_id, to_ref_kind, to_ref_id, edge_kind, ordinal) + VALUES (?, ?, ?, ?, ?) + `).run(args.fromNodeId, args.toRefKind, args.toRefId, args.edgeKind, args.ordinal ?? 0); +} + +export function getEdgesFrom(db: Database.Database, fromNodeId: string): Array<{ to_ref_kind: string; to_ref_id: string; edge_kind: string; ordinal: number }> { + return db.prepare("SELECT to_ref_kind, to_ref_id, edge_kind, ordinal FROM analysis_edges WHERE from_node_id = ?").all(fromNodeId) as any; +} + +export function getEdgesTo(db: Database.Database, toRefKind: string, toRefId: string): Array<{ from_node_id: string; edge_kind: string }> { + return db.prepare("SELECT from_node_id, edge_kind FROM analysis_edges WHERE to_ref_kind = ? AND to_ref_id = ?").all(toRefKind, toRefId) as any; +} + +export function getAnchoredMessageIds(db: Database.Database, nodeId: string): string[] { + return (db.prepare(` + SELECT to_ref_id FROM analysis_edges + WHERE from_node_id = ? AND to_ref_kind = 'message' AND edge_kind = 'anchors' + ORDER BY ordinal ASC + `).all(nodeId) as Array<{ to_ref_id: string }>).map((r) => r.to_ref_id); +} + +// ── analysis_progress ── + +export function getProgress(db: Database.Database, args: { + analyzerId: string; + analyzerVersionId: string; + configId: string; + sessionId: string; +}): ProgressRow | undefined { + return db.prepare(` + SELECT * FROM analysis_progress + WHERE analyzer_id = ? AND analyzer_version_id = ? AND config_id = ? AND session_id = ? + `).get(args.analyzerId, args.analyzerVersionId, args.configId, args.sessionId) as ProgressRow | undefined; +} + +export function upsertProgress(db: Database.Database, args: { + analyzerId: string; + analyzerVersionId: string; + configId: string; + sessionId: string; + cursorJson: string | null; + lastRunId: string | null; + totalAnalyzed: number; + status: "ok" | "in_progress" | "error" | "needs_rerun"; + errorMessage: string | null; +}): void { + db.prepare(` + INSERT INTO analysis_progress ( + analyzer_id, analyzer_version_id, config_id, session_id, + cursor_json, last_run_id, total_analyzed, status, error_message, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(analyzer_id, analyzer_version_id, config_id, session_id) DO UPDATE SET + cursor_json = excluded.cursor_json, + last_run_id = excluded.last_run_id, + total_analyzed = excluded.total_analyzed, + status = excluded.status, + error_message = excluded.error_message, + updated_at = excluded.updated_at + `).run( + args.analyzerId, + args.analyzerVersionId, + args.configId, + args.sessionId, + args.cursorJson, + args.lastRunId, + args.totalAnalyzed, + args.status, + args.errorMessage, + new Date().toISOString(), + ); +} + +// ── Messages ── + +export function getMessage(db: Database.Database, id: string): MessageRow | undefined { + return db.prepare(` + SELECT id, session_id, parent_id, timestamp, role, + content_text, content_thinking, tool_calls, tool_results, meta_json + FROM messages WHERE id = ? + `).get(id) as MessageRow | undefined; +} diff --git a/src/db/queries.ts b/src/db/queries.ts index 01129b6..a98e3da 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -65,13 +65,14 @@ export interface MessageInsert { content_thinking: string | null; tool_calls: string | null; tool_results: string | null; + meta_json?: string | null; } export function insertMessage(db: Database.Database, m: MessageInsert): void { db.prepare(` - INSERT OR IGNORE INTO messages (id, session_id, parent_id, timestamp, role, content_text, content_thinking, tool_calls, tool_results, content_hash) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `).run(m.id, m.session_id, m.parent_id, m.timestamp, m.role, m.content_text, m.content_thinking, m.tool_calls, m.tool_results, null); + INSERT OR IGNORE INTO messages (id, session_id, parent_id, timestamp, role, content_text, content_thinking, tool_calls, tool_results, content_hash, meta_json) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run(m.id, m.session_id, m.parent_id, m.timestamp, m.role, m.content_text, m.content_thinking, m.tool_calls, m.tool_results, null, m.meta_json ?? null); } export function countMessages(db: Database.Database, sessionId: string): number { @@ -82,6 +83,22 @@ export function getSessionMessages(db: Database.Database, sessionId: string): Ar return db.prepare("SELECT role, content_text, content_thinking, tool_calls, timestamp FROM messages WHERE session_id = ? ORDER BY rowid ASC").all(sessionId) as any[]; } +export function getSessionMessagesFull(db: Database.Database, sessionId: string): Array<{ id: string; session_id: string; parent_id: string | null; timestamp: string | null; role: string; content_text: string | null; content_thinking: string | null; tool_calls: string | null; tool_results: string | null; meta_json: string | null }> { + return db.prepare(` + SELECT id, session_id, parent_id, timestamp, role, content_text, content_thinking, + tool_calls, tool_results, meta_json + FROM messages WHERE session_id = ? ORDER BY rowid ASC + `).all(sessionId) as any[]; +} + +export function getMessageById(db: Database.Database, id: string): { id: string; session_id: string; parent_id: string | null; timestamp: string | null; role: string; content_text: string | null; content_thinking: string | null; tool_calls: string | null; tool_results: string | null; meta_json: string | null } | undefined { + return db.prepare(` + SELECT id, session_id, parent_id, timestamp, role, content_text, content_thinking, + tool_calls, tool_results, meta_json + FROM messages WHERE id = ? + `).get(id) as any; +} + // ── Proposals ── export function insertProposal(db: Database.Database, p: Proposal): string { @@ -98,11 +115,46 @@ export function listProposals(db: Database.Database, status?: string): Proposal[ } export function acceptProposal(db: Database.Database, id: string): boolean { - return db.prepare("UPDATE proposals SET status = 'accepted' WHERE id = ? AND status = 'new'").run(id).changes > 0; + // Both 'new' (legacy) and 'open' (framework) are valid pre-accept states. + return db.prepare("UPDATE proposals SET status = 'accepted' WHERE id = ? AND status IN ('new', 'open')").run(id).changes > 0; } export function rejectProposal(db: Database.Database, id: string): boolean { - return db.prepare("UPDATE proposals SET status = 'rejected' WHERE id = ? AND status = 'new'").run(id).changes > 0; + return db.prepare("UPDATE proposals SET status = 'rejected' WHERE id = ? AND status IN ('new', 'open')").run(id).changes > 0; +} + +/** + * List proposals joined with their source analysis_node (if any). + * Used by the proposals command and the tool to render richer + * output: target_type, target_path, title, confidence, etc. + */ +export function listProposalsEnriched(db: Database.Database, status?: string): Array<{ + id: string; + created_at: string; + updated_at: string | null; + session_id: string; + analyzer_id: string | null; + target_type: string | null; + target_path: string | null; + title: string | null; + summary: string; + detail: string | null; + evidence: string | null; + confidence: number | null; + severity: string; + dedup_key: string | null; + status: string; + analysis_node_id: string | null; +}> { + const where = status ? "WHERE status = ?" : ""; + const params = status ? [status] : []; + return db.prepare(` + SELECT id, created_at, updated_at, session_id, analyzer_id, + target_type, target_path, title, summary, detail, evidence, + confidence, severity, dedup_key, status, analysis_node_id + FROM proposals ${where} + ORDER BY created_at DESC + `).all(...params) as any; } export function computeDedupHash(target: string, severity: string, summary: string): string { diff --git a/src/db/schema.ts b/src/db/schema.ts index efa808f..65abfb1 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -1,6 +1,22 @@ -import Database from "better-sqlite3"; +import type Database from "better-sqlite3"; +/** + * Apply schema migrations. Idempotent — safe to call on every command + * start. Existing tables and rows are preserved; new tables and + * columns are added. + * + * Migration history: + * 001 — initial: sessions, messages, proposals + FTS5 + * 002 — analyzer framework: adds meta_json to messages; extends + * proposals with analyzer_id / target_type / target_path / + * title / analysis_node_id / confidence / dedup_key / + * updated_at / evidence_json; creates analyzer_defs, + * analyzer_versions, prompt_registry, analyzer_configs, + * analysis_runs, analysis_nodes, analysis_edges, + * analysis_progress. + */ export function migrate(db: Database.Database): void { + // 001 — base schema db.exec(` CREATE TABLE IF NOT EXISTS sessions ( id TEXT PRIMARY KEY, @@ -71,4 +87,149 @@ export function migrate(db: Database.Database): void { VALUES ('delete', OLD.rowid, OLD.content_text, OLD.content_thinking); END; `); -} \ No newline at end of file + + // 002 — analyzer framework + addColumnIfMissing(db, "messages", "meta_json", "TEXT"); + addColumnIfMissing(db, "proposals", "analysis_node_id", "TEXT"); + addColumnIfMissing(db, "proposals", "analyzer_id", "TEXT"); + addColumnIfMissing(db, "proposals", "target_type", "TEXT"); + addColumnIfMissing(db, "proposals", "target_path", "TEXT"); + addColumnIfMissing(db, "proposals", "title", "TEXT"); + addColumnIfMissing(db, "proposals", "evidence_json", "TEXT"); + addColumnIfMissing(db, "proposals", "confidence", "REAL"); + addColumnIfMissing(db, "proposals", "dedup_key", "TEXT"); + addColumnIfMissing(db, "proposals", "updated_at", "TEXT"); + + db.exec(` + CREATE TABLE IF NOT EXISTS analyzer_defs ( + id TEXT PRIMARY KEY, + label TEXT NOT NULL, + description TEXT, + anchor_span TEXT NOT NULL, + dependencies TEXT NOT NULL DEFAULT '[]', + created_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS analyzer_versions ( + analyzer_id TEXT NOT NULL, + version_id TEXT NOT NULL, + implementation_kind TEXT NOT NULL, + code_ref TEXT, + created_at TEXT NOT NULL, + PRIMARY KEY (analyzer_id, version_id), + FOREIGN KEY (analyzer_id) REFERENCES analyzer_defs(id) + ); + + CREATE TABLE IF NOT EXISTS prompt_registry ( + hash TEXT PRIMARY KEY, + content TEXT NOT NULL, + role TEXT, + created_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS analyzer_configs ( + id TEXT PRIMARY KEY, + analyzer_id TEXT NOT NULL, + config_hash TEXT NOT NULL, + config_json TEXT NOT NULL, + label TEXT, + created_at TEXT NOT NULL, + FOREIGN KEY (analyzer_id) REFERENCES analyzer_defs(id) + ); + + CREATE TABLE IF NOT EXISTS analysis_runs ( + id TEXT PRIMARY KEY, + analyzer_id TEXT NOT NULL, + analyzer_version_id TEXT NOT NULL, + config_id TEXT NOT NULL, + session_id TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'planned', + prompt_bundle_hash TEXT NOT NULL, + started_at TEXT NOT NULL, + finished_at TEXT, + model_spec TEXT, + cost_usd REAL DEFAULT 0, + tokens_used INTEGER DEFAULT 0, + nodes_produced INTEGER DEFAULT 0, + nodes_skipped INTEGER DEFAULT 0, + error_message TEXT, + FOREIGN KEY (analyzer_id, analyzer_version_id) REFERENCES analyzer_versions(analyzer_id, version_id), + FOREIGN KEY (session_id) REFERENCES sessions(id) + ); + + CREATE TABLE IF NOT EXISTS analysis_nodes ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + analyzer_id TEXT NOT NULL, + analyzer_version_id TEXT NOT NULL, + config_id TEXT NOT NULL, + run_id TEXT NOT NULL, + node_kind TEXT NOT NULL, + content_json TEXT NOT NULL, + source_set_hash TEXT NOT NULL, + input_hash TEXT NOT NULL, + created_at TEXT NOT NULL, + model_used TEXT, + cost_usd REAL DEFAULT 0, + tokens_used INTEGER DEFAULT 0, + duration_ms INTEGER, + FOREIGN KEY (run_id) REFERENCES analysis_runs(id), + FOREIGN KEY (session_id) REFERENCES sessions(id) + ); + + CREATE TABLE IF NOT EXISTS analysis_edges ( + from_node_id TEXT NOT NULL, + to_ref_kind TEXT NOT NULL, + to_ref_id TEXT NOT NULL, + edge_kind TEXT NOT NULL, + ordinal INTEGER DEFAULT 0, + PRIMARY KEY (from_node_id, to_ref_kind, to_ref_id, edge_kind, ordinal), + FOREIGN KEY (from_node_id) REFERENCES analysis_nodes(id) + ); + + CREATE TABLE IF NOT EXISTS analysis_progress ( + analyzer_id TEXT NOT NULL, + analyzer_version_id TEXT NOT NULL, + config_id TEXT NOT NULL, + session_id TEXT NOT NULL, + cursor_json TEXT, + last_run_id TEXT, + total_analyzed INTEGER DEFAULT 0, + status TEXT NOT NULL DEFAULT 'ok', + error_message TEXT, + updated_at TEXT NOT NULL, + PRIMARY KEY (analyzer_id, analyzer_version_id, config_id, session_id), + FOREIGN KEY (session_id) REFERENCES sessions(id) + ); + `); + + db.exec(` + CREATE INDEX IF NOT EXISTS idx_runs_session ON analysis_runs(session_id); + CREATE INDEX IF NOT EXISTS idx_runs_status ON analysis_runs(status); + CREATE INDEX IF NOT EXISTS idx_nodes_session ON analysis_nodes(session_id); + CREATE INDEX IF NOT EXISTS idx_nodes_analyzer ON analysis_nodes(analyzer_id, analyzer_version_id); + CREATE INDEX IF NOT EXISTS idx_nodes_kind ON analysis_nodes(node_kind); + CREATE INDEX IF NOT EXISTS idx_nodes_input_hash ON analysis_nodes(input_hash); + CREATE INDEX IF NOT EXISTS idx_nodes_source_hash ON analysis_nodes(source_set_hash); + CREATE INDEX IF NOT EXISTS idx_nodes_config ON analysis_nodes(config_id); + CREATE INDEX IF NOT EXISTS idx_nodes_idempotency + ON analysis_nodes(analyzer_id, analyzer_version_id, config_id, source_set_hash); + CREATE INDEX IF NOT EXISTS idx_edges_from ON analysis_edges(from_node_id); + CREATE INDEX IF NOT EXISTS idx_edges_to ON analysis_edges(to_ref_kind, to_ref_id); + CREATE INDEX IF NOT EXISTS idx_edges_kind ON analysis_edges(edge_kind); + CREATE INDEX IF NOT EXISTS idx_progress_session + ON analysis_progress(analyzer_id, analyzer_version_id, session_id); + CREATE UNIQUE INDEX IF NOT EXISTS idx_config_unique + ON analyzer_configs(analyzer_id, config_hash); + `); +} + +/** + * Add a column to a table if it doesn't already exist. + * SQLite has no ADD COLUMN IF NOT EXISTS, so we inspect pragma. + */ +function addColumnIfMissing(db: Database.Database, table: string, column: string, typeDecl: string): void { + const cols = db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>; + if (cols.some((c) => c.name === column)) return; + db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${typeDecl}`); +} From 74af429690a84a839e37a0b1b947bd8d36e3a86c Mon Sep 17 00:00:00 2001 From: Nicolas Marchildon Date: Tue, 2 Jun 2026 11:11:20 -0400 Subject: [PATCH 2/5] analyze: implement turn-pair-core, turn-pair-llm, session-overview MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit turn-pair-core: deterministic per-pair metrics - 19 properties from §6.3: lengths, correction detection, tool stats, friction score, model/usage capture, compaction boundary - Friction score uses weighted sum of binary signals (correction, tool_failure threshold, retry, thinking, compaction) - Step function on tool failures (>= max_tool_failures => full weight) - Anchors to every message in the pair (user, assistant, tool results) turn-pair-llm: LLM enrichment for high-signal pairs - Filters dependency nodes to correction_detected OR friction_score >= 0.4 - Calls cheap model with structured classify prompt - Refines + consumes the deterministic node; uses_prompt edges - Captures LLM cost/tokens on run and node rows session-overview: session-level analysis with map-reduce - Builds structured digest from messages + pair nodes - Splits into segments when digest > use_map_reduce_over_chars - Map phase on each segment (cheap model), reduce phase on the combined summaries (mid model) - Produces improvement_proposals materialized into the proposals table - consumes both dependency analyzers; uses_prompt edges for map+reduce --- .../analyzers/session-overview/config.ts | 20 + .../analyzers/session-overview/digest.ts | 211 +++++++++ .../analyzers/session-overview/index.ts | 260 +++++++++++ .../analyzers/session-overview/prompt-map.ts | 119 +++++ .../session-overview/prompt-reduce.ts | 130 ++++++ .../analyzers/turn-pair-core/config.ts | 55 +++ src/analyze/analyzers/turn-pair-core/index.ts | 411 ++++++++++++++++++ .../analyzers/turn-pair-core/patterns.ts | 146 +++++++ src/analyze/analyzers/turn-pair-llm/config.ts | 14 + src/analyze/analyzers/turn-pair-llm/index.ts | 269 ++++++++++++ src/analyze/analyzers/turn-pair-llm/prompt.ts | 158 +++++++ 11 files changed, 1793 insertions(+) create mode 100644 src/analyze/analyzers/session-overview/config.ts create mode 100644 src/analyze/analyzers/session-overview/digest.ts create mode 100644 src/analyze/analyzers/session-overview/index.ts create mode 100644 src/analyze/analyzers/session-overview/prompt-map.ts create mode 100644 src/analyze/analyzers/session-overview/prompt-reduce.ts create mode 100644 src/analyze/analyzers/turn-pair-core/config.ts create mode 100644 src/analyze/analyzers/turn-pair-core/index.ts create mode 100644 src/analyze/analyzers/turn-pair-core/patterns.ts create mode 100644 src/analyze/analyzers/turn-pair-llm/config.ts create mode 100644 src/analyze/analyzers/turn-pair-llm/index.ts create mode 100644 src/analyze/analyzers/turn-pair-llm/prompt.ts diff --git a/src/analyze/analyzers/session-overview/config.ts b/src/analyze/analyzers/session-overview/config.ts new file mode 100644 index 0000000..7647d55 --- /dev/null +++ b/src/analyze/analyzers/session-overview/config.ts @@ -0,0 +1,20 @@ +/** + * Default config for the session-overview analyzer. + */ + +export const DEFAULT_SESSION_OVERVIEW_CONFIG = { + /** Maximum input tokens for the digest (rough char count / 4). */ + context_budget_chars: 100_000, + /** If digest exceeds budget, split into segments of this size. */ + segment_chars: 30_000, + /** Max segments to map in a single run (cost cap). */ + max_segments: 8, + /** When to use map-reduce vs single-call. */ + use_map_reduce_over_chars: 60_000, + /** Tool model tier for the map phase. */ + map_tier: "cheap", + /** Tool model tier for the reduce phase. */ + reduce_tier: "mid", +} as const; + +export type SessionOverviewConfig = typeof DEFAULT_SESSION_OVERVIEW_CONFIG; diff --git a/src/analyze/analyzers/session-overview/digest.ts b/src/analyze/analyzers/session-overview/digest.ts new file mode 100644 index 0000000..6d4d8cd --- /dev/null +++ b/src/analyze/analyzers/session-overview/digest.ts @@ -0,0 +1,211 @@ +/** + * Build a structured session digest from turn-pair nodes and + * raw messages. The digest is what the LLM sees, so its shape + * matters: a markdown summary that highlights friction points, + * corrections, and tool failures, while keeping tool-result + * details verbatim only for high-signal events. + */ + +import type { AnalysisNodeRow, MessageRow } from "../../types.js"; + +export interface DigestSegment { + index: number; + text: string; + charCount: number; +} + +export interface DigestResult { + segments: DigestSegment[]; + totalChars: number; + pairCount: number; + frictionCount: number; + compactionCount: number; +} + +interface TurnPairCoreProps { + correction_detected: boolean; + friction_score: number; + tool_failure_count: number; + tool_waste_bytes: number; + correction_type: "explicit" | "implicit" | "repetition" | null; + correction_text: string | null; + tool_names: string[]; + elapsed_seconds: number | null; + model: string | null; +} + +interface TurnPairLlmProps { + sentiment: string; + frustration_level: number; + quality_score: number; + friction_cause: string | null; + friction_summary: string | null; + user_intent: string; + [key: string]: unknown; +} + +function safeParse(s: string | null): T | null { + if (!s) return null; + try { return JSON.parse(s) as T; } catch { return null; } +} + +/** + * Group messages by their position relative to the most recent + * compaction summary. Each group becomes a "phase" of the + * session. Phases before compaction are summarized from the + * compaction text; phases after are detailed. + */ +function groupMessagesByCompaction(messages: MessageRow[]): Array<{ phase: string; messages: MessageRow[] }> { + const phases: Array<{ phase: string; messages: MessageRow[] }> = []; + let current: { phase: string; messages: MessageRow[] } = { phase: "initial", messages: [] }; + let compactionIndex = 0; + for (const m of messages) { + if (m.role === "compactionSummary") { + phases.push(current); + compactionIndex++; + current = { phase: `post-compaction-${compactionIndex}`, messages: [] }; + continue; + } + current.messages.push(m); + } + if (current.messages.length > 0 || phases.length === 0) phases.push(current); + return phases; +} + +function formatPairRow(idx: number, props: TurnPairCoreProps, llm?: TurnPairLlmProps | null): string { + const sentiment = llm?.sentiment ?? "—"; + const cause = llm?.friction_cause ?? "—"; + const elapsed = props.elapsed_seconds != null ? `${props.elapsed_seconds.toFixed(1)}s` : "—"; + const tools = props.tool_names.length > 0 ? props.tool_names.join(",") : "—"; + const corr = props.correction_type ?? "—"; + return `| ${idx} | ${elapsed} | ${sentiment} | ${props.friction_score.toFixed(2)} | ${corr} | ${props.tool_failure_count} | ${tools} | ${cause} |`; +} + +function formatStatBlock(stats: { + totalPairs: number; + frictionPairs: number; + correctionRate: number; + toolFailures: number; + toolWaste: number; + durationSec: number | null; +}): string { + const dur = stats.durationSec != null ? `${stats.durationSec.toFixed(0)}s` : "—"; + return [ + "### Statistics", + `- Total pairs: ${stats.totalPairs}`, + `- Friction pairs (score >= 0.4): ${stats.frictionPairs}`, + `- Correction rate: ${(stats.correctionRate * 100).toFixed(0)}%`, + `- Tool failures: ${stats.toolFailures}`, + `- Tool waste (bytes never referenced): ${stats.toolWaste}`, + `- Session duration: ${dur}`, + ].join("\n"); +} + +/** + * Build the digest from pair nodes and the messages they cover. + * + * Pair nodes MUST be sorted in chronological order. The matching + * messages are read from the framework's edge table; we get them + * passed in already-resolved. + */ +export function buildDigest(args: { + sessionId: string; + messages: MessageRow[]; + pairNodes: AnalysisNodeRow[]; + llmNodes: AnalysisNodeRow[]; +}): DigestResult { + const pairProps: TurnPairCoreProps[] = args.pairNodes.map((n) => safeParse(n.content_json) ?? ({} as TurnPairCoreProps)); + const llmPropsByAnchor = new Map(); + for (const n of args.llmNodes) { + const p = safeParse(n.content_json); + if (p) { + llmPropsByAnchor.set(n.id, p); + } + } + + const phases = groupMessagesByCompaction(args.messages); + const compactions = args.messages.filter((m) => m.role === "compactionSummary"); + + // Compute aggregate stats from pair props + const totalPairs = pairProps.length; + const frictionPairs = pairProps.filter((p) => p.friction_score >= 0.4).length; + const correctionCount = pairProps.filter((p) => p.correction_detected).length; + const toolFailures = pairProps.reduce((s, p) => s + p.tool_failure_count, 0); + const toolWaste = pairProps.reduce((s, p) => s + p.tool_waste_bytes, 0); + const firstTs = args.messages[0]?.timestamp; + const lastTs = args.messages[args.messages.length - 1]?.timestamp; + const durationSec = firstTs && lastTs ? (Date.parse(lastTs) - Date.parse(firstTs)) / 1000 : null; + + // Per-phase digests + const phaseTexts: string[] = []; + phaseTexts.push(`## Session Overview`); + phaseTexts.push(`Session ID: ${args.sessionId}`); + phaseTexts.push(`Messages: ${args.messages.length}, Pairs: ${totalPairs}, Compactions: ${compactions.length}\n`); + + for (let pi = 0; pi < phases.length; pi++) { + const phase = phases[pi]!; + phaseTexts.push(`### Phase ${pi + 1} (${phase.phase})`); + if (pi > 0 && compactions[pi - 1]) { + phaseTexts.push("**Compaction summary (verbatim):**"); + phaseTexts.push(compactions[pi - 1]!.content_text ?? "(empty)"); + phaseTexts.push(""); + } + phaseTexts.push("**Per-pair (chronological):**"); + phaseTexts.push("| # | Elapsed | Sentiment | Friction | Correction | ToolFailures | Tools | Cause |"); + phaseTexts.push("|---|---------|-----------|----------|------------|--------------|-------|-------|"); + // For each pair node, format a row. The pair node IDs match the + // sequence in pairProps. We look up the matching LLM node by id. + for (let i = 0; i < pairProps.length; i++) { + const p = pairProps[i]!; + const node = args.pairNodes[i]!; + const llm = llmPropsByAnchor.get(node.id) ?? null; + phaseTexts.push(formatPairRow(i + 1, p, llm)); + } + phaseTexts.push(""); + } + + phaseTexts.push(formatStatBlock({ + totalPairs, + frictionPairs, + correctionRate: totalPairs > 0 ? correctionCount / totalPairs : 0, + toolFailures, + toolWaste, + durationSec, + })); + + const fullText = phaseTexts.join("\n"); + return { + segments: [{ index: 0, text: fullText, charCount: fullText.length }], + totalChars: fullText.length, + pairCount: totalPairs, + frictionCount: frictionPairs, + compactionCount: compactions.length, + }; +} + +/** + * Split a digest into multiple segments of approximately equal + * character count, with each segment self-contained (header + + * portion of pairs + footer). Used when the digest exceeds the + * model's context budget. + */ +export function splitDigest(digest: DigestResult, segmentChars: number): DigestSegment[] { + if (digest.totalChars <= segmentChars) return digest.segments; + // Simple split: chunk the text evenly, preserving header lines. + const text = digest.segments[0]?.text ?? ""; + const lines = text.split("\n"); + const out: DigestSegment[] = []; + let buf: string[] = []; + let bufChars = 0; + for (const line of lines) { + if (bufChars + line.length > segmentChars && buf.length > 0) { + out.push({ index: out.length, text: buf.join("\n"), charCount: bufChars }); + buf = []; + bufChars = 0; + } + buf.push(line); + bufChars += line.length + 1; + } + if (buf.length > 0) out.push({ index: out.length, text: buf.join("\n"), charCount: bufChars }); + return out; +} diff --git a/src/analyze/analyzers/session-overview/index.ts b/src/analyze/analyzers/session-overview/index.ts new file mode 100644 index 0000000..941da32 --- /dev/null +++ b/src/analyze/analyzers/session-overview/index.ts @@ -0,0 +1,260 @@ +/** + * session-overview — produces one summary node per session, + * consuming turn-pair-core and turn-pair-llm nodes. The summary + * includes a free-text summary, key friction points, a sentiment + * arc, and a list of improvement proposals. Proposals are + * materialized into the `proposals` table by the framework. + * + * Strategy: + * 1. Build a structured digest from messages + pair nodes. + * 2. If the digest fits in `use_map_reduce_over_chars`, do a + * single reduce call (no map phase). + * 3. Otherwise, split the digest into segments, call the map + * prompt on each (cheap model), then call the reduce prompt + * with the merged segment summaries + stats (mid model). + * + * The framework does not pick the model — analyzers pass a tier + * (cheap, mid, expensive) and the framework resolves to a + * concrete model string from `prospector.json`. + */ + +import type { + AnalysisNodeRow, + AnalysisResult, + AnalysisUnit, + Analyzer, + AnalyzerConfig, + AnalyzerDef, + AnalyzerPlanContext, + AnalyzerRunContext, + AnalyzerVersion, + MessageRow, + PromptVersion, +} from "../../types.js"; +import { computeSourceSetHash } from "../../framework.js"; +import { EDGE_KINDS, REF_KINDS } from "../../edge-kinds.js"; +import { fullHash, shortHash } from "../../input-hash.js"; +import { TURN_PAIR_CORE_DEF } from "../turn-pair-core/index.js"; +import { TURN_PAIR_LLM_DEF } from "../turn-pair-llm/index.js"; +import { buildDigest, splitDigest } from "./digest.js"; +import { + SESSION_OVERVIEW_MAP_PROMPT, + buildMapPrompt, + parseMapResponse, + type MapSummary, +} from "./prompt-map.js"; +import { + SESSION_OVERVIEW_REDUCE_PROMPT, + buildReducePrompt, + parseReduceResponse, + type SessionOverviewProperties, +} from "./prompt-reduce.js"; +import { DEFAULT_SESSION_OVERVIEW_CONFIG, type SessionOverviewConfig } from "./config.js"; + +export const SESSION_OVERVIEW_DEF: AnalyzerDef = { + id: "session-overview", + label: "Session-Level Analysis & Proposals", + description: "Produces a session-level summary, key friction points, sentiment arc, and improvement proposals. Consumes turn-pair-core and turn-pair-llm nodes.", + anchorSpan: "full_session", + dependencies: [TURN_PAIR_CORE_DEF.id, TURN_PAIR_LLM_DEF.id], +}; + +export const SESSION_OVERVIEW_VERSION: AnalyzerVersion = { + analyzerId: SESSION_OVERVIEW_DEF.id, + versionId: "0.1.0", + implementationKind: "in_process_llm", + codeRef: "src/analyze/analyzers/session-overview/index.ts", +}; + +const MAP_PROMPT_HASH = shortHash(SESSION_OVERVIEW_MAP_PROMPT); +const MAP_PROMPT_FULL = fullHash(SESSION_OVERVIEW_MAP_PROMPT); +const REDUCE_PROMPT_HASH = shortHash(SESSION_OVERVIEW_REDUCE_PROMPT); +const REDUCE_PROMPT_FULL = fullHash(SESSION_OVERVIEW_REDUCE_PROMPT); + +const SESSION_OVERVIEW_PROMPTS: Record = { + "map-segment": { + hash: MAP_PROMPT_HASH, + content: SESSION_OVERVIEW_MAP_PROMPT, + fullHash: MAP_PROMPT_FULL, + role: "map", + }, + "reduce-summaries": { + hash: REDUCE_PROMPT_HASH, + content: SESSION_OVERVIEW_REDUCE_PROMPT, + fullHash: REDUCE_PROMPT_FULL, + role: "reduce", + }, +}; + +function isPairNode(n: AnalysisNodeRow): boolean { + return n.analyzer_id === TURN_PAIR_CORE_DEF.id; +} +function isLlmNode(n: AnalysisNodeRow): boolean { + return n.analyzer_id === TURN_PAIR_LLM_DEF.id; +} + +async function callMap(digest: string, ctx: AnalyzerRunContext): Promise { + const prompt = buildMapPrompt(digest); + const response = await ctx.llm({ + model: "cheap", + system: "You are a session-segment summarizer. Return JSON only.", + user: prompt, + temperature: 0.0, + maxTokens: 2000, + }); + return parseMapResponse(response.text); +} + +async function callReduce(segmentSummaries: string, stats: string, ctx: AnalyzerRunContext): Promise { + const prompt = buildReducePrompt({ segmentSummaries, stats }); + const response = await ctx.llm({ + model: "mid", + system: "You are a session reducer. Return JSON only.", + user: prompt, + temperature: 0.0, + maxTokens: 4000, + }); + return parseReduceResponse(response.text); +} + +export const sessionOverviewAnalyzer: Analyzer = { + def: SESSION_OVERVIEW_DEF, + version: SESSION_OVERVIEW_VERSION, + prompts: SESSION_OVERVIEW_PROMPTS, + defaultConfig: { + id: "", + analyzerId: SESSION_OVERVIEW_DEF.id, + configJson: DEFAULT_SESSION_OVERVIEW_CONFIG as unknown as Record, + configHash: "", + label: "default", + }, + + async plan(ctx: AnalyzerPlanContext): Promise { + const pairNodes = (ctx.dependencyNodes[TURN_PAIR_CORE_DEF.id] ?? []) + .filter(isPairNode) + .sort((a, b) => a.id.localeCompare(b.id)); + if (pairNodes.length === 0) return []; + + const llmNodes = (ctx.dependencyNodes[TURN_PAIR_LLM_DEF.id] ?? []).filter(isLlmNode); + + const sources = [ + ...pairNodes.map((n) => ({ kind: "analysis_node" as const, id: n.id })), + ...llmNodes.map((n) => ({ kind: "analysis_node" as const, id: n.id })), + ]; + + return [{ + sources, + sourceSetHash: computeSourceSetHash(sources), + anchorKind: "session", + anchorRef: ctx.sessionId, + }]; + }, + + async analyze(unit: AnalysisUnit, ctx: AnalyzerRunContext): Promise { + const config = (ctx.config.configJson as unknown as SessionOverviewConfig) ?? DEFAULT_SESSION_OVERVIEW_CONFIG; + + // Load messages for the session. We re-load here because the + // plan context is per-plan and we want the most up-to-date + // picture in case anything changed. + const messages: MessageRow[] = ctx.getSessionMessages(ctx.run.session_id); + + // Resolve dependency nodes for this session + const pairNodes = ctx.getDependencyNodes(TURN_PAIR_CORE_DEF.id) + .filter(isPairNode) + .sort((a, b) => a.id.localeCompare(b.id)); + const llmNodes = ctx.getDependencyNodes(TURN_PAIR_LLM_DEF.id).filter(isLlmNode); + + const digest = buildDigest({ + sessionId: ctx.run.session_id, + messages, + pairNodes, + llmNodes, + }); + + const useMapReduce = digest.totalChars > config.use_map_reduce_over_chars; + const statsText = JSON.stringify({ + total_pairs: digest.pairCount, + friction_pairs: digest.frictionCount, + compactions: digest.compactionCount, + total_messages: messages.length, + }, null, 2); + + let result: SessionOverviewProperties; + let usedPrompts: string[] = []; + + if (!useMapReduce) { + // Single reduce call with the whole digest as input + const segmentSummaries = JSON.stringify([{ segment: 0, summary: digest.segments[0]?.text ?? "" }]); + result = await callReduce(segmentSummaries, statsText, ctx); + usedPrompts = [REDUCE_PROMPT_HASH]; + } else { + // Map-reduce: split, map each, then reduce + const segments = splitDigest(digest, config.segment_chars).slice(0, config.max_segments); + const mapResults: MapSummary[] = []; + for (const seg of segments) { + const m = await callMap(seg.text, ctx); + mapResults.push(m); + } + const segmentSummaries = JSON.stringify( + mapResults.map((m, i) => ({ segment: i, summary: m.segment_summary, proposals: m.improvement_proposals })), + null, + 2, + ); + result = await callReduce(segmentSummaries, statsText, ctx); + usedPrompts = [MAP_PROMPT_HASH, REDUCE_PROMPT_HASH]; + } + + const edges: AnalysisResult["edges"] = []; + // Anchors to session + edges.push({ + toRefKind: REF_KINDS.SESSION, + toRefId: ctx.run.session_id, + edgeKind: EDGE_KINDS.ANCHORS, + ordinal: 0, + }); + // Consumes all dependency nodes + for (const n of pairNodes) { + edges.push({ + toRefKind: REF_KINDS.ANALYSIS_NODE, + toRefId: n.id, + edgeKind: EDGE_KINDS.CONSUMES, + }); + } + for (const n of llmNodes) { + edges.push({ + toRefKind: REF_KINDS.ANALYSIS_NODE, + toRefId: n.id, + edgeKind: EDGE_KINDS.CONSUMES, + }); + } + // uses_prompt for each prompt + for (const h of usedPrompts) { + edges.push({ + toRefKind: REF_KINDS.PROMPT_VERSION, + toRefId: h, + edgeKind: EDGE_KINDS.USES_PROMPT, + }); + } + + return { + contentJson: result as unknown as Record, + nodeKind: "summary", + anchorKind: "session", + anchorRef: ctx.run.session_id, + edges, + }; + }, +}; + +export { + SESSION_OVERVIEW_MAP_PROMPT, + buildMapPrompt, + parseMapResponse, + SESSION_OVERVIEW_REDUCE_PROMPT, + buildReducePrompt, + parseReduceResponse, + buildDigest, + splitDigest, + DEFAULT_SESSION_OVERVIEW_CONFIG, +}; +export type { SessionOverviewConfig, SessionOverviewProperties, MapSummary }; diff --git a/src/analyze/analyzers/session-overview/prompt-map.ts b/src/analyze/analyzers/session-overview/prompt-map.ts new file mode 100644 index 0000000..098e4d7 --- /dev/null +++ b/src/analyze/analyzers/session-overview/prompt-map.ts @@ -0,0 +1,119 @@ +/** + * Map-phase prompt: given a session digest segment, produce a + * segment-level summary with the same shape used by the reduce + * phase. + */ + +export const SESSION_OVERVIEW_MAP_PROMPT = `You are summarizing a SEGMENT of an AI coding agent session. The segment below is a structured digest of user messages, agent responses, tool calls, and friction signals. + +Produce a JSON object summarizing this segment: + +{ + "segment_summary": "2–3 sentence summary of what the user was trying to accomplish and the agent's overall performance in this segment", + "key_friction_points": [ + { + "description": "short noun phrase", + "severity": "low" | "medium" | "high", + "evidence_pair_index": integer + } + ], + "improvement_proposals": [ + { + "target_type": "agents_md" | "system_md" | "skill" | "extension_prompt" | "tool_output" | "repo_doc" | "config", + "target_path": "path to the file or config that should change", + "title": "short imperative title", + "summary": "one-line description", + "detail": "2–3 sentence proposed change", + "evidence": "the pair excerpt that triggered this", + "confidence": 0.0-1.0, + "severity": "friction" | "correction" | "waste" | "suggestion" | "insight" + } + ], + "sentiment_arc": [ + { "segment": 0, "sentiment": "positive" | "neutral" | "negative" | "frustrated", "key_event": "short phrase" } + ] +} + +Rules: +- Only propose changes that are clearly supported by the digest. +- Be specific. "Be more careful" is bad; "When the user says pnpm, do not run npm" is good. +- Confidence 0.0–1.0 reflects how strongly the digest supports the proposal. +- severity values: friction (user struggled), correction (user explicitly corrected), waste (tool calls or context that didn't help), suggestion (opportunity to improve), insight (observation, not an action item). + +Return JSON only. No markdown fences, no prose. + +Segment digest: +""" +{digest} +"""`; + +export function buildMapPrompt(digest: string): string { + return SESSION_OVERVIEW_MAP_PROMPT.replace("{digest}", digest); +} + +const VALID_TARGET_TYPES = new Set([ + "agents_md", "system_md", "skill", "extension_prompt", "tool_output", "repo_doc", "config", +]); +const VALID_SEVERITIES = new Set(["friction", "correction", "waste", "suggestion", "insight"]); + +export interface MapSummary { + segment_summary: string; + key_friction_points: Array<{ description: string; severity: "low" | "medium" | "high"; evidence_pair_index: number }>; + improvement_proposals: Array<{ + target_type: string; + target_path: string; + title: string; + summary: string; + detail: string; + evidence: string; + confidence: number; + severity: string; + }>; + sentiment_arc: Array<{ segment: number; sentiment: string; key_event: string }>; +} + +export function parseMapResponse(text: string): MapSummary { + const empty: MapSummary = { + segment_summary: "", + key_friction_points: [], + improvement_proposals: [], + sentiment_arc: [], + }; + try { + const t = text.trim().replace(/^```json\s*/i, "").replace(/```$/i, "").trim(); + const obj = JSON.parse(t); + if (!obj || typeof obj !== "object") return empty; + const o = obj as Record; + return { + segment_summary: typeof o.segment_summary === "string" ? o.segment_summary : "", + key_friction_points: Array.isArray(o.key_friction_points) + ? (o.key_friction_points as Array>).map((p) => ({ + description: String(p.description ?? ""), + severity: ["low", "medium", "high"].includes(p.severity as string) ? (p.severity as "low" | "medium" | "high") : "medium", + evidence_pair_index: typeof p.evidence_pair_index === "number" ? p.evidence_pair_index : 0, + })) + : [], + improvement_proposals: Array.isArray(o.improvement_proposals) + ? (o.improvement_proposals as Array>).map((p) => ({ + target_type: VALID_TARGET_TYPES.has(p.target_type as string) ? (p.target_type as string) : "repo_doc", + target_path: String(p.target_path ?? ""), + title: String(p.title ?? ""), + summary: String(p.summary ?? ""), + detail: String(p.detail ?? ""), + evidence: String(p.evidence ?? ""), + confidence: typeof p.confidence === "number" ? p.confidence : 0.5, + severity: VALID_SEVERITIES.has(p.severity as string) ? (p.severity as string) : "suggestion", + })) + : [], + sentiment_arc: Array.isArray(o.sentiment_arc) + ? (o.sentiment_arc as Array>).map((s) => ({ + segment: typeof s.segment === "number" ? s.segment : 0, + sentiment: String(s.sentiment ?? "neutral"), + key_event: String(s.key_event ?? ""), + })) + : [], + }; + } catch { + return empty; + } +} diff --git a/src/analyze/analyzers/session-overview/prompt-reduce.ts b/src/analyze/analyzers/session-overview/prompt-reduce.ts new file mode 100644 index 0000000..6172cab --- /dev/null +++ b/src/analyze/analyzers/session-overview/prompt-reduce.ts @@ -0,0 +1,130 @@ +/** + * Reduce-phase prompt: given a list of segment summaries + * (from the map phase) plus the deterministic stats, produce a + * final session-level analysis with materialized proposals. + */ + +export const SESSION_OVERVIEW_REDUCE_PROMPT = `You are producing the FINAL session-level analysis for an AI coding agent session. You will receive a list of per-segment summaries from a previous pass, plus deterministic session statistics. + +Combine the per-segment summaries into a single session-level analysis. De-duplicate proposals that recur across segments — keep the strongest instance. Return JSON only. + +Schema (return exactly this shape): +{ + "session_summary": "3–6 sentence summary of the entire session", + "key_friction_points": [ + { + "description": "short noun phrase", + "pair_node_id": "id of the originating pair node (use the IDs from the segment summaries if available, else 'unknown')", + "severity": "low" | "medium" | "high" + } + ], + "improvement_proposals": [ + { + "target_type": "agents_md" | "system_md" | "skill" | "extension_prompt" | "tool_output" | "repo_doc" | "config", + "target_path": "path to the file or config that should change", + "title": "short imperative title", + "summary": "one-line description", + "detail": "2–3 sentence proposed change", + "evidence": "the conversation excerpt that triggered this", + "confidence": 0.0-1.0, + "severity": "friction" | "correction" | "waste" | "suggestion" | "insight" + } + ], + "sentiment_arc": [ + { "segment": integer, "sentiment": "positive" | "neutral" | "negative" | "frustrated", "key_event": "short phrase" } + ] +} + +Rules: +- De-duplicate: if two segments propose the same change, keep the one with higher confidence. +- Be conservative. A session that ran smoothly may produce zero proposals. +- Each proposal must point at a concrete, actionable file or config change. +- Confidence reflects how strongly the evidence supports the proposal. + +Per-segment summaries: +""" +{segment_summaries} +""" + +Deterministic session statistics: +""" +{stats} +""" + +Return JSON only. No markdown fences, no prose.`; + +export function buildReducePrompt(args: { + segmentSummaries: string; + stats: string; +}): string { + return SESSION_OVERVIEW_REDUCE_PROMPT + .replace("{segment_summaries}", args.segmentSummaries) + .replace("{stats}", args.stats); +} + +const VALID_TARGET_TYPES = new Set([ + "agents_md", "system_md", "skill", "extension_prompt", "tool_output", "repo_doc", "config", +]); +const VALID_SEVERITIES = new Set(["friction", "correction", "waste", "suggestion", "insight"]); + +export interface SessionOverviewProperties { + session_summary: string; + key_friction_points: Array<{ description: string; pair_node_id: string; severity: "low" | "medium" | "high" }>; + improvement_proposals: Array<{ + target_type: string; + target_path: string; + title: string; + summary: string; + detail: string; + evidence: string; + confidence: number; + severity: string; + }>; + sentiment_arc: Array<{ segment: number; sentiment: string; key_event: string }>; +} + +export function parseReduceResponse(text: string): SessionOverviewProperties { + const empty: SessionOverviewProperties = { + session_summary: "", + key_friction_points: [], + improvement_proposals: [], + sentiment_arc: [], + }; + try { + const t = text.trim().replace(/^```json\s*/i, "").replace(/```$/i, "").trim(); + const obj = JSON.parse(t); + if (!obj || typeof obj !== "object") return empty; + const o = obj as Record; + return { + session_summary: typeof o.session_summary === "string" ? o.session_summary : "", + key_friction_points: Array.isArray(o.key_friction_points) + ? (o.key_friction_points as Array>).map((p) => ({ + description: String(p.description ?? ""), + pair_node_id: String(p.pair_node_id ?? "unknown"), + severity: ["low", "medium", "high"].includes(p.severity as string) ? (p.severity as "low" | "medium" | "high") : "medium", + })) + : [], + improvement_proposals: Array.isArray(o.improvement_proposals) + ? (o.improvement_proposals as Array>).map((p) => ({ + target_type: VALID_TARGET_TYPES.has(p.target_type as string) ? (p.target_type as string) : "repo_doc", + target_path: String(p.target_path ?? ""), + title: String(p.title ?? ""), + summary: String(p.summary ?? ""), + detail: String(p.detail ?? ""), + evidence: String(p.evidence ?? ""), + confidence: typeof p.confidence === "number" ? p.confidence : 0.5, + severity: VALID_SEVERITIES.has(p.severity as string) ? (p.severity as string) : "suggestion", + })) + : [], + sentiment_arc: Array.isArray(o.sentiment_arc) + ? (o.sentiment_arc as Array>).map((s) => ({ + segment: typeof s.segment === "number" ? s.segment : 0, + sentiment: String(s.sentiment ?? "neutral"), + key_event: String(s.key_event ?? ""), + })) + : [], + }; + } catch { + return empty; + } +} diff --git a/src/analyze/analyzers/turn-pair-core/config.ts b/src/analyze/analyzers/turn-pair-core/config.ts new file mode 100644 index 0000000..70a5f67 --- /dev/null +++ b/src/analyze/analyzers/turn-pair-core/config.ts @@ -0,0 +1,55 @@ +/** + * Default config for turn-pair-core analyzer. + * + * The friction_score formula and signal weights are tunable here. + */ + +export const DEFAULT_TURN_PAIR_CORE_CONFIG = { + /** Threshold above which friction is considered "high". */ + friction_threshold: 0.4, + /** Per-signal weights for the friction score. */ + weights: { + correction: 0.45, + tool_failure: 0.25, + retry: 0.2, + thinking_present: 0.05, + compaction_boundary: 0.05, + }, + /** Cap on tool failures counted per pair. */ + max_tool_failures: 3, +} as const; + +export type TurnPairCoreConfig = typeof DEFAULT_TURN_PAIR_CORE_CONFIG; + +/** + * Compute a 0.0–1.0 friction score for a turn pair. + * + * The score is a weighted sum of the binary / count signals clipped + * to [0, 1]. Each weight is in [0, 1] and they should sum to ~1.0 + * by default; if a caller supplies different weights, the score + * simply scales to whatever they sum to. + * + * Tool failures contribute fully once they reach `max_tool_failures` + * (a step function, not a linear ramp). This makes the signal robust + * to long failure cascades. + */ +export function computeFrictionScore( + config: TurnPairCoreConfig, + signals: { + correctionDetected: boolean; + toolFailureCount: number; + retryDetected: boolean; + hasThinking: boolean; + isCompactionBoundary: boolean; + }, +): number { + const w = config.weights; + const failures = signals.toolFailureCount >= config.max_tool_failures ? 1 : 0; + const score = + (signals.correctionDetected ? w.correction : 0) + + failures * w.tool_failure + + (signals.retryDetected ? w.retry : 0) + + (signals.hasThinking ? w.thinking_present : 0) + + (signals.isCompactionBoundary ? w.compaction_boundary : 0); + return Math.max(0, Math.min(1, score)); +} diff --git a/src/analyze/analyzers/turn-pair-core/index.ts b/src/analyze/analyzers/turn-pair-core/index.ts new file mode 100644 index 0000000..e5121f1 --- /dev/null +++ b/src/analyze/analyzers/turn-pair-core/index.ts @@ -0,0 +1,411 @@ +/** + * turn-pair-core — deterministic, per-(user, assistant + tool_results) + * pair analyzer. + * + * Produces a single `metric` node per pair with all 19 properties + * from §6.3 of the design doc. No LLM. Always runnable; the + * cheapest pass. + * + * Edges: + * anchors → each message in the pair (user, assistant, tool results) + */ + +import type { + AnalysisResult, + AnalysisUnit, + Analyzer, + AnalyzerConfig, + AnalyzerDef, + AnalyzerPlanContext, + AnalyzerRunContext, + AnalyzerVersion, + MessageRow, + PromptVersion, + SourceRef, +} from "../../types.js"; +import { computeSourceSetHash } from "../../framework.js"; +import { EDGE_KINDS, REF_KINDS } from "../../edge-kinds.js"; +import { + detectAllCorrectionPatterns, + detectCorrection, + detectRepetition, + extractCorrectionText, +} from "./patterns.js"; +import { DEFAULT_TURN_PAIR_CORE_CONFIG, computeFrictionScore, type TurnPairCoreConfig } from "./config.js";export const TURN_PAIR_CORE_DEF: AnalyzerDef = { + id: "turn-pair-core", + label: "Per-Turn Deterministic Metrics", + description: "Computes per-pair metrics (lengths, friction score, tool stats, correction detection) without using an LLM.", + anchorSpan: "pair", + dependencies: [], +}; + +export const TURN_PAIR_CORE_VERSION: AnalyzerVersion = { + analyzerId: TURN_PAIR_CORE_DEF.id, + versionId: "0.1.0", + implementationKind: "deterministic", + codeRef: "src/analyze/analyzers/turn-pair-core/index.ts", +}; + +export const TURN_PAIR_CORE_PROMPTS: Record = {}; + +export interface TurnPairNode { + user_msg_length: number; + assistant_msg_length: number; + has_thinking: boolean; + thinking_length: number; + correction_detected: boolean; + correction_patterns: string[]; + correction_type: "explicit" | "implicit" | "repetition" | null; + correction_text: string | null; + tool_call_count: number; + tool_names: string[]; + tool_failure_count: number; + tool_failure_details: Array<{ tool_name: string; error_preview: string }>; + tool_waste_bytes: number; + retry_detected: boolean; + elapsed_seconds: number | null; + friction_score: number; + model: string | null; + stop_reason: string | null; + usage_input_tokens: number | null; + usage_output_tokens: number | null; + is_compaction_boundary: boolean; + /** Indices into ctx.messages for traceability. */ + user_index: number; + assistant_index: number; +} + +function findPair(messages: MessageRow[], startAt: number): { user: MessageRow; assistant: MessageRow | null; intervening: MessageRow[]; userIndex: number; assistantIndex: number; endIndex: number } | null { + for (let i = startAt; i < messages.length; i++) { + const m = messages[i]!; + if (m.role !== "user") continue; + + // The pair extends from this user message to (but not including) + // the next user message, or the end of the session. Tool results, + // intermediate assistants, and compaction summaries within are + // "intervening" and belong to this pair. The "assistant response" + // is the LAST assistant in the range, if any. Intervening does + // not include the final assistant (that IS the response). + let j = i + 1; + let lastAssistant: MessageRow | null = null; + let lastAssistantIndex = -1; + while (j < messages.length) { + const n = messages[j]!; + if (n.role === "user") break; + if (n.role === "assistant") { + lastAssistant = n; + lastAssistantIndex = j; + } + j++; + } + const endIndex = j - 1; + const intervening: MessageRow[] = []; + for (let k = i + 1; k <= endIndex && k < messages.length; k++) { + if (k !== lastAssistantIndex) intervening.push(messages[k]!); + } + return { user: m, assistant: lastAssistant, intervening, userIndex: i, assistantIndex: lastAssistantIndex, endIndex }; + } + return null; +} + +function parseMeta(metaJson: string | null): { model?: string; stop_reason?: string; usage?: { input?: number; output?: number } } | null { + if (!metaJson) return null; + try { return JSON.parse(metaJson); } catch { return null; } +} + +function toolName(callJson: string | null): string | null { + if (!callJson) return null; + try { + const parsed = JSON.parse(callJson); + if (Array.isArray(parsed) && parsed[0]?.name) return String(parsed[0].name); + } catch { /* ignore */ } + return null; +} + +function toolResultInfo(resultJson: string | null): { toolName: string; isError: boolean; textLength: number } | null { + if (!resultJson) return null; + try { + const parsed = JSON.parse(resultJson); + if (Array.isArray(parsed) && parsed[0]) { + return { + toolName: String(parsed[0].toolName ?? ""), + isError: Boolean(parsed[0].isError), + textLength: Number(parsed[0].textLength ?? 0), + }; + } + } catch { /* ignore */ } + return null; +} + +function buildToolCallKey(name: string, args: Record): string { + // For "read" we use the path; for "bash" the command; etc. + // Anything else uses a JSON of the args. + const target = (args.path as string) + ?? (args.command as string) + ?? (args.file as string) + ?? JSON.stringify(args); + return `${name}::${target}`; +} + +function previewError(text: string | null): string { + if (!text) return ""; + return text.length > 80 ? text.slice(0, 80) + "..." : text; +} + +export function buildTurnPairNode( + messages: MessageRow[], + userIndex: number, + endIndex: number, + config: TurnPairCoreConfig, +): TurnPairNode | null { + const user = messages[userIndex]; + if (!user || user.role !== "user") return null; + + // Find the last assistant in the range [userIndex+1, endIndex] + let assistant: MessageRow | null = null; + let lastAssistantIndex = -1; + for (let k = userIndex + 1; k <= endIndex && k < messages.length; k++) { + if (messages[k]!.role === "assistant") { + assistant = messages[k]!; + lastAssistantIndex = k; + } + } + if (!assistant) return null; + + // Intervening: everything in [userIndex+1, endIndex] EXCEPT the + // final assistant. This is where tool results, compactions, and + // intermediate assistants live. + const intervening: MessageRow[] = []; + for (let k = userIndex + 1; k <= endIndex && k < messages.length; k++) { + if (k !== lastAssistantIndex) intervening.push(messages[k]!); + } + + const userText = user.content_text; + const assistantText = assistant?.content_text ?? null; + const thinking = assistant?.content_thinking ?? null; + + // Tool calls + let toolCalls: Array<{ name: string; arguments: Record }> = []; + if (assistant?.tool_calls) { + try { toolCalls = JSON.parse(assistant.tool_calls); } catch { toolCalls = []; } + } + + // Tool results + let toolResults: Array<{ toolName: string; isError: boolean; textLength: number }> = []; + for (const m of intervening) { + if (m.role === "toolResult") { + const info = toolResultInfo(m.tool_results); + if (info) toolResults.push(info); + } + } + + // Correction detection + const userPriorIndex = findPriorUserIndex(messages, userIndex); + const priorText = userPriorIndex >= 0 ? messages[userPriorIndex]?.content_text ?? null : null; + const correction = detectCorrection(userText); + const correctionPatterns = detectAllCorrectionPatterns(userText); + const repetition = !correction && detectRepetition(userText, priorText); + const correctionType: TurnPairNode["correction_type"] = correction ? "explicit" : (repetition ? "repetition" : null); + const correctionText = correction && userText ? extractCorrectionText(userText, correction) : null; + + // Tool stats + const toolCallCount = toolCalls.length; + const toolNames = [...new Set(toolCalls.map((c) => c.name))]; + const failures = toolResults.filter((r) => r.isError); + const toolFailureDetails: Array<{ tool_name: string; error_preview: string }> = []; + for (let i = 0; i < failures.length; i++) { + const f = failures[i]!; + const resultMsg = intervening.find((m) => m.role === "toolResult" && toolResultInfo(m.tool_results)?.toolName === f.toolName); + toolFailureDetails.push({ tool_name: f.toolName, error_preview: previewError(resultMsg?.content_text ?? null) }); + } + + // Waste bytes: tool results that are never referenced in the + // assistant's text. We approximate by checking the assistant's + // text for any of the tool result text (cheap: any 30-char + // fragment). A more sophisticated approach would diff structured + // references; this is a useful upper bound. + let toolWasteBytes = 0; + if (assistantText) { + for (let i = 0; i < intervening.length; i++) { + const m = intervening[i]!; + if (m.role !== "toolResult") continue; + const resultLen = m.content_text?.length ?? 0; + if (resultLen === 0) continue; + // Pull a 30-char sample + const sample = m.content_text?.slice(0, 30) ?? ""; + if (sample.length >= 10 && !assistantText.includes(sample)) { + toolWasteBytes += resultLen; + } + } + } + + // Retry detection + let retryDetected = false; + const seenTargets = new Set(); + for (const c of toolCalls) { + const key = buildToolCallKey(c.name, c.arguments); + if (seenTargets.has(key)) { + retryDetected = true; + break; + } + seenTargets.add(key); + } + + // Elapsed seconds + let elapsedSeconds: number | null = null; + if (user.timestamp && assistant?.timestamp) { + const u = Date.parse(user.timestamp); + const a = Date.parse(assistant.timestamp); + if (!isNaN(u) && !isNaN(a)) elapsedSeconds = Math.max(0, (a - u) / 1000); + } + + // Compaction boundary + const isCompactionBoundary = intervening.some((m) => m.role === "compactionSummary") || assistant?.role === "compactionSummary"; + + // Assistant meta + const meta = parseMeta(assistant?.meta_json ?? null); + const model = meta?.model ?? null; + const stopReason = meta?.stop_reason ?? null; + const inputTokens = meta?.usage?.input ?? null; + const outputTokens = meta?.usage?.output ?? null; + + // Friction score + const frictionScore = computeFrictionScore(config, { + correctionDetected: correction !== null || repetition, + toolFailureCount: failures.length, + retryDetected, + hasThinking: thinking != null && thinking.length > 0, + isCompactionBoundary, + }); + + return { + user_msg_length: userText?.length ?? 0, + assistant_msg_length: assistantText?.length ?? 0, + has_thinking: thinking != null && thinking.length > 0, + thinking_length: thinking?.length ?? 0, + correction_detected: correction !== null || repetition, + correction_patterns: correctionPatterns, + correction_type: correctionType, + correction_text: correctionText, + tool_call_count: toolCallCount, + tool_names: toolNames, + tool_failure_count: failures.length, + tool_failure_details: toolFailureDetails, + tool_waste_bytes: toolWasteBytes, + retry_detected: retryDetected, + elapsed_seconds: elapsedSeconds, + friction_score: frictionScore, + model, + stop_reason: stopReason, + usage_input_tokens: inputTokens, + usage_output_tokens: outputTokens, + is_compaction_boundary: isCompactionBoundary, + user_index: userIndex, + assistant_index: lastAssistantIndex, + }; +} + +function findPriorUserIndex(messages: MessageRow[], before: number): number { + for (let i = before - 1; i >= 0; i--) { + if (messages[i]!.role === "user") return i; + } + return -1; +} + +export const turnPairCoreAnalyzer: Analyzer = { + def: TURN_PAIR_CORE_DEF, + version: TURN_PAIR_CORE_VERSION, + prompts: TURN_PAIR_CORE_PROMPTS, + defaultConfig: { + id: "", // resolved at registration + analyzerId: TURN_PAIR_CORE_DEF.id, + configJson: DEFAULT_TURN_PAIR_CORE_CONFIG as unknown as Record, + configHash: "", + label: "default", + }, + + async plan(ctx: AnalyzerPlanContext): Promise { + const units: AnalysisUnit[] = []; + let i = 0; + while (true) { + const pair = findPair(ctx.messages, i); + if (!pair) break; + // Skip pairs with no assistant response — there's nothing + // to measure until the assistant acts. + if (!pair.assistant) break; + + const sources: SourceRef[] = []; + for (let k = pair.userIndex; k <= pair.endIndex && k < ctx.messages.length; k++) { + sources.push({ kind: "message", id: ctx.messages[k]!.id }); + } + + units.push({ + sources, + sourceSetHash: computeSourceSetHash(sources), + anchorKind: "pair", + anchorRef: pair.user.id, + meta: { userIndex: pair.userIndex, endIndex: pair.endIndex }, + }); + + i = pair.endIndex + 1; + } + return units; + }, + + async analyze(unit: AnalysisUnit, ctx: AnalyzerRunContext): Promise { + const userIndex = (unit.meta as { userIndex: number } | undefined)?.userIndex; + const endIndex = (unit.meta as { endIndex: number } | undefined)?.endIndex; + if (typeof userIndex !== "number" || typeof endIndex !== "number") { + throw new Error("turn-pair-core: unit.meta missing userIndex/endIndex"); + } + + const config = (ctx.config.configJson as unknown as TurnPairCoreConfig) ?? DEFAULT_TURN_PAIR_CORE_CONFIG; + + // Reconstruct the in-pair messages by reading from the run + // context's getMessage(). unit.sources is already in order + // (built sequentially in plan()). + const messages: MessageRow[] = []; + for (const src of unit.sources) { + if (src.kind !== "message") continue; + const m = ctx.getMessage(src.id); + if (m) messages.push(m); + } + + const props = buildTurnPairNode(messages, 0, messages.length - 1, config); + if (!props) throw new Error(`turn-pair-core: could not build node for unit at userIndex=${userIndex}`); + + const edges: AnalysisResult["edges"] = []; + for (let k = 0; k < messages.length; k++) { + edges.push({ + toRefKind: REF_KINDS.MESSAGE, + toRefId: messages[k]!.id, + edgeKind: EDGE_KINDS.ANCHORS, + ordinal: k, + }); + } + edges.push({ + toRefKind: REF_KINDS.SESSION, + toRefId: ctx.run.session_id, + edgeKind: EDGE_KINDS.ANCHORS, + ordinal: 999, + }); + + return { + contentJson: props as unknown as Record, + nodeKind: "metric", + anchorKind: "pair", + anchorRef: unit.anchorRef, + edges, + }; + }, +}; + +// Re-exports for unit tests +export { DEFAULT_TURN_PAIR_CORE_CONFIG, computeFrictionScore } from "./config.js"; +export { + detectCorrection, + detectAllCorrectionPatterns, + detectRepetition, + extractCorrectionText, +} from "./patterns.js"; +export type { TurnPairCoreConfig } from "./config.js"; diff --git a/src/analyze/analyzers/turn-pair-core/patterns.ts b/src/analyze/analyzers/turn-pair-core/patterns.ts new file mode 100644 index 0000000..e333f16 --- /dev/null +++ b/src/analyze/analyzers/turn-pair-core/patterns.ts @@ -0,0 +1,146 @@ +/** + * Correction and friction detection patterns. + * + * These regex sets are matched against user message text. They are + * intentionally simple — false positives get filtered by the LLM + * pass in turn-pair-llm. The deterministic pass is meant to flag + * candidates cheaply. + * + * Categories: + * - strong_explicit: clearly corrective ("no, use X", "actually...") + * - weak_explicit: hedged but corrective ("could you try X", "maybe Y") + * - negation: leading negative words that flip intent + * - repetition: same intent, different words + */ + +export type CorrectionType = "explicit" | "implicit" | "repetition" | null; + +export interface CorrectionMatch { + pattern: string; + type: NonNullable; + matched: string; +} + +const STRONG_EXPLICIT: RegExp[] = [ + /\bno[,.\s]+(use|do|that's|that's|that is|it'?s|it is)\b/i, + /\bnot\s+that\b/i, + /\bnot\s+like\s+(that|this)\b/i, + /\bno,\s+\w/i, // "no, do X" + /\bdon'?t\s+(do|use|run|edit|add|remove|change|try)\b/i, + /\bstop\s+(doing|using|running|trying)\b/i, + /\bthat'?s\s+wrong\b/i, + /\bthat\s+is\s+wrong\b/i, + /\bthat'?s\s+not\s+what\b/i, + /\bthat\s+is\s+not\s+what\b/i, + /\bthat'?s\s+incorrect\b/i, + /\bthat'?s\s+incorrect\b/i, + /\binstead\s+of\s+(that|this)\b/i, + /\binstead,?\s+\w/i, + /\bI\s+said\b/i, + /\bI\s+told\s+you\b/i, + /\bI\s+already\s+(said|told|mentioned)\b/i, + /\bI\s+meant\b/i, + /\bactually[,.\s]/i, + /\bI\s+meant\s+to\s+say\b/i, +]; + +const WEAK_EXPLICIT: RegExp[] = [ + /\bcould\s+you\s+(please\s+)?(try|use|do|change|switch)\b/i, + /\bmaybe\s+(we|you|try)\b/i, + /\bperhaps\s+(we|you|try)\b/i, + /\bwhy\s+don'?t\s+you\b/i, + /\bcan\s+you\s+(try|use|do)\s+instead\b/i, + /\bprefer\s+to\s+(use|do)\b/i, + /\bplease\s+(use|do|try)\b/i, + /\bshould\s+(use|do|be)\b/i, + /\bjust\s+\w+\b/i, // "just do X" — often corrective +]; + +const NEGATION: RegExp[] = [ + /^\s*no\b/i, + /^\s*not\b/i, + /^\s*never\b/i, + /^\s*don'?t\b/i, + /^\s*doesn'?t\b/i, + /^\s*didn'?t\b/i, + /^\s*won'?t\b/i, + /^\s*can'?t\b/i, + /^\s*shouldn'?t\b/i, + /^\s*wouldn'?t\b/i, + /^\s*isn'?t\b/i, + /^\s*aren'?t\b/i, +]; + +/** + * Detect a correction in the given user text. Returns the first + * match found, or null. + * + * Strong patterns take priority over weak patterns. Negation is + * only flagged in isolation (the user must lead with a negative + * word, not use one mid-sentence). + */ +export function detectCorrection(text: string | null): CorrectionMatch | null { + if (!text) return null; + for (const re of STRONG_EXPLICIT) { + const m = re.exec(text); + if (m) return { pattern: re.source, type: "explicit", matched: m[0] }; + } + for (const re of WEAK_EXPLICIT) { + const m = re.exec(text); + if (m) return { pattern: re.source, type: "explicit", matched: m[0] }; + } + for (const re of NEGATION) { + const m = re.exec(text); + if (m) return { pattern: re.source, type: "explicit", matched: m[0] }; + } + return null; +} + +/** + * Returns all matched patterns, not just the first. Used by the + * `correction_patterns` field on a turn-pair node. + */ +export function detectAllCorrectionPatterns(text: string | null): string[] { + if (!text) return []; + const matched: string[] = []; + for (const re of STRONG_EXPLICIT) if (re.test(text)) matched.push(re.source); + for (const re of WEAK_EXPLICIT) if (re.test(text)) matched.push(re.source); + for (const re of NEGATION) if (re.test(text)) matched.push(re.source); + return matched; +} + +/** + * Detect if a user message is repeating a prior request without + * new content. We use a simple length-based heuristic: if the + * message is short (< 40 chars) and the prior user message also + * exists in the session, the chance of repetition is high. + * + * A proper "repetition" detector would compare embeddings; this + * is a cheap signal only. + */ +export function detectRepetition(text: string | null, priorUserText: string | null): boolean { + if (!text) return false; + if (text.length > 40) return false; + if (!priorUserText) return false; + const overlap = sharedTokenCount(text, priorUserText); + return overlap >= 2; +} + +function sharedTokenCount(a: string, b: string): number { + const aTokens = new Set(a.toLowerCase().split(/\W+/).filter((t) => t.length > 2)); + const bTokens = new Set(b.toLowerCase().split(/\W+/).filter((t) => t.length > 2)); + let count = 0; + for (const t of aTokens) if (bTokens.has(t)) count++; + return count; +} + +/** + * Extract the corrective instruction itself — the part of the + * message that is corrective. We use a simple heuristic: take the + * substring after the matched pattern. + */ +export function extractCorrectionText(text: string, match: CorrectionMatch): string { + const idx = text.toLowerCase().indexOf(match.matched.toLowerCase()); + if (idx < 0) return text; + return text.slice(idx + match.matched.length).trim().slice(0, 240) || match.matched; +} diff --git a/src/analyze/analyzers/turn-pair-llm/config.ts b/src/analyze/analyzers/turn-pair-llm/config.ts new file mode 100644 index 0000000..bcd7be4 --- /dev/null +++ b/src/analyze/analyzers/turn-pair-llm/config.ts @@ -0,0 +1,14 @@ +/** + * Default config for the turn-pair-llm analyzer. + */ + +export const DEFAULT_TURN_PAIR_LLM_CONFIG = { + /** Minimum friction_score (0–1) from the deterministic pass to qualify. */ + friction_threshold: 0.4, + /** Whether to require correction_detected (in addition to the score). */ + require_correction: false, + /** Maximum turns per session to enrich (skip beyond to bound cost). */ + max_pairs_per_session: 50, +} as const; + +export type TurnPairLlmConfig = typeof DEFAULT_TURN_PAIR_LLM_CONFIG; diff --git a/src/analyze/analyzers/turn-pair-llm/index.ts b/src/analyze/analyzers/turn-pair-llm/index.ts new file mode 100644 index 0000000..7726014 --- /dev/null +++ b/src/analyze/analyzers/turn-pair-llm/index.ts @@ -0,0 +1,269 @@ +/** + * turn-pair-llm — LLM enrichment for high-signal turn pairs. + * + * Depends on turn-pair-core. Filters dependency nodes to those + * flagged as corrections or high friction, then asks an LLM to + * classify sentiment, frustration, friction cause, and quality. + * + * The framework passes a `cheap` model (configured in + * prospector.json's `models.cheap`) via the run context's + * `llm()` function. The analyzer doesn't pick the model. + * + * Edges produced: + * refines → turn-pair-core node + * consumes → turn-pair-core node + * anchors → each message in the underlying pair (inherited) + * uses_prompt → the classification prompt + */ + +import type { + AnalysisNodeRow, + AnalysisResult, + AnalysisUnit, + Analyzer, + AnalyzerConfig, + AnalyzerDef, + AnalyzerPlanContext, + AnalyzerRunContext, + AnalyzerVersion, + PromptVersion, +} from "../../types.js"; +import { computeSourceSetHash } from "../../framework.js"; +import { EDGE_KINDS, REF_KINDS } from "../../edge-kinds.js"; +import { TURN_PAIR_CORE_DEF } from "../turn-pair-core/index.js"; +import { fullHash, shortHash } from "../../input-hash.js"; +import { + TURN_PAIR_LLM_PROMPT, + TURN_PAIR_LLM_PROMPT_NAME, + buildTurnPairLlmPrompt, + parseTurnPairLlmResponse, +} from "./prompt.js"; +import { DEFAULT_TURN_PAIR_LLM_CONFIG, type TurnPairLlmConfig } from "./config.js"; + +export const TURN_PAIR_LLM_DEF: AnalyzerDef = { + id: "turn-pair-llm", + label: "Per-Turn LLM Sentiment & Friction", + description: "Enriches high-signal turn pairs (corrections, high friction) with an LLM classification of sentiment, frustration, friction cause, and quality.", + anchorSpan: "pair", + dependencies: [TURN_PAIR_CORE_DEF.id], +}; + +export const TURN_PAIR_LLM_VERSION: AnalyzerVersion = { + analyzerId: TURN_PAIR_LLM_DEF.id, + versionId: "0.1.0", + implementationKind: "in_process_llm", + codeRef: "src/analyze/analyzers/turn-pair-llm/index.ts", +}; + +const PROMPT_HASH = shortHash(TURN_PAIR_LLM_PROMPT); +const PROMPT_FULL_HASH = fullHash(TURN_PAIR_LLM_PROMPT); + +const TURN_PAIR_LLM_PROMPTS: Record = { + [TURN_PAIR_LLM_PROMPT_NAME]: { + hash: PROMPT_HASH, + content: TURN_PAIR_LLM_PROMPT, + fullHash: PROMPT_FULL_HASH, + role: "classify", + }, +}; + +interface TurnPairCoreProps { + user_msg_length: number; + assistant_msg_length: number; + has_thinking: boolean; + thinking_length: number; + correction_detected: boolean; + correction_patterns: string[]; + correction_type: "explicit" | "implicit" | "repetition" | null; + correction_text: string | null; + tool_call_count: number; + tool_names: string[]; + tool_failure_count: number; + tool_waste_bytes: number; + retry_detected: boolean; + elapsed_seconds: number | null; + friction_score: number; + is_compaction_boundary: boolean; + user_index: number; + assistant_index: number; + [key: string]: unknown; +} + +function isHighSignal(node: AnalysisNodeRow, config: TurnPairLlmConfig): boolean { + let props: TurnPairCoreProps; + try { props = JSON.parse(node.content_json) as TurnPairCoreProps; } catch { return false; } + if (config.require_correction && !props.correction_detected) return false; + return Boolean(props.correction_detected) || props.friction_score >= config.friction_threshold; +} + +export const turnPairLlmAnalyzer: Analyzer = { + def: TURN_PAIR_LLM_DEF, + version: TURN_PAIR_LLM_VERSION, + prompts: TURN_PAIR_LLM_PROMPTS, + defaultConfig: { + id: "", + analyzerId: TURN_PAIR_LLM_DEF.id, + configJson: DEFAULT_TURN_PAIR_LLM_CONFIG as unknown as Record, + configHash: "", + label: "default", + }, + + async plan(ctx: AnalyzerPlanContext): Promise { + const config = (ctx.dependencyNodes[TURN_PAIR_CORE_DEF.id]?.[0] + ? (JSON.parse(ctx.dependencyNodes[TURN_PAIR_CORE_DEF.id]![0]!.content_json) as TurnPairCoreProps) + : null) + ? DEFAULT_TURN_PAIR_LLM_CONFIG + : DEFAULT_TURN_PAIR_LLM_CONFIG; + + // We can't read the config here (plan doesn't have it), so + // we use defaults. The actual filtering happens at run time + // because the config is per-run. + const effectiveConfig = config; + + const pairNodes = ctx.dependencyNodes[TURN_PAIR_CORE_DEF.id] ?? []; + const highSignal = pairNodes + .filter((n) => isHighSignal(n, effectiveConfig)) + .slice(0, DEFAULT_TURN_PAIR_LLM_CONFIG.max_pairs_per_session); + + return highSignal.map((n) => ({ + sources: [{ kind: "analysis_node", id: n.id }], + sourceSetHash: computeSourceSetHash([{ kind: "analysis_node", id: n.id }]), + anchorKind: "analysis_node", + anchorRef: n.id, + meta: { deterministicNodeId: n.id }, + })); + }, + + async analyze(unit: AnalysisUnit, ctx: AnalyzerRunContext): Promise { + const detId = (unit.meta as { deterministicNodeId?: string } | undefined)?.deterministicNodeId; + if (!detId) throw new Error("turn-pair-llm: missing deterministicNodeId in unit meta"); + + const detNode = ctx.getNode(detId); + if (!detNode) throw new Error(`turn-pair-llm: dependency node not found: ${detId}`); + + let detProps: TurnPairCoreProps; + try { detProps = JSON.parse(detNode.content_json) as TurnPairCoreProps; } + catch (e) { + throw new Error(`turn-pair-llm: invalid content_json in dependency node: ${(e as Error).message}`); + } + + // Read the messages anchored to the deterministic node. + const messages = ctx.getAnchoredMessages(detId); + if (messages.length === 0) { + throw new Error("turn-pair-llm: deterministic node has no anchored message edges; cannot build prompt"); + } + + const userMsg = messages.find((m) => m.role === "user"); + const assistantMsg = [...messages].reverse().find((m) => m.role === "assistant"); + if (!userMsg || !assistantMsg) { + throw new Error("turn-pair-llm: cannot find user and assistant messages for pair"); + } + + // Build tool call / result digests + const toolCalls: Array<{ name: string; args: unknown }> = []; + const toolResults: Array<{ tool: string; ok: boolean; preview: string }> = []; + for (const m of messages) { + if (m.role === "assistant" && m.tool_calls) { + try { + const calls = JSON.parse(m.tool_calls); + if (Array.isArray(calls)) { + for (const c of calls) { + toolCalls.push({ name: c.name, args: c.arguments }); + } + } + } catch { /* ignore */ } + } + if (m.role === "toolResult" && m.tool_results) { + try { + const results = JSON.parse(m.tool_results); + if (Array.isArray(results)) { + for (const r of results) { + toolResults.push({ + tool: r.toolName, + ok: !r.isError, + preview: (m.content_text ?? "").slice(0, 200), + }); + } + } + } catch { /* ignore */ } + } + } + + const prompt = buildTurnPairLlmPrompt({ + userText: userMsg.content_text ?? "", + assistantText: assistantMsg.content_text ?? "", + toolCalls: JSON.stringify(toolCalls, null, 2), + toolResults: JSON.stringify(toolResults, null, 2), + friction: { + correction_detected: detProps.correction_detected, + friction_score: detProps.friction_score, + tool_failure_count: detProps.tool_failure_count, + retry_detected: detProps.retry_detected, + thinking_length: detProps.thinking_length, + }, + }); + + const start = Date.now(); + const response = await ctx.llm({ + model: "cheap", // framework resolves to a concrete model + system: "You are a turn-pair classifier. Return JSON only.", + user: prompt, + temperature: 0.0, + maxTokens: 600, + }); + const durationMs = Date.now() - start; + + const classification = parseTurnPairLlmResponse(response.text); + + const edges: AnalysisResult["edges"] = [ + { + toRefKind: REF_KINDS.ANALYSIS_NODE, + toRefId: detId, + edgeKind: EDGE_KINDS.REFINES, + }, + { + toRefKind: REF_KINDS.ANALYSIS_NODE, + toRefId: detId, + edgeKind: EDGE_KINDS.CONSUMES, + }, + { + toRefKind: REF_KINDS.PROMPT_VERSION, + toRefId: PROMPT_HASH, + edgeKind: EDGE_KINDS.USES_PROMPT, + }, + ]; + // Anchor to the same messages as the deterministic node + for (const m of messages) { + edges.push({ + toRefKind: REF_KINDS.MESSAGE, + toRefId: m.id, + edgeKind: EDGE_KINDS.ANCHORS, + }); + } + edges.push({ + toRefKind: REF_KINDS.SESSION, + toRefId: ctx.run.session_id, + edgeKind: EDGE_KINDS.ANCHORS, + ordinal: 999, + }); + + return { + contentJson: classification as unknown as Record, + nodeKind: "classification", + anchorKind: "analysis_node", + anchorRef: detId, + edges, + modelUsed: response.model, + costUsd: response.costUsd, + tokensUsed: response.tokensUsed, + durationMs, + }; + }, +}; + +export { TURN_PAIR_LLM_PROMPT, buildTurnPairLlmPrompt, parseTurnPairLlmResponse } from "./prompt.js"; +export { DEFAULT_TURN_PAIR_LLM_CONFIG } from "./config.js"; +export type { TurnPairLlmConfig } from "./config.js"; + +// Re-exports for tests +export type { TurnPairLlmClassification } from "./prompt.js"; diff --git a/src/analyze/analyzers/turn-pair-llm/prompt.ts b/src/analyze/analyzers/turn-pair-llm/prompt.ts new file mode 100644 index 0000000..524b81f --- /dev/null +++ b/src/analyze/analyzers/turn-pair-llm/prompt.ts @@ -0,0 +1,158 @@ +/** + * Prompt template for the turn-pair-llm analyzer. + * + * Classifies a flagged turn pair along several axes: + * - sentiment + * - frustration level (0–10) + * - correction type + * - friction cause and summary + * - user intent + * - quality score (1–5) + * + * The output is plain JSON, not tool-calling. We parse the response + * with the shared parser below. + */ + +export const TURN_PAIR_LLM_PROMPT_NAME = "classify-turn-pair"; + +export const TURN_PAIR_LLM_PROMPT = `You are analyzing a single turn pair from an AI coding agent session. A "turn pair" is a user message followed by the agent's response (and any tool calls/results in between). + +You will receive: + - The user message text + - The agent's final text response + - A pre-computed "friction summary" flagging whether the turn had corrections, tool failures, retries, or high thinking time + - The tool calls and results during the turn + +Classify this turn pair along these axes. Return ONLY a JSON object — no prose, no markdown fences. + +Schema (return exactly this shape, no extra keys): +{ + "sentiment": "positive" | "neutral" | "negative" | "frustrated", + "frustration_level": 0-10, + "correction_type_llm": "explicit" | "implicit" | "repetition" | null, + "friction_cause": string | null, + "friction_summary": string | null, + "user_intent": string, + "quality_score": 1-5 +} + +Rules: +- sentiment: pick the dominant emotional tone of the user's message and context. +- frustration_level: 0 = no frustration, 10 = extreme frustration. +- correction_type_llm: only set if the deterministic pass flagged correction_detected=true. Use: + "explicit" when the user clearly corrects the agent ("no, use X", "actually...", "that's wrong") + "implicit" when the user course-corrects without explicit pushback ("maybe try X") + "repetition" when the user re-asks the same thing + null otherwise +- friction_cause: a short noun phrase naming the cause (e.g. "wrong_function_name", "missing_test_step", "ambiguous_request"). null if no friction. +- friction_summary: 1–2 sentences explaining the friction. null if no friction. +- user_intent: one sentence describing what the user was trying to accomplish. +- quality_score: 1 = very poor agent response, 5 = excellent. + +User message: +""" +{user_text} +""" + +Agent response: +""" +{assistant_text} +""" + +Tool calls during this turn: +""" +{tool_calls_text} +""" + +Tool results during this turn: +""" +{tool_results_text} +""" + +Pre-computed friction signals: +- correction_detected: {correction_detected} +- friction_score: {friction_score} +- tool_failure_count: {tool_failure_count} +- retry_detected: {retry_detected} +- thinking_length: {thinking_length} + +Return JSON only.`; + +export function buildTurnPairLlmPrompt(args: { + userText: string; + assistantText: string; + toolCalls: string; + toolResults: string; + friction: { + correction_detected: boolean; + friction_score: number; + tool_failure_count: number; + retry_detected: boolean; + thinking_length: number; + }; +}): string { + return TURN_PAIR_LLM_PROMPT + .replace("{user_text}", args.userText) + .replace("{assistant_text}", args.assistantText) + .replace("{tool_calls_text}", args.toolCalls) + .replace("{tool_results_text}", args.toolResults) + .replace("{correction_detected}", String(args.friction.correction_detected)) + .replace("{friction_score}", args.friction.friction_score.toFixed(2)) + .replace("{tool_failure_count}", String(args.friction.tool_failure_count)) + .replace("{retry_detected}", String(args.friction.retry_detected)) + .replace("{thinking_length}", String(args.friction.thinking_length)); +} + +const VALID_SENTIMENTS = new Set(["positive", "neutral", "negative", "frustrated"]); +const VALID_CORRECTIONS = new Set(["explicit", "implicit", "repetition"]); + +export interface TurnPairLlmClassification { + sentiment: "positive" | "neutral" | "negative" | "frustrated"; + frustration_level: number; + correction_type_llm: "explicit" | "implicit" | "repetition" | null; + friction_cause: string | null; + friction_summary: string | null; + user_intent: string; + quality_score: number; +} + +/** + * Parse the LLM's JSON response into a typed classification. + * Defensive against malformed output: returns sensible defaults + * rather than throwing, since classification is a soft signal. + */ +export function parseTurnPairLlmResponse(text: string): TurnPairLlmClassification { + const defaults: TurnPairLlmClassification = { + sentiment: "neutral", + frustration_level: 0, + correction_type_llm: null, + friction_cause: null, + friction_summary: null, + user_intent: "", + quality_score: 3, + }; + try { + // Strip optional code fences + const trimmed = text.trim().replace(/^```json\s*/i, "").replace(/```$/i, "").trim(); + const obj = JSON.parse(trimmed); + if (!obj || typeof obj !== "object") return defaults; + const o = obj as Record; + return { + sentiment: VALID_SENTIMENTS.has(o.sentiment as string) ? (o.sentiment as TurnPairLlmClassification["sentiment"]) : defaults.sentiment, + frustration_level: clampInt(o.frustration_level, 0, 10), + correction_type_llm: VALID_CORRECTIONS.has(o.correction_type_llm as string) ? (o.correction_type_llm as TurnPairLlmClassification["correction_type_llm"]) : null, + friction_cause: typeof o.friction_cause === "string" ? o.friction_cause : null, + friction_summary: typeof o.friction_summary === "string" ? o.friction_summary : null, + user_intent: typeof o.user_intent === "string" ? o.user_intent : "", + quality_score: clampInt(o.quality_score, 1, 5), + }; + } catch { + return defaults; + } +} + +function clampInt(v: unknown, lo: number, hi: number): number { + const n = typeof v === "number" ? Math.round(v) : NaN; + if (isNaN(n)) return lo; + return Math.max(lo, Math.min(hi, n)); +} From fe12c14e87efa6930a38ff0835e2d5477fb68922 Mon Sep 17 00:00:00 2001 From: Nicolas Marchildon Date: Tue, 2 Jun 2026 11:11:30 -0400 Subject: [PATCH 3/5] sync: capture assistant meta (model, usage, stopReason) into messages.meta_json The deterministic turn-pair-core analyzer needs the assistant's model, usage, and stop_reason to populate per-pair metrics. The JSONL parser now extracts these from the assistant message envelope and the sync loop writes them as a JSON meta_json column on the messages table. --- src/sync/index.ts | 1 + src/sync/parser.ts | 20 ++++++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/sync/index.ts b/src/sync/index.ts index 7e91a7a..88af76c 100644 --- a/src/sync/index.ts +++ b/src/sync/index.ts @@ -97,6 +97,7 @@ export function runSync(db: Database.Database, sessionsDir: string): SyncResult content_thinking: entry.thinking, tool_calls: entry.tool_calls ? JSON.stringify(entry.tool_calls) : null, tool_results: entry.tool_results ? JSON.stringify(entry.tool_results) : null, + meta_json: entry.meta ? JSON.stringify(entry.meta) : null, }); msgCount++; } diff --git a/src/sync/parser.ts b/src/sync/parser.ts index c7235f0..1df2f70 100644 --- a/src/sync/parser.ts +++ b/src/sync/parser.ts @@ -19,6 +19,7 @@ export interface ParsedMessage { thinking: string | null; tool_calls: Array<{ name: string; arguments: Record }> | null; tool_results: Array<{ toolCallId: string; toolName: string; isError: boolean; textLength: number }> | null; + meta?: Record | null; }; } @@ -67,6 +68,7 @@ export function parseLine(line: string): ParsedLine | null { let thinking: string | null = null; let tool_calls: ParsedMessage["entry"]["tool_calls"] = null; let tool_results: ParsedMessage["entry"]["tool_results"] = null; + let meta: Record | null = null; if (typeof content === "string") { text = content; @@ -104,9 +106,23 @@ export function parseLine(line: string): ParsedLine | null { }]; } + // Capture assistant / model metadata for analyzers that need + // model, usage, stopReason, etc. + if (msg && typeof msg === "object") { + const m = msg as Record; + if (m.model !== undefined || m.usage !== undefined || m.stopReason !== undefined || m.api !== undefined || m.provider !== undefined) { + meta = {}; + if (m.model !== undefined) meta.model = m.model; + if (m.api !== undefined) meta.api = m.api; + if (m.provider !== undefined) meta.provider = m.provider; + if (m.stopReason !== undefined) meta.stop_reason = m.stopReason; + if (m.usage !== undefined) meta.usage = m.usage; + } + } + return { kind: "message", - entry: { id, parentId, timestamp, role: role as MessageRole, text, thinking, tool_calls, tool_results }, + entry: { id, parentId, timestamp, role: role as MessageRole, text, thinking, tool_calls, tool_results, meta }, }; } @@ -131,7 +147,7 @@ export function parseLine(line: string): ParsedLine | null { return { kind: "message", - entry: { id, parentId, timestamp, role: role as MessageRole, text, thinking: null, tool_calls: null, tool_results: null }, + entry: { id, parentId, timestamp, role: role as MessageRole, text, thinking: null, tool_calls: null, tool_results: null, meta: null }, }; } From f10cadc0fb798fac91114384ec24b64259406fba Mon Sep 17 00:00:00 2001 From: Nicolas Marchildon Date: Tue, 2 Jun 2026 11:11:51 -0400 Subject: [PATCH 4/5] commands: wire framework, drop legacy single-prompt parser - analyze command runs the framework's three default analyzers in order over each unanalyzed session; supports --analyzer and --limit; reports node and proposal counts - proposals command now reads the enriched view (target_type, target_path, title); accept/reject also work on 'open' status - stats command shows analyzer-framework health (registered analyzers, node counts per analyzer, successful runs) - tool gains an 'analyze' action callable from a Pi agent - index.ts installs the default LLM caller (delegates to pi.ai) - drop src/analyze/prompt.ts and parser.ts: their job moved to the per-analyzer prompt modules with proper schemas --- src/analyze/parser.ts | 63 ----------------------------------- src/analyze/prompt.ts | 69 --------------------------------------- src/commands/analyze.ts | 43 ++++++++++++++++-------- src/commands/proposals.ts | 26 +++++++++------ src/commands/stats.ts | 36 ++++++++++++++++++-- src/commands/tool.ts | 43 ++++++++++++++++++++---- src/index.ts | 50 +++++++++++++++++++++++++++- 7 files changed, 165 insertions(+), 165 deletions(-) delete mode 100644 src/analyze/parser.ts delete mode 100644 src/analyze/prompt.ts diff --git a/src/analyze/parser.ts b/src/analyze/parser.ts deleted file mode 100644 index 6c7713f..0000000 --- a/src/analyze/parser.ts +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Parse LLM analysis response into structured proposals. - */ - -export interface ParsedProposal { - target: string; - severity: "friction" | "correction" | "waste" | "suggestion"; - summary: string; - detail: string; - evidence: string; -} - -const VALID_SEVERITIES = new Set(["friction", "correction", "waste", "suggestion"]); - -/** - * Parse LLM tool-call response into typed proposals. - * Handles both tool-call arguments object and plain JSON text. - */ -export function parseAnalysisResponse(response: unknown): ParsedProposal[] { - // If it's already a parsed tool call arguments object - if (response && typeof response === "object" && "proposals" in response) { - const proposals = (response as { proposals: unknown[] }).proposals; - if (Array.isArray(proposals)) { - return proposals.filter(isValidProposal).map(normalizeProposal); - } - } - - // If it's a string, try to extract JSON - if (typeof response === "string") { - const jsonMatch = response.match(/```json\s*([\s\S]*?)```/) ?? - response.match(/(\{[\s\S]*\})/); - if (jsonMatch) { - try { - const parsed = JSON.parse(jsonMatch[1] ?? jsonMatch[0]!); - if (parsed && typeof parsed === "object" && "proposals" in parsed && Array.isArray(parsed.proposals)) { - return parsed.proposals.filter(isValidProposal).map(normalizeProposal); - } - } catch { /* ignore */ } - } - } - - return []; -} - -function isValidProposal(item: unknown): item is Record { - if (!item || typeof item !== "object") return false; - const p = item as Record; - return typeof p.target === "string" && typeof p.summary === "string" && p.target.length > 0 && p.summary.length > 0; -} - -function normalizeProposal(item: Record): ParsedProposal { - const severity = VALID_SEVERITIES.has(item.severity as string) - ? (item.severity as ParsedProposal["severity"]) - : "suggestion"; - - return { - target: String(item.target), - severity, - summary: String(item.summary), - detail: String(item.detail ?? ""), - evidence: String(item.evidence ?? ""), - }; -} \ No newline at end of file diff --git a/src/analyze/prompt.ts b/src/analyze/prompt.ts deleted file mode 100644 index facf044..0000000 --- a/src/analyze/prompt.ts +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Prompt template for session friction/sentiment extraction. - * Uses tool-calling schema for structured LLM output. - */ - -export const ANALYSIS_TOOL_NAME = "submit_proposals"; - -export const ANALYSIS_SYSTEM_PROMPT = `You are a session analyst for an AI coding agent. You review session transcripts and identify friction, corrections, waste, and suggestions for improvement. - -You MUST call the ${ANALYSIS_TOOL_NAME} tool with your findings. Do not respond in plain text. - -Focus on: -1. **Friction**: Moments where the user struggled, repeated themselves, or had to course-correct the agent. -2. **Corrections**: Times the user explicitly corrected the agent ("no, use X", "not like that", "actually..."). -3. **Waste**: Tool calls or context that didn't contribute to the task — large file reads never referenced, failed commands retried without changes. -4. **Suggestions**: Opportunities to improve the agent's configuration, skills, or documentation based on observed patterns. - -Each proposal should target a specific, actionable improvement. Prefer specific, small changes over vague recommendations.`; - -export function buildAnalysisPrompt(transcript: string, sessionProject: string): string { - return `## Session: ${sessionProject} - - -${transcript} - - -Analyze this session transcript for friction, corrections, waste, and suggestions. Call the ${ANALYSIS_TOOL_NAME} tool with your findings.`; -} - -export const ANALYSIS_TOOL_SCHEMA = { - name: ANALYSIS_TOOL_NAME, - description: "Submit proposals for improving the coding agent based on session analysis", - parameters: { - type: "object" as const, - properties: { - proposals: { - type: "array" as const, - items: { - type: "object" as const, - properties: { - target: { - type: "string" as const, - description: "What to change, e.g. 'AGENTS.md § Tool usage' or 'skill/debug-typescript-errors'", - }, - severity: { - type: "string" as const, - enum: ["friction", "correction", "waste", "suggestion"], - description: "The type of finding", - }, - summary: { - type: "string" as const, - description: "One-line description of the proposed change", - }, - detail: { - type: "string" as const, - description: "Full proposal text with context and suggested change", - }, - evidence: { - type: "string" as const, - description: "The session excerpt that triggered this proposal", - }, - }, - required: ["target", "severity", "summary", "detail", "evidence"], - }, - }, - }, - required: ["proposals"], - }, -}; \ No newline at end of file diff --git a/src/commands/analyze.ts b/src/commands/analyze.ts index 1d6aa7a..39144d0 100644 --- a/src/commands/analyze.ts +++ b/src/commands/analyze.ts @@ -1,12 +1,14 @@ import type { ExtensionAPI } from "../pi-stubs.js"; import Database from "better-sqlite3"; import { migrate } from "../db/schema.js"; -import { getUnanalyzedSessions, getSessionMessages, markAnalyzed } from "../db/queries.js"; +import { getUnanalyzedSessions } from "../db/queries.js"; import { getDbPath, loadConfig } from "../config.js"; +import { AnalyzerFramework } from "../analyze/framework.js"; +import { registerDefaults, getDefaultLLMCaller } from "../analyze/defaults.js"; export function registerAnalyzeCommand(pi: ExtensionAPI): void { pi.registerCommand("prospect-analyze", { - description: "Run LLM analysis over unanalyzed sessions to generate proposals", + description: "Run analyzer framework over unanalyzed sessions (turn-pair-core, turn-pair-llm, session-overview)", handler: async (args: string, ctx: { ui: { notify: (msg: string, level: string) => void } }) => { const config = loadConfig(); const parsedArgs = parseArgs(args ?? ""); @@ -35,19 +37,28 @@ export function registerAnalyzeCommand(pi: ExtensionAPI): void { ctx.ui.notify(startMsg, "info"); console.log(startMsg); + const llm = getDefaultLLMCaller(); + const fw = new AnalyzerFramework({ db, llm }); + registerDefaults(fw); + + let totalNodes = 0; let totalProposals = 0; let errors = 0; + const analyzersToRun = parsedArgs.analyzer ? [parsedArgs.analyzer] : ["turn-pair-core", "turn-pair-llm", "session-overview"]; for (const session of unanalyzed) { try { - const messages = getSessionMessages(db, session.id); - if (messages.length < 2) { - markAnalyzed(db, session.id); - continue; + for (const analyzerId of analyzersToRun) { + if (!fw.get(analyzerId)) continue; + const summary = await fw.run(analyzerId, session.id, { model: modelSpec }); + totalNodes += summary.nodesProduced; + if (summary.status === "error") { + errors++; + const errMsg = `Error on session ${session.id} analyzer ${analyzerId}: ${summary.status}`; + ctx.ui.notify(errMsg, "warning"); + console.error(errMsg); + } } - - // TODO: Call LLM via @earendil-works/pi-ai - markAnalyzed(db, session.id); } catch (err) { errors++; const errMsg = `Error on session ${session.id}: ${err}`; @@ -56,7 +67,11 @@ export function registerAnalyzeCommand(pi: ExtensionAPI): void { } } - const doneMsg = `Done. ${unanalyzed.length - errors} analyzed, ${totalProposals} proposals, ${errors} errors.`; + // Count proposals generated + const propCount = (db.prepare("SELECT COUNT(*) as c FROM proposals WHERE status = 'open'").get() as { c: number }).c; + totalProposals = propCount; + + const doneMsg = `Done. ${unanalyzed.length - errors} analyzed, ${totalNodes} nodes, ${totalProposals} open proposals, ${errors} errors.`; ctx.ui.notify(doneMsg, "info"); console.log(doneMsg); } finally { @@ -66,15 +81,17 @@ export function registerAnalyzeCommand(pi: ExtensionAPI): void { }); } -function parseArgs(raw: string): { model?: string; limit?: number } { - const result: { model?: string; limit?: number } = {}; +function parseArgs(raw: string): { model?: string; limit?: number; analyzer?: string } { + const result: { model?: string; limit?: number; analyzer?: string } = {}; const parts = raw.split(/\s+/); for (let i = 0; i < parts.length; i++) { if (parts[i] === "--model" && parts[i + 1]) result.model = parts[++i]; else if (parts[i] === "--limit" && parts[i + 1]) { const n = parseInt(parts[++i]!, 10); if (!isNaN(n)) result.limit = n; + } else if (parts[i] === "--analyzer" && parts[i + 1]) { + result.analyzer = parts[++i]; } } return result; -} \ No newline at end of file +} diff --git a/src/commands/proposals.ts b/src/commands/proposals.ts index 5c7b051..b84789d 100644 --- a/src/commands/proposals.ts +++ b/src/commands/proposals.ts @@ -1,7 +1,7 @@ import type { ExtensionAPI } from "../pi-stubs.js"; import Database from "better-sqlite3"; import { migrate } from "../db/schema.js"; -import { listProposals, acceptProposal, rejectProposal } from "../db/queries.js"; +import { listProposalsEnriched, acceptProposal, rejectProposal } from "../db/queries.js"; import { getDbPath } from "../config.js"; function output(ctx: any, text: string, level: "info" | "warning" | "error" = "info"): void { @@ -9,25 +9,31 @@ function output(ctx: any, text: string, level: "info" | "warning" | "error" = "i console.log(text); } +function renderProposal(p: ReturnType[number]): string { + const short = p.id.slice(0, 8); + const target = p.target_type + ? `${p.target_type}${p.target_path ? `:${p.target_path}` : ""}` + : "(unknown)"; + const title = p.title ? ` — ${p.title}` : ""; + return `[${p.status}] ${short} | ${p.severity} | ${target}${title}\n ${p.summary}`; +} + export function registerProposalsCommand(pi: ExtensionAPI): void { pi.registerCommand("prospect-proposals", { - description: "List proposals (optionally filter by status: new, accepted, rejected)", + description: "List proposals (optionally filter by status: open, accepted, rejected, new)", handler: async (args: string, ctx: any) => { const db = new Database(getDbPath()); migrate(db); try { const status = args?.trim() || undefined; - const proposals = listProposals(db, status); + const proposals = listProposalsEnriched(db, status); if (proposals.length === 0) { output(ctx, "No proposals found."); return; } - const lines = proposals.map((p) => { - const short = p.id.slice(0, 8); - return `[${p.status}] ${short} | ${p.severity} | ${p.target}\n ${p.summary}`; - }); + const lines = proposals.map(renderProposal); output(ctx, `Proposals (${proposals.length}):\n${lines.join("\n")}`); } finally { db.close(); @@ -44,7 +50,7 @@ export function registerProposalsCommand(pi: ExtensionAPI): void { migrate(db); try { const ok = acceptProposal(db, id); - output(ctx, ok ? `Proposal ${id} accepted.` : `Proposal ${id} not found or not in 'new' status.`, ok ? "info" : "warning"); + output(ctx, ok ? `Proposal ${id} accepted.` : `Proposal ${id} not found or not in 'new'/'open' status.`, ok ? "info" : "warning"); } finally { db.close(); } @@ -60,10 +66,10 @@ export function registerProposalsCommand(pi: ExtensionAPI): void { migrate(db); try { const ok = rejectProposal(db, id); - output(ctx, ok ? `Proposal ${id} rejected.` : `Proposal ${id} not found or not in 'new' status.`, ok ? "info" : "warning"); + output(ctx, ok ? `Proposal ${id} rejected.` : `Proposal ${id} not found or not in 'new'/'open' status.`, ok ? "info" : "warning"); } finally { db.close(); } }, }); -} \ No newline at end of file +} diff --git a/src/commands/stats.ts b/src/commands/stats.ts index 05c37b3..4dfe4a2 100644 --- a/src/commands/stats.ts +++ b/src/commands/stats.ts @@ -12,6 +12,21 @@ export function registerStatsCommand(pi: ExtensionAPI): void { migrate(db); try { const s = getStats(db); + + // Analysis framework stats + const analyzerDefs = (db.prepare("SELECT COUNT(*) as c FROM analyzer_defs").get() as { c: number }).c; + const analysisNodes = (db.prepare("SELECT COUNT(*) as c FROM analysis_nodes WHERE node_kind != 'error'").get() as { c: number }).c; + const errorNodes = (db.prepare("SELECT COUNT(*) as c FROM analysis_nodes WHERE node_kind = 'error'").get() as { c: number }).c; + const proposalNodes = (db.prepare("SELECT COUNT(*) as c FROM analysis_nodes WHERE node_kind = 'proposal'").get() as { c: number }).c; + const runs = (db.prepare("SELECT COUNT(*) as c FROM analysis_runs WHERE status = 'ok'").get() as { c: number }).c; + const nodesByAnalyzer = db.prepare(` + SELECT analyzer_id, COUNT(*) as c FROM analysis_nodes + WHERE node_kind != 'error' + GROUP BY analyzer_id + `).all() as Array<{ analyzer_id: string; c: number }>; + + const openProposals = (db.prepare("SELECT COUNT(*) as c FROM proposals WHERE status IN ('new', 'open')").get() as { c: number }).c; + const lines = [ "╔══════════════════════════════════════════╗", "║ ⛏️ Prospector Stats ║", @@ -21,12 +36,27 @@ export function registerStatsCommand(pi: ExtensionAPI): void { ` Messages (user+asst):${s.totalMessages}`, ` Tool results: ${s.totalToolResults}`, ` Sessions analyzed: ${s.messagesProcessed}`, + "", + " Analysis:", + ` Analyzers registered: ${analyzerDefs}`, + ` Analysis nodes: ${analysisNodes}`, + ` Error nodes: ${errorNodes}`, + ` Proposal nodes: ${proposalNodes}`, + ` Successful runs: ${runs}`, + ]; + if (nodesByAnalyzer.length > 0) { + lines.push(" Per-analyzer:"); + for (const r of nodesByAnalyzer) { + lines.push(` ${r.analyzer_id}: ${r.c}`); + } + } + lines.push( "", " Proposals:", - ` new: ${s.proposalsByStatus.new}`, + ` new/open: ${s.proposalsByStatus.new + openProposals}`, ` accepted: ${s.proposalsByStatus.accepted}`, ` rejected: ${s.proposalsByStatus.rejected}`, - ]; + ); const text = lines.join("\n"); ctx.ui.notify(text, "info"); console.log(text); @@ -35,4 +65,4 @@ export function registerStatsCommand(pi: ExtensionAPI): void { } }, }); -} \ No newline at end of file +} diff --git a/src/commands/tool.ts b/src/commands/tool.ts index cad47a5..c71bd1a 100644 --- a/src/commands/tool.ts +++ b/src/commands/tool.ts @@ -3,14 +3,16 @@ import Database from "better-sqlite3"; import { Type } from "typebox"; import { migrate } from "../db/schema.js"; import { runSync } from "../sync/index.js"; -import { getStats, listProposals, acceptProposal, rejectProposal } from "../db/queries.js"; +import { getStats, listProposalsEnriched, acceptProposal, rejectProposal } from "../db/queries.js"; import { getDbPath, getSessionsDir } from "../config.js"; +import { AnalyzerFramework } from "../analyze/framework.js"; +import { registerDefaults, getDefaultLLMCaller } from "../analyze/defaults.js"; export function registerProspectTool(pi: ExtensionAPI): void { pi.registerTool({ name: "prospect", label: "Prospect", - description: "Index sessions, check stats, list/accept/reject proposals. Actions: sync, stats, list_proposals, accept, reject.", + description: "Index sessions, check stats, list/accept/reject proposals, run analyzers. Actions: sync, stats, list_proposals, accept, reject, analyze.", parameters: Type.Object({ action: Type.Union([ Type.Literal("sync"), @@ -18,9 +20,13 @@ export function registerProspectTool(pi: ExtensionAPI): void { Type.Literal("list_proposals"), Type.Literal("accept"), Type.Literal("reject"), + Type.Literal("analyze"), ]), - status: Type.Optional(Type.Union([Type.Literal("new"), Type.Literal("accepted"), Type.Literal("rejected")])), + status: Type.Optional(Type.Union([Type.Literal("new"), Type.Literal("accepted"), Type.Literal("rejected"), Type.Literal("open")])), proposal_id: Type.Optional(Type.String()), + analyzer_id: Type.Optional(Type.String()), + session_id: Type.Optional(Type.String()), + limit: Type.Optional(Type.Integer()), }), async execute(_toolCallId: string, params: Record, _signal: unknown, _onUpdate: unknown, _ctx: unknown) { const db = new Database(getDbPath()); @@ -36,9 +42,15 @@ export function registerProspectTool(pi: ExtensionAPI): void { return { content: [{ type: "text" as const, text: JSON.stringify(stats, null, 2) }], details: stats }; } case "list_proposals": { - const proposals = listProposals(db, params.status as string | undefined); + const proposals = listProposalsEnriched(db, params.status as string | undefined); if (proposals.length === 0) return { content: [{ type: "text" as const, text: "No proposals found." }], details: [] }; - const text = proposals.map((p) => `[${p.status}] ${p.id.slice(0, 8)} | ${p.severity} | ${p.target}\n ${p.summary}`).join("\n\n"); + const text = proposals.map((p) => { + const target = p.target_type + ? `${p.target_type}${p.target_path ? `:${p.target_path}` : ""}` + : "(unknown)"; + const title = p.title ? ` — ${p.title}` : ""; + return `[${p.status}] ${p.id.slice(0, 8)} | ${p.severity} | ${target}${title}\n ${p.summary}`; + }).join("\n\n"); return { content: [{ type: "text" as const, text }], details: proposals }; } case "accept": { @@ -51,10 +63,29 @@ export function registerProspectTool(pi: ExtensionAPI): void { const ok = rejectProposal(db, params.proposal_id as string); return { content: [{ type: "text" as const, text: ok ? `Rejected ${params.proposal_id}` : "Not found or not new" }], details: { ok } }; } + case "analyze": { + const llm = getDefaultLLMCaller(); + const fw = new AnalyzerFramework({ db, llm }); + registerDefaults(fw); + const analyzerId = (params.analyzer_id as string) ?? "turn-pair-core"; + const sessionId = params.session_id as string | undefined; + if (sessionId) { + const r = await fw.run(analyzerId, sessionId); + return { content: [{ type: "text" as const, text: JSON.stringify(r, null, 2) }], details: r }; + } + // Run over all unanalyzed + const sessions = (db.prepare("SELECT id FROM sessions WHERE analyzed_at IS NULL ORDER BY started_at ASC LIMIT ?").all((params.limit as number) ?? 100) as Array<{ id: string }>).map((r) => r.id); + const results: unknown[] = []; + for (const sid of sessions) { + const r = await fw.run(analyzerId, sid); + results.push(r); + } + return { content: [{ type: "text" as const, text: JSON.stringify(results, null, 2) }], details: results }; + } } } finally { db.close(); } }, }); -} \ No newline at end of file +} diff --git a/src/index.ts b/src/index.ts index b551ed0..b9204b1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,11 +4,59 @@ import { registerStatsCommand } from "./commands/stats.js"; import { registerProposalsCommand } from "./commands/proposals.js"; import { registerAnalyzeCommand } from "./commands/analyze.js"; import { registerProspectTool } from "./commands/tool.js"; +import { setDefaultLLMCaller } from "./analyze/defaults.js"; +import type { LLMCaller, LLMRequest } from "./analyze/types.js"; + +/** + * LLM caller that delegates to Pi's model provider. In production + * the host SDK exposes a `pi.ai` namespace; here we use the + * extension's context to call models. In tests this default + * stub is replaced via `setDefaultLLMCaller`. + */ +function makePiLLMCaller(pi: ExtensionAPI): LLMCaller { + return async (request: LLMRequest): Promise<{ + text: string; + model: string; + costUsd: number; + tokensUsed: number; + durationMs: number; + }> => { + const start = Date.now(); + try { + // Delegate to Pi's ai.complete API. The exact shape depends + // on the host SDK; we use a minimal interface that Pi + // should be able to satisfy. Falls back to a stub if the + // host doesn't expose this. + const result = await (pi as any).ai?.complete?.({ + model: request.model, + system: request.system, + prompt: request.user, + ...request.jsonSchema ? { schema: request.jsonSchema } : {}, + }); + return { + text: result?.text ?? "", + model: request.model, + costUsd: result?.cost ?? 0, + tokensUsed: result?.tokens ?? 0, + durationMs: Date.now() - start, + }; + } catch (err) { + return { + text: "", + model: request.model, + costUsd: 0, + tokensUsed: 0, + durationMs: Date.now() - start, + }; + } + }; +} export default function (pi: ExtensionAPI) { + setDefaultLLMCaller(makePiLLMCaller(pi)); registerSyncCommand(pi); registerStatsCommand(pi); registerProposalsCommand(pi); registerAnalyzeCommand(pi); registerProspectTool(pi); -} \ No newline at end of file +} From a88ef77f9c2d0e2628529e31b34b2f5586cea476 Mon Sep 17 00:00:00 2001 From: Nicolas Marchildon Date: Tue, 2 Jun 2026 11:12:04 -0400 Subject: [PATCH 5/5] tests: unit + component coverage for framework, hashing, analyzers Unit tests: - framework-hash: shortHash, fullHash, source_set_hash, prompt bundle hash, input hash, edge-kind validation - turn-pair-patterns: detectCorrection, detectAllCorrectionPatterns, detectRepetition, extractCorrectionText, computeFrictionScore - turn-pair-builder: buildTurnPairNode length/thinking/correction/ tool calls/failures/retry/model/elapsed/waste/compaction - turn-pair-llm: buildTurnPairLlmPrompt, parseTurnPairLlmResponse - session-overview: buildDigest, splitDigest, parseMapResponse, parseReduceResponse, buildMapPrompt, buildReducePrompt Component tests: - framework: registration, idempotent re-run, source-set changes, LLM cost capture, error nodes, proposal materialization, dedup, dependency visibility, crash recovery (stale running runs) - turn-pair-llm: end-to-end enrichment of high-signal pairs - session-overview: end-to-end with materialized proposals - e2e: real fixture sync + full framework flow, verifies meta_json is captured on assistant messages --- tests/component/e2e.test.ts | 106 +++++ tests/component/framework.test.ts | 524 +++++++++++++++++++++++ tests/component/session-overview.test.ts | 141 ++++++ tests/component/turn-pair-llm.test.ts | 114 +++++ tests/unit/framework-hash.test.ts | 184 ++++++++ tests/unit/session-overview.test.ts | 188 ++++++++ tests/unit/turn-pair-builder.test.ts | 266 ++++++++++++ tests/unit/turn-pair-llm.test.ts | 84 ++++ tests/unit/turn-pair-patterns.test.ts | 156 +++++++ 9 files changed, 1763 insertions(+) create mode 100644 tests/component/e2e.test.ts create mode 100644 tests/component/framework.test.ts create mode 100644 tests/component/session-overview.test.ts create mode 100644 tests/component/turn-pair-llm.test.ts create mode 100644 tests/unit/framework-hash.test.ts create mode 100644 tests/unit/session-overview.test.ts create mode 100644 tests/unit/turn-pair-builder.test.ts create mode 100644 tests/unit/turn-pair-llm.test.ts create mode 100644 tests/unit/turn-pair-patterns.test.ts diff --git a/tests/component/e2e.test.ts b/tests/component/e2e.test.ts new file mode 100644 index 0000000..c5b5bb5 --- /dev/null +++ b/tests/component/e2e.test.ts @@ -0,0 +1,106 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import * as os from "node:os"; +import Database from "better-sqlite3"; +import { migrate } from "../../src/db/schema.js"; +import { runSync } from "../../src/sync/index.js"; +import { AnalyzerFramework } from "../../src/analyze/framework.js"; +import { turnPairCoreAnalyzer, turnPairLlmAnalyzer, sessionOverviewAnalyzer } from "../../src/analyze/analyzers/index.js"; +import type { LLMCaller, LLMRequest } from "../../src/analyze/types.js"; + +const FIXTURES = path.resolve(import.meta.dirname, "..", "fixtures"); + +function tempDb(): { db: Database.Database; close: () => void } { + const dbPath = path.join(os.tmpdir(), `prospect-e2e-${Date.now()}-${Math.random().toString(36).slice(2)}.db`); + const db = new Database(dbPath); + migrate(db); + return { db, close: () => { db.close(); try { fs.unlinkSync(dbPath); } catch {} } }; +} + +describe("end-to-end sync + framework", () => { + it("syncs fixtures and produces analysis nodes + proposals", async () => { + const { db, close } = tempDb(); + try { + const sync = runSync(db, FIXTURES); + assert.ok(sync.sessionsProcessed >= 2); + + // Verify meta_json was captured for assistant messages + const assistants = db.prepare( + "SELECT meta_json FROM messages WHERE role = 'assistant' AND meta_json IS NOT NULL LIMIT 1", + ).get() as { meta_json: string } | undefined; + assert.ok(assistants, "at least one assistant message should have meta_json"); + const meta = JSON.parse(assistants!.meta_json); + assert.ok(meta.model, "meta should include model"); + assert.ok(meta.usage, "meta should include usage"); + assert.equal(meta.stop_reason, "toolUse"); + + // Wire framework + const llmCalls: LLMRequest[] = []; + const llm: LLMCaller = async (req) => { + llmCalls.push(req); + if (req.model === "cheap") { + return { + text: JSON.stringify({ + sentiment: "neutral", + frustration_level: 0, + correction_type_llm: null, + friction_cause: null, + friction_summary: null, + user_intent: "ask a question", + quality_score: 4, + }), + model: "stub/cheap", + costUsd: 0, + tokensUsed: 0, + durationMs: 0, + }; + } + return { + text: JSON.stringify({ + session_summary: "User asked how to run tests; agent provided the answer.", + key_friction_points: [], + improvement_proposals: [], + sentiment_arc: [], + }), + model: "stub/mid", + costUsd: 0, + tokensUsed: 0, + durationMs: 0, + }; + }; + + const fw = new AnalyzerFramework({ db, llm }); + fw.register(turnPairCoreAnalyzer); + fw.register(turnPairLlmAnalyzer); + fw.register(sessionOverviewAnalyzer); + + // Run all three over all sessions + const sessions = db.prepare("SELECT id FROM sessions").all() as Array<{ id: string }>; + for (const s of sessions) { + await fw.run("turn-pair-core", s.id); + await fw.run("turn-pair-llm", s.id); + await fw.run("session-overview", s.id); + } + + // Verify there are turn-pair-core metric nodes + const metrics = db.prepare("SELECT COUNT(*) as c FROM analysis_nodes WHERE node_kind = 'metric'").get() as { c: number }; + assert.ok(metrics.c > 0, "should have turn-pair-core metric nodes"); + + // Verify there is at least one session-overview summary + const summaries = db.prepare("SELECT COUNT(*) as c FROM analysis_nodes WHERE node_kind = 'summary'").get() as { c: number }; + assert.equal(summaries.c, sessions.length); + + // Verify edges + const edgeCounts = db.prepare(` + SELECT edge_kind, COUNT(*) as c FROM analysis_edges GROUP BY edge_kind + `).all() as Array<{ edge_kind: string; c: number }>; + const kinds = new Set(edgeCounts.map((e) => e.edge_kind)); + assert.ok(kinds.has("anchors")); + assert.ok(kinds.has("consumes")); + } finally { + close(); + } + }); +}); diff --git a/tests/component/framework.test.ts b/tests/component/framework.test.ts new file mode 100644 index 0000000..75c31fc --- /dev/null +++ b/tests/component/framework.test.ts @@ -0,0 +1,524 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import * as os from "node:os"; +import Database from "better-sqlite3"; +import { migrate } from "../../src/db/schema.js"; +import { AnalyzerFramework } from "../../src/analyze/framework.js"; +import type { + Analyzer, + AnalyzerDef, + AnalysisResult, + AnalysisUnit, + AnalyzerConfig, + AnalyzerPlanContext, + AnalyzerRunContext, + LLMRequest, + LLMResponse, + LLMCaller, + AnalyzerVersion, + PromptVersion, + AnalysisNodeRow, +} from "../../src/analyze/types.js"; +import { REF_KINDS, EDGE_KINDS } from "../../src/analyze/types.js"; + +function tempDb(): { db: Database.Database; close: () => void } { + const dbPath = path.join(os.tmpdir(), `prospect-fw-${Date.now()}-${Math.random().toString(36).slice(2)}.db`); + const db = new Database(dbPath); + migrate(db); + return { db, close: () => { db.close(); try { fs.unlinkSync(dbPath); } catch {} } }; +} + +function seedSession(db: Database.Database, sessionId: string, messages: Array<{ id: string; role: string; text: string }>) { + db.prepare(`INSERT INTO sessions (id, file_path, project, cwd, parent_session, started_at, last_line, last_modified, message_count) VALUES (?, ?, '', '', NULL, ?, 0, 0, 0)`).run(sessionId, `/fake/${sessionId}.jsonl`, "2026-01-01T00:00:00Z"); + for (const m of messages) { + db.prepare(` + INSERT INTO messages (id, session_id, parent_id, timestamp, role, content_text, content_thinking, tool_calls, tool_results, meta_json) + VALUES (?, ?, NULL, ?, ?, ?, NULL, NULL, NULL, NULL) + `).run(m.id, sessionId, "2026-01-01T00:00:00Z", m.role, m.text); + } + db.prepare(`UPDATE sessions SET message_count = ? WHERE id = ?`).run(messages.length, sessionId); +} + +const STUB_LLM: LLMCaller = async (_req: LLMRequest): Promise => { + return { text: "{}", model: "stub/model", costUsd: 0, tokensUsed: 0, durationMs: 0 }; +}; + +function makeSimpleAnalyzer(overrides: Partial<{ + id: string; + dependencies: string[]; + planFn: (ctx: AnalyzerPlanContext) => Promise; + analyzeFn: (unit: AnalysisUnit, ctx: AnalyzerRunContext) => Promise; +}>): Analyzer { + const def: AnalyzerDef = { + id: overrides.id ?? "test-analyzer", + label: "Test", + description: "test", + anchorSpan: "pair", + dependencies: overrides.dependencies ?? [], + }; + const version: AnalyzerVersion = { + analyzerId: def.id, + versionId: "0.0.1", + implementationKind: "deterministic", + }; + const config: AnalyzerConfig = { + id: "", + analyzerId: def.id, + configJson: {}, + configHash: "", + label: "default", + }; + return { + def, + version, + prompts: {} as Record, + defaultConfig: config, + async plan(ctx) { + return overrides.planFn ? overrides.planFn(ctx) : [{ + sources: ctx.messages.map((m) => ({ kind: "message", id: m.id })), + sourceSetHash: "fake-hash", + anchorKind: "session", + anchorRef: ctx.sessionId, + }]; + }, + async analyze(unit, ctx) { + return overrides.analyzeFn + ? overrides.analyzeFn(unit, ctx) + : { + contentJson: { hello: "world" }, + nodeKind: "metric", + anchorKind: "session", + anchorRef: ctx.run.session_id, + edges: [{ + toRefKind: REF_KINDS.SESSION, + toRefId: ctx.run.session_id, + edgeKind: EDGE_KINDS.ANCHORS, + }], + }; + }, + }; +} + +describe("AnalyzerFramework — registration and lookup", () => { + it("registers an analyzer and persists def/version/prompts", () => { + const { db, close } = tempDb(); + try { + const fw = new AnalyzerFramework({ db, llm: STUB_LLM }); + fw.register(makeSimpleAnalyzer({ id: "a1" })); + assert.equal(fw.list().length, 1); + + const def = db.prepare("SELECT * FROM analyzer_defs WHERE id = ?").get("a1") as any; + assert.ok(def); + assert.equal(def.label, "Test"); + + const v = db.prepare("SELECT * FROM analyzer_versions WHERE analyzer_id = ?").get("a1") as any; + assert.ok(v); + assert.equal(v.version_id, "0.0.1"); + } finally { + close(); + } + }); + + it("is idempotent on re-registration", () => { + const { db, close } = tempDb(); + try { + const fw = new AnalyzerFramework({ db, llm: STUB_LLM }); + fw.register(makeSimpleAnalyzer({ id: "a1" })); + fw.register(makeSimpleAnalyzer({ id: "a1" })); + assert.equal(fw.list().length, 1); + } finally { + close(); + } + }); +}); + +describe("AnalyzerFramework — run()", () => { + it("produces a node and a session-anchored edge", async () => { + const { db, close } = tempDb(); + try { + seedSession(db, "s1", [ + { id: "m1", role: "user", text: "hi" }, + { id: "m2", role: "assistant", text: "ok" }, + ]); + + const fw = new AnalyzerFramework({ db, llm: STUB_LLM }); + fw.register(makeSimpleAnalyzer({ id: "a1" })); + const summary = await fw.run("a1", "s1"); + + assert.equal(summary.status, "ok"); + assert.equal(summary.nodesProduced, 1); + assert.equal(summary.nodesSkipped, 0); + + const node = db.prepare("SELECT * FROM analysis_nodes WHERE analyzer_id = ?").get("a1") as any; + assert.ok(node); + assert.equal(node.session_id, "s1"); + assert.equal(node.node_kind, "metric"); + + const edges = db.prepare("SELECT * FROM analysis_edges WHERE from_node_id = ?").all(node.id) as any[]; + assert.ok(edges.length >= 1); + const anchor = edges.find((e: any) => e.to_ref_kind === "session" && e.to_ref_id === "s1"); + assert.ok(anchor); + } finally { + close(); + } + }); + + it("is idempotent on re-run (no new nodes)", async () => { + const { db, close } = tempDb(); + try { + seedSession(db, "s1", [ + { id: "m1", role: "user", text: "hi" }, + { id: "m2", role: "assistant", text: "ok" }, + ]); + const fw = new AnalyzerFramework({ db, llm: STUB_LLM }); + fw.register(makeSimpleAnalyzer({ id: "a1" })); + + const r1 = await fw.run("a1", "s1"); + assert.equal(r1.nodesProduced, 1); + assert.equal(r1.nodesSkipped, 0); + + const r2 = await fw.run("a1", "s1"); + assert.equal(r2.nodesProduced, 0); + assert.equal(r2.nodesSkipped, 1); + + const count = (db.prepare("SELECT COUNT(*) as c FROM analysis_nodes").get() as { c: number }).c; + assert.equal(count, 1); + } finally { + close(); + } + }); + + it("creates a fresh node when source set changes", async () => { + const { db, close } = tempDb(); + try { + seedSession(db, "s1", [ + { id: "m1", role: "user", text: "hi" }, + { id: "m2", role: "assistant", text: "ok" }, + ]); + const fw = new AnalyzerFramework({ db, llm: STUB_LLM }); + // Custom plan that hashes by message count — different sources → different hash + fw.register(makeSimpleAnalyzer({ + id: "a1", + planFn: async (ctx) => [{ + sources: ctx.messages.map((m) => ({ kind: "message", id: m.id })), + sourceSetHash: `hash-${ctx.messages.length}`, + anchorKind: "session", + anchorRef: ctx.sessionId, + }], + })); + + const r1 = await fw.run("a1", "s1"); + assert.equal(r1.nodesProduced, 1); + + // Add a message and re-run + db.prepare(`INSERT INTO messages (id, session_id, parent_id, timestamp, role, content_text) VALUES ('m3', 's1', NULL, '2026-01-01T00:00:05Z', 'user', 'again')`).run(); + const r2 = await fw.run("a1", "s1"); + assert.equal(r2.nodesProduced, 1); + assert.equal(r2.nodesSkipped, 0); + } finally { + close(); + } + }); + + it("captures LLM cost and tokens on the run row", async () => { + const { db, close } = tempDb(); + try { + seedSession(db, "s1", [ + { id: "m1", role: "user", text: "hi" }, + { id: "m2", role: "assistant", text: "ok" }, + ]); + const llm: LLMCaller = async () => ({ text: "x", model: "x/y", costUsd: 0.01, tokensUsed: 100, durationMs: 5 }); + const fw = new AnalyzerFramework({ db, llm }); + fw.register(makeSimpleAnalyzer({ + id: "a1", + analyzeFn: async (_u, ctx) => ({ + contentJson: {}, + nodeKind: "metric", + anchorKind: "session", + anchorRef: ctx.run.session_id, + edges: [], + modelUsed: "x/y", + costUsd: 0.01, + tokensUsed: 100, + }), + })); + await fw.run("a1", "s1"); + + const run = db.prepare("SELECT * FROM analysis_runs WHERE analyzer_id = ?").get("a1") as any; + assert.ok(run); + assert.equal(run.cost_usd, 0.01); + assert.equal(run.tokens_used, 100); + } finally { + close(); + } + }); + + it("inserts an error node if analyze() throws, but continues", async () => { + const { db, close } = tempDb(); + try { + seedSession(db, "s1", [ + { id: "m1", role: "user", text: "hi" }, + { id: "m2", role: "assistant", text: "ok" }, + ]); + const fw = new AnalyzerFramework({ db, llm: STUB_LLM }); + fw.register(makeSimpleAnalyzer({ + id: "a1", + planFn: async (ctx) => [ + { + sources: [{ kind: "message", id: ctx.messages[0]!.id }], + sourceSetHash: "h1", + anchorKind: "session", + anchorRef: ctx.sessionId, + }, + { + sources: [{ kind: "message", id: ctx.messages[1]!.id }], + sourceSetHash: "h2", + anchorKind: "session", + anchorRef: ctx.sessionId, + }, + ], + analyzeFn: async (unit) => { + if (unit.sourceSetHash === "h1") throw new Error("boom"); + return { + contentJson: { ok: true }, + nodeKind: "metric", + anchorKind: "session", + anchorRef: unit.anchorRef ?? "s1", + edges: [], + }; + }, + })); + + const r = await fw.run("a1", "s1"); + assert.equal(r.nodesProduced, 1); + + const errorNode = db.prepare("SELECT * FROM analysis_nodes WHERE node_kind = 'error'").get() as any; + assert.ok(errorNode); + assert.match(JSON.parse(errorNode.content_json).error, /boom/); + + const okNode = db.prepare("SELECT * FROM analysis_nodes WHERE node_kind = 'metric'").get() as any; + assert.ok(okNode); + } finally { + close(); + } + }); +}); + +describe("AnalyzerFramework — proposal materialization", () => { + it("materializes proposals from a summary node's content_json.improvement_proposals", async () => { + const { db, close } = tempDb(); + try { + seedSession(db, "s1", [ + { id: "m1", role: "user", text: "hi" }, + { id: "m2", role: "assistant", text: "ok" }, + ]); + const fw = new AnalyzerFramework({ db, llm: STUB_LLM }); + fw.register(makeSimpleAnalyzer({ + id: "session-overview", + analyzeFn: async (_u, ctx) => ({ + nodeKind: "summary", + anchorKind: "session", + anchorRef: ctx.run.session_id, + edges: [], + contentJson: { + improvement_proposals: [{ + target_type: "skill", + target_path: "skill/foo", + title: "Add a foo skill", + summary: "Tests show confusion about foo", + detail: "Add a skill", + evidence: "User repeatedly asked", + confidence: 0.7, + severity: "suggestion", + }], + }, + }), + })); + + await fw.run("session-overview", "s1"); + + const proposals = db.prepare("SELECT * FROM proposals WHERE session_id = ?").all("s1") as any[]; + assert.equal(proposals.length, 1); + assert.equal(proposals[0].target_type, "skill"); + assert.equal(proposals[0].title, "Add a foo skill"); + assert.equal(proposals[0].status, "open"); + assert.equal(proposals[0].analyzer_id, "session-overview"); + + const proposalNode = db.prepare("SELECT * FROM analysis_nodes WHERE node_kind = 'proposal'").get() as any; + assert.ok(proposalNode); + + const producesEdges = db.prepare("SELECT * FROM analysis_edges WHERE edge_kind = 'produces'").all() as any[]; + assert.ok(producesEdges.length >= 1); + assert.equal(producesEdges[0].to_ref_id, proposalNode.id); + } finally { + close(); + } + }); + + it("dedups proposals on (target_type, target_path, severity, normalized title)", async () => { + const { db, close } = tempDb(); + try { + seedSession(db, "s1", [ + { id: "m1", role: "user", text: "hi" }, + { id: "m2", role: "assistant", text: "ok" }, + ]); + seedSession(db, "s2", [ + { id: "n1", role: "user", text: "hi" }, + { id: "n2", role: "assistant", text: "ok" }, + ]); + const fw = new AnalyzerFramework({ db, llm: STUB_LLM }); + fw.register(makeSimpleAnalyzer({ + id: "session-overview", + analyzeFn: async (_u, ctx) => ({ + nodeKind: "summary", + anchorKind: "session", + anchorRef: ctx.run.session_id, + edges: [], + contentJson: { + improvement_proposals: [{ + target_type: "skill", + target_path: "skill/foo", + title: "Add a Foo Skill.", // trailing punctuation + summary: "s", + detail: "", + evidence: "", + confidence: 0.5, + severity: "suggestion", + }], + }, + }), + })); + + await fw.run("session-overview", "s1"); + await fw.run("session-overview", "s2"); + + const proposals = db.prepare("SELECT * FROM proposals").all() as any[]; + assert.equal(proposals.length, 1, "should dedup to a single open proposal"); + } finally { + close(); + } + }); +}); + +describe("AnalyzerFramework — dependency visibility", () => { + it("inserts an error node when a child reads a non-declared dependency", async () => { + const { db, close } = tempDb(); + try { + seedSession(db, "s1", [ + { id: "m1", role: "user", text: "hi" }, + { id: "m2", role: "assistant", text: "ok" }, + ]); + const fw = new AnalyzerFramework({ db, llm: STUB_LLM }); + fw.register(makeSimpleAnalyzer({ id: "child", dependencies: [] })); + fw.register(makeSimpleAnalyzer({ + id: "parent", + dependencies: ["child"], + analyzeFn: async (_u, ctx) => { + // Try to read a non-declared dependency + ctx.getDependencyNodes("not-declared"); + return { + nodeKind: "metric", + anchorKind: "session", + anchorRef: ctx.run.session_id, + edges: [], + contentJson: {}, + }; + }, + })); + + const r = await fw.run("parent", "s1"); + assert.equal(r.nodesProduced, 0); + assert.equal(r.status, "ok"); + const errorNode = db.prepare("SELECT * FROM analysis_nodes WHERE node_kind = 'error'").get() as any; + assert.ok(errorNode); + const content = JSON.parse(errorNode.content_json); + assert.match(content.error, /did not declare/); + } finally { + close(); + } + }); + + it("exposes own nodes and declared dependency nodes in plan context", async () => { + const { db, close } = tempDb(); + try { + seedSession(db, "s1", [ + { id: "m1", role: "user", text: "hi" }, + { id: "m2", role: "assistant", text: "ok" }, + ]); + const fw = new AnalyzerFramework({ db, llm: STUB_LLM }); + + // First analyzer: produces nodes for the session + fw.register(makeSimpleAnalyzer({ + id: "producer", + analyzeFn: async (_u, ctx) => ({ + nodeKind: "metric", + anchorKind: "session", + anchorRef: ctx.run.session_id, + edges: [], + contentJson: { from: "producer" }, + }), + })); + await fw.run("producer", "s1"); + + // Second analyzer: declares dependency on producer + let observedDeps: Record = {}; + fw.register(makeSimpleAnalyzer({ + id: "consumer", + dependencies: ["producer"], + planFn: async (ctx) => { + observedDeps = ctx.dependencyNodes; + return [{ + sources: ctx.messages.map((m) => ({ kind: "message", id: m.id })), + sourceSetHash: "h", + anchorKind: "session", + anchorRef: ctx.sessionId, + }]; + }, + analyzeFn: async (_u, ctx) => ({ + nodeKind: "summary", + anchorKind: "session", + anchorRef: ctx.run.session_id, + edges: [], + contentJson: {}, + }), + })); + await fw.run("consumer", "s1"); + + assert.ok(observedDeps["producer"]); + assert.equal(observedDeps["producer"]!.length, 1); + } finally { + close(); + } + }); +}); + +describe("AnalyzerFramework — crash recovery", () => { + it("marks stale 'running' runs as 'error'", async () => { + const { db, close } = tempDb(); + try { + seedSession(db, "s1", [ + { id: "m1", role: "user", text: "hi" }, + { id: "m2", role: "assistant", text: "ok" }, + ]); + const fw = new AnalyzerFramework({ db, llm: STUB_LLM }); + fw.register(makeSimpleAnalyzer({ id: "a1" })); + + // Inject a stale 'running' row + db.prepare(` + INSERT INTO analysis_runs (id, analyzer_id, analyzer_version_id, config_id, session_id, status, prompt_bundle_hash, started_at) + VALUES (?, 'a1', '0.0.1', 'c1', 's1', 'running', 'ph', '2026-01-01T00:00:00Z') + `).run("stale-1"); + + const n = fw.recoverStaleRuns(); + assert.equal(n, 1); + + const row = db.prepare("SELECT status, error_message FROM analysis_runs WHERE id = ?").get("stale-1") as any; + assert.equal(row.status, "error"); + assert.match(row.error_message, /stale/); + } finally { + close(); + } + }); +}); diff --git a/tests/component/session-overview.test.ts b/tests/component/session-overview.test.ts new file mode 100644 index 0000000..f40a467 --- /dev/null +++ b/tests/component/session-overview.test.ts @@ -0,0 +1,141 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import * as os from "node:os"; +import Database from "better-sqlite3"; +import { migrate } from "../../src/db/schema.js"; +import { AnalyzerFramework } from "../../src/analyze/framework.js"; +import { turnPairCoreAnalyzer } from "../../src/analyze/analyzers/turn-pair-core/index.js"; +import { turnPairLlmAnalyzer } from "../../src/analyze/analyzers/turn-pair-llm/index.js"; +import { sessionOverviewAnalyzer } from "../../src/analyze/analyzers/session-overview/index.js"; +import type { LLMCaller } from "../../src/analyze/types.js"; + +function tempDb(): { db: Database.Database; close: () => void } { + const dbPath = path.join(os.tmpdir(), `prospect-so-${Date.now()}-${Math.random().toString(36).slice(2)}.db`); + const db = new Database(dbPath); + migrate(db); + return { db, close: () => { db.close(); try { fs.unlinkSync(dbPath); } catch {} } }; +} + +function seedSession(db: Database.Database, sessionId: string, messages: Array<{ id: string; role: string; text: string }>) { + db.prepare(`INSERT INTO sessions (id, file_path, project, cwd, parent_session, started_at, last_line, last_modified, message_count) VALUES (?, ?, '', '', NULL, ?, 0, 0, 0)`).run(sessionId, `/fake/${sessionId}.jsonl`, "2026-01-01T00:00:00Z"); + for (const m of messages) { + db.prepare(` + INSERT INTO messages (id, session_id, parent_id, timestamp, role, content_text, content_thinking, tool_calls, tool_results, meta_json) + VALUES (?, ?, NULL, '2026-01-01T00:00:00Z', ?, ?, NULL, NULL, NULL, NULL) + `).run(m.id, sessionId, m.role, m.text); + } + db.prepare(`UPDATE sessions SET message_count = ? WHERE id = ?`).run(messages.length, sessionId); +} + +describe("session-overview end-to-end", () => { + it("produces a session-anchored summary node and materializes proposals", async () => { + const { db, close } = tempDb(); + try { + seedSession(db, "s1", [ + { id: "u1", role: "user", text: "actually, use pnpm not npm" }, + { id: "a1", role: "assistant", text: "switching" }, + ]); + + const llm: LLMCaller = async (req) => { + if (req.model === "cheap") { + // turn-pair-llm classify + return { + text: JSON.stringify({ + sentiment: "frustrated", + frustration_level: 6, + correction_type_llm: "explicit", + friction_cause: "wrong_pkg_mgr", + friction_summary: "User corrected pnpm", + user_intent: "fix package manager", + quality_score: 3, + }), + model: "stub/cheap", + costUsd: 0.001, + tokensUsed: 50, + durationMs: 5, + }; + } + // mid (reduce) + return { + text: JSON.stringify({ + session_summary: "User wanted pnpm; agent initially used npm and was corrected.", + key_friction_points: [ + { description: "wrong package manager", pair_node_id: "unknown", severity: "high" }, + ], + improvement_proposals: [{ + target_type: "agents_md", + target_path: "~/.pi/agent/AGENTS.md", + title: "Default to project's package manager", + summary: "Read package.json before running install", + detail: "When the user says 'install', read package.json first and use the package manager declared there.", + evidence: "User: actually, use pnpm not npm", + confidence: 0.85, + severity: "correction", + }], + sentiment_arc: [{ segment: 0, sentiment: "frustrated", key_event: "package manager correction" }], + }), + model: "stub/mid", + costUsd: 0.01, + tokensUsed: 200, + durationMs: 20, + }; + }; + + const fw = new AnalyzerFramework({ db, llm }); + fw.register(turnPairCoreAnalyzer); + fw.register(turnPairLlmAnalyzer); + fw.register(sessionOverviewAnalyzer); + + await fw.run("turn-pair-core", "s1"); + await fw.run("turn-pair-llm", "s1"); + const r = await fw.run("session-overview", "s1"); + assert.equal(r.status, "ok"); + assert.equal(r.nodesProduced, 1); + + // Verify summary node + const summary = db.prepare(`SELECT * FROM analysis_nodes WHERE analyzer_id = 'session-overview' AND node_kind = 'summary'`).get() as any; + assert.ok(summary); + assert.equal(summary.node_kind, "summary"); + const content = JSON.parse(summary.content_json); + assert.ok(content.session_summary.length > 0); + assert.equal(content.improvement_proposals.length, 1); + + // Verify proposal materialized + const proposals = db.prepare(`SELECT * FROM proposals WHERE session_id = ?`).all("s1") as any[]; + assert.equal(proposals.length, 1); + assert.equal(proposals[0].target_type, "agents_md"); + assert.equal(proposals[0].title, "Default to project's package manager"); + + // Verify edges + const edges = db.prepare(`SELECT * FROM analysis_edges WHERE from_node_id = ?`).all(summary.id) as any[]; + const edgeKinds = new Set(edges.map((e: any) => e.edge_kind)); + assert.ok(edgeKinds.has("anchors"), "has anchors"); + assert.ok(edgeKinds.has("consumes"), "has consumes"); + assert.ok(edgeKinds.has("uses_prompt"), "has uses_prompt"); + } finally { + close(); + } + }); + + it("skips the session if no turn-pair-core nodes exist", async () => { + const { db, close } = tempDb(); + try { + seedSession(db, "s1", [ + { id: "u1", role: "user", text: "hi" }, + { id: "a1", role: "assistant", text: "hello" }, + ]); + const llm: LLMCaller = async () => ({ text: "{}", model: "x", costUsd: 0, tokensUsed: 0, durationMs: 0 }); + const fw = new AnalyzerFramework({ db, llm }); + fw.register(sessionOverviewAnalyzer); + const r = await fw.run("session-overview", "s1"); + assert.equal(r.nodesProduced, 0); + + const summary = db.prepare(`SELECT * FROM analysis_nodes WHERE analyzer_id = 'session-overview'`).get(); + assert.equal(summary, undefined); + } finally { + close(); + } + }); +}); diff --git a/tests/component/turn-pair-llm.test.ts b/tests/component/turn-pair-llm.test.ts new file mode 100644 index 0000000..ae4a57e --- /dev/null +++ b/tests/component/turn-pair-llm.test.ts @@ -0,0 +1,114 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import * as os from "node:os"; +import Database from "better-sqlite3"; +import { migrate } from "../../src/db/schema.js"; +import { AnalyzerFramework } from "../../src/analyze/framework.js"; +import { turnPairCoreAnalyzer } from "../../src/analyze/analyzers/turn-pair-core/index.js"; +import { turnPairLlmAnalyzer } from "../../src/analyze/analyzers/turn-pair-llm/index.js"; +import type { LLMCaller, LLMRequest } from "../../src/analyze/types.js"; + +function tempDb(): { db: Database.Database; close: () => void } { + const dbPath = path.join(os.tmpdir(), `prospect-tpllm-${Date.now()}-${Math.random().toString(36).slice(2)}.db`); + const db = new Database(dbPath); + migrate(db); + return { db, close: () => { db.close(); try { fs.unlinkSync(dbPath); } catch {} } }; +} + +function seedSession(db: Database.Database, sessionId: string, messages: Array<{ id: string; role: string; text: string; ts?: string }>) { + db.prepare(`INSERT INTO sessions (id, file_path, project, cwd, parent_session, started_at, last_line, last_modified, message_count) VALUES (?, ?, '', '', NULL, ?, 0, 0, 0)`).run(sessionId, `/fake/${sessionId}.jsonl`, "2026-01-01T00:00:00Z"); + for (const m of messages) { + db.prepare(` + INSERT INTO messages (id, session_id, parent_id, timestamp, role, content_text, content_thinking, tool_calls, tool_results, meta_json) + VALUES (?, ?, NULL, ?, ?, ?, NULL, NULL, NULL, NULL) + `).run(m.id, sessionId, m.ts ?? "2026-01-01T00:00:00Z", m.role, m.text); + } + db.prepare(`UPDATE sessions SET message_count = ? WHERE id = ?`).run(messages.length, sessionId); +} + +describe("turn-pair-llm end-to-end", () => { + it("only enriches pairs flagged by the deterministic pass", async () => { + const { db, close } = tempDb(); + try { + // Two pairs: one with correction, one clean + seedSession(db, "s1", [ + { id: "u1", role: "user", text: "actually, use pnpm not npm" }, + { id: "a1", role: "assistant", text: "switching to pnpm" }, + { id: "u2", role: "user", text: "what does the package.json say?" }, + { id: "a2", role: "assistant", text: "it has scripts" }, + ]); + + const llmCalls: LLMRequest[] = []; + const llm: LLMCaller = async (req) => { + llmCalls.push(req); + return { + text: JSON.stringify({ + sentiment: "frustrated", + frustration_level: 6, + correction_type_llm: "explicit", + friction_cause: "wrong_package_manager", + friction_summary: "User corrected pnpm vs npm", + user_intent: "fix package manager", + quality_score: 3, + }), + model: "stub/cheap", + costUsd: 0.001, + tokensUsed: 100, + durationMs: 5, + }; + }; + + const fw = new AnalyzerFramework({ db, llm }); + fw.register(turnPairCoreAnalyzer); + fw.register(turnPairLlmAnalyzer); + + const r1 = await fw.run("turn-pair-core", "s1"); + assert.equal(r1.nodesProduced, 2, "two pairs"); + + const r2 = await fw.run("turn-pair-llm", "s1"); + assert.equal(r2.nodesProduced, 1, "only the corrected pair is enriched"); + assert.equal(llmCalls.length, 1); + + const llmNodes = db.prepare(`SELECT * FROM analysis_nodes WHERE analyzer_id = 'turn-pair-llm'`).all() as any[]; + assert.equal(llmNodes.length, 1); + assert.equal(llmNodes[0].node_kind, "classification"); + + const content = JSON.parse(llmNodes[0].content_json); + assert.equal(content.sentiment, "frustrated"); + assert.equal(content.frustration_level, 6); + + // Verify edges + const edges = db.prepare(`SELECT * FROM analysis_edges WHERE from_node_id = ?`).all(llmNodes[0].id) as any[]; + const edgeKinds = edges.map((e: any) => `${e.to_ref_kind}:${e.edge_kind}`).sort(); + assert.ok(edgeKinds.some((k) => k === "analysis_node:refines"), "has refines edge"); + assert.ok(edgeKinds.some((k) => k === "analysis_node:consumes"), "has consumes edge"); + assert.ok(edgeKinds.some((k) => k === "prompt_version:uses_prompt"), "has uses_prompt edge"); + } finally { + close(); + } + }); + + it("records LLM cost and tokens on the run row", async () => { + const { db, close } = tempDb(); + try { + seedSession(db, "s1", [ + { id: "u1", role: "user", text: "no, use pnpm" }, + { id: "a1", role: "assistant", text: "ok" }, + ]); + const llm: LLMCaller = async () => ({ text: JSON.stringify({ sentiment: "negative" }), model: "x/y", costUsd: 0.005, tokensUsed: 50, durationMs: 10 }); + const fw = new AnalyzerFramework({ db, llm }); + fw.register(turnPairCoreAnalyzer); + fw.register(turnPairLlmAnalyzer); + await fw.run("turn-pair-core", "s1"); + await fw.run("turn-pair-llm", "s1"); + + const run = db.prepare(`SELECT * FROM analysis_runs WHERE analyzer_id = 'turn-pair-llm'`).get() as any; + assert.ok(run.cost_usd >= 0.005); + assert.ok(run.tokens_used >= 50); + } finally { + close(); + } + }); +}); diff --git a/tests/unit/framework-hash.test.ts b/tests/unit/framework-hash.test.ts new file mode 100644 index 0000000..c0e0573 --- /dev/null +++ b/tests/unit/framework-hash.test.ts @@ -0,0 +1,184 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + computeInputHash, + computePromptBundleHash, + computeSourceSetHash, + shortHash, + fullHash, + canonicalJsonStringify, + computeConfigHash, +} from "../../src/analyze/input-hash.js"; +import { + EDGE_KINDS, + EDGE_KIND_LIST, + REF_KINDS, + REF_KIND_LIST, + validateEdge, + isEdgeKind, + isRefKind, +} from "../../src/analyze/edge-kinds.js"; + +describe("shortHash", () => { + it("returns 16 hex chars", () => { + const h = shortHash("hello"); + assert.equal(h.length, 16); + assert.match(h, /^[0-9a-f]{16}$/); + }); + + it("is deterministic", () => { + assert.equal(shortHash("foo"), shortHash("foo")); + }); + + it("changes with input", () => { + assert.notEqual(shortHash("foo"), shortHash("bar")); + }); +}); + +describe("fullHash", () => { + it("returns 64 hex chars", () => { + const h = fullHash("hello"); + assert.equal(h.length, 64); + assert.match(h, /^[0-9a-f]{64}$/); + }); + + it("starts with the short hash", () => { + assert.equal(fullHash("hello").slice(0, 16), shortHash("hello")); + }); +}); + +describe("computeSourceSetHash", () => { + it("is order-independent", () => { + const a = computeSourceSetHash([{ kind: "message", id: "m1" }, { kind: "message", id: "m2" }]); + const b = computeSourceSetHash([{ kind: "message", id: "m2" }, { kind: "message", id: "m1" }]); + assert.equal(a, b); + }); + + it("distinguishes different kinds with the same id", () => { + const a = computeSourceSetHash([{ kind: "message", id: "x" }]); + const b = computeSourceSetHash([{ kind: "analysis_node", id: "x" }]); + assert.notEqual(a, b); + }); + + it("handles empty input", () => { + assert.equal(computeSourceSetHash([]), shortHash("")); + }); +}); + +describe("computePromptBundleHash", () => { + it("is order-independent", () => { + const a = computePromptBundleHash(["hash1", "hash2"]); + const b = computePromptBundleHash(["hash2", "hash1"]); + assert.equal(a, b); + }); + + it("handles empty bundle", () => { + assert.equal(computePromptBundleHash([]), shortHash("")); + }); +}); + +describe("computeInputHash", () => { + it("changes when any component changes", () => { + const base = { + analyzerId: "a", + analyzerVersionId: "v1", + configId: "c1", + promptBundleHash: "p1", + sourceSetHash: "s1", + }; + const h0 = computeInputHash(base); + assert.notEqual(h0, computeInputHash({ ...base, analyzerId: "b" })); + assert.notEqual(h0, computeInputHash({ ...base, analyzerVersionId: "v2" })); + assert.notEqual(h0, computeInputHash({ ...base, configId: "c2" })); + assert.notEqual(h0, computeInputHash({ ...base, promptBundleHash: "p2" })); + assert.notEqual(h0, computeInputHash({ ...base, sourceSetHash: "s2" })); + }); + + it("is deterministic for the same input", () => { + const args = { + analyzerId: "a", + analyzerVersionId: "v1", + configId: "c1", + promptBundleHash: "p1", + sourceSetHash: "s1", + }; + assert.equal(computeInputHash(args), computeInputHash(args)); + }); +}); + +describe("canonicalJsonStringify / computeConfigHash", () => { + it("sorts keys at every level", () => { + const a = canonicalJsonStringify({ b: 1, a: 2, c: { y: 3, x: 4 } }); + const b = canonicalJsonStringify({ c: { x: 4, y: 3 }, a: 2, b: 1 }); + assert.equal(a, b); + }); + + it("handles arrays (preserves order)", () => { + const a = canonicalJsonStringify([3, 1, 2]); + assert.equal(a, "[3,1,2]"); + }); + + it("computeConfigHash is order-independent", () => { + const h1 = computeConfigHash({ a: 1, b: 2 }); + const h2 = computeConfigHash({ b: 2, a: 1 }); + assert.equal(h1, h2); + }); +}); + +describe("edge-kinds", () => { + it("EDGE_KIND_LIST contains the documented kinds", () => { + assert.deepEqual([...EDGE_KIND_LIST].sort(), [ + "anchors", + "consumes", + "produces", + "refines", + "uses_config", + "uses_prompt", + ]); + }); + + it("isEdgeKind accepts valid kinds", () => { + assert.equal(isEdgeKind("anchors"), true); + assert.equal(isEdgeKind("invalid"), false); + assert.equal(isEdgeKind(42), false); + assert.equal(isEdgeKind(null), false); + }); + + it("isRefKind accepts valid kinds", () => { + assert.equal(isRefKind("message"), true); + assert.equal(isRefKind("analysis_node"), true); + assert.equal(isRefKind("nope"), false); + }); + + it("validates anchors → message or session", () => { + validateEdge(EDGE_KINDS.ANCHORS, REF_KINDS.MESSAGE); + validateEdge(EDGE_KINDS.ANCHORS, REF_KINDS.SESSION); + assert.throws(() => validateEdge(EDGE_KINDS.ANCHORS, REF_KINDS.ANALYSIS_NODE), /anchors/); + }); + + it("validates consumes → message or analysis_node", () => { + validateEdge(EDGE_KINDS.CONSUMES, REF_KINDS.MESSAGE); + validateEdge(EDGE_KINDS.CONSUMES, REF_KINDS.ANALYSIS_NODE); + assert.throws(() => validateEdge(EDGE_KINDS.CONSUMES, REF_KINDS.SESSION), /consumes/); + }); + + it("validates refines → analysis_node", () => { + validateEdge(EDGE_KINDS.REFINES, REF_KINDS.ANALYSIS_NODE); + assert.throws(() => validateEdge(EDGE_KINDS.REFINES, REF_KINDS.MESSAGE), /refines/); + }); + + it("validates uses_prompt → prompt_version", () => { + validateEdge(EDGE_KINDS.USES_PROMPT, REF_KINDS.PROMPT_VERSION); + assert.throws(() => validateEdge(EDGE_KINDS.USES_PROMPT, REF_KINDS.ANALYSIS_NODE), /uses_prompt/); + }); + + it("validates uses_config → config_version", () => { + validateEdge(EDGE_KINDS.USES_CONFIG, REF_KINDS.CONFIG_VERSION); + assert.throws(() => validateEdge(EDGE_KINDS.USES_CONFIG, REF_KINDS.ANALYSIS_NODE), /uses_config/); + }); + + it("validates produces → analysis_node", () => { + validateEdge(EDGE_KINDS.PRODUCES, REF_KINDS.ANALYSIS_NODE); + assert.throws(() => validateEdge(EDGE_KINDS.PRODUCES, REF_KINDS.MESSAGE), /produces/); + }); +}); diff --git a/tests/unit/session-overview.test.ts b/tests/unit/session-overview.test.ts new file mode 100644 index 0000000..88993dc --- /dev/null +++ b/tests/unit/session-overview.test.ts @@ -0,0 +1,188 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + buildDigest, + splitDigest, +} from "../../src/analyze/analyzers/session-overview/digest.js"; +import { + buildMapPrompt, + parseMapResponse, +} from "../../src/analyze/analyzers/session-overview/prompt-map.js"; +import { + buildReducePrompt, + parseReduceResponse, +} from "../../src/analyze/analyzers/session-overview/prompt-reduce.js"; +import type { AnalysisNodeRow, MessageRow } from "../../src/analyze/types.js"; + +function makeMessage(id: string, role: string, text: string | null): MessageRow { + return { + id, + session_id: "s", + parent_id: null, + timestamp: "2026-01-01T00:00:00Z", + role: role as MessageRow["role"], + content_text: text, + content_thinking: null, + tool_calls: null, + tool_results: null, + meta_json: null, + }; +} + +function makePairNode(id: string, props: Record): AnalysisNodeRow { + return { + id, + session_id: "s", + analyzer_id: "turn-pair-core", + analyzer_version_id: "0.1.0", + config_id: "c", + run_id: "r", + node_kind: "metric", + content_json: JSON.stringify(props), + source_set_hash: "h", + input_hash: "i", + created_at: "2026-01-01T00:00:00Z", + model_used: null, + cost_usd: 0, + tokens_used: 0, + duration_ms: null, + }; +} + +describe("buildDigest", () => { + it("produces a single-segment digest with header, phases, and stats", () => { + const messages = [ + makeMessage("u1", "user", "actually, use pnpm"), + makeMessage("a1", "assistant", "ok switching"), + ]; + const pairProps = { + correction_detected: true, + friction_score: 0.45, + tool_failure_count: 0, + tool_waste_bytes: 0, + correction_type: "explicit", + correction_text: "use pnpm", + tool_names: ["bash"], + elapsed_seconds: 5.0, + model: "anthropic/claude-sonnet-4-5", + }; + const pairNodes = [makePairNode("p1", pairProps)]; + const digest = buildDigest({ sessionId: "s1", messages, pairNodes, llmNodes: [] }); + assert.equal(digest.segments.length, 1); + assert.ok(digest.segments[0]!.text.includes("Session ID: s1")); + assert.ok(digest.segments[0]!.text.includes("explicit"), "shows correction type"); + assert.ok(digest.segments[0]!.text.includes("0.45"), "shows friction score"); + assert.ok(digest.segments[0]!.text.includes("Statistics")); + assert.ok(digest.segments[0]!.text.includes("Total pairs: 1")); + assert.equal(digest.pairCount, 1); + assert.equal(digest.frictionCount, 1); + }); + + it("counts compactions", () => { + const messages = [ + makeMessage("u1", "user", "hi"), + makeMessage("a1", "assistant", "ok"), + makeMessage("c1", "compactionSummary", "old context"), + makeMessage("u2", "user", "hi again"), + makeMessage("a2", "assistant", "ok again"), + ]; + const digest = buildDigest({ sessionId: "s1", messages, pairNodes: [], llmNodes: [] }); + assert.equal(digest.compactionCount, 1); + }); +}); + +describe("splitDigest", () => { + it("returns single segment when under budget", () => { + const digest = buildDigest({ sessionId: "s", messages: [], pairNodes: [], llmNodes: [] }); + const segs = splitDigest(digest, 10_000); + assert.equal(segs.length, 1); + }); + + it("splits into multiple segments when over budget", () => { + const messages = Array.from({ length: 50 }, (_, i) => makeMessage(`m${i}`, i % 2 === 0 ? "user" : "assistant", `text-${i}-${"x".repeat(100)}`)); + const pairNodes = Array.from({ length: 25 }, (_, i) => makePairNode(`p${i}`, { + correction_detected: i % 3 === 0, + friction_score: i % 5 === 0 ? 0.5 : 0.1, + tool_failure_count: 0, + tool_waste_bytes: 0, + correction_type: null, + correction_text: null, + tool_names: ["read"], + elapsed_seconds: 1, + model: "x", + })); + const digest = buildDigest({ sessionId: "s", messages, pairNodes, llmNodes: [] }); + const segs = splitDigest(digest, 500); + assert.ok(segs.length >= 2, `expected >=2 segments, got ${segs.length}`); + }); +}); + +describe("buildMapPrompt / buildReducePrompt", () => { + it("substitutes the digest placeholder", () => { + const out = buildMapPrompt("hello-marker"); + assert.ok(out.includes("hello-marker")); + assert.equal(out.includes("{digest}"), false); + }); + + it("substitutes segment_summaries and stats", () => { + const out = buildReducePrompt({ segmentSummaries: "ss-marker", stats: "st-marker" }); + assert.ok(out.includes("ss-marker")); + assert.ok(out.includes("st-marker")); + assert.equal(out.includes("{segment_summaries}"), false); + assert.equal(out.includes("{stats}"), false); + }); +}); + +describe("parseMapResponse", () => { + it("parses well-formed JSON", () => { + const text = JSON.stringify({ + segment_summary: "User wanted X, agent did Y", + key_friction_points: [{ description: "wrong_api", severity: "high", evidence_pair_index: 3 }], + improvement_proposals: [{ + target_type: "skill", + target_path: "skill/foo", + title: "Add a foo skill", + summary: "s", + detail: "d", + evidence: "e", + confidence: 0.8, + severity: "suggestion", + }], + sentiment_arc: [{ segment: 0, sentiment: "frustrated", key_event: "agent loop" }], + }); + const r = parseMapResponse(text); + assert.equal(r.segment_summary, "User wanted X, agent did Y"); + assert.equal(r.key_friction_points.length, 1); + assert.equal(r.improvement_proposals.length, 1); + assert.equal(r.sentiment_arc.length, 1); + }); + + it("returns empty on invalid JSON", () => { + const r = parseMapResponse("not json"); + assert.equal(r.segment_summary, ""); + assert.equal(r.improvement_proposals.length, 0); + }); +}); + +describe("parseReduceResponse", () => { + it("parses well-formed JSON", () => { + const text = JSON.stringify({ + session_summary: "Overall summary", + key_friction_points: [], + improvement_proposals: [{ + target_type: "agents_md", + target_path: "~/.pi/agent/AGENTS.md", + title: "Tweak", + summary: "s", + detail: "d", + evidence: "e", + confidence: 0.6, + severity: "suggestion", + }], + sentiment_arc: [], + }); + const r = parseReduceResponse(text); + assert.equal(r.session_summary, "Overall summary"); + assert.equal(r.improvement_proposals.length, 1); + }); +}); diff --git a/tests/unit/turn-pair-builder.test.ts b/tests/unit/turn-pair-builder.test.ts new file mode 100644 index 0000000..c3be367 --- /dev/null +++ b/tests/unit/turn-pair-builder.test.ts @@ -0,0 +1,266 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + buildTurnPairNode, +} from "../../src/analyze/analyzers/turn-pair-core/index.js"; +import { DEFAULT_TURN_PAIR_CORE_CONFIG } from "../../src/analyze/analyzers/turn-pair-core/config.js"; +import type { MessageRow } from "../../src/analyze/types.js"; + +function makeUser(id: string, text: string, ts: string): MessageRow { + return { + id, + session_id: "s", + parent_id: null, + timestamp: ts, + role: "user", + content_text: text, + content_thinking: null, + tool_calls: null, + tool_results: null, + meta_json: null, + }; +} + +function makeAssistant(id: string, text: string, ts: string, opts: { + thinking?: string; + toolCalls?: Array<{ name: string; arguments: Record }>; + model?: string; + usage?: { input?: number; output?: number }; + stopReason?: string; +} = {}): MessageRow { + const meta: Record = {}; + if (opts.model) meta.model = opts.model; + if (opts.usage) meta.usage = opts.usage; + if (opts.stopReason) meta.stop_reason = opts.stopReason; + return { + id, + session_id: "s", + parent_id: null, + timestamp: ts, + role: "assistant", + content_text: text, + content_thinking: opts.thinking ?? null, + tool_calls: opts.toolCalls ? JSON.stringify(opts.toolCalls) : null, + tool_results: null, + meta_json: Object.keys(meta).length > 0 ? JSON.stringify(meta) : null, + }; +} + +function makeToolResult(id: string, toolName: string, text: string, isError: boolean): MessageRow { + const tr = { toolName, isError, textLength: text.length }; + return { + id, + session_id: "s", + parent_id: null, + timestamp: null, + role: "toolResult", + content_text: text, + content_thinking: null, + tool_calls: null, + tool_results: JSON.stringify([tr]), + meta_json: null, + }; +} + +describe("buildTurnPairNode — basic shape", () => { + it("returns null if no user message at the index", () => { + const msgs: MessageRow[] = [makeAssistant("a1", "hi", "2026-01-01T00:00:00Z")]; + const r = buildTurnPairNode(msgs, 0, 0, DEFAULT_TURN_PAIR_CORE_CONFIG); + assert.equal(r, null); + }); + + it("returns null if no assistant follows", () => { + const msgs: MessageRow[] = [makeUser("u1", "hi", "2026-01-01T00:00:00Z")]; + const r = buildTurnPairNode(msgs, 0, 0, DEFAULT_TURN_PAIR_CORE_CONFIG); + assert.equal(r, null); + }); + + it("computes lengths from text", () => { + const msgs = [ + makeUser("u1", "hello world", "2026-01-01T00:00:00Z"), + makeAssistant("a1", "ok", "2026-01-01T00:00:05Z"), + ]; + const r = buildTurnPairNode(msgs, 0, 1, DEFAULT_TURN_PAIR_CORE_CONFIG); + assert.ok(r); + assert.equal(r!.user_msg_length, 11); + assert.equal(r!.assistant_msg_length, 2); + }); + + it("captures thinking when present", () => { + const msgs = [ + makeUser("u1", "hi", "2026-01-01T00:00:00Z"), + makeAssistant("a1", "ok", "2026-01-01T00:00:05Z", { thinking: "let me think" }), + ]; + const r = buildTurnPairNode(msgs, 0, 1, DEFAULT_TURN_PAIR_CORE_CONFIG); + assert.equal(r!.has_thinking, true); + assert.equal(r!.thinking_length, "let me think".length); + }); +}); + +describe("buildTurnPairNode — correction", () => { + it("flags correction_detected with 'actually'", () => { + const msgs = [ + makeUser("u1", "actually, use pnpm not npm", "2026-01-01T00:00:00Z"), + makeAssistant("a1", "ok switching", "2026-01-01T00:00:05Z"), + ]; + const r = buildTurnPairNode(msgs, 0, 1, DEFAULT_TURN_PAIR_CORE_CONFIG); + assert.equal(r!.correction_detected, true); + assert.equal(r!.correction_type, "explicit"); + assert.ok(r!.correction_patterns.length >= 1); + assert.ok(r!.correction_text); + }); + + it("leaves correction_detected false for clean messages", () => { + const msgs = [ + makeUser("u1", "what is the package manager?", "2026-01-01T00:00:00Z"), + makeAssistant("a1", "pnpm", "2026-01-01T00:00:05Z"), + ]; + const r = buildTurnPairNode(msgs, 0, 1, DEFAULT_TURN_PAIR_CORE_CONFIG); + assert.equal(r!.correction_detected, false); + assert.equal(r!.correction_type, null); + }); + + it("raises friction_score when correction detected", () => { + const clean = buildTurnPairNode([ + makeUser("u1", "hello", "2026-01-01T00:00:00Z"), + makeAssistant("a1", "hi", "2026-01-01T00:00:05Z"), + ], 0, 1, DEFAULT_TURN_PAIR_CORE_CONFIG)!; + const corr = buildTurnPairNode([ + makeUser("u2", "actually, use pnpm", "2026-01-01T00:00:00Z"), + makeAssistant("a2", "ok", "2026-01-01T00:00:05Z"), + ], 0, 1, DEFAULT_TURN_PAIR_CORE_CONFIG)!; + assert.ok(corr.friction_score > clean.friction_score); + }); +}); + +describe("buildTurnPairNode — tools", () => { + it("counts tool calls and tool names", () => { + const msgs = [ + makeUser("u1", "look at the file", "2026-01-01T00:00:00Z"), + makeAssistant("a1", "reading", "2026-01-01T00:00:05Z", { + toolCalls: [ + { name: "read", arguments: { path: "/a" } }, + { name: "read", arguments: { path: "/b" } }, + { name: "bash", arguments: { command: "ls" } }, + ], + }), + makeToolResult("tr1", "read", "file contents of A", false), + makeToolResult("tr2", "read", "file contents of B", false), + ]; + // endIndex includes the last tool result in this pair + const r = buildTurnPairNode(msgs, 0, 3, DEFAULT_TURN_PAIR_CORE_CONFIG); + assert.equal(r!.tool_call_count, 3); + assert.deepEqual(r!.tool_names.sort(), ["bash", "read"]); + }); + + it("counts tool failures and reports details", () => { + const msgs = [ + makeUser("u1", "read a", "2026-01-01T00:00:00Z"), + makeAssistant("a1", "trying", "2026-01-01T00:00:05Z", { + toolCalls: [{ name: "read", arguments: { path: "/a" } }], + }), + makeToolResult("tr1", "read", "ERROR: file not found", true), + ]; + const r = buildTurnPairNode(msgs, 0, 2, DEFAULT_TURN_PAIR_CORE_CONFIG); + assert.equal(r!.tool_failure_count, 1); + assert.equal(r!.tool_failure_details[0]!.tool_name, "read"); + assert.match(r!.tool_failure_details[0]!.error_preview, /ERROR/); + }); + + it("detects retry when same tool+target is called twice", () => { + const msgs = [ + makeUser("u1", "read a", "2026-01-01T00:00:00Z"), + makeAssistant("a1", "retrying", "2026-01-01T00:00:05Z", { + toolCalls: [ + { name: "read", arguments: { path: "/a" } }, + { name: "read", arguments: { path: "/a" } }, + ], + }), + ]; + const r = buildTurnPairNode(msgs, 0, 1, DEFAULT_TURN_PAIR_CORE_CONFIG); + assert.equal(r!.retry_detected, true); + }); + + it("does not flag retry when different paths", () => { + const msgs = [ + makeUser("u1", "read a and b", "2026-01-01T00:00:00Z"), + makeAssistant("a1", "ok", "2026-01-01T00:00:05Z", { + toolCalls: [ + { name: "read", arguments: { path: "/a" } }, + { name: "read", arguments: { path: "/b" } }, + ], + }), + ]; + const r = buildTurnPairNode(msgs, 0, 1, DEFAULT_TURN_PAIR_CORE_CONFIG); + assert.equal(r!.retry_detected, false); + }); + + it("captures model and usage from meta", () => { + const msgs = [ + makeUser("u1", "hi", "2026-01-01T00:00:00Z"), + makeAssistant("a1", "ok", "2026-01-01T00:00:05Z", { + model: "anthropic/claude-sonnet-4-5", + usage: { input: 100, output: 50 }, + stopReason: "stop", + }), + ]; + const r = buildTurnPairNode(msgs, 0, 1, DEFAULT_TURN_PAIR_CORE_CONFIG); + assert.equal(r!.model, "anthropic/claude-sonnet-4-5"); + assert.equal(r!.usage_input_tokens, 100); + assert.equal(r!.usage_output_tokens, 50); + assert.equal(r!.stop_reason, "stop"); + }); + + it("computes elapsed_seconds from timestamps", () => { + const msgs = [ + makeUser("u1", "hi", "2026-01-01T00:00:00Z"), + makeAssistant("a1", "ok", "2026-01-01T00:00:10Z"), + ]; + const r = buildTurnPairNode(msgs, 0, 1, DEFAULT_TURN_PAIR_CORE_CONFIG); + assert.equal(r!.elapsed_seconds, 10); + }); +}); + +describe("buildTurnPairNode — waste bytes", () => { + it("counts bytes of tool results never referenced in assistant text", () => { + const sample = "x".repeat(200); + const msgs = [ + makeUser("u1", "check it", "2026-01-01T00:00:00Z"), + makeAssistant("a1", "I looked at the file", "2026-01-01T00:00:05Z", { + toolCalls: [{ name: "read", arguments: { path: "/a" } }], + }), + makeToolResult("tr1", "read", sample, false), + ]; + const r = buildTurnPairNode(msgs, 0, 2, DEFAULT_TURN_PAIR_CORE_CONFIG); + assert.equal(r!.tool_waste_bytes, 200); + }); + + it("does not count bytes when assistant references the result", () => { + const snippet = "function login(user) { return true; }"; + const msgs = [ + makeUser("u1", "show me login", "2026-01-01T00:00:00Z"), + makeAssistant("a1", `the file has ${snippet} inside`, "2026-01-01T00:00:05Z", { + toolCalls: [{ name: "read", arguments: { path: "/a" } }], + }), + makeToolResult("tr1", "read", snippet, false), + ]; + const r = buildTurnPairNode(msgs, 0, 2, DEFAULT_TURN_PAIR_CORE_CONFIG); + assert.equal(r!.tool_waste_bytes, 0); + }); +}); + +describe("buildTurnPairNode — compaction boundary", () => { + it("flags when a compaction summary is in the range", () => { + const msgs: MessageRow[] = [ + makeUser("u1", "hi", "2026-01-01T00:00:00Z"), + { + id: "cs1", session_id: "s", parent_id: null, timestamp: null, + role: "compactionSummary", content_text: "old context", content_thinking: null, + tool_calls: null, tool_results: null, meta_json: null, + }, + makeAssistant("a1", "ok", "2026-01-01T00:00:05Z"), + ]; + const r = buildTurnPairNode(msgs, 0, 2, DEFAULT_TURN_PAIR_CORE_CONFIG); + assert.equal(r!.is_compaction_boundary, true); + }); +}); diff --git a/tests/unit/turn-pair-llm.test.ts b/tests/unit/turn-pair-llm.test.ts new file mode 100644 index 0000000..036c7d5 --- /dev/null +++ b/tests/unit/turn-pair-llm.test.ts @@ -0,0 +1,84 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + buildTurnPairLlmPrompt, + parseTurnPairLlmResponse, +} from "../../src/analyze/analyzers/turn-pair-llm/prompt.js"; + +describe("buildTurnPairLlmPrompt", () => { + it("substitutes every placeholder", () => { + const out = buildTurnPairLlmPrompt({ + userText: "u-marker", + assistantText: "a-marker", + toolCalls: "[]", + toolResults: "[]", + friction: { + correction_detected: true, + friction_score: 0.5, + tool_failure_count: 2, + retry_detected: true, + thinking_length: 100, + }, + }); + assert.ok(out.includes("u-marker")); + assert.ok(out.includes("a-marker")); + assert.ok(out.includes("[]")); + assert.ok(out.includes("true")); + assert.ok(out.includes("0.50")); + // No leftover template placeholders + assert.equal(out.includes("{user_text}"), false); + assert.equal(out.includes("{assistant_text}"), false); + }); +}); + +describe("parseTurnPairLlmResponse", () => { + it("parses well-formed JSON", () => { + const text = JSON.stringify({ + sentiment: "frustrated", + frustration_level: 7, + correction_type_llm: "explicit", + friction_cause: "wrong_function_name", + friction_summary: "User corrected the function name twice.", + user_intent: "Get the agent to use the right helper", + quality_score: 2, + }); + const c = parseTurnPairLlmResponse(text); + assert.equal(c.sentiment, "frustrated"); + assert.equal(c.frustration_level, 7); + assert.equal(c.correction_type_llm, "explicit"); + assert.equal(c.friction_cause, "wrong_function_name"); + assert.equal(c.quality_score, 2); + }); + + it("strips code fences", () => { + const text = "```json\n" + JSON.stringify({ sentiment: "positive" }) + "\n```"; + const c = parseTurnPairLlmResponse(text); + assert.equal(c.sentiment, "positive"); + }); + + it("returns defaults on invalid JSON", () => { + const c = parseTurnPairLlmResponse("not json"); + assert.equal(c.sentiment, "neutral"); + assert.equal(c.frustration_level, 0); + assert.equal(c.quality_score, 3); + }); + + it("clamps frustration_level to [0, 10]", () => { + const c1 = parseTurnPairLlmResponse(JSON.stringify({ frustration_level: 99 })); + assert.equal(c1.frustration_level, 10); + const c2 = parseTurnPairLlmResponse(JSON.stringify({ frustration_level: -5 })); + assert.equal(c2.frustration_level, 0); + }); + + it("clamps quality_score to [1, 5]", () => { + const c1 = parseTurnPairLlmResponse(JSON.stringify({ quality_score: 99 })); + assert.equal(c1.quality_score, 5); + const c2 = parseTurnPairLlmResponse(JSON.stringify({ quality_score: 0 })); + assert.equal(c2.quality_score, 1); + }); + + it("rejects invalid sentiment values", () => { + const c = parseTurnPairLlmResponse(JSON.stringify({ sentiment: "happy" })); + assert.equal(c.sentiment, "neutral"); + }); +}); diff --git a/tests/unit/turn-pair-patterns.test.ts b/tests/unit/turn-pair-patterns.test.ts new file mode 100644 index 0000000..9a40128 --- /dev/null +++ b/tests/unit/turn-pair-patterns.test.ts @@ -0,0 +1,156 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + detectCorrection, + detectAllCorrectionPatterns, + detectRepetition, + extractCorrectionText, +} from "../../src/analyze/analyzers/turn-pair-core/patterns.js"; +import { + DEFAULT_TURN_PAIR_CORE_CONFIG, + computeFrictionScore, +} from "../../src/analyze/analyzers/turn-pair-core/config.js"; + +describe("detectCorrection", () => { + it("flags 'no, use X' as strong explicit", () => { + const m = detectCorrection("no, use pnpm not npm"); + assert.ok(m); + assert.equal(m?.type, "explicit"); + }); + + it("flags 'actually' as strong explicit", () => { + const m = detectCorrection("actually, I meant pnpm"); + assert.ok(m); + assert.equal(m?.type, "explicit"); + }); + + it("flags 'I said' / 'I told you' as strong explicit", () => { + assert.ok(detectCorrection("I said use pnpm")); + assert.ok(detectCorrection("I told you this earlier")); + }); + + it("flags 'that's wrong' as strong explicit", () => { + assert.ok(detectCorrection("that's wrong, the function is foo")); + }); + + it("flags leading negation", () => { + assert.ok(detectCorrection("no")); + assert.ok(detectCorrection("don't do that")); + assert.ok(detectCorrection("never mind, do X instead")); + }); + + it("flags weak correction patterns", () => { + assert.ok(detectCorrection("could you try using the new API?")); + assert.ok(detectCorrection("maybe we should use a different approach")); + }); + + it("returns null for clean messages", () => { + assert.equal(detectCorrection("Hello, can you help me with the auth module?"), null); + assert.equal(detectCorrection("Run the tests please"), null); + }); + + it("returns null for null input", () => { + assert.equal(detectCorrection(null), null); + }); +}); + +describe("detectAllCorrectionPatterns", () => { + it("returns multiple patterns when several match", () => { + const m = detectAllCorrectionPatterns("actually, I said use pnpm, no?"); + assert.ok(m.length >= 2); + }); + + it("returns empty for clean text", () => { + assert.deepEqual(detectAllCorrectionPatterns("Hello world"), []); + }); +}); + +describe("detectRepetition", () => { + it("flags short message with shared tokens as repetition", () => { + assert.equal(detectRepetition("try pnpm install", "run pnpm install please"), true); + }); + + it("does not flag longer messages", () => { + assert.equal(detectRepetition( + "please run pnpm install to update dependencies across the workspace", + "run pnpm install", + ), false); + }); + + it("returns false without prior text", () => { + assert.equal(detectRepetition("try pnpm install", null), false); + }); +}); + +describe("extractCorrectionText", () => { + it("returns text after the matched pattern", () => { + const m = detectCorrection("actually, use pnpm not npm"); + assert.ok(m); + const text = extractCorrectionText("actually, use pnpm not npm", m!); + assert.match(text, /use pnpm not npm/); + }); + + it("caps at 240 chars", () => { + const long = "no, " + "x".repeat(500); + const m = detectCorrection(long); + assert.ok(m); + const text = extractCorrectionText(long, m!); + assert.ok(text.length <= 240); + }); +}); + +describe("computeFrictionScore", () => { + const cfg = DEFAULT_TURN_PAIR_CORE_CONFIG; + + it("is 0 with no signals", () => { + assert.equal(computeFrictionScore(cfg, { + correctionDetected: false, + toolFailureCount: 0, + retryDetected: false, + hasThinking: false, + isCompactionBoundary: false, + }), 0); + }); + + it("is at most 1.0 with all signals", () => { + const score = computeFrictionScore(cfg, { + correctionDetected: true, + toolFailureCount: 100, + retryDetected: true, + hasThinking: true, + isCompactionBoundary: true, + }); + assert.ok(score <= 1.0); + assert.ok(score > 0.5); + }); + + it("is sensitive to correction", () => { + const noCorr = computeFrictionScore(cfg, { + correctionDetected: false, + toolFailureCount: 0, retryDetected: false, hasThinking: false, isCompactionBoundary: false, + }); + const corr = computeFrictionScore(cfg, { + correctionDetected: true, + toolFailureCount: 0, retryDetected: false, hasThinking: false, isCompactionBoundary: false, + }); + assert.ok(corr > noCorr); + }); + + it("caps failures at max_tool_failures (step function)", () => { + const belowCap = computeFrictionScore(cfg, { + correctionDetected: false, toolFailureCount: 2, + retryDetected: false, hasThinking: false, isCompactionBoundary: false, + }); + const atCap = computeFrictionScore(cfg, { + correctionDetected: false, toolFailureCount: 3, + retryDetected: false, hasThinking: false, isCompactionBoundary: false, + }); + const aboveCap = computeFrictionScore(cfg, { + correctionDetected: false, toolFailureCount: 10, + retryDetected: false, hasThinking: false, isCompactionBoundary: false, + }); + assert.equal(belowCap, 0, "below cap should be 0"); + assert.equal(atCap, aboveCap, "at and above cap should match"); + assert.equal(atCap, cfg.weights.tool_failure); + }); +});