diff --git a/packages/indexeddb/src/storage.ts b/packages/indexeddb/src/storage.ts index 7efa254..b72314a 100644 --- a/packages/indexeddb/src/storage.ts +++ b/packages/indexeddb/src/storage.ts @@ -382,6 +382,12 @@ export interface RawMessageSearchHooks { threshold?: number; botId?: string; includeArchived?: boolean; + /** + * Forward `includeDeprecated` from the unified search input so + * supersession-aware callers can opt into deprecated rows for + * audits. Default `false` preserves the current-truth behaviour. + */ + includeDeprecated?: boolean; }): Promise; lexicalSearch?(input: { userId: string; @@ -389,6 +395,12 @@ export interface RawMessageSearchHooks { limit: number; botId?: string; includeArchived?: boolean; + /** + * Forward `includeDeprecated` from the unified search input so + * supersession-aware callers can opt into deprecated rows for + * audits. Default `false` preserves the current-truth behaviour. + */ + includeDeprecated?: boolean; }): Promise; } diff --git a/packages/memory-store/src/config.ts b/packages/memory-store/src/config.ts index be68c0c..09056e7 100644 --- a/packages/memory-store/src/config.ts +++ b/packages/memory-store/src/config.ts @@ -215,6 +215,12 @@ export interface UnifiedSearchDeps { limit: number; threshold: number; botId?: string; + /** + * Forward `includeDeprecated` from `UnifiedMemorySearchInput` so + * supersession-aware callers can opt into deprecated rows for + * audits. Default `false` preserves the current-truth behaviour. + */ + includeDeprecated?: boolean; /** Optional peer scope resolved from `UnifiedMemorySearchInput.peerFilter`. */ peers?: ReadonlyArray; /** Optional `FactType` filter resolved from `UnifiedMemorySearchInput.factTypes`. */ @@ -282,6 +288,12 @@ export interface UnifiedSearchDeps { keywords: string[]; limit: number; botId?: string; + /** + * Forward `includeDeprecated` from `UnifiedMemorySearchInput` so + * supersession-aware callers can opt into deprecated rows for + * audits. Default `false` preserves the current-truth behaviour. + */ + includeDeprecated?: boolean; /** Optional peer scope resolved from `UnifiedMemorySearchInput.peerFilter`. */ peers?: ReadonlyArray; /** Optional `FactType` filter resolved from `UnifiedMemorySearchInput.factTypes`. */ diff --git a/packages/memory-store/src/search/gather-evidence.ts b/packages/memory-store/src/search/gather-evidence.ts index 1d364e0..ac006ee 100644 --- a/packages/memory-store/src/search/gather-evidence.ts +++ b/packages/memory-store/src/search/gather-evidence.ts @@ -396,10 +396,24 @@ export async function gatherEvidence(opts: GatherOptions): Promise * Build the prompt the LLM receives when `synthesize: true` is set. * Mirrors the legacy `reflect.ts` prompt format so callers see identical * behaviour before and after the merge. + * + * Per-fact provenance: when an evidence item carries `metadata.source` + * (e.g. `"meeting://2026-08-15"` written by `opencontext add --source`), + * the line format surfaces it as `source=` so the LLM can attribute + * claims to the originating fact. Items without a `source` metadata field + * fall back to the channel-level `source` (e.g. `"memory"`, `"insight"`) + * and finally `"unknown"`, so the prompt stays well-formed even when an + * upstream tier doesn't carry per-fact provenance. */ export function buildSynthesisPrompt(input: { query: string; - evidence: Array<{ id: string; source: SearchTier; snippet: string; score: number }>; + evidence: Array<{ + id: string; + source: SearchTier; + snippet: string; + score: number; + metadata?: Record; + }>; responseSchema?: Record; }): string { const grouped = new Map(); @@ -415,9 +429,13 @@ export function buildSynthesisPrompt(input: { if (items.length === 0) { continue; } - const lines = items.map( - (item, index) => ` [${index + 1}] (${item.id}, score=${item.score.toFixed(4)}) ${item.snippet}`, - ); + const lines = items.map((item, index) => { + const factSource = + typeof item.metadata?.source === "string" && item.metadata.source.length > 0 + ? item.metadata.source + : (item.source ?? "unknown"); + return ` [${index + 1}] (${item.id}, source=${factSource}, score=${item.score.toFixed(4)}) ${item.snippet}`; + }); sections.push(`## ${tier}\n${lines.join("\n")}`); } @@ -512,6 +530,9 @@ export async function synthesizeAnswer(input: { source: mapEvidenceSourceToTier(item.source), snippet: item.snippet, score: item.score, + // Forward per-fact provenance (e.g. `metadata.source` from + // `opencontext add --source`) so the synthesis prompt can cite it. + metadata: item.metadata, })); const warnings: UnifiedMemorySearchWarning[] = []; diff --git a/packages/memory-store/src/search/unified-search.ts b/packages/memory-store/src/search/unified-search.ts index 6149230..40554dc 100644 --- a/packages/memory-store/src/search/unified-search.ts +++ b/packages/memory-store/src/search/unified-search.ts @@ -328,6 +328,7 @@ async function runSemanticSearchForEmbedding( if (typeof deps.searchRawMessagesAnn === "function") { const searchRawMessagesAnn = deps.searchRawMessagesAnn; const factTypes = input.factTypes?.length ? input.factTypes : undefined; + const includeDeprecated = input.includeDeprecated === true; semantic = ( await Promise.all( filters.map((filter) => @@ -337,6 +338,7 @@ async function runSemanticSearchForEmbedding( limit, threshold, botId: "botId" in filter ? filter.botId : undefined, + includeDeprecated, ...(peerPeers.length > 0 ? { peers: peerPeers } : {}), ...(factTypes ? { factTypes } : {}), ...(runtimeContext ?? {}), @@ -394,6 +396,7 @@ async function runSemanticSearchForEmbedding( queryEmbedding, limit, threshold, + includeDeprecated: input.includeDeprecated === true, ...(peerPeers.length > 0 ? { peers: peerPeers } : {}), ...(factTypes ? { factTypes } : {}), }); @@ -481,6 +484,7 @@ async function runLexicalSearchForKeywords( const filters = input.botIds && input.botIds.length > 0 ? input.botIds : [undefined]; const searchRawMessagesLexical = deps.searchRawMessagesLexical; const factTypes = input.factTypes?.length ? input.factTypes : undefined; + const includeDeprecated = input.includeDeprecated === true; return ( await Promise.all( filters.map((botId) => @@ -489,6 +493,7 @@ async function runLexicalSearchForKeywords( keywords, limit: Math.ceil(limit / filters.length), botId, + includeDeprecated, ...(peerPeers.length > 0 ? { peers: peerPeers } : {}), ...(factTypes ? { factTypes } : {}), ...(runtimeContext ?? {}), @@ -515,6 +520,7 @@ async function runLexicalSearchForKeywords( const { lexicalSearchRawMessages } = await import("../storage/sqlite-raw-message-store"); const filters = input.botIds && input.botIds.length > 0 ? input.botIds : [undefined]; const factTypes = input.factTypes?.length ? input.factTypes : undefined; + const includeDeprecated = input.includeDeprecated === true; return ( await Promise.all( filters.map((botId) => @@ -523,6 +529,7 @@ async function runLexicalSearchForKeywords( keywords, limit: Math.ceil(limit / filters.length), botId, + includeDeprecated, ...(factTypes ? { factTypes } : {}), }), ), @@ -582,6 +589,7 @@ function searchInputToUnified(input: SearchInput): UnifiedMemorySearchInput { reasoningStrategy: input.reasoningStrategy, factTypes: input.factTypes, includeRetrievalDiagnostics: input.includeRetrievalDiagnostics, + includeDeprecated: input.includeDeprecated, }; } @@ -1416,6 +1424,7 @@ export function createUnifiedSearch(deps: UnifiedSearchDeps = {}): UnifiedSearch snippet: hit.content, score: hit.similarity, timestamp: getCandidateTimestamp(hit.metadata), + metadata: hit.metadata, })); const base: SearchOutput = { query, diff --git a/packages/memory-store/src/search/utilities.ts b/packages/memory-store/src/search/utilities.ts index 7eb2b3e..6c2930c 100644 --- a/packages/memory-store/src/search/utilities.ts +++ b/packages/memory-store/src/search/utilities.ts @@ -171,6 +171,15 @@ export interface UnifiedMemorySearchInput { factTypes?: FactType[]; /** Include pre-fusion channel candidates in the response for diagnostics. */ includeRetrievalDiagnostics?: boolean; + /** + * Include messages that have been soft-deprecated + * (`raw_messages.deprecated_at IS NOT NULL`). Default `false` keeps + * `current-truth` retrievals clean: a row that was marked superseded + * via `deprecateMessages` (or the `opencontext deprecate` CLI) is + * excluded from results. Set to `true` for audits and historical + * exploration. + */ + includeDeprecated?: boolean; } export type UnifiedMemoryMergeStrategy = "similarity" | "rrf"; @@ -321,6 +330,15 @@ export interface SearchInput { includeArchivedInsights?: boolean; /** Include pre-fusion retrieval candidates; intended for evaluation/debugging. */ includeRetrievalDiagnostics?: boolean; + /** + * Include messages that have been soft-deprecated + * (`raw_messages.deprecated_at IS NOT NULL`). Default `false` keeps + * `current-truth` retrievals clean: a row that was marked superseded + * via `deprecateMessages` (or the `opencontext deprecate` CLI) is + * excluded from results. Set to `true` for audits and historical + * exploration. + */ + includeDeprecated?: boolean; } export interface SearchEvidence { @@ -329,6 +347,12 @@ export interface SearchEvidence { snippet: string; score: number; timestamp?: number; + /** + * Forwarded from the underlying hit's `metadata`. Surfaces per-fact + * provenance (`metadata.source`, `metadata.factType`, etc.) to the + * synthesis prompt and the CLI's `--context-only` / `--json` output. + */ + metadata?: Record; } export interface SearchOutput { diff --git a/packages/memory-store/src/storage/raw-message-store.ts b/packages/memory-store/src/storage/raw-message-store.ts index 9414c2c..b911b52 100644 --- a/packages/memory-store/src/storage/raw-message-store.ts +++ b/packages/memory-store/src/storage/raw-message-store.ts @@ -40,6 +40,12 @@ export type RawMessageStorageManagerWithSearch = RawMessageStorageManager & { scanLimit?: number; threshold?: number; includeArchived?: boolean; + /** + * Forward `includeDeprecated` from the unified search input so + * supersession-aware callers can opt into deprecated rows for + * audits. Default `false` preserves the current-truth behaviour. + */ + includeDeprecated?: boolean; platform?: string; botId?: string; channel?: string; @@ -52,6 +58,12 @@ export type RawMessageStorageManagerWithSearch = RawMessageStorageManager & { keywords: string[]; limit?: number; includeArchived?: boolean; + /** + * Forward `includeDeprecated` from the unified search input so + * supersession-aware callers can opt into deprecated rows for + * audits. Default `false` preserves the current-truth behaviour. + */ + includeDeprecated?: boolean; platform?: string; botId?: string; }) => Promise; diff --git a/packages/memory-store/src/storage/sqlite-raw-message-store.ts b/packages/memory-store/src/storage/sqlite-raw-message-store.ts index 9a4375d..03c4914 100644 --- a/packages/memory-store/src/storage/sqlite-raw-message-store.ts +++ b/packages/memory-store/src/storage/sqlite-raw-message-store.ts @@ -79,6 +79,12 @@ export async function lexicalSearchRawMessages(input: { keywords: string[]; limit?: number; includeArchived?: boolean; + /** + * Forward `includeDeprecated` from `UnifiedMemorySearchInput` so + * supersession-aware callers can opt into deprecated rows for + * audits. Default `false` preserves the current-truth behaviour. + */ + includeDeprecated?: boolean; platform?: string; botId?: string; factTypes?: Array<"world" | "experience" | "mental_model">; diff --git a/packages/opencontext/src/cli/deprecate.ts b/packages/opencontext/src/cli/deprecate.ts new file mode 100644 index 0000000..82be4e8 --- /dev/null +++ b/packages/opencontext/src/cli/deprecate.ts @@ -0,0 +1,198 @@ +/** + * `opencontext deprecate` — soft-deprecate one or more raw messages + * directly via the active raw-message manager. + * + * Mirrors `opencontext add`'s shape (single-file CLI, JSON envelope on + * `--json`, no LLM roundtrip) but writes `deprecated_at`, + * `deprecation_reason`, and `superseded_by_summary_id` instead of + * inserting new rows. This is the supersession primitive that lets + * `search` return the current-truth version of a decision/fact: once + * the older rows are deprecated, `search --limit 1` returns the + * successor instead of tying on similarity and falling back to a + * lex `(type, id)` tiebreaker. + * + * Idempotent — re-running with the same ids returns 0 affected rows + * (the underlying SQL guards on `WHERE deprecated_at IS NULL`). + * + * Exit codes: + * 0 — at least one row was deprecated (or all inputs were already deprecated, with --json reporting 0) + * 1 — validation error, backend refused, or threw mid-call + * + * Output: + * default human-readable line: "deprecated N message(s): id1, id2" + * --json { ok, exit, count, ids, deprecatedAt, reason?, supersededBySummaryId? } + */ + +import { getRawMessageManager } from "@melandlabs/memory-store"; + +export interface DeprecateOptions { + userId: string; + ids: string[]; + reason?: string; + supersededBySummaryId?: string; + json: boolean; +} + +export interface DeprecateOutput { + ok: boolean; + exit: number; + count: number; + ids: string[]; + deprecatedAt: number; + reason?: string; + supersededBySummaryId?: string; + error?: string; +} + +const logPrefix = "[opencontext/deprecate]"; + +export function parseDeprecateArgs(argv: string[]): DeprecateOptions { + const opts: DeprecateOptions = { + userId: "", + ids: [], + json: false, + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + const next = argv[i + 1]; + const take = () => { + if (next === undefined) { + throw new Error(`${logPrefix} ${arg} requires a value`); + } + i += 1; + return next; + }; + + switch (arg) { + case "--user": + opts.userId = take(); + break; + case "--id": + opts.ids.push(take()); + break; + case "--reason": + opts.reason = take(); + break; + case "--superseded-by": + opts.supersededBySummaryId = take(); + break; + case "--json": + opts.json = true; + break; + case "--help": + case "-h": + printDeprecateHelp(); + process.exit(0); + break; + default: + throw new Error(`${logPrefix} unknown flag: ${arg}`); + } + } + + if (!opts.userId) { + // Mirror `opencontext add`'s default-to-"default" behaviour so ad-hoc + // single-tenant scripts keep working. Multi-user hosts should still + // pass `--user ` explicitly to avoid cross-tenant writes. + opts.userId = "default"; + } + if (opts.ids.length === 0) { + throw new Error(`${logPrefix} --id is required (repeatable)`); + } + return opts; +} + +export async function runDeprecate(opts: DeprecateOptions): Promise { + const manager = await getRawMessageManager(); + if (typeof manager.deprecateMessages !== "function") { + return emit( + opts, + { + ok: false, + exit: 1, + count: 0, + ids: opts.ids, + deprecatedAt: Date.now(), + reason: opts.reason, + supersededBySummaryId: opts.supersededBySummaryId, + error: "active raw-message manager exposes no deprecateMessages", + }, + "error: active raw-message manager exposes no deprecateMessages", + ); + } + + const deprecatedAt = Date.now(); + const count = await manager.deprecateMessages(opts.ids, { + userId: opts.userId, + deprecatedAt, + reason: opts.reason, + supersededBySummaryId: opts.supersededBySummaryId, + }); + const out: DeprecateOutput = { + ok: true, + exit: 0, + count, + ids: opts.ids, + deprecatedAt, + reason: opts.reason, + supersededBySummaryId: opts.supersededBySummaryId, + }; + const humanLine = + count === 0 + ? `no-op: ${opts.ids.length} id(s) already deprecated` + : `deprecated ${count} message${count === 1 ? "" : "s"}: ${opts.ids.join(", ")}`; + return emit(opts, out, humanLine); +} + +function emit(opts: DeprecateOptions, out: DeprecateOutput, humanLine: string): number { + if (opts.json) { + process.stdout.write(`${JSON.stringify(out)}\n`); + } else { + process.stdout.write(`${humanLine}\n`); + } + return out.exit; +} + +function printDeprecateHelp(): void { + console.log(`opencontext deprecate — soft-deprecate raw messages (supersession). + +Marks the given message ids as deprecated: the underlying +'raw_messages.deprecated_at' column is set (plus optional +'deprecation_reason' and 'superseded_by_summary_id'). Once deprecated, a +row is hidden from 'opencontext search' by default — opt back in with +'--include-deprecated' for audits of the supersession chain. + +This is the write-side complement of the existing +'RawMessageStorageManager.deprecateMessages' JS API. After running this +command, 'search --limit 1' for the same query returns the successor +instead of tying on similarity. + +Usage: + opencontext deprecate [options] + +Required: + --id Message id to deprecate (repeatable) + +Identity: + --user User / workspace id (default: "default") + +Supersession metadata: + --reason Free-form deprecation reason (stored in + 'deprecation_reason') + --superseded-by id of the successor message / summary (stored + in 'superseded_by_summary_id') + +Output: + --json Emit JSON envelope instead of a human line + +Examples: + # Mark an old decision as superseded by a new one + opencontext deprecate --user alice --id \\ + --reason "superseded by tRPC migration" --superseded-by + + # Bulk: deprecate every legacy row in a list + opencontext deprecate --user alice --id --id --reason "audit cleanup" + + # Script-friendly + opencontext deprecate --user alice --id --json`); +} diff --git a/packages/opencontext/src/cli/opencontext.ts b/packages/opencontext/src/cli/opencontext.ts index a681385..0cc13bb 100644 --- a/packages/opencontext/src/cli/opencontext.ts +++ b/packages/opencontext/src/cli/opencontext.ts @@ -35,6 +35,7 @@ import { parseOkfArgs, printOkfHelp, startOkf } from "@melandlabs/okf"; import { closeSQLiteVsaStore } from "@melandlabs/sqlite"; import { startHttpServer, startMcpServer } from "../index.js"; import { parseAddArgs, runAdd } from "./add.js"; +import { parseDeprecateArgs, runDeprecate } from "./deprecate.js"; import { parseDoctorArgs, runDoctor } from "./doctor.js"; import { parseListArgs, runList } from "./list.js"; import { parseSearchArgs, runSearch } from "./search.js"; @@ -284,14 +285,17 @@ Usage: opencontext [command] [options] Commands: - mcp Start the MCP server on stdio (default) - http Start the HTTP server - add Append a raw message to the active manager (no LLM roundtrip) - search Unified read with --mode {auto|lex|sem} and --context-only - list Browse raw messages by filter (newest first by default) - stats Report counts from the active raw-message store - doctor Run health checks against the local install - okf OKF v0.2 (Open Knowledge Format) importer / exporter + mcp Start the MCP server on stdio (default) + http Start the HTTP server + add Append a raw message to the active manager (no LLM roundtrip) + deprecate Soft-deprecate raw messages (supersession: hide from search + unless --include-deprecated is set; record --reason and + --superseded-by for the chain) + search Unified read with --mode {auto|lex|sem} and --context-only + list Browse raw messages by filter (newest first by default) + stats Report counts from the active raw-message store + doctor Run health checks against the local install + okf OKF v0.2 (Open Knowledge Format) importer / exporter Run "opencontext --help" for command-specific options. @@ -304,6 +308,8 @@ Examples: opencontext add --user alice --text "Rust achieves memory safety without GC" opencontext search --user alice --query "memory safety" --k 5 opencontext search --user alice --query "x" --context-only + opencontext deprecate --user alice --id --reason "superseded" --superseded-by + opencontext search --user alice --query "memory safety" --include-deprecated --json opencontext list --user alice --since 2026-08-01 --limit 20 opencontext stats --json | jq '.stats.totalMessages' opencontext doctor @@ -550,6 +556,10 @@ async function main(): Promise { process.exit(await runAdd(parseAddArgs(argv.slice(1)))); } + if (head === "deprecate" || head === "DEPRECATE") { + process.exit(await runDeprecate(parseDeprecateArgs(argv.slice(1)))); + } + if (head === "search" || head === "SEARCH") { process.exit(await runSearch(parseSearchArgs(argv.slice(1)))); } diff --git a/packages/opencontext/src/cli/search.test.ts b/packages/opencontext/src/cli/search.test.ts index 0e597fe..7057fbc 100644 --- a/packages/opencontext/src/cli/search.test.ts +++ b/packages/opencontext/src/cli/search.test.ts @@ -57,6 +57,7 @@ describe("parseSearchArgs", () => { contextOnly: false, json: false, explain: false, + includeDeprecated: false, }); }); @@ -106,6 +107,7 @@ describe("parseSearchArgs", () => { contextOnly: true, json: true, explain: true, + includeDeprecated: false, }); }); @@ -323,9 +325,44 @@ describe("runSearch", () => { expect(out).toContain("count=1"); expect(out).toContain("[0.870] memory @ "); expect(out).toContain("id: r1"); + // Per-fact provenance: hits without `metadata.source` fall back to em-dash + // so the prompt stays well-formed even when the upstream tier doesn't + // carry per-fact provenance. + expect(out).toContain("source: —"); expect(out).toContain("discussed the roadmap"); }); + it("--context-only surfaces per-fact metadata.source when present", async () => { + const { __mock } = await getMockStore(); + __mock.search.mockResolvedValueOnce( + makeOutput({ + results: [ + { + id: "r1", + type: "memory", + content: "We use tRPC", + similarity: 0.95, + metadata: { source: "meeting://2026-08-15" }, + }, + ], + evidence: [ + { + id: "r1", + source: "memory", + score: 0.95, + snippet: "We use tRPC", + timestamp: Date.parse("2026-08-15T10:00:00Z"), + metadata: { source: "meeting://2026-08-15" }, + }, + ], + }), + ); + + await runSearch(parseSearchArgs(["--user", "alice", "--query", "x", "--context-only"])); + const out = stdoutChunks.join(""); + expect(out).toContain("source: meeting://2026-08-15"); + }); + it("--json prints the full SearchOutput envelope", async () => { const { __mock } = await getMockStore(); __mock.search.mockResolvedValueOnce(makeOutput({ count: 2 })); @@ -409,4 +446,25 @@ describe("runSearch", () => { expect(parsed.exit).toBe(1); expect(parsed.error).toBe("backend down"); }); + + it("omits includeDeprecated from SearchInput by default", async () => { + const { __mock } = await getMockStore(); + __mock.search.mockResolvedValueOnce(makeOutput()); + + await runSearch(parseSearchArgs(["--user", "alice", "--query", "x"])); + const input = __mock.search.mock.calls[0]?.[0] as SearchInput; + // `includeDeprecated: true` is opt-in. Without the flag the SDK + // should not see the key, so current-truth behaviour is preserved + // for scripts that haven't opted in. + expect(input.includeDeprecated).toBeUndefined(); + }); + + it("--include-deprecated forwards includeDeprecated: true to the SDK", async () => { + const { __mock } = await getMockStore(); + __mock.search.mockResolvedValueOnce(makeOutput()); + + await runSearch(parseSearchArgs(["--user", "alice", "--query", "x", "--include-deprecated"])); + const input = __mock.search.mock.calls[0]?.[0] as SearchInput; + expect(input.includeDeprecated).toBe(true); + }); }); diff --git a/packages/opencontext/src/cli/search.ts b/packages/opencontext/src/cli/search.ts index 463a3a4..556ba92 100644 --- a/packages/opencontext/src/cli/search.ts +++ b/packages/opencontext/src/cli/search.ts @@ -44,6 +44,13 @@ export interface SearchOptions { contextOnly: boolean; json: boolean; explain: boolean; + /** + * Include messages that have been soft-deprecated via + * `opencontext deprecate` (or `deprecateMessages` directly). Default + * `false` keeps `current-truth` retrievals clean; set `true` to audit + * the supersession chain (REST → GraphQL → tRPC, etc.). + */ + includeDeprecated: boolean; } export interface SearchEnvelope { @@ -72,6 +79,7 @@ export function parseSearchArgs(argv: string[]): SearchOptions { contextOnly: false, json: false, explain: false, + includeDeprecated: false, }; for (let i = 0; i < argv.length; i += 1) { @@ -118,6 +126,9 @@ export function parseSearchArgs(argv: string[]): SearchOptions { case "--context-only": opts.contextOnly = true; break; + case "--include-deprecated": + opts.includeDeprecated = true; + break; case "--json": opts.json = true; break; @@ -179,6 +190,10 @@ export async function runSearch(opts: SearchOptions): Promise { sources: opts.mode === "sem" ? (["memory"] as const) : undefined, // Critical: --context-only must never spend an LLM call. synthesize: false, + // Forward supersession opt-in. Default false in SearchInput + // already; we only set it when the user explicitly opts in so + // existing scripts keep their current-truth behaviour. + ...(opts.includeDeprecated ? { includeDeprecated: true } : {}), }; let out: SearchOutput; @@ -233,6 +248,13 @@ function emitContextOnly(opts: SearchOptions, env: SearchEnvelope): number { const ts = ev.timestamp ? new Date(ev.timestamp).toISOString() : "—"; lines.push(`[${ev.score.toFixed(3)}] ${ev.source} @ ${ts}`); lines.push(` id: ${ev.id}`); + // Surface per-fact provenance when the hit carried `metadata.source` + // (set by `opencontext add --source `). Falls back to em-dash + // for legacy rows without provenance so the prompt stays + // well-formed. + const factSource = + typeof ev.metadata?.source === "string" && ev.metadata.source.length > 0 ? ev.metadata.source : "—"; + lines.push(` source: ${factSource}`); lines.push(` ${ev.snippet}`); lines.push(""); } @@ -296,6 +318,9 @@ Filtering: --kind Filter to one fact type (repeatable) --since Inclusive start date for memory timestamps --until Inclusive end date for memory timestamps + --include-deprecated Include messages soft-deprecated via + 'opencontext deprecate' (default: hide). Use + for audits of the supersession chain. Output: --json Emit full SearchOutput as JSON diff --git a/packages/sqlite/src/raw-message-manager.ts b/packages/sqlite/src/raw-message-manager.ts index 7d3555b..d687a36 100644 --- a/packages/sqlite/src/raw-message-manager.ts +++ b/packages/sqlite/src/raw-message-manager.ts @@ -1388,33 +1388,63 @@ export class SQLiteRawMessageManager implements RawMessageStorageManager { input: SQLiteRawMessageSemanticSearchInput, ): SQLiteRawMessageSemanticSearchResult[] { const limit = Math.max(1, Math.floor(input.limit ?? 10)); - const scanLimit = Math.min(4096, Math.max(limit * 4, Math.floor(input.scanLimit ?? limit * 10))); + // sqlite-vec rejects knn queries with k > 4096 ("k value in knn query + // too large"). The widest possible scan is therefore 4096; widening + // past it throws inside sqlite-vec. + const vecKnnMaxK = 4096; + // Bound retries so a heavily-deprecated corpus can't pin us in a + // widening loop. Three doublings (10x → 20x → 40x → 80x limit) + // comfortably out-scans the worst-case "every top-K row is deprecated" + // scenario for typical limits while still returning inside the hard cap. + const maxWideningAttempts = 3; + let currentScanLimit = Math.min( + vecKnnMaxK, + Math.max(limit * 4, Math.floor(input.scanLimit ?? limit * 10)), + ); const threshold = input.threshold ?? 0.7; - const rows = this.db - .prepare(` - SELECT chunk_id, distance - FROM ${this.getChildVectorTableName(input.queryEmbedding.length)} - WHERE embedding MATCH ? - ORDER BY distance - LIMIT ? - `) - .all(floatArrayToBuffer(input.queryEmbedding), scanLimit) as Array<{ - chunk_id: string; - distance: number; - }>; - const distances = new Map(rows.map((row) => [row.chunk_id, row.distance])); - const chunks = this.getSearchChunkRowsByIds(rows.map((row) => row.chunk_id)); - return this.hydrateSemanticChunkRows( - chunks.map((chunk) => ({ - chunk, - similarity: sqliteVectorDistanceToCosineSimilarity( - distances.get(chunk.chunk_id) ?? Number.POSITIVE_INFINITY, - ), - })), - input, - ) - .filter((result) => result.similarity >= threshold) - .slice(0, limit); + for (let attempt = 0; attempt <= maxWideningAttempts; attempt += 1) { + const rows = this.db + .prepare( + ` + SELECT chunk_id, distance + FROM ${this.getChildVectorTableName(input.queryEmbedding.length)} + WHERE embedding MATCH ? + ORDER BY distance + LIMIT ? + `, + ) + .all(floatArrayToBuffer(input.queryEmbedding), currentScanLimit) as Array<{ + chunk_id: string; + distance: number; + }>; + const distances = new Map(rows.map((row) => [row.chunk_id, row.distance])); + const chunks = this.getSearchChunkRowsByIds(rows.map((row) => row.chunk_id)); + const results = this.hydrateSemanticChunkRows( + chunks.map((chunk) => ({ + chunk, + similarity: sqliteVectorDistanceToCosineSimilarity( + distances.get(chunk.chunk_id) ?? Number.POSITIVE_INFINITY, + ), + })), + input, + ).filter((result) => result.similarity >= threshold); + + // If post-filtering (deprecated / archived / peer) ate enough rows + // to underflow `limit`, widen the vec scan and retry — mirroring + // the pattern in `searchMessagesWithVectorTable`. Two early-exit + // conditions: (a) we already have enough rows, (b) the vec scan + // returned fewer rows than we asked for, so widening cannot help. + if (results.length >= limit || rows.length < currentScanLimit) { + return results.slice(0, limit); + } + if (currentScanLimit >= vecKnnMaxK) { + return results.slice(0, limit); + } + currentScanLimit = Math.min(currentScanLimit * 2, vecKnnMaxK); + } + // Unreachable: the loop above always returns. Returning an empty slice + // keeps the signature honest in case the bounds ever change. + return []; } private searchChunksWithStoredEmbeddings( @@ -1943,7 +1973,11 @@ export class SQLiteRawMessageManager implements RawMessageStorageManager { const threshold = input.threshold ?? 0.7; // sqlite-vec rejects knn queries with k > 4096 ("k value in knn query // too large"), so the widening scan must stop there instead of - // doubling past the engine limit and throwing. + // doubling past the engine limit and throwing. `matchesSemanticFilters` + // post-filters deprecated / archived / peer-scoped rows, so when those + // dominate the top-K we widen the vec scan and retry to preserve + // `limit`. This is the parent-level mirror of the widen loop in + // `searchChunksWithVectorTable`; keep both in sync when tuning. const vecKnnMaxK = 4096; let currentScanLimit = scanLimit; while (true) {