Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions packages/indexeddb/src/storage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -382,13 +382,25 @@ export interface RawMessageSearchHooks {
threshold?: number;
botId?: string;
includeArchived?: boolean;
/**
* Forward `includeDeprecated` from the unified search input so
* supersession-aware callers can opt into deprecated rows for
* audits. Default `false` preserves the current-truth behaviour.
*/
includeDeprecated?: boolean;
}): Promise<RawMessageSemanticHit[]>;
lexicalSearch?(input: {
userId: string;
keywords: string[];
limit: number;
botId?: string;
includeArchived?: boolean;
/**
* Forward `includeDeprecated` from the unified search input so
* supersession-aware callers can opt into deprecated rows for
* audits. Default `false` preserves the current-truth behaviour.
*/
includeDeprecated?: boolean;
}): Promise<RawMessageLexicalHit[]>;
}

Expand Down
12 changes: 12 additions & 0 deletions packages/memory-store/src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,12 @@ export interface UnifiedSearchDeps {
limit: number;
threshold: number;
botId?: string;
/**
* Forward `includeDeprecated` from `UnifiedMemorySearchInput` so
* supersession-aware callers can opt into deprecated rows for
* audits. Default `false` preserves the current-truth behaviour.
*/
includeDeprecated?: boolean;
/** Optional peer scope resolved from `UnifiedMemorySearchInput.peerFilter`. */
peers?: ReadonlyArray<Peer>;
/** Optional `FactType` filter resolved from `UnifiedMemorySearchInput.factTypes`. */
Expand Down Expand Up @@ -282,6 +288,12 @@ export interface UnifiedSearchDeps {
keywords: string[];
limit: number;
botId?: string;
/**
* Forward `includeDeprecated` from `UnifiedMemorySearchInput` so
* supersession-aware callers can opt into deprecated rows for
* audits. Default `false` preserves the current-truth behaviour.
*/
includeDeprecated?: boolean;
/** Optional peer scope resolved from `UnifiedMemorySearchInput.peerFilter`. */
peers?: ReadonlyArray<Peer>;
/** Optional `FactType` filter resolved from `UnifiedMemorySearchInput.factTypes`. */
Expand Down
29 changes: 25 additions & 4 deletions packages/memory-store/src/search/gather-evidence.ts
Original file line number Diff line number Diff line change
Expand Up @@ -396,10 +396,24 @@ export async function gatherEvidence(opts: GatherOptions): Promise<GatherResult>
* Build the prompt the LLM receives when `synthesize: true` is set.
* Mirrors the legacy `reflect.ts` prompt format so callers see identical
* behaviour before and after the merge.
*
* Per-fact provenance: when an evidence item carries `metadata.source`
* (e.g. `"meeting://2026-08-15"` written by `opencontext add --source`),
* the line format surfaces it as `source=<uri>` so the LLM can attribute
* claims to the originating fact. Items without a `source` metadata field
* fall back to the channel-level `source` (e.g. `"memory"`, `"insight"`)
* and finally `"unknown"`, so the prompt stays well-formed even when an
* upstream tier doesn't carry per-fact provenance.
*/
export function buildSynthesisPrompt(input: {
query: string;
evidence: Array<{ id: string; source: SearchTier; snippet: string; score: number }>;
evidence: Array<{
id: string;
source: SearchTier;
snippet: string;
score: number;
metadata?: Record<string, unknown>;
}>;
responseSchema?: Record<string, unknown>;
}): string {
const grouped = new Map<SearchTier, typeof input.evidence>();
Expand All @@ -415,9 +429,13 @@ export function buildSynthesisPrompt(input: {
if (items.length === 0) {
continue;
}
const lines = items.map(
(item, index) => ` [${index + 1}] (${item.id}, score=${item.score.toFixed(4)}) ${item.snippet}`,
);
const lines = items.map((item, index) => {
const factSource =
typeof item.metadata?.source === "string" && item.metadata.source.length > 0
? item.metadata.source
: (item.source ?? "unknown");
return ` [${index + 1}] (${item.id}, source=${factSource}, score=${item.score.toFixed(4)}) ${item.snippet}`;
});
sections.push(`## ${tier}\n${lines.join("\n")}`);
}

Expand Down Expand Up @@ -512,6 +530,9 @@ export async function synthesizeAnswer(input: {
source: mapEvidenceSourceToTier(item.source),
snippet: item.snippet,
score: item.score,
// Forward per-fact provenance (e.g. `metadata.source` from
// `opencontext add --source`) so the synthesis prompt can cite it.
metadata: item.metadata,
}));
const warnings: UnifiedMemorySearchWarning[] = [];

Expand Down
9 changes: 9 additions & 0 deletions packages/memory-store/src/search/unified-search.ts
Original file line number Diff line number Diff line change
Expand Up @@ -328,6 +328,7 @@ async function runSemanticSearchForEmbedding(
if (typeof deps.searchRawMessagesAnn === "function") {
const searchRawMessagesAnn = deps.searchRawMessagesAnn;
const factTypes = input.factTypes?.length ? input.factTypes : undefined;
const includeDeprecated = input.includeDeprecated === true;
semantic = (
await Promise.all(
filters.map((filter) =>
Expand All @@ -337,6 +338,7 @@ async function runSemanticSearchForEmbedding(
limit,
threshold,
botId: "botId" in filter ? filter.botId : undefined,
includeDeprecated,
...(peerPeers.length > 0 ? { peers: peerPeers } : {}),
...(factTypes ? { factTypes } : {}),
...(runtimeContext ?? {}),
Expand Down Expand Up @@ -394,6 +396,7 @@ async function runSemanticSearchForEmbedding(
queryEmbedding,
limit,
threshold,
includeDeprecated: input.includeDeprecated === true,
...(peerPeers.length > 0 ? { peers: peerPeers } : {}),
...(factTypes ? { factTypes } : {}),
});
Expand Down Expand Up @@ -481,6 +484,7 @@ async function runLexicalSearchForKeywords(
const filters = input.botIds && input.botIds.length > 0 ? input.botIds : [undefined];
const searchRawMessagesLexical = deps.searchRawMessagesLexical;
const factTypes = input.factTypes?.length ? input.factTypes : undefined;
const includeDeprecated = input.includeDeprecated === true;
return (
await Promise.all(
filters.map((botId) =>
Expand All @@ -489,6 +493,7 @@ async function runLexicalSearchForKeywords(
keywords,
limit: Math.ceil(limit / filters.length),
botId,
includeDeprecated,
...(peerPeers.length > 0 ? { peers: peerPeers } : {}),
...(factTypes ? { factTypes } : {}),
...(runtimeContext ?? {}),
Expand All @@ -515,6 +520,7 @@ async function runLexicalSearchForKeywords(
const { lexicalSearchRawMessages } = await import("../storage/sqlite-raw-message-store");
const filters = input.botIds && input.botIds.length > 0 ? input.botIds : [undefined];
const factTypes = input.factTypes?.length ? input.factTypes : undefined;
const includeDeprecated = input.includeDeprecated === true;
return (
await Promise.all(
filters.map((botId) =>
Expand All @@ -523,6 +529,7 @@ async function runLexicalSearchForKeywords(
keywords,
limit: Math.ceil(limit / filters.length),
botId,
includeDeprecated,
...(factTypes ? { factTypes } : {}),
}),
),
Expand Down Expand Up @@ -582,6 +589,7 @@ function searchInputToUnified(input: SearchInput): UnifiedMemorySearchInput {
reasoningStrategy: input.reasoningStrategy,
factTypes: input.factTypes,
includeRetrievalDiagnostics: input.includeRetrievalDiagnostics,
includeDeprecated: input.includeDeprecated,
};
}

Expand Down Expand Up @@ -1416,6 +1424,7 @@ export function createUnifiedSearch(deps: UnifiedSearchDeps = {}): UnifiedSearch
snippet: hit.content,
score: hit.similarity,
timestamp: getCandidateTimestamp(hit.metadata),
metadata: hit.metadata,
}));
const base: SearchOutput = {
query,
Expand Down
24 changes: 24 additions & 0 deletions packages/memory-store/src/search/utilities.ts
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,15 @@ export interface UnifiedMemorySearchInput {
factTypes?: FactType[];
/** Include pre-fusion channel candidates in the response for diagnostics. */
includeRetrievalDiagnostics?: boolean;
/**
* Include messages that have been soft-deprecated
* (`raw_messages.deprecated_at IS NOT NULL`). Default `false` keeps
* `current-truth` retrievals clean: a row that was marked superseded
* via `deprecateMessages` (or the `opencontext deprecate` CLI) is
* excluded from results. Set to `true` for audits and historical
* exploration.
*/
includeDeprecated?: boolean;
}

export type UnifiedMemoryMergeStrategy = "similarity" | "rrf";
Expand Down Expand Up @@ -321,6 +330,15 @@ export interface SearchInput {
includeArchivedInsights?: boolean;
/** Include pre-fusion retrieval candidates; intended for evaluation/debugging. */
includeRetrievalDiagnostics?: boolean;
/**
* Include messages that have been soft-deprecated
* (`raw_messages.deprecated_at IS NOT NULL`). Default `false` keeps
* `current-truth` retrievals clean: a row that was marked superseded
* via `deprecateMessages` (or the `opencontext deprecate` CLI) is
* excluded from results. Set to `true` for audits and historical
* exploration.
*/
includeDeprecated?: boolean;
}

export interface SearchEvidence {
Expand All @@ -329,6 +347,12 @@ export interface SearchEvidence {
snippet: string;
score: number;
timestamp?: number;
/**
* Forwarded from the underlying hit's `metadata`. Surfaces per-fact
* provenance (`metadata.source`, `metadata.factType`, etc.) to the
* synthesis prompt and the CLI's `--context-only` / `--json` output.
*/
metadata?: Record<string, unknown>;
}

export interface SearchOutput {
Expand Down
12 changes: 12 additions & 0 deletions packages/memory-store/src/storage/raw-message-store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@ export type RawMessageStorageManagerWithSearch = RawMessageStorageManager & {
scanLimit?: number;
threshold?: number;
includeArchived?: boolean;
/**
* Forward `includeDeprecated` from the unified search input so
* supersession-aware callers can opt into deprecated rows for
* audits. Default `false` preserves the current-truth behaviour.
*/
includeDeprecated?: boolean;
platform?: string;
botId?: string;
channel?: string;
Expand All @@ -52,6 +58,12 @@ export type RawMessageStorageManagerWithSearch = RawMessageStorageManager & {
keywords: string[];
limit?: number;
includeArchived?: boolean;
/**
* Forward `includeDeprecated` from the unified search input so
* supersession-aware callers can opt into deprecated rows for
* audits. Default `false` preserves the current-truth behaviour.
*/
includeDeprecated?: boolean;
platform?: string;
botId?: string;
}) => Promise<unknown[]>;
Expand Down
6 changes: 6 additions & 0 deletions packages/memory-store/src/storage/sqlite-raw-message-store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,12 @@ export async function lexicalSearchRawMessages(input: {
keywords: string[];
limit?: number;
includeArchived?: boolean;
/**
* Forward `includeDeprecated` from `UnifiedMemorySearchInput` so
* supersession-aware callers can opt into deprecated rows for
* audits. Default `false` preserves the current-truth behaviour.
*/
includeDeprecated?: boolean;
platform?: string;
botId?: string;
factTypes?: Array<"world" | "experience" | "mental_model">;
Expand Down
Loading
Loading