Skip to content

refactor(sqlite): optimize vector-search, shard-manager, transcript-manager #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ZeR020 merged 3 commits into from
May 7, 2026
Merged

refactor(sqlite): optimize vector-search, shard-manager, transcript-manager #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
53c870a
refactor(sqlite): prepared statement caching, diversity filter optimi…
ZeR020 May 7, 2026
13cfae4
fix(sqlite): preserve token count behavior in optimization
ZeR020 May 7, 2026
28e6955
fix(sqlite): normalize diversity tokenization
ZeR020 May 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 24 additions & 31 deletions src/services/sqlite/shard-manager.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,29 @@ const METADATA_DB_NAME = "metadata.db";
export class ShardManager {
private metadataDb: DatabaseType;
private metadataPath: string;
private activeShardStmt: any;
private allShardsStmt: any;
private scopedShardsStmt: any;
private createShardStmt: any;

constructor() {
this.metadataPath = join(CONFIG.storagePath, METADATA_DB_NAME);
this.metadataDb = connectionManager.getConnection(this.metadataPath);
this.initMetadataDb();
this.activeShardStmt = this.metadataDb.prepare(`
SELECT * FROM shards WHERE scope = ? AND scope_hash = ? AND is_active = 1
ORDER BY shard_index DESC LIMIT 1
`);
this.allShardsStmt = this.metadataDb.prepare(`
SELECT * FROM shards WHERE scope = ? ORDER BY shard_index ASC
`);
this.scopedShardsStmt = this.metadataDb.prepare(`
SELECT * FROM shards WHERE scope = ? AND scope_hash = ? ORDER BY shard_index ASC
`);
this.createShardStmt = this.metadataDb.prepare(`
INSERT INTO shards (scope, scope_hash, shard_index, db_path, vector_count, is_active, created_at)
VALUES (?, ?, ?, ?, 0, 1, ?)
`);
}

private initMetadataDb(): void {
Expand Down Expand Up @@ -53,13 +71,7 @@ export class ShardManager {
}

getActiveShard(scope: "user" | "project", scopeHash: string): ShardInfo | null {
const stmt = this.metadataDb.prepare(`
SELECT * FROM shards
WHERE scope = ? AND scope_hash = ? AND is_active = 1
ORDER BY shard_index DESC LIMIT 1
`);

const row = stmt.get(scope, scopeHash) as any;
const row = this.activeShardStmt.get(scope, scopeHash) as any;
if (!row) return null;

return {
Expand All @@ -75,24 +87,10 @@ export class ShardManager {
}

getAllShards(scope: "user" | "project", scopeHash: string): ShardInfo[] {
let stmt;
let rows;

if (scopeHash === "") {
stmt = this.metadataDb.prepare(`
SELECT * FROM shards
WHERE scope = ?
ORDER BY shard_index ASC
`);
rows = stmt.all(scope) as any[];
} else {
stmt = this.metadataDb.prepare(`
SELECT * FROM shards
WHERE scope = ? AND scope_hash = ?
ORDER BY shard_index ASC
`);
rows = stmt.all(scope, scopeHash) as any[];
}
const rows =
scopeHash === ""
? (this.allShardsStmt.all(scope) as any[])
: (this.scopedShardsStmt.all(scope, scopeHash) as any[]);

return rows.map((row: any) => ({
id: row.id,
Expand All @@ -111,12 +109,7 @@ export class ShardManager {
const storedPath = join(`${scope}s`, basename(fullPath)).replace(/\\/g, "/");
const now = Date.now();

const stmt = this.metadataDb.prepare(`
INSERT INTO shards (scope, scope_hash, shard_index, db_path, vector_count, is_active, created_at)
VALUES (?, ?, ?, ?, 0, 1, ?)
`);

const result = stmt.run(scope, scopeHash, shardIndex, storedPath, now);
const result = this.createShardStmt.run(scope, scopeHash, shardIndex, storedPath, now);

const db = connectionManager.getConnection(fullPath);
this.initShardDb(db);
Expand Down
7 changes: 5 additions & 2 deletions src/services/sqlite/transcript-manager.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,12 @@ function getTranscriptDbPath(): string {
return join(CONFIG.storagePath, "transcripts.db");
}

const WORD_SPLIT_RE = /\s+/;

function approximateTokenCount(text: string): number {
const words = text.trim().split(/\s+/).filter(Boolean).length;
return Math.ceil(words * 1.33);
const trimmed = text.trim();
if (!trimmed) return 0;
return Math.ceil(trimmed.split(WORD_SPLIT_RE).length * 1.33);
}

export class TranscriptManager {
Expand Down
101 changes: 57 additions & 44 deletions src/services/sqlite/vector-search.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ function safeParseMetadata(raw: string | null | undefined): Record<string, unkno
export class VectorSearch {
private readonly backendPromise: Promise<VectorBackend>;
private readonly fallbackBackend: VectorBackend;
private readonly stmtCache = new WeakMap<DatabaseType, Map<string, any>>();

constructor(backend?: VectorBackend, fallbackBackend: VectorBackend = new ExactScanBackend()) {
this.backendPromise = backend
Expand All @@ -39,6 +40,20 @@ export class VectorSearch {
this.fallbackBackend = fallbackBackend;
}

private getStmt(db: DatabaseType, sql: string): any {
let dbCache = this.stmtCache.get(db);
if (!dbCache) {
dbCache = new Map();
this.stmtCache.set(db, dbCache);
}
let stmt = dbCache.get(sql);
if (!stmt) {
stmt = db.prepare(sql);
dbCache.set(sql, stmt);
}
return stmt;
}

private async getBackend(): Promise<VectorBackend> {
return this.backendPromise;
}
Expand All @@ -52,15 +67,18 @@ export class VectorSearch {
* @param shard - Optional shard info for vector backend indexing
*/
async insertVector(db: DatabaseType, record: MemoryRecord, shard?: ShardInfo): Promise<void> {
const insertMemory = db.prepare(`
const insertMemory = this.getStmt(
db,
`
INSERT INTO memories (
id, content, vector, tags_vector, container_tag, tags, type, created_at, updated_at,
metadata, display_name, user_name, user_email, project_path, project_name, git_repo_url,
recency_score, frequency_score, importance_score, utility_score, novelty_score,
confidence_score, interference_penalty, strength, access_count, last_accessed,
store_type, decay_rate
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
`
);

insertMemory.run(
record.id,
Expand Down Expand Up @@ -418,29 +436,36 @@ export class VectorSearch {
const diversityThreshold = CONFIG.retrieval.diversityThreshold || 0.9;
const finalResults: SearchResult[] = [];
const maxResults = CONFIG.retrieval.maxResults || limit;
const wordSetCache = new Map<string, Set<string>>();

function getWordSet(text: string): Set<string> {
let set = wordSetCache.get(text);
if (!set) {
set = new Set(
(text.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? []).filter((w) => w.length > 4)
);
wordSetCache.set(text, set);
}
return set;
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.

for (const candidate of allResults) {
if (finalResults.length >= maxResults) break;

// Only filter by diversity, don't re-penalize scores
let isDiverse = true;
const candidateWords = getWordSet(candidate.memory);
for (const selected of finalResults) {
// Simple text-based diversity check as fallback
const candidateWords = new Set(
candidate.memory
.toLowerCase()
.split(/\s+/)
.filter((w) => w.length > 4)
);
const selectedWords = new Set(
selected.memory
.toLowerCase()
.split(/\s+/)
.filter((w) => w.length > 4)
);
const intersection = [...candidateWords].filter((w) => selectedWords.has(w));
const union = new Set([...candidateWords, ...selectedWords]);
const jaccard = union.size > 0 ? intersection.length / union.size : 0;
const selectedWords = getWordSet(selected.memory);
let intersectionSize = 0;
const smallerSet =
candidateWords.size <= selectedWords.size ? candidateWords : selectedWords;
const largerSet =
candidateWords.size <= selectedWords.size ? selectedWords : candidateWords;
for (const word of smallerSet) {
if (largerSet.has(word)) intersectionSize++;
}
const unionSize = candidateWords.size + selectedWords.size - intersectionSize;
const jaccard = unionSize > 0 ? intersectionSize / unionSize : 0;

if (jaccard > diversityThreshold) {
isDiverse = false;
Expand All @@ -464,7 +489,7 @@ export class VectorSearch {
* @param shard - Optional shard info for backend index cleanup
*/
async deleteVector(db: DatabaseType, memoryId: string, shard?: ShardInfo): Promise<void> {
db.prepare(`DELETE FROM memories WHERE id = ?`).run(memoryId);
this.getStmt(db, `DELETE FROM memories WHERE id = ?`).run(memoryId);

if (shard) {
const backend = await this.getBackend();
Expand All @@ -489,7 +514,7 @@ export class VectorSearch {
shard?: ShardInfo,
tagsVector?: Float32Array
): Promise<void> {
db.prepare(`UPDATE memories SET vector = ?, tags_vector = ? WHERE id = ?`).run(
this.getStmt(db, `UPDATE memories SET vector = ?, tags_vector = ? WHERE id = ?`).run(
toBlob(vector),
toBlob(tagsVector),
memoryId
Expand All @@ -515,22 +540,11 @@ export class VectorSearch {
* @returns Raw database rows
*/
listMemories(db: DatabaseType, containerTag: string, limit: number): any[] {
const stmt = db.prepare(
const sql =
containerTag === ""
? `
SELECT * FROM memories
WHERE is_deprecated = 0
ORDER BY is_pinned DESC, strength DESC, recency_score DESC
LIMIT ?
`
: `
SELECT * FROM memories
WHERE container_tag = ? AND is_deprecated = 0
ORDER BY is_pinned DESC, strength DESC, recency_score DESC
LIMIT ?
`
);

? `SELECT * FROM memories WHERE is_deprecated = 0 ORDER BY is_pinned DESC, strength DESC, recency_score DESC LIMIT ?`
: `SELECT * FROM memories WHERE container_tag = ? AND is_deprecated = 0 ORDER BY is_pinned DESC, strength DESC, recency_score DESC LIMIT ?`;
const stmt = this.getStmt(db, sql);
return (containerTag === "" ? stmt.all(limit) : stmt.all(containerTag, limit)) as any[];
}

Expand All @@ -541,10 +555,10 @@ export class VectorSearch {
* @returns All memory rows ordered by creation time
*/
getAllMemories(db: DatabaseType): any[] {
const stmt = db.prepare(
return this.getStmt(
db,
`SELECT * FROM memories WHERE is_deprecated = 0 ORDER BY created_at DESC`
);
return stmt.all() as any[];
).all() as any[];
}

/**
Expand All @@ -555,8 +569,7 @@ export class VectorSearch {
* @returns The memory row, or null if not found
*/
getMemoryById(db: DatabaseType, memoryId: string): any | null {
const stmt = db.prepare(`SELECT * FROM memories WHERE id = ?`);
return stmt.get(memoryId) as any;
return this.getStmt(db, `SELECT * FROM memories WHERE id = ?`).get(memoryId) as any;
}

/**
Expand Down Expand Up @@ -591,10 +604,10 @@ export class VectorSearch {
* @returns Number of matching memories
*/
countVectors(db: DatabaseType, containerTag: string): number {
const stmt = db.prepare(
const result = this.getStmt(
db,
`SELECT COUNT(*) as count FROM memories WHERE container_tag = ? AND is_deprecated = 0`
);
const result = stmt.get(containerTag) as any;
).get(containerTag) as any;
return result.count;
}

Expand Down
Loading