diff --git a/README.md b/README.md index f2f8c35..265bc5b 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ katok search bm25 "지난주 미팅 자료" --limit 30 --json 각 `search` 명령은 `--limit `(기본 10)으로 반환할 결과 개수를 조절할 수 있습니다. -검색 최신성이 중요하면 검색 전에 항상 `katok doctor --json`의 `freshness`를 확인하세요. 이 기본 doctor는 macOS app data probe를 실행하지 않으므로 권한 prompt 없이 사용할 수 있습니다. `sync_before_search`가 `true`이면 `katok sync --source macos --json`을 먼저 실행하고, `index_before_semantic_search`가 `true`이면 `katok index --json`을 실행한 뒤 semantic search를 사용합니다. +검색 최신성이 중요하면 검색 전에 항상 `katok doctor --json`의 `freshness`를 확인하세요. 이 기본 doctor는 macOS app data probe를 실행하지 않으므로 권한 prompt 없이 사용할 수 있습니다. `sync_before_search`가 `true`이면 `katok sync --source macos --json`을 먼저 실행하고, `index_before_semantic_search`가 `true`이면 `katok index --json`을 실행한 뒤 semantic search를 사용합니다. doctor와 semantic search는 archive revision을 현재 committed index generation과 비교하므로, sync 뒤 index가 오래됐거나 vector ID가 archive와 어긋나면 검색 전에 명시적으로 재인덱싱을 요구합니다. 검색 결과에서 더 넓은 맥락이 필요하면 chunk 명령을 사용합니다. @@ -173,8 +173,12 @@ katok media backfill --kind file --kind video --json `katok search bm25`는 SQLite FTS5 BM25 랭킹을 사용합니다. 여러 단어가 섞인 일반 질의에 적합합니다. +BM25 입력은 FTS5 연산식이 아니라 일반 검색어로 처리됩니다. `+`, `-`, 따옴표, 괄호 같은 문자가 포함되어도 문자 그대로 tokenizer에 전달되며 FTS5 column filter나 boolean 문법으로 실행되지 않습니다. + `katok search semantic`은 EmbeddingGemma로 만든 로컬 벡터 인덱스를 사용합니다. 표현이 정확히 기억나지 않아도 의미가 비슷한 대화를 찾을 수 있습니다. +`katok index`는 새 generation을 완전히 만든 뒤 `CURRENT` 포인터를 원자적으로 교체합니다. 실패하면 이전 generation이 그대로 유지되고 명령은 non-zero로 끝납니다. `--full`은 기존 vector를 재사용하지 않는 완전 rebuild이고, 기본 index는 healthy generation의 동일 vector만 재사용하며 무결성 불일치가 있으면 archive에서 self-heal합니다. + ## EmbeddingGemma 로컬 벡터 검색 `katok index`는 기본값으로 `embeddinggemma-300m-q4`를 앱 프로세스 안에서 실행합니다. diff --git a/skills/katok/SKILL.md b/skills/katok/SKILL.md index ce1ba60..9bb4caf 100644 --- a/skills/katok/SKILL.md +++ b/skills/katok/SKILL.md @@ -68,6 +68,7 @@ environment variable; setting one is silently ignored and the run writes into th 5. Run `katok sync --source macos --json` when `freshness.recommendation.sync_before_search` is `true`, when the user asks for recent messages, or when search freshness matters. 6. Run `katok index --json` before semantic search when `freshness.recommendation.index_before_semantic_search` is `true` or after a sync that should affect vector search. 7. Use `katok search keyword ...`, `katok search bm25 ...`, and `katok search semantic ...` for discovery. + - A search returns a global top-k across rooms (default 10). For a person or topic that may occur across many rooms, raise `--limit` (for example 100) and group the returned hits by the stable `chat_id`; a room outside the first 10 is not an engine omission. 8. Use `katok chunk get ...` only for explicit retrieval. 9. Run `katok doctor --macos-probe --json` only for setup or permission diagnostics, because it may trigger a macOS "access data from other apps" prompt. @@ -94,7 +95,7 @@ Use `katok chunk context --json` to inspect the immediate previous an `katok index` runs the local `embeddinggemma-300m-q4` embedder in-process by default. Do not ask the user to start a Python, Jina, TEI, or local HTTP embedding server. Use `KATOK_EMBEDDER=mock` only for synthetic QA and `KATOK_EMBEDDER=local-test` only when you need deterministic local vector tests without downloading the model. -The index never follows the KakaoTalk database on its own, so a search only ever reflects the last sync. Run `katok sync --source macos --json` before the first query of a session and before any question about recent messages. Skipping it does not return zero results, it silently returns a stale set. Freshness also depends on KakaoTalk itself running (`pgrep -x KakaoTalk`), because the source database receives new messages only while the app is up. +The index never follows the KakaoTalk database on its own, so a search only ever reflects the last sync. Run `katok sync --source macos --json` before the first query of a session and before any question about recent messages. After sync, doctor compares the archive revision with the committed semantic generation and requests indexing when they differ; semantic search also refuses stale or orphaned generations instead of silently searching them. Freshness also depends on KakaoTalk itself running (`pgrep -x KakaoTalk`), because the source database receives new messages only while the app is up. Sync is cheap enough to run often because only chats whose messages changed have their chunk tails recomputed. Three runs still pay the full cost: the first sync on an empty archive, the first sync after `chunk_gap_group_seconds` or `chunk_gap_direct_seconds` changes, and the first sync after an upgrade that bumps the chunker version, which includes the first run against an archive written before the version was recorded. Each of these invalidates every stored chunk once, after which sync returns to the incremental path. The payload reports `rebuilt_chats` and a `timings_ms` breakdown (`read_source`, `upsert_messages`, `rebuild_chunks`), so a slow run can be attributed to a stage instead of guessed at. diff --git a/src/cli.rs b/src/cli.rs index a94105e..2b57f4c 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -46,6 +46,7 @@ pub(crate) enum Commands { touched: bool, }, Index { + /// Rebuild every vector instead of reusing unchanged vectors from the committed generation. #[arg(long)] full: bool, #[arg(long)] diff --git a/src/commands.rs b/src/commands.rs index a20b2fe..59c0ac8 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -9,7 +9,7 @@ use katok::{ }, config::KatokConfig, search::{bm25_search_with_snippet, keyword_search_with_snippet}, - semantic::{semantic_search_live_with_config, semantic_search_with_snippet}, + semantic::semantic_search_live_with_config, transcript::export_transcript, types::SyncTimings, }; @@ -23,6 +23,39 @@ mod media_commands; mod permissions; mod source_adapter; +pub(crate) fn command_requests_json(command: &Commands) -> bool { + match command { + Commands::Doctor { json, .. } + | Commands::Sync { json, .. } + | Commands::Index { json, .. } + | Commands::WipeIndex { json, .. } + | Commands::Chunks { json, .. } + | Commands::Transcript { json, .. } => *json, + Commands::Search { command } => match command { + SearchCommand::Keyword { json, .. } + | SearchCommand::Bm25 { json, .. } + | SearchCommand::Semantic { json, .. } => *json, + }, + Commands::Chunk { command } => match command { + crate::cli::ChunkCommand::Get { json, .. } + | crate::cli::ChunkCommand::Context { json, .. } + | crate::cli::ChunkCommand::Parent { json, .. } => *json, + }, + Commands::Source { command } => match command { + SourceCommand::Chats { json, .. } => *json, + }, + Commands::Media { command } => match command { + crate::cli::MediaCommand::Get { json, .. } + | crate::cli::MediaCommand::Backfill { json, .. } => *json, + }, + Commands::Permissions { command } => match command { + PermissionsCommand::Macos { json, .. } => *json, + }, + #[cfg(all(target_os = "macos", feature = "private-send"))] + Commands::Send { json, .. } => *json, + } +} + pub(crate) fn run( command: Commands, config: KatokConfig, @@ -312,7 +345,7 @@ fn run_doctor( "data_dir": data_dir, "archive": archive_path, "semantic_index": semantic_dir, - "freshness": freshness::load(&data_dir)?, + "freshness": freshness::load(&data_dir, &archive_path, &semantic_dir)?, "local_first": true, "macos": cfg!(target_os = "macos"), "source_adapter": { @@ -520,30 +553,19 @@ fn run_search( print_payload(json, &hits) } SearchCommand::Semantic { query, limit, json } => { - let hits = if std::env::var("KATOK_EMBEDDER").unwrap_or_default() == "mock" { - semantic_search_with_snippet( + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .context("create semantic runtime")?; + let hits = runtime + .block_on(semantic_search_live_with_config( &archive, semantic_dir, &query, limit, - config.snippet_length, - ) - .context("semantic search")? - } else { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .context("create semantic runtime")?; - runtime - .block_on(semantic_search_live_with_config( - &archive, - semantic_dir, - &query, - limit, - config, - )) - .context("semantic search")? - }; + config, + )) + .context("semantic search")?; print_payload(json, &hits) } } diff --git a/src/commands/freshness.rs b/src/commands/freshness.rs index d4a5448..4d98b22 100644 --- a/src/commands/freshness.rs +++ b/src/commands/freshness.rs @@ -29,6 +29,8 @@ pub(crate) struct IndexFreshness { pub(crate) vectorstore: String, pub(crate) semantic_units: String, pub(crate) embedded_texts: usize, + #[serde(default)] + pub(crate) archive_revision: String, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -49,15 +51,33 @@ impl Default for FreshnessRecommendation { } } -pub(crate) fn load(data_dir: &Path) -> Result { +pub(crate) fn load( + data_dir: &Path, + archive_path: &Path, + semantic_dir: &Path, +) -> Result { let path = status_path(data_dir); - if !path.exists() { - return Ok(FreshnessStatus::default()); - } - let bytes = std::fs::read(&path).context("read freshness status")?; - let mut status: FreshnessStatus = - serde_json::from_slice(&bytes).context("parse freshness status")?; - status.recommendation = recommendation(&status); + let mut status = if path.exists() { + let bytes = std::fs::read(&path).context("read freshness status")?; + serde_json::from_slice(&bytes).context("parse freshness status")? + } else { + FreshnessStatus::default() + }; + let committed_cursor = katok::semantic::committed_cursor(semantic_dir); + let committed_error = match &committed_cursor { + Err(katok::Error::SemanticIndexMissing) => None, + Err(error) => Some(error.to_string()), + Ok(_) => None, + }; + status.last_index = committed_cursor.ok().map(|cursor| IndexFreshness { + completed_at: cursor.completed_at, + embedder: cursor.embedder_id, + vectorstore: cursor.vectorstore, + semantic_units: cursor.semantic_units, + embedded_texts: cursor.embedded_texts, + archive_revision: cursor.archive_revision, + }); + status.recommendation = recommendation(&status, archive_path, committed_error.as_deref()); Ok(status) } @@ -67,43 +87,86 @@ pub(crate) fn record_sync( total_messages: usize, chunks: usize, ) -> Result<()> { - let mut status = load(data_dir)?; + let mut status = load_raw(data_dir)?; status.last_sync = Some(SyncFreshness { completed_at: chrono::Utc::now().to_rfc3339(), source: source.to_string(), total_messages, chunks, }); - status.recommendation = recommendation(&status); - save(data_dir, &status) -} - -pub(crate) fn record_index( - data_dir: &Path, - embedder: &str, - vectorstore: &str, - semantic_units: &str, - embedded_texts: usize, -) -> Result<()> { - let mut status = load(data_dir)?; - status.last_index = Some(IndexFreshness { - completed_at: chrono::Utc::now().to_rfc3339(), - embedder: embedder.to_string(), - vectorstore: vectorstore.to_string(), - semantic_units: semantic_units.to_string(), - embedded_texts, - }); - status.recommendation = recommendation(&status); + status.recommendation = recommendation_from_records(&status); save(data_dir, &status) } fn save(data_dir: &Path, status: &FreshnessStatus) -> Result<()> { katok::paths::ensure_private_dir(data_dir).context("create data directory")?; let bytes = serde_json::to_vec_pretty(status).context("serialize freshness status")?; - std::fs::write(status_path(data_dir), bytes).context("write freshness status") + let path = status_path(data_dir); + let temporary = data_dir.join(format!(".status.json.{}", std::process::id())); + std::fs::write(&temporary, bytes).context("write freshness status staging file")?; + std::fs::rename(&temporary, path).context("publish freshness status") +} + +fn load_raw(data_dir: &Path) -> Result { + let path = status_path(data_dir); + if !path.exists() { + return Ok(FreshnessStatus::default()); + } + let bytes = std::fs::read(&path).context("read freshness status")?; + serde_json::from_slice(&bytes).context("parse freshness status") +} + +fn recommendation( + status: &FreshnessStatus, + archive_path: &Path, + committed_error: Option<&str>, +) -> FreshnessRecommendation { + let base = recommendation_from_records(status); + if base.sync_before_search { + return base; + } + if let Some(error) = committed_error { + return FreshnessRecommendation { + sync_before_search: false, + index_before_semantic_search: true, + reason: format!( + "semantic index is corrupt ({error}); run katok index --json before semantic search" + ), + }; + } + if status.last_index.is_none() { + return base; + } + if !archive_path.is_file() { + return FreshnessRecommendation { + sync_before_search: true, + index_before_semantic_search: true, + reason: "archive is missing; run katok sync --source macos --json, then katok index --json before search".to_string(), + }; + } + let current_revision = katok::archive::Archive::open(archive_path) + .and_then(|archive| katok::semantic::archive_revision(&archive)); + let committed_revision = status + .last_index + .as_ref() + .map(|index| index.archive_revision.as_str()); + match (current_revision, committed_revision) { + (Ok(current), Some(committed)) if current == committed => base, + (Ok(_), Some(_)) => FreshnessRecommendation { + sync_before_search: false, + index_before_semantic_search: true, + reason: "archive revision is newer than the committed semantic index; run katok index --json before semantic search".to_string(), + }, + (Err(error), Some(_)) => FreshnessRecommendation { + sync_before_search: false, + index_before_semantic_search: true, + reason: format!("archive revision could not be read ({error}); repair the archive before semantic search"), + }, + (_, None) => base, + } } -fn recommendation(status: &FreshnessStatus) -> FreshnessRecommendation { +fn recommendation_from_records(status: &FreshnessStatus) -> FreshnessRecommendation { let sync_before_search = status.last_sync.is_none(); let index_before_semantic_search = status.last_index.is_none(); let reason = if sync_before_search { diff --git a/src/commands/index_commands.rs b/src/commands/index_commands.rs index afe136d..7598f0e 100644 --- a/src/commands/index_commands.rs +++ b/src/commands/index_commands.rs @@ -1,10 +1,9 @@ -use crate::commands::freshness; use crate::support::print_payload; use anyhow::{Context, Result}; use katok::{ archive::Archive, config::KatokConfig, - semantic::{index_semantic_live, planned_semantic_documents, write_semantic_documents}, + semantic::{index_semantic_live, planned_semantic_documents}, }; use std::path::Path; @@ -15,16 +14,14 @@ pub(crate) fn run( config: &KatokConfig, archive_path: &Path, semantic_dir: &Path, - data_dir: &Path, + _data_dir: &Path, ) -> Result<()> { + if !archive_path.is_file() { + anyhow::bail!("archive is missing; run katok sync before katok index"); + } let archive = Archive::open(archive_path).context("open archive")?; let chunks = archive.all_chunks().context("load chunks")?; - let documents = planned_semantic_documents(&archive, semantic_dir).context("plan documents")?; - let written = if dry_run { - 0 - } else if std::env::var("KATOK_EMBEDDER").unwrap_or_default() == "mock" { - write_semantic_documents(&archive, semantic_dir).context("write semantic documents")? - } else { + if !dry_run { return run_live_index(LiveIndexInput { full, dry_run, @@ -32,30 +29,20 @@ pub(crate) fn run( config, archive: &archive, semantic_dir, - data_dir, candidate_chunks: chunks.len(), - documents, }); - }; + } + let documents = planned_semantic_documents(&archive, semantic_dir).context("plan documents")?; let payload = serde_json::json!({ "full": full, "dry_run": dry_run, "candidate_chunks": chunks.len(), - "written_documents": written, - "embedding_calls": if dry_run { 0 } else { chunks.len() }, + "written_documents": 0, + "embedding_calls": 0, "documents": documents, "embedder": config.embedder_model, "semantic_units": "parent_windows" }); - if !dry_run { - freshness::record_index( - data_dir, - &config.embedder_model, - "documents", - "parent_windows", - chunks.len(), - )?; - } print_payload(json, &payload) } @@ -66,9 +53,7 @@ struct LiveIndexInput<'a> { config: &'a KatokConfig, archive: &'a Archive, semantic_dir: &'a Path, - data_dir: &'a Path, candidate_chunks: usize, - documents: Vec, } fn run_live_index(input: LiveIndexInput<'_>) -> Result<()> { @@ -81,15 +66,13 @@ fn run_live_index(input: LiveIndexInput<'_>) -> Result<()> { input.archive, input.semantic_dir, input.config, + input.full, )) .context("index semantic documents")?; - freshness::record_index( - input.data_dir, - report.embedder, - report.vectorstore, - report.semantic_units, - report.embedded_texts, - )?; + let generation = katok::semantic::current_generation(input.semantic_dir) + .context("resolve committed semantic generation")?; + let documents = planned_semantic_documents(input.archive, &generation) + .context("report semantic documents")?; let payload = serde_json::json!({ "full": input.full, "dry_run": input.dry_run, @@ -97,10 +80,14 @@ fn run_live_index(input: LiveIndexInput<'_>) -> Result<()> { "written_documents": report.written_documents, "embedding_calls": report.embedding_calls, "embedded_texts": report.embedded_texts, - "documents": input.documents, + "documents": documents, "embedder": report.embedder, "vectorstore": report.vectorstore, - "semantic_units": report.semantic_units + "semantic_units": report.semantic_units, + "archive_revision": report.archive_revision, + "reused_vectors": report.reused_vectors, + "self_healed": report.self_healed, + "cleanup_warnings": report.cleanup_warnings }); print_payload(input.json, &payload) } diff --git a/src/lib.rs b/src/lib.rs index e8f69bd..b7eff4e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -37,6 +37,10 @@ pub enum Error { MissingChunk(String), #[error("semantic index has never been synced")] SemanticIndexMissing, + #[error("semantic index is stale or corrupt: {0}; re-run katok index")] + SemanticIndexStale(String), + #[error("semantic index rebuild is already running: {0}")] + SemanticIndexBusy(std::path::PathBuf), #[error("invalid semantic path: {0}")] InvalidSemanticPath(std::path::PathBuf), #[error("unsupported source adapter: {0}")] diff --git a/src/main.rs b/src/main.rs index 78efd65..413f84d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,8 +10,34 @@ mod cli; mod commands; mod support; -fn main() -> Result<()> { +fn main() { let cli = Cli::parse(); + let json = commands::command_requests_json(&cli.command); + if let Err(error) = run(cli) { + if json { + let code = error_code(&error); + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "ok": false, + "error": { + "code": code, + "message": error.to_string(), + "cause": format!("{error:#}") + } + })) + .unwrap_or_else(|_| { + "{\"ok\":false,\"error\":{\"code\":\"serialization_failed\"}}".to_string() + }) + ); + } else { + eprintln!("Error: {error:#}"); + } + std::process::exit(1); + } +} + +fn run(cli: Cli) -> Result<()> { let config = KatokConfig::load(cli.config.as_deref()).context("load config")?; let data_dir = match cli.data_dir { Some(path) => path, @@ -27,3 +53,14 @@ fn main() -> Result<()> { commands::run(cli.command, config, data_dir, archive_path, semantic_dir) } + +fn error_code(error: &anyhow::Error) -> &'static str { + match error.downcast_ref::() { + Some(katok::Error::SemanticIndexMissing) => "semantic_index_missing", + Some(katok::Error::SemanticIndexStale(_)) => "semantic_index_stale", + Some(katok::Error::SemanticIndexBusy(_)) => "semantic_index_busy", + Some(katok::Error::EmptyQuery) => "empty_query", + Some(katok::Error::Sql(_)) => "sqlite_error", + _ => "command_failed", + } +} diff --git a/src/search.rs b/src/search.rs index 17fc7f3..1b9c9f3 100644 --- a/src/search.rs +++ b/src/search.rs @@ -47,6 +47,7 @@ pub fn bm25_search_with_snippet( if query.trim().is_empty() { return Err(Error::EmptyQuery); } + let fts_query = literal_fts_query(query); let mut stmt = archive .connection() .prepare( @@ -59,7 +60,7 @@ pub fn bm25_search_with_snippet( ) .map_err(Error::Sql)?; let ids = stmt - .query_map(params![query.trim(), limit as i64], |row| { + .query_map(params![fts_query, limit as i64], |row| { row.get::<_, String>(0) }) .map_err(Error::Sql)? @@ -68,6 +69,14 @@ pub fn bm25_search_with_snippet( hydrate_hits(archive, ids, "bm25", query, snippet_length) } +fn literal_fts_query(query: &str) -> String { + query + .split_whitespace() + .map(|term| format!("\"{}\"", term.replace('"', "\"\""))) + .collect::>() + .join(" ") +} + pub(crate) fn hydrate_hits( archive: &Archive, ids: Vec, @@ -84,6 +93,7 @@ pub(crate) fn hydrate_hits( unit: "micro_chunk", rank: idx + 1, chunk_id: chunk.chunk_id, + chat_id: chunk.chat_id, chat_name: chunk.chat_name, sender_nickname: chunk.sender_nickname, started_at: chunk.started_at, @@ -115,6 +125,7 @@ pub(crate) fn hydrate_parent_hits( unit: "parent_window", rank: idx + 1, chunk_id: parent.parent_id, + chat_id: parent.chat_id, chat_name: parent.chat_name, sender_nickname: "multiple".to_string(), started_at: parent.started_at, diff --git a/src/semantic.rs b/src/semantic.rs index 7578958..dda4e73 100644 --- a/src/semantic.rs +++ b/src/semantic.rs @@ -3,8 +3,13 @@ mod live; mod mock; mod store; +use crate::{archive::Archive, Error, Result}; +use sha2::{Digest, Sha256}; +use std::path::{Path, PathBuf}; + pub use live::{ - index_semantic_live, semantic_search_live_with_config, SemanticIndexReport, STORE_DIR, + committed_cursor, index_semantic_live, semantic_search_live_with_config, SemanticIndexReport, + STORE_DIR, }; pub use mock::{ planned_semantic_documents, semantic_search, semantic_search_with_snippet, @@ -13,11 +18,62 @@ pub use mock::{ pub(crate) const CHUNK_SCHEMA_ID: &str = "katok-kakao-parent-window-v1"; pub(crate) const SOURCE_ID: &str = "katok-kakao-parent-windows"; +pub const CURRENT_FILE: &str = "CURRENT"; +pub const GENERATIONS_DIR: &str = "generations"; pub fn semantic_source_dir(root: &std::path::Path) -> std::path::PathBuf { root.join("source").join("chunks") } +pub fn archive_revision(archive: &Archive) -> Result { + let parents = archive.all_parent_chunks()?; + let mut material = String::new(); + for parent in parents { + material.push_str(&parent.parent_id); + material.push('\0'); + material.push_str(&content_hash(&parent.text)); + material.push('\0'); + } + Ok(content_hash(&material)) +} + +pub fn current_generation(root: &Path) -> Result { + let pointer = root.join(CURRENT_FILE); + let generation = std::fs::read_to_string(&pointer) + .map_err(|error| { + if error.kind() == std::io::ErrorKind::NotFound { + Error::SemanticIndexMissing + } else { + Error::Io(error) + } + })? + .trim() + .to_string(); + if generation.is_empty() + || generation.contains('/') + || generation.contains('\\') + || generation == "." + || generation == ".." + { + return Err(Error::SemanticIndexStale( + "CURRENT contains an invalid generation id".to_string(), + )); + } + let path = root.join(GENERATIONS_DIR).join(generation); + if !path.is_dir() { + return Err(Error::SemanticIndexStale(format!( + "CURRENT generation is missing at {}", + path.display() + ))); + } + Ok(path) +} + +pub(crate) fn content_hash(content: &str) -> String { + let digest = Sha256::digest(content.as_bytes()); + digest.iter().map(|byte| format!("{byte:02x}")).collect() +} + pub(crate) fn document_path(dir: &std::path::Path, chunk_id: &str) -> std::path::PathBuf { dir.join(format!("{chunk_id}.md")) } diff --git a/src/semantic/live.rs b/src/semantic/live.rs index 5e9ec1b..923b5d8 100644 --- a/src/semantic/live.rs +++ b/src/semantic/live.rs @@ -2,15 +2,16 @@ use crate::{ archive::Archive, config::KatokConfig, search::hydrate_parent_hits, types::SearchHit, Error, Result, }; +use rusqlite::Connection; use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; -use std::path::Path; +use std::{path::Path, time::Duration}; use super::{ + archive_revision, content_hash, current_generation, embedder::create_embedder, mock::write_semantic_documents_plain, store::{LocalVectorStore, VectorUpsert}, - CHUNK_SCHEMA_ID, SOURCE_ID, + CHUNK_SCHEMA_ID, CURRENT_FILE, GENERATIONS_DIR, SOURCE_ID, }; pub const STORE_DIR: &str = "store"; @@ -23,53 +24,123 @@ pub struct SemanticIndexReport { pub embedder: &'static str, pub vectorstore: &'static str, pub semantic_units: &'static str, + pub archive_revision: String, + pub reused_vectors: usize, + pub self_healed: bool, + pub cleanup_warnings: Vec, } #[derive(Debug, Clone, Serialize, Deserialize)] -struct SemanticCursor { +pub struct SemanticCursor { source_id: String, - last_synced_at: String, - seen_token: String, + pub completed_at: String, + pub archive_revision: String, chunk_schema_id: String, - embedder_id: String, - vectorstore: String, + pub embedder_id: String, + pub vectorstore: String, + pub semantic_units: String, + pub embedded_texts: usize, } pub async fn index_semantic_live( archive: &Archive, - dir: &Path, + root: &Path, config: &KatokConfig, + full: bool, ) -> Result { - crate::paths::ensure_private_dir(dir)?; - let written = write_semantic_documents_plain(archive, dir)?; + crate::paths::ensure_private_dir(root)?; + crate::paths::ensure_private_dir(&root.join(GENERATIONS_DIR))?; + let _writer = IndexWriterGuard::acquire(root)?; + let revision = archive_revision(archive)?; + let generation_id = format!( + "gen-{}-{}-{}", + &revision[..16], + std::process::id(), + chrono::Utc::now().timestamp_nanos_opt().unwrap_or_default() + ); + let staging = root + .join(GENERATIONS_DIR) + .join(format!(".{generation_id}.staging")); + let mut staging_guard = StagingGuard::new(staging.clone()); + crate::paths::ensure_private_dir(&staging)?; + let written = write_semantic_documents_plain(archive, &staging)?; let parents = archive.all_parent_chunks()?; - let seen_token = semantic_seen_token(&parents); - let store = LocalVectorStore::open(&dir.join(STORE_DIR), usize::from(config.vector_dimension))?; + let store = LocalVectorStore::open( + &staging.join(STORE_DIR), + usize::from(config.vector_dimension), + )?; let mut embedder = create_embedder(config)?; + let prior_generation = current_generation(root); + let mut self_healed = false; + let prior_store = if full { + None + } else { + match prior_generation { + Ok(generation) => match validate_generation( + archive, + &generation, + embedder.id(), + config.vector_dimension, + ) { + Ok(()) => Some(LocalVectorStore::open_existing( + &generation.join(STORE_DIR), + usize::from(config.vector_dimension), + )?), + Err(Error::SemanticIndexStale(_) | Error::Embedding(_) | Error::Sql(_)) => { + self_healed = true; + None + } + Err(error) => return Err(error), + }, + Err(Error::SemanticIndexMissing | Error::SemanticIndexStale(_)) => None, + Err(error) => return Err(error), + } + }; let mut pending = Vec::new(); + let mut reused_vectors = 0usize; for parent in parents { let hash = content_hash(&parent.text); let heading_path = format!("{} / parent window", parent.chat_name); - match store.fetch(&parent.parent_id)? { - Some(stored) if stored.content_hash == hash => { - store.mark_seen(&stored.chunk_id, &seen_token, &heading_path)?; - } - Some(_) | None => pending.push(PendingChunk { + if let Some(stored) = prior_store + .as_ref() + .and_then(|prior| prior.fetch(&parent.parent_id).transpose()) + .transpose()? + .filter(|stored| stored.content_hash == hash) + { + reused_vectors += 1; + store.upsert(&VectorUpsert { chunk_id: parent.parent_id, content_hash: hash, - seen_token: seen_token.clone(), + seen_token: revision.clone(), + heading_path, + vector: stored.vector, + })?; + } else { + pending.push(PendingChunk { + chunk_id: parent.parent_id, + content_hash: hash, + seen_token: revision.clone(), heading_path, text: parent.text, - }), + }); } } let embedded_texts = pending.len(); let batch_size = config.embedding_batch_size.max(1); let embedding_calls = embed_pending(&store, &mut *embedder, &pending, batch_size)?; - store.delete_stale(&seen_token)?; - save_cursor(dir, &seen_token, embedder.id())?; + save_cursor(&staging, &revision, embedder.id(), embedded_texts)?; + validate_generation(archive, &staging, embedder.id(), config.vector_dimension)?; + + let generation = root.join(GENERATIONS_DIR).join(&generation_id); + std::fs::rename(&staging, &generation).map_err(Error::Io)?; + staging_guard.disarm(); + if let Err(error) = publish_current(root, &generation_id) { + let _ = std::fs::remove_dir_all(&generation); + return Err(error); + } + let cleanup_warnings = cleanup_old_generations(root, &generation_id); Ok(SemanticIndexReport { written_documents: written, @@ -78,12 +149,16 @@ pub async fn index_semantic_live( embedder: embedder.id(), vectorstore: "local", semantic_units: "parent_windows", + archive_revision: revision, + reused_vectors, + self_healed, + cleanup_warnings, }) } pub async fn semantic_search_live_with_config( archive: &Archive, - dir: &Path, + root: &Path, query: &str, limit: usize, config: &KatokConfig, @@ -91,20 +166,67 @@ pub async fn semantic_search_live_with_config( if query.trim().is_empty() { return Err(Error::EmptyQuery); } - if !dir.join("cursor.json").exists() { - return Err(Error::SemanticIndexMissing); - } - let cursor = load_cursor(dir)?; + let generation = current_generation(root)?; + let cursor = load_cursor(&generation)?; let mut embedder = create_embedder(config)?; - validate_cursor(&cursor, embedder.id())?; - let store = LocalVectorStore::open(&dir.join(STORE_DIR), usize::from(config.vector_dimension))?; + validate_generation(archive, &generation, embedder.id(), config.vector_dimension)?; + let store = LocalVectorStore::open_existing( + &generation.join(STORE_DIR), + usize::from(config.vector_dimension), + )?; let vector = embedder.embed_query(query)?; let ids = store .search(&vector, limit)? .into_iter() .map(|hit| hit.chunk_id) .collect::>(); - hydrate_parent_hits(archive, ids, "semantic", query, config.snippet_length) + if cursor.archive_revision != archive_revision(archive)? { + return Err(Error::SemanticIndexStale( + "archive changed while semantic search was starting".to_string(), + )); + } + match hydrate_parent_hits(archive, ids, "semantic", query, config.snippet_length) { + Err(Error::MissingChunk(id)) => Err(Error::SemanticIndexStale(format!( + "vector references missing archive window {id}" + ))), + result => result, + } +} + +pub fn committed_cursor(root: &Path) -> Result { + load_cursor(¤t_generation(root)?) +} + +fn validate_generation( + archive: &Archive, + generation: &Path, + embedder_id: &str, + dimension: u16, +) -> Result<()> { + let cursor = load_cursor(generation)?; + validate_cursor(&cursor, embedder_id)?; + let current_revision = archive_revision(archive)?; + if cursor.archive_revision != current_revision { + return Err(Error::SemanticIndexStale(format!( + "archive revision is {}, index revision is {}", + current_revision, cursor.archive_revision + ))); + } + let mut expected = archive + .all_parent_chunks()? + .into_iter() + .map(|parent| (parent.parent_id, content_hash(&parent.text))) + .collect::>(); + expected.sort(); + let actual = + LocalVectorStore::open_existing(&generation.join(STORE_DIR), usize::from(dimension))? + .content_pairs()?; + if actual != expected { + return Err(Error::SemanticIndexStale( + "vector ids or content hashes do not match the archive".to_string(), + )); + } + Ok(()) } #[derive(Debug, Clone)] @@ -148,58 +270,123 @@ fn embed_pending( Ok(pending.len().div_ceil(batch_size)) } -fn save_cursor(dir: &Path, seen_token: &str, embedder_id: &str) -> Result<()> { +fn save_cursor(dir: &Path, revision: &str, embedder_id: &str, embedded_texts: usize) -> Result<()> { let cursor = SemanticCursor { source_id: SOURCE_ID.to_string(), - last_synced_at: chrono::Utc::now().to_rfc3339(), - seen_token: seen_token.to_string(), + completed_at: chrono::Utc::now().to_rfc3339(), + archive_revision: revision.to_string(), chunk_schema_id: CHUNK_SCHEMA_ID.to_string(), embedder_id: embedder_id.to_string(), vectorstore: "local".to_string(), + semantic_units: "parent_windows".to_string(), + embedded_texts, }; let json = serde_json::to_vec_pretty(&cursor).map_err(Error::Json)?; std::fs::write(dir.join("cursor.json"), json).map_err(Error::Io) } fn load_cursor(dir: &Path) -> Result { - let content = std::fs::read(dir.join("cursor.json")).map_err(Error::Io)?; - serde_json::from_slice(&content).map_err(Error::Json) + let content = std::fs::read(dir.join("cursor.json")) + .map_err(|error| Error::SemanticIndexStale(format!("cannot read cursor: {error}")))?; + serde_json::from_slice(&content) + .map_err(|error| Error::SemanticIndexStale(format!("cannot parse cursor: {error}"))) } fn validate_cursor(cursor: &SemanticCursor, embedder_id: &str) -> Result<()> { - if cursor.source_id != SOURCE_ID { - return stale_index_error("source", &cursor.source_id, SOURCE_ID); + for (field, actual, expected) in [ + ("source", cursor.source_id.as_str(), SOURCE_ID), + ("schema", cursor.chunk_schema_id.as_str(), CHUNK_SCHEMA_ID), + ("vectorstore", cursor.vectorstore.as_str(), "local"), + ("embedder", cursor.embedder_id.as_str(), embedder_id), + ] { + if actual != expected { + return Err(Error::SemanticIndexStale(format!( + "{field} is {actual}, expected {expected}" + ))); + } } - if cursor.chunk_schema_id != CHUNK_SCHEMA_ID { - return stale_index_error("schema", &cursor.chunk_schema_id, CHUNK_SCHEMA_ID); + Ok(()) +} + +fn publish_current(root: &Path, generation_id: &str) -> Result<()> { + let temporary = root.join(format!(".{CURRENT_FILE}.{}", std::process::id())); + std::fs::write(&temporary, format!("{generation_id}\n")).map_err(Error::Io)?; + std::fs::rename(&temporary, root.join(CURRENT_FILE)).map_err(Error::Io) +} + +fn cleanup_old_generations(root: &Path, current: &str) -> Vec { + let generations = root.join(GENERATIONS_DIR); + let entries = match std::fs::read_dir(&generations) { + Ok(entries) => entries, + Err(error) => { + return vec![format!( + "cannot scan old semantic generations at {}: {error}", + generations.display() + )]; + } + }; + let mut warnings = Vec::new(); + for entry in entries { + let entry = match entry { + Ok(entry) => entry, + Err(error) => { + warnings.push(format!("cannot read a semantic generation entry: {error}")); + continue; + } + }; + if entry.file_name() == current { + continue; + } + if let Err(error) = std::fs::remove_dir_all(entry.path()) { + warnings.push(format!( + "cannot remove old semantic generation {}: {error}", + entry.path().display() + )); + } } - if cursor.vectorstore != "local" { - return stale_index_error("vectorstore", &cursor.vectorstore, "local"); + warnings +} + +struct IndexWriterGuard { + conn: Connection, +} + +impl IndexWriterGuard { + fn acquire(root: &Path) -> Result { + let path = root.join("index-writer.sqlite3"); + let conn = Connection::open(&path).map_err(Error::Sql)?; + conn.busy_timeout(Duration::ZERO).map_err(Error::Sql)?; + conn.execute_batch("BEGIN EXCLUSIVE") + .map_err(|_| Error::SemanticIndexBusy(path))?; + Ok(Self { conn }) } - if cursor.embedder_id != embedder_id { - return stale_index_error("embedder", &cursor.embedder_id, embedder_id); +} + +impl Drop for IndexWriterGuard { + fn drop(&mut self) { + let _ = self.conn.execute_batch("ROLLBACK"); } - Ok(()) } -fn stale_index_error(field: &str, actual: &str, expected: &str) -> Result<()> { - Err(Error::Embedding(format!( - "semantic index {field} is {actual}, expected {expected}; re-run katok index" - ))) +struct StagingGuard { + path: std::path::PathBuf, + armed: bool, } -fn semantic_seen_token(chunks: &[crate::types::ParentChunk]) -> String { - let mut material = String::new(); - for chunk in chunks { - material.push_str(&chunk.parent_id); - material.push('\0'); - material.push_str(&content_hash(&chunk.text)); - material.push('\0'); +impl StagingGuard { + fn new(path: std::path::PathBuf) -> Self { + Self { path, armed: true } + } + + fn disarm(&mut self) { + self.armed = false; } - content_hash(&material) } -fn content_hash(content: &str) -> String { - let digest = Sha256::digest(content.as_bytes()); - digest.iter().map(|byte| format!("{byte:02x}")).collect() +impl Drop for StagingGuard { + fn drop(&mut self) { + if self.armed { + let _ = std::fs::remove_dir_all(&self.path); + } + } } diff --git a/src/semantic/store.rs b/src/semantic/store.rs index 871e556..ab9d9f1 100644 --- a/src/semantic/store.rs +++ b/src/semantic/store.rs @@ -1,5 +1,5 @@ use crate::{Error, Result}; -use rusqlite::{params, Connection}; +use rusqlite::{params, Connection, OpenFlags}; use std::path::Path; pub(crate) struct LocalVectorStore { @@ -9,8 +9,8 @@ pub(crate) struct LocalVectorStore { #[derive(Debug, Clone)] pub(crate) struct StoredVector { - pub chunk_id: String, pub content_hash: String, + pub vector: Vec, } #[derive(Debug, Clone)] @@ -38,21 +38,68 @@ impl LocalVectorStore { Ok(Self { conn, dimension }) } + pub(crate) fn open_existing(dir: &Path, dimension: usize) -> Result { + let path = dir.join("vectors.sqlite3"); + if !path.is_file() { + return Err(Error::SemanticIndexStale(format!( + "vector store is missing at {}", + path.display() + ))); + } + let conn = Connection::open_with_flags(&path, OpenFlags::SQLITE_OPEN_READ_ONLY) + .map_err(Error::Sql)?; + Ok(Self { conn, dimension }) + } + pub(crate) fn fetch(&self, chunk_id: &str) -> Result> { let mut statement = self .conn - .prepare("SELECT chunk_id, content_hash FROM vectors WHERE chunk_id = ?1") + .prepare("SELECT chunk_id, content_hash, vector FROM vectors WHERE chunk_id = ?1") .map_err(Error::Sql)?; let mut rows = statement.query(params![chunk_id]).map_err(Error::Sql)?; let Some(row) = rows.next().map_err(Error::Sql)? else { return Ok(None); }; Ok(Some(StoredVector { - chunk_id: row.get(0).map_err(Error::Sql)?, content_hash: row.get(1).map_err(Error::Sql)?, + vector: decode_vector( + &row.get::<_, Vec>(2).map_err(Error::Sql)?, + self.dimension, + )?, })) } + pub(crate) fn content_pairs(&self) -> Result> { + let mut statement = self + .conn + .prepare("SELECT chunk_id, content_hash, vector FROM vectors ORDER BY chunk_id") + .map_err(Error::Sql)?; + let pairs = statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Vec>(2)?, + )) + }) + .map_err(Error::Sql)? + .collect::, _>>() + .map_err(Error::Sql)?; + pairs + .into_iter() + .map(|(chunk_id, content_hash, vector)| { + let expected = self.dimension * std::mem::size_of::(); + if vector.len() != expected { + return Err(Error::SemanticIndexStale(format!( + "vector {chunk_id} has {} bytes, expected {expected}", + vector.len() + ))); + } + Ok((chunk_id, content_hash)) + }) + .collect() + } + pub(crate) fn upsert(&self, item: &VectorUpsert) -> Result<()> { if item.vector.len() != self.dimension { return Err(Error::Embedding(format!( @@ -84,31 +131,6 @@ impl LocalVectorStore { Ok(()) } - pub(crate) fn mark_seen( - &self, - chunk_id: &str, - seen_token: &str, - heading_path: &str, - ) -> Result<()> { - self.conn - .execute( - "UPDATE vectors SET seen_token = ?2, heading_path = ?3 WHERE chunk_id = ?1", - params![chunk_id, seen_token, heading_path], - ) - .map_err(Error::Sql)?; - Ok(()) - } - - pub(crate) fn delete_stale(&self, seen_token: &str) -> Result<()> { - self.conn - .execute( - "DELETE FROM vectors WHERE seen_token != ?1", - params![seen_token], - ) - .map_err(Error::Sql)?; - Ok(()) - } - pub(crate) fn search(&self, query: &[f32], limit: usize) -> Result> { if query.len() != self.dimension { return Err(Error::Embedding(format!( diff --git a/src/types.rs b/src/types.rs index c5fa1dd..1d38d1e 100644 --- a/src/types.rs +++ b/src/types.rs @@ -242,6 +242,7 @@ pub struct SearchHit { pub unit: &'static str, pub rank: usize, pub chunk_id: String, + pub chat_id: String, pub chat_name: String, pub sender_nickname: String, pub started_at: String, diff --git a/tests/cli_behaviors.rs b/tests/cli_behaviors.rs index 0074ce7..a098aa7 100644 --- a/tests/cli_behaviors.rs +++ b/tests/cli_behaviors.rs @@ -165,7 +165,7 @@ fn cli_reports_semantic_index_states_when_embedder_is_local_test_or_mocked() { ]) .assert() .failure() - .stderr(predicate::str::contains( + .stdout(predicate::str::contains( "semantic index has never been synced", )); @@ -278,7 +278,7 @@ fn cli_lists_gap_chunks_and_applies_chunk_output_flags() { ]) .assert() .failure() - .stderr(predicate::str::contains("chunk not found")); + .stdout(predicate::str::contains("chunk not found")); } #[test] @@ -297,7 +297,7 @@ fn cli_rejects_malformed_config_and_missing_kakaocli_without_private_dump() { ]) .assert() .failure() - .stderr(predicate::str::contains("config parse error")); + .stdout(predicate::str::contains("config parse error")); // Force kakaocli to be absent from PATH so the failure is deterministic // regardless of whether the host has kakaocli installed. @@ -307,7 +307,7 @@ fn cli_rejects_malformed_config_and_missing_kakaocli_without_private_dump() { .args(["source", "chats", "--source", "kakaocli", "--json"]) .assert() .failure() - .stderr(predicate::str::contains("kakaocli not found on PATH")); + .stdout(predicate::str::contains("kakaocli not found on PATH")); } #[test] diff --git a/tests/config_wiring.rs b/tests/config_wiring.rs index 23f03e1..33504d5 100644 --- a/tests/config_wiring.rs +++ b/tests/config_wiring.rs @@ -63,7 +63,13 @@ fn cli_honors_configured_semantic_dir_when_indexing() { .assert() .success(); - let document_dir = data_dir.join("custom-semantic/source/chunks"); + let semantic_dir = data_dir.join("custom-semantic"); + let generation = + std::fs::read_to_string(semantic_dir.join("CURRENT")).expect("read current generation"); + let document_dir = semantic_dir + .join("generations") + .join(generation.trim()) + .join("source/chunks"); let documents = std::fs::read_dir(document_dir) .expect("read semantic docs") .collect::, _>>() diff --git a/tests/gap_behaviors.rs b/tests/gap_behaviors.rs index de19578..8437637 100644 --- a/tests/gap_behaviors.rs +++ b/tests/gap_behaviors.rs @@ -24,7 +24,7 @@ fn cli_handles_plan_gap_edges_when_exercised() { ]) .assert() .failure() - .stderr(predicate::str::contains( + .stdout(predicate::str::contains( "semantic index has never been synced", )); @@ -41,8 +41,8 @@ fn cli_handles_plan_gap_edges_when_exercised() { ]) .assert() .failure() - .stderr(predicate::str::contains("fixture parse error on line 2")) - .stderr(predicate::str::contains("PRIVATE-MALFORMED-BODY").not()); + .stdout(predicate::str::contains("fixture parse error on line 2")) + .stdout(predicate::str::contains("PRIVATE-MALFORMED-BODY").not()); Command::cargo_bin("katok") .expect("katok binary") @@ -91,14 +91,14 @@ fn cli_handles_plan_gap_edges_when_exercised() { .success() .stdout(predicate::str::contains("\"written_documents\": 1")) .stdout(predicate::str::contains("window_")) - .stdout(predicate::str::contains("embeddinggemma-300m-q4")); + .stdout(predicate::str::contains("embeddinggemma/local-test")); Command::cargo_bin("katok") .expect("katok binary") .args(["--data-dir", data_dir, "search", "bm25", "", "--json"]) .assert() .failure() - .stderr(predicate::str::contains("empty query")); + .stdout(predicate::str::contains("empty query")); Command::cargo_bin("katok") .expect("katok binary") @@ -112,7 +112,7 @@ fn cli_handles_plan_gap_edges_when_exercised() { ]) .assert() .failure() - .stderr(predicate::str::contains("chunk not found")); + .stdout(predicate::str::contains("chunk not found")); Command::cargo_bin("katok") .expect("katok binary") diff --git a/tests/hierarchical_behaviors.rs b/tests/hierarchical_behaviors.rs index b710cb6..8d63a0f 100644 --- a/tests/hierarchical_behaviors.rs +++ b/tests/hierarchical_behaviors.rs @@ -135,16 +135,20 @@ fn cli_rejects_stale_micro_chunk_semantic_cursor_when_searching() { .success(); let semantic_dir = data_dir.join("semantic"); - std::fs::create_dir_all(&semantic_dir).expect("create semantic dir"); + let generation_dir = semantic_dir.join("generations/stale"); + std::fs::create_dir_all(&generation_dir).expect("create semantic dir"); + std::fs::write(semantic_dir.join("CURRENT"), "stale\n").expect("write current pointer"); std::fs::write( - semantic_dir.join("cursor.json"), + generation_dir.join("cursor.json"), r#"{ "source_id": "katok-kakao-parent-windows", - "last_synced_at": "2026-01-01T00:00:00Z", - "seen_token": "old", + "completed_at": "2026-01-01T00:00:00Z", + "archive_revision": "old", "chunk_schema_id": "katok-kakao-chunk-v1", "embedder_id": "embeddinggemma/local-test", - "vectorstore": "local" + "vectorstore": "local", + "semantic_units": "parent_windows", + "embedded_texts": 0 }"#, ) .expect("write stale cursor"); @@ -162,6 +166,6 @@ fn cli_rejects_stale_micro_chunk_semantic_cursor_when_searching() { ]) .assert() .failure() - .stderr(predicate::str::contains("re-run katok index")) - .stderr(predicate::str::contains("katok-kakao-chunk-v1")); + .stdout(predicate::str::contains("re-run katok index")) + .stdout(predicate::str::contains("katok-kakao-chunk-v1")); } diff --git a/tests/kakaocli_adapter.rs b/tests/kakaocli_adapter.rs index 800ac6c..8ac1c10 100644 --- a/tests/kakaocli_adapter.rs +++ b/tests/kakaocli_adapter.rs @@ -56,14 +56,14 @@ fn cli_surfaces_kakaocli_failure_detail_instead_of_generic_message() { .failure(); assert - .stderr(predicate::str::contains("kakaocli chats failed")) - .stderr(predicate::str::contains( + .stdout(predicate::str::contains("kakaocli chats failed")) + .stdout(predicate::str::contains( "SQL error: prepare: file is not a database", )) // Must not regress to the old misleading message. - .stderr(predicate::str::contains("not found or not configured").not()) + .stdout(predicate::str::contains("not found or not configured").not()) // Must not leak any message body, even on the error path. - .stderr(predicate::str::contains("합성 카카오 메시지").not()); + .stdout(predicate::str::contains("합성 카카오 메시지").not()); } #[cfg(unix)] diff --git a/tests/live_semantic.rs b/tests/live_semantic.rs index 561cb5a..fb1ad82 100644 --- a/tests/live_semantic.rs +++ b/tests/live_semantic.rs @@ -79,8 +79,13 @@ fn live_semantic_cli_indexes_local_embeddings_and_searches_without_endpoint() { .success() .stdout(predicate::str::contains("chunk_2aeac4db0a04ceb2")); - assert!(data_dir.join("semantic/store").exists()); - assert!(data_dir.join("semantic/cursor.json").exists()); + let generation = std::fs::read_to_string(data_dir.join("semantic/CURRENT")) + .expect("read current generation"); + let generation = data_dir + .join("semantic/generations") + .join(generation.trim()); + assert!(generation.join("store").exists()); + assert!(generation.join("cursor.json").exists()); } #[test] @@ -106,7 +111,7 @@ fn live_semantic_cli_rejects_stale_remote_embedding_endpoint_config() { ]) .assert() .failure() - .stderr(predicate::str::contains( + .stdout(predicate::str::contains( "unknown field `embedder_base_url`", )); } diff --git a/tests/search_index_recovery.rs b/tests/search_index_recovery.rs new file mode 100644 index 0000000..af4734b --- /dev/null +++ b/tests/search_index_recovery.rs @@ -0,0 +1,303 @@ +use assert_cmd::Command; +use predicates::prelude::*; +use serde_json::Value; +use std::path::{Path, PathBuf}; + +fn fixture(name: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/kakao") + .join(name) +} + +fn run_json(data_dir: &Path, args: &[&str]) -> Value { + let mut command = Command::cargo_bin("katok").expect("katok binary"); + command + .env("KATOK_EMBEDDER", "local-test") + .arg("--data-dir") + .arg(data_dir); + command.args(args); + let output = command.assert().success().get_output().stdout.clone(); + serde_json::from_slice(&output).expect("valid json output") +} + +fn current_generation(data_dir: &Path) -> PathBuf { + let semantic = data_dir.join("semantic"); + let id = std::fs::read_to_string(semantic.join("CURRENT")).expect("CURRENT"); + semantic.join("generations").join(id.trim()) +} + +#[test] +fn doctor_and_search_reject_an_archive_newer_than_the_index() { + let dir = tempfile::tempdir().expect("tempdir"); + let data_dir = dir.path().join("data"); + let newer = dir.path().join("newer.jsonl"); + let mut rows = std::fs::read_to_string(fixture("replies.jsonl")).expect("fixture"); + rows.push_str("{\"account_hash\":\"acct-synthetic\",\"chat_id\":\"chat-new\",\"chat_name\":\"Synthetic New\",\"chat_type\":\"group\",\"message_id\":\"new-1\",\"sender_id\":\"u3\",\"sender_nickname\":\"지수\",\"timestamp\":\"2026-01-02T09:00:00Z\",\"text\":\"새 인덱스 토큰\",\"message_type\":\"text\",\"reply_to_message_id\":null}\n"); + std::fs::write(&newer, rows).expect("newer fixture"); + + run_json( + &data_dir, + &[ + "sync", + "--source", + "fixture", + fixture("replies.jsonl").to_str().unwrap(), + "--json", + ], + ); + run_json(&data_dir, &["index", "--json"]); + run_json( + &data_dir, + &[ + "sync", + "--source", + "fixture", + newer.to_str().unwrap(), + "--json", + ], + ); + + let doctor = run_json(&data_dir, &["doctor", "--json"]); + assert_eq!( + doctor["freshness"]["recommendation"]["index_before_semantic_search"], + true + ); + + Command::cargo_bin("katok") + .expect("katok binary") + .env("KATOK_EMBEDDER", "local-test") + .arg("--data-dir") + .arg(&data_dir) + .args(["search", "semantic", "새 인덱스 토큰", "--json"]) + .assert() + .failure() + .stdout(predicate::str::contains("semantic index is stale")); +} + +#[test] +fn index_self_heals_an_orphan_generation_and_full_never_reuses_vectors() { + let dir = tempfile::tempdir().expect("tempdir"); + let data_dir = dir.path().join("data"); + run_json( + &data_dir, + &[ + "sync", + "--source", + "fixture", + fixture("replies.jsonl").to_str().unwrap(), + "--json", + ], + ); + run_json(&data_dir, &["index", "--json"]); + + let generation = current_generation(&data_dir); + let store = + rusqlite::Connection::open(generation.join("store/vectors.sqlite3")).expect("store"); + store.execute( + "INSERT INTO vectors(chunk_id, content_hash, seen_token, heading_path, vector) VALUES (?1, ?2, ?3, ?4, ?5)", + rusqlite::params!["window_orphan", "stale", "stale", "Synthetic", vec![0_u8; 768 * 4]], + ).expect("insert orphan"); + drop(store); + + let report = run_json(&data_dir, &["index", "--json"]); + assert_eq!(report["self_healed"], true); + let search = run_json(&data_dir, &["search", "semantic", "회의 보고서", "--json"]); + assert!(search.as_array().is_some_and(|hits| !hits.is_empty())); + + let full = run_json(&data_dir, &["index", "--full", "--json"]); + assert_eq!(full["full"], true); + assert_eq!(full["reused_vectors"], 0); + assert!(full["embedded_texts"] + .as_u64() + .is_some_and(|count| count > 0)); + let generations = std::fs::read_dir(data_dir.join("semantic/generations")) + .expect("generation directory") + .collect::, _>>() + .expect("generation entries"); + assert_eq!(generations.len(), 1); +} + +#[test] +fn failed_rebuild_keeps_the_committed_generation_and_returns_json_error() { + let dir = tempfile::tempdir().expect("tempdir"); + let data_dir = dir.path().join("data"); + run_json( + &data_dir, + &[ + "sync", + "--source", + "fixture", + fixture("replies.jsonl").to_str().unwrap(), + "--json", + ], + ); + run_json(&data_dir, &["index", "--json"]); + let current_before = std::fs::read(data_dir.join("semantic/CURRENT")).expect("CURRENT before"); + let doctor_before = run_json(&data_dir, &["doctor", "--json"]); + let config = dir.path().join("bad.toml"); + std::fs::write(&config, "vector_dimension = 0\n").expect("bad config"); + + Command::cargo_bin("katok") + .expect("katok binary") + .env("KATOK_EMBEDDER", "local-test") + .arg("--config") + .arg(&config) + .arg("--data-dir") + .arg(&data_dir) + .args(["index", "--full", "--json"]) + .assert() + .failure() + .stdout(predicate::str::contains("\"ok\": false")) + .stdout(predicate::str::contains( + "embedding dimension must be nonzero", + )); + + assert_eq!( + std::fs::read(data_dir.join("semantic/CURRENT")).unwrap(), + current_before + ); + let doctor_after = run_json(&data_dir, &["doctor", "--json"]); + assert_eq!( + doctor_after["freshness"]["last_index"], + doctor_before["freshness"]["last_index"] + ); +} + +#[test] +fn doctor_reports_a_corrupt_generation_pointer_instead_of_hiding_it_as_missing() { + let dir = tempfile::tempdir().expect("tempdir"); + let data_dir = dir.path().join("data"); + run_json( + &data_dir, + &[ + "sync", + "--source", + "fixture", + fixture("replies.jsonl").to_str().unwrap(), + "--json", + ], + ); + run_json(&data_dir, &["index", "--json"]); + std::fs::write(data_dir.join("semantic/CURRENT"), "../outside\n").expect("corrupt CURRENT"); + + let doctor = run_json(&data_dir, &["doctor", "--json"]); + assert_eq!( + doctor["freshness"]["recommendation"]["index_before_semantic_search"], + true + ); + assert!(doctor["freshness"]["recommendation"]["reason"] + .as_str() + .is_some_and(|reason| reason.contains("CURRENT") && reason.contains("corrupt"))); +} + +#[test] +fn bm25_treats_fts_punctuation_as_literal_user_text() { + let dir = tempfile::tempdir().expect("tempdir"); + let data_dir = dir.path().join("data"); + let fixture = dir.path().join("special.jsonl"); + std::fs::write(&fixture, "{\"account_hash\":\"acct-synthetic\",\"chat_id\":\"chat-special\",\"chat_name\":\"Synthetic Special\",\"chat_type\":\"group\",\"message_id\":\"special-1\",\"sender_id\":\"u1\",\"sender_nickname\":\"민지\",\"timestamp\":\"2026-01-01T09:00:00Z\",\"text\":\"NTRU+ SMAUG-T Golden KAT exact phrase\",\"message_type\":\"text\",\"reply_to_message_id\":null}\n").expect("fixture"); + run_json( + &data_dir, + &[ + "sync", + "--source", + "fixture", + fixture.to_str().unwrap(), + "--json", + ], + ); + + for query in [ + "NTRU+ SMAUG-T Golden KAT", + "\"Golden\" KAT", + "Golden: KAT", + "Golden* KAT", + ] { + Command::cargo_bin("katok") + .expect("katok binary") + .arg("--data-dir") + .arg(&data_dir) + .args(["search", "bm25", query, "--json"]) + .assert() + .success() + .stdout(predicate::str::contains("\"chat_id\": \"chat-special\"")); + } +} + +#[test] +fn concurrent_index_writers_fail_loudly_instead_of_publishing_two_generations() { + let dir = tempfile::tempdir().expect("tempdir"); + let data_dir = dir.path().join("data"); + run_json( + &data_dir, + &[ + "sync", + "--source", + "fixture", + fixture("replies.jsonl").to_str().unwrap(), + "--json", + ], + ); + let semantic = data_dir.join("semantic"); + std::fs::create_dir_all(&semantic).expect("semantic dir"); + let lock = rusqlite::Connection::open(semantic.join("index-writer.sqlite3")).expect("lock db"); + lock.execute_batch("BEGIN EXCLUSIVE") + .expect("exclusive lock"); + + let mut command = Command::cargo_bin("katok").expect("katok binary"); + let output = command + .env("KATOK_EMBEDDER", "local-test") + .arg("--data-dir") + .arg(&data_dir) + .args(["index", "--json"]) + .output() + .expect("run index"); + assert!(!output.status.success()); + let json: Value = serde_json::from_slice(&output.stdout).expect("json error"); + assert!(json["error"]["cause"] + .as_str() + .is_some_and(|cause| cause.contains("already running"))); + assert!(!semantic.join("CURRENT").exists()); +} + +#[test] +fn cross_room_search_can_expand_beyond_the_global_default_top_ten_and_group_by_chat_id() { + let dir = tempfile::tempdir().expect("tempdir"); + let data_dir = dir.path().join("data"); + let fixture = dir.path().join("many-rooms.jsonl"); + let rows = (0..12) + .map(|room| format!("{{\"account_hash\":\"acct-synthetic\",\"chat_id\":\"chat-{room:02}\",\"chat_name\":\"Synthetic Room {room:02}\",\"chat_type\":\"group\",\"message_id\":\"m-{room:02}\",\"sender_id\":\"u1\",\"sender_nickname\":\"민지\",\"timestamp\":\"2026-01-01T09:{room:02}:00Z\",\"text\":\"다중방인물 검색\",\"message_type\":\"text\",\"reply_to_message_id\":null}}\n")) + .collect::(); + std::fs::write(&fixture, rows).expect("fixture"); + run_json( + &data_dir, + &[ + "sync", + "--source", + "fixture", + fixture.to_str().unwrap(), + "--json", + ], + ); + + let default_hits = run_json(&data_dir, &["search", "keyword", "다중방인물", "--json"]); + assert_eq!(default_hits.as_array().map(Vec::len), Some(10)); + let expanded = run_json( + &data_dir, + &[ + "search", + "keyword", + "다중방인물", + "--limit", + "100", + "--json", + ], + ); + let chat_ids = expanded + .as_array() + .expect("hits") + .iter() + .filter_map(|hit| hit["chat_id"].as_str()) + .collect::>(); + assert_eq!(chat_ids.len(), 12); +}