diff --git a/.github/workflows/chinese-fts.yml b/.github/workflows/cjk-fts.yml similarity index 84% rename from .github/workflows/chinese-fts.yml rename to .github/workflows/cjk-fts.yml index 96150bf6e..2873a9415 100644 --- a/.github/workflows/chinese-fts.yml +++ b/.github/workflows/cjk-fts.yml @@ -1,9 +1,9 @@ -name: Chinese FTS +name: CJK FTS on: pull_request: paths: - - ".github/workflows/chinese-fts.yml" + - ".github/workflows/cjk-fts.yml" - "Makefile" - "internal/db/**" - "internal/service/**" @@ -12,7 +12,7 @@ on: push: branches: [main] paths: - - ".github/workflows/chinese-fts.yml" + - ".github/workflows/cjk-fts.yml" - "Makefile" - "internal/db/**" - "internal/service/**" @@ -27,8 +27,8 @@ permissions: contents: read jobs: - test-chinese-fts: - name: Go Test (Chinese FTS sidecar) + test-cjk-fts: + name: Go Test (CJK FTS sidecar) runs-on: ubuntu-latest timeout-minutes: 25 steps: @@ -47,7 +47,7 @@ jobs: - name: Build pinned simple and cppjieba sidecar run: bash scripts/build-simple-fts.sh dist/agentsview-simple - - name: Run database, service, and MCP tests with Chinese FTS enabled + - name: Run database, service, and MCP tests with CJK FTS enabled env: AGENTSVIEW_SIMPLE_DIR: ${{ github.workspace }}/dist/agentsview-simple CGO_ENABLED: "1" diff --git a/Makefile b/Makefile index 2a1795342..37477e0bf 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ AIR_BIN := $(shell if command -v air >/dev/null 2>&1; then command -v air; \ elif [ -x "$(GOPATH_FIRST)/bin/air" ]; then printf "%s" "$(GOPATH_FIRST)/bin/air"; \ fi) -.PHONY: build build-release install install-chinese-fts simple-fts frontend frontend-dev dev check-air air-install desktop-dev desktop-build desktop-macos-app desktop-macos-dmg desktop-windows-installer desktop-linux-appimage desktop-app docs-install docs-build docs-serve docs-check docs-screenshots docs-assets-branch docs-generated-assets-branch docs-deploy-staging docs-deploy test test-short test-evalingest bench-backends bench-gate bench-gate-config bench-pg-usage test-postgres test-postgres-ci test-s3 postgres-up postgres-down test-ssh test-ssh-ci ssh-up ssh-down e2e e2e-duckdb vet lint lint-ci lint-golangci lint-golangci-ci nilaway nilaway-golangci-build lint-tools tidy clean release release-darwin-arm64 release-darwin-amd64 release-linux-amd64 install-hooks ensure-embed-dir pricing-snapshot sqlite-vec-header dev-snapshot help check-timing-budgets +.PHONY: build build-release install install-cjk-fts simple-fts frontend frontend-dev dev check-air air-install desktop-dev desktop-build desktop-macos-app desktop-macos-dmg desktop-windows-installer desktop-linux-appimage desktop-app docs-install docs-build docs-serve docs-check docs-screenshots docs-assets-branch docs-generated-assets-branch docs-deploy-staging docs-deploy test test-short test-evalingest bench-backends bench-gate bench-gate-config bench-pg-usage test-postgres test-postgres-ci test-s3 postgres-up postgres-down test-ssh test-ssh-ci ssh-up ssh-down e2e e2e-duckdb vet lint lint-ci lint-golangci lint-golangci-ci nilaway nilaway-golangci-build lint-tools tidy clean release release-darwin-arm64 release-darwin-amd64 release-linux-amd64 install-hooks ensure-embed-dir pricing-snapshot sqlite-vec-header dev-snapshot help check-timing-budgets # Ensure go:embed has at least one file (no-op if frontend is built) ensure-embed-dir: @@ -102,8 +102,8 @@ install: build-release simple-fts: bash scripts/build-simple-fts.sh dist/agentsview-simple -# Install the binary and its Chinese-search sidecar in sibling bin/lib trees. -install-chinese-fts: install simple-fts +# Install the binary and its CJK-search sidecar in sibling bin/lib trees. +install-cjk-fts: install simple-fts @if [ -d "$(HOME)/.local/bin" ]; then \ INSTALL_DIR="$(HOME)/.local/bin"; \ else \ @@ -127,7 +127,7 @@ install-chinese-fts: install simple-fts for name in simple-LICENSE cppjieba-LICENSE; do \ install -m 0644 "dist/agentsview-simple/licenses/$$name" "$$SIMPLE_DIR/licenses/$$name"; \ done; \ - echo "Installed Chinese FTS sidecar to $$SIMPLE_DIR" + echo "Installed CJK FTS sidecar to $$SIMPLE_DIR" # Build frontend SPA and copy into embed directory frontend: @@ -653,7 +653,7 @@ help: @echo " build-release - Release build (optimized, stripped)" @echo " pricing-snapshot - Restore LiteLLM snapshot from artifact branch" @echo " install - Build and install to ~/.local/bin or GOPATH" - @echo " install-chinese-fts - Install agentsview with the optional Chinese FTS sidecar" + @echo " install-cjk-fts - Install agentsview with the optional CJK FTS sidecar" @echo " simple-fts - Build the pinned simple/cppjieba SQLite extension" @echo "" @echo " dev - Run Go server with live reload via air (use with frontend-dev)" diff --git a/README.md b/README.md index 79cb93214..93f197e6a 100644 --- a/README.md +++ b/README.md @@ -311,8 +311,9 @@ agentsview stats --include-git-outcomes | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | | ![Search](https://agentsview.io/assets/generated/screenshots/search-results.png) | ![Heatmap](https://agentsview.io/assets/generated/screenshots/heatmap.png) | -- **Full-text search** across all message content (FTS5), with optional - `simple`/cppjieba tokenization for Chinese queries +- **Full-text search** across all message content (FTS5), with optional CJK + character and phrase search for Chinese, Japanese, and Korean text in + SQLite, including Chinese word segmentation through `simple`/cppjieba - **Semantic search** (opt-in) -- index session content with any OpenAI-compatible embeddings endpoint and search by meaning with `agentsview session search --semantic` or `--hybrid`; every content-search diff --git a/docs/changelog.md b/docs/changelog.md index 67985b2c2..2b8a58059 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -13,10 +13,12 @@ description: Release history for AgentsView files are gone remain in the archive. (#1677) - Browse and search Open Code Review sessions, including review comments, tools, thinking, recorded token usage, and resumed reviews. (#1660) -- Search Chinese words and individual characters in SQLite message content - with an optional tokenizer sidecar. HTTP, CLI, and MCP search use word - segmentation; ASCII-only searches keep English stemming. Install with - `make install-chinese-fts`. (#1491) +- Search Chinese, Japanese, and Korean text in SQLite message content with + the optional CJK full-text search sidecar. Japanese kana and Korean Hangul + queries preserve character order within each search term; Han-only queries + use Chinese word segmentation. Quoted phrases preserve order in all three + languages. ASCII-only searches keep English stemming. Install with + `make install-cjk-fts`. (#1491) - Browse and search Tau sessions, including the active conversation branch, thinking, tools, session names, and recorded token usage. (#1661) - Configure session directories and alternate homes in `[agents.]` tables diff --git a/docs/configuration.md b/docs/configuration.md index 29de6dccf..75c8ca1da 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1309,17 +1309,40 @@ pulled in from PostgreSQL sync or copied from other archives. ## Database The SQLite database uses WAL mode for concurrent reads and includes FTS5 -full-text search indexes on message content. To add Chinese word, phrase, and -single-character matching, build and install the pinned `simple`/cppjieba -sidecar with `make install-chinese-fts`. Building it requires Git, CMake -3.19 or newer, and a C++14 compiler. AgentsView discovers it next to the binary +full-text search indexes on message content. + +### CJK full-text search + +To add character and phrase matching for Chinese, Japanese, and Korean text, +including Chinese word segmentation, build and install the pinned +`simple`/cppjieba sidecar with `make install-cjk-fts`. Building it requires Git, +CMake 3.19 or newer, and a C++14 compiler. AgentsView discovers it next to the binary or under the sibling `lib/agentsview/simple` directory. A custom path can be selected with `AGENTSVIEW_SIMPLE_DIR`. -The sidecar adds a parallel `messages_chinese_fts` index and routes only CJK -queries through it. ASCII-only searches continue to use the existing Porter -index, so searches such as `run` retain English stemming. The Chinese index is -derived data: if the sidecar is removed, AgentsView drops that optional index +The sidecar indexes individual CJK characters and routes queries containing Han, +Hiragana, Katakana, or Hangul through that index. Query preparation depends on +the scripts in the query: + +- Queries containing Japanese kana or Korean Hangul preserve character order + and adjacency within each whitespace-separated search term. For example, + `かな` does not match `なか`, and `검색` does not match separate occurrences + of `검` and `색`. Separate terms can match anywhere in the same message. +- Queries containing Han without kana or Hangul use Chinese word segmentation + through cppjieba. Japanese queries written entirely in kanji take this same + path because the scripts alone do not distinguish the languages. Quote a + kanji phrase, such as `"検索方法"`, to require its characters in order. +- A leading double quote opts into explicit FTS5 expressions, including phrases + and operators, in any language. For example, `"검색 기능"` requires the + two terms together, while `검색 기능` allows intervening text. + +Japanese and Korean matching is character-based; it does not analyze grammatical +word forms or expand readings, romanizations, or spelling variants. Chinese word +segmentation also stays off when a query mixes Han with kana or Hangul. + +ASCII-only searches continue to use the existing Porter index, so searches such +as `run` retain English stemming. The `messages_cjk_fts` index is derived data: +if the sidecar is removed, AgentsView drops that optional index and continues with the standard FTS5 path; reinstalling the sidecar backfills it on the next writable open. AgentsView fingerprints the native library and all cppjieba dictionaries, atomically rebuilding the index when that fingerprint @@ -1327,7 +1350,7 @@ changes. Writers running with another fingerprint leave a freshness marker instead of mixing incompatible token streams. Pinyin expansion is disabled in the derived index because ASCII-only queries continue to use the Porter index. -Chinese word segmentation is specific to SQLite message search, including the +CJK full-text search is specific to SQLite message search, including the HTTP, CLI, and MCP search paths. PostgreSQL/CockroachDB and DuckDB do not load this SQLite extension and keep their existing search behavior. Substring and regular-expression searches are unchanged. Session search result snippets @@ -1338,18 +1361,18 @@ The first backfill, a changed fingerprint, or any pending session requires a full index rebuild before startup completes. AgentsView logs this wait. The freshness ledger stores session IDs rather than old message IDs and token content, so it cannot remove stale entries for individual replaced or deleted -messages. Removing the sidecar drops the Chinese index but retains the -`messages_chinese_fts_pending_sessions` ledger and three persistent session +messages. Removing the sidecar drops the CJK index but retains the +`messages_cjk_fts_pending_sessions` ledger and three persistent session triggers. The ledger holds at most one row per touched session ID until the -next successful Chinese index rebuild clears it. +next successful CJK index rebuild clears it. Index maintenance uses TEMP triggers on the writer connection. Writes made without these triggers or with another sidecar fingerprint leave the index -stale. Chinese search then falls back to standard FTS5 and logs a warning once +stale. CJK search then falls back to standard FTS5 and logs a warning once per database handle. Reopening the archive with the sidecar restores the index and its triggers. -**Schema tables:** +### Schema tables | Table | Purpose | | -------------------- | ---------------------------------------------------------------------------- | @@ -1363,7 +1386,7 @@ and its triggers. | `stats` | Aggregate counts (session_count, message_count) | | `skipped_files` | Cache of non-interactive session files | | `messages_fts` | FTS5 virtual table for full-text search | -| `messages_chinese_fts` | Optional FTS5 index using the `simple` Chinese tokenizer | +| `messages_cjk_fts` | Optional CJK FTS5 index using the `simple` character tokenizer | The database is automatically migrated on startup when the schema changes. When the stored data version is stale, AgentsView preserves the existing database and diff --git a/internal/db/chinese_fts_runtime.go b/internal/db/cjk_fts_runtime.go similarity index 66% rename from internal/db/chinese_fts_runtime.go rename to internal/db/cjk_fts_runtime.go index 02c1a485c..352aaf258 100644 --- a/internal/db/chinese_fts_runtime.go +++ b/internal/db/cjk_fts_runtime.go @@ -16,48 +16,48 @@ import ( const simpleFTSDirEnv = "AGENTSVIEW_SIMPLE_DIR" const ( - chineseFTSFingerprintStatsKey = "messages_chinese_fts_fingerprint_v1" - chineseFTSSchemaVersion = "messages-chinese-fts-v3" + cjkFTSFingerprintStatsKey = "messages_cjk_fts_fingerprint_v1" + cjkFTSSchemaVersion = "messages-cjk-fts-v3" ) var ( simpleFTSRuntimeConfig, simpleFTSRuntimeErr = discoverSimpleFTSRuntime() ) -const schemaChineseFTSPendingSessions = ` -CREATE TABLE IF NOT EXISTS messages_chinese_fts_pending_sessions ( +const schemaCJKFTSPendingSessions = ` +CREATE TABLE IF NOT EXISTS messages_cjk_fts_pending_sessions ( session_id TEXT PRIMARY KEY, generation INTEGER NOT NULL CHECK (generation > 0) ); -DROP TRIGGER IF EXISTS sessions_chinese_pending_bi; -DROP TRIGGER IF EXISTS sessions_chinese_pending_bu; -DROP TRIGGER IF EXISTS sessions_chinese_pending_bd; +DROP TRIGGER IF EXISTS sessions_cjk_pending_bi; +DROP TRIGGER IF EXISTS sessions_cjk_pending_bu; +DROP TRIGGER IF EXISTS sessions_cjk_pending_bd; -CREATE TRIGGER sessions_chinese_pending_bi +CREATE TRIGGER sessions_cjk_pending_bi BEFORE INSERT ON sessions WHEN NOT EXISTS (SELECT 1 FROM sessions WHERE id = new.id) BEGIN - INSERT INTO messages_chinese_fts_pending_sessions(session_id, generation) + INSERT INTO messages_cjk_fts_pending_sessions(session_id, generation) VALUES(new.id, 1) ON CONFLICT(session_id) DO UPDATE SET - generation = messages_chinese_fts_pending_sessions.generation + 1; + generation = messages_cjk_fts_pending_sessions.generation + 1; END; -CREATE TRIGGER sessions_chinese_pending_bu +CREATE TRIGGER sessions_cjk_pending_bu BEFORE UPDATE OF transcript_revision ON sessions WHEN old.transcript_revision IS NOT new.transcript_revision BEGIN - INSERT INTO messages_chinese_fts_pending_sessions(session_id, generation) + INSERT INTO messages_cjk_fts_pending_sessions(session_id, generation) VALUES(new.id, 1) ON CONFLICT(session_id) DO UPDATE SET - generation = messages_chinese_fts_pending_sessions.generation + 1; + generation = messages_cjk_fts_pending_sessions.generation + 1; END; -CREATE TRIGGER sessions_chinese_pending_bd +CREATE TRIGGER sessions_cjk_pending_bd BEFORE DELETE ON sessions BEGIN - INSERT INTO messages_chinese_fts_pending_sessions(session_id, generation) + INSERT INTO messages_cjk_fts_pending_sessions(session_id, generation) VALUES(old.id, 1) ON CONFLICT(session_id) DO UPDATE SET - generation = messages_chinese_fts_pending_sessions.generation + 1; + generation = messages_cjk_fts_pending_sessions.generation + 1; END; ` @@ -178,7 +178,7 @@ func fingerprintSimpleFTSRuntime( libraryPath, dictionaryPath string, ) (string, error) { h := sha256.New() - _, _ = io.WriteString(h, chineseFTSSchemaVersion+"\n") + _, _ = io.WriteString(h, cjkFTSSchemaVersion+"\n") type fingerprintFile struct { name string @@ -216,7 +216,7 @@ func fingerprintSimpleFTSRuntime( } _, _ = io.WriteString(h, "\x00") } - return fmt.Sprintf("%s:%x", chineseFTSSchemaVersion, h.Sum(nil)), nil + return fmt.Sprintf("%s:%x", cjkFTSSchemaVersion, h.Sum(nil)), nil } func simpleFTSLibraryName(goos string) (string, error) { @@ -243,32 +243,32 @@ func requireRegularFile(path string) error { return nil } -type chineseFTSTransactor interface { +type cjkFTSTransactor interface { BeginTx(context.Context, *sql.TxOptions) (*sql.Tx, error) } -// ensureChineseFTS atomically reconciles the derived Chinese index with the +// ensureCJKFTS atomically reconciles the derived CJK index with the // loaded extension and dictionaries. The table, complete backfill, fingerprint, // and connection-local maintenance triggers become visible together. -func ensureChineseFTS( - ctx context.Context, conn chineseFTSTransactor, forceRebuild bool, +func ensureCJKFTS( + ctx context.Context, conn cjkFTSTransactor, forceRebuild bool, ) error { tx, err := conn.BeginTx(ctx, nil) if err != nil { - return fmt.Errorf("beginning Chinese FTS transaction: %w", err) + return fmt.Errorf("beginning CJK FTS transaction: %w", err) } defer func() { _ = tx.Rollback() }() for _, trigger := range []string{ - "messages_chinese_ai", - "messages_chinese_ad", - "messages_chinese_au", - "sessions_chinese_pending_ai", - "sessions_chinese_pending_au", - "sessions_chinese_pending_ad", + "messages_cjk_ai", + "messages_cjk_ad", + "messages_cjk_au", + "sessions_cjk_pending_ai", + "sessions_cjk_pending_au", + "sessions_cjk_pending_ad", } { if _, err := tx.ExecContext(ctx, "DROP TRIGGER IF EXISTS "+trigger); err != nil { - return fmt.Errorf("dropping Chinese FTS trigger %s: %w", trigger, err) + return fmt.Errorf("dropping CJK FTS trigger %s: %w", trigger, err) } } @@ -276,9 +276,9 @@ func ensureChineseFTS( if err := tx.QueryRowContext(ctx, ` SELECT EXISTS( SELECT 1 FROM sqlite_master - WHERE type = 'table' AND name = 'messages_chinese_fts' + WHERE type = 'table' AND name = 'messages_cjk_fts' )`).Scan(&tableExists); err != nil { - return fmt.Errorf("checking Chinese FTS table: %w", err) + return fmt.Errorf("checking CJK FTS table: %w", err) } var pendingTableExists bool @@ -286,48 +286,48 @@ func ensureChineseFTS( SELECT EXISTS( SELECT 1 FROM sqlite_master WHERE type = 'table' - AND name = 'messages_chinese_fts_pending_sessions' + AND name = 'messages_cjk_fts_pending_sessions' )`).Scan(&pendingTableExists); err != nil { - return fmt.Errorf("checking Chinese FTS freshness ledger table: %w", err) + return fmt.Errorf("checking CJK FTS freshness ledger table: %w", err) } trackFreshness := simpleFTSRuntimeConfig.available() || tableExists || pendingTableExists if !trackFreshness { if _, err := tx.ExecContext( - ctx, "DELETE FROM stats WHERE key = ?", chineseFTSFingerprintStatsKey, + ctx, "DELETE FROM stats WHERE key = ?", cjkFTSFingerprintStatsKey, ); err != nil { - return fmt.Errorf("clearing orphaned Chinese FTS fingerprint: %w", err) + return fmt.Errorf("clearing orphaned CJK FTS fingerprint: %w", err) } if err := tx.Commit(); err != nil { - return fmt.Errorf("committing disabled Chinese FTS state: %w", err) + return fmt.Errorf("committing disabled CJK FTS state: %w", err) } return nil } - if _, err := tx.ExecContext(ctx, schemaChineseFTSPendingSessions); err != nil { - return fmt.Errorf("installing Chinese FTS freshness ledger: %w", err) + if _, err := tx.ExecContext(ctx, schemaCJKFTSPendingSessions); err != nil { + return fmt.Errorf("installing CJK FTS freshness ledger: %w", err) } var pendingSessions int if err := tx.QueryRowContext(ctx, - "SELECT count(*) FROM messages_chinese_fts_pending_sessions", + "SELECT count(*) FROM messages_cjk_fts_pending_sessions", ).Scan(&pendingSessions); err != nil { - return fmt.Errorf("checking Chinese FTS freshness ledger: %w", err) + return fmt.Errorf("checking CJK FTS freshness ledger: %w", err) } if !simpleFTSRuntimeConfig.available() { if tableExists { - if _, err := tx.ExecContext(ctx, "DROP TABLE messages_chinese_fts"); err != nil { - return fmt.Errorf("dropping unavailable Chinese FTS: %w", err) + if _, err := tx.ExecContext(ctx, "DROP TABLE messages_cjk_fts"); err != nil { + return fmt.Errorf("dropping unavailable CJK FTS: %w", err) } } if _, err := tx.ExecContext( - ctx, "DELETE FROM stats WHERE key = ?", chineseFTSFingerprintStatsKey, + ctx, "DELETE FROM stats WHERE key = ?", cjkFTSFingerprintStatsKey, ); err != nil { - return fmt.Errorf("clearing Chinese FTS fingerprint: %w", err) + return fmt.Errorf("clearing CJK FTS fingerprint: %w", err) } if err := tx.Commit(); err != nil { - return fmt.Errorf("committing Chinese FTS removal: %w", err) + return fmt.Errorf("committing CJK FTS removal: %w", err) } return nil } @@ -335,53 +335,53 @@ func ensureChineseFTS( var storedFingerprint string fingerprintErr := tx.QueryRowContext(ctx, "SELECT CAST(value AS TEXT) FROM stats WHERE key = ?", - chineseFTSFingerprintStatsKey, + cjkFTSFingerprintStatsKey, ).Scan(&storedFingerprint) if fingerprintErr != nil && !errors.Is(fingerprintErr, sql.ErrNoRows) { - return fmt.Errorf("reading Chinese FTS fingerprint: %w", fingerprintErr) + return fmt.Errorf("reading CJK FTS fingerprint: %w", fingerprintErr) } current := tableExists && fingerprintErr == nil && pendingSessions == 0 && storedFingerprint == simpleFTSRuntimeConfig.fingerprint if forceRebuild || !current { - log.Print("rebuilding Chinese FTS index; startup waits for the full message scan to finish") + log.Print("rebuilding CJK FTS index; startup waits for the full message scan to finish") if tableExists { - if _, err := tx.ExecContext(ctx, "DROP TABLE messages_chinese_fts"); err != nil { - return fmt.Errorf("dropping stale Chinese FTS: %w", err) + if _, err := tx.ExecContext(ctx, "DROP TABLE messages_cjk_fts"); err != nil { + return fmt.Errorf("dropping stale CJK FTS: %w", err) } } - if _, err := tx.ExecContext(ctx, schemaChineseFTS); err != nil { - return fmt.Errorf("creating Chinese FTS: %w", err) + if _, err := tx.ExecContext(ctx, schemaCJKFTS); err != nil { + return fmt.Errorf("creating CJK FTS: %w", err) } if _, err := tx.ExecContext(ctx, - "INSERT INTO messages_chinese_fts(messages_chinese_fts) VALUES('rebuild')", + "INSERT INTO messages_cjk_fts(messages_cjk_fts) VALUES('rebuild')", ); err != nil { - return fmt.Errorf("backfilling Chinese FTS: %w", err) + return fmt.Errorf("backfilling CJK FTS: %w", err) } if _, err := tx.ExecContext(ctx, ` INSERT INTO stats (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`, - chineseFTSFingerprintStatsKey, + cjkFTSFingerprintStatsKey, simpleFTSRuntimeConfig.fingerprint, ); err != nil { - return fmt.Errorf("storing Chinese FTS fingerprint: %w", err) + return fmt.Errorf("storing CJK FTS fingerprint: %w", err) } if _, err := tx.ExecContext( - ctx, "DELETE FROM messages_chinese_fts_pending_sessions", + ctx, "DELETE FROM messages_cjk_fts_pending_sessions", ); err != nil { - return fmt.Errorf("clearing Chinese FTS freshness ledger: %w", err) + return fmt.Errorf("clearing CJK FTS freshness ledger: %w", err) } } - if _, err := tx.ExecContext(ctx, schemaChineseFTSTriggers); err != nil { - return fmt.Errorf("installing Chinese FTS triggers: %w", err) + if _, err := tx.ExecContext(ctx, schemaCJKFTSTriggers); err != nil { + return fmt.Errorf("installing CJK FTS triggers: %w", err) } if err := tx.Commit(); err != nil { - return fmt.Errorf("committing Chinese FTS transaction: %w", err) + return fmt.Errorf("committing CJK FTS transaction: %w", err) } return nil } -func installChineseFTSTriggers(conn *sql.DB) error { - return ensureChineseFTS(context.Background(), conn, false) +func installCJKFTSTriggers(conn *sql.DB) error { + return ensureCJKFTS(context.Background(), conn, false) } diff --git a/internal/db/chinese_fts_search.go b/internal/db/cjk_fts_search.go similarity index 72% rename from internal/db/chinese_fts_search.go rename to internal/db/cjk_fts_search.go index b364cc2ac..6184ea987 100644 --- a/internal/db/chinese_fts_search.go +++ b/internal/db/cjk_fts_search.go @@ -25,22 +25,31 @@ func (db *DB) prepareMessageFTSQuery( match: prepared, plain: StripFTSQuotes(prepared), } - if prepared == "" || !containsCJK(trimmed) || !db.HasChineseFTS() { + if prepared == "" || !containsCJK(trimmed) || !db.HasCJKFTS() { return query, nil } - query.table = "messages_chinese_fts" + query.table = "messages_cjk_fts" if strings.HasPrefix(trimmed, `"`) { // A leading quote is the established opt-in for an explicit FTS5 // expression. Preserve phrases, operators, and grouping verbatim. query.match = prepared return query, nil } + if strings.ContainsFunc(trimmed, func(r rune) bool { + return unicode.In(r, unicode.Hiragana, unicode.Katakana, unicode.Hangul) + }) { + // Jieba turns kana and Hangul queries into independent character + // matches. Keep each whitespace-delimited term as a phrase in the + // character index instead. Han-only queries remain Chinese-segmented; + // callers can quote Japanese kanji phrases to preserve their order. + return query, nil + } conn, err := db.getReader().Conn(ctx) if err != nil { return messageFTSQuery{}, fmt.Errorf( - "acquiring Chinese FTS query connection: %w", err, + "acquiring CJK FTS query connection: %w", err, ) } defer conn.Close() @@ -52,12 +61,12 @@ func (db *DB) prepareMessageFTSQuery( simpleFTSJiebaMu.Unlock() if err != nil { return messageFTSQuery{}, fmt.Errorf( - "preparing Chinese FTS query: %w", err, + "preparing CJK FTS query: %w", err, ) } if strings.TrimSpace(query.match) == "" { return messageFTSQuery{}, &SearchInputError{ - Msg: "search: Chinese FTS query is empty after tokenization", + Msg: "search: CJK FTS query is empty after tokenization", } } // jieba_query joins terms with AND and may add an unquoted prefix '*'. diff --git a/internal/db/cjk_fts_search_test.go b/internal/db/cjk_fts_search_test.go new file mode 100644 index 000000000..929081e60 --- /dev/null +++ b/internal/db/cjk_fts_search_test.go @@ -0,0 +1,60 @@ +package db + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// These searches exercise the store's choice of query preparation against the +// real optional index. Chinese segmentation must not turn kana or Hangul terms +// into unordered character matches. +func TestCJKFTSJapaneseAndKoreanSearch(t *testing.T) { + if !simpleFTSRuntimeConfig.available() { + t.Skip("simple FTS5 runtime is not installed for this test process") + } + for _, tc := range []struct { + name string + query string + match string + miss string + }{ + {"hiragana order", "かな", "かなを探します。", "なかを探します。"}, + {"katakana adjacency", "カタカナ", "カタカナを探します。", "カナとカタを探します。"}, + {"kanji and kana", "検索方法を", "検索方法を説明します。", "方法を変えて検索します。"}, + {"quoted kanji", `"検索方法"`, "検索方法を説明します。", "方法を変えて検索します。"}, + {"single kana", "ぬ", "いぬを探します。", "ねこを探します。"}, + {"hangul adjacency", "검색", "검색합니다.", "색상 검토입니다."}, + {"hangul order", "검색", "검색합니다.", "색검합니다."}, + {"single hangul syllable", "검", "검색합니다.", "색상입니다."}, + {"japanese with latin", "SQLite 検索します", "SQLiteで検索します。", "SQLiteで検索し、別の作業をします。"}, + {"korean with latin", "SQLite 검색", "SQLite로 검색합니다.", "SQLite 색상 검토입니다."}, + {"separate japanese terms", "かな カタカナ", "カタカナとかなを探します。", "カタカナを探します。"}, + {"separate korean terms", "검색 기능", "기능을 추가해 검색합니다.", "검색합니다."}, + {"quoted japanese phrase", `"かな カタカナ"`, "かな カタカナを探します。", "カタカナとかなを探します。"}, + {"quoted korean phrase", `"검색 기능"`, "검색 기능을 추가합니다.", "기능을 추가해 검색합니다."}, + } { + t.Run(tc.name, func(t *testing.T) { + d := testDB(t) + seedSearchSession(t, d, "match", "proj", [][2]string{{"user", tc.match}}) + seedSearchSession(t, d, "miss", "proj", [][2]string{{"user", tc.miss}}) + + results, err := d.Search(context.Background(), SearchFilter{Query: tc.query, Limit: 20}) + require.NoError(t, err) + require.Len(t, results.Results, 1, "session search must preserve term adjacency and order") + assert.Equal(t, "match", results.Results[0].SessionID) + + content, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: tc.query, + Mode: "fts", + Sources: []string{"messages"}, + Limit: 20, + }) + require.NoError(t, err) + require.Len(t, content.Matches, 1, "content search must use the same CJK query preparation") + assert.Equal(t, "match", content.Matches[0].SessionID) + }) + } +} diff --git a/internal/db/chinese_fts_test.go b/internal/db/cjk_fts_test.go similarity index 86% rename from internal/db/chinese_fts_test.go rename to internal/db/cjk_fts_test.go index c293d8013..da700e71b 100644 --- a/internal/db/chinese_fts_test.go +++ b/internal/db/cjk_fts_test.go @@ -82,7 +82,7 @@ func TestDiscoverSimpleFTSRuntimeExplicitDirIsValidated(t *testing.T) { assert.Contains(t, err.Error(), simpleFTSDirEnv) } -func TestChineseFTSSearch(t *testing.T) { +func TestCJKFTSChineseSearch(t *testing.T) { if !simpleFTSRuntimeConfig.available() { t.Skip("simple FTS5 runtime is not installed for this test process") } @@ -103,7 +103,7 @@ func TestChineseFTSSearch(t *testing.T) { var pending int require.NoError(t, d.getReader().QueryRow( - "SELECT count(*) FROM messages_chinese_fts_pending_sessions", + "SELECT count(*) FROM messages_cjk_fts_pending_sessions", ).Scan(&pending)) assert.Zero(t, pending) @@ -116,8 +116,8 @@ func TestChineseFTSSearch(t *testing.T) { require.NoError(t, err) var pinyinHits int require.NoError(t, d.getReader().QueryRow( - `SELECT count(*) FROM messages_chinese_fts - WHERE messages_chinese_fts MATCH ?`, pinyinMatch, + `SELECT count(*) FROM messages_cjk_fts + WHERE messages_cjk_fts MATCH ?`, pinyinMatch, ).Scan(&pinyinHits)) assert.Zero(t, pinyinHits) @@ -213,24 +213,24 @@ func TestChineseFTSSearch(t *testing.T) { var storedFingerprint string require.NoError(t, d.getReader().QueryRow( "SELECT CAST(value AS TEXT) FROM stats WHERE key = ?", - chineseFTSFingerprintStatsKey, + cjkFTSFingerprintStatsKey, ).Scan(&storedFingerprint)) assert.Equal(t, simpleFTSRuntimeConfig.fingerprint, storedFingerprint) // Simulate a pre-fix partial build: the table exists without the atomic // completion fingerprint. Reopen must replace and backfill it. _, err = d.getWriter().Exec(` - DROP TRIGGER IF EXISTS messages_chinese_ai; - DROP TRIGGER IF EXISTS messages_chinese_ad; - DROP TRIGGER IF EXISTS messages_chinese_au; - DROP TABLE messages_chinese_fts; - CREATE VIRTUAL TABLE messages_chinese_fts USING fts5( + DROP TRIGGER IF EXISTS messages_cjk_ai; + DROP TRIGGER IF EXISTS messages_cjk_ad; + DROP TRIGGER IF EXISTS messages_cjk_au; + DROP TABLE messages_cjk_fts; + CREATE VIRTUAL TABLE messages_cjk_fts USING fts5( content, content='messages', content_rowid='id', tokenize='simple' ); - DELETE FROM stats WHERE key = '` + chineseFTSFingerprintStatsKey + `'`) + DELETE FROM stats WHERE key = '` + cjkFTSFingerprintStatsKey + `'`) require.NoError(t, err) require.NoError(t, d.Reopen()) repaired, err := d.SearchContent(context.Background(), ContentSearchFilter{ @@ -245,12 +245,12 @@ func TestChineseFTSSearch(t *testing.T) { _, err = d.getWriter().Exec( "UPDATE stats SET value = 'stale' WHERE key = ?", - chineseFTSFingerprintStatsKey, + cjkFTSFingerprintStatsKey, ) require.NoError(t, err) - assert.False(t, d.HasChineseFTS()) + assert.False(t, d.HasCJKFTS()) require.NoError(t, d.Reopen()) - assert.True(t, d.HasChineseFTS()) + assert.True(t, d.HasCJKFTS()) require.NoError(t, d.CloseWriter()) require.NoError(t, d.ReopenWriter()) @@ -282,7 +282,7 @@ func TestChineseFTSSearch(t *testing.T) { assert.Equal(t, "swapped", swapped.Matches[0].SessionID) } -func TestChineseFTSContentSnippetCentersOnMatch(t *testing.T) { +func TestCJKFTSContentSnippetCentersOnMatch(t *testing.T) { if !simpleFTSRuntimeConfig.available() { t.Skip("simple FTS5 runtime is not installed for this test process") } @@ -311,7 +311,7 @@ func TestChineseFTSContentSnippetCentersOnMatch(t *testing.T) { } } -func TestChineseFTSTableCanBeDroppedWithoutExtension(t *testing.T) { +func TestCJKFTSTableCanBeDroppedWithoutExtension(t *testing.T) { if !simpleFTSRuntimeConfig.available() { t.Skip("simple FTS5 runtime is not installed for this test process") } @@ -323,11 +323,11 @@ func TestChineseFTSTableCanBeDroppedWithoutExtension(t *testing.T) { raw, err := sql.Open("sqlite3", makeDSN(path, false)) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, raw.Close()) }) - _, err = raw.Exec("DROP TABLE messages_chinese_fts") + _, err = raw.Exec("DROP TABLE messages_cjk_fts") require.NoError(t, err) } -func TestChineseFTSRebuildsAfterLegacyWriter(t *testing.T) { +func TestCJKFTSRebuildsAfterLegacyWriter(t *testing.T) { if !simpleFTSRuntimeConfig.available() { t.Skip("simple FTS5 runtime is not installed for this test process") } @@ -357,32 +357,32 @@ func TestChineseFTSRebuildsAfterLegacyWriter(t *testing.T) { var pending int require.NoError(t, raw.QueryRow( - "SELECT count(*) FROM messages_chinese_fts_pending_sessions", + "SELECT count(*) FROM messages_cjk_fts_pending_sessions", ).Scan(&pending)) assert.Equal(t, 1, pending) require.NoError(t, raw.Close()) - require.False(t, d.HasChineseFTS()) + require.False(t, d.HasCJKFTS()) // A local metadata update does not repair the other writer's message index. name := "Renamed session" require.NoError(t, d.RenameSession("legacy", &name)) - require.False(t, d.HasChineseFTS(), "renaming must preserve the stale marker") + require.False(t, d.HasCJKFTS(), "renaming must preserve the stale marker") insertSession(t, d, "legacy", "proj", func(s *Session) { s.UserMessageCount = 2 }) - require.False(t, d.HasChineseFTS(), "upserts must preserve the stale marker") + require.False(t, d.HasCJKFTS(), "upserts must preserve the stale marker") require.NoError(t, d.InsertMessages([]Message{{ SessionID: "legacy", Ordinal: 1, Role: "assistant", Content: "本地追加的消息。", }})) - require.False(t, d.HasChineseFTS(), "appends do not repair earlier stale content") + require.False(t, d.HasCJKFTS(), "appends do not repair earlier stale content") var output bytes.Buffer previousWriter := log.Writer() log.SetOutput(&output) t.Cleanup(func() { log.SetOutput(previousWriter) }) require.NoError(t, d.Reopen()) - assert.Contains(t, output.String(), "rebuilding Chinese FTS index; startup waits") + assert.Contains(t, output.String(), "rebuilding CJK FTS index; startup waits") page, err := d.SearchContent(context.Background(), ContentSearchFilter{ Pattern: "旧版本写入", Mode: "fts", @@ -394,12 +394,12 @@ func TestChineseFTSRebuildsAfterLegacyWriter(t *testing.T) { assert.Equal(t, "legacy", page.Matches[0].SessionID) require.NoError(t, d.getReader().QueryRow( - "SELECT count(*) FROM messages_chinese_fts_pending_sessions", + "SELECT count(*) FROM messages_cjk_fts_pending_sessions", ).Scan(&pending)) assert.Zero(t, pending) } -func TestChineseFTSForeignFingerprintDefersMaintenance(t *testing.T) { +func TestCJKFTSForeignFingerprintDefersMaintenance(t *testing.T) { if !simpleFTSRuntimeConfig.available() { t.Skip("simple FTS5 runtime is not installed for this test process") } @@ -414,12 +414,12 @@ func TestChineseFTSForeignFingerprintDefersMaintenance(t *testing.T) { _, err := d.getWriter().Exec( "UPDATE stats SET value = 'foreign-runtime' WHERE key = ?", - chineseFTSFingerprintStatsKey, + cjkFTSFingerprintStatsKey, ) require.NoError(t, err) - assert.False(t, d.HasChineseFTS()) - assert.False(t, d.HasChineseFTS()) - assert.Equal(t, 1, strings.Count(output.String(), "Chinese FTS unavailable or stale")) + assert.False(t, d.HasCJKFTS()) + assert.False(t, d.HasCJKFTS()) + assert.Equal(t, 1, strings.Count(output.String(), "CJK FTS unavailable or stale")) require.NoError(t, d.Update(func(tx *sql.Tx) error { if _, err := tx.Exec( @@ -439,7 +439,7 @@ func TestChineseFTSForeignFingerprintDefersMaintenance(t *testing.T) { var pending int require.NoError(t, d.getReader().QueryRow( - "SELECT count(*) FROM messages_chinese_fts_pending_sessions", + "SELECT count(*) FROM messages_cjk_fts_pending_sessions", ).Scan(&pending)) assert.Equal(t, 1, pending) @@ -453,13 +453,13 @@ func TestChineseFTSForeignFingerprintDefersMaintenance(t *testing.T) { var staleMatches int require.NoError(t, d.getReader().QueryRow( - `SELECT count(*) FROM messages_chinese_fts - WHERE messages_chinese_fts MATCH ?`, match, + `SELECT count(*) FROM messages_cjk_fts + WHERE messages_cjk_fts MATCH ?`, match, ).Scan(&staleMatches)) assert.Zero(t, staleMatches) require.NoError(t, d.Reopen()) - assert.True(t, d.HasChineseFTS()) + assert.True(t, d.HasCJKFTS()) page, err := d.SearchContent(context.Background(), ContentSearchFilter{ Pattern: "跨版本写入", Mode: "fts", @@ -470,12 +470,12 @@ func TestChineseFTSForeignFingerprintDefersMaintenance(t *testing.T) { require.NotEmpty(t, page.Matches) assert.Equal(t, "foreign-runtime", page.Matches[0].SessionID) require.NoError(t, d.getReader().QueryRow( - "SELECT count(*) FROM messages_chinese_fts_pending_sessions", + "SELECT count(*) FROM messages_cjk_fts_pending_sessions", ).Scan(&pending)) assert.Zero(t, pending) } -func TestChineseFTSJiebaConfigurationSerializesWithQueries(t *testing.T) { +func TestCJKFTSJiebaConfigurationSerializesWithQueries(t *testing.T) { if !simpleFTSRuntimeConfig.available() { t.Skip("simple FTS5 runtime is not installed for this test process") } @@ -529,7 +529,7 @@ func TestChineseFTSJiebaConfigurationSerializesWithQueries(t *testing.T) { } // Session upserts and no-op recall inserts must leave a healthy index usable. -func TestChineseFTSSurvivesSessionResyncUpsert(t *testing.T) { +func TestCJKFTSSurvivesSessionResyncUpsert(t *testing.T) { if !simpleFTSRuntimeConfig.available() { t.Skip("simple FTS5 runtime is not installed for this test process") } @@ -537,7 +537,7 @@ func TestChineseFTSSurvivesSessionResyncUpsert(t *testing.T) { seedSearchSession(t, d, "resync", "proj", [][2]string{ {"user", "中文搜索必须在重新同步之后仍然可用。"}, }) - require.True(t, d.HasChineseFTS(), "Chinese FTS live after the first write") + require.True(t, d.HasCJKFTS(), "CJK FTS live after the first write") // Re-upsert the same session id, leaving transcript_revision alone. This // is the shape of every ordinary resync of an unchanged session. @@ -552,10 +552,10 @@ func TestChineseFTSSurvivesSessionResyncUpsert(t *testing.T) { var pending int require.NoError(t, d.getReader().QueryRow( - "SELECT count(*) FROM messages_chinese_fts_pending_sessions", + "SELECT count(*) FROM messages_cjk_fts_pending_sessions", ).Scan(&pending)) assert.Zero(t, pending, "resync upsert must not strand a pending row") - assert.True(t, d.HasChineseFTS(), "Chinese FTS stays live across a resync") + assert.True(t, d.HasCJKFTS(), "CJK FTS stays live across a resync") page, err := d.SearchContent(context.Background(), ContentSearchFilter{ Pattern: "重新同步", @@ -568,12 +568,12 @@ func TestChineseFTSSurvivesSessionResyncUpsert(t *testing.T) { assert.Equal(t, "resync", page.Matches[0].SessionID) } -// TestChineseFTSSurvivesCompaction pins that staged archive compaction keeps -// the optional Chinese index queryable. Compaction rebuilds the archive into a +// TestCJKFTSSurvivesCompaction pins that staged archive compaction keeps +// the optional CJK index queryable. Compaction rebuilds the archive into a // candidate file through maintenance connections that do not load the // tokenizer sidecar, and then swaps that candidate in, so the index surviving // the round trip is worth holding still. -func TestChineseFTSSurvivesCompaction(t *testing.T) { +func TestCJKFTSSurvivesCompaction(t *testing.T) { if !simpleFTSRuntimeConfig.available() { t.Skip("simple FTS5 runtime is not installed for this test process") } @@ -581,14 +581,14 @@ func TestChineseFTSSurvivesCompaction(t *testing.T) { seedSearchSession(t, d, "compact-cn", "proj", [][2]string{ {"user", "压缩之后中文索引必须继续可用。"}, }) - require.True(t, d.HasChineseFTS(), "Chinese FTS live before compaction") + require.True(t, d.HasCJKFTS(), "CJK FTS live before compaction") _, err := d.Compact(context.Background(), CompactOptions{ StagingDir: t.TempDir(), }) - require.NoError(t, err, "compaction must not fail on an archive with a Chinese index") + require.NoError(t, err, "compaction must not fail on an archive with a CJK index") - assert.True(t, d.HasChineseFTS(), "Chinese FTS still live after compaction") + assert.True(t, d.HasCJKFTS(), "CJK FTS still live after compaction") page, err := d.SearchContent(context.Background(), ContentSearchFilter{ Pattern: "中文索引", Mode: "fts", diff --git a/internal/db/db.go b/internal/db/db.go index bed900d2f..cb1c06095 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -556,17 +556,17 @@ CREATE TRIGGER IF NOT EXISTS messages_ad AFTER DELETE ON messages BEGIN END; ` -const chineseFTSRuntimeMatchesSQL = `EXISTS ( +const cjkFTSRuntimeMatchesSQL = `EXISTS ( SELECT 1 FROM main.stats - WHERE key = '` + chineseFTSFingerprintStatsKey + `' - AND CAST(value AS TEXT) = agentsview_chinese_fts_fingerprint() + WHERE key = '` + cjkFTSFingerprintStatsKey + `' + AND CAST(value AS TEXT) = agentsview_cjk_fts_fingerprint() )` -const messagesChineseADTriggerDDL = ` -CREATE TEMP TRIGGER IF NOT EXISTS messages_chinese_ad +const messagesCJKADTriggerDDL = ` +CREATE TEMP TRIGGER IF NOT EXISTS messages_cjk_ad AFTER DELETE ON main.messages -WHEN ` + chineseFTSRuntimeMatchesSQL + ` BEGIN - INSERT INTO messages_chinese_fts(messages_chinese_fts, rowid, content) +WHEN ` + cjkFTSRuntimeMatchesSQL + ` BEGIN + INSERT INTO messages_cjk_fts(messages_cjk_fts, rowid, content) VALUES('delete', old.id, old.content); END; ` @@ -590,8 +590,8 @@ CREATE TRIGGER IF NOT EXISTS messages_au AFTER UPDATE ON messages BEGIN END; ` -const schemaChineseFTS = ` -CREATE VIRTUAL TABLE IF NOT EXISTS messages_chinese_fts USING fts5( +const schemaCJKFTS = ` +CREATE VIRTUAL TABLE IF NOT EXISTS messages_cjk_fts USING fts5( content, content='messages', content_rowid='id', @@ -599,19 +599,19 @@ CREATE VIRTUAL TABLE IF NOT EXISTS messages_chinese_fts USING fts5( ); ` -const schemaChineseFTSTriggers = ` -CREATE TEMP TRIGGER IF NOT EXISTS messages_chinese_ai +const schemaCJKFTSTriggers = ` +CREATE TEMP TRIGGER IF NOT EXISTS messages_cjk_ai AFTER INSERT ON main.messages -WHEN ` + chineseFTSRuntimeMatchesSQL + ` BEGIN - INSERT INTO messages_chinese_fts(rowid, content) VALUES (new.id, new.content); +WHEN ` + cjkFTSRuntimeMatchesSQL + ` BEGIN + INSERT INTO messages_cjk_fts(rowid, content) VALUES (new.id, new.content); END; -` + messagesChineseADTriggerDDL + ` -CREATE TEMP TRIGGER IF NOT EXISTS messages_chinese_au +` + messagesCJKADTriggerDDL + ` +CREATE TEMP TRIGGER IF NOT EXISTS messages_cjk_au AFTER UPDATE ON main.messages -WHEN ` + chineseFTSRuntimeMatchesSQL + ` BEGIN - INSERT INTO messages_chinese_fts(messages_chinese_fts, rowid, content) +WHEN ` + cjkFTSRuntimeMatchesSQL + ` BEGIN + INSERT INTO messages_cjk_fts(messages_cjk_fts, rowid, content) VALUES('delete', old.id, old.content); - INSERT INTO messages_chinese_fts(rowid, content) VALUES (new.id, new.content); + INSERT INTO messages_cjk_fts(rowid, content) VALUES (new.id, new.content); END; -- The persistent BEFORE triggers mark a session pending without consulting @@ -620,23 +620,23 @@ END; -- 1, which was created by this write. Higher generations include an earlier -- unmaintained write and must survive until the index is rebuilt. The BEFORE -- INSERT trigger ignores existing sessions, so an upsert marks at most once. -CREATE TEMP TRIGGER IF NOT EXISTS sessions_chinese_pending_ai +CREATE TEMP TRIGGER IF NOT EXISTS sessions_cjk_pending_ai AFTER INSERT ON main.sessions -WHEN ` + chineseFTSRuntimeMatchesSQL + ` BEGIN - DELETE FROM messages_chinese_fts_pending_sessions +WHEN ` + cjkFTSRuntimeMatchesSQL + ` BEGIN + DELETE FROM messages_cjk_fts_pending_sessions WHERE session_id = new.id AND generation = 1; END; -CREATE TEMP TRIGGER IF NOT EXISTS sessions_chinese_pending_au +CREATE TEMP TRIGGER IF NOT EXISTS sessions_cjk_pending_au AFTER UPDATE OF transcript_revision ON main.sessions WHEN old.transcript_revision IS NOT new.transcript_revision - AND ` + chineseFTSRuntimeMatchesSQL + ` BEGIN - DELETE FROM messages_chinese_fts_pending_sessions + AND ` + cjkFTSRuntimeMatchesSQL + ` BEGIN + DELETE FROM messages_cjk_fts_pending_sessions WHERE session_id = new.id AND generation = 1; END; -CREATE TEMP TRIGGER IF NOT EXISTS sessions_chinese_pending_ad +CREATE TEMP TRIGGER IF NOT EXISTS sessions_cjk_pending_ad AFTER DELETE ON main.sessions -WHEN ` + chineseFTSRuntimeMatchesSQL + ` BEGIN - DELETE FROM messages_chinese_fts_pending_sessions +WHEN ` + cjkFTSRuntimeMatchesSQL + ` BEGIN + DELETE FROM messages_cjk_fts_pending_sessions WHERE session_id = old.id AND generation = 1; END; ` @@ -797,7 +797,7 @@ type DB struct { // session history. messagesLoadCount atomic.Int64 - chineseFTSUnavailableLog sync.Once + cjkFTSUnavailableLog sync.Once } // MessagesLoadCount returns the total number of GetAllMessages calls the @@ -4434,13 +4434,13 @@ func (db *DB) DropFTS() error { db.mu.Lock() defer db.mu.Unlock() stmts := []string{ - "DROP TRIGGER IF EXISTS messages_chinese_ai", - "DROP TRIGGER IF EXISTS messages_chinese_ad", - "DROP TRIGGER IF EXISTS messages_chinese_au", - "DROP TRIGGER IF EXISTS sessions_chinese_pending_ai", - "DROP TRIGGER IF EXISTS sessions_chinese_pending_au", - "DROP TRIGGER IF EXISTS sessions_chinese_pending_ad", - "DROP TABLE IF EXISTS messages_chinese_fts", + "DROP TRIGGER IF EXISTS messages_cjk_ai", + "DROP TRIGGER IF EXISTS messages_cjk_ad", + "DROP TRIGGER IF EXISTS messages_cjk_au", + "DROP TRIGGER IF EXISTS sessions_cjk_pending_ai", + "DROP TRIGGER IF EXISTS sessions_cjk_pending_au", + "DROP TRIGGER IF EXISTS sessions_cjk_pending_ad", + "DROP TABLE IF EXISTS messages_cjk_fts", "DROP TRIGGER IF EXISTS messages_ai", "DROP TRIGGER IF EXISTS messages_ad", "DROP TRIGGER IF EXISTS messages_au", @@ -4453,9 +4453,9 @@ func (db *DB) DropFTS() error { } } if _, err := w.Exec( - "DELETE FROM stats WHERE key = ?", chineseFTSFingerprintStatsKey, + "DELETE FROM stats WHERE key = ?", cjkFTSFingerprintStatsKey, ); err != nil { - return fmt.Errorf("clearing Chinese fts fingerprint: %w", err) + return fmt.Errorf("clearing CJK fts fingerprint: %w", err) } return nil } @@ -4476,8 +4476,8 @@ func (db *DB) RebuildFTS() error { if err != nil { return fmt.Errorf("rebuild fts index: %w", err) } - if err := ensureChineseFTS(context.Background(), w, true); err != nil { - return fmt.Errorf("rebuild Chinese fts index: %w", err) + if err := ensureCJKFTS(context.Background(), w, true); err != nil { + return fmt.Errorf("rebuild CJK fts index: %w", err) } return nil } @@ -4539,23 +4539,23 @@ func (db *DB) HasFTS() bool { return err == nil } -// HasChineseFTS reports whether the optional simple-tokenized message index is +// HasCJKFTS reports whether the optional simple-tokenized message index is // loaded and queryable on this database connection. -func (db *DB) HasChineseFTS() (available bool) { +func (db *DB) HasCJKFTS() (available bool) { if !simpleFTSRuntimeConfig.available() { return false } defer func() { if !available { - db.chineseFTSUnavailableLog.Do(func() { - log.Print("Chinese FTS unavailable or stale; using standard FTS5 until the archive is reopened") + db.cjkFTSUnavailableLog.Do(func() { + log.Print("CJK FTS unavailable or stale; using standard FTS5 until the archive is reopened") }) } }() var storedFingerprint string if err := db.getReader().QueryRow( "SELECT CAST(value AS TEXT) FROM stats WHERE key = ?", - chineseFTSFingerprintStatsKey, + cjkFTSFingerprintStatsKey, ).Scan(&storedFingerprint); err != nil || storedFingerprint != simpleFTSRuntimeConfig.fingerprint { return false @@ -4563,13 +4563,13 @@ func (db *DB) HasChineseFTS() (available bool) { var hasPendingSessions bool if err := db.getReader().QueryRow(` SELECT EXISTS( - SELECT 1 FROM messages_chinese_fts_pending_sessions LIMIT 1 + SELECT 1 FROM messages_cjk_fts_pending_sessions LIMIT 1 )`, ).Scan(&hasPendingSessions); err != nil || hasPendingSessions { return false } _, err := db.getReader().Exec( - "SELECT 1 FROM messages_chinese_fts LIMIT 1", + "SELECT 1 FROM messages_cjk_fts LIMIT 1", ) return err == nil } @@ -4656,8 +4656,8 @@ func (db *DB) init(ctx context.Context) error { } } - if err := ensureChineseFTS(ctx, w, false); err != nil { - return fmt.Errorf("initializing Chinese FTS: %w", err) + if err := ensureCJKFTS(ctx, w, false); err != nil { + return fmt.Errorf("initializing CJK FTS: %w", err) } var recallFTSCount int @@ -5033,7 +5033,7 @@ func (db *DB) reopenLockedWithBarrier(keepWriterBarrier bool) error { writer.Close() return fmt.Errorf("configuring reopened wal: %w", err) } - if err := installChineseFTSTriggers(writer); err != nil { + if err := installCJKFTSTriggers(writer); err != nil { writer.Close() return fmt.Errorf("configuring reopened writer: %w", err) } @@ -5168,7 +5168,7 @@ func (db *DB) ReopenWriter() error { writer.Close() return fmt.Errorf("configuring reopened wal: %w", err) } - if err := installChineseFTSTriggers(writer); err != nil { + if err := installCJKFTSTriggers(writer); err != nil { writer.Close() return fmt.Errorf("configuring reopened writer: %w", err) } diff --git a/internal/db/messages.go b/internal/db/messages.go index 6b9341b36..87c481017 100644 --- a/internal/db/messages.go +++ b/internal/db/messages.go @@ -2111,24 +2111,24 @@ func sessionHasFTSTableTx(tx transactionQueries, table string) (bool, error) { return ftsCount > 0, nil } -func sessionHasCurrentChineseFTSTx( +func sessionHasCurrentCJKFTSTx( tx transactionQueries, ) (bool, error) { - exists, err := sessionHasFTSTableTx(tx, "messages_chinese_fts") + exists, err := sessionHasFTSTableTx(tx, "messages_cjk_fts") if err != nil || !exists || !simpleFTSRuntimeConfig.available() { return false, err } var storedFingerprint string err = tx.QueryRow( "SELECT CAST(value AS TEXT) FROM stats WHERE key = ?", - chineseFTSFingerprintStatsKey, + cjkFTSFingerprintStatsKey, ).Scan(&storedFingerprint) if errors.Is(err, sql.ErrNoRows) { return false, nil } if err != nil { return false, fmt.Errorf( - "reading Chinese fts fingerprint: %w", err, + "reading CJK fts fingerprint: %w", err, ) } return storedFingerprint == simpleFTSRuntimeConfig.fingerprint, nil @@ -2143,14 +2143,14 @@ func deleteSessionMessageRowsTx( deleteDDL string }{ {"messages_fts", "messages_ad", messagesADTriggerDDL}, - {"messages_chinese_fts", "messages_chinese_ad", messagesChineseADTriggerDDL}, + {"messages_cjk_fts", "messages_cjk_ad", messagesCJKADTriggerDDL}, } active := tables[:0] for _, table := range tables { var exists bool var err error - if table.name == "messages_chinese_fts" { - exists, err = sessionHasCurrentChineseFTSTx(tx) + if table.name == "messages_cjk_fts" { + exists, err = sessionHasCurrentCJKFTSTx(tx) } else { exists, err = sessionHasFTSTableTx(tx, table.name) } diff --git a/internal/db/prepared_testdb_test.go b/internal/db/prepared_testdb_test.go index b8d5abfe3..01dd33a60 100644 --- a/internal/db/prepared_testdb_test.go +++ b/internal/db/prepared_testdb_test.go @@ -19,9 +19,9 @@ func OpenPreparedTestDB(path string) (*DB, error) { writer.Close() return nil, fmt.Errorf("configuring prepared test wal: %w", err) } - if err := installChineseFTSTriggers(writer); err != nil { + if err := installCJKFTSTriggers(writer); err != nil { writer.Close() - return nil, fmt.Errorf("configuring prepared test Chinese FTS: %w", err) + return nil, fmt.Errorf("configuring prepared test CJK FTS: %w", err) } reader, err := sql.Open(sqliteArchiveDriverName, makeDSN(path, true)) diff --git a/internal/db/sqlite_driver.go b/internal/db/sqlite_driver.go index 59caea8b8..6fe17d9c8 100644 --- a/internal/db/sqlite_driver.go +++ b/internal/db/sqlite_driver.go @@ -61,11 +61,11 @@ func configureArchiveSQLiteConnection(conn *sqlite3.SQLiteConn) error { return err } if err := conn.RegisterFunc( - "agentsview_chinese_fts_fingerprint", + "agentsview_cjk_fts_fingerprint", func() string { return simpleFTSRuntimeConfig.fingerprint }, true, ); err != nil { - return fmt.Errorf("registering Chinese FTS fingerprint: %w", err) + return fmt.Errorf("registering CJK FTS fingerprint: %w", err) } if !simpleFTSRuntimeConfig.available() { return nil diff --git a/internal/mcp/tools_test.go b/internal/mcp/tools_test.go index 9cf6202f6..17ceb0fad 100644 --- a/internal/mcp/tools_test.go +++ b/internal/mcp/tools_test.go @@ -83,7 +83,7 @@ func TestSearchSessions_ReturnsHitsWithOrdinal(t *testing.T) { func TestSearchSessions_ChineseSegmentation(t *testing.T) { ts, d := newTestToolset(t) - if !d.HasChineseFTS() { + if !d.HasCJKFTS() { t.Skip("simple FTS5 runtime is not installed for this test process") } seedFTSSession(t, d, "chinese", "proj", @@ -99,6 +99,32 @@ func TestSearchSessions_ChineseSegmentation(t *testing.T) { assert.Empty(t, phrase.Results, "explicit phrases must retain word adjacency") } +func TestSearchSessions_JapaneseAndKoreanTerms(t *testing.T) { + for _, tc := range []struct { + name string + query string + match string + miss string + }{ + {"japanese", "かな", "かなを探します。", "なかを探します。"}, + {"korean", "검색", "검색합니다.", "색상 검토입니다."}, + } { + t.Run(tc.name, func(t *testing.T) { + ts, d := newTestToolset(t) + if !d.HasCJKFTS() { + t.Skip("simple FTS5 runtime is not installed for this test process") + } + seedFTSSession(t, d, "match", "proj", tc.match, "2024-06-15T10:00:00Z") + seedFTSSession(t, d, "miss", "proj", tc.miss, "2024-06-15T10:00:00Z") + + out := mustSearch(t, ts, searchSessionsIn{Query: tc.query}) + require.Len(t, out.Results, 1) + assert.Equal(t, "match", out.Results[0].SessionID) + assert.Contains(t, out.Results[0].Snippet, ""+tc.query+"") + }) + } +} + func TestSearchSessions_QuerySyntax(t *testing.T) { ts, d := newTestToolset(t) seedFTSSession(t, d, "terms", "proj", diff --git a/internal/service/search_chinese_test.go b/internal/service/search_chinese_test.go index 10a4c2607..b2a550845 100644 --- a/internal/service/search_chinese_test.go +++ b/internal/service/search_chinese_test.go @@ -25,7 +25,7 @@ func TestDirectSearchSegmentsChineseQuery(t *testing.T) { dbtest.UserMsg(id, 0, "这是全文的搜索实现说明。"), dbtest.AsstMsg(id, 1, "understood"), }, dbtest.WithMessageCounts(3, 2)) - if !d.HasChineseFTS() { + if !d.HasCJKFTS() { t.Skip("simple FTS5 runtime is not installed for this test process") } be := service.NewDirectBackend(d, nil)