Add Tokenizer Mode Support for Seshat Search Index

jest.config.cjs

@@ -0,0 +1,19 @@
+module.exports = {
+    preset: "ts-jest/presets/default-esm",
+    testEnvironment: "node",
+    testMatch: ["**/test/unit-tests/**/*.test.ts"],
+    moduleFileExtensions: ["ts", "js"],
+    extensionsToTreatAsEsm: [".ts"],
+    moduleNameMapper: {
+        "^(\\.{1,2}/.*)\\.js$": "$1",
+    },
+    transform: {
+        "^.+\\.ts$": [
+            "ts-jest",
+            {
+                useESM: true,
+                isolatedModules: true,
+            },
+        ],
+    },
+};

package.json

@@ -54,6 +54,7 @@
         "clean": "rimraf webapp.asar dist packages deploys lib",
         "hak": "tsx scripts/hak/index.ts",
         "test": "playwright test",
+        "test:unit": "NODE_OPTIONS='--experimental-vm-modules' jest",
         "test:open": "yarn test --ui",
         "test:screenshots:build": "docker build playwright -t element-desktop-playwright --platform linux/amd64",
         "test:screenshots:run": "docker run --rm --network host -v $(pwd):/work/element-desktop -v element-desktop-playwright:/work/element-desktop/node_modules -v /var/run/docker.sock:/var/run/docker.sock --platform linux/amd64 -it element-desktop-playwright",
@@ -80,6 +81,7 @@
         "@stylistic/eslint-plugin": "^5.0.0",
         "@types/auto-launch": "^5.0.1",
         "@types/counterpart": "^0.18.1",
+        "@types/jest": "^30.0.0",
         "@types/minimist": "^1.2.1",
         "@types/node": "18.19.130",
         "@types/pacote": "^11.1.1",
@@ -101,6 +103,7 @@
         "eslint-plugin-unicorn": "^56.0.0",
         "glob": "^13.0.0",
         "husky": "^9.1.6",
+        "jest": "^30.2.0",
         "knip": "^5.0.0",
         "lint-staged": "^16.0.0",
         "matrix-web-i18n": "^3.2.1",
@@ -111,6 +114,7 @@
         "prettier": "^3.0.0",
         "rimraf": "^6.0.0",
         "tar": "^7.0.0",
+        "ts-jest": "^29.4.6",
         "tsx": "^4.19.2",
         "typescript": "5.9.3"
     },

src/@types/matrix-seshat.d.ts

@@ -9,6 +9,9 @@ declare module "matrix-seshat" {
     interface IConfig {
         language?: string;
         passphrase?: string;
+        tokenizerMode?: "ngram" | "language";
+        ngramMinSize?: number;
+        ngramMaxSize?: number;
     }
 
     /* eslint-disable camelcase */

src/seshat-config.ts

@@ -0,0 +1,29 @@
+/*
+Copyright 2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+/**
+ * Create Seshat configuration based on tokenizer mode.
+ *
+ * @param tokenizerMode - The tokenizer mode: "ngram" for N-gram tokenization (CJK languages),
+ *                        or "language" for standard language-based tokenization.
+ * @returns Configuration object for Seshat initialization.
+ */
+export function createSeshatConfig(tokenizerMode?: string): {
+    tokenizerMode: "ngram" | "language";
+    ngramMinSize?: number;
+    ngramMaxSize?: number;
+} {
+    if (tokenizerMode === "ngram") {
+        return {
+            tokenizerMode: "ngram",
+            ngramMinSize: 2,
+            ngramMaxSize: 4,
+        };
+    }
+    // Default to language-based tokenizer
+    return { tokenizerMode: "language" };
+}

src/seshat.ts

@@ -17,6 +17,7 @@ import type {
 import IpcMainEvent = Electron.IpcMainEvent;
 import { randomArray } from "./utils.js";
 import Store from "./store.js";
+import { createSeshatConfig } from "./seshat-config.js";
 
 let seshatSupported = false;
 let Seshat: typeof SeshatType;
@@ -40,6 +41,7 @@ try {
 let eventIndex: SeshatType | null = null;
 
 const seshatDefaultPassphrase = "DEFAULT_PASSPHRASE";
+
 async function getOrCreatePassphrase(store: Store, key: string): Promise<string> {
     try {
         const storedPassphrase = await store.getSecret(key);
@@ -103,13 +105,15 @@ ipcMain.on("seshat", async function (_ev: IpcMainEvent, payload): Promise<void>
             if (eventIndex === null) {
                 const userId = args[0];
                 const deviceId = args[1];
+                const tokenizerMode = args[2] as string | undefined;
                 const passphraseKey = `seshat|${userId}|${deviceId}`;
 
                 const passphrase = await getOrCreatePassphrase(store, passphraseKey);
+                const seshatConfig = createSeshatConfig(tokenizerMode);
 
                 try {
                     await afs.mkdir(eventStorePath, { recursive: true });
-                    eventIndex = new Seshat(eventStorePath, { passphrase });
+                    eventIndex = new Seshat(eventStorePath, { passphrase, ...seshatConfig });
                 } catch (e) {
                     if (e instanceof ReindexError) {
                         // If this is a reindex error, the index schema
@@ -118,6 +122,7 @@ ipcMain.on("seshat", async function (_ev: IpcMainEvent, payload): Promise<void>
                         // database again.
                         const recoveryIndex = new SeshatRecovery(eventStorePath, {
                             passphrase,
+                            ...seshatConfig,
                         });
 
                         const userVersion = await recoveryIndex.getUserVersion();
@@ -131,10 +136,19 @@ ipcMain.on("seshat", async function (_ev: IpcMainEvent, payload): Promise<void>
                             await recoveryIndex.reindex();
                         }
 
-                        eventIndex = new Seshat(eventStorePath, { passphrase });
+                        eventIndex = new Seshat(eventStorePath, { passphrase, ...seshatConfig });
                     } else {
-                        sendError(payload.id, <Error>e);
-                        return;
+                        // Schema mismatch or other errors - delete and recreate the database
+                        console.warn("Failed to open Seshat database, deleting and recreating:", e);
+                        await deleteContents(eventStorePath);
+                        try {
+                            eventIndex = new Seshat(eventStorePath, { passphrase, ...seshatConfig });
+                            // Return that the database was recreated so element-web can re-add checkpoints
+                            ret = { wasRecreated: true };
+                        } catch (e2) {
+                            sendError(payload.id, <Error>e2);
+                            return;
+                        }
                     }
                 }
             }

test/unit-tests/seshat.test.ts

@@ -0,0 +1,40 @@
+/*
+Copyright 2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+import { createSeshatConfig } from "../../src/seshat-config";
+
+describe("createSeshatConfig", () => {
+    it("returns ngram config when tokenizerMode is 'ngram'", () => {
+        const config = createSeshatConfig("ngram");
+        expect(config).toEqual({
+            tokenizerMode: "ngram",
+            ngramMinSize: 2,
+            ngramMaxSize: 4,
+        });
+    });
+
+    it("returns language config when tokenizerMode is 'language'", () => {
+        const config = createSeshatConfig("language");
+        expect(config).toEqual({
+            tokenizerMode: "language",
+        });
+    });
+
+    it("defaults to language config when tokenizerMode is undefined", () => {
+        const config = createSeshatConfig(undefined);
+        expect(config).toEqual({
+            tokenizerMode: "language",
+        });
+    });
+
+    it("defaults to language config when tokenizerMode is unknown", () => {
+        const config = createSeshatConfig("unknown");
+        expect(config).toEqual({
+            tokenizerMode: "language",
+        });
+    });
+});