Add Tokenizer Mode Support for Message Search

src/async-components/views/dialogs/eventindex/ConfirmTokenizerChangeDialog.tsx

@@ -0,0 +1,60 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+import React from "react";
+
+import BaseDialog from "../../../../components/views/dialogs/BaseDialog";
+import Spinner from "../../../../components/views/elements/Spinner";
+import DialogButtons from "../../../../components/views/elements/DialogButtons";
+import { _t } from "../../../../languageHandler";
+import EventIndexPeg from "../../../../indexing/EventIndexPeg";
+
+interface IProps {
+    onFinished: (confirmed?: boolean) => void;
+}
+
+interface IState {
+    deleting: boolean;
+}
+
+/*
+ * Confirmation dialog for deleting the database when tokenizer mode is changed.
+ */
+export default class ConfirmTokenizerChangeDialog extends React.Component<IProps, IState> {
+    public constructor(props: IProps) {
+        super(props);
+        this.state = {
+            deleting: false,
+        };
+    }
+
+    private readonly onConfirm = async (): Promise<void> => {
+        this.setState({
+            deleting: true,
+        });
+
+        await EventIndexPeg.deleteEventIndex();
+        this.props.onFinished(true);
+    };
+
+    public render(): React.ReactNode {
+        return (
+            <BaseDialog onFinished={this.props.onFinished} title={_t("common|are_you_sure")}>
+                {_t("settings|security|tokenizer_mode_change_warning")}
+                {this.state.deleting ? <Spinner /> : <div />}
+                <DialogButtons
+                    primaryButton={_t("action|ok")}
+                    onPrimaryButtonClick={this.onConfirm}
+                    primaryButtonClass="danger"
+                    cancelButtonClass="warning"
+                    onCancel={this.props.onFinished}
+                    disabled={this.state.deleting}
+                />
+            </BaseDialog>
+        );
+    }
+}

src/async-components/views/dialogs/eventindex/ManageEventIndexDialog.tsx

@@ -46,6 +46,12 @@ interface IState {
 
     /** Time to sleep between crawlwer passes, in milliseconds. */
     crawlerSleepTime: number;
+
+    /** Tokenizer mode for search indexing. */
+    tokenizerMode: string;
+
+    /** Initial tokenizer mode when dialog was opened. */
+    initialTokenizerMode: string;
 }
 
 /*
@@ -55,6 +61,7 @@ export default class ManageEventIndexDialog extends React.Component<IProps, ISta
     public constructor(props: IProps) {
         super(props);
 
+        const initialTokenizerMode = SettingsStore.getValueAt(SettingLevel.DEVICE, "tokenizerMode");
         this.state = {
             eventIndexSize: 0,
             eventCount: 0,
@@ -63,6 +70,8 @@ export default class ManageEventIndexDialog extends React.Component<IProps, ISta
             roomCount: 0,
             currentRoom: null,
             crawlerSleepTime: SettingsStore.getValueAt(SettingLevel.DEVICE, "crawlerSleepTime"),
+            tokenizerMode: initialTokenizerMode,
+            initialTokenizerMode: initialTokenizerMode,
         };
     }
 
@@ -125,6 +134,51 @@ export default class ManageEventIndexDialog extends React.Component<IProps, ISta
         SettingsStore.setValue("crawlerSleepTime", null, SettingLevel.DEVICE, e.target.value);
     };
 
+    private readonly onTokenizerModeChange = (e: ChangeEvent<HTMLSelectElement>): void => {
+        this.setState({ tokenizerMode: e.target.value });
+        // Don't save to settings yet - wait for Done button
+    };
+
+    private readonly onDone = async (): Promise<void> => {
+        // Check if tokenizer mode has changed
+        if (this.state.tokenizerMode !== this.state.initialTokenizerMode) {
+            // Show confirmation dialog
+            const ConfirmTokenizerChangeDialog = (await import("./ConfirmTokenizerChangeDialog")).default;
+            Modal.createDialog(
+                ConfirmTokenizerChangeDialog,
+                {
+                    onFinished: async (confirmed?: boolean) => {
+                        if (confirmed) {
+                            // Save the tokenizer mode setting
+                            SettingsStore.setValue(
+                                "tokenizerMode",
+                                null,
+                                SettingLevel.DEVICE,
+                                this.state.tokenizerMode,
+                            );
+                        } else {
+                            // User cancelled - revert tokenizer mode to initial value
+                            this.setState((prevState) => ({ tokenizerMode: prevState.initialTokenizerMode }));
+                            SettingsStore.setValue(
+                                "tokenizerMode",
+                                null,
+                                SettingLevel.DEVICE,
+                                this.state.initialTokenizerMode,
+                            );
+                        }
+                        this.props.onFinished();
+                    },
+                },
+                undefined,
+                /* priority = */ false,
+                /* static = */ true,
+            );
+        } else {
+            // No change, just close the dialog
+            this.props.onFinished();
+        }
+    };
+
     public render(): React.ReactNode {
         const brand = SdkConfig.get().brand;
 
@@ -165,6 +219,18 @@ export default class ManageEventIndexDialog extends React.Component<IProps, ISta
                         value={this.state.crawlerSleepTime.toString()}
                         onChange={this.onCrawlerSleepTimeChange}
                     />
+                    <Field
+                        element="select"
+                        label={_t("settings|security|tokenizer_mode")}
+                        value={this.state.tokenizerMode}
+                        onChange={this.onTokenizerModeChange}
+                    >
+                        <option value="ngram">{_t("settings|security|tokenizer_mode_ngram")}</option>
+                        <option value="language">{_t("settings|security|tokenizer_mode_language")}</option>
+                    </Field>
+                    <div className="mx_SettingsTab_subsectionText">
+                        {_t("settings|security|tokenizer_mode_description")}
+                    </div>
                 </div>
             </div>
         );
@@ -178,7 +244,7 @@ export default class ManageEventIndexDialog extends React.Component<IProps, ISta
                 {eventIndexingSettings}
                 <DialogButtons
                     primaryButton={_t("action|done")}
-                    onPrimaryButtonClick={this.props.onFinished}
+                    onPrimaryButtonClick={this.onDone}
                     primaryButtonClass="primary"
                     cancelButton={_t("action|disable")}
                     onCancel={this.onDisable}

src/i18n/strings/en_EN.json

@@ -2974,7 +2974,12 @@
             "message_search_unsupported_web": "%(brand)s can't securely cache encrypted messages locally while running in a web browser. Use <desktopLink>%(brand)s Desktop</desktopLink> for encrypted messages to appear in search results.",
             "record_session_details": "Record the client name, version, and url to recognise sessions more easily in session manager",
             "send_analytics": "Send analytics data",
-            "strict_encryption": "Only send messages to verified users"
+            "strict_encryption": "Only send messages to verified users",
+            "tokenizer_mode": "Search tokenizer mode",
+            "tokenizer_mode_change_warning": "Changing the tokenizer mode requires deleting the existing search index. The index will be rebuilt automatically. Do you want to continue?",
+            "tokenizer_mode_description": "Language-based: Single language support fixed to UI language. Only works with some languages (English, German, etc.). N-gram: Supports all languages including those without word boundaries. Works with mixed languages. Changing this setting will rebuild your search index.",
+            "tokenizer_mode_language": "Language-based (single language support based on UI settings)",
+            "tokenizer_mode_ngram": "N-gram (Multi-language support)"
         },
         "send_read_receipts": "Send read receipts",
         "send_read_receipts_unsupported": "Your server doesn't support disabling sending read receipts.",

src/indexing/BaseEventIndexManager.ts

@@ -87,11 +87,16 @@ export default abstract class BaseEventIndexManager {
      *
      * @param {string} userId The event that should be added to the index.
      * @param {string} deviceId The profile of the event sender at the
+     * @param {string} tokenizerMode The tokenizer mode to use ("ngram" or "language")
      *
      * @return {Promise} A promise that will resolve when the event index is
-     * initialized.
+     * initialized. Returns { wasRecreated: true } if the database was recreated.
      */
-    public async initEventIndex(userId: string, deviceId: string): Promise<void> {
+    public async initEventIndex(
+        userId: string,
+        deviceId: string,
+        tokenizerMode?: string,
+    ): Promise<{ wasRecreated?: boolean } | void> {
         throw new Error("Unimplemented");
     }
 

src/indexing/EventIndex.ts

@@ -74,6 +74,8 @@ export default class EventIndex extends EventEmitter {
      * The current checkpoint that the crawler is working on.
      */
     private currentCheckpoint: ICrawlerCheckpoint | null = null;
+    // Flag to force adding initial checkpoints (e.g., after database recreation)
+    private forceAddInitialCheckpoints = false;
 
     /**
      * True if we need to add the initial checkpoints for encrypted rooms, once we've completed a sync.
@@ -106,6 +108,14 @@ export default class EventIndex extends EventEmitter {
         this.registerListeners();
     }
 
+    /**
+     * Mark that initial checkpoints should be added on next sync.
+     * This is used when the database is recreated (e.g., schema change).
+     */
+    public setForceAddInitialCheckpoints(force: boolean): void {
+        this.forceAddInitialCheckpoints = force;
+    }
+
     /**
      * Register event listeners that are necessary for the event index to work.
      */
@@ -204,8 +214,10 @@ export default class EventIndex extends EventEmitter {
             if (!indexManager) return;
 
             // If the index was empty when we first started up, add the initial checkpoints, to back-populate the index.
-            if (this.needsInitialCheckpoints) {
+            // Also check forceAddInitialCheckpoints flag (used when database is recreated, e.g., schema change)
+            if (this.needsInitialCheckpoints || this.forceAddInitialCheckpoints) {
                 await this.addInitialCheckpoints();
+                this.forceAddInitialCheckpoints = false;
             }
 
             // Start the crawler if it's not already running.
@@ -1017,7 +1029,7 @@ export default class EventIndex extends EventEmitter {
         };
 
         const encryptedRooms = rooms.filter(isRoomEncrypted);
-        encryptedRooms.forEach((room, index) => {
+        encryptedRooms.forEach((room) => {
             totalRooms.add(room.roomId);
         });
 

src/indexing/EventIndexPeg.ts

@@ -54,7 +54,8 @@ export class EventIndexPeg {
             return false;
         }
 
-        if (!SettingsStore.getValueAt(SettingLevel.DEVICE, "enableEventIndexing")) {
+        const enableEventIndexing = SettingsStore.getValueAt(SettingLevel.DEVICE, "enableEventIndexing");
+        if (!enableEventIndexing) {
             logger.log("EventIndex: Event indexing is disabled, not initializing");
             return false;
         }
@@ -78,20 +79,31 @@ export class EventIndexPeg {
 
         const userId = client.getUserId()!;
         const deviceId = client.getDeviceId()!;
+        const tokenizerMode = SettingsStore.getValueAt(SettingLevel.DEVICE, "tokenizerMode");
 
         try {
-            await indexManager.initEventIndex(userId, deviceId);
+            const initResult = await indexManager.initEventIndex(userId, deviceId, tokenizerMode);
+
+            // If the database was recreated (e.g., due to schema change), force re-adding checkpoints
+            if (initResult && typeof initResult === "object" && initResult.wasRecreated) {
+                logger.log("EventIndex: Database was recreated, will force add initial checkpoints");
+                index.setForceAddInitialCheckpoints(true);
+            }
 
             const userVersion = await indexManager.getUserVersion();
             const eventIndexIsEmpty = await indexManager.isEventIndexEmpty();
 
             if (eventIndexIsEmpty) {
                 await indexManager.setUserVersion(INDEX_VERSION);
+                // Force adding initial checkpoints because limited timeline events
+                // may add checkpoints before onSync is called
+                logger.log("EventIndex: Index is empty, will force add initial checkpoints");
+                index.setForceAddInitialCheckpoints(true);
             } else if (userVersion === 0 && !eventIndexIsEmpty) {
                 await indexManager.closeEventIndex();
                 await this.deleteEventIndex();
 
-                await indexManager.initEventIndex(userId, deviceId);
+                await indexManager.initEventIndex(userId, deviceId, tokenizerMode);
                 await indexManager.setUserVersion(INDEX_VERSION);
             }
 

src/settings/Settings.tsx

@@ -337,6 +337,7 @@ export interface Settings {
     "RightPanel.phases": IBaseSetting<IRightPanelForRoomStored | null>;
     "enableEventIndexing": IBaseSetting<boolean>;
     "crawlerSleepTime": IBaseSetting<number>;
+    "tokenizerMode": IBaseSetting<string>;
     "showCallButtonsInComposer": IBaseSetting<boolean>;
     "ircDisplayNameWidth": IBaseSetting<number>;
     "layout": IBaseSetting<Layout>;
@@ -1235,6 +1236,11 @@ export const SETTINGS: Settings = {
         displayName: _td("settings|security|message_search_sleep_time"),
         default: 3000,
     },
+    "tokenizerMode": {
+        supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS,
+        displayName: _td("settings|security|tokenizer_mode"),
+        default: "language",
+    },
     "showCallButtonsInComposer": {
         // Dev note: This is no longer "in composer" but is instead "in room header".
         // TODO: Rename with settings v3

src/vector/platform/SeshatIndexManager.ts

@@ -28,8 +28,12 @@ export class SeshatIndexManager extends BaseEventIndexManager {
         return this.ipc.call("supportsEventIndexing");
     }
 
-    public async initEventIndex(userId: string, deviceId: string): Promise<void> {
-        return this.ipc.call("initEventIndex", userId, deviceId);
+    public async initEventIndex(
+        userId: string,
+        deviceId: string,
+        tokenizerMode?: string,
+    ): Promise<{ wasRecreated?: boolean } | void> {
+        return this.ipc.call("initEventIndex", userId, deviceId, tokenizerMode);
     }
 
     public async addEventToIndex(ev: IMatrixEvent, profile: IMatrixProfile): Promise<void> {

test/unit-tests/async-components/dialogs/eventindex/ConfirmTokenizerChangeDialog-test.tsx

@@ -0,0 +1,33 @@
+/*
+Copyright 2025 New Vector Ltd.
+Copyright 2020, 2021 The Matrix.org Foundation C.I.C.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+import React from "react";
+import { fireEvent, render, screen } from "jest-matrix-react";
+
+import ConfirmTokenizerChangeDialog from "../../../../../src/async-components/views/dialogs/eventindex/ConfirmTokenizerChangeDialog";
+import EventIndexPeg from "../../../../../src/indexing/EventIndexPeg";
+import { flushPromises } from "../../../../test-utils";
+
+describe("<ConfirmTokenizerChangeDialog />", () => {
+    afterEach(() => {
+        jest.restoreAllMocks();
+    });
+
+    it("deletes the event index and finishes when confirmed", async () => {
+        const onFinished = jest.fn();
+        jest.spyOn(EventIndexPeg, "deleteEventIndex").mockResolvedValue(undefined);
+
+        render(<ConfirmTokenizerChangeDialog onFinished={onFinished} />);
+
+        fireEvent.click(screen.getByRole("button", { name: /ok/i }));
+        await flushPromises();
+
+        expect(EventIndexPeg.deleteEventIndex).toHaveBeenCalled();
+        expect(onFinished).toHaveBeenCalledWith(true);
+    });
+});