fix(input): segment Thai composing text by word boundaries

app/src/main/java/helium314/keyboard/latin/inputlogic/InputLogic.java

@@ -71,6 +71,7 @@
 import helium314.keyboard.latin.utils.TextRange;
 import helium314.keyboard.latin.utils.TimestampKt;
 
+import java.text.BreakIterator;
 import java.util.ArrayList;
 import java.util.Locale;
 import java.util.TreeSet;
@@ -83,6 +84,9 @@ public final class InputLogic {
     private static final String TAG = InputLogic.class.getSimpleName();
     private static final char INLINE_EMOJI_SEARCH_MARKER = ':';
     private static final int[] EMPTY_CODE_POINTS = new int[0];
+    private static final Locale THAI_LOCALE = Locale.forLanguageTag("th");
+    private static final ThreadLocal<BreakIterator> THAI_WORD_BREAK_ITERATOR =
+            ThreadLocal.withInitial(() -> BreakIterator.getWordInstance(THAI_LOCALE));
 
     // TODO : Remove this member when we can.
     final LatinIME mLatinIME;
@@ -1086,6 +1090,7 @@ private void handleNonSeparatorEvent(final Event event, final SettingsValues set
             if (mWordComposer.isSingleLetter()) {
                 mWordComposer.setCapitalizedModeAtStartComposingTime(inputTransaction.getShiftState());
             }
+            maybeCommitCompletedWordSegments(settingsValues);
             setComposingTextInternal(getTextWithUnderline(mWordComposer.getTypedWord()), 1);
         } else {
             final boolean swapWeakSpace = tryStripSpaceAndReturnWhetherShouldSwapInstead(event, inputTransaction);
@@ -1105,6 +1110,54 @@ private void handleNonSeparatorEvent(final Event event, final SettingsValues set
         inputTransaction.setRequiresUpdateSuggestions();
     }
 
+    private void maybeCommitCompletedWordSegments(final SettingsValues settingsValues) {
+        if (!ScriptUtils.needsWordSegmentation(settingsValues.mLocale)
+                || settingsValues.mSpacingAndPunctuations.mCurrentLanguageHasSpaces) {
+            return;
+        }
+
+        final String typedWord = mWordComposer.getTypedWord();
+        final int length = typedWord.length();
+        if (length <= 1) {
+            return;
+        }
+
+        final BreakIterator iterator = THAI_WORD_BREAK_ITERATOR.get();
+        iterator.setText(typedWord);
+        int segmentStart = iterator.first();
+        int wordBoundary = iterator.next();
+        boolean didCommitSegment = false;
+        while (wordBoundary != BreakIterator.DONE && wordBoundary < length) {
+            final String completedWordSegment = typedWord.substring(segmentStart, wordBoundary);
+            if (!TextUtils.isEmpty(completedWordSegment)) {
+                commitCompletedWordSegment(settingsValues, completedWordSegment);
+                didCommitSegment = true;
+            }
+            segmentStart = wordBoundary;
+            wordBoundary = iterator.next();
+        }
+        if (!didCommitSegment) {
+            return;
+        }
+
+        final String remainingWord = typedWord.substring(segmentStart);
+        final int[] codePoints = StringUtils.toCodePointArray(remainingWord);
+        mWordComposer.setComposingWord(codePoints,
+                mLatinIME.getCoordinatesForCurrentKeyboard(codePoints));
+    }
+
+    private void commitCompletedWordSegment(final SettingsValues settingsValues,
+            final String completedWordSegment) {
+        final NgramContext ngramContext = getNgramContextFromNthPreviousWordForSuggestion(
+                settingsValues.mSpacingAndPunctuations, 2);
+        mConnection.commitText(completedWordSegment, 1);
+        performAdditionToUserHistoryDictionary(settingsValues, completedWordSegment, ngramContext);
+        mLastComposedWord = new LastComposedWord(new ArrayList<>(), null, completedWordSegment,
+                completedWordSegment, LastComposedWord.NOT_A_SEPARATOR, ngramContext,
+                CapsMode.OFF);
+        StatsUtils.onWordCommitUserTyped(completedWordSegment, mWordComposer.isBatchMode());
+    }
+
     private boolean isCursorAtStartOrAfterSeparator(SettingsValues settingsValues) {
         var codePointBeforeCursor = mConnection.getCodePointBeforeCursor();
         return codePointBeforeCursor == Constants.NOT_A_CODE
@@ -1121,10 +1174,12 @@ private void handleSeparatorEvent(final Event event, final InputTransaction inpu
         final int codePoint = event.getCodePoint();
         final SettingsValues settingsValues = inputTransaction.getSettingsValues();
         final boolean wasComposingWord = mWordComposer.isComposingWord();
+        final boolean needsSegmentation = ScriptUtils.needsWordSegmentation(settingsValues.mLocale);
         // We avoid sending spaces in languages without spaces if we were composing.
         final boolean shouldAvoidSendingCode = Constants.CODE_SPACE == codePoint
                 && !settingsValues.mSpacingAndPunctuations.mCurrentLanguageHasSpaces
-                && wasComposingWord;
+                && wasComposingWord
+                && !needsSegmentation;
 
         if (mWordComposer.isCursorFrontOrMiddleOfComposingWord()) {
             // If we are in the middle of a recorrection, we need to commit the recorrection

app/src/main/java/helium314/keyboard/latin/utils/ScriptUtils.kt

@@ -186,6 +186,14 @@ object ScriptUtils {
         }
     }
 
+    /**
+     * Returns true if the locale uses a script that requires explicit word segmentation.
+     * Currently returns true for Thai only.
+     */
+    @JvmStatic
+    fun needsWordSegmentation(locale: Locale): Boolean =
+        locale.language == "th"
+
     @JvmStatic
     fun isScriptRtl(script: String): Boolean {
         return when (script) {

app/src/test/java/helium314/keyboard/latin/InputLogicTest.kt

@@ -18,7 +18,9 @@ import helium314.keyboard.event.Event
 import helium314.keyboard.keyboard.KeyboardSwitcher
 import helium314.keyboard.keyboard.MainKeyboardView
 import helium314.keyboard.keyboard.internal.keyboard_parser.floris.KeyCode
+import helium314.keyboard.latin.ShadowFacilitator2.Companion.addedWords
 import helium314.keyboard.latin.ShadowFacilitator2.Companion.lastAddedWord
+import helium314.keyboard.latin.ShadowFacilitator2.Companion.ngramContexts
 import helium314.keyboard.latin.SuggestedWords.SuggestedWordInfo
 import helium314.keyboard.latin.common.Constants
 import helium314.keyboard.latin.common.LocaleUtils.constructLocale
@@ -91,6 +93,45 @@ class InputLogicTest {
         assertEquals("", composingText)
     }
 
+    @Test fun `english space-separated typing remains unchanged`() {
+        chainInput("hello")
+        assertEquals("hello", composingText)
+        input(' ')
+        assertEquals("hello ", text)
+        assertEquals("", composingText)
+    }
+
+    @Test fun `single thai segment remains composing`() {
+        useThaiSubtype()
+        chainInput("ไทย")
+        assertEquals("ไทย", composingText)
+        assertEquals(emptyList(), addedWords)
+    }
+
+    @Test fun `thai segments commit individually and preserve context order`() {
+        useThaiSubtype()
+        chainInput("ภาษาไทยดี")
+        assertEquals("ภาษาไทยดี", text)
+        assertEquals("ดี", composingText)
+        assertEquals(listOf("ภาษา", "ไทย"), addedWords)
+        assertEquals(listOf("<S>", "ภาษา"), ngramContexts)
+    }
+
+    @Test fun `space after segmented thai input inserts one space`() {
+        useThaiSubtype()
+        chainInput("ภาษาไทยดี")
+        input(' ')
+        assertEquals("ภาษาไทยดี ", text)
+        assertEquals("", composingText)
+    }
+
+    @Test fun `non-thai no-space language remains unsegmented`() {
+        latinIME.switchToSubtype(SubtypeSettings.getResourceSubtypesForLocale("ko".constructLocale()).first())
+        chainInput("한국어")
+        assertEquals("한국어", composingText)
+        assertEquals(emptyList(), addedWords)
+    }
+
     @Test fun delete() {
         setText("hello there ")
         functionalKeyPress(KeyCode.DELETE)
@@ -728,13 +769,20 @@ class InputLogicTest {
         currentScript = ScriptUtils.SCRIPT_LATIN
         ShadowInputMethodService.reset()
         lastAddedWord = ""
+        addedWords.clear()
+        ngramContexts.clear()
 
         // reset settings
         latinIME.prefs().edit { clear() }
 
         setText("") // (re)sets selection and composing word
     }
 
+    private fun useThaiSubtype() {
+        latinIME.switchToSubtype(SubtypeSettings.getResourceSubtypesForLocale("th".constructLocale()).first())
+        currentScript = ScriptUtils.SCRIPT_THAI
+    }
+
     private fun chainInput(text: String) = text.forEach { input(it.code) }
 
     private fun input(char: Char) = input(char.code)
@@ -971,8 +1019,12 @@ class ShadowFacilitator2 {
                          ngramContext: NgramContext, timeStampInSeconds: Long,
                          blockPotentiallyOffensive: Boolean) {
         lastAddedWord = suggestion
+        addedWords.add(suggestion)
+        ngramContexts.add(ngramContext.extractPrevWordsContext())
     }
     companion object {
         var lastAddedWord = ""
+        val addedWords = mutableListOf<String>()
+        val ngramContexts = mutableListOf<String>()
     }
 }

app/src/test/java/helium314/keyboard/latin/ScriptUtilsTest.kt

@@ -5,9 +5,12 @@ import helium314.keyboard.latin.common.LocaleUtils.constructLocale
 import helium314.keyboard.latin.utils.ScriptUtils.SCRIPT_CYRILLIC
 import helium314.keyboard.latin.utils.ScriptUtils.SCRIPT_DEVANAGARI
 import helium314.keyboard.latin.utils.ScriptUtils.SCRIPT_LATIN
+import helium314.keyboard.latin.utils.ScriptUtils.needsWordSegmentation
 import helium314.keyboard.latin.utils.ScriptUtils.script
 import kotlin.test.Test
 import kotlin.test.assertEquals
+import kotlin.test.assertFalse
+import kotlin.test.assertTrue
 
 class ScriptUtilsTest {
     @Test fun defaultScript() {
@@ -18,4 +21,18 @@ class ScriptUtilsTest {
         assertEquals(SCRIPT_CYRILLIC, "mk".constructLocale().script())
         assertEquals(SCRIPT_CYRILLIC, "fr-Cyrl".constructLocale().script())
     }
+
+    @Test fun needsWordSegmentationThai() {
+        assertTrue(needsWordSegmentation("th".constructLocale()))
+    }
+
+    @Test fun needsWordSegmentationNonThai() {
+        assertFalse(needsWordSegmentation("en".constructLocale()))
+        assertFalse(needsWordSegmentation("ja".constructLocale()))
+        assertFalse(needsWordSegmentation("zh".constructLocale()))
+        assertFalse(needsWordSegmentation("lo".constructLocale()))
+        assertFalse(needsWordSegmentation("km".constructLocale()))
+        assertFalse(needsWordSegmentation("ko".constructLocale()))
+        assertFalse(needsWordSegmentation("my".constructLocale()))
+    }
 }