diff --git a/internal/search/rerank/tokens.go b/internal/search/rerank/tokens.go index 58249cae3..3f94b1796 100644 --- a/internal/search/rerank/tokens.go +++ b/internal/search/rerank/tokens.go @@ -3,6 +3,7 @@ package rerank import ( "strings" "unicode" + "unicode/utf8" ) // tokenize lowercases and splits a string on non-alphanumeric and @@ -45,9 +46,9 @@ func tokenize(s string) []string { // SCREAMING → Camel split: keep the last upper as // the start of the next token: HTTPHeader → HTTP + // Header. - if unicode.IsUpper(r) && unicode.IsUpper(prev) && i+1 < len(s) { - next := []rune(s[i:])[1] - if unicode.IsLower(next) { + if unicode.IsUpper(r) && unicode.IsUpper(prev) { + next, size := utf8.DecodeRuneInString(s[i+utf8.RuneLen(r):]) + if size > 0 && unicode.IsLower(next) { flush() } } diff --git a/internal/search/rerank/tokens_test.go b/internal/search/rerank/tokens_test.go new file mode 100644 index 000000000..e85aff017 --- /dev/null +++ b/internal/search/rerank/tokens_test.go @@ -0,0 +1,33 @@ +package rerank + +import ( + "reflect" + "testing" +) + +func TestTokenize(t *testing.T) { + cases := []struct { + in string + want []string + }{ + {"", nil}, + {"ParseHTTPHeader", []string{"parse", "http", "header"}}, + {"validate_user_token", []string{"validate", "user", "token"}}, + {"HTTPHeader", []string{"http", "header"}}, + {"простой текст", []string{"простой", "текст"}}, + // A word ending in consecutive uppercase runes where the last + // one is multi-byte used to panic with "index out of range [1] + // with length 1": the lookahead guard compared byte offsets, so + // []rune(s[i:]) could hold a single rune. + {"ТЕКСТ", []string{"текст"}}, + {"ПРОСТОЙ ТЕКСТ", []string{"простой", "текст"}}, + {"CAFÉ", []string{"café"}}, + // SCREAMING → Camel split still works across a multi-byte run. + {"ЖКХStatus", []string{"жкх", "status"}}, + } + for _, tc := range cases { + if got := tokenize(tc.in); !reflect.DeepEqual(got, tc.want) { + t.Errorf("tokenize(%q) = %v, want %v", tc.in, got, tc.want) + } + } +}