fix(search): stop the rerank tokenizer panicking on trailing multi-byte uppercase runes

internal/search/rerank/tokens.go

@@ -3,6 +3,7 @@ package rerank
 import (
 	"strings"
 	"unicode"
+	"unicode/utf8"
 )
 
 // tokenize lowercases and splits a string on non-alphanumeric and
@@ -45,9 +46,9 @@ func tokenize(s string) []string {
 				// SCREAMING → Camel split: keep the last upper as
 				// the start of the next token: HTTPHeader → HTTP +
 				// Header.
-				if unicode.IsUpper(r) && unicode.IsUpper(prev) && i+1 < len(s) {
-					next := []rune(s[i:])[1]
-					if unicode.IsLower(next) {
+				if unicode.IsUpper(r) && unicode.IsUpper(prev) {
+					next, size := utf8.DecodeRuneInString(s[i+utf8.RuneLen(r):])
+					if size > 0 && unicode.IsLower(next) {
 						flush()
 					}
 				}

internal/search/rerank/tokens_test.go

@@ -0,0 +1,33 @@
+package rerank
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestTokenize(t *testing.T) {
+	cases := []struct {
+		in   string
+		want []string
+	}{
+		{"", nil},
+		{"ParseHTTPHeader", []string{"parse", "http", "header"}},
+		{"validate_user_token", []string{"validate", "user", "token"}},
+		{"HTTPHeader", []string{"http", "header"}},
+		{"простой текст", []string{"простой", "текст"}},
+		// A word ending in consecutive uppercase runes where the last
+		// one is multi-byte used to panic with "index out of range [1]
+		// with length 1": the lookahead guard compared byte offsets, so
+		// []rune(s[i:]) could hold a single rune.
+		{"ТЕКСТ", []string{"текст"}},
+		{"ПРОСТОЙ ТЕКСТ", []string{"простой", "текст"}},
+		{"CAFÉ", []string{"café"}},
+		// SCREAMING → Camel split still works across a multi-byte run.
+		{"ЖКХStatus", []string{"жкх", "status"}},
+	}
+	for _, tc := range cases {
+		if got := tokenize(tc.in); !reflect.DeepEqual(got, tc.want) {
+			t.Errorf("tokenize(%q) = %v, want %v", tc.in, got, tc.want)
+		}
+	}
+}