Keep the last character of a vocab line that has no newline

src/f5_tts/model/utils.py

@@ -124,7 +124,7 @@ def get_tokenizer(dataset_name, tokenizer: str = "pinyin"):
         with open(tokenizer_path, "r", encoding="utf-8") as f:
             vocab_char_map = {}
             for i, char in enumerate(f):
-                vocab_char_map[char[:-1]] = i
+                vocab_char_map[char.rstrip("\r\n")] = i
         vocab_size = len(vocab_char_map)
         assert vocab_char_map[" "] == 0, "make sure space is of idx 0 in vocab.txt, cuz 0 is used for unknown char"
 
@@ -136,7 +136,7 @@ def get_tokenizer(dataset_name, tokenizer: str = "pinyin"):
         with open(dataset_name, "r", encoding="utf-8") as f:
             vocab_char_map = {}
             for i, char in enumerate(f):
-                vocab_char_map[char[:-1]] = i
+                vocab_char_map[char.rstrip("\r\n")] = i
         vocab_size = len(vocab_char_map)
 
     return vocab_char_map, vocab_size

tests/test_get_tokenizer.py

@@ -0,0 +1,33 @@
+"""The last vocab line keeps every character when the file has no final newline."""
+
+import importlib.util
+from pathlib import Path
+
+
+def _get_tokenizer():
+    path = Path(__file__).resolve().parents[1] / "src" / "f5_tts" / "model" / "utils.py"
+    spec = importlib.util.spec_from_file_location("f5_tts_model_utils_under_test", path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module.get_tokenizer
+
+
+def test_last_vocab_entry_keeps_its_last_character(tmp_path: Path) -> None:
+    get_tokenizer = _get_tokenizer()
+    vocab = tmp_path / "vocab.txt"
+    vocab.write_bytes(b" \nni3")
+
+    vocab_char_map, vocab_size = get_tokenizer(str(vocab), tokenizer="custom")
+
+    assert vocab_char_map[" "] == 0
+    assert vocab_char_map["ni3"] == 1
+    assert vocab_size == 2
+
+    vocab.write_bytes(b" \nni3\n")
+    vocab_char_map, vocab_size = get_tokenizer(str(vocab), tokenizer="custom")
+    assert vocab_char_map["ni3"] == 1
+    assert vocab_size == 2
+
+    vocab.write_bytes(b" \nx")
+    vocab_char_map, _ = get_tokenizer(str(vocab), tokenizer="custom")
+    assert vocab_char_map["x"] == 1