From 9df6f41fddd4e9ca61fcd00da0542dad0c4831c2 Mon Sep 17 00:00:00 2001
From: Sasha Mitchell <sash.t.mitchell@gmail.com>
Date: Tue, 29 Sep 2026 00:04:18 +0700
Subject: [PATCH] Add a test for a vocab line with no trailing newline.

---
 src/f5_tts/model/utils.py   |  4 ++--
 tests/test_get_tokenizer.py | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 2 deletions(-)
 create mode 100644 tests/test_get_tokenizer.py

diff --git a/src/f5_tts/model/utils.py b/src/f5_tts/model/utils.py
index ff3495626..99f7c19f5 100644
--- a/src/f5_tts/model/utils.py
+++ b/src/f5_tts/model/utils.py
@@ -124,7 +124,7 @@ def get_tokenizer(dataset_name, tokenizer: str = "pinyin"):
         with open(tokenizer_path, "r", encoding="utf-8") as f:
             vocab_char_map = {}
             for i, char in enumerate(f):
-                vocab_char_map[char[:-1]] = i
+                vocab_char_map[char.rstrip("\r\n")] = i
         vocab_size = len(vocab_char_map)
         assert vocab_char_map[" "] == 0, "make sure space is of idx 0 in vocab.txt, cuz 0 is used for unknown char"
 
@@ -136,7 +136,7 @@ def get_tokenizer(dataset_name, tokenizer: str = "pinyin"):
         with open(dataset_name, "r", encoding="utf-8") as f:
             vocab_char_map = {}
             for i, char in enumerate(f):
-                vocab_char_map[char[:-1]] = i
+                vocab_char_map[char.rstrip("\r\n")] = i
         vocab_size = len(vocab_char_map)
 
     return vocab_char_map, vocab_size
diff --git a/tests/test_get_tokenizer.py b/tests/test_get_tokenizer.py
new file mode 100644
index 000000000..529c2f205
--- /dev/null
+++ b/tests/test_get_tokenizer.py
@@ -0,0 +1,33 @@
+"""The last vocab line keeps every character when the file has no final newline."""
+
+import importlib.util
+from pathlib import Path
+
+
+def _get_tokenizer():
+    path = Path(__file__).resolve().parents[1] / "src" / "f5_tts" / "model" / "utils.py"
+    spec = importlib.util.spec_from_file_location("f5_tts_model_utils_under_test", path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module.get_tokenizer
+
+
+def test_last_vocab_entry_keeps_its_last_character(tmp_path: Path) -> None:
+    get_tokenizer = _get_tokenizer()
+    vocab = tmp_path / "vocab.txt"
+    vocab.write_bytes(b" \nni3")
+
+    vocab_char_map, vocab_size = get_tokenizer(str(vocab), tokenizer="custom")
+
+    assert vocab_char_map[" "] == 0
+    assert vocab_char_map["ni3"] == 1
+    assert vocab_size == 2
+
+    vocab.write_bytes(b" \nni3\n")
+    vocab_char_map, vocab_size = get_tokenizer(str(vocab), tokenizer="custom")
+    assert vocab_char_map["ni3"] == 1
+    assert vocab_size == 2
+
+    vocab.write_bytes(b" \nx")
+    vocab_char_map, _ = get_tokenizer(str(vocab), tokenizer="custom")
+    assert vocab_char_map["x"] == 1