fix missing token issue

TECH4DX · Oct 17, 2022 · 949f97a · 949f97a
1 parent 0ad085f
commit 949f97a
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/convert-pt-to-ggml.py b/convert-pt-to-ggml.py
@@ -271,7 +271,7 @@ def bytes_to_unicode():
 fout.write(struct.pack("i", len(tokens)))
 
 for key in tokens:
-    text = bytearray([byte_decoder[c] for c in key]).decode('utf-8', errors='replace').encode('utf-8')
+    text = bytearray([byte_decoder[c] for c in key])
     fout.write(struct.pack("i", len(text)))
     fout.write(text)