models : change default encoding to utf8 (ggerganov#605)

TECH4DX · Mar 22, 2023 · 992aa2c · 992aa2c
1 parent 4aa3bcf
commit 992aa2c
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/models/convert-h5-to-ggml.py b/models/convert-h5-to-ggml.py
@@ -79,11 +79,11 @@ def bytes_to_unicode():
 dir_whisper = sys.argv[2]
 dir_out     = sys.argv[3]
 
-with open(dir_model + "/vocab.json", "r") as f:
+with open(dir_model + "/vocab.json", "r", encoding="utf8") as f:
     encoder = json.load(f)
-with open(dir_model + "/added_tokens.json", "r") as f:
+with open(dir_model + "/added_tokens.json", "r", encoding="utf8") as f:
     encoder_added = json.load(f)
-with open(dir_model + "/config.json", "r") as f:
+with open(dir_model + "/config.json", "r", encoding="utf8") as f:
     hparams = json.load(f)
 
 model = WhisperForConditionalGeneration.from_pretrained(dir_model)