save tokenizer

MeetKai · Nov 12, 2024 · bfaed17 · bfaed17
1 parent f636615
commit bfaed17
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/functionary/train/train_lora.py b/functionary/train/train_lora.py
@@ -435,6 +435,7 @@ def compute_metrics(eval_preds):
 
     if training_args.local_rank == 0:
         model.save_pretrained(training_args.output_dir, state_dict=state_dict)
+        tokenizer.save_pretrained(training_args.output_dir)
 
 
 if __name__ == "__main__":