another fix

stanford-crfm · Jan 17, 2025 · 0b149b0 · 0b149b0
1 parent fa049d6
commit 0b149b0
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/src/levanter/data/text.py b/src/levanter/data/text.py
@@ -955,6 +955,9 @@ def preprocess_chat_example(batch, tokenizer: PreTrainedTokenizerBase, should_ap
 
     # Tokenize sources to get lengths
     sources_tokenized = tokenizer(sources, padding=False, truncation=True)
+
+    if should_append_eos:
+        targets = [t + tokenizer.eos_token for t in targets]
 
     # Combine for full examples
     full_examples = [f"{s}{t}" for s, t in zip(sources, targets)]