add wikitext eval

XMaster96 · Nov 21, 2020 · a2b731b · a2b731b
1 parent 4cb5221
commit a2b731b
Showing 1 changed file with 2 additions and 5 deletions.
diff --git a/data/encoders.py b/data/encoders.py
@@ -21,11 +21,8 @@ def fetch_encoder(params):
 
 
 # GPT2Tokenizer and Tokenizer have different ways of fetching token ids
-def encode(encoder, text, gpt=True):
-    if gpt:
-        result = encoder.encode(text, max_length=None)
-    else:
-        result = encoder.encode(text)
+def encode(encoder, text):
+    result = encoder.encode(text)
     if isinstance(result, list):
         return result
     return result.ids