quick fixes (#207)

eole/modules/rope.py

@@ -236,9 +236,9 @@ def update(self, maxseqlen, step=0, prefetch=1024, reset=False, positions=None):
         elif hasattr(self, "cos") and self.cos.size(0) >= max(offset + (step or 0), 0) + maxseqlen:
             return self.cos, self.sin
         if self.mode == "1d":
-            cos, sin = self.forward_1d(maxseqlen, step=step, prefetch=prefetch, offset=offset)
+            cos, sin = self.forward_1d(maxseqlen, step=(step or 0), prefetch=prefetch, offset=offset)
         elif self.mode == "2d":
-            cos, sin = self.forward_2d(maxseqlen, step=step, prefetch=prefetch, positions=positions)
+            cos, sin = self.forward_2d(maxseqlen, step=(step or 0), prefetch=prefetch, positions=positions)
         else:
             raise NotImplementedError
         self.register_buffer("cos", cos, persistent=False)

eole/transforms/tokenize.py

@@ -520,6 +520,7 @@ def hex_to_char(match):
         tokenizer = self.load_models[side]
         if self.gpt2_pretok:
             sentence = "".join(tokens)
+            sentence = sentence.replace("｟", "ï½Ł").replace("｠", "ï½ł")
             detokenized = bytearray([self.revtable[c] for c in sentence]).decode("utf-8", errors="replace")
         else:
             detokenized = tokenizer.detokenize(tokens)

eole/transforms/tokenize_id.py

@@ -96,7 +96,7 @@ def tokenize_string(self, string, side="src", is_train=False):
             kwargs = {"max_length": max_length, "truncation": True}
         else:
             kwargs = {}
-        tokens = self.tokenizers[side].encode(string, **kwargs)
+        tokens = self.tokenizers[side].encode(string.replace(DefaultTokens.SEP, "\n"), **kwargs)
         return tokens
 
     def apply(self, example, is_train=False, stats=None, **kwargs):