feature:completed bpe and base

.gitignore

@@ -333,4 +333,5 @@ __pycache__/
 *.pyd
 *.bz2
 
-*.venv
+*.venv
+venv/

openverifiablellm/tokenizer/base.py

@@ -19,13 +19,30 @@ def __init__(self, vocab_size: int, min_frequency: int):
 
     @abstractmethod
     def train(self, text_file: Path, save_path: Path):
-        """Train tokenizer and save model."""
+        """Train tokenizer on a text corpus and save artifacts to save_path."""
+        pass
+
+    @abstractmethod
+    def encode(self, text: str) -> list[int]:
+        """Encode text into a list of integer token ids."""
+        pass
+
+    @abstractmethod
+    def decode(self, ids: list[int]) -> str:
+        """Decode a list of integer token ids back into text."""
+        pass
+
+    @abstractmethod
+    def load(self, tokenizer_dir: Path):
+        """Load a previously trained tokenizer from disk."""
         pass
 
     @abstractmethod
     def get_vocab_path(self, tokenizer_dir: Path) -> Path:
+        """Return path to the vocabulary file."""
         pass
 
     @abstractmethod
-    def get_merges_path(self, tokenizer_dir: Path):
+    def get_merges_path(self, tokenizer_dir: Path) -> Path | None:
+        """Return path to the merges file, or None if not applicable."""
         pass

openverifiablellm/tokenizer/bpe_tokenizer.py

@@ -8,7 +8,43 @@
 
 
 class BPETokenizer(BaseTokenizer):
+    """
+    Byte-level BPE tokenizer implementation.
+
+    Wraps HuggingFace's ByteLevelBPETokenizer and implements
+    the full BaseTokenizer interface including train, encode,
+    decode, and load.
+    """
+
+    def __init__(self, vocab_size: int, min_frequency: int):
+        super().__init__(vocab_size, min_frequency)
+        self._tokenizer = None
+
+    # ------------------------------------------------------------------
+    # Training
+    # ------------------------------------------------------------------
+
     def train(self, text_file: Path, save_path: Path):
+        """
+        Train BPE tokenizer on text corpus and save artifacts.
+
+        Args:
+            text_file: Path to training text corpus.
+            save_path: Directory to save vocab.json and merges.txt.
+
+        Raises:
+            FileNotFoundError: If text_file does not exist or is not a file.
+        """
+
+        text_file = Path(text_file)
+        save_path = Path(save_path)
+
+        if not text_file.is_file():
+            raise FileNotFoundError(
+                f"Training file not found at {text_file}. Please provide a valid text corpus file."
+            )
+
+        save_path.mkdir(parents=True, exist_ok=True)
 
         tokenizer = ByteLevelBPETokenizer()
 
@@ -21,8 +57,109 @@ def train(self, text_file: Path, save_path: Path):
 
         tokenizer.save_model(str(save_path))
 
+        self._tokenizer = tokenizer
+
+    # ------------------------------------------------------------------
+    # Encode / Decode
+    # ------------------------------------------------------------------
+
+    def encode(self, text: str) -> list[int]:
+        """
+        Encode text into a list of token ids.
+
+        Args:
+            text: Input string to tokenize.
+
+        Returns:
+            List of integer token ids.
+
+        Raises:
+            RuntimeError: If tokenizer has not been trained or loaded.
+        """
+
+        self._check_loaded()
+        return self._tokenizer.encode(text).ids
+
+    def decode(self, ids: list[int]) -> str:
+        """
+        Decode a list of token ids back into text.
+
+        Args:
+            ids: List of integer token ids.
+
+        Returns:
+            Decoded string.
+
+        Raises:
+            RuntimeError: If tokenizer has not been trained or loaded.
+        """
+
+        self._check_loaded()
+        return self._tokenizer.decode(ids, skip_special_tokens=False)
+
+    # ------------------------------------------------------------------
+    # Load
+    # ------------------------------------------------------------------
+
+    def load(self, tokenizer_dir: Path):
+        """
+        Load a previously trained BPE tokenizer from disk.
+
+        Args:
+            tokenizer_dir: Directory containing vocab.json and merges.txt.
+
+        Raises:
+            FileNotFoundError: If vocab.json or merges.txt are not found.
+        """
+
+        tokenizer_dir = Path(tokenizer_dir)
+
+        vocab_path = tokenizer_dir / "vocab.json"
+        merges_path = tokenizer_dir / "merges.txt"
+
+        if not vocab_path.is_file():
+            raise FileNotFoundError(
+                f"vocab.json not found at {vocab_path}. Please train the tokenizer first."
+            )
+
+        if not merges_path.is_file():
+            raise FileNotFoundError(
+                f"merges.txt not found at {merges_path}. Please train the tokenizer first."
+            )
+
+        self._tokenizer = ByteLevelBPETokenizer(
+            vocab=str(vocab_path),
+            merges=str(merges_path),
+        )
+
+        # 🔧 Fix: re-register special tokens after loading
+        self._tokenizer.add_special_tokens(SPECIAL_TOKENS)
+
+    # ------------------------------------------------------------------
+    # Artifact paths
+    # ------------------------------------------------------------------
+
     def get_vocab_path(self, tokenizer_dir: Path) -> Path:
-        return tokenizer_dir / "vocab.json"
+        """Return path to vocab.json file."""
+        return Path(tokenizer_dir) / "vocab.json"
 
     def get_merges_path(self, tokenizer_dir: Path) -> Path:
-        return tokenizer_dir / "merges.txt"
+        """Return path to merges.txt file."""
+        return Path(tokenizer_dir) / "merges.txt"
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _check_loaded(self):
+        """
+        Check that tokenizer is loaded before encode/decode.
+
+        Raises:
+            RuntimeError: If tokenizer has not been trained or loaded.
+        """
+
+        if self._tokenizer is None:
+            raise RuntimeError(
+                "BPE tokenizer is not loaded. Call train() or load() before encode/decode."
+            )

openverifiablellm/verify.py

@@ -386,15 +386,17 @@ def verify_preprocessing(
             "environment_hash",
             expected=manifest.get("environment_hash"),
             actual=current_env["environment_hash"],
-            detail="Environment fingerprint comparison"
+            detail="Environment fingerprint comparison",
         )
     else:
-        report.add(CheckResult(
-            name="environment_hash",
-            status=CheckStatus.SKIP,
-            detail="Field absent from manifest (older version)"
-        ))
-
+        report.add(
+            CheckResult(
+                name="environment_hash",
+                status=CheckStatus.SKIP,
+                detail="Field absent from manifest (older version)",
+            )
+        )
+
     # 4. Re-run preprocessing in an isolated temp directory
     tmp_dir = Path(tempfile.mkdtemp(prefix="ovllm_verify_"))
     try: