[FIX] : Add Unit Tests for create_tokenizer and load_merkle_proof

openverifiablellm/tokenizer/__init__.py

@@ -1,6 +1,8 @@
+from .factory import create_tokenizer
 from .train import hash_tokenizer_config, train_tokenizer
 
 __all__ = [
     "train_tokenizer",
     "hash_tokenizer_config",
+    "create_tokenizer",
 ]

openverifiablellm/tokenizer/sentencepiece_tokenizer.py

@@ -1,6 +1,12 @@
 from pathlib import Path
 
-import sentencepiece as spm
+import warnings
+
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore", category=DeprecationWarning)
+    # SWIG-generated modules (like sentencepiece on python 3.11+) emit deprecation warnings
+    # scoping the suppression here prevents it from spamming our test output
+    import sentencepiece as spm
 
 from .base import BaseTokenizer
 

pyproject.toml

@@ -23,6 +23,7 @@ include = ["openverifiablellm*"]
 dev = [
     "pytest>=7.0",
     "ruff>=0.15.4",
+    "numpy",
 ]
 
 [tool.ruff]
@@ -32,3 +33,7 @@ target-version = "py39"
 [tool.ruff.lint]
 select = ["E", "F", "I"]
 ignore = ["E501"]
+
+[tool.pytest.ini_options]
+filterwarnings = [
+]

tests/test_tokenizer.py

@@ -3,9 +3,12 @@
 import pytest
 
 from openverifiablellm.tokenizer import (
+    create_tokenizer,
     hash_tokenizer_config,
     train_tokenizer,
 )
+from openverifiablellm.tokenizer.bpe_tokenizer import BPETokenizer
+from openverifiablellm.tokenizer.sentencepiece_tokenizer import SentencePieceTokenizer
 
 
 @pytest.fixture
@@ -166,3 +169,26 @@ def test_hash_tokenizer_missing_merges(tmp_path):
 
     with pytest.raises(FileNotFoundError):
         hash_tokenizer_config(tokenizer_path)
+
+
+# ---------------------------------------------------------------------
+# create_tokenizer Tests
+# ---------------------------------------------------------------------
+
+
+def test_create_tokenizer_bpe():
+    """Test that create_tokenizer returns a BPETokenizer for 'bpe'."""
+    tokenizer = create_tokenizer("bpe", vocab_size=1000, min_frequency=2)
+    assert isinstance(tokenizer, BPETokenizer)
+
+
+def test_create_tokenizer_sentencepiece():
+    """Test that create_tokenizer returns a SentencePieceTokenizer for 'sentencepiece'."""
+    tokenizer = create_tokenizer("sentencepiece", vocab_size=1000, min_frequency=2)
+    assert isinstance(tokenizer, SentencePieceTokenizer)
+
+
+def test_create_tokenizer_invalid():
+    """Test that create_tokenizer raises a ValueError for invalid types."""
+    with pytest.raises(ValueError, match="Unsupported tokenizer: invalid"):
+        create_tokenizer("invalid", vocab_size=1000, min_frequency=2)

tests/test_util.py

@@ -236,6 +236,27 @@ def test_merkle_root_empty_file(tmp_path):
     assert root == expected
 
 
+def test_compute_merkle_root_multi_chunk_hardcoded(tmp_path):
+    file = tmp_path / "data.txt"
+    # 3 chunks of 8 bytes each
+    chunk1 = b"chunk__1"
+    chunk2 = b"chunk__2"
+    chunk3 = b"chunk__3"
+    file.write_bytes(chunk1 + chunk2 + chunk3)
+
+    h1 = hashlib.sha256(chunk1).digest()
+    h2 = hashlib.sha256(chunk2).digest()
+    h3 = hashlib.sha256(chunk3).digest()
+
+    h12 = hashlib.sha256(h1 + h2).digest()
+    h33 = hashlib.sha256(h3 + h3).digest()
+
+    expected_root = hashlib.sha256(h12 + h33).hexdigest()
+
+    actual_root = utils.compute_merkle_root(file, chunk_size=8)
+    assert actual_root == expected_root
+
+
 # --------------- Merkle proof generation ------------------------------------
 
 
@@ -283,3 +304,35 @@ def test_export_and_load_merkle_proof(tmp_path):
         chunk_data=chunk,
         expected_root=root,
     )
+
+
+# --------------- load_merkle_proof tests ------------------------------------
+
+
+def test_load_merkle_proof_valid_file(tmp_path):
+    proof_data = {
+        "chunk_index": 1,
+        "chunk_size": 8,
+        "proof": [["00" * 32, True]],
+    }
+    proof_file = tmp_path / "proof.json"
+    proof_file.write_text(json.dumps(proof_data))
+
+    loaded_proof = utils.load_merkle_proof(proof_file)
+
+    assert loaded_proof == proof_data
+
+
+def test_load_merkle_proof_missing_file(tmp_path):
+    proof_file = tmp_path / "missing.json"
+
+    with pytest.raises(FileNotFoundError):
+        utils.load_merkle_proof(proof_file)
+
+
+def test_load_merkle_proof_invalid_json(tmp_path):
+    proof_file = tmp_path / "invalid.json"
+    proof_file.write_text("{invalid json}")
+
+    with pytest.raises(json.JSONDecodeError):
+        utils.load_merkle_proof(proof_file)

tests/test_verify.py

@@ -57,7 +57,7 @@ def run_preprocessing(tmp_dir: Path, dump: Path) -> None:
     original = os.getcwd()
     os.chdir(tmp_dir)
     try:
-        utils.extract_text_from_xml(dump)
+        utils.extract_text_from_xml(dump, write_manifest=True)
     finally:
         os.chdir(original)