From 9fa9789388e8a6cecb042471e3cb59997014e8f2 Mon Sep 17 00:00:00 2001 From: Abhishey Date: Thu, 19 Mar 2026 16:49:59 +0530 Subject: [PATCH 1/2] build: bump tokenizers to >=0.21.0 for Windows ARM64 support --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 96523a0..3c34d29 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ requires-python = ">=3.9" dependencies = [ "defusedxml", "sentencepiece", - "tokenizers==0.15.2" + "tokenizers>=0.21.0", ] [tool.setuptools.packages.find] From 6f67592711f8e654ffac364a84d479805c0276c4 Mon Sep 17 00:00:00 2001 From: Abhishey Date: Thu, 19 Mar 2026 17:13:59 +0530 Subject: [PATCH 2/2] chore: add upper bound to tokenizers for stability --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3c34d29..2302297 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ requires-python = ">=3.9" dependencies = [ "defusedxml", "sentencepiece", - "tokenizers>=0.21.0", + "tokenizers>=0.21.0,<1.0.0", ] [tool.setuptools.packages.find]