feat: add multi-language bindings for tiktoken using polywrap wasm wrapper

Cargo.toml

@@ -2,6 +2,7 @@
 resolver = "1"
 members = [
     "tiktoken-rs",
+    "bindings/wrap"
 ]
 
 [workspace.package]

bindings/python/.gitignore

@@ -0,0 +1,26 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# production
+/build
+
+# misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+__pycache__/
+.pytest_cache/

bindings/python/.nvmrc

@@ -0,0 +1 @@
+v18.15.0

bindings/python/.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+  "python.formatting.provider": "black",
+  "python.analysis.typeCheckingMode": "strict",
+  "python.defaultInterpreterPath": "/Users/niraj/Library/Caches/pypoetry/virtualenvs/tiktoken-OAolDqPj-py3.10"
+}

bindings/python/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "template-app-python",
+  "private": true,
+  "scripts": {
+    "codegen": "npx polywrap codegen",
+    "test": "poetry run python -m tiktoken"
+  },
+  "devDependencies": {
+    "polywrap": "0.11.3"
+  }
+}

bindings/python/polywrap.graphql

@@ -0,0 +1 @@
+#import * into Tiktoken from "fs/../wrap/build"

bindings/python/polywrap.yaml

@@ -0,0 +1,6 @@
+format: 0.4.0
+project:
+  name: Tiktoken
+  type: app/python
+source:
+  schema: ./polywrap.graphql

bindings/python/pyproject.toml

@@ -0,0 +1,13 @@
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry]
+name = "tiktoken"
+version = "0.1.0"
+description = "Python Tiktoken SDK"
+authors = ["Cesar <[email protected]>", "Niraj <[email protected]>"]
+
+[tool.poetry.dependencies]
+python = "^3.10"
+polywrap = "0.1.2"

bindings/python/tiktoken/__init__.py

@@ -0,0 +1,5 @@
+from .wrap import Tiktoken
+
+__all__ = [
+    "Tiktoken",
+]

bindings/python/tiktoken/__main__.py

@@ -0,0 +1,21 @@
+from .wrap import Tiktoken, TiktokenTokenizer
+
+
+if __name__ == "__main__":
+    tiktoken = Tiktoken()
+
+    print("encode...")
+    encoded = tiktoken.encode_with_special_tokens({
+        "tokenizer": TiktokenTokenizer.P50kBase,
+        "text": "This is a test         with a lot of spaces"
+    })
+
+    print("decode...")
+    decoded = tiktoken.decode({
+        "tokenizer": TiktokenTokenizer.P50kBase,
+        "tokens": encoded
+    })
+
+    print("encoded:", encoded)
+    print("decoded:", decoded)
+    assert decoded == "This is a test         with a lot of spaces"

bindings/python/tiktoken/wrap/__init__.py

@@ -0,0 +1 @@
+from .types import *