Fix caching for other APIs as well (#23)

microsoft · Oct 13, 2023 · 3c6fcb9 · 3c6fcb9
1 parent 6c7d0d2
commit 3c6fcb9
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 3 deletions.
diff --git a/tokenizer_ts/package-lock.json b/tokenizer_ts/package-lock.json
diff --git a/tokenizer_ts/package.json b/tokenizer_ts/package.json
@@ -2,7 +2,7 @@
   "name": "@microsoft/tiktokenizer",
   "displayName": "tiktokenizer",
   "description": "Tokenizer for OpenAI large language models.",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "author": {
     "name": "Microsoft Corporation"
   },

diff --git a/tokenizer_ts/src/tikTokenizer.ts b/tokenizer_ts/src/tikTokenizer.ts
@@ -253,6 +253,7 @@ export class TikTokenizer {
         const bytes = this.textEncoder.encode(piece);
         const token = this.encoder!.get(uint8ArrayToString(bytes));
         if (token !== undefined) {
+          this.cache.set(piece, [token]);
           tokenCount++;
           if (tokenCount <= maxTokenCount) {
             encodeLength += piece.length;
@@ -397,6 +398,7 @@ export class TikTokenizer {
             const bytes = new TextEncoder().encode(piece);
             const token = this.encoder!.get(uint8ArrayToString(bytes));
             if (token !== undefined) {
+              this.cache.set(piece, [token]);
               tokenCount++;
               encodeLength += piece.length;
               tokenIds.push(token);