Add support for gpt4o with o200k_base

Sources/Tiktoken/Model.swift

@@ -20,6 +20,7 @@ enum Model {
 private extension Model {
     static let MODEL_PREFIX_TO_ENCODING: [String: String] = [
         // chat
+        "gpt-4o-": "o200k_base",
         "gpt-4-": "cl100k_base",  // e.g., gpt-4-0314, etc., plus gpt-4-32k
         "gpt-3.5-turbo-": "cl100k_base",  // e.g, gpt-3.5-turbo-0301, -0401, etc.
     ]
@@ -28,6 +29,7 @@ private extension Model {
         // chat
         "gpt-4": "cl100k_base",
         "gpt-3.5-turbo": "cl100k_base",
+        "gpt-4o": "o200k_base",
         // text
         "text-davinci-003": "p50k_base",
         "text-davinci-002": "p50k_base",

Sources/Tiktoken/Vocab.swift

@@ -76,6 +76,25 @@ public extension Vocab {
                 "<|endofprompt|>": 100276
               ])
     }
-
-    static var all: [Vocab] = [.gpt2, .r50kBase, .p50kBase, .p50kEdit, .cl100kBase]
+
+    static var o200kBase: Vocab {
+        .init(name: "o200k_base",
+              url: "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken",
+              pattern:
+                String([
+                    #"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?"#,
+                    #"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?"#,
+                    #"\p{N}{1,3}"#,
+                    #" ?[^\s\p{L}\p{N}]+[\r\n/]*"#,
+                    #"\s*[\r\n]+"#,
+                    #"\s+(?!\S)"#,
+                    #"\s+"#,
+                ].joined(separator: "|")),
+              specialTokens: [
+                "<|endoftext|>": 199999,
+                "<|endofprompt|>": 200018
+              ])
+    }
+
+    static var all: [Vocab] = [.gpt2, .r50kBase, .p50kBase, .p50kEdit, .cl100kBase, .o200kBase]
 }

Tests/TiktokenTests/TiktokenTests.swift

@@ -27,4 +27,11 @@ final class TiktokenTests: XCTestCase {
         let output = try XCTUnwrap(encoder?.encode(value: input))
         XCTAssertEqual(output, expected)
     }
+
+    func testGivenGPT4oEncodeAndDecode() async throws {
+        let input = "hello world"
+        let encoder = try await sut.getEncoding("gpt-4o")
+        let output = try XCTUnwrap(encoder?.encode(value: input))
+        XCTAssertEqual(encoder?.decode(value: output), input)
+    }
 }