diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 572fe2c..25e3fea 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,8 +32,8 @@ jobs: rustc --version cargo --version cargo build --release -p netsaur - deno run -Ar https://deno.land/x/wasmbuild@0.15.6/main.ts -p netsaur --out src/backends/wasm/lib - deno run -Ar https://deno.land/x/wasmbuild@0.15.6/main.ts -p netsaur-tokenizers --out tokenizers/lib + deno run -Ar jsr:@deno/wasmbuild@0.17.1 -p netsaur --out src/backends/wasm/lib + deno run -Ar jsr:@deno/wasmbuild@0.17.1 -p netsaur-tokenizers --out tokenizers/lib - name: Release uses: softprops/action-gh-release@master env: diff --git a/Cargo.lock b/Cargo.lock index fc22356..042786f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -219,9 +219,9 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "js-sys" -version = "0.3.67" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -344,7 +344,7 @@ dependencies = [ [[package]] name = "netsaur" -version = "0.3.1" +version = "0.3.1-patch" dependencies = [ "getrandom", "js-sys", @@ -358,7 +358,7 @@ dependencies = [ [[package]] name = "netsaur-gpu" -version = "0.3.1" +version = "0.3.1-patch" dependencies = [ "cudarc", "ndarray", @@ -371,7 +371,7 @@ dependencies = [ [[package]] name = "netsaur-tokenizers" -version = "0.3.1" +version = "0.3.1-patch" dependencies = [ "getrandom", "js-sys", @@ -761,9 +761,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.90" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -771,9 +771,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.90" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", @@ -786,9 +786,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.90" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -796,9 +796,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.90" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", @@ -809,6 +809,6 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.90" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" diff --git a/README.md b/README.md index 31da1c9..5be11fe 100644 --- a/README.md +++ b/README.md @@ -254,7 +254,7 @@ console.log(`1 xor 1 = ${out4[0]} (should be close to 0)`); ### Documentation The full documentation for Netsaur can be found -[here](https://deno.land/x/netsaur@0.3.1/mod.ts). +[here](https://deno.land/x/netsaur@0.3.1-patch/mod.ts). ### License diff --git a/crates/core-gpu/Cargo.toml b/crates/core-gpu/Cargo.toml index cfe10e8..db058c3 100644 --- a/crates/core-gpu/Cargo.toml +++ b/crates/core-gpu/Cargo.toml @@ -1,7 +1,7 @@ [package] edition = "2021" name = "netsaur-gpu" -version = "0.3.1" +version = "0.3.1-patch" [lib] crate-type = ["cdylib"] diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 2083381..9da1efe 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -1,7 +1,7 @@ [package] edition = "2021" name = "netsaur" -version = "0.3.1" +version = "0.3.1-patch" [lib] crate-type = ["cdylib"] @@ -14,6 +14,6 @@ serde_json = "1.0" safetensors = "0.4.0" [target.'cfg(target_arch = "wasm32")'.dependencies] -wasm-bindgen = "=0.2.90" +wasm-bindgen = "0.2.92" getrandom = { version = "0.2", features = ["js"] } -js-sys = "0.3.67" +js-sys = "0.3.69" diff --git a/crates/tokenizers/Cargo.toml b/crates/tokenizers/Cargo.toml index 22f1a1e..1f8a351 100644 --- a/crates/tokenizers/Cargo.toml +++ b/crates/tokenizers/Cargo.toml @@ -1,7 +1,7 @@ [package] edition = "2021" name = "netsaur-tokenizers" -version = "0.3.1" +version = "0.3.1-patch" [lib] crate-type = ["cdylib"] @@ -13,6 +13,6 @@ serde = {version = "1.0", features = ["derive"]} serde_json = "1.0" serde-wasm-bindgen = "0.6.0" tokenizers = { version="0.14.1", default-features=false, features = ["unstable_wasm"]} -wasm-bindgen = "=0.2.90" +wasm-bindgen = "0.2.92" getrandom = { version = "0.2", features = ["js"] } -js-sys = "0.3.67" +js-sys = "0.3.69" diff --git a/deno.json b/deno.json index 55c543e..db57725 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "name": "@denosaurs/netsaur", - "version": "0.3.2", + "version": "0.3.2-patch", "exports": { ".": "./mod.ts", "./tokenizers": "./tokenizers/mod.ts", @@ -28,7 +28,7 @@ "build": "deno task build:cpu && deno task build:wasm && deno task build:tokenizers", "build:cpu": "cargo build --release -p netsaur", "build:gpu": "cargo build --release -p netsaur-gpu", - "build:wasm": "deno run -Ar https://deno.land/x/wasmbuild@0.15.6/main.ts -p netsaur --out src/backends/wasm/lib", - "build:tokenizers": "deno run -Ar https://deno.land/x/wasmbuild@0.15.6/main.ts -p netsaur-tokenizers --out tokenizers/lib" + "build:wasm": "deno run -Ar jsr:@deno/wasmbuild@0.17.1 -p netsaur --out src/backends/wasm/lib", + "build:tokenizers": "deno run -Ar jsr:@deno/wasmbuild@0.17.1 -p netsaur-tokenizers --out tokenizers/lib" } } diff --git a/examples/visualize.ipynb b/examples/visualize.ipynb index cba0443..92ff4d8 100644 --- a/examples/visualize.ipynb +++ b/examples/visualize.ipynb @@ -74,7 +74,7 @@ " setupBackend,\n", " SigmoidLayer,\n", " tensor2D,\n", - "} from \"https://deno.land/x/netsaur@0.3.1/mod.ts\";\n", + "} from \"https://deno.land/x/netsaur@0.3.1-patch/mod.ts\";\n", "\n", "await setupBackend(AUTO);\n", "\n", diff --git a/src/backends/cpu/mod.ts b/src/backends/cpu/mod.ts index 938caa8..11bd85a 100644 --- a/src/backends/cpu/mod.ts +++ b/src/backends/cpu/mod.ts @@ -15,7 +15,7 @@ const options: FetchOptions = { name: "netsaur", url: new URL(import.meta.url).protocol !== "file:" ? new URL( - "https://github.com/denosaurs/netsaur/releases/download/0.3.2/", + "https://github.com/denosaurs/netsaur/releases/download/0.3.2-patch/", import.meta.url, ) : "./target/release/", diff --git a/src/backends/gpu/mod.ts b/src/backends/gpu/mod.ts index 3178206..ddb1c08 100644 --- a/src/backends/gpu/mod.ts +++ b/src/backends/gpu/mod.ts @@ -15,7 +15,7 @@ const options: FetchOptions = { name: "netsaur_gpu", url: new URL(import.meta.url).protocol !== "file:" ? new URL( - "https://github.com/denosaurs/netsaur/releases/download/0.3.2/", + "https://github.com/denosaurs/netsaur/releases/download/0.3.2-patch/", import.meta.url, ) : "./target/release/", diff --git a/src/backends/wasm/lib/netsaur.generated.d.ts b/src/backends/wasm/lib/netsaur.generated.d.ts new file mode 100644 index 0000000..7019bba --- /dev/null +++ b/src/backends/wasm/lib/netsaur.generated.d.ts @@ -0,0 +1,65 @@ +// deno-lint-ignore-file +// deno-fmt-ignore-file + +export interface InstantiateResult { + instance: WebAssembly.Instance; + exports: { + wasm_backend_create: typeof wasm_backend_create; + wasm_backend_train: typeof wasm_backend_train; + wasm_backend_predict: typeof wasm_backend_predict; + wasm_backend_save: typeof wasm_backend_save; + wasm_backend_load: typeof wasm_backend_load + }; +} + +/** Gets if the Wasm module has been instantiated. */ +export function isInstantiated(): boolean; + +/** Options for instantiating a Wasm instance. */ +export interface InstantiateOptions { + /** Optional url to the Wasm file to instantiate. */ + url?: URL; + /** Callback to decompress the raw Wasm file bytes before instantiating. */ + decompress?: (bytes: Uint8Array) => Uint8Array; +} + +/** Instantiates an instance of the Wasm module returning its functions. +* @remarks It is safe to call this multiple times and once successfully +* loaded it will always return a reference to the same object. */ +export function instantiate(opts?: InstantiateOptions): Promise; + +/** Instantiates an instance of the Wasm module along with its exports. + * @remarks It is safe to call this multiple times and once successfully + * loaded it will always return a reference to the same object. */ +export function instantiateWithInstance(opts?: InstantiateOptions): Promise; + +/** +* @param {string} config +* @param {Array} shape +* @returns {number} +*/ +export function wasm_backend_create(config: string, shape: Array): number; +/** +* @param {number} id +* @param {(Float32Array)[]} buffers +* @param {string} options +*/ +export function wasm_backend_train(id: number, buffers: (Float32Array)[], options: string): void; +/** +* @param {number} id +* @param {Float32Array} buffer +* @param {string} options +* @returns {Float32Array} +*/ +export function wasm_backend_predict(id: number, buffer: Float32Array, options: string): Float32Array; +/** +* @param {number} id +* @returns {Uint8Array} +*/ +export function wasm_backend_save(id: number): Uint8Array; +/** +* @param {Uint8Array} buffer +* @param {Array} shape +* @returns {number} +*/ +export function wasm_backend_load(buffer: Uint8Array, shape: Array): number; diff --git a/src/backends/wasm/lib/netsaur.generated.js b/src/backends/wasm/lib/netsaur.generated.js index 0fcf6d7..5e33c00 100644 --- a/src/backends/wasm/lib/netsaur.generated.js +++ b/src/backends/wasm/lib/netsaur.generated.js @@ -2,7 +2,9 @@ // @ts-nocheck: generated // deno-lint-ignore-file // deno-fmt-ignore-file -// source-hash: ac65da5f94f2c624c75bd431d5aee05293df6fab +/// + +// source-hash: c0144020ff3bc5a20072ca9f8a0a0e02af6c6eb7 let wasm; let cachedInt32Memory0; @@ -106,6 +108,7 @@ function passStringToWasm0(arg, malloc, realloc) { const ret = encodeString(arg, view); offset += ret.written; + ptr = realloc(ptr, len, offset, 1) >>> 0; } WASM_VECTOR_LEN = offset; @@ -274,11 +277,11 @@ const imports = { getObject(arg0).getRandomValues(getObject(arg1)); }, arguments); }, - __wbg_newnoargs_5859b6d41c6fe9f7: function (arg0, arg1) { + __wbg_newnoargs_e258087cd0daa0ea: function (arg0, arg1) { const ret = new Function(getStringFromWasm0(arg0, arg1)); return addHeapObject(ret); }, - __wbg_call_a79f1973a4f07d5e: function () { + __wbg_call_27c0f87801dedf93: function () { return handleError(function (arg0, arg1) { const ret = getObject(arg0).call(getObject(arg1)); return addHeapObject(ret); @@ -288,25 +291,25 @@ const imports = { const ret = getObject(arg0); return addHeapObject(ret); }, - __wbg_self_086b5302bcafb962: function () { + __wbg_self_ce0dbfc45cf2f5be: function () { return handleError(function () { const ret = self.self; return addHeapObject(ret); }, arguments); }, - __wbg_window_132fa5d7546f1de5: function () { + __wbg_window_c6fb939a7f436783: function () { return handleError(function () { const ret = window.window; return addHeapObject(ret); }, arguments); }, - __wbg_globalThis_e5f801a37ad7d07b: function () { + __wbg_globalThis_d1e6af4856ba331b: function () { return handleError(function () { const ret = globalThis.globalThis; return addHeapObject(ret); }, arguments); }, - __wbg_global_f9a61fce4af6b7c1: function () { + __wbg_global_207b558942527489: function () { return handleError(function () { const ret = global.global; return addHeapObject(ret); @@ -316,23 +319,23 @@ const imports = { const ret = getObject(arg0) === undefined; return ret; }, - __wbg_set_379b27f1d5f1bf9c: function (arg0, arg1, arg2) { + __wbg_set_d4638f722068f043: function (arg0, arg1, arg2) { getObject(arg0)[arg1 >>> 0] = takeObject(arg2); }, - __wbg_setlength_f34414d42869b943: function (arg0, arg1) { + __wbg_setlength_ef7b0804fcb1df8d: function (arg0, arg1) { getObject(arg0).length = arg1 >>> 0; }, - __wbg_call_f6a2bc58c19c53c6: function () { + __wbg_call_b3ca7c6051f9bec1: function () { return handleError(function (arg0, arg1, arg2) { const ret = getObject(arg0).call(getObject(arg1), getObject(arg2)); return addHeapObject(ret); }, arguments); }, - __wbg_buffer_5d1b598a01b41a42: function (arg0) { + __wbg_buffer_12d079cc21e14bdb: function (arg0) { const ret = getObject(arg0).buffer; return addHeapObject(ret); }, - __wbg_newwithbyteoffsetandlength_d695c7957788f922: function ( + __wbg_newwithbyteoffsetandlength_aa4a17c33a06e5cb: function ( arg0, arg1, arg2, @@ -340,18 +343,18 @@ const imports = { const ret = new Uint8Array(getObject(arg0), arg1 >>> 0, arg2 >>> 0); return addHeapObject(ret); }, - __wbg_new_ace717933ad7117f: function (arg0) { + __wbg_new_63b92bc8671ed464: function (arg0) { const ret = new Uint8Array(getObject(arg0)); return addHeapObject(ret); }, - __wbg_set_74906aa30864df5a: function (arg0, arg1, arg2) { + __wbg_set_a47bac70306a19a7: function (arg0, arg1, arg2) { getObject(arg0).set(getObject(arg1), arg2 >>> 0); }, - __wbg_length_f0764416ba5bb237: function (arg0) { + __wbg_length_c20a40f15020d68a: function (arg0) { const ret = getObject(arg0).length; return ret; }, - __wbg_newwithbyteoffsetandlength_21163b4dfcbc673c: function ( + __wbg_newwithbyteoffsetandlength_4a659d079a1650e0: function ( arg0, arg1, arg2, @@ -359,22 +362,22 @@ const imports = { const ret = new Float32Array(getObject(arg0), arg1 >>> 0, arg2 >>> 0); return addHeapObject(ret); }, - __wbg_new_9d3ed79326710a1c: function (arg0) { + __wbg_new_9efabd6b6d2ce46d: function (arg0) { const ret = new Float32Array(getObject(arg0)); return addHeapObject(ret); }, - __wbg_set_2967d31ab8d0cef7: function (arg0, arg1, arg2) { + __wbg_set_bd975934d1b1fddb: function (arg0, arg1, arg2) { getObject(arg0).set(getObject(arg1), arg2 >>> 0); }, - __wbg_length_94345229ddb57245: function (arg0) { + __wbg_length_d25bbcbc3367f684: function (arg0) { const ret = getObject(arg0).length; return ret; }, - __wbg_newwithlength_728575f3bba9959b: function (arg0) { + __wbg_newwithlength_e9b4878cebadb3d3: function (arg0) { const ret = new Uint8Array(arg0 >>> 0); return addHeapObject(ret); }, - __wbg_subarray_7f7a652672800851: function (arg0, arg1, arg2) { + __wbg_subarray_a1f73cd4b5b42fe1: function (arg0, arg1, arg2) { const ret = getObject(arg0).subarray(arg1 >>> 0, arg2 >>> 0); return addHeapObject(ret); }, @@ -388,56 +391,23 @@ const imports = { }, }; -/** - * @callback WasmBuildDecompressCallback - * @param {Uint8Array} compressed - * @returns {Uint8Array} decompressed - */ - -/** - * @callback WasmBuildCacheCallback - * @param {URL} url - * @param {WasmBuildDecompressCallback | undefined} decompress - * @returns {Promise} - */ - -/** - * @typedef WasmBuildLoaderOptions - * @property {WebAssembly.Imports | undefined} imports - The Wasm module's imports. - * @property {WasmBuildCacheCallback} [cache] - A function that caches the Wasm module to - * a local path so that a network request isn't required on every load. - * - * Returns an ArrayBuffer with the bytes on download success, but cache save failure. - */ - class WasmBuildLoader { - /** @type {WasmBuildLoaderOptions} */ #options; - /** @type {Promise | undefined} */ #lastLoadPromise; - /** @type {WebAssembly.WebAssemblyInstantiatedSource | undefined} */ #instantiated; - /** @param {WasmBuildLoaderOptions} options */ constructor(options) { this.#options = options; } - /** @returns {WebAssembly.Instance | undefined} */ get instance() { return this.#instantiated?.instance; } - /** @returns {WebAssembly.Module | undefined} */ get module() { return this.#instantiated?.module; } - /** - * @param {URL} url - * @param {WasmBuildDecompressCallback | undefined} decompress - * @returns {Promise} - */ load( url, decompress, @@ -457,10 +427,6 @@ class WasmBuildLoader { return this.#lastLoadPromise; } - /** - * @param {URL} url - * @param {WasmBuildDecompressCallback | undefined} decompress - */ async #instantiate(url, decompress) { const imports = this.#options.imports; if (this.#options.cache != null && url.protocol !== "file:") { @@ -483,8 +449,7 @@ class WasmBuildLoader { const isFile = url.protocol === "file:"; // make file urls work in Node via dnt - const isNode = - (/** @type {any} */ (globalThis)).process?.versions?.node != null; + const isNode = globalThis.process?.versions?.node != null; if (isFile && typeof Deno !== "object") { throw new Error( "Loading local files are not supported in this environment", @@ -513,12 +478,7 @@ class WasmBuildLoader { wasmResponse.headers.get("content-type")?.toLowerCase() .startsWith("application/wasm") ) { - return WebAssembly.instantiateStreaming( - // Cast to any so there's no type checking issues with dnt - // (https://github.com/denoland/wasmbuild/issues/92) - /** @type {any} */ (wasmResponse), - imports, - ); + return WebAssembly.instantiateStreaming(wasmResponse, imports); } else { return WebAssembly.instantiate( await wasmResponse.arrayBuffer(), @@ -531,65 +491,19 @@ class WasmBuildLoader { } } } - -/** @param {URL | string} url */ -async function fetchWithRetries(url, maxRetries = 5) { - let sleepMs = 250; - let iterationCount = 0; - while (true) { - iterationCount++; - try { - const res = await fetch(url); - if (res.ok || iterationCount > maxRetries) { - return res; - } - } catch (err) { - if (iterationCount > maxRetries) { - throw err; - } - } - console.warn(`Failed fetching. Retrying in ${sleepMs}ms...`); - await new Promise((resolve) => setTimeout(resolve, sleepMs)); - sleepMs = Math.min(sleepMs * 2, 10_000); - } -} const isNodeOrDeno = typeof Deno === "object" || (typeof process !== "undefined" && process.versions != null && process.versions.node != null); const loader = new WasmBuildLoader({ imports, - cache: isNodeOrDeno - ? (await import("https://deno.land/x/wasmbuild@0.15.6/loader/cache.ts")) - .cacheToLocalDir - : undefined, + cache: isNodeOrDeno ? cacheToLocalDir : undefined, }); -/** - * Options for instantiating a Wasm instance. - * @typedef {Object} InstantiateOptions - * @property {URL=} url - Optional url to the Wasm file to instantiate. - * @property {DecompressCallback=} decompress - Callback to decompress the - * raw Wasm file bytes before instantiating. - */ -/** Instantiates an instance of the Wasm module returning its functions. - * @remarks It is safe to call this multiple times and once successfully - * loaded it will always return a reference to the same object. - * @param {InstantiateOptions=} opts - */ export async function instantiate(opts) { return (await instantiateWithInstance(opts)).exports; } -/** Instantiates an instance of the Wasm module along with its exports. - * @remarks It is safe to call this multiple times and once successfully - * loaded it will always return a reference to the same object. - * @param {InstantiateOptions=} opts - * @returns {Promise<{ - * instance: WebAssembly.Instance; - * exports: { wasm_backend_create: typeof wasm_backend_create; wasm_backend_train: typeof wasm_backend_train; wasm_backend_predict: typeof wasm_backend_predict; wasm_backend_save: typeof wasm_backend_save; wasm_backend_load: typeof wasm_backend_load } - * }>} - */ export async function instantiateWithInstance(opts) { const { instance } = await loader.load( opts?.url ?? new URL("netsaur_bg.wasm", import.meta.url), @@ -614,7 +528,171 @@ function getWasmInstanceExports() { }; } -/** Gets if the Wasm module has been instantiated. */ export function isInstantiated() { return loader.instance != null; } +export async function cacheToLocalDir(url, decompress) { + const localPath = await getUrlLocalPath(url); + if (localPath == null) { + return undefined; + } + if (!await exists(localPath)) { + const fileBytes = decompress(new Uint8Array(await getUrlBytes(url))); + try { + await Deno.writeFile(localPath, fileBytes); + } catch { + // ignore and return the wasm bytes + return fileBytes; + } + } + return toFileUrl(localPath); +} +async function getUrlLocalPath(url) { + try { + const dataDirPath = await getInitializedLocalDataDirPath(); + const hash = await getUrlHash(url); + return `${dataDirPath}/${hash}.wasm`; + } catch { + return undefined; + } +} +async function getInitializedLocalDataDirPath() { + const dataDir = localDataDir(); + if (dataDir == null) { + throw new Error(`Could not find local data directory.`); + } + const dirPath = `${dataDir}/deno-wasmbuild`; + await ensureDir(dirPath); + return dirPath; +} +async function exists(filePath) { + try { + await Deno.lstat(filePath); + return true; + } catch (error) { + if (error instanceof Deno.errors.NotFound) { + return false; + } + throw error; + } +} +async function ensureDir(dir) { + try { + const fileInfo = await Deno.lstat(dir); + if (!fileInfo.isDirectory) { + throw new Error(`Path was not a directory '${dir}'`); + } + } catch (err) { + if (err instanceof Deno.errors.NotFound) { + // if dir not exists. then create it. + await Deno.mkdir(dir, { recursive: true }); + return; + } + throw err; + } +} +async function getUrlHash(url) { + // Taken from MDN: https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest + const hashBuffer = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(url.href), + ); + // convert buffer to byte array + const hashArray = Array.from(new Uint8Array(hashBuffer)); + // convert bytes to hex string + const hashHex = hashArray + .map((b) => b.toString(16).padStart(2, "0")) + .join(""); + return hashHex; +} +async function getUrlBytes(url) { + const response = await fetchWithRetries(url); + return await response.arrayBuffer(); +} +// the below is extracted from deno_std/path +const WHITESPACE_ENCODINGS = { + "\u0009": "%09", + "\u000A": "%0A", + "\u000B": "%0B", + "\u000C": "%0C", + "\u000D": "%0D", + "\u0020": "%20", +}; +function encodeWhitespace(string) { + return string.replaceAll(/[\s]/g, (c) => { + return WHITESPACE_ENCODINGS[c] ?? c; + }); +} +function toFileUrl(path) { + return Deno.build.os === "windows" + ? windowsToFileUrl(path) + : posixToFileUrl(path); +} +function posixToFileUrl(path) { + const url = new URL("file:///"); + url.pathname = encodeWhitespace( + path.replace(/%/g, "%25").replace(/\\/g, "%5C"), + ); + return url; +} +function windowsToFileUrl(path) { + const [, hostname, pathname] = path.match( + /^(?:[/\\]{2}([^/\\]+)(?=[/\\](?:[^/\\]|$)))?(.*)/, + ); + const url = new URL("file:///"); + url.pathname = encodeWhitespace(pathname.replace(/%/g, "%25")); + if (hostname != null && hostname != "localhost") { + url.hostname = hostname; + if (!url.hostname) { + throw new TypeError("Invalid hostname."); + } + } + return url; +} +export async function fetchWithRetries(url, maxRetries = 5) { + let sleepMs = 250; + let iterationCount = 0; + while (true) { + iterationCount++; + try { + const res = await fetch(url); + if (res.ok || iterationCount > maxRetries) { + return res; + } + } catch (err) { + if (iterationCount > maxRetries) { + throw err; + } + } + console.warn(`Failed fetching. Retrying in ${sleepMs}ms...`); + await new Promise((resolve) => setTimeout(resolve, sleepMs)); + sleepMs = Math.min(sleepMs * 2, 10000); + } +} +// MIT License - Copyright (c) justjavac. +// https://github.com/justjavac/deno_dirs/blob/e8c001bbef558f08fd486d444af391729b0b8068/data_local_dir/mod.ts +function localDataDir() { + switch (Deno.build.os) { + case "linux": { + const xdg = Deno.env.get("XDG_DATA_HOME"); + if (xdg) { + return xdg; + } + const home = Deno.env.get("HOME"); + if (home) { + return `${home}/.local/share`; + } + break; + } + case "darwin": { + const home = Deno.env.get("HOME"); + if (home) { + return `${home}/Library/Application Support`; + } + break; + } + case "windows": + return Deno.env.get("LOCALAPPDATA") ?? undefined; + } + return undefined; +} diff --git a/src/backends/wasm/lib/netsaur_bg.wasm b/src/backends/wasm/lib/netsaur_bg.wasm index 1409079..cf21c42 100644 Binary files a/src/backends/wasm/lib/netsaur_bg.wasm and b/src/backends/wasm/lib/netsaur_bg.wasm differ diff --git a/src/backends/wasm/mod.ts b/src/backends/wasm/mod.ts index 179c492..91b1db7 100644 --- a/src/backends/wasm/mod.ts +++ b/src/backends/wasm/mod.ts @@ -22,7 +22,7 @@ export class WASMInstance { await instantiate({ url: new URL(import.meta.url).protocol !== "file:" ? new URL( - "https://github.com/denosaurs/netsaur/releases/download/0.3.2/netsaur_bg.wasm", + "https://github.com/denosaurs/netsaur/releases/download/0.3.2-patch/netsaur_bg.wasm", import.meta.url, ) : undefined, diff --git a/tokenizers/mod.ts b/tokenizers/mod.ts index a796574..cda505d 100644 --- a/tokenizers/mod.ts +++ b/tokenizers/mod.ts @@ -16,7 +16,7 @@ export async function init(): Promise { await instantiate({ url: new URL(import.meta.url).protocol !== "file:" ? new URL( - "https://github.com/denosaurs/netsaur/releases/download/0.3.2/netsaur_tokenizers_bg.wasm", + "https://github.com/denosaurs/netsaur/releases/download/0.3.2-patch/netsaur_tokenizers_bg.wasm", import.meta.url, ) : undefined,