vlcn-io · ikusteu · Feb 7, 2024 · Feb 7, 2024 · Feb 7, 2024 · Feb 7, 2024
diff --git a/.github/ISSUE_TEMPLATE/-do-not-post-anything-other-than-a-bug-report.md b/.github/ISSUE_TEMPLATE/-do-not-post-anything-other-than-a-bug-report.md
@@ -0,0 +1,10 @@
+---
+name: " Do not post anything other than a bug report"
+about: Issues are only for possible bugs in project code.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,10 @@
+---
+name: Bug report
+about: Report potential problems in project code
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+In this project, Issues are only for possible bugs in project code. Please don't ask for help debugging _your_ code; I have plenty of challenges debugging _my_ code. 🥲
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -10,7 +10,7 @@ on:
     branches: [ "master" ]
 
 env:
-  EM_VERSION: 3.1.45
+  EM_VERSION: 3.1.61
   EM_CACHE_FOLDER: 'emsdk-cache'
 
 jobs:
@@ -20,21 +20,28 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [16.x]
+        node-version: [20.x]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node-version }}
         cache: 'npm'
-
+
+    - uses: browser-actions/setup-chrome@v1
+      id: setup-chrome
+      with:
+        chrome-version: 129
+    - run: |
+          ${{ steps.setup-chrome.outputs.chrome-path }} --version
+
     # Install yarn dependencies.
     - name: Get yarn cache directory path
       id: yarn-cache-dir-path
       run: echo "dir=$(yarn config get cacheFolder)" >> $GITHUB_OUTPUT
-    - uses: actions/cache@v3
+    - uses: actions/cache@v4
       id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
       with:
         path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
@@ -44,16 +51,16 @@ jobs:
     - run: yarn install
 
     - name: Test with checked-in WASM files
-      run: yarn test --reporters progress
+      run: yarn test
 
     # Install EMSDK
     - name: Setup Emscripten
       id: cache-system-libraries
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{env.EM_CACHE_FOLDER}}
         key: ${{env.EM_VERSION}}-${{ runner.os }}
-    - uses: mymindstorm/setup-emsdk@v12
+    - uses: mymindstorm/setup-emsdk@v14
       with:
         version: ${{env.EM_VERSION}}
         actions-cache-folder: ${{env.EM_CACHE_FOLDER}}
@@ -62,4 +69,4 @@ jobs:
     - name: Test WASM build
       run: |
         make clean && make
-        yarn test --reporters progress
+        yarn test
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,7 @@
 .DS_Store
 
+node_modules/
+
 # https://yarnpkg.com/getting-started/qa/#which-files-should-be-gitignored
 .yarn/*
 !.yarn/patches
@@ -10,15 +12,9 @@
 .pnp.*
 *.o
 
+.turbo/
 /cache
 /debug
 /deps
 /tmp
-
-.DS_Store
-
-node_modules/
-.turbo/
-/dist-xl.zip
-/dist-xl
 /dist
diff --git a/.yarn/releases/yarn-3.1.1.cjs b/.yarn/releases/yarn-3.1.1.cjs
diff --git a/.yarn/releases/yarn-4.0.2.cjs b/.yarn/releases/yarn-4.0.2.cjs
diff --git a/.yarnrc.yml b/.yarnrc.yml
@@ -0,0 +1,5 @@
+compressionLevel: mixed
+
+enableGlobalCache: false
+
+yarnPath: .yarn/releases/yarn-4.0.2.cjs
diff --git a/Makefile b/Makefile
@@ -1,6 +1,6 @@
 # dependencies
-SQLITE_VERSION = version-3.45.0
-SQLITE_TARBALL_URL = https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=${SQLITE_VERSION}
+SQLITE_VERSION = version-3.50.1
+SQLITE_TARBALL_URL = https://www.sqlite.org/src/tarball/$(SQLITE_VERSION)/sqlite.tar.gz
 
 EXTENSION_FUNCTIONS = extension-functions.c
 EXTENSION_FUNCTIONS_URL = https://www.sqlite.org/contrib/download/extension-functions.c?get=25
@@ -15,13 +15,21 @@ CFILES_EXTRA = \
 CFILES = \
 	sqlite3-extra.c \
 	extension-functions.c \
+	main.c \
 	libauthorizer.c \
 	libfunction.c \
-	libmodule.c \
+	libhook.c \
 	libprogress.c \
 	libvfs.c \
 	$(CFILES_EXTRA)
 
+JSFILES = \
+	src/libauthorizer.js \
+	src/libfunction.js \
+	src/libhook.js \
+	src/libprogress.js \
+	src/libvfs.js
+
 dir.crsql := ./crsql/src
 
 vpath %.c src
@@ -32,6 +40,7 @@ vpath %.c $(dir.crsql)
 EXPORTED_FUNCTIONS = src/exported_functions.json
 EXPORTED_RUNTIME_METHODS = src/extra_exported_runtime_methods.json
 ASYNCIFY_IMPORTS = src/asyncify_imports.json
+JSPI_EXPORTS = src/jspi_exports.json
 
 # intermediate files
 RS_LIB = crsql_bundle
@@ -65,29 +74,30 @@ EMFLAGS_COMMON = \
 	-s INVOKE_RUN \
 	-s ENVIRONMENT="web,worker" \
 	-s STACK_SIZE=512KB \
+	-s WASM_BIGINT=0 \
 	$(EMFLAGS_EXTRA)
 
 EMFLAGS_DEBUG = \
 	-s ASSERTIONS=1 \
-	-g \
+	-g -O1 \
 	$(EMFLAGS_COMMON)
 
 EMFLAGS_DIST = \
 	-O3 \
 	-flto \
-	--closure 1 \
 	$(EMFLAGS_COMMON)
 
 EMFLAGS_INTERFACES = \
 	-s EXPORTED_FUNCTIONS=@$(EXPORTED_FUNCTIONS) \
 	-s EXPORTED_RUNTIME_METHODS=@$(EXPORTED_RUNTIME_METHODS)
 
 EMFLAGS_LIBRARIES = \
-	--js-library src/libauthorizer.js \
-	--js-library src/libfunction.js \
-	--js-library src/libmodule.js \
-	--js-library src/libprogress.js \
-	--js-library src/libvfs.js
+	--js-library src/libadapters.js \
+	--post-js src/libauthorizer.js \
+	--post-js src/libfunction.js \
+	--post-js src/libhook.js \
+	--post-js src/libprogress.js \
+	--post-js src/libvfs.js
 
 EMFLAGS_ASYNCIFY_COMMON = \
 	-s ASYNCIFY \
@@ -101,13 +111,20 @@ EMFLAGS_ASYNCIFY_DIST = \
 	$(EMFLAGS_ASYNCIFY_COMMON) \
 	-s ASYNCIFY_STACK_SIZE=16384
 
+EMFLAGS_JSPI = \
+	-s JSPI \
+	-s ASYNCIFY_IMPORTS=@src/asyncify_imports.json \
+	-s JSPI_EXPORTS=@src/jspi_exports.json
+
+# NOTE: The tests expect the default page size to be 8192 (not 4096 like the sqlite3 default)
 WASQLITE_EXTRA_DEFINES = \
 	-DSQLITE_EXTRA_INIT=core_init \
 	-DSQLITE_ENABLE_FTS5 \
 	-DSQLITE_OMIT_UTF16 \
 	-DSQLITE_ENABLE_BYTECODE_VTAB \
 	-DDEFAULT_CACHE_SIZE=8000 \
-	-DCRSQLITE_WASM
+	-DCRSQLITE_WASM \
+	-DSQLITE_DEFAULT_PAGE_SIZE=8192
 
 # https://www.sqlite.org/compile.html
 WASQLITE_DEFINES = \
@@ -138,12 +155,12 @@ crsqlite-extra: $(sqlite3.extra.c)
 
 .PHONY: clean
 clean:
-	rm -rf dist dist-xl debug tmp
+	rm -rf dist debug tmp
 	rm -f *.o
 
 .PHONY: spotless
 spotless:
-	rm -rf dist dist-xl debug tmp deps cache
+	rm -rf dist debug tmp deps cache
 
 ## cache
 .PHONY: clean-cache
@@ -159,8 +176,6 @@ cache/$(EXTENSION_FUNCTIONS):
 clean-deps:
 	rm -rf deps
 
-.PHONY: deps
-
 deps/$(SQLITE_VERSION)/sqlite3.h deps/$(SQLITE_VERSION)/sqlite3.c:
 	mkdir -p cache/$(SQLITE_VERSION)
 	curl -LsS $(SQLITE_TARBALL_URL) | tar -xzf - -C cache/$(SQLITE_VERSION)/ --strip-components=1
@@ -208,23 +223,32 @@ clean-debug:
 	rm -rf debug
 
 .PHONY: debug
-debug: debug/crsqlite-sync.mjs debug/crsqlite.mjs
+debug: debug/crsqlite-sync.mjs debug/crsqlite.mjs debug/crsqlite-jspi.mjs
 
-debug/crsqlite-sync.mjs: $(OBJ_FILES_DEBUG) $(RS_DEBUG_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
+debug/crsqlite-sync.mjs: $(OBJ_FILES_DEBUG) $(JSFILES) $(RS_DEBUG_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
 	mkdir -p debug
 	$(EMCC) $(EMFLAGS_DEBUG) \
 	  $(EMFLAGS_INTERFACES) \
 	  $(EMFLAGS_LIBRARIES) \
-		$(RS_WASM_TGT_DIR)/debug/deps/*.bc \
+	  $(RS_WASM_TGT_DIR)/debug/deps/*.bc \
 	  $(OBJ_FILES_DEBUG) -o $@
 
-debug/crsqlite.mjs: $(OBJ_FILES_DEBUG) $(RS_DEBUG_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
+debug/crsqlite.mjs: $(OBJ_FILES_DEBUG) $(JSFILES) $(RS_DEBUG_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
 	mkdir -p debug
 	$(EMCC) $(EMFLAGS_DEBUG) \
 	  $(EMFLAGS_INTERFACES) \
 	  $(EMFLAGS_LIBRARIES) \
 	  $(EMFLAGS_ASYNCIFY_DEBUG) \
-		$(RS_WASM_TGT_DIR)/debug/deps/*.bc \
+	  $(RS_WASM_TGT_DIR)/debug/deps/*.bc \
+	  $(OBJ_FILES_DEBUG) -o $@
+
+debug/crsqlite-jspi.mjs: $(OBJ_FILES_DEBUG) $(JSFILES) $(RS_RELEASE_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
+	mkdir -p debug
+	$(EMCC) $(EMFLAGS_DEBUG) \
+	  $(EMFLAGS_INTERFACES) \
+	  $(EMFLAGS_LIBRARIES) \
+	  $(EMFLAGS_JSPI) \
+	  $(RS_WASM_TGT_DIR)/release/deps/*.bc \
 	  $(OBJ_FILES_DEBUG) -o $@
 
 ## dist
@@ -233,24 +257,32 @@ clean-dist:
 	rm -rf dist
 
 .PHONY: dist
-dist: deps dist/crsqlite-sync.mjs dist/crsqlite.mjs
+dist: dist/crsqlite-sync.mjs dist/crsqlite.mjs dist/crsqlite-jspi.mjs
 
 FORCE: ;
 
-dist/crsqlite-sync.mjs: $(OBJ_FILES_DIST) $(RS_RELEASE_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
+dist/crsqlite-sync.mjs: $(OBJ_FILES_DIST) $(JSFILES) $(RS_RELEASE_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
 	mkdir -p dist
 	$(EMCC) $(EMFLAGS_DIST) \
 	  $(EMFLAGS_INTERFACES) \
 	  $(EMFLAGS_LIBRARIES) \
-		$(RS_WASM_TGT_DIR)/release/deps/*.bc \
+	  $(RS_WASM_TGT_DIR)/release/deps/*.bc \
 	  $(OBJ_FILES_DIST) -o $@
 
-dist/crsqlite.mjs: $(OBJ_FILES_DIST) $(RS_RELEASE_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
+dist/crsqlite.mjs: $(OBJ_FILES_DIST) $(JSFILES) $(RS_RELEASE_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
 	mkdir -p dist
 	$(EMCC) $(EMFLAGS_DIST) \
 	  $(EMFLAGS_INTERFACES) \
 	  $(EMFLAGS_LIBRARIES) \
 	  $(EMFLAGS_ASYNCIFY_DIST) \
-		$(RS_WASM_TGT_DIR)/release/deps/*.bc \
+	  $(RS_WASM_TGT_DIR)/release/deps/*.bc \
 	  $(OBJ_FILES_DIST) -o $@
 
+dist/crsqlite-jspi.mjs: $(OBJ_FILES_DIST) $(JSFILES) $(RS_RELEASE_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
+	mkdir -p dist
+	$(EMCC) $(EMFLAGS_DIST) \
+	  $(EMFLAGS_INTERFACES) \
+	  $(EMFLAGS_LIBRARIES) \
+	  $(EMFLAGS_JSPI) \
+	  $(RS_WASM_TGT_DIR)/release/deps/*.bc \
+	  $(OBJ_FILES_DIST) -o $@
diff --git a/README.md b/README.md
@@ -1,11 +1,11 @@
-[![wa-sqlite CI](https://github.com/rhashimoto/wa-sqlite/actions/workflows/ci.yml/badge.svg?branch=breaking-changes)](https://github.com/rhashimoto/wa-sqlite/actions/workflows/ci.yml?branch=breaking-changes)
+[![wa-sqlite CI](https://github.com/rhashimoto/wa-sqlite/actions/workflows/ci.yml/badge.svg)](https://github.com/rhashimoto/wa-sqlite/actions/workflows/ci.yml)
 
 # wa-sqlite
-This is a WebAssembly build of SQLite with experimental support for writing SQLite virtual filesystems and virtual table modules completely in Javascript. This allows alternative browser storage options such as IndexedDB and File System Access. Applications can opt to use either a synchronous or asynchronous (using Asyncify) SQLite library build (an asynchronous build is required for asynchronous extensions).
+This is a WebAssembly build of SQLite with support for writing SQLite virtual filesystems completely in Javascript. This allows alternative browser storage options such as IndexedDB and Origin Private File System. Applications can opt to use either a synchronous or asynchronous (using Asyncify or JSPI) SQLite library build (an asynchronous build is required for asynchronous extensions).
 
-[IndexedDB](https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/IDBMinimalVFS.js) and [Origin Private File System](https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/OriginPrivateFileSystemVFS.js) virtual file systems and a [virtual table module that accesses Javascript arrays](https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/ArrayModule.js) are among the examples provided as proof of concept.
+IndexedDB and several Origin Private File System virtual file systems are among the examples provided as proof of concept. A table comparing the different VFS classes is [here](https://github.com/rhashimoto/wa-sqlite/tree/master/src/examples#vfs-comparison).
 
-[Try the demo](https://rhashimoto.github.io/wa-sqlite/demo/) or run [benchmarks](https://rhashimoto.github.io/wa-sqlite/demo/benchmarks.html) with a modern desktop web browser. More information is available in the [FAQ](https://github.com/rhashimoto/wa-sqlite/issues?q=is%3Aissue+label%3Afaq+), [discussion forums](https://github.com/rhashimoto/wa-sqlite/discussions), and [API reference](https://rhashimoto.github.io/wa-sqlite/docs/).
+[Try the demo](https://rhashimoto.github.io/wa-sqlite/demo/?build=asyncify&config=IDBBatchAtomicVFS&reset) or run [benchmarks](https://rhashimoto.github.io/wa-sqlite/demo/benchmarks/?config=asyncify,IDBBatchAtomicVFS;asyncify,IDBMirrorVFS;default,AccessHandlePoolVFS;default,OPFSCoopSyncVFS;asyncify,OPFSAdaptiveVFS;asyncify,OPFSPermutedVFS) with a modern desktop web browser. More information is available in the [FAQ](https://github.com/rhashimoto/wa-sqlite/issues?q=is%3Aissue+label%3Afaq+), [discussion forums](https://github.com/rhashimoto/wa-sqlite/discussions), and [API reference](https://rhashimoto.github.io/wa-sqlite/docs/).
 
 ## Build
 The primary motivation for this project is to enable additions to SQLite with only Javascript. Most developers should be able to use the pre-built artifacts in
@@ -19,20 +19,20 @@ If you do want to build yourself, here are the prerequisites:
 
 * Building on Debian Linux is known to work, compatibility with other platforms is unknown.
 * `yarn` - If you use a different package manager (e.g. `npm`) then file paths in the demo will need adjustment.
-* [Emscripten SDK](https://emscripten.org/docs/getting_started/downloads.html) 3.1.25+.
+* [Emscripten SDK](https://emscripten.org/docs/getting_started/downloads.html) 3.1.61+.
 * `curl`, `make`, `openssl`, `sed`, `tclsh`, `unzip`
 
 Here are the build steps:
 * Make sure `emcc` works.
 * `git clone [email protected]:rhashimoto/wa-sqlite.git`
 * `cd wa-sqlite`
 * `yarn install`
-* `make` (or `yarn prepack`)
+* `make`
 
-The default build produces ES6 modules + WASM, [synchronous and asynchronous](https://github.com/rhashimoto/wa-sqlite/issues/7) (using Asyncify) in `dist/`.
+The default build produces ES6 modules + WASM, [synchronous and asynchronous](https://github.com/rhashimoto/wa-sqlite/issues/7) (using Asyncify and JSPI) in `dist/`.
 
 ## API
-Javascript wrappers for core SQLITE C API functions (and some others) are provided. Some convenience functions are also provided to reduce boilerplate. Here's sample code to load the library and call the API:
+Javascript wrappers for core SQLITE C API functions (and some others) are provided. Some convenience functions are also provided to reduce boilerplate. Here is sample code to load the library and call the API:
 
 ```javascript
   import SQLiteESMFactory from 'wa-sqlite/dist/crsqlite-sync.mjs';
@@ -51,25 +51,24 @@ Javascript wrappers for core SQLITE C API functions (and some others) are provid
   hello();
 ```
 
-The [implementation of `sqlite3.exec`](https://github.com/rhashimoto/wa-sqlite/blob/b5824ac0031da81712bee42671a917b252737c45/src/sqlite-api.js#L422-L434) may be of interest to anyone wanting more fine-grained use of SQLite statement objects (e.g. for binding parameters, explicit column datatypes, etc.).
+There is a slightly more complicated example [here](https://github.com/rhashimoto/wa-sqlite/tree/master/demo/hello) that also shows how to use a virtual filesystem (VFS) for persistent storage.
+
+The [implementation of `sqlite3.exec`](https://github.com/rhashimoto/wa-sqlite/blob/eb6e62584b2864d5029f51c6afe155d71ba0caa8/src/sqlite-api.js#L409-L418) may be of interest to anyone wanting more fine-grained use of SQLite statement objects (e.g. for binding parameters, explicit column datatypes, etc.).
 
 [API reference](https://rhashimoto.github.io/wa-sqlite/docs/)
 
 ## Demo
 To serve the demo directly from the source tree:
 * `yarn start`
-* Open a browser on http://localhost:8000/demo/
-
-The demo page provides access to databases on multiple VFS implementations, including IndexedDB (which is the only one persistent across page loads and multiple tabs). In addition, in each database there is a SQLite module named "array" that provides some historical stock data from a common Javascript array - use it for virtual tables in SQL like this:
-```sql
-CREATE VIRTUAL TABLE IF NOT EXISTS goog USING array;
+* Open a browser on http://localhost:8000/demo/?build=asyncify&config=IDBBatchAtomicVFS&reset
 
--- Use it directly out of the Javascript array:
-SELECT * FROM goog LIMIT 5;
+The demo page provides access to databases on multiple VFS implementations. Query parameters on the demo page URL can be used to specify the configuration and initial state:
 
--- Copy into a native table (on the current VFS):
-CREATE TABLE IF NOT EXISTS tbl AS SELECT * FROM goog;
-```
+| Parameter | Purpose | Values | Default |
+|----|----|----|----|
+| build | Emscripten build type | default, asyncify, jspi | default |
+| config | select VFS | MemoryVFS, MemoryAsyncVFS, IDBBatchAtomicVFS, IDBMirrorVFS, AccessHandlePoolVFS, OPFSAdaptiveVFS, OPFSAnyContextVFS, OPFSCoopSyncVFS, OPFSPermutedVFS | uses SQLite internal memory |
+| reset | clear persistent storage | | |
 
 For convenience, if any text region is selected in the editor, only that region will be executed. In addition, the editor contents are restored across page reloads using browser localStorage.