diff --git a/CHANGELOG.md b/CHANGELOG.md index 69383d7..f015a3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +## [0.1.1] - 2026-07-11 + +### Fixed +- Read Zarr **v2** stores over HTTP/S3/GCS/Azure by consuming their consolidated `.zmetadata`. `read_zarr`, `read_zarr_metadata`, and `read_zarr_groups` previously errored on *every* remote v2 store (`remote Zarr store has no consolidated metadata in zarr.json`) because `list_array_names_remote` only understood the Zarr v3 `consolidated_metadata` block. Object stores can't list directories, so v2's separate `.zmetadata` object is the only way to enumerate arrays remotely; the reader now falls back to it. Public v2 stores such as Pangeo GPCP and ARCO-ERA5 read directly from their URL. Covered by `test/test_http_integration.py` (v2 `.zmetadata`, v3 `consolidated_metadata`, and an OME-Zarr bioimage — `array_path` selection plus a nested label image — over a loopback HTTP server; `make test_http_debug`) and by `test/test_http_integration_real.py`, which reads the live public Pangeo GPCP v2 store end-to-end (`make test_http_real`, network-gated). The OME-Zarr fixture is now written with consolidated metadata so it is readable over HTTP as well as locally. +- Read OME-Zarr images by `array_path` even when the store has no consolidated metadata. `array_path=` now opens the requested array directly instead of first listing the whole store, and dimension names fall back to the OME `multiscales.axes` when the array carries neither `dimension_names` nor `_ARRAY_DIMENSIONS`. Real public bioimage stores (e.g. the IDR) now read via `read_zarr(url, array_path='0')` — covered by a network-gated test in `test/test_http_integration_real.py`. +- An array selected by `array_path` exposes its data as a `value` column, so numeric levels (`0`) and nested paths (`labels/nuclei/0`) no longer need to be double-quoted as SQL identifiers. +- `dims=` is now a `LIST(VARCHAR)` — `read_zarr(store, dims=['time','lat','lon'])`, the idiomatic SQL form. It was previously a `VARCHAR` that accepted only a comma-separated or JSON-array *string* (and the SQL-list form errored at bind with `expected ident`). +- Docs now use real, runnable examples verified against the extension — the public GPCP store in `README.md`, plus this repo's fixtures in `docs/README.md` and `docs/ome-zarr.md` — replacing the `path/to/...` and `image.ome.zarr` placeholders. + +## [0.1.0] - 2026-07-07 + ### Fixed - Rename the crate/extension from `duckdb_zarr` to `zarr` throughout (Cargo package name, `Makefile` `EXTENSION_NAME`, `MainDistributionPipeline.yml`, tests, docs). The prior partial rename only updated `description.yml` and added a `zarr_init_c_api` alias; native community builds still produced and looked for `duckdb_zarr` artifacts because the Rust crate name (and thus the compiled library filename) didn't match, and the `Makefile`'s plain `EXTENSION_NAME=duckdb_zarr` assignment overrode the `EXTENSION_NAME` env var the CI distribution workflow passes in. - CI: align DuckDB to v1.5.4 across all three version sites — crate `=1.10504.0`, workflow `duckdb_version`, and `Makefile` `TARGET_DUCKDB_VERSION` (the last stamps the extension metadata `duckdb_version`; with `USE_UNSTABLE_C_API=1` the loader requires an *exact* match) — so the built extension loads in the `duckdb_sqllogictest` test runner, which had moved to 1.5.4. Fixes macOS-arm64/Windows test-load version-mismatch failures. Wrapped now-`unsafe` `FlatVector::as_mut_ptr` calls per the 1.10504.0 API. diff --git a/Cargo.lock b/Cargo.lock index e8c4227..5f5534e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3657,7 +3657,7 @@ dependencies = [ [[package]] name = "zarr" -version = "0.1.0" +version = "0.1.1" dependencies = [ "base64", "duckdb", diff --git a/Cargo.toml b/Cargo.toml index d6aa1cd..bd67f50 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zarr" -version = "0.1.0" +version = "0.1.1" edition = "2021" [lib] diff --git a/Makefile b/Makefile index e4f3f5a..07e176c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean clean_all clippy fmt fmt-check generate_fixtures lint release-check render-community-descriptor +.PHONY: clean clean_all clippy fmt fmt-check generate_fixtures lint release-check render-community-descriptor test_http test_http_debug test_http_release test_http_real PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) @@ -39,6 +39,28 @@ test: test_debug test_debug: generate_fixtures test_extension_debug test_release: generate_fixtures test_extension_release +# HTTP integration tests. Build the extension first (make debug / make release). +# pytest runs via uv, so no venv setup is needed; duckdb is pinned to the target +# DuckDB version. Both files run together as one suite: +# test_http_integration.py — loopback http.server + synthetic fixtures +# (deterministic; v2 .zmetadata + v3 consolidated_metadata) +# test_http_integration_real.py — reads a real public v2 store over the internet +# (network-marked; skips when the store is unreachable) +# test_http_real runs only the real-data file. +test_http: test_http_debug +test_http_debug: generate_fixtures + uv run --with pytest --with 'duckdb==$(TARGET_DUCKDB_VERSION:v%=%)' \ + pytest test/test_http_integration.py test/test_http_integration_real.py \ + --extension build/debug/$(EXTENSION_NAME).duckdb_extension -v +test_http_release: generate_fixtures + uv run --with pytest --with 'duckdb==$(TARGET_DUCKDB_VERSION:v%=%)' \ + pytest test/test_http_integration.py test/test_http_integration_real.py \ + --extension build/release/$(EXTENSION_NAME).duckdb_extension -v +test_http_real: + uv run --with pytest --with 'duckdb==$(TARGET_DUCKDB_VERSION:v%=%)' \ + pytest test/test_http_integration_real.py \ + --extension build/debug/$(EXTENSION_NAME).duckdb_extension -v + fmt: cargo fmt --all diff --git a/README.md b/README.md index 51863e8..da766df 100644 --- a/README.md +++ b/README.md @@ -5,30 +5,45 @@ A Rust DuckDB extension that lets you query [Zarr](https://zarr.dev/) stores with SQL — in the same spirit as [xarray-sql](https://github.com/alxmrs/xarray-sql) and [zarr-datafusion](https://lib.rs/crates/zarr-datafusion), but as a first-class DuckDB extension with no external query engine. ```sql --- Local store: path ending in .zarr is intercepted automatically +-- Query a public Zarr store straight from its URL. GPCP is a multi-group store +-- (a precip variable plus CF bounds), so pick the group with dims= — a list of +-- dimension names. +SELECT time, latitude, longitude, precip +FROM read_zarr( + 'https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/gpcp-feedstock/gpcp.zarr', + dims=['time','latitude','longitude'] +) +LIMIT 10; + +-- Inspect a store's arrays — a local path or a URL +SELECT name, role, dtype, shape +FROM read_zarr_metadata('https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/gpcp-feedstock/gpcp.zarr'); + +-- Single-group stores need no dims=; a path/URL ending in .zarr is intercepted +-- automatically. (These use this repo's fixtures — run `make generate_fixtures` first.) SELECT lat, lon, AVG(temperature) -FROM 'era5.zarr' +FROM 'test/fixtures/xarray_tutorial/float_baseline.zarr' GROUP BY lat, lon; --- HTTP/HTTPS stores also work via replacement scan -SELECT * FROM 'https://example.com/data.zarr'; +-- Select one array by its store-relative path (e.g. an OME-Zarr resolution level or nested label) +SELECT * FROM read_zarr('test/fixtures/bioimage/ome_zarr/synthetic_multichannel.ome.zarr', array_path='0'); --- Explicit table function with optional dims= for multi-group stores -SELECT * FROM read_zarr('path/to/store.zarr', dims=['time', 'lat', 'lon']); - --- Select one array by its store-relative path (including nested arrays) -SELECT * FROM read_zarr('image.ome.zarr', array_path='labels/nuclei/0'); - --- Inspect arrays in a store -SELECT name, role, dtype, shape FROM read_zarr_metadata('path/to/store.zarr'); - --- List dimension groups -SELECT * FROM read_zarr_groups('path/to/store.zarr'); +-- List a store's dimension groups +SELECT dims, shape, data_vars FROM read_zarr_groups('test/fixtures/xarray_tutorial/multi_dim_group.zarr'); ``` See [docs/design.md](docs/design.md) for the full design and [docs/ome-zarr.md](docs/ome-zarr.md) for a small bioimage example. +## Remote stores + +Read directly from a URL — HTTP/HTTPS, S3, GCS, or Azure. Object stores can't list +directories, so a remote store must carry **consolidated metadata**: a Zarr v2 +`.zmetadata` object or a Zarr v3 `consolidated_metadata` block. Most published +datasets already do (e.g. anything written by xarray with `consolidated=True`), so +they read straight from their URL as shown above. S3/GCS/Azure credentials come from +DuckDB's secrets manager (`CREATE SECRET ... TYPE S3`). + ## Status Active development. Phases 1–3 are implemented: @@ -52,10 +67,14 @@ Requires: Rust toolchain, Python 3.11+, make, git. ## Testing ```shell -make test_debug # or make test_release +make test_debug # SQLLogicTest suite over local stores (or make test_release) +make test_http # HTTP integration tests: synthetic loopback + a real public store ``` -Tests are in `test/sql/` (SQLLogicTest format). Fixtures are generated automatically by `make test_*`. To regenerate manually: +SQLLogicTest cases live in `test/sql/`. The HTTP suite (`test/test_http_integration*.py`) +reads stores over HTTP: synthetic v2 `.zmetadata` and v3 `consolidated_metadata` fixtures +via a loopback server, plus the live public GPCP store (network-gated — skipped when +unreachable). Fixtures are generated automatically by `make test_*`. To regenerate manually: ```shell make generate_fixtures diff --git a/description.yml b/description.yml index aeb7185..536a4bf 100644 --- a/description.yml +++ b/description.yml @@ -1,7 +1,7 @@ extension: name: zarr description: Query Zarr array stores (v2 and v3) with SQL — read xarray-convention datasets as tables - version: 0.1.0 + version: 0.1.1 language: Rust build: cargo requires_toolchains: "rust;python3" @@ -29,7 +29,8 @@ docs: **Functions** - `read_zarr(path)` — scan all rows from a single-group Zarr store. - - `read_zarr(path, dims=['time','lat','lon'])` — select one dim group from a multi-group store. + - `read_zarr(path, dims=['time','lat','lon'])` — select one dim group from a multi-group store + (`dims` is a list of dimension names). - `read_zarr_metadata(path)` — inspect array names, dtypes, shapes, and roles. - `read_zarr_groups(path)` — enumerate distinct dimension groups in a store. @@ -47,6 +48,7 @@ docs: - Zarr v2 and v3 - Codecs: gzip, zstd, blosc, crc32c, sharding, transpose, bytes (big/little endian) - CF conventions: `scale_factor`/`add_offset` packed integers, `_FillValue`/`missing_value` null masking - - Local filesystem, HTTP/HTTPS, S3, GCS, and Azure Blob Storage - (consolidated metadata required for remote array listing; S3/GCS/Azure credentials - are read from DuckDB's secrets manager — `CREATE SECRET ... TYPE S3` etc.) + - Local filesystem, HTTP/HTTPS, S3, GCS, and Azure Blob Storage. Remote stores must carry + consolidated metadata — a Zarr v2 `.zmetadata` object or a Zarr v3 `consolidated_metadata` + block — because object stores cannot list directories. S3/GCS/Azure credentials are read + from DuckDB's secrets manager — `CREATE SECRET ... TYPE S3` etc. diff --git a/docs/README.md b/docs/README.md index bf1266d..5a87ccc 100644 --- a/docs/README.md +++ b/docs/README.md @@ -41,15 +41,16 @@ This project is related to [xarray-sql](https://github.com/alxmrs/xarray-sql) an -- Load the extension LOAD 'zarr'; --- Read metadata from a Zarr array -SELECT * FROM read_zarr_metadata('/path/to/my/array.zarr'); +-- Read metadata from a Zarr store (a local path or a URL) +SELECT * FROM read_zarr_metadata('test/fixtures/xarray_tutorial/float_baseline.zarr'); --- Read array data as a table -SELECT * FROM read_zarr('/path/to/my/array.zarr'); +-- Read a store as a table +SELECT * FROM read_zarr('test/fixtures/xarray_tutorial/float_baseline.zarr'); --- Query specific dimensions or filter data -SELECT * FROM read_zarr('/path/to/my/array.zarr') -WHERE dimension_0 > 100 AND dimension_1 < 50; +-- Filter with plain SQL on the coordinate columns +SELECT time, lat, lon, temperature +FROM read_zarr('test/fixtures/xarray_tutorial/float_baseline.zarr') +WHERE lat > 0 AND lon < 180; ``` For a small bioimage walkthrough, see [Querying OME-Zarr](ome-zarr.md). @@ -63,19 +64,19 @@ Use metadata discovery first to see the available store-relative paths: ```sql SELECT name, dims, shape, dtype -FROM read_zarr_metadata('/path/to/image.ome.zarr'); +FROM read_zarr_metadata('test/fixtures/bioimage/ome_zarr/synthetic_multichannel.ome.zarr'); ``` Then pass `array_path=` when you want a specific image or label array: ```sql -SELECT c, AVG("0") AS mean_intensity -FROM read_zarr('/path/to/image.ome.zarr', array_path='0') +SELECT c, AVG(value) AS mean_intensity +FROM read_zarr('test/fixtures/bioimage/ome_zarr/synthetic_multichannel.ome.zarr', array_path='0') GROUP BY c; -SELECT "labels/nuclei/0" AS label, COUNT(*) AS pixels -FROM read_zarr('/path/to/image.ome.zarr', array_path='labels/nuclei/0') -WHERE "labels/nuclei/0" > 0 +SELECT value AS label, COUNT(*) AS pixels +FROM read_zarr('test/fixtures/bioimage/ome_zarr/synthetic_multichannel.ome.zarr', array_path='labels/nuclei/0') +WHERE value > 0 GROUP BY label; ``` diff --git a/docs/ome-zarr.md b/docs/ome-zarr.md index c09c409..c3b1162 100644 --- a/docs/ome-zarr.md +++ b/docs/ome-zarr.md @@ -1,36 +1,58 @@ # Querying OME-Zarr OME-Zarr images commonly contain several resolution levels and nested label -arrays. Start by listing the available arrays: +arrays. The examples below run against this repo's small synthetic fixture +(`make generate_fixtures` first). Start by listing the available arrays: ```sql SELECT name, dims, shape, dtype -FROM read_zarr_metadata('image.ome.zarr'); +FROM read_zarr_metadata('test/fixtures/bioimage/ome_zarr/synthetic_multichannel.ome.zarr'); ``` The `array_path` argument is optional. Use it when a store has multiple resolution levels, nested labels, or otherwise ambiguous array groups. The path is the same store-relative path reported by `read_zarr_metadata`. -Then select a resolution level and aggregate a region by channel: +Then select a resolution level and aggregate by channel: ```sql -SELECT c, AVG("0") AS mean_intensity -FROM read_zarr('image.ome.zarr', array_path='0') -WHERE y BETWEEN 100 AND 199 - AND x BETWEEN 200 AND 299 +SELECT c, AVG(value) AS mean_intensity +FROM read_zarr('test/fixtures/bioimage/ome_zarr/synthetic_multichannel.ome.zarr', array_path='0') GROUP BY c; ``` -Here, `0` is the conventional path for the highest-resolution level. Xarray -dimension names such as `c`, `y`, and `x` become SQL columns. Numeric and nested -array names must be double-quoted when referenced as columns. +Here, `0` is the conventional path for the highest-resolution level. When you +select a single array with `array_path`, its data is exposed as a `value` column +and its xarray dimension names (`c`, `y`, `x`) become the other columns — add a +`WHERE y BETWEEN ... AND x BETWEEN ...` clause to aggregate a sub-region. Nested label arrays use the same selector: ```sql -SELECT "labels/nuclei/0" AS label, COUNT(*) AS pixels -FROM read_zarr('image.ome.zarr', array_path='labels/nuclei/0') -WHERE "labels/nuclei/0" > 0 +SELECT value AS label, COUNT(*) AS pixels +FROM read_zarr('test/fixtures/bioimage/ome_zarr/synthetic_multichannel.ome.zarr', array_path='labels/nuclei/0') +WHERE value > 0 GROUP BY label; ``` + +## Remote OME-Zarr + +Public OME-Zarr images can be read straight from a URL. Bioimage stores usually +lack consolidated metadata, so whole-store operations (`read_zarr_metadata`, or +`read_zarr` without `array_path`) aren't available remotely — but a specific +resolution level reads by `array_path`, with `c`/`z`/`y`/`x` recovered from the +OME `multiscales.axes`: + +```sql +-- A public image from the Image Data Resource (IDR) +SELECT c, z, y, x, value AS intensity +FROM read_zarr( + 'https://uk1s3.embassy.ebi.ac.uk/idr/zarr/v0.4/idr0062A/6001240.zarr', + array_path='0' +) +LIMIT 10; +``` + +Filters and aggregates over a full pyramid level scan every chunk (there is no +sub-chunk filter pushdown yet), so keep remote exploratory queries bounded with +`LIMIT` or use a coarse resolution level. diff --git a/examples/demo.zarr/lat/c/0 b/examples/demo.zarr/lat/c/0 new file mode 100644 index 0000000..9e4e0ff Binary files /dev/null and b/examples/demo.zarr/lat/c/0 differ diff --git a/examples/demo.zarr/lat/zarr.json b/examples/demo.zarr/lat/zarr.json new file mode 100644 index 0000000..4ad7efe --- /dev/null +++ b/examples/demo.zarr/lat/zarr.json @@ -0,0 +1,45 @@ +{ + "shape": [ + 4 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 4 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "lat" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/examples/demo.zarr/lon/c/0 b/examples/demo.zarr/lon/c/0 new file mode 100644 index 0000000..d504c63 Binary files /dev/null and b/examples/demo.zarr/lon/c/0 differ diff --git a/examples/demo.zarr/lon/zarr.json b/examples/demo.zarr/lon/zarr.json new file mode 100644 index 0000000..5d05204 --- /dev/null +++ b/examples/demo.zarr/lon/zarr.json @@ -0,0 +1,45 @@ +{ + "shape": [ + 5 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 5 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "lon" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/examples/demo.zarr/temperature/c/0/0/0 b/examples/demo.zarr/temperature/c/0/0/0 new file mode 100644 index 0000000..8dc6e32 Binary files /dev/null and b/examples/demo.zarr/temperature/c/0/0/0 differ diff --git a/examples/demo.zarr/temperature/zarr.json b/examples/demo.zarr/temperature/zarr.json new file mode 100644 index 0000000..24ff901 --- /dev/null +++ b/examples/demo.zarr/temperature/zarr.json @@ -0,0 +1,51 @@ +{ + "shape": [ + 3, + 4, + 5 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 3, + 4, + 5 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "lat", + "lon" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/examples/demo.zarr/time/c/0 b/examples/demo.zarr/time/c/0 new file mode 100644 index 0000000..4355680 Binary files /dev/null and b/examples/demo.zarr/time/c/0 differ diff --git a/examples/demo.zarr/time/zarr.json b/examples/demo.zarr/time/zarr.json new file mode 100644 index 0000000..7f0caa8 --- /dev/null +++ b/examples/demo.zarr/time/zarr.json @@ -0,0 +1,43 @@ +{ + "shape": [ + 3 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 3 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": {}, + "dimension_names": [ + "time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/examples/demo.zarr/zarr.json b/examples/demo.zarr/zarr.json new file mode 100644 index 0000000..c110534 --- /dev/null +++ b/examples/demo.zarr/zarr.json @@ -0,0 +1,195 @@ +{ + "attributes": {}, + "zarr_format": 3, + "consolidated_metadata": { + "kind": "inline", + "must_understand": false, + "metadata": { + "lat": { + "shape": [ + 4 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 4 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "lat" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "lon": { + "shape": [ + 5 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 5 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "lon" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "temperature": { + "shape": [ + 3, + 4, + 5 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 3, + 4, + 5 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "lat", + "lon" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "time": { + "shape": [ + 3 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 3 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": {}, + "dimension_names": [ + "time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + } + } + }, + "node_type": "group" +} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 775729c..0d2515c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "duckdb-zarr" -version = "0.1.0" +version = "0.1.1" requires-python = ">=3.11" dependencies = [ "duckdb", @@ -21,3 +21,8 @@ dev = [ [tool.uv] package = false + +[tool.pytest.ini_options] +markers = [ + "network: reads a real public store over the internet (skipped when unreachable)", +] diff --git a/scripts/generate_fixtures.py b/scripts/generate_fixtures.py index 3edaf33..f8f7af0 100644 --- a/scripts/generate_fixtures.py +++ b/scripts/generate_fixtures.py @@ -216,6 +216,13 @@ def main() -> None: ) print(f" wrote nested labels to {dest}") + # Consolidate the OME-Zarr store so it is also readable over HTTP. Remote + # (object-store) listing needs consolidated metadata; local reads scan + # directories and ignore it. Idempotent — skipped once already consolidated. + if "consolidated_metadata" not in (dest / "zarr.json").read_text(): + zarr.consolidate_metadata(str(dest)) + print(f" consolidated {dest}") + # ── float_baseline (synthetic) ─────────────────────────────────────────── # True float32 baseline with no packing, no sentinels. # Tests: basic read_zarr, coord classification, plain numeric copy path. @@ -551,6 +558,49 @@ def main() -> None: dest, zarr_format=2, consolidated=False, encoding=v2_enc) print(f" wrote {dest}") + # ── consolidated_{v2,v3}_http ──────────────────────────────────────────── + # Remote (HTTP/object) stores cannot list directories, so the Rust reader + # enumerates arrays from consolidated metadata. These two fixtures cover both + # remote code paths in list_array_names_remote (see test/test_http_integration.py): + # * v2 → a separate `.zmetadata` object + # * v3 → a `consolidated_metadata` block embedded in `zarr.json` + # Same 8×6×12 (time, lat, lon) grid so the HTTP assertions are shared. + rng = np.random.default_rng(0) + http_data = rng.standard_normal((8, 6, 12)).astype("float32") + http_lat = np.linspace(-90.0, 90.0, 6) + http_lon = np.linspace(0.0, 360.0, 12, endpoint=False) + http_time = np.arange(8, dtype="int64") + http_da = xr.DataArray( + http_data, dims=["time", "lat", "lon"], + coords={"time": http_time, "lat": http_lat, "lon": http_lon}, + attrs={"units": "K", "long_name": "temperature"}) + http_ds = xr.Dataset({"temperature": http_da}) + + print("consolidated_v2_http (zarr v2 + .zmetadata)...") + dest = FIXTURES / "consolidated_v2_http.zarr" + if (dest / ".zmetadata").exists(): + print(f" (cached) {dest}") + else: + if dest.exists(): + _rmtree(dest) + # gzip (not blosc): blosc build fails on macOS Tahoe; matches float_baseline_v2. + v2_enc = {v: {"compressor": {"id": "gzip", "level": 1}} + for v in ("temperature", "lat", "lon", "time")} + http_ds.to_zarr(dest, zarr_format=2, consolidated=True, encoding=v2_enc) + print(f" wrote {dest}") + + print("consolidated_v3_http (zarr v3 + consolidated_metadata)...") + dest = FIXTURES / "consolidated_v3_http.zarr" + root_meta = dest / "zarr.json" + if root_meta.exists() and "consolidated_metadata" in root_meta.read_text(): + print(f" (cached) {dest}") + else: + if dest.exists(): + _rmtree(dest) + http_ds.to_zarr(dest, zarr_format=3, consolidated=False) + zarr.consolidate_metadata(str(dest)) + print(f" wrote {dest}") + print("\nAll fixtures written.") diff --git a/src/read_zarr.rs b/src/read_zarr.rs index 1a15ddf..f516621 100644 --- a/src/read_zarr.rs +++ b/src/read_zarr.rs @@ -2,8 +2,8 @@ use std::collections::HashMap; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Mutex; -use duckdb::core::{DataChunkHandle, LogicalTypeId}; -use duckdb::vtab::{BindInfo, InitInfo, TableFunctionInfo, VTab}; +use duckdb::core::{DataChunkHandle, LogicalTypeHandle, LogicalTypeId}; +use duckdb::vtab::{BindInfo, InitInfo, TableFunctionInfo, VTab, Value}; use crate::zarr_reader::meta::{ build_column_defs, build_work_units, dim_group_for_array, dimension_names, extract_file_system, @@ -74,10 +74,11 @@ impl VTab for ReadZarrVTab { let path_val = bind.get_parameter(0); let store_path = path_val.to_string(); - // Optional dims= named parameter: JSON array string e.g. '["time","lat","lon"]' + // Optional dims= named parameter: a list of dimension names, + // e.g. read_zarr(store, dims=['time','lat','lon']). let requested_dims: Option> = bind .get_named_parameter("dims") - .map(|v| parse_dims_param(&v.to_string())) + .map(parse_dims_param) .transpose()?; let array_path = bind .get_named_parameter("array_path") @@ -92,14 +93,19 @@ impl VTab for ReadZarrVTab { let fs = unsafe { extract_file_system(bind) }; let store = open_store(&store_path, Some(fs))?; - let array_names = crate::zarr_reader::meta::list_array_names(&store_path, &store)?; - - if array_names.is_empty() { - return Err(format!("no Zarr arrays found in '{store_path}'").into()); - } - if let Some(requested) = requested_array { - let array_name = select_array_name(&array_names, &requested)?; + // Only this one array is needed. Listing requires consolidated metadata + // on remote stores, so here it is best-effort: when available it lets + // coordinate arrays be resolved; when not (e.g. an OME-Zarr store served + // over HTTP without consolidation) the array still reads by array_path, + // with dimensions synthesized as integer indices. + let array_names = + crate::zarr_reader::meta::list_array_names(&store_path, &store).unwrap_or_default(); + let array_name = if array_names.is_empty() { + requested.trim().trim_matches('/').to_string() + } else { + select_array_name(&array_names, &requested)? + }; let group = dim_group_for_array(&store, &array_names, &array_name)?; if let Some(dims) = requested_dims { if group.dims != dims { @@ -113,6 +119,11 @@ impl VTab for ReadZarrVTab { return finish_bind(bind, store, &group); } + let array_names = crate::zarr_reader::meta::list_array_names(&store_path, &store)?; + if array_names.is_empty() { + return Err(format!("no Zarr arrays found in '{store_path}'").into()); + } + let (dim_groups, _coord_names) = infer_dim_groups(&store, &array_names)?; if dim_groups.is_empty() { @@ -244,7 +255,10 @@ impl VTab for ReadZarrVTab { fn named_parameters() -> Option> { Some(vec![ - ("dims".to_string(), LogicalTypeId::Varchar.into()), + ( + "dims".to_string(), + LogicalTypeHandle::list(&LogicalTypeId::Varchar.into()), + ), ("array".to_string(), LogicalTypeId::Varchar.into()), ("array_path".to_string(), LogicalTypeId::Varchar.into()), ]) @@ -255,22 +269,22 @@ impl VTab for ReadZarrVTab { // Helpers // --------------------------------------------------------------------------- -/// Parse a `dims` named-parameter value into an ordered list of dimension names. +/// Extract the ordered dimension names from a `dims` named-parameter value. /// -/// Accepts either a JSON array (`'["time","lat","lon"]'`) or a plain -/// comma-separated string (`'time,lat,lon'`). -fn parse_dims_param(s: &str) -> Result, Box> { - let trimmed = s.trim(); - if trimmed.starts_with('[') { - let arr: Vec = serde_json::from_str(trimmed)?; - Ok(arr) - } else { - Ok(trimmed - .split(',') - .map(|d| d.trim().to_string()) - .filter(|d| !d.is_empty()) - .collect()) - } +/// `dims` is a `LIST(VARCHAR)`, e.g. `read_zarr(store, dims=['time','lat','lon'])`. +/// A single data-variable column selected by `array_path` has a numeric level +/// name (`0`) or a nested store-relative path (`labels/nuclei/0`), both of which +/// otherwise need SQL double-quoting. Surface it as `value` instead. Ordinary +/// variable names (`temperature`, `precip`) are left unchanged. +fn needs_value_alias(name: &str) -> bool { + name.contains('/') || name.parse::().is_ok() +} + +fn parse_dims_param(value: Value) -> Result, Box> { + let items = value + .to_list() + .ok_or("dims must be a list of dimension names, e.g. dims=['time','lat','lon']")?; + Ok(items.iter().map(|item| item.to_string()).collect()) } fn finish_bind( @@ -291,10 +305,19 @@ fn finish_bind( let columns = build_column_defs(&store, group, &coord_arrays)?; - // Register output columns with DuckDB. + // Register output columns with DuckDB. When a single array is selected by + // array_path, its name is a numeric level (`0`) or a nested store-relative + // path (`labels/nuclei/0`); surface that value column as `value` so callers + // don't have to double-quote it. Decoding still keys off `col.name`. + let single_data_var = columns.iter().filter(|c| !c.is_coord).count() == 1; for col in &columns { let duckdb_type = col.on_disk_dtype.to_duckdb_type(&col.encoding); - bind.add_result_column(&col.name, duckdb_type); + let display_name = if single_data_var && !col.is_coord && needs_value_alias(&col.name) { + "value" + } else { + col.name.as_str() + }; + bind.add_result_column(display_name, duckdb_type); } // Pre-open data variable arrays once at bind time. diff --git a/src/zarr_reader/meta.rs b/src/zarr_reader/meta.rs index a146d38..06f9a2c 100644 --- a/src/zarr_reader/meta.rs +++ b/src/zarr_reader/meta.rs @@ -9,7 +9,7 @@ use duckdb::ffi::{ }; use zarrs::array::Array; use zarrs::filesystem::FilesystemStore; -use zarrs::storage::ReadableStorageTraits; +use zarrs::storage::{ReadableStorageTraits, StoreKey}; use super::duckdb_store::DuckDbStore; use super::types::{ @@ -84,7 +84,9 @@ pub fn open_store( /// List the store-relative paths of all arrays in the Zarr hierarchy. /// /// - Local paths: recursively scans directories containing `zarr.json` (v3) or `.zarray` (v2). -/// - Remote paths (HTTP/HTTPS/S3/GCS/Azure): reads consolidated metadata from the root zarr.json. +/// - Remote paths (HTTP/HTTPS/S3/GCS/Azure): enumerates arrays from consolidated metadata, +/// since object stores cannot list directories — a v3 `consolidated_metadata` block in +/// `zarr.json`, or a v2 `.zmetadata` object. pub fn list_array_names( store_path: &str, store: &ZarrStore, @@ -156,25 +158,62 @@ fn list_array_names_remote(store: &ZarrStore) -> Result, Box = consolidated - .metadata - .iter() - .filter_map(|(path, meta)| { - let name = path.trim_start_matches('/'); - if matches!(meta, NodeMetadata::Array(_)) { - Some(name.to_string()) - } else { - None - } - }) + if let Some(consolidated) = group.consolidated_metadata() { + let mut names: Vec = consolidated + .metadata + .iter() + .filter_map(|(path, meta)| { + let name = path.trim_start_matches('/'); + if matches!(meta, NodeMetadata::Array(_)) { + Some(name.to_string()) + } else { + None + } + }) + .collect(); + names.sort(); + return Ok(names); + } + + // Zarr v2: consolidated metadata lives in a separate `.zmetadata` object. + // HTTP/object stores cannot list directories, so `.zmetadata` is the only + // way to enumerate a v2 store's arrays remotely. + if let Some(names) = list_array_names_zmetadata(store)? { + return Ok(names); + } + + Err( + "remote Zarr store has no consolidated metadata: found neither a v3 \ + `consolidated_metadata` block in `zarr.json` nor a v2 `.zmetadata` object. \ + Re-write the store with consolidated metadata \ + (xarray: `ds.to_zarr(store, consolidated=True)`)." + .into(), + ) +} + +/// Enumerate array names from a Zarr v2 consolidated-metadata (`.zmetadata`) object. +/// +/// Returns `Ok(None)` when the store has no `.zmetadata`, so the caller can emit a +/// single clear error. Arrays are the entries whose key ends in `/.zarray`. +fn list_array_names_zmetadata( + store: &ZarrStore, +) -> Result>, Box> { + let Some(bytes) = store.get(&StoreKey::new(".zmetadata")?)? else { + return Ok(None); + }; + let doc: serde_json::Value = serde_json::from_slice(&bytes)?; + let metadata = doc + .get("metadata") + .and_then(serde_json::Value::as_object) + .ok_or("`.zmetadata` is missing its `metadata` object")?; + let mut names: Vec = metadata + .keys() + .filter_map(|key| key.strip_suffix("/.zarray").map(str::to_string)) .collect(); names.sort(); - Ok(names) + Ok(Some(names)) } /// Open one array by name from the store. @@ -213,8 +252,16 @@ pub fn dim_group_for_array( array_name: &str, ) -> Result> { let arr = open_array(store, array_name)?; - let dims = dimension_names(&arr, array_name)?; let shape = arr.shape().to_vec(); + // Named dimensions come from xarray metadata; fall back to OME-Zarr + // `multiscales.axes` (matched by rank) for stores that carry neither + // `dimension_names` nor `_ARRAY_DIMENSIONS` on the array itself. + let dims = match dimension_names(&arr, array_name) { + Ok(dims) => dims, + Err(err) => ome_axis_names(store, array_name) + .filter(|axes| axes.len() == shape.len()) + .ok_or(err)?, + }; let first_chunk = vec![0u64; shape.len()]; let chunk_shape = arr .chunk_shape(&first_chunk)? @@ -235,6 +282,52 @@ pub fn dim_group_for_array( }) } +/// OME-Zarr fallback for dimension names. +/// +/// Real OME-Zarr images record axis names in the parent group's +/// `multiscales.axes` rather than in per-array `dimension_names` / +/// `_ARRAY_DIMENSIONS`. This resolves them by matching the array to its +/// `multiscales.datasets[].path` entry, recovering `c`/`z`/`y`/`x` for +/// `array_path` reads of stores like those in the IDR. +fn ome_axis_names(store: &ZarrStore, array_name: &str) -> Option> { + use zarrs::group::Group; + + let (group_path, dataset) = match array_name.rsplit_once('/') { + Some((group, ds)) => (format!("/{group}"), ds), + None => ("/".to_string(), array_name), + }; + let group = Group::open(store.clone(), &group_path).ok()?; + let multiscales = group.attributes().get("multiscales")?.as_array()?; + for ms in multiscales { + let matches_dataset = ms + .get("datasets") + .and_then(serde_json::Value::as_array) + .is_some_and(|datasets| { + datasets + .iter() + .any(|d| d.get("path").and_then(serde_json::Value::as_str) == Some(dataset)) + }); + if !matches_dataset { + continue; + } + let Some(axes) = ms.get("axes").and_then(serde_json::Value::as_array) else { + continue; + }; + let names: Vec = axes + .iter() + .filter_map(|axis| { + axis.get("name") + .and_then(serde_json::Value::as_str) + .map(String::from) + }) + .collect(); + if !names.is_empty() { + return Some(names); + } + } + None +} + fn parent_path(name: &str) -> &str { name.rsplit_once('/') .map(|(parent, _)| parent) diff --git a/test/sql/bioimage_ome_zarr.test b/test/sql/bioimage_ome_zarr.test index 74e7c9e..62875e2 100644 --- a/test/sql/bioimage_ome_zarr.test +++ b/test/sql/bioimage_ome_zarr.test @@ -21,7 +21,7 @@ FROM read_zarr( # Named xarray dimensions make channel and spatial queries self-describing. query II rowsort -SELECT c, SUM("0") +SELECT c, SUM(value) FROM read_zarr( 'test/fixtures/bioimage/ome_zarr/synthetic_multichannel.ome.zarr', array_path = '0' @@ -33,7 +33,7 @@ GROUP BY c; # Query a spatial region from the DNA channel using image-axis names. query II -SELECT COUNT(*), SUM("0") +SELECT COUNT(*), SUM(value) FROM read_zarr( 'test/fixtures/bioimage/ome_zarr/synthetic_multichannel.ome.zarr', array_path = '0' @@ -60,7 +60,7 @@ labels/nuclei/0 data # array_path= selects a nested image-label array while retaining named image axes. query III -SELECT COUNT(*), COUNT(*) FILTER (WHERE "labels/nuclei/0" > 0), SUM("labels/nuclei/0") +SELECT COUNT(*), COUNT(*) FILTER (WHERE value > 0), SUM(value) FROM read_zarr( 'test/fixtures/bioimage/ome_zarr/synthetic_multichannel.ome.zarr', array_path = 'labels/nuclei/0' diff --git a/test/sql/read_zarr.test b/test/sql/read_zarr.test index b01cf10..c79f072 100644 --- a/test/sql/read_zarr.test +++ b/test/sql/read_zarr.test @@ -215,25 +215,19 @@ SELECT COUNT(*) FROM read_zarr_groups('test/fixtures/xarray_tutorial/multi_dim_g # dims= selects the surface group (time, lat, lon): 4*5*8 = 160 rows. query I -SELECT COUNT(*) FROM read_zarr('test/fixtures/xarray_tutorial/multi_dim_group.zarr', dims='["time","lat","lon"]'); +SELECT COUNT(*) FROM read_zarr('test/fixtures/xarray_tutorial/multi_dim_group.zarr', dims=['time','lat','lon']); ---- 160 # dims= selects the atmosphere group (time, level, lat, lon): 4*3*5*8 = 480 rows. query I -SELECT COUNT(*) FROM read_zarr('test/fixtures/xarray_tutorial/multi_dim_group.zarr', dims='["time","level","lat","lon"]'); +SELECT COUNT(*) FROM read_zarr('test/fixtures/xarray_tutorial/multi_dim_group.zarr', dims=['time','level','lat','lon']); ---- 480 -# dims= also accepts comma-separated form. -query I -SELECT COUNT(*) FROM read_zarr('test/fixtures/xarray_tutorial/multi_dim_group.zarr', dims='time,lat,lon'); ----- -160 - # dims= with no match → error. statement error -SELECT * FROM read_zarr('test/fixtures/xarray_tutorial/multi_dim_group.zarr', dims='["x","y"]') LIMIT 1; +SELECT * FROM read_zarr('test/fixtures/xarray_tutorial/multi_dim_group.zarr', dims=['x','y']) LIMIT 1; ---- no dimension group matches diff --git a/test/sql/realistic_queries.test b/test/sql/realistic_queries.test index 90e5a56..54566fa 100644 --- a/test/sql/realistic_queries.test +++ b/test/sql/realistic_queries.test @@ -260,9 +260,9 @@ CROSS JOIN # Inner join: pair surface (t2m) with atmosphere (temp) at 1000 hPa. query I SELECT COUNT(*) FROM - read_zarr('test/fixtures/xarray_tutorial/multi_dim_group.zarr', dims='["time","lat","lon"]') s + read_zarr('test/fixtures/xarray_tutorial/multi_dim_group.zarr', dims=['time','lat','lon']) s JOIN - read_zarr('test/fixtures/xarray_tutorial/multi_dim_group.zarr', dims='["time","level","lat","lon"]') a + read_zarr('test/fixtures/xarray_tutorial/multi_dim_group.zarr', dims=['time','level','lat','lon']) a ON s.time = a.time AND s.lat = a.lat AND s.lon = a.lon WHERE a.level = 1000.0; ---- @@ -272,8 +272,8 @@ WHERE a.level = 1000.0; query I SELECT COUNT(*) FROM ( SELECT s.t2m, a.temp - FROM read_zarr('test/fixtures/xarray_tutorial/multi_dim_group.zarr', dims='["time","lat","lon"]') s - JOIN read_zarr('test/fixtures/xarray_tutorial/multi_dim_group.zarr', dims='["time","level","lat","lon"]') a + FROM read_zarr('test/fixtures/xarray_tutorial/multi_dim_group.zarr', dims=['time','lat','lon']) s + JOIN read_zarr('test/fixtures/xarray_tutorial/multi_dim_group.zarr', dims=['time','level','lat','lon']) a ON s.time = a.time AND s.lat = a.lat AND s.lon = a.lon WHERE a.level = 1000.0 ); diff --git a/test/test_http_integration.py b/test/test_http_integration.py new file mode 100644 index 0000000..9667f8a --- /dev/null +++ b/test/test_http_integration.py @@ -0,0 +1,149 @@ +""" +HTTP integration tests for duckdb-zarr's remote reader. + +Remote (HTTP/object) stores cannot list directories, so ``read_zarr`` enumerates +arrays from consolidated metadata. Two mechanisms are exercised here: + + * Zarr v2 — a separate ``.zmetadata`` object (consolidated_v2_http.zarr) + * Zarr v3 — a ``consolidated_metadata`` block (consolidated_v3_http.zarr) + +Both fixtures carry the same 8×6×12 (time, lat, lon) grid, so the assertions are +shared across the two remote paths. The stores are served over the loopback +``http.server`` started by the ``http_server_url`` fixture in conftest.py. + +Run with: + make test_http_debug + # or directly: + pytest test/test_http_integration.py \ + --extension build/debug/zarr.duckdb_extension +""" +import pathlib + +import duckdb +import pytest + +ROOT = pathlib.Path(__file__).parent.parent + +# Store-relative path (under the repo root the http server is rooted at) → id. +FIXTURES = { + "v2_zmetadata": "test/fixtures/xarray_tutorial/consolidated_v2_http.zarr", + "v3_consolidated": "test/fixtures/xarray_tutorial/consolidated_v3_http.zarr", +} + + +@pytest.fixture(scope="module") +def con(extension_path): + """DuckDB connection with the zarr extension loaded, shared across the module.""" + c = duckdb.connect(config={"allow_unsigned_extensions": True}) + c.execute(f"LOAD '{extension_path}'") + return c + + +@pytest.fixture(params=sorted(FIXTURES), ids=sorted(FIXTURES)) +def store(request, http_server_url): + """Full HTTP URL for each consolidated fixture (parametrized over v2 and v3).""" + rel = FIXTURES[request.param] + if not (ROOT / rel).exists(): + pytest.fail( + f"HTTP fixture not found: {rel}\n" + "Run: python scripts/generate_fixtures.py" + ) + return f"{http_server_url}/{rel}" + + +# ── read_zarr over HTTP ───────────────────────────────────────────────────── + +def test_row_count(con, store): + assert con.execute(f"SELECT COUNT(*) FROM read_zarr('{store}')").fetchone()[0] == 576 + + +def test_distinct_coords(con, store): + row = con.execute( + f"SELECT COUNT(DISTINCT lat), COUNT(DISTINCT lon) FROM read_zarr('{store}')" + ).fetchone() + assert row == (6, 12) + + +def test_lat_range(con, store): + row = con.execute(f"SELECT MIN(lat), MAX(lat) FROM read_zarr('{store}')").fetchone() + assert row == (-90.0, 90.0) + + +def test_temperature_non_null(con, store): + got = con.execute( + f"SELECT COUNT(*) FROM read_zarr('{store}') WHERE temperature IS NOT NULL" + ).fetchone()[0] + assert got == 576 + + +def test_projection_pushdown(con, store): + """Projecting a single column must still return correct values over HTTP.""" + got = con.execute( + f"SELECT COUNT(DISTINCT lat) FROM (SELECT lat FROM read_zarr('{store}'))" + ).fetchone()[0] + assert got == 6 + + +# ── replacement scan (bare path ending in .zarr) ──────────────────────────── + +def test_replacement_scan(con, store): + assert con.execute(f"SELECT COUNT(*) FROM '{store}'").fetchone()[0] == 576 + + +# ── read_zarr_metadata over HTTP ──────────────────────────────────────────── + +def test_metadata_roles(con, store): + rows = con.execute( + f"SELECT name, role FROM read_zarr_metadata('{store}') ORDER BY name" + ).fetchall() + assert rows == [ + ("lat", "coord"), + ("lon", "coord"), + ("temperature", "data"), + ("time", "coord"), + ] + + +# ── OME-Zarr over HTTP ────────────────────────────────────────────────────── +# The same synthetic OME-NGFF bioimage the SQLLogicTest suite reads locally, now +# read remotely. It carries consolidated metadata (see scripts/generate_fixtures.py), +# so its arrays — including the nested label image — are discoverable over HTTP. + +OME_FIXTURE = "test/fixtures/bioimage/ome_zarr/synthetic_multichannel.ome.zarr" + + +@pytest.fixture +def ome_store(http_server_url): + if not (ROOT / OME_FIXTURE).exists(): + pytest.fail( + f"OME fixture not found: {OME_FIXTURE}\n" + "Run: python scripts/generate_fixtures.py" + ) + return f"{http_server_url}/{OME_FIXTURE}" + + +def test_ome_metadata(con, ome_store): + """The image level and the nested label array are both enumerated over HTTP.""" + rows = con.execute( + f"SELECT name, role FROM read_zarr_metadata('{ome_store}') ORDER BY name" + ).fetchall() + assert rows == [("0", "data"), ("labels/nuclei/0", "data")] + + +def test_ome_array_path_channels(con, ome_store): + """array_path selects a resolution level; per-channel means match the fixture.""" + rows = con.execute( + f"SELECT c, AVG(value) FROM read_zarr('{ome_store}', array_path='0') " + "GROUP BY c ORDER BY c" + ).fetchall() + assert rows == [(0, 6.5), (1, 106.5)] + + +def test_ome_nested_label(con, ome_store): + """The nested label image is readable by its store-relative array_path.""" + rows = con.execute( + "SELECT value AS label, COUNT(*) " + f"FROM read_zarr('{ome_store}', array_path='labels/nuclei/0') " + "GROUP BY label ORDER BY label" + ).fetchall() + assert rows == [(0, 6), (1, 3), (2, 3)] diff --git a/test/test_http_integration_real.py b/test/test_http_integration_real.py new file mode 100644 index 0000000..95d5eb6 --- /dev/null +++ b/test/test_http_integration_real.py @@ -0,0 +1,117 @@ +""" +Real-data HTTP integration tests. + +These read an *actual public* Zarr **v2** store over HTTPS — the exact scenario +the `.zmetadata` remote fix targets — so they validate the work end-to-end +against real-world data the synthetic loopback fixtures can't capture: a real +consolidated `.zmetadata`, blosc compression, CF bounds variables, and large +chunked arrays. + +They hit the network, so they are marked `network` and skip automatically when +the store is unreachable (offline / CI without egress). + +Run with: + make test_http_real + # or directly: + pytest test/test_http_integration_real.py \ + --extension build/debug/zarr.duckdb_extension +""" +import urllib.request + +import duckdb +import pytest + +# Pangeo GPCP daily precipitation: Zarr v2 + consolidated `.zmetadata`, +# blosc-compressed, with CF bounds variables (lat_bounds/lon_bounds/time_bounds) +# that make it a multi-group store — a realistic public target for the reader. +GPCP = "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/gpcp-feedstock/gpcp.zarr" +PRECIP_DIMS = "['time','latitude','longitude']" + +pytestmark = pytest.mark.network + + +@pytest.fixture(scope="module") +def con(extension_path): + """DuckDB connection with the zarr extension loaded.""" + c = duckdb.connect(config={"allow_unsigned_extensions": True}) + c.execute(f"LOAD '{extension_path}'") + return c + + +@pytest.fixture(scope="module") +def gpcp(): + """Skip the whole module when the public store is unreachable.""" + try: + with urllib.request.urlopen(f"{GPCP}/.zmetadata", timeout=20) as resp: + if resp.status != 200: + pytest.skip(f"GPCP store returned HTTP {resp.status}") + except Exception as exc: # DNS/timeout/connection/HTTP error → offline + pytest.skip(f"GPCP store unreachable: {exc}") + return GPCP + + +def test_metadata_enumerates_v2_arrays(con, gpcp): + """read_zarr_metadata enumerates arrays from the remote v2 `.zmetadata`. + + This is the crux of the fix: without `.zmetadata` support the reader can't + list any arrays in a remote v2 store and the query errors at bind. + """ + roles = dict( + con.execute(f"SELECT name, role FROM read_zarr_metadata('{gpcp}')").fetchall() + ) + assert roles.get("precip") == "data" + assert roles.get("latitude") == "coord" + assert roles.get("longitude") == "coord" + assert roles.get("time") == "coord" + + +def test_coordinate_pivot(con, gpcp): + """The coordinate pivot works over HTTP (projection skips the precip chunk).""" + row = con.execute( + "SELECT MIN(latitude), COUNT(*) FROM (" + f" SELECT latitude FROM read_zarr('{gpcp}', dims={PRECIP_DIMS}) LIMIT 500" + ")" + ).fetchone() + assert row[0] == -90.0 # GPCP latitude starts at the south pole + assert row[1] == 500 + + +def test_precip_decode(con, gpcp): + """End-to-end decode of a real blosc-compressed v2 chunk fetched over HTTP.""" + total, non_null = con.execute( + "SELECT COUNT(*), COUNT(precip) FROM (" + f" SELECT precip FROM read_zarr('{gpcp}', dims={PRECIP_DIMS}) LIMIT 200" + ")" + ).fetchone() + assert total == 200 + assert non_null >= 1 # at least some real precip values decoded (rest may be fill/NULL) + + +# ── Real public OME-Zarr (IDR) via array_path ─────────────────────────────── +# A real bioimage store with NO consolidated metadata and NO per-array dimension +# metadata — exactly the case array_path (open the array without listing the +# store) plus OME multiscales.axes dimension names unlock. +IDR_OME = "https://uk1s3.embassy.ebi.ac.uk/idr/zarr/v0.4/idr0062A/6001240.zarr" + + +@pytest.fixture(scope="module") +def idr(): + """Skip when the IDR OME-Zarr store is unreachable.""" + try: + with urllib.request.urlopen(f"{IDR_OME}/0/.zarray", timeout=20) as resp: + if resp.status != 200: + pytest.skip(f"IDR store returned HTTP {resp.status}") + except Exception as exc: + pytest.skip(f"IDR store unreachable: {exc}") + return IDR_OME + + +def test_idr_ome_zarr_array_path(con, idr): + """A real OME-Zarr image with no consolidated metadata reads by array_path; + the c/z/y/x dimensions are recovered from OME multiscales.axes.""" + rows = con.execute( + f"SELECT c, z, y, x, value FROM read_zarr('{idr}', array_path='0') LIMIT 100" + ).fetchall() + assert len(rows) == 100 + assert all(r[0] == 0 and r[1] == 0 for r in rows) # first chunk: channel 0, z 0 + assert all(r[4] is not None for r in rows) # uint16 intensities decoded diff --git a/uv.lock b/uv.lock index e56511c..2d6b32b 100644 --- a/uv.lock +++ b/uv.lock @@ -170,7 +170,7 @@ wheels = [ [[package]] name = "duckdb-zarr" -version = "0.1.0" +version = "0.1.1" source = { virtual = "." } dependencies = [ { name = "duckdb" },