Merge branch 'zarr-developers:main' into main

Author: d-v-b

.github/workflows/codspeed.yml

@@ -33,6 +33,8 @@ jobs:
         version: '1.16.5'
     - name: Run the benchmarks
       uses: CodSpeedHQ/action@f99becdce5e5d51fd556489ebef684f4ecfd6286 # v4.18.5
+      env:
+        ZARR_BENCHMARK_CLEAR_CACHE: '1'
       with:
           mode: walltime
           run: hatch run test.py3.12-minimal:pytest tests/benchmarks --codspeed

.github/workflows/test.yml

@@ -178,6 +178,8 @@ jobs:
     - name: Install Hatch
       run: python -m pip install hatch==1.16.5
     - name: Run Benchmarks
+      env:
+        ZARR_BENCHMARK_CLEAR_CACHE: '1'
       run: |
         hatch env run --env "test.py3.13-minimal" run-benchmark
 

changes/4172.misc.md

@@ -0,0 +1,7 @@
+Improved `CoordinateIndexer` construction for large, sorted, in-bounds, one-dimensional integer
+coordinate selections over regular chunk grids (e.g. `arr.get_coordinate_selection(sorted_idx)`,
+`arr.vindex[sorted_idx]`, and the gather behind sparse/CSR row selections). When boundary searching
+is estimated to be cheaper than processing every coordinate, per-chunk projections are now built
+with `searchsorted`, making index construction ~15x faster for large gathers. Sparse sorted
+selections spanning many chunks relative to their coordinate count, as well as unsorted, negative,
+multi-dimensional, and irregular-grid selections, continue to use the existing implementation.

changes/4179.bugfix.md

@@ -0,0 +1 @@
+Fixed the opt-in `FusedCodecPipeline` for sharded arrays whose inner or index codec chain contains a codec implementing only the async codec interface (no `SupportsSyncCodec`). Such arrays previously raised `TypeError: All codecs must implement SupportsSyncCodec` on both read and write; the pipeline now declines its synchronous fast path for them and falls back to the async path, matching the behavior of the default `BatchedCodecPipeline`. Fully sync-capable codec chains keep the fast path unchanged.

changes/4183.bugfix.md

@@ -0,0 +1,3 @@
+Fixed `TypeError: unhashable type: 'writeable void-scalar'` when writing to sharded arrays whose fill value is a `np.void` scalar, e.g. arrays with a structured dtype.
+
+`ArraySpec` equality and hashing now compare the fill value by its byte representation rather than numeric equality. As a result, two specs with a `NaN` (or `NaT`) fill value now compare equal, while fill values of `-0.0` and `0.0` now compare unequal. This also restores the sharding codec's per-chunk spec cache, which had been disabled because of this bug.

changes/4199.bugfix.md

@@ -0,0 +1 @@
+The end-to-end benchmarks no longer invoke `sudo` to drop the OS page cache during a regular `pytest` run. Cache clearing is now opt-in via the `ZARR_BENCHMARK_CLEAR_CACHE` environment variable, which the benchmark CI jobs set.

src/zarr/abc/codec.py

@@ -82,6 +82,20 @@ def _decode_sync(self, chunk_data: CO, chunk_spec: ArraySpec) -> CI: ...
     def _encode_sync(self, chunk_data: CI, chunk_spec: ArraySpec) -> CO | None: ...
 
 
+def _codec_supports_sync(codec: object) -> bool:
+    """Whether `codec` can actually run on a synchronous (no event loop) path.
+
+    Structural membership in `SupportsSyncCodec` is necessary but not always
+    sufficient: a codec can provide `_decode_sync`/`_encode_sync` whose ability
+    to run depends on runtime configuration the type system cannot see.
+    `ShardingCodec` is the canonical case — its sync methods delegate to its
+    configured inner and index codec chains, so they only work when every codec
+    in those chains is itself sync-capable. Such codecs opt out dynamically via
+    a `_sync_capable` attribute/property (absent means capable).
+    """
+    return isinstance(codec, SupportsSyncCodec) and getattr(codec, "_sync_capable", True)
+
+
 class BaseCodec[CI: CodecInput, CO: CodecOutput](Metadata):
     """Generic base class for codecs.
 

src/zarr/codecs/sharding.py

@@ -14,7 +14,7 @@
     ArrayBytesCodecPartialEncodeMixin,
     Codec,
     CodecPipeline,
-    SupportsSyncCodec,
+    _codec_supports_sync,
 )
 from zarr.abc.store import (
     ByteGetter,
@@ -411,11 +411,7 @@ def __init__(
         object.__setattr__(self, "subchunk_write_order", subchunk_write_order)
 
         # Use instance-local lru_cache to avoid memory leaks
-
-        # numpy void scalars are not hashable, which means an array spec with a fill value that is
-        # a numpy void scalar will break the lru_cache. This is commented for now but should be
-        # fixed. See https://github.com/zarr-developers/zarr-python/issues/3054
-        # object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec))
+        object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec))
         object.__setattr__(self, "_get_index_chunk_spec", lru_cache()(self._get_index_chunk_spec))
         object.__setattr__(self, "_get_chunks_per_shard", lru_cache()(self._get_chunks_per_shard))
         object.__setattr__(self, "_shard_index_size", lru_cache()(self._shard_index_size))
@@ -441,7 +437,7 @@ def __setstate__(self, state: dict[str, Any]) -> None:
         object.__setattr__(self, "subchunk_write_order", state["subchunk_write_order"])
 
         # Use instance-local lru_cache to avoid memory leaks
-        # object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec))
+        object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec))
         object.__setattr__(self, "_get_index_chunk_spec", lru_cache()(self._get_index_chunk_spec))
         object.__setattr__(self, "_get_chunks_per_shard", lru_cache()(self._get_chunks_per_shard))
         object.__setattr__(self, "_shard_index_size", lru_cache()(self._shard_index_size))
@@ -1406,8 +1402,30 @@ def _is_complete_shard_write(
             is_complete_chunk for *_, is_complete_chunk in indexed_chunks
         )
 
+    @property
+    def _sync_capable(self) -> bool:
+        """Dynamic opt-out consulted by `_codec_supports_sync` / `ChunkTransform`.
+
+        This codec structurally satisfies `SupportsSyncCodec`, but every sync
+        method (`_decode_sync`, `_encode_sync`, `_decode_partial_sync`,
+        `_encode_partial_sync`) delegates to the inner and index codec chains
+        through `ChunkTransform`, so it can only run synchronously when every
+        codec in BOTH chains is itself sync-capable. Reporting False here makes
+        `ChunkTransform` construction raise, which in turn makes
+        `FusedCodecPipeline.evolve_from_array_spec` set `sync_transform=None` —
+        the whole pipeline then declines the sync fast path and routes through
+        the async paths (partial shard decode / async fallback write), exactly
+        as it does for an async-only TOP-level codec or a non-sync store.
+        """
+        return self._inner_codecs_sync_capable() and self._index_codecs_sync_capable()
+
+    def _inner_codecs_sync_capable(self) -> bool:
+        # _codec_supports_sync (not bare isinstance) so a nested sharding codec
+        # with an async-only inner chain propagates its opt-out outward.
+        return all(_codec_supports_sync(c) for c in self.codecs)
+
     def _index_codecs_sync_capable(self) -> bool:
-        return all(isinstance(c, SupportsSyncCodec) for c in self.index_codecs)
+        return all(_codec_supports_sync(c) for c in self.index_codecs)
 
     async def _decode_shard_index(
         self, index_bytes: Buffer, chunks_per_shard: tuple[int, ...]

src/zarr/core/array_spec.py

@@ -3,6 +3,8 @@
 from dataclasses import dataclass, fields
 from typing import TYPE_CHECKING, Any, Literal, Self, TypedDict, cast
 
+import numpy as np
+
 from zarr.core.common import (
     MemoryOrder,
     parse_bool,
@@ -132,7 +134,7 @@ def parse_array_config(data: ArrayConfigLike | None) -> ArrayConfig:
         return ArrayConfig.from_dict(data)
 
 
-@dataclass(frozen=True)
+@dataclass(frozen=True, eq=False)
 class ArraySpec:
     shape: tuple[int, ...]
     dtype: ZDType[TBaseDType, TBaseScalar]
@@ -157,6 +159,24 @@ def __init__(
         object.__setattr__(self, "config", config)
         object.__setattr__(self, "prototype", prototype)
 
+    def _key(self) -> tuple[object, ...]:
+        """Returns the tuple used for equality/hash identity."""
+        fill_value = self.fill_value
+        if isinstance(fill_value, np.generic):
+            # fill_values should be byte-identical, otherwise they correspond to different values in memory / on disk.
+            # Importantly, this ensures np.nan == np.nan, NaT == NaT, and -0.0 != 0.0.
+            # It also fixes np.void fill_values being unhashable (#3054).
+            fill_value = fill_value.tobytes()
+        return (self.shape, self.dtype, fill_value, self.config, self.prototype)
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, ArraySpec):
+            return NotImplemented
+        return self._key() == other._key()
+
+    def __hash__(self) -> int:
+        return hash(self._key())
+
     @property
     def ndim(self) -> int:
         return len(self.shape)

src/zarr/core/chunk_utils.py

@@ -3,7 +3,7 @@
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any, cast
 
-from zarr.abc.codec import GetResult, SupportsSyncCodec
+from zarr.abc.codec import GetResult, SupportsSyncCodec, _codec_supports_sync
 from zarr.core.indexing import is_scalar
 
 if TYPE_CHECKING:
@@ -240,7 +240,13 @@ class ChunkTransform:
     def __post_init__(self) -> None:
         from zarr.core.codec_pipeline import codecs_from_list
 
-        non_sync = [c for c in self.codecs if not isinstance(c, SupportsSyncCodec)]
+        # _codec_supports_sync, not a bare isinstance check: a codec can satisfy
+        # the SupportsSyncCodec protocol structurally yet be unable to run
+        # synchronously (ShardingCodec whose inner/index chain contains an
+        # async-only codec). Such codecs opt out via `_sync_capable`, and the
+        # TypeError here is what makes FusedCodecPipeline.evolve_from_array_spec
+        # decline the sync fast path and fall back to the async pipeline.
+        non_sync = [c for c in self.codecs if not _codec_supports_sync(c)]
         if non_sync:
             names = ", ".join(type(c).__name__ for c in non_sync)
             raise TypeError(