Skip to content

Commit 051ffac

Browse files
committed
Add int64 fallback for dictionary index width
Cap coordinate dictionary indices at the correct width for any cardinality: the selection now tiers int8 → int16 → int32 → int64 at exact `MAX + 1` boundaries (indices run 0..n-1, so a signed max M holds n = M+1). The int64 fallback keeps astronomically large coordinate axes representable instead of silently overflowing a 32-bit index. Follows the adaptive-key-width note in jayendra13's zarr-datafusion. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019VuSeCio99NcME5eubcN3N
1 parent 6eed3c3 commit 051ffac

2 files changed

Lines changed: 33 additions & 4 deletions

File tree

tests/test_dict_coords.py

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,32 @@
1111

1212
import numpy as np
1313
import pyarrow as pa
14+
import pytest
1415
import xarray as xr
1516

1617
from xarray_sql import XarrayContext
17-
from xarray_sql.df import _parse_schema, block_slices, iter_record_batches
18+
from xarray_sql.df import (
19+
_coord_index_type,
20+
_parse_schema,
21+
block_slices,
22+
iter_record_batches,
23+
)
24+
25+
26+
@pytest.mark.parametrize(
27+
"n_values, expected",
28+
[
29+
(1, pa.int8()),
30+
(128, pa.int8()), # int8 holds indices 0..127
31+
(129, pa.int16()),
32+
(32768, pa.int16()), # int16 holds indices 0..32767
33+
(32769, pa.int32()),
34+
(2**31, pa.int32()), # int32 holds indices 0..2**31-1
35+
(2**31 + 1, pa.int64()), # fallback keeps huge axes representable
36+
],
37+
)
38+
def test_coord_index_type_boundaries(n_values, expected):
39+
assert _coord_index_type(n_values) == expected
1840

1941

2042
def _grid() -> xr.Dataset:

xarray_sql/df.py

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -403,12 +403,19 @@ def _coord_index_type(n_values: int) -> pa.DataType:
403403
partition's cardinality). Narrower indices mean fewer bytes moved and cheaper
404404
group/join hashing: a 721-point latitude or 1440-point longitude fits int16,
405405
while a multi-million-step time axis needs int32.
406+
407+
Indices run ``0 .. n_values - 1``, so a signed type with maximum ``M`` holds
408+
a cardinality of ``M + 1``. The ``int64`` fallback keeps astronomically large
409+
coordinate axes representable rather than silently overflowing a 32-bit index
410+
(cf. the adaptive-key-width note in jayendra13's zarr-datafusion).
406411
"""
407-
if n_values <= np.iinfo(np.int8).max:
412+
if n_values <= np.iinfo(np.int8).max + 1:
408413
return pa.int8()
409-
if n_values <= np.iinfo(np.int16).max:
414+
if n_values <= np.iinfo(np.int16).max + 1:
410415
return pa.int16()
411-
return pa.int32()
416+
if n_values <= np.iinfo(np.int32).max + 1:
417+
return pa.int32()
418+
return pa.int64()
412419

413420

414421
def _as_dictionary_field(field: pa.Field, n_values: int) -> pa.Field:

0 commit comments

Comments
 (0)