@@ -19,7 +19,7 @@ def _get_chunk_slicer(
1919 if dim in chunk_index :
2020 which_chunk = chunk_index [dim ]
2121 return slice (
22- chunk_bounds [dim ][which_chunk ], chunk_bounds [dim ][which_chunk + 1 ]
22+ chunk_bounds [dim ][which_chunk ], chunk_bounds [dim ][which_chunk + 1 ]
2323 )
2424 return slice (None )
2525
@@ -38,19 +38,20 @@ def block_slices(ds: xr.Dataset, chunks: Chunks = None) -> t.Iterator[Block]:
3838
3939 # chunks is Dict[str, Tuple[int, ...]] from xarray
4040 chunk_bounds = {
41- dim : np .cumsum ((0 ,) + tuple (c )) # type: ignore[arg-type]
42- for dim , c in chunks .items ()
41+ dim : np .cumsum ((0 ,) + tuple (c )) # type: ignore[arg-type]
42+ for dim , c in chunks .items ()
4343 }
44- ichunk = {dim : range (len (tuple (c ))) for dim , c in
45- chunks .items ()} # type: ignore[arg-type]
44+ ichunk = {
45+ dim : range (len (tuple (c ))) for dim , c in chunks .items () # type: ignore
46+ } # type: ignore[arg-type]
4647 ick , icv = zip (* ichunk .items ()) # Makes same order of keys and val.
4748 chunk_idxs = (dict (zip (ick , i )) for i in itertools .product (* icv ))
4849 blocks = (
49- {
50- dim : _get_chunk_slicer (dim , chunk_index , chunk_bounds )
51- for dim in ds .dims
52- }
53- for chunk_index in chunk_idxs
50+ {
51+ dim : _get_chunk_slicer (dim , chunk_index , chunk_bounds )
52+ for dim in ds .dims
53+ }
54+ for chunk_index in chunk_idxs
5455 )
5556 yield from blocks
5657
@@ -139,7 +140,7 @@ def from_map(
139140 pa_table = pa .Table .from_pandas (df )
140141 except Exception as e :
141142 raise ValueError (
142- f"Cannot convert function result to PyArrow Table: { e } "
143+ f"Cannot convert function result to PyArrow Table: { e } "
143144 )
144145
145146 results .append (pa_table )
@@ -199,13 +200,15 @@ def _parse_schema(ds) -> pa.Schema:
199200# return from_map_batched(pivot_block, blocks, schema=schema)
200201
201202
202-
203203def read_xarray (ds : xr .Dataset , chunks : Chunks = None ) -> pa .dataset .Dataset :
204204 fst = next (iter (ds .values ())).dims
205205 assert all (
206- da .dims == fst for da in ds .values ()
206+ da .dims == fst for da in ds .values ()
207207 ), "All dimensions must be equal. Please filter data_vars in the Dataset."
208- def _in_memory_ds_from (batches : t .Iterable , schema_ : pa .Schema ) -> pa .dataset .InMemoryDataset :
208+
209+ def _in_memory_ds_from (
210+ batches : t .Iterable , schema_ : pa .Schema
211+ ) -> pa .dataset .InMemoryDataset :
209212 reader = pa .RecordBatchReader .from_batches (schema_ , batches )
210213 return pa .dataset .InMemoryDataset (source = reader , schema = schema_ )
211214
@@ -218,7 +221,7 @@ def pivot_block(b: Block) -> pa.RecordBatch:
218221 blocks = block_slices (ds , chunks )
219222
220223 children = [
221- _in_memory_ds_from (pivot_block (block ), schema ) for block in blocks
224+ _in_memory_ds_from (pivot_block (block ), schema ) for block in blocks
222225 ]
223226
224227 return pa .dataset .UnionDataset (schema , children )
0 commit comments