Skip to content

Commit 35f910f

Browse files
authoredMar 3, 2024
Turning on Dask-Expr (#28)
This PR enables dask-expr with copy on write semantics in Dask Dataframes. Fixes #14.
1 parent 1de22ff commit 35f910f

9 files changed

+3327
-3
lines changed
 

‎perf_tests/compute_air.py-2024-03-03T07:42:36+05:30.svg

+415
Loading

‎perf_tests/compute_air.py-2024-03-03T07:50:18+05:30.svg

+415
Loading

‎perf_tests/groupby_air.py-2024-03-03T07:43:22+05:30.svg

+415
Loading

‎perf_tests/groupby_air.py-2024-03-03T07:50:06+05:30.svg

+415
Loading

‎perf_tests/groupby_air_full.py-2024-03-03T07:44:17+05:30.svg

+415
Loading

‎perf_tests/groupby_air_full.py-2024-03-03T07:49:53+05:30.svg

+415
Loading

‎perf_tests/sanity.py-2024-03-03T07:45:47+05:30.svg

+415
Loading

‎perf_tests/sanity.py-2024-03-03T07:49:44+05:30.svg

+415
Loading

‎qarray/df.py

+7-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import itertools
22
import typing as t
33

4+
import dask
45
import dask.dataframe as dd
56
import numpy as np
67
import pandas as pd
@@ -12,6 +13,12 @@
1213
Block = t.Dict[str, slice]
1314
Chunks = t.Dict[str, int]
1415

16+
# Turn on Dask-Expr
17+
dask.config.set({'dataframe.query-planning-warning': False})
18+
dask.config.set({"dataframe.query-planning": True})
19+
# Turn on Copy-On-Write (needs Pandas 2.0).
20+
pd.options.mode.copy_on_write = True
21+
1522

1623
# Borrowed from Xarray
1724
def _get_chunk_slicer(dim: t.Hashable, chunk_index: t.Mapping,
@@ -123,6 +130,3 @@ def f(b: Block) -> pd.DataFrame:
123130
divisions=divisions,
124131
token=token,
125132
)
126-
127-
# TODO(alxmrs): Try dask expressions dataframe:
128-
# https://github.com/dask-contrib/dask-expr

0 commit comments

Comments
 (0)
Please sign in to comment.