From 7d4954580cc71f61bf7b37c0fed9353a804f7c3b Mon Sep 17 00:00:00 2001 From: Andrew Huang Date: Thu, 19 Mar 2026 14:44:45 -0700 Subject: [PATCH 01/13] Add req --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 6bfb1d0..9c6faaf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ [project.optional-dependencies] test = [ + "cftime>", "pytest", "xarray[io]", "gcsfs", From e8ba7218e349bee12c0eaef49c876d59924a4df2 Mon Sep 17 00:00:00 2001 From: Andrew Huang Date: Thu, 26 Mar 2026 12:19:54 -0700 Subject: [PATCH 02/13] Add docs infra --- .github/workflows/docs.yml | 52 +++++++++++++++ .gitignore | 2 + CONTRIBUTING.md | 2 + docs/assets/logo.svg | 11 ++++ docs/contributing.md | 1 + docs/examples.md | 23 +++++++ docs/index.md | 1 + docs/reference/xarray_sql.md | 8 +++ pixi.toml | 76 ++++++++++++++++++++++ pyproject.toml | 7 ++ xarray_sql/df.py | 23 +++---- xarray_sql/reader.py | 6 +- zensical.toml | 122 +++++++++++++++++++++++++++++++++++ 13 files changed, 320 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/docs.yml create mode 100644 docs/assets/logo.svg create mode 100644 docs/contributing.md create mode 100644 docs/examples.md create mode 100644 docs/index.md create mode 100644 docs/reference/xarray_sql.md create mode 100644 pixi.toml create mode 100644 zensical.toml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..3a149a4 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,52 @@ +name: Build documentation + +on: + push: + branches: + - main + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Pixi + uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659 # v0.8.1 + with: + environments: docs + + - name: Build documentation + run: pixi run -e docs docs-build + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: ./site + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index c5e693e..7e5bb31 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ test_data .chainlink .claude CHANGELOG.md +*.ipynb +/site diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d173793..c64d18f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,12 +26,14 @@ reading [Xarray's contributing guide](https://docs.xarray.dev/en/stable/contribu Thanks so much for your contribution! For a volunteer led project, we so appreciate your help. A few things to keep in mind: + - Please be nice. We assume good intent from you, and we ask you to do the same for us. - Development in this project will be slow if not sporadic. Reviews will come as time allows. - Every contribution, big or small, matters and deserves credit. Here are a few requests for your development process: + - We require all code to be formatted with `pyink` and type-checked with `mypy`. These checks run automatically via pre-commit hooks (see Developer setup above). If you need to run them manually: diff --git a/docs/assets/logo.svg b/docs/assets/logo.svg new file mode 100644 index 0000000..8636807 --- /dev/null +++ b/docs/assets/logo.svg @@ -0,0 +1,11 @@ + + + xarray + -sql + diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100644 index 0000000..ea38c9b --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1 @@ +--8<-- "CONTRIBUTING.md" diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 0000000..cff0f33 --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,23 @@ +# Examples + +```python +import xarray as xr +import xarray_sql as xql + +ds = xr.tutorial.open_dataset('air_temperature') + +ctx = xql.XarrayContext() +ctx.from_dataset('air', ds, chunks=dict(time=24)) + +result = ctx.sql(''' + SELECT + "lat", "lon", AVG("air") as air_avg + FROM + "air" + GROUP BY + "lat", "lon" +''') + +df = result.to_pandas() +df.head() +``` diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..612c7a5 --- /dev/null +++ b/docs/index.md @@ -0,0 +1 @@ +--8<-- "README.md" diff --git a/docs/reference/xarray_sql.md b/docs/reference/xarray_sql.md new file mode 100644 index 0000000..bef077c --- /dev/null +++ b/docs/reference/xarray_sql.md @@ -0,0 +1,8 @@ +# xarray-sql + +::: xarray_sql + options: + show_root_heading: true + show_source: false + members: true + show_submodules: true diff --git a/pixi.toml b/pixi.toml new file mode 100644 index 0000000..5a863df --- /dev/null +++ b/pixi.toml @@ -0,0 +1,76 @@ +[workspace] +name = "xarray-sql" +channels = ["conda-forge"] +platforms = ["osx-arm64", "osx-64", "linux-64", "win-64"] + +[tasks] +postinstall = "pip install --no-build-isolation --no-deps --disable-pip-version-check -e ." +postinstall-maturin = "maturin develop --release" + +[dependencies] +python = ">=3.10" +pip = "*" +maturin = ">=1.9,<2.0" +setuptools = ">=61" + +[feature.test.dependencies] +pytest = ">=6" +pytest-cov = "*" +mypy = "*" +xarray = ">=2024.7.0" +dask = ">=2024.8.0" +cftime = "*" + +[feature.test.pypi-dependencies] +datafusion = "==52.0.0" + +[feature.test.tasks] +test = "pytest" +test-coverage = "pytest --cov=xarray_sql --cov-report=xml --cov-report=term-missing" + +[feature.build.dependencies] +python-build = "*" +twine = "*" +wheel = "*" +maturin = ">=1.9,<2.0" + +[feature.build.tasks] +build-wheel = "maturin build --release --strip" +check-wheel = "twine check target/wheels/*" + +[feature.lint.dependencies] +pre-commit = "*" + +[feature.lint.tasks] +lint = "pre-commit run --all-files" +lint-install = "pre-commit install" + +[feature.py310.dependencies] +python = "3.10.*" + +[feature.py311.dependencies] +python = "3.11.*" + +[feature.py312.dependencies] +python = "3.12.*" + +[feature.py313.dependencies] +python = "3.13.*" + +[feature.docs.dependencies] +zensical = "*" +mkdocstrings-python = "*" + +[feature.docs.tasks] +docs = "zensical serve" +docs-build = "zensical build" + +[environments] +default = ["test"] +py310 = ["py310", "test"] +py311 = ["py311", "test"] +py312 = ["py312", "test"] +py313 = ["py313", "test"] +docs = ["docs"] +build = ["build"] +lint = { features = ["lint"], no-default-feature = true } diff --git a/pyproject.toml b/pyproject.toml index 9c6faaf..c1ec9bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,13 @@ test = [ "xarray[io]", "gcsfs", ] +dev = [ + "zensical", + "mkdocstrings[python]", + "pre-commit", + "pytest", + "watchfiles", +] [project.urls] Homepage = "https://github.com/alxmrs/xarray-sql" diff --git a/xarray_sql/df.py b/xarray_sql/df.py index 756de20..85a7b1a 100644 --- a/xarray_sql/df.py +++ b/xarray_sql/df.py @@ -64,10 +64,10 @@ def _block_len(block: Block) -> int: def from_map_batched( func: Callable[..., pd.DataFrame], - *iterables, + *iterables: Any, args: tuple | None = None, schema: pa.Schema = None, - **kwargs, + **kwargs: Any, ) -> pa.RecordBatchReader: """Create a PyArrow RecordBatchReader by mapping a function over iterables. @@ -76,8 +76,8 @@ def from_map_batched( RecordBatches which are created via the `func` one-at-a-time. Args: - func: Function to apply to each element of the iterables. Currently, the function - must return a Pandas DataFrame. + func: Function to apply to each element of the iterables. Currently, the + function must return a Pandas DataFrame. *iterables: Iterable objects to map the function over. schema: Optional schema needed for the RecordBatchReader. args: Additional positional arguments to pass to func. @@ -98,7 +98,7 @@ def map_batches(): def from_map( - func: Callable, *iterables, args: tuple | None = None, **kwargs + func: Callable, *iterables: Any, args: tuple | None = None, **kwargs: Any ) -> pa.Table: """Create a PyArrow Table by mapping a function over iterables. @@ -358,12 +358,13 @@ def partition_metadata( blocks: List of block slices from block_slices(). Returns: - List of dicts mapping dimension name to (min_value, max_value, dtype_str) - tuples. - - For datetime64, values are nanoseconds since Unix epoch (int64), - dtype_str is "timestamp_ns" - - For numeric types, values are Python int or float, dtype_str is - "int64" or "float64" + List of dicts mapping dimension name to + (min_value, max_value, dtype_str) tuples. + + - For datetime64, values are nanoseconds since Unix epoch + (int64), dtype_str is "timestamp_ns" + - For numeric types, values are Python int or float, + dtype_str is "int64" or "float64" Note: If a partition has an empty slice for a dimension, that dimension is diff --git a/xarray_sql/reader.py b/xarray_sql/reader.py index 01bb024..1460242 100644 --- a/xarray_sql/reader.py +++ b/xarray_sql/reader.py @@ -173,9 +173,9 @@ def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> pa.RecordBatchReader: Args: ds: An Xarray Dataset. All `data_vars` must share the same dimensions. - chunks: Xarray-like chunks. If not provided, will default to the Dataset's - chunks. The product of the chunk sizes becomes the standard length of each - dataframe partition. + chunks: Xarray-like chunks. If not provided, will default to the + Dataset's chunks. The product of the chunk sizes becomes the + standard length of each dataframe partition. Returns: A PyArrow RecordBatchReader, which is a table representation of the input diff --git a/zensical.toml b/zensical.toml new file mode 100644 index 0000000..21bce73 --- /dev/null +++ b/zensical.toml @@ -0,0 +1,122 @@ +[project] +site_name = "xarray-sql" +site_description = "Query Xarray with SQL" +site_author = "Alexander Merose" +site_url = "https://alxmrs.github.io/xarray-sql" +repo_url = "https://github.com/alxmrs/xarray-sql" +repo_name = "alxmrs/xarray-sql" +edit_uri = "edit/main/docs/" +nav = [ + {"Home" = "index.md"}, + {"Examples" = "examples.md"}, + {"Contributing" = "contributing.md"}, + {"Reference" = "reference/xarray_sql.md"} +] + +# Theme configuration +[project.theme] +variant = "modern" +logo = "assets/logo.svg" +features = [ + "announce.dismiss", + "content.action.edit", + "content.action.view", + "content.code.annotate", + "content.code.copy", + "content.tooltips", + "navigation.footer", + "navigation.indexes", + "navigation.instant", + "navigation.instant.prefetch", + "navigation.instant.progress", + "navigation.sections", + "navigation.top", + "navigation.tracking", + "search.highlight", + "search.share", + "search.suggest", + "toc.follow" +] + +# Color palette - light mode +[[project.theme.palette]] +media = "(prefers-color-scheme: light)" +scheme = "default" +primary = "custom" +accent = "blue" + +[project.theme.palette.toggle] +icon = "material/brightness-7" +name = "Switch to dark mode" + +# Color palette - dark mode +[[project.theme.palette]] +media = "(prefers-color-scheme: dark)" +scheme = "slate" +primary = "custom" +accent = "black" + +[project.theme.palette.toggle] +icon = "material/brightness-4" +name = "Switch to light mode" + +# Markdown extensions +[project.markdown_extensions.toc] +permalink = true + +[project.markdown_extensions.pymdownx.highlight] +anchor_linenums = true +line_spans = "__span" +pygments_lang_class = true + +[project.markdown_extensions.pymdownx.superfences] + +[project.markdown_extensions.pymdownx.tasklist] +custom_checkbox = true + +[project.markdown_extensions.pymdownx.inlinehilite] + +[project.markdown_extensions.admonition] + +[project.markdown_extensions.pymdownx.snippets] +url_download = true +base_path = ["."] + +[project.markdown_extensions.pymdownx.tabbed] +alternate_style = true + +[project.markdown_extensions.pymdownx.emoji] +emoji_index = "zensical.extensions.emoji.twemoji" +emoji_generator = "zensical.extensions.emoji.to_svg" + +[project.markdown_extensions.attr_list] + +[project.markdown_extensions.md_in_html] + +[project.markdown_extensions.abbr] + +[project.markdown_extensions.def_list] + +# Plugins +[project.plugins] +search = {} +typeset = {} + +[project.plugins.mkdocstrings.handlers.python] +paths = ["."] + +[project.plugins.mkdocstrings.handlers.python.options] +docstring_style = "google" +show_if_no_docstring = true +filters = ["!^_"] + +# Extra configuration +[project.extra] + +[[project.extra.social]] +icon = "fontawesome/brands/github" +link = "https://github.com/alxmrs/xarray-sql" + +[[project.extra.social]] +icon = "fontawesome/brands/python" +link = "https://pypi.org/project/xarray-sql" From 20a7b72f956175596fc7c8eb61766dc3999b33d6 Mon Sep 17 00:00:00 2001 From: Andrew Huang Date: Thu, 26 Mar 2026 12:54:49 -0700 Subject: [PATCH 03/13] update logo --- docs/assets/logo.svg | 87 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 77 insertions(+), 10 deletions(-) diff --git a/docs/assets/logo.svg b/docs/assets/logo.svg index 8636807..06e3f7e 100644 --- a/docs/assets/logo.svg +++ b/docs/assets/logo.svg @@ -1,11 +1,78 @@ - - - xarray - -sql + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + SQL + From 913cab02292b297a061b788e950204b4feb1d795 Mon Sep 17 00:00:00 2001 From: Andrew Huang Date: Thu, 26 Mar 2026 13:08:59 -0700 Subject: [PATCH 04/13] update logo --- docs/assets/logo.svg | 157 ++++++++++++++++++++++++++----------------- 1 file changed, 95 insertions(+), 62 deletions(-) diff --git a/docs/assets/logo.svg b/docs/assets/logo.svg index 06e3f7e..213e6b1 100644 --- a/docs/assets/logo.svg +++ b/docs/assets/logo.svg @@ -1,7 +1,39 @@ - - -