Add experimental XarraySource

docs/configuration/sources.md

@@ -37,6 +37,7 @@ ui.servable()
 | Oracle | Oracle via SQLAlchemy |
 | MSSQL | Microsoft SQL Server via SQLAlchemy |
 | Intake | Data catalogs |
+| Xarray | NetCDF and Zarr scientific datasets |
 
 ## Database connections
 
@@ -173,6 +174,28 @@ source = DuckDBSource(
 
 1. Required for HTTP/S3 access
 
+### Xarray datasets
+
+Use `XarraySource` when your data already lives in NetCDF or Zarr and you want
+to preserve coordinate-aware filtering before flattening to a DataFrame.
+
+``` py title="Xarray NetCDF source"
+from lumen.sources.xarray import XarraySource
+import lumen.ai as lmai
+
+source = XarraySource(
+    uri="air_temperature.nc",
+    filterable_coords=["time", "lat", "lon"],
+    max_rows=5000,
+)
+
+ui = lmai.ExplorerUI(data=source)
+ui.servable()
+```
+
+Set `dataset_format="zarr"` for suffixless stores, or leave the default `auto`
+format detection for normal NetCDF and local Zarr directories.
+
 ### Multiple sources
 
 ``` py title="Mix sources"

docs/configuration/spec/sources.md

@@ -26,9 +26,57 @@ sources:
 | `intake` | Intake catalog entries | Data catalogs |
 | `rest` | REST API endpoints | API data |
 | `live` | Live website status checks | Website monitoring |
+| `xarray` | NetCDF and Zarr datasets | Labeled n-dimensional scientific data |
 
 This guide focuses on `file` sources (most common). See Lumen's source reference for other types.
 
+## Xarray sources
+
+`xarray` sources expose each `Dataset.data_var` as a logical table. Queries filter
+coordinates with xarray selection semantics, and results are flattened to pandas
+DataFrames so they can flow through normal Lumen pipelines and views.
+
+### Basic xarray spec
+
+```yaml
+sources:
+  forecast:
+    type: xarray
+    uri: ./air_temperature.nc
+    filterable_coords: [time, lat, lon]
+    max_rows: 5000
+
+pipelines:
+  air:
+    source: forecast
+    table: air
+    filters:
+      - type: widget
+        field: time
+      - type: widget
+        field: lat
+```
+
+Use `uri` for NetCDF or Zarr-backed datasets, or construct `XarraySource(dataset=...)`
+directly in Python when you already have an in-memory `xarray.Dataset`.
+
+### Key parameters
+
+- `uri`: Local path, `file://` URI, or remote URI to an xarray-readable dataset
+- `dataset_format`: `auto`, `netcdf`, or `zarr`; set explicitly for suffixless stores
+- `load_kwargs`: Passed through to `xr.open_dataset()` or `xr.open_zarr()`
+- `filterable_coords`: Restricts which 1D coordinates appear in schema and queries
+- `max_rows`: Rejects oversized flattened results before DataFrame materialization
+
+### Current behavior
+
+- Each data variable becomes its own table
+- Only 1D coordinates are queryable
+- `get_schema()` includes `__len__` and supports sampled schemas via `limit` and `shuffle`
+- Zarr stores can be selected explicitly with `dataset_format: zarr` or inferred from local store markers
+
+See [examples/xarray_air_temperature.yaml](../../../examples/xarray_air_temperature.yaml) for a minimal review-friendly spec.
+
 ## File sources
 
 FileSource loads data from files in various formats:

examples/xarray_air_temperature.yaml

@@ -0,0 +1,29 @@
+config:
+  title: Xarray Air Temperature
+  reloadable: false
+
+sources:
+  air_temperature:
+    type: xarray
+    uri: ./air_temperature.nc
+    filterable_coords: [time, lat, lon]
+    max_rows: 5000
+
+pipelines:
+  air_temperature:
+    source: air_temperature
+    table: air
+    filters:
+      - type: widget
+        field: time
+      - type: widget
+        field: lat
+
+layouts:
+  - title: Air Temperature
+    pipeline: air_temperature
+    views:
+      - type: table
+        page_size: 20
+        sizing_mode: stretch_width
+    sizing_mode: stretch_width

examples/xarray_air_temperature_demo.py

@@ -0,0 +1,51 @@
+"""
+Small manual demo for the experimental XarraySource.
+
+This script uses xarray's tutorial dataset, so it requires:
+- xarray
+- pooch
+- network access on first run to download the dataset
+"""
+
+from __future__ import annotations
+
+import sys
+
+from pathlib import Path
+
+import xarray as xr
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+from lumen.sources.xarray import XarraySource
+
+
+def main() -> None:
+    ds = xr.tutorial.open_dataset("air_temperature")
+    source = XarraySource(dataset=ds)
+
+    try:
+        table = source.get_tables()[0]
+
+        print("Tables:")
+        print(source.get_tables())
+
+        print("\nSchema keys:")
+        print(list(source.get_schema(table).keys()))
+
+        print("\nMetadata:")
+        print(source.get_metadata(table))
+
+        print("\nFiltered result (lat=(30.0, 60.0), time='2013-01-01'):")
+        result = source.get(table, lat=(30.0, 60.0), time="2013-01-01")
+        print(result.head())
+        print(f"\nReturned {len(result)} rows")
+    finally:
+        ds.close()
+        source.close()
+
+
+if __name__ == "__main__":
+    main()

lumen/sources/__init__.py

@@ -1 +1,16 @@
+from importlib import import_module
+from importlib.util import find_spec
+
 from .base import *
+from .base import __all__ as _base_all
+
+__all__ = list(_base_all)
+
+if find_spec("xarray") is not None:
+    __all__.append("XarraySource")
+
+
+def __getattr__(name):
+    if name == "XarraySource" and find_spec("xarray") is not None:
+        return import_module(f"{__name__}.xarray").XarraySource
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")