fix(data_catalog): reading from private bucket

.github/workflows/sonar.yml

@@ -42,6 +42,16 @@ jobs:
       - uses: prefix-dev/setup-pixi@v0.10.2
         with:
           pixi-version: "v0.76.0"
+      - name: Create aws profile for minio access
+        env:
+          ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY_READONLY_HYDROMT_DATA }}
+          SECRET_KEY: ${{ secrets.MINIO_SECRET_KEY_READONLY_HYDROMT_DATA }}
+        run: |
+          mkdir -p ~/.aws
+          pixi run aws configure set aws_access_key_id     "$ACCESS_KEY" --profile hydromt-data
+          pixi run aws configure set aws_secret_access_key "$SECRET_KEY" --profile hydromt-data
+          pixi run aws configure set region eu-west-1 --profile hydromt-data
+          pixi run aws configure set endpoint_url https://s3.deltares.nl --profile hydromt-data
       - name: Test
         run: pixi run --locked test-cov
       - name: SonarQube Scan

.github/workflows/tests.yml

@@ -46,6 +46,16 @@ jobs:
         with:
           pixi-version: "v0.76.0"
           environments: ${{ matrix.dependencies }}-py${{ matrix.python-version }}
+      - name: Create aws profile for minio access
+        env:
+          ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY_READONLY_HYDROMT_DATA }}
+          SECRET_KEY: ${{ secrets.MINIO_SECRET_KEY_READONLY_HYDROMT_DATA }}
+        run: |
+          mkdir -p ~/.aws
+          pixi run aws configure set aws_access_key_id     "$ACCESS_KEY" --profile hydromt-data
+          pixi run aws configure set aws_secret_access_key "$SECRET_KEY" --profile hydromt-data
+          pixi run aws configure set region eu-west-1 --profile hydromt-data
+          pixi run aws configure set endpoint_url https://s3.deltares.nl --profile hydromt-data
       - name: Test
         env:
           PYTEST_ADDOPTS: ${{ matrix.dependencies == 'low' && '-p no:warnings' || '' }}

hydromt/_compat.py

@@ -7,6 +7,7 @@
 HAS_OPENPYXL = False
 HAS_PYET = False
 HAS_S3FS = False
+HAS_BOTO3 = False
 
 try:
     import gcsfs
@@ -43,6 +44,14 @@
 except ImportError:
     pass
 
+try:
+    import boto3
+
+    HAS_BOTO3 = True
+except ImportError:
+    pass
+
+
 # entrypoints in standard library only compatible from 3.10 onwards
 py_version = sys.version_info
 if py_version[0] >= 3 and py_version[1] >= 10:

hydromt/data_catalog/drivers/dataset/dataset_driver.py

@@ -21,7 +21,7 @@ class DatasetDriver(BaseDriver, ABC):
     @abstractmethod
     def read(
         self, uris: list[str], *, handle_nodata: NoDataStrategy = NoDataStrategy.RAISE
-    ) -> xr.Dataset:
+    ) -> xr.Dataset | None:
         """
         Read data from one or more URIs into an xarray Dataset.
 
@@ -37,8 +37,8 @@ def read(
 
         Returns
         -------
-        xr.Dataset
-            The loaded dataset.
+        xr.Dataset | None
+            The loaded dataset, or None if no data was found and the strategy allows.
         """
         ...
 

hydromt/data_catalog/drivers/dataset/xarray_driver.py

@@ -14,8 +14,9 @@
 from hydromt.data_catalog.drivers.xarray_options import (
     XarrayDriverOptions,
     XarrayIOFormat,
+    _read_xarray,
 )
-from hydromt.error import NoDataStrategy, exec_nodata_strat
+from hydromt.error import NoDataStrategy
 
 logger = logging.getLogger(__name__)
 
@@ -42,7 +43,7 @@ class DatasetXarrayDriver(DatasetDriver):
 
     def read(
         self, uris: list[str], *, handle_nodata: NoDataStrategy = NoDataStrategy.RAISE
-    ) -> xr.Dataset:
+    ) -> xr.Dataset | None:
         """
         Read zarr or netCDF data into an xarray Dataset.
 
@@ -60,45 +61,21 @@ def read(
 
         Returns
         -------
-        xr.Dataset
-            The dataset read from the source files.
+        xr.Dataset | None
+            The dataset read from the source files, or None if no data was found and the strategy allows.
 
         Raises
         ------
         ValueError
             If the provided files have mixed or unsupported extensions.
         """
-        preprocessor = self.options.get_preprocessor()
-        filtered_uris, io_format = self.options.filter_uris_by_format(uris)
-
-        # Read and merge
-        if io_format == XarrayIOFormat.ZARR:
-            datasets = [
-                preprocessor(xr.open_zarr(_uri, **self.options.get_kwargs()))
-                for _uri in filtered_uris
-            ]
-            ds: xr.Dataset = xr.merge(datasets)
-        elif io_format == XarrayIOFormat.NETCDF4:
-            ds: xr.Dataset = xr.open_mfdataset(
-                filtered_uris,
-                decode_coords="all",
-                preprocess=preprocessor,
-                **self.options.get_kwargs(),
-                decode_timedelta=True,
-            )
-        else:
-            raise ValueError(
-                f"Unknown extension for DatasetXarrayDriver: {self.options.get_reading_ext(uris[0])}"
-            )
-
-        for variable in ds.data_vars:
-            if ds[variable].size == 0:
-                exec_nodata_strat(
-                    f"No data from driver: '{self.name}' for variable: '{variable}'",
-                    strategy=handle_nodata,
-                )
-                return None  # handle_nodata == ignore
-        return ds
+        return _read_xarray(
+            uris=uris,
+            options=self.options,
+            filesystem=self.filesystem,
+            driver_name=self.name,
+            handle_nodata=handle_nodata,
+        )
 
     def write(
         self,

hydromt/data_catalog/drivers/geodataset/geodataset_driver.py

@@ -35,7 +35,7 @@ def read(
         mask: Geom | None = None,
         predicate: Predicate = "intersects",
         metadata: SourceMetadata | None = None,
-    ) -> xr.Dataset:
+    ) -> xr.Dataset | None:
         """
         Read in data to an xarray Dataset.
 

hydromt/data_catalog/drivers/geodataset/xarray_driver.py

@@ -17,8 +17,9 @@
 from hydromt.data_catalog.drivers.xarray_options import (
     XarrayDriverOptions,
     XarrayIOFormat,
+    _read_xarray,
 )
-from hydromt.error import NoDataStrategy, exec_nodata_strat
+from hydromt.error import NoDataStrategy
 from hydromt.typing import (
     Geom,
     Predicate,
@@ -54,7 +55,7 @@ def read(
         mask: Geom | None = None,
         predicate: Predicate = "intersects",
         metadata: SourceMetadata | None = None,
-    ) -> xr.Dataset:
+    ) -> xr.Dataset | None:
         """
         Read in data to an xarray Dataset.
 
@@ -89,37 +90,13 @@ def read(
                 "metadata": metadata,
             },
         )
-        preprocessor = self.options.get_preprocessor()
-        filtered_uris, io_format = self.options.filter_uris_by_format(uris)
-
-        # Read and merge
-        if io_format == XarrayIOFormat.ZARR:
-            datasets = [
-                preprocessor(xr.open_zarr(_uri, **self.options.get_kwargs()))
-                for _uri in filtered_uris
-            ]
-            ds: xr.Dataset = xr.merge(datasets)
-        elif io_format == XarrayIOFormat.NETCDF4:
-            ds: xr.Dataset = xr.open_mfdataset(
-                filtered_uris,
-                decode_coords="all",
-                preprocess=preprocessor,
-                **self.options.get_kwargs(),
-                decode_timedelta=True,
-            )
-        else:
-            raise ValueError(
-                f"Unknown extension for GeoDatasetXarrayDriver: {self.options.get_reading_ext(uris[0])} "
-            )
-
-        for variable in ds.data_vars:
-            if ds[variable].size == 0:
-                exec_nodata_strat(
-                    f"No data from driver: '{self.name}' for variable: '{variable}'",
-                    strategy=handle_nodata,
-                )
-                return None  # handle_nodata == ignore
-        return ds
+        return _read_xarray(
+            uris=uris,
+            options=self.options,
+            filesystem=self.filesystem,
+            driver_name=self.name,
+            handle_nodata=handle_nodata,
+        )
 
     def write(
         self,

hydromt/data_catalog/drivers/raster/raster_dataset_driver.py

@@ -35,7 +35,7 @@ def read(
         zoom: Zoom | None = None,
         chunks: dict[str, Any] | None = None,
         metadata: SourceMetadata | None = None,
-    ) -> xr.Dataset:
+    ) -> xr.Dataset | None:
         """
         Read raster data from one or more URIs into an xarray Dataset.
 
@@ -61,8 +61,8 @@ def read(
 
         Returns
         -------
-        xr.Dataset
-            The loaded raster dataset.
+        xr.Dataset | None
+            The loaded raster dataset, or None if no data was found and the strategy allows.
 
         """
         ...

hydromt/data_catalog/drivers/raster/raster_xarray_driver.py

@@ -5,7 +5,6 @@
 from typing import Any, ClassVar
 
 import xarray as xr
-from aiohttp.client_exceptions import ClientResponseError
 from pydantic import Field
 
 from hydromt._utils.unused_kwargs import _warn_on_unused_kwargs
@@ -18,8 +17,9 @@
 from hydromt.data_catalog.drivers.xarray_options import (
     XarrayDriverOptions,
     XarrayIOFormat,
+    _read_xarray,
 )
-from hydromt.error import NoDataStrategy, exec_nodata_strat
+from hydromt.error import NoDataStrategy
 from hydromt.typing import (
     Geom,
     SourceMetadata,
@@ -59,7 +59,7 @@ def read(
         zoom: Zoom | None = None,
         chunks: dict[str, Any] | None = None,
         metadata: SourceMetadata | None = None,
-    ) -> xr.Dataset:
+    ) -> xr.Dataset | None:
         """
         Read zarr or netCDF raster data into an xarray Dataset.
 
@@ -87,8 +87,8 @@ def read(
 
         Returns
         -------
-        xr.Dataset
-            The merged xarray Dataset.
+        xr.Dataset | None
+            The merged xarray Dataset, or None if no data was found and the strategy allows.
 
         Raises
         ------
@@ -114,37 +114,13 @@ def read(
         if len(uris) == 0:
             return None  # handle_nodata == ignore
 
-        preprocessor = self.options.get_preprocessor()
-        filtered_uris, io_format = self.options.filter_uris_by_format(uris)
-
-        # Read and merge
-        if io_format == XarrayIOFormat.ZARR:
-            datasets = [
-                preprocessor(ds)
-                for ds in self._open_zarrs(filtered_uris, self.options.get_kwargs())
-            ]
-            ds: xr.Dataset = xr.merge(datasets)
-        elif io_format == XarrayIOFormat.NETCDF4:
-            ds: xr.Dataset = xr.open_mfdataset(
-                filtered_uris,
-                decode_coords="all",
-                preprocess=preprocessor,
-                **self.options.get_kwargs(),
-                decode_timedelta=True,
-            )
-        else:
-            raise ValueError(
-                f"Unknown extension for RasterDatasetXarrayDriver: {self.options.get_reading_ext(uris[0])} "
-            )
-
-        for variable in ds.data_vars:
-            if ds[variable].size == 0:
-                exec_nodata_strat(
-                    f"No data from driver: '{self.name}' for variable: '{variable}'",
-                    strategy=handle_nodata,
-                )
-                return None  # handle_nodata == ignore
-        return ds
+        return _read_xarray(
+            uris=uris,
+            options=self.options,
+            filesystem=self.filesystem,
+            driver_name=self.name,
+            handle_nodata=handle_nodata,
+        )
 
     def write(
         self,
@@ -194,19 +170,3 @@ def write(
             data.to_netcdf(path, **write_kwargs)
 
         return Path(path)
-
-    @staticmethod
-    def _open_zarrs(uris: list[str], read_kwargs: dict[str, Any]) -> list[xr.Dataset]:
-        """Open multiple zarr datasets with error handling."""
-        datasets = []
-        for _uri in uris:
-            try:
-                ds = xr.open_zarr(_uri, **read_kwargs)
-                datasets.append(ds)
-            except ClientResponseError as e:
-                if e.status == 401:
-                    raise PermissionError(
-                        f"Unauthorized access to {_uri}. Check your credentials."
-                    ) from e
-                raise
-        return datasets