From 032e853c05cd9c60d77ee3706a15d4fd8db45704 Mon Sep 17 00:00:00 2001 From: Karnak Oza Date: Sat, 11 Jul 2026 22:43:24 +0530 Subject: [PATCH] Add Google Earth Engine accessor for satellite imagery (#53) - Implement GoogleEarthEngineAccessor with NDVI, NDWI, NDBI, and NDBI-change methods covering the use cases in #53 (vegetation coverage/breeding sites, urban density, informal settlement growth) - Register earthengine-api as an optional extra and add entry point so the accessor is discoverable via the standard source registry - Add tests covering initialization, list_countries, and indicator sign/range sanity checks against real-world reference points - Raise a clear ValueError when no cloud-free Landsat imagery is available for a given area/date range, instead of a cryptic band-not-found error Closes #53 --- README.md | 20 +- docs/sources/google_earth_engine.md | 3 + pyproject.toml | 6 +- .../sources/google_earth_engine.py | 294 ++++++++++++++++++ tests/test_accessors.py | 52 ++++ 5 files changed, 367 insertions(+), 8 deletions(-) create mode 100644 docs/sources/google_earth_engine.md create mode 100644 src/epidatasets/sources/google_earth_engine.py diff --git a/README.md b/README.md index 7a599b7..5c6e1f9 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ --- -> A Python library providing unified access to **24 epidemiological data sources** from around the world, with a plugin registry, CLI, and optional extras for specialized data. +> A Python library providing unified access to **25 epidemiological data sources** from around the world, with a plugin registry, CLI, and optional extras for specialized data. ## 📋 Table of Contents @@ -79,7 +79,7 @@ **epidatasets** provides: -- **Unified interface** — A single `get_source()` API to access 24 data sources worldwide +- **Unified interface** — A single `get_source()` API to access 25 data sources worldwide - **Plugin registry** — Sources are discovered at runtime via `entry_points`, making it easy to extend - **Optional extras** — Install only the dependencies you need (`pip install epidatasets[who,brazil]`) - **CLI** — Command-line tool for listing sources, inspecting metadata, and querying countries @@ -109,6 +109,9 @@ pip install epidatasets[eurostat] # Climate/environmental data (Copernicus CDS) pip install epidatasets[climate] +# Google Earth Engine satellite imagery +pip install epidatasets[earthengine] + # Geospatial visualization pip install epidatasets[geo] @@ -176,7 +179,7 @@ epidemiological-datasets/ │ ├── _base.py # BaseAccessor ABC │ ├── _registry.py # Plugin registry (entry_points) │ ├── cli.py # CLI (typer) -│ ├── sources/ # 24 data source accessors +│ ├── sources/ # 25 data source accessors │ │ ├── __init__.py │ │ ├── africa_cdc.py │ │ ├── cdc_opendata.py @@ -188,6 +191,7 @@ epidemiological-datasets/ │ │ ├── epipulse.py │ │ ├── eurostat.py │ │ ├── global_health.py +│ │ ├── google_earth_engine.py │ │ ├── healthdata_gov.py │ │ ├── india_idsp.py │ │ ├── infodengue_api.py @@ -219,7 +223,7 @@ epidemiological-datasets/ │ ├── index.md │ ├── installation.md │ ├── quickstart.md -│ ├── sources/ # Per-source API docs (24 pages) +│ ├── sources/ # Per-source API docs (25 pages) │ ├── api/ # API reference │ │ ├── base.md │ │ ├── registry.md @@ -245,6 +249,7 @@ epidemiological-datasets/ | [Global.health](https://global.health/) | Pandemic linelist data | Varies | Open | `epidatasets.sources.global_health` | | [Malaria Atlas Project](https://malariaatlas.org/) | Malaria prevalence & vector data | Annual | Open | `epidatasets.sources.malaria_atlas` | | [Copernicus Climate Data Store](https://cds.climate.copernicus.eu/) | Environmental & climate data | Varies | Open | `epidatasets.sources.copernicus_cds` | +| [Google Earth Engine](https://earthengine.google.com/) | Satellite imagery (Landsat, Sentinel-2, MODIS) — vegetation, built-up, water indices | Varies (per-scene) | Free registration | `epidatasets.sources.google_earth_engine` | | [Pathoplexus](https://pathoplexus.org/) | Pathogen genomic data | Continuous | Open | `epidatasets.sources.pathoplexus` | | [InfoDengue](https://info.dengue.mat.br/) | Dengue surveillance (Brazil) | Weekly | Open | `epidatasets.sources.infodengue_api` | @@ -508,6 +513,7 @@ owid_covid = owid.get_covid_data( | `epipulse` | `EpiPulseAccessor` | — | ECDC EpiPulse surveillance portal | | `eurostat` | `EurostatAccessor` | `[eurostat]` | EU health statistics | | `global_health` | `GlobalHealthAccessor` | — | Global.health pandemic linelist data | +| `google_earth_engine` | `GoogleEarthEngineAccessor` | `[earthengine]` | Google Earth Engine satellite imagery indices | | `healthdata_gov` | `HealthDataGovAccessor` | — | US HealthData.gov | | `india_idsp` | `IndiaIDSPAccessor` | — | India IDSP disease surveillance | | `infodengue` | `InfoDengueAPI` | — | InfoDengue dengue surveillance (Brazil) | @@ -527,7 +533,7 @@ owid_covid = owid.get_covid_data( ### What is epidatasets? -A Python library providing a unified interface to 24 epidemiological data sources worldwide, installable via `pip install epidatasets`. +A Python library providing a unified interface to 25 epidemiological data sources worldwide, installable via `pip install epidatasets`. ### Do I need to install all optional dependencies? @@ -597,9 +603,9 @@ We welcome contributions! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for deta ## 📊 Statistics -- **Data sources:** 24 registered (via plugin registry) +- **Data sources:** 25 registered (via plugin registry) - **Countries covered:** 100+ -- **Optional extras:** 10 (`who`, `brazil`, `eurostat`, `climate`, `geo`, `viz`, `genomics`, `cli`, `worldbank`, `search`) +- **Optional extras:** 11 (`who`, `brazil`, `eurostat`, `climate`, `earthengine`, `geo`, `viz`, `genomics`, `cli`, `worldbank`, `search`) - **Example notebooks:** 28+ - **Documentation:** [epidatasets.readthedocs.io](https://epidatasets.readthedocs.io) diff --git a/docs/sources/google_earth_engine.md b/docs/sources/google_earth_engine.md new file mode 100644 index 0000000..1fa7121 --- /dev/null +++ b/docs/sources/google_earth_engine.md @@ -0,0 +1,3 @@ +# Google Earth Engine + +::: epidatasets.sources.google_earth_engine.GoogleEarthEngineAccessor diff --git a/pyproject.toml b/pyproject.toml index bdc459d..a7bb716 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,9 @@ brazil = [ eurostat = [ "eurostat>=1.1.1", ] +earthengine = [ + "earthengine-api>=0.1.400", +] climate = [ "cdsapi>=0.7.7", "xarray>=2026.4.0", @@ -95,7 +98,7 @@ pdf = [ "pdfplumber>=0.10.0", ] all = [ - "epidatasets[who,brazil,eurostat,climate,geo,viz,genomics,cli,worldbank,search,scraping,pdf]", + "epidatasets[who,brazil,eurostat,climate,geo,viz,genomics,cli,worldbank,search,scraping,pdf,earthengine]", ] dev = [ "pytest>=7.4.0", @@ -137,6 +140,7 @@ ecdc_atlas = "epidatasets.sources.ecdc_atlas:ECDCAtlasAccessor" epipulse = "epidatasets.sources.epipulse:EpiPulseAccessor" eurostat = "epidatasets.sources.eurostat:EurostatAccessor" global_health = "epidatasets.sources.global_health:GlobalHealthAccessor" +google_earth_engine = "epidatasets.sources.google_earth_engine:GoogleEarthEngineAccessor" healthdata_gov = "epidatasets.sources.healthdata_gov:HealthDataGovAccessor" india_idsp = "epidatasets.sources.india_idsp:IndiaIDSPAccessor" infodengue = "epidatasets.sources.infodengue_api:InfoDengueAPI" diff --git a/src/epidatasets/sources/google_earth_engine.py b/src/epidatasets/sources/google_earth_engine.py new file mode 100644 index 0000000..64ece66 --- /dev/null +++ b/src/epidatasets/sources/google_earth_engine.py @@ -0,0 +1,294 @@ +""" +Google Earth Engine (GEE) Accessor + +Provides access to planetary-scale satellite imagery for epidemiological +modeling, including Landsat, Sentinel-2, and MODIS derived indicators. + +Data Source: https://earthengine.google.com/ +API: Earth Engine Python API +Documentation: https://developers.google.com/earth-engine + +Requirements: + - Google account with Earth Engine access (register at + https://signup.earthengine.google.com/, free for research/nonprofit use) + - earthengine-api Python library: pip install earthengine-api + - One-time interactive auth: run `earthengine authenticate` in a shell, + or call ee.Authenticate() once from Python + +Key Datasets: + - LANDSAT/LC08/C02/T1_L2: Landsat 8 Collection 2, Level 2 (surface reflectance) + - COPERNICUS/S2_SR_HARMONIZED: Sentinel-2 surface reflectance + - MODIS/061/MOD13Q1: MODIS vegetation indices (16-day, 250m) + - USDOS/LSIB_SIMPLE/2017: Simplified country boundaries + +Use Cases: + - Vegetation coverage / mosquito breeding habitat proxy (NDVI) + - Urban density analysis (NDBI - built-up index) + - Informal settlement growth / detection (NDBI change over time) + - Standing water proxy relevant to breeding sites (NDWI) + +Notes on quotas: + This accessor returns *reduced scalar values* (e.g. mean NDVI over a + region) rather than raw raster exports, to stay well within Earth + Engine's interactive-use quotas. Raster/export workflows + (Export.image.toDrive) are intentionally out of scope for this + accessor and would need separate, explicitly-batched handling. + +Author: +License: MIT +""" + +from __future__ import annotations + +import logging +import os +from typing import ClassVar + +import pandas as pd + +from epidatasets._base import BaseAccessor + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Optional import - earthengine-api is required only if using this accessor +try: + import ee + + HAS_EE = True +except ImportError: + HAS_EE = False + logger.warning( + "earthengine-api not installed. Install with: pip install earthengine-api" + ) + + +class GoogleEarthEngineAccessor(BaseAccessor): + """ + Accessor for Google Earth Engine (GEE) satellite imagery and derived indices. + + Provides reduced (scalar) indicator values -- NDVI, NDBI, NDWI -- over a + point or region, for use as features in epidemiological models (e.g. + vegetation/breeding-site proxies, urban density, settlement growth). + + Setup: + 1. Register at https://signup.earthengine.google.com/ + 2. Install earthengine-api: pip install earthengine-api + 3. Authenticate once: run `earthengine authenticate` in a terminal + 4. (Optional) set an EE_PROJECT env var to your GEE cloud project ID + + Example: + >>> from epidatasets.sources.google_earth_engine import GoogleEarthEngineAccessor + >>> gee = GoogleEarthEngineAccessor() + >>> + >>> # Mean NDVI near Rio de Janeiro over a date range + >>> ndvi = gee.get_ndvi( + ... lon=-43.2, lat=-22.9, buffer_m=10000, + ... start_date='2021-03-01', end_date='2021-03-31', + ... ) + >>> + >>> # Built-up index (urban density proxy) for the same area + >>> ndbi = gee.get_built_up_index( + ... lon=-43.2, lat=-22.9, buffer_m=10000, + ... start_date='2021-03-01', end_date='2021-03-31', + ... ) + """ + + source_name: ClassVar[str] = "google_earth_engine" + source_description: ClassVar[str] = ( + "Google Earth Engine satellite imagery (Landsat, Sentinel-2, MODIS) " + "and derived vegetation/built-up indices for epidemiological modeling" + ) + source_url: ClassVar[str] = "https://earthengine.google.com/" + + # Image collections used by this accessor + COLLECTIONS = { + "landsat8_sr": "LANDSAT/LC08/C02/T1_L2", + "sentinel2_sr": "COPERNICUS/S2_SR_HARMONIZED", + "modis_vi": "MODIS/061/MOD13Q1", + "countries": "USDOS/LSIB_SIMPLE/2017", + } + + # Landsat 8 Collection 2 L2 band names for red/NIR/SWIR + LANDSAT8_BANDS = {"red": "SR_B4", "nir": "SR_B5", "swir": "SR_B6", "green": "SR_B3"} + + def __init__( + self, + project: str | None = None, + cloud_cover_max: int = 20, + ): + """ + Initialize the Google Earth Engine accessor. + + Args: + project: GEE cloud project ID. Falls back to EE_PROJECT env var. + Required for accounts registered under the newer + project-based GEE access model. + cloud_cover_max: Max acceptable cloud cover percentage (0-100) + when filtering Landsat/Sentinel-2 image collections. + + Raises: + ImportError: If earthengine-api is not installed. + RuntimeError: If Earth Engine initialization fails (e.g. not + authenticated yet -- run `earthengine authenticate`). + """ + if not HAS_EE: + raise ImportError( + "earthengine-api is required. Install with: " + "pip install earthengine-api\n" + "Then run `earthengine authenticate` once in a terminal." + ) + + self.project = project or os.getenv("EE_PROJECT") + self.cloud_cover_max = cloud_cover_max + + try: + ee.Initialize(project=self.project) + logger.info( + "Earth Engine initialized successfully (project=%s)", self.project + ) + except Exception as e: + raise RuntimeError( + "Failed to initialize Earth Engine. Make sure you've run " + "`earthengine authenticate` and, if required, set EE_PROJECT " + f"to a valid GEE cloud project. Original error: {e}" + ) from e + + # ------------------------------------------------------------------ + # Required by BaseAccessor + # ------------------------------------------------------------------ + def list_countries(self) -> pd.DataFrame: + """Return countries available via GEE's simplified LSIB boundaries. + + Returns + ------- + pd.DataFrame + Columns ``country_code`` (FIPS 10-4, as provided by LSIB) and + ``country_name``. + """ + fc = ee.FeatureCollection(self.COLLECTIONS["countries"]) + names = fc.aggregate_array("country_na").getInfo() + codes = fc.aggregate_array("country_co").getInfo() + return pd.DataFrame({"country_code": codes, "country_name": names}) + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + def _aoi(self, lon: float, lat: float, buffer_m: float) -> ee.Geometry: + return ee.Geometry.Point([lon, lat]).buffer(buffer_m) # type: ignore[no-any-return] + + def _landsat_composite( + self, aoi: ee.Geometry, start_date: str, end_date: str + ) -> ee.Image: + """Cloud-filtered median composite from Landsat 8 SR over a date range.""" + collection = ( + ee.ImageCollection(self.COLLECTIONS["landsat8_sr"]) + .filterBounds(aoi) + .filterDate(start_date, end_date) + .filter(ee.Filter.lt("CLOUD_COVER", self.cloud_cover_max)) + ) + count = collection.size().getInfo() + if count == 0: + raise ValueError( + f"No Landsat 8 images found for this area/date range " + f"({start_date} to {end_date}) with cloud cover < {self.cloud_cover_max}%. " + f"Try widening the date range or raising cloud_cover_max." + ) + return collection.median() # type: ignore[no-any-return] + + def _reduce_mean( + self, image: ee.Image, band_name: str, aoi: ee.Geometry, scale: int + ) -> float | None: + result = ( + image.select(band_name) + .reduceRegion( + reducer=ee.Reducer.mean(), geometry=aoi, scale=scale, maxPixels=1e9 + ) + .get(band_name) + ) + value = result.getInfo() + return float(value) if value is not None else None + + # ------------------------------------------------------------------ + # Vegetation coverage / mosquito habitat proxy + # ------------------------------------------------------------------ + def get_ndvi( + self, + lon: float, + lat: float, + start_date: str, + end_date: str, + buffer_m: float = 10000, + scale: int = 30, + ) -> float | None: + """Mean NDVI over a buffered point -- vegetation coverage proxy + relevant to mosquito breeding habitat likelihood.""" + aoi = self._aoi(lon, lat, buffer_m) + image = self._landsat_composite(aoi, start_date, end_date) + ndvi = image.normalizedDifference( + [self.LANDSAT8_BANDS["nir"], self.LANDSAT8_BANDS["red"]] + ).rename("NDVI") + return self._reduce_mean(ndvi, "NDVI", aoi, scale) + + def get_ndwi( + self, + lon: float, + lat: float, + start_date: str, + end_date: str, + buffer_m: float = 10000, + scale: int = 30, + ) -> float | None: + """Mean NDWI over a buffered point -- standing-water proxy, + relevant to identifying potential mosquito breeding sites.""" + aoi = self._aoi(lon, lat, buffer_m) + image = self._landsat_composite(aoi, start_date, end_date) + ndwi = image.normalizedDifference( + [self.LANDSAT8_BANDS["green"], self.LANDSAT8_BANDS["nir"]] + ).rename("NDWI") + return self._reduce_mean(ndwi, "NDWI", aoi, scale) + + # ------------------------------------------------------------------ + # Urban density / informal settlement detection + # ------------------------------------------------------------------ + def get_built_up_index( + self, + lon: float, + lat: float, + start_date: str, + end_date: str, + buffer_m: float = 10000, + scale: int = 30, + ) -> float | None: + """Mean NDBI (built-up index) over a buffered point -- urban + density proxy.""" + aoi = self._aoi(lon, lat, buffer_m) + image = self._landsat_composite(aoi, start_date, end_date) + ndbi = image.normalizedDifference( + [self.LANDSAT8_BANDS["swir"], self.LANDSAT8_BANDS["nir"]] + ).rename("NDBI") + return self._reduce_mean(ndbi, "NDBI", aoi, scale) + + def get_built_up_change( + self, + lon: float, + lat: float, + start_date_before: str, + end_date_before: str, + start_date_after: str, + end_date_after: str, + buffer_m: float = 10000, + scale: int = 30, + ) -> float | None: + """Change in mean NDBI between two periods -- positive values + indicate new/growing built-up area, useful as an informal + settlement growth signal.""" + before = self.get_built_up_index( + lon, lat, start_date_before, end_date_before, buffer_m, scale + ) + after = self.get_built_up_index( + lon, lat, start_date_after, end_date_after, buffer_m, scale + ) + if before is None or after is None: + return None + return after - before diff --git a/tests/test_accessors.py b/tests/test_accessors.py index 2cdd923..ce4024d 100644 --- a/tests/test_accessors.py +++ b/tests/test_accessors.py @@ -888,6 +888,58 @@ def test_invalid_disease_raises(self, accessor): with pytest.raises(ValueError): accessor.get_disease_data("NonExistentDisease") +class TestGoogleEarthEngine: + @pytest.fixture + def accessor(self): + from epidatasets.sources.google_earth_engine import GoogleEarthEngineAccessor + return GoogleEarthEngineAccessor(project=os.getenv("EE_PROJECT")) + + def test_initialization(self, accessor): + assert accessor is not None + assert accessor.source_name == "google_earth_engine" + + @requires_external_api + def test_list_countries(self, accessor): + countries = accessor.list_countries() + assert isinstance(countries, pd.DataFrame) + assert len(countries) > 100 + assert "country_code" in countries.columns + assert "country_name" in countries.columns + + @requires_external_api + def test_get_ndvi_urban_area(self, accessor): + """Downtown São Paulo: expect NDVI in a valid, low-vegetation range.""" + ndvi = accessor.get_ndvi( + lon=-46.63, lat=-23.55, + start_date="2021-03-01", end_date="2021-03-31", + ) + assert ndvi is not None + assert -1.0 <= ndvi <= 1.0 + + @requires_external_api + def test_get_built_up_index_forest_vs_urban(self, accessor): + """NDBI should be lower (more negative) for dense forest than for a dense urban core.""" + forest_ndbi = accessor.get_built_up_index( + lon=-62.5, lat=-4.0, + start_date="2021-01-01", end_date="2021-12-31", + ) + urban_ndbi = accessor.get_built_up_index( + lon=-46.63, lat=-23.55, + start_date="2021-03-01", end_date="2021-03-31", + ) + assert forest_ndbi is not None + assert urban_ndbi is not None + assert forest_ndbi < urban_ndbi + + @requires_external_api + def test_no_imagery_raises_clear_error(self, accessor): + """A 1-day window over a persistently cloudy region should raise a + clear ValueError, not a cryptic band-name crash.""" + with pytest.raises(ValueError, match="No Landsat 8 images found"): + accessor.get_ndvi( + lon=-62.5, lat=-4.0, + start_date="2021-03-01", end_date="2021-03-02", + ) class TestSmoke: def test_package_import(self):