From c7f224f066d7d2eca3a67c3d937325b2df952acc Mon Sep 17 00:00:00 2001 From: dfguerrerom Date: Tue, 4 Aug 2026 11:34:59 +0200 Subject: [PATCH] cleanup: render class rasters through pysepal's add_raster Categorical rendering was never app-specific: the discrete LUT, the server-side registration that keeps per-class alpha, and the COG preparation now live in SepalMap.add_raster behind class_colors. SbaeMap just inherits it, so the local add_class_raster, _optimize_for_tiles, _build_class_colormap and scripts/tiling.py all go, and the inert LOCALTILESERVER_HOST knob goes with the TileClient call that carried it. classification_layer was assigned twice and read nowhere; dropped. Needs pysepal 3.9.0. --- component/scripts/logging_config.py | 2 +- component/scripts/tiling.py | 231 ---------------------------- component/tile/upload.py | 6 +- component/widget/analysis_tab.py | 10 +- component/widget/map.py | 143 +---------------- sepal_environment.yml | 3 +- tests/test_analysis_ui_widgets.py | 6 +- tests/test_map_colormap.py | 21 --- tests/test_tiling_tools.py | 68 -------- 9 files changed, 15 insertions(+), 475 deletions(-) delete mode 100644 component/scripts/tiling.py delete mode 100644 tests/test_map_colormap.py delete mode 100644 tests/test_tiling_tools.py diff --git a/component/scripts/logging_config.py b/component/scripts/logging_config.py index c66373b..1da510e 100644 --- a/component/scripts/logging_config.py +++ b/component/scripts/logging_config.py @@ -17,7 +17,7 @@ # rio-tiler emits ``NoOverviewWarning`` once per tile when a source raster has no # overviews. The raster-add paths now build overviews up front -# (``SbaeMap.add_class_raster`` -> ``prepare_for_tiles``), so this warning is +# (``SepalMap.add_raster`` -> ``prepare_for_tiles``), so this warning is # non-actionable noise here; silence it so any raster that still slips through a # fallback path cannot flood the logs with hundreds of identical lines. try: diff --git a/component/scripts/tiling.py b/component/scripts/tiling.py deleted file mode 100644 index 93c5cb6..0000000 --- a/component/scripts/tiling.py +++ /dev/null @@ -1,231 +0,0 @@ -# file: tiling_prepare.py -import hashlib -import os -import pathlib -import shutil -import subprocess -import sys - -import rasterio as rio - - -def _hash_for_cache(path: str) -> str: - st = os.stat(path) - h = hashlib.sha1() - h.update(path.encode()) - h.update(str(st.st_size).encode()) - h.update(str(int(st.st_mtime)).encode()) - return h.hexdigest()[:16] - - -def _find_tool(name: str) -> str | None: - """Absolute path to a GDAL CLI tool, or ``None`` when it isn't installed. - - A conda/micromamba env launched by absolute path (a Jupyter kernel, SEPAL) - keeps these binaries in ``bin/`` next to ``sys.executable`` while that - directory stays off PATH, so the bare-name lookup fails there and the COG - route is skipped for the slower rasterio fallback. - """ - found = shutil.which(name) - if found: - return found - sibling = pathlib.Path(sys.executable).parent / name - return str(sibling) if sibling.exists() else None - - -def _tool(name: str) -> str: - """Resolved tool path, falling back to the bare name so subprocess names it.""" - return _find_tool(name) or name - - -def _gdal_ok(): - return all(_find_tool(name) for name in ("gdalinfo", "gdal_translate", "gdaladdo")) - - -def _is_categorical(ds: rio.io.DatasetReader) -> bool: - # crude heuristic: integer dtype and few unique categories indicated by colormap or QL metadata - if ds.count != 1: - return False - if ds.dtypes[0].startswith(("int8", "uint8", "int16", "uint16", "int32", "uint32")): - return True - return False - - -def _has_overviews(ds): - return any(ds.overviews(i + 1) for i in range(ds.count)) - - -def _is_tiled(ds): - # block_shapes is None on some drivers; treat as not tiled - try: - bs = ds.block_shapes - return bs and all((b[0] > 1 and b[1] > 1) for b in bs) - except Exception: - return False - - -def _needs_reproject(ds, target_epsg: int | None): - if not target_epsg or not ds.crs: - return False - try: - return ds.crs.to_epsg() != target_epsg - except Exception: - return True - - -def _target_overview_levels(width, height, block=512): - # pyramid down to roughly block size - levels = [] - longest = max(width, height) - lvl = 2 - while longest / lvl > block: - levels.append(lvl) - lvl *= 2 - return levels or [2, 4, 8, 16] - - -def analyze_tif(path: str) -> dict: - with rio.open(path) as ds: - return { - "path": path, - "crs": str(ds.crs), - "epsg": (ds.crs.to_epsg() if ds.crs else None), - "width": ds.width, - "height": ds.height, - "bands": ds.count, - "dtype": ds.dtypes[0], - "tiled": _is_tiled(ds), - "overviews": [ds.overviews(i + 1) for i in range(ds.count)], - "categorical_guess": _is_categorical(ds), - } - - -def _build_overviews_inplace(path: str, categorical: bool): - resamp = "NEAREST" if categorical else "AVERAGE" - with rio.open(path) as ds: - levels = _target_overview_levels(ds.width, ds.height) - # Rasterio can build in place too: - try: - with rio.open(path, "r+") as ds: - ds.build_overviews(levels, resampling=resamp.lower()) - ds.update_tags(ns="rio_overview", resampling=resamp.lower()) - except Exception: - # fallback to gdaladdo if rasterio fails - gdaladdo = _find_tool("gdaladdo") - if not gdaladdo: - raise - cmd = [ - gdaladdo, - "-r", - resamp, - "--config", - "COMPRESS_OVERVIEW", - "DEFLATE", - "--config", - "PREDICTOR_OVERVIEW", - "2", - path, - *map(str, levels), - ] - subprocess.run(cmd, check=True) - - -def _translate_to_cog(src: str, dst: str, resampling: str, block=512): - cmd = [ - _tool("gdal_translate"), - src, - dst, - "-of", - "COG", - "-co", - "COMPRESS=DEFLATE", - "-co", - "LEVEL=6", - "-co", - "PREDICTOR=2", - "-co", - f"BLOCKSIZE={block}", - "-co", - "NUM_THREADS=ALL_CPUS", - "-co", - f"RESAMPLING={resampling}", - ] - subprocess.run(cmd, check=True) - - -def _warp_to_epsg(src: str, dst: str, epsg: int, resampling: str, block=512): - cmd = [ - _tool("gdalwarp"), - "-overwrite", - "-t_srs", - f"EPSG:{epsg}", - "-r", - resampling, - "-multi", - "-wo", - "NUM_THREADS=ALL_CPUS", - "-co", - "TILED=YES", - "-co", - f"BLOCKXSIZE={block}", - "-co", - f"BLOCKYSIZE={block}", - "-co", - "COMPRESS=DEFLATE", - "-co", - "PREDICTOR=2", - "-co", - "BIGTIFF=IF_SAFER", - src, - dst, - ] - subprocess.run(cmd, check=True, capture_output=True, text=True) - - -def prepare_for_tiles( - path: str, - cache_dir: str | None = None, - warp_to_3857: bool = False, - force: bool = False, -) -> dict: - """Returns dict: {"path": optimized_path, "report": analysis_dict}.""" - path = os.path.abspath(path) - rep = analyze_tif(path) - categorical = rep["categorical_guess"] - resamp = "NEAREST" if categorical else "AVERAGE" - # Open once (context-managed) instead of leaking a handle per rio.open call. - with rio.open(path) as _ds: - need_reproj = _needs_reproject(_ds, 3857) if warp_to_3857 else False - good_enough = rep["tiled"] and _has_overviews(_ds) and not need_reproj - - if good_enough and not force: - return {"path": path, "report": rep} - - cache_dir = cache_dir or os.path.join(pathlib.Path.home(), ".cache", "localtiles") - os.makedirs(cache_dir, exist_ok=True) - tag = _hash_for_cache(path) - tmp_base = os.path.join(cache_dir, f"{os.path.basename(path)}.{tag}") - - if _gdal_ok(): - # Prefer building a clean COG (and reprojection if requested) - out = tmp_base + (".3857.cog.tif" if warp_to_3857 else ".cog.tif") - if warp_to_3857: - # first warp to an intermediate tiled TIFF, then translate to COG - inter = tmp_base + ".warp.tif" - _warp_to_epsg(path, inter, 3857, resamp) - _translate_to_cog(inter, out, resamp) - try: - os.remove(inter) - except (OSError, PermissionError): - pass - else: - _translate_to_cog(path, out, resamp) - final_rep = analyze_tif(out) - return {"path": out, "report": final_rep} - else: - # No GDAL CLI: do the minimum—build overviews in place or copy to temp and add overviews - dst = tmp_base + ".ovr.tif" - shutil.copy2(path, dst) - _build_overviews_inplace(dst, categorical) - final_rep = analyze_tif(dst) - return {"path": dst, "report": final_rep} diff --git a/component/tile/upload.py b/component/tile/upload.py index ce057eb..ff2297c 100644 --- a/component/tile/upload.py +++ b/component/tile/upload.py @@ -4,6 +4,7 @@ from typing import Any, Dict import solara +from pysepal.mapping import prepare_for_tiles from sepal_ui.sepalwidgets.file_input import FileInputComponent from sepal_ui.solara.notifications import use_notifications @@ -14,7 +15,6 @@ get_file_info, is_raster_file, ) -from component.scripts.tiling import prepare_for_tiles from component.widget.map import SbaeMap logger = logging.getLogger("sbae.upload") @@ -34,11 +34,11 @@ def add_optimized_raster_to_map(): and status == "adding_to_map" and sampling_method == "stratified" ): - sbae_map.add_class_raster( + sbae_map.add_raster( optimized_path, - app_state.class_colors.value, layer_name="Classification Map", key="clas", + class_colors=app_state.class_colors.value, ) app_state.raster_optimization_status.value = "finished" diff --git a/component/widget/analysis_tab.py b/component/widget/analysis_tab.py index c04e7e2..0761c41 100644 --- a/component/widget/analysis_tab.py +++ b/component/widget/analysis_tab.py @@ -164,7 +164,7 @@ def derive_map_source(state, sbae_map=None): state.analysis_reference_df.value = ref_out # Standalone mode never runs the design-step upload that populates - # class_colors, so it's empty here -- without this, add_class_raster falls + # class_colors, so it's empty here -- without this, add_raster falls # back to a continuous colormap and the map renders near-black. Derive it # from the raster; guarded so a real design-step palette is kept. if not state.class_colors.value: @@ -176,11 +176,11 @@ def derive_map_source(state, sbae_map=None): if sbae_map is not None: try: - sbae_map.add_class_raster( + sbae_map.add_raster( raster, - state.class_colors.value or {}, - "Classification (analysis)", - "clas_an", + layer_name="Classification (analysis)", + key="clas_an", + class_colors=state.class_colors.value or {}, ) # Reference points are drawn by the panel's render thread (from # analysis_reference_df, for every source) on their own layer. diff --git a/component/widget/map.py b/component/widget/map.py index f2e552a..acc88fa 100644 --- a/component/widget/map.py +++ b/component/widget/map.py @@ -1,15 +1,12 @@ import logging -import os import shutil import pandas as pd import solara -from localtileserver import TileClient, get_leaflet_tile_layer from pysepal.scripts.scratch import scratch_dir from sepal_ui.mapping import SepalMap from sepal_ui.sepalwidgets.vue_app import ThemeToggle -from component.scripts.logging_config import quiet_tile_server_logs from component.scripts.vector_tiles import ( CORRECT_COLOR, INCORRECT_COLOR, @@ -29,12 +26,6 @@ _REFERENCE_LEGEND_LABEL = "Reference point" -def _hex_to_rgb(hex_color: str) -> tuple: - """Convert '#rrggbb' to an (r, g, b) tuple.""" - h = hex_color.lstrip("#") - return tuple(int(h[i : i + 2], 16) for i in (0, 2, 4)) - - def _points_signature(df): """Cheap content signature of a points DataFrame (``None`` when empty). @@ -71,20 +62,6 @@ def _compose_points_legend( return legend -def _build_class_colormap(class_colors: dict) -> dict: - """Build a discrete {pixel_value: (r, g, b, a)} LUT for a categorical raster. - - Every class present in ``class_colors`` is rendered opaque, including code 0 - and codes above 255 -- area calculation treats those as valid, sampleable - classes, so they must be visible on the map. Values not in ``class_colors`` - render transparent (background). - """ - colormap = {i: (0, 0, 0, 0) for i in range(256)} - for code, hex_color in class_colors.items(): - colormap[int(code)] = (*_hex_to_rgb(hex_color), 255) - return colormap - - class SbaeMap(SepalMap): """SBAE Map class extending SepalMap for map visualization and interactions.""" @@ -93,7 +70,6 @@ def __init__(self, theme_toggle: ThemeToggle, gee: bool = False, min_zoom: int = fullscreen=True, theme_toggle=theme_toggle, gee=gee, min_zoom=min_zoom ) - self.classification_layer = None self.sample_points_layer = None self.sample_points_dir = None self._pending_points_dir = None @@ -109,123 +85,6 @@ def __init__(self, theme_toggle: ThemeToggle, gee: bool = False, min_zoom: int = # re-entry remounts the render task). self._reference_points_sig = None - def _optimize_for_tiles(self, path) -> str: - """Return a tiling-optimized (cached COG with overviews) path. - - rio-tiler reads full-resolution pixels -- and warns ``NoOverviewWarning`` - -- for every tile when the source has no overviews, so low-zoom tiles are - slow. ``prepare_for_tiles`` is a fast no-op when ``path`` is already a - tiled COG with overviews (the design step pre-optimizes off-thread); it - only does real work for raw rasters such as the analysis classification - map, which are added off the UI thread. Best-effort: on failure, serve the - raw raster (tiling still works, just slower). - """ - from component.scripts.tiling import prepare_for_tiles - - try: - return prepare_for_tiles(str(path))["path"] - except Exception as e: - logger.warning( - "Tiling optimization failed for %s (%s); serving the raw raster.", - path, - e, - ) - return str(path) - - def add_class_raster( - self, - path, - class_colors, - layer_name: str = "Classification Map", - key: str = "clas", - opacity: float = 1.0, - fit_bounds: bool = True, - ): - """Add a categorical raster with exact per-class colors. - - Unlike ``add_raster`` (which applies a continuous inferno colormap - stretched across the value range and renders sparse/low-value class - maps as black), this builds a discrete lookup table so each class - value gets its own color. Any value not in ``class_colors`` is rendered - transparent (background); classes 0 and > 255 are colored like any other. - - Args: - path: path to the (optimized) raster file. - class_colors: mapping of class code -> '#rrggbb' hex color. - layer_name: display name of the layer. - key: unequivocal key of the layer (for later removal). - opacity: layer opacity, default 1.0. - fit_bounds: whether to recenter/zoom onto the raster. - """ - # Build (or reuse a cached) COG with overviews before tiling: rio-tiler - # otherwise reads full-res pixels and logs NoOverviewWarning per tile. - tile_path = self._optimize_for_tiles(path) - - if not class_colors: - logger.warning( - "add_class_raster called without class_colors; " - "falling back to add_raster for %s", - path, - ) - return self.add_raster( - tile_path, - layer_name=layer_name, - key=key, - opacity=opacity, - fit_bounds=fit_bounds, - ) - - # Discrete LUT: transparent everywhere unless it's a known class. Every - # class in class_colors is colored, including code 0 and codes > 255. - colormap = _build_class_colormap(class_colors) - - # localtileserver won't accept a raw {value: rgba} dict as `colormap`; - # it must be registered server-side first, which yields a "custom:" - # key. This is the only path that preserves per-class alpha (the - # matplotlib-Colormap path forces alpha=1, losing transparency). - try: - from localtileserver.tiler.palettes import register_colormap - - colormap_arg = register_colormap(colormap) - except Exception: - logger.warning( - "localtileserver register_colormap unavailable; falling back " - "to add_raster (classes may render dark) for %s", - path, - ) - return self.add_raster( - tile_path, - layer_name=layer_name, - key=key, - opacity=opacity, - fit_bounds=fit_bounds, - ) - - # Bind the tile server to a reachable interface when serving the app - # over the network (e.g. Solara --host over Tailscale). Defaults to - # loopback for local dev; set LOCALTILESERVER_HOST=0.0.0.0 (or the - # tailnet IP) plus LOCALTILESERVER_CLIENT_HOST for remote access. - client = TileClient( - tile_path, host=os.environ.get("LOCALTILESERVER_HOST", "127.0.0.1") - ) - quiet_tile_server_logs() - layer = get_leaflet_tile_layer( - client, - colormap=colormap_arg, - name=layer_name, - opacity=opacity, - max_zoom=20, - ) - self.add_layer(layer, key=key) - self.classification_layer = layer - layer.raster = str(path) - - if fit_bounds: - self.center = client.center() - self.zoom = client.default_zoom - - return layer - async def build_sample_points_layer( self, points_data, @@ -289,7 +148,7 @@ def attach_sample_points_layer(self, layer): Mutates the map, so the main thread is preferred. ``add_sample_points`` deliberately calls this off the UI thread anyway, matching the - pre-existing off-thread ``add_class_raster`` mutation in + pre-existing off-thread ``add_raster`` mutation in ``analysis_tab.py``. """ old_dir = getattr(self, "sample_points_dir", None) diff --git a/sepal_environment.yml b/sepal_environment.yml index 9a0f44e..cca0dd2 100644 --- a/sepal_environment.yml +++ b/sepal_environment.yml @@ -24,7 +24,8 @@ dependencies: # tippecanoe builds the PMTiles vector tiles for the map sample-points layer. - tippecanoe>=2 - pip: - - pysepal>=3.8.1 + # 3.9.0 is the first release with add_raster(class_colors=...) + - pysepal>=3.9.0 - git+https://github.com/openforis/earthengine-api.git@v1.6.14#egg=earthengine-api&subdirectory=python - git+https://github.com/SerafiniJose/jupyter-loopback.git@fix/voila-localhost-proxy-probe - ipecharts>=1.0.0 diff --git a/tests/test_analysis_ui_widgets.py b/tests/test_analysis_ui_widgets.py index 9c37034..9df7e4a 100644 --- a/tests/test_analysis_ui_widgets.py +++ b/tests/test_analysis_ui_widgets.py @@ -246,17 +246,17 @@ def test_analysis_panel_accepts_sbae_map(): class _FakeSbaeMap: - """Records add_class_raster/add_sample_points calls instead of a real map.""" + """Records add_raster/add_sample_points calls instead of a real map.""" def __init__(self): self.class_raster_calls = [] self.sample_points_calls = [] self.reference_points_calls = [] - def add_class_raster(self, path, class_colors, layer_name, key): + def add_raster(self, image, layer_name=None, key=None, class_colors=None, **kwargs): self.class_raster_calls.append( { - "path": path, + "path": image, "class_colors": class_colors, "layer_name": layer_name, "key": key, diff --git a/tests/test_map_colormap.py b/tests/test_map_colormap.py deleted file mode 100644 index 74961cd..0000000 --- a/tests/test_map_colormap.py +++ /dev/null @@ -1,21 +0,0 @@ -"""The categorical colormap must show every sampleable class (incl. 0 and >255).""" - -from component.widget.map import _build_class_colormap - - -def test_colormap_colors_class_zero(): - cm = _build_class_colormap({0: "#ff0000", 5: "#00ff00"}) - assert cm[0] == (255, 0, 0, 255) # class 0 is a real class, not background - assert cm[5] == (0, 255, 0, 255) - - -def test_colormap_colors_codes_above_255(): - cm = _build_class_colormap({300: "#0000ff", 1024: "#ffffff"}) - assert cm[300] == (0, 0, 255, 255) - assert cm[1024] == (255, 255, 255, 255) - - -def test_colormap_leaves_unknown_values_transparent(): - cm = _build_class_colormap({5: "#00ff00"}) - assert cm[7] == (0, 0, 0, 0) - assert cm[5] == (0, 255, 0, 255) diff --git a/tests/test_tiling_tools.py b/tests/test_tiling_tools.py deleted file mode 100644 index 95c6b60..0000000 --- a/tests/test_tiling_tools.py +++ /dev/null @@ -1,68 +0,0 @@ -"""GDAL CLI lookup must reach the venv's bin/ even when it is off PATH. - -SEPAL launches the app from a micromamba venv by absolute path, so the binaries -sitting next to ``sys.executable`` are invisible to a bare-name lookup and the -COG route would be skipped for the slower rasterio fallback. -""" - -import sys - -from component.scripts.tiling import _find_tool, _gdal_ok, _tool - - -def test_find_tool_prefers_path(monkeypatch, tmp_path): - on_path = tmp_path / "usr" / "bin" - on_path.mkdir(parents=True) - (on_path / "gdalinfo").touch(mode=0o755) - monkeypatch.setenv("PATH", str(on_path)) - - assert _find_tool("gdalinfo") == str(on_path / "gdalinfo") - - -def test_find_tool_falls_back_to_interpreter_sibling(monkeypatch, tmp_path): - bindir = tmp_path / "bin" - bindir.mkdir() - (bindir / "gdal_translate").touch(mode=0o755) - monkeypatch.setattr(sys, "executable", str(bindir / "python3")) - monkeypatch.setenv("PATH", "/nonexistent") - - assert _find_tool("gdal_translate") == str(bindir / "gdal_translate") - - -def test_find_tool_returns_none_when_missing(monkeypatch, tmp_path): - bindir = tmp_path / "bin" - bindir.mkdir() # no sibling binary - monkeypatch.setattr(sys, "executable", str(bindir / "python3")) - monkeypatch.setenv("PATH", "/nonexistent") - - assert _find_tool("gdaladdo") is None - - -def test_tool_falls_back_to_the_bare_name(monkeypatch, tmp_path): - # subprocess then raises FileNotFoundError naming the tool, as before. - monkeypatch.setattr(sys, "executable", str(tmp_path / "python3")) - monkeypatch.setenv("PATH", "/nonexistent") - - assert _tool("gdalwarp") == "gdalwarp" - - -def test_gdal_ok_is_false_when_a_tool_is_missing(monkeypatch, tmp_path): - bindir = tmp_path / "bin" - bindir.mkdir() - (bindir / "gdalinfo").touch(mode=0o755) # only one of the three - monkeypatch.setattr(sys, "executable", str(bindir / "python3")) - monkeypatch.setenv("PATH", "/nonexistent") - - assert _gdal_ok() is False - - -def test_gdal_ok_is_true_from_the_interpreter_sibling(monkeypatch, tmp_path): - bindir = tmp_path / "bin" - bindir.mkdir() - for name in ("gdalinfo", "gdal_translate", "gdaladdo"): - (bindir / name).touch(mode=0o755) - monkeypatch.setattr(sys, "executable", str(bindir / "python3")) - monkeypatch.setenv("PATH", "/nonexistent") - - # the SEPAL case: nothing on PATH, everything next to the interpreter - assert _gdal_ok() is True