From e41bfa4beb253f8fb2d8f8f98f9b17cb481e36d8 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Tue, 17 Dec 2024 18:48:53 +0100 Subject: [PATCH] Issue #346 Some more ProcessArgs porting for less boilerplate code and better/earlier error messages --- openeo_driver/ProcessGraphDeserializer.py | 234 ++++++++-------------- tests/test_views_execute.py | 2 +- 2 files changed, 89 insertions(+), 147 deletions(-) diff --git a/openeo_driver/ProcessGraphDeserializer.py b/openeo_driver/ProcessGraphDeserializer.py index 0109115b..6f08a28c 100644 --- a/openeo_driver/ProcessGraphDeserializer.py +++ b/openeo_driver/ProcessGraphDeserializer.py @@ -13,7 +13,7 @@ import time import warnings from pathlib import Path -from typing import Any, Callable, Dict, Iterable, List, Tuple, Union, Sequence +from typing import Any, Callable, Dict, Iterable, List, Tuple, Union, Sequence, Optional import geopandas as gpd import numpy as np @@ -936,22 +936,18 @@ def save_result(args: Dict, env: EvalEnv) -> SaveResult: # TODO: return type no @process_registry_100.add_function(spec=read_spec("openeo-processes/experimental/save_ml_model.json")) @process_registry_2xx.add_function(spec=read_spec("openeo-processes/experimental/save_ml_model.json")) -def save_ml_model(args: dict, env: EvalEnv) -> MlModelResult: - data: DriverMlModel = extract_arg(args, "data", process_id="save_ml_model") - if not isinstance(data, DriverMlModel): - raise ProcessParameterInvalidException( - parameter="data", process="save_ml_model", reason=f"Invalid data type {type(data)!r} expected raster-cube." - ) - options = args.get("options", {}) +def save_ml_model(args: ProcessArgs, env: EvalEnv) -> MlModelResult: + data = args.get_required("data", expected_type=DriverMlModel) + options = args.get_optional("options", default={}, expected_type=dict) return MlModelResult(ml_model=data, options=options) @process_registry_100.add_function(spec=read_spec("openeo-processes/experimental/load_ml_model.json")) @process_registry_2xx.add_function(spec=read_spec("openeo-processes/experimental/load_ml_model.json")) -def load_ml_model(args: dict, env: EvalEnv) -> DriverMlModel: +def load_ml_model(args: ProcessArgs, env: EvalEnv) -> DriverMlModel: if env.get(ENV_DRY_RUN_TRACER): return DriverMlModel() - job_id = extract_arg(args, "id") + job_id = args.get_required("id", expected_type=str) return env.backend_implementation.load_ml_model(job_id) @@ -1187,51 +1183,34 @@ def add_dimension(args: ProcessArgs, env: EvalEnv) -> DriverDataCube: @process -def drop_dimension(args: dict, env: EvalEnv) -> DriverDataCube: - data_cube = extract_arg(args, 'data') - if not isinstance(data_cube, DriverDataCube): - raise ProcessParameterInvalidException( - parameter="data", process="drop_dimension", - reason=f"Invalid data type {type(data_cube)!r} expected raster-cube." - ) - return data_cube.drop_dimension(name=extract_arg(args, 'name')) +def drop_dimension(args: ProcessArgs, env: EvalEnv) -> DriverDataCube: + cube: DriverDataCube = args.get_required("data", expected_type=DriverDataCube) + name: str = args.get_required("name", expected_type=str) + return cube.drop_dimension(name=name) @process -def dimension_labels(args: dict, env: EvalEnv) -> DriverDataCube: - data_cube = extract_arg(args, 'data') - if not isinstance(data_cube, DriverDataCube): - raise ProcessParameterInvalidException( - parameter="data", process="dimension_labels", - reason=f"Invalid data type {type(data_cube)!r} expected raster-cube." - ) - return data_cube.dimension_labels(dimension=extract_arg(args, 'dimension')) +def dimension_labels(args: ProcessArgs, env: EvalEnv) -> List[str]: + cube: DriverDataCube = args.get_required("data", expected_type=DriverDataCube) + dimension: str = args.get_required("dimension", expected_type=str) + return cube.dimension_labels(dimension=dimension) @process -def rename_dimension(args: dict, env: EvalEnv) -> DriverDataCube: - data_cube = extract_arg(args, 'data') - if not isinstance(data_cube, DriverDataCube): - raise ProcessParameterInvalidException( - parameter="data", process="rename_dimension", - reason=f"Invalid data type {type(data_cube)!r} expected raster-cube." - ) - return data_cube.rename_dimension(source=extract_arg(args, 'source'),target=extract_arg(args, 'target')) +def rename_dimension(args: ProcessArgs, env: EvalEnv) -> DriverDataCube: + cube: DriverDataCube = args.get_required("data", expected_type=DriverDataCube) + source: str = args.get_required("source", expected_type=str) + target: str = args.get_required("target", expected_type=str) + return cube.rename_dimension(source=source, target=target) @process -def rename_labels(args: dict, env: EvalEnv) -> DriverDataCube: - data_cube = extract_arg(args, 'data') - if not isinstance(data_cube, DriverDataCube): - raise ProcessParameterInvalidException( - parameter="data", process="rename_labels", - reason=f"Invalid data type {type(data_cube)!r} expected raster-cube." - ) - return data_cube.rename_labels( - dimension=extract_arg(args, 'dimension'), - target=extract_arg(args, 'target'), - source=args.get('source',[]) - ) +def rename_labels(args: ProcessArgs, env: EvalEnv) -> DriverDataCube: + cube: DriverDataCube = args.get_required("data", expected_type=DriverDataCube) + dimension: str = args.get_required("dimension", expected_type=str) + target: List = args.get_required("target", expected_type=list) + source: Optional[list] = args.get_optional("source", default=None, expected_type=list) + return cube.rename_labels(dimension=dimension, target=target, source=source) @process @@ -1377,14 +1356,10 @@ def aggregate_spatial(args: ProcessArgs, env: EvalEnv) -> DriverDataCube: @process -def mask(args: dict, env: EvalEnv) -> DriverDataCube: - cube = extract_arg(args, 'data') - if not isinstance(cube, DriverDataCube): - raise ProcessParameterInvalidException( - parameter="data", process="mask", reason=f"Invalid data type {type(cube)!r} expected raster-cube." - ) - mask = extract_arg(args, 'mask') - replacement = args.get('replacement', None) +def mask(args: ProcessArgs, env: EvalEnv) -> DriverDataCube: + cube: DriverDataCube = args.get_required("data", expected_type=DriverDataCube) + mask: DriverDataCube = args.get_required("mask", expected_type=DriverDataCube) + replacement = args.get_optional("replacement", default=None) return cube.mask(mask=mask, replacement=replacement) @@ -1416,7 +1391,10 @@ def mask_polygon(args: dict, env: EvalEnv) -> DriverDataCube: return image_collection -def _extract_temporal_extent(args: dict, field="extent", process_id="filter_temporal") -> Tuple[str, str]: +def _extract_temporal_extent( + args: Union[dict, ProcessArgs], field="extent", process_id="filter_temporal" +) -> Tuple[str, str]: + # TODO #346: make this a ProcessArgs method? extent = extract_arg(args, name=field, process_id=process_id) if len(extent) != 2: raise ProcessParameterInvalidException( @@ -1441,29 +1419,27 @@ def _extract_temporal_extent(args: dict, field="extent", process_id="filter_temp @process -def filter_temporal(args: dict, env: EvalEnv) -> DriverDataCube: - cube = extract_arg(args, 'data') - if not isinstance(cube, DriverDataCube): - raise ProcessParameterInvalidException( - parameter="data", process="filter_temporal", - reason=f"Invalid data type {type(cube)!r} expected raster-cube." - ) +def filter_temporal(args: ProcessArgs, env: EvalEnv) -> DriverDataCube: + cube: DriverDataCube = args.get_required("data", expected_type=DriverDataCube) extent = _extract_temporal_extent(args, field="extent", process_id="filter_temporal") return cube.filter_temporal(start=extent[0], end=extent[1]) + @process_registry_100.add_function(spec=read_spec("openeo-processes/1.x/proposals/filter_labels.json")) @process_registry_2xx.add_function(spec=read_spec("openeo-processes/2.x/proposals/filter_labels.json")) -def filter_labels(args: dict, env: EvalEnv) -> DriverDataCube: - cube = extract_arg(args, 'data') - if not isinstance(cube, DriverDataCube): - raise ProcessParameterInvalidException( - parameter="data", process="filter_labels", - reason=f"Invalid data type {type(cube)!r} expected cube." - ) +def filter_labels(args: ProcessArgs, env: EvalEnv) -> DriverDataCube: + cube: DriverDataCube = args.get_required("data", expected_type=DriverDataCube) + # TODO: validation that condition is a process graph construct + condition = args.get_required("condition", expected_type=dict) + dimension = args.get_required("dimension", expected_type=str) + context = args.get_optional("context", default=None) + return cube.filter_labels(condition=condition, dimension=dimension, context=context, env=env) - return cube.filter_labels(condition=extract_arg(args,"condition"),dimension=extract_arg(args,"dimension"),context=args.get("context",None),env=env) -def _extract_bbox_extent(args: dict, field="extent", process_id="filter_bbox", handle_geojson=False) -> dict: +def _extract_bbox_extent( + args: Union[dict, ProcessArgs], field="extent", process_id="filter_bbox", handle_geojson=False +) -> dict: + # TODO #346: make this a ProcessArgs method? extent = extract_arg(args, name=field, process_id=process_id) if handle_geojson and extent.get("type") in [ "Polygon", @@ -1488,24 +1464,16 @@ def _extract_bbox_extent(args: dict, field="extent", process_id="filter_bbox", h @process -def filter_bbox(args: Dict, env: EvalEnv) -> DriverDataCube: - cube = extract_arg(args, 'data') - if not isinstance(cube, DriverDataCube): - raise ProcessParameterInvalidException( - parameter="data", process="filter_bbox", reason=f"Invalid data type {type(cube)!r} expected raster-cube." - ) +def filter_bbox(args: ProcessArgs, env: EvalEnv) -> DriverDataCube: + cube: DriverDataCube = args.get_required("data", expected_type=DriverDataCube) spatial_extent = _extract_bbox_extent(args, "extent", process_id="filter_bbox") return cube.filter_bbox(**spatial_extent) @process -def filter_spatial(args: Dict, env: EvalEnv) -> DriverDataCube: - cube = extract_arg(args, 'data') - geometries = extract_arg(args, 'geometries') - if not isinstance(cube, DriverDataCube): - raise ProcessParameterInvalidException( - parameter="data", process="filter_spatial", reason=f"Invalid data type {type(cube)!r} expected raster-cube." - ) +def filter_spatial(args: ProcessArgs, env: EvalEnv) -> DriverDataCube: + cube: DriverDataCube = args.get_required("data", expected_type=DriverDataCube) + geometries = args.get_required("geometries") if isinstance(geometries, dict): if "type" in geometries and geometries["type"] != "GeometryCollection": @@ -1534,32 +1502,22 @@ def filter_spatial(args: Dict, env: EvalEnv) -> DriverDataCube: @process -def filter_bands(args: Dict, env: EvalEnv) -> Union[DriverDataCube, DriverVectorCube]: - cube: Union[DriverDataCube, DriverVectorCube] = extract_arg(args, "data") - if not isinstance(cube, DriverDataCube) and not isinstance(cube, DriverVectorCube): - raise ProcessParameterInvalidException( - parameter="data", process="filter_bands", reason=f"Invalid data type {type(cube)!r} expected raster-cube." - ) - bands = extract_arg(args, "bands", process_id="filter_bands") +def filter_bands(args: ProcessArgs, env: EvalEnv) -> Union[DriverDataCube, DriverVectorCube]: + cube: Union[DriverDataCube, DriverVectorCube] = args.get_required( + "data", expected_type=(DriverDataCube, DriverVectorCube) + ) + bands = args.get_required("bands", expected_type=list) return cube.filter_bands(bands=bands) @process -def apply_kernel(args: Dict, env: EvalEnv) -> DriverDataCube: - image_collection = extract_arg(args, 'data') - kernel = np.asarray(extract_arg(args, 'kernel')) - factor = args.get('factor', 1.0) - border = args.get('border', 0) - if not isinstance(image_collection, DriverDataCube): - raise ProcessParameterInvalidException( - parameter="data", process="apply_kernel", - reason=f"Invalid data type {type(image_collection)!r} expected raster-cube." - ) - if border == "0": - # R-client sends `0` border as a string - border = 0 - replace_invalid = args.get('replace_invalid', 0) - return image_collection.apply_kernel(kernel=kernel, factor=factor, border=border, replace_invalid=replace_invalid) +def apply_kernel(args: ProcessArgs, env: EvalEnv) -> DriverDataCube: + cube: DriverDataCube = args.get_required("data", expected_type=DriverDataCube) + kernel = np.asarray(args.get_required("kernel", expected_type=list)) + factor = args.get_optional("factor", default=1.0, expected_type=(int, float)) + border = args.get_optional("border", default=0, expected_type=int) + replace_invalid = args.get_optional("replace_invalid", default=0, expected_type=(int, float)) + return cube.apply_kernel(kernel=kernel, factor=factor, border=border, replace_invalid=replace_invalid) @process @@ -1698,20 +1656,17 @@ def run_udf(args: dict, env: EvalEnv): @process -def linear_scale_range(args: dict, env: EvalEnv) -> DriverDataCube: - image_collection = extract_arg(args, 'x') - - inputMin = extract_arg(args, "inputMin") - inputMax = extract_arg(args, "inputMax") - outputMax = args.get("outputMax", 1.0) - outputMin = args.get("outputMin", 0.0) - if not isinstance(image_collection, DriverDataCube): - raise ProcessParameterInvalidException( - parameter="data", process="linear_scale_range", - reason=f"Invalid data type {type(image_collection)!r} expected raster-cube." - ) - - return image_collection.linear_scale_range(inputMin, inputMax, outputMin, outputMax) +def linear_scale_range(args: ProcessArgs, env: EvalEnv) -> DriverDataCube: + # TODO: eliminate this top-level linear_scale_range process implementation (should be used as `apply` callback) + _log.warning("DEPRECATED: linear_scale_range usage directly on cube is deprecated/non-standard.") + cube: DriverDataCube = args.get_required("x", expected_type=DriverDataCube) + # Note: non-standard camelCase parameter names (https://github.com/Open-EO/openeo-processes/issues/302) + input_min = args.get_required("inputMin") + input_max = args.get_required("inputMax") + output_min = args.get_optional("outputMin", default=0.0) + output_max = args.get_optional("outputMax", default=1.0) + # TODO linear_scale_range is defined on GeopysparkDataCube, but not on DriverDataCube + return cube.linear_scale_range(input_min, input_max, output_min, output_max) @process @@ -1991,14 +1946,10 @@ def get_geometries(args: Dict, env: EvalEnv) -> Union[DelayedVector, dict]: .param('data', description="A raster data cube.", schema={"type": "object", "subtype": "raster-cube"}) .returns("vector-cube", schema={"type": "object", "subtype": "vector-cube"}) ) -def raster_to_vector(args: Dict, env: EvalEnv): - image_collection = extract_arg(args, 'data') - if not isinstance(image_collection, DriverDataCube): - raise ProcessParameterInvalidException( - parameter="data", process="raster_to_vector", - reason=f"Invalid data type {type(image_collection)!r} expected raster-cube." - ) - return image_collection.raster_to_vector() +def raster_to_vector(args: ProcessArgs, env: EvalEnv): + cube: DriverDataCube = args.get_required("data", expected_type=DriverDataCube) + # TODO: raster_to_vector is only defined on GeopysparkDataCube, not DriverDataCube + return cube.raster_to_vector() @non_standard_process( @@ -2238,13 +2189,8 @@ def discard_result(args: ProcessArgs, env: EvalEnv): @process_registry_100.add_function(spec=read_spec("openeo-processes/experimental/mask_scl_dilation.json")) @process_registry_2xx.add_function(spec=read_spec("openeo-processes/experimental/mask_scl_dilation.json")) -def mask_scl_dilation(args: Dict, env: EvalEnv): - cube: DriverDataCube = extract_arg(args, 'data') - if not isinstance(cube, DriverDataCube): - raise ProcessParameterInvalidException( - parameter="data", process="mask_scl_dilation", - reason=f"Invalid data type {type(cube)!r} expected raster-cube." - ) +def mask_scl_dilation(args: ProcessArgs, env: EvalEnv): + cube: DriverDataCube = args.get_required("data", expected_type=DriverDataCube) if hasattr(cube, "mask_scl_dilation"): the_args = args.copy() del the_args["data"] @@ -2275,13 +2221,8 @@ def to_scl_dilation_mask(args: ProcessArgs, env: EvalEnv): @process_registry_100.add_function(spec=read_spec("openeo-processes/experimental/mask_l1c.json")) @process_registry_2xx.add_function(spec=read_spec("openeo-processes/experimental/mask_l1c.json")) -def mask_l1c(args: Dict, env: EvalEnv): - cube: DriverDataCube = extract_arg(args, 'data') - if not isinstance(cube, DriverDataCube): - raise ProcessParameterInvalidException( - parameter="data", process="mask_l1c", - reason=f"Invalid data type {type(cube)!r} expected raster-cube." - ) +def mask_l1c(args: ProcessArgs, env: EvalEnv): + cube: DriverDataCube = args.get_required("data", expected_type=DriverDataCube) if hasattr(cube, "mask_l1c"): return cube.mask_l1c() else: @@ -2376,10 +2317,11 @@ def load_result(args: ProcessArgs, env: EvalEnv) -> DriverDataCube: @process_registry_100.add_function(spec=read_spec("openeo-processes/1.x/proposals/inspect.json")) @process_registry_2xx.add_function(spec=read_spec("openeo-processes/2.x/proposals/inspect.json")) -def inspect(args: dict, env: EvalEnv): - data = extract_arg(args, "data") - message = args.get("message", "") - level = args.get("level", "info") +def inspect(args: ProcessArgs, env: EvalEnv): + data = args.get_required("data") + message = args.get_optional("message", default="") + code = args.get_optional("code", default="User") + level = args.get_optional("level", default="info") if message: _log.log(level=logging.getLevelName(level.upper()), msg=message) data_message = str(data) diff --git a/tests/test_views_execute.py b/tests/test_views_execute.py index bb0894e2..11bbec6a 100644 --- a/tests/test_views_execute.py +++ b/tests/test_views_execute.py @@ -3156,7 +3156,7 @@ def test_execute_no_cube_dynamic_args(api): assert kwargs["factor"] == 7.75 -@pytest.mark.parametrize(["border", "expected"], [(0, 0), ("0", 0), ]) +@pytest.mark.parametrize(["border", "expected"], [(0, 0)]) def test_execute_apply_kernel_border(api, border, expected): pg = { "lc1": {'process_id': 'load_collection', 'arguments': {'id': 'S2_FOOBAR'}},