diff --git a/CHANGELOG.md b/CHANGELOG.md index e96b3487..5c03342a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ and start a new "In Progress" section above it. - `DiskWorkspace`: support unified asset keys ([Open-EO/openeo-geopyspark-driver#1111](https://github.com/Open-EO/openeo-geopyspark-driver/issues/1111)) - Support persisting results metadata URI in job registry ([Open-EO/openeo-geopyspark-driver#1255](https://github.com/Open-EO/openeo-geopyspark-driver/issues/1255)) - More fine-grained `convert_node` cache control ([Open-EO/openeo-geopyspark-driver#1331](https://github.com/Open-EO/openeo-geopyspark-driver/issues/1331)/[#422](https://github.com/Open-EO/openeo-python-driver/pull/422)) +- `get_result_metadata` can return items ([Open-EO/openeo-geopyspark-driver#1111](https://github.com/Open-EO/openeo-geopyspark-driver/issues/1111)) ## 0.134.0 diff --git a/openeo_driver/views.py b/openeo_driver/views.py index 44022c5c..1b17d55f 100644 --- a/openeo_driver/views.py +++ b/openeo_driver/views.py @@ -1160,16 +1160,20 @@ def job_results_canonical_url() -> str: if TREAT_JOB_RESULTS_V100_LIKE_V110 or requested_api_version().at_least("1.1.0"): ml_model_metadata = None - def job_result_item_url(item_id) -> str: + def job_result_item_url(item_id, is11 = False) -> str: signer = get_backend_config().url_signer + + method_start = ".get_job_result_item" + if is11: + method_start = method_start + "11" if not signer: - return url_for(".get_job_result_item", job_id=job_id, item_id=item_id, _external=True) + return url_for(method_start, job_id=job_id, item_id=item_id, _external=True) expires = signer.get_expires() secure_key = signer.sign_job_item(job_id=job_id, user_id=user_id, item_id=item_id, expires=expires) user_base64 = user_id_b64_encode(user_id) return url_for( - ".get_job_result_item_signed", + method_start + "_signed", job_id=job_id, user_base64=user_base64, secure_key=secure_key, @@ -1178,19 +1182,27 @@ def job_result_item_url(item_id) -> str: _external=True, ) - for filename, metadata in result_assets.items(): - if ("data" in metadata.get("roles", []) and - any(media_type in metadata.get("type", "") for media_type in - ["geotiff", "netcdf", "text/csv", "application/parquet"])): - links.append( - {"rel": "item", "href": job_result_item_url(item_id=filename), "type": stac_item_media_type} - ) - elif metadata.get("ml_model_metadata", False): - # TODO: Currently we only support one ml_model per batch job. - ml_model_metadata = metadata + + if len(result_metadata.items) > 0 : + for item_id in result_metadata.items.keys(): links.append( - {"rel": "item", "href": job_result_item_url(item_id=filename), "type": "application/json"} + {"rel": "item", "href": job_result_item_url(item_id=item_id, is11=True), "type": stac_item_media_type} ) + else: + + for filename, metadata in result_assets.items(): + if ("data" in metadata.get("roles", []) and + any(media_type in metadata.get("type", "") for media_type in + ["geotiff", "netcdf", "text/csv", "application/parquet"])): + links.append( + {"rel": "item", "href": job_result_item_url(item_id=filename), "type": stac_item_media_type} + ) + elif metadata.get("ml_model_metadata", False): + # TODO: Currently we only support one ml_model per batch job. + ml_model_metadata = metadata + links.append( + {"rel": "item", "href": job_result_item_url(item_id=filename), "type": "application/json"} + ) result = dict_no_none( { @@ -1357,11 +1369,118 @@ def get_job_result_item_signed(job_id, user_base64, secure_key, item_id): signer.verify_job_item(signature=secure_key, job_id=job_id, user_id=user_id, item_id=item_id, expires=expires) return _get_job_result_item(job_id, item_id, user_id) + @api_endpoint + @blueprint.route('/jobs//results/items11///', methods=['GET']) + def get_job_result_item11_signed(job_id, user_base64, secure_key, item_id): + expires = request.args.get('expires') + signer = get_backend_config().url_signer + user_id = user_id_b64_decode(user_base64) + signer.verify_job_item(signature=secure_key, job_id=job_id, user_id=user_id, item_id=item_id, expires=expires) + return _get_job_result_item11(job_id, item_id, user_id) + @blueprint.route('/jobs//results/items/', methods=['GET']) @auth_handler.requires_bearer_auth def get_job_result_item(job_id: str, item_id: str, user: User) -> flask.Response: return _get_job_result_item(job_id, item_id, user.user_id) + @api_endpoint(version=ComparableVersion("1.1.0").or_higher) + @blueprint.route('/jobs//results/items11/', methods=['GET']) + @auth_handler.requires_bearer_auth + def get_job_result_item11(job_id: str, item_id: str, user: User) -> flask.Response: + return _get_job_result_item11(job_id, item_id, user.user_id) + + def _get_job_result_item11(job_id, item_id, user_id): + if item_id == DriverMlModel.METADATA_FILE_NAME: + return _download_ml_model_metadata(job_id, item_id, user_id) + + metadata = backend_implementation.batch_jobs.get_result_metadata( + job_id=job_id, user_id=user_id + ) + + if item_id not in metadata.items: + raise OpenEOApiException("Item with id {item_id!r} not found in job {job_id!r}".format(item_id=item_id, job_id=job_id), status_code=404) + item_metadata = metadata.items.get(item_id,None) + + job_info = backend_implementation.batch_jobs.get_job_info(job_id, user_id) + + assets = {} + for asset_key, asset in item_metadata.get("assets", {}).items(): + assets[asset_key] = _asset_object(job_id, user_id, asset_key, asset, job_info) + + geometry = item_metadata.get("geometry", job_info.geometry) + bbox = item_metadata.get("bbox", job_info.bbox) + + properties = item_metadata.get("properties", {"datetime": item_metadata.get("datetime")}) + if properties["datetime"] is None: + to_datetime = Rfc3339(propagate_none=True).datetime + + start_datetime = item_metadata.get("start_datetime") or to_datetime(job_info.start_datetime) + end_datetime = item_metadata.get("end_datetime") or to_datetime(job_info.end_datetime) + + if start_datetime == end_datetime: + properties["datetime"] = start_datetime + else: + if start_datetime: + properties["start_datetime"] = start_datetime + if end_datetime: + properties["end_datetime"] = end_datetime + + if job_info.proj_shape: + properties["proj:shape"] = job_info.proj_shape + if job_info.proj_bbox: + properties["proj:bbox"] = job_info.proj_bbox + if job_info.epsg: + properties["proj:epsg"] = job_info.epsg + + if job_info.proj_bbox and job_info.epsg: + if not bbox: + bbox = BoundingBox.from_wsen_tuple(job_info.proj_bbox, job_info.epsg).reproject(4326).as_wsen_tuple() + if not geometry: + geometry = BoundingBox.from_wsen_tuple(job_info.proj_bbox, job_info.epsg).as_polygon() + geometry = mapping(reproject_geometry(geometry, CRS.from_epsg(job_info.epsg), CRS.from_epsg(4326))) + + stac_item = { + "type": "Feature", + "stac_version": "1.0.0", + "stac_extensions": [ + STAC_EXTENSION.EO_V110, + STAC_EXTENSION.FILEINFO, + STAC_EXTENSION.PROJECTION, + ], + "id": item_id, + "geometry": geometry, + "bbox": bbox, + "properties": properties, + "links": [ + { + "rel": "self", + # MUST be absolute + "href": url_for(".get_job_result_item", job_id=job_id, item_id=item_id, _external=True), + "type": stac_item_media_type, + }, + { + "rel": "collection", + "href": url_for(".list_job_results", job_id=job_id, _external=True), # SHOULD be absolute + "type": "application/json", + }, + ], + "assets": assets, + "collection": job_id, + } + # Add optional items, if they are present. + stac_item.update( + **dict_no_none( + { + "epsg": job_info.epsg, + } + ) + ) + + resp = jsonify(stac_item) + resp.mimetype = stac_item_media_type + return resp + + def _get_job_result_item(job_id, item_id, user_id): if item_id == DriverMlModel.METADATA_FILE_NAME: return _download_ml_model_metadata(job_id, item_id, user_id) diff --git a/tests/test_views.py b/tests/test_views.py index 79b6382e..c8e10939 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -2847,6 +2847,207 @@ def test_get_job_results_custom_links(self, api100): } ) + @pytest.mark.parametrize("backend_config_overrides", [{"url_signer": UrlSigner(secret="123&@#")}]) + def test_get_job_results_from_stac_1_1_items(self, api110, backend_config_overrides): + job_id = "07024ee9-7847-4b8a-b260-6c879a2b3cdc" + + with self._fresh_job_registry(): + dummy_backend.DummyBatchJobs._update_status(job_id=job_id, user_id=TEST_USER, status="finished") + dummy_backend.DummyBatchJobs.set_result_metadata( + job_id=job_id, + user_id=TEST_USER, + metadata=BatchJobResultMetadata( + items={ + "5d2db643-5cc3-4b27-8ef3-11f7d203b221_2023-12-31T21:41:00Z": { + "geometry": { + "coordinates": [ + [ + [3.359808992021044, 51.08284561357965], + [3.359808992021044, 51.88641704215104], + [4.690166134878123, 51.88641704215104], + [4.690166134878123, 51.08284561357965], + [3.359808992021044, 51.08284561357965], + ] + ], + "type": "Polygon", + }, + "assets": { + "openEO": { + "datetime": "2023-12-31T21:41:00Z", + "roles": ["data"], + "bbox": [ + 3.359808992021044, + 51.08284561357965, + 4.690166134878123, + 51.88641704215104, + ], + "geometry": { + "coordinates": [ + [ + [3.359808992021044, 51.08284561357965], + [3.359808992021044, 51.88641704215104], + [4.690166134878123, 51.88641704215104], + [4.690166134878123, 51.08284561357965], + [3.359808992021044, 51.08284561357965], + ] + ], + "type": "Polygon", + }, + "href": "openEO_20231231T214100Z.tif", + "nodata": "nan", + "type": "image/tiff; application=geotiff", + "bands": [ + {"name": "LST", "common_name": "surface_temperature", "aliases": ["LST_in:LST"]} + ], + "raster:bands": [ + { + "name": "LST", + "statistics": { + "valid_percent": 66.88, + "maximum": 281.04800415039, + "stddev": 19.598456945276, + "minimum": 224.46798706055, + "mean": 259.57087672984, + }, + } + ], + } + }, + "id": "5d2db643-5cc3-4b27-8ef3-11f7d203b221_2023-12-31T21:41:00Z", + "properties": {"datetime": "2023-12-31T21:41:00Z"}, + "bbox": [3.359808992021044, 51.08284561357965, 4.690166134878123, 51.88641704215104], + } + } + ), + ) + + resp = api110.get(f"/jobs/{job_id}/results", headers=self.AUTH_HEADER).assert_status_code(200) + + item_links = [link for link in resp.json["links"] if link["rel"] == "item"] + assert len(item_links) == 1, "expected exactly one item link in STAC Collection" + + item_link = item_links[0] + assert item_link["href"] == f"http://oeo.net/openeo/1.1.0/jobs/{job_id}/results/items11/TXIuVGVzdA==/6dfcff9f3d3d760f1cfca269ccc245fb/5d2db643-5cc3-4b27-8ef3-11f7d203b221_2023-12-31T21:41:00Z" + + @pytest.mark.parametrize("backend_config_overrides", [{"url_signer": UrlSigner(secret="123&@#")}]) + def test_get_stac_1_1_item(self, api110, backend_implementation, backend_config_overrides): + job_id = "07024ee9-7847-4b8a-b260-6c879a2b3cdc" + item_id = "5d2db643-5cc3-4b27-8ef3-11f7d203b221_2023-12-31T21:41:00Z" + with self._fresh_job_registry(): + dummy_backend.DummyBatchJobs.set_result_metadata( + job_id=job_id, + user_id=TEST_USER, + metadata=BatchJobResultMetadata( + items={ + "5d2db643-5cc3-4b27-8ef3-11f7d203b221_2023-12-31T21:41:00Z": { + "geometry": { + "coordinates": [ + [ + [3.359808992021044, 51.08284561357965], + [3.359808992021044, 51.88641704215104], + [4.690166134878123, 51.88641704215104], + [4.690166134878123, 51.08284561357965], + [3.359808992021044, 51.08284561357965], + ] + ], + "type": "Polygon", + }, + "assets": { + "openEO": { + "datetime": "2023-12-31T21:41:00Z", + "roles": ["data"], + "bbox": [ + 3.359808992021044, + 51.08284561357965, + 4.690166134878123, + 51.88641704215104, + ], + "geometry": { + "coordinates": [ + [ + [3.359808992021044, 51.08284561357965], + [3.359808992021044, 51.88641704215104], + [4.690166134878123, 51.88641704215104], + [4.690166134878123, 51.08284561357965], + [3.359808992021044, 51.08284561357965], + ] + ], + "type": "Polygon", + }, + "href": "s3://openeo-data-staging-waw4-1/batch_jobs/j-250605095828442799fdde3c29b5b047/openEO_20231231T214100Z.tif", + "nodata": "nan", + "type": "image/tiff; application=geotiff", + "bands": [ + {"name": "LST", "common_name": "surface_temperature", "aliases": ["LST_in:LST"]} + ], + "raster:bands": [ + { + "name": "LST", + "statistics": { + "valid_percent": 66.88, + "maximum": 281.04800415039, + "stddev": 19.598456945276, + "minimum": 224.46798706055, + "mean": 259.57087672984, + }, + } + ], + } + }, + "id": "5d2db643-5cc3-4b27-8ef3-11f7d203b221_2023-12-31T21:41:00Z", + "properties": {"datetime": "2023-12-31T21:41:00Z"}, + "bbox": [3.359808992021044, 51.08284561357965, 4.690166134878123, 51.88641704215104], + } + } + ), + ) + resp = api110.get( + f"/jobs/{job_id}/results/items11/{item_id}", headers=self.AUTH_HEADER + ) + + resp_data = resp.assert_status_code(200).json + assert resp_data == { + 'assets': { + 'openEO': DictSubSet({ + 'href': 'http://oeo.net/openeo/1.1.0/jobs/07024ee9-7847-4b8a-b260-6c879a2b3cdc/results/assets/TXIuVGVzdA==/0a577c1d63aed6a562c29a5bd4f535fe/openEO', + 'roles': ['data'], + 'type': 'image/tiff; application=geotiff', + 'title': 'openEO', + }) + }, + 'bbox': [3.359808992021044, 51.08284561357965, 4.690166134878123, 51.88641704215104], + 'collection': '07024ee9-7847-4b8a-b260-6c879a2b3cdc', + 'geometry': { + 'coordinates': [[ + [3.359808992021044, 51.08284561357965], + [3.359808992021044, 51.88641704215104], + [4.690166134878123, 51.88641704215104], + [4.690166134878123, 51.08284561357965], + [3.359808992021044, 51.08284561357965] + ]], + 'type': 'Polygon' + }, + 'id': '5d2db643-5cc3-4b27-8ef3-11f7d203b221_2023-12-31T21:41:00Z', + 'links': [ + { + 'href': 'http://oeo.net/openeo/1.1.0/jobs/07024ee9-7847-4b8a-b260-6c879a2b3cdc/results/items/5d2db643-5cc3-4b27-8ef3-11f7d203b221_2023-12-31T21:41:00Z', + 'rel': 'self', + 'type': 'application/geo+json' + }, + { + 'href': 'http://oeo.net/openeo/1.1.0/jobs/07024ee9-7847-4b8a-b260-6c879a2b3cdc/results', + 'rel': 'collection', + 'type': 'application/json' + } + ], + 'properties': {'datetime': '2023-12-31T21:41:00Z'}, + 'stac_extensions': ['https://stac-extensions.github.io/eo/v1.1.0/schema.json', + 'https://stac-extensions.github.io/file/v2.1.0/schema.json', + 'https://stac-extensions.github.io/projection/v1.1.0/schema.json'], + 'stac_version': '1.0.0', + 'type': 'Feature' + } + def test_get_job_results_invalid_job(self, api): api.get("/jobs/deadbeef-f00/results", headers=self.AUTH_HEADER).assert_error(404, "JobNotFound")