From 6c9f632399f6c7b4ae53de77c671612f3cd1acf5 Mon Sep 17 00:00:00 2001 From: ghiggi Date: Tue, 25 Mar 2025 17:38:52 +0100 Subject: [PATCH 1/2] Add MCH --- radar_api/checks.py | 4 ++-- radar_api/etc/network/FMI.yaml | 1 + radar_api/etc/network/IDEAM.yaml | 1 + radar_api/etc/network/MCH.yaml | 11 +++++++++++ radar_api/etc/network/NEXRAD.yaml | 1 + radar_api/etc/radar/MCH/MLA.yaml | 8 ++++++++ radar_api/etc/radar/MCH/MLD.yaml | 8 ++++++++ radar_api/etc/radar/MCH/MLL.yaml | 8 ++++++++ radar_api/etc/radar/MCH/MLP.yaml | 8 ++++++++ radar_api/etc/radar/MCH/MLW.yaml | 8 ++++++++ radar_api/filter.py | 4 ++-- radar_api/io.py | 14 +++++++------- radar_api/readers.py | 17 ++++++++++++++--- radar_api/search.py | 18 +++++++++++++++--- radar_api/tests/test_io.py | 8 ++++++-- 15 files changed, 100 insertions(+), 19 deletions(-) create mode 100644 radar_api/etc/network/MCH.yaml create mode 100644 radar_api/etc/radar/MCH/MLA.yaml create mode 100644 radar_api/etc/radar/MCH/MLD.yaml create mode 100644 radar_api/etc/radar/MCH/MLL.yaml create mode 100644 radar_api/etc/radar/MCH/MLP.yaml create mode 100644 radar_api/etc/radar/MCH/MLW.yaml diff --git a/radar_api/checks.py b/radar_api/checks.py index 5029559..14ae9ad 100644 --- a/radar_api/checks.py +++ b/radar_api/checks.py @@ -85,7 +85,7 @@ def check_radar(radar, network): if not isinstance(radar, str): raise TypeError("Specify 'radar' as a string.") check_network(network) - valid_radars = available_radars() + valid_radars = available_radars(only_public=False) if radar not in valid_radars: raise ValueError(f"Invalid {network} radar {radar}. Available radars: {valid_radars}") return radar @@ -98,7 +98,7 @@ def check_network(network): if not isinstance(network, str): raise TypeError("Specify 'network' as a string.") - valid_networks = available_networks() + valid_networks = available_networks(only_public=False) if network not in valid_networks: raise ValueError(f"Invalid network {network}. Available networks: {valid_networks}") return network diff --git a/radar_api/etc/network/FMI.yaml b/radar_api/etc/network/FMI.yaml index 8bc0c82..b9f48c2 100644 --- a/radar_api/etc/network/FMI.yaml +++ b/radar_api/etc/network/FMI.yaml @@ -1,5 +1,6 @@ network: "FMI" description: "Finnish Meteorological Institute radars" +public_data: True cloud_directory_pattern: "s3://fmi-opendata-radar-volume-hdf5/{time:%Y}/{time:%m}/{time:%d}/{radar:s}" local_directory_pattern: "{base_dir}/FMI/{time:%Y}/{time:%m}/{time:%d}/{time:%H}/{radar:s}" filename_patterns: diff --git a/radar_api/etc/network/IDEAM.yaml b/radar_api/etc/network/IDEAM.yaml index 8946f16..1b5414c 100644 --- a/radar_api/etc/network/IDEAM.yaml +++ b/radar_api/etc/network/IDEAM.yaml @@ -1,5 +1,6 @@ network: "IDEAM" description: "Colombian weather radar network" +public_data: True cloud_directory_pattern: "s3://s3-radaresideam/l2_data/{time:%Y}/{time:%m}/{time:%d}/{radar:s}" local_directory_pattern: "{base_dir}/IDEAM/{time:%Y}/{time:%m}/{time:%d}/{time:%H}/{radar:s}" filename_patterns: diff --git a/radar_api/etc/network/MCH.yaml b/radar_api/etc/network/MCH.yaml new file mode 100644 index 0000000..2d7e42b --- /dev/null +++ b/radar_api/etc/network/MCH.yaml @@ -0,0 +1,11 @@ +network: MCH +description: MeteoSwiss Rad4Alp network +public_data: False +cloud_directory_pattern: null +local_directory_pattern: "/store_new/mch/msrad/radar/swiss/data/{time:%Y}/{time:%y}{time:%j}/{radar:3s}{time:%y}{time:%j}.zip" +filename_patterns: + - "{radar_acronym:3s}{start_time:%y%j%H%M}0U.{volume_identifier:3s}" +pyart_reader: read_metranet +xradar_reader: null +xradar_engine: null + diff --git a/radar_api/etc/network/NEXRAD.yaml b/radar_api/etc/network/NEXRAD.yaml index d38c1d3..5cd548d 100644 --- a/radar_api/etc/network/NEXRAD.yaml +++ b/radar_api/etc/network/NEXRAD.yaml @@ -1,5 +1,6 @@ network: "NEXRAD" description: "NOAA NEXRAD radar network" +public_data: True cloud_directory_pattern: "s3://noaa-nexrad-level2/{time:%Y}/{time:%m}/{time:%d}/{radar:s}" local_directory_pattern: "{base_dir}/NEXRAD/{time:%Y}/{time:%m}/{time:%d}/{time:%H}/{radar:s}" filename_patterns: diff --git a/radar_api/etc/radar/MCH/MLA.yaml b/radar_api/etc/radar/MCH/MLA.yaml new file mode 100644 index 0000000..d5fc7e8 --- /dev/null +++ b/radar_api/etc/radar/MCH/MLA.yaml @@ -0,0 +1,8 @@ +icao: "" +radar_name: "Albis" +start_time: "2017-01-01 00:00:00" +end_time: "" +latitude: 47.284332 +longitude: 8.512 +altitude: 938 +radar_band: C diff --git a/radar_api/etc/radar/MCH/MLD.yaml b/radar_api/etc/radar/MCH/MLD.yaml new file mode 100644 index 0000000..a668817 --- /dev/null +++ b/radar_api/etc/radar/MCH/MLD.yaml @@ -0,0 +1,8 @@ +icao: "" +radar_name: "Dole" +start_time: "2017-01-01 00:00:00" +end_time: "" +latitude: 46.425114 +longitude: 6.099415 +altitude: 1682 +radar_band: C diff --git a/radar_api/etc/radar/MCH/MLL.yaml b/radar_api/etc/radar/MCH/MLL.yaml new file mode 100644 index 0000000..4f26f42 --- /dev/null +++ b/radar_api/etc/radar/MCH/MLL.yaml @@ -0,0 +1,8 @@ +icao: "" +radar_name: "Lema" +start_time: "2017-01-01 00:00:00" +end_time: "" +latitude: 46.04076 +longitude: 8.833217 +altitude: 1626 +radar_band: C diff --git a/radar_api/etc/radar/MCH/MLP.yaml b/radar_api/etc/radar/MCH/MLP.yaml new file mode 100644 index 0000000..4034f1b --- /dev/null +++ b/radar_api/etc/radar/MCH/MLP.yaml @@ -0,0 +1,8 @@ +icao: "" +radar_name: "Plaine Morte" +start_time: "2018-01-01 00:00:00" +end_time: "" +latitude: 46.370647 +longitude: 7.486552 +altitude: 2937 +radar_band: C diff --git a/radar_api/etc/radar/MCH/MLW.yaml b/radar_api/etc/radar/MCH/MLW.yaml new file mode 100644 index 0000000..b7443dd --- /dev/null +++ b/radar_api/etc/radar/MCH/MLW.yaml @@ -0,0 +1,8 @@ +icao: "" +radar_name: "Weissfluhgipfel" +start_time: "2019-01-01 00:00:00" +end_time: "" +latitude: 46.834972 +longitude: 9.794458 +altitude: 2850 +radar_band: C diff --git a/radar_api/filter.py b/radar_api/filter.py index fdc9409..4ef0ba4 100644 --- a/radar_api/filter.py +++ b/radar_api/filter.py @@ -40,8 +40,8 @@ def is_file_within_time(start_time, end_time, file_start_time, file_end_time): # - Case 2 # s e # | | - # -------- - is_case2 = file_start_time >= start_time and file_end_time < end_time + # (--)---------(--) + is_case2 = file_start_time >= start_time and file_end_time <= end_time # - Case 3 # s e # | | diff --git a/radar_api/io.py b/radar_api/io.py index 976979a..60c9664 100644 --- a/radar_api/io.py +++ b/radar_api/io.py @@ -62,7 +62,7 @@ def get_radar_config_filepath(network, radar): return filepath -def available_networks(): +def available_networks(only_public=True): """Get list of available networks.""" network_config_path = get_network_config_path() networks_config_filenames = os.listdir(network_config_path) @@ -82,10 +82,10 @@ def _get_network_radars(network, start_time=None, end_time=None): return radars -def available_radars(network=None, start_time=None, end_time=None): +def available_radars(network=None, start_time=None, end_time=None, only_public=True): """Get list of available radars.""" if network is None: - networks = available_networks() + networks = available_networks(only_public=only_public) list_radars = [ _get_network_radars(network=network, start_time=start_time, end_time=end_time) for network in networks ] @@ -194,10 +194,10 @@ def is_radar_available(network, radar, start_time=None, end_time=None): ) -def get_network_database(network): +def get_network_database(network, only_public=True): """Retrieve the radar network database.""" list_info = [] - for radar in available_radars(network=network): + for radar in available_radars(network=network, only_public=only_public): try: radar_info_path = get_radar_config_filepath(network=network, radar=radar) radar_info = read_yaml(radar_info_path) @@ -211,9 +211,9 @@ def get_network_database(network): return pd.DataFrame(list_info) -def get_database(): +def get_database(only_public=True): """Retrieve the RADAR-API database.""" - list_df = [get_network_database(network) for network in available_networks()] + list_df = [get_network_database(network) for network in available_networks(only_public=only_public)] return pd.concat(list_df) diff --git a/radar_api/readers.py b/radar_api/readers.py index 5b499fc..c60ecba 100644 --- a/radar_api/readers.py +++ b/radar_api/readers.py @@ -72,18 +72,29 @@ def get_xradar_datatree_reader(network): """Return the xradar datatree reader.""" import xradar.io - func = getattr(xradar.io, get_network_info(network)["xradar_reader"]) + xradar_reader_name = get_network_info(network)["xradar_reader"] + if xradar_reader_name is None: + raise NotImplementedError(f"No xradar reader is yet available for network {network}.") + func = getattr(xradar.io, xradar_reader_name) return func def get_pyart_reader(network): """Return the pyart reader.""" + import pyart.aux_io import pyart.io + pyart_reader_name = get_network_info(network)["pyart_reader"] + if pyart_reader_name is None: + raise NotImplementedError(f"No pyart reader is yet available for network {network}.") + try: - func = getattr(pyart.io, get_network_info(network)["pyart_reader"]) + func = getattr(pyart.io, pyart_reader_name) except AttributeError: - func = getattr(pyart.aux_io, get_network_info(network)["pyart_reader"]) + try: + func = getattr(pyart.aux_io, pyart_reader_name) + except AttributeError: + raise NotImplementedError(f"The pyart reader {pyart_reader_name} is not available in your pyart library.") return func diff --git a/radar_api/search.py b/radar_api/search.py index 93d3903..31ca272 100644 --- a/radar_api/search.py +++ b/radar_api/search.py @@ -24,6 +24,8 @@ # SOFTWARE. """This module provides functions for searching files on local disk and cloud buckets.""" import datetime +import os +import zipfile import pandas as pd from trollsift import Parser @@ -111,12 +113,22 @@ def get_directories_paths(start_time, end_time, network, radar, protocol, base_d return paths +def _list_files_within_zip(zip_filepath): + """Return the paths of files within a zip file.""" + with zipfile.ZipFile(zip_filepath, "r") as zf: + filenames = zf.namelist() + filepaths = [os.path.join(zip_filepath, fname) for fname in filenames] + return filepaths + + def _try_list_files(fs, dir_path): + """Return filepaths within a given directory (or zip file).""" try: - fpaths = fs.ls(dir_path) + if not dir_path.endswith(".zip"): + return fs.ls(dir_path) + return _list_files_within_zip(dir_path) except Exception: - fpaths = [] - return fpaths + return [] def find_files( diff --git a/radar_api/tests/test_io.py b/radar_api/tests/test_io.py index efdc61e..e99f6a4 100644 --- a/radar_api/tests/test_io.py +++ b/radar_api/tests/test_io.py @@ -87,20 +87,24 @@ def test_get_radar_config_filepath(network): def test_available_networks(): """Test available_networks.""" - nets = available_networks() - assert "NEXRAD" in nets + networks = available_networks() + assert isinstance(networks, list) + assert len(networks) > 0 + assert "NEXRAD" in networks def test_available_radars_all_networks(): """Test available_radars().""" radars = available_radars() assert isinstance(radars, list) + assert len(radars) > 0 def test_available_radars_single_network(): """Test available_radars(network).""" radars = available_radars("NEXRAD") assert isinstance(radars, list) + assert len(radars) > 0 def test_get_network_info(): From e4072fadcb907d7072ea680e5a8e11f878fdc206 Mon Sep 17 00:00:00 2001 From: ghiggi Date: Fri, 22 Aug 2025 11:12:17 +0200 Subject: [PATCH 2/2] Add commit to reverse maybe in future --- .../etc/network/{MCH.yaml => MCH_CSCS.yaml} | 0 radar_api/etc/network/MCH_LTE.yaml | 12 ++++ .../etc/radar/{MCH => MCH_CSCS}/MLA.yaml | 0 .../etc/radar/{MCH => MCH_CSCS}/MLD.yaml | 0 .../etc/radar/{MCH => MCH_CSCS}/MLL.yaml | 0 .../etc/radar/{MCH => MCH_CSCS}/MLP.yaml | 0 .../etc/radar/{MCH => MCH_CSCS}/MLW.yaml | 0 radar_api/etc/radar/MCH_LTE/MLA.yaml | 8 +++ radar_api/etc/radar/MCH_LTE/MLD.yaml | 8 +++ radar_api/etc/radar/MCH_LTE/MLL.yaml | 8 +++ radar_api/etc/radar/MCH_LTE/MLP.yaml | 8 +++ radar_api/etc/radar/MCH_LTE/MLW.yaml | 8 +++ radar_api/utils/xradar.py | 66 ++++++++++++++++++- 13 files changed, 116 insertions(+), 2 deletions(-) rename radar_api/etc/network/{MCH.yaml => MCH_CSCS.yaml} (100%) create mode 100644 radar_api/etc/network/MCH_LTE.yaml rename radar_api/etc/radar/{MCH => MCH_CSCS}/MLA.yaml (100%) rename radar_api/etc/radar/{MCH => MCH_CSCS}/MLD.yaml (100%) rename radar_api/etc/radar/{MCH => MCH_CSCS}/MLL.yaml (100%) rename radar_api/etc/radar/{MCH => MCH_CSCS}/MLP.yaml (100%) rename radar_api/etc/radar/{MCH => MCH_CSCS}/MLW.yaml (100%) create mode 100644 radar_api/etc/radar/MCH_LTE/MLA.yaml create mode 100644 radar_api/etc/radar/MCH_LTE/MLD.yaml create mode 100644 radar_api/etc/radar/MCH_LTE/MLL.yaml create mode 100644 radar_api/etc/radar/MCH_LTE/MLP.yaml create mode 100644 radar_api/etc/radar/MCH_LTE/MLW.yaml diff --git a/radar_api/etc/network/MCH.yaml b/radar_api/etc/network/MCH_CSCS.yaml similarity index 100% rename from radar_api/etc/network/MCH.yaml rename to radar_api/etc/network/MCH_CSCS.yaml diff --git a/radar_api/etc/network/MCH_LTE.yaml b/radar_api/etc/network/MCH_LTE.yaml new file mode 100644 index 0000000..21d6dbe --- /dev/null +++ b/radar_api/etc/network/MCH_LTE.yaml @@ -0,0 +1,12 @@ +network: MCH_LTE +description: MeteoSwiss Rad4Alp network +public_data: False +cloud_directory_pattern: null +local_directory_pattern: "{base_dir}/MCH/{time:%Y}/{time:%m}/{time:%d}/{time:%H}/{radar:s}" +filename_patterns: + - "{radar_acronym:3s}{start_time:%y%j%H%M}0U.{volume_identifier:3s}" +pyart_reader: read_metranet +xradar_reader: null +xradar_engine: null + + diff --git a/radar_api/etc/radar/MCH/MLA.yaml b/radar_api/etc/radar/MCH_CSCS/MLA.yaml similarity index 100% rename from radar_api/etc/radar/MCH/MLA.yaml rename to radar_api/etc/radar/MCH_CSCS/MLA.yaml diff --git a/radar_api/etc/radar/MCH/MLD.yaml b/radar_api/etc/radar/MCH_CSCS/MLD.yaml similarity index 100% rename from radar_api/etc/radar/MCH/MLD.yaml rename to radar_api/etc/radar/MCH_CSCS/MLD.yaml diff --git a/radar_api/etc/radar/MCH/MLL.yaml b/radar_api/etc/radar/MCH_CSCS/MLL.yaml similarity index 100% rename from radar_api/etc/radar/MCH/MLL.yaml rename to radar_api/etc/radar/MCH_CSCS/MLL.yaml diff --git a/radar_api/etc/radar/MCH/MLP.yaml b/radar_api/etc/radar/MCH_CSCS/MLP.yaml similarity index 100% rename from radar_api/etc/radar/MCH/MLP.yaml rename to radar_api/etc/radar/MCH_CSCS/MLP.yaml diff --git a/radar_api/etc/radar/MCH/MLW.yaml b/radar_api/etc/radar/MCH_CSCS/MLW.yaml similarity index 100% rename from radar_api/etc/radar/MCH/MLW.yaml rename to radar_api/etc/radar/MCH_CSCS/MLW.yaml diff --git a/radar_api/etc/radar/MCH_LTE/MLA.yaml b/radar_api/etc/radar/MCH_LTE/MLA.yaml new file mode 100644 index 0000000..d5fc7e8 --- /dev/null +++ b/radar_api/etc/radar/MCH_LTE/MLA.yaml @@ -0,0 +1,8 @@ +icao: "" +radar_name: "Albis" +start_time: "2017-01-01 00:00:00" +end_time: "" +latitude: 47.284332 +longitude: 8.512 +altitude: 938 +radar_band: C diff --git a/radar_api/etc/radar/MCH_LTE/MLD.yaml b/radar_api/etc/radar/MCH_LTE/MLD.yaml new file mode 100644 index 0000000..a668817 --- /dev/null +++ b/radar_api/etc/radar/MCH_LTE/MLD.yaml @@ -0,0 +1,8 @@ +icao: "" +radar_name: "Dole" +start_time: "2017-01-01 00:00:00" +end_time: "" +latitude: 46.425114 +longitude: 6.099415 +altitude: 1682 +radar_band: C diff --git a/radar_api/etc/radar/MCH_LTE/MLL.yaml b/radar_api/etc/radar/MCH_LTE/MLL.yaml new file mode 100644 index 0000000..4f26f42 --- /dev/null +++ b/radar_api/etc/radar/MCH_LTE/MLL.yaml @@ -0,0 +1,8 @@ +icao: "" +radar_name: "Lema" +start_time: "2017-01-01 00:00:00" +end_time: "" +latitude: 46.04076 +longitude: 8.833217 +altitude: 1626 +radar_band: C diff --git a/radar_api/etc/radar/MCH_LTE/MLP.yaml b/radar_api/etc/radar/MCH_LTE/MLP.yaml new file mode 100644 index 0000000..4034f1b --- /dev/null +++ b/radar_api/etc/radar/MCH_LTE/MLP.yaml @@ -0,0 +1,8 @@ +icao: "" +radar_name: "Plaine Morte" +start_time: "2018-01-01 00:00:00" +end_time: "" +latitude: 46.370647 +longitude: 7.486552 +altitude: 2937 +radar_band: C diff --git a/radar_api/etc/radar/MCH_LTE/MLW.yaml b/radar_api/etc/radar/MCH_LTE/MLW.yaml new file mode 100644 index 0000000..b7443dd --- /dev/null +++ b/radar_api/etc/radar/MCH_LTE/MLW.yaml @@ -0,0 +1,8 @@ +icao: "" +radar_name: "Weissfluhgipfel" +start_time: "2019-01-01 00:00:00" +end_time: "" +latitude: 46.834972 +longitude: 9.794458 +altitude: 2850 +radar_band: C diff --git a/radar_api/utils/xradar.py b/radar_api/utils/xradar.py index d209d20..d1b4314 100644 --- a/radar_api/utils/xradar.py +++ b/radar_api/utils/xradar.py @@ -48,8 +48,9 @@ def _get_sweep_dataset(radar_obj, sweep): fields = list(radar_obj.fields) for field_name in fields: arr = _get_field_array(radar_obj, sweep, field_name) - dims = radar_obj.fields[field_name]["coordinates"].split(" ")[1:] - dict_da[field_name] = xr.DataArray(arr, dims=dims) + if "coordinates" in radar_obj.fields[field_name]: + dims = radar_obj.fields[field_name]["coordinates"].split(" ")[1:] + dict_da[field_name] = xr.DataArray(arr, dims=dims) ds = xr.Dataset(dict_da) # Add coords coords_dict = { @@ -98,3 +99,64 @@ def get_nexrad_datatree_from_pyart(radar_obj): dt[coord] = value return dt + + +def _get_sweep_dataset_mch(radar_obj, sweep): + dict_da = {} + fields = list(radar_obj.fields) + for field_name in fields: + arr = _get_field_array(radar_obj, sweep=0, field_name=field_name) + if "coordinates" in radar_obj.fields[field_name]: + dims = radar_obj.fields[field_name]["coordinates"].split(" ")[1:] + dict_da[field_name] = xr.DataArray(arr, dims=dims) + ds = xr.Dataset(dict_da) + # Add coords + coords_dict = { + "azimuth": ("azimuth", radar_obj.get_azimuth(0)), + "elevation": ("azimuth", radar_obj.get_elevation(0)), + "range": ("range", radar_obj.range["data"][: ds.sizes["range"]]), + "time": ("azimuth", radar_obj.time["data"][radar_obj.get_slice(0)]), + } + # Add other coordinates + coords_dict.update(_get_radar_location(radar_obj)) + coords_dict["sweep_number"] = sweep + coords_dict["sweep_mode"] = radar_obj.sweep_mode["data"][0] + coords_dict["sweep_fixed_angle"] = radar_obj.fixed_angle["data"][0] + + ds = ds.assign_coords(coords_dict) + ds["time"].attrs["units"] = radar_obj.time["units"] + + # Decode time + ds = xr.decode_cf(ds, decode_times=True) + return ds + + +def get_mch_datatree_from_pyart(radar_obj): + """Convert a pyart object to xradar datatree.""" + # Define renaming dictionary to CF-Radials2 + # --> https://github.com/openradar/xradar/blob/830d86b1c6290f1dce0e73c60a1d3b819735f906/xradar/model.py#L385 + # --> Currently set same range for all sweeps ! + # --> Currently do not copy metadata and variable attributes ! + dict_var_naming = { + "reflectivity": "DBZH", + "differential_reflectivity": "ZDR", + "uncorrected_cross_correlation_ratio": "RHOHV", + "uncorrected_differential_phase": "PHIDP", + "spectrum_width": "WRADH", + "velocity": "VRADH", + # reflectivity_hh_clut + # reflectivity_vv + # signal_to_noise_ratio + } + dict_ds = {} + for sweep in radar_obj.sweep_number["data"]: + sweep_name = f"sweep_{sweep}" + ds = _get_sweep_dataset_mch(radar_obj, sweep=sweep) + rename_dict = {k: v for k, v in dict_var_naming.items() if k in ds} + dict_ds[sweep_name] = ds.rename(rename_dict) + dt = xr.DataTree.from_dict(dict_ds) + # Add geolocation + for coord, value in _get_radar_location(radar_obj).items(): + dt[coord] = value + + return dt \ No newline at end of file