Enable specification of local archive without public access

radar_api/checks.py

@@ -85,7 +85,7 @@ def check_radar(radar, network):
     if not isinstance(radar, str):
         raise TypeError("Specify 'radar' as a string.")
     check_network(network)
-    valid_radars = available_radars()
+    valid_radars = available_radars(only_public=False)
     if radar not in valid_radars:
         raise ValueError(f"Invalid {network} radar {radar}. Available radars: {valid_radars}")
     return radar
@@ -98,7 +98,7 @@ def check_network(network):
     if not isinstance(network, str):
         raise TypeError("Specify 'network' as a string.")
 
-    valid_networks = available_networks()
+    valid_networks = available_networks(only_public=False)
     if network not in valid_networks:
         raise ValueError(f"Invalid network {network}. Available networks: {valid_networks}")
     return network

radar_api/etc/network/FMI.yaml

@@ -1,5 +1,6 @@
 network: "FMI"
 description: "Finnish Meteorological Institute radars"
+public_data: True
 cloud_directory_pattern: "s3://fmi-opendata-radar-volume-hdf5/{time:%Y}/{time:%m}/{time:%d}/{radar:s}"
 local_directory_pattern: "{base_dir}/FMI/{time:%Y}/{time:%m}/{time:%d}/{time:%H}/{radar:s}"
 filename_patterns:

radar_api/etc/network/IDEAM.yaml

@@ -1,5 +1,6 @@
 network: "IDEAM"
 description: "Colombian weather radar network"
+public_data: True
 cloud_directory_pattern: "s3://s3-radaresideam/l2_data/{time:%Y}/{time:%m}/{time:%d}/{radar:s}"
 local_directory_pattern: "{base_dir}/IDEAM/{time:%Y}/{time:%m}/{time:%d}/{time:%H}/{radar:s}"
 filename_patterns:

radar_api/etc/network/MCH_CSCS.yaml

@@ -0,0 +1,11 @@
+network: MCH
+description: MeteoSwiss Rad4Alp network
+public_data: False
+cloud_directory_pattern: null 
+local_directory_pattern: "/store_new/mch/msrad/radar/swiss/data/{time:%Y}/{time:%y}{time:%j}/{radar:3s}{time:%y}{time:%j}.zip" 
+filename_patterns:
+  - "{radar_acronym:3s}{start_time:%y%j%H%M}0U.{volume_identifier:3s}"
+pyart_reader: read_metranet
+xradar_reader: null
+xradar_engine: null
+

radar_api/etc/network/MCH_LTE.yaml

@@ -0,0 +1,12 @@
+network: MCH_LTE
+description: MeteoSwiss Rad4Alp network
+public_data: False
+cloud_directory_pattern: null 
+local_directory_pattern: "{base_dir}/MCH/{time:%Y}/{time:%m}/{time:%d}/{time:%H}/{radar:s}"
+filename_patterns:
+  - "{radar_acronym:3s}{start_time:%y%j%H%M}0U.{volume_identifier:3s}"
+pyart_reader: read_metranet
+xradar_reader: null
+xradar_engine: null
+
+

radar_api/etc/network/NEXRAD.yaml

@@ -1,5 +1,6 @@
 network: "NEXRAD"
 description: "NOAA NEXRAD radar network"
+public_data: True
 cloud_directory_pattern: "s3://noaa-nexrad-level2/{time:%Y}/{time:%m}/{time:%d}/{radar:s}"
 local_directory_pattern: "{base_dir}/NEXRAD/{time:%Y}/{time:%m}/{time:%d}/{time:%H}/{radar:s}"
 filename_patterns:

radar_api/etc/radar/MCH_CSCS/MLA.yaml

@@ -0,0 +1,8 @@
+icao: ""
+radar_name: "Albis"
+start_time: "2017-01-01 00:00:00"
+end_time: ""
+latitude: 47.284332
+longitude: 8.512
+altitude: 938
+radar_band: C

radar_api/etc/radar/MCH_CSCS/MLD.yaml

@@ -0,0 +1,8 @@
+icao: ""
+radar_name: "Dole"
+start_time: "2017-01-01 00:00:00"
+end_time: ""
+latitude: 46.425114
+longitude: 6.099415
+altitude: 1682
+radar_band: C

radar_api/etc/radar/MCH_CSCS/MLL.yaml

@@ -0,0 +1,8 @@
+icao: ""
+radar_name: "Lema"
+start_time: "2017-01-01 00:00:00"
+end_time: ""
+latitude: 46.04076
+longitude: 8.833217
+altitude: 1626
+radar_band: C

radar_api/etc/radar/MCH_CSCS/MLP.yaml

@@ -0,0 +1,8 @@
+icao: ""
+radar_name: "Plaine Morte"
+start_time: "2018-01-01 00:00:00"
+end_time: ""
+latitude: 46.370647
+longitude: 7.486552
+altitude: 2937
+radar_band: C

radar_api/etc/radar/MCH_CSCS/MLW.yaml

@@ -0,0 +1,8 @@
+icao: ""
+radar_name: "Weissfluhgipfel"
+start_time: "2019-01-01 00:00:00"
+end_time: ""
+latitude: 46.834972
+longitude: 9.794458
+altitude: 2850
+radar_band: C

radar_api/etc/radar/MCH_LTE/MLA.yaml

@@ -0,0 +1,8 @@
+icao: ""
+radar_name: "Albis"
+start_time: "2017-01-01 00:00:00"
+end_time: ""
+latitude: 47.284332
+longitude: 8.512
+altitude: 938
+radar_band: C

radar_api/etc/radar/MCH_LTE/MLD.yaml

@@ -0,0 +1,8 @@
+icao: ""
+radar_name: "Dole"
+start_time: "2017-01-01 00:00:00"
+end_time: ""
+latitude: 46.425114
+longitude: 6.099415
+altitude: 1682
+radar_band: C

radar_api/etc/radar/MCH_LTE/MLL.yaml

@@ -0,0 +1,8 @@
+icao: ""
+radar_name: "Lema"
+start_time: "2017-01-01 00:00:00"
+end_time: ""
+latitude: 46.04076
+longitude: 8.833217
+altitude: 1626
+radar_band: C

radar_api/etc/radar/MCH_LTE/MLP.yaml

@@ -0,0 +1,8 @@
+icao: ""
+radar_name: "Plaine Morte"
+start_time: "2018-01-01 00:00:00"
+end_time: ""
+latitude: 46.370647
+longitude: 7.486552
+altitude: 2937
+radar_band: C

radar_api/etc/radar/MCH_LTE/MLW.yaml

@@ -0,0 +1,8 @@
+icao: ""
+radar_name: "Weissfluhgipfel"
+start_time: "2019-01-01 00:00:00"
+end_time: ""
+latitude: 46.834972
+longitude: 9.794458
+altitude: 2850
+radar_band: C

radar_api/filter.py

@@ -40,8 +40,8 @@ def is_file_within_time(start_time, end_time, file_start_time, file_end_time):
     # - Case 2
     #     s               e
     #     |               |
-    #          --------
-    is_case2 = file_start_time >= start_time and file_end_time < end_time
+    #     (--)---------(--)
+    is_case2 = file_start_time >= start_time and file_end_time <= end_time
     # - Case 3
     #     s               e
     #     |               |

radar_api/io.py

@@ -62,7 +62,7 @@ def get_radar_config_filepath(network, radar):
     return filepath
 
 
-def available_networks():
+def available_networks(only_public=True):
     """Get list of available networks."""
     network_config_path = get_network_config_path()
     networks_config_filenames = os.listdir(network_config_path)
@@ -82,10 +82,10 @@ def _get_network_radars(network, start_time=None, end_time=None):
     return radars
 
 
-def available_radars(network=None, start_time=None, end_time=None):
+def available_radars(network=None, start_time=None, end_time=None, only_public=True):
     """Get list of available radars."""
     if network is None:
-        networks = available_networks()
+        networks = available_networks(only_public=only_public)
         list_radars = [
             _get_network_radars(network=network, start_time=start_time, end_time=end_time) for network in networks
         ]
@@ -194,10 +194,10 @@ def is_radar_available(network, radar, start_time=None, end_time=None):
     )
 
 
-def get_network_database(network):
+def get_network_database(network, only_public=True):
     """Retrieve the radar network database."""
     list_info = []
-    for radar in available_radars(network=network):
+    for radar in available_radars(network=network, only_public=only_public):
         try:
             radar_info_path = get_radar_config_filepath(network=network, radar=radar)
             radar_info = read_yaml(radar_info_path)
@@ -211,9 +211,9 @@ def get_network_database(network):
     return pd.DataFrame(list_info)
 
 
-def get_database():
+def get_database(only_public=True):
     """Retrieve the RADAR-API database."""
-    list_df = [get_network_database(network) for network in available_networks()]
+    list_df = [get_network_database(network) for network in available_networks(only_public=only_public)]
     return pd.concat(list_df)
 
 

radar_api/readers.py

@@ -72,18 +72,29 @@ def get_xradar_datatree_reader(network):
     """Return the xradar datatree reader."""
     import xradar.io
 
-    func = getattr(xradar.io, get_network_info(network)["xradar_reader"])
+    xradar_reader_name = get_network_info(network)["xradar_reader"]
+    if xradar_reader_name is None:
+        raise NotImplementedError(f"No xradar reader is yet available for network {network}.")
+    func = getattr(xradar.io, xradar_reader_name)
     return func
 
 
 def get_pyart_reader(network):
     """Return the pyart reader."""
+    import pyart.aux_io
     import pyart.io
 
+    pyart_reader_name = get_network_info(network)["pyart_reader"]
+    if pyart_reader_name is None:
+        raise NotImplementedError(f"No pyart reader is yet available for network {network}.")
+
     try:
-        func = getattr(pyart.io, get_network_info(network)["pyart_reader"])
+        func = getattr(pyart.io, pyart_reader_name)
     except AttributeError:
-        func = getattr(pyart.aux_io, get_network_info(network)["pyart_reader"])
+        try:
+            func = getattr(pyart.aux_io, pyart_reader_name)
+        except AttributeError:
+            raise NotImplementedError(f"The pyart reader {pyart_reader_name} is not available in your pyart library.")
     return func
 
 

radar_api/search.py

@@ -24,6 +24,8 @@
 # SOFTWARE.
 """This module provides functions for searching files on local disk and cloud buckets."""
 import datetime
+import os
+import zipfile
 
 import pandas as pd
 from trollsift import Parser
@@ -111,12 +113,22 @@ def get_directories_paths(start_time, end_time, network, radar, protocol, base_d
     return paths
 
 
+def _list_files_within_zip(zip_filepath):
+    """Return the paths of files within a zip file."""
+    with zipfile.ZipFile(zip_filepath, "r") as zf:
+        filenames = zf.namelist()
+    filepaths = [os.path.join(zip_filepath, fname) for fname in filenames]
+    return filepaths
+
+
 def _try_list_files(fs, dir_path):
+    """Return filepaths within a given directory (or zip file)."""
     try:
-        fpaths = fs.ls(dir_path)
+        if not dir_path.endswith(".zip"):
+            return fs.ls(dir_path)
+        return _list_files_within_zip(dir_path)
     except Exception:
-        fpaths = []
-    return fpaths
+        return []
 
 
 def find_files(

radar_api/tests/test_io.py

@@ -87,20 +87,24 @@ def test_get_radar_config_filepath(network):
 
 def test_available_networks():
     """Test available_networks."""
-    nets = available_networks()
-    assert "NEXRAD" in nets
+    networks = available_networks()
+    assert isinstance(networks, list)
+    assert len(networks) > 0
+    assert "NEXRAD" in networks
 
 
 def test_available_radars_all_networks():
     """Test available_radars()."""
     radars = available_radars()
     assert isinstance(radars, list)
+    assert len(radars) > 0
 
 
 def test_available_radars_single_network():
     """Test available_radars(network)."""
     radars = available_radars("NEXRAD")
     assert isinstance(radars, list)
+    assert len(radars) > 0
 
 
 def test_get_network_info():

radar_api/utils/xradar.py

@@ -48,8 +48,9 @@ def _get_sweep_dataset(radar_obj, sweep):
     fields = list(radar_obj.fields)
     for field_name in fields:
         arr = _get_field_array(radar_obj, sweep, field_name)
-        dims = radar_obj.fields[field_name]["coordinates"].split(" ")[1:]
-        dict_da[field_name] = xr.DataArray(arr, dims=dims)
+        if "coordinates" in radar_obj.fields[field_name]:
+            dims = radar_obj.fields[field_name]["coordinates"].split(" ")[1:]
+            dict_da[field_name] = xr.DataArray(arr, dims=dims)
     ds = xr.Dataset(dict_da)
     # Add coords
     coords_dict = {
@@ -98,3 +99,64 @@ def get_nexrad_datatree_from_pyart(radar_obj):
         dt[coord] = value
 
     return dt
+
+
+def _get_sweep_dataset_mch(radar_obj, sweep):
+    dict_da = {}
+    fields = list(radar_obj.fields)
+    for field_name in fields:
+        arr = _get_field_array(radar_obj, sweep=0, field_name=field_name)
+        if "coordinates" in radar_obj.fields[field_name]:
+            dims = radar_obj.fields[field_name]["coordinates"].split(" ")[1:]
+            dict_da[field_name] = xr.DataArray(arr, dims=dims)
+    ds = xr.Dataset(dict_da)
+    # Add coords
+    coords_dict = {
+        "azimuth": ("azimuth", radar_obj.get_azimuth(0)),
+        "elevation": ("azimuth", radar_obj.get_elevation(0)),
+        "range": ("range", radar_obj.range["data"][: ds.sizes["range"]]),
+        "time": ("azimuth", radar_obj.time["data"][radar_obj.get_slice(0)]),
+    }
+    # Add other coordinates
+    coords_dict.update(_get_radar_location(radar_obj))
+    coords_dict["sweep_number"] = sweep
+    coords_dict["sweep_mode"] = radar_obj.sweep_mode["data"][0]
+    coords_dict["sweep_fixed_angle"] = radar_obj.fixed_angle["data"][0]
+
+    ds = ds.assign_coords(coords_dict)
+    ds["time"].attrs["units"] = radar_obj.time["units"]
+
+    # Decode time
+    ds = xr.decode_cf(ds, decode_times=True)
+    return ds
+
+
+def get_mch_datatree_from_pyart(radar_obj):
+    """Convert a pyart object to xradar datatree."""
+    # Define renaming dictionary to CF-Radials2
+    # --> https://github.com/openradar/xradar/blob/830d86b1c6290f1dce0e73c60a1d3b819735f906/xradar/model.py#L385
+    # --> Currently set same range for all sweeps !
+    # --> Currently do not copy metadata and variable attributes !
+    dict_var_naming = {
+        "reflectivity": "DBZH",
+        "differential_reflectivity": "ZDR",
+        "uncorrected_cross_correlation_ratio": "RHOHV",
+        "uncorrected_differential_phase": "PHIDP",
+        "spectrum_width": "WRADH",
+        "velocity": "VRADH",
+        # reflectivity_hh_clut
+        # reflectivity_vv
+        # signal_to_noise_ratio
+    }
+    dict_ds = {}
+    for sweep in radar_obj.sweep_number["data"]:
+        sweep_name = f"sweep_{sweep}"
+        ds = _get_sweep_dataset_mch(radar_obj, sweep=sweep)
+        rename_dict = {k: v for k, v in dict_var_naming.items() if k in ds}
+        dict_ds[sweep_name] = ds.rename(rename_dict)
+    dt = xr.DataTree.from_dict(dict_ds)
+    # Add geolocation
+    for coord, value in _get_radar_location(radar_obj).items():
+        dt[coord] = value
+
+    return dt