From a35b1dec238970f6be8118febb763bc601098897 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Mon, 4 Sep 2023 23:41:36 +0200 Subject: [PATCH 01/48] First version of the l1b reader for the Arctic Weather Satellite Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/aws_l1b_nc.yaml | 425 ++++++++++++++++++++++++++++++ satpy/readers/aws_l1b.py | 311 ++++++++++++++++++++++ 2 files changed, 736 insertions(+) create mode 100644 satpy/etc/readers/aws_l1b_nc.yaml create mode 100644 satpy/readers/aws_l1b.py diff --git a/satpy/etc/readers/aws_l1b_nc.yaml b/satpy/etc/readers/aws_l1b_nc.yaml new file mode 100644 index 0000000000..86ed11bc16 --- /dev/null +++ b/satpy/etc/readers/aws_l1b_nc.yaml @@ -0,0 +1,425 @@ +reader: + name: aws_l1b_nc + short_name: AWS L1B RAD NetCDF4 + long_name: AWS L1B Radiance (NetCDF4) + description: Reader for the ESA AWS (Arctic Weather Satellite) Sounder level-1b files in netCDF4. + reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader + sensors: [aws,] + status: Beta + default_channels: [] + + data_identification_keys: + name: + required: true + frequency_double_sideband: + type: !!python/name:satpy.readers.pmw_channels_definitions.FrequencyDoubleSideBand + frequency_range: + type: !!python/name:satpy.readers.pmw_channels_definitions.FrequencyRange + resolution: + polarization: + enum: + - QH + - QV + calibration: + enum: + - brightness_temperature + transitive: true + modifiers: + required: true + default: [] + type: !!python/name:satpy.dataset.ModifierTuple + +datasets: + '1': + name: '1' + frequency_range: + central: 50.3 + bandwidth: 0.180 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [lon_horn_1, lat_horn_1] + file_type: aws_l1b_nc + file_key: data/calibration/aws_toa_brightness_temperature + '2': + name: '2' + frequency_range: + central: 52.8 + bandwidth: 0.400 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [lon_horn_1, lat_horn_1] + file_type: aws_l1b_nc + file_key: data/calibration/aws_toa_brightness_temperature + '3': + name: '3' + frequency_range: + central: 53.246 + bandwidth: 0.300 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [lon_horn_1, lat_horn_1] + file_type: aws_l1b_nc + file_key: data/calibration/aws_toa_brightness_temperature + '4': + name: '4' + frequency_range: + central: 53.596 + bandwidth: 0.370 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [lon_horn_1, lat_horn_1] + file_type: aws_l1b_nc + file_key: data/calibration/aws_toa_brightness_temperature + '5': + name: '5' + frequency_range: + central: 54.4 + bandwidth: 0.400 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [lon_horn_1, lat_horn_1] + file_type: aws_l1b_nc + file_key: data/calibration/aws_toa_brightness_temperature + '6': + name: '6' + frequency_range: + central: 54.94 + bandwidth: 0.400 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [lon_horn_1, lat_horn_1] + file_type: aws_l1b_nc + file_key: data/calibration/aws_toa_brightness_temperature + '7': + name: '7' + frequency_range: + central: 55.5 + bandwidth: 0.330 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [lon_horn_1, lat_horn_1] + file_type: aws_l1b_nc + file_key: data/calibration/aws_toa_brightness_temperature + '8': + name: '8' + frequency_range: + central: 57.290344 + bandwidth: 0.330 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [lon_horn_1, lat_horn_1] + file_type: aws_l1b_nc + file_key: data/calibration/aws_toa_brightness_temperature + '9': + name: '9' + frequency_range: + central: 89.0 + bandwidth: 4.0 + unit: GHz + polarization: 'QV' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [lon_horn_2, lat_horn_2] + file_type: aws_l1b_nc + file_key: data/calibration/aws_toa_brightness_temperature + '10': + name: '10' + frequency_range: + central: 165.5 + bandwidth: 2.700 + unit: GHz + polarization: 'QH' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [lon_horn_3, lat_horn_3] + file_type: aws_l1b_nc + file_key: data/calibration/aws_toa_brightness_temperature + '11': + name: '11' + frequency_range: + central: 176.311 + bandwidth: 2.0 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [lon_horn_3, lat_horn_3] + file_type: aws_l1b_nc + file_key: data/calibration/aws_toa_brightness_temperature + '12': + name: '12' + frequency_range: + central: 178.811 + bandwidth: 2.0 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [lon_horn_3, lat_horn_3] + file_type: aws_l1b_nc + file_key: data/calibration/aws_toa_brightness_temperature + '13': + name: '13' + frequency_range: + central: 180.311 + bandwidth: 1.0 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [lon_horn_3, lat_horn_3] + file_type: aws_l1b_nc + file_key: data/calibration/aws_toa_brightness_temperature + '14': + name: '14' + frequency_range: + central: 181.511 + bandwidth: 1.0 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [lon_horn_3, lat_horn_3] + file_type: aws_l1b_nc + file_key: data/calibration/aws_toa_brightness_temperature + '15': + name: '15' + frequency_range: + central: 182.311 + bandwidth: 0.5 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [lon_horn_3, lat_horn_3] + file_type: aws_l1b_nc + file_key: data/calibration/aws_toa_brightness_temperature + '16': + name: '16' + frequency_double_sideband: + central: 325.15 + side: 1.2 + bandwidth: 0.8 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [lon_horn_4, lat_horn_4] + file_type: aws_l1b_nc + file_key: data/calibration/aws_toa_brightness_temperature + '17': + name: '17' + frequency_double_sideband: + central: 325.15 + side: 2.4 + bandwidth: 1.2 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [lon_horn_4, lat_horn_4] + file_type: aws_l1b_nc + file_key: data/calibration/aws_toa_brightness_temperature + '18': + name: '18' + frequency_double_sideband: + central: 325.15 + side: 4.1 + bandwidth: 1.8 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [lon_horn_4, lat_horn_4] + file_type: aws_l1b_nc + file_key: data/calibration/aws_toa_brightness_temperature + '19': + name: '19' + frequency_double_sideband: + central: 325.15 + side: 6.6 + bandwidth: 2.8 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [lon_horn_4, lat_horn_4] + file_type: aws_l1b_nc + file_key: data/calibration/aws_toa_brightness_temperature + +# --- Coordinates --- + + lon_horn_1: + name: lon_horn_1 + file_type: aws_l1b_nc + file_key: aws_lon + standard_name: longitude + units: degrees_east + n_horns: 0 + + lat_horn_1: + name: lat_horn_1 + file_type: aws_l1b_nc + file_key: aws_lat + standard_name: latitude + units: degrees_north + n_horns: 0 + + lon_horn_2: + name: lon_horn_2 + file_type: aws_l1b_nc + file_key: data/navigation/aws_lon + standard_name: longitude + units: degrees_east + n_horns: 1 + + lat_horn_2: + name: lat_horn_2 + file_type: aws_l1b_nc + file_key: data/navigation/aws_lat + standard_name: latitude + units: degrees_north + n_horns: 1 + + lon_horn_3: + name: lon_horn_3 + file_type: aws_l1b_nc + file_key: data/navigation/aws_lon + standard_name: longitude + units: degrees_east + n_horns: 2 + + lat_horn_3: + name: lat_horn_3 + file_type: aws_l1b_nc + file_key: data/navigation/aws_lat + standard_name: latitude + units: degrees_north + n_horns: 2 + + lon_horn_4: + name: lon_horn_4 + file_type: aws_l1b_nc + file_key: data/navigation/aws_lon + standard_name: longitude + units: degrees_east + n_horns: 3 + + lat_horn_4: + name: lat_horn_4 + file_type: aws_l1b_nc + file_key: data/navigation/aws_lat + standard_name: latitude + units: degrees_north + n_horns: 3 + + aws_lat: + name: aws_lat + file_type: aws_l1b_nc + file_key: data/navigation/aws_lat + standard_name: latitude + units: degrees_north + + aws_lon: + name: aws_lon + file_type: aws_l1b_nc + file_key: data/navigation/aws_lon + standard_name: longitude + units: degrees_east + +# --- Navigation data --- + + solar_azimuth: + name: solar_azimuth + standard_name: solar_azimuth_angle + file_type: aws_l1b_nc + file_key: data/navigation/aws_solar_azimuth_angle + coordinates: + - aws_lon + - aws_lat + solar_zenith: + name: solar_zenith + standard_name: solar_zenith_angle + file_type: aws_l1b_nc + file_key: data/navigation/aws_solar_zenith_angle + coordinates: + - aws_lon + - aws_lat + satellite_azimuth: + name: satellite_azimuth + standard_name: satellite_azimuth_angle + file_type: aws_l1b_nc + file_key: data/navigation/aws_satellite_azimuth_angle + coordinates: + - aws_lon + - aws_lat + satellite_zenith: + name: satellite_zenith + standard_name: satellite_zenith_angle + file_type: aws_l1b_nc + file_key: data/navigation/aws_satellite_zenith_angle + coordinates: + - aws_lon + - aws_lat + + +file_types: + aws_l1b_nc: + # W_XX-OHB-Unknown,SAT,1-AWS-1B-RAD_C_OHB_20230707124607_G_D_20220621090100_20220621090618_T_B____.nc + # W_XX-OHB-Stockholm,SAT,AWS1-MWR-1B-RAD_C_OHB_20230823161321_G_D_20240115111111_20240115125434_T_B____.nc + file_reader: !!python/name:satpy.readers.aws_l1b.AWSL1BFile + file_patterns: ['W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc'] diff --git a/satpy/readers/aws_l1b.py b/satpy/readers/aws_l1b.py new file mode 100644 index 0000000000..02df6734fc --- /dev/null +++ b/satpy/readers/aws_l1b.py @@ -0,0 +1,311 @@ +# Copyright (c) 2023 Pytroll Developers + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Reader for the Arctic Weather Satellite (AWS) Sounder level-1b data. + +Test data provided by ESA August 23, 2023. +""" + +import logging +from datetime import datetime + +import dask.array as da +import numpy as np +import xarray as xr +from netCDF4 import default_fillvals + +from .netcdf_utils import NetCDF4FileHandler + +logger = logging.getLogger(__name__) + +DUMMY_STARTTIME = datetime(2023, 7, 7, 12, 0) +DUMMY_ENDTIME = datetime(2023, 7, 7, 12, 10) +# dict containing all available auxiliary data parameters to be read using the index map. Keys are the +# parameter name and values are the paths to the variable inside the netcdf + +AUX_DATA = { + 'scantime_utc': 'data/navigation/aws_scantime_utc', + 'solar_azimuth': 'data/navigation/aws_solar_azimuth_angle', + 'solar_zenith': 'data/navigation/aws_solar_zenith_angle', + 'satellite_azimuth': 'data/navigation/aws_satellite_azimuth_angle', + 'satellite_zenith': 'data/navigation/aws_satellite_zenith_angle', + 'surface_type': 'data/navigation/aws_surface_type', + 'terrain_elevation': 'data/navigation/aws_terrain_elevation', + 'aws_lat': 'data/navigation/aws_lat', + 'aws_lon': 'data/navigation/aws_lon', +} + +AWS_CHANNEL_NAMES_TO_NUMBER = {'1': 1, '2': 2, '3': 3, '4': 4, + '5': 5, '6': 6, '7': 7, '8': 8, + '9': 9, '10': 10, '11': 11, '12': 12, + '13': 13, '14': 14, '15': 15, '16': 16, + '17': 17, '18': 18, '19': 19} + +AWS_CHANNEL_NAMES = list(AWS_CHANNEL_NAMES_TO_NUMBER.keys()) +AWS_CHANNELS = set(AWS_CHANNEL_NAMES) + + +def get_channel_index_from_name(chname): + """Get the AWS channel index from the channel name.""" + chindex = AWS_CHANNEL_NAMES_TO_NUMBER.get(chname, 0) - 1 + if 0 <= chindex < 19: + return chindex + raise AttributeError(f"Channel name {chname!r} not supported") + + +def _get_aux_data_name_from_dsname(dsname): + aux_data_name = [key for key in AUX_DATA.keys() if key in dsname] + if len(aux_data_name) > 0: + return aux_data_name[0] + + +class AWSL1BFile(NetCDF4FileHandler): + """Class implementing the AWS L1b Filehandler. + + This class implements the ESA Arctic Weather Satellite (AWS) Level-1b + NetCDF reader. It is designed to be used through the :class:`~satpy.Scene` + class using the :mod:`~satpy.Scene.load` method with the reader + ``"aws_l1b_nc"``. + + """ + + _platform_name_translate = { + "": "AWS", + } + + def __init__(self, filename, filename_info, filetype_info): + """Initialize file handler.""" + xarray_kwargs = {'decode_times': False} + super().__init__(filename, filename_info, + filetype_info, + xarray_kwargs=xarray_kwargs, + cache_var_size=10000, + cache_handle=True) + logger.debug('Reading: {}'.format(self.filename)) + logger.debug('Start: {}'.format(self.start_time)) + logger.debug('End: {}'.format(self.end_time)) + + self._cache = {} + + self._channel_names = AWS_CHANNEL_NAMES + + @property + def start_time(self): + """Get start time.""" + try: + return datetime.strptime(self['/attr/sensing_start_time_utc'], + '%Y-%m-%d %H:%M:%S.%f') + except ValueError: + return DUMMY_STARTTIME + + @property + def end_time(self): + """Get end time.""" + try: + return datetime.strptime(self['/attr/sensing_end_time_utc'], + '%Y-%m-%d %H:%M:%S.%f') + except ValueError: + return DUMMY_ENDTIME + + @property + def sensor(self): + """Get the sensor name.""" + return self['/attr/instrument'] + + @property + def platform_name(self): + """Get the platform name.""" + return self._platform_name_translate.get(self['/attr/Spacecraft']) + + @property + def sub_satellite_longitude_start(self): + """Get the longitude of sub-satellite point at start of the product.""" + return self['status/satellite/subsat_longitude_start'].data.item() + + @property + def sub_satellite_latitude_start(self): + """Get the latitude of sub-satellite point at start of the product.""" + return self['status/satellite/subsat_latitude_start'].data.item() + + @property + def sub_satellite_longitude_end(self): + """Get the longitude of sub-satellite point at end of the product.""" + return self['status/satellite/subsat_longitude_end'].data.item() + + @property + def sub_satellite_latitude_end(self): + """Get the latitude of sub-satellite point at end of the product.""" + return self['status/satellite/subsat_latitude_end'].data.item() + + def get_dataset(self, dataset_id, dataset_info): + """Get dataset using file_key in dataset_info.""" + logger.debug('Reading {} from {}'.format(dataset_id['name'], self.filename)) + + var_key = dataset_info['file_key'] + # if _get_aux_data_name_from_dsname(dataset_id['name']) is not None: + if _get_aux_data_name_from_dsname(var_key) is not None: + nhorn = dataset_info['n_horns'] + variable = self._get_dataset_aux_data(var_key, nhorn) # (dataset_id['name']) + elif dataset_id['name'] in AWS_CHANNELS: + logger.debug(f'Reading in file to get dataset with key {var_key}.') + variable = self._get_dataset_channel(dataset_id, dataset_info) + else: + logger.warning(f'Could not find key {var_key} in NetCDF file, no valid Dataset created') # noqa: E501 + return None + + variable = self._manage_attributes(variable, dataset_info) + variable = self._drop_coords(variable) + variable = self._standardize_dims(variable) + + if dataset_info['standard_name'] in ['longitude', 'latitude']: + lon_or_lat = xr.DataArray( + variable.data[:, :], + attrs=variable.attrs, + dims=(variable.dims[0], variable.dims[1]) + ) + variable = lon_or_lat + + return variable + + @staticmethod + def _standardize_dims(variable): + """Standardize dims to y, x.""" + if 'n_scans' in variable.dims: + variable = variable.rename({'n_fovs': 'x', 'n_scans': 'y'}) + if variable.dims[0] == 'x': + variable = variable.transpose('y', 'x') + return variable + + @staticmethod + def _drop_coords(variable): + """Drop coords that are not in dims.""" + for coord in variable.coords: + if coord not in variable.dims: + variable = variable.drop_vars(coord) + return variable + + def _manage_attributes(self, variable, dataset_info): + """Manage attributes of the dataset.""" + variable.attrs.setdefault('units', None) + variable.attrs.update(dataset_info) + variable.attrs.update(self._get_global_attributes()) + return variable + + def _get_dataset_channel(self, key, dataset_info): + """Load dataset corresponding to channel measurement. + + Load a dataset when the key refers to a measurand, whether uncalibrated + (counts) or calibrated in terms of brightness temperature or radiance. + + """ + # Get the dataset + # Get metadata for given dataset + grp_pth = dataset_info['file_key'] + channel_index = get_channel_index_from_name(key['name']) + + # data = self[grp_pth][:, :, channel_index] + data = self[grp_pth][channel_index, :, :] + data = data.transpose() + # This transposition should not be needed were the format following the EPS-SG format spec!! + attrs = data.attrs.copy() + + fv = attrs.pop( + "FillValue", + default_fillvals.get(data.dtype.str[1:], np.nan)) + vr = attrs.get("valid_range", [-np.inf, np.inf]) + + if key['calibration'] == "counts": + attrs["_FillValue"] = fv + nfv = fv + else: + nfv = np.nan + data = data.where(data >= vr[0], nfv) + data = data.where(data <= vr[1], nfv) + + # Manage the attributes of the dataset + data.attrs.setdefault('units', None) + data.attrs.update(dataset_info) + + dataset_attrs = getattr(data, 'attrs', {}) + dataset_attrs.update(dataset_info) + dataset_attrs.update({ + "platform_name": self.platform_name, + "sensor": self.sensor, + "orbital_parameters": {'sub_satellite_latitude_start': self.sub_satellite_latitude_start, + 'sub_satellite_longitude_start': self.sub_satellite_longitude_start, + 'sub_satellite_latitude_end': self.sub_satellite_latitude_end, + 'sub_satellite_longitude_end': self.sub_satellite_longitude_end}, + }) + + try: + dataset_attrs.update(key.to_dict()) + except AttributeError: + dataset_attrs.update(key) + + data.attrs.update(dataset_attrs) + return data + + def _get_dataset_aux_data(self, dsname, nhorn): + """Get the auxiliary data arrays using the index map.""" + # Geolocation and navigation data: + if dsname in ['aws_lat', 'aws_lon', + 'solar_azimuth', 'solar_zenith', + 'satellite_azimuth', 'satellite_zenith', + 'surface_type', 'terrain_elevation']: + var_key = AUX_DATA.get(dsname) + else: + raise NotImplementedError(f"Dataset {dsname!r} not supported!") + + try: + variable = self[var_key][nhorn, :, :] + except KeyError: + logger.exception("Could not find key %s in NetCDF file, no valid Dataset created", var_key) + raise + + # Scale the data: + if 'scale_factor' in variable.attrs and 'add_offset' in variable.attrs: + missing_value = variable.attrs['missing_value'] + variable.data = da.where(variable.data == missing_value, np.nan, + variable.data * variable.attrs['scale_factor'] + variable.attrs['add_offset']) + + return variable + + def _get_global_attributes(self): + """Create a dictionary of global attributes.""" + return { + 'filename': self.filename, + 'start_time': self.start_time, + 'end_time': self.end_time, + 'spacecraft_name': self.platform_name, + 'sensor': self.sensor, + 'filename_start_time': self.filename_info['start_time'], + 'filename_end_time': self.filename_info['end_time'], + 'platform_name': self.platform_name, + 'quality_group': self._get_quality_attributes(), + } + + def _get_quality_attributes(self): + """Get quality attributes.""" + quality_group = self['quality'] + quality_dict = {} + for key in quality_group: + # Add the values (as Numpy array) of each variable in the group + # where possible + try: + quality_dict[key] = quality_group[key].values + except ValueError: + quality_dict[key] = None + + quality_dict.update(quality_group.attrs) + return quality_dict From 4d6fee9f788a6d8b0bb5a2a36ae2171565d4ea22 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 22 Sep 2023 10:30:41 +0200 Subject: [PATCH 02/48] Fix longitudes to be bbetween -180 and 180 and other fixes for geo-location and angles Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/aws_l1b_nc.yaml | 125 ++++++++++++++++++++---------- satpy/readers/aws_l1b.py | 30 ++++--- 2 files changed, 106 insertions(+), 49 deletions(-) diff --git a/satpy/etc/readers/aws_l1b_nc.yaml b/satpy/etc/readers/aws_l1b_nc.yaml index 86ed11bc16..810ad3eb5a 100644 --- a/satpy/etc/readers/aws_l1b_nc.yaml +++ b/satpy/etc/readers/aws_l1b_nc.yaml @@ -322,7 +322,7 @@ datasets: lon_horn_2: name: lon_horn_2 file_type: aws_l1b_nc - file_key: data/navigation/aws_lon + file_key: aws_lon standard_name: longitude units: degrees_east n_horns: 1 @@ -330,7 +330,7 @@ datasets: lat_horn_2: name: lat_horn_2 file_type: aws_l1b_nc - file_key: data/navigation/aws_lat + file_key: aws_lat standard_name: latitude units: degrees_north n_horns: 1 @@ -338,7 +338,7 @@ datasets: lon_horn_3: name: lon_horn_3 file_type: aws_l1b_nc - file_key: data/navigation/aws_lon + file_key: aws_lon standard_name: longitude units: degrees_east n_horns: 2 @@ -346,7 +346,7 @@ datasets: lat_horn_3: name: lat_horn_3 file_type: aws_l1b_nc - file_key: data/navigation/aws_lat + file_key: aws_lat standard_name: latitude units: degrees_north n_horns: 2 @@ -354,7 +354,7 @@ datasets: lon_horn_4: name: lon_horn_4 file_type: aws_l1b_nc - file_key: data/navigation/aws_lon + file_key: aws_lon standard_name: longitude units: degrees_east n_horns: 3 @@ -362,64 +362,109 @@ datasets: lat_horn_4: name: lat_horn_4 file_type: aws_l1b_nc - file_key: data/navigation/aws_lat + file_key: aws_lat standard_name: latitude units: degrees_north n_horns: 3 - aws_lat: - name: aws_lat - file_type: aws_l1b_nc - file_key: data/navigation/aws_lat - standard_name: latitude - units: degrees_north + # aws_lat: + # name: aws_lat + # file_type: aws_l1b_nc + # file_key: data/navigation/aws_lat + # standard_name: latitude + # units: degrees_north - aws_lon: - name: aws_lon - file_type: aws_l1b_nc - file_key: data/navigation/aws_lon - standard_name: longitude - units: degrees_east + # aws_lon: + # name: aws_lon + # file_type: aws_l1b_nc + # file_key: data/navigation/aws_lon + # standard_name: longitude + # units: degrees_east # --- Navigation data --- - solar_azimuth: - name: solar_azimuth - standard_name: solar_azimuth_angle + solar_azimuth_horn_1: + name: solar_azimuth_horn_1 file_type: aws_l1b_nc file_key: data/navigation/aws_solar_azimuth_angle + standard_name: solar_azimuth_angle + n_horns: 0 coordinates: - - aws_lon - - aws_lat - solar_zenith: - name: solar_zenith - standard_name: solar_zenith_angle + - lon_horn_1 + - lat_horn_1 + + # solar_zenith: + # name: solar_zenith + # standard_name: solar_zenith_angle + # file_type: aws_l1b_nc + # file_key: data/navigation/aws_solar_zenith_angle + # coordinates: + # - aws_lon + # - aws_lat + # satellite_azimuth: + # name: satellite_azimuth + # standard_name: satellite_azimuth_angle + # file_type: aws_l1b_nc + # file_key: data/navigation/aws_satellite_azimuth_angle + # coordinates: + # - aws_lon + # - aws_lat + # satellite_zenith: + # name: satellite_zenith + # standard_name: satellite_zenith_angle + # file_type: aws_l1b_nc + # file_key: data/navigation/aws_satellite_zenith_angle + # coordinates: + # - aws_lon + # - aws_lat + + satellite_zenith_horn_1: + name: satellite_zenith_horn_1 file_type: aws_l1b_nc - file_key: data/navigation/aws_solar_zenith_angle + file_key: data/navigation/aws_satellite_zenith_angle + standard_name: satellite_zenith_angle + n_horns: 0 coordinates: - - aws_lon - - aws_lat - satellite_azimuth: - name: satellite_azimuth - standard_name: satellite_azimuth_angle + - lon_horn_1 + - lat_horn_1 + + satellite_zenith_horn_2: + name: satellite_zenith_horn_2 file_type: aws_l1b_nc - file_key: data/navigation/aws_satellite_azimuth_angle + file_key: data/navigation/aws_satellite_zenith_angle + standard_name: satellite_zenith_angle + n_horns: 1 coordinates: - - aws_lon - - aws_lat - satellite_zenith: - name: satellite_zenith + - lon_horn_2 + - lat_horn_2 + + satellite_zenith_horn_3: + name: satellite_zenith_horn_3 + file_type: aws_l1b_nc + file_key: data/navigation/aws_satellite_zenith_angle standard_name: satellite_zenith_angle + n_horns: 2 + coordinates: + - lon_horn_3 + - lat_horn_3 + + satellite_zenith_horn_4: + name: satellite_zenith_horn_4 file_type: aws_l1b_nc file_key: data/navigation/aws_satellite_zenith_angle + standard_name: satellite_zenith_angle + n_horns: 3 coordinates: - - aws_lon - - aws_lat + - lon_horn_4 + - lat_horn_4 file_types: aws_l1b_nc: # W_XX-OHB-Unknown,SAT,1-AWS-1B-RAD_C_OHB_20230707124607_G_D_20220621090100_20220621090618_T_B____.nc # W_XX-OHB-Stockholm,SAT,AWS1-MWR-1B-RAD_C_OHB_20230823161321_G_D_20240115111111_20240115125434_T_B____.nc + # W_XX-OHB-Stockholm,SAT,AWS1-MWR-1B-RAD_C_OHB_20230816120142_G_D_20240115111111_20240115125434_T_B____radsim.nc file_reader: !!python/name:satpy.readers.aws_l1b.AWSL1BFile - file_patterns: ['W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc'] + file_patterns: [ + 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc', + 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____radsim.nc'] diff --git a/satpy/readers/aws_l1b.py b/satpy/readers/aws_l1b.py index 02df6734fc..aa34852f09 100644 --- a/satpy/readers/aws_l1b.py +++ b/satpy/readers/aws_l1b.py @@ -36,14 +36,16 @@ AUX_DATA = { 'scantime_utc': 'data/navigation/aws_scantime_utc', - 'solar_azimuth': 'data/navigation/aws_solar_azimuth_angle', - 'solar_zenith': 'data/navigation/aws_solar_zenith_angle', - 'satellite_azimuth': 'data/navigation/aws_satellite_azimuth_angle', - 'satellite_zenith': 'data/navigation/aws_satellite_zenith_angle', + 'solar_azimuth_angle': 'data/navigation/aws_solar_azimuth_angle', + 'solar_zenith_angle': 'data/navigation/aws_solar_zenith_angle', + 'satellite_azimuth_angle': 'data/navigation/aws_satellite_azimuth_angle', + 'satellite_zenith_angle': 'data/navigation/aws_satellite_zenith_angle', 'surface_type': 'data/navigation/aws_surface_type', 'terrain_elevation': 'data/navigation/aws_terrain_elevation', 'aws_lat': 'data/navigation/aws_lat', 'aws_lon': 'data/navigation/aws_lon', + 'latitude': 'data/navigation/aws_lat', + 'longitude': 'data/navigation/aws_lon', } AWS_CHANNEL_NAMES_TO_NUMBER = {'1': 1, '2': 2, '3': 3, '4': 4, @@ -156,7 +158,10 @@ def get_dataset(self, dataset_id, dataset_info): # if _get_aux_data_name_from_dsname(dataset_id['name']) is not None: if _get_aux_data_name_from_dsname(var_key) is not None: nhorn = dataset_info['n_horns'] - variable = self._get_dataset_aux_data(var_key, nhorn) # (dataset_id['name']) + standard_name = dataset_info['standard_name'] + + # variable = self._get_dataset_aux_data(var_key, nhorn) # (dataset_id['name']) + variable = self._get_dataset_aux_data(standard_name, nhorn) # (dataset_id['name']) elif dataset_id['name'] in AWS_CHANNELS: logger.debug(f'Reading in file to get dataset with key {var_key}.') variable = self._get_dataset_channel(dataset_id, dataset_info) @@ -169,8 +174,11 @@ def get_dataset(self, dataset_id, dataset_info): variable = self._standardize_dims(variable) if dataset_info['standard_name'] in ['longitude', 'latitude']: + data = variable.data[:, :] + if dataset_info['standard_name'] in ['longitude']: + data = self._scale_lons(data) lon_or_lat = xr.DataArray( - variable.data[:, :], + data, attrs=variable.attrs, dims=(variable.dims[0], variable.dims[1]) ) @@ -178,6 +186,10 @@ def get_dataset(self, dataset_id, dataset_info): return variable + @staticmethod + def _scale_lons(lons): + return xr.where(lons > 180, lons - 360, lons) + @staticmethod def _standardize_dims(variable): """Standardize dims to y, x.""" @@ -259,9 +271,9 @@ def _get_dataset_channel(self, key, dataset_info): def _get_dataset_aux_data(self, dsname, nhorn): """Get the auxiliary data arrays using the index map.""" # Geolocation and navigation data: - if dsname in ['aws_lat', 'aws_lon', - 'solar_azimuth', 'solar_zenith', - 'satellite_azimuth', 'satellite_zenith', + if dsname in ['latitude', 'longitude', + 'solar_azimuth_angle', 'solar_zenith_angle', + 'satellite_azimuth_angle', 'satellite_zenith_angle', 'surface_type', 'terrain_elevation']: var_key = AUX_DATA.get(dsname) else: From 5c3a7d3703d100cae41f51441861e4ce4924201d Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 22 Sep 2023 10:33:06 +0200 Subject: [PATCH 03/48] Add some first basic humidity RGB recipes Signed-off-by: Adam.Dybbroe --- satpy/etc/composites/aws.yaml | 18 ++++++++++++++++++ satpy/etc/enhancements/aws.yaml | 29 +++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 satpy/etc/composites/aws.yaml create mode 100644 satpy/etc/enhancements/aws.yaml diff --git a/satpy/etc/composites/aws.yaml b/satpy/etc/composites/aws.yaml new file mode 100644 index 0000000000..77d2014794 --- /dev/null +++ b/satpy/etc/composites/aws.yaml @@ -0,0 +1,18 @@ +sensor_name: aws + +composites: + mw183_humidity: + compositor: !!python/name:satpy.composites.RGBCompositor + prerequisites: + - name: '15' + - name: '13' + - name: '11' + standard_name: mw183_humidity + + mw183_humidity_surface: + compositor: !!python/name:satpy.composites.RGBCompositor + prerequisites: + - name: '9' + - name: '10' + - name: '15' + standard_name: mw183_humidity_surface diff --git a/satpy/etc/enhancements/aws.yaml b/satpy/etc/enhancements/aws.yaml new file mode 100644 index 0000000000..c997a11350 --- /dev/null +++ b/satpy/etc/enhancements/aws.yaml @@ -0,0 +1,29 @@ +enhancements: + + mw183_humidity: + standard_name: mw183_humidity + operations: + - name: inverse + method: !!python/name:satpy.enhancements.invert + args: + - [true, true, true] + - name: stretch + method: !!python/name:satpy.enhancements.stretch + kwargs: {stretch: linear} + - name: gamma + method: !!python/name:satpy.enhancements.gamma + kwargs: {gamma: 1.2} + + mw183_humidity_surface: + standard_name: mw183_humidity_surface + operations: + - name: inverse + method: !!python/name:satpy.enhancements.invert + args: + - [true, true, true] + - name: stretch + method: !!python/name:satpy.enhancements.stretch + kwargs: {stretch: linear} + - name: gamma + method: !!python/name:satpy.enhancements.gamma + kwargs: {gamma: 1.2} From 8f0d1bf006afdaf3a1ca29095078440b08544502 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 6 Nov 2023 20:56:26 +0100 Subject: [PATCH 04/48] Start working on tests for aws l1b --- satpy/etc/readers/aws_l1b_nc.yaml | 164 +++++-- satpy/readers/aws_l1b.py | 558 ++++++++++++----------- satpy/tests/reader_tests/test_aws_l1b.py | 128 ++++++ 3 files changed, 546 insertions(+), 304 deletions(-) create mode 100644 satpy/tests/reader_tests/test_aws_l1b.py diff --git a/satpy/etc/readers/aws_l1b_nc.yaml b/satpy/etc/readers/aws_l1b_nc.yaml index 810ad3eb5a..0989579a8f 100644 --- a/satpy/etc/readers/aws_l1b_nc.yaml +++ b/satpy/etc/readers/aws_l1b_nc.yaml @@ -306,80 +306,67 @@ datasets: lon_horn_1: name: lon_horn_1 file_type: aws_l1b_nc - file_key: aws_lon standard_name: longitude units: degrees_east n_horns: 0 + file_key: data/navigation/aws_lon + lat_horn_1: name: lat_horn_1 file_type: aws_l1b_nc - file_key: aws_lat standard_name: latitude units: degrees_north n_horns: 0 + file_key: data/navigation/aws_lat lon_horn_2: name: lon_horn_2 file_type: aws_l1b_nc - file_key: aws_lon standard_name: longitude units: degrees_east n_horns: 1 + file_key: data/navigation/aws_lon lat_horn_2: name: lat_horn_2 file_type: aws_l1b_nc - file_key: aws_lat standard_name: latitude units: degrees_north n_horns: 1 + file_key: data/navigation/aws_lat lon_horn_3: name: lon_horn_3 file_type: aws_l1b_nc - file_key: aws_lon standard_name: longitude units: degrees_east n_horns: 2 + file_key: data/navigation/aws_lon lat_horn_3: name: lat_horn_3 file_type: aws_l1b_nc - file_key: aws_lat standard_name: latitude units: degrees_north n_horns: 2 + file_key: data/navigation/aws_lat lon_horn_4: name: lon_horn_4 file_type: aws_l1b_nc - file_key: aws_lon standard_name: longitude units: degrees_east n_horns: 3 + file_key: data/navigation/aws_lon lat_horn_4: name: lat_horn_4 file_type: aws_l1b_nc - file_key: aws_lat standard_name: latitude units: degrees_north n_horns: 3 - - # aws_lat: - # name: aws_lat - # file_type: aws_l1b_nc - # file_key: data/navigation/aws_lat - # standard_name: latitude - # units: degrees_north - - # aws_lon: - # name: aws_lon - # file_type: aws_l1b_nc - # file_key: data/navigation/aws_lon - # standard_name: longitude - # units: degrees_east + file_key: data/navigation/aws_lat # --- Navigation data --- @@ -393,30 +380,115 @@ datasets: - lon_horn_1 - lat_horn_1 - # solar_zenith: - # name: solar_zenith - # standard_name: solar_zenith_angle - # file_type: aws_l1b_nc - # file_key: data/navigation/aws_solar_zenith_angle - # coordinates: - # - aws_lon - # - aws_lat - # satellite_azimuth: - # name: satellite_azimuth - # standard_name: satellite_azimuth_angle - # file_type: aws_l1b_nc - # file_key: data/navigation/aws_satellite_azimuth_angle - # coordinates: - # - aws_lon - # - aws_lat - # satellite_zenith: - # name: satellite_zenith - # standard_name: satellite_zenith_angle - # file_type: aws_l1b_nc - # file_key: data/navigation/aws_satellite_zenith_angle - # coordinates: - # - aws_lon - # - aws_lat + solar_azimuth_horn_2: + name: solar_azimuth_horn_2 + file_type: aws_l1b_nc + file_key: data/navigation/aws_solar_azimuth_angle + standard_name: solar_azimuth_angle + n_horns: 1 + coordinates: + - lon_horn_1 + - lat_horn_1 + + solar_azimuth_horn_3: + name: solar_azimuth_horn_3 + file_type: aws_l1b_nc + file_key: data/navigation/aws_solar_azimuth_angle + standard_name: solar_azimuth_angle + n_horns: 2 + coordinates: + - lon_horn_1 + - lat_horn_1 + + solar_azimuth_horn_4: + name: solar_azimuth_horn_4 + file_type: aws_l1b_nc + file_key: data/navigation/aws_solar_azimuth_angle + standard_name: solar_azimuth_angle + n_horns: 3 + coordinates: + - lon_horn_1 + - lat_horn_1 + + solar_zenith_horn_1: + name: solar_zenith_horn_1 + file_type: aws_l1b_nc + file_key: data/navigation/aws_solar_zenith_angle + standard_name: solar_zenith_angle + n_horns: 0 + coordinates: + - lon_horn_1 + - lat_horn_1 + + solar_zenith_horn_2: + name: solar_zenith_horn_2 + file_type: aws_l1b_nc + file_key: data/navigation/aws_solar_zenith_angle + standard_name: solar_zenith_angle + n_horns: 1 + coordinates: + - lon_horn_1 + - lat_horn_1 + + solar_zenith_horn_3: + name: solar_zenith_horn_3 + file_type: aws_l1b_nc + file_key: data/navigation/aws_solar_zenith_angle + standard_name: solar_zenith_angle + n_horns: 2 + coordinates: + - lon_horn_1 + - lat_horn_1 + + solar_zenith_horn_4: + name: solar_zenith_horn_4 + file_type: aws_l1b_nc + file_key: data/navigation/aws_solar_zenith_angle + standard_name: solar_zenith_angle + n_horns: 3 + coordinates: + - lon_horn_1 + - lat_horn_1 + + satellite_azimuth_horn_1: + name: satellite_azimuth_horn_1 + file_type: aws_l1b_nc + file_key: data/navigation/aws_satellite_azimuth_angle + standard_name: satellite_azimuth_angle + n_horns: 0 + coordinates: + - lon_horn_1 + - lat_horn_1 + + satellite_azimuth_horn_2: + name: satellite_azimuth_horn_2 + file_type: aws_l1b_nc + file_key: data/navigation/aws_satellite_azimuth_angle + standard_name: satellite_azimuth_angle + n_horns: 1 + coordinates: + - lon_horn_2 + - lat_horn_2 + + satellite_azimuth_horn_3: + name: satellite_azimuth_horn_3 + file_type: aws_l1b_nc + file_key: data/navigation/aws_satellite_azimuth_angle + standard_name: satellite_azimuth_angle + n_horns: 2 + coordinates: + - lon_horn_3 + - lat_horn_3 + + satellite_azimuth_horn_4: + name: satellite_azimuth_horn_4 + file_type: aws_l1b_nc + file_key: data/navigation/aws_satellite_azimuth_angle + standard_name: satellite_azimuth_angle + n_horns: 3 + coordinates: + - lon_horn_4 + - lat_horn_4 satellite_zenith_horn_1: name: satellite_zenith_horn_1 diff --git a/satpy/readers/aws_l1b.py b/satpy/readers/aws_l1b.py index aa34852f09..aef8df1a5e 100644 --- a/satpy/readers/aws_l1b.py +++ b/satpy/readers/aws_l1b.py @@ -18,36 +18,41 @@ """ import logging -from datetime import datetime -import dask.array as da -import numpy as np +# import dask.array as da +# import numpy as np import xarray as xr -from netCDF4 import default_fillvals from .netcdf_utils import NetCDF4FileHandler -logger = logging.getLogger(__name__) +# from datetime import datetime + +# from netCDF4 import default_fillvals -DUMMY_STARTTIME = datetime(2023, 7, 7, 12, 0) -DUMMY_ENDTIME = datetime(2023, 7, 7, 12, 10) -# dict containing all available auxiliary data parameters to be read using the index map. Keys are the -# parameter name and values are the paths to the variable inside the netcdf -AUX_DATA = { - 'scantime_utc': 'data/navigation/aws_scantime_utc', - 'solar_azimuth_angle': 'data/navigation/aws_solar_azimuth_angle', - 'solar_zenith_angle': 'data/navigation/aws_solar_zenith_angle', - 'satellite_azimuth_angle': 'data/navigation/aws_satellite_azimuth_angle', - 'satellite_zenith_angle': 'data/navigation/aws_satellite_zenith_angle', - 'surface_type': 'data/navigation/aws_surface_type', - 'terrain_elevation': 'data/navigation/aws_terrain_elevation', - 'aws_lat': 'data/navigation/aws_lat', - 'aws_lon': 'data/navigation/aws_lon', - 'latitude': 'data/navigation/aws_lat', - 'longitude': 'data/navigation/aws_lon', -} +logger = logging.getLogger(__name__) +DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f" + +# DUMMY_STARTTIME = datetime(2023, 7, 7, 12, 0) +# DUMMY_ENDTIME = datetime(2023, 7, 7, 12, 10) +# # dict containing all available auxiliary data parameters to be read using the index map. Keys are the +# # parameter name and values are the paths to the variable inside the netcdf +# +# AUX_DATA = { +# 'scantime_utc': 'data/navigation/aws_scantime_utc', +# 'solar_azimuth_angle': 'data/navigation/aws_solar_azimuth_angle', +# 'solar_zenith_angle': 'data/navigation/aws_solar_zenith_angle', +# 'satellite_azimuth_angle': 'data/navigation/aws_satellite_azimuth_angle', +# 'satellite_zenith_angle': 'data/navigation/aws_satellite_zenith_angle', +# 'surface_type': 'data/navigation/aws_surface_type', +# 'terrain_elevation': 'data/navigation/aws_terrain_elevation', +# 'aws_lat': 'data/navigation/aws_lat', +# 'aws_lon': 'data/navigation/aws_lon', +# 'latitude': 'data/navigation/aws_lat', +# 'longitude': 'data/navigation/aws_lon', +# } +# AWS_CHANNEL_NAMES_TO_NUMBER = {'1': 1, '2': 2, '3': 3, '4': 4, '5': 5, '6': 6, '7': 7, '8': 8, '9': 9, '10': 10, '11': 11, '12': 12, @@ -55,21 +60,23 @@ '17': 17, '18': 18, '19': 19} AWS_CHANNEL_NAMES = list(AWS_CHANNEL_NAMES_TO_NUMBER.keys()) -AWS_CHANNELS = set(AWS_CHANNEL_NAMES) - - -def get_channel_index_from_name(chname): - """Get the AWS channel index from the channel name.""" - chindex = AWS_CHANNEL_NAMES_TO_NUMBER.get(chname, 0) - 1 - if 0 <= chindex < 19: - return chindex - raise AttributeError(f"Channel name {chname!r} not supported") - - -def _get_aux_data_name_from_dsname(dsname): - aux_data_name = [key for key in AUX_DATA.keys() if key in dsname] - if len(aux_data_name) > 0: - return aux_data_name[0] +# AWS_CHANNELS = set(AWS_CHANNEL_NAMES) +# +# +# def get_channel_index_from_name(chname): +# """Get the AWS channel index from the channel name.""" +# chindex = AWS_CHANNEL_NAMES_TO_NUMBER.get(chname, 0) - 1 +# if 0 <= chindex < 19: +# return chindex +# raise AttributeError(f"Channel name {chname!r} not supported") +# +# +# def _get_aux_data_name_from_dsname(dsname): +# aux_data_name = [key for key in AUX_DATA.keys() if key in dsname] +# if len(aux_data_name) > 0: +# return aux_data_name[0] +# +# class AWSL1BFile(NetCDF4FileHandler): @@ -82,43 +89,20 @@ class using the :mod:`~satpy.Scene.load` method with the reader """ - _platform_name_translate = { - "": "AWS", - } - - def __init__(self, filename, filename_info, filetype_info): - """Initialize file handler.""" - xarray_kwargs = {'decode_times': False} - super().__init__(filename, filename_info, - filetype_info, - xarray_kwargs=xarray_kwargs, - cache_var_size=10000, - cache_handle=True) - logger.debug('Reading: {}'.format(self.filename)) - logger.debug('Start: {}'.format(self.start_time)) - logger.debug('End: {}'.format(self.end_time)) - - self._cache = {} - - self._channel_names = AWS_CHANNEL_NAMES + def __init__(self, filename, filename_info, filetype_info, auto_maskandscale=True): + """Initialize the handler.""" + super().__init__(filename, filename_info, filetype_info) + self.filename_info = filename_info @property def start_time(self): - """Get start time.""" - try: - return datetime.strptime(self['/attr/sensing_start_time_utc'], - '%Y-%m-%d %H:%M:%S.%f') - except ValueError: - return DUMMY_STARTTIME + """Get the start time.""" + return self.filename_info["start_time"] @property def end_time(self): - """Get end time.""" - try: - return datetime.strptime(self['/attr/sensing_end_time_utc'], - '%Y-%m-%d %H:%M:%S.%f') - except ValueError: - return DUMMY_ENDTIME + """Get the end time.""" + return self.filename_info["end_time"] @property def sensor(self): @@ -128,196 +112,254 @@ def sensor(self): @property def platform_name(self): """Get the platform name.""" - return self._platform_name_translate.get(self['/attr/Spacecraft']) - - @property - def sub_satellite_longitude_start(self): - """Get the longitude of sub-satellite point at start of the product.""" - return self['status/satellite/subsat_longitude_start'].data.item() - - @property - def sub_satellite_latitude_start(self): - """Get the latitude of sub-satellite point at start of the product.""" - return self['status/satellite/subsat_latitude_start'].data.item() - - @property - def sub_satellite_longitude_end(self): - """Get the longitude of sub-satellite point at end of the product.""" - return self['status/satellite/subsat_longitude_end'].data.item() - - @property - def sub_satellite_latitude_end(self): - """Get the latitude of sub-satellite point at end of the product.""" - return self['status/satellite/subsat_latitude_end'].data.item() + return self.filename_info["platform_name"] def get_dataset(self, dataset_id, dataset_info): - """Get dataset using file_key in dataset_info.""" - logger.debug('Reading {} from {}'.format(dataset_id['name'], self.filename)) - - var_key = dataset_info['file_key'] - # if _get_aux_data_name_from_dsname(dataset_id['name']) is not None: - if _get_aux_data_name_from_dsname(var_key) is not None: - nhorn = dataset_info['n_horns'] - standard_name = dataset_info['standard_name'] - - # variable = self._get_dataset_aux_data(var_key, nhorn) # (dataset_id['name']) - variable = self._get_dataset_aux_data(standard_name, nhorn) # (dataset_id['name']) - elif dataset_id['name'] in AWS_CHANNELS: - logger.debug(f'Reading in file to get dataset with key {var_key}.') - variable = self._get_dataset_channel(dataset_id, dataset_info) - else: - logger.warning(f'Could not find key {var_key} in NetCDF file, no valid Dataset created') # noqa: E501 - return None - - variable = self._manage_attributes(variable, dataset_info) - variable = self._drop_coords(variable) - variable = self._standardize_dims(variable) - - if dataset_info['standard_name'] in ['longitude', 'latitude']: - data = variable.data[:, :] - if dataset_info['standard_name'] in ['longitude']: - data = self._scale_lons(data) - lon_or_lat = xr.DataArray( - data, - attrs=variable.attrs, - dims=(variable.dims[0], variable.dims[1]) - ) - variable = lon_or_lat - - return variable - - @staticmethod - def _scale_lons(lons): - return xr.where(lons > 180, lons - 360, lons) - - @staticmethod - def _standardize_dims(variable): - """Standardize dims to y, x.""" - if 'n_scans' in variable.dims: - variable = variable.rename({'n_fovs': 'x', 'n_scans': 'y'}) - if variable.dims[0] == 'x': - variable = variable.transpose('y', 'x') - return variable - - @staticmethod - def _drop_coords(variable): - """Drop coords that are not in dims.""" - for coord in variable.coords: - if coord not in variable.dims: - variable = variable.drop_vars(coord) - return variable - - def _manage_attributes(self, variable, dataset_info): - """Manage attributes of the dataset.""" - variable.attrs.setdefault('units', None) - variable.attrs.update(dataset_info) - variable.attrs.update(self._get_global_attributes()) - return variable - - def _get_dataset_channel(self, key, dataset_info): - """Load dataset corresponding to channel measurement. - - Load a dataset when the key refers to a measurand, whether uncalibrated - (counts) or calibrated in terms of brightness temperature or radiance. - - """ - # Get the dataset - # Get metadata for given dataset - grp_pth = dataset_info['file_key'] - channel_index = get_channel_index_from_name(key['name']) - - # data = self[grp_pth][:, :, channel_index] - data = self[grp_pth][channel_index, :, :] - data = data.transpose() - # This transposition should not be needed were the format following the EPS-SG format spec!! - attrs = data.attrs.copy() - - fv = attrs.pop( - "FillValue", - default_fillvals.get(data.dtype.str[1:], np.nan)) - vr = attrs.get("valid_range", [-np.inf, np.inf]) - - if key['calibration'] == "counts": - attrs["_FillValue"] = fv - nfv = fv + """Get the data.""" + if dataset_id["name"] in AWS_CHANNEL_NAMES: + channel_data = self[dataset_info["file_key"]] + channel_data.coords["n_channels"] = AWS_CHANNEL_NAMES + data_array = channel_data.sel(n_channels=dataset_id["name"]) + elif (dataset_id["name"].startswith("lon") or dataset_id["name"].startswith("lat") + or dataset_id["name"].startswith("solar_azimuth") + or dataset_id["name"].startswith("solar_zenith") + or dataset_id["name"].startswith("satellite_azimuth") + or dataset_id["name"].startswith("satellite_zenith")): + channel_data = self[dataset_info["file_key"]] + channel_data.coords["n_geo_groups"] = [0, 1, 2, 3] + n_horn = dataset_info["n_horns"] + data_array = channel_data.sel(n_geo_groups=n_horn) else: - nfv = np.nan - data = data.where(data >= vr[0], nfv) - data = data.where(data <= vr[1], nfv) - - # Manage the attributes of the dataset - data.attrs.setdefault('units', None) - data.attrs.update(dataset_info) - - dataset_attrs = getattr(data, 'attrs', {}) - dataset_attrs.update(dataset_info) - dataset_attrs.update({ - "platform_name": self.platform_name, - "sensor": self.sensor, - "orbital_parameters": {'sub_satellite_latitude_start': self.sub_satellite_latitude_start, - 'sub_satellite_longitude_start': self.sub_satellite_longitude_start, - 'sub_satellite_latitude_end': self.sub_satellite_latitude_end, - 'sub_satellite_longitude_end': self.sub_satellite_longitude_end}, - }) - - try: - dataset_attrs.update(key.to_dict()) - except AttributeError: - dataset_attrs.update(key) - - data.attrs.update(dataset_attrs) - return data - - def _get_dataset_aux_data(self, dsname, nhorn): - """Get the auxiliary data arrays using the index map.""" - # Geolocation and navigation data: - if dsname in ['latitude', 'longitude', - 'solar_azimuth_angle', 'solar_zenith_angle', - 'satellite_azimuth_angle', 'satellite_zenith_angle', - 'surface_type', 'terrain_elevation']: - var_key = AUX_DATA.get(dsname) - else: - raise NotImplementedError(f"Dataset {dsname!r} not supported!") - - try: - variable = self[var_key][nhorn, :, :] - except KeyError: - logger.exception("Could not find key %s in NetCDF file, no valid Dataset created", var_key) - raise - - # Scale the data: - if 'scale_factor' in variable.attrs and 'add_offset' in variable.attrs: - missing_value = variable.attrs['missing_value'] - variable.data = da.where(variable.data == missing_value, np.nan, - variable.data * variable.attrs['scale_factor'] + variable.attrs['add_offset']) - - return variable - - def _get_global_attributes(self): - """Create a dictionary of global attributes.""" - return { - 'filename': self.filename, - 'start_time': self.start_time, - 'end_time': self.end_time, - 'spacecraft_name': self.platform_name, - 'sensor': self.sensor, - 'filename_start_time': self.filename_info['start_time'], - 'filename_end_time': self.filename_info['end_time'], - 'platform_name': self.platform_name, - 'quality_group': self._get_quality_attributes(), - } - - def _get_quality_attributes(self): - """Get quality attributes.""" - quality_group = self['quality'] - quality_dict = {} - for key in quality_group: - # Add the values (as Numpy array) of each variable in the group - # where possible - try: - quality_dict[key] = quality_group[key].values - except ValueError: - quality_dict[key] = None - - quality_dict.update(quality_group.attrs) - return quality_dict + raise NotImplementedError + + if "scale_factor" in data_array.attrs and "add_offset" in data_array.attrs: + with xr.set_options(keep_attrs=True): + data_array = data_array * data_array.attrs["scale_factor"] + data_array.attrs["add_offset"] + data_array.attrs.pop("scale_factor") + data_array.attrs.pop("add_offset") + # if "missing_value" in data_array.attrs: + # with xr.set_options(keep_attrs=True): + # data_array = data_array.where(data_array != data_array.attrs["missing_value"]) + return data_array + + +# class AWSL1BFile(NetCDF4FileHandler): +# """Class implementing the AWS L1b Filehandler. +# +# This class implements the ESA Arctic Weather Satellite (AWS) Level-1b +# NetCDF reader. It is designed to be used through the :class:`~satpy.Scene` +# class using the :mod:`~satpy.Scene.load` method with the reader +# ``"aws_l1b_nc"``. +# +# """ +# +# _platform_name_translate = { +# "": "AWS", +# } +# +# def __init__(self, filename, filename_info, filetype_info): +# """Initialize file handler.""" +# xarray_kwargs = {'decode_times': False} +# super().__init__(filename, filename_info, +# filetype_info, +# xarray_kwargs=xarray_kwargs, +# cache_var_size=10000, +# cache_handle=True) +# logger.debug('Reading: {}'.format(self.filename)) +# logger.debug('Start: {}'.format(self.start_time)) +# logger.debug('End: {}'.format(self.end_time)) +## +# self._channel_names = AWS_CHANNEL_NAMES + +# +# @property +# def sub_satellite_longitude_start(self): +# """Get the longitude of sub-satellite point at start of the product.""" +# return self['status/satellite/subsat_longitude_start'].data.item() +# +# @property +# def sub_satellite_latitude_start(self): +# """Get the latitude of sub-satellite point at start of the product.""" +# return self['status/satellite/subsat_latitude_start'].data.item() +# +# @property +# def sub_satellite_longitude_end(self): +# """Get the longitude of sub-satellite point at end of the product.""" +# return self['status/satellite/subsat_longitude_end'].data.item() +# +# @property +# def sub_satellite_latitude_end(self): +# """Get the latitude of sub-satellite point at end of the product.""" +# return self['status/satellite/subsat_latitude_end'].data.item() +# +# def get_dataset(self, dataset_id, dataset_info): +# """Get dataset using file_key in dataset_info.""" +# logger.debug('Reading {} from {}'.format(dataset_id['name'], self.filename)) +# +# var_key = dataset_info['file_key'] +# # if _get_aux_data_name_from_dsname(dataset_id['name']) is not None: +# if _get_aux_data_name_from_dsname(var_key) is not None: +# nhorn = dataset_info['n_horns'] +# standard_name = dataset_info['standard_name'] +# +# # variable = self._get_dataset_aux_data(var_key, nhorn) # (dataset_id['name']) +# variable = self._get_dataset_aux_data(standard_name, nhorn) # (dataset_id['name']) +# elif dataset_id['name'] in AWS_CHANNELS: +# logger.debug(f'Reading in file to get dataset with key {var_key}.') +# variable = self._get_dataset_channel(dataset_id, dataset_info) +# else: +# logger.warning(f'Could not find key {var_key} in NetCDF file, no valid Dataset created') # noqa: E501 +# return None +# +# variable = self._manage_attributes(variable, dataset_info) +# variable = self._drop_coords(variable) +# variable = self._standardize_dims(variable) +# +# if dataset_info['standard_name'] in ['longitude', 'latitude']: +# data = variable.data[:, :] +# if dataset_info['standard_name'] in ['longitude']: +# data = self._scale_lons(data) +# lon_or_lat = xr.DataArray( +# data, +# attrs=variable.attrs, +# dims=(variable.dims[0], variable.dims[1]) +# ) +# variable = lon_or_lat +# +# return variable +# +# @staticmethod +# def _scale_lons(lons): +# return xr.where(lons > 180, lons - 360, lons) +# +# @staticmethod +# def _standardize_dims(variable): +# """Standardize dims to y, x.""" +# if 'n_scans' in variable.dims: +# variable = variable.rename({'n_fovs': 'x', 'n_scans': 'y'}) +# if variable.dims[0] == 'x': +# variable = variable.transpose('y', 'x') +# return variable +# +# @staticmethod +# def _drop_coords(variable): +# """Drop coords that are not in dims.""" +# for coord in variable.coords: +# if coord not in variable.dims: +# variable = variable.drop_vars(coord) +# return variable +# +# def _manage_attributes(self, variable, dataset_info): +# """Manage attributes of the dataset.""" +# variable.attrs.setdefault('units', None) +# variable.attrs.update(dataset_info) +# variable.attrs.update(self._get_global_attributes()) +# return variable +# +# def _get_dataset_channel(self, key, dataset_info): +# """Load dataset corresponding to channel measurement. +# +# Load a dataset when the key refers to a measurand, whether uncalibrated +# (counts) or calibrated in terms of brightness temperature or radiance. +# +# """ +# # Get the dataset +# # Get metadata for given dataset +# grp_pth = dataset_info['file_key'] +# channel_index = get_channel_index_from_name(key['name']) +# +# # data = self[grp_pth][:, :, channel_index] +# data = self[grp_pth][channel_index, :, :] +# data = data.transpose() +# # This transposition should not be needed were the format following the EPS-SG format spec!! +# attrs = data.attrs.copy() +# +# fv = attrs.pop( +# "FillValue", +# default_fillvals.get(data.dtype.str[1:], np.nan)) +# vr = attrs.get("valid_range", [-np.inf, np.inf]) +# +# if key['calibration'] == "counts": +# attrs["_FillValue"] = fv +# nfv = fv +# else: +# nfv = np.nan +# data = data.where(data >= vr[0], nfv) +# data = data.where(data <= vr[1], nfv) +# +# # Manage the attributes of the dataset +# data.attrs.setdefault('units', None) +# data.attrs.update(dataset_info) +# +# dataset_attrs = getattr(data, 'attrs', {}) +# dataset_attrs.update(dataset_info) +# dataset_attrs.update({ +# "platform_name": self.platform_name, +# "sensor": self.sensor, +# "orbital_parameters": {'sub_satellite_latitude_start': self.sub_satellite_latitude_start, +# 'sub_satellite_longitude_start': self.sub_satellite_longitude_start, +# 'sub_satellite_latitude_end': self.sub_satellite_latitude_end, +# 'sub_satellite_longitude_end': self.sub_satellite_longitude_end}, +# }) +# +# try: +# dataset_attrs.update(key.to_dict()) +# except AttributeError: +# dataset_attrs.update(key) +# +# data.attrs.update(dataset_attrs) +# return data +# +# def _get_dataset_aux_data(self, dsname, nhorn): +# """Get the auxiliary data arrays using the index map.""" +# # Geolocation and navigation data: +# if dsname in ['latitude', 'longitude', +# 'solar_azimuth_angle', 'solar_zenith_angle', +# 'satellite_azimuth_angle', 'satellite_zenith_angle', +# 'surface_type', 'terrain_elevation']: +# var_key = AUX_DATA.get(dsname) +# else: +# raise NotImplementedError(f"Dataset {dsname!r} not supported!") +# +# try: +# variable = self[var_key][nhorn, :, :] +# except KeyError: +# logger.exception("Could not find key %s in NetCDF file, no valid Dataset created", var_key) +# raise +# +# # Scale the data: +# if 'scale_factor' in variable.attrs and 'add_offset' in variable.attrs: +# missing_value = variable.attrs['missing_value'] +# variable.data = da.where(variable.data == missing_value, np.nan, +# variable.data * variable.attrs['scale_factor'] + variable.attrs['add_offset']) +# +# return variable +# +# def _get_global_attributes(self): +# """Create a dictionary of global attributes.""" +# return { +# 'filename': self.filename, +# 'start_time': self.start_time, +# 'end_time': self.end_time, +# 'spacecraft_name': self.platform_name, +# 'sensor': self.sensor, +# 'filename_start_time': self.filename_info['start_time'], +# 'filename_end_time': self.filename_info['end_time'], +# 'platform_name': self.platform_name, +# 'quality_group': self._get_quality_attributes(), +# } +# +# def _get_quality_attributes(self): +# """Get quality attributes.""" +# quality_group = self['quality'] +# quality_dict = {} +# for key in quality_group: +# # Add the values (as Numpy array) of each variable in the group +# # where possible +# try: +# quality_dict[key] = quality_group[key].values +# except ValueError: +# quality_dict[key] = None +# +# quality_dict.update(quality_group.attrs) +# return quality_dict diff --git a/satpy/tests/reader_tests/test_aws_l1b.py b/satpy/tests/reader_tests/test_aws_l1b.py new file mode 100644 index 0000000000..f3df0b077a --- /dev/null +++ b/satpy/tests/reader_tests/test_aws_l1b.py @@ -0,0 +1,128 @@ +"""Tests for aws l1b filehandlers.""" + +import os +from datetime import datetime, timedelta +from random import randrange + +import numpy as np +import pytest +import xarray as xr +from datatree import DataTree +from trollsift import compose, parse + +from satpy.readers.aws_l1b import DATETIME_FORMAT, AWSL1BFile + +platform_name = "AWS1" +file_pattern = "W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc" # noqa +fake_data = xr.DataArray(np.random.randint(0, 700000, size=19*5*5).reshape((19, 5, 5)), + dims=["n_channels", "n_fovs", "n_scans"]) +fake_lon_data = xr.DataArray(np.random.randint(0, 3599999, size=25 * 4).reshape((4, 5, 5)), + dims=["n_geo_groups", "n_fovs", "n_scans"]) +fake_lat_data = xr.DataArray(np.random.randint(-900000, 900000, size=25 * 4).reshape((4, 5, 5)), + dims=["n_geo_groups", "n_fovs", "n_scans"]) +fake_sun_azi_data = xr.DataArray(np.random.randint(0, 36000, size=25 * 4).reshape((4, 5, 5)), + dims=["n_geo_groups", "n_fovs", "n_scans"]) +fake_sun_zen_data = xr.DataArray(np.random.randint(0, 36000, size=25 * 4).reshape((4, 5, 5)), + dims=["n_geo_groups", "n_fovs", "n_scans"]) +fake_sat_azi_data = xr.DataArray(np.random.randint(0, 36000, size=25 * 4).reshape((4, 5, 5)), + dims=["n_geo_groups", "n_fovs", "n_scans"]) +fake_sat_zen_data = xr.DataArray(np.random.randint(0, 36000, size=25 * 4).reshape((4, 5, 5)), + dims=["n_geo_groups", "n_fovs", "n_scans"]) + + +def random_date(start, end): + """Create a random datetime between two datetimes.""" + delta = end - start + int_delta = (delta.days * 24 * 60 * 60) + delta.seconds + random_second = randrange(int_delta) + return start + timedelta(seconds=random_second) + + +@pytest.fixture(scope="session") +def aws_file(tmp_path_factory): + """Create an AWS file.""" + ds = DataTree() + start_time = random_date(datetime(2024, 6, 1), datetime(2030, 6, 1)) + ds.attrs["sensing_start_time_utc"] = start_time.strftime(DATETIME_FORMAT) + end_time = random_date(datetime(2024, 6, 1), datetime(2030, 6, 1)) + ds.attrs["sensing_end_time_utc"] = end_time.strftime(DATETIME_FORMAT) + processing_time = random_date(datetime(2024, 6, 1), datetime(2030, 6, 1)) + + instrument = "AWS" + ds.attrs["instrument"] = instrument + ds["data/calibration/aws_toa_brightness_temperature"] = fake_data + ds["data/calibration/aws_toa_brightness_temperature"].attrs["scale_factor"] = 0.001 + ds["data/calibration/aws_toa_brightness_temperature"].attrs["add_offset"] = 0.0 + ds["data/navigation/aws_lon"] = fake_lon_data + ds["data/navigation/aws_lat"] = fake_lat_data + ds["data/navigation/aws_solar_azimuth_angle"] = fake_sun_azi_data + ds["data/navigation/aws_solar_zenith_angle"] = fake_sun_zen_data + ds["data/navigation/aws_satellite_azimuth_angle"] = fake_sat_azi_data + ds["data/navigation/aws_satellite_zenith_angle"] = fake_sat_zen_data + + tmp_dir = tmp_path_factory.mktemp("aws_l1b_tests") + filename = tmp_dir / compose(file_pattern, dict(start_time=start_time, end_time=end_time, + processing_time=processing_time, platform_name=platform_name)) + + ds.to_netcdf(filename) + return filename + + +@pytest.fixture +def aws_handler(aws_file): + """Create an aws filehandler.""" + filename_info = parse(file_pattern, os.path.basename(aws_file)) + return AWSL1BFile(aws_file, filename_info, dict()) + + +def test_start_end_time(aws_file): + """Test that start and end times are read correctly.""" + filename_info = parse(file_pattern, os.path.basename(aws_file)) + handler = AWSL1BFile(aws_file, filename_info, dict()) + + assert handler.start_time == filename_info["start_time"] + assert handler.end_time == filename_info["end_time"] + + +def test_metadata(aws_handler): + """Test that the metadata is read correctly.""" + assert aws_handler.sensor == "AWS" + assert aws_handler.platform_name == platform_name + + +def test_get_channel_data(aws_handler): + """Test retrieving the channel data.""" + did = dict(name="1") + dataset_info = dict(file_key="data/calibration/aws_toa_brightness_temperature") + np.testing.assert_allclose(aws_handler.get_dataset(did, dataset_info), fake_data.isel(n_channels=0) * 0.001) + + +@pytest.mark.parametrize(["id_name", "file_key", "fake_array"], + [("lon_horn_1", "data/navigation/aws_lon", fake_lon_data), + ("lat_horn_1", "data/navigation/aws_lat", fake_lat_data), + ("solar_azimuth_horn_1", "data/navigation/aws_solar_azimuth_angle", fake_sun_azi_data), + ("solar_zenith_horn_1", "data/navigation/aws_solar_zenith_angle", fake_sun_zen_data), + ("satellite_azimuth_horn_1", "data/navigation/aws_satellite_azimuth_angle", + fake_sat_azi_data), + ("satellite_zenith_horn_1", "data/navigation/aws_satellite_zenith_angle", + fake_sat_zen_data)]) +def test_get_navigation_data(aws_handler, id_name, file_key, fake_array): + """Test retrieving the angles_data.""" + did = dict(name=id_name) + dataset_info = dict(file_key=file_key, n_horns=0) + np.testing.assert_allclose(aws_handler.get_dataset(did, dataset_info), fake_array.isel(n_geo_groups=0)) + + +# def test_channel_is_masked_and_scaled(): +# pass + +# def test_navigation_is_scaled_and_scaled(): +# pass + + +# def test_orbital_parameters_are_provided(): +# pass + + +# def test_coords_contain_xy(): +# pass From 7191df150afadafd1b861b3ae2239103dd418232 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Tue, 7 Nov 2023 09:52:49 +0100 Subject: [PATCH 05/48] Finalize aws_l1b refactoring and tests --- satpy/etc/readers/aws_l1b_nc.yaml | 297 ++++++------------- satpy/readers/aws_l1b.py | 344 +++++------------------ satpy/tests/reader_tests/test_aws_l1b.py | 82 ++++-- 3 files changed, 214 insertions(+), 509 deletions(-) diff --git a/satpy/etc/readers/aws_l1b_nc.yaml b/satpy/etc/readers/aws_l1b_nc.yaml index 0989579a8f..adee00ed2a 100644 --- a/satpy/etc/readers/aws_l1b_nc.yaml +++ b/satpy/etc/readers/aws_l1b_nc.yaml @@ -20,6 +20,12 @@ reader: enum: - QH - QV + horn: + enum: + - "1" + - "2" + - "3" + - "4" calibration: enum: - brightness_temperature @@ -29,6 +35,21 @@ reader: default: [] type: !!python/name:satpy.dataset.ModifierTuple + coord_identification_keys: + name: + required: true + resolution: + polarization: + enum: + - QH + - QV + horn: + enum: + - "1" + - "2" + - "3" + - "4" + datasets: '1': name: '1' @@ -41,7 +62,8 @@ datasets: calibration: brightness_temperature: standard_name: toa_brightness_temperature - coordinates: [lon_horn_1, lat_horn_1] + horn: "1" + coordinates: [longitude, latitude] file_type: aws_l1b_nc file_key: data/calibration/aws_toa_brightness_temperature '2': @@ -55,7 +77,8 @@ datasets: calibration: brightness_temperature: standard_name: toa_brightness_temperature - coordinates: [lon_horn_1, lat_horn_1] + horn: "1" + coordinates: [longitude, latitude] file_type: aws_l1b_nc file_key: data/calibration/aws_toa_brightness_temperature '3': @@ -69,7 +92,8 @@ datasets: calibration: brightness_temperature: standard_name: toa_brightness_temperature - coordinates: [lon_horn_1, lat_horn_1] + horn: "1" + coordinates: [longitude, latitude] file_type: aws_l1b_nc file_key: data/calibration/aws_toa_brightness_temperature '4': @@ -83,7 +107,8 @@ datasets: calibration: brightness_temperature: standard_name: toa_brightness_temperature - coordinates: [lon_horn_1, lat_horn_1] + horn: "1" + coordinates: [longitude, latitude] file_type: aws_l1b_nc file_key: data/calibration/aws_toa_brightness_temperature '5': @@ -97,7 +122,8 @@ datasets: calibration: brightness_temperature: standard_name: toa_brightness_temperature - coordinates: [lon_horn_1, lat_horn_1] + horn: "1" + coordinates: [longitude, latitude] file_type: aws_l1b_nc file_key: data/calibration/aws_toa_brightness_temperature '6': @@ -111,7 +137,8 @@ datasets: calibration: brightness_temperature: standard_name: toa_brightness_temperature - coordinates: [lon_horn_1, lat_horn_1] + horn: "1" + coordinates: [longitude, latitude] file_type: aws_l1b_nc file_key: data/calibration/aws_toa_brightness_temperature '7': @@ -125,7 +152,8 @@ datasets: calibration: brightness_temperature: standard_name: toa_brightness_temperature - coordinates: [lon_horn_1, lat_horn_1] + horn: "1" + coordinates: [longitude, latitude] file_type: aws_l1b_nc file_key: data/calibration/aws_toa_brightness_temperature '8': @@ -139,7 +167,8 @@ datasets: calibration: brightness_temperature: standard_name: toa_brightness_temperature - coordinates: [lon_horn_1, lat_horn_1] + horn: "1" + coordinates: [longitude, latitude] file_type: aws_l1b_nc file_key: data/calibration/aws_toa_brightness_temperature '9': @@ -153,7 +182,8 @@ datasets: calibration: brightness_temperature: standard_name: toa_brightness_temperature - coordinates: [lon_horn_2, lat_horn_2] + horn: "2" + coordinates: [longitude, latitude] file_type: aws_l1b_nc file_key: data/calibration/aws_toa_brightness_temperature '10': @@ -167,7 +197,8 @@ datasets: calibration: brightness_temperature: standard_name: toa_brightness_temperature - coordinates: [lon_horn_3, lat_horn_3] + horn: "3" + coordinates: [longitude, latitude] file_type: aws_l1b_nc file_key: data/calibration/aws_toa_brightness_temperature '11': @@ -181,7 +212,8 @@ datasets: calibration: brightness_temperature: standard_name: toa_brightness_temperature - coordinates: [lon_horn_3, lat_horn_3] + horn: "3" + coordinates: [longitude, latitude] file_type: aws_l1b_nc file_key: data/calibration/aws_toa_brightness_temperature '12': @@ -195,7 +227,8 @@ datasets: calibration: brightness_temperature: standard_name: toa_brightness_temperature - coordinates: [lon_horn_3, lat_horn_3] + horn: "3" + coordinates: [longitude, latitude] file_type: aws_l1b_nc file_key: data/calibration/aws_toa_brightness_temperature '13': @@ -209,7 +242,8 @@ datasets: calibration: brightness_temperature: standard_name: toa_brightness_temperature - coordinates: [lon_horn_3, lat_horn_3] + horn: "3" + coordinates: [longitude, latitude] file_type: aws_l1b_nc file_key: data/calibration/aws_toa_brightness_temperature '14': @@ -223,7 +257,8 @@ datasets: calibration: brightness_temperature: standard_name: toa_brightness_temperature - coordinates: [lon_horn_3, lat_horn_3] + horn: "3" + coordinates: [longitude, latitude] file_type: aws_l1b_nc file_key: data/calibration/aws_toa_brightness_temperature '15': @@ -237,7 +272,8 @@ datasets: calibration: brightness_temperature: standard_name: toa_brightness_temperature - coordinates: [lon_horn_3, lat_horn_3] + horn: "3" + coordinates: [longitude, latitude] file_type: aws_l1b_nc file_key: data/calibration/aws_toa_brightness_temperature '16': @@ -252,7 +288,8 @@ datasets: calibration: brightness_temperature: standard_name: toa_brightness_temperature - coordinates: [lon_horn_4, lat_horn_4] + horn: "4" + coordinates: [longitude, latitude] file_type: aws_l1b_nc file_key: data/calibration/aws_toa_brightness_temperature '17': @@ -267,7 +304,8 @@ datasets: calibration: brightness_temperature: standard_name: toa_brightness_temperature - coordinates: [lon_horn_4, lat_horn_4] + horn: "4" + coordinates: [longitude, latitude] file_type: aws_l1b_nc file_key: data/calibration/aws_toa_brightness_temperature '18': @@ -282,7 +320,8 @@ datasets: calibration: brightness_temperature: standard_name: toa_brightness_temperature - coordinates: [lon_horn_4, lat_horn_4] + horn: "4" + coordinates: [longitude, latitude] file_type: aws_l1b_nc file_key: data/calibration/aws_toa_brightness_temperature '19': @@ -297,238 +336,72 @@ datasets: calibration: brightness_temperature: standard_name: toa_brightness_temperature - coordinates: [lon_horn_4, lat_horn_4] + horn: "4" + coordinates: [longitude, latitude] file_type: aws_l1b_nc file_key: data/calibration/aws_toa_brightness_temperature # --- Coordinates --- - lon_horn_1: - name: lon_horn_1 + longitude: + name: longitude file_type: aws_l1b_nc standard_name: longitude units: degrees_east - n_horns: 0 + horn: ["1", "2", "3", "4"] file_key: data/navigation/aws_lon - lat_horn_1: - name: lat_horn_1 - file_type: aws_l1b_nc - standard_name: latitude - units: degrees_north - n_horns: 0 - file_key: data/navigation/aws_lat - - lon_horn_2: - name: lon_horn_2 - file_type: aws_l1b_nc - standard_name: longitude - units: degrees_east - n_horns: 1 - file_key: data/navigation/aws_lon - - lat_horn_2: - name: lat_horn_2 - file_type: aws_l1b_nc - standard_name: latitude - units: degrees_north - n_horns: 1 - file_key: data/navigation/aws_lat - - lon_horn_3: - name: lon_horn_3 - file_type: aws_l1b_nc - standard_name: longitude - units: degrees_east - n_horns: 2 - file_key: data/navigation/aws_lon - - lat_horn_3: - name: lat_horn_3 + latitude: + name: latitude file_type: aws_l1b_nc standard_name: latitude units: degrees_north - n_horns: 2 + horn: ["1", "2", "3", "4"] file_key: data/navigation/aws_lat - lon_horn_4: - name: lon_horn_4 - file_type: aws_l1b_nc - standard_name: longitude - units: degrees_east - n_horns: 3 - file_key: data/navigation/aws_lon - - lat_horn_4: - name: lat_horn_4 - file_type: aws_l1b_nc - standard_name: latitude - units: degrees_north - n_horns: 3 - file_key: data/navigation/aws_lat # --- Navigation data --- - solar_azimuth_horn_1: - name: solar_azimuth_horn_1 - file_type: aws_l1b_nc - file_key: data/navigation/aws_solar_azimuth_angle - standard_name: solar_azimuth_angle - n_horns: 0 - coordinates: - - lon_horn_1 - - lat_horn_1 - - solar_azimuth_horn_2: - name: solar_azimuth_horn_2 + solar_azimuth: + name: solar_azimuth file_type: aws_l1b_nc file_key: data/navigation/aws_solar_azimuth_angle standard_name: solar_azimuth_angle - n_horns: 1 + horn: ["1", "2", "3", "4"] coordinates: - - lon_horn_1 - - lat_horn_1 + - longitude + - latitude - solar_azimuth_horn_3: - name: solar_azimuth_horn_3 - file_type: aws_l1b_nc - file_key: data/navigation/aws_solar_azimuth_angle - standard_name: solar_azimuth_angle - n_horns: 2 - coordinates: - - lon_horn_1 - - lat_horn_1 - - solar_azimuth_horn_4: - name: solar_azimuth_horn_4 - file_type: aws_l1b_nc - file_key: data/navigation/aws_solar_azimuth_angle - standard_name: solar_azimuth_angle - n_horns: 3 - coordinates: - - lon_horn_1 - - lat_horn_1 - - solar_zenith_horn_1: - name: solar_zenith_horn_1 + solar_zenith: + name: solar_zenith file_type: aws_l1b_nc file_key: data/navigation/aws_solar_zenith_angle standard_name: solar_zenith_angle - n_horns: 0 + horn: ["1", "2", "3", "4"] coordinates: - - lon_horn_1 - - lat_horn_1 + - longitude + - latitude - solar_zenith_horn_2: - name: solar_zenith_horn_2 - file_type: aws_l1b_nc - file_key: data/navigation/aws_solar_zenith_angle - standard_name: solar_zenith_angle - n_horns: 1 - coordinates: - - lon_horn_1 - - lat_horn_1 - - solar_zenith_horn_3: - name: solar_zenith_horn_3 - file_type: aws_l1b_nc - file_key: data/navigation/aws_solar_zenith_angle - standard_name: solar_zenith_angle - n_horns: 2 - coordinates: - - lon_horn_1 - - lat_horn_1 - - solar_zenith_horn_4: - name: solar_zenith_horn_4 - file_type: aws_l1b_nc - file_key: data/navigation/aws_solar_zenith_angle - standard_name: solar_zenith_angle - n_horns: 3 - coordinates: - - lon_horn_1 - - lat_horn_1 - - satellite_azimuth_horn_1: - name: satellite_azimuth_horn_1 + satellite_azimuth: + name: satellite_azimuth file_type: aws_l1b_nc file_key: data/navigation/aws_satellite_azimuth_angle standard_name: satellite_azimuth_angle - n_horns: 0 - coordinates: - - lon_horn_1 - - lat_horn_1 - - satellite_azimuth_horn_2: - name: satellite_azimuth_horn_2 - file_type: aws_l1b_nc - file_key: data/navigation/aws_satellite_azimuth_angle - standard_name: satellite_azimuth_angle - n_horns: 1 - coordinates: - - lon_horn_2 - - lat_horn_2 - - satellite_azimuth_horn_3: - name: satellite_azimuth_horn_3 - file_type: aws_l1b_nc - file_key: data/navigation/aws_satellite_azimuth_angle - standard_name: satellite_azimuth_angle - n_horns: 2 - coordinates: - - lon_horn_3 - - lat_horn_3 - - satellite_azimuth_horn_4: - name: satellite_azimuth_horn_4 - file_type: aws_l1b_nc - file_key: data/navigation/aws_satellite_azimuth_angle - standard_name: satellite_azimuth_angle - n_horns: 3 - coordinates: - - lon_horn_4 - - lat_horn_4 - - satellite_zenith_horn_1: - name: satellite_zenith_horn_1 - file_type: aws_l1b_nc - file_key: data/navigation/aws_satellite_zenith_angle - standard_name: satellite_zenith_angle - n_horns: 0 - coordinates: - - lon_horn_1 - - lat_horn_1 - - satellite_zenith_horn_2: - name: satellite_zenith_horn_2 - file_type: aws_l1b_nc - file_key: data/navigation/aws_satellite_zenith_angle - standard_name: satellite_zenith_angle - n_horns: 1 - coordinates: - - lon_horn_2 - - lat_horn_2 - - satellite_zenith_horn_3: - name: satellite_zenith_horn_3 - file_type: aws_l1b_nc - file_key: data/navigation/aws_satellite_zenith_angle - standard_name: satellite_zenith_angle - n_horns: 2 + horn: ["1", "2", "3", "4"] coordinates: - - lon_horn_3 - - lat_horn_3 + - longitude + - latitude - satellite_zenith_horn_4: - name: satellite_zenith_horn_4 + satellite_zenith: + name: satellite_zenith file_type: aws_l1b_nc file_key: data/navigation/aws_satellite_zenith_angle standard_name: satellite_zenith_angle - n_horns: 3 + horn: ["1", "2", "3", "4"] coordinates: - - lon_horn_4 - - lat_horn_4 + - longitude + - latitude file_types: diff --git a/satpy/readers/aws_l1b.py b/satpy/readers/aws_l1b.py index aef8df1a5e..465c4b86a2 100644 --- a/satpy/readers/aws_l1b.py +++ b/satpy/readers/aws_l1b.py @@ -19,40 +19,14 @@ import logging -# import dask.array as da -# import numpy as np import xarray as xr from .netcdf_utils import NetCDF4FileHandler -# from datetime import datetime - -# from netCDF4 import default_fillvals - - logger = logging.getLogger(__name__) DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f" -# DUMMY_STARTTIME = datetime(2023, 7, 7, 12, 0) -# DUMMY_ENDTIME = datetime(2023, 7, 7, 12, 10) -# # dict containing all available auxiliary data parameters to be read using the index map. Keys are the -# # parameter name and values are the paths to the variable inside the netcdf -# -# AUX_DATA = { -# 'scantime_utc': 'data/navigation/aws_scantime_utc', -# 'solar_azimuth_angle': 'data/navigation/aws_solar_azimuth_angle', -# 'solar_zenith_angle': 'data/navigation/aws_solar_zenith_angle', -# 'satellite_azimuth_angle': 'data/navigation/aws_satellite_azimuth_angle', -# 'satellite_zenith_angle': 'data/navigation/aws_satellite_zenith_angle', -# 'surface_type': 'data/navigation/aws_surface_type', -# 'terrain_elevation': 'data/navigation/aws_terrain_elevation', -# 'aws_lat': 'data/navigation/aws_lat', -# 'aws_lon': 'data/navigation/aws_lon', -# 'latitude': 'data/navigation/aws_lat', -# 'longitude': 'data/navigation/aws_lon', -# } -# AWS_CHANNEL_NAMES_TO_NUMBER = {'1': 1, '2': 2, '3': 3, '4': 4, '5': 5, '6': 6, '7': 7, '8': 8, '9': 9, '10': 10, '11': 11, '12': 12, @@ -60,23 +34,6 @@ '17': 17, '18': 18, '19': 19} AWS_CHANNEL_NAMES = list(AWS_CHANNEL_NAMES_TO_NUMBER.keys()) -# AWS_CHANNELS = set(AWS_CHANNEL_NAMES) -# -# -# def get_channel_index_from_name(chname): -# """Get the AWS channel index from the channel name.""" -# chindex = AWS_CHANNEL_NAMES_TO_NUMBER.get(chname, 0) - 1 -# if 0 <= chindex < 19: -# return chindex -# raise AttributeError(f"Channel name {chname!r} not supported") -# -# -# def _get_aux_data_name_from_dsname(dsname): -# aux_data_name = [key for key in AUX_DATA.keys() if key in dsname] -# if len(aux_data_name) > 0: -# return aux_data_name[0] -# -# class AWSL1BFile(NetCDF4FileHandler): @@ -91,7 +48,9 @@ class using the :mod:`~satpy.Scene.load` method with the reader def __init__(self, filename, filename_info, filetype_info, auto_maskandscale=True): """Initialize the handler.""" - super().__init__(filename, filename_info, filetype_info) + super().__init__(filename, filename_info, filetype_info, + cache_var_size=10000, + cache_handle=True) self.filename_info = filename_info @property @@ -114,240 +73,87 @@ def platform_name(self): """Get the platform name.""" return self.filename_info["platform_name"] + @property + def sub_satellite_longitude_start(self): + """Get the longitude of sub-satellite point at start of the product.""" + return self['status/satellite/subsat_longitude_start'].data.item() + + @property + def sub_satellite_latitude_start(self): + """Get the latitude of sub-satellite point at start of the product.""" + return self['status/satellite/subsat_latitude_start'].data.item() + + @property + def sub_satellite_longitude_end(self): + """Get the longitude of sub-satellite point at end of the product.""" + return self['status/satellite/subsat_longitude_end'].data.item() + + @property + def sub_satellite_latitude_end(self): + """Get the latitude of sub-satellite point at end of the product.""" + return self['status/satellite/subsat_latitude_end'].data.item() + def get_dataset(self, dataset_id, dataset_info): """Get the data.""" if dataset_id["name"] in AWS_CHANNEL_NAMES: - channel_data = self[dataset_info["file_key"]] - channel_data.coords["n_channels"] = AWS_CHANNEL_NAMES - data_array = channel_data.sel(n_channels=dataset_id["name"]) - elif (dataset_id["name"].startswith("lon") or dataset_id["name"].startswith("lat") - or dataset_id["name"].startswith("solar_azimuth") - or dataset_id["name"].startswith("solar_zenith") - or dataset_id["name"].startswith("satellite_azimuth") - or dataset_id["name"].startswith("satellite_zenith")): - channel_data = self[dataset_info["file_key"]] - channel_data.coords["n_geo_groups"] = [0, 1, 2, 3] - n_horn = dataset_info["n_horns"] - data_array = channel_data.sel(n_geo_groups=n_horn) + data_array = self._get_channel_data(dataset_id, dataset_info) + elif (dataset_id["name"] in ["longitude", "latitude", + "solar_azimuth", "solar_zenith", + "satellite_zenith", "satellite_azimuth"]): + data_array = self._get_navigation_data(dataset_id, dataset_info) else: raise NotImplementedError - if "scale_factor" in data_array.attrs and "add_offset" in data_array.attrs: - with xr.set_options(keep_attrs=True): - data_array = data_array * data_array.attrs["scale_factor"] + data_array.attrs["add_offset"] - data_array.attrs.pop("scale_factor") - data_array.attrs.pop("add_offset") - # if "missing_value" in data_array.attrs: - # with xr.set_options(keep_attrs=True): - # data_array = data_array.where(data_array != data_array.attrs["missing_value"]) - return data_array + data_array = mask_and_scale(data_array) + if dataset_id["name"] == "longitude": + data_array = data_array.where(data_array <= 180, data_array - 360) + data_array.attrs.update(dataset_info) -# class AWSL1BFile(NetCDF4FileHandler): -# """Class implementing the AWS L1b Filehandler. -# -# This class implements the ESA Arctic Weather Satellite (AWS) Level-1b -# NetCDF reader. It is designed to be used through the :class:`~satpy.Scene` -# class using the :mod:`~satpy.Scene.load` method with the reader -# ``"aws_l1b_nc"``. -# -# """ -# -# _platform_name_translate = { -# "": "AWS", -# } -# -# def __init__(self, filename, filename_info, filetype_info): -# """Initialize file handler.""" -# xarray_kwargs = {'decode_times': False} -# super().__init__(filename, filename_info, -# filetype_info, -# xarray_kwargs=xarray_kwargs, -# cache_var_size=10000, -# cache_handle=True) -# logger.debug('Reading: {}'.format(self.filename)) -# logger.debug('Start: {}'.format(self.start_time)) -# logger.debug('End: {}'.format(self.end_time)) -## -# self._channel_names = AWS_CHANNEL_NAMES + data_array.attrs["orbital_parameters"] = {'sub_satellite_latitude_start': self.sub_satellite_latitude_start, + 'sub_satellite_longitude_start': self.sub_satellite_longitude_start, + 'sub_satellite_latitude_end': self.sub_satellite_latitude_end, + 'sub_satellite_longitude_end': self.sub_satellite_longitude_end} + + data_array.attrs["platform_name"] = self.platform_name + data_array.attrs["sensor"] = self.sensor + return data_array + + def _get_channel_data(self, dataset_id, dataset_info): + channel_data = self[dataset_info["file_key"]] + channel_data.coords["n_channels"] = AWS_CHANNEL_NAMES + channel_data = channel_data.rename({'n_fovs': 'x', 'n_scans': 'y'}) + return channel_data.sel(n_channels=dataset_id["name"]).drop_vars("n_channels") + + def _get_navigation_data(self, dataset_id, dataset_info): + geo_data = self[dataset_info["file_key"]] + geo_data.coords["n_geo_groups"] = ["1", "2", "3", "4"] + geo_data = geo_data.rename({'n_fovs': 'x', 'n_scans': 'y'}) + horn = dataset_id["horn"].name + return geo_data.sel(n_geo_groups=horn).drop_vars("n_geo_groups") + + +def mask_and_scale(data_array): + """Mask then scale the data array.""" + if "missing_value" in data_array.attrs: + with xr.set_options(keep_attrs=True): + data_array = data_array.where(data_array != data_array.attrs["missing_value"]) + data_array.attrs.pop("missing_value") + if "valid_max" in data_array.attrs: + with xr.set_options(keep_attrs=True): + data_array = data_array.where(data_array <= data_array.attrs["valid_max"]) + data_array.attrs.pop("valid_max") + if "valid_min" in data_array.attrs: + with xr.set_options(keep_attrs=True): + data_array = data_array.where(data_array >= data_array.attrs["valid_min"]) + data_array.attrs.pop("valid_min") + if "scale_factor" in data_array.attrs and "add_offset" in data_array.attrs: + with xr.set_options(keep_attrs=True): + data_array = data_array * data_array.attrs["scale_factor"] + data_array.attrs["add_offset"] + data_array.attrs.pop("scale_factor") + data_array.attrs.pop("add_offset") + return data_array -# -# @property -# def sub_satellite_longitude_start(self): -# """Get the longitude of sub-satellite point at start of the product.""" -# return self['status/satellite/subsat_longitude_start'].data.item() -# -# @property -# def sub_satellite_latitude_start(self): -# """Get the latitude of sub-satellite point at start of the product.""" -# return self['status/satellite/subsat_latitude_start'].data.item() -# -# @property -# def sub_satellite_longitude_end(self): -# """Get the longitude of sub-satellite point at end of the product.""" -# return self['status/satellite/subsat_longitude_end'].data.item() -# -# @property -# def sub_satellite_latitude_end(self): -# """Get the latitude of sub-satellite point at end of the product.""" -# return self['status/satellite/subsat_latitude_end'].data.item() -# -# def get_dataset(self, dataset_id, dataset_info): -# """Get dataset using file_key in dataset_info.""" -# logger.debug('Reading {} from {}'.format(dataset_id['name'], self.filename)) -# -# var_key = dataset_info['file_key'] -# # if _get_aux_data_name_from_dsname(dataset_id['name']) is not None: -# if _get_aux_data_name_from_dsname(var_key) is not None: -# nhorn = dataset_info['n_horns'] -# standard_name = dataset_info['standard_name'] -# -# # variable = self._get_dataset_aux_data(var_key, nhorn) # (dataset_id['name']) -# variable = self._get_dataset_aux_data(standard_name, nhorn) # (dataset_id['name']) -# elif dataset_id['name'] in AWS_CHANNELS: -# logger.debug(f'Reading in file to get dataset with key {var_key}.') -# variable = self._get_dataset_channel(dataset_id, dataset_info) -# else: -# logger.warning(f'Could not find key {var_key} in NetCDF file, no valid Dataset created') # noqa: E501 -# return None -# -# variable = self._manage_attributes(variable, dataset_info) -# variable = self._drop_coords(variable) -# variable = self._standardize_dims(variable) -# -# if dataset_info['standard_name'] in ['longitude', 'latitude']: -# data = variable.data[:, :] -# if dataset_info['standard_name'] in ['longitude']: -# data = self._scale_lons(data) -# lon_or_lat = xr.DataArray( -# data, -# attrs=variable.attrs, -# dims=(variable.dims[0], variable.dims[1]) -# ) -# variable = lon_or_lat -# -# return variable -# -# @staticmethod -# def _scale_lons(lons): -# return xr.where(lons > 180, lons - 360, lons) -# -# @staticmethod -# def _standardize_dims(variable): -# """Standardize dims to y, x.""" -# if 'n_scans' in variable.dims: -# variable = variable.rename({'n_fovs': 'x', 'n_scans': 'y'}) -# if variable.dims[0] == 'x': -# variable = variable.transpose('y', 'x') -# return variable -# -# @staticmethod -# def _drop_coords(variable): -# """Drop coords that are not in dims.""" -# for coord in variable.coords: -# if coord not in variable.dims: -# variable = variable.drop_vars(coord) -# return variable -# -# def _manage_attributes(self, variable, dataset_info): -# """Manage attributes of the dataset.""" -# variable.attrs.setdefault('units', None) -# variable.attrs.update(dataset_info) -# variable.attrs.update(self._get_global_attributes()) -# return variable -# -# def _get_dataset_channel(self, key, dataset_info): -# """Load dataset corresponding to channel measurement. -# -# Load a dataset when the key refers to a measurand, whether uncalibrated -# (counts) or calibrated in terms of brightness temperature or radiance. -# -# """ -# # Get the dataset -# # Get metadata for given dataset -# grp_pth = dataset_info['file_key'] -# channel_index = get_channel_index_from_name(key['name']) -# -# # data = self[grp_pth][:, :, channel_index] -# data = self[grp_pth][channel_index, :, :] -# data = data.transpose() -# # This transposition should not be needed were the format following the EPS-SG format spec!! -# attrs = data.attrs.copy() -# -# fv = attrs.pop( -# "FillValue", -# default_fillvals.get(data.dtype.str[1:], np.nan)) -# vr = attrs.get("valid_range", [-np.inf, np.inf]) -# -# if key['calibration'] == "counts": -# attrs["_FillValue"] = fv -# nfv = fv -# else: -# nfv = np.nan -# data = data.where(data >= vr[0], nfv) -# data = data.where(data <= vr[1], nfv) -# -# # Manage the attributes of the dataset -# data.attrs.setdefault('units', None) -# data.attrs.update(dataset_info) -# -# dataset_attrs = getattr(data, 'attrs', {}) -# dataset_attrs.update(dataset_info) -# dataset_attrs.update({ -# "platform_name": self.platform_name, -# "sensor": self.sensor, -# "orbital_parameters": {'sub_satellite_latitude_start': self.sub_satellite_latitude_start, -# 'sub_satellite_longitude_start': self.sub_satellite_longitude_start, -# 'sub_satellite_latitude_end': self.sub_satellite_latitude_end, -# 'sub_satellite_longitude_end': self.sub_satellite_longitude_end}, -# }) -# -# try: -# dataset_attrs.update(key.to_dict()) -# except AttributeError: -# dataset_attrs.update(key) -# -# data.attrs.update(dataset_attrs) -# return data -# -# def _get_dataset_aux_data(self, dsname, nhorn): -# """Get the auxiliary data arrays using the index map.""" -# # Geolocation and navigation data: -# if dsname in ['latitude', 'longitude', -# 'solar_azimuth_angle', 'solar_zenith_angle', -# 'satellite_azimuth_angle', 'satellite_zenith_angle', -# 'surface_type', 'terrain_elevation']: -# var_key = AUX_DATA.get(dsname) -# else: -# raise NotImplementedError(f"Dataset {dsname!r} not supported!") -# -# try: -# variable = self[var_key][nhorn, :, :] -# except KeyError: -# logger.exception("Could not find key %s in NetCDF file, no valid Dataset created", var_key) -# raise -# -# # Scale the data: -# if 'scale_factor' in variable.attrs and 'add_offset' in variable.attrs: -# missing_value = variable.attrs['missing_value'] -# variable.data = da.where(variable.data == missing_value, np.nan, -# variable.data * variable.attrs['scale_factor'] + variable.attrs['add_offset']) -# -# return variable -# -# def _get_global_attributes(self): -# """Create a dictionary of global attributes.""" -# return { -# 'filename': self.filename, -# 'start_time': self.start_time, -# 'end_time': self.end_time, -# 'spacecraft_name': self.platform_name, -# 'sensor': self.sensor, -# 'filename_start_time': self.filename_info['start_time'], -# 'filename_end_time': self.filename_info['end_time'], -# 'platform_name': self.platform_name, -# 'quality_group': self._get_quality_attributes(), -# } # # def _get_quality_attributes(self): # """Get quality attributes.""" diff --git a/satpy/tests/reader_tests/test_aws_l1b.py b/satpy/tests/reader_tests/test_aws_l1b.py index f3df0b077a..7ab910d194 100644 --- a/satpy/tests/reader_tests/test_aws_l1b.py +++ b/satpy/tests/reader_tests/test_aws_l1b.py @@ -2,6 +2,7 @@ import os from datetime import datetime, timedelta +from enum import Enum from random import randrange import numpy as np @@ -14,7 +15,12 @@ platform_name = "AWS1" file_pattern = "W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc" # noqa -fake_data = xr.DataArray(np.random.randint(0, 700000, size=19*5*5).reshape((19, 5, 5)), +fake_data_np = np.random.randint(0, 700000, size=19*5*5).reshape((19, 5, 5)) +fake_data_np[0, 0, 0] = -2147483648 +fake_data_np[0, 0, 1] = 700000 + 10 +fake_data_np[0, 0, 2] = -10 + +fake_data = xr.DataArray(fake_data_np, dims=["n_channels", "n_fovs", "n_scans"]) fake_lon_data = xr.DataArray(np.random.randint(0, 3599999, size=25 * 4).reshape((4, 5, 5)), dims=["n_geo_groups", "n_fovs", "n_scans"]) @@ -53,12 +59,22 @@ def aws_file(tmp_path_factory): ds["data/calibration/aws_toa_brightness_temperature"] = fake_data ds["data/calibration/aws_toa_brightness_temperature"].attrs["scale_factor"] = 0.001 ds["data/calibration/aws_toa_brightness_temperature"].attrs["add_offset"] = 0.0 + ds["data/calibration/aws_toa_brightness_temperature"].attrs["missing_value"] = -2147483648 + ds["data/calibration/aws_toa_brightness_temperature"].attrs["valid_min"] = 0 + ds["data/calibration/aws_toa_brightness_temperature"].attrs["valid_max"] = 700000 + ds["data/navigation/aws_lon"] = fake_lon_data + ds["data/navigation/aws_lon"].attrs["scale_factor"] = 1e-4 + ds["data/navigation/aws_lon"].attrs["add_offset"] = 0.0 ds["data/navigation/aws_lat"] = fake_lat_data ds["data/navigation/aws_solar_azimuth_angle"] = fake_sun_azi_data ds["data/navigation/aws_solar_zenith_angle"] = fake_sun_zen_data ds["data/navigation/aws_satellite_azimuth_angle"] = fake_sat_azi_data ds["data/navigation/aws_satellite_zenith_angle"] = fake_sat_zen_data + ds['status/satellite/subsat_latitude_end'] = np.array(22.39) + ds['status/satellite/subsat_longitude_start'] = np.array(304.79) + ds['status/satellite/subsat_latitude_start'] = np.array(55.41) + ds['status/satellite/subsat_longitude_end'] = np.array(296.79) tmp_dir = tmp_path_factory.mktemp("aws_l1b_tests") filename = tmp_dir / compose(file_pattern, dict(start_time=start_time, end_time=end_time, @@ -94,35 +110,45 @@ def test_get_channel_data(aws_handler): """Test retrieving the channel data.""" did = dict(name="1") dataset_info = dict(file_key="data/calibration/aws_toa_brightness_temperature") - np.testing.assert_allclose(aws_handler.get_dataset(did, dataset_info), fake_data.isel(n_channels=0) * 0.001) + expected = fake_data.isel(n_channels=0) + expected = expected.where(expected != -2147483648) + expected = expected.where(expected <= 700000) + expected = expected.where(expected >= 0) + expected = expected * 0.001 + res = aws_handler.get_dataset(did, dataset_info) + np.testing.assert_allclose(res, expected) + assert "x" in res.dims + assert "y" in res.dims + assert "orbital_parameters" in res.attrs + assert res.attrs["orbital_parameters"]["sub_satellite_longitude_end"] == 296.79 + assert res.dims == ("x", "y") + assert "n_channels" not in res.coords + assert res.attrs["sensor"] == "AWS" + assert res.attrs["platform_name"] == "AWS1" @pytest.mark.parametrize(["id_name", "file_key", "fake_array"], - [("lon_horn_1", "data/navigation/aws_lon", fake_lon_data), - ("lat_horn_1", "data/navigation/aws_lat", fake_lat_data), - ("solar_azimuth_horn_1", "data/navigation/aws_solar_azimuth_angle", fake_sun_azi_data), - ("solar_zenith_horn_1", "data/navigation/aws_solar_zenith_angle", fake_sun_zen_data), - ("satellite_azimuth_horn_1", "data/navigation/aws_satellite_azimuth_angle", - fake_sat_azi_data), - ("satellite_zenith_horn_1", "data/navigation/aws_satellite_zenith_angle", - fake_sat_zen_data)]) + [("longitude", "data/navigation/aws_lon", fake_lon_data * 1e-4), + ("latitude", "data/navigation/aws_lat", fake_lat_data), + ("solar_azimuth", "data/navigation/aws_solar_azimuth_angle", fake_sun_azi_data), + ("solar_zenith", "data/navigation/aws_solar_zenith_angle", fake_sun_zen_data), + ("satellite_azimuth", "data/navigation/aws_satellite_azimuth_angle", fake_sat_azi_data), + ("satellite_zenith", "data/navigation/aws_satellite_zenith_angle", fake_sat_zen_data)]) def test_get_navigation_data(aws_handler, id_name, file_key, fake_array): """Test retrieving the angles_data.""" - did = dict(name=id_name) - dataset_info = dict(file_key=file_key, n_horns=0) - np.testing.assert_allclose(aws_handler.get_dataset(did, dataset_info), fake_array.isel(n_geo_groups=0)) - - -# def test_channel_is_masked_and_scaled(): -# pass - -# def test_navigation_is_scaled_and_scaled(): -# pass - - -# def test_orbital_parameters_are_provided(): -# pass - - -# def test_coords_contain_xy(): -# pass + Horn = Enum("Horn", ["1", "2", "3", "4"]) + did = dict(name=id_name, horn=Horn["1"]) + dataset_info = dict(file_key=file_key, standard_name=id_name) + res = aws_handler.get_dataset(did, dataset_info) + if id_name == "longitude": + fake_array = fake_array.where(fake_array <= 180, fake_array - 360) + + np.testing.assert_allclose(res, fake_array.isel(n_geo_groups=0)) + assert "x" in res.dims + assert "y" in res.dims + assert "orbital_parameters" in res.attrs + assert res.dims == ("x", "y") + assert "standard_name" in res.attrs + assert "n_geo_groups" not in res.coords + if id_name == "longitude": + assert res.max() <= 180 From e526ff5f1f6716dc68063638fe4fa7acc03139b6 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Tue, 7 Nov 2023 09:57:58 +0100 Subject: [PATCH 06/48] Replace single quotes with double quotes --- satpy/readers/aws_l1b.py | 32 ++++++++++++------------ satpy/tests/reader_tests/test_aws_l1b.py | 12 ++++----- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/satpy/readers/aws_l1b.py b/satpy/readers/aws_l1b.py index 465c4b86a2..afeae3b80e 100644 --- a/satpy/readers/aws_l1b.py +++ b/satpy/readers/aws_l1b.py @@ -27,11 +27,11 @@ DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f" -AWS_CHANNEL_NAMES_TO_NUMBER = {'1': 1, '2': 2, '3': 3, '4': 4, - '5': 5, '6': 6, '7': 7, '8': 8, - '9': 9, '10': 10, '11': 11, '12': 12, - '13': 13, '14': 14, '15': 15, '16': 16, - '17': 17, '18': 18, '19': 19} +AWS_CHANNEL_NAMES_TO_NUMBER = {"1": 1, "2": 2, "3": 3, "4": 4, + "5": 5, "6": 6, "7": 7, "8": 8, + "9": 9, "10": 10, "11": 11, "12": 12, + "13": 13, "14": 14, "15": 15, "16": 16, + "17": 17, "18": 18, "19": 19} AWS_CHANNEL_NAMES = list(AWS_CHANNEL_NAMES_TO_NUMBER.keys()) @@ -66,7 +66,7 @@ def end_time(self): @property def sensor(self): """Get the sensor name.""" - return self['/attr/instrument'] + return self["/attr/instrument"] @property def platform_name(self): @@ -76,22 +76,22 @@ def platform_name(self): @property def sub_satellite_longitude_start(self): """Get the longitude of sub-satellite point at start of the product.""" - return self['status/satellite/subsat_longitude_start'].data.item() + return self["status/satellite/subsat_longitude_start"].data.item() @property def sub_satellite_latitude_start(self): """Get the latitude of sub-satellite point at start of the product.""" - return self['status/satellite/subsat_latitude_start'].data.item() + return self["status/satellite/subsat_latitude_start"].data.item() @property def sub_satellite_longitude_end(self): """Get the longitude of sub-satellite point at end of the product.""" - return self['status/satellite/subsat_longitude_end'].data.item() + return self["status/satellite/subsat_longitude_end"].data.item() @property def sub_satellite_latitude_end(self): """Get the latitude of sub-satellite point at end of the product.""" - return self['status/satellite/subsat_latitude_end'].data.item() + return self["status/satellite/subsat_latitude_end"].data.item() def get_dataset(self, dataset_id, dataset_info): """Get the data.""" @@ -110,10 +110,10 @@ def get_dataset(self, dataset_id, dataset_info): data_array.attrs.update(dataset_info) - data_array.attrs["orbital_parameters"] = {'sub_satellite_latitude_start': self.sub_satellite_latitude_start, - 'sub_satellite_longitude_start': self.sub_satellite_longitude_start, - 'sub_satellite_latitude_end': self.sub_satellite_latitude_end, - 'sub_satellite_longitude_end': self.sub_satellite_longitude_end} + data_array.attrs["orbital_parameters"] = {"sub_satellite_latitude_start": self.sub_satellite_latitude_start, + "sub_satellite_longitude_start": self.sub_satellite_longitude_start, + "sub_satellite_latitude_end": self.sub_satellite_latitude_end, + "sub_satellite_longitude_end": self.sub_satellite_longitude_end} data_array.attrs["platform_name"] = self.platform_name data_array.attrs["sensor"] = self.sensor @@ -122,13 +122,13 @@ def get_dataset(self, dataset_id, dataset_info): def _get_channel_data(self, dataset_id, dataset_info): channel_data = self[dataset_info["file_key"]] channel_data.coords["n_channels"] = AWS_CHANNEL_NAMES - channel_data = channel_data.rename({'n_fovs': 'x', 'n_scans': 'y'}) + channel_data = channel_data.rename({"n_fovs": "x", "n_scans": "y"}) return channel_data.sel(n_channels=dataset_id["name"]).drop_vars("n_channels") def _get_navigation_data(self, dataset_id, dataset_info): geo_data = self[dataset_info["file_key"]] geo_data.coords["n_geo_groups"] = ["1", "2", "3", "4"] - geo_data = geo_data.rename({'n_fovs': 'x', 'n_scans': 'y'}) + geo_data = geo_data.rename({"n_fovs": "x", "n_scans": "y"}) horn = dataset_id["horn"].name return geo_data.sel(n_geo_groups=horn).drop_vars("n_geo_groups") diff --git a/satpy/tests/reader_tests/test_aws_l1b.py b/satpy/tests/reader_tests/test_aws_l1b.py index 7ab910d194..480c692b57 100644 --- a/satpy/tests/reader_tests/test_aws_l1b.py +++ b/satpy/tests/reader_tests/test_aws_l1b.py @@ -71,10 +71,10 @@ def aws_file(tmp_path_factory): ds["data/navigation/aws_solar_zenith_angle"] = fake_sun_zen_data ds["data/navigation/aws_satellite_azimuth_angle"] = fake_sat_azi_data ds["data/navigation/aws_satellite_zenith_angle"] = fake_sat_zen_data - ds['status/satellite/subsat_latitude_end'] = np.array(22.39) - ds['status/satellite/subsat_longitude_start'] = np.array(304.79) - ds['status/satellite/subsat_latitude_start'] = np.array(55.41) - ds['status/satellite/subsat_longitude_end'] = np.array(296.79) + ds["status/satellite/subsat_latitude_end"] = np.array(22.39) + ds["status/satellite/subsat_longitude_start"] = np.array(304.79) + ds["status/satellite/subsat_latitude_start"] = np.array(55.41) + ds["status/satellite/subsat_longitude_end"] = np.array(296.79) tmp_dir = tmp_path_factory.mktemp("aws_l1b_tests") filename = tmp_dir / compose(file_pattern, dict(start_time=start_time, end_time=end_time, @@ -84,7 +84,7 @@ def aws_file(tmp_path_factory): return filename -@pytest.fixture +@pytest.fixture() def aws_handler(aws_file): """Create an aws filehandler.""" filename_info = parse(file_pattern, os.path.basename(aws_file)) @@ -127,7 +127,7 @@ def test_get_channel_data(aws_handler): assert res.attrs["platform_name"] == "AWS1" -@pytest.mark.parametrize(["id_name", "file_key", "fake_array"], +@pytest.mark.parametrize(("id_name", "file_key", "fake_array"), [("longitude", "data/navigation/aws_lon", fake_lon_data * 1e-4), ("latitude", "data/navigation/aws_lat", fake_lat_data), ("solar_azimuth", "data/navigation/aws_solar_azimuth_angle", fake_sun_azi_data), From f03971533795e871342299e33fa5536dd88380b1 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 8 Nov 2023 09:32:35 +0100 Subject: [PATCH 07/48] Fix aws yaml file --- satpy/etc/readers/aws_l1b_nc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/etc/readers/aws_l1b_nc.yaml b/satpy/etc/readers/aws_l1b_nc.yaml index adee00ed2a..d1d5ea8c5f 100644 --- a/satpy/etc/readers/aws_l1b_nc.yaml +++ b/satpy/etc/readers/aws_l1b_nc.yaml @@ -6,7 +6,7 @@ reader: reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader sensors: [aws,] status: Beta - default_channels: [] + supports_fsspec: false data_identification_keys: name: From 47821d6b5bbe509fc5e044dcd8e48a1233a04c56 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 30 Nov 2023 15:17:02 +0100 Subject: [PATCH 08/48] Fix aws reader according to review comments --- satpy/readers/aws_l1b.py | 24 +----------------------- satpy/tests/reader_tests/test_aws_l1b.py | 3 +++ 2 files changed, 4 insertions(+), 23 deletions(-) diff --git a/satpy/readers/aws_l1b.py b/satpy/readers/aws_l1b.py index afeae3b80e..b23dd4119b 100644 --- a/satpy/readers/aws_l1b.py +++ b/satpy/readers/aws_l1b.py @@ -27,13 +27,7 @@ DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f" -AWS_CHANNEL_NAMES_TO_NUMBER = {"1": 1, "2": 2, "3": 3, "4": 4, - "5": 5, "6": 6, "7": 7, "8": 8, - "9": 9, "10": 10, "11": 11, "12": 12, - "13": 13, "14": 14, "15": 15, "16": 16, - "17": 17, "18": 18, "19": 19} - -AWS_CHANNEL_NAMES = list(AWS_CHANNEL_NAMES_TO_NUMBER.keys()) +AWS_CHANNEL_NAMES = list(str(i) for i in range(1, 20)) class AWSL1BFile(NetCDF4FileHandler): @@ -153,19 +147,3 @@ def mask_and_scale(data_array): data_array.attrs.pop("scale_factor") data_array.attrs.pop("add_offset") return data_array - -# -# def _get_quality_attributes(self): -# """Get quality attributes.""" -# quality_group = self['quality'] -# quality_dict = {} -# for key in quality_group: -# # Add the values (as Numpy array) of each variable in the group -# # where possible -# try: -# quality_dict[key] = quality_group[key].values -# except ValueError: -# quality_dict[key] = None -# -# quality_dict.update(quality_group.attrs) -# return quality_dict diff --git a/satpy/tests/reader_tests/test_aws_l1b.py b/satpy/tests/reader_tests/test_aws_l1b.py index 480c692b57..abbf517ab8 100644 --- a/satpy/tests/reader_tests/test_aws_l1b.py +++ b/satpy/tests/reader_tests/test_aws_l1b.py @@ -111,9 +111,12 @@ def test_get_channel_data(aws_handler): did = dict(name="1") dataset_info = dict(file_key="data/calibration/aws_toa_brightness_temperature") expected = fake_data.isel(n_channels=0) + # mask no_data value expected = expected.where(expected != -2147483648) + # mask outside the valid range expected = expected.where(expected <= 700000) expected = expected.where(expected >= 0) + # "calibrate" expected = expected * 0.001 res = aws_handler.get_dataset(did, dataset_info) np.testing.assert_allclose(res, expected) From dea6a67946f29a1622f535e42d6f4fb7d7fa0b94 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Tue, 1 Oct 2024 20:09:04 +0200 Subject: [PATCH 09/48] Fixed for latest real-data (not yet final format) Signed-off-by: Adam.Dybbroe --- pyproject.toml | 1 + satpy/readers/aws_l1b.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 196ae6a462..679a01a625 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,7 @@ hsaf_grib = ["pygrib"] remote_reading = ["fsspec"] insat_3d = ["xarray-datatree"] gms5-vissr_l1b = ["numba"] +aws_l1b = ["xarray-datatree"] # Writers: cf = ["h5netcdf >= 0.7.3"] awips_tiled = ["netCDF4 >= 1.1.8"] diff --git a/satpy/readers/aws_l1b.py b/satpy/readers/aws_l1b.py index b23dd4119b..cb421d4482 100644 --- a/satpy/readers/aws_l1b.py +++ b/satpy/readers/aws_l1b.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Pytroll Developers +# Copyright (c) 2023, 2024 Pytroll Developers # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -67,6 +67,16 @@ def platform_name(self): """Get the platform name.""" return self.filename_info["platform_name"] + @property + def orbit_start(self): + """Get the orbit number for the start of data.""" + return int(self["/attr/orbit_start"]) + + @property + def orbit_end(self): + """Get the orbit number for the end of data.""" + return int(self["/attr/orbit_end"]) + @property def sub_satellite_longitude_start(self): """Get the longitude of sub-satellite point at start of the product.""" @@ -111,6 +121,7 @@ def get_dataset(self, dataset_id, dataset_info): data_array.attrs["platform_name"] = self.platform_name data_array.attrs["sensor"] = self.sensor + data_array.attrs["orbit_number"] = self.orbit_start return data_array def _get_channel_data(self, dataset_id, dataset_info): From 70bbc862edf29d02ef4ec11248e7e15786c5af7d Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Tue, 1 Oct 2024 20:12:03 +0200 Subject: [PATCH 10/48] Changing the humidity-surface RGB to consider the lowest peaking absorption channel for the blue band rather than highest Signed-off-by: Adam.Dybbroe --- satpy/etc/composites/atms.yaml | 2 +- satpy/etc/composites/aws.yaml | 18 +++++++++++++++++- satpy/etc/enhancements/aws.yaml | 28 ++++++++++++++++++++++++++++ satpy/etc/readers/atms_sdr_hdf5.yaml | 2 +- 4 files changed, 47 insertions(+), 3 deletions(-) diff --git a/satpy/etc/composites/atms.yaml b/satpy/etc/composites/atms.yaml index 27afd5d2b8..624f0bc93b 100644 --- a/satpy/etc/composites/atms.yaml +++ b/satpy/etc/composites/atms.yaml @@ -14,5 +14,5 @@ composites: prerequisites: - name: '16' - name: '17' - - name: '22' + - name: '18' standard_name: mw183_humidity_surface diff --git a/satpy/etc/composites/aws.yaml b/satpy/etc/composites/aws.yaml index 77d2014794..55af749d89 100644 --- a/satpy/etc/composites/aws.yaml +++ b/satpy/etc/composites/aws.yaml @@ -14,5 +14,21 @@ composites: prerequisites: - name: '9' - name: '10' - - name: '15' + - name: '12' standard_name: mw183_humidity_surface + + mw325_humidity_surface: + compositor: !!python/name:satpy.composites.RGBCompositor + prerequisites: + - name: '9' + - name: '10' + - name: '19' + standard_name: mw325_humidity_surface + + mw325_humidity: + compositor: !!python/name:satpy.composites.RGBCompositor + prerequisites: + - name: '16' + - name: '18' + - name: '19' + standard_name: mw325_humidity diff --git a/satpy/etc/enhancements/aws.yaml b/satpy/etc/enhancements/aws.yaml index c997a11350..4d79b0ae11 100644 --- a/satpy/etc/enhancements/aws.yaml +++ b/satpy/etc/enhancements/aws.yaml @@ -27,3 +27,31 @@ enhancements: - name: gamma method: !!python/name:satpy.enhancements.gamma kwargs: {gamma: 1.2} + + mw325_humidity: + standard_name: mw325_humidity + operations: + - name: inverse + method: !!python/name:satpy.enhancements.invert + args: + - [true, true, true] + - name: stretch + method: !!python/name:satpy.enhancements.stretch + kwargs: {stretch: linear} + - name: gamma + method: !!python/name:satpy.enhancements.gamma + kwargs: {gamma: 1.2} + + mw325_humidity_surface: + standard_name: mw325_humidity_surface + operations: + - name: inverse + method: !!python/name:satpy.enhancements.invert + args: + - [true, true, true] + - name: stretch + method: !!python/name:satpy.enhancements.stretch + kwargs: {stretch: linear} + - name: gamma + method: !!python/name:satpy.enhancements.gamma + kwargs: {gamma: 1.2} diff --git a/satpy/etc/readers/atms_sdr_hdf5.yaml b/satpy/etc/readers/atms_sdr_hdf5.yaml index fa8e4105ce..6ad1a1e0a9 100644 --- a/satpy/etc/readers/atms_sdr_hdf5.yaml +++ b/satpy/etc/readers/atms_sdr_hdf5.yaml @@ -33,7 +33,7 @@ reader: file_types: atms_sdr_hdf5: file_reader: !!python/name:satpy.readers.atms_sdr_hdf5.ATMS_SDR_FileHandler - file_patterns: ['SATMS_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_time:%H%M%S%f}_b{orbit:5d}_c{creation_time:%Y%m%d%H%M%S%f}_{source}.h5', 'GATMO_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_time:%H%M%S%f}_b{orbit:5d}_c{creation_time:%Y%m%d%H%M%S%f}_{source}.h5'] + file_patterns: ['SATMS_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_time:%H%M%S%f}_b{orbit:5d}_c{creation_time:%Y%m%d%H%M%S%f}_{source}.h5', 'GATMO_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_time:%H%M%S%f}_b{orbit:5d}_c{creation_time:%Y%m%d%H%M%S%f}_{source}.h5','GATMO-SATMS_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S%f}_e{end_time:%H%M%S%f}_b{orbit:5d}_c{creation_time:%Y%m%d%H%M%S%f}_{source}.h5'] # Example filenames # GATMO_j01_d20221220_t0910240_e0921356_b26361_c20221220100456680030_cspp_dev.h5 # SATMS_j01_d20221220_t0910240_e0921356_b26361_c20221220100456348770_cspp_dev.h5 From a84d874f8093f91e65f0bdb859c66021a3cb7a49 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Thu, 10 Oct 2024 17:52:17 +0200 Subject: [PATCH 11/48] A first draft fix of the viewing geometry per feed horn Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/aws_l1b_nc.yaml | 147 ++++++++++++++++++++--- satpy/readers/aws_l1b.py | 22 +++- satpy/tests/reader_tests/test_aws_l1b.py | 53 ++++++-- 3 files changed, 192 insertions(+), 30 deletions(-) diff --git a/satpy/etc/readers/aws_l1b_nc.yaml b/satpy/etc/readers/aws_l1b_nc.yaml index d1d5ea8c5f..68e395d31d 100644 --- a/satpy/etc/readers/aws_l1b_nc.yaml +++ b/satpy/etc/readers/aws_l1b_nc.yaml @@ -363,46 +363,165 @@ datasets: # --- Navigation data --- - solar_azimuth: - name: solar_azimuth + solar_azimuth_horn1: + name: solar_azimuth_horn1 file_type: aws_l1b_nc file_key: data/navigation/aws_solar_azimuth_angle standard_name: solar_azimuth_angle - horn: ["1", "2", "3", "4"] + horn: "1" + coordinates: + - longitude + - latitude + + solar_azimuth_horn2: + name: solar_azimuth_horn2 + file_type: aws_l1b_nc + file_key: data/navigation/aws_solar_azimuth_angle + standard_name: solar_azimuth_angle + horn: "2" coordinates: - longitude - latitude - solar_zenith: - name: solar_zenith + solar_azimuth_horn3: + name: solar_azimuth_horn3 + file_type: aws_l1b_nc + file_key: data/navigation/aws_solar_azimuth_angle + standard_name: solar_azimuth_angle + horn: "3" + coordinates: + - longitude + - latitude + + solar_azimuth_horn4: + name: solar_azimuth_horn4 + file_type: aws_l1b_nc + file_key: data/navigation/aws_solar_azimuth_angle + standard_name: solar_azimuth_angle + horn: "4" + coordinates: + - longitude + - latitude + + solar_zenith_horn1: + name: solar_zenith_horn1 file_type: aws_l1b_nc file_key: data/navigation/aws_solar_zenith_angle standard_name: solar_zenith_angle - horn: ["1", "2", "3", "4"] + horn: "1" coordinates: - longitude - latitude - satellite_azimuth: - name: satellite_azimuth + solar_zenith_horn2: + name: solar_zenith_horn2 file_type: aws_l1b_nc - file_key: data/navigation/aws_satellite_azimuth_angle - standard_name: satellite_azimuth_angle - horn: ["1", "2", "3", "4"] + file_key: data/navigation/aws_solar_zenith_angle + standard_name: solar_zenith_angle + horn: "2" coordinates: - longitude - latitude - satellite_zenith: - name: satellite_zenith + solar_zenith_horn3: + name: solar_zenith_horn3 + file_type: aws_l1b_nc + file_key: data/navigation/aws_solar_zenith_angle + standard_name: solar_zenith_angle + horn: "3" + coordinates: + - longitude + - latitude + + solar_zenith_horn4: + name: solar_zenith_horn4 + file_type: aws_l1b_nc + file_key: data/navigation/aws_solar_zenith_angle + standard_name: solar_zenith_angle + horn: "4" + coordinates: + - longitude + - latitude + + satellite_zenith_horn1: + name: satellite_zenith_horn1 file_type: aws_l1b_nc file_key: data/navigation/aws_satellite_zenith_angle standard_name: satellite_zenith_angle - horn: ["1", "2", "3", "4"] + horn: "1" coordinates: - longitude - latitude + satellite_zenith_horn2: + name: satellite_zenith_horn2 + file_type: aws_l1b_nc + file_key: data/navigation/aws_satellite_zenith_angle + standard_name: satellite_zenith_angle + horn: "2" + coordinates: + - longitude + - latitude + + satellite_zenith_horn3: + name: satellite_zenith_horn3 + file_type: aws_l1b_nc + file_key: data/navigation/aws_satellite_zenith_angle + standard_name: satellite_zenith_angle + horn: "3" + coordinates: + - longitude + - latitude + + satellite_zenith_horn4: + name: satellite_zenith_horn4 + file_type: aws_l1b_nc + file_key: data/navigation/aws_satellite_zenith_angle + standard_name: satellite_zenith_angle + horn: "4" + coordinates: + - longitude + - latitude + + satellite_azimuth_horn1: + name: satellite_azimuth_horn1 + file_type: aws_l1b_nc + file_key: data/navigation/aws_satellite_azimuth_angle + standard_name: satellite_azimuth_angle + horn: "1" + coordinates: + - longitude + - latitude + + satellite_azimuth_horn2: + name: satellite_azimuth_horn2 + file_type: aws_l1b_nc + file_key: data/navigation/aws_satellite_azimuth_angle + standard_name: satellite_azimuth_angle + horn: "2" + coordinates: + - longitude + - latitude + + satellite_azimuth_horn3: + name: satellite_azimuth_horn3 + file_type: aws_l1b_nc + file_key: data/navigation/aws_satellite_azimuth_angle + standard_name: satellite_azimuth_angle + horn: "3" + coordinates: + - longitude + - latitude + + satellite_azimuth_horn4: + name: satellite_azimuth_horn4 + file_type: aws_l1b_nc + file_key: data/navigation/aws_satellite_azimuth_angle + standard_name: satellite_azimuth_angle + horn: "4" + coordinates: + - longitude + - latitude file_types: aws_l1b_nc: diff --git a/satpy/readers/aws_l1b.py b/satpy/readers/aws_l1b.py index cb421d4482..ce07c209e1 100644 --- a/satpy/readers/aws_l1b.py +++ b/satpy/readers/aws_l1b.py @@ -101,9 +101,25 @@ def get_dataset(self, dataset_id, dataset_info): """Get the data.""" if dataset_id["name"] in AWS_CHANNEL_NAMES: data_array = self._get_channel_data(dataset_id, dataset_info) - elif (dataset_id["name"] in ["longitude", "latitude", - "solar_azimuth", "solar_zenith", - "satellite_zenith", "satellite_azimuth"]): + elif dataset_id["name"] in ["satellite_zenith_horn1", + "satellite_zenith_horn2", + "satellite_zenith_horn3", + "satellite_zenith_horn4", + "solar_azimuth_horn1", + "solar_azimuth_horn2", + "solar_azimuth_horn3", + "solar_azimuth_horn4", + "solar_zenith_horn1", + "solar_zenith_horn2", + "solar_zenith_horn3", + "solar_zenith_horn4", + "satellite_azimuth_horn1", + "satellite_azimuth_horn2", + "satellite_azimuth_horn3", + "satellite_azimuth_horn4"]: + data_array = self._get_navigation_data(dataset_id, dataset_info) + + elif dataset_id["name"] in ["longitude", "latitude"]: data_array = self._get_navigation_data(dataset_id, dataset_info) else: raise NotImplementedError diff --git a/satpy/tests/reader_tests/test_aws_l1b.py b/satpy/tests/reader_tests/test_aws_l1b.py index abbf517ab8..a9ed72a211 100644 --- a/satpy/tests/reader_tests/test_aws_l1b.py +++ b/satpy/tests/reader_tests/test_aws_l1b.py @@ -15,24 +15,27 @@ platform_name = "AWS1" file_pattern = "W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc" # noqa -fake_data_np = np.random.randint(0, 700000, size=19*5*5).reshape((19, 5, 5)) + +rng = np.random.default_rng() + +fake_data_np = rng.integers(0, 700000, size=19*5*5).reshape((19, 5, 5)) fake_data_np[0, 0, 0] = -2147483648 fake_data_np[0, 0, 1] = 700000 + 10 fake_data_np[0, 0, 2] = -10 fake_data = xr.DataArray(fake_data_np, dims=["n_channels", "n_fovs", "n_scans"]) -fake_lon_data = xr.DataArray(np.random.randint(0, 3599999, size=25 * 4).reshape((4, 5, 5)), +fake_lon_data = xr.DataArray(rng.integers(0, 3599999, size=25 * 4).reshape((4, 5, 5)), dims=["n_geo_groups", "n_fovs", "n_scans"]) -fake_lat_data = xr.DataArray(np.random.randint(-900000, 900000, size=25 * 4).reshape((4, 5, 5)), +fake_lat_data = xr.DataArray(rng.integers(-900000, 900000, size=25 * 4).reshape((4, 5, 5)), dims=["n_geo_groups", "n_fovs", "n_scans"]) -fake_sun_azi_data = xr.DataArray(np.random.randint(0, 36000, size=25 * 4).reshape((4, 5, 5)), +fake_sun_azi_data = xr.DataArray(rng.integers(0, 36000, size=25 * 4).reshape((4, 5, 5)), dims=["n_geo_groups", "n_fovs", "n_scans"]) -fake_sun_zen_data = xr.DataArray(np.random.randint(0, 36000, size=25 * 4).reshape((4, 5, 5)), +fake_sun_zen_data = xr.DataArray(rng.integers(0, 36000, size=25 * 4).reshape((4, 5, 5)), dims=["n_geo_groups", "n_fovs", "n_scans"]) -fake_sat_azi_data = xr.DataArray(np.random.randint(0, 36000, size=25 * 4).reshape((4, 5, 5)), +fake_sat_azi_data = xr.DataArray(rng.integers(0, 36000, size=25 * 4).reshape((4, 5, 5)), dims=["n_geo_groups", "n_fovs", "n_scans"]) -fake_sat_zen_data = xr.DataArray(np.random.randint(0, 36000, size=25 * 4).reshape((4, 5, 5)), +fake_sat_zen_data = xr.DataArray(rng.integers(0, 36000, size=25 * 4).reshape((4, 5, 5)), dims=["n_geo_groups", "n_fovs", "n_scans"]) @@ -56,6 +59,8 @@ def aws_file(tmp_path_factory): instrument = "AWS" ds.attrs["instrument"] = instrument + ds.attrs["orbit_start"] = 9991 + ds.attrs["orbit_end"] = 9992 ds["data/calibration/aws_toa_brightness_temperature"] = fake_data ds["data/calibration/aws_toa_brightness_temperature"].attrs["scale_factor"] = 0.001 ds["data/calibration/aws_toa_brightness_temperature"].attrs["add_offset"] = 0.0 @@ -76,6 +81,7 @@ def aws_file(tmp_path_factory): ds["status/satellite/subsat_latitude_start"] = np.array(55.41) ds["status/satellite/subsat_longitude_end"] = np.array(296.79) + tmp_dir = tmp_path_factory.mktemp("aws_l1b_tests") filename = tmp_dir / compose(file_pattern, dict(start_time=start_time, end_time=end_time, processing_time=processing_time, platform_name=platform_name)) @@ -84,7 +90,7 @@ def aws_file(tmp_path_factory): return filename -@pytest.fixture() +@pytest.fixture def aws_handler(aws_file): """Create an aws filehandler.""" filename_info = parse(file_pattern, os.path.basename(aws_file)) @@ -133,12 +139,9 @@ def test_get_channel_data(aws_handler): @pytest.mark.parametrize(("id_name", "file_key", "fake_array"), [("longitude", "data/navigation/aws_lon", fake_lon_data * 1e-4), ("latitude", "data/navigation/aws_lat", fake_lat_data), - ("solar_azimuth", "data/navigation/aws_solar_azimuth_angle", fake_sun_azi_data), - ("solar_zenith", "data/navigation/aws_solar_zenith_angle", fake_sun_zen_data), - ("satellite_azimuth", "data/navigation/aws_satellite_azimuth_angle", fake_sat_azi_data), - ("satellite_zenith", "data/navigation/aws_satellite_zenith_angle", fake_sat_zen_data)]) + ]) def test_get_navigation_data(aws_handler, id_name, file_key, fake_array): - """Test retrieving the angles_data.""" + """Test retrieving the geolocation (lon-lat) data.""" Horn = Enum("Horn", ["1", "2", "3", "4"]) did = dict(name=id_name, horn=Horn["1"]) dataset_info = dict(file_key=file_key, standard_name=id_name) @@ -155,3 +158,27 @@ def test_get_navigation_data(aws_handler, id_name, file_key, fake_array): assert "n_geo_groups" not in res.coords if id_name == "longitude": assert res.max() <= 180 + + +@pytest.mark.parametrize(("id_name", "file_key", "fake_array"), + [("solar_azimuth_horn1", "data/navigation/aws_solar_azimuth_angle", fake_sun_azi_data), + ("solar_zenith_horn1", "data/navigation/aws_solar_zenith_angle", fake_sun_zen_data), + ("satellite_azimuth_horn1", "data/navigation/aws_satellite_azimuth_angle", fake_sat_azi_data), + ("satellite_zenith_horn1", "data/navigation/aws_satellite_zenith_angle", fake_sat_zen_data)]) +def test_get_viewing_geometry_data(aws_handler, id_name, file_key, fake_array): + """Test retrieving the angles_data.""" + Horn = Enum("Horn", ["1", "2", "3", "4"]) + did = dict(name=id_name, horn=Horn["1"]) + + dataset_info = dict(file_key=file_key, standard_name=id_name) + res = aws_handler.get_dataset(did, dataset_info) + + np.testing.assert_allclose(res, fake_array.isel(n_geo_groups=0)) + assert "x" in res.dims + assert "y" in res.dims + assert "orbital_parameters" in res.attrs + assert res.dims == ("x", "y") + assert "standard_name" in res.attrs + assert "n_geo_groups" not in res.coords + if id_name == "longitude": + assert res.max() <= 180 From c1b2d9d233e674a514c2ab0649fbfbc246c2babe Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 11 Dec 2024 10:33:55 +0100 Subject: [PATCH 12/48] Add support for reading the EUMETSAT AWS/EPS-Sterna L1b data Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/aws_l1b_nc.yaml | 1 + satpy/etc/readers/eps_sterna_l1b_nc.yaml | 532 +++++++++++++++++++++++ satpy/readers/aws_l1b.py | 15 +- 3 files changed, 546 insertions(+), 2 deletions(-) create mode 100644 satpy/etc/readers/eps_sterna_l1b_nc.yaml diff --git a/satpy/etc/readers/aws_l1b_nc.yaml b/satpy/etc/readers/aws_l1b_nc.yaml index 68e395d31d..0d6df8dc4e 100644 --- a/satpy/etc/readers/aws_l1b_nc.yaml +++ b/satpy/etc/readers/aws_l1b_nc.yaml @@ -531,4 +531,5 @@ file_types: file_reader: !!python/name:satpy.readers.aws_l1b.AWSL1BFile file_patterns: [ 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc', + 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB__{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc', 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____radsim.nc'] diff --git a/satpy/etc/readers/eps_sterna_l1b_nc.yaml b/satpy/etc/readers/eps_sterna_l1b_nc.yaml new file mode 100644 index 0000000000..026aefc918 --- /dev/null +++ b/satpy/etc/readers/eps_sterna_l1b_nc.yaml @@ -0,0 +1,532 @@ +reader: + name: aws_l1b_nc + short_name: AWS L1B RAD NetCDF4 + long_name: AWS L1B Radiance (NetCDF4) + description: Reader for the EUMETSAT EPS-Sterna Sounder level-1b files in netCDF4. + reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader + sensors: [aws,] + status: Beta + supports_fsspec: false + + data_identification_keys: + name: + required: true + frequency_double_sideband: + type: !!python/name:satpy.readers.pmw_channels_definitions.FrequencyDoubleSideBand + frequency_range: + type: !!python/name:satpy.readers.pmw_channels_definitions.FrequencyRange + resolution: + polarization: + enum: + - QH + - QV + horn: + enum: + - "1" + - "2" + - "3" + - "4" + calibration: + enum: + - brightness_temperature + transitive: true + modifiers: + required: true + default: [] + type: !!python/name:satpy.dataset.ModifierTuple + + coord_identification_keys: + name: + required: true + resolution: + polarization: + enum: + - QH + - QV + horn: + enum: + - "1" + - "2" + - "3" + - "4" + +datasets: + '1': + name: '1' + frequency_range: + central: 50.3 + bandwidth: 0.180 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + horn: "1" + coordinates: [longitude, latitude] + file_type: eps_sterna_l1b_nc + file_key: data/calibration/toa_brightness_temperature + '2': + name: '2' + frequency_range: + central: 52.8 + bandwidth: 0.400 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + horn: "1" + coordinates: [longitude, latitude] + file_type: eps_sterna_l1b_nc + file_key: data/calibration/toa_brightness_temperature + '3': + name: '3' + frequency_range: + central: 53.246 + bandwidth: 0.300 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + horn: "1" + coordinates: [longitude, latitude] + file_type: eps_sterna_l1b_nc + file_key: data/calibration/toa_brightness_temperature + '4': + name: '4' + frequency_range: + central: 53.596 + bandwidth: 0.370 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + horn: "1" + coordinates: [longitude, latitude] + file_type: eps_sterna_l1b_nc + file_key: data/calibration/toa_brightness_temperature + '5': + name: '5' + frequency_range: + central: 54.4 + bandwidth: 0.400 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + horn: "1" + coordinates: [longitude, latitude] + file_type: eps_sterna_l1b_nc + file_key: data/calibration/toa_brightness_temperature + '6': + name: '6' + frequency_range: + central: 54.94 + bandwidth: 0.400 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + horn: "1" + coordinates: [longitude, latitude] + file_type: eps_sterna_l1b_nc + file_key: data/calibration/toa_brightness_temperature + '7': + name: '7' + frequency_range: + central: 55.5 + bandwidth: 0.330 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + horn: "1" + coordinates: [longitude, latitude] + file_type: eps_sterna_l1b_nc + file_key: data/calibration/toa_brightness_temperature + '8': + name: '8' + frequency_range: + central: 57.290344 + bandwidth: 0.330 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + horn: "1" + coordinates: [longitude, latitude] + file_type: eps_sterna_l1b_nc + file_key: data/calibration/toa_brightness_temperature + '9': + name: '9' + frequency_range: + central: 89.0 + bandwidth: 4.0 + unit: GHz + polarization: 'QV' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + horn: "2" + coordinates: [longitude, latitude] + file_type: eps_sterna_l1b_nc + file_key: data/calibration/toa_brightness_temperature + '10': + name: '10' + frequency_range: + central: 165.5 + bandwidth: 2.700 + unit: GHz + polarization: 'QH' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + horn: "3" + coordinates: [longitude, latitude] + file_type: eps_sterna_l1b_nc + file_key: data/calibration/toa_brightness_temperature + '11': + name: '11' + frequency_range: + central: 176.311 + bandwidth: 2.0 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + horn: "3" + coordinates: [longitude, latitude] + file_type: eps_sterna_l1b_nc + file_key: data/calibration/toa_brightness_temperature + '12': + name: '12' + frequency_range: + central: 178.811 + bandwidth: 2.0 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + horn: "3" + coordinates: [longitude, latitude] + file_type: eps_sterna_l1b_nc + file_key: data/calibration/toa_brightness_temperature + '13': + name: '13' + frequency_range: + central: 180.311 + bandwidth: 1.0 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + horn: "3" + coordinates: [longitude, latitude] + file_type: eps_sterna_l1b_nc + file_key: data/calibration/toa_brightness_temperature + '14': + name: '14' + frequency_range: + central: 181.511 + bandwidth: 1.0 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + horn: "3" + coordinates: [longitude, latitude] + file_type: eps_sterna_l1b_nc + file_key: data/calibration/toa_brightness_temperature + '15': + name: '15' + frequency_range: + central: 182.311 + bandwidth: 0.5 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + horn: "3" + coordinates: [longitude, latitude] + file_type: eps_sterna_l1b_nc + file_key: data/calibration/toa_brightness_temperature + '16': + name: '16' + frequency_double_sideband: + central: 325.15 + side: 1.2 + bandwidth: 0.8 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + horn: "4" + coordinates: [longitude, latitude] + file_type: eps_sterna_l1b_nc + file_key: data/calibration/toa_brightness_temperature + '17': + name: '17' + frequency_double_sideband: + central: 325.15 + side: 2.4 + bandwidth: 1.2 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + horn: "4" + coordinates: [longitude, latitude] + file_type: eps_sterna_l1b_nc + file_key: data/calibration/toa_brightness_temperature + '18': + name: '18' + frequency_double_sideband: + central: 325.15 + side: 4.1 + bandwidth: 1.8 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + horn: "4" + coordinates: [longitude, latitude] + file_type: eps_sterna_l1b_nc + file_key: data/calibration/toa_brightness_temperature + '19': + name: '19' + frequency_double_sideband: + central: 325.15 + side: 6.6 + bandwidth: 2.8 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + horn: "4" + coordinates: [longitude, latitude] + file_type: eps_sterna_l1b_nc + file_key: data/calibration/toa_brightness_temperature + +# --- Coordinates --- + + longitude: + name: longitude + file_type: eps_sterna_l1b_nc + standard_name: longitude + units: degrees_east + horn: ["1", "2", "3", "4"] + file_key: data/navigation/longitude + + + latitude: + name: latitude + file_type: eps_sterna_l1b_nc + standard_name: latitude + units: degrees_north + horn: ["1", "2", "3", "4"] + file_key: data/navigation/latitude + + +# --- Navigation data --- + + solar_azimuth_horn1: + name: solar_azimuth_horn1 + file_type: eps_sterna_l1b_nc + file_key: data/navigation/solar_azimuth_angle + standard_name: solar_azimuth_angle + horn: "1" + coordinates: + - longitude + - latitude + + solar_azimuth_horn2: + name: solar_azimuth_horn2 + file_type: eps_sterna_l1b_nc + file_key: data/navigation/solar_azimuth_angle + standard_name: solar_azimuth_angle + horn: "2" + coordinates: + - longitude + - latitude + + solar_azimuth_horn3: + name: solar_azimuth_horn3 + file_type: eps_sterna_l1b_nc + file_key: data/navigation/solar_azimuth_angle + standard_name: solar_azimuth_angle + horn: "3" + coordinates: + - longitude + - latitude + + solar_azimuth_horn4: + name: solar_azimuth_horn4 + file_type: eps_sterna_l1b_nc + file_key: data/navigation/solar_azimuth_angle + standard_name: solar_azimuth_angle + horn: "4" + coordinates: + - longitude + - latitude + + solar_zenith_horn1: + name: solar_zenith_horn1 + file_type: eps_sterna_l1b_nc + file_key: data/navigation/solar_zenith_angle + standard_name: solar_zenith_angle + horn: "1" + coordinates: + - longitude + - latitude + + solar_zenith_horn2: + name: solar_zenith_horn2 + file_type: eps_sterna_l1b_nc + file_key: data/navigation/solar_zenith_angle + standard_name: solar_zenith_angle + horn: "2" + coordinates: + - longitude + - latitude + + solar_zenith_horn3: + name: solar_zenith_horn3 + file_type: eps_sterna_l1b_nc + file_key: data/navigation/solar_zenith_angle + standard_name: solar_zenith_angle + horn: "3" + coordinates: + - longitude + - latitude + + solar_zenith_horn4: + name: solar_zenith_horn4 + file_type: eps_sterna_l1b_nc + file_key: data/navigation/solar_zenith_angle + standard_name: solar_zenith_angle + horn: "4" + coordinates: + - longitude + - latitude + + satellite_zenith_horn1: + name: satellite_zenith_horn1 + file_type: eps_sterna_l1b_nc + file_key: data/navigation/satellite_zenith_angle + standard_name: satellite_zenith_angle + horn: "1" + coordinates: + - longitude + - latitude + + satellite_zenith_horn2: + name: satellite_zenith_horn2 + file_type: eps_sterna_l1b_nc + file_key: data/navigation/satellite_zenith_angle + standard_name: satellite_zenith_angle + horn: "2" + coordinates: + - longitude + - latitude + + satellite_zenith_horn3: + name: satellite_zenith_horn3 + file_type: eps_sterna_l1b_nc + file_key: data/navigation/satellite_zenith_angle + standard_name: satellite_zenith_angle + horn: "3" + coordinates: + - longitude + - latitude + + satellite_zenith_horn4: + name: satellite_zenith_horn4 + file_type: eps_sterna_l1b_nc + file_key: data/navigation/satellite_zenith_angle + standard_name: satellite_zenith_angle + horn: "4" + coordinates: + - longitude + - latitude + + satellite_azimuth_horn1: + name: satellite_azimuth_horn1 + file_type: eps_sterna_l1b_nc + file_key: data/navigation/satellite_azimuth_angle + standard_name: satellite_azimuth_angle + horn: "1" + coordinates: + - longitude + - latitude + + satellite_azimuth_horn2: + name: satellite_azimuth_horn2 + file_type: eps_sterna_l1b_nc + file_key: data/navigation/satellite_azimuth_angle + standard_name: satellite_azimuth_angle + horn: "2" + coordinates: + - longitude + - latitude + + satellite_azimuth_horn3: + name: satellite_azimuth_horn3 + file_type: eps_sterna_l1b_nc + file_key: data/navigation/satellite_azimuth_angle + standard_name: satellite_azimuth_angle + horn: "3" + coordinates: + - longitude + - latitude + + satellite_azimuth_horn4: + name: satellite_azimuth_horn4 + file_type: eps_sterna_l1b_nc + file_key: data/navigation/satellite_azimuth_angle + standard_name: satellite_azimuth_angle + horn: "4" + coordinates: + - longitude + - latitude + +file_types: + eps_sterna_l1b_nc: + # W_XX-EUMETSAT-Darmstadt,SAT,AWS1-MWR-1B-RAD_C_EUMT_20241121085911_G_D_20241109234502_20241110004559_T_N____.nc + file_reader: !!python/name:satpy.readers.aws_l1b.AWSL1BFile + file_patterns: [ + 'W_XX-EUMETSAT-Darmstadt,SAT,{platform_name}-MWR-1B-RAD_C_EUMT_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_N____.nc', + ] diff --git a/satpy/readers/aws_l1b.py b/satpy/readers/aws_l1b.py index ce07c209e1..9023f6f0ab 100644 --- a/satpy/readers/aws_l1b.py +++ b/satpy/readers/aws_l1b.py @@ -15,6 +15,10 @@ """Reader for the Arctic Weather Satellite (AWS) Sounder level-1b data. Test data provided by ESA August 23, 2023. + +Sample data for five orbits in September 2024 provided by ESA to the Science +Advisory Group for MWS and AWS, November 26, 2024. + """ import logging @@ -47,6 +51,11 @@ def __init__(self, filename, filename_info, filetype_info, auto_maskandscale=Tru cache_handle=True) self.filename_info = filename_info + if filetype_info["file_type"].startswith("eps_sterna"): + self._feed_horn_group_name = "n_feedhorns" + else: + self._feed_horn_group_name = "n_geo_groups" + @property def start_time(self): """Get the start time.""" @@ -147,11 +156,13 @@ def _get_channel_data(self, dataset_id, dataset_info): return channel_data.sel(n_channels=dataset_id["name"]).drop_vars("n_channels") def _get_navigation_data(self, dataset_id, dataset_info): + """Get the navigation (geolocation) data for one feed horn.""" geo_data = self[dataset_info["file_key"]] - geo_data.coords["n_geo_groups"] = ["1", "2", "3", "4"] + geo_data.coords[self._feed_horn_group_name] = ["1", "2", "3", "4"] geo_data = geo_data.rename({"n_fovs": "x", "n_scans": "y"}) horn = dataset_id["horn"].name - return geo_data.sel(n_geo_groups=horn).drop_vars("n_geo_groups") + _selection = {self._feed_horn_group_name: horn} + return geo_data.sel(_selection).drop_vars(self._feed_horn_group_name) def mask_and_scale(data_array): From b2775eb03564c1e283311bee710e415df3bbcffd Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 11 Dec 2024 10:39:38 +0100 Subject: [PATCH 13/48] Add support for reading ESA AWS L1c Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/aws_l1c_nc.yaml | 375 ++++++++++++++++++++++++++++++ satpy/readers/aws_l1c.py | 122 ++++++++++ 2 files changed, 497 insertions(+) create mode 100644 satpy/etc/readers/aws_l1c_nc.yaml create mode 100644 satpy/readers/aws_l1c.py diff --git a/satpy/etc/readers/aws_l1c_nc.yaml b/satpy/etc/readers/aws_l1c_nc.yaml new file mode 100644 index 0000000000..8abd9150ef --- /dev/null +++ b/satpy/etc/readers/aws_l1c_nc.yaml @@ -0,0 +1,375 @@ +reader: + name: aws_l1c_nc + short_name: AWS L1C RAD NetCDF4 + long_name: AWS L1C Radiance (NetCDF4) + description: Reader for the ESA AWS (Arctic Weather Satellite) Sounder level-1c files in netCDF4. + reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader + sensors: [aws,] + status: Beta + supports_fsspec: false + + data_identification_keys: + name: + required: true + frequency_double_sideband: + type: !!python/name:satpy.readers.pmw_channels_definitions.FrequencyDoubleSideBand + frequency_range: + type: !!python/name:satpy.readers.pmw_channels_definitions.FrequencyRange + resolution: + polarization: + enum: + - QH + - QV + calibration: + enum: + - brightness_temperature + transitive: true + modifiers: + required: true + default: [] + type: !!python/name:satpy.dataset.ModifierTuple + + coord_identification_keys: + name: + required: true + resolution: + polarization: + enum: + - QH + - QV + +datasets: + '1': + name: '1' + frequency_range: + central: 50.3 + bandwidth: 0.180 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '2': + name: '2' + frequency_range: + central: 52.8 + bandwidth: 0.400 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '3': + name: '3' + frequency_range: + central: 53.246 + bandwidth: 0.300 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '4': + name: '4' + frequency_range: + central: 53.596 + bandwidth: 0.370 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '5': + name: '5' + frequency_range: + central: 54.4 + bandwidth: 0.400 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '6': + name: '6' + frequency_range: + central: 54.94 + bandwidth: 0.400 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '7': + name: '7' + frequency_range: + central: 55.5 + bandwidth: 0.330 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '8': + name: '8' + frequency_range: + central: 57.290344 + bandwidth: 0.330 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '9': + name: '9' + frequency_range: + central: 89.0 + bandwidth: 4.0 + unit: GHz + polarization: 'QV' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '10': + name: '10' + frequency_range: + central: 165.5 + bandwidth: 2.700 + unit: GHz + polarization: 'QH' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '11': + name: '11' + frequency_range: + central: 176.311 + bandwidth: 2.0 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '12': + name: '12' + frequency_range: + central: 178.811 + bandwidth: 2.0 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '13': + name: '13' + frequency_range: + central: 180.311 + bandwidth: 1.0 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '14': + name: '14' + frequency_range: + central: 181.511 + bandwidth: 1.0 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '15': + name: '15' + frequency_range: + central: 182.311 + bandwidth: 0.5 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '16': + name: '16' + frequency_double_sideband: + central: 325.15 + side: 1.2 + bandwidth: 0.8 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '17': + name: '17' + frequency_double_sideband: + central: 325.15 + side: 2.4 + bandwidth: 1.2 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '18': + name: '18' + frequency_double_sideband: + central: 325.15 + side: 4.1 + bandwidth: 1.8 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '19': + name: '19' + frequency_double_sideband: + central: 325.15 + side: 6.6 + bandwidth: 2.8 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + +# --- Coordinates --- + + longitude: + name: longitude + file_type: aws_l1c_nc + standard_name: longitude + units: degrees_east + file_key: data/navigation/aws_lon + + latitude: + name: latitude + file_type: aws_l1c_nc + standard_name: latitude + units: degrees_north + file_key: data/navigation/aws_lat + +# --- Navigation data --- + + solar_azimuth: + name: solar_azimuth + file_type: aws_l1c_nc + file_key: data/navigation/aws_solar_azimuth_angle + standard_name: solar_azimuth_angle + coordinates: + - longitude + - latitude + + solar_zenith: + name: solar_zenith + file_type: aws_l1c_nc + file_key: data/navigation/aws_solar_zenith_angle + standard_name: solar_zenith_angle + coordinates: + - longitude + - latitude + + satellite_azimuth: + name: satellite_azimuth + file_type: aws_l1c_nc + file_key: data/navigation/aws_satellite_azimuth_angle + standard_name: satellite_azimuth_angle + coordinates: + - longitude + - latitude + + satellite_zenith: + name: satellite_zenith + file_type: aws_l1c_nc + file_key: data/navigation/aws_satellite_zenith_angle + standard_name: satellite_zenith_angle + coordinates: + - longitude + - latitude + +file_types: + aws_l1c_nc: + # W_XX-OHB-Unknown,SAT,1-AWS-1B-RAD_C_OHB_20230707124607_G_D_20220621090100_20220621090618_T_B____.nc + # W_XX-OHB-Stockholm,SAT,AWS1-MWR-1B-RAD_C_OHB_20230823161321_G_D_20240115111111_20240115125434_T_B____.nc + # W_XX-OHB-Stockholm,SAT,AWS1-MWR-1B-RAD_C_OHB_20230816120142_G_D_20240115111111_20240115125434_T_B____radsim.nc + file_reader: !!python/name:satpy.readers.aws_l1c.AWSL1CFile + file_patterns: [ + 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1C-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc', + 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1C-RAD_C_OHB__{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc',] diff --git a/satpy/readers/aws_l1c.py b/satpy/readers/aws_l1c.py new file mode 100644 index 0000000000..b4566c63f4 --- /dev/null +++ b/satpy/readers/aws_l1c.py @@ -0,0 +1,122 @@ +# Copyright (c) 2023, 2024 Pytroll Developers + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Reader for the Arctic Weather Satellite (AWS) Sounder level-1c data. + +Sample data provided by ESA September 27, 2024. +""" + +import logging + +import xarray as xr + +from .netcdf_utils import NetCDF4FileHandler + +logger = logging.getLogger(__name__) + +DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f" + +AWS_CHANNEL_NAMES = list(str(i) for i in range(1, 20)) + + +class AWSL1CFile(NetCDF4FileHandler): + """Class implementing the AWS L1c Filehandler. + + This class implements the ESA Arctic Weather Satellite (AWS) Level-1b + NetCDF reader. It is designed to be used through the :class:`~satpy.Scene` + class using the :mod:`~satpy.Scene.load` method with the reader + ``"aws_l1c_nc"``. + + """ + + def __init__(self, filename, filename_info, filetype_info, auto_maskandscale=True): + """Initialize the handler.""" + super().__init__(filename, filename_info, filetype_info, + cache_var_size=10000, + cache_handle=True) + self.filename_info = filename_info + + @property + def start_time(self): + """Get the start time.""" + return self.filename_info["start_time"] + + @property + def end_time(self): + """Get the end time.""" + return self.filename_info["end_time"] + + @property + def sensor(self): + """Get the sensor name.""" + return "MWR" + + @property + def platform_name(self): + """Get the platform name.""" + return self.filename_info["platform_name"] + + def get_dataset(self, dataset_id, dataset_info): + """Get the data.""" + if dataset_id["name"] in AWS_CHANNEL_NAMES: + data_array = self._get_channel_data(dataset_id, dataset_info) + elif (dataset_id["name"] in ["longitude", "latitude", + "solar_azimuth", "solar_zenith", + "satellite_zenith", "satellite_azimuth"]): + data_array = self._get_navigation_data(dataset_id, dataset_info) + else: + raise NotImplementedError + + data_array = mask_and_scale(data_array) + if dataset_id["name"] == "longitude": + data_array = data_array.where(data_array <= 180, data_array - 360) + + data_array.attrs.update(dataset_info) + + data_array.attrs["platform_name"] = self.platform_name + data_array.attrs["sensor"] = self.sensor + return data_array + + def _get_channel_data(self, dataset_id, dataset_info): + channel_data = self[dataset_info["file_key"]] + channel_data.coords["n_channels"] = AWS_CHANNEL_NAMES + channel_data = channel_data.rename({"n_fovs": "x", "n_scans": "y"}) + return channel_data.sel(n_channels=dataset_id["name"]).drop_vars("n_channels") + + def _get_navigation_data(self, dataset_id, dataset_info): + geo_data = self[dataset_info["file_key"]] + geo_data = geo_data.rename({"n_fovs": "x", "n_scans": "y"}) + return geo_data + + +def mask_and_scale(data_array): + """Mask then scale the data array.""" + if "missing_value" in data_array.attrs: + with xr.set_options(keep_attrs=True): + data_array = data_array.where(data_array != data_array.attrs["missing_value"]) + data_array.attrs.pop("missing_value") + if "valid_max" in data_array.attrs: + with xr.set_options(keep_attrs=True): + data_array = data_array.where(data_array <= data_array.attrs["valid_max"]) + data_array.attrs.pop("valid_max") + if "valid_min" in data_array.attrs: + with xr.set_options(keep_attrs=True): + data_array = data_array.where(data_array >= data_array.attrs["valid_min"]) + data_array.attrs.pop("valid_min") + if "scale_factor" in data_array.attrs and "add_offset" in data_array.attrs: + with xr.set_options(keep_attrs=True): + data_array = data_array * data_array.attrs["scale_factor"] + data_array.attrs["add_offset"] + data_array.attrs.pop("scale_factor") + data_array.attrs.pop("add_offset") + return data_array From 41e7975ebf567b26789e625bd3aab56048c8b825 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 11 Dec 2024 11:22:17 +0100 Subject: [PATCH 14/48] Fix the tests Signed-off-by: Adam.Dybbroe --- satpy/tests/reader_tests/test_aws_l1b.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/satpy/tests/reader_tests/test_aws_l1b.py b/satpy/tests/reader_tests/test_aws_l1b.py index a9ed72a211..4bb5e6f0ca 100644 --- a/satpy/tests/reader_tests/test_aws_l1b.py +++ b/satpy/tests/reader_tests/test_aws_l1b.py @@ -51,9 +51,9 @@ def random_date(start, end): def aws_file(tmp_path_factory): """Create an AWS file.""" ds = DataTree() - start_time = random_date(datetime(2024, 6, 1), datetime(2030, 6, 1)) + start_time = datetime(2024, 9, 1, 12, 0) ds.attrs["sensing_start_time_utc"] = start_time.strftime(DATETIME_FORMAT) - end_time = random_date(datetime(2024, 6, 1), datetime(2030, 6, 1)) + end_time = datetime(2024, 9, 1, 12, 15) ds.attrs["sensing_end_time_utc"] = end_time.strftime(DATETIME_FORMAT) processing_time = random_date(datetime(2024, 6, 1), datetime(2030, 6, 1)) @@ -94,16 +94,15 @@ def aws_file(tmp_path_factory): def aws_handler(aws_file): """Create an aws filehandler.""" filename_info = parse(file_pattern, os.path.basename(aws_file)) - return AWSL1BFile(aws_file, filename_info, dict()) + filetype_info = dict() + filetype_info["file_type"] = "aws_l1b" + return AWSL1BFile(aws_file, filename_info, filetype_info) -def test_start_end_time(aws_file): +def test_start_end_time(aws_handler): """Test that start and end times are read correctly.""" - filename_info = parse(file_pattern, os.path.basename(aws_file)) - handler = AWSL1BFile(aws_file, filename_info, dict()) - - assert handler.start_time == filename_info["start_time"] - assert handler.end_time == filename_info["end_time"] + assert aws_handler.start_time == datetime(2024, 9, 1, 12, 0) + assert aws_handler.end_time == datetime(2024, 9, 1, 12, 15) def test_metadata(aws_handler): From 441ee06f6cb039d19f3221cf6749978611995f5c Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 11 Dec 2024 18:58:51 +0100 Subject: [PATCH 15/48] Adapt test to latest AWS l1b format Signed-off-by: Adam.Dybbroe --- satpy/tests/reader_tests/test_aws_l1b.py | 44 +++++++++++------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/satpy/tests/reader_tests/test_aws_l1b.py b/satpy/tests/reader_tests/test_aws_l1b.py index 4bb5e6f0ca..d3c45e4e3d 100644 --- a/satpy/tests/reader_tests/test_aws_l1b.py +++ b/satpy/tests/reader_tests/test_aws_l1b.py @@ -18,25 +18,22 @@ rng = np.random.default_rng() -fake_data_np = rng.integers(0, 700000, size=19*5*5).reshape((19, 5, 5)) +fake_data_np = rng.integers(0, 700000, size=10*145*19).reshape((10, 145, 19)) fake_data_np[0, 0, 0] = -2147483648 -fake_data_np[0, 0, 1] = 700000 + 10 -fake_data_np[0, 0, 2] = -10 - -fake_data = xr.DataArray(fake_data_np, - dims=["n_channels", "n_fovs", "n_scans"]) -fake_lon_data = xr.DataArray(rng.integers(0, 3599999, size=25 * 4).reshape((4, 5, 5)), - dims=["n_geo_groups", "n_fovs", "n_scans"]) -fake_lat_data = xr.DataArray(rng.integers(-900000, 900000, size=25 * 4).reshape((4, 5, 5)), - dims=["n_geo_groups", "n_fovs", "n_scans"]) -fake_sun_azi_data = xr.DataArray(rng.integers(0, 36000, size=25 * 4).reshape((4, 5, 5)), - dims=["n_geo_groups", "n_fovs", "n_scans"]) -fake_sun_zen_data = xr.DataArray(rng.integers(0, 36000, size=25 * 4).reshape((4, 5, 5)), - dims=["n_geo_groups", "n_fovs", "n_scans"]) -fake_sat_azi_data = xr.DataArray(rng.integers(0, 36000, size=25 * 4).reshape((4, 5, 5)), - dims=["n_geo_groups", "n_fovs", "n_scans"]) -fake_sat_zen_data = xr.DataArray(rng.integers(0, 36000, size=25 * 4).reshape((4, 5, 5)), - dims=["n_geo_groups", "n_fovs", "n_scans"]) +fake_data_np[1, 0, 0] = 700000 + 10 +fake_data_np[2, 0, 0] = -10 + +ARRAY_DIMS = ["n_scans", "n_fovs", "n_channels"] +fake_data = xr.DataArray(fake_data_np, dims=ARRAY_DIMS) + +GEO_DIMS = ["n_scans", "n_fovs", "n_geo_groups"] +GEO_SIZE = 10*145*4 +fake_lon_data = xr.DataArray(rng.integers(0, 3599999, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) +fake_lat_data = xr.DataArray(rng.integers(-900000, 900000, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) +fake_sun_azi_data = xr.DataArray(rng.integers(0, 36000, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) +fake_sun_zen_data = xr.DataArray(rng.integers(0, 36000, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) +fake_sat_azi_data = xr.DataArray(rng.integers(0, 36000, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) +fake_sat_zen_data = xr.DataArray(rng.integers(0, 36000, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) def random_date(start, end): @@ -81,7 +78,6 @@ def aws_file(tmp_path_factory): ds["status/satellite/subsat_latitude_start"] = np.array(55.41) ds["status/satellite/subsat_longitude_end"] = np.array(296.79) - tmp_dir = tmp_path_factory.mktemp("aws_l1b_tests") filename = tmp_dir / compose(file_pattern, dict(start_time=start_time, end_time=end_time, processing_time=processing_time, platform_name=platform_name)) @@ -129,7 +125,7 @@ def test_get_channel_data(aws_handler): assert "y" in res.dims assert "orbital_parameters" in res.attrs assert res.attrs["orbital_parameters"]["sub_satellite_longitude_end"] == 296.79 - assert res.dims == ("x", "y") + assert res.dims == ("y", "x") assert "n_channels" not in res.coords assert res.attrs["sensor"] == "AWS" assert res.attrs["platform_name"] == "AWS1" @@ -152,7 +148,7 @@ def test_get_navigation_data(aws_handler, id_name, file_key, fake_array): assert "x" in res.dims assert "y" in res.dims assert "orbital_parameters" in res.attrs - assert res.dims == ("x", "y") + assert res.dims == ("y", "x") assert "standard_name" in res.attrs assert "n_geo_groups" not in res.coords if id_name == "longitude": @@ -167,16 +163,16 @@ def test_get_navigation_data(aws_handler, id_name, file_key, fake_array): def test_get_viewing_geometry_data(aws_handler, id_name, file_key, fake_array): """Test retrieving the angles_data.""" Horn = Enum("Horn", ["1", "2", "3", "4"]) - did = dict(name=id_name, horn=Horn["1"]) + dset_id = dict(name=id_name, horn=Horn["1"]) dataset_info = dict(file_key=file_key, standard_name=id_name) - res = aws_handler.get_dataset(did, dataset_info) + res = aws_handler.get_dataset(dset_id, dataset_info) np.testing.assert_allclose(res, fake_array.isel(n_geo_groups=0)) assert "x" in res.dims assert "y" in res.dims assert "orbital_parameters" in res.attrs - assert res.dims == ("x", "y") + assert res.dims == ("y", "x") assert "standard_name" in res.attrs assert "n_geo_groups" not in res.coords if id_name == "longitude": From 1437cd193f0b8bde8a93053ddbdbd3699d49d20c Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 11 Dec 2024 19:58:31 +0100 Subject: [PATCH 16/48] Fix tests and add some basic RGB recipes for AWS Signed-off-by: Adam.Dybbroe --- satpy/etc/composites/aws.yaml | 20 ++++++++++++++++---- satpy/etc/enhancements/generic.yaml | 15 +++++++++++++++ satpy/etc/readers/aws_l1b_nc.yaml | 2 +- satpy/etc/readers/eps_sterna_l1b_nc.yaml | 4 ++-- 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/satpy/etc/composites/aws.yaml b/satpy/etc/composites/aws.yaml index 55af749d89..ba38a69d02 100644 --- a/satpy/etc/composites/aws.yaml +++ b/satpy/etc/composites/aws.yaml @@ -2,12 +2,12 @@ sensor_name: aws composites: mw183_humidity: - compositor: !!python/name:satpy.composites.RGBCompositor + standard_name: mw183_humidity + compositor: !!python/name:satpy.composites.GenericCompositor prerequisites: - - name: '15' - - name: '13' - name: '11' - standard_name: mw183_humidity + - name: '13' + - name: '15' mw183_humidity_surface: compositor: !!python/name:satpy.composites.RGBCompositor @@ -32,3 +32,15 @@ composites: - name: '18' - name: '19' standard_name: mw325_humidity + + ch1_tbs_colors: + compositor: !!python/name:satpy.composites.SingleBandCompositor + prerequisites: + - name: '1' + standard_name: tbs_colors + + ch10_tbs_colors: + compositor: !!python/name:satpy.composites.SingleBandCompositor + prerequisites: + - name: '10' + standard_name: tbs_colors diff --git a/satpy/etc/enhancements/generic.yaml b/satpy/etc/enhancements/generic.yaml index cdfb7851ad..8989d44152 100644 --- a/satpy/etc/enhancements/generic.yaml +++ b/satpy/etc/enhancements/generic.yaml @@ -1289,3 +1289,18 @@ enhancements: image_ready: standard_name: image_ready operations: [] + + mw183_humidity: + # matches AWS + standard_name: mw183_humidity + operations: + - name: stretch + method: !!python/name:satpy.enhancements.stretch + kwargs: + stretch: crude + min_stretch: [290, 290, 290] + max_stretch: [190, 190, 190] + - name: gamma + method: !!python/name:satpy.enhancements.gamma + kwargs: + gamma: [1.5, 1.2, 1.2] diff --git a/satpy/etc/readers/aws_l1b_nc.yaml b/satpy/etc/readers/aws_l1b_nc.yaml index 0d6df8dc4e..1f1341e0e6 100644 --- a/satpy/etc/readers/aws_l1b_nc.yaml +++ b/satpy/etc/readers/aws_l1b_nc.yaml @@ -530,6 +530,6 @@ file_types: # W_XX-OHB-Stockholm,SAT,AWS1-MWR-1B-RAD_C_OHB_20230816120142_G_D_20240115111111_20240115125434_T_B____radsim.nc file_reader: !!python/name:satpy.readers.aws_l1b.AWSL1BFile file_patterns: [ - 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc', + 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{disposition_mode:1s}_{processing_mode:1s}____.nc', 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB__{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc', 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____radsim.nc'] diff --git a/satpy/etc/readers/eps_sterna_l1b_nc.yaml b/satpy/etc/readers/eps_sterna_l1b_nc.yaml index 026aefc918..3313703ea4 100644 --- a/satpy/etc/readers/eps_sterna_l1b_nc.yaml +++ b/satpy/etc/readers/eps_sterna_l1b_nc.yaml @@ -1,5 +1,5 @@ reader: - name: aws_l1b_nc + name: eps_sterna_l1b_nc short_name: AWS L1B RAD NetCDF4 long_name: AWS L1B Radiance (NetCDF4) description: Reader for the EUMETSAT EPS-Sterna Sounder level-1b files in netCDF4. @@ -528,5 +528,5 @@ file_types: # W_XX-EUMETSAT-Darmstadt,SAT,AWS1-MWR-1B-RAD_C_EUMT_20241121085911_G_D_20241109234502_20241110004559_T_N____.nc file_reader: !!python/name:satpy.readers.aws_l1b.AWSL1BFile file_patterns: [ - 'W_XX-EUMETSAT-Darmstadt,SAT,{platform_name}-MWR-1B-RAD_C_EUMT_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_N____.nc', + 'W_XX-EUMETSAT-Darmstadt,SAT,{platform_name}-MWR-1B-RAD_C_EUMT_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{disposition_mode:1s}_{processing_mode:1s}____.nc', ] From 23fcf3c7121d3289fdc1b727379740be427b113f Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Thu, 12 Dec 2024 11:29:13 +0100 Subject: [PATCH 17/48] Fix for xarray > 2024.09, and leave stand alone datatree Signed-off-by: Adam.Dybbroe --- satpy/tests/reader_tests/test_aws_l1b.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/tests/reader_tests/test_aws_l1b.py b/satpy/tests/reader_tests/test_aws_l1b.py index d3c45e4e3d..214f1e1599 100644 --- a/satpy/tests/reader_tests/test_aws_l1b.py +++ b/satpy/tests/reader_tests/test_aws_l1b.py @@ -8,8 +8,8 @@ import numpy as np import pytest import xarray as xr -from datatree import DataTree from trollsift import compose, parse +from xarray import DataTree from satpy.readers.aws_l1b import DATETIME_FORMAT, AWSL1BFile From b8b08e47308a423136a89b206c5814e7e1215a71 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 13 Dec 2024 14:34:42 +0100 Subject: [PATCH 18/48] First version of the l1b reader for the Arctic Weather Satellite Signed-off-by: Adam.Dybbroe # Conflicts: # satpy/etc/readers/aws_l1b_nc.yaml # satpy/readers/aws_l1b.py --- satpy/readers/aws_l1b.py | 1 - 1 file changed, 1 deletion(-) diff --git a/satpy/readers/aws_l1b.py b/satpy/readers/aws_l1b.py index 9023f6f0ab..b1ba4dabd2 100644 --- a/satpy/readers/aws_l1b.py +++ b/satpy/readers/aws_l1b.py @@ -43,7 +43,6 @@ class using the :mod:`~satpy.Scene.load` method with the reader ``"aws_l1b_nc"``. """ - def __init__(self, filename, filename_info, filetype_info, auto_maskandscale=True): """Initialize the handler.""" super().__init__(filename, filename_info, filetype_info, From 0c280046886bec208ead8b40877a99bb892e6353 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Sun, 15 Dec 2024 22:14:53 +0100 Subject: [PATCH 19/48] Fix the naming - The AWS/EPS-Sterna sensor is named "MWR" Signed-off-by: Adam.Dybbroe --- .../{aws_l1b_nc.yaml => aws1_mwr_l1b_nc.yaml} | 12 +- satpy/etc/readers/aws_l1c_nc.yaml | 375 ------------------ ...l1b_nc.yaml => eps_sterna_mwr_l1b_nc.yaml} | 2 +- satpy/readers/{aws_l1b.py => mwr_l1b.py} | 22 +- satpy/readers/{aws_l1c.py => mwr_l1c.py} | 2 +- satpy/tests/reader_tests/conftest.py | 3 +- .../{test_aws_l1b.py => test_aws1_mwr_l1b.py} | 59 ++- .../reader_tests/test_eps_sterna_mwr_l1b.py | 130 ++++++ 8 files changed, 190 insertions(+), 415 deletions(-) rename satpy/etc/readers/{aws_l1b_nc.yaml => aws1_mwr_l1b_nc.yaml} (98%) delete mode 100644 satpy/etc/readers/aws_l1c_nc.yaml rename satpy/etc/readers/{eps_sterna_l1b_nc.yaml => eps_sterna_mwr_l1b_nc.yaml} (99%) rename satpy/readers/{aws_l1b.py => mwr_l1b.py} (91%) rename satpy/readers/{aws_l1c.py => mwr_l1c.py} (97%) rename satpy/tests/reader_tests/{test_aws_l1b.py => test_aws1_mwr_l1b.py} (79%) create mode 100644 satpy/tests/reader_tests/test_eps_sterna_mwr_l1b.py diff --git a/satpy/etc/readers/aws_l1b_nc.yaml b/satpy/etc/readers/aws1_mwr_l1b_nc.yaml similarity index 98% rename from satpy/etc/readers/aws_l1b_nc.yaml rename to satpy/etc/readers/aws1_mwr_l1b_nc.yaml index 1f1341e0e6..7fc88332b2 100644 --- a/satpy/etc/readers/aws_l1b_nc.yaml +++ b/satpy/etc/readers/aws1_mwr_l1b_nc.yaml @@ -1,10 +1,10 @@ reader: - name: aws_l1b_nc - short_name: AWS L1B RAD NetCDF4 - long_name: AWS L1B Radiance (NetCDF4) - description: Reader for the ESA AWS (Arctic Weather Satellite) Sounder level-1b files in netCDF4. + name: aws1_mwr_l1b_nc + short_name: AWS1 MWR L1B RAD NetCDF4 + long_name: AWS1 MWR L1B Radiance (NetCDF4) + description: Reader for the ESA AWS (Arctic Weather Satellite) Micorwave Radiometer (MWR) level-1b files in netCDF4. reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader - sensors: [aws,] + sensors: [mwr,] status: Beta supports_fsspec: false @@ -528,7 +528,7 @@ file_types: # W_XX-OHB-Unknown,SAT,1-AWS-1B-RAD_C_OHB_20230707124607_G_D_20220621090100_20220621090618_T_B____.nc # W_XX-OHB-Stockholm,SAT,AWS1-MWR-1B-RAD_C_OHB_20230823161321_G_D_20240115111111_20240115125434_T_B____.nc # W_XX-OHB-Stockholm,SAT,AWS1-MWR-1B-RAD_C_OHB_20230816120142_G_D_20240115111111_20240115125434_T_B____radsim.nc - file_reader: !!python/name:satpy.readers.aws_l1b.AWSL1BFile + file_reader: !!python/name:satpy.readers.mwr_l1b.AWS_EPS_Sterna_MWR_L1BFile file_patterns: [ 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{disposition_mode:1s}_{processing_mode:1s}____.nc', 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB__{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc', diff --git a/satpy/etc/readers/aws_l1c_nc.yaml b/satpy/etc/readers/aws_l1c_nc.yaml deleted file mode 100644 index 8abd9150ef..0000000000 --- a/satpy/etc/readers/aws_l1c_nc.yaml +++ /dev/null @@ -1,375 +0,0 @@ -reader: - name: aws_l1c_nc - short_name: AWS L1C RAD NetCDF4 - long_name: AWS L1C Radiance (NetCDF4) - description: Reader for the ESA AWS (Arctic Weather Satellite) Sounder level-1c files in netCDF4. - reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader - sensors: [aws,] - status: Beta - supports_fsspec: false - - data_identification_keys: - name: - required: true - frequency_double_sideband: - type: !!python/name:satpy.readers.pmw_channels_definitions.FrequencyDoubleSideBand - frequency_range: - type: !!python/name:satpy.readers.pmw_channels_definitions.FrequencyRange - resolution: - polarization: - enum: - - QH - - QV - calibration: - enum: - - brightness_temperature - transitive: true - modifiers: - required: true - default: [] - type: !!python/name:satpy.dataset.ModifierTuple - - coord_identification_keys: - name: - required: true - resolution: - polarization: - enum: - - QH - - QV - -datasets: - '1': - name: '1' - frequency_range: - central: 50.3 - bandwidth: 0.180 - unit: GHz - polarization: 'QH' - resolution: 40000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - coordinates: [longitude, latitude] - file_type: aws_l1c_nc - file_key: data/calibration/aws_toa_brightness_temperature - '2': - name: '2' - frequency_range: - central: 52.8 - bandwidth: 0.400 - unit: GHz - polarization: 'QH' - resolution: 40000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - coordinates: [longitude, latitude] - file_type: aws_l1c_nc - file_key: data/calibration/aws_toa_brightness_temperature - '3': - name: '3' - frequency_range: - central: 53.246 - bandwidth: 0.300 - unit: GHz - polarization: 'QH' - resolution: 40000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - coordinates: [longitude, latitude] - file_type: aws_l1c_nc - file_key: data/calibration/aws_toa_brightness_temperature - '4': - name: '4' - frequency_range: - central: 53.596 - bandwidth: 0.370 - unit: GHz - polarization: 'QH' - resolution: 40000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - coordinates: [longitude, latitude] - file_type: aws_l1c_nc - file_key: data/calibration/aws_toa_brightness_temperature - '5': - name: '5' - frequency_range: - central: 54.4 - bandwidth: 0.400 - unit: GHz - polarization: 'QH' - resolution: 40000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - coordinates: [longitude, latitude] - file_type: aws_l1c_nc - file_key: data/calibration/aws_toa_brightness_temperature - '6': - name: '6' - frequency_range: - central: 54.94 - bandwidth: 0.400 - unit: GHz - polarization: 'QH' - resolution: 40000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - coordinates: [longitude, latitude] - file_type: aws_l1c_nc - file_key: data/calibration/aws_toa_brightness_temperature - '7': - name: '7' - frequency_range: - central: 55.5 - bandwidth: 0.330 - unit: GHz - polarization: 'QH' - resolution: 40000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - coordinates: [longitude, latitude] - file_type: aws_l1c_nc - file_key: data/calibration/aws_toa_brightness_temperature - '8': - name: '8' - frequency_range: - central: 57.290344 - bandwidth: 0.330 - unit: GHz - polarization: 'QH' - resolution: 40000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - coordinates: [longitude, latitude] - file_type: aws_l1c_nc - file_key: data/calibration/aws_toa_brightness_temperature - '9': - name: '9' - frequency_range: - central: 89.0 - bandwidth: 4.0 - unit: GHz - polarization: 'QV' - resolution: 20000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - coordinates: [longitude, latitude] - file_type: aws_l1c_nc - file_key: data/calibration/aws_toa_brightness_temperature - '10': - name: '10' - frequency_range: - central: 165.5 - bandwidth: 2.700 - unit: GHz - polarization: 'QH' - resolution: 20000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - coordinates: [longitude, latitude] - file_type: aws_l1c_nc - file_key: data/calibration/aws_toa_brightness_temperature - '11': - name: '11' - frequency_range: - central: 176.311 - bandwidth: 2.0 - unit: GHz - polarization: 'QV' - resolution: 10000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - coordinates: [longitude, latitude] - file_type: aws_l1c_nc - file_key: data/calibration/aws_toa_brightness_temperature - '12': - name: '12' - frequency_range: - central: 178.811 - bandwidth: 2.0 - unit: GHz - polarization: 'QV' - resolution: 10000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - coordinates: [longitude, latitude] - file_type: aws_l1c_nc - file_key: data/calibration/aws_toa_brightness_temperature - '13': - name: '13' - frequency_range: - central: 180.311 - bandwidth: 1.0 - unit: GHz - polarization: 'QV' - resolution: 10000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - coordinates: [longitude, latitude] - file_type: aws_l1c_nc - file_key: data/calibration/aws_toa_brightness_temperature - '14': - name: '14' - frequency_range: - central: 181.511 - bandwidth: 1.0 - unit: GHz - polarization: 'QV' - resolution: 10000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - coordinates: [longitude, latitude] - file_type: aws_l1c_nc - file_key: data/calibration/aws_toa_brightness_temperature - '15': - name: '15' - frequency_range: - central: 182.311 - bandwidth: 0.5 - unit: GHz - polarization: 'QV' - resolution: 10000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - coordinates: [longitude, latitude] - file_type: aws_l1c_nc - file_key: data/calibration/aws_toa_brightness_temperature - '16': - name: '16' - frequency_double_sideband: - central: 325.15 - side: 1.2 - bandwidth: 0.8 - unit: GHz - polarization: 'QV' - resolution: 10000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - coordinates: [longitude, latitude] - file_type: aws_l1c_nc - file_key: data/calibration/aws_toa_brightness_temperature - '17': - name: '17' - frequency_double_sideband: - central: 325.15 - side: 2.4 - bandwidth: 1.2 - unit: GHz - polarization: 'QV' - resolution: 10000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - coordinates: [longitude, latitude] - file_type: aws_l1c_nc - file_key: data/calibration/aws_toa_brightness_temperature - '18': - name: '18' - frequency_double_sideband: - central: 325.15 - side: 4.1 - bandwidth: 1.8 - unit: GHz - polarization: 'QV' - resolution: 10000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - coordinates: [longitude, latitude] - file_type: aws_l1c_nc - file_key: data/calibration/aws_toa_brightness_temperature - '19': - name: '19' - frequency_double_sideband: - central: 325.15 - side: 6.6 - bandwidth: 2.8 - unit: GHz - polarization: 'QV' - resolution: 10000 - calibration: - brightness_temperature: - standard_name: toa_brightness_temperature - coordinates: [longitude, latitude] - file_type: aws_l1c_nc - file_key: data/calibration/aws_toa_brightness_temperature - -# --- Coordinates --- - - longitude: - name: longitude - file_type: aws_l1c_nc - standard_name: longitude - units: degrees_east - file_key: data/navigation/aws_lon - - latitude: - name: latitude - file_type: aws_l1c_nc - standard_name: latitude - units: degrees_north - file_key: data/navigation/aws_lat - -# --- Navigation data --- - - solar_azimuth: - name: solar_azimuth - file_type: aws_l1c_nc - file_key: data/navigation/aws_solar_azimuth_angle - standard_name: solar_azimuth_angle - coordinates: - - longitude - - latitude - - solar_zenith: - name: solar_zenith - file_type: aws_l1c_nc - file_key: data/navigation/aws_solar_zenith_angle - standard_name: solar_zenith_angle - coordinates: - - longitude - - latitude - - satellite_azimuth: - name: satellite_azimuth - file_type: aws_l1c_nc - file_key: data/navigation/aws_satellite_azimuth_angle - standard_name: satellite_azimuth_angle - coordinates: - - longitude - - latitude - - satellite_zenith: - name: satellite_zenith - file_type: aws_l1c_nc - file_key: data/navigation/aws_satellite_zenith_angle - standard_name: satellite_zenith_angle - coordinates: - - longitude - - latitude - -file_types: - aws_l1c_nc: - # W_XX-OHB-Unknown,SAT,1-AWS-1B-RAD_C_OHB_20230707124607_G_D_20220621090100_20220621090618_T_B____.nc - # W_XX-OHB-Stockholm,SAT,AWS1-MWR-1B-RAD_C_OHB_20230823161321_G_D_20240115111111_20240115125434_T_B____.nc - # W_XX-OHB-Stockholm,SAT,AWS1-MWR-1B-RAD_C_OHB_20230816120142_G_D_20240115111111_20240115125434_T_B____radsim.nc - file_reader: !!python/name:satpy.readers.aws_l1c.AWSL1CFile - file_patterns: [ - 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1C-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc', - 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1C-RAD_C_OHB__{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc',] diff --git a/satpy/etc/readers/eps_sterna_l1b_nc.yaml b/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml similarity index 99% rename from satpy/etc/readers/eps_sterna_l1b_nc.yaml rename to satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml index 3313703ea4..6cf6776d84 100644 --- a/satpy/etc/readers/eps_sterna_l1b_nc.yaml +++ b/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml @@ -526,7 +526,7 @@ datasets: file_types: eps_sterna_l1b_nc: # W_XX-EUMETSAT-Darmstadt,SAT,AWS1-MWR-1B-RAD_C_EUMT_20241121085911_G_D_20241109234502_20241110004559_T_N____.nc - file_reader: !!python/name:satpy.readers.aws_l1b.AWSL1BFile + file_reader: !!python/name:satpy.readers.aws_l1b.AWS_EPS_Sterna_MWR_L1BFile file_patterns: [ 'W_XX-EUMETSAT-Darmstadt,SAT,{platform_name}-MWR-1B-RAD_C_EUMT_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{disposition_mode:1s}_{processing_mode:1s}____.nc', ] diff --git a/satpy/readers/aws_l1b.py b/satpy/readers/mwr_l1b.py similarity index 91% rename from satpy/readers/aws_l1b.py rename to satpy/readers/mwr_l1b.py index b1ba4dabd2..d398cd94cd 100644 --- a/satpy/readers/aws_l1b.py +++ b/satpy/readers/mwr_l1b.py @@ -12,13 +12,17 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""Reader for the Arctic Weather Satellite (AWS) Sounder level-1b data. +"""Reader for the level-1b data from the MWR sounder onboard AWS and EPS-STerna. -Test data provided by ESA August 23, 2023. +AWS = Arctic Weather Satellite. MWR = Microwave Radiometer. + +AWS test data provided by ESA August 23, 2023. Sample data for five orbits in September 2024 provided by ESA to the Science Advisory Group for MWS and AWS, November 26, 2024. +Sample EPS-Sterna l1b format AWS data from 16 orbits the 9th of November 2024. + """ import logging @@ -34,13 +38,13 @@ AWS_CHANNEL_NAMES = list(str(i) for i in range(1, 20)) -class AWSL1BFile(NetCDF4FileHandler): - """Class implementing the AWS L1b Filehandler. +class AWS_EPS_Sterna_MWR_L1BFile(NetCDF4FileHandler): + """Class implementing the AWS/EPS-Sterna MWR L1b Filehandler. - This class implements the ESA Arctic Weather Satellite (AWS) Level-1b - NetCDF reader. It is designed to be used through the :class:`~satpy.Scene` - class using the :mod:`~satpy.Scene.load` method with the reader - ``"aws_l1b_nc"``. + This class implements the ESA Arctic Weather Satellite (AWS) and EPS-Sterna + MWR Level-1b NetCDF reader. It is designed to be used through the + :class:`~satpy.Scene` class using the :mod:`~satpy.Scene.load` method with + the reader ``"mwr_l1b_nc"``. """ def __init__(self, filename, filename_info, filetype_info, auto_maskandscale=True): @@ -130,7 +134,7 @@ def get_dataset(self, dataset_id, dataset_info): elif dataset_id["name"] in ["longitude", "latitude"]: data_array = self._get_navigation_data(dataset_id, dataset_info) else: - raise NotImplementedError + raise NotImplementedError(f"Dataset {dataset_id['name']} not available or not supported yet!") data_array = mask_and_scale(data_array) if dataset_id["name"] == "longitude": diff --git a/satpy/readers/aws_l1c.py b/satpy/readers/mwr_l1c.py similarity index 97% rename from satpy/readers/aws_l1c.py rename to satpy/readers/mwr_l1c.py index b4566c63f4..5dac981d5c 100644 --- a/satpy/readers/aws_l1c.py +++ b/satpy/readers/mwr_l1c.py @@ -76,7 +76,7 @@ def get_dataset(self, dataset_id, dataset_info): "satellite_zenith", "satellite_azimuth"]): data_array = self._get_navigation_data(dataset_id, dataset_info) else: - raise NotImplementedError + raise NotImplementedError(f"Dataset {dataset_id['name']} not available or not supported yet!") data_array = mask_and_scale(data_array) if dataset_id["name"] == "longitude": diff --git a/satpy/tests/reader_tests/conftest.py b/satpy/tests/reader_tests/conftest.py index 8f6f572494..c1be06b29c 100644 --- a/satpy/tests/reader_tests/conftest.py +++ b/satpy/tests/reader_tests/conftest.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2021 Satpy developers +# Copyright (c) 2021, 2024 Satpy developers # # This file is part of satpy. # @@ -15,4 +15,5 @@ # # You should have received a copy of the GNU General Public License along with # satpy. If not, see . + """Setup and configuration for all reader tests.""" diff --git a/satpy/tests/reader_tests/test_aws_l1b.py b/satpy/tests/reader_tests/test_aws1_mwr_l1b.py similarity index 79% rename from satpy/tests/reader_tests/test_aws_l1b.py rename to satpy/tests/reader_tests/test_aws1_mwr_l1b.py index 214f1e1599..817f0f23d5 100644 --- a/satpy/tests/reader_tests/test_aws_l1b.py +++ b/satpy/tests/reader_tests/test_aws1_mwr_l1b.py @@ -11,7 +11,7 @@ from trollsift import compose, parse from xarray import DataTree -from satpy.readers.aws_l1b import DATETIME_FORMAT, AWSL1BFile +from satpy.readers.mwr_l1b import DATETIME_FORMAT, AWS_EPS_Sterna_MWR_L1BFile platform_name = "AWS1" file_pattern = "W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc" # noqa @@ -45,8 +45,8 @@ def random_date(start, end): @pytest.fixture(scope="session") -def aws_file(tmp_path_factory): - """Create an AWS file.""" +def aws_mwr_file(tmp_path_factory): + """Create an AWS MWR l1b file.""" ds = DataTree() start_time = datetime(2024, 9, 1, 12, 0) ds.attrs["sensing_start_time_utc"] = start_time.strftime(DATETIME_FORMAT) @@ -54,7 +54,7 @@ def aws_file(tmp_path_factory): ds.attrs["sensing_end_time_utc"] = end_time.strftime(DATETIME_FORMAT) processing_time = random_date(datetime(2024, 6, 1), datetime(2030, 6, 1)) - instrument = "AWS" + instrument = "MWR" ds.attrs["instrument"] = instrument ds.attrs["orbit_start"] = 9991 ds.attrs["orbit_end"] = 9992 @@ -87,27 +87,33 @@ def aws_file(tmp_path_factory): @pytest.fixture -def aws_handler(aws_file): - """Create an aws filehandler.""" - filename_info = parse(file_pattern, os.path.basename(aws_file)) +def mwr_handler(aws_mwr_file): + """Create an AWS MWR filehandler.""" + filename_info = parse(file_pattern, os.path.basename(aws_mwr_file)) filetype_info = dict() - filetype_info["file_type"] = "aws_l1b" - return AWSL1BFile(aws_file, filename_info, filetype_info) + filetype_info["file_type"] = "aws1_mwr_l1b" + return AWS_EPS_Sterna_MWR_L1BFile(aws_mwr_file, filename_info, filetype_info) -def test_start_end_time(aws_handler): +def test_start_end_time(mwr_handler): """Test that start and end times are read correctly.""" - assert aws_handler.start_time == datetime(2024, 9, 1, 12, 0) - assert aws_handler.end_time == datetime(2024, 9, 1, 12, 15) + assert mwr_handler.start_time == datetime(2024, 9, 1, 12, 0) + assert mwr_handler.end_time == datetime(2024, 9, 1, 12, 15) -def test_metadata(aws_handler): +def test_orbit_number_start_end(mwr_handler): + """Test that start and end orbit number is read correctly.""" + assert mwr_handler.orbit_start == 9991 + assert mwr_handler.orbit_end == 9992 + + +def test_metadata(mwr_handler): """Test that the metadata is read correctly.""" - assert aws_handler.sensor == "AWS" - assert aws_handler.platform_name == platform_name + assert mwr_handler.sensor == "MWR" + assert mwr_handler.platform_name == platform_name -def test_get_channel_data(aws_handler): +def test_get_channel_data(mwr_handler): """Test retrieving the channel data.""" did = dict(name="1") dataset_info = dict(file_key="data/calibration/aws_toa_brightness_temperature") @@ -119,7 +125,7 @@ def test_get_channel_data(aws_handler): expected = expected.where(expected >= 0) # "calibrate" expected = expected * 0.001 - res = aws_handler.get_dataset(did, dataset_info) + res = mwr_handler.get_dataset(did, dataset_info) np.testing.assert_allclose(res, expected) assert "x" in res.dims assert "y" in res.dims @@ -127,7 +133,7 @@ def test_get_channel_data(aws_handler): assert res.attrs["orbital_parameters"]["sub_satellite_longitude_end"] == 296.79 assert res.dims == ("y", "x") assert "n_channels" not in res.coords - assert res.attrs["sensor"] == "AWS" + assert res.attrs["sensor"] == "MWR" assert res.attrs["platform_name"] == "AWS1" @@ -135,12 +141,12 @@ def test_get_channel_data(aws_handler): [("longitude", "data/navigation/aws_lon", fake_lon_data * 1e-4), ("latitude", "data/navigation/aws_lat", fake_lat_data), ]) -def test_get_navigation_data(aws_handler, id_name, file_key, fake_array): +def test_get_navigation_data(mwr_handler, id_name, file_key, fake_array): """Test retrieving the geolocation (lon-lat) data.""" Horn = Enum("Horn", ["1", "2", "3", "4"]) did = dict(name=id_name, horn=Horn["1"]) dataset_info = dict(file_key=file_key, standard_name=id_name) - res = aws_handler.get_dataset(did, dataset_info) + res = mwr_handler.get_dataset(did, dataset_info) if id_name == "longitude": fake_array = fake_array.where(fake_array <= 180, fake_array - 360) @@ -160,13 +166,13 @@ def test_get_navigation_data(aws_handler, id_name, file_key, fake_array): ("solar_zenith_horn1", "data/navigation/aws_solar_zenith_angle", fake_sun_zen_data), ("satellite_azimuth_horn1", "data/navigation/aws_satellite_azimuth_angle", fake_sat_azi_data), ("satellite_zenith_horn1", "data/navigation/aws_satellite_zenith_angle", fake_sat_zen_data)]) -def test_get_viewing_geometry_data(aws_handler, id_name, file_key, fake_array): +def test_get_viewing_geometry_data(mwr_handler, id_name, file_key, fake_array): """Test retrieving the angles_data.""" Horn = Enum("Horn", ["1", "2", "3", "4"]) dset_id = dict(name=id_name, horn=Horn["1"]) dataset_info = dict(file_key=file_key, standard_name=id_name) - res = aws_handler.get_dataset(dset_id, dataset_info) + res = mwr_handler.get_dataset(dset_id, dataset_info) np.testing.assert_allclose(res, fake_array.isel(n_geo_groups=0)) assert "x" in res.dims @@ -177,3 +183,12 @@ def test_get_viewing_geometry_data(aws_handler, id_name, file_key, fake_array): assert "n_geo_groups" not in res.coords if id_name == "longitude": assert res.max() <= 180 + +def test_try_get_data_not_in_file(mwr_handler): + """Test retrieving a data field that is not available in the file.""" + did = dict(name="toa_brightness_temperature") + dataset_info = dict(file_key="data/calibration/toa_brightness_temperature") + + match_str = "Dataset toa_brightness_temperature not available or not supported yet!" + with pytest.raises(NotImplementedError, match=match_str): + _ = mwr_handler.get_dataset(did, dataset_info) diff --git a/satpy/tests/reader_tests/test_eps_sterna_mwr_l1b.py b/satpy/tests/reader_tests/test_eps_sterna_mwr_l1b.py new file mode 100644 index 0000000000..38c6d3aa78 --- /dev/null +++ b/satpy/tests/reader_tests/test_eps_sterna_mwr_l1b.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2024 Satpy developers + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Tests for the EPS-Sterna MWR l1b filehandlers.""" + +import os +from datetime import datetime +from enum import Enum + +import numpy as np +import pytest +import xarray as xr +from trollsift import compose, parse +from xarray import DataTree + +from satpy.readers.mwr_l1b import DATETIME_FORMAT, AWS_EPS_Sterna_MWR_L1BFile +from satpy.tests.reader_tests.test_aws1_mwr_l1b import random_date + +platform_name = "AWS1" +# W_XX-EUMETSAT-Darmstadt,SAT,AWS1-MWR-1B-RAD_C_EUMT_20241121085911_G_D_20241109234502_20241110004559_T_N____.nc +file_pattern = "W_XX-EUMETSAT-Darmstadt,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc" # noqa + +rng = np.random.default_rng() + +fake_data_np = rng.integers(0, 700000, size=10*145*19).reshape((10, 145, 19)) +fake_data_np[0, 0, 0] = -2147483648 +fake_data_np[1, 0, 0] = 700000 + 10 +fake_data_np[2, 0, 0] = -10 + +ARRAY_DIMS = ["n_scans", "n_fovs", "n_channels"] +fake_data = xr.DataArray(fake_data_np, dims=ARRAY_DIMS) + +GEO_DIMS = ["n_scans", "n_fovs", "n_feedhorns"] +GEO_SIZE = 10*145*4 +fake_lon_data = xr.DataArray(rng.integers(0, 3599999, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) +fake_lat_data = xr.DataArray(rng.integers(-900000, 900000, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) +fake_sun_azi_data = xr.DataArray(rng.integers(0, 36000, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) +fake_sun_zen_data = xr.DataArray(rng.integers(0, 36000, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) +fake_sat_azi_data = xr.DataArray(rng.integers(0, 36000, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) +fake_sat_zen_data = xr.DataArray(rng.integers(0, 36000, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) + + +@pytest.fixture(scope="session") +def eps_sterna_mwr_file(tmp_path_factory): + """Create an EPS-Sterna MWR l1b file.""" + ds = DataTree() + start_time = datetime(2024, 9, 1, 12, 0) + ds.attrs["sensing_start_time_utc"] = start_time.strftime(DATETIME_FORMAT) + end_time = datetime(2024, 9, 1, 12, 15) + ds.attrs["sensing_end_time_utc"] = end_time.strftime(DATETIME_FORMAT) + processing_time = random_date(datetime(2024, 6, 1), datetime(2030, 6, 1)) + + instrument = "MWR" + ds.attrs["instrument"] = instrument + ds.attrs["orbit_start"] = 9991 + ds.attrs["orbit_end"] = 9992 + ds["data/calibration/toa_brightness_temperature"] = fake_data + ds["data/calibration/toa_brightness_temperature"].attrs["scale_factor"] = 0.001 + ds["data/calibration/toa_brightness_temperature"].attrs["add_offset"] = 0.0 + ds["data/calibration/toa_brightness_temperature"].attrs["missing_value"] = -2147483648 + ds["data/calibration/toa_brightness_temperature"].attrs["valid_min"] = 0 + ds["data/calibration/toa_brightness_temperature"].attrs["valid_max"] = 700000 + + ds["data/navigation/longitude"] = fake_lon_data + ds["data/navigation/longitude"].attrs["scale_factor"] = 1e-4 + ds["data/navigation/longitude"].attrs["add_offset"] = 0.0 + ds["data/navigation/latitude"] = fake_lat_data + ds["data/navigation/solar_azimuth_angle"] = fake_sun_azi_data + ds["data/navigation/solar_zenith_angle"] = fake_sun_zen_data + ds["data/navigation/satellite_azimuth_angle"] = fake_sat_azi_data + ds["data/navigation/satellite_zenith_angle"] = fake_sat_zen_data + ds["status/satellite/subsat_latitude_end"] = np.array(22.39) + ds["status/satellite/subsat_longitude_start"] = np.array(304.79) + ds["status/satellite/subsat_latitude_start"] = np.array(55.41) + ds["status/satellite/subsat_longitude_end"] = np.array(296.79) + + tmp_dir = tmp_path_factory.mktemp("eps_sterna_mwr_l1b_tests") + filename = tmp_dir / compose(file_pattern, dict(start_time=start_time, end_time=end_time, + processing_time=processing_time, platform_name=platform_name)) + + ds.to_netcdf(filename) + return filename + + +@pytest.fixture +def mwr_handler(eps_sterna_mwr_file): + """Create an EPS-Sterna MWR filehandler.""" + filename_info = parse(file_pattern, os.path.basename(eps_sterna_mwr_file)) + filetype_info = dict() + filetype_info["file_type"] = "eps_sterna_mwr_l1b" + return AWS_EPS_Sterna_MWR_L1BFile(eps_sterna_mwr_file, filename_info, filetype_info) + + +@pytest.mark.parametrize(("id_name", "file_key", "fake_array"), + [("longitude", "data/navigation/longitude", fake_lon_data * 1e-4), + ("latitude", "data/navigation/latitude", fake_lat_data), + ]) +def test_get_navigation_data(mwr_handler, id_name, file_key, fake_array): + """Test retrieving the geolocation (lon-lat) data.""" + Horn = Enum("Horn", ["1", "2", "3", "4"]) + did = dict(name=id_name, horn=Horn["1"]) + dataset_info = dict(file_key=file_key, standard_name=id_name) + res = mwr_handler.get_dataset(did, dataset_info) + if id_name == "longitude": + fake_array = fake_array.where(fake_array <= 180, fake_array - 360) + + np.testing.assert_allclose(res, fake_array.isel(n_feedhorns=0)) + assert "x" in res.dims + assert "y" in res.dims + assert "orbital_parameters" in res.attrs + assert res.dims == ("y", "x") + assert "standard_name" in res.attrs + assert "n_feedhorns" not in res.coords + if id_name == "longitude": + assert res.max() <= 180 From 9ec30bca409c581a25eb83fd2907719cac9db808 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Mon, 16 Dec 2024 15:44:16 +0100 Subject: [PATCH 20/48] Refactor tests and fix naming for AWS/EPS-Sterna Radiometer = MWR Signed-off-by: Adam.Dybbroe --- pyproject.toml | 1 - satpy/etc/composites/{aws.yaml => mwr.yaml} | 0 satpy/etc/readers/aws1_mwr_l1c_nc.yaml | 375 ++++++++++++++++++ satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml | 4 +- satpy/readers/mwr_l1c.py | 6 +- satpy/tests/reader_tests/conftest.py | 169 ++++++++ satpy/tests/reader_tests/test_aws1_mwr_l1b.py | 136 ++----- .../reader_tests/test_eps_sterna_mwr_l1b.py | 99 +---- 8 files changed, 598 insertions(+), 192 deletions(-) rename satpy/etc/composites/{aws.yaml => mwr.yaml} (100%) create mode 100644 satpy/etc/readers/aws1_mwr_l1c_nc.yaml diff --git a/pyproject.toml b/pyproject.toml index 33b2d4de98..9ed0eda02d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,6 @@ hsaf_grib = ["pygrib"] remote_reading = ["fsspec"] insat_3d = ["xarray>=2024.10.0"] gms5-vissr_l1b = ["numba"] -aws_l1b = ["xarray-datatree"] # Writers: cf = ["h5netcdf >= 0.7.3"] awips_tiled = ["netCDF4 >= 1.1.8"] diff --git a/satpy/etc/composites/aws.yaml b/satpy/etc/composites/mwr.yaml similarity index 100% rename from satpy/etc/composites/aws.yaml rename to satpy/etc/composites/mwr.yaml diff --git a/satpy/etc/readers/aws1_mwr_l1c_nc.yaml b/satpy/etc/readers/aws1_mwr_l1c_nc.yaml new file mode 100644 index 0000000000..7df360ce4a --- /dev/null +++ b/satpy/etc/readers/aws1_mwr_l1c_nc.yaml @@ -0,0 +1,375 @@ +reader: + name: aws_l1c_nc + short_name: AWS L1C RAD NetCDF4 + long_name: AWS L1C Radiance (NetCDF4) + description: Reader for the ESA AWS (Arctic Weather Satellite) MWR level-1c files in netCDF4. + reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader + sensors: [mwr,] + status: Beta + supports_fsspec: false + + data_identification_keys: + name: + required: true + frequency_double_sideband: + type: !!python/name:satpy.readers.pmw_channels_definitions.FrequencyDoubleSideBand + frequency_range: + type: !!python/name:satpy.readers.pmw_channels_definitions.FrequencyRange + resolution: + polarization: + enum: + - QH + - QV + calibration: + enum: + - brightness_temperature + transitive: true + modifiers: + required: true + default: [] + type: !!python/name:satpy.dataset.ModifierTuple + + coord_identification_keys: + name: + required: true + resolution: + polarization: + enum: + - QH + - QV + +datasets: + '1': + name: '1' + frequency_range: + central: 50.3 + bandwidth: 0.180 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '2': + name: '2' + frequency_range: + central: 52.8 + bandwidth: 0.400 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '3': + name: '3' + frequency_range: + central: 53.246 + bandwidth: 0.300 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '4': + name: '4' + frequency_range: + central: 53.596 + bandwidth: 0.370 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '5': + name: '5' + frequency_range: + central: 54.4 + bandwidth: 0.400 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '6': + name: '6' + frequency_range: + central: 54.94 + bandwidth: 0.400 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '7': + name: '7' + frequency_range: + central: 55.5 + bandwidth: 0.330 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '8': + name: '8' + frequency_range: + central: 57.290344 + bandwidth: 0.330 + unit: GHz + polarization: 'QH' + resolution: 40000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '9': + name: '9' + frequency_range: + central: 89.0 + bandwidth: 4.0 + unit: GHz + polarization: 'QV' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '10': + name: '10' + frequency_range: + central: 165.5 + bandwidth: 2.700 + unit: GHz + polarization: 'QH' + resolution: 20000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '11': + name: '11' + frequency_range: + central: 176.311 + bandwidth: 2.0 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '12': + name: '12' + frequency_range: + central: 178.811 + bandwidth: 2.0 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '13': + name: '13' + frequency_range: + central: 180.311 + bandwidth: 1.0 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '14': + name: '14' + frequency_range: + central: 181.511 + bandwidth: 1.0 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '15': + name: '15' + frequency_range: + central: 182.311 + bandwidth: 0.5 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '16': + name: '16' + frequency_double_sideband: + central: 325.15 + side: 1.2 + bandwidth: 0.8 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '17': + name: '17' + frequency_double_sideband: + central: 325.15 + side: 2.4 + bandwidth: 1.2 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '18': + name: '18' + frequency_double_sideband: + central: 325.15 + side: 4.1 + bandwidth: 1.8 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + '19': + name: '19' + frequency_double_sideband: + central: 325.15 + side: 6.6 + bandwidth: 2.8 + unit: GHz + polarization: 'QV' + resolution: 10000 + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + coordinates: [longitude, latitude] + file_type: aws_l1c_nc + file_key: data/calibration/aws_toa_brightness_temperature + +# --- Coordinates --- + + longitude: + name: longitude + file_type: aws_l1c_nc + standard_name: longitude + units: degrees_east + file_key: data/navigation/aws_lon + + latitude: + name: latitude + file_type: aws_l1c_nc + standard_name: latitude + units: degrees_north + file_key: data/navigation/aws_lat + +# --- Navigation data --- + + solar_azimuth: + name: solar_azimuth + file_type: aws_l1c_nc + file_key: data/navigation/aws_solar_azimuth_angle + standard_name: solar_azimuth_angle + coordinates: + - longitude + - latitude + + solar_zenith: + name: solar_zenith + file_type: aws_l1c_nc + file_key: data/navigation/aws_solar_zenith_angle + standard_name: solar_zenith_angle + coordinates: + - longitude + - latitude + + satellite_azimuth: + name: satellite_azimuth + file_type: aws_l1c_nc + file_key: data/navigation/aws_satellite_azimuth_angle + standard_name: satellite_azimuth_angle + coordinates: + - longitude + - latitude + + satellite_zenith: + name: satellite_zenith + file_type: aws_l1c_nc + file_key: data/navigation/aws_satellite_zenith_angle + standard_name: satellite_zenith_angle + coordinates: + - longitude + - latitude + +file_types: + aws_l1c_nc: + # W_XX-OHB-Unknown,SAT,1-AWS-1B-RAD_C_OHB_20230707124607_G_D_20220621090100_20220621090618_T_B____.nc + # W_XX-OHB-Stockholm,SAT,AWS1-MWR-1B-RAD_C_OHB_20230823161321_G_D_20240115111111_20240115125434_T_B____.nc + # W_XX-OHB-Stockholm,SAT,AWS1-MWR-1B-RAD_C_OHB_20230816120142_G_D_20240115111111_20240115125434_T_B____radsim.nc + file_reader: !!python/name:satpy.readers.mwr_l1c.AWSL1CFile + file_patterns: [ + 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1C-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc', + 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1C-RAD_C_OHB__{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc',] diff --git a/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml b/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml index 6cf6776d84..c3a665b9c4 100644 --- a/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml +++ b/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml @@ -4,7 +4,7 @@ reader: long_name: AWS L1B Radiance (NetCDF4) description: Reader for the EUMETSAT EPS-Sterna Sounder level-1b files in netCDF4. reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader - sensors: [aws,] + sensors: [mwr,] status: Beta supports_fsspec: false @@ -526,7 +526,7 @@ datasets: file_types: eps_sterna_l1b_nc: # W_XX-EUMETSAT-Darmstadt,SAT,AWS1-MWR-1B-RAD_C_EUMT_20241121085911_G_D_20241109234502_20241110004559_T_N____.nc - file_reader: !!python/name:satpy.readers.aws_l1b.AWS_EPS_Sterna_MWR_L1BFile + file_reader: !!python/name:satpy.readers.mwr_l1b.AWS_EPS_Sterna_MWR_L1BFile file_patterns: [ 'W_XX-EUMETSAT-Darmstadt,SAT,{platform_name}-MWR-1B-RAD_C_EUMT_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{disposition_mode:1s}_{processing_mode:1s}____.nc', ] diff --git a/satpy/readers/mwr_l1c.py b/satpy/readers/mwr_l1c.py index 5dac981d5c..9acfc604a7 100644 --- a/satpy/readers/mwr_l1c.py +++ b/satpy/readers/mwr_l1c.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, 2024 Pytroll Developers +# Copyright (c) 2024 Pytroll Developers # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,7 +12,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""Reader for the Arctic Weather Satellite (AWS) Sounder level-1c data. +"""Reader for the Arctic Weather Satellite (AWS) MWR level-1c data. + +MWR = Microwaver Radiometer, onboard AWS and EPS-Sterna Sample data provided by ESA September 27, 2024. """ diff --git a/satpy/tests/reader_tests/conftest.py b/satpy/tests/reader_tests/conftest.py index c1be06b29c..65c1303865 100644 --- a/satpy/tests/reader_tests/conftest.py +++ b/satpy/tests/reader_tests/conftest.py @@ -17,3 +17,172 @@ # satpy. If not, see . """Setup and configuration for all reader tests.""" + +import os +from datetime import datetime, timedelta +from random import randrange + +import numpy as np +import pytest +import xarray as xr +from trollsift import compose, parse +from xarray import DataTree + +from satpy.readers.mwr_l1b import DATETIME_FORMAT, AWS_EPS_Sterna_MWR_L1BFile + +platform_name = "AWS1" +# W_XX-EUMETSAT-Darmstadt,SAT,AWS1-MWR-1B-RAD_C_EUMT_20241121085911_G_D_20241109234502_20241110004559_T_N____.nc +eumetsat_file_pattern = "W_XX-EUMETSAT-Darmstadt,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc" # noqa + +esa_file_pattern = "W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc" # noqa + +rng = np.random.default_rng() + +def random_date(start, end): + """Create a random datetime between two datetimes.""" + delta = end - start + int_delta = (delta.days * 24 * 60 * 60) + delta.seconds + random_second = randrange(int_delta) + return start + timedelta(seconds=random_second) + + +@pytest.fixture(scope="session") +def fake_mwr_data_array(): + """Return a fake AWS/EPS-Sterna MWR l1b data array.""" + fake_data_np = rng.integers(0, 700000, size=10*145*19).reshape((10, 145, 19)) + fake_data_np[0, 0, 0] = -2147483648 + fake_data_np[1, 0, 0] = 700000 + 10 + fake_data_np[2, 0, 0] = -10 + array_dims = ["n_scans", "n_fovs", "n_channels"] + return xr.DataArray(fake_data_np, dims=array_dims) + + +def make_fake_angles(geo_size, geo_dims): + """Return fake sun-satellite angle array.""" + maxval = 36000 + dummy_array = (np.arange(0, geo_size) * maxval/geo_size).astype("int32") + return xr.DataArray(dummy_array.reshape((10, 145, 4)), dims=geo_dims) + + +def make_fake_mwr_lonlats(geo_size, geo_dims): + """Return fake geolocation data arrays.""" + maxval = 3600000 + dummy_array = (np.arange(0, geo_size) * maxval/geo_size).astype("int32") + fake_lon_data = xr.DataArray(dummy_array.reshape((10, 145, 4)), dims=geo_dims) + maxval = 1800000 + dummy_array = (np.arange(0, geo_size) * maxval/geo_size - maxval/2).astype("int32") + fake_lat_data = xr.DataArray(dummy_array.reshape((10, 145, 4)), dims=geo_dims) + return (fake_lon_data, fake_lat_data) + + +@pytest.fixture(scope="module") +def eps_sterna_mwr_file(tmp_path_factory, fake_mwr_data_array): + """Create an EPS-Sterna MWR l1b file.""" + geo_dims = ["n_scans", "n_fovs", "n_feedhorns"] + geo_size = 10*145*4 + + ds = DataTree() + start_time = datetime(2024, 9, 1, 12, 0) + ds.attrs["sensing_start_time_utc"] = start_time.strftime(DATETIME_FORMAT) + end_time = datetime(2024, 9, 1, 12, 15) + ds.attrs["sensing_end_time_utc"] = end_time.strftime(DATETIME_FORMAT) + processing_time = random_date(datetime(2024, 9, 1, 13), datetime(2030, 6, 1)) + + instrument = "MWR" + ds.attrs["instrument"] = instrument + ds.attrs["orbit_start"] = 9991 + ds.attrs["orbit_end"] = 9992 + ds["data/calibration/toa_brightness_temperature"] = fake_mwr_data_array + ds["data/calibration/toa_brightness_temperature"].attrs["scale_factor"] = 0.001 + ds["data/calibration/toa_brightness_temperature"].attrs["add_offset"] = 0.0 + ds["data/calibration/toa_brightness_temperature"].attrs["missing_value"] = -2147483648 + ds["data/calibration/toa_brightness_temperature"].attrs["valid_min"] = 0 + ds["data/calibration/toa_brightness_temperature"].attrs["valid_max"] = 700000 + + fake_lon_data, fake_lat_data = make_fake_mwr_lonlats(geo_size, geo_dims) + + ds["data/navigation/longitude"] = fake_lon_data + ds["data/navigation/longitude"].attrs["scale_factor"] = 1e-4 + ds["data/navigation/longitude"].attrs["add_offset"] = 0.0 + ds["data/navigation/latitude"] = fake_lat_data + ds["data/navigation/solar_azimuth_angle"] = make_fake_angles(geo_size, geo_dims) + ds["data/navigation/solar_zenith_angle"] = make_fake_angles(geo_size, geo_dims) + ds["data/navigation/satellite_azimuth_angle"] = make_fake_angles(geo_size, geo_dims) + ds["data/navigation/satellite_zenith_angle"] = make_fake_angles(geo_size, geo_dims) + ds["status/satellite/subsat_latitude_end"] = np.array(22.39) + ds["status/satellite/subsat_longitude_start"] = np.array(304.79) + ds["status/satellite/subsat_latitude_start"] = np.array(55.41) + ds["status/satellite/subsat_longitude_end"] = np.array(296.79) + + tmp_dir = tmp_path_factory.mktemp("eps_sterna_mwr_l1b_tests") + filename = tmp_dir / compose(eumetsat_file_pattern, dict(start_time=start_time, end_time=end_time, + processing_time=processing_time, + platform_name=platform_name)) + + ds.to_netcdf(filename) + return filename + + +@pytest.fixture +def eps_sterna_mwr_handler(eps_sterna_mwr_file): + """Create an EPS-Sterna MWR filehandler.""" + filename_info = parse(eumetsat_file_pattern, os.path.basename(eps_sterna_mwr_file)) + filetype_info = dict() + filetype_info["file_type"] = "eps_sterna_mwr_l1b" + return AWS_EPS_Sterna_MWR_L1BFile(eps_sterna_mwr_file, filename_info, filetype_info) + + +@pytest.fixture(scope="session") +def aws_mwr_file(tmp_path_factory, fake_mwr_data_array): + """Create an AWS MWR l1b file.""" + geo_dims = ["n_scans", "n_fovs", "n_geo_groups"] + geo_size = 10*145*4 + + ds = DataTree() + start_time = datetime(2024, 9, 1, 12, 0) + ds.attrs["sensing_start_time_utc"] = start_time.strftime(DATETIME_FORMAT) + end_time = datetime(2024, 9, 1, 12, 15) + ds.attrs["sensing_end_time_utc"] = end_time.strftime(DATETIME_FORMAT) + processing_time = random_date(datetime(2024, 6, 1), datetime(2030, 6, 1)) + + instrument = "MWR" + ds.attrs["instrument"] = instrument + ds.attrs["orbit_start"] = 9991 + ds.attrs["orbit_end"] = 9992 + ds["data/calibration/aws_toa_brightness_temperature"] = fake_mwr_data_array + ds["data/calibration/aws_toa_brightness_temperature"].attrs["scale_factor"] = 0.001 + ds["data/calibration/aws_toa_brightness_temperature"].attrs["add_offset"] = 0.0 + ds["data/calibration/aws_toa_brightness_temperature"].attrs["missing_value"] = -2147483648 + ds["data/calibration/aws_toa_brightness_temperature"].attrs["valid_min"] = 0 + ds["data/calibration/aws_toa_brightness_temperature"].attrs["valid_max"] = 700000 + + fake_lon_data, fake_lat_data = make_fake_mwr_lonlats(geo_size, geo_dims) + + ds["data/navigation/aws_lon"] = fake_lon_data + ds["data/navigation/aws_lon"].attrs["scale_factor"] = 1e-4 + ds["data/navigation/aws_lon"].attrs["add_offset"] = 0.0 + ds["data/navigation/aws_lat"] = fake_lat_data + ds["data/navigation/aws_solar_azimuth_angle"] = make_fake_angles(geo_size, geo_dims) + ds["data/navigation/aws_solar_zenith_angle"] = make_fake_angles(geo_size, geo_dims) + ds["data/navigation/aws_satellite_azimuth_angle"] = make_fake_angles(geo_size, geo_dims) + ds["data/navigation/aws_satellite_zenith_angle"] = make_fake_angles(geo_size, geo_dims) + ds["status/satellite/subsat_latitude_end"] = np.array(22.39) + ds["status/satellite/subsat_longitude_start"] = np.array(304.79) + ds["status/satellite/subsat_latitude_start"] = np.array(55.41) + ds["status/satellite/subsat_longitude_end"] = np.array(296.79) + + tmp_dir = tmp_path_factory.mktemp("aws_l1b_tests") + filename = tmp_dir / compose(esa_file_pattern, dict(start_time=start_time, end_time=end_time, + processing_time=processing_time, platform_name=platform_name)) + + ds.to_netcdf(filename) + return filename + + +@pytest.fixture +def aws_mwr_handler(aws_mwr_file): + """Create an AWS MWR filehandler.""" + filename_info = parse(esa_file_pattern, os.path.basename(aws_mwr_file)) + filetype_info = dict() + filetype_info["file_type"] = "aws1_mwr_l1b" + return AWS_EPS_Sterna_MWR_L1BFile(aws_mwr_file, filename_info, filetype_info) diff --git a/satpy/tests/reader_tests/test_aws1_mwr_l1b.py b/satpy/tests/reader_tests/test_aws1_mwr_l1b.py index 817f0f23d5..6441715223 100644 --- a/satpy/tests/reader_tests/test_aws1_mwr_l1b.py +++ b/satpy/tests/reader_tests/test_aws1_mwr_l1b.py @@ -1,123 +1,50 @@ """Tests for aws l1b filehandlers.""" -import os -from datetime import datetime, timedelta +from datetime import datetime from enum import Enum -from random import randrange import numpy as np import pytest -import xarray as xr -from trollsift import compose, parse -from xarray import DataTree -from satpy.readers.mwr_l1b import DATETIME_FORMAT, AWS_EPS_Sterna_MWR_L1BFile +from satpy.tests.reader_tests.conftest import make_fake_angles, make_fake_mwr_lonlats platform_name = "AWS1" file_pattern = "W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc" # noqa -rng = np.random.default_rng() - -fake_data_np = rng.integers(0, 700000, size=10*145*19).reshape((10, 145, 19)) -fake_data_np[0, 0, 0] = -2147483648 -fake_data_np[1, 0, 0] = 700000 + 10 -fake_data_np[2, 0, 0] = -10 - -ARRAY_DIMS = ["n_scans", "n_fovs", "n_channels"] -fake_data = xr.DataArray(fake_data_np, dims=ARRAY_DIMS) - -GEO_DIMS = ["n_scans", "n_fovs", "n_geo_groups"] -GEO_SIZE = 10*145*4 -fake_lon_data = xr.DataArray(rng.integers(0, 3599999, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) -fake_lat_data = xr.DataArray(rng.integers(-900000, 900000, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) -fake_sun_azi_data = xr.DataArray(rng.integers(0, 36000, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) -fake_sun_zen_data = xr.DataArray(rng.integers(0, 36000, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) -fake_sat_azi_data = xr.DataArray(rng.integers(0, 36000, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) -fake_sat_zen_data = xr.DataArray(rng.integers(0, 36000, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) - - -def random_date(start, end): - """Create a random datetime between two datetimes.""" - delta = end - start - int_delta = (delta.days * 24 * 60 * 60) + delta.seconds - random_second = randrange(int_delta) - return start + timedelta(seconds=random_second) - - -@pytest.fixture(scope="session") -def aws_mwr_file(tmp_path_factory): - """Create an AWS MWR l1b file.""" - ds = DataTree() - start_time = datetime(2024, 9, 1, 12, 0) - ds.attrs["sensing_start_time_utc"] = start_time.strftime(DATETIME_FORMAT) - end_time = datetime(2024, 9, 1, 12, 15) - ds.attrs["sensing_end_time_utc"] = end_time.strftime(DATETIME_FORMAT) - processing_time = random_date(datetime(2024, 6, 1), datetime(2030, 6, 1)) - - instrument = "MWR" - ds.attrs["instrument"] = instrument - ds.attrs["orbit_start"] = 9991 - ds.attrs["orbit_end"] = 9992 - ds["data/calibration/aws_toa_brightness_temperature"] = fake_data - ds["data/calibration/aws_toa_brightness_temperature"].attrs["scale_factor"] = 0.001 - ds["data/calibration/aws_toa_brightness_temperature"].attrs["add_offset"] = 0.0 - ds["data/calibration/aws_toa_brightness_temperature"].attrs["missing_value"] = -2147483648 - ds["data/calibration/aws_toa_brightness_temperature"].attrs["valid_min"] = 0 - ds["data/calibration/aws_toa_brightness_temperature"].attrs["valid_max"] = 700000 - - ds["data/navigation/aws_lon"] = fake_lon_data - ds["data/navigation/aws_lon"].attrs["scale_factor"] = 1e-4 - ds["data/navigation/aws_lon"].attrs["add_offset"] = 0.0 - ds["data/navigation/aws_lat"] = fake_lat_data - ds["data/navigation/aws_solar_azimuth_angle"] = fake_sun_azi_data - ds["data/navigation/aws_solar_zenith_angle"] = fake_sun_zen_data - ds["data/navigation/aws_satellite_azimuth_angle"] = fake_sat_azi_data - ds["data/navigation/aws_satellite_zenith_angle"] = fake_sat_zen_data - ds["status/satellite/subsat_latitude_end"] = np.array(22.39) - ds["status/satellite/subsat_longitude_start"] = np.array(304.79) - ds["status/satellite/subsat_latitude_start"] = np.array(55.41) - ds["status/satellite/subsat_longitude_end"] = np.array(296.79) - - tmp_dir = tmp_path_factory.mktemp("aws_l1b_tests") - filename = tmp_dir / compose(file_pattern, dict(start_time=start_time, end_time=end_time, - processing_time=processing_time, platform_name=platform_name)) - - ds.to_netcdf(filename) - return filename - - -@pytest.fixture -def mwr_handler(aws_mwr_file): - """Create an AWS MWR filehandler.""" - filename_info = parse(file_pattern, os.path.basename(aws_mwr_file)) - filetype_info = dict() - filetype_info["file_type"] = "aws1_mwr_l1b" - return AWS_EPS_Sterna_MWR_L1BFile(aws_mwr_file, filename_info, filetype_info) - - -def test_start_end_time(mwr_handler): + +geo_dims = ["n_scans", "n_fovs", "n_geo_groups"] +geo_size = 10*145*4 +fake_lon_data, fake_lat_data = make_fake_mwr_lonlats(geo_size, geo_dims) +fake_sun_azi_data = make_fake_angles(geo_size, geo_dims) +fake_sun_zen_data = make_fake_angles(geo_size, geo_dims) +fake_sat_azi_data = make_fake_angles(geo_size, geo_dims) +fake_sat_zen_data = make_fake_angles(geo_size, geo_dims) + + + +def test_start_end_time(aws_mwr_handler): """Test that start and end times are read correctly.""" - assert mwr_handler.start_time == datetime(2024, 9, 1, 12, 0) - assert mwr_handler.end_time == datetime(2024, 9, 1, 12, 15) + assert aws_mwr_handler.start_time == datetime(2024, 9, 1, 12, 0) + assert aws_mwr_handler.end_time == datetime(2024, 9, 1, 12, 15) -def test_orbit_number_start_end(mwr_handler): +def test_orbit_number_start_end(aws_mwr_handler): """Test that start and end orbit number is read correctly.""" - assert mwr_handler.orbit_start == 9991 - assert mwr_handler.orbit_end == 9992 + assert aws_mwr_handler.orbit_start == 9991 + assert aws_mwr_handler.orbit_end == 9992 -def test_metadata(mwr_handler): +def test_metadata(aws_mwr_handler): """Test that the metadata is read correctly.""" - assert mwr_handler.sensor == "MWR" - assert mwr_handler.platform_name == platform_name + assert aws_mwr_handler.sensor == "MWR" + assert aws_mwr_handler.platform_name == platform_name -def test_get_channel_data(mwr_handler): +def test_get_channel_data(aws_mwr_handler, fake_mwr_data_array): """Test retrieving the channel data.""" did = dict(name="1") dataset_info = dict(file_key="data/calibration/aws_toa_brightness_temperature") - expected = fake_data.isel(n_channels=0) + expected = fake_mwr_data_array.isel(n_channels=0) # mask no_data value expected = expected.where(expected != -2147483648) # mask outside the valid range @@ -125,7 +52,7 @@ def test_get_channel_data(mwr_handler): expected = expected.where(expected >= 0) # "calibrate" expected = expected * 0.001 - res = mwr_handler.get_dataset(did, dataset_info) + res = aws_mwr_handler.get_dataset(did, dataset_info) np.testing.assert_allclose(res, expected) assert "x" in res.dims assert "y" in res.dims @@ -141,12 +68,12 @@ def test_get_channel_data(mwr_handler): [("longitude", "data/navigation/aws_lon", fake_lon_data * 1e-4), ("latitude", "data/navigation/aws_lat", fake_lat_data), ]) -def test_get_navigation_data(mwr_handler, id_name, file_key, fake_array): +def test_get_navigation_data(aws_mwr_handler, id_name, file_key, fake_array): """Test retrieving the geolocation (lon-lat) data.""" Horn = Enum("Horn", ["1", "2", "3", "4"]) did = dict(name=id_name, horn=Horn["1"]) dataset_info = dict(file_key=file_key, standard_name=id_name) - res = mwr_handler.get_dataset(did, dataset_info) + res = aws_mwr_handler.get_dataset(did, dataset_info) if id_name == "longitude": fake_array = fake_array.where(fake_array <= 180, fake_array - 360) @@ -166,13 +93,13 @@ def test_get_navigation_data(mwr_handler, id_name, file_key, fake_array): ("solar_zenith_horn1", "data/navigation/aws_solar_zenith_angle", fake_sun_zen_data), ("satellite_azimuth_horn1", "data/navigation/aws_satellite_azimuth_angle", fake_sat_azi_data), ("satellite_zenith_horn1", "data/navigation/aws_satellite_zenith_angle", fake_sat_zen_data)]) -def test_get_viewing_geometry_data(mwr_handler, id_name, file_key, fake_array): +def test_get_viewing_geometry_data(aws_mwr_handler, id_name, file_key, fake_array): """Test retrieving the angles_data.""" Horn = Enum("Horn", ["1", "2", "3", "4"]) dset_id = dict(name=id_name, horn=Horn["1"]) dataset_info = dict(file_key=file_key, standard_name=id_name) - res = mwr_handler.get_dataset(dset_id, dataset_info) + res = aws_mwr_handler.get_dataset(dset_id, dataset_info) np.testing.assert_allclose(res, fake_array.isel(n_geo_groups=0)) assert "x" in res.dims @@ -184,11 +111,12 @@ def test_get_viewing_geometry_data(mwr_handler, id_name, file_key, fake_array): if id_name == "longitude": assert res.max() <= 180 -def test_try_get_data_not_in_file(mwr_handler): + +def test_try_get_data_not_in_file(aws_mwr_handler): """Test retrieving a data field that is not available in the file.""" did = dict(name="toa_brightness_temperature") dataset_info = dict(file_key="data/calibration/toa_brightness_temperature") match_str = "Dataset toa_brightness_temperature not available or not supported yet!" with pytest.raises(NotImplementedError, match=match_str): - _ = mwr_handler.get_dataset(did, dataset_info) + _ = aws_mwr_handler.get_dataset(did, dataset_info) diff --git a/satpy/tests/reader_tests/test_eps_sterna_mwr_l1b.py b/satpy/tests/reader_tests/test_eps_sterna_mwr_l1b.py index 38c6d3aa78..0620bc8437 100644 --- a/satpy/tests/reader_tests/test_eps_sterna_mwr_l1b.py +++ b/satpy/tests/reader_tests/test_eps_sterna_mwr_l1b.py @@ -18,104 +18,27 @@ """Tests for the EPS-Sterna MWR l1b filehandlers.""" -import os -from datetime import datetime from enum import Enum import numpy as np import pytest -import xarray as xr -from trollsift import compose, parse -from xarray import DataTree -from satpy.readers.mwr_l1b import DATETIME_FORMAT, AWS_EPS_Sterna_MWR_L1BFile -from satpy.tests.reader_tests.test_aws1_mwr_l1b import random_date - -platform_name = "AWS1" -# W_XX-EUMETSAT-Darmstadt,SAT,AWS1-MWR-1B-RAD_C_EUMT_20241121085911_G_D_20241109234502_20241110004559_T_N____.nc -file_pattern = "W_XX-EUMETSAT-Darmstadt,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc" # noqa - -rng = np.random.default_rng() - -fake_data_np = rng.integers(0, 700000, size=10*145*19).reshape((10, 145, 19)) -fake_data_np[0, 0, 0] = -2147483648 -fake_data_np[1, 0, 0] = 700000 + 10 -fake_data_np[2, 0, 0] = -10 - -ARRAY_DIMS = ["n_scans", "n_fovs", "n_channels"] -fake_data = xr.DataArray(fake_data_np, dims=ARRAY_DIMS) - -GEO_DIMS = ["n_scans", "n_fovs", "n_feedhorns"] -GEO_SIZE = 10*145*4 -fake_lon_data = xr.DataArray(rng.integers(0, 3599999, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) -fake_lat_data = xr.DataArray(rng.integers(-900000, 900000, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) -fake_sun_azi_data = xr.DataArray(rng.integers(0, 36000, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) -fake_sun_zen_data = xr.DataArray(rng.integers(0, 36000, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) -fake_sat_azi_data = xr.DataArray(rng.integers(0, 36000, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) -fake_sat_zen_data = xr.DataArray(rng.integers(0, 36000, size=GEO_SIZE).reshape((10, 145, 4)), dims=GEO_DIMS) - - -@pytest.fixture(scope="session") -def eps_sterna_mwr_file(tmp_path_factory): - """Create an EPS-Sterna MWR l1b file.""" - ds = DataTree() - start_time = datetime(2024, 9, 1, 12, 0) - ds.attrs["sensing_start_time_utc"] = start_time.strftime(DATETIME_FORMAT) - end_time = datetime(2024, 9, 1, 12, 15) - ds.attrs["sensing_end_time_utc"] = end_time.strftime(DATETIME_FORMAT) - processing_time = random_date(datetime(2024, 6, 1), datetime(2030, 6, 1)) - - instrument = "MWR" - ds.attrs["instrument"] = instrument - ds.attrs["orbit_start"] = 9991 - ds.attrs["orbit_end"] = 9992 - ds["data/calibration/toa_brightness_temperature"] = fake_data - ds["data/calibration/toa_brightness_temperature"].attrs["scale_factor"] = 0.001 - ds["data/calibration/toa_brightness_temperature"].attrs["add_offset"] = 0.0 - ds["data/calibration/toa_brightness_temperature"].attrs["missing_value"] = -2147483648 - ds["data/calibration/toa_brightness_temperature"].attrs["valid_min"] = 0 - ds["data/calibration/toa_brightness_temperature"].attrs["valid_max"] = 700000 - - ds["data/navigation/longitude"] = fake_lon_data - ds["data/navigation/longitude"].attrs["scale_factor"] = 1e-4 - ds["data/navigation/longitude"].attrs["add_offset"] = 0.0 - ds["data/navigation/latitude"] = fake_lat_data - ds["data/navigation/solar_azimuth_angle"] = fake_sun_azi_data - ds["data/navigation/solar_zenith_angle"] = fake_sun_zen_data - ds["data/navigation/satellite_azimuth_angle"] = fake_sat_azi_data - ds["data/navigation/satellite_zenith_angle"] = fake_sat_zen_data - ds["status/satellite/subsat_latitude_end"] = np.array(22.39) - ds["status/satellite/subsat_longitude_start"] = np.array(304.79) - ds["status/satellite/subsat_latitude_start"] = np.array(55.41) - ds["status/satellite/subsat_longitude_end"] = np.array(296.79) - - tmp_dir = tmp_path_factory.mktemp("eps_sterna_mwr_l1b_tests") - filename = tmp_dir / compose(file_pattern, dict(start_time=start_time, end_time=end_time, - processing_time=processing_time, platform_name=platform_name)) - - ds.to_netcdf(filename) - return filename - - -@pytest.fixture -def mwr_handler(eps_sterna_mwr_file): - """Create an EPS-Sterna MWR filehandler.""" - filename_info = parse(file_pattern, os.path.basename(eps_sterna_mwr_file)) - filetype_info = dict() - filetype_info["file_type"] = "eps_sterna_mwr_l1b" - return AWS_EPS_Sterna_MWR_L1BFile(eps_sterna_mwr_file, filename_info, filetype_info) +from satpy.tests.reader_tests.conftest import make_fake_mwr_lonlats +geo_dims = ["n_scans", "n_fovs", "n_feedhorns"] +geo_size = 10*145*4 +fake_lon_data, fake_lat_data = make_fake_mwr_lonlats(geo_size, geo_dims) @pytest.mark.parametrize(("id_name", "file_key", "fake_array"), [("longitude", "data/navigation/longitude", fake_lon_data * 1e-4), ("latitude", "data/navigation/latitude", fake_lat_data), ]) -def test_get_navigation_data(mwr_handler, id_name, file_key, fake_array): +def test_get_navigation_data(eps_sterna_mwr_handler, id_name, file_key, fake_array): """Test retrieving the geolocation (lon-lat) data.""" Horn = Enum("Horn", ["1", "2", "3", "4"]) did = dict(name=id_name, horn=Horn["1"]) dataset_info = dict(file_key=file_key, standard_name=id_name) - res = mwr_handler.get_dataset(did, dataset_info) + res = eps_sterna_mwr_handler.get_dataset(did, dataset_info) if id_name == "longitude": fake_array = fake_array.where(fake_array <= 180, fake_array - 360) @@ -128,3 +51,13 @@ def test_get_navigation_data(mwr_handler, id_name, file_key, fake_array): assert "n_feedhorns" not in res.coords if id_name == "longitude": assert res.max() <= 180 + + +def test_try_get_data_not_in_file(eps_sterna_mwr_handler): + """Test retrieving a data field that is not available in the file.""" + did = dict(name="aws_toa_brightness_temperature") + dataset_info = dict(file_key="data/calibration/aws_toa_brightness_temperature") + + match_str = "Dataset aws_toa_brightness_temperature not available or not supported yet!" + with pytest.raises(NotImplementedError, match=match_str): + _ = eps_sterna_mwr_handler.get_dataset(did, dataset_info) From bf450fd217b9b8112b064d07f9ee5ac36127cb0d Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Mon, 16 Dec 2024 15:48:26 +0100 Subject: [PATCH 21/48] Revert back to the old (current) version Signed-off-by: Adam.Dybbroe --- satpy/etc/composites/atms.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/etc/composites/atms.yaml b/satpy/etc/composites/atms.yaml index 624f0bc93b..27afd5d2b8 100644 --- a/satpy/etc/composites/atms.yaml +++ b/satpy/etc/composites/atms.yaml @@ -14,5 +14,5 @@ composites: prerequisites: - name: '16' - name: '17' - - name: '18' + - name: '22' standard_name: mw183_humidity_surface From 8bee181393d397e8f7c50f0ac0cfbcff2127d9ab Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Mon, 16 Dec 2024 16:07:06 +0100 Subject: [PATCH 22/48] Bugfix Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/aws1_mwr_l1c_nc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/etc/readers/aws1_mwr_l1c_nc.yaml b/satpy/etc/readers/aws1_mwr_l1c_nc.yaml index 7df360ce4a..2a679ec182 100644 --- a/satpy/etc/readers/aws1_mwr_l1c_nc.yaml +++ b/satpy/etc/readers/aws1_mwr_l1c_nc.yaml @@ -1,5 +1,5 @@ reader: - name: aws_l1c_nc + name: aws1_mwr_l1c_nc short_name: AWS L1C RAD NetCDF4 long_name: AWS L1C Radiance (NetCDF4) description: Reader for the ESA AWS (Arctic Weather Satellite) MWR level-1c files in netCDF4. From 3badb8d00f64dd5c3597a7990d45ec4d0f1fa2f7 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Mon, 16 Dec 2024 17:22:38 +0100 Subject: [PATCH 23/48] Bugfix Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml b/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml index c3a665b9c4..9544a9ce3e 100644 --- a/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml +++ b/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml @@ -1,5 +1,5 @@ reader: - name: eps_sterna_l1b_nc + name: eps_sterna_mwr_l1b_nc short_name: AWS L1B RAD NetCDF4 long_name: AWS L1B Radiance (NetCDF4) description: Reader for the EUMETSAT EPS-Sterna Sounder level-1b files in netCDF4. From a6414c96af9e8a152836244193557c9d86d7ce53 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Mon, 16 Dec 2024 21:45:01 +0100 Subject: [PATCH 24/48] Refactor and share code between the l1b and l1c readers Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/aws1_mwr_l1c_nc.yaml | 2 +- satpy/readers/mwr_l1b.py | 12 ++--- satpy/readers/mwr_l1c.py | 64 ++------------------------ 3 files changed, 9 insertions(+), 69 deletions(-) diff --git a/satpy/etc/readers/aws1_mwr_l1c_nc.yaml b/satpy/etc/readers/aws1_mwr_l1c_nc.yaml index 2a679ec182..ad2acd4dc3 100644 --- a/satpy/etc/readers/aws1_mwr_l1c_nc.yaml +++ b/satpy/etc/readers/aws1_mwr_l1c_nc.yaml @@ -369,7 +369,7 @@ file_types: # W_XX-OHB-Unknown,SAT,1-AWS-1B-RAD_C_OHB_20230707124607_G_D_20220621090100_20220621090618_T_B____.nc # W_XX-OHB-Stockholm,SAT,AWS1-MWR-1B-RAD_C_OHB_20230823161321_G_D_20240115111111_20240115125434_T_B____.nc # W_XX-OHB-Stockholm,SAT,AWS1-MWR-1B-RAD_C_OHB_20230816120142_G_D_20240115111111_20240115125434_T_B____radsim.nc - file_reader: !!python/name:satpy.readers.mwr_l1c.AWSL1CFile + file_reader: !!python/name:satpy.readers.mwr_l1c.AWS_MWR_L1CFile file_patterns: [ 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1C-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc', 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1C-RAD_C_OHB__{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc',] diff --git a/satpy/readers/mwr_l1b.py b/satpy/readers/mwr_l1b.py index d398cd94cd..6767b68eb9 100644 --- a/satpy/readers/mwr_l1b.py +++ b/satpy/readers/mwr_l1b.py @@ -25,17 +25,11 @@ """ -import logging - import xarray as xr from .netcdf_utils import NetCDF4FileHandler -logger = logging.getLogger(__name__) - -DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f" - -AWS_CHANNEL_NAMES = list(str(i) for i in range(1, 20)) +MWR_CHANNEL_NAMES = list(str(i) for i in range(1, 20)) class AWS_EPS_Sterna_MWR_L1BFile(NetCDF4FileHandler): @@ -111,7 +105,7 @@ def sub_satellite_latitude_end(self): def get_dataset(self, dataset_id, dataset_info): """Get the data.""" - if dataset_id["name"] in AWS_CHANNEL_NAMES: + if dataset_id["name"] in MWR_CHANNEL_NAMES: data_array = self._get_channel_data(dataset_id, dataset_info) elif dataset_id["name"] in ["satellite_zenith_horn1", "satellite_zenith_horn2", @@ -154,7 +148,7 @@ def get_dataset(self, dataset_id, dataset_info): def _get_channel_data(self, dataset_id, dataset_info): channel_data = self[dataset_info["file_key"]] - channel_data.coords["n_channels"] = AWS_CHANNEL_NAMES + channel_data.coords["n_channels"] = MWR_CHANNEL_NAMES channel_data = channel_data.rename({"n_fovs": "x", "n_scans": "y"}) return channel_data.sel(n_channels=dataset_id["name"]).drop_vars("n_channels") diff --git a/satpy/readers/mwr_l1c.py b/satpy/readers/mwr_l1c.py index 9acfc604a7..6bc0320dc8 100644 --- a/satpy/readers/mwr_l1c.py +++ b/satpy/readers/mwr_l1c.py @@ -19,20 +19,10 @@ Sample data provided by ESA September 27, 2024. """ -import logging +from satpy.readers.mwr_l1b import MWR_CHANNEL_NAMES, AWS_EPS_Sterna_MWR_L1BFile, mask_and_scale -import xarray as xr -from .netcdf_utils import NetCDF4FileHandler - -logger = logging.getLogger(__name__) - -DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f" - -AWS_CHANNEL_NAMES = list(str(i) for i in range(1, 20)) - - -class AWSL1CFile(NetCDF4FileHandler): +class AWS_MWR_L1CFile(AWS_EPS_Sterna_MWR_L1BFile): """Class implementing the AWS L1c Filehandler. This class implements the ESA Arctic Weather Satellite (AWS) Level-1b @@ -41,37 +31,19 @@ class using the :mod:`~satpy.Scene.load` method with the reader ``"aws_l1c_nc"``. """ - def __init__(self, filename, filename_info, filetype_info, auto_maskandscale=True): """Initialize the handler.""" - super().__init__(filename, filename_info, filetype_info, - cache_var_size=10000, - cache_handle=True) + super().__init__(filename, filename_info, filetype_info, auto_maskandscale) self.filename_info = filename_info - @property - def start_time(self): - """Get the start time.""" - return self.filename_info["start_time"] - - @property - def end_time(self): - """Get the end time.""" - return self.filename_info["end_time"] - @property def sensor(self): """Get the sensor name.""" return "MWR" - @property - def platform_name(self): - """Get the platform name.""" - return self.filename_info["platform_name"] - def get_dataset(self, dataset_id, dataset_info): """Get the data.""" - if dataset_id["name"] in AWS_CHANNEL_NAMES: + if dataset_id["name"] in MWR_CHANNEL_NAMES: data_array = self._get_channel_data(dataset_id, dataset_info) elif (dataset_id["name"] in ["longitude", "latitude", "solar_azimuth", "solar_zenith", @@ -90,35 +62,9 @@ def get_dataset(self, dataset_id, dataset_info): data_array.attrs["sensor"] = self.sensor return data_array - def _get_channel_data(self, dataset_id, dataset_info): - channel_data = self[dataset_info["file_key"]] - channel_data.coords["n_channels"] = AWS_CHANNEL_NAMES - channel_data = channel_data.rename({"n_fovs": "x", "n_scans": "y"}) - return channel_data.sel(n_channels=dataset_id["name"]).drop_vars("n_channels") def _get_navigation_data(self, dataset_id, dataset_info): + """Get the navigation (geolocation) data.""" geo_data = self[dataset_info["file_key"]] geo_data = geo_data.rename({"n_fovs": "x", "n_scans": "y"}) return geo_data - - -def mask_and_scale(data_array): - """Mask then scale the data array.""" - if "missing_value" in data_array.attrs: - with xr.set_options(keep_attrs=True): - data_array = data_array.where(data_array != data_array.attrs["missing_value"]) - data_array.attrs.pop("missing_value") - if "valid_max" in data_array.attrs: - with xr.set_options(keep_attrs=True): - data_array = data_array.where(data_array <= data_array.attrs["valid_max"]) - data_array.attrs.pop("valid_max") - if "valid_min" in data_array.attrs: - with xr.set_options(keep_attrs=True): - data_array = data_array.where(data_array >= data_array.attrs["valid_min"]) - data_array.attrs.pop("valid_min") - if "scale_factor" in data_array.attrs and "add_offset" in data_array.attrs: - with xr.set_options(keep_attrs=True): - data_array = data_array * data_array.attrs["scale_factor"] + data_array.attrs["add_offset"] - data_array.attrs.pop("scale_factor") - data_array.attrs.pop("add_offset") - return data_array From a5d472a51396f228f4070ce11348d1c3b351f8a9 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Mon, 16 Dec 2024 21:49:02 +0100 Subject: [PATCH 25/48] Fix tests Signed-off-by: Adam.Dybbroe --- satpy/tests/reader_tests/conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/satpy/tests/reader_tests/conftest.py b/satpy/tests/reader_tests/conftest.py index 65c1303865..9e168518f4 100644 --- a/satpy/tests/reader_tests/conftest.py +++ b/satpy/tests/reader_tests/conftest.py @@ -28,7 +28,9 @@ from trollsift import compose, parse from xarray import DataTree -from satpy.readers.mwr_l1b import DATETIME_FORMAT, AWS_EPS_Sterna_MWR_L1BFile +from satpy.readers.mwr_l1b import AWS_EPS_Sterna_MWR_L1BFile + +DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f" platform_name = "AWS1" # W_XX-EUMETSAT-Darmstadt,SAT,AWS1-MWR-1B-RAD_C_EUMT_20241121085911_G_D_20241109234502_20241110004559_T_N____.nc From 55c999e0e65858bb518422f35e98b9137fcd6151 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 18 Dec 2024 20:44:21 +0100 Subject: [PATCH 26/48] Refactor and add test coverage for the ESA AWS Level-1c reader Signed-off-by: Adam.Dybbroe --- satpy/readers/mwr_l1b.py | 52 ++++++---- satpy/readers/mwr_l1c.py | 4 +- satpy/tests/reader_tests/conftest.py | 73 +++++++++++++- satpy/tests/reader_tests/test_aws1_mwr_l1b.py | 2 - satpy/tests/reader_tests/test_aws1_mwr_l1c.py | 99 +++++++++++++++++++ 5 files changed, 204 insertions(+), 26 deletions(-) create mode 100644 satpy/tests/reader_tests/test_aws1_mwr_l1c.py diff --git a/satpy/readers/mwr_l1b.py b/satpy/readers/mwr_l1b.py index 6767b68eb9..03a8b9f122 100644 --- a/satpy/readers/mwr_l1b.py +++ b/satpy/readers/mwr_l1b.py @@ -32,15 +32,9 @@ MWR_CHANNEL_NAMES = list(str(i) for i in range(1, 20)) -class AWS_EPS_Sterna_MWR_L1BFile(NetCDF4FileHandler): - """Class implementing the AWS/EPS-Sterna MWR L1b Filehandler. - - This class implements the ESA Arctic Weather Satellite (AWS) and EPS-Sterna - MWR Level-1b NetCDF reader. It is designed to be used through the - :class:`~satpy.Scene` class using the :mod:`~satpy.Scene.load` method with - the reader ``"mwr_l1b_nc"``. +class AWS_EPS_Sterna_BaseFileHandler(NetCDF4FileHandler): + """Base class implementing the AWS/EPS-Sterna MWR Level-1b&c Filehandlers.""" - """ def __init__(self, filename, filename_info, filetype_info, auto_maskandscale=True): """Initialize the handler.""" super().__init__(filename, filename_info, filetype_info, @@ -48,11 +42,6 @@ def __init__(self, filename, filename_info, filetype_info, auto_maskandscale=Tru cache_handle=True) self.filename_info = filename_info - if filetype_info["file_type"].startswith("eps_sterna"): - self._feed_horn_group_name = "n_feedhorns" - else: - self._feed_horn_group_name = "n_geo_groups" - @property def start_time(self): """Get the start time.""" @@ -83,6 +72,37 @@ def orbit_end(self): """Get the orbit number for the end of data.""" return int(self["/attr/orbit_end"]) + def get_dataset(self, dataset_id, dataset_info): + """Get the data.""" + raise NotImplementedError("This is not implemented in the Base class.") + + def _get_channel_data(self, dataset_id, dataset_info): + channel_data = self[dataset_info["file_key"]] + channel_data.coords["n_channels"] = MWR_CHANNEL_NAMES + channel_data = channel_data.rename({"n_fovs": "x", "n_scans": "y"}) + return channel_data.sel(n_channels=dataset_id["name"]).drop_vars("n_channels") + + + +class AWS_EPS_Sterna_MWR_L1BFile(AWS_EPS_Sterna_BaseFileHandler): + """Class implementing the AWS/EPS-Sterna MWR L1b Filehandler. + + This class implements the ESA Arctic Weather Satellite (AWS) and EPS-Sterna + MWR Level-1b NetCDF reader. It is designed to be used through the + :class:`~satpy.Scene` class using the :mod:`~satpy.Scene.load` method with + the reader ``"mwr_l1b_nc"``. + + """ + def __init__(self, filename, filename_info, filetype_info, auto_maskandscale=True): + """Initialize the handler.""" + super().__init__(filename, filename_info, filetype_info, auto_maskandscale) + + if filetype_info["file_type"].startswith("eps_sterna"): + self._feed_horn_group_name = "n_feedhorns" + else: + self._feed_horn_group_name = "n_geo_groups" + + @property def sub_satellite_longitude_start(self): """Get the longitude of sub-satellite point at start of the product.""" @@ -146,12 +166,6 @@ def get_dataset(self, dataset_id, dataset_info): data_array.attrs["orbit_number"] = self.orbit_start return data_array - def _get_channel_data(self, dataset_id, dataset_info): - channel_data = self[dataset_info["file_key"]] - channel_data.coords["n_channels"] = MWR_CHANNEL_NAMES - channel_data = channel_data.rename({"n_fovs": "x", "n_scans": "y"}) - return channel_data.sel(n_channels=dataset_id["name"]).drop_vars("n_channels") - def _get_navigation_data(self, dataset_id, dataset_info): """Get the navigation (geolocation) data for one feed horn.""" geo_data = self[dataset_info["file_key"]] diff --git a/satpy/readers/mwr_l1c.py b/satpy/readers/mwr_l1c.py index 6bc0320dc8..1b6f7269f4 100644 --- a/satpy/readers/mwr_l1c.py +++ b/satpy/readers/mwr_l1c.py @@ -19,10 +19,10 @@ Sample data provided by ESA September 27, 2024. """ -from satpy.readers.mwr_l1b import MWR_CHANNEL_NAMES, AWS_EPS_Sterna_MWR_L1BFile, mask_and_scale +from satpy.readers.mwr_l1b import MWR_CHANNEL_NAMES, AWS_EPS_Sterna_BaseFileHandler, mask_and_scale -class AWS_MWR_L1CFile(AWS_EPS_Sterna_MWR_L1BFile): +class AWS_MWR_L1CFile(AWS_EPS_Sterna_BaseFileHandler): """Class implementing the AWS L1c Filehandler. This class implements the ESA Arctic Weather Satellite (AWS) Level-1b diff --git a/satpy/tests/reader_tests/conftest.py b/satpy/tests/reader_tests/conftest.py index 9e168518f4..a541da2be5 100644 --- a/satpy/tests/reader_tests/conftest.py +++ b/satpy/tests/reader_tests/conftest.py @@ -29,6 +29,7 @@ from xarray import DataTree from satpy.readers.mwr_l1b import AWS_EPS_Sterna_MWR_L1BFile +from satpy.readers.mwr_l1c import AWS_MWR_L1CFile DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f" @@ -38,6 +39,9 @@ esa_file_pattern = "W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc" # noqa +esa_l1c_file_pattern = "W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1C-RAD_C_OHB__{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc" # noqa + + rng = np.random.default_rng() def random_date(start, end): @@ -59,15 +63,15 @@ def fake_mwr_data_array(): return xr.DataArray(fake_data_np, dims=array_dims) -def make_fake_angles(geo_size, geo_dims): +def make_fake_angles(geo_size, geo_dims, shape=(10, 145, 4)): """Return fake sun-satellite angle array.""" maxval = 36000 dummy_array = (np.arange(0, geo_size) * maxval/geo_size).astype("int32") - return xr.DataArray(dummy_array.reshape((10, 145, 4)), dims=geo_dims) + return xr.DataArray(dummy_array.reshape(shape), dims=geo_dims) def make_fake_mwr_lonlats(geo_size, geo_dims): - """Return fake geolocation data arrays.""" + """Return fake geolocation data arrays for all 4 MWR horns.""" maxval = 3600000 dummy_array = (np.arange(0, geo_size) * maxval/geo_size).astype("int32") fake_lon_data = xr.DataArray(dummy_array.reshape((10, 145, 4)), dims=geo_dims) @@ -77,6 +81,17 @@ def make_fake_mwr_lonlats(geo_size, geo_dims): return (fake_lon_data, fake_lat_data) +def make_fake_mwr_l1c_lonlats(geo_size, geo_dims): + """Return fake level-1c geolocation data arrays.""" + maxval = 3600000 + dummy_array = (np.arange(0, geo_size) * maxval/geo_size).astype("int32") + fake_lon_data = xr.DataArray(dummy_array.reshape((10, 145)), dims=geo_dims) + maxval = 1800000 + dummy_array = (np.arange(0, geo_size) * maxval/geo_size - maxval/2).astype("int32") + fake_lat_data = xr.DataArray(dummy_array.reshape((10, 145)), dims=geo_dims) + return (fake_lon_data, fake_lat_data) + + @pytest.fixture(scope="module") def eps_sterna_mwr_file(tmp_path_factory, fake_mwr_data_array): """Create an EPS-Sterna MWR l1b file.""" @@ -188,3 +203,55 @@ def aws_mwr_handler(aws_mwr_file): filetype_info = dict() filetype_info["file_type"] = "aws1_mwr_l1b" return AWS_EPS_Sterna_MWR_L1BFile(aws_mwr_file, filename_info, filetype_info) + + +@pytest.fixture(scope="session") +def aws_mwr_l1c_file(tmp_path_factory, fake_mwr_data_array): + """Create an AWS MWR l1c file.""" + geo_dims = ["n_scans", "n_fovs"] + geo_size = 10*145 + + ds = DataTree() + start_time = datetime(2024, 9, 1, 12, 0) + ds.attrs["sensing_start_time_utc"] = start_time.strftime(DATETIME_FORMAT) + end_time = datetime(2024, 9, 1, 12, 15) + ds.attrs["sensing_end_time_utc"] = end_time.strftime(DATETIME_FORMAT) + processing_time = random_date(datetime(2024, 6, 1), datetime(2030, 6, 1)) + + ds.attrs["instrument"] = "MWR" + ds.attrs["orbit_start"] = 9991 + ds.attrs["orbit_end"] = 9992 + ds["data/calibration/aws_toa_brightness_temperature"] = fake_mwr_data_array + ds["data/calibration/aws_toa_brightness_temperature"].attrs["scale_factor"] = 0.001 + ds["data/calibration/aws_toa_brightness_temperature"].attrs["add_offset"] = 0.0 + ds["data/calibration/aws_toa_brightness_temperature"].attrs["missing_value"] = -2147483648 + ds["data/calibration/aws_toa_brightness_temperature"].attrs["valid_min"] = 0 + ds["data/calibration/aws_toa_brightness_temperature"].attrs["valid_max"] = 700000 + + fake_lon_data, fake_lat_data = make_fake_mwr_l1c_lonlats(geo_size, geo_dims) + + ds["data/navigation/aws_lon"] = fake_lon_data + ds["data/navigation/aws_lon"].attrs["scale_factor"] = 1e-4 + ds["data/navigation/aws_lon"].attrs["add_offset"] = 0.0 + ds["data/navigation/aws_lat"] = fake_lat_data + ds["data/navigation/aws_solar_azimuth_angle"] = make_fake_angles(geo_size, geo_dims, shape=(10, 145)) + ds["data/navigation/aws_solar_zenith_angle"] = make_fake_angles(geo_size, geo_dims, shape=(10, 145)) + ds["data/navigation/aws_satellite_azimuth_angle"] = make_fake_angles(geo_size, geo_dims, shape=(10, 145)) + ds["data/navigation/aws_satellite_zenith_angle"] = make_fake_angles(geo_size, geo_dims, shape=(10, 145)) + + tmp_dir = tmp_path_factory.mktemp("aws_l1c_tests") + filename = tmp_dir / compose(esa_l1c_file_pattern, dict(start_time=start_time, end_time=end_time, + processing_time=processing_time, + platform_name=platform_name)) + + ds.to_netcdf(filename) + return filename + + +@pytest.fixture +def aws_mwr_l1c_handler(aws_mwr_l1c_file): + """Create an AWS MWR level-1c filehandler.""" + filename_info = parse(esa_l1c_file_pattern, os.path.basename(aws_mwr_l1c_file)) + filetype_info = dict() + filetype_info["file_type"] = "aws1_mwr_l1c" + return AWS_MWR_L1CFile(aws_mwr_l1c_file, filename_info, filetype_info) diff --git a/satpy/tests/reader_tests/test_aws1_mwr_l1b.py b/satpy/tests/reader_tests/test_aws1_mwr_l1b.py index 6441715223..9a789bd08f 100644 --- a/satpy/tests/reader_tests/test_aws1_mwr_l1b.py +++ b/satpy/tests/reader_tests/test_aws1_mwr_l1b.py @@ -108,8 +108,6 @@ def test_get_viewing_geometry_data(aws_mwr_handler, id_name, file_key, fake_arra assert res.dims == ("y", "x") assert "standard_name" in res.attrs assert "n_geo_groups" not in res.coords - if id_name == "longitude": - assert res.max() <= 180 def test_try_get_data_not_in_file(aws_mwr_handler): diff --git a/satpy/tests/reader_tests/test_aws1_mwr_l1c.py b/satpy/tests/reader_tests/test_aws1_mwr_l1c.py new file mode 100644 index 0000000000..fc4118bc34 --- /dev/null +++ b/satpy/tests/reader_tests/test_aws1_mwr_l1c.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2024 Satpy developers + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Tests for ESA Arctic Weather Satellite (AWS) level-1c file reading.""" + + + +import numpy as np +import pytest + +from satpy.tests.reader_tests.conftest import make_fake_angles, make_fake_mwr_l1c_lonlats + +platform_name = "AWS1" +# W_XX-OHB-Stockholm,SAT,AWS1-MWR-1C-RAD_C_OHB__20241126183628_G_D_20240913222540_20240914000332_T_B____.nc +file_pattern = "W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1C-RAD_C_OHB__{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc" # noqa + + +geo_dims = ["n_scans", "n_fovs"] +geo_size = 10*145 +fake_lon_data, fake_lat_data = make_fake_mwr_l1c_lonlats(geo_size, geo_dims) +fake_sun_azi_data = make_fake_angles(geo_size, geo_dims, shape=(10, 145)) +fake_sun_zen_data = make_fake_angles(geo_size, geo_dims, shape=(10, 145)) +fake_sat_azi_data = make_fake_angles(geo_size, geo_dims, shape=(10, 145)) +fake_sat_zen_data = make_fake_angles(geo_size, geo_dims, shape=(10, 145)) + + +def test_get_channel_data(aws_mwr_l1c_handler, fake_mwr_data_array): + """Test retrieving the channel data.""" + did = dict(name="1") + dataset_info = dict(file_key="data/calibration/aws_toa_brightness_temperature") + expected = fake_mwr_data_array.isel(n_channels=0) + # mask no_data value + expected = expected.where(expected != -2147483648) + # mask outside the valid range + expected = expected.where(expected <= 700000) + expected = expected.where(expected >= 0) + # "calibrate" + expected = expected * 0.001 + res = aws_mwr_l1c_handler.get_dataset(did, dataset_info) + np.testing.assert_allclose(res, expected) + assert "x" in res.dims + assert "y" in res.dims + assert res.dims == ("y", "x") + assert "n_channels" not in res.coords + assert res.attrs["sensor"] == "MWR" + assert res.attrs["platform_name"] == "AWS1" + + +@pytest.mark.parametrize(("id_name", "file_key", "fake_array"), + [("longitude", "data/navigation/aws_lon", fake_lon_data * 1e-4), + ("latitude", "data/navigation/aws_lat", fake_lat_data), + ]) +def test_get_navigation_data(aws_mwr_l1c_handler, id_name, file_key, fake_array): + """Test retrieving the geolocation (lon, lat) data.""" + did = dict(name=id_name) + dataset_info = dict(file_key=file_key, standard_name=id_name) + res = aws_mwr_l1c_handler.get_dataset(did, dataset_info) + if id_name == "longitude": + fake_array = fake_array.where(fake_array <= 180, fake_array - 360) + + np.testing.assert_allclose(res, fake_array) + assert "x" in res.dims + assert "y" in res.dims + assert res.dims == ("y", "x") + assert "standard_name" in res.attrs + if id_name == "longitude": + assert res.max() <= 180 + + +@pytest.mark.parametrize(("id_name", "file_key", "fake_array"), + [("solar_azimuth", "data/navigation/aws_solar_azimuth_angle", fake_sun_azi_data), + ("solar_zenith", "data/navigation/aws_solar_zenith_angle", fake_sun_zen_data), + ("satellite_azimuth", "data/navigation/aws_satellite_azimuth_angle", fake_sat_azi_data), + ("satellite_zenith", "data/navigation/aws_satellite_zenith_angle", fake_sat_zen_data)]) +def test_get_viewing_geometry_data(aws_mwr_l1c_handler, id_name, file_key, fake_array): + """Test retrieving the angles_data.""" + dset_id = dict(name=id_name) + dataset_info = dict(file_key=file_key, standard_name=id_name) + res = aws_mwr_l1c_handler.get_dataset(dset_id, dataset_info) + np.testing.assert_allclose(res, fake_array) + assert "x" in res.dims + assert "y" in res.dims + assert res.dims == ("y", "x") + assert "standard_name" in res.attrs From 110cd9c4f0fceca8ea805a22d58da85f08fada84 Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Thu, 19 Dec 2024 12:44:54 +0100 Subject: [PATCH 27/48] Update satpy/etc/composites/mwr.yaml Co-authored-by: Panu Lahtinen --- satpy/etc/composites/mwr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/etc/composites/mwr.yaml b/satpy/etc/composites/mwr.yaml index ba38a69d02..907185838a 100644 --- a/satpy/etc/composites/mwr.yaml +++ b/satpy/etc/composites/mwr.yaml @@ -1,4 +1,4 @@ -sensor_name: aws +sensor_name: mwr composites: mw183_humidity: From 7347fb0bfe01b622b3c5ba15ae5a62905ae2e37b Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Thu, 19 Dec 2024 12:46:31 +0100 Subject: [PATCH 28/48] Update satpy/tests/reader_tests/conftest.py Co-authored-by: Panu Lahtinen --- satpy/tests/reader_tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/tests/reader_tests/conftest.py b/satpy/tests/reader_tests/conftest.py index a541da2be5..aeec133f30 100644 --- a/satpy/tests/reader_tests/conftest.py +++ b/satpy/tests/reader_tests/conftest.py @@ -96,7 +96,7 @@ def make_fake_mwr_l1c_lonlats(geo_size, geo_dims): def eps_sterna_mwr_file(tmp_path_factory, fake_mwr_data_array): """Create an EPS-Sterna MWR l1b file.""" geo_dims = ["n_scans", "n_fovs", "n_feedhorns"] - geo_size = 10*145*4 + geo_size = 10 * 145 * 4 ds = DataTree() start_time = datetime(2024, 9, 1, 12, 0) From 1cf434c6b7d412b3bf9d7824fc03ca0f57a47eb6 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Thu, 19 Dec 2024 12:58:01 +0100 Subject: [PATCH 29/48] Fix file naming and secure composite recipe consistency Signed-off-by: Adam.Dybbroe --- satpy/etc/enhancements/generic.yaml | 2 +- satpy/etc/enhancements/{aws.yaml => mwr.yaml} | 12 +++---- satpy/readers/mwr_l1b.py | 33 ++++++++++--------- 3 files changed, 24 insertions(+), 23 deletions(-) rename satpy/etc/enhancements/{aws.yaml => mwr.yaml} (89%) diff --git a/satpy/etc/enhancements/generic.yaml b/satpy/etc/enhancements/generic.yaml index 8989d44152..ff6e500560 100644 --- a/satpy/etc/enhancements/generic.yaml +++ b/satpy/etc/enhancements/generic.yaml @@ -1291,7 +1291,7 @@ enhancements: operations: [] mw183_humidity: - # matches AWS + # matches EPS-Sterna and AWS MWR standard_name: mw183_humidity operations: - name: stretch diff --git a/satpy/etc/enhancements/aws.yaml b/satpy/etc/enhancements/mwr.yaml similarity index 89% rename from satpy/etc/enhancements/aws.yaml rename to satpy/etc/enhancements/mwr.yaml index 4d79b0ae11..da32d6a499 100644 --- a/satpy/etc/enhancements/aws.yaml +++ b/satpy/etc/enhancements/mwr.yaml @@ -3,16 +3,16 @@ enhancements: mw183_humidity: standard_name: mw183_humidity operations: - - name: inverse - method: !!python/name:satpy.enhancements.invert - args: - - [true, true, true] - name: stretch method: !!python/name:satpy.enhancements.stretch - kwargs: {stretch: linear} + kwargs: + stretch: crude + min_stretch: [290, 290, 290] + max_stretch: [190, 190, 190] - name: gamma method: !!python/name:satpy.enhancements.gamma - kwargs: {gamma: 1.2} + kwargs: + gamma: [1.5, 1.2, 1.2] mw183_humidity_surface: standard_name: mw183_humidity_surface diff --git a/satpy/readers/mwr_l1b.py b/satpy/readers/mwr_l1b.py index 03a8b9f122..5eac8c2699 100644 --- a/satpy/readers/mwr_l1b.py +++ b/satpy/readers/mwr_l1b.py @@ -31,6 +31,22 @@ MWR_CHANNEL_NAMES = list(str(i) for i in range(1, 20)) +NAVIGATION_DATASET_NAMES = ["satellite_zenith_horn1", + "satellite_zenith_horn2", + "satellite_zenith_horn3", + "satellite_zenith_horn4", + "solar_azimuth_horn1", + "solar_azimuth_horn2", + "solar_azimuth_horn3", + "solar_azimuth_horn4", + "solar_zenith_horn1", + "solar_zenith_horn2", + "solar_zenith_horn3", + "solar_zenith_horn4", + "satellite_azimuth_horn1", + "satellite_azimuth_horn2", + "satellite_azimuth_horn3", + "satellite_azimuth_horn4"] class AWS_EPS_Sterna_BaseFileHandler(NetCDF4FileHandler): """Base class implementing the AWS/EPS-Sterna MWR Level-1b&c Filehandlers.""" @@ -127,22 +143,7 @@ def get_dataset(self, dataset_id, dataset_info): """Get the data.""" if dataset_id["name"] in MWR_CHANNEL_NAMES: data_array = self._get_channel_data(dataset_id, dataset_info) - elif dataset_id["name"] in ["satellite_zenith_horn1", - "satellite_zenith_horn2", - "satellite_zenith_horn3", - "satellite_zenith_horn4", - "solar_azimuth_horn1", - "solar_azimuth_horn2", - "solar_azimuth_horn3", - "solar_azimuth_horn4", - "solar_zenith_horn1", - "solar_zenith_horn2", - "solar_zenith_horn3", - "solar_zenith_horn4", - "satellite_azimuth_horn1", - "satellite_azimuth_horn2", - "satellite_azimuth_horn3", - "satellite_azimuth_horn4"]: + elif dataset_id["name"] in NAVIGATION_DATASET_NAMES: data_array = self._get_navigation_data(dataset_id, dataset_info) elif dataset_id["name"] in ["longitude", "latitude"]: From 1e49dd5333d98cfc1c185291ea8748e0df1df23a Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Thu, 19 Dec 2024 12:59:18 +0100 Subject: [PATCH 30/48] Update satpy/readers/mwr_l1b.py Co-authored-by: Panu Lahtinen --- satpy/readers/mwr_l1b.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/readers/mwr_l1b.py b/satpy/readers/mwr_l1b.py index 03a8b9f122..0d3a5e7568 100644 --- a/satpy/readers/mwr_l1b.py +++ b/satpy/readers/mwr_l1b.py @@ -29,7 +29,7 @@ from .netcdf_utils import NetCDF4FileHandler -MWR_CHANNEL_NAMES = list(str(i) for i in range(1, 20)) +MWR_CHANNEL_NAMES = [str(i) for i in range(1, 20)] class AWS_EPS_Sterna_BaseFileHandler(NetCDF4FileHandler): From 62c3b3637950542921fbb3f8ab30463303a01c22 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Thu, 19 Dec 2024 13:04:05 +0100 Subject: [PATCH 31/48] Remove redundant MWR specific enhancement Signed-off-by: Adam.Dybbroe --- satpy/etc/enhancements/generic.yaml | 2 +- satpy/etc/enhancements/mwr.yaml | 14 -------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/satpy/etc/enhancements/generic.yaml b/satpy/etc/enhancements/generic.yaml index ff6e500560..088a59f625 100644 --- a/satpy/etc/enhancements/generic.yaml +++ b/satpy/etc/enhancements/generic.yaml @@ -1291,7 +1291,7 @@ enhancements: operations: [] mw183_humidity: - # matches EPS-Sterna and AWS MWR + # matches EPS-Sterna and AWS MWR, and ATMS and MHS standard_name: mw183_humidity operations: - name: stretch diff --git a/satpy/etc/enhancements/mwr.yaml b/satpy/etc/enhancements/mwr.yaml index da32d6a499..26b18b2549 100644 --- a/satpy/etc/enhancements/mwr.yaml +++ b/satpy/etc/enhancements/mwr.yaml @@ -1,19 +1,5 @@ enhancements: - mw183_humidity: - standard_name: mw183_humidity - operations: - - name: stretch - method: !!python/name:satpy.enhancements.stretch - kwargs: - stretch: crude - min_stretch: [290, 290, 290] - max_stretch: [190, 190, 190] - - name: gamma - method: !!python/name:satpy.enhancements.gamma - kwargs: - gamma: [1.5, 1.2, 1.2] - mw183_humidity_surface: standard_name: mw183_humidity_surface operations: From b2f0e1410e068bec64dc58546930b1f291986d0b Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Thu, 19 Dec 2024 13:09:14 +0100 Subject: [PATCH 32/48] Update satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml Co-authored-by: Panu Lahtinen --- satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml b/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml index 9544a9ce3e..97a8e8888c 100644 --- a/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml +++ b/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml @@ -2,7 +2,7 @@ reader: name: eps_sterna_mwr_l1b_nc short_name: AWS L1B RAD NetCDF4 long_name: AWS L1B Radiance (NetCDF4) - description: Reader for the EUMETSAT EPS-Sterna Sounder level-1b files in netCDF4. + description: Reader for the EUMETSAT EPS-Sterna radiometer level-1b files in netCDF4. reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader sensors: [mwr,] status: Beta From 1ef7cc380875d83cb6989e84e0b8287f3739c6ba Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Thu, 19 Dec 2024 14:00:02 +0100 Subject: [PATCH 33/48] Fix spelling Co-authored-by: Panu Lahtinen --- satpy/etc/readers/aws1_mwr_l1b_nc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/etc/readers/aws1_mwr_l1b_nc.yaml b/satpy/etc/readers/aws1_mwr_l1b_nc.yaml index 7fc88332b2..03978afe49 100644 --- a/satpy/etc/readers/aws1_mwr_l1b_nc.yaml +++ b/satpy/etc/readers/aws1_mwr_l1b_nc.yaml @@ -2,7 +2,7 @@ reader: name: aws1_mwr_l1b_nc short_name: AWS1 MWR L1B RAD NetCDF4 long_name: AWS1 MWR L1B Radiance (NetCDF4) - description: Reader for the ESA AWS (Arctic Weather Satellite) Micorwave Radiometer (MWR) level-1b files in netCDF4. + description: Reader for the ESA AWS (Arctic Weather Satellite) Microwave Radiometer (MWR) level-1b files in netCDF4. reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader sensors: [mwr,] status: Beta From afe175b138571d8bdd07c59e3278f1d606c375a0 Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Thu, 19 Dec 2024 14:00:23 +0100 Subject: [PATCH 34/48] Update satpy/tests/reader_tests/test_aws1_mwr_l1c.py Co-authored-by: Panu Lahtinen --- satpy/tests/reader_tests/test_aws1_mwr_l1c.py | 1 - 1 file changed, 1 deletion(-) diff --git a/satpy/tests/reader_tests/test_aws1_mwr_l1c.py b/satpy/tests/reader_tests/test_aws1_mwr_l1c.py index fc4118bc34..14ed54d38e 100644 --- a/satpy/tests/reader_tests/test_aws1_mwr_l1c.py +++ b/satpy/tests/reader_tests/test_aws1_mwr_l1c.py @@ -19,7 +19,6 @@ """Tests for ESA Arctic Weather Satellite (AWS) level-1c file reading.""" - import numpy as np import pytest From 3a71be5f2c1393116ba652bec8bcd817eca472af Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Thu, 19 Dec 2024 14:01:04 +0100 Subject: [PATCH 35/48] Update satpy/readers/mwr_l1c.py Co-authored-by: Panu Lahtinen --- satpy/readers/mwr_l1c.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/readers/mwr_l1c.py b/satpy/readers/mwr_l1c.py index 1b6f7269f4..ae93c72414 100644 --- a/satpy/readers/mwr_l1c.py +++ b/satpy/readers/mwr_l1c.py @@ -46,8 +46,8 @@ def get_dataset(self, dataset_id, dataset_info): if dataset_id["name"] in MWR_CHANNEL_NAMES: data_array = self._get_channel_data(dataset_id, dataset_info) elif (dataset_id["name"] in ["longitude", "latitude", - "solar_azimuth", "solar_zenith", - "satellite_zenith", "satellite_azimuth"]): + "solar_azimuth_angle", "solar_zenith_angle", + "satellite_zenith_angle", "satellite_azimuth_angle"]): data_array = self._get_navigation_data(dataset_id, dataset_info) else: raise NotImplementedError(f"Dataset {dataset_id['name']} not available or not supported yet!") From 49fb56453a09b41688e6bdc82e91cd450f73556f Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Thu, 19 Dec 2024 14:01:33 +0100 Subject: [PATCH 36/48] Update satpy/tests/reader_tests/test_aws1_mwr_l1c.py Co-authored-by: Panu Lahtinen --- satpy/tests/reader_tests/test_aws1_mwr_l1c.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/satpy/tests/reader_tests/test_aws1_mwr_l1c.py b/satpy/tests/reader_tests/test_aws1_mwr_l1c.py index 14ed54d38e..41d097aca4 100644 --- a/satpy/tests/reader_tests/test_aws1_mwr_l1c.py +++ b/satpy/tests/reader_tests/test_aws1_mwr_l1c.py @@ -82,10 +82,10 @@ def test_get_navigation_data(aws_mwr_l1c_handler, id_name, file_key, fake_array) @pytest.mark.parametrize(("id_name", "file_key", "fake_array"), - [("solar_azimuth", "data/navigation/aws_solar_azimuth_angle", fake_sun_azi_data), - ("solar_zenith", "data/navigation/aws_solar_zenith_angle", fake_sun_zen_data), - ("satellite_azimuth", "data/navigation/aws_satellite_azimuth_angle", fake_sat_azi_data), - ("satellite_zenith", "data/navigation/aws_satellite_zenith_angle", fake_sat_zen_data)]) + [("solar_azimuth_angle", "data/navigation/aws_solar_azimuth_angle", fake_sun_azi_data), + ("solar_zenith_angle", "data/navigation/aws_solar_zenith_angle", fake_sun_zen_data), + ("satellite_azimuth_angle", "data/navigation/aws_satellite_azimuth_angle", fake_sat_azi_data), + ("satellite_zenith_angle", "data/navigation/aws_satellite_zenith_angle", fake_sat_zen_data)]) def test_get_viewing_geometry_data(aws_mwr_l1c_handler, id_name, file_key, fake_array): """Test retrieving the angles_data.""" dset_id = dict(name=id_name) From ba126ccac97066be03848d5a31e76032bda72429 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Thu, 19 Dec 2024 15:21:41 +0100 Subject: [PATCH 37/48] Fix sensor name and adjust yaml files accordingly + fix a few issues raised in the PR review Signed-off-by: Adam.Dybbroe --- satpy/etc/composites/mwr.yaml | 6 ++-- satpy/etc/enhancements/mwr.yaml | 32 ++---------------- satpy/etc/readers/aws1_mwr_l1c_nc.yaml | 3 -- satpy/readers/mwr_l1b.py | 4 ++- satpy/readers/mwr_l1c.py | 4 ++- satpy/tests/reader_tests/conftest.py | 33 ++++++++++--------- satpy/tests/reader_tests/test_aws1_mwr_l1b.py | 10 +++--- satpy/tests/reader_tests/test_aws1_mwr_l1c.py | 2 +- 8 files changed, 34 insertions(+), 60 deletions(-) diff --git a/satpy/etc/composites/mwr.yaml b/satpy/etc/composites/mwr.yaml index 907185838a..d959faa632 100644 --- a/satpy/etc/composites/mwr.yaml +++ b/satpy/etc/composites/mwr.yaml @@ -15,7 +15,7 @@ composites: - name: '9' - name: '10' - name: '12' - standard_name: mw183_humidity_surface + standard_name: mw_humidity_surface mw325_humidity_surface: compositor: !!python/name:satpy.composites.RGBCompositor @@ -23,7 +23,7 @@ composites: - name: '9' - name: '10' - name: '19' - standard_name: mw325_humidity_surface + standard_name: mw_humidity_surface mw325_humidity: compositor: !!python/name:satpy.composites.RGBCompositor @@ -31,7 +31,7 @@ composites: - name: '16' - name: '18' - name: '19' - standard_name: mw325_humidity + standard_name: mw_humidity_surface ch1_tbs_colors: compositor: !!python/name:satpy.composites.SingleBandCompositor diff --git a/satpy/etc/enhancements/mwr.yaml b/satpy/etc/enhancements/mwr.yaml index 26b18b2549..4a9cff6354 100644 --- a/satpy/etc/enhancements/mwr.yaml +++ b/satpy/etc/enhancements/mwr.yaml @@ -1,35 +1,7 @@ enhancements: - mw183_humidity_surface: - standard_name: mw183_humidity_surface - operations: - - name: inverse - method: !!python/name:satpy.enhancements.invert - args: - - [true, true, true] - - name: stretch - method: !!python/name:satpy.enhancements.stretch - kwargs: {stretch: linear} - - name: gamma - method: !!python/name:satpy.enhancements.gamma - kwargs: {gamma: 1.2} - - mw325_humidity: - standard_name: mw325_humidity - operations: - - name: inverse - method: !!python/name:satpy.enhancements.invert - args: - - [true, true, true] - - name: stretch - method: !!python/name:satpy.enhancements.stretch - kwargs: {stretch: linear} - - name: gamma - method: !!python/name:satpy.enhancements.gamma - kwargs: {gamma: 1.2} - - mw325_humidity_surface: - standard_name: mw325_humidity_surface + mw_humidity_surface: + standard_name: mw_humidity_surface operations: - name: inverse method: !!python/name:satpy.enhancements.invert diff --git a/satpy/etc/readers/aws1_mwr_l1c_nc.yaml b/satpy/etc/readers/aws1_mwr_l1c_nc.yaml index ad2acd4dc3..4a6215a5a1 100644 --- a/satpy/etc/readers/aws1_mwr_l1c_nc.yaml +++ b/satpy/etc/readers/aws1_mwr_l1c_nc.yaml @@ -366,9 +366,6 @@ datasets: file_types: aws_l1c_nc: - # W_XX-OHB-Unknown,SAT,1-AWS-1B-RAD_C_OHB_20230707124607_G_D_20220621090100_20220621090618_T_B____.nc - # W_XX-OHB-Stockholm,SAT,AWS1-MWR-1B-RAD_C_OHB_20230823161321_G_D_20240115111111_20240115125434_T_B____.nc - # W_XX-OHB-Stockholm,SAT,AWS1-MWR-1B-RAD_C_OHB_20230816120142_G_D_20240115111111_20240115125434_T_B____radsim.nc file_reader: !!python/name:satpy.readers.mwr_l1c.AWS_MWR_L1CFile file_patterns: [ 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1C-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc', diff --git a/satpy/readers/mwr_l1b.py b/satpy/readers/mwr_l1b.py index 362641306b..7f77f6dbf9 100644 --- a/satpy/readers/mwr_l1b.py +++ b/satpy/readers/mwr_l1b.py @@ -71,7 +71,9 @@ def end_time(self): @property def sensor(self): """Get the sensor name.""" - return self["/attr/instrument"] + # This should have been self["/attr/instrument"] + # But the sensor name is currently incorrect in the ESA level-1b files + return "mwr" @property def platform_name(self): diff --git a/satpy/readers/mwr_l1c.py b/satpy/readers/mwr_l1c.py index 1b6f7269f4..dba26d45a5 100644 --- a/satpy/readers/mwr_l1c.py +++ b/satpy/readers/mwr_l1c.py @@ -39,7 +39,9 @@ def __init__(self, filename, filename_info, filetype_info, auto_maskandscale=Tru @property def sensor(self): """Get the sensor name.""" - return "MWR" + # This should have been self["/attr/instrument"] + # But the sensor name is currently incorrect in the ESA level-1b files + return "mwr" def get_dataset(self, dataset_id, dataset_info): """Get the data.""" diff --git a/satpy/tests/reader_tests/conftest.py b/satpy/tests/reader_tests/conftest.py index aeec133f30..83a6291288 100644 --- a/satpy/tests/reader_tests/conftest.py +++ b/satpy/tests/reader_tests/conftest.py @@ -19,7 +19,8 @@ """Setup and configuration for all reader tests.""" import os -from datetime import datetime, timedelta +from datetime import datetime as dt +from datetime import timedelta from random import randrange import numpy as np @@ -99,11 +100,11 @@ def eps_sterna_mwr_file(tmp_path_factory, fake_mwr_data_array): geo_size = 10 * 145 * 4 ds = DataTree() - start_time = datetime(2024, 9, 1, 12, 0) + start_time = dt(2024, 9, 1, 12, 0) ds.attrs["sensing_start_time_utc"] = start_time.strftime(DATETIME_FORMAT) - end_time = datetime(2024, 9, 1, 12, 15) + end_time = dt(2024, 9, 1, 12, 15) ds.attrs["sensing_end_time_utc"] = end_time.strftime(DATETIME_FORMAT) - processing_time = random_date(datetime(2024, 9, 1, 13), datetime(2030, 6, 1)) + processing_time = random_date(dt(2024, 9, 1, 13), dt(2030, 6, 1)) instrument = "MWR" ds.attrs["instrument"] = instrument @@ -140,7 +141,7 @@ def eps_sterna_mwr_file(tmp_path_factory, fake_mwr_data_array): return filename -@pytest.fixture +@pytest.fixture(scope="module") def eps_sterna_mwr_handler(eps_sterna_mwr_file): """Create an EPS-Sterna MWR filehandler.""" filename_info = parse(eumetsat_file_pattern, os.path.basename(eps_sterna_mwr_file)) @@ -149,18 +150,18 @@ def eps_sterna_mwr_handler(eps_sterna_mwr_file): return AWS_EPS_Sterna_MWR_L1BFile(eps_sterna_mwr_file, filename_info, filetype_info) -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def aws_mwr_file(tmp_path_factory, fake_mwr_data_array): """Create an AWS MWR l1b file.""" geo_dims = ["n_scans", "n_fovs", "n_geo_groups"] - geo_size = 10*145*4 + geo_size = 10 * 145 * 4 ds = DataTree() - start_time = datetime(2024, 9, 1, 12, 0) + start_time = dt(2024, 9, 1, 12, 0) ds.attrs["sensing_start_time_utc"] = start_time.strftime(DATETIME_FORMAT) - end_time = datetime(2024, 9, 1, 12, 15) + end_time = dt(2024, 9, 1, 12, 15) ds.attrs["sensing_end_time_utc"] = end_time.strftime(DATETIME_FORMAT) - processing_time = random_date(datetime(2024, 6, 1), datetime(2030, 6, 1)) + processing_time = random_date(dt(2024, 6, 1), dt(2030, 6, 1)) instrument = "MWR" ds.attrs["instrument"] = instrument @@ -196,7 +197,7 @@ def aws_mwr_file(tmp_path_factory, fake_mwr_data_array): return filename -@pytest.fixture +@pytest.fixture(scope="module") def aws_mwr_handler(aws_mwr_file): """Create an AWS MWR filehandler.""" filename_info = parse(esa_file_pattern, os.path.basename(aws_mwr_file)) @@ -205,18 +206,18 @@ def aws_mwr_handler(aws_mwr_file): return AWS_EPS_Sterna_MWR_L1BFile(aws_mwr_file, filename_info, filetype_info) -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def aws_mwr_l1c_file(tmp_path_factory, fake_mwr_data_array): """Create an AWS MWR l1c file.""" geo_dims = ["n_scans", "n_fovs"] geo_size = 10*145 ds = DataTree() - start_time = datetime(2024, 9, 1, 12, 0) + start_time = dt(2024, 9, 1, 12, 0) ds.attrs["sensing_start_time_utc"] = start_time.strftime(DATETIME_FORMAT) - end_time = datetime(2024, 9, 1, 12, 15) + end_time = dt(2024, 9, 1, 12, 15) ds.attrs["sensing_end_time_utc"] = end_time.strftime(DATETIME_FORMAT) - processing_time = random_date(datetime(2024, 6, 1), datetime(2030, 6, 1)) + processing_time = random_date(dt(2024, 6, 1), dt(2030, 6, 1)) ds.attrs["instrument"] = "MWR" ds.attrs["orbit_start"] = 9991 @@ -248,7 +249,7 @@ def aws_mwr_l1c_file(tmp_path_factory, fake_mwr_data_array): return filename -@pytest.fixture +@pytest.fixture(scope="module") def aws_mwr_l1c_handler(aws_mwr_l1c_file): """Create an AWS MWR level-1c filehandler.""" filename_info = parse(esa_l1c_file_pattern, os.path.basename(aws_mwr_l1c_file)) diff --git a/satpy/tests/reader_tests/test_aws1_mwr_l1b.py b/satpy/tests/reader_tests/test_aws1_mwr_l1b.py index 9a789bd08f..a88cd8e062 100644 --- a/satpy/tests/reader_tests/test_aws1_mwr_l1b.py +++ b/satpy/tests/reader_tests/test_aws1_mwr_l1b.py @@ -1,6 +1,6 @@ """Tests for aws l1b filehandlers.""" -from datetime import datetime +from datetime import datetime as dt from enum import Enum import numpy as np @@ -24,8 +24,8 @@ def test_start_end_time(aws_mwr_handler): """Test that start and end times are read correctly.""" - assert aws_mwr_handler.start_time == datetime(2024, 9, 1, 12, 0) - assert aws_mwr_handler.end_time == datetime(2024, 9, 1, 12, 15) + assert aws_mwr_handler.start_time == dt(2024, 9, 1, 12, 0) + assert aws_mwr_handler.end_time == dt(2024, 9, 1, 12, 15) def test_orbit_number_start_end(aws_mwr_handler): @@ -36,7 +36,7 @@ def test_orbit_number_start_end(aws_mwr_handler): def test_metadata(aws_mwr_handler): """Test that the metadata is read correctly.""" - assert aws_mwr_handler.sensor == "MWR" + assert aws_mwr_handler.sensor == "mwr" assert aws_mwr_handler.platform_name == platform_name @@ -60,7 +60,7 @@ def test_get_channel_data(aws_mwr_handler, fake_mwr_data_array): assert res.attrs["orbital_parameters"]["sub_satellite_longitude_end"] == 296.79 assert res.dims == ("y", "x") assert "n_channels" not in res.coords - assert res.attrs["sensor"] == "MWR" + assert res.attrs["sensor"] == "mwr" assert res.attrs["platform_name"] == "AWS1" diff --git a/satpy/tests/reader_tests/test_aws1_mwr_l1c.py b/satpy/tests/reader_tests/test_aws1_mwr_l1c.py index fc4118bc34..86ee5600d7 100644 --- a/satpy/tests/reader_tests/test_aws1_mwr_l1c.py +++ b/satpy/tests/reader_tests/test_aws1_mwr_l1c.py @@ -57,7 +57,7 @@ def test_get_channel_data(aws_mwr_l1c_handler, fake_mwr_data_array): assert "y" in res.dims assert res.dims == ("y", "x") assert "n_channels" not in res.coords - assert res.attrs["sensor"] == "MWR" + assert res.attrs["sensor"] == "mwr" assert res.attrs["platform_name"] == "AWS1" From 0eb261a505fb7931010a95fc0e0647e2b6d30a24 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Thu, 19 Dec 2024 15:30:45 +0100 Subject: [PATCH 38/48] Fix file name pattern Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/aws1_mwr_l1b_nc.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/satpy/etc/readers/aws1_mwr_l1b_nc.yaml b/satpy/etc/readers/aws1_mwr_l1b_nc.yaml index 03978afe49..a232621ba9 100644 --- a/satpy/etc/readers/aws1_mwr_l1b_nc.yaml +++ b/satpy/etc/readers/aws1_mwr_l1b_nc.yaml @@ -527,9 +527,7 @@ file_types: aws_l1b_nc: # W_XX-OHB-Unknown,SAT,1-AWS-1B-RAD_C_OHB_20230707124607_G_D_20220621090100_20220621090618_T_B____.nc # W_XX-OHB-Stockholm,SAT,AWS1-MWR-1B-RAD_C_OHB_20230823161321_G_D_20240115111111_20240115125434_T_B____.nc - # W_XX-OHB-Stockholm,SAT,AWS1-MWR-1B-RAD_C_OHB_20230816120142_G_D_20240115111111_20240115125434_T_B____radsim.nc file_reader: !!python/name:satpy.readers.mwr_l1b.AWS_EPS_Sterna_MWR_L1BFile file_patterns: [ - 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{disposition_mode:1s}_{processing_mode:1s}____.nc', - 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB__{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc', - 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____radsim.nc'] + 'W_{country:2s}-{organisation:s}-{location:s},SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{disposition_mode:1s}_{processing_mode:1s}____.nc', + 'W_{country:2s}-{organisation:s}-{location:s},SAT,{platform_name}-MWR-1B-RAD_C_OHB__{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{disposition_mode:1s}_{processing_mode:1s}____.nc'] From 05d952ee9cb037f7e1e33fd77ad1b657e805f9fe Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Thu, 19 Dec 2024 21:01:07 +0100 Subject: [PATCH 39/48] Refactor the AWS/EPS-STerna tests Signed-off-by: Adam.Dybbroe --- satpy/tests/reader_tests/conftest.py | 181 +++++++++++++-------------- 1 file changed, 90 insertions(+), 91 deletions(-) diff --git a/satpy/tests/reader_tests/conftest.py b/satpy/tests/reader_tests/conftest.py index 83a6291288..c78cf5ae1b 100644 --- a/satpy/tests/reader_tests/conftest.py +++ b/satpy/tests/reader_tests/conftest.py @@ -36,11 +36,7 @@ platform_name = "AWS1" # W_XX-EUMETSAT-Darmstadt,SAT,AWS1-MWR-1B-RAD_C_EUMT_20241121085911_G_D_20241109234502_20241110004559_T_N____.nc -eumetsat_file_pattern = "W_XX-EUMETSAT-Darmstadt,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc" # noqa - -esa_file_pattern = "W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc" # noqa - -esa_l1c_file_pattern = "W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1C-RAD_C_OHB__{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc" # noqa +file_pattern = "W_{country:2s}-{organisation:s}-{location:s},SAT,{platform_name}-MWR-{processing_level}-RAD_C_{originator:4s}_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc" # noqa rng = np.random.default_rng() @@ -93,10 +89,20 @@ def make_fake_mwr_l1c_lonlats(geo_size, geo_dims): return (fake_lon_data, fake_lat_data) -@pytest.fixture(scope="module") -def eps_sterna_mwr_file(tmp_path_factory, fake_mwr_data_array): - """Create an EPS-Sterna MWR l1b file.""" - geo_dims = ["n_scans", "n_fovs", "n_feedhorns"] +def aws_eps_sterna_mwr_l1bfile(fake_mwr_data_array, eps_sterna=True): + """Create an AWS and EPS-Sterna MWR l1b file.""" + if eps_sterna: + n_feedhorns="n_feedhorns" + prefix = "" + longitude_attr = "longitude" + latitude_attr = "latitude" + else: + n_feedhorns="n_geo_groups" + prefix = "aws_" + longitude_attr = "aws_lon" + latitude_attr = "aws_lat" + + geo_dims = ["n_scans", "n_fovs", n_feedhorns] geo_size = 10 * 145 * 4 ds = DataTree() @@ -104,113 +110,84 @@ def eps_sterna_mwr_file(tmp_path_factory, fake_mwr_data_array): ds.attrs["sensing_start_time_utc"] = start_time.strftime(DATETIME_FORMAT) end_time = dt(2024, 9, 1, 12, 15) ds.attrs["sensing_end_time_utc"] = end_time.strftime(DATETIME_FORMAT) - processing_time = random_date(dt(2024, 9, 1, 13), dt(2030, 6, 1)) instrument = "MWR" ds.attrs["instrument"] = instrument ds.attrs["orbit_start"] = 9991 ds.attrs["orbit_end"] = 9992 - ds["data/calibration/toa_brightness_temperature"] = fake_mwr_data_array - ds["data/calibration/toa_brightness_temperature"].attrs["scale_factor"] = 0.001 - ds["data/calibration/toa_brightness_temperature"].attrs["add_offset"] = 0.0 - ds["data/calibration/toa_brightness_temperature"].attrs["missing_value"] = -2147483648 - ds["data/calibration/toa_brightness_temperature"].attrs["valid_min"] = 0 - ds["data/calibration/toa_brightness_temperature"].attrs["valid_max"] = 700000 + dset_name = f"data/calibration/{prefix}toa_brightness_temperature" + ds[dset_name] = fake_mwr_data_array + ds[dset_name].attrs["scale_factor"] = 0.001 + ds[dset_name].attrs["add_offset"] = 0.0 + ds[dset_name].attrs["missing_value"] = -2147483648 + ds[dset_name].attrs["valid_min"] = 0 + ds[dset_name].attrs["valid_max"] = 700000 fake_lon_data, fake_lat_data = make_fake_mwr_lonlats(geo_size, geo_dims) - ds["data/navigation/longitude"] = fake_lon_data - ds["data/navigation/longitude"].attrs["scale_factor"] = 1e-4 - ds["data/navigation/longitude"].attrs["add_offset"] = 0.0 - ds["data/navigation/latitude"] = fake_lat_data - ds["data/navigation/solar_azimuth_angle"] = make_fake_angles(geo_size, geo_dims) - ds["data/navigation/solar_zenith_angle"] = make_fake_angles(geo_size, geo_dims) - ds["data/navigation/satellite_azimuth_angle"] = make_fake_angles(geo_size, geo_dims) - ds["data/navigation/satellite_zenith_angle"] = make_fake_angles(geo_size, geo_dims) + ds[f"data/navigation/{longitude_attr}"] = fake_lon_data + ds[f"data/navigation/{longitude_attr}"].attrs["scale_factor"] = 1e-4 + ds[f"data/navigation/{longitude_attr}"].attrs["add_offset"] = 0.0 + ds[f"data/navigation/{latitude_attr}"] = fake_lat_data + ds[f"data/navigation/{prefix}solar_azimuth_angle"] = make_fake_angles(geo_size, geo_dims) + ds[f"data/navigation/{prefix}solar_zenith_angle"] = make_fake_angles(geo_size, geo_dims) + ds[f"data/navigation/{prefix}satellite_azimuth_angle"] = make_fake_angles(geo_size, geo_dims) + ds[f"data/navigation/{prefix}satellite_zenith_angle"] = make_fake_angles(geo_size, geo_dims) ds["status/satellite/subsat_latitude_end"] = np.array(22.39) ds["status/satellite/subsat_longitude_start"] = np.array(304.79) ds["status/satellite/subsat_latitude_start"] = np.array(55.41) ds["status/satellite/subsat_longitude_end"] = np.array(296.79) - tmp_dir = tmp_path_factory.mktemp("eps_sterna_mwr_l1b_tests") - filename = tmp_dir / compose(eumetsat_file_pattern, dict(start_time=start_time, end_time=end_time, - processing_time=processing_time, - platform_name=platform_name)) - - ds.to_netcdf(filename) - return filename - + return ds @pytest.fixture(scope="module") -def eps_sterna_mwr_handler(eps_sterna_mwr_file): - """Create an EPS-Sterna MWR filehandler.""" - filename_info = parse(eumetsat_file_pattern, os.path.basename(eps_sterna_mwr_file)) - filetype_info = dict() - filetype_info["file_type"] = "eps_sterna_mwr_l1b" - return AWS_EPS_Sterna_MWR_L1BFile(eps_sterna_mwr_file, filename_info, filetype_info) +def eps_sterna_mwr_file(tmp_path_factory, fake_mwr_data_array): + """Create an EPS-Sterna MWR l1b file.""" + ds = aws_eps_sterna_mwr_l1bfile(fake_mwr_data_array, eps_sterna=True) + tmp_dir = tmp_path_factory.mktemp("eps_sterna_mwr_l1b_tests") + start_time = dt.fromisoformat(ds.attrs["sensing_start_time_utc"]) + end_time = dt.fromisoformat(ds.attrs["sensing_end_time_utc"]) + platform_name = "AWS1" + processing_time = random_date(dt(2024, 9, 1, 13), dt(2030, 6, 1)) + filename = tmp_dir / compose(file_pattern, dict(country="XX", + organisation="EUMETSAT", + location="Darmstadt", + processing_level="1B", + originator="EUMT", + start_time=start_time, end_time=end_time, + processing_time=processing_time, + platform_name=platform_name)) + ds.to_netcdf(filename) + return filename @pytest.fixture(scope="module") def aws_mwr_file(tmp_path_factory, fake_mwr_data_array): """Create an AWS MWR l1b file.""" - geo_dims = ["n_scans", "n_fovs", "n_geo_groups"] - geo_size = 10 * 145 * 4 - - ds = DataTree() - start_time = dt(2024, 9, 1, 12, 0) - ds.attrs["sensing_start_time_utc"] = start_time.strftime(DATETIME_FORMAT) - end_time = dt(2024, 9, 1, 12, 15) - ds.attrs["sensing_end_time_utc"] = end_time.strftime(DATETIME_FORMAT) - processing_time = random_date(dt(2024, 6, 1), dt(2030, 6, 1)) - - instrument = "MWR" - ds.attrs["instrument"] = instrument - ds.attrs["orbit_start"] = 9991 - ds.attrs["orbit_end"] = 9992 - ds["data/calibration/aws_toa_brightness_temperature"] = fake_mwr_data_array - ds["data/calibration/aws_toa_brightness_temperature"].attrs["scale_factor"] = 0.001 - ds["data/calibration/aws_toa_brightness_temperature"].attrs["add_offset"] = 0.0 - ds["data/calibration/aws_toa_brightness_temperature"].attrs["missing_value"] = -2147483648 - ds["data/calibration/aws_toa_brightness_temperature"].attrs["valid_min"] = 0 - ds["data/calibration/aws_toa_brightness_temperature"].attrs["valid_max"] = 700000 - - fake_lon_data, fake_lat_data = make_fake_mwr_lonlats(geo_size, geo_dims) - - ds["data/navigation/aws_lon"] = fake_lon_data - ds["data/navigation/aws_lon"].attrs["scale_factor"] = 1e-4 - ds["data/navigation/aws_lon"].attrs["add_offset"] = 0.0 - ds["data/navigation/aws_lat"] = fake_lat_data - ds["data/navigation/aws_solar_azimuth_angle"] = make_fake_angles(geo_size, geo_dims) - ds["data/navigation/aws_solar_zenith_angle"] = make_fake_angles(geo_size, geo_dims) - ds["data/navigation/aws_satellite_azimuth_angle"] = make_fake_angles(geo_size, geo_dims) - ds["data/navigation/aws_satellite_zenith_angle"] = make_fake_angles(geo_size, geo_dims) - ds["status/satellite/subsat_latitude_end"] = np.array(22.39) - ds["status/satellite/subsat_longitude_start"] = np.array(304.79) - ds["status/satellite/subsat_latitude_start"] = np.array(55.41) - ds["status/satellite/subsat_longitude_end"] = np.array(296.79) + ds = aws_eps_sterna_mwr_l1bfile(fake_mwr_data_array, eps_sterna=False) tmp_dir = tmp_path_factory.mktemp("aws_l1b_tests") - filename = tmp_dir / compose(esa_file_pattern, dict(start_time=start_time, end_time=end_time, - processing_time=processing_time, platform_name=platform_name)) - + start_time = dt.fromisoformat(ds.attrs["sensing_start_time_utc"]) + end_time = dt.fromisoformat(ds.attrs["sensing_end_time_utc"]) + platform_name = "AWS1" + processing_time = random_date(dt(2024, 9, 1, 13), dt(2030, 6, 1)) + filename = tmp_dir / compose(file_pattern, dict(country="SE", + organisation="SMHI", + location="Norrkoping", + processing_level="1B", + originator="SMHI", + start_time=start_time, end_time=end_time, + processing_time=processing_time, + platform_name=platform_name)) ds.to_netcdf(filename) return filename -@pytest.fixture(scope="module") -def aws_mwr_handler(aws_mwr_file): - """Create an AWS MWR filehandler.""" - filename_info = parse(esa_file_pattern, os.path.basename(aws_mwr_file)) - filetype_info = dict() - filetype_info["file_type"] = "aws1_mwr_l1b" - return AWS_EPS_Sterna_MWR_L1BFile(aws_mwr_file, filename_info, filetype_info) - - @pytest.fixture(scope="module") def aws_mwr_l1c_file(tmp_path_factory, fake_mwr_data_array): """Create an AWS MWR l1c file.""" geo_dims = ["n_scans", "n_fovs"] - geo_size = 10*145 + geo_size = 10 * 145 ds = DataTree() start_time = dt(2024, 9, 1, 12, 0) @@ -241,18 +218,40 @@ def aws_mwr_l1c_file(tmp_path_factory, fake_mwr_data_array): ds["data/navigation/aws_satellite_zenith_angle"] = make_fake_angles(geo_size, geo_dims, shape=(10, 145)) tmp_dir = tmp_path_factory.mktemp("aws_l1c_tests") - filename = tmp_dir / compose(esa_l1c_file_pattern, dict(start_time=start_time, end_time=end_time, - processing_time=processing_time, - platform_name=platform_name)) - + filename = tmp_dir / compose(file_pattern, dict(country="SE", + organisation="SMHI", + location="Norrkoping", + processing_level="1C", + originator="SMHI", + start_time=start_time, end_time=end_time, + processing_time=processing_time, + platform_name=platform_name)) ds.to_netcdf(filename) return filename +@pytest.fixture(scope="module") +def eps_sterna_mwr_handler(eps_sterna_mwr_file): + """Create an EPS-Sterna MWR filehandler.""" + filename_info = parse(file_pattern, os.path.basename(eps_sterna_mwr_file)) + filetype_info = dict() + filetype_info["file_type"] = "eps_sterna_mwr_l1b" + return AWS_EPS_Sterna_MWR_L1BFile(eps_sterna_mwr_file, filename_info, filetype_info) + + +@pytest.fixture(scope="module") +def aws_mwr_handler(aws_mwr_file): + """Create an AWS MWR filehandler.""" + filename_info = parse(file_pattern, os.path.basename(aws_mwr_file)) + filetype_info = dict() + filetype_info["file_type"] = "aws1_mwr_l1b" + return AWS_EPS_Sterna_MWR_L1BFile(aws_mwr_file, filename_info, filetype_info) + + @pytest.fixture(scope="module") def aws_mwr_l1c_handler(aws_mwr_l1c_file): """Create an AWS MWR level-1c filehandler.""" - filename_info = parse(esa_l1c_file_pattern, os.path.basename(aws_mwr_l1c_file)) + filename_info = parse(file_pattern, os.path.basename(aws_mwr_l1c_file)) filetype_info = dict() filetype_info["file_type"] = "aws1_mwr_l1c" return AWS_MWR_L1CFile(aws_mwr_l1c_file, filename_info, filetype_info) From 152b45ec947a3a5fbc250f34ba2a340e55d6469e Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 20 Dec 2024 12:05:07 +0100 Subject: [PATCH 40/48] Fixed datetime import syntax - use custom practice Signed-off-by: Adam.Dybbroe --- satpy/tests/reader_tests/conftest.py | 24 +++++++++---------- satpy/tests/reader_tests/test_aws1_mwr_l1b.py | 6 ++--- satpy/tests/reader_tests/test_aws1_mwr_l1c.py | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/satpy/tests/reader_tests/conftest.py b/satpy/tests/reader_tests/conftest.py index c78cf5ae1b..241f00f28f 100644 --- a/satpy/tests/reader_tests/conftest.py +++ b/satpy/tests/reader_tests/conftest.py @@ -18,8 +18,8 @@ """Setup and configuration for all reader tests.""" +import datetime as dt import os -from datetime import datetime as dt from datetime import timedelta from random import randrange @@ -106,9 +106,9 @@ def aws_eps_sterna_mwr_l1bfile(fake_mwr_data_array, eps_sterna=True): geo_size = 10 * 145 * 4 ds = DataTree() - start_time = dt(2024, 9, 1, 12, 0) + start_time = dt.datetime(2024, 9, 1, 12, 0) ds.attrs["sensing_start_time_utc"] = start_time.strftime(DATETIME_FORMAT) - end_time = dt(2024, 9, 1, 12, 15) + end_time = dt.datetime(2024, 9, 1, 12, 15) ds.attrs["sensing_end_time_utc"] = end_time.strftime(DATETIME_FORMAT) instrument = "MWR" @@ -146,10 +146,10 @@ def eps_sterna_mwr_file(tmp_path_factory, fake_mwr_data_array): ds = aws_eps_sterna_mwr_l1bfile(fake_mwr_data_array, eps_sterna=True) tmp_dir = tmp_path_factory.mktemp("eps_sterna_mwr_l1b_tests") - start_time = dt.fromisoformat(ds.attrs["sensing_start_time_utc"]) - end_time = dt.fromisoformat(ds.attrs["sensing_end_time_utc"]) + start_time = dt.datetime.fromisoformat(ds.attrs["sensing_start_time_utc"]) + end_time = dt.datetime.fromisoformat(ds.attrs["sensing_end_time_utc"]) platform_name = "AWS1" - processing_time = random_date(dt(2024, 9, 1, 13), dt(2030, 6, 1)) + processing_time = random_date(dt.datetime(2024, 9, 1, 13), dt.datetime(2030, 6, 1)) filename = tmp_dir / compose(file_pattern, dict(country="XX", organisation="EUMETSAT", location="Darmstadt", @@ -167,10 +167,10 @@ def aws_mwr_file(tmp_path_factory, fake_mwr_data_array): ds = aws_eps_sterna_mwr_l1bfile(fake_mwr_data_array, eps_sterna=False) tmp_dir = tmp_path_factory.mktemp("aws_l1b_tests") - start_time = dt.fromisoformat(ds.attrs["sensing_start_time_utc"]) - end_time = dt.fromisoformat(ds.attrs["sensing_end_time_utc"]) + start_time = dt.datetime.fromisoformat(ds.attrs["sensing_start_time_utc"]) + end_time = dt.datetime.fromisoformat(ds.attrs["sensing_end_time_utc"]) platform_name = "AWS1" - processing_time = random_date(dt(2024, 9, 1, 13), dt(2030, 6, 1)) + processing_time = random_date(dt.datetime(2024, 9, 1, 13), dt.datetime(2030, 6, 1)) filename = tmp_dir / compose(file_pattern, dict(country="SE", organisation="SMHI", location="Norrkoping", @@ -190,11 +190,11 @@ def aws_mwr_l1c_file(tmp_path_factory, fake_mwr_data_array): geo_size = 10 * 145 ds = DataTree() - start_time = dt(2024, 9, 1, 12, 0) + start_time = dt.datetime(2024, 9, 1, 12, 0) ds.attrs["sensing_start_time_utc"] = start_time.strftime(DATETIME_FORMAT) - end_time = dt(2024, 9, 1, 12, 15) + end_time = dt.datetime(2024, 9, 1, 12, 15) ds.attrs["sensing_end_time_utc"] = end_time.strftime(DATETIME_FORMAT) - processing_time = random_date(dt(2024, 6, 1), dt(2030, 6, 1)) + processing_time = random_date(dt.datetime(2024, 6, 1), dt.datetime(2030, 6, 1)) ds.attrs["instrument"] = "MWR" ds.attrs["orbit_start"] = 9991 diff --git a/satpy/tests/reader_tests/test_aws1_mwr_l1b.py b/satpy/tests/reader_tests/test_aws1_mwr_l1b.py index a88cd8e062..cfecfd4a66 100644 --- a/satpy/tests/reader_tests/test_aws1_mwr_l1b.py +++ b/satpy/tests/reader_tests/test_aws1_mwr_l1b.py @@ -1,6 +1,6 @@ """Tests for aws l1b filehandlers.""" -from datetime import datetime as dt +import datetime as dt from enum import Enum import numpy as np @@ -24,8 +24,8 @@ def test_start_end_time(aws_mwr_handler): """Test that start and end times are read correctly.""" - assert aws_mwr_handler.start_time == dt(2024, 9, 1, 12, 0) - assert aws_mwr_handler.end_time == dt(2024, 9, 1, 12, 15) + assert aws_mwr_handler.start_time == dt.datetime(2024, 9, 1, 12, 0) + assert aws_mwr_handler.end_time == dt.datetime(2024, 9, 1, 12, 15) def test_orbit_number_start_end(aws_mwr_handler): diff --git a/satpy/tests/reader_tests/test_aws1_mwr_l1c.py b/satpy/tests/reader_tests/test_aws1_mwr_l1c.py index 5ef44a9636..9fe034a76d 100644 --- a/satpy/tests/reader_tests/test_aws1_mwr_l1c.py +++ b/satpy/tests/reader_tests/test_aws1_mwr_l1c.py @@ -30,7 +30,7 @@ geo_dims = ["n_scans", "n_fovs"] -geo_size = 10*145 +geo_size = 10 * 145 fake_lon_data, fake_lat_data = make_fake_mwr_l1c_lonlats(geo_size, geo_dims) fake_sun_azi_data = make_fake_angles(geo_size, geo_dims, shape=(10, 145)) fake_sun_zen_data = make_fake_angles(geo_size, geo_dims, shape=(10, 145)) From 4ad6ef4ec4bcc9344255a88336ab9ed8a2f3a4db Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 20 Dec 2024 12:07:44 +0100 Subject: [PATCH 41/48] Remove redundant timedelta import Signed-off-by: Adam.Dybbroe --- satpy/tests/reader_tests/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/satpy/tests/reader_tests/conftest.py b/satpy/tests/reader_tests/conftest.py index 241f00f28f..cfee6d4523 100644 --- a/satpy/tests/reader_tests/conftest.py +++ b/satpy/tests/reader_tests/conftest.py @@ -20,7 +20,6 @@ import datetime as dt import os -from datetime import timedelta from random import randrange import numpy as np @@ -46,7 +45,7 @@ def random_date(start, end): delta = end - start int_delta = (delta.days * 24 * 60 * 60) + delta.seconds random_second = randrange(int_delta) - return start + timedelta(seconds=random_second) + return start + dt.timedelta(seconds=random_second) @pytest.fixture(scope="session") From 2b8aecb238e12d37a854c045af59e587ac059da8 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 20 Dec 2024 12:18:36 +0100 Subject: [PATCH 42/48] Clean up and improve file patterns for AWS/EPS-Sterna Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/aws1_mwr_l1b_nc.yaml | 4 ++-- satpy/etc/readers/aws1_mwr_l1c_nc.yaml | 4 ++-- satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/satpy/etc/readers/aws1_mwr_l1b_nc.yaml b/satpy/etc/readers/aws1_mwr_l1b_nc.yaml index a232621ba9..bbc4a567e7 100644 --- a/satpy/etc/readers/aws1_mwr_l1b_nc.yaml +++ b/satpy/etc/readers/aws1_mwr_l1b_nc.yaml @@ -529,5 +529,5 @@ file_types: # W_XX-OHB-Stockholm,SAT,AWS1-MWR-1B-RAD_C_OHB_20230823161321_G_D_20240115111111_20240115125434_T_B____.nc file_reader: !!python/name:satpy.readers.mwr_l1b.AWS_EPS_Sterna_MWR_L1BFile file_patterns: [ - 'W_{country:2s}-{organisation:s}-{location:s},SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{disposition_mode:1s}_{processing_mode:1s}____.nc', - 'W_{country:2s}-{organisation:s}-{location:s},SAT,{platform_name}-MWR-1B-RAD_C_OHB__{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{disposition_mode:1s}_{processing_mode:1s}____.nc'] + 'W_{country:2s}-{organisation:s}-{location:s},SAT,{platform_name}-MWR-1B-RAD_C_{originator:4s}_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{disposition_mode:1s}_{processing_mode:1s}____.nc' + ] diff --git a/satpy/etc/readers/aws1_mwr_l1c_nc.yaml b/satpy/etc/readers/aws1_mwr_l1c_nc.yaml index 4a6215a5a1..e62b88d54d 100644 --- a/satpy/etc/readers/aws1_mwr_l1c_nc.yaml +++ b/satpy/etc/readers/aws1_mwr_l1c_nc.yaml @@ -368,5 +368,5 @@ file_types: aws_l1c_nc: file_reader: !!python/name:satpy.readers.mwr_l1c.AWS_MWR_L1CFile file_patterns: [ - 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1C-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc', - 'W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1C-RAD_C_OHB__{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc',] + 'W_{country:2s}-{organisation:s}-{location:s},SAT,{platform_name}-MWR-1C-RAD_C_{originator:4s}_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{disposition_mode:1s}_{processing_mode:1s}____.nc' + ] diff --git a/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml b/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml index 97a8e8888c..efc7dc8bc9 100644 --- a/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml +++ b/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml @@ -528,5 +528,5 @@ file_types: # W_XX-EUMETSAT-Darmstadt,SAT,AWS1-MWR-1B-RAD_C_EUMT_20241121085911_G_D_20241109234502_20241110004559_T_N____.nc file_reader: !!python/name:satpy.readers.mwr_l1b.AWS_EPS_Sterna_MWR_L1BFile file_patterns: [ - 'W_XX-EUMETSAT-Darmstadt,SAT,{platform_name}-MWR-1B-RAD_C_EUMT_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{disposition_mode:1s}_{processing_mode:1s}____.nc', + 'W_{country:2s}-{organisation:s}-{location:s},SAT,{platform_name}-MWR-1B-RAD_C_{originator:4s}_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{disposition_mode:1s}_{processing_mode:1s}____.nc' ] From cd21dd258f2f8545332cdd69f5672eee179c0acd Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 20 Dec 2024 13:26:50 +0100 Subject: [PATCH 43/48] Fix short/long names and make improvements raised in the review Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/aws1_mwr_l1b_nc.yaml | 2 +- satpy/etc/readers/aws1_mwr_l1c_nc.yaml | 20 +++++++++---------- satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml | 4 ++-- satpy/readers/mwr_l1c.py | 2 +- satpy/tests/reader_tests/conftest.py | 2 +- satpy/tests/reader_tests/test_aws1_mwr_l1b.py | 7 +++---- satpy/tests/reader_tests/test_aws1_mwr_l1c.py | 7 ++----- .../reader_tests/test_eps_sterna_mwr_l1b.py | 5 +++++ 8 files changed, 25 insertions(+), 24 deletions(-) diff --git a/satpy/etc/readers/aws1_mwr_l1b_nc.yaml b/satpy/etc/readers/aws1_mwr_l1b_nc.yaml index bbc4a567e7..4b3cccc648 100644 --- a/satpy/etc/readers/aws1_mwr_l1b_nc.yaml +++ b/satpy/etc/readers/aws1_mwr_l1b_nc.yaml @@ -1,6 +1,6 @@ reader: name: aws1_mwr_l1b_nc - short_name: AWS1 MWR L1B RAD NetCDF4 + short_name: AWS1 MWR L1B long_name: AWS1 MWR L1B Radiance (NetCDF4) description: Reader for the ESA AWS (Arctic Weather Satellite) Microwave Radiometer (MWR) level-1b files in netCDF4. reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader diff --git a/satpy/etc/readers/aws1_mwr_l1c_nc.yaml b/satpy/etc/readers/aws1_mwr_l1c_nc.yaml index e62b88d54d..e27cc99a2d 100644 --- a/satpy/etc/readers/aws1_mwr_l1c_nc.yaml +++ b/satpy/etc/readers/aws1_mwr_l1c_nc.yaml @@ -1,7 +1,7 @@ reader: name: aws1_mwr_l1c_nc - short_name: AWS L1C RAD NetCDF4 - long_name: AWS L1C Radiance (NetCDF4) + short_name: AWS1 MWR L1C + long_name: AWS1 MWR L1C Radiance (NetCDF4) description: Reader for the ESA AWS (Arctic Weather Satellite) MWR level-1c files in netCDF4. reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader sensors: [mwr,] @@ -328,8 +328,8 @@ datasets: # --- Navigation data --- - solar_azimuth: - name: solar_azimuth + solar_azimuth_angle: + name: solar_azimuth_angle file_type: aws_l1c_nc file_key: data/navigation/aws_solar_azimuth_angle standard_name: solar_azimuth_angle @@ -337,8 +337,8 @@ datasets: - longitude - latitude - solar_zenith: - name: solar_zenith + solar_zenith_angle: + name: solar_zenith_angle file_type: aws_l1c_nc file_key: data/navigation/aws_solar_zenith_angle standard_name: solar_zenith_angle @@ -346,8 +346,8 @@ datasets: - longitude - latitude - satellite_azimuth: - name: satellite_azimuth + satellite_azimuth_angle: + name: satellite_azimuth_angle file_type: aws_l1c_nc file_key: data/navigation/aws_satellite_azimuth_angle standard_name: satellite_azimuth_angle @@ -355,8 +355,8 @@ datasets: - longitude - latitude - satellite_zenith: - name: satellite_zenith + satellite_zenith_angle: + name: satellite_zenith_angle file_type: aws_l1c_nc file_key: data/navigation/aws_satellite_zenith_angle standard_name: satellite_zenith_angle diff --git a/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml b/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml index efc7dc8bc9..fe5bea93b2 100644 --- a/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml +++ b/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml @@ -1,7 +1,7 @@ reader: name: eps_sterna_mwr_l1b_nc - short_name: AWS L1B RAD NetCDF4 - long_name: AWS L1B Radiance (NetCDF4) + short_name: EPS-Sterna MWR L1B + long_name: EPS-Sterna MWR L1B Radiance (NetCDF4) description: Reader for the EUMETSAT EPS-Sterna radiometer level-1b files in netCDF4. reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader sensors: [mwr,] diff --git a/satpy/readers/mwr_l1c.py b/satpy/readers/mwr_l1c.py index d14ab6783a..0182b1d77c 100644 --- a/satpy/readers/mwr_l1c.py +++ b/satpy/readers/mwr_l1c.py @@ -14,7 +14,7 @@ # along with this program. If not, see . """Reader for the Arctic Weather Satellite (AWS) MWR level-1c data. -MWR = Microwaver Radiometer, onboard AWS and EPS-Sterna +MWR = Microwave Radiometer, onboard AWS and EPS-Sterna Sample data provided by ESA September 27, 2024. """ diff --git a/satpy/tests/reader_tests/conftest.py b/satpy/tests/reader_tests/conftest.py index cfee6d4523..92e9ef5445 100644 --- a/satpy/tests/reader_tests/conftest.py +++ b/satpy/tests/reader_tests/conftest.py @@ -147,7 +147,7 @@ def eps_sterna_mwr_file(tmp_path_factory, fake_mwr_data_array): tmp_dir = tmp_path_factory.mktemp("eps_sterna_mwr_l1b_tests") start_time = dt.datetime.fromisoformat(ds.attrs["sensing_start_time_utc"]) end_time = dt.datetime.fromisoformat(ds.attrs["sensing_end_time_utc"]) - platform_name = "AWS1" + platform_name = "ST01" processing_time = random_date(dt.datetime(2024, 9, 1, 13), dt.datetime(2030, 6, 1)) filename = tmp_dir / compose(file_pattern, dict(country="XX", organisation="EUMETSAT", diff --git a/satpy/tests/reader_tests/test_aws1_mwr_l1b.py b/satpy/tests/reader_tests/test_aws1_mwr_l1b.py index cfecfd4a66..59c429cca5 100644 --- a/satpy/tests/reader_tests/test_aws1_mwr_l1b.py +++ b/satpy/tests/reader_tests/test_aws1_mwr_l1b.py @@ -8,8 +8,7 @@ from satpy.tests.reader_tests.conftest import make_fake_angles, make_fake_mwr_lonlats -platform_name = "AWS1" -file_pattern = "W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1B-RAD_C_OHB_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc" # noqa +PLATFORM_NAME = "AWS1" geo_dims = ["n_scans", "n_fovs", "n_geo_groups"] @@ -37,7 +36,7 @@ def test_orbit_number_start_end(aws_mwr_handler): def test_metadata(aws_mwr_handler): """Test that the metadata is read correctly.""" assert aws_mwr_handler.sensor == "mwr" - assert aws_mwr_handler.platform_name == platform_name + assert aws_mwr_handler.platform_name == PLATFORM_NAME def test_get_channel_data(aws_mwr_handler, fake_mwr_data_array): @@ -61,7 +60,7 @@ def test_get_channel_data(aws_mwr_handler, fake_mwr_data_array): assert res.dims == ("y", "x") assert "n_channels" not in res.coords assert res.attrs["sensor"] == "mwr" - assert res.attrs["platform_name"] == "AWS1" + assert res.attrs["platform_name"] == PLATFORM_NAME @pytest.mark.parametrize(("id_name", "file_key", "fake_array"), diff --git a/satpy/tests/reader_tests/test_aws1_mwr_l1c.py b/satpy/tests/reader_tests/test_aws1_mwr_l1c.py index 9fe034a76d..7ebca463fa 100644 --- a/satpy/tests/reader_tests/test_aws1_mwr_l1c.py +++ b/satpy/tests/reader_tests/test_aws1_mwr_l1c.py @@ -24,10 +24,7 @@ from satpy.tests.reader_tests.conftest import make_fake_angles, make_fake_mwr_l1c_lonlats -platform_name = "AWS1" -# W_XX-OHB-Stockholm,SAT,AWS1-MWR-1C-RAD_C_OHB__20241126183628_G_D_20240913222540_20240914000332_T_B____.nc -file_pattern = "W_XX-OHB-Stockholm,SAT,{platform_name}-MWR-1C-RAD_C_OHB__{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_T_B____.nc" # noqa - +PLATFORM_NAME = "AWS1" geo_dims = ["n_scans", "n_fovs"] geo_size = 10 * 145 @@ -57,7 +54,7 @@ def test_get_channel_data(aws_mwr_l1c_handler, fake_mwr_data_array): assert res.dims == ("y", "x") assert "n_channels" not in res.coords assert res.attrs["sensor"] == "mwr" - assert res.attrs["platform_name"] == "AWS1" + assert res.attrs["platform_name"] == PLATFORM_NAME @pytest.mark.parametrize(("id_name", "file_key", "fake_array"), diff --git a/satpy/tests/reader_tests/test_eps_sterna_mwr_l1b.py b/satpy/tests/reader_tests/test_eps_sterna_mwr_l1b.py index 0620bc8437..bd9be5b694 100644 --- a/satpy/tests/reader_tests/test_eps_sterna_mwr_l1b.py +++ b/satpy/tests/reader_tests/test_eps_sterna_mwr_l1b.py @@ -61,3 +61,8 @@ def test_try_get_data_not_in_file(eps_sterna_mwr_handler): match_str = "Dataset aws_toa_brightness_temperature not available or not supported yet!" with pytest.raises(NotImplementedError, match=match_str): _ = eps_sterna_mwr_handler.get_dataset(did, dataset_info) + +def test_metadata(eps_sterna_mwr_handler): + """Test that the metadata is read correctly.""" + assert eps_sterna_mwr_handler.sensor == "mwr" + assert eps_sterna_mwr_handler.platform_name == "ST01" From a70375726f4b81dd80cf79a01c6e436add1278c1 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 20 Dec 2024 14:02:43 +0100 Subject: [PATCH 44/48] Add missing enhancement Signed-off-by: Adam.Dybbroe --- satpy/etc/enhancements/mwr.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/satpy/etc/enhancements/mwr.yaml b/satpy/etc/enhancements/mwr.yaml index 4a9cff6354..6c39440de8 100644 --- a/satpy/etc/enhancements/mwr.yaml +++ b/satpy/etc/enhancements/mwr.yaml @@ -13,3 +13,12 @@ enhancements: - name: gamma method: !!python/name:satpy.enhancements.gamma kwargs: {gamma: 1.2} + + tbs_colors: + standard_name: tbs_colors + operations: + - name: colorize + method: !!python/name:satpy.enhancements.colorize + kwargs: + palettes: + - {colors: spectral, min_value: 280, max_value: 180} From a17b8b33306ccf432239c18d18c1c7876c984ec8 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Sun, 5 Jan 2025 21:24:34 +0100 Subject: [PATCH 45/48] Refactor AWS/EPS-Sterna l1b file fixtures Signed-off-by: Adam.Dybbroe --- satpy/tests/reader_tests/conftest.py | 58 +++++++++++++--------------- 1 file changed, 26 insertions(+), 32 deletions(-) diff --git a/satpy/tests/reader_tests/conftest.py b/satpy/tests/reader_tests/conftest.py index 92e9ef5445..a6459d96d2 100644 --- a/satpy/tests/reader_tests/conftest.py +++ b/satpy/tests/reader_tests/conftest.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2021, 2024 Satpy developers +# Copyright (c) 2021, 2024, 2025 Satpy developers # # This file is part of satpy. # @@ -139,47 +139,41 @@ def aws_eps_sterna_mwr_l1bfile(fake_mwr_data_array, eps_sterna=True): return ds -@pytest.fixture(scope="module") -def eps_sterna_mwr_file(tmp_path_factory, fake_mwr_data_array): - """Create an EPS-Sterna MWR l1b file.""" - ds = aws_eps_sterna_mwr_l1bfile(fake_mwr_data_array, eps_sterna=True) - tmp_dir = tmp_path_factory.mktemp("eps_sterna_mwr_l1b_tests") +def create_mwr_file(tmpdir, data_array, eps_sterna=False): + """Create an AWS or EPS-Sterna MWR l1b file.""" + ds = aws_eps_sterna_mwr_l1bfile(data_array, eps_sterna=eps_sterna) start_time = dt.datetime.fromisoformat(ds.attrs["sensing_start_time_utc"]) end_time = dt.datetime.fromisoformat(ds.attrs["sensing_end_time_utc"]) - platform_name = "ST01" + if eps_sterna: + platform_name = "ST01" + else: + platform_name = "AWS1" + processing_time = random_date(dt.datetime(2024, 9, 1, 13), dt.datetime(2030, 6, 1)) - filename = tmp_dir / compose(file_pattern, dict(country="XX", - organisation="EUMETSAT", - location="Darmstadt", - processing_level="1B", - originator="EUMT", - start_time=start_time, end_time=end_time, - processing_time=processing_time, - platform_name=platform_name)) + filename = tmpdir / compose(file_pattern, dict(country="XX", + organisation="EUMETSAT", + location="Darmstadt", + processing_level="1B", + originator="EUMT", + start_time=start_time, end_time=end_time, + processing_time=processing_time, + platform_name=platform_name)) ds.to_netcdf(filename) return filename +@pytest.fixture(scope="module") +def eps_sterna_mwr_file(tmp_path_factory, fake_mwr_data_array): + """Create an EPS-Sterna MWR l1b file.""" + tmpdir = tmp_path_factory.mktemp("eps_sterna_mwr_l1b_tests") + return create_mwr_file(tmpdir, fake_mwr_data_array, eps_sterna=True) + + @pytest.fixture(scope="module") def aws_mwr_file(tmp_path_factory, fake_mwr_data_array): """Create an AWS MWR l1b file.""" - ds = aws_eps_sterna_mwr_l1bfile(fake_mwr_data_array, eps_sterna=False) - - tmp_dir = tmp_path_factory.mktemp("aws_l1b_tests") - start_time = dt.datetime.fromisoformat(ds.attrs["sensing_start_time_utc"]) - end_time = dt.datetime.fromisoformat(ds.attrs["sensing_end_time_utc"]) - platform_name = "AWS1" - processing_time = random_date(dt.datetime(2024, 9, 1, 13), dt.datetime(2030, 6, 1)) - filename = tmp_dir / compose(file_pattern, dict(country="SE", - organisation="SMHI", - location="Norrkoping", - processing_level="1B", - originator="SMHI", - start_time=start_time, end_time=end_time, - processing_time=processing_time, - platform_name=platform_name)) - ds.to_netcdf(filename) - return filename + tmpdir = tmp_path_factory.mktemp("aws_l1b_tests") + return create_mwr_file(tmpdir, fake_mwr_data_array, eps_sterna=False) @pytest.fixture(scope="module") From c7d5fe9602a99618a38ae1b5eb5e0de49770298e Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Mon, 6 Jan 2025 19:04:22 +0100 Subject: [PATCH 46/48] Refactoring tests reusing the l1b file creation part also for level-1c Signed-off-by: Adam.Dybbroe --- satpy/tests/reader_tests/conftest.py | 92 ++++++------------- satpy/tests/reader_tests/test_aws1_mwr_l1b.py | 11 ++- .../reader_tests/test_eps_sterna_mwr_l1b.py | 6 +- 3 files changed, 39 insertions(+), 70 deletions(-) diff --git a/satpy/tests/reader_tests/conftest.py b/satpy/tests/reader_tests/conftest.py index a6459d96d2..ddb7bde8d2 100644 --- a/satpy/tests/reader_tests/conftest.py +++ b/satpy/tests/reader_tests/conftest.py @@ -59,21 +59,21 @@ def fake_mwr_data_array(): return xr.DataArray(fake_data_np, dims=array_dims) -def make_fake_angles(geo_size, geo_dims, shape=(10, 145, 4)): +def make_fake_angles(geo_size, geo_dims, shape): """Return fake sun-satellite angle array.""" maxval = 36000 dummy_array = (np.arange(0, geo_size) * maxval/geo_size).astype("int32") return xr.DataArray(dummy_array.reshape(shape), dims=geo_dims) -def make_fake_mwr_lonlats(geo_size, geo_dims): +def make_fake_mwr_lonlats(geo_size, geo_dims, shape): """Return fake geolocation data arrays for all 4 MWR horns.""" maxval = 3600000 dummy_array = (np.arange(0, geo_size) * maxval/geo_size).astype("int32") - fake_lon_data = xr.DataArray(dummy_array.reshape((10, 145, 4)), dims=geo_dims) + fake_lon_data = xr.DataArray(dummy_array.reshape(shape), dims=geo_dims) maxval = 1800000 dummy_array = (np.arange(0, geo_size) * maxval/geo_size - maxval/2).astype("int32") - fake_lat_data = xr.DataArray(dummy_array.reshape((10, 145, 4)), dims=geo_dims) + fake_lat_data = xr.DataArray(dummy_array.reshape(shape), dims=geo_dims) return (fake_lon_data, fake_lat_data) @@ -88,7 +88,7 @@ def make_fake_mwr_l1c_lonlats(geo_size, geo_dims): return (fake_lon_data, fake_lat_data) -def aws_eps_sterna_mwr_l1bfile(fake_mwr_data_array, eps_sterna=True): +def aws_eps_sterna_mwr_level1_file(fake_mwr_data_array, eps_sterna=True, l1b=True): """Create an AWS and EPS-Sterna MWR l1b file.""" if eps_sterna: n_feedhorns="n_feedhorns" @@ -101,8 +101,14 @@ def aws_eps_sterna_mwr_l1bfile(fake_mwr_data_array, eps_sterna=True): longitude_attr = "aws_lon" latitude_attr = "aws_lat" - geo_dims = ["n_scans", "n_fovs", n_feedhorns] - geo_size = 10 * 145 * 4 + if l1b: + geo_dims = ["n_scans", "n_fovs", n_feedhorns] + geo_size = 10 * 145 * 4 + shape = (10, 145, 4) + else: + geo_dims = ["n_scans", "n_fovs"] + geo_size = 10 * 145 + shape = (10, 145) ds = DataTree() start_time = dt.datetime(2024, 9, 1, 12, 0) @@ -110,8 +116,7 @@ def aws_eps_sterna_mwr_l1bfile(fake_mwr_data_array, eps_sterna=True): end_time = dt.datetime(2024, 9, 1, 12, 15) ds.attrs["sensing_end_time_utc"] = end_time.strftime(DATETIME_FORMAT) - instrument = "MWR" - ds.attrs["instrument"] = instrument + ds.attrs["instrument"] = "MWR" ds.attrs["orbit_start"] = 9991 ds.attrs["orbit_end"] = 9992 dset_name = f"data/calibration/{prefix}toa_brightness_temperature" @@ -122,27 +127,28 @@ def aws_eps_sterna_mwr_l1bfile(fake_mwr_data_array, eps_sterna=True): ds[dset_name].attrs["valid_min"] = 0 ds[dset_name].attrs["valid_max"] = 700000 - fake_lon_data, fake_lat_data = make_fake_mwr_lonlats(geo_size, geo_dims) + fake_lon_data, fake_lat_data = make_fake_mwr_lonlats(geo_size, geo_dims, shape) ds[f"data/navigation/{longitude_attr}"] = fake_lon_data ds[f"data/navigation/{longitude_attr}"].attrs["scale_factor"] = 1e-4 ds[f"data/navigation/{longitude_attr}"].attrs["add_offset"] = 0.0 ds[f"data/navigation/{latitude_attr}"] = fake_lat_data - ds[f"data/navigation/{prefix}solar_azimuth_angle"] = make_fake_angles(geo_size, geo_dims) - ds[f"data/navigation/{prefix}solar_zenith_angle"] = make_fake_angles(geo_size, geo_dims) - ds[f"data/navigation/{prefix}satellite_azimuth_angle"] = make_fake_angles(geo_size, geo_dims) - ds[f"data/navigation/{prefix}satellite_zenith_angle"] = make_fake_angles(geo_size, geo_dims) - ds["status/satellite/subsat_latitude_end"] = np.array(22.39) - ds["status/satellite/subsat_longitude_start"] = np.array(304.79) - ds["status/satellite/subsat_latitude_start"] = np.array(55.41) - ds["status/satellite/subsat_longitude_end"] = np.array(296.79) + ds[f"data/navigation/{prefix}solar_azimuth_angle"] = make_fake_angles(geo_size, geo_dims, shape) + ds[f"data/navigation/{prefix}solar_zenith_angle"] = make_fake_angles(geo_size, geo_dims, shape) + ds[f"data/navigation/{prefix}satellite_azimuth_angle"] = make_fake_angles(geo_size, geo_dims, shape) + ds[f"data/navigation/{prefix}satellite_zenith_angle"] = make_fake_angles(geo_size, geo_dims, shape) + if l1b: + ds["status/satellite/subsat_latitude_end"] = np.array(22.39) + ds["status/satellite/subsat_longitude_start"] = np.array(304.79) + ds["status/satellite/subsat_latitude_start"] = np.array(55.41) + ds["status/satellite/subsat_longitude_end"] = np.array(296.79) return ds -def create_mwr_file(tmpdir, data_array, eps_sterna=False): - """Create an AWS or EPS-Sterna MWR l1b file.""" - ds = aws_eps_sterna_mwr_l1bfile(data_array, eps_sterna=eps_sterna) +def create_mwr_file(tmpdir, data_array, eps_sterna=False, l1b=True): + """Create an AWS or EPS-Sterna MWR l1b (or level-1c) file.""" + ds = aws_eps_sterna_mwr_level1_file(data_array, eps_sterna=eps_sterna, l1b=l1b) start_time = dt.datetime.fromisoformat(ds.attrs["sensing_start_time_utc"]) end_time = dt.datetime.fromisoformat(ds.attrs["sensing_end_time_utc"]) if eps_sterna: @@ -179,48 +185,8 @@ def aws_mwr_file(tmp_path_factory, fake_mwr_data_array): @pytest.fixture(scope="module") def aws_mwr_l1c_file(tmp_path_factory, fake_mwr_data_array): """Create an AWS MWR l1c file.""" - geo_dims = ["n_scans", "n_fovs"] - geo_size = 10 * 145 - - ds = DataTree() - start_time = dt.datetime(2024, 9, 1, 12, 0) - ds.attrs["sensing_start_time_utc"] = start_time.strftime(DATETIME_FORMAT) - end_time = dt.datetime(2024, 9, 1, 12, 15) - ds.attrs["sensing_end_time_utc"] = end_time.strftime(DATETIME_FORMAT) - processing_time = random_date(dt.datetime(2024, 6, 1), dt.datetime(2030, 6, 1)) - - ds.attrs["instrument"] = "MWR" - ds.attrs["orbit_start"] = 9991 - ds.attrs["orbit_end"] = 9992 - ds["data/calibration/aws_toa_brightness_temperature"] = fake_mwr_data_array - ds["data/calibration/aws_toa_brightness_temperature"].attrs["scale_factor"] = 0.001 - ds["data/calibration/aws_toa_brightness_temperature"].attrs["add_offset"] = 0.0 - ds["data/calibration/aws_toa_brightness_temperature"].attrs["missing_value"] = -2147483648 - ds["data/calibration/aws_toa_brightness_temperature"].attrs["valid_min"] = 0 - ds["data/calibration/aws_toa_brightness_temperature"].attrs["valid_max"] = 700000 - - fake_lon_data, fake_lat_data = make_fake_mwr_l1c_lonlats(geo_size, geo_dims) - - ds["data/navigation/aws_lon"] = fake_lon_data - ds["data/navigation/aws_lon"].attrs["scale_factor"] = 1e-4 - ds["data/navigation/aws_lon"].attrs["add_offset"] = 0.0 - ds["data/navigation/aws_lat"] = fake_lat_data - ds["data/navigation/aws_solar_azimuth_angle"] = make_fake_angles(geo_size, geo_dims, shape=(10, 145)) - ds["data/navigation/aws_solar_zenith_angle"] = make_fake_angles(geo_size, geo_dims, shape=(10, 145)) - ds["data/navigation/aws_satellite_azimuth_angle"] = make_fake_angles(geo_size, geo_dims, shape=(10, 145)) - ds["data/navigation/aws_satellite_zenith_angle"] = make_fake_angles(geo_size, geo_dims, shape=(10, 145)) - - tmp_dir = tmp_path_factory.mktemp("aws_l1c_tests") - filename = tmp_dir / compose(file_pattern, dict(country="SE", - organisation="SMHI", - location="Norrkoping", - processing_level="1C", - originator="SMHI", - start_time=start_time, end_time=end_time, - processing_time=processing_time, - platform_name=platform_name)) - ds.to_netcdf(filename) - return filename + tmpdir = tmp_path_factory.mktemp("aws_l1c_tests") + return create_mwr_file(tmpdir, fake_mwr_data_array, eps_sterna=False, l1b=False) @pytest.fixture(scope="module") diff --git a/satpy/tests/reader_tests/test_aws1_mwr_l1b.py b/satpy/tests/reader_tests/test_aws1_mwr_l1b.py index 59c429cca5..e388b6186e 100644 --- a/satpy/tests/reader_tests/test_aws1_mwr_l1b.py +++ b/satpy/tests/reader_tests/test_aws1_mwr_l1b.py @@ -13,11 +13,12 @@ geo_dims = ["n_scans", "n_fovs", "n_geo_groups"] geo_size = 10*145*4 -fake_lon_data, fake_lat_data = make_fake_mwr_lonlats(geo_size, geo_dims) -fake_sun_azi_data = make_fake_angles(geo_size, geo_dims) -fake_sun_zen_data = make_fake_angles(geo_size, geo_dims) -fake_sat_azi_data = make_fake_angles(geo_size, geo_dims) -fake_sat_zen_data = make_fake_angles(geo_size, geo_dims) +shape = (10, 145, 4) +fake_lon_data, fake_lat_data = make_fake_mwr_lonlats(geo_size, geo_dims, shape) +fake_sun_azi_data = make_fake_angles(geo_size, geo_dims, shape) +fake_sun_zen_data = make_fake_angles(geo_size, geo_dims, shape) +fake_sat_azi_data = make_fake_angles(geo_size, geo_dims, shape) +fake_sat_zen_data = make_fake_angles(geo_size, geo_dims, shape) diff --git a/satpy/tests/reader_tests/test_eps_sterna_mwr_l1b.py b/satpy/tests/reader_tests/test_eps_sterna_mwr_l1b.py index bd9be5b694..43abdf74d4 100644 --- a/satpy/tests/reader_tests/test_eps_sterna_mwr_l1b.py +++ b/satpy/tests/reader_tests/test_eps_sterna_mwr_l1b.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2024 Satpy developers +# Copyright (c) 2024, 2025 Satpy developers # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -27,7 +27,9 @@ geo_dims = ["n_scans", "n_fovs", "n_feedhorns"] geo_size = 10*145*4 -fake_lon_data, fake_lat_data = make_fake_mwr_lonlats(geo_size, geo_dims) +shape = (10, 145, 4) +fake_lon_data, fake_lat_data = make_fake_mwr_lonlats(geo_size, geo_dims, shape) + @pytest.mark.parametrize(("id_name", "file_key", "fake_array"), [("longitude", "data/navigation/longitude", fake_lon_data * 1e-4), From d4654d157e7e4b4eabebea41f6f357eca0bea123 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Tue, 7 Jan 2025 09:05:38 +0100 Subject: [PATCH 47/48] Fix right processing level in tmp file name Signed-off-by: Adam.Dybbroe --- satpy/tests/reader_tests/conftest.py | 9 ++++----- satpy/tests/reader_tests/test_aws1_mwr_l1c.py | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/satpy/tests/reader_tests/conftest.py b/satpy/tests/reader_tests/conftest.py index ddb7bde8d2..f4b9844851 100644 --- a/satpy/tests/reader_tests/conftest.py +++ b/satpy/tests/reader_tests/conftest.py @@ -151,16 +151,15 @@ def create_mwr_file(tmpdir, data_array, eps_sterna=False, l1b=True): ds = aws_eps_sterna_mwr_level1_file(data_array, eps_sterna=eps_sterna, l1b=l1b) start_time = dt.datetime.fromisoformat(ds.attrs["sensing_start_time_utc"]) end_time = dt.datetime.fromisoformat(ds.attrs["sensing_end_time_utc"]) - if eps_sterna: - platform_name = "ST01" - else: - platform_name = "AWS1" + + platform_name = "ST01" if eps_sterna else "AWS1" + processing_level = "1B" if l1b else "1C" processing_time = random_date(dt.datetime(2024, 9, 1, 13), dt.datetime(2030, 6, 1)) filename = tmpdir / compose(file_pattern, dict(country="XX", organisation="EUMETSAT", location="Darmstadt", - processing_level="1B", + processing_level=processing_level, originator="EUMT", start_time=start_time, end_time=end_time, processing_time=processing_time, diff --git a/satpy/tests/reader_tests/test_aws1_mwr_l1c.py b/satpy/tests/reader_tests/test_aws1_mwr_l1c.py index 7ebca463fa..10c499838d 100644 --- a/satpy/tests/reader_tests/test_aws1_mwr_l1c.py +++ b/satpy/tests/reader_tests/test_aws1_mwr_l1c.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2024 Satpy developers +# Copyright (c) 2024-2025 Satpy developers # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From 0b9760c4ecee4d47642cb4e83bb78e2099b36b61 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Mon, 13 Jan 2025 15:08:19 +0100 Subject: [PATCH 48/48] Improvements following review comments. Signed-off-by: Adam.Dybbroe --- satpy/etc/composites/microwave.yaml | 1 + satpy/etc/composites/mwr.yaml | 2 +- satpy/etc/readers/aws1_mwr_l1b_nc.yaml | 5 +- satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml | 1 + satpy/readers/mwr_l1b.py | 54 +++++++++++++------- satpy/readers/mwr_l1c.py | 25 ++++++++- satpy/tests/reader_tests/conftest.py | 5 +- 7 files changed, 71 insertions(+), 22 deletions(-) create mode 100644 satpy/etc/composites/microwave.yaml diff --git a/satpy/etc/composites/microwave.yaml b/satpy/etc/composites/microwave.yaml new file mode 100644 index 0000000000..9cc6789cbd --- /dev/null +++ b/satpy/etc/composites/microwave.yaml @@ -0,0 +1 @@ +sensor_name: microwave diff --git a/satpy/etc/composites/mwr.yaml b/satpy/etc/composites/mwr.yaml index d959faa632..5f10986d4c 100644 --- a/satpy/etc/composites/mwr.yaml +++ b/satpy/etc/composites/mwr.yaml @@ -1,4 +1,4 @@ -sensor_name: mwr +sensor_name: microwave/mwr composites: mw183_humidity: diff --git a/satpy/etc/readers/aws1_mwr_l1b_nc.yaml b/satpy/etc/readers/aws1_mwr_l1b_nc.yaml index 4b3cccc648..f487d58265 100644 --- a/satpy/etc/readers/aws1_mwr_l1b_nc.yaml +++ b/satpy/etc/readers/aws1_mwr_l1b_nc.yaml @@ -527,7 +527,10 @@ file_types: aws_l1b_nc: # W_XX-OHB-Unknown,SAT,1-AWS-1B-RAD_C_OHB_20230707124607_G_D_20220621090100_20220621090618_T_B____.nc # W_XX-OHB-Stockholm,SAT,AWS1-MWR-1B-RAD_C_OHB_20230823161321_G_D_20240115111111_20240115125434_T_B____.nc + # W_NO-KSAT-Tromso,SAT,AWS1-MWR-1B-RAD_C_OHB__20250110134851_G_O_20250110114708_20250110132329_C_N____.nc file_reader: !!python/name:satpy.readers.mwr_l1b.AWS_EPS_Sterna_MWR_L1BFile file_patterns: [ - 'W_{country:2s}-{organisation:s}-{location:s},SAT,{platform_name}-MWR-1B-RAD_C_{originator:4s}_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{disposition_mode:1s}_{processing_mode:1s}____.nc' + 'W_{country:2s}-{organisation:s}-{location:s},SAT,{platform_name}-MWR-1B-RAD_C_{originator:4s}_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{disposition_mode:1s}_{processing_mode:1s}____.nc', + 'W_{country:2s}-{organisation:s}-{location:s},SAT,{platform_name}-MWR-1B-RAD_C_{originator:4s}_{processing_time:%Y%m%d%H%M%S}_G_O_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{disposition_mode:1s}_{processing_mode:1s}____.nc' ] + feed_horn_group_name: n_geo_groups diff --git a/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml b/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml index fe5bea93b2..d08113270a 100644 --- a/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml +++ b/satpy/etc/readers/eps_sterna_mwr_l1b_nc.yaml @@ -530,3 +530,4 @@ file_types: file_patterns: [ 'W_{country:2s}-{organisation:s}-{location:s},SAT,{platform_name}-MWR-1B-RAD_C_{originator:4s}_{processing_time:%Y%m%d%H%M%S}_G_D_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{disposition_mode:1s}_{processing_mode:1s}____.nc' ] + feed_horn_group_name: n_feedhorns diff --git a/satpy/readers/mwr_l1b.py b/satpy/readers/mwr_l1b.py index 7f77f6dbf9..173c729ed8 100644 --- a/satpy/readers/mwr_l1b.py +++ b/satpy/readers/mwr_l1b.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, 2024 Pytroll Developers +# Copyright (c) 2023 - 2025 Pytroll Developers # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -23,6 +23,36 @@ Sample EPS-Sterna l1b format AWS data from 16 orbits the 9th of November 2024. +Continous feed (though restricted to the SAG members and selected European +users/evaluators) in the EUMETSAT Data Store of global AWS data from January +9th, 2025. + +Example: +-------- +Here is an example how to read the data in satpy: + +.. code-block:: python + + from satpy import Scene + from glob import glob + + filenames = glob("data/W_NO-KSAT-Tromso,SAT,AWS1-MWR-1B-RAD_C_OHB__*_G_O_20250110114708*.nc" + scn = Scene(filenames=filenames, reader='aws1_mwr_l1b_nc') + + composites = ['mw183_humidity'] + dataset_names = composites + ['1'] + + scn.load(dataset_names) + print(scn['1']) + scn.show('mw183_humidity') + + +As the file format for the EPS Sterna Level-1b is slightly different from the +ESA format, reading the EPS Sterna level-1b data uses a different reader, named +`eps_sterna_mwr_l1b_nc`. So, if specifying the reader name as in the above code +example, please provide the actual name for that data: eps_sterna_mwr_l1b_nc. + + """ import xarray as xr @@ -46,7 +76,9 @@ "satellite_azimuth_horn1", "satellite_azimuth_horn2", "satellite_azimuth_horn3", - "satellite_azimuth_horn4"] + "satellite_azimuth_horn4", + "longitude", + "latitude"] class AWS_EPS_Sterna_BaseFileHandler(NetCDF4FileHandler): """Base class implementing the AWS/EPS-Sterna MWR Level-1b&c Filehandlers.""" @@ -103,23 +135,12 @@ def _get_channel_data(self, dataset_id, dataset_info): class AWS_EPS_Sterna_MWR_L1BFile(AWS_EPS_Sterna_BaseFileHandler): - """Class implementing the AWS/EPS-Sterna MWR L1b Filehandler. + """Class implementing the AWS/EPS-Sterna MWR L1b Filehandler.""" - This class implements the ESA Arctic Weather Satellite (AWS) and EPS-Sterna - MWR Level-1b NetCDF reader. It is designed to be used through the - :class:`~satpy.Scene` class using the :mod:`~satpy.Scene.load` method with - the reader ``"mwr_l1b_nc"``. - - """ def __init__(self, filename, filename_info, filetype_info, auto_maskandscale=True): """Initialize the handler.""" super().__init__(filename, filename_info, filetype_info, auto_maskandscale) - - if filetype_info["file_type"].startswith("eps_sterna"): - self._feed_horn_group_name = "n_feedhorns" - else: - self._feed_horn_group_name = "n_geo_groups" - + self._feed_horn_group_name = filetype_info.get("feed_horn_group_name") @property def sub_satellite_longitude_start(self): @@ -147,9 +168,6 @@ def get_dataset(self, dataset_id, dataset_info): data_array = self._get_channel_data(dataset_id, dataset_info) elif dataset_id["name"] in NAVIGATION_DATASET_NAMES: data_array = self._get_navigation_data(dataset_id, dataset_info) - - elif dataset_id["name"] in ["longitude", "latitude"]: - data_array = self._get_navigation_data(dataset_id, dataset_info) else: raise NotImplementedError(f"Dataset {dataset_id['name']} not available or not supported yet!") diff --git a/satpy/readers/mwr_l1c.py b/satpy/readers/mwr_l1c.py index 0182b1d77c..3d429fd8f5 100644 --- a/satpy/readers/mwr_l1c.py +++ b/satpy/readers/mwr_l1c.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024 Pytroll Developers +# Copyright (c) 2024 - 2025 Pytroll Developers # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,8 +17,31 @@ MWR = Microwave Radiometer, onboard AWS and EPS-Sterna Sample data provided by ESA September 27, 2024. + + +Example: +-------- +Here is an example how to read the data in satpy: + +.. code-block:: python + + from satpy import Scene + from glob import glob + + filenames = glob("data/W_XX-OHB-Stockholm,SAT,AWS1-MWR-1C-RAD_C_OHB_*20240913204851_*.nc") + + scn = Scene(filenames=filenames, reader='aws1_mwr_l1c_nc') + + composites = ['mw183_humidity'] + dataset_names = composites + ['1'] + + scn.load(dataset_names) + print(scn['1']) + scn.show('mw183_humidity') + """ + from satpy.readers.mwr_l1b import MWR_CHANNEL_NAMES, AWS_EPS_Sterna_BaseFileHandler, mask_and_scale diff --git a/satpy/tests/reader_tests/conftest.py b/satpy/tests/reader_tests/conftest.py index f4b9844851..00742574ef 100644 --- a/satpy/tests/reader_tests/conftest.py +++ b/satpy/tests/reader_tests/conftest.py @@ -48,7 +48,7 @@ def random_date(start, end): return start + dt.timedelta(seconds=random_second) -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def fake_mwr_data_array(): """Return a fake AWS/EPS-Sterna MWR l1b data array.""" fake_data_np = rng.integers(0, 700000, size=10*145*19).reshape((10, 145, 19)) @@ -194,6 +194,7 @@ def eps_sterna_mwr_handler(eps_sterna_mwr_file): filename_info = parse(file_pattern, os.path.basename(eps_sterna_mwr_file)) filetype_info = dict() filetype_info["file_type"] = "eps_sterna_mwr_l1b" + filetype_info["feed_horn_group_name"] = "n_feedhorns" return AWS_EPS_Sterna_MWR_L1BFile(eps_sterna_mwr_file, filename_info, filetype_info) @@ -203,6 +204,7 @@ def aws_mwr_handler(aws_mwr_file): filename_info = parse(file_pattern, os.path.basename(aws_mwr_file)) filetype_info = dict() filetype_info["file_type"] = "aws1_mwr_l1b" + filetype_info["feed_horn_group_name"] = "n_geo_groups" return AWS_EPS_Sterna_MWR_L1BFile(aws_mwr_file, filename_info, filetype_info) @@ -212,4 +214,5 @@ def aws_mwr_l1c_handler(aws_mwr_l1c_file): filename_info = parse(file_pattern, os.path.basename(aws_mwr_l1c_file)) filetype_info = dict() filetype_info["file_type"] = "aws1_mwr_l1c" + filetype_info["feed_horn_group_name"] = None return AWS_MWR_L1CFile(aws_mwr_l1c_file, filename_info, filetype_info)