diff --git a/utils/b2i/b2iconverter/ioda_variables.py b/utils/b2i/b2iconverter/ioda_variables.py index d898f81d..26b604cc 100644 --- a/utils/b2i/b2iconverter/ioda_variables.py +++ b/utils/b2i/b2iconverter/ioda_variables.py @@ -4,6 +4,10 @@ from .ioda_metadata import IODAMetadata from .ioda_addl_vars import IODAAdditionalVariables +# Error values are hard-coded in the converters using functions +# like the set_salinity_error below +# These values are overwritten at run time. + class IODAVariables: def __init__(self): diff --git a/utils/b2i/bufr2ioda_insitu_profile_pirata.py b/utils/b2i/bufr2ioda_insitu_profile_pirata.py deleted file mode 100755 index 34f1af4b..00000000 --- a/utils/b2i/bufr2ioda_insitu_profile_pirata.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 - -import sys -from b2iconverter.util import parse_arguments -from b2iconverter.bufr2ioda_config import Bufr2iodaConfig -from b2iconverter.bufr2ioda_converter import Bufr2ioda_Converter -from mbuoyb_tropical_ioda_variables import MbuoybTropicalIODAVariables -from wmo_codes import * - -import numpy as np - - -platform_description = 'PIRATA Tropical mooring profiles from mbuoyb: temperature and salinity' - - -class PirataConfig(Bufr2iodaConfig): - def ioda_filename(self): - return f"{self.cycle_type}.t{self.hh}z.insitu_profile_pirata.{self.cycle_datetime}.nc" - - -class PirataIODAVariables(MbuoybTropicalIODAVariables): - def filter(self): - super().filter() - mask = [True if int(rpid) in PIRATA else False for rpid in self.metadata.stationID] - self.metadata.filter(mask) - self.temp = self.temp[mask] - self.saln = self.saln[mask] - - -if __name__ == '__main__': - - script_name, config_file, log_file, test_file = parse_arguments() - - bufr2ioda_config = PirataConfig( - script_name, - config_file, - platform_description) - - ioda_vars = PirataIODAVariables() - ioda_vars.set_temperature_var_name("waterTemperature") - ioda_vars.set_temperature_error(0.02) - ioda_vars.set_salinity_var_name("salinity") - ioda_vars.set_salinity_error(0.01) - - tropical = Bufr2ioda_Converter(bufr2ioda_config, ioda_vars, log_file) - - tropical.run() - - if test_file: - result = tropical.test(test_file) - sys.exit(result) diff --git a/utils/b2i/bufr2ioda_insitu_profile_pirata_saln.py b/utils/b2i/bufr2ioda_insitu_profile_pirata_saln.py new file mode 100755 index 00000000..498d48f7 --- /dev/null +++ b/utils/b2i/bufr2ioda_insitu_profile_pirata_saln.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 + +import sys +from b2iconverter.util import parse_arguments +from b2iconverter.bufr2ioda_config import Bufr2iodaConfig +from b2iconverter.bufr2ioda_converter import Bufr2ioda_Converter +from b2iconverter.ioda_variables import IODAVariables +from b2iconverter.ioda_metadata import IODAMetadata +from b2iconverter.ioda_addl_vars import IODAAdditionalVariables +from wmo_codes import * + +import numpy as np + + +platform_description = 'PIRATA Tropical mooring profiles from mbuoyb: salinity' + + +class PirataConfig(Bufr2iodaConfig): + def ioda_filename(self): + return f"{self.cycle_type}.t{self.hh}z.insitu_profile_pirata_saln.{self.cycle_datetime}.nc" + + +class PirataIODAVariables(IODAVariables): + def __init__(self): + self.construct() + self.metadata = IODAMetadata() + self.additional_vars = PirataAdditionalVariables(self) + + def build_query(self): + q = super().build_query() + q.add('latitude', '*/CLATH') + q.add('longitude', '*/CLONH') + q.add('stationID', '*/RPID') + q.add('depth', '*/IDMSMDBS/BBYSTSL/DBSS') + q.add('saln', '*/IDMSMDBS/BBYSTSL/SALN') + return q + + def set_obs_from_query_result(self, r): + self.saln = r.get('saln', group_by='depth') + + def filter(self): + super().filter() + mask = [True if int(rpid) in PIRATA else False for rpid in self.metadata.stationID] + mask = mask & self.SalinityFilter() + self.metadata.filter(mask) + self.saln = self.saln[mask] + + def write_to_ioda_file(self, obsspace): + self.metadata.write_to_ioda_file(obsspace) + self.additional_vars.write_to_ioda_file(obsspace) + self.write_obs_value_s(obsspace) + + def log_obs(self, logger): + self.log_salinity(logger) + + +class PirataAdditionalVariables(IODAAdditionalVariables): + def construct(self): + n = len(self.ioda_vars.metadata.lon) + self.PreQC = (np.ma.masked_array(np.full(n, 0))).astype(np.int32) + self.ObsError_saln = \ + np.float32(np.ma.masked_array(np.full(n, self.ioda_vars.S_error))) + self.compute_ocean_basin() + + def write_to_ioda_file(self, obsspace): + self.write_preqc(obsspace, self.ioda_vars.S_name) + self.write_obs_errorS(obsspace) + self.write_ocean_basin(obsspace) + + def log(self, logger): + self.log_preqc(logger) + self.log_obs_error_saln(logger) + self.log_ocean_basin(logger) + + +if __name__ == '__main__': + + script_name, config_file, log_file, test_file = parse_arguments() + + bufr2ioda_config = PirataConfig( + script_name, + config_file, + platform_description) + + ioda_vars = PirataIODAVariables() + ioda_vars.set_salinity_var_name("salinity") + ioda_vars.set_salinity_error(0.01) + + tropical = Bufr2ioda_Converter(bufr2ioda_config, ioda_vars, log_file) + + tropical.run() + + if test_file: + result = tropical.test(test_file) + sys.exit(result) diff --git a/utils/b2i/bufr2ioda_insitu_profile_pirata_temp.py b/utils/b2i/bufr2ioda_insitu_profile_pirata_temp.py new file mode 100755 index 00000000..87cd43d2 --- /dev/null +++ b/utils/b2i/bufr2ioda_insitu_profile_pirata_temp.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 + +import sys +from b2iconverter.util import parse_arguments +from b2iconverter.bufr2ioda_config import Bufr2iodaConfig +from b2iconverter.bufr2ioda_converter import Bufr2ioda_Converter +from b2iconverter.ioda_variables import IODAVariables +from b2iconverter.ioda_metadata import IODAMetadata +from b2iconverter.ioda_addl_vars import IODAAdditionalVariables +from wmo_codes import * + +import numpy as np + + +platform_description = 'PIRATA Tropical mooring profiles from mbuoyb: temperature' + + +class PirataConfig(Bufr2iodaConfig): + def ioda_filename(self): + return f"{self.cycle_type}.t{self.hh}z.insitu_profile_pirata_temp.{self.cycle_datetime}.nc" + + +class PirataIODAVariables(IODAVariables): + def __init__(self): + self.construct() + self.metadata = IODAMetadata() + self.additional_vars = PirataAdditionalVariables(self) + + def build_query(self): + q = super().build_query() + q.add('latitude', '*/CLATH') + q.add('longitude', '*/CLONH') + q.add('stationID', '*/RPID') + q.add('depth', '*/IDMSMDBS/BBYSTSL/DBSS') + q.add('temp', '*/IDMSMDBS/BBYSTSL/SST1') + return q + + def set_obs_from_query_result(self, r): + self.temp = r.get('temp', group_by='depth') + self.temp -= 273.15 + + def filter(self): + super().filter() + mask = [True if int(rpid) in PIRATA else False for rpid in self.metadata.stationID] + mask = mask & self.TemperatureFilter() + self.metadata.filter(mask) + self.temp = self.temp[mask] + + def write_to_ioda_file(self, obsspace): + self.metadata.write_to_ioda_file(obsspace) + self.additional_vars.write_to_ioda_file(obsspace) + self.write_obs_value_t(obsspace) + + def log_obs(self, logger): + self.log_temperature(logger) + + +class PirataAdditionalVariables(IODAAdditionalVariables): + def construct(self): + n = len(self.ioda_vars.metadata.lon) + self.PreQC = (np.ma.masked_array(np.full(n, 0))).astype(np.int32) + self.ObsError_temp = \ + np.float32(np.ma.masked_array(np.full(n, self.ioda_vars.T_error))) + self.compute_ocean_basin() + + def write_to_ioda_file(self, obsspace): + self.write_preqc(obsspace, self.ioda_vars.T_name) + self.write_obs_errorT(obsspace) + self.write_ocean_basin(obsspace) + + def log(self, logger): + self.log_preqc(logger) + self.log_obs_error_temp(logger) + self.log_ocean_basin(logger) + + + +if __name__ == '__main__': + + script_name, config_file, log_file, test_file = parse_arguments() + + bufr2ioda_config = PirataConfig( + script_name, + config_file, + platform_description) + + ioda_vars = PirataIODAVariables() + ioda_vars.set_temperature_var_name("waterTemperature") + ioda_vars.set_temperature_error(0.02) + + tropical = Bufr2ioda_Converter(bufr2ioda_config, ioda_vars, log_file) + + tropical.run() + + if test_file: + result = tropical.test(test_file) + sys.exit(result) diff --git a/utils/b2i/bufr2ioda_insitu_profile_rama.py b/utils/b2i/bufr2ioda_insitu_profile_rama.py deleted file mode 100755 index 049944bc..00000000 --- a/utils/b2i/bufr2ioda_insitu_profile_rama.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 - -import sys -from b2iconverter.util import parse_arguments -from b2iconverter.bufr2ioda_config import Bufr2iodaConfig -from b2iconverter.bufr2ioda_converter import Bufr2ioda_Converter -from mbuoyb_tropical_ioda_variables import MbuoybTropicalIODAVariables -from wmo_codes import * - - -platform_description = 'RAMA Tropical mooring profiles from mbuoyb: temperature and salinity' - - -class RamaConfig(Bufr2iodaConfig): - def ioda_filename(self): - return f"{self.cycle_type}.t{self.hh}z.insitu_profile_rama.{self.cycle_datetime}.nc" - - -class RamaIODAVariables(MbuoybTropicalIODAVariables): - def filter(self): - super().filter() - mask = is_rama(self.metadata.stationID) - self.metadata.filter(mask) - self.temp = self.temp[mask] - self.saln = self.saln[mask] - - -if __name__ == '__main__': - - script_name, config_file, log_file, test_file = parse_arguments() - - bufr2ioda_config = RamaConfig( - script_name, - config_file, - platform_description) - - ioda_vars = RamaIODAVariables() - ioda_vars.set_temperature_var_name("waterTemperature") - ioda_vars.set_temperature_error(0.02) - ioda_vars.set_salinity_var_name("salinity") - ioda_vars.set_salinity_error(0.01) - - tropical = Bufr2ioda_Converter(bufr2ioda_config, ioda_vars, log_file) - - tropical.run() - - if test_file: - result = tropical.test(test_file) - sys.exit(result) diff --git a/utils/b2i/bufr2ioda_insitu_profile_rama_saln.py b/utils/b2i/bufr2ioda_insitu_profile_rama_saln.py new file mode 100755 index 00000000..d7eb0dd8 --- /dev/null +++ b/utils/b2i/bufr2ioda_insitu_profile_rama_saln.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 + +import sys +from b2iconverter.util import parse_arguments +from b2iconverter.bufr2ioda_config import Bufr2iodaConfig +from b2iconverter.bufr2ioda_converter import Bufr2ioda_Converter +from b2iconverter.ioda_variables import IODAVariables +from b2iconverter.ioda_metadata import IODAMetadata +from b2iconverter.ioda_addl_vars import IODAAdditionalVariables +from wmo_codes import * + +import numpy as np + + +platform_description = 'RAMA Tropical mooring profiles from mbuoyb: salinity' + + +class RamaConfig(Bufr2iodaConfig): + def ioda_filename(self): + return f"{self.cycle_type}.t{self.hh}z.insitu_profile_rama_saln.{self.cycle_datetime}.nc" + + +class RamaIODAVariables(IODAVariables): + def __init__(self): + self.construct() + self.metadata = IODAMetadata() + self.additional_vars = RamaAdditionalVariables(self) + + def build_query(self): + q = super().build_query() + q.add('latitude', '*/CLATH') + q.add('longitude', '*/CLONH') + q.add('stationID', '*/RPID') + q.add('depth', '*/IDMSMDBS/BBYSTSL/DBSS') + q.add('saln', '*/IDMSMDBS/BBYSTSL/SALN') + return q + + def set_obs_from_query_result(self, r): + self.saln = r.get('saln', group_by='depth') + + def filter(self): + super().filter() + mask = [True if int(rpid) in RAMA else False for rpid in self.metadata.stationID] + mask = mask & self.SalinityFilter() + self.metadata.filter(mask) + self.saln = self.saln[mask] + + def write_to_ioda_file(self, obsspace): + self.metadata.write_to_ioda_file(obsspace) + self.additional_vars.write_to_ioda_file(obsspace) + self.write_obs_value_s(obsspace) + + def log_obs(self, logger): + self.log_salinity(logger) + + +class RamaAdditionalVariables(IODAAdditionalVariables): + def construct(self): + n = len(self.ioda_vars.metadata.lon) + self.PreQC = (np.ma.masked_array(np.full(n, 0))).astype(np.int32) + self.ObsError_saln = \ + np.float32(np.ma.masked_array(np.full(n, self.ioda_vars.S_error))) + self.compute_ocean_basin() + + def write_to_ioda_file(self, obsspace): + self.write_preqc(obsspace, self.ioda_vars.S_name) + self.write_obs_errorS(obsspace) + self.write_ocean_basin(obsspace) + + def log(self, logger): + self.log_preqc(logger) + self.log_obs_error_saln(logger) + self.log_ocean_basin(logger) + + +if __name__ == '__main__': + + script_name, config_file, log_file, test_file = parse_arguments() + + bufr2ioda_config = RamaConfig( + script_name, + config_file, + platform_description) + + ioda_vars = RamaIODAVariables() + ioda_vars.set_salinity_var_name("salinity") + ioda_vars.set_salinity_error(0.01) + + tropical = Bufr2ioda_Converter(bufr2ioda_config, ioda_vars, log_file) + + tropical.run() + + if test_file: + result = tropical.test(test_file) + sys.exit(result) diff --git a/utils/b2i/bufr2ioda_insitu_profile_rama_temp.py b/utils/b2i/bufr2ioda_insitu_profile_rama_temp.py new file mode 100755 index 00000000..a9762231 --- /dev/null +++ b/utils/b2i/bufr2ioda_insitu_profile_rama_temp.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +import sys +from b2iconverter.util import parse_arguments +from b2iconverter.bufr2ioda_config import Bufr2iodaConfig +from b2iconverter.bufr2ioda_converter import Bufr2ioda_Converter +from b2iconverter.ioda_variables import IODAVariables +from b2iconverter.ioda_metadata import IODAMetadata +from b2iconverter.ioda_addl_vars import IODAAdditionalVariables +from wmo_codes import * + +import numpy as np + + +platform_description = 'RAMA Tropical mooring profiles from mbuoyb: temperature' + + +class RamaConfig(Bufr2iodaConfig): + def ioda_filename(self): + return f"{self.cycle_type}.t{self.hh}z.insitu_profile_rama_temp.{self.cycle_datetime}.nc" + + +class RamaIODAVariables(IODAVariables): + def __init__(self): + self.construct() + self.metadata = IODAMetadata() + self.additional_vars = RamaAdditionalVariables(self) + + def build_query(self): + q = super().build_query() + q.add('latitude', '*/CLATH') + q.add('longitude', '*/CLONH') + q.add('stationID', '*/RPID') + q.add('depth', '*/IDMSMDBS/BBYSTSL/DBSS') + q.add('temp', '*/IDMSMDBS/BBYSTSL/SST1') + return q + + def set_obs_from_query_result(self, r): + self.temp = r.get('temp', group_by='depth') + self.temp -= 273.15 + + def filter(self): + super().filter() + mask = [True if int(rpid) in RAMA else False for rpid in self.metadata.stationID] + mask = mask & self.TemperatureFilter() + self.metadata.filter(mask) + self.temp = self.temp[mask] + + def write_to_ioda_file(self, obsspace): + self.metadata.write_to_ioda_file(obsspace) + self.additional_vars.write_to_ioda_file(obsspace) + self.write_obs_value_t(obsspace) + + def log_obs(self, logger): + self.log_temperature(logger) + + +class RamaAdditionalVariables(IODAAdditionalVariables): + def construct(self): + n = len(self.ioda_vars.metadata.lon) + self.PreQC = (np.ma.masked_array(np.full(n, 0))).astype(np.int32) + self.ObsError_temp = \ + np.float32(np.ma.masked_array(np.full(n, self.ioda_vars.T_error))) + self.compute_ocean_basin() + + def write_to_ioda_file(self, obsspace): + self.write_preqc(obsspace, self.ioda_vars.T_name) + self.write_obs_errorT(obsspace) + self.write_ocean_basin(obsspace) + + def log(self, logger): + self.log_preqc(logger) + self.log_obs_error_temp(logger) + self.log_ocean_basin(logger) + + +if __name__ == '__main__': + + script_name, config_file, log_file, test_file = parse_arguments() + + bufr2ioda_config = RamaConfig( + script_name, + config_file, + platform_description) + + ioda_vars = RamaIODAVariables() + ioda_vars.set_temperature_var_name("waterTemperature") + ioda_vars.set_temperature_error(0.02) + + tropical = Bufr2ioda_Converter(bufr2ioda_config, ioda_vars, log_file) + + tropical.run() + + if test_file: + result = tropical.test(test_file) + sys.exit(result) diff --git a/utils/b2i/bufr2ioda_insitu_profile_taotriton.py b/utils/b2i/bufr2ioda_insitu_profile_taotriton.py deleted file mode 100755 index 69c8d244..00000000 --- a/utils/b2i/bufr2ioda_insitu_profile_taotriton.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 - -import sys -from b2iconverter.util import parse_arguments -from b2iconverter.bufr2ioda_config import Bufr2iodaConfig -from b2iconverter.bufr2ioda_converter import Bufr2ioda_Converter -from mbuoyb_tropical_ioda_variables import MbuoybTropicalIODAVariables -from wmo_codes import * - - -platform_description = 'TAO/TRITON Tropical mooring profiles from mbuoyb: temperature and salinity' - - -class TaotritonConfig(Bufr2iodaConfig): - def ioda_filename(self): - return f"{self.cycle_type}.t{self.hh}z.insitu_profile_taotriton.{self.cycle_datetime}.nc" - - -class TaotritonIODAVariables(MbuoybTropicalIODAVariables): - def filter(self): - super().filter() - mask = [True if int(rpid) in TAO_TRITON else False for rpid in self.metadata.stationID] - self.metadata.filter(mask) - self.temp = self.temp[mask] - self.saln = self.saln[mask] - - -if __name__ == '__main__': - - script_name, config_file, log_file, test_file = parse_arguments() - - bufr2ioda_config = TaotritonConfig( - script_name, - config_file, - platform_description) - - ioda_vars = TaotritonIODAVariables() - ioda_vars.set_temperature_var_name("waterTemperature") - ioda_vars.set_temperature_error(0.02) - ioda_vars.set_salinity_var_name("salinity") - ioda_vars.set_salinity_error(0.01) - - tropical = Bufr2ioda_Converter(bufr2ioda_config, ioda_vars, log_file) - - tropical.run() - - if test_file: - result = tropical.test(test_file) - sys.exit(result) diff --git a/utils/b2i/bufr2ioda_insitu_profile_taotriton_saln.py b/utils/b2i/bufr2ioda_insitu_profile_taotriton_saln.py new file mode 100755 index 00000000..b11acd30 --- /dev/null +++ b/utils/b2i/bufr2ioda_insitu_profile_taotriton_saln.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 + +import sys +from b2iconverter.util import parse_arguments +from b2iconverter.bufr2ioda_config import Bufr2iodaConfig +from b2iconverter.bufr2ioda_converter import Bufr2ioda_Converter +from b2iconverter.ioda_variables import IODAVariables +from b2iconverter.ioda_metadata import IODAMetadata +from b2iconverter.ioda_addl_vars import IODAAdditionalVariables +from wmo_codes import * + +import numpy as np + + +platform_description = 'TAO/TRITON Tropical mooring profiles from mbuoyb: salinity' + + +class TaotritonConfig(Bufr2iodaConfig): + def ioda_filename(self): + return f"{self.cycle_type}.t{self.hh}z.insitu_profile_taotriton_saln.{self.cycle_datetime}.nc" + + +class TaotritonIODAVariables(IODAVariables): + def __init__(self): + self.construct() + self.metadata = IODAMetadata() + self.additional_vars = TaotritonAdditionalVariables(self) + + def build_query(self): + q = super().build_query() + q.add('latitude', '*/CLATH') + q.add('longitude', '*/CLONH') + q.add('stationID', '*/RPID') + q.add('depth', '*/IDMSMDBS/BBYSTSL/DBSS') + q.add('saln', '*/IDMSMDBS/BBYSTSL/SALN') + return q + + def set_obs_from_query_result(self, r): + self.saln = r.get('saln', group_by='depth') + + def filter(self): + super().filter() + mask = [True if int(rpid) in TAO_TRITON else False for rpid in self.metadata.stationID] + mask = mask & self.SalinityFilter() + self.metadata.filter(mask) + self.saln = self.saln[mask] + + def write_to_ioda_file(self, obsspace): + self.metadata.write_to_ioda_file(obsspace) + self.additional_vars.write_to_ioda_file(obsspace) + self.write_obs_value_s(obsspace) + + def log_obs(self, logger): + self.log_salinity(logger) + + +class TaotritonAdditionalVariables(IODAAdditionalVariables): + def construct(self): + n = len(self.ioda_vars.metadata.lon) + self.PreQC = (np.ma.masked_array(np.full(n, 0))).astype(np.int32) + self.ObsError_saln = \ + np.float32(np.ma.masked_array(np.full(n, self.ioda_vars.S_error))) + self.compute_ocean_basin() + + def write_to_ioda_file(self, obsspace): + self.write_preqc(obsspace, self.ioda_vars.S_name) + self.write_obs_errorS(obsspace) + self.write_ocean_basin(obsspace) + + def log(self, logger): + self.log_preqc(logger) + self.log_obs_error_saln(logger) + self.log_ocean_basin(logger) + + +if __name__ == '__main__': + + script_name, config_file, log_file, test_file = parse_arguments() + + bufr2ioda_config = TaotritonConfig( + script_name, + config_file, + platform_description) + + ioda_vars = TaotritonIODAVariables() + ioda_vars.set_salinity_var_name("salinity") + ioda_vars.set_salinity_error(0.01) + + tropical = Bufr2ioda_Converter(bufr2ioda_config, ioda_vars, log_file) + + tropical.run() + + if test_file: + result = tropical.test(test_file) + sys.exit(result) diff --git a/utils/b2i/bufr2ioda_insitu_profile_taotriton_temp.py b/utils/b2i/bufr2ioda_insitu_profile_taotriton_temp.py new file mode 100755 index 00000000..29b36f50 --- /dev/null +++ b/utils/b2i/bufr2ioda_insitu_profile_taotriton_temp.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +import sys +from b2iconverter.util import parse_arguments +from b2iconverter.bufr2ioda_config import Bufr2iodaConfig +from b2iconverter.bufr2ioda_converter import Bufr2ioda_Converter +from b2iconverter.ioda_variables import IODAVariables +from b2iconverter.ioda_metadata import IODAMetadata +from b2iconverter.ioda_addl_vars import IODAAdditionalVariables +from wmo_codes import * + +import numpy as np + + +platform_description = 'TAO/TRITON Tropical mooring profiles from mbuoyb: temperature' + + +class TaotritonConfig(Bufr2iodaConfig): + def ioda_filename(self): + return f"{self.cycle_type}.t{self.hh}z.insitu_profile_taotriton_temp.{self.cycle_datetime}.nc" + + +class TaotritonIODAVariables(IODAVariables): + def __init__(self): + self.construct() + self.metadata = IODAMetadata() + self.additional_vars = TaotritonAdditionalVariables(self) + + def build_query(self): + q = super().build_query() + q.add('latitude', '*/CLATH') + q.add('longitude', '*/CLONH') + q.add('stationID', '*/RPID') + q.add('depth', '*/IDMSMDBS/BBYSTSL/DBSS') + q.add('temp', '*/IDMSMDBS/BBYSTSL/SST1') + return q + + def set_obs_from_query_result(self, r): + self.temp = r.get('temp', group_by='depth') + self.temp -= 273.15 + + def filter(self): + super().filter() + mask = [True if int(rpid) in TAO_TRITON else False for rpid in self.metadata.stationID] + mask = mask & self.TemperatureFilter() + self.metadata.filter(mask) + self.temp = self.temp[mask] + + def write_to_ioda_file(self, obsspace): + self.metadata.write_to_ioda_file(obsspace) + self.additional_vars.write_to_ioda_file(obsspace) + self.write_obs_value_t(obsspace) + + def log_obs(self, logger): + self.log_temperature(logger) + + +class TaotritonAdditionalVariables(IODAAdditionalVariables): + def construct(self): + n = len(self.ioda_vars.metadata.lon) + self.PreQC = (np.ma.masked_array(np.full(n, 0))).astype(np.int32) + self.ObsError_temp = \ + np.float32(np.ma.masked_array(np.full(n, self.ioda_vars.T_error))) + self.compute_ocean_basin() + + def write_to_ioda_file(self, obsspace): + self.write_preqc(obsspace, self.ioda_vars.T_name) + self.write_obs_errorT(obsspace) + self.write_ocean_basin(obsspace) + + def log(self, logger): + self.log_preqc(logger) + self.log_obs_error_temp(logger) + self.log_ocean_basin(logger) + + +if __name__ == '__main__': + + script_name, config_file, log_file, test_file = parse_arguments() + + bufr2ioda_config = TaotritonConfig( + script_name, + config_file, + platform_description) + + ioda_vars = TaotritonIODAVariables() + ioda_vars.set_temperature_var_name("waterTemperature") + ioda_vars.set_temperature_error(0.02) + + tropical = Bufr2ioda_Converter(bufr2ioda_config, ioda_vars, log_file) + + tropical.run() + + if test_file: + result = tropical.test(test_file) + sys.exit(result) diff --git a/utils/b2i/mbuoyb_tropical_ioda_variables.py b/utils/b2i/mbuoyb_tropical_ioda_variables.py deleted file mode 100644 index fdd3a65b..00000000 --- a/utils/b2i/mbuoyb_tropical_ioda_variables.py +++ /dev/null @@ -1,27 +0,0 @@ -import numpy as np -import bufr -from b2iconverter.ioda_variables import IODAVariables -from b2iconverter.ioda_metadata import IODAMetadata -from b2iconverter.ioda_addl_vars import IODAAdditionalVariables -from b2iconverter.util import log_variable, compute_hash -from wmo_codes import * - - -class MbuoybTropicalIODAVariables(IODAVariables): - - def build_query(self): - q = super().build_query() - q.add('latitude', '*/CLATH') - q.add('longitude', '*/CLONH') - q.add('stationID', '*/RPID') - q.add('depth', '*/IDMSMDBS/BBYSTSL/DBSS') - q.add('temp', '*/IDMSMDBS/BBYSTSL/SST1') - q.add('saln', '*/IDMSMDBS/BBYSTSL/SALN') - return q - - def filter(self): - super().filter() - mask = self.TemperatureFilter() & self.SalinityFilter() - self.metadata.filter(mask) - self.temp = self.temp[mask] - self.saln = self.saln[mask] diff --git a/utils/b2i/test/CMakeLists.txt b/utils/b2i/test/CMakeLists.txt index cf3f24e0..2ee7090a 100644 --- a/utils/b2i/test/CMakeLists.txt +++ b/utils/b2i/test/CMakeLists.txt @@ -3,35 +3,55 @@ find_package(Python REQUIRED) string(REGEX REPLACE "^([0-9]+\\.[0-9]+).*" "\\1" PYTHON_MAJOR_MINOR ${Python_VERSION}) set(PYIODACONV_DIR "${CMAKE_BINARY_DIR}/lib/python${PYTHON_MAJOR_MINOR}:${CMAKE_BINARY_DIR}/lib/python${PYTHON_MAJOR_MINOR}/site-packages") -# Define test input YAML files -list(APPEND b2i_test_input - testinput/bufr2ioda_insitu_profile_argo_2021063006.yaml.in - testinput/bufr2ioda_insitu_profile_bathy_2021063006.yaml.in - testinput/bufr2ioda_insitu_profile_glider_2021063006.yaml.in - testinput/bufr2ioda_insitu_profile_tesac_2021063006.yaml.in - testinput/bufr2ioda_insitu_profile_xbtctd_2021063006.yaml.in - testinput/bufr2ioda_insitu_surface_trkob_2021063006.yaml.in - testinput/bufr2ioda_insitu_profile_taotriton_2025061900.yaml.in - testinput/bufr2ioda_insitu_profile_pirata_2025061900.yaml.in - testinput/bufr2ioda_insitu_profile_rama_2025061900.yaml.in - testinput/bufr2ioda_insitu_surface_ndbc_2025061900.yaml.in - testinput/bufr2ioda_insitu_surface_dbuoyb_drifter_2019010700.yaml.in +set(B2I_TEST_TABLE + profile argo subpfl 2021063006 06 + profile glider subpfl 2021063006 06 + + profile bathy bathy 2021063006 06 + profile tesac tesac 2021063006 06 + profile xbtctd xbtctd 2021063006 06 + + profile taotriton_temp mbuoyb 2025061900 00 + profile taotriton_saln mbuoyb 2025061900 00 + profile pirata_temp mbuoyb 2025061900 00 + profile pirata_saln mbuoyb 2025061900 00 + profile rama_temp mbuoyb 2025061900 00 + profile rama_saln mbuoyb 2025061900 00 + surface ndbc mbuoyb 2025061900 00 + + surface trkob trkob 2021063006 06 + surface dbuoyb_drifter dbuoyb 2019010700 00 ) -# Define reference output files -list(APPEND b2i_test_ref - testref/bufr2ioda_insitu_profile_argo_2021063006.ref - testref/bufr2ioda_insitu_profile_bathy_2021063006.ref - testref/bufr2ioda_insitu_profile_glider_2021063006.ref - testref/bufr2ioda_insitu_profile_tesac_2021063006.ref - testref/bufr2ioda_insitu_profile_xbtctd_2021063006.ref - testref/bufr2ioda_insitu_surface_trkob_2021063006.ref - testref/bufr2ioda_insitu_profile_taotriton_2025061900.ref - testref/bufr2ioda_insitu_profile_pirata_2025061900.ref - testref/bufr2ioda_insitu_profile_rama_2025061900.ref - testref/bufr2ioda_insitu_surface_ndbc_2025061900.ref - testref/bufr2ioda_insitu_surface_dbuoyb_drifter_2019010700.ref -) +set(b2i_test_input "") +set(b2i_test_ref "") + +list(LENGTH B2I_TEST_TABLE TABLE_LEN) + +math(EXPR NROWS "${TABLE_LEN} / 5") +math(EXPR TEST_TABLE_LAST_ROW "${NROWS} - 1") + +foreach(i RANGE 0 ${TEST_TABLE_LAST_ROW}) + math(EXPR idx0 "5*${i}") + math(EXPR idx1 "5*${i} + 1") + math(EXPR idx2 "5*${i} + 2") + math(EXPR idx3 "5*${i} + 3") + + list(GET B2I_TEST_TABLE ${idx0} category) + list(GET B2I_TEST_TABLE ${idx1} name) + list(GET B2I_TEST_TABLE ${idx2} bufr) + list(GET B2I_TEST_TABLE ${idx3} date) + + set(test "bufr2ioda_insitu_${category}_${name}_${date}") + + list(APPEND b2i_test_input + "testinput/${test}.yaml.in" + ) + + list(APPEND b2i_test_ref + "testref/${test}.ref" + ) +endforeach() # Set up b2i-specific test directories set(B2I_TEST_DIR ${CMAKE_CURRENT_BINARY_DIR}/b2i/test) @@ -134,68 +154,63 @@ else() endif() -# Function to add b2i tests -function(ADD_B2I_TEST testname testbufr) - # set(CONFIG_TYPE "json") - set(CONFIG_TYPE "yaml") - - if (testbufr STREQUAL "mbuoyb") - set(DATE "2025061900") - set(CYCLE "00") - else() - if (testbufr STREQUAL "dbuoyb") - set(DATE "2019010700") - set(CYCLE "00") - else() - set(DATE "2021063006") - set(CYCLE "06") - endif() - endif() +function(ADD_B2I_TEST category name bufr date cycle) + + set(TEST "bufr2ioda_insitu_${category}_${name}") + set(TESTREF_FILE "${TEST}_${date}.ref") - set(TEST "bufr2ioda_insitu_${testname}") - set(TESTREF_FILE "${TEST}_${DATE}.ref") set(BUFR_INPUT_DIR ${B2I_TEST_DIR}) set(IODA_OUTPUT_DIR ${B2I_TESTOUTPUT_DIR}) - set(BUFR_TEST_FILE "${DATE}-gdas.t${CYCLE}z.${testbufr}.tm00.bufr_d") + + set(BUFR_TEST_FILE "${date}-gdas.t${cycle}z.${bufr}.tm00.bufr_d") set(BUFR_FILE "${BUFR_TEST_DIR}/${BUFR_TEST_FILE}") if(NOT EXISTS ${BUFR_FILE}) message(WARNING "BUFR file ${BUFR_FILE} not found; skipping test ${TEST}") return() endif() + file(COPY ${BUFR_FILE} DESTINATION ${BUFR_INPUT_DIR}) - set(CONFIG_FILE_NAME "${TEST}_${DATE}.${CONFIG_TYPE}") - set(CONFIG_FILE_IN "${B2I_TESTINPUT_DIR}/${TEST}_${DATE}.${CONFIG_TYPE}.in") + set(CONFIG_FILE_NAME "${TEST}_${date}.yaml") + set(CONFIG_FILE_IN "${B2I_TESTINPUT_DIR}/${TEST}_${date}.yaml.in") set(CONFIG_FILE "${B2I_TEST_DIR}/${CONFIG_FILE_NAME}") + CREATE_CONFIG_FILE( - ${CONFIG_FILE_IN} - ${CONFIG_FILE} - ${BUFR_INPUT_DIR} - ${IODA_OUTPUT_DIR} - ${OCEAN_BASIN_FILE} - ) + ${CONFIG_FILE_IN} + ${CONFIG_FILE} + ${BUFR_INPUT_DIR} + ${IODA_OUTPUT_DIR} + ${OCEAN_BASIN_FILE} + ) add_test(NAME test_b2i_${TEST} - COMMAND ${B2I_SCRIPT_DIR}/${TEST}.py -c ${CONFIG_FILE} -t ${B2I_TESTREF_DIR}/${TESTREF_FILE} - WORKING_DIRECTORY ${B2I_TEST_DIR}) + COMMAND ${B2I_SCRIPT_DIR}/${TEST}.py + -c ${CONFIG_FILE} + -t ${B2I_TESTREF_DIR}/${TESTREF_FILE} + WORKING_DIRECTORY ${B2I_TEST_DIR}) + set_property(TEST test_b2i_${TEST} - APPEND PROPERTY - ENVIRONMENT "PYTHONPATH=${PYIODACONV_DIR}:$ENV{PYTHONPATH}") - # message("Created test test_b2i_${TEST}") + APPEND PROPERTY + ENVIRONMENT "PYTHONPATH=${PYIODACONV_DIR}:$ENV{PYTHONPATH}") endfunction() -# Add tests if prerequisites are met if(GENERATE_B2I_TESTS) - ADD_B2I_TEST("profile_argo" "subpfl") - ADD_B2I_TEST("profile_bathy" "bathy") - ADD_B2I_TEST("profile_glider" "subpfl") - ADD_B2I_TEST("profile_tesac" "tesac") - ADD_B2I_TEST("profile_rama" "mbuoyb") - ADD_B2I_TEST("profile_pirata" "mbuoyb") - ADD_B2I_TEST("profile_taotriton" "mbuoyb") - ADD_B2I_TEST("profile_xbtctd" "xbtctd") - ADD_B2I_TEST("surface_ndbc" "mbuoyb") - ADD_B2I_TEST("surface_trkob" "trkob") - ADD_B2I_TEST("surface_dbuoyb_drifter" "dbuoyb") + + foreach(i RANGE 0 ${TEST_TABLE_LAST_ROW}) + math(EXPR idx0 "5*${i}") + math(EXPR idx1 "5*${i} + 1") + math(EXPR idx2 "5*${i} + 2") + math(EXPR idx3 "5*${i} + 3") + math(EXPR idx4 "5*${i} + 4") + + list(GET B2I_TEST_TABLE ${idx0} category) + list(GET B2I_TEST_TABLE ${idx1} name) + list(GET B2I_TEST_TABLE ${idx2} bufr) + list(GET B2I_TEST_TABLE ${idx3} date) + list(GET B2I_TEST_TABLE ${idx4} cycle) + + ADD_B2I_TEST(${category} ${name} ${bufr} ${date} ${cycle}) + endforeach() + endif() diff --git a/utils/b2i/test/testinput/bufr2ioda_insitu_profile_pirata_2025061900.yaml.in b/utils/b2i/test/testinput/bufr2ioda_insitu_profile_pirata_saln_2025061900.yaml.in similarity index 76% rename from utils/b2i/test/testinput/bufr2ioda_insitu_profile_pirata_2025061900.yaml.in rename to utils/b2i/test/testinput/bufr2ioda_insitu_profile_pirata_saln_2025061900.yaml.in index 8cd0df83..e16def55 100644 --- a/utils/b2i/test/testinput/bufr2ioda_insitu_profile_pirata_2025061900.yaml.in +++ b/utils/b2i/test/testinput/bufr2ioda_insitu_profile_pirata_saln_2025061900.yaml.in @@ -8,5 +8,5 @@ cycle_datetime: '2025061900' dump_directory: __BUFRINPUTDIR__ ioda_directory: __IODAOUTPUTDIR__ ocean_basin: __OCEANBASIN__ -data_description: 6-hrly in situ PIRATA tropical mooring profiles +data_description: 6-hrly in situ PIRATA tropical mooring salinity profiles data_provider: U.S. NOAA diff --git a/utils/b2i/test/testinput/bufr2ioda_insitu_profile_pirata_temp_2025061900.yaml.in b/utils/b2i/test/testinput/bufr2ioda_insitu_profile_pirata_temp_2025061900.yaml.in new file mode 100644 index 00000000..a7964424 --- /dev/null +++ b/utils/b2i/test/testinput/bufr2ioda_insitu_profile_pirata_temp_2025061900.yaml.in @@ -0,0 +1,12 @@ +--- +data_format: mbuoyb +subsets: mbuoyb +source: NCEP data tank +data_type: pirata +cycle_type: gdas +cycle_datetime: '2025061900' +dump_directory: __BUFRINPUTDIR__ +ioda_directory: __IODAOUTPUTDIR__ +ocean_basin: __OCEANBASIN__ +data_description: 6-hrly in situ PIRATA tropical mooring temperature profiles +data_provider: U.S. NOAA diff --git a/utils/b2i/test/testinput/bufr2ioda_insitu_profile_rama_2025061900.yaml.in b/utils/b2i/test/testinput/bufr2ioda_insitu_profile_rama_saln_2025061900.yaml.in similarity index 77% rename from utils/b2i/test/testinput/bufr2ioda_insitu_profile_rama_2025061900.yaml.in rename to utils/b2i/test/testinput/bufr2ioda_insitu_profile_rama_saln_2025061900.yaml.in index 2d4623fd..e0d701ee 100644 --- a/utils/b2i/test/testinput/bufr2ioda_insitu_profile_rama_2025061900.yaml.in +++ b/utils/b2i/test/testinput/bufr2ioda_insitu_profile_rama_saln_2025061900.yaml.in @@ -8,5 +8,5 @@ cycle_datetime: '2025061900' dump_directory: __BUFRINPUTDIR__ ioda_directory: __IODAOUTPUTDIR__ ocean_basin: __OCEANBASIN__ -data_description: 6-hrly in situ RAMA tropical mooring profiles +data_description: 6-hrly in situ RAMA tropical mooring salinity profiles data_provider: U.S. NOAA diff --git a/utils/b2i/test/testinput/bufr2ioda_insitu_profile_rama_temp_2025061900.yaml.in b/utils/b2i/test/testinput/bufr2ioda_insitu_profile_rama_temp_2025061900.yaml.in new file mode 100644 index 00000000..d7e8a0de --- /dev/null +++ b/utils/b2i/test/testinput/bufr2ioda_insitu_profile_rama_temp_2025061900.yaml.in @@ -0,0 +1,12 @@ +--- +data_format: mbuoyb +subsets: mbuoyb +source: NCEP data tank +data_type: rama +cycle_type: gdas +cycle_datetime: '2025061900' +dump_directory: __BUFRINPUTDIR__ +ioda_directory: __IODAOUTPUTDIR__ +ocean_basin: __OCEANBASIN__ +data_description: 6-hrly in situ RAMA tropical mooring temperature profiles +data_provider: U.S. NOAA diff --git a/utils/b2i/test/testinput/bufr2ioda_insitu_profile_taotriton_2025061900.yaml.in b/utils/b2i/test/testinput/bufr2ioda_insitu_profile_taotriton_saln_2025061900.yaml.in similarity index 75% rename from utils/b2i/test/testinput/bufr2ioda_insitu_profile_taotriton_2025061900.yaml.in rename to utils/b2i/test/testinput/bufr2ioda_insitu_profile_taotriton_saln_2025061900.yaml.in index 5ff187ed..600b0f1f 100644 --- a/utils/b2i/test/testinput/bufr2ioda_insitu_profile_taotriton_2025061900.yaml.in +++ b/utils/b2i/test/testinput/bufr2ioda_insitu_profile_taotriton_saln_2025061900.yaml.in @@ -8,5 +8,5 @@ cycle_datetime: '2025061900' dump_directory: __BUFRINPUTDIR__ ioda_directory: __IODAOUTPUTDIR__ ocean_basin: __OCEANBASIN__ -data_description: 6-hrly in situ TAO/TRITON tropical mooring profiles +data_description: 6-hrly in situ TAO/TRITON tropical mooring salinity profiles data_provider: U.S. NOAA diff --git a/utils/b2i/test/testinput/bufr2ioda_insitu_profile_taotriton_temp_2025061900.yaml.in b/utils/b2i/test/testinput/bufr2ioda_insitu_profile_taotriton_temp_2025061900.yaml.in new file mode 100644 index 00000000..a32d1ac8 --- /dev/null +++ b/utils/b2i/test/testinput/bufr2ioda_insitu_profile_taotriton_temp_2025061900.yaml.in @@ -0,0 +1,12 @@ +--- +data_format: mbuoyb +subsets: mbuoyb +source: NCEP data tank +data_type: taotriton +cycle_type: gdas +cycle_datetime: '2025061900' +dump_directory: __BUFRINPUTDIR__ +ioda_directory: __IODAOUTPUTDIR__ +ocean_basin: __OCEANBASIN__ +data_description: 6-hrly in situ TAO/TRITON tropical mooring temperature profiles +data_provider: U.S. NOAA diff --git a/utils/b2i/test/testref/bufr2ioda_insitu_profile_pirata_2025061900.ref b/utils/b2i/test/testref/bufr2ioda_insitu_profile_pirata_saln_2025061900.ref similarity index 77% rename from utils/b2i/test/testref/bufr2ioda_insitu_profile_pirata_2025061900.ref rename to utils/b2i/test/testref/bufr2ioda_insitu_profile_pirata_saln_2025061900.ref index dae2720f..af90c5e9 100644 --- a/utils/b2i/test/testref/bufr2ioda_insitu_profile_pirata_2025061900.ref +++ b/utils/b2i/test/testref/bufr2ioda_insitu_profile_pirata_saln_2025061900.ref @@ -10,14 +10,9 @@ depth: 178, float32 min, max = 1.0, 120.0 depth hash = 2a58150c94fc54ae394c7365f0b22daa03ea83f79e395f74c07a666feb0e750c stationID: 178,