converters for marine da

dump/mapping/b2i_marine/CMakeLists.txt

@@ -0,0 +1,197 @@
+# Copy the bkg files
+# ------------------
+set(TESTDATA ${PROJECT_BINARY_DIR}/test/testdata )
+
+
+# Symlink test input yaml files
+# -----------------------------
+# create testinput dir
+set (TESTINPUT_DIR ${PROJECT_BINARY_DIR}/test/marine/testinput)
+
+file(MAKE_DIRECTORY ${TESTINPUT_DIR})
+
+# symlink
+foreach(FILENAME ${test_input})
+	get_filename_component(filename ${FILENAME} NAME)
+	execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink
+		${FILENAME}
+		${TESTINPUT_DIR}
+	)
+endforeach(FILENAME)
+
+# install
+install(FILES ${test_input}
+        DESTINATION "test/testinput/")
+
+
+###########################################################################
+# bufr to ioda tests:
+###########################################################################
+
+find_package(Python REQUIRED)
+# Extract the major and minor version (e.g., "3.10" from "3.10.13")
+string(REGEX REPLACE "^([0-9]+\\.[0-9]+).*" "\\1" PYTHON_MAJOR_MINOR ${Python_VERSION})
+set(PYIODACONV_DIR "${PROJECT_SOURCE_DIR}/build/lib/python${PYTHON_MAJOR_MINOR}/")
+
+set(TEST_WORKING_DIR ${PROJECT_BINARY_DIR}/test/marine)
+set(MARINE_BUFR2IODA_DIR ${PROJECT_SOURCE_DIR}/ush/ioda/bufr2ioda/marine)
+# set(MARINE_BUFR2IODA_DIR ${MARINE_BUFR2IODA_DIR}/newb2i)
+set(MARINE_BUFR2IODA_DIR ${MARINE_BUFR2IODA_DIR}/b2iron)
+set(CONFIG_DIR ${PROJECT_SOURCE_DIR}/test/marine/testinput)
+set(TESTREF_DIR ${PROJECT_SOURCE_DIR}/test/marine/testref)
+
+
+# prepare a test.yaml file from test.yaml.in by replacing
+# placeholder patterns 
+# __BUFRINPUTDIR__ 
+# __IODAOUTPUTDIR__ 
+# __OCEANBASIN__
+# __CONFIGDIR__
+# with actual directory paths
+function(CREATE_CONFIG_FILE 
+	test_config_in 
+	test_config_out 
+	bufr_input_dir 
+	ioda_output_dir
+	ocean_basin_file
+)
+	file(READ "${test_config_in}" file_content)
+	string(REPLACE "__BUFRINPUTDIR__" "\"${bufr_input_dir}\"" temp_content "${file_content}")
+	string(REPLACE "__IODAOUTPUTDIR__" "\"${ioda_output_dir}\"" temp_content2 "${temp_content}")
+	string(REPLACE "__OCEANBASIN__" "\"${ocean_basin_file}\"" temp_content3 "${temp_content2}")
+	string(REPLACE "__CONFIGDIR__" ${CONFIG_DIR} temp_content4 "${temp_content3}")
+	file(WRITE "${test_config_out}" "${temp_content4}")
+endfunction()
+
+
+function(CHECK_AND_SET_PATH PATH1 PATH2 RESULT_VAR)
+    # Check if PATH1 exists
+    if(EXISTS ${PATH1})
+        set(${RESULT_VAR} ${PATH1} PARENT_SCOPE)
+        set(${RESULT_VAR}_EXISTS TRUE PARENT_SCOPE)
+        return()
+    endif()
+
+    # Check if PATH2 exists
+    if(EXISTS ${PATH2})
+        set(${RESULT_VAR} ${PATH2} PARENT_SCOPE)
+        set(${RESULT_VAR}_EXISTS TRUE PARENT_SCOPE)
+        return()
+    endif()
+
+    # If neither path exists
+    set(${RESULT_VAR} "" PARENT_SCOPE)
+    set(${RESULT_VAR}_EXISTS FALSE PARENT_SCOPE)
+endfunction()
+
+
+# find the input files
+set(BUFR_TEST_DIR_ORION
+	"/work/noaa/da/marineda/gfs-marine/data/obs/ci/bufr"
+)
+set(BUFR_TEST_DIR_HERA
+	"/scratch1/NCEPDEV/da/common/ci/bufr"
+)
+CHECK_AND_SET_PATH(
+	${BUFR_TEST_DIR_ORION} 
+	${BUFR_TEST_DIR_HERA} 
+	BUFR_TEST_DIR
+)
+if (NOT BUFR_TEST_DIR_EXISTS)
+	message(WARNING "BUFR test file directory not found -- bufr to ioda tests not generated.")
+	set(GENERATE_BUFR2IODA_TESTS FALSE)
+else()
+    # message(STATUS "Found bufr test directory: ${BUFR_TEST_DIR}")
+
+	set(OCEAN_BASIN_FILE_HERA
+		"/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/common/RECCAP2_region_masks_all_v20221025.nc"
+	)
+	set(OCEAN_BASIN_FILE_ORION
+		"/work/noaa/global/glopara/fix/gdas/soca/20240802/common/RECCAP2_region_masks_all_v20221025.nc"
+	)
+	CHECK_AND_SET_PATH(
+		${OCEAN_BASIN_FILE_ORION} 
+		${OCEAN_BASIN_FILE_HERA} 
+		OCEAN_BASIN_FILE
+	)
+	if (NOT OCEAN_BASIN_FILE_EXISTS)
+		message(WARNING "Ocean basin data file not found -- bufr to ioda tests not generated.")
+		set(GENERATE_BUFR2IODA_TESTS FALSE)
+	endif()
+	# message(STATUS "Found ocean basin data in ${OCEAN_BASIN_FILE}")
+	set(GENERATE_BUFR2IODA_TESTS TRUE)
+endif()
+
+
+function(ADD_INSITU_TEST testname testbufr)
+	# set(CONFIG_TYPE "json")
+	set(CONFIG_TYPE "yaml")
+
+	set(DATE "2019010700")
+	set(CYCLE "00")
+	# if (testbufr STREQUAL "dbuoy")
+		# set(DATE "2019010700")
+		# set(CYCLE "00")
+	# else()
+		# set(DATE "2021063006")
+		# set(CYCLE "06")
+	# endif()
+
+	set(TESTEXEC "bufr2ioda_insitu_${testname}.py")
+	set(TEST "bufr2ioda_insitu_${testname}_${testbufr}")
+
+	set(TESTREF_FILE "${TEST}_${DATE}.ref")
+
+	# stage the input file to directory ${BUFR_INPUT_DIR}
+	set(BUFR_INPUT_DIR ${TEST_WORKING_DIR})
+	set(BUFR_TEST_FILE "${DATE}-gdas.t${CYCLE}z.${testbufr}.tm00.bufr_d")
+	set(BUFR_FILE "${BUFR_TEST_DIR}/${BUFR_TEST_FILE}")
+	if (NOT EXISTS ${BUFR_FILE})
+		message(WARNING "BUFR file ${BUFR_FILE} not found, test not generated")
+		return()
+	endif()
+	file(COPY ${BUFR_FILE} DESTINATION ${BUFR_INPUT_DIR})
+
+	# stage the config file
+	set(CONFIG_FILE_NAME ${TEST}_${DATE}.${CONFIG_TYPE})
+	set(CONFIG_FILE_IN "${CONFIG_DIR}/${CONFIG_FILE_NAME}.in")
+	set(CONFIG_FILE "${TEST_WORKING_DIR}/${CONFIG_FILE_NAME}")
+	set(IODA_OUTPUT_DIR ${TEST_WORKING_DIR})
+	CREATE_CONFIG_FILE(
+		${CONFIG_FILE_IN}
+		${CONFIG_FILE}
+		${BUFR_INPUT_DIR}
+		${IODA_OUTPUT_DIR}
+		${OCEAN_BASIN_FILE}
+	)
+
+	add_test(
+		NAME test_gdasapp_${TEST}
+		COMMAND ${MARINE_BUFR2IODA_DIR}/${TESTEXEC} -c ${CONFIG_FILE} -t ${TESTREF_DIR}/${TESTREF_FILE}
+		WORKING_DIRECTORY ${TEST_WORKING_DIR}
+	)
+	set_property(
+		TEST test_gdasapp_${TEST}
+		APPEND PROPERTY 
+			ENVIRONMENT "PYTHONPATH=${PYIODACONV_DIR}:$ENV{PYTHONPATH}"
+	)
+endfunction()
+
+
+
+if (GENERATE_BUFR2IODA_TESTS)
+	ADD_INSITU_TEST("profile_argo" "subpfl")
+	ADD_INSITU_TEST("profile_bathy" "bathy")
+	ADD_INSITU_TEST("profile_glider" "subpfl")
+	ADD_INSITU_TEST("profile_tesac" "tesac")
+	ADD_INSITU_TEST("profile_tropical" "dbuoy")
+	ADD_INSITU_TEST("profile_tropical" "mbuoyb")
+	ADD_INSITU_TEST("profile_xbtctd" "xbtctd")
+    ADD_INSITU_TEST("surface_altkob" "altkob")
+    ADD_INSITU_TEST("surface_cstgd" "cstgd")
+	ADD_INSITU_TEST("surface_drifter" "dbuoy")
+	ADD_INSITU_TEST("surface_drifter" "dbuoyb")
+	ADD_INSITU_TEST("surface_lcman" "lcman")
+    ADD_INSITU_TEST("surface_shipsu" "shipsu")
+	ADD_INSITU_TEST("surface_trkob" "trkob")
+endif()

dump/mapping/b2i_marine/README

@@ -0,0 +1,7 @@
+12 converters, 14 tests
+marine insitu profiles and surface obs
+from gdasapp
+
+each test is run using a converter and 2 configuration yaml files
+for example, a converter can be run to extract tropical moorings from
+either dbuoy or mbouyb

dump/mapping/b2i_marine/b2ibase/b2i.py

@@ -0,0 +1,73 @@
+import sys
+import os
+import tempfile
+from .util import run_diff
+
+
+class B2I:
+    def __init__(self, config, data, logger):
+        self.config = config
+        self.data = data
+        self.logger = logger
+
+    def read_from_bufr(self):
+        map_yaml_file = self.config.data_description_filepath()
+        self.logger.debug(f"read_from_bufr: using mapping file {map_yaml_file}")
+        bufr_file_path = self.config.bufr_filepath()
+        self.logger.debug(f"reading bufr file {bufr_file_path}")
+        self.data.read_from_bufr(bufr_file_path, map_yaml_file)
+        self.logger.debug(f"read from bufr: data size = {self.data.get_data_size()}")
+        if self.data.get_data_size() == 0:
+            self.logger.warning("No data read -- exiting")
+            sys.exit()
+
+    def process_data(self):
+        self.data.process_data()
+        self.logger.debug(f"processed data: data size = {self.data.get_data_size()}")
+        if self.data.get_data_size() == 0:
+            self.logger.warning("No data read -- exiting")
+            sys.exit()
+        ocean_file_path = self.config.ocean_basin_nc_file_path()
+        self.data.add_ocean_basin(ocean_file_path)
+        # print("process_data: variables:")
+        # self.data.print_var_names()
+
+    def write_to_ioda_file(self):
+        map_yaml_file = self.config.data_description_filepath()
+        self.logger.debug(f"write_to_ioda_file: using mapping file {map_yaml_file}")
+        iodafile_path = self.config.ioda_filepath()
+        path, fname = os.path.split(iodafile_path)
+        os.makedirs(path, exist_ok=True)
+        self.logger.debug(f"writing ioda file {iodafile_path}")
+        self.data.write_to_ioda_file(iodafile_path, map_yaml_file, self.config)
+
+    def run(self):
+        self.read_from_bufr()
+        self.process_data()
+        self.write_to_ioda_file()
+        self.data.log(self.logger)
+
+    def test(self, test_file):
+        with tempfile.NamedTemporaryFile(delete=False, suffix='.log') as temp_log_file:
+            temp_log_file_name = temp_log_file.name
+            self.logger.debug(f"TEST: created a temporary log file {temp_log_file_name}")
+
+            self.logger.disable_logging()
+            self.logger.enable_test_file_logging(temp_log_file_name)
+            self.data.log(self.logger)
+            self.logger.disable_test_file_logging()
+            self.logger.enable_logging()
+
+            if os.path.exists(test_file):
+                self.logger.debug(f"TEST: running diff with reference file {test_file}")
+            else:
+                self.logger.error(f"TEST: reference file not found: {test_file}")
+                return 1    # failure
+
+            result = run_diff(temp_log_file_name, test_file, self.logger)
+            if result:
+                self.logger.error(f"TEST ERROR: files are different")
+            else:
+                self.logger.info(f"TEST passed: files are identical")
+
+            return result

dump/mapping/b2i_marine/b2ibase/config.py

@@ -0,0 +1,71 @@
+import json
+import yaml
+import os
+import sys
+
+
+# configuration file can be either json or yaml
+# this config class provides the functions that determine
+# the names and the paths of the bufr input and the ioda output files
+# these functions can be overridden in the converter
+
+class Config:
+    def __init__(self, config_file, logger):
+        self.logger = logger
+        self.logger.debug(f"Reading configuration file {config_file}")
+        _, file_extension = os.path.splitext(config_file)
+        if file_extension == ".json":
+            with open(config_file, "r") as file:
+                config = json.load(file)
+            self.read_config(config)
+        elif file_extension == ".yaml":
+            with open(config_file, "r") as file:
+                config = yaml.safe_load(file)
+            self.read_config(config)
+        else:
+            self.logger.critical(f"Unknown file extension = {file_extension}")
+            sys.exit(1)
+
+    def read_config(self, config):
+        # Get parameters from configuration
+        self.data_format = config["data_format"]
+        self.source = config["source"]
+        self.data_type = config["data_type"]
+        self.data_provider = config["data_provider"]
+        self.cycle_type = config["cycle_type"]
+        self.cycle_datetime = config["cycle_datetime"]
+        self.dump_dir = config["dump_directory"]
+        self.ioda_dir = config["ioda_directory"]
+        self.ocean_basin = config["ocean_basin"]
+        self.data_description = config["data_description"]
+        self.data_description_file = config["data_description_file"]
+        self.platform_description = config["platform_description"]
+
+        self.yyyymmdd = self.cycle_datetime[0:8]
+        self.hh = self.cycle_datetime[8:10]
+
+        # General Information
+        self.converter = 'BUFR to IODA Converter'
+
+    def data_description_filepath(self):
+        return self.data_description_file
+
+    def ocean_basin_nc_file_path(self):
+        return self.ocean_basin
+
+    def bufr_filename(self):
+        return f"{self.cycle_datetime}-{self.cycle_type}.t{self.hh}z.{self.data_format}.tm00.bufr_d"
+
+    def bufr_filepath(self):
+        return os.path.join(self.dump_dir, self.bufr_filename())
+
+    def ioda_filename(self):
+        return f"{self.cycle_type}.t{self.hh}z.insitu_profile_{self.data_format}.{self.cycle_datetime}.nc4"
+
+    def ioda_filepath(self):
+        return os.path.join(self.ioda_dir, self.ioda_filename())
+
+    # def create_ioda_attributes(self, description, date_range):
+        # date_range_str = date_range[0] + ", " + date_range[1]
+        # description.add_global(name='datetimeRange', value=date_range_str)
+        # self.logger.debug("Created ioda global attributes")