diff --git a/parm/gsi_to_ioda.xml.j2 b/parm/gsi_to_ioda.xml.j2 index 47327f74..ee43bef3 100644 --- a/parm/gsi_to_ioda.xml.j2 +++ b/parm/gsi_to_ioda.xml.j2 @@ -33,11 +33,10 @@ obsforge_gdas_gsi_to_ioda_@H {{ ACCOUNT }} {{ QUEUE }} - {{ PARTITION }} {{ WALLTIME_GSI_TO_IODA }} {{ TASK_GEOM_GSI_TO_IODA }} {{ MEMORY_GSI_TO_IODA }} - --export=NONE + -l place=vscatter:shared {{ COMROOT }}/{{ PSLOT }}/logs/@Y@m@d@H/gdas_gsi_to_ioda.log @@ -73,11 +72,10 @@ obsforge_gdas_atmos_bufr_dump_@H {{ ACCOUNT }} {{ QUEUE }} - {{ PARTITION }} {{ WALLTIME_ATMOS_BUFR_DUMP }} {{ TASK_GEOM_ATMOS_BUFR_DUMP }} {{ MEMORY_ATMOS_BUFR_DUMP }} - --export=NONE + -l place=vscatter:shared {{ COMROOT }}/{{ PSLOT }}/logs/@Y@m@d@H/gdas_atmos_bufr_dump.log @@ -112,11 +110,10 @@ obsforge_gdas_stage_output_@H {{ ACCOUNT }} {{ QUEUE }} - {{ PARTITION }} {{ WALLTIME_STAGE_OUTPUT }} {{ TASK_GEOM_STAGE_OUTPUT }} {{ MEMORY_STAGE_OUTPUT }} - --export=NONE + -l place=vscatter:shared {{ COMROOT }}/{{ PSLOT }}/logs/@Y@m@d@H/gdas_stage_output.log @@ -136,8 +133,8 @@ - gdas_gsi_to_ioda - gdas_atmos_bufr_dump + + diff --git a/ush/python/pyobsforge/task/gsi_to_ioda.py b/ush/python/pyobsforge/task/gsi_to_ioda.py index bafb8738..5236ef0c 100644 --- a/ush/python/pyobsforge/task/gsi_to_ioda.py +++ b/ush/python/pyobsforge/task/gsi_to_ioda.py @@ -312,7 +312,7 @@ def convert_bias_correction_files(self) -> None: except Exception as e: raise WorkflowException(f"An error occurred during execution of {exec_cmd}:\n{e}") from e - # Create satellite tarball and copy to COMOUT + # Create bias tarball and copy to COMOUT comout = os.path.join(self.task_config['COMROOT'], self.task_config['PSLOT'], f"{self.task_config.RUN}.{self.task_config.current_cycle.strftime('%Y%m%d')}", @@ -320,24 +320,32 @@ def convert_bias_correction_files(self) -> None: 'atmos_gsi') if not os.path.exists(comout): FileHandler({'mkdir': [comout]}).sync() - tarball_out = os.path.join(comout, f"{self.task_config.APREFIX}rad_varbc_params.tar") + tarball_out = os.path.join(comout, f"{self.task_config.APREFIX}varbc_params.tar") with tarfile.open(tarball_out, "w") as tar: for sat in satlist: - bias_file = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}radiance_{sat}.satbias.gsi.nc') - if os.path.exists(bias_file): - logger.info(f"Adding {bias_file} to tarball") - tar.add(bias_file, arcname=os.path.basename(bias_file)) - tlapse_file = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}radiance_{sat}.tlapse.gsi.txt') - if os.path.exists(tlapse_file): + bias_file_in = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}radiance_{sat}.satbias.gsi.nc') + bias_file = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}radiance_{sat}.bias.nc') + cov_file = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}radiance_{sat}.bias_cov.nc') + if os.path.exists(bias_file_in): + logger.info(f"Adding {bias_file_in} to tarball as {os.path.basename(bias_file)}") + tar.add(bias_file_in, arcname=os.path.basename(bias_file)) + # add the same file with a different name for the covariance file + # (this is needed for the UFO varbc reader to read in the covariance info) + tar.add(bias_file_in, arcname=os.path.basename(cov_file)) + tlapse_file_in = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}radiance_{sat}.tlapse.gsi.txt') + tlapse_file = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}radiance_{sat}.tlapse.txt') + if os.path.exists(tlapse_file_in): logger.info(f"Adding {tlapse_file} to tarball") - tar.add(tlapse_file, arcname=os.path.basename(tlapse_file)) - logger.info(f"Finished creating bias correction tarball at {tarball_out}") + tar.add(tlapse_file_in, arcname=os.path.basename(tlapse_file)) + # also add the aircraft bias file to the tarball + acft_bias_file_in = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}aircraft_bias.gsi.nc') + acft_bias_file = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}aircraft.bias.nc') + acft_cov_file = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}aircraft.bias_cov.nc') + if os.path.exists(acft_bias_file_in): + logger.info(f"Adding {acft_bias_file_in} to tarball") + tar.add(acft_bias_file_in, arcname=os.path.basename(acft_bias_file)) + # add the same file with a different name for the covariance file + # (this is needed for the UFO varbc reader to read in the covariance info) + tar.add(acft_bias_file_in, arcname=os.path.basename(acft_cov_file)) - # copy aircraft bias file to COMOUT - acft_bias_file = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}aircraft_bias.gsi.nc') - if os.path.exists(acft_bias_file): - dest = os.path.join(comout, os.path.basename(acft_bias_file)) - FileHandler({'copy_opt': [[acft_bias_file, dest]]}).sync() - logger.info(f"Copied aircraft bias file to {dest}") - else: - logger.warning(f"Aircraft bias file {acft_bias_file} does not exist, skipping copy to COMOUT") + logger.info(f"Finished creating bias correction tarball at {tarball_out}") diff --git a/ush/python/pyobsforge/task/stage_output.py b/ush/python/pyobsforge/task/stage_output.py index 5edfbd0b..373052d8 100644 --- a/ush/python/pyobsforge/task/stage_output.py +++ b/ush/python/pyobsforge/task/stage_output.py @@ -114,10 +114,13 @@ def run(self): # Copy obsforge BUFR status log file bufr_status_log_src = os.path.join(self.task_config.COMIN_ATMOS_OBSFORGE, - "obsforge_atmos_bufr_status.log") + f"{self.task_config.OPREFIX}obsforge_atmos_bufr_status.log") bufr_status_log_dest = os.path.join(self.task_config.COMOUT_ATMOS_OBS, - "obsforge_atmos_bufr_status.log") + f"{self.task_config.OPREFIX}obsforge_atmos_bufr_status.log") if os.path.exists(bufr_status_log_src): + # If no observation files were copied, ensure the output directory exists + if not copy_list: + os.makedirs(self.task_config.COMOUT_ATMOS_OBS, exist_ok=True) FileHandler({'copy': [[bufr_status_log_src, bufr_status_log_dest]]}).sync() logger.info(f"Copied BUFR status log from {bufr_status_log_src} to {bufr_status_log_dest}") else: @@ -127,9 +130,9 @@ def run(self): bias_correction_config = self.task_config.get('bias correction', {}) if bias_correction_config.get('source') == 'GSI': bc_src_file = os.path.join(self.task_config.COMIN_ATMOS_GSI, - f"{self.task_config.OPREFIX}rad_varbc_params.tar") + f"{self.task_config.OPREFIX}varbc_params.tar") bc_dest_file = os.path.join(self.task_config.COMOUT_ATMOS_BC, - f"{self.task_config.OPREFIX}rad_varbc_params.tar") + f"{self.task_config.OPREFIX}varbc_params.tar") if os.path.exists(bc_src_file): FileHandler({'mkdir': [self.task_config.COMOUT_ATMOS_BC], 'copy': [[bc_src_file, bc_dest_file]]}).sync()