diff --git a/parm/gsi_to_ioda.xml.j2 b/parm/gsi_to_ioda.xml.j2
index 47327f74..ee43bef3 100644
--- a/parm/gsi_to_ioda.xml.j2
+++ b/parm/gsi_to_ioda.xml.j2
@@ -33,11 +33,10 @@
obsforge_gdas_gsi_to_ioda_@H
{{ ACCOUNT }}
{{ QUEUE }}
- {{ PARTITION }}
{{ WALLTIME_GSI_TO_IODA }}
{{ TASK_GEOM_GSI_TO_IODA }}
{{ MEMORY_GSI_TO_IODA }}
- --export=NONE
+ -l place=vscatter:shared
{{ COMROOT }}/{{ PSLOT }}/logs/@Y@m@d@H/gdas_gsi_to_ioda.log
@@ -73,11 +72,10 @@
obsforge_gdas_atmos_bufr_dump_@H
{{ ACCOUNT }}
{{ QUEUE }}
- {{ PARTITION }}
{{ WALLTIME_ATMOS_BUFR_DUMP }}
{{ TASK_GEOM_ATMOS_BUFR_DUMP }}
{{ MEMORY_ATMOS_BUFR_DUMP }}
- --export=NONE
+ -l place=vscatter:shared
{{ COMROOT }}/{{ PSLOT }}/logs/@Y@m@d@H/gdas_atmos_bufr_dump.log
@@ -112,11 +110,10 @@
obsforge_gdas_stage_output_@H
{{ ACCOUNT }}
{{ QUEUE }}
- {{ PARTITION }}
{{ WALLTIME_STAGE_OUTPUT }}
{{ TASK_GEOM_STAGE_OUTPUT }}
{{ MEMORY_STAGE_OUTPUT }}
- --export=NONE
+ -l place=vscatter:shared
{{ COMROOT }}/{{ PSLOT }}/logs/@Y@m@d@H/gdas_stage_output.log
@@ -136,8 +133,8 @@
- gdas_gsi_to_ioda
- gdas_atmos_bufr_dump
+
+
diff --git a/ush/python/pyobsforge/task/gsi_to_ioda.py b/ush/python/pyobsforge/task/gsi_to_ioda.py
index bafb8738..5236ef0c 100644
--- a/ush/python/pyobsforge/task/gsi_to_ioda.py
+++ b/ush/python/pyobsforge/task/gsi_to_ioda.py
@@ -312,7 +312,7 @@ def convert_bias_correction_files(self) -> None:
except Exception as e:
raise WorkflowException(f"An error occurred during execution of {exec_cmd}:\n{e}") from e
- # Create satellite tarball and copy to COMOUT
+ # Create bias tarball and copy to COMOUT
comout = os.path.join(self.task_config['COMROOT'],
self.task_config['PSLOT'],
f"{self.task_config.RUN}.{self.task_config.current_cycle.strftime('%Y%m%d')}",
@@ -320,24 +320,32 @@ def convert_bias_correction_files(self) -> None:
'atmos_gsi')
if not os.path.exists(comout):
FileHandler({'mkdir': [comout]}).sync()
- tarball_out = os.path.join(comout, f"{self.task_config.APREFIX}rad_varbc_params.tar")
+ tarball_out = os.path.join(comout, f"{self.task_config.APREFIX}varbc_params.tar")
with tarfile.open(tarball_out, "w") as tar:
for sat in satlist:
- bias_file = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}radiance_{sat}.satbias.gsi.nc')
- if os.path.exists(bias_file):
- logger.info(f"Adding {bias_file} to tarball")
- tar.add(bias_file, arcname=os.path.basename(bias_file))
- tlapse_file = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}radiance_{sat}.tlapse.gsi.txt')
- if os.path.exists(tlapse_file):
+ bias_file_in = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}radiance_{sat}.satbias.gsi.nc')
+ bias_file = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}radiance_{sat}.bias.nc')
+ cov_file = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}radiance_{sat}.bias_cov.nc')
+ if os.path.exists(bias_file_in):
+ logger.info(f"Adding {bias_file_in} to tarball as {os.path.basename(bias_file)}")
+ tar.add(bias_file_in, arcname=os.path.basename(bias_file))
+ # add the same file with a different name for the covariance file
+ # (this is needed for the UFO varbc reader to read in the covariance info)
+ tar.add(bias_file_in, arcname=os.path.basename(cov_file))
+ tlapse_file_in = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}radiance_{sat}.tlapse.gsi.txt')
+ tlapse_file = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}radiance_{sat}.tlapse.txt')
+ if os.path.exists(tlapse_file_in):
logger.info(f"Adding {tlapse_file} to tarball")
- tar.add(tlapse_file, arcname=os.path.basename(tlapse_file))
- logger.info(f"Finished creating bias correction tarball at {tarball_out}")
+ tar.add(tlapse_file_in, arcname=os.path.basename(tlapse_file))
+ # also add the aircraft bias file to the tarball
+ acft_bias_file_in = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}aircraft_bias.gsi.nc')
+ acft_bias_file = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}aircraft.bias.nc')
+ acft_cov_file = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}aircraft.bias_cov.nc')
+ if os.path.exists(acft_bias_file_in):
+ logger.info(f"Adding {acft_bias_file_in} to tarball")
+ tar.add(acft_bias_file_in, arcname=os.path.basename(acft_bias_file))
+ # add the same file with a different name for the covariance file
+ # (this is needed for the UFO varbc reader to read in the covariance info)
+ tar.add(acft_bias_file_in, arcname=os.path.basename(acft_cov_file))
- # copy aircraft bias file to COMOUT
- acft_bias_file = os.path.join(bias_dir_path, f'{self.task_config["APREFIX"]}aircraft_bias.gsi.nc')
- if os.path.exists(acft_bias_file):
- dest = os.path.join(comout, os.path.basename(acft_bias_file))
- FileHandler({'copy_opt': [[acft_bias_file, dest]]}).sync()
- logger.info(f"Copied aircraft bias file to {dest}")
- else:
- logger.warning(f"Aircraft bias file {acft_bias_file} does not exist, skipping copy to COMOUT")
+ logger.info(f"Finished creating bias correction tarball at {tarball_out}")
diff --git a/ush/python/pyobsforge/task/stage_output.py b/ush/python/pyobsforge/task/stage_output.py
index 5edfbd0b..373052d8 100644
--- a/ush/python/pyobsforge/task/stage_output.py
+++ b/ush/python/pyobsforge/task/stage_output.py
@@ -114,10 +114,13 @@ def run(self):
# Copy obsforge BUFR status log file
bufr_status_log_src = os.path.join(self.task_config.COMIN_ATMOS_OBSFORGE,
- "obsforge_atmos_bufr_status.log")
+ f"{self.task_config.OPREFIX}obsforge_atmos_bufr_status.log")
bufr_status_log_dest = os.path.join(self.task_config.COMOUT_ATMOS_OBS,
- "obsforge_atmos_bufr_status.log")
+ f"{self.task_config.OPREFIX}obsforge_atmos_bufr_status.log")
if os.path.exists(bufr_status_log_src):
+ # If no observation files were copied, ensure the output directory exists
+ if not copy_list:
+ os.makedirs(self.task_config.COMOUT_ATMOS_OBS, exist_ok=True)
FileHandler({'copy': [[bufr_status_log_src, bufr_status_log_dest]]}).sync()
logger.info(f"Copied BUFR status log from {bufr_status_log_src} to {bufr_status_log_dest}")
else:
@@ -127,9 +130,9 @@ def run(self):
bias_correction_config = self.task_config.get('bias correction', {})
if bias_correction_config.get('source') == 'GSI':
bc_src_file = os.path.join(self.task_config.COMIN_ATMOS_GSI,
- f"{self.task_config.OPREFIX}rad_varbc_params.tar")
+ f"{self.task_config.OPREFIX}varbc_params.tar")
bc_dest_file = os.path.join(self.task_config.COMOUT_ATMOS_BC,
- f"{self.task_config.OPREFIX}rad_varbc_params.tar")
+ f"{self.task_config.OPREFIX}varbc_params.tar")
if os.path.exists(bc_src_file):
FileHandler({'mkdir': [self.task_config.COMOUT_ATMOS_BC],
'copy': [[bc_src_file, bc_dest_file]]}).sync()