Refactor Local Archiving: Move YAML Logic to archive_vars.py and Handle Ensemble/Deterministic Archiving

jobs/JGLOBAL_ARCHIVE_VRFY

@@ -2,26 +2,9 @@
 
 source "${HOMEgfs}/ush/jjob_header.sh" -e "arch_vrfy" -c "base arch_vrfy"
 
-##############################################
-# Set variables used in the script
-##############################################
-YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
-    COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \
-    COMIN_ATMOS_GENESIS:COM_ATMOS_GENESIS_TMPL \
-    COMIN_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL \
-    COMIN_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL \
-    COMIN_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \
-    COMIN_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL \
-    COMIN_OBS:COM_OBS_TMPL \
-    COMOUT_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL
-
-for grid in "0p25" "0p50" "1p00"; do
-    YMD=${PDY} HH=${cyc} GRID=${grid} declare_from_tmpl -rx \
-        "COMIN_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL"
-done
-
 ###############################################################
 # Run archive script
+# Note: COM paths are calculated within the Python script
 ###############################################################
 
 ${GLOBALARCHIVESH:-${SCRgfs}/exglobal_archive_vrfy.py}
@@ -30,10 +13,6 @@ if [[ ${err} -ne 0 ]]; then
     exit "${err}"
 fi
 
-##############################################
-# End JOB SPECIFIC work
-##############################################
-
 ##############################################
 # Final processing
 ##############################################

jobs/JGLOBAL_ENS_ARCHIVE_VRFY

@@ -2,16 +2,9 @@
 
 source "${HOMEgfs}/ush/jjob_header.sh" -e "earc_vrfy" -c "base earc_vrfy"
 
-##############################################
-# Set variables used in the script
-##############################################
-MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
-    COMIN_ATMOS_ANALYSIS_ENSSTAT:COM_ATMOS_ANALYSIS_TMPL \
-    COMIN_ATMOS_HISTORY_ENSSTAT:COM_ATMOS_HISTORY_TMPL \
-    COMIN_SNOW_ANALYSIS_ENSSTAT:COM_SNOW_ANALYSIS_TMPL
-
 ###############################################################
 # Run archive script
+# Note: COM paths are calculated within the Python script
 ###############################################################
 
 "${SCRgfs}/exgdas_enkf_earc_vrfy.py"
@@ -20,12 +13,6 @@ if [[ ${err} -ne 0 ]]; then
     exit "${err}"
 fi
 
-###############################################################
-
-##############################################
-# End JOB SPECIFIC work
-##############################################
-
 ##############################################
 # Final processing
 ##############################################

parm/archive/enkf_arcdir.yaml.j2

@@ -0,0 +1,22 @@
+# Variables provided by archive_vars.py:
+# - cycle_HH, cycle_YMDH, cycle_YMD, head
+# - ARCDIR
+# - All COMIN_* paths
+
+# Ensemble (EnKF) archiving template
+# Used for: enkfgdas, enkfgfs, enkfgcafs, enkfgcdas
+
+mkdir:
+    - "{{ ARCDIR }}"
+
+copy_req:
+    # Ensemble analysis statistics - REQUIRED
+    {% if DO_JEDIATMENS == True %}
+    - ["{{ COMIN_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}atmensstat",
+       "{{ ARCDIR }}/atmensstat.{{ RUN }}.{{ cycle_YMDH }}"]
+    {% else %}
+    - ["{{ COMIN_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}enkfstat.txt",
+       "{{ ARCDIR }}/enkfstat.{{ RUN }}.{{ cycle_YMDH }}"]
+    - ["{{ COMIN_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}gsistat_ensmean.txt",
+       "{{ ARCDIR }}/gsistat.{{ RUN }}.{{ cycle_YMDH }}.ensmean"]
+    {% endif %}

parm/archive/gcafs_arcdir.yaml.j2

@@ -1,107 +1,55 @@
-{% set cycle_HH = current_cycle | strftime("%H") %}
-{% set cycle_YMDH = current_cycle | to_YMDH %}
-{% set cycle_YMD = current_cycle | to_YMD %}
-{% set head = RUN + ".t" + cycle_HH + "z." %}
+# Variables provided by archive_vars.py:
+# - cycle_HH, cycle_YMDH, cycle_YMD, head
+# - VFYARC
+# - All COMIN_* paths
 
-# Select data to store in the ARCDIR and VFYARC from deterministic runs
-# This file set will contain all source-destination pairs to send to the FileHandler for copying
-{% set file_set = [] %}
+# Deterministic GCAFS/GCDAS archiving template
+# Used for: gcafs, gcdas
 
-# Declare the VFYARC where Fit2Obs data will be sent
-{% set VFYARC = ROTDIR ~ "/vrfyarch" %}
-
-# Deterministic files
-{% if "enkf" not in RUN %}
-    # Common files to be added to both the gcafs and gcdas keys below
-    {% set det_files = [] %}
-
-    # Deterministic analysis files (generated for cycled experiments)
-    {% set det_anl_files = [] %}
-
-    {% if DO_AERO_ANL %}
-        {% do det_anl_files.append([COMIN_CHEM_ANALYSIS ~ "/" ~ head ~ "aerostat.tgz",
-                                    ARCDIR ~ "/aerostat." ~ RUN ~ "." ~ cycle_YMDH ~ ".tgz"]) %}
-    {% endif %}
-
-    {% if DO_PREP_OBS_AERO == True %}
-        {% do det_anl_files.append([COMIN_OBS ~ "/" ~ head ~ "aeroobs",
-                                    ARCDIR ~ "/aeroobs." ~ RUN ~ "." ~ cycle_YMDH]) %}
-        {% do det_anl_files.append([COMIN_OBS ~ "/" ~ head ~ "aeroawobs",
-                                    ARCDIR ~ "/aeroawobs." ~ RUN ~ "." ~ cycle_YMDH]) %}
-    {% endif %}
+mkdir:
+    - "{{ ARCDIR }}"
+{% if DO_FIT2OBS == True %}
+    - "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}"
+{% endif %}
 
-    # GCAFS-specific files
-    {% set gfs_files = [] %}
+copy_req:
+{% if RUN == "gcafs" %}
+    # GCAFS forecast files - REQUIRED
     {% for fhr in range(0, FHMAX_GFS + 1, FHOUT_GFS) %}
-        {% do gfs_files.append([COMIN_ATMOS_GRIB_1p00 ~ "/" ~ head ~ "pres_a.1p00.f" ~ '%03d'|format(fhr) ~ ".grib2",
-                                ARCDIR ~ "/pgbf" ~ '%02d'|format(fhr) ~ "." ~ RUN ~ "." ~ cycle_YMDH ~ ".grib2"]) %}
+    - ["{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_a.1p00.f{{ '%03d'|format(fhr) }}.grib2",
+       "{{ ARCDIR }}/pgbf{{ '%02d'|format(fhr) }}.{{ RUN }}.{{ cycle_YMDH }}.grib2"]
     {% endfor %}
 
-    # GCAFS Fit2Obs data
-    {% set fit2obs_files = [] %}
-    {% for fhr in range(0, FHMAX_FITS + 1, 6) %}
-        {% set sfcfile = "/" + head + "sfc.f" + '%03d'|format(fhr) + ".nc" %}
-        {% set sigfile = "/" + head + "atm.f" + '%03d'|format(fhr) + ".nc" %}
-        {% do fit2obs_files.append([COMIN_ATMOS_HISTORY ~ "/" ~ sfcfile,
-                                    VFYARC ~ "/" ~ RUN ~ "." ~ cycle_YMD ~ "/" ~ cycle_HH ~ "/" ~ sfcfile ]) %}
-        {% do fit2obs_files.append([COMIN_ATMOS_HISTORY ~ "/" ~ sigfile,
-                                    VFYARC ~ "/" ~ RUN ~ "." ~ cycle_YMD ~ "/" ~ cycle_HH ~ "/" ~ sigfile ]) %}
-    {% endfor %}
+    {% if DO_FIT2OBS == True %}
+    # GCAFS Fit2Obs data - REQUIRED if DO_FIT2OBS is enabled
+        {% for fhr in range(0, FHMAX_FITS + 1, 6) %}
+    - ["{{ COMIN_ATMOS_HISTORY }}/{{ head }}sfc.f{{ '%03d'|format(fhr) }}.nc",
+       "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}/{{ head }}sfc.f{{ '%03d'|format(fhr) }}.nc"]
+    - ["{{ COMIN_ATMOS_HISTORY }}/{{ head }}atm.f{{ '%03d'|format(fhr) }}.nc",
+       "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}/{{ head }}atm.f{{ '%03d'|format(fhr) }}.nc"]
+        {% endfor %}
+    {% endif %}
+{% endif %}
 
-    # GCDAS-specific files
-    {% set gdas_files = [] %}
+{% if RUN == "gcdas" %}
+    # GCDAS forecast files - REQUIRED
     {% for fhr in range(0, FHMAX + 1, FHOUT) %}
-        {% do gdas_files.append([COMIN_ATMOS_GRIB_1p00 ~ "/" ~ head ~ "pres_a.1p00.f" ~ '%03d'|format(fhr) ~ ".grib2",
-                                 ARCDIR ~ "/pgbf" ~ '%02d'|format(fhr) ~ "." ~ RUN ~ "." ~ cycle_YMDH ~ ".grib2"]) %}
+    - ["{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_a.1p00.f{{ '%03d'|format(fhr) }}.grib2",
+       "{{ ARCDIR }}/pgbf{{ '%02d'|format(fhr) }}.{{ RUN }}.{{ cycle_YMDH }}.grib2"]
     {% endfor %}
+{% endif %}
 
-    # Now append the necessary file pairs to file_set
-    # Common deterministic files
-    {% set file_set = file_set + det_files %}
-    {% if MODE == "cycled" %}
-        {% set file_set = file_set + det_anl_files %}
-    {% endif %}
-
-    # Run-specific deterministic files
-    {% if RUN == "gcafs" %}
-        {% set file_set = file_set + gfs_files %}
-        # Fit2Obs files
-        {% if DO_FIT2OBS == True %}
-            {% set file_set = file_set + fit2obs_files %}
-        {% endif %}
-    {% elif RUN == "gcdas" %}
-        {% set file_set = file_set + gdas_files %}
+{% if MODE == "cycled" %}
+    # Deterministic analysis files (cycled mode only) - REQUIRED
+    {% if DO_AERO_ANL %}
+    - ["{{ COMIN_CHEM_ANALYSIS }}/{{ head }}aerostat.tgz",
+       "{{ ARCDIR }}/aerostat.{{ RUN }}.{{ cycle_YMDH }}.tgz"]
     {% endif %}
 
-{% else %}  # End of deterministic files
-
-    # Ensemble analysis files
-    {% set enkf_files = [] %}
-    {% if DO_JEDIATMENS == True %}
-        {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "atmensstat",
-                                 ARCDIR ~ "/atmensstat." ~ RUN ~ "." ~ cycle_YMDH ]) %}
-    {% else %}
-        {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "enkfstat.txt",
-                                 ARCDIR ~ "/enkfstat." ~ RUN ~ "." ~ cycle_YMDH ]) %}
-        {% do enkf_files.append([COMIN_ATMOS_ANALYSIS_ENSSTAT ~ "/" ~ head ~ "gsistat_ensmean.txt",
-                                 ARCDIR ~ "/gsistat." ~ RUN ~ "." ~ cycle_YMDH ~ ".ensmean"]) %}
+    {% if DO_PREP_OBS_AERO == True %}
+    - ["{{ COMIN_OBS }}/{{ head }}aeroobs",
+       "{{ ARCDIR }}/aeroobs.{{ RUN }}.{{ cycle_YMDH }}"]
+    - ["{{ COMIN_OBS }}/{{ head }}aeroawobs",
+       "{{ ARCDIR }}/aeroawobs.{{ RUN }}.{{ cycle_YMDH }}"]
     {% endif %}
-
-    # Construct the final file set
-    {% set file_set = file_set + enkf_files %}
-
 {% endif %}
-
-
-# Actually write the yaml
-mkdir:
-    - "{{ ARCDIR }}"
-
-    {% if DO_FIT2OBS == True %}
-    - "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}"
-    {% endif %}
-
-copy:
-   {% for source_dest_pair in file_set %}
-   - {{ source_dest_pair }}
-   {% endfor %}

parm/archive/gefs_arcdir.yaml.j2

@@ -1,36 +1,17 @@
-{% set cycle_HH = current_cycle | strftime("%H") %}
-{% set cycle_YMDH = current_cycle | to_YMDH %}
-{% set cycle_YMD = current_cycle | to_YMD %}
-{% set head = RUN + ".t" + cycle_HH + "z." %}
+# Variables provided by archive_vars.py:
+# - cycle_HH, cycle_YMDH, cycle_YMD, head
+# - VFYARC (archive directory)
+# - COMIN_ATMOS_ENSSTAT_1p00 (calculated in Python with MEMDIR='ensstat')
 
-# Declare the GEFS_ARCH where atmos data will be sent
-{% set GEFS_ARCH = ROTDIR ~ "/gefsarch" %}
-
-{% set file_set = [] %}
-
-{% set tmpl_dict = ({ '${ROTDIR}':ROTDIR,
-                      '${RUN}':RUN,
-                      '${YMD}':cycle_YMD,
-                      '${HH}':cycle_HH,
-                      '${GRID}': '1p00',
-                      '${MEMDIR}': 'ensstat' }) %}
-
-{% set COMIN_ATMOS_ENSSTAT_1p00 = COM_ATMOS_GRIB_GRID_TMPL | replace_tmpl(tmpl_dict) %}
+# Create directories first
+mkdir:
+    - "{{ VFYARC }}"
 
-# Select ensstat files to copy to the arcdir
-{% set ensstat_files = [] %}
-{% if path_exists(COMIN_ATMOS_ENSSTAT_1p00) %}
+# Define all source-destination pairs for archiving
+# Use copy_req for files that MUST exist (raise error if missing)
+copy_req:
+    # GEFS ensemble mean forecast files - REQUIRED
     {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %}
-        {% do ensstat_files.append([COMIN_ATMOS_ENSSTAT_1p00 ~ "/" ~ head ~ "mean.pres_." ~
-                                    "1p00" ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2",
-                                    GEFS_ARCH]) %}
+    - ["{{ COMIN_ATMOS_ENSSTAT_1p00 }}/{{ head }}mean.pres_.1p00.f{{ '%03d'|format(fhr) }}.grib2",
+       "{{ VFYARC }}/{{ head }}mean.pres_.1p00.f{{ '%03d'|format(fhr) }}.grib2"]
     {% endfor %}
-{% endif %}
-{% set file_set = ensstat_files %}
-# Actually write the yaml
-mkdir:
-    - "{{ GEFS_ARCH }}"
-copy:
-   {% for source_dest_pair in file_set %}
-   - {{ source_dest_pair }}
-   {% endfor %}