Add VIIRS obs processing to dev/gcafs.v1

.github/workflows/ci_unit_tests.yaml

@@ -63,7 +63,7 @@ jobs:
         sudo chmod -R a+w /scratch3/NCEPDEV
         source rocoto-${{ env.ROCOTO_VERSION }}/rocoto_path.sh
         cd global-workflow/sorc
-        git submodule update --init -j 2 ufs_model.fd gsi_monitor.fd wxflow
+        git submodule update --init -j 2 ufs_model.fd wxflow
         ./link_workflow.sh
         export PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${PWD}/wxflow/src"
         # Create test data directory for unit tests

.gitmodules

@@ -8,13 +8,12 @@
 [submodule "sorc/gfs_utils.fd"]
   path = sorc/gfs_utils.fd
   url = https://github.com/NOAA-EMC/gfs-utils
+[submodule "sorc/gdas.cd"]
+  path = sorc/gdas.cd
+  url = https://github.com/NOAA-EMC/GDASApp.git
 [submodule "sorc/ufs_utils.fd"]
   path = sorc/ufs_utils.fd
   url = https://github.com/ufs-community/UFS_UTILS.git
-[submodule "sorc/gdas.cd"]
-	path = sorc/gcdas.cd
-  url = https://github.com/NOAA-EMC/GDASApp.git
-  ignore = dirty
 [submodule "sorc/gsi_utils.fd"]
   path = sorc/gsi_utils.fd
   url = https://github.com/NOAA-EMC/GSI-Utils.git

dev/jobs/JGCDAS_PREPARE_OBS

@@ -0,0 +1,44 @@
+#! /usr/bin/env bash
+
+source "${HOMEgcafs}/ush/preamble.sh"
+source "${HOMEgcafs}/ush/jjob_header.sh" -e "prepareobs" -c "base prepareobs"
+
+##############################################
+# Set variables used in the script
+##############################################
+export COMOUT_OBS="${COMROOT}/${RUN}.${PDY}/${cyc}/obs/chem"
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${DUMPAODPY:-${HOMEgcafs}/scripts/exgcdas_prepare_obs.py}
+${EXSCRIPT}
+status=$?
+if [[ ${status} -ne 0 ]]; then
+    exit "${status}"
+fi
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]]; then
+    cat "${pgmout}"
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit
+if [[ "${KEEPDATA}" == "NO" ]]; then
+    rm -rf "${DATA}"
+fi
+
+exit 0

dev/jobs/JGLOBAL_OFFLINE_ATMOS_ANALYSIS

@@ -22,7 +22,8 @@ export GPREFIX="${GDUMP}.t${gcyc}z."
 
 RUN="gdas" DUMP="gdas" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
     COMIN_OBSPROC:COM_OBSPROC_TMPL \
-    COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
+    COMINgfs:COM_GFS_TMPL
+export COMINgfs_ATMOS_ANALYSIS=${COMINgfs}/atmos/
 
 YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
     COMOUT_OBS:COM_OBS_TMPL \

dev/parm/config/gcafs/config.prepareobs

@@ -0,0 +1,9 @@
+#!/bin/bash -x
+
+########## config.prepareobs ##########
+
+echo "BEGIN: config.prepareobs"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" prepareobs
+echo "END: config.prepareobs"

dev/parm/config/gcafs/config.resources

@@ -18,6 +18,7 @@ if (( $# != 1 )); then
     echo "snowanl esnowanl"
     echo "prep_emissions"
     echo "aeroanlinit aeroanlvar aeroanlfinal aeroanlgenb"
+    echo "prepareobs"
     echo "offlineanl"
     echo "anal sfcanl analcalc analdiag anlstat fcst echgres"
     echo "upp atmos_products"
@@ -363,6 +364,15 @@ case ${step} in
     tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
     ;;
 
+  "prepareobs")
+    ntasks=80
+    walltime="00:20:00"
+    threads_per_task=1
+    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    memory="3072M"
+    ;;
+
+
   "aeroanlinit")
     # below lines are for creating JEDI YAML
     case "${CASE}" in

dev/scripts/exgcdas_prepare_obs.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+# exgcdas_prepare_obs.py
+# This script will collect and preprocess
+# aerosol optical depth observations for
+# global aerosol assimilation
+import os
+
+from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, parse_j2yaml
+from pyobsforge.task.aero_prepobs import AerosolObsPrep
+
+# Initialize root logger
+logger = Logger(level='DEBUG', colored_log=True)
+
+
+if __name__ == '__main__':
+
+    # Take configuration from environment and cast it as python dictionary
+    config_env = cast_strdict_as_dtypedict(os.environ)
+    # Take configuration from YAML file to augment/append config dict
+    config_yaml = parse_j2yaml(os.path.join(config_env['HOMEgcafs'], 'parm', 'chem', 'prepare_obs.yaml'), config_env)
+    # Combine configs together
+    config = AttrDict(**config_env, **config_yaml['aoddump'])
+
+    aeroObs = AerosolObsPrep(config)
+    aeroObs.initialize()
+    aeroObs.execute()
+    aeroObs.finalize()