Add VIIRS obs processing to dev/gcafs.v1

dev/jobs/JGCDAS_PREPARE_OBS

@@ -0,0 +1,43 @@
+#! /usr/bin/env bash
+
+source "${HOMEgcafs}/ush/preamble.sh"
+source "${HOMEgcafs}/ush/jjob_header.sh" -e "prepareobs" -c "base prepareobs"
+
+##############################################
+# Set variables used in the script
+##############################################
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${DUMPAODPY:-${HOMEgcafs}/scripts/exgcdas_prepare_obs.py}
+${EXSCRIPT}
+status=$?
+if [[ ${status} -ne 0 ]]; then
+    exit "${status}"
+fi
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]]; then
+    cat "${pgmout}"
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit
+if [[ "${KEEPDATA}" == "NO" ]]; then
+    rm -rf "${DATA}"
+fi
+
+exit 0

dev/parm/config/gcafs/config.prepareobs

@@ -0,0 +1,9 @@
+#!/bin/bash -x
+
+########## config.prepareobs ##########
+
+echo "BEGIN: config.prepareobs"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" prepareobs
+echo "END: config.prepareobs"

dev/parm/config/gcafs/config.resources

@@ -18,6 +18,7 @@ if (( $# != 1 )); then
     echo "snowanl esnowanl"
     echo "prep_emissions"
     echo "aeroanlinit aeroanlvar aeroanlfinal aeroanlgenb"
+    echo "prepareobs"
     echo "offlineanl"
     echo "anal sfcanl analcalc analdiag anlstat fcst echgres"
     echo "upp atmos_products"
@@ -363,6 +364,15 @@ case ${step} in
     tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
     ;;
 
+  "prepareobs")
+    ntasks=80
+    walltime="00:20:00"
+    threads_per_task=1
+    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    memory="3072M"
+    ;;
+
+
   "aeroanlinit")
     # below lines are for creating JEDI YAML
     case "${CASE}" in

dev/scripts/exgcdas_prepare_obs.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+# exgcdas_prepare_obs.py
+# This script will collect and preprocess
+# aerosol optical depth observations for
+# global aerosol assimilation
+import os
+
+from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, parse_j2yaml
+from pyobsforge.task.aero_prepobs import AerosolObsPrep
+
+# Initialize root logger
+logger = Logger(level='DEBUG', colored_log=True)
+
+
+if __name__ == '__main__':
+
+    # Take configuration from environment and cast it as python dictionary
+    config_env = cast_strdict_as_dtypedict(os.environ)
+    # Take configuration from YAML file to augment/append config dict
+    config_yaml = parse_j2yaml(os.path.join(config_env['HOMEgcafs'], 'parm', 'chem', 'prepare_obs.yaml'), config_env)
+    # Combine configs together
+    config = AttrDict(**config_env, **config_yaml['aoddump'])
+
+    aeroObs = AerosolObsPrep(config)
+    aeroObs.initialize()
+    aeroObs.execute()
+    aeroObs.finalize()

dev/ush/setup_gcafs_for_nco.py

@@ -279,6 +279,44 @@ def setup_gcafs_for_nco():
         num_replacements = replace_gfs_with_gcafs(file_path)
         print(f"Modified {file_path}: {num_replacements} replacements made.")
 
+    # Go through the ush directory and replace FOOgfs with FOOgcafs in all scripts
+    ush_dir = os.path.join(global_workflow_dir, 'ush')
+    for root, _, files in os.walk(ush_dir):
+        for file in files:
+            # Only process .sh and .py files
+            if file.endswith(('.sh', '.py')):
+                file_path = os.path.join(root, file)
+                num_replacements = replace_gfs_with_gcafs(file_path)
+                if num_replacements > 0:
+                    print(f"Modified {file_path}: {num_replacements} replacements made.")
 
+    # Go through the dev/ush and dev/workflow directories and replace FOOgfs with FOOgcafs in all scripts
+    # and YAMLs in dev/ci/cases
+    for subdir in ['dev/ush', 'dev/workflow', 'dev/ci/cases']:
+        dir_path = os.path.join(global_workflow_dir, subdir)
+        for root, _, files in os.walk(dir_path):
+            for file in files:
+                # Skip the current script to avoid self-modification
+                if file == 'setup_gcafs_for_nco.py':
+                    continue
+                # Only process .sh and .py scripts and YAML files
+                if file.endswith(('.sh', '.py', '.yaml')):
+                    file_path = os.path.join(root, file)
+                    num_replacements = replace_gfs_with_gcafs(file_path)
+                    if num_replacements > 0:
+                        print(f"Modified {file_path}: {num_replacements} replacements made.")
+
+    # Go through the dev/parm/config/gcafs directory and replace FOOgfs with FOOgcafs in all config files
+    config_gcafs_dir = os.path.join(global_workflow_dir, 'dev', 'parm', 'config', 'gcafs')
+    for root, _, files in os.walk(config_gcafs_dir):
+        for file in files:
+            # Only process config files
+            if file.startswith('config.'):
+                file_path = os.path.join(root, file)
+                num_replacements = replace_gfs_with_gcafs(file_path)
+                if num_replacements > 0:
+                    print(f"Modified {file_path}: {num_replacements} replacements made.")
+
+
 if __name__ == "__main__":
     setup_gcafs_for_nco()

ecf/defs/gcafs_v1.def

@@ -7,8 +7,12 @@ suite gcafs_v1
     edit QUEUE_ARCH 'dev_transfer'
     edit PROJENVIR 'DEV'
     edit MACHINE_SITE 'development'
-    edit ENVIR 'prod'
-    edit EXPDIR '%PACKAGEHOME%/dev/parm/config/gcafs'
+    edit ENVIR 'para'
+    edit EXPDIR '%PACKAGEHOME%/parm/config/gcafs'
+    edit COMROOT '/lfs/h2/emc/ptmp/cory.r.martin/gcafs/com'
+    edit DATAROOT '/lfs/h2/emc/ptmp/cory.r.martin/gcafs/dataroot'
+    edit ECF_INCLUDE '/lfs/h2/emc/da/noscrub/cory.r.martin/gcafs/git/release/gcafs.v1.0.0/ecf/include'
+    edit PDY '20260127'
     family 00
       edit CYC '00'
       family gcafs

ecf/include/head.h

@@ -1,3 +1,4 @@
+echo "HELLO CRM"
 date
 hostname
 set -xe  # print commands as they are executed and enable signal trapping
@@ -59,6 +60,12 @@ if [ -d /apps/ops/prod ]; then # On WCOSS2
   echo "Listing modules from head.h:"
   module list
   set -x
+  if [[ ! " ops.prod ops.para " =~ " $(whoami) " ]]; then
+    echo "Allow over-riding defaults for developers"
+    if [ -n "%COMROOT:%" ]; then export COMROOT="%COMROOT:%"; fi
+    if [ -n "%DATAROOT:%" ]; then export DATAROOT="%DATAROOT:%"; fi
+    if [ -n "%DCOMROOT:%" ]; then export DCOMROOT="%DCOMROOT:%"; fi
+  fi
 fi
 
 timeout 300 ecflow_client --init=${ECF_RID}

ecf/scripts/gcdas/chem/analysis/jgcdas_prepare_obs.ecf

@@ -32,91 +32,13 @@ module use /apps/ops/test/spack-stack-nco-1.9/modulefiles/Core
 module load stack-oneapi/2024.2.1
 module load stack-cray-mpich/8.1.29
 module load stack-python/3.11.7
-module load cmake/3.27.9
 module load craype/2.7.17
 module load cray-pals/1.3.2
-
-module load git/2.47.0
-module load git-lfs/3.5.1
-
-module load zstd/1.5.6
-module load pigz/2.8
-module load tar/1.34
-module load gettext/0.22.5
-module load curl/8.10.1
-module load hdf5/1.14.3
-module load parallel-netcdf/1.12.3
-module load netcdf-c/4.9.2
-module load nccmp/1.9.0.1
-module load netcdf-fortran/4.6.1
-module load nco/5.2.4
-module load parallelio/2.6.2
-module load wget/1.21.1
-module load boost/1.84.0
-module load ecbuild/3.7.2
-module load eccodes/2.33.0
-module load eigen/3.4.0
-module load openblas/0.3.24
-module load eckit/1.28.3
-module load fckit/0.13.2
-module load fms/2024.02
-module load python-venv/1.0
-module load py-pyyaml/6.0.2
-module load intel-oneapi-runtime/2024.2.1
-module load glibc/2.31
-module load esmf/8.8.0
-module load atlas/0.40.0
-module load sp/2.5.0
-module load ip/5.1.0
-module load gsl-lite/0.37.0
-module load libjpeg/2.1.0
-module load krb5/1.21.1-1
-module load libtirpc/1.3.3
-module load hdf/4.2.15
-module load jedi-cmake/1.4.0
-module load libpng/1.6.37
-module load libxt/1.3.0
-module load libxmu/1.2.1
-module load libxpm/3.5.17
-module load libxaw/1.0.16
-module load udunits/2.2.28
-module load ncview/2.1.9
-module load netcdf-cxx4/4.3.1
-module load json/3.11.3
-#module load crtm/v2.4_jedi
 module load prod_util/2.0.14
-module load grib-util/1.4.0
-
-module load py-numpy/1.26.4
-module load bufr/12.1.0
-
-module load py-markupsafe/2.1.3
-module load py-jinja2/3.1.4
-module load py-cftime/1.0.3.4
-module load py-certifi/2023.7.22
-module load py-netcdf4/1.7.1.post2
-module load py-pybind11/2.13.5
-module load py-setuptools/63.4.3
-module load py-pycodestyle/2.11.0
-module load py-pyyaml/6.0.2
-module load py-scipy/1.14.1
-
-module load py-setuptools/63.4.3
-module load py-tzdata/2023.3
-module load py-pytz/2023.3
-module load py-six/1.16.0
-module load py-python-dateutil/2.8.2
-module load py-pandas/2.2.3
-module load py-packaging/24.1
-module load py-xarray/2024.7.0
-module load py-f90nml/1.4.3
-module load py-pip/23.1.2
-module load py-click/8.1.7
-module load py-wheel/0.41.2
 
-ncdump=$(command -v ncdump)
-NETCDF=$(echo "${ncdump}" | cut -d " " -f 3)
-export NETCDF
+#ncdump=$(command -v ncdump)
+#NETCDF=$(echo "${ncdump}" | cut -d " " -f 3)
+#export NETCDF
 
 module list
 
@@ -125,6 +47,7 @@ module list
 #############################################################
 export cyc=%CYC%
 export cycle=t%CYC%z
+export EXPDIR=%EXPDIR%
 
 if [[ -d "${HOMEgcafs}/sorc/wxflow/src" ]]; then
   PYTHONPATH="${HOMEgcafs}/sorc/wxflow/src${PYTHONPATH:+:${PYTHONPATH}}"

parm/chem/prepare_obs.yaml

@@ -0,0 +1,9 @@
+aoddump:
+  provider: VIIRSAOD
+  platforms: ['npp', 'j01', 'n21'] # note j01==n20
+  binning_stride: 33
+  binning_min_number_of_obs: 32
+  binning_cressman_radius: 25
+  thinning_threshold: 0
+  channel: 4
+  preqc: 0