Spec cloud

.gitignore

@@ -1,5 +1,4 @@
 *.o
-Makefile
 Makefile.in
 /ar-lib
 /mdate-sh

.gitlab-ci.yml

@@ -0,0 +1,78 @@
+build:gfdl-ws:
+  stage: build
+  tags:
+    - gfdl-ws
+  script:
+    - autoreconf --install
+    - ./configure --prefix /home/fms/opt/gfdl-grtcode CFLAGS="-g -O2 -Wall -Wextra -pedantic -std=c99 -fopenmp" CPPFLAGS="-I/home/Raymond.Menzel/local/include" LDFLAGS="-fopenmp -L/home/Raymond.Menzel/local/lib"
+    - make
+  artifacts:
+    untracked: true
+    expire_in: 1 days
+
+
+test:gfdl-ws:
+  stage: test
+  tags:
+    - gfdl-ws
+  variables:
+    LD_LIBRARY_PATH: /home/Raymond.Menzel/local/lib
+  script:
+    - make check
+  dependencies:
+    - build:gfdl-ws
+
+
+deploy:gfdl-ws:
+  stage: deploy
+  environment: gfdl-ws
+  only:
+    - master
+  tags:
+    - gfdl-ws_deploy
+  script:
+    - make install
+  dependencies:
+    - build:gfdl-ws
+
+
+build:lscsky50-d:
+  stage: build
+  tags:
+    - skylake
+  script:
+    - source /opt/lmod/lmod/init/bash
+    - module load intel/2019_up2
+    - autoreconf --install
+    - ./configure --prefix /opt/gfdl-grtcode CC="icc" CFLAGS="-g -O2 -qopenmp" CPPFLAGS="-I/opt/hdf5/1.10.1/INTEL/include -I/opt/netcdf/4.6.1/INTEL/include" LDFLAGS="-qopenmp -L/opt/hdf5/1.10.1/INTEL/lib -L/opt/netcdf/4.6.1/INTEL/lib64" CXX="icc"
+    - make
+  artifacts:
+    untracked: true
+    expire_in: 1 days
+
+
+test:lscsky50-d:
+  stage: test
+  tags:
+    - skylake
+  script:
+    - source /opt/lmod/lmod/init/bash
+    - module load intel/2019_up2 netcdf/4.6.1
+    - make check
+  dependencies:
+    - build:lscsky50-d
+
+
+deploy:lscsky50-d:
+  stage: deploy
+  environment: skylake
+  only:
+    - master
+  tags:
+    - skylake
+  script:
+    - source /opt/lmod/lmod/init/bash
+    - module load intel/2019_up2
+    - make install
+  dependencies:
+    - build:lscsky50-d

GRTworkflow/build.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+
+# Get the directory where the environment script lives.
+workflow_home="/ncrc/home1/Jing.Feng/scripts/grtcode/GRTworkflow" #"$( dirname -- "$( readlink -f -- "$0"; )"; )"
+
+# Parse the command line arguments to see if the user wants to use a custom version of grtcode.
+argument_list="$0 [-h|--help] [-p path to grtcode repository]"
+default_grtcode_repository="$( readlink -f -- $workflow_home/../grtcode )"
+grtcode_repository="$default_grtcode_repository"
+while [[ $# -gt 0 ]]; do
+  argument="$1"
+  case $argument in
+    -h|--help)
+      printf "Usage: $argument_list\n"
+      printf "\nOptional arguments:"
+      printf "-h, --help:    Print this message.\n"
+      printf "-p <path>:     Path to a grtcode repository.\n"
+      exit 0
+    ;;
+    -p)
+      shift
+      if [ "$#" -gt 0 ]; then
+        grtcode_repository="$1"
+        shift
+      else
+        printf "Error: please specify the path to the grtcode repository.\n"
+        printf "Usage: $argument_list\n"
+        exit 1
+      fi
+    ;;
+    *)
+      printf "Error: Unknown argument.\n"
+      printf "Usage: $argument_list\n"
+      exit 1
+    ;;
+  esac
+done
+grtcode_repository="$( readlink -f -- $grtcode_repository )"
+commit="$( git -C $grtcode_repository rev-parse HEAD )"
+remote_url="$( git -C $grtcode_repository config --get remote.origin.url )"
+local_changes="$( git -C $grtcode_repository status --porcelain=v1 2>/dev/null | wc -l )"
+printf "Using grtcode repository: $grtcode_repository\n"
+printf "  [$remote_url @ $commit]\n"
+if [ "$local_changes" -gt 0 ]; then
+  printf "Warning: $grtcode_repository has local changes.\n"
+fi
+
+# If you are not using the basic installation, then build your custom version.
+  # Source the environment script.
+  source /ncrc/home1/Jing.Feng/scripts/grtcode/GRTworkflow/environment.sh
+
+  # Prepare to build the libraries and executables.
+  current_location="$PWD"
+  cd $grtcode_repository
+  mkdir -p build
+
+  # Clean and then re-build the executables.
+  makefile="$grtcode_repository/Makefile"
+  make -f $makefile clean
+  CC="${CC}" \
+    CPPFLAGS="${CPPFLAGS}" \
+    CFLAGS="${CFLAGS} -diag-disable=10441" \
+    CXX="${CXX}" \
+    CXXFLAGS="${CXXFLAGS} -diag-disable=10441" \
+    LDFLAGS="${LDFLAGS}" \
+    MPICC="${MPICC}" \
+    MPICXX="${MPICXX}" \
+    make -f $makefile all rfmip-irf era5
+  printf "Built executable in $grtcode_repository/build\n"
+
+# Check which executables exist.
+printf "Checking executables:\n"
+for executable in circ rfmip-irf era5; do
+  if [ -x "$grtcode_repository/build/$executable" ]; then
+    printf "\t$grtcode_repository/build/$executable ... created successfully.\n"
+  else
+    printf "\t$grtcode_repository/build/$executable ... failed.\n"
+  fi
+done
+
+if [ "$USER" == "Raymond.Menzel" ]; then
+  # Build the combiner.
+  $CC -o $workflow_home/grtcode-results-combiner -g -O3 -qopenmp -diag-disable=10441 \
+    -I$grtcode_repository/utilities/src \
+    $workflow_home/grtcode-results-combiner.c \
+    $grtcode_repository/build/argparse.o -lnetcdf
+fi

GRTworkflow/combine.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+#SBATCH --job-name=grt-combine
+#SBATCH --account=gfdl_m
+#SBATCH --partition=batch
+#SBATCH --clusters=c5
+#SBATCH --qos=normal
+#SBATCH --nodes=1
+#SBATCH --time=00:05:00
+#SBATCH --array=0-247
+#SBATCH --output=logs/slurm-%A_%a.out
+#SBATCH --error=logs/slurm-%A_%a.err
+
+module load cray-python/3.11.5
+
+# Define experiment and year arrays
+exps=(control PI co2_PI ch4_PI n2o_PI cfc12eq_PI hfc134aeq_PI co2_2xPI co2_4xPI)
+years=($(seq 2001 2024))
+
+# Calculate (exp, year) index from SLURM_ARRAY_TASK_ID
+exp_index=$((SLURM_ARRAY_TASK_ID / ${#years[@]}))
+year_index=$((SLURM_ARRAY_TASK_ID % ${#years[@]}))
+
+exp=${exps[$exp_index]}
+year=${years[$year_index]}
+
+workdir="/gpfs/f5/gfdl_m/scratch/Jing.Feng/work/grtcode-era5"
+script="/ncrc/home1/Jing.Feng/scripts/GRTworkflow/combiner.py"
+outdir="$workdir/$exp"
+
+echo "[$(date)] Task $SLURM_ARRAY_TASK_ID — Running $exp $year"
+
+# Run the combiner using srun
+python3 "$script" --workdir "$workdir" --year "$year" --nametag "$exp"
+
+# Only delete files if the command above succeeded
+if [ $? -eq 0 ]; then
+    rm -f "$outdir/${year}"*.nc
+    echo "[$(date)] Success: Removed $outdir/${year}*.nc"
+else
+    echo "[$(date)] Failure: Skipped removal for $exp $year"
+fi
+

GRTworkflow/combiner.py

@@ -0,0 +1,80 @@
+import os
+import glob
+import xarray as xr
+import argparse
+
+def combine_netcdf_files(input_files, output_file):
+    """
+    Combine NetCDF files by concatenating along the 'lon' dimension.
+    Only variables starting with 'rl' are included, and longitude is globally sorted.
+    """
+    datasets = []
+    for file in input_files:
+        ds = xr.open_dataset(file)
+
+        # Filter variables starting with 'rl'
+        rl_vars = {}
+        for var in ds.data_vars:
+            if var.startswith('rl'):
+                data = ds[var]
+                if '_FillValue' in data.attrs and np.isnan(data.attrs['_FillValue']):
+                    data.attrs['_FillValue'] = -9999.0
+                if data.dtype == 'float64':
+                    data = data.astype('float32')
+                rl_vars[var] = data
+
+        rl_ds = xr.Dataset(rl_vars, coords=ds.coords)
+        datasets.append(rl_ds)
+
+    # Concatenate and globally sort by 'lon'
+    combined_ds = xr.concat(
+        datasets,
+        dim='lon',
+        data_vars='all',
+        coords='all',
+        compat='override',
+        combine_attrs='drop'
+    ).sortby('lon')
+
+    # Save to NetCDF
+    combined_ds['lon'].attrs['long_name'] = 'longitude'
+    combined_ds['lon'].attrs['units'] = 'degrees_E'
+    combined_ds['lon'].attrs['axis'] = 'X'
+
+    # --- Dimension reordering section ---
+    target_order = ("time", "lw_wavenumber", "lat", "lon")
+    reordered_vars = {}
+
+    for varname in combined_ds.data_vars:
+        var = combined_ds[varname]
+
+        if var.ndim <= 1:
+            reordered_vars[varname] = var
+            continue
+
+        if all(dim in var.dims for dim in target_order):
+            reordered_vars[varname] = var.transpose(*target_order)
+        else:
+            print(f"Skipping variable '{varname}' — unexpected dimensions: {var.dims}")
+            reordered_vars[varname] = var  # Keep as-is
+
+    final_ds = xr.Dataset(reordered_vars, coords=combined_ds.coords)
+
+    # Save final output
+    final_ds.to_netcdf(output_file)
+
+    print("Final longitude ordering:", combined_ds['lon'].values)
+    print(f"Combined NetCDF file saved to {output_file}")
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--workdir', type=str, required=True, help='Path to the working directory')
+    parser.add_argument('--year', type=int, required=True, help='Year of the data to combine')
+    parser.add_argument('--nametag', type=str, required=True, help='experiment name')
+    args = parser.parse_args()
+
+    file_pattern = f"{args.workdir}/{args.nametag}/{args.year}.era5-fluxes.nc.*"
+    output_file = f"{args.workdir}/{args.year}.{args.nametag}-cleansky-spectra.nc"
+
+    input_files = sorted(glob.glob(file_pattern))
+    combine_netcdf_files(input_files, output_file)

GRTworkflow/comine_multiple.sh

@@ -0,0 +1,31 @@
+module load cray-python/3.11.5
+
+# Define experiment and year arrays
+exps=(control PI co2_PI ch4_PI n2o_PI cfc12eq_PI hfc134aeq_PI co2_2xPI co2_4xPI)
+years=($(seq 2001 2024))
+
+# Calculate (exp, year) index from SLURM_ARRAY_TASK_ID
+exp_index=$((SLURM_ARRAY_TASK_ID / ${#years[@]}))
+year_index=$((SLURM_ARRAY_TASK_ID % ${#years[@]}))
+
+workdir="/gpfs/f5/gfdl_m/scratch/Jing.Feng/work/grtcode-era5"
+script="/ncrc/home1/Jing.Feng/scripts/GRTworkflow/combiner.py"
+
+
+outdir="$workdir/$exp"
+
+for exp in "${exps[@]}"; do
+    for year in "${years[@]}"; do
+        outdir="$workdir/$exp"
+        echo "[$(date)] Running srun for $exp $year"
+
+        srun python3 "$script" --workdir "$workdir" --year "$year" --nametag "$exp"
+
+        if [ $? -eq 0 ]; then
+            rm -f "$outdir/${year}"*.nc
+            echo "[$(date)] Success: Removed $outdir/${year}*.nc"
+        else
+            echo "[$(date)] Failure: Skipped removal for $exp $year"
+        fi
+    done
+done

GRTworkflow/environment.sh

@@ -0,0 +1,32 @@
+#!/bin/bash -e
+
+source $MODULESHOME/init/bash
+
+module unload cray-netcdf cray-hdf5 fre
+module unload PrgEnv-pgi PrgEnv-intel PrgEnv-gnu PrgEnv-cray
+module load PrgEnv-intel/8.6.0
+module unload intel intel-classic intel-oneapi
+module load intel-classic/2023.2.0
+module load cray-hdf5/1.12.2.11
+module load cray-netcdf/4.9.0.9
+module unload cray-libsci
+module load cray-libsci/24.11.0
+
+export KMP_STACKSIZE="512m"
+export NC_BLKSZ="1M"
+export F_UFMTENDIAN="big"
+
+PREFIX="/gpfs/f5/gfdl_m/scratch/Jing.Feng/line-by-line"
+CC="cc"
+CPPFLAGS="-I${PREFIX}/include"
+CFLAGS="-g -O3 -qopenmp"
+MPICC="cc"
+CXX="CC"
+CXXFLAGS="-g -O3 -qopenmp"
+MPICXX="CC"
+LDFLAGS="-L${PREFIX}/lib"
+
+#debug
+#CPPFLAGS="-I${PREFIX}/include"
+#CFLAGS="-g -O0 -fno-omit-frame-pointer -qopenmp"
+#CXXFLAGS="-g -O0 -fno-omit-frame-pointer -qopenmp"

GRTworkflow/foo

@@ -0,0 +1,32 @@
+#!/bin/bash -ex
+
+# Load the software environment.
+source /gpfs/f5/gfdl_m/scratch/Raymond.Menzel/grtcode-2025.02/workflow/environment.sh
+
+# Run GRTCODE.
+index=1
+export OMP_NUM_THREADS=128
+    /gpfs/f5/gfdl_m/scratch/Raymond.Menzel/grtcode-2025.02/workflow/../grtcode/build/era5 \
+    /gpfs/f5/gfdl_m/scratch/Raymond.Menzel/grtcode-2025.02/workflow/../grtcode/grtcode-data/HITRAN_files/hitran2016.par \
+    /gpfs/f5/gfdl_m/scratch/Raymond.Menzel/grtcode-2025.02/workflow/../grtcode/grtcode-data/solar_flux/solar_flux.csv \
+    /gpfs/f5/gfdl_m/scratch/Raymond.Menzel/grtcode-2025.02/workflow/../grtcode/grtcode-data/rfmip-irf/multiple_input4MIPs_radiation_RFMIP_UColorado-RFMIP-1-2_none.nc/1979-era5.nc \
+     \
+    -clean \
+    -year 1979 \
+    -H2O -CO2 -O3 -N2O -CH4 -CO -O2 \
+    -h2o-ctm /gpfs/f5/gfdl_m/scratch/Raymond.Menzel/grtcode-2025.02/workflow/../grtcode/grtcode-data/water_vapor_continuum \
+    -o3-ctm /gpfs/f5/gfdl_m/scratch/Raymond.Menzel/grtcode-2025.02/workflow/../grtcode/grtcode-data/ozone_continuum/ozone_continuum.csv \
+    -CFC-11 /gpfs/f5/gfdl_m/scratch/Raymond.Menzel/grtcode-2025.02/workflow/../grtcode/grtcode-data/cfc_cross_sections/CFC-11_absorption_cross_sections.csv \
+    -CFC-12 /gpfs/f5/gfdl_m/scratch/Raymond.Menzel/grtcode-2025.02/workflow/../grtcode/grtcode-data/cfc_cross_sections/CFC-12_absorption_cross_sections.csv \
+    -N2-N2 /gpfs/f5/gfdl_m/scratch/Raymond.Menzel/grtcode-2025.02/workflow/../grtcode/grtcode-data/collision_induced_absorption/N2-N2.csv \
+    -O2-N2 /gpfs/f5/gfdl_m/scratch/Raymond.Menzel/grtcode-2025.02/workflow/../grtcode/grtcode-data/collision_induced_absorption/O2-N2.csv \
+    -O2-O2 /gpfs/f5/gfdl_m/scratch/Raymond.Menzel/grtcode-2025.02/workflow/../grtcode/grtcode-data/collision_induced_absorption/O2-O2.csv \
+    -x $(( ($index-1)*(144/24) )) \
+    -X $(( ($index*(144/24)-1) )) \
+    -y 0 -Y 0 \
+    -o 1979.era5-fluxes.nc.$index \
+    -beta-path /gpfs/f5/gfdl_m/scratch/Raymond.Menzel/grtcode-2025.02/workflow/../grtcode/grtcode-data/clouds/beta_distribution.nc \
+    -ice-path /gpfs/f5/gfdl_m/scratch/Raymond.Menzel/grtcode-2025.02/workflow/../grtcode/grtcode-data/clouds/fu_liou.nc \
+    -liquid-path /gpfs/f5/gfdl_m/scratch/Raymond.Menzel/grtcode-2025.02/workflow/../grtcode/grtcode-data/clouds/hu_stamnes.nc \
+    -flux-at-level 12 \
+    |& tee 1979.era5.log.$index &