Add climo wrapper

.github/workflows/build_conda.yml

@@ -10,36 +10,51 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build:
+  condabuild:
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/noaa-gfdl/fre-cli:miniconda24.7.1_gcc14.2.0
+      image: ghcr.io/noaa-gfdl/fre-cli:miniconda24_gcc14_v2
     steps:
       - name: Checkout Files
         uses: actions/checkout@v4
         with:
           submodules: 'recursive'
 
-      - name: Run Conda to Build
+      - name: Configure Conda
         run: |
-          # append the reqd channels
+          echo "removing defaults, appending reqd channels"
           conda config --append channels conda-forge
           conda config --append channels noaa-gfdl
-
-          # remove any default channels
           conda config --remove channels defaults
 
-          # just in case
-          conda config --show channels
+          echo "setting strict channel priority"
+          conda config --set channel_priority strict
+
+          echo "setting anaconda_upload to no"
+          conda config --set anaconda_upload no
 
-          # install conda-build and conda-verify
-          conda install conda-build conda-verify
+          echo "printing conda config just in case"
+          conda config --show
+
+      - name: Update Conda and Conda Package Indices
+        run: |
+          echo "updating conda and package channel indices for conda-forge, noaa-gfdl"
+          conda update -y conda
+          conda update -y --all --override-channels -c conda-forge
+          conda update -y --all --override-channels -c noaa-gfdl
 
-          # conda build
+      - name: Conda install conda-build
+        run: |
+          echo "conda install conda-build"
+          conda install conda-forge::conda-build
+
+      - name: Build fre-cli Conda Package
+        run: |
+          echo "conda building fre-cli package and outputting as a tarball"
           mkdir -p /app/fre-cli-tarball
           conda build --package-format tar.bz2 --output-folder /app/fre-cli-tarball .
 
-      - name: Upload fre-cli tarball
+      - name: Upload fre-cli Tarball
         uses: actions/upload-artifact@v4
         with:
           name: fre-cli-tarball

.github/workflows/create_test_conda_env.yml

@@ -14,7 +14,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build-linux:
+  condaenvcreate:
     runs-on: ubuntu-latest
     container:
       image: ghcr.io/noaa-gfdl/fre-cli:miniconda24_gcc14_v2
@@ -24,36 +24,103 @@ jobs:
         with:
           submodules: 'recursive'
 
-      - name: Create fre-cli environment
+      - name: Configure Conda
         run: |
-          # create env holding all deps, the env cant be explicitly activated in CI/CD
-          conda env create -f environment.yml --name fre-cli
+          echo "removing defaults, appending reqd channels"
+          conda config --append channels conda-forge
+          conda config --append channels noaa-gfdl
+          conda config --remove channels defaults
 
-          # sets CONDA to wherever it may be on the image
-          source /root/.bashrc
+          echo "setting strict channel priority"
+          conda config --set channel_priority strict
 
-          # add conda env's executables and mkmf to github's PATH equiv.
-          echo $CONDA/envs/fre-cli/bin >> $GITHUB_PATH
+          echo "printing conda config just in case"
+          conda config --show
 
-          # use *conda environment's pip* to install fre-cli, called w/ full path as module for explicitness
-          $CONDA/envs/fre-cli/bin/python -m pip install --prefix $CONDA/envs/fre-cli .
+      - name: Update Conda and Channel Package Indices
+        run: |
+          echo "updating conda and package channel indices for conda-forge, noaa-gfdl"
+          conda update -y conda
+          conda update -y --all --override-channels -c conda-forge
+          conda update -y --all --override-channels -c noaa-gfdl
+
+      - name: Create fre-cli Environment
+        run: |
+          echo "creating fre-cli env holding all deps"
+          conda env create -y -f environment.yml --name fre-cli
 
-      - name: Run pytest in fre-cli environment
+      - name: Activate fre-cli Environment and Install fre-cli
         run: |
-          # add spack installed binaries to front of path so that conda's netcdf/hdf5 installs don't break compilation tests
+          echo "sourcing /some/path/to/conda/etc/profile.d/conda.sh"
+          source /opt/software/linux-rocky9-zen2/gcc-11.4.1/miniconda3-24.7.1-akg5caxhoqv3oasili3pgukbyryceq74/etc/profile.d/conda.sh
+
+          echo "conda initializing"
+          conda init
+
+          echo "listing packages installed in the fre-cli environment"
+          conda env list
+          conda list -n fre-cli
+
+          echo "activating the fre-cli environment"
+          conda activate fre-cli
+
+          echo "pip installing fre-cli into the environment"
+          python -m pip install .
+
+          echo "listing packages installed again after pip installing fre-cli"
+          conda list -n fre-cli
+          which fre
+
+      - name: Install pytest, pylint, and pytest-cov into fre-cli Env
+        run: |
+          echo "sourcing /some/path/to/conda/etc/profile.d/conda.sh"
+          source /opt/software/linux-rocky9-zen2/gcc-11.4.1/miniconda3-24.7.1-akg5caxhoqv3oasili3pgukbyryceq74/etc/profile.d/conda.sh
+
+          echo "conda initializing"
+          conda init
+
+          echo "activating the fre-cli environment"
+          conda activate fre-cli
+
+          echo "conda install testing-reqs"
+          conda install conda-forge::pytest conda-forge::pylint conda-forge::pytest-cov
+
+      - name: Run pytest In fre-cli Env
+        run: |
+          echo "sourcing /some/path/to/conda/etc/profile.d/conda.sh"
+          source /opt/software/linux-rocky9-zen2/gcc-11.4.1/miniconda3-24.7.1-akg5caxhoqv3oasili3pgukbyryceq74/etc/profile.d/conda.sh
+
+          echo "conda initializing"
+          conda init
+
+          echo "activating the fre-cli environment"
+          conda activate fre-cli
+
+          echo "adding spack installed binaries of netcdf/hdf5 to front of path, so conda installed ones don't break compilation tests"
           export path_save=$PATH
           export PATH="/opt/views/view/bin:$PATH"
+          echo $PATH
 
-          # run pytest
-          pytest --durations=20 --log-level INFO --cov-report=xml --cov-config=coveragerc --cov=fre fre/
+          echo "running pytest, will print 20 longest durations"
+          pytest --durations=20 --log-level DEBUG --cov-report=xml --cov-config=coveragerc --cov=fre fre/
 
-          # restore original path and install genbadge to generate coverage badge based on xml
+          echo "restoring original PATH"
           export PATH="$path_save"
+          echo $PATH
 
-      - name: Run pylint in fre-cli environment
+      - name: Run pylint On fre/ In fre-cli Env
         run: |
-          # run pylint, ignored modules avoid warnings arising from code internal to those modules
-          pylint --fail-under 7.10 --max-line-length 120 --max-args 6 -ry --ignored-modules netCDF4,cmor fre/
+          echo "sourcing /some/path/to/conda/etc/profile.d/conda.sh"
+          source /opt/software/linux-rocky9-zen2/gcc-11.4.1/miniconda3-24.7.1-akg5caxhoqv3oasili3pgukbyryceq74/etc/profile.d/conda.sh
+
+          echo "conda initializing"
+          conda init
+
+          echo "activating the fre-cli environment"
+          conda activate fre-cli
+
+          echo "running pylint on fre"
+          pylint --fail-under 7.50 --max-line-length 120 --max-args 6 --max-positional-arguments 6 -ry --ignored-modules netCDF4,cmor fre/
 
       - name: Upload results to Codecov
         uses: codecov/codecov-action@v5

.github/workflows/publish_conda.yml

@@ -3,24 +3,53 @@ on:
   push:
     branches:
       - main
+
+
+
+
+
+
 jobs:
-  publish:
+  condapublish:
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/noaa-gfdl/fre-cli:miniconda24.7.1_gcc14.2.0
+      image: ghcr.io/noaa-gfdl/fre-cli:miniconda24_gcc14_v2
     steps:
       - name: Checkout Files
         uses: actions/checkout@v4
         with:
           submodules: 'recursive'
 
-      - name: Run Conda to Build and Publish
+      - name: Configure Conda
         run: |
+          echo "removing defaults, appending reqd channels"
           conda config --append channels conda-forge
           conda config --append channels noaa-gfdl
           conda config --remove channels defaults
-          conda config --show channels
-          conda install conda-build anaconda-client conda-verify
-          export ANACONDA_API_TOKEN=${{ secrets.ANACONDA_TOKEN }}
+
+          echo "setting strict channel priority"
+          conda config --set channel_priority strict
+
+          echo "setting anaconda_upload to yes"
           conda config --set anaconda_upload yes
+
+          echo "printing conda config just in case"
+          conda config --show
+
+      - name: Update Conda and Conda Package Indices
+        run: |
+          echo "updating conda and package channel indices for conda-forge, noaa-gfdl"
+          conda update -y conda
+          conda update -y --all --override-channels -c conda-forge
+          conda update -y --all --override-channels -c noaa-gfdl
+
+      - name: Conda install conda-build
+        run: |
+          echo "conda install conda-build"
+          conda install conda-forge::conda-build
+
+      - name: Build fre-cli Conda Package
+        run: |
+          echo "conda building fre-cli package and outputting as a tarball"
+          export ANACONDA_API_TOKEN=${{ secrets.ANACONDA_TOKEN }}
           conda build .

.github/workflows/spell_check.yml

@@ -1,7 +1,11 @@
 name: 'Check spelling'
 on:
   pull_request:
+    branches:
+      - main
   push:
+    branches:
+      - main
 
 jobs:
   spellcheck:

environment.yml

@@ -4,25 +4,24 @@ channels:
   - noaa-gfdl
 dependencies:
   - python=3.11.*
-  - pip
-  - noaa-gfdl::analysis_scripts
-  - noaa-gfdl::catalogbuilder>=2.0.2
-  - noaa-gfdl::fre-nctools
+  - noaa-gfdl::analysis_scripts==0.0.1
+  - noaa-gfdl::catalogbuilder==2.0.3
+  - noaa-gfdl::fre-nctools==2022.02.01
+  - conda-forge::cdo>=2
   - conda-forge::cftime
   - conda-forge::click>=8.2
-  - conda-forge::cylc-flow>=8.2.0
+  - conda-forge::cmor=3.11.*
+  - conda-forge::cylc-flow>=8.2
   - conda-forge::cylc-rose
-  - conda-forge::cmor>=3.11.0
-  - conda-forge::cdo>=2.0.0
-  - conda-forge::jinja2=3.0.*
+  - conda-forge::jinja2>=3
   - conda-forge::jsonschema
   - conda-forge::metomi-rose
   - conda-forge::nccmp
-  - conda-forge::numpy<2
-  - conda-forge::pylint
+  - conda-forge::numpy=1.26.*
   - conda-forge::pytest
   - conda-forge::pytest-cov
+  - conda-forge::pylint
   - conda-forge::python-cdo
   - conda-forge::pyyaml
-  - conda-forge::xarray
-  - conda-forge::netcdf4
+  - conda-forge::xarray>=2024.*
+  - conda-forge::netcdf4=1.7.*

fre/app/freapp.py

@@ -6,7 +6,9 @@
 
 from .mask_atmos_plevel.mask_atmos_plevel import mask_atmos_plevel_subtool
 from .generate_time_averages.generate_time_averages import generate
+from .generate_time_averages.wrapper import generate_wrapper
 from .regrid_xy.regrid_xy import regrid_xy
+from .generate_time_averages.combine import combine
 from .remap_pp_components.remap_pp_components import remap_pp_components
 
 @click.group(help=click.style(" - app subcommands", fg=(250,154,90)))
@@ -158,3 +160,80 @@
     start_time = time.perf_counter()
     generate(inf, outf, pkg, var, unwgt, avg_type)
     click.echo(f'Finished in total time {round(time.perf_counter() - start_time , 2)} second(s)')
+
+@app_cli.command()
+@click.option("--cycle-point",
+              type = str,
+              required = True,
+              help = "Beginning cycle-point in ISO8601")
+@click.option("--dir", 'dir_',
+              type = str,
+              required = True,
+              help = "Root directory containing the shards")
+@click.option("--sources",
+              type = str,
+              required = True,
+              help = "Sources (history file) input file, comma-separated")
+@click.option("--output-interval",
+              type = str,
+              required = True,
+              help = "ISO interval of the desired climatology")
+@click.option("--input-interval",
+              type = str,
+              required = True,
+              help = "ISO interval of the input timeseries")
+@click.option("--grid",
+              type = str,
+              required = True,
+              help = "Grid label corresponding to the shards directory, e.g. native, regrid-xy/180_288.conserve_order2")
+
+@click.option("--frequency",
+              type = str,
+              required = True,
+              help = "Frequency of desired climatology: 'mon' or 'yr'")
+@click.option("-p", "--pkg",
+              type = click.Choice(["cdo","fre-nctools","fre-python-tools"]),
+              default = "cdo",
+              help = "Time average approach")
+def gen_time_averages_wrapper(cycle_point, dir_, sources, output_interval, input_interval, grid, frequency, pkg):
+    """
+    Wrapper for climatology tool.
+    Timeaverages all variables for a desired cycle point, source, and grid.
+    """
+    sources_list = sources.split(',')
+    generate_wrapper(cycle_point, dir_, sources_list, output_interval, input_interval, grid, frequency, pkg)
+
+@app_cli.command()
+@click.option("--in-dir",
+              type = str,
+              required = True,
+              help = "Input directory")
+@click.option("--out-dir",
+              type = str,
+              required = True,
+              help = "Output directory")
+@click.option("--component",
+              type = str,
+              required = True,
+              help = "Component name to combine")
+@click.option("--begin",
+              type = int,
+              required = True,
+              help = "Beginning year")
+@click.option("--end",
+              type = int,
+              required = True,
+              help = "Ending year")
+@click.option("--frequency",
+              type = str,
+              required = True,
+              help = "Climatology frequency; 'mon' or 'yr'")
+@click.option("--interval",
+              type = str,
+              required = True,
+              help = "Climatology interval in ISO8601")
+def combine_time_averages(in_dir, out_dir, component, begin, end, frequency, interval):
+    """
+    Combine per-variable climatologies into one file
+    """
+    combine(in_dir, out_dir, component, begin, end, frequency, interval)

fre/app/generate_time_averages/__init__.py

@@ -1,3 +1,3 @@
 '''required for generate_time_averages module import functionality'''
-__all__ = ['generate_time_averages', 'timeAverager',
+__all__ = ['generate_time_averages', 'timeAverager', 'wrapper', 'combine',
            'frenctoolsTimeAverager', 'cdoTimeAverager', 'frepytoolsTimeAverager']