Add climo wrapper

fre/app/freapp.py

@@ -6,7 +6,9 @@
 
 from .mask_atmos_plevel import mask_atmos_plevel_subtool
 from .generate_time_averages.generate_time_averages import generate
+from .generate_time_averages.wrapper import generate_wrapper
 from .regrid_xy.regrid_xy import regrid_xy
+from .generate_time_averages.combine import combine
 
 @click.group(help=click.style(" - app subcommands", fg=(250,154,90)))
 def app_cli():
@@ -111,5 +113,77 @@ def gen_time_averages(inf, outf, pkg, var, unwgt, avg_type):
     generate(inf, outf, pkg, var, unwgt, avg_type)
     click.echo(f'Finished in total time {round(time.perf_counter() - start_time , 2)} second(s)')
 
+@app_cli.command()
+@click.option("--cycle-point",
+              type = str,
+              required = True,
+              help = "Beginning cycle-point in ISO8601")
+@click.option("--dir",
+              type = str,
+              required = True,
+              help = "Root directory containing the shards")
+@click.option("--sources",
+              type = str,
+              required = True,
+              help = "Sources (history file) input file, comma-separated")
+@click.option("--output-interval",
+              type = str,
+              required = True,
+              help = "ISO interval of the desired climatology")
+@click.option("--input-interval",
+              type = str,
+              required = True,
+              help = "ISO interval of the input timeseries")
+@click.option("--grid",
+              type = str,
+              required = True,
+              help = "Grid label corresponding to the shards directory (e.g. 'native' and 'regrid-xy/180_288.conserve_order2'")
+@click.option("--frequency",
+              type = str,
+              required = True,
+              help = "Frequency of desired climatology: 'mon' or 'yr'")
+def gen_time_averages_wrapper(cycle_point, dir, sources, output_interval, input_interval, grid, frequency):
+    """
+    Wrapper for climatology tool.
+    Timeaverages all variables for a desired cycle point, source, and grid.
+    """
+    sources_list = sources.split(',')
+    generate_wrapper(cycle_point, dir, sources_list, output_interval, input_interval, grid, frequency)
+
+@app_cli.command()
+@click.option("--in-dir",
+              type = str,
+              required = True,
+              help = "Input directory")
+@click.option("--out-dir",
+              type = str,
+              required = True,
+              help = "Output directory")
+@click.option("--component",
+              type = str,
+              required = True,
+              help = "Component name to combine")
+@click.option("--begin",
+              type = str,
+              required = True,
+              help = "Beginning year")
+@click.option("--end",
+              type = str,
+              required = True,
+              help = "Ending year")
+@click.option("--frequency",
+              type = str,
+              required = True,
+              help = "Climatology frequency; 'mon' or 'yr'")
+@click.option("--interval",
+              type = str,
+              required = True,
+              help = "Climatology interval in ISO8601")
+def combine_time_averages(in_dir, out_dir, component, begin, end, frequency, interval):
+    """
+    Combine per-variable climatologies into one file
+    """
+    combine(in_dir, out_dir, component, begin, end, frequency, interval)
+
 if __name__ == "__main__":
     app_cli()

fre/app/generate_time_averages/__init__.py

@@ -1,3 +1,3 @@
 '''required for generate_time_averages module import functionality'''
-__all__ = ['generate_time_averages', 'timeAverager',
+__all__ = ['generate_time_averages', 'timeAverager', 'wrapper',
            'frenctoolsTimeAverager', 'cdoTimeAverager', 'frepytoolsTimeAverager']

fre/app/generate_time_averages/combine.py

@@ -0,0 +1,110 @@
+import logging
+from pathlib import Path
+import glob
+import os
+import subprocess
+import metomi.isodatetime.parsers
+
+fre_logger = logging.getLogger(__name__)
+duration_parser = metomi.isodatetime.parsers.DurationParser()
+
+def form_bronx_directory_name(frequency: str, interval: str) -> str:
+    """
+    Form the legacy Bronx timeaverage directory name
+    given a frequency and interval.
+
+    :param frequency: Frequency of the climatology
+    :type frequency: 'mon' or 'yr'
+    :param interval: Interval of the climatology
+    :type interval: ISO8601 duration
+    :return: Corresponding Bronx directory name
+    :rtype: str
+    """
+
+    if frequency == "mon":
+        frequency_label = "monthly"
+    elif frequency == "yr":
+        frequency_label = "annual"
+    else:
+        raise ValueError(f"Frequency '{frequency}' not recognized")
+    interval_object = duration_parser.parse(interval)
+    return frequency_label + '_' + str(interval_object.years) + 'yr'
+
+
+def check_glob(target: str) -> None:
+    """
+    Verify that at least one file is resolved by the glob.
+    Raises FileNotFoundError if no files are found.
+
+    :param target: Glob target to resolve
+    :type target: str
+    :rtype: None
+    """
+    files = glob.glob(target)
+    if len(files) >= 1:
+        fre_logger.debug(f"{target} has {len(files)} files")
+    else:
+        raise FileNotFoundError(f"{target} resolves to no files")
+
+
+def combine(root_in_dir: str, root_out_dir: str, component: str, begin: str, end: str, frequency: str, interval: str) -> None:
+    """
+    Combine per-variable climatologies into one file.
+
+    :param root_in_dir: Root timeaverage shards directory, up to the "av"
+    :param type: str
+    :param root_out_dir: Root output postprocess directory, up to the "pp"
+    :param type: str
+    :param component: Component to process
+    :param type: str
+    :param begin: Beginning of the climatology
+    :param type: YYYY
+    :param end: Ending of the climatology
+    :param type: YYYY
+    :param frequency: Sampling type of the climatology
+    :param type: 'mon' or 'yr'
+    :param interval: Length of the climatology
+    :param type: ISO8601 duration
+    :rtype: None
+    """
+    if frequency == "yr":
+        frequency_iso = "P1Y"
+    elif frequency == "mon":
+        frequency_iso = "P1M"
+    else:
+        raise ValueError(f"Frequency '{frequency}' not known")
+    outdir = Path(root_out_dir) / component / "av" / form_bronx_directory_name(frequency, interval)
+    fre_logger.debug(f"Output dir = '{outdir}'")
+    outdir.mkdir(exist_ok=True, parents=True)
+
+    if begin == end:
+        date_string = f"{begin:04d}"
+    else:
+        date_string = f"{begin:04d}-{end:04d}"
+
+    indir = Path(root_in_dir) / frequency_iso / interval
+    fre_logger.debug(f"Input dir = '{indir}'")
+    gotta_go_back_here = os.getcwd()
+    os.chdir(indir)
+
+    if frequency == 'yr':
+        source = component + '.' + date_string + '.*.nc'
+        target = component + '.' + date_string + '.nc'
+        check_glob(source)
+        subprocess.run(['cdo', '-O', 'merge', source, target], check=True)
+        fre_logger.debug(f"Output file created: {target}")
+        fre_logger.debug(f"Copying to {outdir}")
+        subprocess.run(['cp', '-v', target, outdir], check=True)
+    elif frequency == 'mon':
+        for MM in range(1,13):
+            source = f"{component}.{date_string}.*.{MM:02d}.nc"
+            target = f"{component}.{date_string}.{MM:02d}.nc"
+            check_glob(source)
+            subprocess.run(['cdo', '-O', 'merge', source, target], check=True)
+            fre_logger.debug(f"Output file created: {target}")
+            fre_logger.debug(f"Copying to {outdir}")
+            subprocess.run(['cp', '-v', target, outdir], check=True)
+    else:
+        raise ValueError(f"Frequency '{frequency}' not known")
+
+    os.chdir(gotta_go_back_here)

fre/app/generate_time_averages/frenctoolsTimeAverager.py

@@ -7,6 +7,7 @@
 import logging
 import subprocess
 import shutil
+from pathlib import Path
 
 fre_logger = logging.getLogger(__name__)
 
@@ -50,7 +51,7 @@ def generate_timavg(self, infile=None, outfile=None):
         #Recursive call if month is selcted for climatology. by Avery Kiihne
         if self.avg_type == 'month':
             monthly_nc_dir = f"monthly_nc_files"    #Folder that new monthly input files are put 
-            output_dir = f"monthly_output_files"    #Folder for the results, split by month
+            output_dir = Path(outfile).parent       #Save output in the user-specified location
             os.makedirs(monthly_nc_dir, exist_ok=True)   #create directory if it does not exist
             os.makedirs(output_dir, exist_ok=True)
             #Extract unique months from the infile 
@@ -59,7 +60,7 @@ def generate_timavg(self, infile=None, outfile=None):
 
             #Dictionary to store output filenames by month
             nc_month_file_paths = {month_index: os.path.join(monthly_nc_dir, f"all_years.{month_index}.nc") for month_index in month_indices}
-            month_output_file_paths = {month_index: os.path.join(output_dir, f"{outfile}_.{month_index}.nc") for month_index in month_indices}
+            month_output_file_paths = {month_index: os.path.join(output_dir, f"{Path(outfile).stem}.{month_index:02d}.nc") for month_index in month_indices}
 
             cdo = Cdo()
             #Loop through each month and select the corresponding data

fre/app/generate_time_averages/generate_time_averages.py

@@ -15,10 +15,11 @@ def generate_time_average(infile = None, outfile = None,
     #Use cdo to merge multiple files if present
     merged = False
     if type(infile).__name__=='list' and len(infile)> 1:   #multiple files case. Generates one combined file
+        infile_str = [str(item) for item in infile]
         from cdo import Cdo
         _cdo=Cdo()
         merged_file = "merged_output.nc"
-        _cdo.mergetime(input=' '.join(infile), output=merged_file)
+        _cdo.mergetime(input=' '.join(infile_str), output=merged_file)
         multi_file = infile   #preserve the original file names for later
         infile = merged_file
         merged = True

fre/app/generate_time_averages/tests/test_combine.py

@@ -0,0 +1,98 @@
+import pytest
+import subprocess
+from pathlib import Path
+import xarray as xr
+#import metomi.isodatetimeparsers.parsers
+
+from .. import combine
+
+@pytest.fixture()
+def create_annual_per_variable_climatologies(tmp_path):
+    """
+    Create per-variable climatologies.
+
+    in/atmos/P1Y/P2Y/alb_sfc.nc
+    in/atmos/P1Y/P2Y/aliq.nc
+    """
+    # path to input files
+    input_file_dir = Path('fre/tests/test_files/climatology/outputs/annual')
+    input_files = [
+        input_file_dir / 'atmos.1980-1981.alb_sfc.cdl',
+        input_file_dir / 'atmos.1980-1981.aliq.cdl'
+    ]
+    for file_ in input_files:
+        assert file_.exists()
+
+    # write netcdf files
+    output_dir = Path(tmp_path, 'in', 'atmos', 'P1Y', 'P2Y')
+    output_dir.mkdir(parents=True)
+
+    # write netcdfs from the cdfs
+    for file_ in input_files:
+        output_file = output_dir / file_.stem
+        output_file = Path(str(output_file) + '.nc')
+        command = ['ncgen', '-o', output_file, file_]
+        sp = subprocess.run(command, check=True)
+        assert sp.returncode == 0
+        assert output_file.exists()
+
+    yield tmp_path
+
+@pytest.fixture()
+def create_monthly_per_variable_climatologies(tmp_path):
+    """
+    Create per-variable climatologies.
+
+    in/atmos/P1M/P2Y/alb_sfc.[01-12].nc
+    in/atmos/P1M/P2Y/aliq.[01-12].nc
+    """
+    # path to input CDL files
+    input_dir = Path('fre/tests/test_files/climatology/outputs/monthly')
+
+    # two variables
+    input_basenames = [
+        'atmos.1980-1981.alb_sfc',
+        'atmos.1980-1981.aliq'
+    ]
+
+    # path to test output NC files
+    output_dir = Path(tmp_path, 'in', 'atmos', 'P1M', 'P2Y')
+    output_dir.mkdir(parents=True)
+
+    # write netcdf files
+    for i in range(1,13):
+        for name in input_basenames:
+            input_file = input_dir / f"{name}.{i:02d}.cdl"
+            assert input_file.exists()
+
+            output_file = output_dir / f"{name}.{i:02d}.nc"
+            command = ['ncgen', '-o', output_file, input_file]
+            sp = subprocess.run(command, check=True)
+            assert sp.returncode == 0
+            assert output_file.exists()
+
+    yield tmp_path
+
+def test_combine_annual_av(create_annual_per_variable_climatologies):
+    """
+    Combine per-variable annual climatologies into combined annual climatology file
+    """
+
+    combine.combine(create_annual_per_variable_climatologies / 'in' / 'atmos', create_annual_per_variable_climatologies / 'out', 'atmos', 1980, 1981, 'yr', 'P2Y')
+
+    output_dir = Path(create_annual_per_variable_climatologies, 'out', 'atmos', 'av', 'annual_2yr')
+    output_file = output_dir / 'atmos.1980-1981.nc'
+
+    assert output_file.exists()
+
+def test_combine_monthly_av(create_monthly_per_variable_climatologies):
+    """
+    Combine per-variable monthly climatologies into combined monthly climatology file
+    """
+
+    combine.combine(create_monthly_per_variable_climatologies / 'in' / 'atmos', create_monthly_per_variable_climatologies / 'out', 'atmos', 1980, 1981, 'mon', 'P2Y')
+
+    output_dir = Path(create_monthly_per_variable_climatologies, 'out', 'atmos', 'av', 'monthly_2yr')
+    for i in range(1,13):
+        output_file = output_dir / f'atmos.1980-1981.{i:02d}.nc'
+        assert output_file.exists()