diff --git a/dev/parm/config/gefs/config.nsst b/dev/parm/config/gefs/config.nsst.j2 similarity index 96% rename from dev/parm/config/gefs/config.nsst rename to dev/parm/config/gefs/config.nsst.j2 index f9a29d44d3a..04abadc91dc 100644 --- a/dev/parm/config/gefs/config.nsst +++ b/dev/parm/config/gefs/config.nsst.j2 @@ -11,7 +11,7 @@ echo "BEGIN: config.nsst" export NST_MODEL=2 # nstf_name(2) : NST_SPINUP : 0 = OFF, 1 = ON, -export NST_SPINUP=0 +export NST_SPINUP="{{ NST_SPINUP }}" # nstf_name(3) : NST_RESV (Reserved, NSST Analysis) : 0 = OFF, 1 = ON export NST_RESV=0 diff --git a/dev/parm/config/gefs/config.oceanice_products b/dev/parm/config/gefs/config.oceanice_products index 48483613f43..7b30de84bc5 100644 --- a/dev/parm/config/gefs/config.oceanice_products +++ b/dev/parm/config/gefs/config.oceanice_products @@ -7,9 +7,10 @@ echo "BEGIN: config.oceanice_products" # Get task specific resources source "${EXPDIR}/config.resources" oceanice_products +export write_grib2=False +export write_netcdf=False export OCEANICEPRODUCTS_CONFIG="${PARMgfs}/post/oceanice_products_${NET}.yaml" # Maximum number of rocoto tasks per member export MAX_TASKS=25 - echo "END: config.oceanice_products" diff --git a/dev/parm/config/gefs/config.ufs b/dev/parm/config/gefs/config.ufs index 04924ab1283..33c06d70d0a 100644 --- a/dev/parm/config/gefs/config.ufs +++ b/dev/parm/config/gefs/config.ufs @@ -311,7 +311,7 @@ if [[ "${skip_mom6}" == "false" ]]; then TOPOEDITS="ufs.topo_edits_011818.nc" case ${RUN} in gfs|gefs|sfs) - MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" + MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_30L.nc" MOM6_DIAG_MISVAL="-1e34" ;; gdas|enkfgdas|enkfgfs) @@ -339,7 +339,7 @@ if [[ "${skip_mom6}" == "false" ]]; then eps_imesh="1.0e-1" case ${RUN} in gfs|gefs|sfs) - MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" + MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_30L.nc" MOM6_DIAG_MISVAL="-1e34" ;; gdas|enkfgdas|enkfgfs) @@ -368,7 +368,7 @@ if [[ "${skip_mom6}" == "false" ]]; then eps_imesh="1.0e-1" case ${RUN} in gfs|gefs|sfs) - MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" + MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_30L.nc" MOM6_DIAG_MISVAL="-1e34" ;; gdas|enkfgdas|enkfgfs) diff --git a/dev/parm/config/gefs/yaml/defaults.yaml b/dev/parm/config/gefs/yaml/defaults.yaml index e20890d5582..b302a8bf30b 100644 --- a/dev/parm/config/gefs/yaml/defaults.yaml +++ b/dev/parm/config/gefs/yaml/defaults.yaml @@ -29,6 +29,8 @@ fcst: MONO: "non-mono" ocn: MOM6_INTERP_ICS: "NO" +nsst: + NST_SPINUP: 0 # config.aero has just a system-specific path to add. # This is handled by the setup_expt.py, but it has to be told to write to it. aero: {} diff --git a/dev/parm/config/gfs/config.oceanice_products b/dev/parm/config/gfs/config.oceanice_products index 6cce26997ce..7f1d94bf0b9 100644 --- a/dev/parm/config/gfs/config.oceanice_products +++ b/dev/parm/config/gfs/config.oceanice_products @@ -10,6 +10,8 @@ source "${EXPDIR}/config.resources" oceanice_products # Maximum number of rocoto tasks per member export MAX_TASKS=25 +export write_grib2=False +export write_netcdf=False export OCEANICEPRODUCTS_CONFIG="${PARMgfs}/post/oceanice_products_gfs.yaml" # No. of forecast hours to process in a single job diff --git a/dev/parm/config/gfs/config.ufs b/dev/parm/config/gfs/config.ufs index 6c883d66bf7..411b0461b46 100644 --- a/dev/parm/config/gfs/config.ufs +++ b/dev/parm/config/gfs/config.ufs @@ -451,7 +451,7 @@ if [[ "${skip_mom6}" == "false" ]]; then TOPOEDITS="ufs.topo_edits_011818.nc" case ${RUN} in gfs|gefs|sfs|gcafs) - MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" + MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_30L.nc" MOM6_DIAG_MISVAL="-1e34" ;; gdas|enkfgdas|enkfgfs) @@ -479,7 +479,7 @@ if [[ "${skip_mom6}" == "false" ]]; then eps_imesh="1.0e-1" case ${RUN} in gfs|gefs|sfs|gcafs) - MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" + MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_30L.nc" MOM6_DIAG_MISVAL="-1e34" ;; gdas|enkfgdas|enkfgfs) @@ -516,7 +516,7 @@ if [[ "${skip_mom6}" == "false" ]]; then eps_imesh="1.0e-1" case ${RUN} in gfs|gefs|sfs|gcafs) - MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" + MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_30L.nc" MOM6_DIAG_MISVAL="-1e34" ;; gdas|enkfgdas|enkfgfs) diff --git a/dev/parm/config/sfs/config.base.j2 b/dev/parm/config/sfs/config.base.j2 index 32a26ab5f8a..bff4c3be6ce 100644 --- a/dev/parm/config/sfs/config.base.j2 +++ b/dev/parm/config/sfs/config.base.j2 @@ -318,23 +318,8 @@ export DO_METP="NO" # Run METPLUS jobs - set METPLUS settings in config. export DO_FIT2OBS="NO" # Run fit to observations package # Archiving options -export HPSSARCH="{{ HPSSARCH }}" # save data to HPSS archive -export LOCALARCH="{{ LOCALARCH }}" # save data to local archive -if [[ ${HPSSARCH} = "YES" ]] && [[ ${LOCALARCH} = "YES" ]]; then - echo "Both HPSS and local archiving selected. Please choose one or the other." - exit 3 -elif [[ ${HPSSARCH} = "YES" ]] || [[ ${LOCALARCH} = "YES" ]]; then - export DO_ARCHCOM="YES" -else - export DO_ARCHCOM="NO" -fi -export ARCH_CYC=00 # Archive data at this cycle for warm start and/or forecast-only capabilities -export ARCH_WARMICFREQ=4 # Archive frequency in days for warm start capability -export ARCH_FCSTICFREQ=1 # Archive frequency in days for gdas and gfs forecast-only capability -export ARCH_EXPDIR='YES' # Archive the EXPDIR configs, XML, and database -export ARCH_EXPDIR_FREQ=0 # How often to archive the EXPDIR in hours or 0 for first and last cycle only -export ARCH_HASHES='YES' # Archive the hashes of the GW and submodules and 'git status' for each; requires ARCH_EXPDIR -export ARCH_DIFFS='NO' # Archive the output of 'git diff' for the GW; requires ARCH_EXPDIR +export DO_ARCHCOM="NO" # Tar and archive the COM directories +export ARCHCOM_TO="{{ ARCHCOM_TO }}" # Valid options are hpss, globus_hpss, and local # Number of regional collectives to create soundings for export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9} diff --git a/dev/parm/config/sfs/config.nsst b/dev/parm/config/sfs/config.nsst deleted file mode 120000 index 3a015c1d65f..00000000000 --- a/dev/parm/config/sfs/config.nsst +++ /dev/null @@ -1 +0,0 @@ -../gefs/config.nsst \ No newline at end of file diff --git a/dev/parm/config/sfs/config.nsst.j2 b/dev/parm/config/sfs/config.nsst.j2 new file mode 100644 index 00000000000..137e785eb65 --- /dev/null +++ b/dev/parm/config/sfs/config.nsst.j2 @@ -0,0 +1,32 @@ +#! /usr/bin/env bash + +########## config.nsst ########## +# NSST specific + +echo "BEGIN: config.nsst" + +# NSST parameters contained within nstf_name + +# nstf_name(1) : NST_MODEL (NSST Model) : 0 = OFF, 1 = ON but uncoupled, 2 = ON and coupled +export NST_MODEL=2 + +# nstf_name(2) : NST_SPINUP : 0 = OFF, 1 = ON, +export NST_SPINUP="{{ NST_SPINUP }}" + +# nstf_name(3) : NST_RESV (Reserved, NSST Analysis) : 0 = OFF, 1 = ON +export NST_RESV=0 + +# nstf_name(4,5) : ZSEA1, ZSEA2 the two depths to apply vertical average (bias correction) +export ZSEA1=0 +export ZSEA2=0 + +export NST_GSI=3 # default 0: No NST info at all; + # 1: Input NST info but not used in GSI; + # 2: Input NST info, used in CRTM simulation, no Tr analysis + # 3: Input NST info, used in both CRTM simulation and Tr analysis +export NSTINFO=0 # number of elements added in obs. data array (default = 0) +if [[ ${NST_GSI} -gt 0 ]]; then + export NSTINFO=4 +fi + +echo "END: config.nsst" diff --git a/dev/parm/config/sfs/yaml/defaults.yaml b/dev/parm/config/sfs/yaml/defaults.yaml index c322c9a0142..305183fdf88 100644 --- a/dev/parm/config/sfs/yaml/defaults.yaml +++ b/dev/parm/config/sfs/yaml/defaults.yaml @@ -29,6 +29,8 @@ fcst: MONO: "mono" ocn: MOM6_INTERP_ICS: "NO" +nsst: + NST_SPINUP: 0 # config.aero has just a system-specific path to add. # This is handled by the setup_expt.py, but it has to be told to write to it. aero: {} diff --git a/dev/workflow/applications/sfs.py b/dev/workflow/applications/sfs.py index d08e028fdcc..89c6f4e9788 100644 --- a/dev/workflow/applications/sfs.py +++ b/dev/workflow/applications/sfs.py @@ -93,6 +93,8 @@ def _get_app_configs(self, run): if options['do_archcom']: configs += ['arch_tars'] + if options['do_globusarch']: + configs += ['globus'] configs += ['arch_vrfy', 'cleanup'] @@ -164,7 +166,12 @@ def get_task_names(self): if options['do_extractvars']: tasks += ['extractvars'] - # TODO: Add archive + # TODO add archiving for SFS + # if options['do_archcom']: + # tasks += ['arch_tars'] + # if options['do_globusarch']: + # tasks += ['globus'] + tasks += ['cleanup'] return {f"{self.run}": tasks} diff --git a/dev/workflow/rocoto/gfs_tasks.py b/dev/workflow/rocoto/gfs_tasks.py index df2467a85fd..cc0ac3c80a3 100644 --- a/dev/workflow/rocoto/gfs_tasks.py +++ b/dev/workflow/rocoto/gfs_tasks.py @@ -2228,12 +2228,12 @@ def arch_tars(self): tarball_types.append('chem') if self.options['do_ocean']: - tarball_types.extend(['ocean_6hravg', 'ocean_grib2', 'gfs_flux_1p00']) + tarball_types.extend(['ocean_6hravg', 'ocean_native', 'gfs_flux_1p00']) if self.options.get('do_jediocnvar', False) and self.app_config.mode == 'cycled': tarball_types.append('gfsocean_analysis') if self.options['do_ice']: - tarball_types.extend(['ice_6hravg', 'ice_grib2']) + tarball_types.extend(['ice_6hravg', 'ice_native']) if self.options['do_bufrsnd']: tarball_types.append('gfs_downstream') diff --git a/dev/workflow/rocoto/sfs_tasks.py b/dev/workflow/rocoto/sfs_tasks.py index 9113a5d9bb4..2a8906e073e 100644 --- a/dev/workflow/rocoto/sfs_tasks.py +++ b/dev/workflow/rocoto/sfs_tasks.py @@ -612,6 +612,30 @@ def arch_tars(self): return task + # Globus transfer for HPSS archiving + def globus(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.run}_arch_tars'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + resources = self.get_resource('globus') + task_name = f'{self.run}_globus_arch' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.run, + 'command': f'{self.HOMEgfs}/dev/jobs/globus_arch.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + def cleanup(self): deps = [] dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_prod'} @@ -637,6 +661,13 @@ def cleanup(self): if self.options['do_extractvars']: dep_dict = {'type': 'metatask', 'name': f'{self.run}_extractvars'} deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_archcom']: + if self.options['do_globusarch']: + dep_dict = {'type': 'task', 'name': f'{self.run}_globus_arch'} + else: + dep_dict = {'type': 'task', 'name': f'{self.run}_arch_tars'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) dependencies = rocoto.create_dependency(dep=deps, dep_condition='and') resources = self.get_resource('cleanup') task_name = f'{self.run}_cleanup' diff --git a/jobs/JGLOBAL_ARCHIVE_TARS b/jobs/JGLOBAL_ARCHIVE_TARS index 9614ecb18e2..67b45b993ab 100755 --- a/jobs/JGLOBAL_ARCHIVE_TARS +++ b/jobs/JGLOBAL_ARCHIVE_TARS @@ -32,6 +32,7 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_ICE_INPUT:COM_ICE_INPUT_TMPL \ COMIN_ICE_RESTART:COM_ICE_RESTART_TMPL \ COMIN_ICE_GRIB:COM_ICE_GRIB_TMPL \ + COMIN_ICE_NETCDF:COM_ICE_NETCDF_TMPL \ COMIN_OBS:COM_OBS_TMPL \ COMIN_TOP:COM_TOP_TMPL \ COMIN_OCEAN_HISTORY:COM_OCEAN_HISTORY_TMPL \ diff --git a/parm/archive/ice_native.yaml.j2 b/parm/archive/ice_native.yaml.j2 new file mode 100644 index 00000000000..546650a0225 --- /dev/null +++ b/parm/archive/ice_native.yaml.j2 @@ -0,0 +1,7 @@ +ice_native: + name: "ICE_NATIVE" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_native_subset.tar" + required: + {% for fhr in range(FHOUT_ICE_GFS, FHMAX_GFS + FHOUT_ICE_GFS, FHOUT_ICE_GFS) %} + - "{{ COMIN_ICE_NETCDF | relpath(ROTDIR) }}/native/{{ RUN }}.t{{ cycle_HH }}z.native.f{{ '%03d' % fhr }}.nc" + {% endfor %} diff --git a/parm/archive/ocean_native.yaml.j2 b/parm/archive/ocean_native.yaml.j2 new file mode 100644 index 00000000000..51c039c06bb --- /dev/null +++ b/parm/archive/ocean_native.yaml.j2 @@ -0,0 +1,7 @@ +ocean_native: + name: "OCEAN_NATIVE" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_native_subset.tar" + required: + {% for fhr in range(FHOUT_OCN_GFS, FHMAX_GFS + FHOUT_OCN_GFS, FHOUT_OCN_GFS) %} + - "{{ COMIN_OCEAN_NETCDF | relpath(ROTDIR) }}/native/{{ RUN }}.t{{ cycle_HH }}z.native.f{{ '%03d' % fhr }}.nc" + {% endfor %} diff --git a/parm/post/oceanice_products_gefs.yaml b/parm/post/oceanice_products_gefs.yaml index b2651fdc331..a797efa6364 100644 --- a/parm/post/oceanice_products_gefs.yaml +++ b/parm/post/oceanice_products_gefs.yaml @@ -1,16 +1,17 @@ ocnicepost: executable: "ocnicepost.x" namelist: - write_grib2: True - write_netcdf: False + write_grib2: {{ write_grib2 }} + write_netcdf: {{ write_netcdf }} debug: False fix_data: mkdir: - "{{ DATA }}" copy: - - ["{{ EXECgfs }}/ocnicepost.x", "{{ DATA }}/"] - ["{{ PARMgfs }}/post/ocnicepost.nml.jinja2", "{{ DATA }}/"] - - ["{{ PARMgfs }}/post/{{ component }}_gefs.csv", "{{ DATA }}/{{ component }}.csv"] + {% if write_grib2 or write_netcdf %} + - ["{{ PARMgfs }}/post/{{ component }}_{{ RUN }}.csv", "{{ DATA }}/{{ component }}.csv"] + - ["{{ EXECgfs }}/ocnicepost.x", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Bu.to.Ct.bilinear.nc", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Cu.to.Ct.bilinear.nc", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Cv.to.Ct.bilinear.nc", "{{ DATA }}/"] @@ -19,6 +20,7 @@ ocnicepost: - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Ct.to.rect.{{ grid }}.conserve.nc", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/template.global.{{ grid }}.gb2", "{{ DATA }}/"] {% endfor %} + {% endif %} ocean: namelist: @@ -28,11 +30,15 @@ ocean: cosvar: "cos_rot" angvar: "" {% if model_grid == 'mx025' or model_grid == 'mx050' or model_grid == 'mx100' %} - ocean_levels: [5, 15, 25, 35, 45, 55, 65, 75, 85, 95, 105, 115, 125, 135, 145, 155, 165, 175, 185, 195, 205, 215, 226, 241, 267, 309, 374, 467, 594, 757, 960, 1204, 1490, 1817, 2184, 2587, 3024, 3489, 3977, 4481] + ocean_levels: [1, 3, 5, 10, 20, 30, 50, 100, 200, 500, 1000] {% elif model_grid == 'mx500' %} - ocean_levels: [5, 15, 25, 35, 45, 55, 65, 75, 85, 95, 105, 115, 125, 135, 145, 155, 165, 175, 185, 195, 205, 215, 226, 241, 267] + ocean_levels: [2.5, 7.5, 12.5, 17.5, 116] {% endif %} - subset: ['SSH', 'SST', 'SSS', 'speed', 'MLD_003', 'latent', 'sensible', 'SW', 'LW', 'LwLatSens', 'Heat_PmE', 'SSU', 'SSV', 'taux', 'tauy', 'temp', 'tob', 'so', 'uo', 'vo'] + subset: + variables: ['SSH', 'SST', 'SSS', 'speed', 'MLD_003', 'latent', 'sensible', 'SW', 'LW', 'LwLatSens', 'Heat_PmE', 'SSU', 'SSV', 'taux', 'tauy', 'temp', 'tob', 'so', 'uo', 'vo'] + compress_with: 'zlib' + compress: True + compress_level: 9 data_in: copy: - ["{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ocean.nc"] @@ -40,14 +46,23 @@ ocean: mkdir: - "{{ COMOUT_OCEAN_NETCDF }}/native" {% for grid in product_grids %} + {% if write_netcdf %} - "{{ COMOUT_OCEAN_NETCDF }}/{{ grid }}" + {% endif %} + {% if write_grib2 %} - "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}" + {% endif %} {% endfor %} copy: - - ["{{ DATA }}/ocean_subset.nc", "{{ COMOUT_OCEAN_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.tripolar.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ocean_subset.nc", "{{ COMOUT_OCEAN_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] {% for grid in product_grids %} + {% if write_netcdf %} + - ["{{ DATA }}/ocean.{{ grid }}.nc", "{{ COMOUT_OCEAN_NETCDF }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.nc"] + {% endif %} + {% if write_grib2 %} - ["{{ DATA }}/ocean.{{ grid }}.grib2", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] - ["{{ DATA }}/ocean.{{ grid }}.grib2.idx", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] + {% endif %} {% endfor %} ice: @@ -57,7 +72,11 @@ ice: sinvar: "" cosvar: "" angvar: "ANGLET" - subset: ['hi_1', 'hs_1', 'aice_1', 'Tsfc_1', 'uvel_1', 'vvel_1', 'frzmlt_1', 'albsni_1'] + subset: + variables: ['hi_1', 'hs_1', 'aice_1', 'Tsfc_1', 'uvel_1', 'vvel_1', 'frzmlt_1', 'albsni_1', 'tmask', 'tarea'] + compress_with: 'zlib' + compress: True + compress_level: 9 data_in: copy: - ["{{ COMIN_ICE_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ice.nc"] @@ -65,12 +84,21 @@ ice: mkdir: - "{{ COMOUT_ICE_NETCDF }}/native" {% for grid in product_grids %} + {% if write_netcdf %} - "{{ COMOUT_ICE_NETCDF }}/{{ grid }}" + {% endif %} + {% if write_grib2 %} - "{{ COMOUT_ICE_GRIB }}/{{ grid }}" + {% endif %} {% endfor %} copy: - - ["{{ DATA }}/ice_subset.nc", "{{ COMOUT_ICE_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.tripolar.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ice_subset.nc", "{{ COMOUT_ICE_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] {% for grid in product_grids %} + {% if write_netcdf %} + - ["{{ DATA }}/ice.{{ grid }}.nc", "{{ COMOUT_ICE_NETCDF }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.nc"] + {% endif %} + {% if write_grib2 %} - ["{{ DATA }}/ice.{{ grid }}.grib2", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] - ["{{ DATA }}/ice.{{ grid }}.grib2.idx", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] + {% endif %} {% endfor %} diff --git a/parm/post/oceanice_products_gfs.yaml b/parm/post/oceanice_products_gfs.yaml index 9604b71560e..6d8355fb491 100644 --- a/parm/post/oceanice_products_gfs.yaml +++ b/parm/post/oceanice_products_gfs.yaml @@ -1,16 +1,17 @@ ocnicepost: executable: "ocnicepost.x" namelist: - write_grib2: True - write_netcdf: False + write_grib2: {{ write_grib2 }} + write_netcdf: {{ write_netcdf }} debug: False fix_data: mkdir: - "{{ DATA }}" copy: - - ["{{ EXECgfs }}/ocnicepost.x", "{{ DATA }}/"] - ["{{ PARMgfs }}/post/ocnicepost.nml.jinja2", "{{ DATA }}/"] - - ["{{ PARMgfs }}/post/{{ component }}_gfs.csv", "{{ DATA }}/{{ component }}.csv"] + {% if write_grib2 or write_netcdf %} + - ["{{ PARMgfs }}/post/{{ component }}_{{ RUN }}.csv", "{{ DATA }}/{{ component }}.csv"] + - ["{{ EXECgfs }}/ocnicepost.x", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Bu.to.Ct.bilinear.nc", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Cu.to.Ct.bilinear.nc", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Cv.to.Ct.bilinear.nc", "{{ DATA }}/"] @@ -19,6 +20,7 @@ ocnicepost: - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Ct.to.rect.{{ grid }}.conserve.nc", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/template.global.{{ grid }}.gb2", "{{ DATA }}/"] {% endfor %} + {% endif %} ocean: namelist: @@ -28,25 +30,39 @@ ocean: cosvar: "cos_rot" angvar: "" {% if model_grid == 'mx025' or model_grid == 'mx050' or model_grid == 'mx100' %} - ocean_levels: [5, 15, 25, 35, 45, 55, 65, 75, 85, 95, 105, 115, 125, 135, 145, 155, 165, 175, 185, 195, 205, 215, 226, 241, 267, 309, 374, 467, 594, 757, 960, 1204, 1490, 1817, 2184, 2587, 3024, 3489, 3977, 4481] + ocean_levels: [1, 3, 5, 10, 20, 30, 50, 100, 200, 500, 1000] {% elif model_grid == 'mx500' %} - ocean_levels: [5, 15, 25, 35, 45, 55, 65, 75, 85, 95, 105, 115, 125, 135, 145, 155, 165, 175, 185, 195, 205, 215, 226, 241, 267] + ocean_levels: [2.5, 7.5, 12.5, 17.5, 116] {% endif %} - subset: ['SSH', 'SST', 'SSS', 'speed', 'MLD_003', 'latent', 'sensible', 'SW', 'LW', 'LwLatSens', 'Heat_PmE', 'SSU', 'SSV', 'taux', 'tauy', 'temp', 'so', 'uo', 'vo'] + subset: + variables: ['SSH', 'MLD_003', 'SST', 'SSS', 'temp', 'tob', 'so', 'uo', 'vo'] + compress_with: 'zlib' + compress: True + compress_level: 9 data_in: copy: - - ["{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.6hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ocean.nc"] + - ["{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ocean.nc"] data_out: mkdir: - - "{{ COMOUT_OCEAN_NETCDF }}" + - "{{ COMOUT_OCEAN_NETCDF }}/native" {% for grid in product_grids %} + {% if write_netcdf %} + - "{{ COMOUT_OCEAN_NETCDF }}/{{ grid }}" + {% endif %} + {% if write_grib2 %} - "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}" + {% endif %} {% endfor %} copy: - - ["{{ DATA }}/ocean_subset.nc", "{{ COMOUT_OCEAN_NETCDF }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.tripolar.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ocean_subset.nc", "{{ COMOUT_OCEAN_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] {% for grid in product_grids %} + {% if write_netcdf %} + - ["{{ DATA }}/ocean.{{ grid }}.nc", "{{ COMOUT_OCEAN_NETCDF }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.nc"] + {% endif %} + {% if write_grib2 %} - ["{{ DATA }}/ocean.{{ grid }}.grib2", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] - ["{{ DATA }}/ocean.{{ grid }}.grib2.idx", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] + {% endif %} {% endfor %} ice: @@ -56,19 +72,33 @@ ice: sinvar: "" cosvar: "" angvar: "ANGLET" - subset: ['hi_h', 'hs_h', 'aice_h', 'Tsfc_h', 'uvel_h', 'vvel_h', 'frzmlt_h', 'albsni_h'] + subset: + variables: ['hi_h', 'hs_h', 'aice_h', 'Tsfc_h', 'uvel_h', 'vvel_h', 'albsni_h', 'tarea', 'tmask'] + compress_with: 'zlib' + compress: True + compress_level: 9 data_in: copy: - - ["{{ COMIN_ICE_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.6hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ice.nc"] + - ["{{ COMIN_ICE_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ice.nc"] data_out: mkdir: - - "{{ COMOUT_ICE_NETCDF }}" + - "{{ COMOUT_ICE_NETCDF }}/native" {% for grid in product_grids %} + {% if write_netcdf %} + - "{{ COMOUT_ICE_NETCDF }}/{{ grid }}" + {% endif %} + {% if write_grib2 %} - "{{ COMOUT_ICE_GRIB }}/{{ grid }}" + {% endif %} {% endfor %} copy: - - ["{{ DATA }}/ice_subset.nc", "{{ COMOUT_ICE_NETCDF }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.tripolar.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ice_subset.nc", "{{ COMOUT_ICE_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] {% for grid in product_grids %} + {% if write_netcdf %} + - ["{{ DATA }}/ice.{{ grid }}.nc", "{{ COMOUT_ICE_NETCDF }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.nc"] + {% endif %} + {% if write_grib2 %} - ["{{ DATA }}/ice.{{ grid }}.grib2", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] - ["{{ DATA }}/ice.{{ grid }}.grib2.idx", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] + {% endif %} {% endfor %} diff --git a/parm/post/oceanice_products_sfs.yaml b/parm/post/oceanice_products_sfs.yaml index e38e545dce3..086af354844 100644 --- a/parm/post/oceanice_products_sfs.yaml +++ b/parm/post/oceanice_products_sfs.yaml @@ -1,16 +1,17 @@ ocnicepost: executable: "ocnicepost.x" namelist: - write_grib2: True - write_netcdf: False + write_grib2: {{ write_grib2 }} + write_netcdf: {{ write_netcdf }} debug: False fix_data: mkdir: - "{{ DATA }}" copy: - - ["{{ EXECgfs }}/ocnicepost.x", "{{ DATA }}/"] - ["{{ PARMgfs }}/post/ocnicepost.nml.jinja2", "{{ DATA }}/"] - - ["{{ PARMgfs }}/post/{{ component }}_gfs.csv", "{{ DATA }}/{{ component }}.csv"] + {% if write_grib2 or write_netcdf %} + - ["{{ PARMgfs }}/post/{{ component }}_{{ RUN }}.csv", "{{ DATA }}/{{ component }}.csv"] + - ["{{ EXECgfs }}/ocnicepost.x", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Bu.to.Ct.bilinear.nc", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Cu.to.Ct.bilinear.nc", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Cv.to.Ct.bilinear.nc", "{{ DATA }}/"] @@ -19,6 +20,7 @@ ocnicepost: - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Ct.to.rect.{{ grid }}.conserve.nc", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/template.global.{{ grid }}.gb2", "{{ DATA }}/"] {% endfor %} + {% endif %} ocean: namelist: @@ -28,11 +30,15 @@ ocean: cosvar: "cos_rot" angvar: "" {% if model_grid == 'mx025' or model_grid == 'mx050' or model_grid == 'mx100' %} - ocean_levels: [5, 15, 25, 35, 45, 55, 65, 75, 85, 95, 105, 115, 125, 135, 145, 155, 165, 175, 185, 195, 205, 215, 226, 241, 267, 309, 374, 467, 594, 757, 960, 1204, 1490, 1817, 2184, 2587, 3024, 3489, 3977, 4481] + ocean_levels: [1, 3, 5, 10, 20, 30, 50, 100, 200, 500, 1000] {% elif model_grid == 'mx500' %} ocean_levels: [5, 15, 25, 35, 45, 55, 65, 75, 85, 95, 105, 115, 125, 135, 145, 155, 165, 175, 185, 195, 205, 215, 226, 241, 267] {% endif %} - subset: ['SSH', 'SST', 'SSS', 'speed', 'MLD_003', 'latent', 'sensible', 'SW', 'LW', 'LwLatSens', 'Heat_PmE', 'SSU', 'SSV', 'taux', 'tauy', 'temp', 'tob', 'so', 'uo', 'vo'] + subset: + variables: ['SSH', 'MLD_003', 'temp', 'tob', 'so', 'uo', 'vo'] + compress_with: 'zlib' + compress: True + compress_level: 9 data_in: copy: - ["{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ocean.nc"] @@ -40,14 +46,23 @@ ocean: mkdir: - "{{ COMOUT_OCEAN_NETCDF }}/native" {% for grid in product_grids %} + {% if write_netcdf %} - "{{ COMOUT_OCEAN_NETCDF }}/{{ grid }}" + {% endif %} + {% if write_grib2 %} - "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}" + {% endif %} {% endfor %} copy: - - ["{{ DATA }}/ocean_subset.nc", "{{ COMOUT_OCEAN_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.tripolar.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ocean_subset.nc", "{{ COMOUT_OCEAN_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] {% for grid in product_grids %} + {% if write_netcdf %} + - ["{{ DATA }}/ocean.{{ grid }}.nc", "{{ COMOUT_OCEAN_NETCDF }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.nc"] + {% endif %} + {% if write_grib2 %} - ["{{ DATA }}/ocean.{{ grid }}.grib2", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] - ["{{ DATA }}/ocean.{{ grid }}.grib2.idx", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] + {% endif %} {% endfor %} ice: @@ -57,7 +72,11 @@ ice: sinvar: "" cosvar: "" angvar: "ANGLET" - subset: ['hi_h', 'hs_h', 'aice_h', 'Tsfc_h', 'uvel_h', 'vvel_h', 'frzmlt_h', 'albsni_h'] + subset: + variables: ['hi_h', 'hs_h', 'aice_h', 'Tsfc_h', 'uvel_h', 'vvel_h', 'albsni_h', 'tmask', 'tarea'] + compress_with: 'zlib' + compress: True + compress_level: 9 data_in: copy: - ["{{ COMIN_ICE_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ice.nc"] @@ -65,12 +84,21 @@ ice: mkdir: - "{{ COMOUT_ICE_NETCDF }}/native" {% for grid in product_grids %} + {% if write_netcdf %} - "{{ COMOUT_ICE_NETCDF }}/{{ grid }}" + {% endif %} + {% if write_grib2 %} - "{{ COMOUT_ICE_GRIB }}/{{ grid }}" + {% endif %} {% endfor %} copy: - - ["{{ DATA }}/ice_subset.nc", "{{ COMOUT_ICE_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.tripolar.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ice_subset.nc", "{{ COMOUT_ICE_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] {% for grid in product_grids %} + {% if write_netcdf %} + - ["{{ DATA }}/ice.{{ grid }}.nc", "{{ COMOUT_ICE_NETCDF }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.nc"] + {% endif %} + {% if write_grib2 %} - ["{{ DATA }}/ice.{{ grid }}.grib2", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] - ["{{ DATA }}/ice.{{ grid }}.grib2.idx", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] + {% endif %} {% endfor %} diff --git a/parm/ufs/fv3/diag_table b/parm/ufs/fv3/diag_table index ba4f9c793d5..ffc60816b83 100644 --- a/parm/ufs/fv3/diag_table +++ b/parm/ufs/fv3/diag_table @@ -49,10 +49,10 @@ "ocean_model", "SW", "SW", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 "ocean_model", "LW", "LW", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 "ocean_model", "evap", "evap", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 -"ocean_model", "lprec", "lprec", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +#"ocean_model", "lprec", "lprec", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 "ocean_model", "lrunoff", "lrunoff", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 #"ocean_model", "frunoff", "frunoff", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 -"ocean_model", "fprec", "fprec", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +#"ocean_model", "fprec", "fprec", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 "ocean_model", "LwLatSens", "LwLatSens", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 "ocean_model", "Heat_PmE", "Heat_PmE", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 diff --git a/ush/python/pygfs/task/oceanice_products.py b/ush/python/pygfs/task/oceanice_products.py index d319608ad14..775d13e07f9 100644 --- a/ush/python/pygfs/task/oceanice_products.py +++ b/ush/python/pygfs/task/oceanice_products.py @@ -170,11 +170,13 @@ def execute(config: Dict, product_grid: str) -> None: None """ - # Run the ocnicepost.x executable - OceanIceProducts.interp(config.DATA, config.APRUN_OCNICEPOST, exec_name="ocnicepost.x") + # Run the ocnicepost.x executable if interpolated variables are wanted + if config.oceanice_yaml.ocnicepost.namelist.write_netcdf or config.oceanice_yaml.ocnicepost.namelist.write_grib2: + OceanIceProducts.interp(config.DATA, config.APRUN_OCNICEPOST, exec_name="ocnicepost.x") - # Index the interpolated grib2 file - OceanIceProducts.index(config, product_grid) + if config.oceanice_yaml.ocnicepost.namelist.write_grib2: + # Index the interpolated grib2 file + OceanIceProducts.index(config, product_grid) @staticmethod @logit(logger) @@ -274,22 +276,40 @@ def subset(config: Dict) -> None: input_file = f"{config.component}.nc" output_file = f"{config.component}_subset.nc" - varlist = config.oceanice_yaml[config.component].subset + + varlist = config.oceanice_yaml[config.component].subset.variables logger.info(f"Subsetting {varlist} from {input_file} to {output_file}") try: # open the netcdf file ds = xr.open_dataset(input_file) - - # subset the variables - ds_subset = ds[varlist] + if config.component == 'ice': + # subset the variables + ds_subset = ds[varlist] + # remove coords that were carried from original file but not used + ds_subset = ds_subset.drop_vars('ELON', errors='ignore') + ds_subset = ds_subset.drop_vars('ELAT', errors='ignore') + ds_subset = ds_subset.drop_vars('NLON', errors='ignore') + ds_subset = ds_subset.drop_vars('NLAT', errors='ignore') + + if config.component == 'ocean': + # subset ocean variables for z_levels in products + levels = config.oceanice_yaml.ocean.namelist.ocean_levels + ds_subset = ds[varlist].sel(z_l=levels) # save global attributes from the old netcdf file into new netcdf file ds_subset.attrs = ds.attrs - # save subsetted variables to a new netcdf file - ds_subset.to_netcdf(output_file) + # save subsetted variables to a new netcdf file and compress + if config.oceanice_yaml[config.component].subset.compress: + compress_with = config.oceanice_yaml[config.component].subset.compress_with + compress_level = config.oceanice_yaml[config.component].subset.compress_level + default_compression = {compress_with: True, "complevel": int(compress_level)} + compress_encoding = {var_name: default_compression for var_name in ds_subset.data_vars} + ds_subset.to_netcdf(output_file, encoding=compress_encoding) + else: + ds_subset.to_netcdf(output_file) except FileNotFoundError: logger.exception(f"FATAL ERROR: Input file not found: {input_file}") @@ -326,6 +346,5 @@ def finalize(config: Dict) -> None: # Copy "component" specific generated data to COM/ directory data_out = config.oceanice_yaml[config.component].data_out - logger.info(f"Copy processed data to COM/ directory") FileHandler(data_out).sync() diff --git a/versions/fix.ver b/versions/fix.ver index fa2b4311fad..d48482e93ae 100644 --- a/versions/fix.ver +++ b/versions/fix.ver @@ -17,7 +17,7 @@ export gdas_snow_ver=20241210 export glwu_ver=20220805 export gsi_ver=20251105 export lut_ver=20220805 -export mom6_ver=20240416 +export mom6_ver=20250128 export orog_ver=20240917 export reg2grb2_ver=20220805 export sfc_climo_ver=20220805