Skip to content

Commit 7a8e2b0

Browse files
danholdawaydanholdaway
authored
New config and ready for coupled mode (#127)
* start new config * Update suites-hofx.yaml * Update suites.yaml * Add new class framework for yamls * restructure default * end of day * add more config options * add fixed options * reorganize the config directories * add more options and improve checking * reorg the static config files * towards yaml comments * towards writing commented dict * comment written to dictionary * finalize location of the prep_config files * coding norms * when files are used still add to config * Optionally pass model to task * jinja stuff * pure string write * generate a coupled flow.cylc with different times * templating of flow.cylc * rename config files * rename sat channel files * refactor the config class * make model names consistent * stage working * getbackground working * remove jedi_config task and put into the runjediexe task * all tasks working manually * remove old suites directory * allow swell_create to call the prep config in one step * suite file bugs * hofx flow working * routine clean up * clean up and codestyle * add a welcome message * move minor version up * YAML norms * YAML norms * YAML norms * YAML norms * use env variables for CiCd id and root * coding norms * clean up some files Co-authored-by: danholdaway <[email protected]>
1 parent 716a6a2 commit 7a8e2b0

File tree

150 files changed

+2053
-2254
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

150 files changed

+2053
-2254
lines changed

‎setup.cfg

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[metadata]
2+
version = attr: swell.__version__

‎setup.py

+11-4
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,14 @@
1212

1313
# --------------------------------------------------------------------------------------------------
1414

15+
import os.path
1516
import setuptools
1617

1718
setuptools.setup(
1819
name='swell',
19-
version='1.0.8',
2020
author='NASA Global Modeling and Assimilation Office',
2121
description='Workflow suites, tasks and configuration for coupled data assimilation',
22-
url='https://github.com/danholdaway/swell',
22+
url='https://github.com/geos-esm/swell',
2323
package_dir={'': 'src'},
2424
packages=setuptools.find_packages(where='src'),
2525
classifiers=[
@@ -33,6 +33,7 @@
3333
python_requires='>=3.6',
3434
install_requires=[
3535
'click',
36+
'jinja2>=3.0.3',
3637
'pyyaml>=6.0',
3738
'pycodestyle>=2.8.0',
3839
'pandas>=1.4.0',
@@ -42,17 +43,23 @@
4243
package_data={
4344
'': [
4445
'deployment/platforms/*/modules*',
45-
'deployment/platforms/*/r2d2_config.yaml',
46+
'deployment/platforms/*/*.yaml',
47+
'suites/*',
4648
'suites/*/*',
47-
'configuration/*.yaml',
49+
'suites/*/*/*',
50+
'configuration/*',
4851
'configuration/*/*',
52+
'configuration/*/*/*',
53+
'configuration/*/*/*/*',
54+
'configuration/*/*/*/*/*',
4955
],
5056
},
5157
include_package_data=True,
5258
entry_points={
5359
'console_scripts': [
5460
'swell_task = swell.tasks.base.task_base:main',
5561
'swell_create_experiment = swell.deployment.bin.swell_create_experiment:main',
62+
'swell_prepare_experiment_config = swell.deployment.bin.swell_prepare_config:main',
5663
'swell_launch_experiment = swell.deployment.bin.swell_launch_experiment:main',
5764
'swell_sat_db_processing = swell.deployment.bin.swell_sat_db_processing:main',
5865
],

‎src/swell/__init__.py

+3
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,6 @@
77
import os
88

99
repo_directory = os.path.dirname(__file__)
10+
11+
# Set the version for swell
12+
__version__ = '1.1.0'

‎src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/background.yaml

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
datetime: '{{local_background_time_iso}}'
2+
filetype: cube sphere history
3+
provider: geos
4+
datapath: ''
5+
filenames: ['{{cycle_dir}}/bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4',
6+
'{{experiment_dir}}/stage/Data/bkg/geos.crtmsrf.{{horizontal_resolution}}.nc4']
7+
state variables: [u,v,ua,va,t,delp,q,qi,ql,qr,qs,o3ppmv,phis,
8+
qls,qcn,cfcn,frocean,frland,varflt,ustar,bstar,
9+
zpbl,cm,ct,cq,kcbl,tsm,khl,khu,frlake,frseaice,vtype,
10+
stype,vfrac,sheleg,ts,soilt,soilm,u10m,v10m]

‎src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/geometry.yaml

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
fms initialization:
2+
namelist filename: '{{experiment_dir}}/stage/Data/fv3files/fmsmpp.nml'
3+
field table filename: '{{experiment_dir}}/stage/Data/fv3files/field_table_gmao'
4+
akbk: '{{experiment_dir}}/stage/Data/fv3files/akbk{{vertical_resolution}}.nc4'
5+
layout: [{{npx_proc}},{{npy_proc}}]
6+
npx: {{horizontal_resolution}}
7+
npy: {{horizontal_resolution}}
8+
npz: {{vertical_resolution}}
9+
field metadata override: '{{experiment_dir}}/stage/Data/fieldmetadata/geos.yaml'

‎src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/getvalues.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
variable change:
2+
variable change name: Model2GeoVaLs

‎src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/pseudo-model.yaml

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
name: PSEUDO
2+
tstep: {{background_frequency}}
3+
filetype: cube sphere history
4+
provider: geos
5+
datapath: ''
6+
filenames: ['{{cycle_dir}}/bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4',
7+
'{{experiment_dir}}/stage/Data/bkg/geos.crtmsrf.{{horizontal_resolution}}.nc4']
8+
model variables: [u,v,ua,va,t,delp,q,qi,ql,qr,qs,o3ppmv,phis,qls,qcn,cfcn,frocean,frland,varflt,ustar,
9+
bstar,zpbl,cm,ct,cq,kcbl,tsm,khl,khu,frlake,frseaice,vtype,stype,vfrac,sheleg,
10+
ts,soilt,soilm,u10m,v10m]

‎src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/r2d2.yaml

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
fetch:
2+
an:
3+
- file_type: [bkg]
4+
filename: '{{cycle_dir}}/bkg.%Y%m%dT%H%M%SZ.nc4'
5+
fc:
6+
- file_type: [bkg]
7+
filename: '{{cycle_dir}}/bkg.%Y%m%dT%H%M%SZ.nc4'
8+
store:
9+
fc:
10+
- file_type: [bkg]

‎src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/model/stage.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
- copy_files:
2+
directories:
3+
- ['/discover/nobackup/drholdaw/JediSwell/bundle/latest/fv3-jedi/test/Data/fieldmetadata/*', '{{experiment_dir}}/stage/Data/fieldmetadata/']
4+
- ['/discover/nobackup/drholdaw/JediSwell/bundle/latest/fv3-jedi/test/Data/fv3files/*', '{{experiment_dir}}/stage/Data/fv3files/']
5+
link_files:
6+
directories:
7+
- ['/discover/nobackup/drholdaw/JediData/GEOS_CRTM_Surface/geos.crtmsrf.{{horizontal_resolution}}.nc4', '{{experiment_dir}}/stage/Data/bkg/']

‎src/swell/configuration/observation_operators/aircraft.yaml ‎src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/aircraft.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,15 @@ obs space:
55
obsdatain:
66
engine:
77
type: H5File
8-
obsfile: $(cycle_dir)/aircraft.{{window_begin}}.nc4
8+
obsfile: '{{cycle_dir}}/aircraft.{{window_begin}}.nc4'
99
obsgrouping:
1010
group variables: ["station_id"]
1111
sort variable: "air_pressure"
1212
sort order: "descending"
1313
obsdataout:
1414
engine:
1515
type: H5File
16-
obsfile: $(cycle_dir)/$(experiment_id).aircraft.{{window_begin}}.nc4
16+
obsfile: '{{cycle_dir}}/{{experiment_id}}.aircraft.{{window_begin}}.nc4'
1717
simulated variables: [eastward_wind, northward_wind, air_temperature]
1818
#obs filters:
1919
##--------------------------------------------------------------------------------------------------------------------

‎src/swell/configuration/observation_operators/airs_aqua.yaml ‎src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/airs_aqua.yaml

+5-5
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@ obs space:
33
obsdatain:
44
engine:
55
type: H5File
6-
obsfile: $(cycle_dir)/airs_aqua.{{window_begin}}.nc4
6+
obsfile: '{{cycle_dir}}/airs_aqua.{{window_begin}}.nc4'
77
obsdataout:
88
engine:
99
type: H5File
10-
obsfile: $(cycle_dir)/$(experiment_id).airs_aqua.{{window_begin}}.nc4
10+
obsfile: '{{cycle_dir}}/{{experiment_id}}.airs_aqua.{{window_begin}}.nc4'
1111
simulated variables: [brightness_temperature]
1212
channels: &airs_aqua_channels 1, 6, 7, 10, 11, 15, 16, 17, 20, 21, 22, 24,
1313
27, 28, 30, 36, 39, 40, 42, 51, 52, 54, 55, 56, 59, 62, 63, 68, 69, 71,
@@ -37,15 +37,15 @@ obs operator:
3737
obs options:
3838
Sensor_ID: airs_aqua
3939
EndianType: little_endian
40-
CoefficientPath: $(crtm_coeff_dir)
40+
CoefficientPath: '{{crtm_coeff_dir}}'
4141
obs bias:
42-
input file: $(cycle_dir)/airs_aqua.{{background_time}}.satbias.nc4
42+
input file: '{{cycle_dir}}/airs_aqua.{{background_time}}.satbias.nc4'
4343
variational bc:
4444
predictors:
4545
- name: constant
4646
- name: lapse_rate
4747
order: 2
48-
tlapse: &airs_aqua_tlapse $(cycle_dir)/airs_aqua.{{background_time}}.tlapse.txt
48+
tlapse: &airs_aqua_tlapse '{{cycle_dir}}/airs_aqua.{{background_time}}.tlapse.txt'
4949
- name: lapse_rate
5050
tlapse: *airs_aqua_tlapse
5151
- name: emissivity

‎src/swell/configuration/observation_operators/amsr2_gcom-w1.yaml ‎src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsr2_gcom-w1.yaml

+5-5
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@ obs space:
33
obsdatain:
44
engine:
55
type: H5File
6-
obsfile: $(cycle_dir)/amsr2_gcom-w1.{{window_begin}}.nc4
6+
obsfile: '{{cycle_dir}}/amsr2_gcom-w1.{{window_begin}}.nc4'
77
obsdataout:
88
engine:
99
type: H5File
10-
obsfile: $(cycle_dir)/$(experiment_id).amsr2_gcom-w1.{{window_begin}}.nc4
10+
obsfile: '{{cycle_dir}}/{{experiment_id}}.amsr2_gcom-w1.{{window_begin}}.nc4'
1111
simulated variables: [brightness_temperature]
1212
channels: &amsr2_gcom-w1_channels 1-14
1313
obs operator:
@@ -21,15 +21,15 @@ obs operator:
2121
obs options:
2222
Sensor_ID: amsr2_gcom-w1
2323
EndianType: little_endian
24-
CoefficientPath: $(crtm_coeff_dir)
24+
CoefficientPath: '{{crtm_coeff_dir}}'
2525
obs bias:
26-
input file: $(cycle_dir)/amsr2_gcom-w1.{{background_time}}.satbias.nc4
26+
input file: '{{cycle_dir}}/amsr2_gcom-w1.{{background_time}}.satbias.nc4'
2727
variational bc:
2828
predictors:
2929
- name: constant
3030
- name: lapse_rate
3131
order: 2
32-
tlapse: &amsr2_gcom-w1_tlapse $(cycle_dir)/amsr2_gcom-w1.{{background_time}}.tlapse.txt
32+
tlapse: &amsr2_gcom-w1_tlapse '{{cycle_dir}}/amsr2_gcom-w1.{{background_time}}.tlapse.txt'
3333
- name: lapse_rate
3434
tlapse: *amsr2_gcom-w1_tlapse
3535
- name: emissivity

‎src/swell/configuration/observation_operators/amsua_aqua.yaml ‎src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_aqua.yaml

+5-5
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@ obs space:
33
obsdatain:
44
engine:
55
type: H5File
6-
obsfile: $(cycle_dir)/amsua_aqua.{{window_begin}}.nc4
6+
obsfile: '{{cycle_dir}}/amsua_aqua.{{window_begin}}.nc4'
77
obsdataout:
88
engine:
99
type: H5File
10-
obsfile: $(cycle_dir)/$(experiment_id).amsua_aqua.{{window_begin}}.nc4
10+
obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_aqua.{{window_begin}}.nc4'
1111
simulated variables: [brightness_temperature]
1212
channels: &amsua_aqua_channels 1-15
1313
obs operator:
@@ -18,9 +18,9 @@ obs operator:
1818
obs options:
1919
Sensor_ID: &Sensor_ID amsua_aqua
2020
EndianType: little_endian
21-
CoefficientPath: $(crtm_coeff_dir)
21+
CoefficientPath: '{{crtm_coeff_dir}}'
2222
obs bias:
23-
input file: $(cycle_dir)/amsua_aqua.{{background_time}}.satbias.nc4
23+
input file: '{{cycle_dir}}/amsua_aqua.{{background_time}}.satbias.nc4'
2424
variational bc:
2525
predictors:
2626
- name: constant
@@ -30,7 +30,7 @@ obs bias:
3030
clwdif_ch314: 2
3131
- name: lapse_rate
3232
order: 2
33-
tlapse: &amsua_aqua_tlapse $(cycle_dir)/amsua_aqua.{{background_time}}.tlapse.txt
33+
tlapse: &amsua_aqua_tlapse '{{cycle_dir}}/amsua_aqua.{{background_time}}.tlapse.txt'
3434
- name: lapse_rate
3535
tlapse: *amsua_aqua_tlapse
3636
- name: emissivity

‎src/swell/configuration/observation_operators/amsua_metop-a.yaml ‎src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_metop-a.yaml

+5-5
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@ obs space:
33
obsdatain:
44
engine:
55
type: H5File
6-
obsfile: $(cycle_dir)/amsua_metop-a.{{window_begin}}.nc4
6+
obsfile: '{{cycle_dir}}/amsua_metop-a.{{window_begin}}.nc4'
77
obsdataout:
88
engine:
99
type: H5File
10-
obsfile: $(cycle_dir)/$(experiment_id).amsua_metop-a.{{window_begin}}.nc4
10+
obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_metop-a.{{window_begin}}.nc4'
1111
simulated variables: [brightness_temperature]
1212
channels: &amsua_metop-a_channels 1-15
1313
obs operator:
@@ -18,9 +18,9 @@ obs operator:
1818
obs options:
1919
Sensor_ID: &Sensor_ID amsua_metop-a
2020
EndianType: little_endian
21-
CoefficientPath: $(crtm_coeff_dir)
21+
CoefficientPath: '{{crtm_coeff_dir}}'
2222
obs bias:
23-
input file: $(cycle_dir)/amsua_metop-a.{{background_time}}.satbias.nc4
23+
input file: '{{cycle_dir}}/amsua_metop-a.{{background_time}}.satbias.nc4'
2424
channels without bc: 14
2525
variational bc:
2626
predictors:
@@ -31,7 +31,7 @@ obs bias:
3131
clwdif_ch314: 2
3232
- name: lapse_rate
3333
order: 2
34-
tlapse: &amsua_metop-a_tlapse $(cycle_dir)/amsua_metop-a.{{background_time}}.tlapse.txt
34+
tlapse: &amsua_metop-a_tlapse '{{cycle_dir}}/amsua_metop-a.{{background_time}}.tlapse.txt'
3535
- name: lapse_rate
3636
tlapse: *amsua_metop-a_tlapse
3737
- name: emissivity

‎src/swell/configuration/observation_operators/amsua_metop-b.yaml ‎src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_metop-b.yaml

+5-5
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@ obs space:
33
obsdatain:
44
engine:
55
type: H5File
6-
obsfile: $(cycle_dir)/amsua_metop-b.{{window_begin}}.nc4
6+
obsfile: '{{cycle_dir}}/amsua_metop-b.{{window_begin}}.nc4'
77
obsdataout:
88
engine:
99
type: H5File
10-
obsfile: $(cycle_dir)/$(experiment_id).amsua_metop-b.{{window_begin}}.nc4
10+
obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_metop-b.{{window_begin}}.nc4'
1111
simulated variables: [brightness_temperature]
1212
channels: &amsua_metop-b_channels 1-15
1313
obs operator:
@@ -18,9 +18,9 @@ obs operator:
1818
obs options:
1919
Sensor_ID: &Sensor_ID amsua_metop-b
2020
EndianType: little_endian
21-
CoefficientPath: $(crtm_coeff_dir)
21+
CoefficientPath: '{{crtm_coeff_dir}}'
2222
obs bias:
23-
input file: $(cycle_dir)/amsua_metop-b.{{background_time}}.satbias.nc4
23+
input file: '{{cycle_dir}}/amsua_metop-b.{{background_time}}.satbias.nc4'
2424
channels without bc: 14
2525
variational bc:
2626
predictors:
@@ -31,7 +31,7 @@ obs bias:
3131
clwdif_ch314: 2
3232
- name: lapse_rate
3333
order: 2
34-
tlapse: &amsua_metop-b_tlapse $(cycle_dir)/amsua_metop-b.{{background_time}}.tlapse.txt
34+
tlapse: &amsua_metop-b_tlapse '{{cycle_dir}}/amsua_metop-b.{{background_time}}.tlapse.txt'
3535
- name: lapse_rate
3636
tlapse: *amsua_metop-b_tlapse
3737
- name: emissivity

‎src/swell/configuration/observation_operators/amsua_metop-c.yaml ‎src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_metop-c.yaml

+5-5
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@ obs space:
33
obsdatain:
44
engine:
55
type: H5File
6-
obsfile: $(cycle_dir)/amsua_metop-c.{{window_begin}}.nc4
6+
obsfile: '{{cycle_dir}}/amsua_metop-c.{{window_begin}}.nc4'
77
obsdataout:
88
engine:
99
type: H5File
10-
obsfile: $(cycle_dir)/$(experiment_id).amsua_metop-c.{{window_begin}}.nc4
10+
obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_metop-c.{{window_begin}}.nc4'
1111
simulated variables: [brightness_temperature]
1212
channels: &amsua_metop-c_channels 1-15
1313
obs operator:
@@ -18,9 +18,9 @@ obs operator:
1818
obs options:
1919
Sensor_ID: &Sensor_ID amsua_metop-c
2020
EndianType: little_endian
21-
CoefficientPath: $(crtm_coeff_dir)
21+
CoefficientPath: '{{crtm_coeff_dir}}'
2222
obs bias:
23-
input file: $(cycle_dir)/amsua_metop-c.{{background_time}}.satbias.nc4
23+
input file: '{{cycle_dir}}/amsua_metop-c.{{background_time}}.satbias.nc4'
2424
channels without bc: 14
2525
variational bc:
2626
predictors:
@@ -31,7 +31,7 @@ obs bias:
3131
clwdif_ch314: 2
3232
- name: lapse_rate
3333
order: 2
34-
tlapse: &amsua_metop-c_tlapse $(cycle_dir)/amsua_metop-c.{{background_time}}.tlapse.txt
34+
tlapse: &amsua_metop-c_tlapse '{{cycle_dir}}/amsua_metop-c.{{background_time}}.tlapse.txt'
3535
- name: lapse_rate
3636
tlapse: *amsua_metop-c_tlapse
3737
- name: emissivity

‎src/swell/configuration/observation_operators/amsua_n15.yaml ‎src/swell/configuration/jedi/fv3-jedi/geos_atmosphere/observations/amsua_n15.yaml

+5-5
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@ obs space:
33
obsdatain:
44
engine:
55
type: H5File
6-
obsfile: $(cycle_dir)/amsua_n15.{{window_begin}}.nc4
6+
obsfile: '{{cycle_dir}}/amsua_n15.{{window_begin}}.nc4'
77
obsdataout:
88
engine:
99
type: H5File
10-
obsfile: $(cycle_dir)/$(experiment_id).amsua_n15.{{window_begin}}.nc4
10+
obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_n15.{{window_begin}}.nc4'
1111
simulated variables: [brightness_temperature]
1212
channels: &amsua_n15_channels 1-15
1313
obs operator:
@@ -18,9 +18,9 @@ obs operator:
1818
obs options:
1919
Sensor_ID: &Sensor_ID amsua_n15
2020
EndianType: little_endian
21-
CoefficientPath: $(crtm_coeff_dir)
21+
CoefficientPath: '{{crtm_coeff_dir}}'
2222
obs bias:
23-
input file: $(cycle_dir)/amsua_n15.{{background_time}}.satbias.nc4
23+
input file: '{{cycle_dir}}/amsua_n15.{{background_time}}.satbias.nc4'
2424
channels without bc: 14
2525
variational bc:
2626
predictors:
@@ -31,7 +31,7 @@ obs bias:
3131
clwdif_ch314: 2
3232
- name: lapse_rate
3333
order: 2
34-
tlapse: &amsua_n15_tlapse $(cycle_dir)/amsua_n15.{{background_time}}.tlapse.txt
34+
tlapse: &amsua_n15_tlapse '{{cycle_dir}}/amsua_n15.{{background_time}}.tlapse.txt'
3535
- name: lapse_rate
3636
tlapse: *amsua_n15_tlapse
3737
- name: emissivity

0 commit comments

Comments
 (0)