From 80d9cc310b2f8009059f80222dd5cc19d3443a90 Mon Sep 17 00:00:00 2001 From: SGallagherMet Date: Thu, 18 Sep 2025 16:10:35 +0100 Subject: [PATCH 1/9] Use custom cylc triggers to skip 'baking' when 'parbake' didn't find any recipes to configure. --- .../app/parbake_recipes/bin/parbake.py | 26 ++++++++++++-- src/CSET/cset_workflow/flow.cylc | 15 ++++++-- tests/workflow_utils/test_parbake_recipes.py | 34 +++++++++++++++++++ 3 files changed, 71 insertions(+), 4 deletions(-) diff --git a/src/CSET/cset_workflow/app/parbake_recipes/bin/parbake.py b/src/CSET/cset_workflow/app/parbake_recipes/bin/parbake.py index 565901a27..3044cafae 100755 --- a/src/CSET/cset_workflow/app/parbake_recipes/bin/parbake.py +++ b/src/CSET/cset_workflow/app/parbake_recipes/bin/parbake.py @@ -17,13 +17,16 @@ import json import os +import subprocess from base64 import b64decode from pathlib import Path from CSET.recipes import load_recipes -def parbake_all(variables: dict, rose_datac: Path, share_dir: Path, aggregation: bool): +def parbake_all( + variables: dict, rose_datac: Path, share_dir: Path, aggregation: bool +) -> int: """Generate and parbake recipes from configuration.""" # Gather all recipes into a big list. recipes = list(load_recipes(variables)) @@ -31,9 +34,12 @@ def parbake_all(variables: dict, rose_datac: Path, share_dir: Path, aggregation: if not recipes: raise ValueError("At least one recipe should be enabled.") # Parbake all recipes remaining after filtering aggregation recipes. + recipe_count = 0 for recipe in filter(lambda r: r.aggregation == aggregation, recipes): print(f"Parbaking {recipe}", flush=True) recipe.parbake(rose_datac, share_dir) + recipe_count += 1 + return recipe_count def main(): @@ -44,7 +50,23 @@ def main(): share_dir = Path(os.environ["CYLC_WORKFLOW_SHARE_DIR"]) aggregation = bool(os.getenv("DO_CASE_AGGREGATION")) # Parbake recipes for cycle. - parbake_all(variables, rose_datac, share_dir, aggregation) + recipe_count = parbake_all(variables, rose_datac, share_dir, aggregation) + + # If running under cylc, notify cylc of task completion. + cylc_workflow_id = os.environ.get("CYLC_WORKFLOW_ID", None) + cylc_task_job = os.environ.get("CYLC_TASK_JOB", None) + if cylc_workflow_id and cylc_task_job: + message_command = [ + "cylc", + "message", + "--", + cylc_workflow_id, + cylc_task_job, + ] + if recipe_count > 0: + subprocess.run(message_command + ["start baking"]) + else: + subprocess.run(message_command + ["skip baking"]) if __name__ == "__main__": # pragma: no cover diff --git a/src/CSET/cset_workflow/flow.cylc b/src/CSET/cset_workflow/flow.cylc index ecfd22104..5d6dbc8f3 100644 --- a/src/CSET/cset_workflow/flow.cylc +++ b/src/CSET/cset_workflow/flow.cylc @@ -46,14 +46,18 @@ final cycle point = {{CSET_TRIAL_END_DATE}} # Analysis from each forecast. {{CSET_TRIAL_CYCLE_PERIOD}} = """ setup_complete[^] => FETCH_DATA:succeed-all => fetch_complete - fetch_complete & parbake_recipes => bake_recipes => cycle_complete + fetch_complete & parbake_recipes:start_baking? => bake_recipes? + + parbake_recipes:skip_baking? | bake_recipes? => cycle_complete """ {% endif %} # Only runs on the final cycle. R1/$ = """ # Run aggregation recipes. - fetch_complete & parbake_aggregation_recipes => bake_aggregation_recipes => cycle_complete + fetch_complete & parbake_aggregation_recipes:start_baking? => bake_aggregation_recipes? + + parbake_aggregation_recipes:skip_baking? | bake_aggregation_recipes? => cycle_complete # Finalise website and cleanup. cycle_complete => finish_website => send_email cycle_complete => housekeeping @@ -97,6 +101,11 @@ final cycle point = {{CSET_TRIAL_END_DATE}} [[[environment]]] ANALYSIS_LENGTH = {{ANALYSIS_LENGTH}} + [[PARBAKE]] + [[[outputs]]] + start_baking='start baking' + skip_baking='skip baking' + [[METPLUS]] [[[environment]]] {% if METPLUS_GRID_STAT|default(False) %} @@ -153,6 +162,7 @@ final cycle point = {{CSET_TRIAL_END_DATE}} [[parbake_recipes]] # Parbake all the recipes for this cycle. + inherit=PARBAKE script = rose task-run -v --app-key=parbake_recipes execution time limit = PT5M [[[directives]]] @@ -163,6 +173,7 @@ final cycle point = {{CSET_TRIAL_END_DATE}} [[parbake_aggregation_recipes]] # Parbake all the aggregation recipes. + inherit=PARBAKE script = rose task-run -v --app-key=parbake_recipes execution time limit = PT5M [[[directives]]] diff --git a/tests/workflow_utils/test_parbake_recipes.py b/tests/workflow_utils/test_parbake_recipes.py index 894f0655f..367b4441a 100644 --- a/tests/workflow_utils/test_parbake_recipes.py +++ b/tests/workflow_utils/test_parbake_recipes.py @@ -14,6 +14,7 @@ """Tests for parbake_recipe workflow utility.""" +import subprocess from pathlib import Path import pytest @@ -25,14 +26,28 @@ def test_main(monkeypatch): """Check parbake.main() invokes parbake_all correctly.""" function_ran = False + recipes_parbaked = 0 + cylc_message_ran = False + cylc_message = "" def mock_parbake_all(variables, rose_datac, share_dir, aggregation): nonlocal function_ran + nonlocal recipes_parbaked function_ran = True assert variables == {"variable": "value"} assert rose_datac == Path("/share/cycle/20000101T0000Z") assert share_dir == Path("/share") assert isinstance(aggregation, bool) + return recipes_parbaked + + def mock_run(cmd, **kwargs): + nonlocal cylc_message + nonlocal cylc_message_ran + cylc_message_ran = True + assert cmd[0:3] == ["cylc", "message", "--"] + assert cmd[3] == "test-workflow" + assert cmd[4] == "test-job" + assert cmd[5] == cylc_message monkeypatch.setattr(parbake, "parbake_all", mock_parbake_all) @@ -51,6 +66,25 @@ def mock_parbake_all(variables, rose_datac, share_dir, aggregation): parbake.main() assert function_ran, "Function did not run!" + # Retry with cylc environment variables set. + monkeypatch.setattr(subprocess, "run", mock_run) + monkeypatch.setenv("CYLC_WORKFLOW_ID", "test-workflow") + monkeypatch.setenv("CYLC_TASK_JOB", "test-job") + + # No recipes parbaked. + function_ran = False + recipes_parbaked = 0 + cylc_message = "skip baking" + parbake.main() + assert cylc_message_ran, "Cylc message function did not run!" + + # Some recipes parbaked. + function_ran = False + recipes_parbaked = 3 + cylc_message = "start baking" + parbake.main() + assert cylc_message_ran, "Cylc message function did not run!" + def test_parbake_all_none_enabled(tmp_working_dir, monkeypatch): """Error when no recipes are enabled.""" From 347eac9db24958eb43ba2ab95b6ccaa3b43f9fd7 Mon Sep 17 00:00:00 2001 From: SGallagherMet Date: Wed, 24 Sep 2025 17:20:53 +0100 Subject: [PATCH 2/9] Remove extra whitespace Co-authored-by: James Frost --- src/CSET/cset_workflow/flow.cylc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/CSET/cset_workflow/flow.cylc b/src/CSET/cset_workflow/flow.cylc index 5d6dbc8f3..ab3e54d0f 100644 --- a/src/CSET/cset_workflow/flow.cylc +++ b/src/CSET/cset_workflow/flow.cylc @@ -47,7 +47,6 @@ final cycle point = {{CSET_TRIAL_END_DATE}} {{CSET_TRIAL_CYCLE_PERIOD}} = """ setup_complete[^] => FETCH_DATA:succeed-all => fetch_complete fetch_complete & parbake_recipes:start_baking? => bake_recipes? - parbake_recipes:skip_baking? | bake_recipes? => cycle_complete """ {% endif %} @@ -56,7 +55,6 @@ final cycle point = {{CSET_TRIAL_END_DATE}} R1/$ = """ # Run aggregation recipes. fetch_complete & parbake_aggregation_recipes:start_baking? => bake_aggregation_recipes? - parbake_aggregation_recipes:skip_baking? | bake_aggregation_recipes? => cycle_complete # Finalise website and cleanup. cycle_complete => finish_website => send_email From b337348cdc580a37bc045d0c0f18b68b4c2c096c Mon Sep 17 00:00:00 2001 From: SGallagherMet Date: Wed, 24 Sep 2025 17:27:42 +0100 Subject: [PATCH 3/9] Moved shared parbake configuration in to PARBAKE family. --- src/CSET/cset_workflow/flow.cylc | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/CSET/cset_workflow/flow.cylc b/src/CSET/cset_workflow/flow.cylc index ab3e54d0f..15bddde30 100644 --- a/src/CSET/cset_workflow/flow.cylc +++ b/src/CSET/cset_workflow/flow.cylc @@ -100,6 +100,13 @@ final cycle point = {{CSET_TRIAL_END_DATE}} ANALYSIS_LENGTH = {{ANALYSIS_LENGTH}} [[PARBAKE]] + script = rose task-run -v --app-key=parbake_recipes + execution time limit = PT5M + [[[directives]]] + --ntasks=1 + --mem=500 + [[[environment]]] + ENCODED_ROSE_SUITE_VARIABLES = {{b64json(ROSE_SUITE_VARIABLES)}} [[[outputs]]] start_baking='start baking' skip_baking='skip baking' @@ -161,24 +168,11 @@ final cycle point = {{CSET_TRIAL_END_DATE}} [[parbake_recipes]] # Parbake all the recipes for this cycle. inherit=PARBAKE - script = rose task-run -v --app-key=parbake_recipes - execution time limit = PT5M - [[[directives]]] - --ntasks=1 - --mem=500 - [[[environment]]] - ENCODED_ROSE_SUITE_VARIABLES = {{b64json(ROSE_SUITE_VARIABLES)}} [[parbake_aggregation_recipes]] # Parbake all the aggregation recipes. inherit=PARBAKE - script = rose task-run -v --app-key=parbake_recipes - execution time limit = PT5M - [[[directives]]] - --ntasks=1 - --mem=500 [[[environment]]] - ENCODED_ROSE_SUITE_VARIABLES = {{b64json(ROSE_SUITE_VARIABLES)}} DO_CASE_AGGREGATION = True [[bake_recipes]] From b5ebbae67225d5f59d1870a527a59fcaa7a35f1c Mon Sep 17 00:00:00 2001 From: SGallagherMet Date: Wed, 24 Sep 2025 17:35:57 +0100 Subject: [PATCH 4/9] Add custom triggers to case_study section of graphing. --- src/CSET/cset_workflow/flow.cylc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/CSET/cset_workflow/flow.cylc b/src/CSET/cset_workflow/flow.cylc index 15bddde30..e77df7236 100644 --- a/src/CSET/cset_workflow/flow.cylc +++ b/src/CSET/cset_workflow/flow.cylc @@ -39,7 +39,8 @@ final cycle point = {{CSET_TRIAL_END_DATE}} {% for date in CSET_CASE_DATES %} R1/{{date}} = """ setup_complete[^] => FETCH_DATA:succeed-all => fetch_complete - fetch_complete & parbake_recipes => bake_recipes => cycle_complete + fetch_complete & parbake_recipes:start_baking? => bake_recipes? + parbake_recipes:skip_baking? | bake_recipes? => cycle_complete """ {% endfor %} {% elif CSET_CYCLING_MODE == "trial" %} From e8ba1a2d903c21feec853ca02d3feca9d5b9bd06 Mon Sep 17 00:00:00 2001 From: James Frost Date: Tue, 11 Nov 2025 09:05:23 +0000 Subject: [PATCH 5/9] Apply suggestions from code review --- .../cset_workflow/app/parbake_recipes/bin/parbake.py | 10 +++++----- src/CSET/cset_workflow/flow.cylc | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/CSET/cset_workflow/app/parbake_recipes/bin/parbake.py b/src/CSET/cset_workflow/app/parbake_recipes/bin/parbake.py index 3044cafae..667f0c0ce 100755 --- a/src/CSET/cset_workflow/app/parbake_recipes/bin/parbake.py +++ b/src/CSET/cset_workflow/app/parbake_recipes/bin/parbake.py @@ -53,8 +53,8 @@ def main(): recipe_count = parbake_all(variables, rose_datac, share_dir, aggregation) # If running under cylc, notify cylc of task completion. - cylc_workflow_id = os.environ.get("CYLC_WORKFLOW_ID", None) - cylc_task_job = os.environ.get("CYLC_TASK_JOB", None) + cylc_workflow_id = os.getenv("CYLC_WORKFLOW_ID") + cylc_task_job = os.getenv("CYLC_TASK_JOB") if cylc_workflow_id and cylc_task_job: message_command = [ "cylc", @@ -63,10 +63,10 @@ def main(): cylc_workflow_id, cylc_task_job, ] - if recipe_count > 0: - subprocess.run(message_command + ["start baking"]) + if recipe_count: + subprocess.run(message_command + ["start baking"], check=True) else: - subprocess.run(message_command + ["skip baking"]) + subprocess.run(message_command + ["skip baking"], check=True) if __name__ == "__main__": # pragma: no cover diff --git a/src/CSET/cset_workflow/flow.cylc b/src/CSET/cset_workflow/flow.cylc index 42d0312b2..d884cdac6 100644 --- a/src/CSET/cset_workflow/flow.cylc +++ b/src/CSET/cset_workflow/flow.cylc @@ -104,7 +104,7 @@ final cycle point = {{CSET_TRIAL_END_DATE}} script = rose task-run -v --app-key=parbake_recipes execution time limit = PT5M [[[environment]]] - ENCODED_ROSE_SUITE_VARIABLES = {{b64json(ROSE_SUITE_VARIABLES)}} + ENCODED_ROSE_SUITE_VARIABLES = {{b64json(ROSE_SUITE_VARIABLES)}} [[[outputs]]] start_baking='start baking' skip_baking='skip baking' From b8005691b84e5813d44a30aea72242955ac27788 Mon Sep 17 00:00:00 2001 From: James Frost Date: Tue, 25 Nov 2025 15:10:25 +0000 Subject: [PATCH 6/9] Working version based on suicide triggers --- src/CSET/cset_workflow/flow.cylc | 15 +- .../cset_workflow/rose-suite.conf.big_data | 294 ++++++++++++++++++ 2 files changed, 303 insertions(+), 6 deletions(-) create mode 100644 src/CSET/cset_workflow/rose-suite.conf.big_data diff --git a/src/CSET/cset_workflow/flow.cylc b/src/CSET/cset_workflow/flow.cylc index 598095b34..6815ae012 100644 --- a/src/CSET/cset_workflow/flow.cylc +++ b/src/CSET/cset_workflow/flow.cylc @@ -39,24 +39,26 @@ final cycle point = {{CSET_TRIAL_END_DATE}} {% for date in CSET_CASE_DATES %} R1/{{date}} = """ setup_complete[^] => FETCH_DATA:succeed-all => fetch_complete - fetch_complete & parbake_recipes:start_baking? => bake_recipes? - parbake_recipes:skip_baking? | bake_recipes? => cycle_complete + fetch_complete & parbake_recipes:start_baking? => bake_recipes + parbake_recipes:skip_baking? => ! bake_recipes + parbake_recipes:skip_baking? | bake_recipes => cycle_complete """ {% endfor %} {% elif CSET_CYCLING_MODE == "trial" %} # Analysis from each forecast. {{CSET_TRIAL_CYCLE_PERIOD}} = """ setup_complete[^] => FETCH_DATA:succeed-all => fetch_complete - fetch_complete & parbake_recipes:start_baking? => bake_recipes? - parbake_recipes:skip_baking? | bake_recipes? => cycle_complete + fetch_complete & parbake_recipes:start_baking? => bake_recipes + parbake_recipes:skip_baking? => ! bake_recipes + parbake_recipes:skip_baking? | bake_recipes => cycle_complete """ {% endif %} # Only runs on the final cycle. R1/$ = """ # Run aggregation recipes. - fetch_complete & parbake_aggregation_recipes:start_baking? => bake_aggregation_recipes? - parbake_aggregation_recipes:skip_baking? | bake_aggregation_recipes? => cycle_complete + fetch_complete & parbake_aggregation_recipes:start_baking? => bake_aggregation_recipes + parbake_aggregation_recipes:skip_baking? | bake_aggregation_recipes => cycle_complete # Finalise website and cleanup. cycle_complete => finish_website => send_email cycle_complete => housekeeping @@ -103,6 +105,7 @@ final cycle point = {{CSET_TRIAL_END_DATE}} [[PARBAKE]] script = rose task-run -v --app-key=parbake_recipes execution time limit = PT5M + completion = succeeded and (start_baking or skip_baking) [[[environment]]] ENCODED_ROSE_SUITE_VARIABLES = {{b64json(ROSE_SUITE_VARIABLES)}} [[[outputs]]] diff --git a/src/CSET/cset_workflow/rose-suite.conf.big_data b/src/CSET/cset_workflow/rose-suite.conf.big_data new file mode 100644 index 000000000..a13446aca --- /dev/null +++ b/src/CSET/cset_workflow/rose-suite.conf.big_data @@ -0,0 +1,294 @@ +[template variables] +AIR_FROST_PRESENCE_DOMAIN_TIMESERIES=False +AIR_FROST_PRESENCE_SPATIAL_DIFFERENCE_PLOT=False +AIR_FROST_PRESENCE_SPATIAL_PLOT=False +ANALYSIS_LENGTH="PT96H" +!!AOA_CYCLIC=False +AOA_DIAG=False +!!AOA_PLEV=[] +AVIATION_COLOUR_STATE=False +AVIATION_COLOUR_STATE_CLOUD_BASE=False +AVIATION_COLOUR_STATE_VISIBILITY=False +AVIATION_FOG_PRESENCE_DOMAIN_MEAN_TIMESERIES=False +AVIATION_FOG_PRESENCE_SPATIAL_DIFFERENCE=False +AVIATION_FOG_PRESENCE_SPATIAL_PLOT=False +BASIC_QQ_PLOT=False +CLOUD_BASE_HEIGHT_LESS_THAN_50_M_DOMAIN_MEAN_TIMESERIES=False +CLOUD_BASE_HEIGHT_LESS_THAN_50_M_SPATIAL_DIFFERENCE=False +CLOUD_BASE_HEIGHT_LESS_THAN_50_M_SPATIAL_PLOT=False +COLORBAR_FILE="" +!!CONDA_METPLUS_VENV_LOCATION="" +CONDA_PATH="" +!!COORDINATE_LIST="" +CSET_CASE_DATES=["20210713T0000Z"] +CSET_CYCLING_MODE="case_study" +CSET_ENV_SEPARATE_MET=False +CSET_ENV_USE_MODULES=False +CSET_MODEL_COUNT=2 +!!CSET_TRIAL_CYCLE_PERIOD="" +!!CSET_TRIAL_END_DATE="" +!!CSET_TRIAL_START_DATE="" +DAILY_09_MAXIMUM_TEMPERATURE_SPATIAL_DIFFERENCE=False +DAILY_09_MAXIMUM_TEMPERATURE_SPATIAL_PLOT=False +DAILY_09_MINIMUM_TEMPERATURE_SPATIAL_DIFFERENCE=False +DAILY_09_MINIMUM_TEMPERATURE_SPATIAL_PLOT=False +DAILY_LIGHTNING_PRESENCE_SPATIAL_DIFFERENCE=False +DAILY_LIGHTNING_PRESENCE_SPATIAL_PLOT=False +DETERMINISTIC_PLOT_CAPE_RATIO=False +DETERMINISTIC_PLOT_INFLOW_PROPERTIES=False +EXTRACT_MLEVEL_TRANSECT=False +EXTRACT_PLEVEL_TRANSECT=False +FOG_PRESENCE_DOMAIN_MEAN_TIMESERIES=False +FOG_PRESENCE_SPATIAL_DIFFERENCE=False +FOG_PRESENCE_SPATIAL_PLOT=False +GROUND_FROST_PRESENCE_DOMAIN_MEAN_TIMESERIES=False +GROUND_FROST_PRESENCE_SPATIAL_DIFFERENCE=False +GROUND_FROST_PRESENCE_SPATIAL_PLOT=False +HISTOGRAM_METHOD="normalised_frequency" +HISTOGRAM_MLEVEL_FIELD=False +HISTOGRAM_MLEVEL_FIELD_AGGREGATION=False,False,False,False +HISTOGRAM_MLEVEL_FIELD_SEQUENCE=False +HISTOGRAM_PLEVEL_FIELD=False +HISTOGRAM_PLEVEL_FIELD_AGGREGATION=False,False,False,False +HISTOGRAM_PLEVEL_FIELD_SEQUENCE=False +HISTOGRAM_SURFACE_FIELD=False +HISTOGRAM_SURFACE_FIELD_AGGREGATION=False,False,False,False +HISTOGRAM_SURFACE_FIELD_SEQUENCE=False +HISTOGRAM_TYPE="step" +HOUSEKEEPING_MODE=2 +!!LATITUDE_POINT=0 +!!LATLON_IN_TYPE="realworld" +!!LEVELS_A=[] +!!LEVELS_B=[] +LIGHTNING_PRESENCE_DOMAIN_TIME_SERIES=False +LIGHTNING_PRESENCE_SPATIAL_DIFFERENCE_PLOT=False +LIGHTNING_PRESENCE_SPATIAL_PLOT=False +LOGLEVEL="DEBUG" +!!LONGITUDE_POINT=0 +MEAN_STRUCTURAL_SIMILARITY_MLEVEL=False +MEAN_STRUCTURAL_SIMILARITY_PLEVEL_FIELD=False +MEAN_STRUCTURAL_SIMILARITY_SURFACE_FIELD=False +!!METPLUS_BASE="" +METPLUS_GRID_STAT=False +METPLUS_POINT_STAT=False +!!MET_INSTALL_DIR="" +!!MET_LIBRARIES="" +!!MLEVEL_TRANSECT_FINISHCOORDS= +!!MLEVEL_TRANSECT_STARTCOORDS= +!!MODEL_FIELDS_A=[] +!!MODEL_FIELDS_B=[] +MODEL_LEVELS=[] +MODEL_LEVEL_FIELDS=["humidity_mixing_ratio", "eastward_wind_at_cell_centres", "northward_wind_at_cell_centres", "cloud_ice_mixing_ratio", "air_potential_temperature", "combined_cloud_amount", "exner_pressure_at_cell_interfaces"] +!!MODULES_LIST= +!!MODULES_PURGE=True +!!ONE_TO_ONE=False +!!PLEVEL_TRANSECT_FINISHCOORDS=4.2,2.5 +!!PLEVEL_TRANSECT_STARTCOORDS=-1.2,-2.5 +PLOTTING_PROJECTION="" +PLOT_RESOLUTION=100 +PRESSURE_LEVELS=[1000, 850, 500, 250] +PRESSURE_LEVEL_FIELDS=["relative_humidity_wrt_ice_at_pressure_levels", "vapour_specific_humidity_at_pressure_levels_for_climate_averaging", "relative_humidity_wrt_water_at_pressure_levels","zonal_wind_at_pressure_levels","meridional_wind_at_pressure_levels","vertical_wind_at_pressure_levels","temperature_at_pressure_levels","geopotential_height_at_pressure_levels"] +!!PROB_TEMPERATURE_CONDITION=["gt", "eq", "lt"] +!!PROB_TEMPERATURE_THRESHOLD=[273.15, 283.15, 278.15] +PROFILE_MLEVEL=False +PROFILE_MLEVEL_AGGREGATION=False,False,False,False +PROFILE_PLEVEL=False +PROFILE_PLEVEL_AGGREGATION=False,False,False,False +RAIN_PRESENCE=True,True +RAIN_PRESENCE_DOMAIN_MEAN_TIMESERIES=False +RAIN_PRESENCE_SPATIAL_DIFFERENCE=False +RAIN_PRESENCE_SPATIAL_PLOT=False +SCREEN_LEVEL_TEMPERATURE_PROBABILITIES=False +!!SCREEN_LEVEL_TEMPERATURE_SPATIAL_PROBABILITY_WITHOUT_CONTROL_MEMBER=False +SELECT_SUBAREA=True +SFC_GALE_FORCE_WINDS_PRESENCE_DOMAIN_MEAN_TIMESERIES=False +SFC_GALE_FORCE_WINDS_PRESENCE_SPATIAL=False +SFC_GALE_FORCE_WINDS_PRESENCE_SPATIAL_DIFFERENCE=False +SFC_HURRICANE_FORCE_WINDS_PRESENCE_DOMAIN_MEAN_TIMESERIES=False +SFC_HURRICANE_FORCE_WINDS_PRESENCE_SPATIAL=False +SFC_HURRICANE_FORCE_WINDS_PRESENCE_SPATIAL_DIFFERENCE=False +SFC_STORM_FORCE_WINDS_PRESENCE_DOMAIN_MEAN_TIMESERIES=False +SFC_STORM_FORCE_WINDS_PRESENCE_SPATIAL=False +SFC_STORM_FORCE_WINDS_PRESENCE_SPATIAL_DIFFERENCE=False +SFC_WIND_BEAUFORT_SCALE_DOMAIN_MEAN_TIMESERIES=False +SFC_WIND_BEAUFORT_SCALE_SPATIAL=False +SFC_WIND_BEAUFORT_SCALE_SPATIAL_DIFFERENCE=False +SFC_WIND_BEAUFORT_SCALE_SPATIAL_PLOT=True +SHOW_CASE_STUDY_AGGREGATE_OPTIONS=True +!!SINGLE_POINT_METHOD="Nearest" +SITE="metoffice" +SKIP_WRITE=True +SNOW_PRESENCE_DOMAIN_MEAN_TIMESERIES=False +SNOW_PRESENCE_SPATIAL_DIFFERENCE=False +SNOW_PRESENCE_SPATIAL_PLOT=False +SPATIAL_DIFFERENCE_MLEVEL_FIELD=False +SPATIAL_DIFFERENCE_MLEVEL_FIELD_AGGREGATION=False,False,False,False +SPATIAL_DIFFERENCE_PLEVEL_FIELD=False +SPATIAL_DIFFERENCE_PLEVEL_FIELD_AGGREGATION=False,False,False,False +SPATIAL_DIFFERENCE_PLEVEL_THETA_E=True +SPATIAL_DIFFERENCE_SURFACE_FIELD=False +SPATIAL_DIFFERENCE_SURFACE_FIELD_AGGREGATION=False,False,False,False +SPATIAL_MLEVEL_FIELD=False +SPATIAL_MLEVEL_FIELD_AGGREGATION=False,False,False,False +SPATIAL_MLEVEL_FIELD_METHOD=[""] +SPATIAL_PLEVEL_FIELD=False +SPATIAL_PLEVEL_FIELD_AGGREGATION=False,False,False,False +SPATIAL_PLEVEL_FIELD_METHOD=["SEQ"] +SPATIAL_PLEVEL_THETA_E=True +SPATIAL_STRUCTURAL_SIMILARITY_MLEVEL=False +SPATIAL_STRUCTURAL_SIMILARITY_PLEVEL_FIELD=False +SPATIAL_STRUCTURAL_SIMILARITY_SURFACE_FIELD=False +SPATIAL_SURFACE_FIELD=False +SPATIAL_SURFACE_FIELD_AGGREGATION=False,False,False,False +SPATIAL_SURFACE_FIELD_METHOD=["SEQ"] +SPECTRUM_MLEVEL_FIELD=False +SPECTRUM_MLEVEL_FIELD_SEQUENCE=False +SPECTRUM_PLEVEL_FIELD=False +SPECTRUM_PLEVEL_FIELD_SEQUENCE=False +SPECTRUM_SURFACE_FIELD=False +SPECTRUM_SURFACE_FIELD_SEQUENCE=False +SUBAREA_EXTENT=44,61,-18,23 +SUBAREA_TYPE="realworld" +SURFACE_FIELDS=["eastward_wind_at_10m"] +SURFACE_SINGLE_POINT_TIME_SERIES=False +!!SURFACE_SYNOP_DIFFS=False +!!SURFACE_SYNOP_FIELDS=[] +!!SURFACE_SYNOP_INTERVAL="" +!!SURFACE_SYNOP_MODEL_FIELDS=[] +SURFACE_SYNOP_OBS=False +!!SURFACE_SYNOP_OFFSET="" +THICK_FOG_PRESENCE_DOMAIN_MEAN_TIMESERIES=False +THICK_FOG_PRESENCE_SPATIAL_DIFFERENCE=False +THICK_FOG_PRESENCE_SPATIAL_PLOT=False +TIMESERIES_MLEVEL_FIELD=False +TIMESERIES_MLEVEL_FIELD_AGGREGATION=False,False,False,False +TIMESERIES_PLEVEL_FIELD=False +TIMESERIES_PLEVEL_FIELD_AGGREGATION=False,False,False,False +TIMESERIES_SURFACE_FIELD=False +TIMESERIES_SURFACE_FIELD_AGGREGATION=False,False,False,False +!!USE_WMO_STATION_NUMBERS=False +!!VERTICAL_COORDINATE_A=[] +!!VERTICAL_COORDINATE_B=[] +WEB_ADDR="https://wwwspice/~$USER/" +WEB_DIR="$HOME/public_html/CSET_afmhot" +!!WMO_BLOCK_STTN_NMBRS=[] +!!m10_analysis_offset="PT0H" +!!m10_data_path="" +!!m10_data_period="PT1H" +!!m10_data_source="filesystem" +!!m10_date_type="initiation" +!!m10_name="" +!!m11_analysis_offset="PT0H" +!!m11_data_path="" +!!m11_data_period="PT1H" +!!m11_data_source="filesystem" +!!m11_date_type="initiation" +!!m11_name="" +!!m12_analysis_offset="PT0H" +!!m12_data_path="" +!!m12_data_period="PT1H" +!!m12_data_source="filesystem" +!!m12_date_type="initiation" +!!m12_name="" +!!m13_analysis_offset="PT0H" +!!m13_data_path="" +!!m13_data_period="PT1H" +!!m13_data_source="filesystem" +!!m13_date_type="initiation" +!!m13_name="" +!!m14_analysis_offset="PT0H" +!!m14_data_path="" +!!m14_data_period="PT1H" +!!m14_data_source="filesystem" +!!m14_date_type="initiation" +!!m14_name="" +!!m15_analysis_offset="PT0H" +!!m15_data_path="" +!!m15_data_period="PT1H" +!!m15_data_source="filesystem" +!!m15_date_type="initiation" +!!m15_name="" +!!m16_analysis_offset="PT0H" +!!m16_data_path="" +!!m16_data_period="PT1H" +!!m16_data_source="filesystem" +!!m16_date_type="initiation" +!!m16_name="" +!!m17_analysis_offset="PT0H" +!!m17_data_path="" +!!m17_data_period="PT1H" +!!m17_data_source="filesystem" +!!m17_date_type="initiation" +!!m17_name="" +!!m18_analysis_offset="PT0H" +!!m18_data_path="" +!!m18_data_period="PT1H" +!!m18_data_source="filesystem" +!!m18_date_type="initiation" +!!m18_name="" +!!m19_analysis_offset="PT0H" +!!m19_data_path="" +!!m19_data_period="PT1H" +!!m19_data_source="filesystem" +!!m19_date_type="initiation" +!!m19_name="" +m1_analysis_offset="PT0H" +m1_data_path="/data/scratch/david.flack/Ahr_valley_Global_trailblazer/5k/%Y%m%dT%H%MZ_*.pp" +m1_data_period="PT6H" +m1_data_source="filesystem" +m1_date_type="initiation" +m1_name="5 km" +!!m20_analysis_offset="PT0H" +!!m20_data_path="" +!!m20_data_period="PT1H" +!!m20_data_source="filesystem" +!!m20_date_type="initiation" +!!m20_name="" +m2_analysis_offset="PT0H" +m2_data_path="/data/scratch/david.flack/Ahr_valley_Global_trailblazer/10k/%Y%m%dT%H%MZ_*.pp" +m2_data_period="PT6H" +m2_data_source="filesystem" +m2_date_type="initiation" +m2_name="10 km" +!!m3_analysis_offset="PT0H" +!!m3_data_path="" +!!m3_data_period="PT1H" +!!m3_data_source="filesystem" +!!m3_date_type="initiation" +!!m3_name="" +!!m4_analysis_offset="PT0H" +!!m4_data_path="" +!!m4_data_period="PT1H" +!!m4_data_source="filesystem" +!!m4_date_type="initiation" +!!m4_name="" +!!m5_analysis_offset="PT0H" +!!m5_data_path="" +!!m5_data_period="PT1H" +!!m5_data_source="filesystem" +!!m5_date_type="initiation" +!!m5_name="" +!!m6_analysis_offset="PT0H" +!!m6_data_path="" +!!m6_data_period="PT1H" +!!m6_data_source="filesystem" +!!m6_date_type="initiation" +!!m6_name="" +!!m7_analysis_offset="PT0H" +!!m7_data_path="" +!!m7_data_period="PT1H" +!!m7_data_source="filesystem" +!!m7_date_type="initiation" +!!m7_name="" +!!m8_analysis_offset="PT0H" +!!m8_data_path="" +!!m8_data_period="PT1H" +!!m8_data_source="filesystem" +!!m8_date_type="initiation" +!!m8_name="" +!!m9_analysis_offset="PT0H" +!!m9_data_path="" +!!m9_data_period="PT1H" +!!m9_data_source="filesystem" +!!m9_date_type="initiation" +!!m9_name="" From af918e876f3f8bb80ea5ab4f63c5e5f3381c217e Mon Sep 17 00:00:00 2001 From: James Frost Date: Tue, 25 Nov 2025 15:15:52 +0000 Subject: [PATCH 7/9] Finish setup before running parbake This is mainly to ensure we have validated our environment before using it. --- src/CSET/cset_workflow/flow.cylc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/CSET/cset_workflow/flow.cylc b/src/CSET/cset_workflow/flow.cylc index 6815ae012..268e926bf 100644 --- a/src/CSET/cset_workflow/flow.cylc +++ b/src/CSET/cset_workflow/flow.cylc @@ -38,6 +38,7 @@ final cycle point = {{CSET_TRIAL_END_DATE}} # Runs for every forecast initiation time to process the data in parallel. {% for date in CSET_CASE_DATES %} R1/{{date}} = """ + setup_complete[^] => parbake_recipes setup_complete[^] => FETCH_DATA:succeed-all => fetch_complete fetch_complete & parbake_recipes:start_baking? => bake_recipes parbake_recipes:skip_baking? => ! bake_recipes @@ -47,6 +48,7 @@ final cycle point = {{CSET_TRIAL_END_DATE}} {% elif CSET_CYCLING_MODE == "trial" %} # Analysis from each forecast. {{CSET_TRIAL_CYCLE_PERIOD}} = """ + setup_complete[^] => parbake_recipes setup_complete[^] => FETCH_DATA:succeed-all => fetch_complete fetch_complete & parbake_recipes:start_baking? => bake_recipes parbake_recipes:skip_baking? => ! bake_recipes @@ -57,6 +59,7 @@ final cycle point = {{CSET_TRIAL_END_DATE}} # Only runs on the final cycle. R1/$ = """ # Run aggregation recipes. + setup_complete[^] => parbake_aggregation_recipes fetch_complete & parbake_aggregation_recipes:start_baking? => bake_aggregation_recipes parbake_aggregation_recipes:skip_baking? | bake_aggregation_recipes => cycle_complete # Finalise website and cleanup. From cb6533d4baf8b10de8d524d3d9a50d572adfa52a Mon Sep 17 00:00:00 2001 From: James Frost Date: Tue, 25 Nov 2025 16:06:17 +0000 Subject: [PATCH 8/9] fixup! Working version based on suicide triggers --- .../cset_workflow/rose-suite.conf.big_data | 294 ------------------ 1 file changed, 294 deletions(-) delete mode 100644 src/CSET/cset_workflow/rose-suite.conf.big_data diff --git a/src/CSET/cset_workflow/rose-suite.conf.big_data b/src/CSET/cset_workflow/rose-suite.conf.big_data deleted file mode 100644 index a13446aca..000000000 --- a/src/CSET/cset_workflow/rose-suite.conf.big_data +++ /dev/null @@ -1,294 +0,0 @@ -[template variables] -AIR_FROST_PRESENCE_DOMAIN_TIMESERIES=False -AIR_FROST_PRESENCE_SPATIAL_DIFFERENCE_PLOT=False -AIR_FROST_PRESENCE_SPATIAL_PLOT=False -ANALYSIS_LENGTH="PT96H" -!!AOA_CYCLIC=False -AOA_DIAG=False -!!AOA_PLEV=[] -AVIATION_COLOUR_STATE=False -AVIATION_COLOUR_STATE_CLOUD_BASE=False -AVIATION_COLOUR_STATE_VISIBILITY=False -AVIATION_FOG_PRESENCE_DOMAIN_MEAN_TIMESERIES=False -AVIATION_FOG_PRESENCE_SPATIAL_DIFFERENCE=False -AVIATION_FOG_PRESENCE_SPATIAL_PLOT=False -BASIC_QQ_PLOT=False -CLOUD_BASE_HEIGHT_LESS_THAN_50_M_DOMAIN_MEAN_TIMESERIES=False -CLOUD_BASE_HEIGHT_LESS_THAN_50_M_SPATIAL_DIFFERENCE=False -CLOUD_BASE_HEIGHT_LESS_THAN_50_M_SPATIAL_PLOT=False -COLORBAR_FILE="" -!!CONDA_METPLUS_VENV_LOCATION="" -CONDA_PATH="" -!!COORDINATE_LIST="" -CSET_CASE_DATES=["20210713T0000Z"] -CSET_CYCLING_MODE="case_study" -CSET_ENV_SEPARATE_MET=False -CSET_ENV_USE_MODULES=False -CSET_MODEL_COUNT=2 -!!CSET_TRIAL_CYCLE_PERIOD="" -!!CSET_TRIAL_END_DATE="" -!!CSET_TRIAL_START_DATE="" -DAILY_09_MAXIMUM_TEMPERATURE_SPATIAL_DIFFERENCE=False -DAILY_09_MAXIMUM_TEMPERATURE_SPATIAL_PLOT=False -DAILY_09_MINIMUM_TEMPERATURE_SPATIAL_DIFFERENCE=False -DAILY_09_MINIMUM_TEMPERATURE_SPATIAL_PLOT=False -DAILY_LIGHTNING_PRESENCE_SPATIAL_DIFFERENCE=False -DAILY_LIGHTNING_PRESENCE_SPATIAL_PLOT=False -DETERMINISTIC_PLOT_CAPE_RATIO=False -DETERMINISTIC_PLOT_INFLOW_PROPERTIES=False -EXTRACT_MLEVEL_TRANSECT=False -EXTRACT_PLEVEL_TRANSECT=False -FOG_PRESENCE_DOMAIN_MEAN_TIMESERIES=False -FOG_PRESENCE_SPATIAL_DIFFERENCE=False -FOG_PRESENCE_SPATIAL_PLOT=False -GROUND_FROST_PRESENCE_DOMAIN_MEAN_TIMESERIES=False -GROUND_FROST_PRESENCE_SPATIAL_DIFFERENCE=False -GROUND_FROST_PRESENCE_SPATIAL_PLOT=False -HISTOGRAM_METHOD="normalised_frequency" -HISTOGRAM_MLEVEL_FIELD=False -HISTOGRAM_MLEVEL_FIELD_AGGREGATION=False,False,False,False -HISTOGRAM_MLEVEL_FIELD_SEQUENCE=False -HISTOGRAM_PLEVEL_FIELD=False -HISTOGRAM_PLEVEL_FIELD_AGGREGATION=False,False,False,False -HISTOGRAM_PLEVEL_FIELD_SEQUENCE=False -HISTOGRAM_SURFACE_FIELD=False -HISTOGRAM_SURFACE_FIELD_AGGREGATION=False,False,False,False -HISTOGRAM_SURFACE_FIELD_SEQUENCE=False -HISTOGRAM_TYPE="step" -HOUSEKEEPING_MODE=2 -!!LATITUDE_POINT=0 -!!LATLON_IN_TYPE="realworld" -!!LEVELS_A=[] -!!LEVELS_B=[] -LIGHTNING_PRESENCE_DOMAIN_TIME_SERIES=False -LIGHTNING_PRESENCE_SPATIAL_DIFFERENCE_PLOT=False -LIGHTNING_PRESENCE_SPATIAL_PLOT=False -LOGLEVEL="DEBUG" -!!LONGITUDE_POINT=0 -MEAN_STRUCTURAL_SIMILARITY_MLEVEL=False -MEAN_STRUCTURAL_SIMILARITY_PLEVEL_FIELD=False -MEAN_STRUCTURAL_SIMILARITY_SURFACE_FIELD=False -!!METPLUS_BASE="" -METPLUS_GRID_STAT=False -METPLUS_POINT_STAT=False -!!MET_INSTALL_DIR="" -!!MET_LIBRARIES="" -!!MLEVEL_TRANSECT_FINISHCOORDS= -!!MLEVEL_TRANSECT_STARTCOORDS= -!!MODEL_FIELDS_A=[] -!!MODEL_FIELDS_B=[] -MODEL_LEVELS=[] -MODEL_LEVEL_FIELDS=["humidity_mixing_ratio", "eastward_wind_at_cell_centres", "northward_wind_at_cell_centres", "cloud_ice_mixing_ratio", "air_potential_temperature", "combined_cloud_amount", "exner_pressure_at_cell_interfaces"] -!!MODULES_LIST= -!!MODULES_PURGE=True -!!ONE_TO_ONE=False -!!PLEVEL_TRANSECT_FINISHCOORDS=4.2,2.5 -!!PLEVEL_TRANSECT_STARTCOORDS=-1.2,-2.5 -PLOTTING_PROJECTION="" -PLOT_RESOLUTION=100 -PRESSURE_LEVELS=[1000, 850, 500, 250] -PRESSURE_LEVEL_FIELDS=["relative_humidity_wrt_ice_at_pressure_levels", "vapour_specific_humidity_at_pressure_levels_for_climate_averaging", "relative_humidity_wrt_water_at_pressure_levels","zonal_wind_at_pressure_levels","meridional_wind_at_pressure_levels","vertical_wind_at_pressure_levels","temperature_at_pressure_levels","geopotential_height_at_pressure_levels"] -!!PROB_TEMPERATURE_CONDITION=["gt", "eq", "lt"] -!!PROB_TEMPERATURE_THRESHOLD=[273.15, 283.15, 278.15] -PROFILE_MLEVEL=False -PROFILE_MLEVEL_AGGREGATION=False,False,False,False -PROFILE_PLEVEL=False -PROFILE_PLEVEL_AGGREGATION=False,False,False,False -RAIN_PRESENCE=True,True -RAIN_PRESENCE_DOMAIN_MEAN_TIMESERIES=False -RAIN_PRESENCE_SPATIAL_DIFFERENCE=False -RAIN_PRESENCE_SPATIAL_PLOT=False -SCREEN_LEVEL_TEMPERATURE_PROBABILITIES=False -!!SCREEN_LEVEL_TEMPERATURE_SPATIAL_PROBABILITY_WITHOUT_CONTROL_MEMBER=False -SELECT_SUBAREA=True -SFC_GALE_FORCE_WINDS_PRESENCE_DOMAIN_MEAN_TIMESERIES=False -SFC_GALE_FORCE_WINDS_PRESENCE_SPATIAL=False -SFC_GALE_FORCE_WINDS_PRESENCE_SPATIAL_DIFFERENCE=False -SFC_HURRICANE_FORCE_WINDS_PRESENCE_DOMAIN_MEAN_TIMESERIES=False -SFC_HURRICANE_FORCE_WINDS_PRESENCE_SPATIAL=False -SFC_HURRICANE_FORCE_WINDS_PRESENCE_SPATIAL_DIFFERENCE=False -SFC_STORM_FORCE_WINDS_PRESENCE_DOMAIN_MEAN_TIMESERIES=False -SFC_STORM_FORCE_WINDS_PRESENCE_SPATIAL=False -SFC_STORM_FORCE_WINDS_PRESENCE_SPATIAL_DIFFERENCE=False -SFC_WIND_BEAUFORT_SCALE_DOMAIN_MEAN_TIMESERIES=False -SFC_WIND_BEAUFORT_SCALE_SPATIAL=False -SFC_WIND_BEAUFORT_SCALE_SPATIAL_DIFFERENCE=False -SFC_WIND_BEAUFORT_SCALE_SPATIAL_PLOT=True -SHOW_CASE_STUDY_AGGREGATE_OPTIONS=True -!!SINGLE_POINT_METHOD="Nearest" -SITE="metoffice" -SKIP_WRITE=True -SNOW_PRESENCE_DOMAIN_MEAN_TIMESERIES=False -SNOW_PRESENCE_SPATIAL_DIFFERENCE=False -SNOW_PRESENCE_SPATIAL_PLOT=False -SPATIAL_DIFFERENCE_MLEVEL_FIELD=False -SPATIAL_DIFFERENCE_MLEVEL_FIELD_AGGREGATION=False,False,False,False -SPATIAL_DIFFERENCE_PLEVEL_FIELD=False -SPATIAL_DIFFERENCE_PLEVEL_FIELD_AGGREGATION=False,False,False,False -SPATIAL_DIFFERENCE_PLEVEL_THETA_E=True -SPATIAL_DIFFERENCE_SURFACE_FIELD=False -SPATIAL_DIFFERENCE_SURFACE_FIELD_AGGREGATION=False,False,False,False -SPATIAL_MLEVEL_FIELD=False -SPATIAL_MLEVEL_FIELD_AGGREGATION=False,False,False,False -SPATIAL_MLEVEL_FIELD_METHOD=[""] -SPATIAL_PLEVEL_FIELD=False -SPATIAL_PLEVEL_FIELD_AGGREGATION=False,False,False,False -SPATIAL_PLEVEL_FIELD_METHOD=["SEQ"] -SPATIAL_PLEVEL_THETA_E=True -SPATIAL_STRUCTURAL_SIMILARITY_MLEVEL=False -SPATIAL_STRUCTURAL_SIMILARITY_PLEVEL_FIELD=False -SPATIAL_STRUCTURAL_SIMILARITY_SURFACE_FIELD=False -SPATIAL_SURFACE_FIELD=False -SPATIAL_SURFACE_FIELD_AGGREGATION=False,False,False,False -SPATIAL_SURFACE_FIELD_METHOD=["SEQ"] -SPECTRUM_MLEVEL_FIELD=False -SPECTRUM_MLEVEL_FIELD_SEQUENCE=False -SPECTRUM_PLEVEL_FIELD=False -SPECTRUM_PLEVEL_FIELD_SEQUENCE=False -SPECTRUM_SURFACE_FIELD=False -SPECTRUM_SURFACE_FIELD_SEQUENCE=False -SUBAREA_EXTENT=44,61,-18,23 -SUBAREA_TYPE="realworld" -SURFACE_FIELDS=["eastward_wind_at_10m"] -SURFACE_SINGLE_POINT_TIME_SERIES=False -!!SURFACE_SYNOP_DIFFS=False -!!SURFACE_SYNOP_FIELDS=[] -!!SURFACE_SYNOP_INTERVAL="" -!!SURFACE_SYNOP_MODEL_FIELDS=[] -SURFACE_SYNOP_OBS=False -!!SURFACE_SYNOP_OFFSET="" -THICK_FOG_PRESENCE_DOMAIN_MEAN_TIMESERIES=False -THICK_FOG_PRESENCE_SPATIAL_DIFFERENCE=False -THICK_FOG_PRESENCE_SPATIAL_PLOT=False -TIMESERIES_MLEVEL_FIELD=False -TIMESERIES_MLEVEL_FIELD_AGGREGATION=False,False,False,False -TIMESERIES_PLEVEL_FIELD=False -TIMESERIES_PLEVEL_FIELD_AGGREGATION=False,False,False,False -TIMESERIES_SURFACE_FIELD=False -TIMESERIES_SURFACE_FIELD_AGGREGATION=False,False,False,False -!!USE_WMO_STATION_NUMBERS=False -!!VERTICAL_COORDINATE_A=[] -!!VERTICAL_COORDINATE_B=[] -WEB_ADDR="https://wwwspice/~$USER/" -WEB_DIR="$HOME/public_html/CSET_afmhot" -!!WMO_BLOCK_STTN_NMBRS=[] -!!m10_analysis_offset="PT0H" -!!m10_data_path="" -!!m10_data_period="PT1H" -!!m10_data_source="filesystem" -!!m10_date_type="initiation" -!!m10_name="" -!!m11_analysis_offset="PT0H" -!!m11_data_path="" -!!m11_data_period="PT1H" -!!m11_data_source="filesystem" -!!m11_date_type="initiation" -!!m11_name="" -!!m12_analysis_offset="PT0H" -!!m12_data_path="" -!!m12_data_period="PT1H" -!!m12_data_source="filesystem" -!!m12_date_type="initiation" -!!m12_name="" -!!m13_analysis_offset="PT0H" -!!m13_data_path="" -!!m13_data_period="PT1H" -!!m13_data_source="filesystem" -!!m13_date_type="initiation" -!!m13_name="" -!!m14_analysis_offset="PT0H" -!!m14_data_path="" -!!m14_data_period="PT1H" -!!m14_data_source="filesystem" -!!m14_date_type="initiation" -!!m14_name="" -!!m15_analysis_offset="PT0H" -!!m15_data_path="" -!!m15_data_period="PT1H" -!!m15_data_source="filesystem" -!!m15_date_type="initiation" -!!m15_name="" -!!m16_analysis_offset="PT0H" -!!m16_data_path="" -!!m16_data_period="PT1H" -!!m16_data_source="filesystem" -!!m16_date_type="initiation" -!!m16_name="" -!!m17_analysis_offset="PT0H" -!!m17_data_path="" -!!m17_data_period="PT1H" -!!m17_data_source="filesystem" -!!m17_date_type="initiation" -!!m17_name="" -!!m18_analysis_offset="PT0H" -!!m18_data_path="" -!!m18_data_period="PT1H" -!!m18_data_source="filesystem" -!!m18_date_type="initiation" -!!m18_name="" -!!m19_analysis_offset="PT0H" -!!m19_data_path="" -!!m19_data_period="PT1H" -!!m19_data_source="filesystem" -!!m19_date_type="initiation" -!!m19_name="" -m1_analysis_offset="PT0H" -m1_data_path="/data/scratch/david.flack/Ahr_valley_Global_trailblazer/5k/%Y%m%dT%H%MZ_*.pp" -m1_data_period="PT6H" -m1_data_source="filesystem" -m1_date_type="initiation" -m1_name="5 km" -!!m20_analysis_offset="PT0H" -!!m20_data_path="" -!!m20_data_period="PT1H" -!!m20_data_source="filesystem" -!!m20_date_type="initiation" -!!m20_name="" -m2_analysis_offset="PT0H" -m2_data_path="/data/scratch/david.flack/Ahr_valley_Global_trailblazer/10k/%Y%m%dT%H%MZ_*.pp" -m2_data_period="PT6H" -m2_data_source="filesystem" -m2_date_type="initiation" -m2_name="10 km" -!!m3_analysis_offset="PT0H" -!!m3_data_path="" -!!m3_data_period="PT1H" -!!m3_data_source="filesystem" -!!m3_date_type="initiation" -!!m3_name="" -!!m4_analysis_offset="PT0H" -!!m4_data_path="" -!!m4_data_period="PT1H" -!!m4_data_source="filesystem" -!!m4_date_type="initiation" -!!m4_name="" -!!m5_analysis_offset="PT0H" -!!m5_data_path="" -!!m5_data_period="PT1H" -!!m5_data_source="filesystem" -!!m5_date_type="initiation" -!!m5_name="" -!!m6_analysis_offset="PT0H" -!!m6_data_path="" -!!m6_data_period="PT1H" -!!m6_data_source="filesystem" -!!m6_date_type="initiation" -!!m6_name="" -!!m7_analysis_offset="PT0H" -!!m7_data_path="" -!!m7_data_period="PT1H" -!!m7_data_source="filesystem" -!!m7_date_type="initiation" -!!m7_name="" -!!m8_analysis_offset="PT0H" -!!m8_data_path="" -!!m8_data_period="PT1H" -!!m8_data_source="filesystem" -!!m8_date_type="initiation" -!!m8_name="" -!!m9_analysis_offset="PT0H" -!!m9_data_path="" -!!m9_data_period="PT1H" -!!m9_data_source="filesystem" -!!m9_date_type="initiation" -!!m9_name="" From 3ac71261bccb885f4992f5c257a298f4420428e0 Mon Sep 17 00:00:00 2001 From: James Frost Date: Wed, 10 Dec 2025 13:29:03 +0000 Subject: [PATCH 9/9] Suicide aggregation bake if set to skip --- src/CSET/cset_workflow/flow.cylc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/CSET/cset_workflow/flow.cylc b/src/CSET/cset_workflow/flow.cylc index 5b4adf8c8..435d2b848 100644 --- a/src/CSET/cset_workflow/flow.cylc +++ b/src/CSET/cset_workflow/flow.cylc @@ -61,6 +61,7 @@ final cycle point = {{CSET_TRIAL_END_DATE}} # Run aggregation recipes. setup_complete[^] => parbake_aggregation_recipes fetch_complete & parbake_aggregation_recipes:start_baking? => bake_aggregation_recipes + parbake_aggregation_recipes:skip_baking? => ! bake_aggregation_recipes parbake_aggregation_recipes:skip_baking? | bake_aggregation_recipes => cycle_complete # Finalise website and cleanup. cycle_complete => finish_website => send_email