diff --git a/.conda/benchcab-dev.yaml b/.conda/benchcab-dev.yaml index 91a1770..ef0c95c 100644 --- a/.conda/benchcab-dev.yaml +++ b/.conda/benchcab-dev.yaml @@ -17,7 +17,7 @@ dependencies: - gitpython - jinja2 - hpcpy>=0.5.0 - - meorg_client>=0.3.1 + - meorg_client>=0.5.0 # CI - pytest-cov # Dev Dependencies diff --git a/.conda/meta.yaml b/.conda/meta.yaml index ba2f8a8..da256dd 100644 --- a/.conda/meta.yaml +++ b/.conda/meta.yaml @@ -30,4 +30,4 @@ requirements: - gitpython - jinja2 - hpcpy>=0.5.0 - - meorg_client>=0.3.1 + - meorg_client>=0.5.0 diff --git a/docs/user_guide/config_options.md b/docs/user_guide/config_options.md index b5803fe..2c83ed4 100644 --- a/docs/user_guide/config_options.md +++ b/docs/user_guide/config_options.md @@ -68,7 +68,6 @@ fluxsite: walltime: 06:00:00 storage: [scratch/a00, gdata/xy11] multiprocess: True - meorg_model_output_id: XXXXXXXX ``` ### [experiment](#experiment) @@ -164,14 +163,6 @@ fluxsites: ``` -### [meorg_model_output_id](#meorg_model_output_id) - -: **Default:** False, _optional key_. :octicons-dash-24: The unique Model Output ID from modelevaluation.org to which output files will be automatically uploaded for analysis. - -A separate upload job will be submitted at the successful completion of benchcab tasks if this key is present, however, the validity is not checked by benchcab at this stage. - -Note: It is the user's responsbility to ensure the model output is configured on modelevaluation.org. - ## spatial Contains settings specific to spatial tests. @@ -381,6 +372,34 @@ realisations: : **Default:** _required key, no default_. :octicons-dash-24: Specify the local checkout path of CABLE branch. +### [meorg_output_name](#meorg_output_name) + + +: **Default:** unset, _optional key_. :octicons-dash-24: Chosen as the model name for one of the realisations, if the user wants to upload the Model Output to me.org for further analysis. A `base32` format hash derived from `model_profile_id` and `$USER` is appended to the model name. + +Note: It is the user's responsbility to ensure the model output name does not clash with existing names belonging to other users on modelevaluation.org. The realisation name is set via `name` if provided, otherwise the default realisation name of the `Repo`. + +The model output name should also follow the Github issue branch format (i.e. it should start with a digit, with words separated by dashes). Finally, the maximum number of characters allowed for `meorg_output_name` is 50. + +This key is _optional_. No default. + +```yaml +realisations: + - repo: + git: + branch: 123-my-branch + meorg_output_name: True + - repo: + git: + branch: 456-my-branch +``` +f(mo_name, user, profile) +123-my-branch-34akg9 # Add by default + + + + + ### [name](#name) : **Default:** base name of [branch_path](#+repo.svn.branch_path) if an SVN repository is given; the branch name if a git repository is given; the folder name if a local path is given, _optional key_. :octicons-dash-24: An alias name used internally by `benchcab` for the branch. The `name` key also specifies the directory name of the source code when retrieving from SVN, GitHub or local. @@ -506,7 +525,7 @@ codecov: ## meorg_bin -: **Default:** False, _optional key. :octicons-dash-24: Specifies the absolute system path to the ME.org client executable. In the absence of this key it will be inferred from the same directory as benchcab should `meorg_model_output_id` be set in `fluxsite` above. +: **Default:** False, _optional key. :octicons-dash-24: Specifies the absolute system path to the ME.org client executable. In the absence of this key it will be inferred from the same directory as benchcab should `meorg_output_name` be set in `realisations` above. ``` yaml diff --git a/docs/user_guide/use_cases.md b/docs/user_guide/use_cases.md index 123c2ef..cb939dd 100644 --- a/docs/user_guide/use_cases.md +++ b/docs/user_guide/use_cases.md @@ -35,17 +35,15 @@ realisations: - repo: git: branch: main + meorg_output_name: True # (1) - repo: git: branch: XXXXX - patch: # (1) + patch: # (2) cable: cable_user: existing_feature: YYYY -fluxsite: - meorg_model_output_id: ZZZZ # (2) - modules: [ intel-compiler/2021.1.1, netcdf/4.7.4, @@ -53,8 +51,8 @@ modules: [ ] ``` -1. Use the option names and values as implemented in the cable namelist file. -2. You need to setup your environment for meorg_client before using this feature. +1. You need to setup your environment for meorg_client before using this feature. +2. Use the option names and values as implemented in the cable namelist file. The evaluation results will be on modelevaluation.org accessible from the Model Output page you've specified @@ -74,17 +72,15 @@ realisations: cable: cable_user: existing_feature: YYYY + meorg_output_name: True # (2) - repo: git: branch: XXXXX - patch: # (2) + patch: # (3) cable: cable_user: existing_feature: YYYY -fluxsite: - meorg_model_output_id: ZZZZ # (3) - modules: [ intel-compiler/2021.1.1, netcdf/4.7.4, @@ -93,8 +89,8 @@ modules: [ ``` 1. Use the option names and values as implemented in the cable namelist file. -2. Use the option names and values as implemented in the cable namelist file. -3. You need to setup your environment for meorg_client before using this feature. +2. You need to setup your environment for meorg_client before using this feature. +3. Use the option names and values as implemented in the cable namelist file. The evaluation results will be on modelevaluation.org accessible from the Model Output page you've specified @@ -110,13 +106,11 @@ realisations: - repo: git: branch: main + meorg_output_name: True # (2) - repo: git: branch: XXXXX -fluxsite: - meorg_model_output_id: ZZZZ # (1) - modules: [ intel-compiler/2021.1.1, netcdf/4.7.4, @@ -141,21 +135,21 @@ realisations: - repo: git: branch: main + meorg_output_name: True # (2) - repo: - name: my-feature-off # (2) + name: my-feature-off # (3) local: - path: XXXXX # (3) + path: XXXXX # (4) - repo: name: my-feature-on local: path: XXXXX - patch: # (4) + patch: # (5) cable: cable_user: new_feature: YYYY fluxsite: - meorg_model_output_id: ZZZZ # (5) pbs: # (6) ncpus: 8 mem: 16GB @@ -169,10 +163,10 @@ modules: [ ``` 1. Testing at one flux site only to save time and resources. -2. We are using the same branch twice so we need to name each occurrence differently. -3. Give the full path to your local CABLE repository with your code changes. -4. Use the option names and values as implemented in the cable namelist file. -5. You need to setup your environment for meorg_client before using this feature. +2. You need to setup your environment for meorg_client before using this feature. +3. We are using the same branch twice so we need to name each occurrence differently. +4. Give the full path to your local CABLE repository with your code changes. +5. Use the option names and values as implemented in the cable namelist file. 6. You can reduce the requested resources to reduce the cost of the test. Comparisons of R0 and R1 should show bitwise agreement. R2 and R0 (and R1) comparison on modelevaluation.org shows the impact of the changes. \ No newline at end of file diff --git a/src/benchcab/benchcab.py b/src/benchcab/benchcab.py index e07d06b..5377a17 100644 --- a/src/benchcab/benchcab.py +++ b/src/benchcab/benchcab.py @@ -245,13 +245,14 @@ def fluxsite_submit_job(self, config_path: str, skip: list[str]) -> None: logger.info("The NetCDF output for each task is written to:") logger.info(f"{internal.FLUXSITE_DIRS['OUTPUT']}/_out.nc") - # Upload to meorg by default - bm.do_meorg( - config, - upload_dir=internal.FLUXSITE_DIRS["OUTPUT"], - benchcab_bin=str(self.benchcab_exe_path), - benchcab_job_id=job_id, - ) + # Upload to meorg if meorg_output_name optional key is passed + if config.get("meorg_output_name") is not None: + bm.do_meorg( + config, + upload_dir=internal.FLUXSITE_DIRS["OUTPUT"], + benchcab_bin=str(self.benchcab_exe_path), + benchcab_job_id=job_id, + ) def gen_codecov(self, config_path: str): """Endpoint for `benchcab codecov`.""" diff --git a/src/benchcab/config.py b/src/benchcab/config.py index 14766e1..8cb10f5 100644 --- a/src/benchcab/config.py +++ b/src/benchcab/config.py @@ -6,10 +6,16 @@ from pathlib import Path import yaml +import copy from cerberus import Validator - +import base64 +import hashlib import benchcab.utils as bu +from benchcab.internal import MEORG_PROFILE from benchcab import internal +from benchcab.utils.repo import create_repo +from benchcab.model import Model +from typing import Optional class ConfigValidationError(Exception): @@ -82,7 +88,7 @@ def read_optional_key(config: dict): Parameters ---------- config : dict - The configuration file with with/without optional keys + The configuration file with, or without optional keys """ if "project" not in config: @@ -119,12 +125,100 @@ def read_optional_key(config: dict): config["fluxsite"]["pbs"] = internal.FLUXSITE_DEFAULT_PBS | config["fluxsite"].get( "pbs", {} ) - config["fluxsite"]["meorg_model_output_id"] = config["fluxsite"].get( - "meorg_model_output_id", internal.FLUXSITE_DEFAULT_MEORG_MODEL_OUTPUT_ID - ) config["codecov"] = config.get("codecov", False) + return config + + +def is_valid_meorg_output_name(name: str) -> Optional[str]: + """Validate model output name against github issue standards. + + Standard: - + + Parameters + ---------- + name: str + The model output name + + Returns + ------- + Optional[str] + If model output name does not meet standard, then return error message + + """ + if len(name) == 0: + return "Model output name is empty\n" + + msg = "" + + if len(name) > 50: + msg += "The length of model output name must be shorter than 50 characters. E.g.: 1-length-is-20-chars\n" + + if " " in name: + msg += "Model output name cannot have spaces. It should use dashes (-) to separate words. E.g. 123-word1-word2\n" + + name_keywords = name.split("-") + + if not name_keywords[0].isdigit(): + msg += "Model output name does not start with number, E.g. 123-number-before-word\n" + + if len(name_keywords) == 1: + msg += "Model output name does not contain keyword after number, E.g. 123-keyword\n" + + if msg == "": + return None + + return f"Errors present when validating model output name:\n{msg}" + + +def add_meorg_output_name(config: dict): + """Determine model output name from realisations. + + Parameters + ---------- + config : dict + The configuration file with optional keys + + """ + # pure function + config = copy.deepcopy(config) + + mo_names = [True for r in config["realisations"] if r.get("meorg_output_name")] + + if len(mo_names) > 1: + msg = "More than 1 value set as true" + raise AssertionError(msg) + + mo_names = "" + for r in config["realisations"]: + # `meorg_output_name` decided either via `name` parameter in a realisation, + # otherwise via `Repo` branch name + repo = create_repo( + spec=r["repo"], + path=internal.SRC_DIR / (r["name"] if r.get("name") else Path()), + ) + mo_name = Model(repo, name=r.get("name")).name + + mo_names += mo_name + if r.pop("meorg_output_name", None): + msg = is_valid_meorg_output_name(mo_name) + if msg is not None: + raise Exception(msg) + + config["meorg_output_name"] = mo_name + + if "meorg_output_name" in config: + user = os.getenv("USER") + mo_name_hash_input = f"{mo_names}{MEORG_PROFILE['id']}{user}" + # hash in bytes form + mo_name_hash_b = hashlib.sha1(mo_name_hash_input.encode()) + # Convert to str and take first 6 characters + mo_name_hash = base64.b32encode(mo_name_hash_b.digest()).decode()[:6] + config["meorg_output_name"] += f"_{mo_name_hash}" + + return config + def read_config_file(config_path: str) -> dict: """Load the config file in a dict. @@ -154,6 +248,8 @@ def read_config(config_path: str) -> dict: ---------- config_path : str Path to the configuration file. + is_meorg: str + Whether workflow includes meorg job submission. If true, determine the model output name Returns ------- @@ -169,7 +265,8 @@ def read_config(config_path: str) -> dict: # Read configuration file config = read_config_file(config_path) # Populate configuration dict with optional keys - read_optional_key(config) - # Validate and return. + config = read_optional_key(config) + # Validate. validate_config(config) + config = add_meorg_output_name(config) return config diff --git a/src/benchcab/data/config-schema.yml b/src/benchcab/data/config-schema.yml index 654d1a5..d5c6dba 100644 --- a/src/benchcab/data/config-schema.yml +++ b/src/benchcab/data/config-schema.yml @@ -51,6 +51,10 @@ realisations: path: type: "string" required: true + meorg_output_name: + nullable: true + type: "boolean" + required: false name: nullable: true type: "string" @@ -107,12 +111,6 @@ fluxsite: schema: type: "string" required: false - meorg_model_output_id: - type: - - "boolean" - - "string" - required: false - default: false spatial: type: "dict" diff --git a/src/benchcab/data/meorg_jobscript.j2 b/src/benchcab/data/meorg_jobscript.j2 index 1c89caa..31cf89d 100644 --- a/src/benchcab/data/meorg_jobscript.j2 +++ b/src/benchcab/data/meorg_jobscript.j2 @@ -18,26 +18,53 @@ set -ev # Set some things DATA_DIR={{data_dir}} NUM_THREADS={{num_threads}} -MODEL_OUTPUT_ID={{model_output_id}} CACHE_DELAY={{cache_delay}} MEORG_BIN={{meorg_bin}} +MODEL_PROFILE_ID={{ model_prof_id }} +meorg_output_name={{ mo.name }} +MODEL_OUTPUT_ARGS=() -{% if purge_outputs %} -# Purge existing model outputs -echo "Purging existing outputs from $MODEL_OUTPUT_ID" -$MEORG_BIN file detach_all $MODEL_OUTPUT_ID +# Create new model output entity +MODEL_OUTPUT_ARGS+="--state-selection {{ mo.state_selection }}" +MODEL_OUTPUT_ARGS+=" --parameter-selection {{ mo.parameter_selection }}" +{% if mo.is_bundle %} +MODEL_OUTPUT_ARGS+=" --is-bundle" {% endif %} +echo "Querying whether $meorg_output_name already exists on me.org" +MODEL_OUTPUT_ID=$($MEORG_BIN output query $meorg_output_name | head -n 1 ) +if [ ! -z "${MODEL_OUTPUT_ID}" ] ; then +# Re-run analysis on the same model output ID, cleaning up existing files +echo "Deleting existing files from model output ID" +$MEORG_BIN file delete_all $MODEL_OUTPUT_ID +echo -n "Updated" +else +echo -n "Created" +fi + +echo " $meorg_output_name on me.org and given this ID: $MODEL_OUTPUT_ID" + +MODEL_OUTPUT_ID="$($MEORG_BIN output create $MODEL_PROFILE_ID $meorg_output_name $MODEL_OUTPUT_ARGS | head -n 1 | awk '{print $NF}')" +echo "Add experiments to model output" +$MEORG_BIN experiment update $MODEL_OUTPUT_ID {{ model_exp_ids|join(',') }} + # Upload the data echo "Uploading data to $MODEL_OUTPUT_ID" -$MEORG_BIN file upload $DATA_DIR/*.nc -n $NUM_THREADS --attach_to $MODEL_OUTPUT_ID +$MEORG_BIN file upload $DATA_DIR/*.nc -n $NUM_THREADS $MODEL_OUTPUT_ID # Wait for the cache to transfer to the object store. echo "Waiting for object store transfer ($CACHE_DELAY sec)" sleep $CACHE_DELAY +{% for exp_id in model_exp_ids %} +echo "Add benchmarks to model output" +$MEORG_BIN benchmark update $MODEL_OUTPUT_ID {{ exp_id }} {{ model_benchmark_ids|join(',') }} + # Trigger the analysis echo "Triggering analysis on $MODEL_OUTPUT_ID" -$MEORG_BIN analysis start $MODEL_OUTPUT_ID +$MEORG_BIN analysis start $MODEL_OUTPUT_ID {{ exp_id }} + +{% endfor %} -echo "DONE" \ No newline at end of file +MEORG_BASE_URL_DEV="${MEORG_BASE_URL_DEV:-https://modelevaluation.org/api/}" +echo "Files transferred to me.org. Analysis in progress. Open ${MEORG_BASE_URL_DEV}/display/${MODEL_OUTPUT_ID} to see results" \ No newline at end of file diff --git a/src/benchcab/data/test/config-basic.yml b/src/benchcab/data/test/config-basic.yml index 846e64f..9ab4457 100644 --- a/src/benchcab/data/test/config-basic.yml +++ b/src/benchcab/data/test/config-basic.yml @@ -20,7 +20,7 @@ realisations: - repo: svn: - branch_path: trunk + branch_path: 123-sample - repo: svn: branch_path: branches/Users/ccc561/v3.0-YP-changes diff --git a/src/benchcab/data/test/config-optional.yml b/src/benchcab/data/test/config-optional.yml index a732a82..915433d 100644 --- a/src/benchcab/data/test/config-optional.yml +++ b/src/benchcab/data/test/config-optional.yml @@ -3,7 +3,6 @@ project: xp65 fluxsite: experiment: AU-Tum - meorg_model_output_id: False multiprocess: False pbs: ncpus: 6 @@ -29,8 +28,9 @@ science_configurations: realisations: - repo: svn: - branch_path: trunk - name: svn_trunk + branch_path: 123-sample + name: 123-sample-optional + meorg_output_name: True - repo: svn: branch_path: branches/Users/ccc561/v3.0-YP-changes diff --git a/src/benchcab/data/test/integration_meorg.sh b/src/benchcab/data/test/integration_meorg.sh index e90c531..2ccc317 100644 --- a/src/benchcab/data/test/integration_meorg.sh +++ b/src/benchcab/data/test/integration_meorg.sh @@ -31,6 +31,7 @@ realisations: - repo: local: path: $CABLE_DIR + meorg_output_name: true - repo: git: branch: main @@ -47,7 +48,6 @@ fluxsite: - scratch/$PROJECT - gdata/$PROJECT # This ID is currently configured on the me.org server. - meorg_model_output_id: Sss7qupAHEZ8ovbCv EOL benchcab run -v diff --git a/src/benchcab/internal.py b/src/benchcab/internal.py index 3fd1ea2..358f96c 100644 --- a/src/benchcab/internal.py +++ b/src/benchcab/internal.py @@ -251,8 +251,42 @@ ], } +# Map experiment with their IDs and benchmarks +# For now, each experiment is associated with 3 benchmarks (1lin, 3km27, LSTM) +MEORG_EXPERIMENT_ID_MAP = { + "AU-Tum": { + "experiment": "jwN9jNMWLEzbT2i9D", + "benchmarks": ["J9BBQCJdsuehsmMf2", "N5X2rjmp96baXrrJ3", "Q7Xu6yGGYdzvvAwbn"], + }, + "AU-How": { + "experiment": "XfC6MTEMm23C4m4iL", + "benchmarks": ["tdrQrKmaihmWdZSZu", "qZWhR3g7JfGhKWPa7", "p2SFiZdQw6ChQK6pr"], + }, + "FI-Hyy": { + "experiment": "nXpDC2Yt7RhhwSKor", + "benchmarks": ["Ym7gwY4k2J2pvDKDJ", "xYA3tSrL2bCeEmvai", "kXFt8mCMtHG4rsnJz"], + }, + "US-Var": { + "experiment": "sD9N2dKx4Jca8B82T", + "benchmarks": ["NbMEBX4sPNHNYkTtq", "X3FoGtYvWmjCyRHGd", "uejBLuHnf4RxAqZXH"], + }, + "US-Whs": { + "experiment": "aWDKqBoTe88ssinuc", + "benchmarks": ["QWsdgXGCWYx7HobXJ", "C42GurGaYDdSRrc2x", "zdnCDJXJzuSheP6T5"], + }, + "five-site-test": { + "experiment": "Nb37QxkAz3FczWDd7", + "benchmarks": ["PP4rFWJGiixFZP8q4", "8kWgyuSkwAKyghsFp", "DYWQuYvxZDgEsp4iX"], + }, + "forty-two-site-test": { + "experiment": "s6k22L3WajmiS9uGv", + "benchmarks": ["zKRrfM7bJpxWPcQ3L", "LMvzc2WL5Qa5jKTpv", "D3XqYwQgH88Tx6NCW"], + }, +} + +MEORG_PROFILE = {"name": "CABLE", "id": "nFcjg4qqHGPkB9sqE"} + FLUXSITE_DEFAULT_EXPERIMENT = "forty-two-site-test" -FLUXSITE_DEFAULT_MEORG_MODEL_OUTPUT_ID = False OPTIONAL_COMMANDS = ["fluxsite-bitwise-cmp", "gen_codecov"] diff --git a/src/benchcab/utils/meorg.py b/src/benchcab/utils/meorg.py index 9ee35df..de17920 100644 --- a/src/benchcab/utils/meorg.py +++ b/src/benchcab/utils/meorg.py @@ -5,12 +5,16 @@ from hpcpy import get_client from meorg_client.client import Client as MeorgClient + import benchcab.utils as bu -from benchcab.internal import MEORG_CLIENT +from benchcab.internal import MEORG_CLIENT, MEORG_PROFILE, MEORG_EXPERIMENT_ID_MAP +from benchcab.utils import interpolate_file_template -def do_meorg(config: dict, upload_dir: str, benchcab_bin: str, benchcab_job_id: str): - """Perform the upload of model outputs to modelevaluation.org +def do_meorg( + config: dict, upload_dir: str, benchcab_bin: str, benchcab_job_id: str = None +): + """Perform the upload of model outputs to modelevaluation.org. Parameters ---------- @@ -29,12 +33,12 @@ def do_meorg(config: dict, upload_dir: str, benchcab_bin: str, benchcab_job_id: """ logger = bu.get_logger() - model_output_id = config["fluxsite"]["meorg_model_output_id"] + meorg_output_name = config["meorg_output_name"] num_threads = MEORG_CLIENT["num_threads"] # Check if a model output id has been assigned - if model_output_id == False: - logger.info("No model_output_id found in fluxsite configuration.") + if config.get("meorg_output_name") is None: + logger.info("No meorg_output_name resolved in configuration.") logger.info("NOT uploading to modelevaluation.org") return False @@ -60,7 +64,7 @@ def do_meorg(config: dict, upload_dir: str, benchcab_bin: str, benchcab_job_id: if MeorgClient().is_initialised() == False: logger.warn( - "A model_output_id has been supplied, but the meorg_client is not initialised." + "A meorg_output_name has been supplied, but the meorg_client is not initialised." ) logger.warn( "To initialise, run `meorg initialise` in the installation environment." @@ -69,17 +73,27 @@ def do_meorg(config: dict, upload_dir: str, benchcab_bin: str, benchcab_job_id: "Once initialised, the outputs from this run can be uploaded with the following command:" ) logger.warn( - f"meorg file upload {upload_dir}/*.nc -n {num_threads} --attach_to {model_output_id}" + f"meorg file upload {upload_dir}/*.nc -n {num_threads} --attach_to {meorg_output_name}" ) logger.warn("Then the analysis can be triggered with:") - logger.warn(f"meorg analysis start {model_output_id}") + logger.warn(f"meorg analysis start {meorg_output_name}") return False # Finally, attempt the upload! else: + experiment = config["fluxsite"]["experiment"] logger.info("Uploading outputs to modelevaluation.org") + mo = { + "state_selection": "default", + "parameter_selection": "automated", + "is_bundle": True, + "name": config["meorg_output_name"], + } + model_exp_id = MEORG_EXPERIMENT_ID_MAP[experiment]["experiment"] + model_benchmark_ids = MEORG_EXPERIMENT_ID_MAP[experiment]["benchmarks"] + # Submit the outputs client = get_client() meorg_jobid = client.submit( @@ -88,7 +102,10 @@ def do_meorg(config: dict, upload_dir: str, benchcab_bin: str, benchcab_job_id: dry_run=False, depends_on=benchcab_job_id, # Interpolate into the job script - model_output_id=model_output_id, + mo=mo, + model_prof_id=MEORG_PROFILE["id"], + model_exp_ids=[model_exp_id], + model_benchmark_ids=model_benchmark_ids, data_dir=upload_dir, cache_delay=MEORG_CLIENT["cache_delay"], mem=MEORG_CLIENT["mem"], diff --git a/tests/test_config.py b/tests/test_config.py index a7fa98a..3403b54 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -23,6 +23,7 @@ def _set_project_env_variable(monkeypatch): # Clear existing environment variables first with mock.patch.dict(os.environ, clear=True): monkeypatch.setenv("PROJECT", OPTIONAL_CONFIG_PROJECT) + monkeypatch.setenv("USER", "test") yield @@ -53,7 +54,7 @@ def no_optional_config() -> dict: return { "modules": ["intel-compiler/2021.1.1", "netcdf/4.7.4", "openmpi/4.1.0"], "realisations": [ - {"repo": {"svn": {"branch_path": "trunk"}}}, + {"repo": {"svn": {"branch_path": "123-sample"}}}, { "repo": { "svn": {"branch_path": "branches/Users/ccc561/v3.0-YP-changes"} @@ -75,7 +76,6 @@ def all_optional_default_config(no_optional_config) -> dict: "experiment": bi.FLUXSITE_DEFAULT_EXPERIMENT, "multiprocess": bi.FLUXSITE_DEFAULT_MULTIPROCESS, "pbs": bi.FLUXSITE_DEFAULT_PBS, - "meorg_model_output_id": bi.FLUXSITE_DEFAULT_MEORG_MODEL_OUTPUT_ID, }, "science_configurations": bi.DEFAULT_SCIENCE_CONFIGURATIONS, "spatial": { @@ -107,7 +107,6 @@ def all_optional_custom_config(no_optional_config) -> dict: "walltime": "10:00:00", "storage": ["scratch/$PROJECT"], }, - "meorg_model_output_id": False, }, "science_configurations": [ { @@ -124,11 +123,13 @@ def all_optional_custom_config(no_optional_config) -> dict: }, "codecov": True, } - branch_names = ["svn_trunk", "git_branch"] + branch_names = ["123-sample-optional", "git_branch"] for c_r, b_n in zip(config["realisations"], branch_names): c_r["name"] = b_n + config["realisations"][0]["meorg_output_name"] = True + return config @@ -199,15 +200,69 @@ def test_no_project_name( ) +def test_add_meorg_output_name(all_optional_custom_config): + """Test addition of correct model output name.""" + output_config = bc.add_meorg_output_name(all_optional_custom_config) + + del all_optional_custom_config["realisations"][0]["meorg_output_name"] + all_optional_custom_config = all_optional_custom_config | { + "meorg_output_name": "123-sample-optional_7J3IEJ" + } + assert output_config == all_optional_custom_config + + +def test_empty_meorg_output_name(): + """Test validating empty model output name.""" + msg = bc.is_valid_meorg_output_name("") + assert msg == "Model output name is empty\n" + + +def test_valid_output_name(): + """Test validating correct model output name.""" + meorg_output_name = "123-sample-issue" + msg = bc.is_valid_meorg_output_name(meorg_output_name) + assert msg is None + + @pytest.mark.parametrize( - ("config_str", "output_config"), + ("meorg_output_name", "output_msg"), [ - ("config-basic.yml", "all_optional_default_config"), - ("config-optional.yml", "all_optional_custom_config"), + ( + f"123-{'l'*48}", + "The length of model output name must be shorter than 50 characters. E.g.: 1-length-is-20-chars\n", + ), + ( + "123-fsd f", + "Model output name cannot have spaces. It should use dashes (-) to separate words. E.g. 123-word1-word2\n", + ), + ( + "hello-123", + "Model output name does not start with number, E.g. 123-number-before-word\n", + ), + ( + "123", + "Model output name does not contain keyword after number, E.g. 123-keyword\n", + ), ], - indirect=["config_str"], ) -def test_read_config(config_path, output_config, request): - """Test overall behaviour of read_config.""" +def test_invalid_output_name(meorg_output_name, output_msg): + """Test validating incorrect model output name.""" + output_msg = f"Errors present when validating model output name:\n{output_msg}" + msg = bc.is_valid_meorg_output_name(meorg_output_name) + assert msg == output_msg + + +@pytest.mark.parametrize("config_str", ["config-basic.yml"], indirect=True) +def test_read_basic_config(config_path, all_optional_default_config): + config = bc.read_config(config_path) + assert pformat(config) == pformat(all_optional_default_config) + + +@pytest.mark.parametrize("config_str", ["config-optional.yml"], indirect=True) +def test_read_optional_config(config_path, all_optional_custom_config): + output_config = all_optional_custom_config | { + "meorg_output_name": "123-sample-optional_7J3IEJ" + } + del output_config["realisations"][0]["meorg_output_name"] config = bc.read_config(config_path) - assert pformat(config) == pformat(request.getfixturevalue(output_config)) + assert pformat(config) == pformat(output_config)