Support for Model output endpoint me.org

.conda/benchcab-dev.yaml

@@ -17,7 +17,7 @@ dependencies:
   - gitpython
   - jinja2
   - hpcpy>=0.5.0
-  - meorg_client>=0.3.1
+  - meorg_client>=0.5.0
   # CI
   - pytest-cov
   # Dev Dependencies

.conda/meta.yaml

@@ -30,4 +30,4 @@ requirements:
         - gitpython
         - jinja2
         - hpcpy>=0.5.0
-        - meorg_client>=0.3.1
+        - meorg_client>=0.5.0

docs/user_guide/config_options.md

@@ -68,7 +68,6 @@ fluxsite:
     walltime: 06:00:00
     storage: [scratch/a00, gdata/xy11]
   multiprocess: True
-  meorg_model_output_id: XXXXXXXX
 ```
 
 ### [experiment](#experiment)
@@ -164,14 +163,6 @@ fluxsites:
 
 ```
 
-### [meorg_model_output_id](#meorg_model_output_id)
-
-: **Default:** False, _optional key_. :octicons-dash-24: The unique Model Output ID from modelevaluation.org to which output files will be automatically uploaded for analysis.
-
-A separate upload job will be submitted at the successful completion of benchcab tasks if this key is present, however, the validity is not checked by benchcab at this stage.
-
-Note: It is the user's responsbility to ensure the model output is configured on modelevaluation.org.
-
 ## spatial
 
 Contains settings specific to spatial tests.
@@ -381,6 +372,25 @@ realisations:
 
 : **Default:** _required key, no default_. :octicons-dash-24: Specify the local checkout path of CABLE branch.
 
+### [meorg_output_name](#meorg_output_name)
+
+
+: **Default:** unset, _optional key_. :octicons-dash-24: Chosen as the model name for one of the realisations. This would be the Model Output to which output files will be automatically uploaded for analysis.  The user must set only one of the realisations keys as `true` for the name to be chosen.
+
+Note: It is the user's responsbility to ensure the model output name does not clash with existing names belonging to other users on modelevaluation.org.
+
+The model output name should also follow the Github issue branch format (i.e. it should start with a digit, with words separated by dashes). Finally, the maximum number of characters allowed for `meorg_output_name` is 50.
+
+This key is _optional_. No default.
+
+```yaml
+realisations:
+  - repo:
+      git:
+        branch: 123-my-branch
+    model_output_name: True
+```
+
 ### [name](#name)
 
 : **Default:** base name of [branch_path](#+repo.svn.branch_path) if an SVN repository is given; the branch name if a git repository is given; the folder name if a local path is given, _optional key_. :octicons-dash-24: An alias name used internally by `benchcab` for the branch. The `name` key also specifies the directory name of the source code when retrieving from SVN, GitHub or local.
@@ -506,7 +516,7 @@ codecov:
 
 ## meorg_bin
 
-: **Default:** False, _optional key. :octicons-dash-24: Specifies the absolute system path to the ME.org client executable. In the absence of this key it will be inferred from the same directory as benchcab should `meorg_model_output_id` be set in `fluxsite` above.
+: **Default:** False, _optional key. :octicons-dash-24: Specifies the absolute system path to the ME.org client executable. In the absence of this key it will be inferred from the same directory as benchcab should `meorg_output_name` be set in `realisations` above.
 
 ``` yaml
 

docs/user_guide/use_cases.md

@@ -35,26 +35,24 @@ realisations:
   - repo:
       git:
         branch: main
+    meorg_output_name: True # (1)
   - repo:
       git:
         branch: XXXXX
-    patch: # (1)
+    patch: # (2)
         cable:
             cable_user:
                 existing_feature: YYYY
 
-fluxsite:
-    meorg_model_output_id: ZZZZ # (2)
-
 modules: [
   intel-compiler/2021.1.1,
   netcdf/4.7.4,
   openmpi/4.1.0
 ]
 ```
 
-1. Use the option names and values as implemented in the cable namelist file.
-2. You need to setup your environment for meorg_client before using this feature. 
+1. You need to setup your environment for meorg_client before using this feature. 
+2. Use the option names and values as implemented in the cable namelist file.
 
 The evaluation results will be on modelevaluation.org accessible from the Model Output page you've specified
 
@@ -74,17 +72,15 @@ realisations:
         cable:
             cable_user:
                 existing_feature: YYYY
+    meorg_output_name: True # (2)
   - repo:
       git:
         branch: XXXXX
-    patch: # (2)
+    patch: # (3)
         cable:
             cable_user:
                 existing_feature: YYYY
 
-fluxsite:
-    meorg_model_output_id: ZZZZ # (3)
-
 modules: [
   intel-compiler/2021.1.1,
   netcdf/4.7.4,
@@ -93,8 +89,8 @@ modules: [
 ```
 
 1. Use the option names and values as implemented in the cable namelist file.
-2. Use the option names and values as implemented in the cable namelist file.
-3. You need to setup your environment for meorg_client before using this feature. 
+2. You need to setup your environment for meorg_client before using this feature. 
+3. Use the option names and values as implemented in the cable namelist file.
 
 The evaluation results will be on modelevaluation.org accessible from the Model Output page you've specified
 
@@ -110,13 +106,11 @@ realisations:
   - repo:
       git:
         branch: main
+    meorg_output_name: True # (2)
   - repo:
       git:
         branch: XXXXX
 
-fluxsite:
-    meorg_model_output_id: ZZZZ # (1)
-
 modules: [
   intel-compiler/2021.1.1,
   netcdf/4.7.4,
@@ -141,21 +135,21 @@ realisations:
   - repo:
       git:
         branch: main
+    meorg_output_name: True # (2)
   - repo:
-      name: my-feature-off # (2)
+      name: my-feature-off # (3)
       local:
-        path: XXXXX # (3)
+        path: XXXXX # (4)
   - repo:
       name: my-feature-on
       local:
         path: XXXXX
-    patch: # (4)
+    patch: # (5)
         cable:
             cable_user:
                 new_feature: YYYY
 
 fluxsite:
-    meorg_model_output_id: ZZZZ # (5)
     pbs: # (6)
       ncpus: 8
       mem: 16GB
@@ -169,10 +163,10 @@ modules: [
 ```
 
 1. Testing at one flux site only to save time and resources.
-2. We are using the same branch twice so we need to name each occurrence differently.
-3. Give the full path to your local CABLE repository with your code changes.
-4. Use the option names and values as implemented in the cable namelist file.
-5. You need to setup your environment for meorg_client before using this feature.
+2. You need to setup your environment for meorg_client before using this feature.
+3. We are using the same branch twice so we need to name each occurrence differently.
+4. Give the full path to your local CABLE repository with your code changes.
+5. Use the option names and values as implemented in the cable namelist file.
 6. You can reduce the requested resources to reduce the cost of the test.
 
 Comparisons of R0 and R1 should show bitwise agreement. R2 and R0 (and R1) comparison on modelevaluation.org shows the impact of the changes.

src/benchcab/benchcab.py

@@ -245,13 +245,14 @@ def fluxsite_submit_job(self, config_path: str, skip: list[str]) -> None:
         logger.info("The NetCDF output for each task is written to:")
         logger.info(f"{internal.FLUXSITE_DIRS['OUTPUT']}/<task_name>_out.nc")
 
-        # Upload to meorg by default
-        bm.do_meorg(
-            config,
-            upload_dir=internal.FLUXSITE_DIRS["OUTPUT"],
-            benchcab_bin=str(self.benchcab_exe_path),
-            benchcab_job_id=job_id,
-        )
+        # Upload to meorg if meorg_output_name optional key is passed
+        if config.get("meorg_output_name") is not None:
+            bm.do_meorg(
+                config,
+                upload_dir=internal.FLUXSITE_DIRS["OUTPUT"],
+                benchcab_bin=str(self.benchcab_exe_path),
+                benchcab_job_id=job_id,
+            )
 
     def gen_codecov(self, config_path: str):
         """Endpoint for `benchcab codecov`."""

src/benchcab/config.py

@@ -6,10 +6,14 @@
 from pathlib import Path
 
 import yaml
+import copy
 from cerberus import Validator
 
 import benchcab.utils as bu
 from benchcab import internal
+from benchcab.utils.repo import create_repo
+from benchcab.model import Model
+from typing import Optional
 
 
 class ConfigValidationError(Exception):
@@ -82,7 +86,7 @@ def read_optional_key(config: dict):
     Parameters
     ----------
     config : dict
-        The configuration file with with/without optional keys
+        The configuration file with, or without optional keys
 
     """
     if "project" not in config:
@@ -119,12 +123,90 @@ def read_optional_key(config: dict):
     config["fluxsite"]["pbs"] = internal.FLUXSITE_DEFAULT_PBS | config["fluxsite"].get(
         "pbs", {}
     )
-    config["fluxsite"]["meorg_model_output_id"] = config["fluxsite"].get(
-        "meorg_model_output_id", internal.FLUXSITE_DEFAULT_MEORG_MODEL_OUTPUT_ID
-    )
 
     config["codecov"] = config.get("codecov", False)
 
+    return config
+
+
+def is_valid_meorg_output_name(name: str) -> Optional[str]:
+    """Validate model output name against github issue standards.
+
+    Standard: <digit>-<words-sep-by-dashes>
+
+    Parameters
+    ----------
+    name: str
+        The model output name
+
+    Returns
+    -------
+    Optional[str]
+        If model output name does not meet standard, then return error message
+
+    """
+    if len(name) == 0:
+        return "Model output name is empty\n"
+
+    msg = ""
+
+    if len(name) > 50:
+        msg += "The length of model output name must be shorter than 50 characters. E.g.: 1-length-is-20-chars\n"
+
+    if " " in name:
+        msg += "Model output name cannot have spaces. It should use dashes (-) to separate words. E.g. 123-word1-word2\n"
+
+    name_keywords = name.split("-")
+
+    if not name_keywords[0].isdigit():
+        msg += "Model output name does not start with number, E.g. 123-number-before-word\n"
+
+    if len(name_keywords) == 1:
+        msg += "Model output name does not contain keyword after number, E.g. 123-keyword\n"
+
+    if msg == "":
+        return None
+
+    return f"Errors present when validating model output name:\n{msg}"
+
+
+def add_meorg_output_name(config: dict):
+    """Determine model output name from realisations.
+
+    Parameters
+    ----------
+    config : dict
+        The configuration file with optional keys
+
+    """
+    # pure function
+    config = copy.deepcopy(config)
+
+    mo_names = [True for r in config["realisations"] if r.get("meorg_output_name")]
+
+    if len(mo_names) > 1:
+        msg = "More than 1 value set as true"
+        raise AssertionError(msg)
+
+    for r in config["realisations"]:
+        if r.pop("meorg_output_name", None):
+            # `meorg_output_name` decided either via `name` parameter in a realisation,
+            # otherwise via `Repo` branch name
+            repo = create_repo(
+                spec=r["repo"],
+                path=internal.SRC_DIR / (r["name"] if r.get("name") else Path()),
+            )
+            mo_name = Model(repo, name=r.get("name")).name
+
+            msg = is_valid_meorg_output_name(mo_name)
+
+            if msg is not None:
+                raise Exception(msg)
+
+            config["meorg_output_name"] = mo_name
+
+    return config
+
 
 def read_config_file(config_path: str) -> dict:
     """Load the config file in a dict.
@@ -154,6 +236,8 @@ def read_config(config_path: str) -> dict:
     ----------
     config_path : str
         Path to the configuration file.
+    is_meorg: str
+        Whether workflow includes meorg job submission. If true, determine the model output name
 
     Returns
     -------
@@ -169,7 +253,8 @@ def read_config(config_path: str) -> dict:
     # Read configuration file
     config = read_config_file(config_path)
     # Populate configuration dict with optional keys
-    read_optional_key(config)
-    # Validate and return.
+    config = read_optional_key(config)
+    # Validate.
     validate_config(config)
+    config = add_meorg_output_name(config)
     return config

src/benchcab/data/config-schema.yml

@@ -51,6 +51,10 @@ realisations:
               path:
                 type: "string"
                 required: true
+      meorg_output_name:
+        nullable: true
+        type: "boolean"
+        required: false
       name:
         nullable: true
         type: "string"
@@ -107,12 +111,6 @@ fluxsite:
           schema:
             type: "string"
             required: false
-    meorg_model_output_id:
-      type: 
-       - "boolean"
-       - "string"
-      required: false
-      default: false
 
 spatial:
   type: "dict"