Support for Model output endpoint me.org

docs/user_guide/config_options.md

@@ -372,7 +372,7 @@ realisations:
 
 : **Default:** _required key, no default_. :octicons-dash-24: Specify the local checkout path of CABLE branch.
 
-### [meorg_model_output_name](#meorg_model_output_name)
+### [model_output_name](#model_output_name)
 
 : **Default:** False :octicons-dash-24: Chosen as the model name for one of the realisations. This would be the Model Output to which output files will be automatically uploaded for analysis.  The user must set only one of the realisations keys as `true` for the name to be chosen.
 

src/benchcab/config.py

@@ -13,6 +13,7 @@
 from benchcab import internal
 from benchcab.utils.repo import create_repo
 from benchcab.model import Model
+from typing import Optional
 
 
 class ConfigValidationError(Exception):
@@ -85,7 +86,7 @@ def read_optional_key(config: dict):
     Parameters
     ----------
     config : dict
-        The configuration file with without optional keys
+        The configuration file with, or without optional keys
 
     """
     if "project" not in config:
@@ -128,13 +129,47 @@ def read_optional_key(config: dict):
     return config
 
 
+def is_valid_model_output_name(name: str) -> Optional[str]:
+    """Validate model output name against github issue standards.
+
+    Standard: <digit>-<words-sep-by-dashes>
+
+    Parameters
+    ----------
+    name: str
+        The model output name
+
+    Returns
+    -------
+    Optional[str]
+        If model output name does not meet standard, then return error message
+
+    """
+    if len(name) == 0:
+        return "Model output name is empty"
+
+    if len(name) > 255:
+        return "Model output name has length more than allowed limit on me.org (255)"
+
+    if " " in name:
+        return "Model output name cannot have spaces"
+
+    name_keywords = name.split("-")
+
+    if not name_keywords[0].isdigit():
+        return "Model output name does not start with number"
+
+    if len(name_keywords) == 1:
+        return "Model output name does not contain keyword after number"
+
+
 def add_model_output_name(config: dict):
     """Determine model output name from realisations.
 
     Parameters
     ----------
     config : dict
-        The configuration file with with optional keys
+        The configuration file with optional keys
 
     """
     # pure function
@@ -145,11 +180,23 @@ def add_model_output_name(config: dict):
         assert not is_model_output_name
         if r.pop("model_output_name", None):
             is_model_output_name = True
-            repo = create_repo(
-                spec=r["repo"],
-                path=internal.SRC_DIR / (r["name"] if r.get("name") else Path()),
-            )
-            config["model_output_name"] = Model(repo).name
+
+            mo_name = None
+            if r.get("name"):
+                mo_name = r["name"]
+            else:
+                repo = create_repo(
+                    spec=r["repo"],
+                    path=internal.SRC_DIR / (r["name"] if r.get("name") else Path()),
+                )
+                mo_name = Model(repo).name
+
+            msg = is_valid_model_output_name(mo_name)
+
+            if msg is not None:
+                raise Exception(msg)
+
+            config["model_output_name"] = mo_name
             break
     assert is_model_output_name
     return config

src/benchcab/data/meorg_jobscript.j2

@@ -31,15 +31,19 @@ MODEL_OUTPUT_ARGS+=" --parameter-selection {{ mo.parameter_selection }}"
 MODEL_OUTPUT_ARGS+=" --is-bundle"
 {% endif %}
 
+echo "Querying whether $MODEL_OUTPUT_NAME already exists on me.org"
 MODEL_OUTPUT_ID=$($MEORG_BIN output query $MODEL_OUTPUT_NAME | head -n 1 )
 if [ ! -z "${MODEL_OUTPUT_ID}" ] ; then
+# Re-run analysis on the same model output ID, cleaning up existing files
 echo "Deleting existing files from model output ID"
 $MEORG_BIN file delete_all $MODEL_OUTPUT_ID
-echo "Updated model output ID"
+echo -n "Updated"
 else
-echo "Create new model output ID"
+echo -n "Created"
 fi
 
+echo " $MODEL_OUTPUT_NAME on me.org and given this ID: $MODEL_OUTPUT_ID"
+
 MODEL_OUTPUT_ID="$($MEORG_BIN output create $MODEL_PROFILE_ID $MODEL_OUTPUT_NAME $MODEL_OUTPUT_ARGS | head -n 1 | awk '{print $NF}')"
 echo "Add experiments to model output"
 $MEORG_BIN experiment update $MODEL_OUTPUT_ID {{ model_exp_ids|join(',') }}
@@ -53,7 +57,7 @@ echo "Waiting for object store transfer ($CACHE_DELAY sec)"
 sleep $CACHE_DELAY
 
 {% for exp_id in model_exp_ids %} 
-echo "Replace benchmarks to model output"
+echo "Add benchmarks to model output"
 $MEORG_BIN benchmark update $MODEL_OUTPUT_ID {{ exp_id }} {{ model_benchmark_ids|join(',') }}
 
 # Trigger the analysis
@@ -62,4 +66,5 @@ $MEORG_BIN analysis start $MODEL_OUTPUT_ID {{ exp_id }}
 
 {% endfor %} 
 
-echo "DONE"
+MEORG_BASE_URL_DEV="${MEORG_BASE_URL_DEV:-https://modelevaluation.org/api/}"
+echo "Files transferred to me.org. Analysis in progress. Open ${MEORG_BASE_URL_DEV}/display/${MODEL_OUTPUT_ID} to see results"

src/benchcab/data/test/config-basic.yml

@@ -20,7 +20,7 @@
 realisations:
   - repo:
       svn:
-        branch_path: trunk
+        branch_path: 123-sample
     model_output_name: True
   - repo:
       svn:

src/benchcab/data/test/config-optional.yml

@@ -28,8 +28,8 @@ science_configurations:
 realisations:
   - repo:
       svn:
-        branch_path: trunk
-    name: svn_trunk
+        branch_path: 123-sample
+    name: 123-sample-optional
     model_output_name: True
   - repo:
       svn:

src/benchcab/internal.py

@@ -284,6 +284,7 @@
     },
 }
 
+MEORG_PROFILE = {"name": "CABLE", "id": "nFcjg4qqHGPkB9sqE"}
 
 FLUXSITE_DEFAULT_EXPERIMENT = "forty-two-site-test"
 
@@ -318,5 +319,3 @@ def get_met_forcing_file_names(experiment: str) -> list[str]:
     walltime="01:00:00",
     storage=["gdata/ks32", "gdata/hh5", "gdata/wd9", "gdata/rp23"],
 )
-
-MEORG_PROFILE = {"name": "CABLE", "id": "nFcjg4qqHGPkB9sqE"}

tests/test_config.py

@@ -53,7 +53,7 @@ def no_optional_config() -> dict:
     return {
         "modules": ["intel-compiler/2021.1.1", "netcdf/4.7.4", "openmpi/4.1.0"],
         "realisations": [
-            {"repo": {"svn": {"branch_path": "trunk"}}, "model_output_name": True},
+            {"repo": {"svn": {"branch_path": "123-sample"}}, "model_output_name": True},
             {
                 "repo": {
                     "svn": {"branch_path": "branches/Users/ccc561/v3.0-YP-changes"}
@@ -122,7 +122,7 @@ def all_optional_custom_config(no_optional_config) -> dict:
         },
         "codecov": True,
     }
-    branch_names = ["svn_trunk", "git_branch"]
+    branch_names = ["123-sample-optional", "git_branch"]
 
     for c_r, b_n in zip(config["realisations"], branch_names):
         c_r["name"] = b_n
@@ -202,22 +202,48 @@ def test_add_model_output_name(no_optional_config):
     output_config = bc.add_model_output_name(no_optional_config)
 
     del no_optional_config["realisations"][0]["model_output_name"]
-    no_optional_config = no_optional_config | {"model_output_name": "trunk"}
+    no_optional_config = no_optional_config | {"model_output_name": "123-sample"}
     assert output_config == no_optional_config
 
 
+def test_valid_valid_output_name():
+    """Test reading config for a file that may/may not exist."""
+    model_output_name = "123-sample-issue"
+    msg = bc.is_valid_model_output_name(model_output_name)
+    assert msg is None
+
+
+@pytest.mark.parametrize(
+    ("model_output_name", "output_msg"),
+    [
+        ("", "Model output name is empty"),
+        (
+            "l" * 256,
+            "Model output name has length more than allowed limit on me.org (255)",
+        ),
+        ("123-fsd f", "Model output name cannot have spaces"),
+        ("hello-123", "Model output name does not start with number"),
+        ("123", "Model output name does not contain keyword after number"),
+    ],
+)
+def test_invalid_valid_output_name(model_output_name, output_msg):
+    """Test reading config for a file that may/may not exist."""
+    msg = bc.is_valid_model_output_name(model_output_name)
+    assert msg == output_msg
+
+
 @pytest.mark.parametrize(
-    ("config_str", "output_config_str"),
+    ("config_str", "model_output_name", "output_config"),
     [
-        ("config-basic.yml", "all_optional_default_config"),
-        ("config-optional.yml", "all_optional_custom_config"),
+        ("config-basic.yml", "123-sample", "all_optional_default_config"),
+        ("config-optional.yml", "123-sample-optional", "all_optional_custom_config"),
     ],
     indirect=["config_str"],
 )
-def test_read_config(config_path, output_config_str, request):
+def test_read_config(request, config_path, model_output_name, output_config):
     """Test overall behaviour of read_config."""
-    output_config = request.getfixturevalue(output_config_str) | {
-        "model_output_name": "trunk"
+    output_config = request.getfixturevalue(output_config) | {
+        "model_output_name": model_output_name
     }
     del output_config["realisations"][0]["model_output_name"]
     config = bc.read_config(config_path)