Merge dts-manifest-schema changes into develop

deployment/conf/deployment.cfg

@@ -4,4 +4,4 @@ DATA_DIR = /kb/deployment/lib/src/data/bulk/
 AUTH_URL = https://ci.kbase.us/services/auth/api/V2/token
 CONCIERGE_PATH = /kbaseconcierge
 FILE_EXTENSION_MAPPINGS = /kb/deployment/conf/supported_apps_w_extensions.json
-DTS_MANIFEST_SCHEMA = /kb/deployment/import_specifications/schema/dts_manifest_schema.json
+DTS_MANIFEST_SCHEMA = /kb/module/import_specifications/schema/dts_manifest_schema.json

staging_service/import_specifications/individual_parsers.py

@@ -2,6 +2,7 @@
 Contains parser functions for use with the file parser framework.
 """
 
+from collections import defaultdict
 import csv
 import json
 import math
@@ -60,6 +61,11 @@
     "null",
 ]
 
+_DTS_INSTRUCTIONS_KEY = "instructions"
+_DTS_INSTRUCTIONS_DATATYPE_KEY = "data_type"
+_DTS_INSTRUCTIONS_PARAMETERS_KEY = "parameters"
+_DTS_INSTRUCTIONS_OBJECTS_KEY = "objects"
+
 
 class _ParseException(Exception):
     pass
@@ -351,15 +357,27 @@ def parse_dts_manifest(path: Path, validator: Draft202012Validator) -> ParseResu
     results = {}
     try:
         with open(path, "r") as manifest:
-            manifest_json = json.load(manifest)
-        if not isinstance(manifest_json, dict):
+            manifest = json.load(manifest)
+        if not isinstance(manifest, dict):
             return _error(Error(ErrorType.PARSE_FAIL, "Manifest is not a dictionary", spcsrc))
-        for err in validator.iter_errors(manifest_json):
+        for err in validator.iter_errors(manifest):
             err_str = err.message
             err_path = list(err.absolute_path)
             if err_path:
                 err_str += f" at {err_path}"
             errors.append(Error(ErrorType.PARSE_FAIL, err_str, spcsrc))
+        if not errors:
+            results = defaultdict(list)
+            instructions = manifest[_DTS_INSTRUCTIONS_KEY]
+            for resource_obj in instructions[_DTS_INSTRUCTIONS_OBJECTS_KEY]:
+                datatype = resource_obj[_DTS_INSTRUCTIONS_DATATYPE_KEY]
+                parameters = frozendict(resource_obj[_DTS_INSTRUCTIONS_PARAMETERS_KEY])
+                results[datatype].append(parameters)
+            # Re-package results as a dict of {datatype: ParseResult}
+            results = {
+                datatype: ParseResult(spcsrc, tuple(paramlist))
+                for datatype, paramlist in results.items()
+            }
     except json.JSONDecodeError:
         return _error(Error(ErrorType.PARSE_FAIL, "File must be in JSON format", spcsrc))
     except FileNotFoundError:

tests/import_specifications/test_data/manifest_errors.json

@@ -0,0 +1,81 @@
+{
+    "name": "manifest",
+    "resources": [
+      {
+        "id": "JDP:555518eb0d8785178e712d88",
+        "name": "61564.assembled",
+        "path": "img/submissions/61564/61564.assembled.gff",
+        "format": "gff",
+        "media_type": "text/plain",
+        "bytes": 455161,
+        "hash": "",
+        "credit": {
+          "comment": "",
+          "content_url": "",
+          "contributors": null,
+          "credit_metadata_source": "",
+          "dates": null,
+          "descriptions": null,
+          "funding": null,
+          "identifier": "JDP:555518eb0d8785178e712d88",
+          "license": {
+            "id": "",
+            "url": ""
+          },
+          "publisher": {
+            "organization_id": "",
+            "organization_name": ""
+          },
+          "related_identifiers": null,
+          "resource_type": "dataset",
+          "titles": null,
+          "url": "",
+          "version": ""
+        }
+      },
+      {
+        "id": "JDP:555518eb0d8785178e712d84",
+        "name": "61564.assembled",
+        "path": "img/submissions/61564/61564.assembled.fna",
+        "format": "fasta",
+        "media_type": "text/plain",
+        "bytes": 6354414,
+        "hash": "",
+        "credit": {
+          "comment": "",
+          "content_url": "",
+          "contributors": null,
+          "credit_metadata_source": "",
+          "dates": null,
+          "descriptions": null,
+          "funding": null,
+          "identifier": "JDP:555518eb0d8785178e712d84",
+          "license": {
+            "id": "",
+            "url": ""
+          },
+          "publisher": {
+            "organization_id": "",
+            "organization_name": ""
+          },
+          "related_identifiers": null,
+          "resource_type": "dataset",
+          "titles": null,
+          "url": "",
+          "version": ""
+        }
+      }
+    ],
+    "instructions": {
+      "protocol": "KBase narrative import",
+      "objects": [
+        {
+          "data_type": "gff_metagenome",
+          "parameters": {
+            "param1": "value1",
+            "param2": "value2"
+          }
+        }
+      ]
+    }
+  }

tests/import_specifications/test_data/manifest_multiple.json

@@ -0,0 +1,62 @@
+{
+    "name": "manifest",
+    "resources": [
+      {
+        "id": "JDP:555518eb0d8785178e712d88",
+        "name": "61564.assembled",
+        "path": "img/submissions/61564/61564.assembled.gff",
+        "format": "gff"
+      },
+      {
+        "id": "JDP:555518eb0d8785178e712d84",
+        "name": "61564.assembled",
+        "path": "img/submissions/61564/61564.assembled.fna",
+        "format": "fasta"
+      },
+      {
+        "id": "JDP:555518ec0d8785178e712d9f",
+        "name": "61567.assembled",
+        "path": "img/submissions/61567/61567.assembled.gff",
+        "format": "gff"
+      },
+      {
+        "id": "JDP:555518ec0d8785178e712d9b",
+        "name": "61567.assembled",
+        "path": "img/submissions/61567/61567.assembled.fna",
+        "format": "fasta"
+      }
+    ],
+    "instructions": {
+      "protocol": "KBase narrative import",
+      "objects": [
+        {
+          "data_type": "gff_metagenome",
+          "parameters": {
+            "mg_param1": "value1",
+            "mg_param2": "value2"
+          }
+        },
+        {
+          "data_type": "gff_metagenome",
+          "parameters": {
+            "mg_param1": "value3",
+            "mg_param2": "value4"
+          }
+        },
+        {
+          "data_type": "gff_genome",
+          "parameters": {
+            "gen_param1": "value1",
+            "gen_param2": "value2"
+          }
+        },
+        {
+          "data_type": "gff_genome",
+          "parameters": {
+            "gen_param1": "value3",
+            "gen_param2": "value4"
+          }
+        }
+      ]
+    }
+  }

tests/import_specifications/test_data/manifest_small.json

@@ -31,13 +31,6 @@
         "titles": null,
         "url": "",
         "version": ""
-      },
-      "instructions": {
-        "data_type": "gff_metagenome",
-        "parameters": {
-          "param1": "value1",
-          "param2": "value2"
-        }
       }
     },
     {
@@ -70,13 +63,6 @@
         "titles": null,
         "url": "",
         "version": ""
-      },
-      "instructions": {
-        "data_type": "gff_metagenome",
-        "parameters": {
-          "param1": "value1",
-          "param2": "value2"
-        }
       }
     },
     {
@@ -109,14 +95,33 @@
         "titles": null,
         "url": "",
         "version": ""
-      },
-      "instructions": {
+      }
+    }
+  ],
+  "instructions": {
+    "protocol": "KBase narrative import",
+    "objects": [
+      {
         "data_type": "gff_metagenome",
         "parameters": {
           "param1": "value1",
           "param2": "value2"
         }
+      },
+      {
+        "data_type": "gff_metagenome",
+        "parameters": {
+          "param1": "value3",
+          "param2": "value4"
+        }
+      },
+      {
+        "data_type": "gff_metagenome",
+        "parameters": {
+          "param1": "value5",
+          "param2": "value6"
+        }
       }
-    }
-  ]
+    ]
+  }
 }