Use gear id instead of name (alg) in rules

Author: ambrussimon

api/jobs/gears.py

@@ -8,7 +8,6 @@
 import datetime
 from jsonschema import Draft4Validator, ValidationError
 import gears as gear_tools
-import pymongo
 
 from .. import config
 from .jobs import Job
@@ -39,17 +38,10 @@ def get_gears():
     return map(lambda x: x['original'], cursor)
 
 def get_gear(_id):
-    return config.db.gears.find_one({'_id': bson.ObjectId(_id)})
-
-def get_gear_by_name(name):
-
-    # Find a gear from the list by name
-    gear_doc = list(config.db.gears.find({'gear.name': name}).sort('created', pymongo.DESCENDING))
-
-    if len(gear_doc) == 0 :
-        raise APINotFoundException('Unknown gear ' + name)
-
-    return gear_doc[0]
+    gear = config.db.gears.find_one({'_id': bson.ObjectId(_id)})
+    if gear is None:
+        raise APINotFoundException('Cannot find gear {}'.format(_id))
+    return gear
 
 def get_invocation_schema(gear):
     return gear_tools.derive_invocation_schema(gear['gear'])

api/jobs/handlers.py

@@ -24,7 +24,7 @@
 
 from ..auth.apikeys import JobApiKey
 
-from .gears import validate_gear_config, get_gears, get_gear, get_invocation_schema, remove_gear, upsert_gear, suggest_container, get_gear_by_name, check_for_gear_insertion
+from .gears import validate_gear_config, get_gears, get_gear, get_invocation_schema, remove_gear, upsert_gear, suggest_container, check_for_gear_insertion
 from .jobs import Job, Logs
 from .batch import check_state, update
 from .queue import Queue
@@ -181,10 +181,7 @@ def post(self, cid):
 
         validate_data(doc, 'rule-new.json', 'input', 'POST', optional=True)
         validate_regexes(doc)
-        try:
-            get_gear_by_name(doc['alg'])
-        except APINotFoundException:
-            self.abort(400, 'Cannot find gear for alg {}, alg not valid'.format(doc['alg']))
+        get_gear(doc['gear_id'])
 
         doc['project_id'] = cid
 
@@ -234,11 +231,8 @@ def put(self, cid, rid):
         updates = self.request.json
         validate_data(updates, 'rule-update.json', 'input', 'POST', optional=True)
         validate_regexes(updates)
-        if updates.get('alg'):
-            try:
-                get_gear_by_name(updates['alg'])
-            except APINotFoundException:
-                self.abort(400, 'Cannot find gear for alg {}, alg not valid'.format(updates['alg']))
+        if updates.get('gear_id'):
+            get_gear(updates['gear_id'])
 
         doc.update(updates)
         config.db.project_rules.replace_one({'_id': bson.ObjectId(rid)}, doc)

api/jobs/rules.py

@@ -138,24 +138,25 @@ def eval_rule(rule, file_, container):
 
     return True
 
-def queue_job_legacy(algorithm_id, input_):
+def queue_job_legacy(gear_id, input_):
     """
     Tie together logic used from the no-manifest, single-file era.
     Takes a single FileReference instead of a map.
     """
 
-    gear = gears.get_gear_by_name(algorithm_id)
+    gear = gears.get_gear(gear_id)
 
     if len(gear['gear']['inputs']) != 1:
-        raise Exception("Legacy gear enqueue attempt of " + algorithm_id + " failed: must have exactly 1 input in manifest")
+        raise Exception("Legacy gear enqueue attempt of " + gear_id + " failed: must have exactly 1 input in manifest")
 
     input_name = gear['gear']['inputs'].keys()[0]
 
     inputs = {
         input_name: input_
     }
 
-    job = Job(str(gear['_id']), inputs, tags=['auto', algorithm_id])
+    gear_tag = gear['gear']['name'] + '-' + gear_id
+    job = Job(str(gear['_id']), inputs, tags=['auto', gear_tag])
     return job
 
 def find_type_in_container(container, type_):
@@ -184,22 +185,23 @@ def create_potential_jobs(db, container, container_type, file_):
 
         if 'from_failed_job' not in file_ and eval_rule(rule, file_, container):
 
-            alg_name = rule['alg']
+            gear_id = rule['gear_id']
+            gear = gears.get_gear(gear_id)
+            gear_tag = gear['gear']['name'] + '-' + gear_id
 
             if rule.get('match') is None:
                 input_ = FileReference(type=container_type, id=str(container['_id']), name=file_['name'])
-                job = queue_job_legacy(alg_name, input_)
+                job = queue_job_legacy(gear_id, input_)
             else:
                 inputs = { }
 
                 for input_name, match_type in rule['match'].iteritems():
                     match = find_type_in_container(container, match_type)
                     if match is None:
-                        raise Exception("No type " + match_type + " found for alg rule " + alg_name + " that should have been satisfied")
+                        raise Exception("No type " + match_type + " found for alg rule " + gear_tag + " that should have been satisfied")
                     inputs[input_name] = FileReference(type=container_type, id=str(container['_id']), name=match['name'])
 
-                gear = gears.get_gear_by_name(alg_name)
-                job = Job(str(gear['_id']), inputs, tags=['auto', alg_name])
+                job = Job(str(gear['_id']), inputs, tags=['auto', gear_tag])
 
             potential_jobs.append({
                 'job': job,
@@ -248,7 +250,7 @@ def create_jobs(db, container_before, container_after, container_type):
         job_map = pj['job'].map()
         Queue.enqueue_job(job_map, origin)
 
-        spawned_jobs.append(pj['rule']['alg'])
+        spawned_jobs.append(pj['rule']['gear_id'])
 
     return spawned_jobs
 

raml/examples/input/rule-new.json

@@ -1,7 +1,7 @@
 {
   "_id": "5a12f2923306be0016179f47",
   "project_id": "57e452791cff88b85f9f9c97",
-  "alg": "dcm2niix",
+  "gear_id": "580925ce9e512c57dc8a103c",
   "name": "dcm2niix",
   "all": [
     {

raml/examples/output/rule-list.json

@@ -1,6 +1,6 @@
 [
   {
-    "alg": "dcm2niix",
+    "gear_id": "580925ce9e512c57dc8a103b",
     "all": [
       {
         "regex": true,
@@ -17,7 +17,7 @@
     "any": []
   },
   {
-    "alg": "dicom-mr-classifier",
+    "gear_id": "580925ce9e512c57dc8a103c",
     "all": [
       {
         "type": "file.type",
@@ -34,7 +34,7 @@
     "any": []
   },
   {
-    "alg": "mriqc",
+    "gear_id": "580925ce9e512c57dc8a103d",
     "all": [
       {
         "type": "file.type",

raml/examples/rules_list.json

@@ -1,6 +1,6 @@
 [
     {
-        "alg" : "dicom_mr_classifier",
+        "gear_id": "580925ce9e512c57dc8a103c",
         "all" : [
             [
                 "file.type",

raml/schemas/definitions/rule.json

@@ -29,7 +29,7 @@
             "properties": {
                 "_id":        { "type": "string" },
                 "project_id": { "type": "string" },
-                "alg":        { "type": "string" },
+                "gear_id":    { "type": "string" },
                 "name":       { "type": "string" },
                 "any":        { "$ref": "#/definitions/rule-items" },
                 "all":        { "$ref": "#/definitions/rule-items" },
@@ -42,7 +42,7 @@
             "type": "object",
             "properties": {
                 "_id":      { "type": "string" },
-                "alg":      { "type": "string" },
+                "gear_id":  { "type": "string" },
                 "name":     { "type": "string" },
                 "any":      { "$ref": "#/definitions/rule-items" },
                 "all":      { "$ref": "#/definitions/rule-items" },

raml/schemas/input/rule-new.json

@@ -3,5 +3,5 @@
     "title": "Rule",
     "type": "object",
     "allOf": [{"$ref": "../definitions/rule.json#/definitions/rule-input"}],
-    "required": ["alg", "name", "any", "all"]
+    "required": ["gear_id", "name", "any", "all"]
 }

tests/integration_tests/python/test_jobs.py

@@ -60,7 +60,7 @@ def test_jobs(data_builder, default_payload, as_public, as_user, as_admin, as_ro
     job0 = copy.deepcopy(job_data)
     job0['gear_id'] = '000000000000000000000000'
     r = as_admin.post('/jobs/add', json=job0)
-    assert r.status_code == 400
+    assert r.status_code == 404
 
     # add job with explicit destination
     r = as_admin.post('/jobs/add', json=job_data)
@@ -353,15 +353,14 @@ def test_failed_job_output(data_builder, default_payload, as_user, as_admin, as_
     }
     gear = data_builder.create_gear(gear=gear_doc)
     gear2 = data_builder.create_gear()
-    gear2_name = as_admin.get('/gears/' + gear2).json()['gear']['name']
     project = data_builder.create_project()
     session = data_builder.create_session()
     acquisition = data_builder.create_acquisition()
     assert as_admin.post('/acquisitions/' + acquisition + '/files', files=file_form('test.zip')).ok
 
     # create rule for text files
     r = as_admin.post('/projects/' + project + '/rules', json={
-        'alg': gear2_name,
+        'gear_id': gear2,
         'name': 'text-trigger',
         'any': [],
         'all': [{'type': 'file.type', 'value': 'text'}]