pass a UserClientSet into SDKMethodRunner

README.md

@@ -52,6 +52,20 @@ PYTHONPATH=.:lib:test pytest --cov-report=xml --cov lib/execution_engine2/ --ver
 ## To run a specific test file via PyCharm
 See [Testing with Pycharm](docs/testing_with_pycharm.md)
 
+## To run pre-commit hooks
+
+`exec` into the docker container as before and switch to the `/ee2` directory.
+
+```
+pip install pre-commit
+pre-commit install
+pre-commit run --all-files
+```
+
+To remove the pre commit hooks:
+```
+pre-commit uninstall
+```
 
 
 ## Test Running Options  

lib/execution_engine2/execution_engine2Impl.py

@@ -6,6 +6,7 @@
 
 from lib.execution_engine2.db.MongoUtil import MongoUtil
 from lib.execution_engine2.sdk.SDKMethodRunner import SDKMethodRunner
+from execution_engine2.utils.APIHelpers import GenerateFromConfig
 
 
 #END_HEADER
@@ -59,9 +60,7 @@ def __init__(self, config):
             maxsize=self.ADMIN_ROLES_CACHE_SIZE, ttl=self.ADMIN_ROLES_CACHE_EXPIRE_TIME
         )
         self.mongo_util = MongoUtil(config)
-
-
-
+        self.gen_cfg = GenerateFromConfig(config)
         #END_CONSTRUCTOR
         pass
 
@@ -227,7 +226,8 @@ def run_job(self, ctx, params):
         # return variables are: job_id
         #BEGIN run_job
         mr = SDKMethodRunner(
-            self.config, user_id=ctx.get("user_id"), token=ctx.get("token"),
+            self.config,
+            user_clients=self.gen_cfg.get_user_clients(ctx),
             job_permission_cache=self.job_permission_cache,
             admin_permissions_cache=self.admin_permissions_cache, mongo_util=self.mongo_util
         )
@@ -298,7 +298,8 @@ def run_job_batch(self, ctx, params, batch_params):
         # return variables are: job_ids
         #BEGIN run_job_batch
         mr = SDKMethodRunner(
-            self.config, user_id=ctx.get("user_id"), token=ctx.get("token"),
+            self.config,
+            user_clients=self.gen_cfg.get_user_clients(ctx),
             job_permission_cache=self.job_permission_cache,
             admin_permissions_cache=self.admin_permissions_cache, mongo_util=self.mongo_util
         )
@@ -326,7 +327,8 @@ def abandon_children(self, ctx, params):
         # return variables are: parent_and_child_ids
         #BEGIN abandon_children
         mr = SDKMethodRunner(
-            self.config, user_id=ctx.get("user_id"), token=ctx.get("token"),
+            self.config,
+            user_clients=self.gen_cfg.get_user_clients(ctx),
             job_permission_cache=self.job_permission_cache,
             admin_permissions_cache=self.admin_permissions_cache, mongo_util=self.mongo_util
         )
@@ -408,7 +410,8 @@ def run_job_concierge(self, ctx, params, concierge_params):
         # return variables are: job_id
         #BEGIN run_job_concierge
         mr = SDKMethodRunner(
-            self.config, user_id=ctx.get("user_id"), token=ctx.get("token"),
+            self.config,
+            user_clients=self.gen_cfg.get_user_clients(ctx),
             mongo_util=self.mongo_util
         )
         job_id = mr.run_job_concierge(params=params,concierge_params=concierge_params)
@@ -478,8 +481,7 @@ def get_job_params(self, ctx, params):
         #BEGIN get_job_params
         mr = SDKMethodRunner(
             self.config,
-            user_id=ctx.get("user_id"),
-            token=ctx.get("token"),
+            user_clients=self.gen_cfg.get_user_clients(ctx),
             job_permission_cache=self.job_permission_cache,
             admin_permissions_cache=self.admin_permissions_cache,
             mongo_util=self.mongo_util
@@ -508,8 +510,7 @@ def update_job_status(self, ctx, params):
         #BEGIN update_job_status
         mr = SDKMethodRunner(
             self.config,
-            user_id=ctx.get("user_id"),
-            token=ctx.get("token"),
+            user_clients=self.gen_cfg.get_user_clients(ctx),
             job_permission_cache=self.job_permission_cache,
             admin_permissions_cache=self.admin_permissions_cache,
             mongo_util=self.mongo_util
@@ -547,8 +548,7 @@ def add_job_logs(self, ctx, params, lines):
         #BEGIN add_job_logs
         mr = SDKMethodRunner(
             self.config,
-            user_id=ctx.get("user_id"),
-            token=ctx.get("token"),
+            user_clients=self.gen_cfg.get_user_clients(ctx),
             job_permission_cache=self.job_permission_cache,
             admin_permissions_cache=self.admin_permissions_cache,
             mongo_util=self.mongo_util
@@ -599,8 +599,7 @@ def get_job_logs(self, ctx, params):
 
         mr = SDKMethodRunner(
             self.config,
-            user_id=ctx.get("user_id"),
-            token=ctx.get("token"),
+            user_clients=self.gen_cfg.get_user_clients(ctx),
             job_permission_cache=self.job_permission_cache,
             admin_permissions_cache=self.admin_permissions_cache,
             mongo_util=self.mongo_util
@@ -641,8 +640,7 @@ def finish_job(self, ctx, params):
         #BEGIN finish_job
         mr = SDKMethodRunner(
             self.config,
-            user_id=ctx.get("user_id"),
-            token=ctx.get("token"),
+            user_clients=self.gen_cfg.get_user_clients(ctx),
             job_permission_cache=self.job_permission_cache,
             admin_permissions_cache=self.admin_permissions_cache,
             mongo_util=self.mongo_util
@@ -671,8 +669,7 @@ def start_job(self, ctx, params):
         #BEGIN start_job
         mr = SDKMethodRunner(
             self.config,
-            user_id=ctx.get("user_id"),
-            token=ctx.get("token"),
+            user_clients=self.gen_cfg.get_user_clients(ctx),
             job_permission_cache=self.job_permission_cache,
             admin_permissions_cache=self.admin_permissions_cache,
             mongo_util=self.mongo_util
@@ -784,7 +781,8 @@ def check_job(self, ctx, params):
         # return variables are: job_state
         #BEGIN check_job
         mr = SDKMethodRunner(
-            self.config, user_id=ctx.get("user_id"), token=ctx.get("token"),
+            self.config,
+            user_clients=self.gen_cfg.get_user_clients(ctx),
             mongo_util=self.mongo_util
         )
         job_state = mr.check_job(
@@ -990,7 +988,8 @@ def check_job_batch(self, ctx, params):
         # return variables are: returnVal
         #BEGIN check_job_batch
         mr = SDKMethodRunner(
-            self.config, user_id=ctx.get("user_id"), token=ctx.get("token"),
+            self.config,
+            user_clients=self.gen_cfg.get_user_clients(ctx),
             mongo_util=self.mongo_util
         )
         returnVal = mr.check_job_batch(
@@ -1108,7 +1107,8 @@ def check_jobs(self, ctx, params):
         # return variables are: returnVal
         #BEGIN check_jobs
         mr = SDKMethodRunner(
-            self.config, user_id=ctx.get("user_id"), token=ctx.get("token"),
+            self.config,
+            user_clients=self.gen_cfg.get_user_clients(ctx),
             mongo_util=self.mongo_util
         )
         returnVal = mr.check_jobs(
@@ -1228,7 +1228,8 @@ def check_workspace_jobs(self, ctx, params):
         # ctx is the context object
         # return variables are: returnVal
         #BEGIN check_workspace_jobs
-        mr = SDKMethodRunner(self.config, user_id=ctx["user_id"], token=ctx["token"],
+        mr = SDKMethodRunner(self.config,
+                             user_clients=self.gen_cfg.get_user_clients(ctx),
                              mongo_util=self.mongo_util)
         returnVal = mr.check_workspace_jobs(
             params.get("workspace_id"),
@@ -1261,8 +1262,7 @@ def cancel_job(self, ctx, params):
         #BEGIN cancel_job
         mr = SDKMethodRunner(
             self.config,
-            user_id=ctx.get("user_id"),
-            token=ctx.get("token"),
+            user_clients=self.gen_cfg.get_user_clients(ctx),
             job_permission_cache=self.job_permission_cache,
             admin_permissions_cache=self.admin_permissions_cache,
             mongo_util=self.mongo_util
@@ -1300,7 +1300,8 @@ def check_job_canceled(self, ctx, params):
         # return variables are: result
         #BEGIN check_job_canceled
         mr = SDKMethodRunner(
-            self.config, user_id=ctx.get("user_id"), token=ctx.get("token"),
+            self.config,
+            user_clients=self.gen_cfg.get_user_clients(ctx),
             mongo_util=self.mongo_util
         )
         result = mr.check_job_canceled(job_id=params["job_id"], as_admin=params.get('as_admin'))
@@ -1327,8 +1328,7 @@ def get_job_status(self, ctx, params):
         #BEGIN get_job_status
         mr = SDKMethodRunner(
             self.config,
-            user_id=ctx.get("user_id"),
-            token=ctx.get("token"),
+            user_clients=self.gen_cfg.get_user_clients(ctx),
             job_permission_cache=self.job_permission_cache,
             admin_permissions_cache=self.admin_permissions_cache,
             mongo_util=self.mongo_util
@@ -1455,7 +1455,8 @@ def check_jobs_date_range_for_user(self, ctx, params):
         # return variables are: returnVal
         #BEGIN check_jobs_date_range_for_user
         mr = SDKMethodRunner(
-            self.config, user_id=ctx.get("user_id"), token=ctx.get("token"),
+            self.config,
+            user_clients=self.gen_cfg.get_user_clients(ctx),
             mongo_util=self.mongo_util
         )
         returnVal = mr.check_jobs_date_range_for_user(
@@ -1590,7 +1591,8 @@ def check_jobs_date_range_for_all(self, ctx, params):
         # return variables are: returnVal
         #BEGIN check_jobs_date_range_for_all
         mr = SDKMethodRunner(
-            self.config, user_id=ctx.get("user_id"), token=ctx.get("token"),
+            self.config,
+            user_clients=self.gen_cfg.get_user_clients(ctx),
             mongo_util=self.mongo_util
         )
         returnVal = mr.check_jobs_date_range_for_user(
@@ -1624,7 +1626,8 @@ def handle_held_job(self, ctx, cluster_id):
         # return variables are: returnVal
         #BEGIN handle_held_job
         mr = SDKMethodRunner(
-            self.config, user_id=ctx.get("user_id"), token=ctx.get("token"),
+            self.config,
+            user_clients=self.gen_cfg.get_user_clients(ctx),
             mongo_util=self.mongo_util
         )
         returnVal = mr.handle_held_job(cluster_id=cluster_id)
@@ -1646,7 +1649,8 @@ def is_admin(self, ctx):
         # return variables are: returnVal
         #BEGIN is_admin
         mr = SDKMethodRunner(
-            self.config, user_id=ctx.get("user_id"), token=ctx.get("token"),
+            self.config,
+            user_clients=self.get_cfg.get_user_clients(ctx),
             mongo_util=self.mongo_util
         )
         returnVal = mr.check_is_admin()
@@ -1671,7 +1675,8 @@ def get_admin_permission(self, ctx):
         # return variables are: returnVal
         #BEGIN get_admin_permission
         mr = SDKMethodRunner(
-            self.config, user_id=ctx.get("user_id"), token=ctx.get("token"),
+            self.config,
+            user_clients=self.get_cfg.get_user_clients(ctx),
             mongo_util=self.mongo_util
         )
         returnVal = mr.get_admin_permission()
@@ -1692,6 +1697,7 @@ def get_client_groups(self, ctx):
         # ctx is the context object
         # return variables are: client_groups
         #BEGIN get_client_groups
+        # TODO I think this needs to be actually extracted from the config file
         client_groups = ['njs', 'bigmem', 'bigmemlong', 'extreme', 'concierge', 'hpc', 'kb_upload',
                          'terabyte', 'multi_tb', 'kb_upload_bulk']
         #END get_client_groups

lib/execution_engine2/sdk/EE2Authentication.py

@@ -5,6 +5,7 @@
 from lib.execution_engine2.authorization.authstrategy import can_read_job, can_write_job
 from lib.execution_engine2.authorization.roles import AdminAuthUtil
 from lib.execution_engine2.db.models.models import Job
+from execution_engine2.sdk.EE2Constants import ADMIN_READ_ROLE, ADMIN_WRITE_ROLE
 
 
 class JobPermissions(Enum):
@@ -33,12 +34,12 @@ def _lookup_admin_permissions(self):
         aau = AdminAuthUtil(self.sdkmr.auth_url, self.sdkmr.admin_roles)
         p = aau.get_admin_role(
             token=self.sdkmr.token,
-            read_role=self.sdkmr.ADMIN_READ_ROLE,
-            write_role=self.sdkmr.ADMIN_WRITE_ROLE,
+            read_role=ADMIN_READ_ROLE,
+            write_role=ADMIN_WRITE_ROLE,
         )
-        if p == self.sdkmr.ADMIN_READ_ROLE:
+        if p == ADMIN_READ_ROLE:
             return AdminPermissions.READ
-        elif p == self.sdkmr.ADMIN_WRITE_ROLE:
+        elif p == ADMIN_WRITE_ROLE:
             return AdminPermissions.WRITE
         else:
             return AdminPermissions.NONE
@@ -149,16 +150,12 @@ def test_job_permissions(
         perm = False
         try:
             if level.value == JobPermissions.READ.value:
-                perm = can_read_job(
-                    job, self.sdkmr.user_id, self.sdkmr.get_workspace_auth()
-                )
+                perm = can_read_job(job, self.sdkmr.user_id, self.sdkmr.workspace_auth)
                 self._update_job_permission_cache(
                     job_id, self.sdkmr.user_id, level, perm
                 )
             elif level.value == JobPermissions.WRITE.value:
-                perm = can_write_job(
-                    job, self.sdkmr.user_id, self.sdkmr.get_workspace_auth()
-                )
+                perm = can_write_job(job, self.sdkmr.user_id, self.sdkmr.workspace_auth)
                 self._update_job_permission_cache(
                     job_id, self.sdkmr.user_id, level, perm
                 )

lib/execution_engine2/sdk/EE2Constants.py

@@ -9,6 +9,10 @@
 KBASE_CONCIERGE_USERNAME = "kbaseconcierge"
 CONCIERGE_CLIENTGROUP = "kbase_concierge"
 
+# these also probably should be configurable.
+ADMIN_READ_ROLE = "EE2_ADMIN_RO"
+ADMIN_WRITE_ROLE = "EE2_ADMIN"
+
 
 class JobError(NamedTuple):
     name: str

lib/execution_engine2/sdk/EE2Runjob.py

@@ -143,7 +143,7 @@ def _check_ws_objects(self, source_objects) -> None:
 
     def _check_workspace_permissions(self, wsid):
         if wsid:
-            if not self.sdkmr.get_workspace_auth().can_write(wsid):
+            if not self.sdkmr.workspace_auth.can_write(wsid):
                 self.logger.debug(
                     f"User {self.sdkmr.user_id} doesn't have permission to run jobs in workspace {wsid}."
                 )
@@ -152,7 +152,7 @@ def _check_workspace_permissions(self, wsid):
                 )
 
     def _check_workspace_permissions_list(self, wsids):
-        perms = self.sdkmr.get_workspace_auth().can_write_list(wsids)
+        perms = self.sdkmr.workspace_auth.can_write_list(wsids)
         bad_ws = [key for key in perms.keys() if perms[key] is False]
         if bad_ws:
             self.logger.debug(

lib/execution_engine2/sdk/EE2Status.py

@@ -445,7 +445,7 @@ def check_jobs(
                     "Checking for read permission to: {}".format(job_ids)
                 )
                 perms = can_read_jobs(
-                    jobs, self.sdkmr.user_id, self.sdkmr.get_workspace_auth()
+                    jobs, self.sdkmr.user_id, self.sdkmr.workspace_auth
                 )
             except RuntimeError as e:
                 self.sdkmr.logger.error(
@@ -502,8 +502,7 @@ def check_workspace_jobs(self, workspace_id, exclude_fields=None, return_list=No
         if exclude_fields is None:
             exclude_fields = []
 
-        ws_auth = self.sdkmr.get_workspace_auth()
-        if not ws_auth.can_read(workspace_id):
+        if not self.sdkmr.workspace_auth.can_read(workspace_id):
             self.sdkmr.logger.debug(
                 f"User {self.sdkmr.user_id} doesn't have permission to read jobs in workspace {workspace_id}."
             )