Auth OIDC

.github/workflows/qiita-ci.yml

@@ -64,7 +64,7 @@ jobs:
           # Setting up main qiita conda environment
           conda config --add channels conda-forge
           conda deactivate
-          conda create --quiet --yes -n qiita python=3.9 pip libgfortran numpy nginx cython redis
+          conda create --quiet --yes -n qiita python=3.9 pip libgfortran numpy nginx cython redis pycurl
           conda env list
           conda activate qiita
           pip install -U pip

qiita_core/configuration_manager.py

@@ -127,6 +127,32 @@ class ConfigurationManager(object):
         The email address a user should write to when asking for help
     sysadmin_email : str
         The email address, Qiita sends internal notifications to a sys admin
+    None (=internal user authentication) or one or several 'oidc_' sections
+    to use external identity providers (IdP) with following values:
+    client_id : str
+        The name you registered Qiita with at the external IdP
+    client_secret : str
+        A secret string with which Qiita identifies at the external IdP (not
+        all IdPs need a secret)
+    redirect_endpoint : str
+        The internal Qiita endpoint the IdP shall redirect the user after
+        logging in
+    wellknown_uri : str
+        The URL of the well-known json document, specifying how API end points
+        like 'authorize', 'token' or 'userinfo' are defined. See e.g.
+        https://swagger.io/docs/specification/authentication/
+            openid-connect-discovery/
+    label : str
+        A speaking label for the Identity Provider
+    scope : str
+        The scope, i.e. fields about a user, which Qiita requests from the
+        Identity Provider, e.g. "profile email eduperson_orcid".
+        Will be automatically extended by the scope "openid", to enable the
+        "authorize_code" OIDC flow.
+    logo : str
+        Optional. Name of a file in qiita_pet/static/img that shall be
+        displayed for login through Service Provider, instead of a plain
+        button
 
     Raises
     ------
@@ -162,6 +188,7 @@ def __init__(self):
         self._get_vamps(config)
         self._get_portal(config)
         self._iframe(config)
+        self._get_oidc(config)
 
     def _get_main(self, config):
         """Get the configuration of the main section"""
@@ -390,3 +417,43 @@ def _get_portal(self, config):
 
     def _iframe(self, config):
         self.iframe_qiimp = config.get('iframe', 'QIIMP', fallback=None)
+
+    def _get_oidc(self, config):
+        """Get the configuration of the open ID connect section(s)
+           User can provide multiple sections with naming schema oidc_foo where
+           foo is the name of an Identity Provider - Qiita can handle multiple
+           Identity Providers simultaneously.
+           """
+        PREFIX = 'oidc_'
+        self.oidc = dict()
+        for section_name in config.sections():
+            if section_name.startswith(PREFIX):
+                provider = dict()
+                provider['client_id'] = config.get(
+                    section_name, 'CLIENT_ID', fallback=None)
+                provider['client_secret'] = config.get(
+                    section_name, 'CLIENT_SECRET', fallback=None)
+                provider['redirect_endpoint'] = config.get(
+                    section_name, 'REDIRECT_ENDPOINT')
+                if provider['redirect_endpoint']:
+                    if not provider['redirect_endpoint'].startswith('/'):
+                        provider['redirect_endpoint'] = '/%s' % provider[
+                            'redirect_endpoint']
+                    if provider['redirect_endpoint'].endswith('/'):
+                        provider['redirect_endpoint'] = provider[
+                            'redirect_endpoint'][:-1]
+                provider['wellknown_uri'] = config.get(
+                    section_name, 'WELLKNOWN_URI')
+                provider['label'] = config.get(section_name, 'LABEL')
+                if not provider['label']:
+                    # fallback, if no label is provided
+                    provider['label'] = section_name[len(PREFIX):]
+                self.oidc[section_name[len(PREFIX):]] = provider
+                provider['scope'] = config.get(
+                    section_name, 'SCOPE', fallback=None)
+                if not provider['scope']:
+                    provider['scope'] = 'openid'
+                if 'openid' not in provider['scope']:
+                    provider['scope'] = 'openid %s' % provider['scope']
+                provider['logo'] = config.get(
+                    section_name, 'LOGO', fallback=None)

qiita_core/support_files/config_test.cfg

@@ -196,3 +196,68 @@ STATS_MAP_CENTER_LONGITUDE =
 # On May 2024, we removed QIIMP from the code base but we will leave this
 # section in case we need to add access to another iframe in the future; note
 # that the qiita-terms are also accessed via iframe but this is internal
+
+# --------------------- External Identity Provider settings --------------------
+# user authentication happens per default within Qiita, i.e. when a user logs in,
+# the stored password hash and email address is compared against what a user
+# just provided. You might however, use an external identity provider (IdP) to
+# authenticate the user like
+#    google: https://developers.google.com/identity/protocols/oauth2 or
+#    github: https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/authorizing-oauth-apps or
+#    self hosted keycloak: https://www.keycloak.org/
+# Thus, you don't have to deal with user verification, reset passwords, ...
+# Authorization (i.e. if the authorized user is allowed to use Qiita or which
+# user level he/she gets assigned is an independent process. You can even use
+# multiple independent external identity providers!
+# Qiita currently only support the "open ID connect" protocol with the implicit flow.
+# Each identity provider comes as its own config section [oidc_foo] and needs
+# to specify the following five fields:
+#
+# Typical identity provider manage multiple "realms" and specific "clients" per realm
+# You need to contact your IdP and register Qiita as a new "client". The IdP will
+# provide you with the correct values.
+#
+# The authorization protocol requires three steps to obtain user information:
+# 1) you identify as the correct client and ask the IdP for a request code
+#    You have to forward the user to the login page of your IdP. To let the IdP
+#    know how to come back to Qiita, you need to provide a redirect URL
+# 2) you exchange the code for a user token
+# 3) you obtain information about the user for the obtaines user token
+# Typically, each step is implemented as a separate URL endpoint
+#
+# To activate IdP: comment out the following config section
+
+#[oidc_academicid]
+#
+## client ID for Qiita as registered at your Identity Provider of choice
+#CLIENT_ID = gi-qiita-prod
+#
+## client secret to verify Qiita as the correct client. Not all IdPs require
+## a client secret!
+#CLIENT_SECRET = verySecretString
+#
+## redirect URL (end point in your Qiita instance), to which the IdP redirects
+## after user types in his/her credentials. If you don't want to change code in
+## qiita_pet/webserver.py the URL must follow the pattern:
+## base_URL/auth/login_OIDC/foo where foo is the name of this config section
+## without the oidc_ prefix!
+#REDIRECT_ENDPOINT = /auth/login_OIDC/localkeycloak
+#
+## The URL of the well-known json document, specifying how API end points
+## like 'authorize', 'token' or 'userinfo' are defined. See e.g.
+## https://swagger.io/docs/specification/authentication/
+##    openid-connect-discovery/
+#WELLKNOWN_URI = https://keycloak.sso.gwdg.de/.well-known/openid-configuration
+#
+## a speaking label for the Identity Provider. Section name is used if empty.
+#LABEL = GWDG Academic Cloud
+#
+## The scope, i.e. fields about a user, which Qiita requests from the
+## Identity Provider, e.g. "profile email eduperson_orcid".
+## Will be automatically extended by the scope "openid", to enable the
+## "authorize_code" OIDC flow.
+#SCOPE = openid
+#
+##Optional. Name of a file in qiita_pet/static/img that shall be
+##displayed for login through Service Provider, instead of a plain button
+#LOGO = oidc_lifescienceAAI.png

qiita_core/tests/test_configuration_manager.py

@@ -289,6 +289,55 @@ def test_get_portal_latlong(self):
         obs._get_portal(self.conf)
         self.assertEqual(obs.stats_map_center_longitude, -105.24827)
 
+    def test_get_oidc(self):
+        SECTION_NAME = 'oidc_academicid'
+        obs = ConfigurationManager()
+        self.assertTrue(len(obs.oidc), 1)
+        self.assertTrue(obs.oidc.keys(), [SECTION_NAME])
+
+        # assert endpoint starts with /
+        self.conf.set(SECTION_NAME, 'REDIRECT_ENDPOINT', 'auth/something')
+        obs._get_oidc(self.conf)
+        self.assertEqual(obs.oidc['academicid']['redirect_endpoint'],
+                         '/auth/something')
+
+        # assert endpoint does not end with /
+        self.conf.set(SECTION_NAME, 'REDIRECT_ENDPOINT', 'auth/something/')
+        obs._get_oidc(self.conf)
+        self.assertEqual(obs.oidc['academicid']['redirect_endpoint'],
+                         '/auth/something')
+
+        self.conf.set(SECTION_NAME, 'CLIENT_ID', 'foo')
+        obs._get_oidc(self.conf)
+        self.assertEqual(obs.oidc['academicid']['client_id'], "foo")
+
+        self.assertTrue('gwdg.de' in obs.oidc['academicid']['wellknown_uri'])
+
+        self.assertEqual(obs.oidc['academicid']['label'],
+                         'GWDG Academic Cloud')
+        # test fallback, if no label is provided
+        self.conf.set(SECTION_NAME, 'LABEL', '')
+        obs._get_oidc(self.conf)
+        self.assertEqual(obs.oidc['academicid']['label'], 'academicid')
+
+        self.assertEqual(obs.oidc['academicid']['scope'], 'openid')
+        # test fallback, if no scope is provided
+        self.conf.set(SECTION_NAME, 'SCOPE', '')
+        obs._get_oidc(self.conf)
+        self.assertEqual(obs.oidc['academicid']['scope'], 'openid')
+
+        # test if scope will be automatically extended with 'openid'
+        self.conf.set(SECTION_NAME, 'SCOPE', 'email affiliation')
+        obs._get_oidc(self.conf)
+        self.assertTrue('openid' in obs.oidc['academicid']['scope'].split())
+
+        self.assertEqual(obs.oidc['academicid']['logo'],
+                         'oidc_lifescienceAAI.png')
+        # test fallback, if no scope is provided
+        self.conf.remove_option(SECTION_NAME, 'LOGO')
+        obs._get_oidc(self.conf)
+        self.assertEqual(obs.oidc['academicid']['logo'], None)
+
 
 CONF = """
 # ------------------------------ Main settings --------------------------------
@@ -471,6 +520,42 @@ def test_get_portal_latlong(self):
 
 # ----------------------------- iframes settings ---------------------------
 [iframe]
+
+# ------------------- External Identity Provider settings ------------------
+[oidc_academicid]
+
+# client ID for Qiita as registered at your Identity Provider of choice
+CLIENT_ID = gi-qiita-prod
+
+# client secret to verify Qiita as the correct client. Not all IdPs require
+# a client secret.
+CLIENT_SECRET = verySecretString
+
+# redirect URL (end point in your Qiita instance), to which the IdP redirects
+# after user types in his/her credentials. If you don't want to change code in
+# qiita_pet/webserver.py the URL must follow the pattern:
+# base_URL/auth/login_OIDC/foo where foo is the name of this config section
+# without the oidc_ prefix!
+REDIRECT_ENDPOINT = /auth/login_OIDC/academicid
+
+# The URL of the well-known json document, specifying how API end points
+# like 'authorize', 'token' or 'userinfo' are defined. See e.g.
+# https://swagger.io/docs/specification/authentication/
+#    openid-connect-discovery/
+WELLKNOWN_URI = https://keycloak.sso.gwdg.de/.well-known/openid-configuration
+
+# a speaking label for the Identity Provider. Section name is used if empty.
+LABEL = GWDG Academic Cloud
+
+# The scope, i.e. fields about a user, which Qiita requests from the
+# Identity Provider, e.g. "profile email eduperson_orcid".
+# Will be automatically extended by the scope "openid", to enable the
+# "authorize_code" OIDC flow.
+SCOPE = openid
+
+# Optional. Name of a file in qiita_pet/static/img that shall be
+# displayed for login through Service Provider, instead of a plain button
+LOGO = oidc_lifescienceAAI.png
 """
 
 if __name__ == '__main__':

qiita_db/environment_manager.py

@@ -7,6 +7,7 @@
 # -----------------------------------------------------------------------------
 
 from os.path import abspath, dirname, join, exists, basename, splitext
+from shutil import copytree
 from functools import partial
 from os import mkdir
 import gzip
@@ -127,6 +128,36 @@ def _download_reference_files():
         _insert_processed_params(ref)
 
 
+def create_mountpoints():
+    r"""In a fresh qiita setup, sub-directories under
+        qiita_config.base_data_dir might not yet exist. To avoid failing in
+        later steps, they are created here.
+    """
+    with qdb.sql_connection.TRN:
+        sql = """SELECT DISTINCT mountpoint FROM qiita.data_directory
+                 WHERE active = TRUE"""
+        qdb.sql_connection.TRN.add(sql)
+        created_subdirs = []
+        for mountpoint in qdb.sql_connection.TRN.execute_fetchflatten():
+            for (ddid, subdir) in qdb.util.get_mountpoint(mountpoint,
+                                                          retrieve_all=True):
+                if not exists(join(qiita_config.base_data_dir, subdir)):
+                    if qiita_config.test_environment:
+                        # if in test mode, we want to potentially fill the
+                        # new directory with according test data
+                        copytree(get_support_file('test_data', mountpoint),
+                                 join(qiita_config.base_data_dir, subdir))
+                    else:
+                        # in production mode, an empty directory is created
+                        mkdir(join(qiita_config.base_data_dir, subdir))
+                    created_subdirs.append(subdir)
+
+        if len(created_subdirs) > 0:
+            print("Created %i sub-directories as 'mount points':\n%s"
+                  % (len(created_subdirs),
+                     ''.join(map(lambda x: ' - %s\n' % x, created_subdirs))))
+
+
 def make_environment(load_ontologies, download_reference, add_demo_user):
     r"""Creates the new environment specified in the configuration
 
@@ -397,6 +428,9 @@ def patch(patches_dir=PATCHES_DIR, verbose=False, test=False):
         with qdb.sql_connection.TRN:
             _populate_test_db()
 
+    # create mountpoints as subdirectories in BASE_DATA_DIR
+    create_mountpoints()
+
     patch_update_sql = "UPDATE settings SET current_patch = %s"
     for sql_patch_fp in sql_patch_files[next_patch_index:]:
         sql_patch_filename = basename(sql_patch_fp)

qiita_db/processing_job.py

@@ -1105,13 +1105,23 @@ def submit(self, parent_job_id=None, dependent_jobs_list=None):
                     # organized into n 'queues' or 'chains', and
                     # will all run simultaneously.
                     for dependent in dependent_jobs_list:
+                        # register dependent job as queued to make qiita
+                        # aware of this child process
+                        dependent._set_status('queued')
+
+                        dep_software = dependent.command.software
+                        dep_job_dir = join(qdb.util.get_work_base_dir(),
+                                           dependent.id)
                         p = Process(target=launcher['function'],
-                                    args=(plugin_env_script,
-                                          plugin_start_script,
+                                    args=(dep_software.environment_script,
+                                          dep_software.start_script,
                                           url,
-                                          self.id,
-                                          job_dir))
+                                          dependent.id,
+                                          dep_job_dir))
                         p.start()
+                        # assign the child process ID as external id to
+                        # the dependent
+                        dependent.external_id = p.pid
             else:
                 error = ("execute_in_process must be defined",
                          "as either true or false")

qiita_db/support_files/test_data/FASTQ/blank.txt

@@ -0,0 +1 @@
+