From aab4750e89e1d93a904ef6ebacdeae526d85fd87 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 08:26:34 +0100 Subject: [PATCH 01/61] extended configuration manager with optional OIDC sections --- qiita_core/configuration_manager.py | 51 +++++++++++++++++ qiita_core/support_files/config_test.cfg | 55 +++++++++++++++++++ .../tests/test_configuration_manager.py | 52 ++++++++++++++++++ 3 files changed, 158 insertions(+) diff --git a/qiita_core/configuration_manager.py b/qiita_core/configuration_manager.py index 4c26583d9..5490e28e2 100644 --- a/qiita_core/configuration_manager.py +++ b/qiita_core/configuration_manager.py @@ -117,6 +117,24 @@ class ConfigurationManager(object): The script used to start the plugins plugin_dir : str The path to the directory containing the plugin configuration files + None (=internal user authentication) or one or several 'oidc_' sections + to use external identity providers (IdP) with following values: + client_id : str + The name you registered Qiita with at the external IdP + client_secret : str + A secret string with which Qiita identifies at the external IdP (not + all IdPs need a secret) + redirect_endpoint : str + The internal Qiita endpoint the IdP shall redirect the user after + logging in + authorize_url : str + The URL of the IdP to obtain a code (step 1) + accesstoken_url : str + The URL of the IdP to exchange the code from step 1 for an access + token (step 2) + userinfo_url : str + The URL of the IdP to obtain information about the user, like email, + username, ... Raises ------ @@ -152,6 +170,7 @@ def __init__(self): self._get_vamps(config) self._get_portal(config) self._iframe(config) + self._get_oidc(config) def _get_main(self, config): """Get the configuration of the main section""" @@ -319,3 +338,35 @@ def _get_portal(self, config): def _iframe(self, config): self.iframe_qiimp = config.get('iframe', 'QIIMP') + + def _get_oidc(self, config): + """Get the configuration of the open ID connect section(s) + User can provide multiple sections with naming schema oidc_foo where + foo is the name of an Identity Provider - Qiita can handle multiple + Identity Providers simultaneously. + """ + PREFIX = 'oidc_' + self.oidc = dict() + for section_name in config.sections(): + if section_name.startswith(PREFIX): + provider = dict() + provider['client_id'] = config.get( + section_name, 'CLIENT_ID', fallback=None) + provider['client_secret'] = config.get( + section_name, 'CLIENT_SECRET', fallback=None) + provider['redirect_endpoint'] = config.get( + section_name, 'REDIRECT_ENDPOINT') + if provider['redirect_endpoint']: + if not provider['redirect_endpoint'].startswith('/'): + provider['redirect_endpoint'] = '/%s' % provider[ + 'redirect_endpoint'] + if provider['redirect_endpoint'].endswith('/'): + provider['redirect_endpoint'] = provider[ + 'redirect_endpoint'][:-1] + provider['authorize_url'] = config.get( + section_name, 'AUTHORIZE_URL') + provider['accesstoken_url'] = config.get( + section_name, 'ACCESS_TOKEN_URL') + provider['userinfo_url'] = config.get( + section_name, 'USERINFO_URL') + self.oidc[section_name[len(PREFIX):]] = provider diff --git a/qiita_core/support_files/config_test.cfg b/qiita_core/support_files/config_test.cfg index e6423f609..0d8cb2435 100644 --- a/qiita_core/support_files/config_test.cfg +++ b/qiita_core/support_files/config_test.cfg @@ -183,3 +183,58 @@ PORTAL_FP = # The real world QIIMP will always need to be accessed with https because Qiita # runs on https too QIIMP = https://localhost:8898/ + + +# --------------------- External Identity Provider settings -------------------- +# user authentication happens per default within Qiita, i.e. when a user logs in, +# the stored password hash and email address is compared against what a user +# just provided. You might however, use an external identity provider (IdP) to +# authenticate the user like +# google: https://developers.google.com/identity/protocols/oauth2 or +# github: https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/authorizing-oauth-apps or +# self hosted keycloak: https://www.keycloak.org/ +# Thus, you don't have to deal with user verification, reset passwords, ... +# Authorization (i.e. if the authorized user is allowed to use Qiita or which +# user level he/she gets assigned is an independent process. You can even use +# multiple independent external identity providers! +# Qiita currently only support the "open ID connect" protocol with the implicit flow. +# Each identity provider comes as its own config section [oidc_foo] and needs +# to specify the following five fields: +# +# Typical identity provider manage multiple "realms" and specific "clients" per realm +# You need to contact your IdP and register Qiita as a new "client". The IdP will +# provide you with the correct values. +# +# The authorization protocol requires three steps to obtain user information: +# 1) you identify as the correct client and ask the IdP for a request code +# You have to forward the user to the login page of your IdP. To let the IdP +# know how to come back to Qiita, you need to provide a redirect URL +# 2) you exchange the code for a user token +# 3) you obtain information about the user for the obtaines user token +# Typically, each step is implemented as a separate URL endpoint +# +# To activate IdP: comment out the following config section + +#[oidc_academicid] +# +## client ID for Qiita as registered at your Identity Provider of choice +#CLIENT_ID = gi-qiita-prod +# +## client secret to verify Qiita as the correct client. Not all IdPs require this +#CLIENT_SECRET = 5M6zKl8SKrlnRP4tPgtrgZpCpcYCj7uK +# +## redirect URL (end point in your Qiita instance), to which the IdP redirects +## after user types in his/her credentials. If you don't want to change code in +## qiita_pet/webserver.py the URL must follow the pattern: +## base_URL/auth/login_OIDC/foo where foo is the name of this config section +## without the oidc_ prefix! +#REDIRECT_ENDPOINT = /auth/login_OIDC/localkeycloak +# +## URL for step 1: obtain code +#AUTHORIZE_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/auth +# +## URL for step 2: obtain user token +#ACCESS_TOKEN_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/token +# +## URL for step 3: obtain user infos +#USERINFO_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/userinfo diff --git a/qiita_core/tests/test_configuration_manager.py b/qiita_core/tests/test_configuration_manager.py index 2a60f353b..765c300ac 100644 --- a/qiita_core/tests/test_configuration_manager.py +++ b/qiita_core/tests/test_configuration_manager.py @@ -214,6 +214,32 @@ def test_get_portal(self): obs._get_portal(self.conf) self.assertEqual(obs.portal_dir, "/gold_portal") + def test_get_postgres(self): + SECTION_NAME = 'oidc_academicid' + obs = ConfigurationManager() + self.assertTrue(len(obs.oidc), 1) + self.assertTrue(obs.oidc.keys(), [SECTION_NAME]) + + # assert endpoint starts with / + self.conf.set(SECTION_NAME, 'REDIRECT_ENDPOINT', 'auth/something') + obs._get_oidc(self.conf) + self.assertEqual(obs.oidc['academicid']['redirect_endpoint'], + '/auth/something') + + # assert endpoint does not end with / + self.conf.set(SECTION_NAME, 'REDIRECT_ENDPOINT', 'auth/something/') + obs._get_oidc(self.conf) + self.assertEqual(obs.oidc['academicid']['redirect_endpoint'], + '/auth/something') + + self.conf.set(SECTION_NAME, 'CLIENT_ID', 'foo') + obs._get_oidc(self.conf) + self.assertEqual(obs.oidc['academicid']['client_id'], "foo") + + self.assertTrue('gwdg.de' in obs.oidc['academicid']['authorize_url']) + self.assertTrue('gwdg.de' in obs.oidc['academicid']['accesstoken_url']) + self.assertTrue('gwdg.de' in obs.oidc['academicid']['userinfo_url']) + CONF = """ # ------------------------------ Main settings -------------------------------- @@ -383,6 +409,32 @@ def test_get_portal(self): # ----------------------------- iframes settings --------------------------- [iframe] QIIMP = https://localhost:8898/ + +# ------------------- External Identity Provider settings ------------------ +[oidc_academicid] + +# client ID for Qiita as registered at your Identity Provider of choice +CLIENT_ID = gi-qiita-prod + +# client secret to verify Qiita as the correct client. Not all IdPs require this +CLIENT_SECRET = 5M6zKl8SKrlnRP4tPgtrgZpCpcYCj7uK + +# redirect URL (end point in your Qiita instance), to which the IdP redirects +# after user types in his/her credentials. If you don't want to change code in +# qiita_pet/webserver.py the URL must follow the pattern: +# base_URL/auth/login_OIDC/foo where foo is the name of this config section +# without the oidc_ prefix! +REDIRECT_ENDPOINT = /auth/login_OIDC/academicid + +# URL for step 1: obtain code +AUTHORIZE_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/auth + +# URL for step 2: obtain user token +ACCESS_TOKEN_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/token + +# URL for step 3: obtain user infos +USERINFO_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/userinfo + """ if __name__ == '__main__': From 49b0448bb2fcaabd4b0c46b5444a27b9953afdf5 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 08:32:30 +0100 Subject: [PATCH 02/61] flake8 --- qiita_core/support_files/config_test.cfg | 3 ++- qiita_core/tests/test_configuration_manager.py | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/qiita_core/support_files/config_test.cfg b/qiita_core/support_files/config_test.cfg index 0d8cb2435..3ff881c01 100644 --- a/qiita_core/support_files/config_test.cfg +++ b/qiita_core/support_files/config_test.cfg @@ -220,7 +220,8 @@ QIIMP = https://localhost:8898/ ## client ID for Qiita as registered at your Identity Provider of choice #CLIENT_ID = gi-qiita-prod # -## client secret to verify Qiita as the correct client. Not all IdPs require this +## client secret to verify Qiita as the correct client. Not all IdPs require +## a client secret! #CLIENT_SECRET = 5M6zKl8SKrlnRP4tPgtrgZpCpcYCj7uK # ## redirect URL (end point in your Qiita instance), to which the IdP redirects diff --git a/qiita_core/tests/test_configuration_manager.py b/qiita_core/tests/test_configuration_manager.py index 765c300ac..2d7407ad5 100644 --- a/qiita_core/tests/test_configuration_manager.py +++ b/qiita_core/tests/test_configuration_manager.py @@ -214,7 +214,7 @@ def test_get_portal(self): obs._get_portal(self.conf) self.assertEqual(obs.portal_dir, "/gold_portal") - def test_get_postgres(self): + def test_get_oidc(self): SECTION_NAME = 'oidc_academicid' obs = ConfigurationManager() self.assertTrue(len(obs.oidc), 1) @@ -416,7 +416,8 @@ def test_get_postgres(self): # client ID for Qiita as registered at your Identity Provider of choice CLIENT_ID = gi-qiita-prod -# client secret to verify Qiita as the correct client. Not all IdPs require this +# client secret to verify Qiita as the correct client. Not all IdPs require +# a client secret. CLIENT_SECRET = 5M6zKl8SKrlnRP4tPgtrgZpCpcYCj7uK # redirect URL (end point in your Qiita instance), to which the IdP redirects @@ -427,14 +428,13 @@ def test_get_postgres(self): REDIRECT_ENDPOINT = /auth/login_OIDC/academicid # URL for step 1: obtain code -AUTHORIZE_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/auth +AUTHORIZE_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/auth # URL for step 2: obtain user token -ACCESS_TOKEN_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/token +ACCESS_TOKEN_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/token # URL for step 3: obtain user infos -USERINFO_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/userinfo - +USERINFO_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/userinfo """ if __name__ == '__main__': From 28406012b453418ae06031a66094ea351ed524b5 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 12:12:45 +0100 Subject: [PATCH 03/61] also provide a label for a speaking name of the identity provider --- qiita_core/configuration_manager.py | 6 ++++++ qiita_core/support_files/config_test.cfg | 3 +++ qiita_core/tests/test_configuration_manager.py | 10 ++++++++++ 3 files changed, 19 insertions(+) diff --git a/qiita_core/configuration_manager.py b/qiita_core/configuration_manager.py index 5490e28e2..a8c6a9799 100644 --- a/qiita_core/configuration_manager.py +++ b/qiita_core/configuration_manager.py @@ -135,6 +135,8 @@ class ConfigurationManager(object): userinfo_url : str The URL of the IdP to obtain information about the user, like email, username, ... + label : str + A speaking label for the Identity Provider Raises ------ @@ -369,4 +371,8 @@ def _get_oidc(self, config): section_name, 'ACCESS_TOKEN_URL') provider['userinfo_url'] = config.get( section_name, 'USERINFO_URL') + provider['label'] = config.get(section_name, 'LABEL') + if not provider['label']: + # fallback, if no label is provided + provider['label'] = section_name[len(PREFIX):] self.oidc[section_name[len(PREFIX):]] = provider diff --git a/qiita_core/support_files/config_test.cfg b/qiita_core/support_files/config_test.cfg index 3ff881c01..dccfc8abf 100644 --- a/qiita_core/support_files/config_test.cfg +++ b/qiita_core/support_files/config_test.cfg @@ -239,3 +239,6 @@ QIIMP = https://localhost:8898/ # ## URL for step 3: obtain user infos #USERINFO_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/userinfo +# +## a speaking label for the Identity Provider. Section name is used if empty. +#LABEL = GWDG Academic Cloud diff --git a/qiita_core/tests/test_configuration_manager.py b/qiita_core/tests/test_configuration_manager.py index 2d7407ad5..5c2044f4d 100644 --- a/qiita_core/tests/test_configuration_manager.py +++ b/qiita_core/tests/test_configuration_manager.py @@ -240,6 +240,13 @@ def test_get_oidc(self): self.assertTrue('gwdg.de' in obs.oidc['academicid']['accesstoken_url']) self.assertTrue('gwdg.de' in obs.oidc['academicid']['userinfo_url']) + self.assertEqual(obs.oidc['academicid']['label'], + 'GWDG Academic Cloud') + # test fallback, if no label is provided + self.conf.set(SECTION_NAME, 'LABEL', '') + obs._get_oidc(self.conf) + self.assertEqual(obs.oidc['academicid']['label'], 'academicid') + CONF = """ # ------------------------------ Main settings -------------------------------- @@ -435,6 +442,9 @@ def test_get_oidc(self): # URL for step 3: obtain user infos USERINFO_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/userinfo + +# a speaking label for the Identity Provider. Section name is used if empty. +LABEL = GWDG Academic Cloud """ if __name__ == '__main__': From f1c91491799173e9a81aecf00834e9f6a0bb126e Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 12:13:20 +0100 Subject: [PATCH 04/61] start implementing the OIDC dance --- qiita_pet/handlers/auth_handlers.py | 19 ++++++++++++++ qiita_pet/templates/sitebase.html | 39 +++++++++++++++++++++++++++++ qiita_pet/webserver.py | 14 ++++++++++- 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/qiita_pet/handlers/auth_handlers.py b/qiita_pet/handlers/auth_handlers.py index 38759e77f..7e8762a7b 100644 --- a/qiita_pet/handlers/auth_handlers.py +++ b/qiita_pet/handlers/auth_handlers.py @@ -7,6 +7,7 @@ # ----------------------------------------------------------------------------- from tornado.escape import url_escape, json_encode +from tornado.auth import OAuth2Mixin from qiita_pet.handlers.base_handlers import BaseHandler from qiita_core.qiita_settings import qiita_config, r_client @@ -175,3 +176,21 @@ class AuthLogoutHandler(BaseHandler): def get(self): self.clear_cookie("user") self.redirect("%s/" % qiita_config.portal_dir) + + +class KeycloakMixin(OAuth2Mixin): + pass + + +class AuthLoginOIDCHandler(BaseHandler, KeycloakMixin): + async def get(self, login): + code = self.get_argument('code', False) + if code: + # step 2: we got a code and now want to exchange it for a user + # access token + print("step2") + pass + else: + # step 1: no code from IdP yet, thus retrieve one now + print("step1") + pass diff --git a/qiita_pet/templates/sitebase.html b/qiita_pet/templates/sitebase.html index 03e5dd1d0..091ad4ac7 100644 --- a/qiita_pet/templates/sitebase.html +++ b/qiita_pet/templates/sitebase.html @@ -160,6 +160,18 @@ // Based on http://codepen.io/willvincent/pen/LbeKKW // and https://datatables.net/examples/api/row_details.html + {% if len(qiita_config.oidc) > 0 %} + $('.modal').css({ + 'top': '30%', + 'margin-top': '-'+($('.modal').height() / 2)+'px', + 'margin-left': '+'+($('.modal').width() / 3)+'px'}); + $('#qiita_signin_modal').click(function(e){ + e.preventDefault(); + $('.qiita_auth_signin_options').modal('hide'); + window.location.href = '{% raw qiita_config.portal_dir %}'; + }); + {% end %} + Vue.component('data-table-processing-jobs', { template: '
', props: ['jobs'], @@ -443,6 +455,33 @@ + + {% elif len(qiita_config.oidc) == 1 %} + {% for idp in qiita_config.oidc %} +
+ +
+ {% end %} + + {% elif len(qiita_config.oidc) > 1 %} +
    + + +
{% else %}
diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py index 84d71431a..27e4b0d37 100644 --- a/qiita_pet/webserver.py +++ b/qiita_pet/webserver.py @@ -20,7 +20,8 @@ from qiita_pet.handlers.base_handlers import ( MainHandler, NoPageHandler, IFrame) from qiita_pet.handlers.auth_handlers import ( - AuthCreateHandler, AuthLoginHandler, AuthLogoutHandler, AuthVerifyHandler) + AuthCreateHandler, AuthLoginHandler, AuthLogoutHandler, AuthVerifyHandler, + AuthLoginOIDCHandler) from qiita_pet.handlers.user_handlers import ( ChangeForgotPasswordHandler, ForgotPasswordHandler, UserProfileHandler, UserMessagesHander, UserJobs) @@ -237,6 +238,16 @@ def __init__(self): (r"/qiita_db/studies/(.*)", APIStudiesListing) ] + # only expose open id connect endpoints iff at least one was configured + # through the settings file + if len(qiita_config.oidc) > 0: + handlers.extend([ + (r"/auth/login_OIDC/(.*)", AuthLoginOIDCHandler) + #(r"/admin/user_authorization/", AdminOIDCUserAuthorization), + #(r"/admin/user_authorizationAjax/", + # AdminOIDCUserAuthorizationAjax), + ]) + # rest endpoints handlers.extend(REST_ENDPOINTS) @@ -268,4 +279,5 @@ def __init__(self): "cookie_secret": qiita_config.cookie_secret, "login_url": "%s/auth/login/" % qiita_config.portal_dir, } + tornado.web.Application.__init__(self, handlers, **settings) From 2eb6d0852dcda70e5d06d0a3680b7643b1d875b7 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 12:57:55 +0100 Subject: [PATCH 05/61] modal not necessary, if only one provider was defined --- qiita_pet/templates/sitebase.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_pet/templates/sitebase.html b/qiita_pet/templates/sitebase.html index 091ad4ac7..65d8e6d68 100644 --- a/qiita_pet/templates/sitebase.html +++ b/qiita_pet/templates/sitebase.html @@ -160,7 +160,7 @@ // Based on http://codepen.io/willvincent/pen/LbeKKW // and https://datatables.net/examples/api/row_details.html - {% if len(qiita_config.oidc) > 0 %} + {% if len(qiita_config.oidc) > 1 %} $('.modal').css({ 'top': '30%', 'margin-top': '-'+($('.modal').height() / 2)+'px', From 48ca02ab616e39d1fd9941ca110dca487ded14f4 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 12:58:19 +0100 Subject: [PATCH 06/61] error handling of provider not in config file --- qiita_pet/handlers/auth_handlers.py | 71 +++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/qiita_pet/handlers/auth_handlers.py b/qiita_pet/handlers/auth_handlers.py index 7e8762a7b..f8fbcbde7 100644 --- a/qiita_pet/handlers/auth_handlers.py +++ b/qiita_pet/handlers/auth_handlers.py @@ -21,6 +21,44 @@ # login code modified from https://gist.github.com/guillaumevincent/4771570 + + + +import os +import datetime +import inspect + +INDENT = 0 + +def inc_indent(): + global INDENT + INDENT += 1 +def dec_indent(): + global INDENT + INDENT -= 1 +def stefan(msg, inc=0, fn='oidc.log'): + global INDENT + if inc < 0: + dec_indent() + fp='/homes/sjanssen/bcf_qiita/Logs/%s' % fn + current_time = datetime.datetime.now() + pid = os.getpid() + FH = open(fp, 'a') + source = inspect.stack()[1] + FH.write("%stime=%s, pid=%s, fct=%s, file=%s, msg=%s\n" % ( + "~~" * INDENT, + current_time, + pid, + source.function, + source.filename, + msg)) + FH.close() + if inc > 0: + inc_indent() + + + + class AuthCreateHandler(BaseHandler): """User Creation""" def get(self): @@ -183,14 +221,39 @@ class KeycloakMixin(OAuth2Mixin): class AuthLoginOIDCHandler(BaseHandler, KeycloakMixin): - async def get(self, login): + async def post(self, login): code = self.get_argument('code', False) + + # Qiita might be configured for multiple identity providers. We learn + # which one the user chose through different name attributes of the + # html form button + idp = None + msg = "" + if hasattr(self, 'path_args') and (len(self.path_args) > 0) and \ + (self.path_args[0] != None) and (self.path_args[0] != ""): + idp = self.path_args[0] + else: + msg = 'External Identity Provider not specified!' + if idp not in qiita_config.oidc.keys(): + msg = 'Unknown Identity Provider "%s", please check config file.' % idp + + if idp is None: + self.render("index.html", message=msg, level='warning') + + stefan("code is %s, idp = %s" % (code, idp), inc=+1) if code: # step 2: we got a code and now want to exchange it for a user # access token - print("step2") + stefan("step2") pass else: # step 1: no code from IdP yet, thus retrieve one now - print("step1") - pass + stefan("step1") + self.authorize_redirect( + redirect_uri=qiita_config.base_url + qiita_config.oidc[idp]['redirect_endpoint'], + client_id=qiita_config.oidc[idp]['client_id'], + client_secret=qiita_config.oidc[idp]['client_secret'], + response_type='code', + scope=['openid'] + ) + stefan("ende", inc=-1) From dc4bd20da13c085d8a798f52c8d2d32f59002547 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 15:46:14 +0100 Subject: [PATCH 07/61] adding pycurl package to enable tornado curl_httpclients --- .github/workflows/qiita-ci.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 6c3b06be1..79323d9ea 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -64,7 +64,7 @@ jobs: # Setting up main qiita conda environment conda config --add channels conda-forge conda deactivate - conda create --quiet --yes -n qiita python=3.9 pip libgfortran numpy nginx cython redis + conda create --quiet --yes -n qiita python=3.9 pip libgfortran numpy nginx cython redis pycurl conda env list conda activate qiita pip install -U pip diff --git a/setup.py b/setup.py index caa72a46f..ed4c2c090 100644 --- a/setup.py +++ b/setup.py @@ -109,7 +109,7 @@ 'openpyxl', 'sphinx-bootstrap-theme', 'Sphinx<3.0', 'gitpython', 'redbiom', 'pyzmq', 'sphinx_rtd_theme', 'paramiko', 'seaborn', 'matplotlib', 'scipy<=1.10.1', - 'nose', + 'nose', 'pycurl', 'flake8', 'six', 'qiita-files @ https://github.com/' 'qiita-spots/qiita-files/archive/master.zip', 'mock', 'python-jose', 'markdown2', 'iteration_utilities', From e1f3c13cf872961a383f22ea02718d8d3e2e4cf5 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 15:46:40 +0100 Subject: [PATCH 08/61] a new method to create a user, if information do not need to be entered manually, but are obtained from an external identity provider --- qiita_db/user.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/qiita_db/user.py b/qiita_db/user.py index b4beac184..59f13c78c 100644 --- a/qiita_db/user.py +++ b/qiita_db/user.py @@ -244,6 +244,45 @@ def create(cls, email, password, info=None): return cls(email) + @classmethod + def create_oidc(cls, email): + """Creates a new user with information obtained from an external + identity provider + + Parameters + ---------- + email : str + The user's email fetched from the User Info of the Identity Provider + upon successful authentication. + + Raises + ------ + IncorrectEmailError + Email string given is not a valid email + """ + if not validate_email(email): + raise IncorrectEmailError("Bad email given: %s" % email) + info = {} + # email and password are minimal needed information, password indicates + # OIDC user registration purely for admins + info['email'] = email + info['password'] = "not_necessary_due_to_OIDC" + #verify code is necessary to manually authorize users on the admin page + info['user_verify_code'] = "OIDC" + qdb.util.check_table_cols(info, cls._table) + columns = info.keys() + values = [info[col] for col in columns] + + # create user + sql = "INSERT INTO qiita.{0} ({1}) VALUES ({2})".format( + cls._table, ','.join(columns), ','.join(['%s'] * len(values))) + + qdb.sql_connection.TRN.add(sql, values) + + qdb.sql_connection.TRN.execute() + + return cls(email) + @classmethod def verify_code(cls, email, code, code_type): """Verify that a code and email match From 48f09a5a96637d25493798181c0c9efd244097bb Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 15:59:09 +0100 Subject: [PATCH 09/61] full OIDC dance implemented --- qiita_db/user.py | 7 +- qiita_pet/handlers/auth_handlers.py | 209 +++++++++++++++++++++------- 2 files changed, 159 insertions(+), 57 deletions(-) diff --git a/qiita_db/user.py b/qiita_db/user.py index 59f13c78c..df2028b66 100644 --- a/qiita_db/user.py +++ b/qiita_db/user.py @@ -252,8 +252,8 @@ def create_oidc(cls, email): Parameters ---------- email : str - The user's email fetched from the User Info of the Identity Provider - upon successful authentication. + The user's email fetched from the User Info of the Identity + Provider upon successful authentication. Raises ------ @@ -267,7 +267,8 @@ def create_oidc(cls, email): # OIDC user registration purely for admins info['email'] = email info['password'] = "not_necessary_due_to_OIDC" - #verify code is necessary to manually authorize users on the admin page + # verify code is necessary to manually authorize users on the admin + # page info['user_verify_code'] = "OIDC" qdb.util.check_table_cols(info, cls._table) columns = info.keys() diff --git a/qiita_pet/handlers/auth_handlers.py b/qiita_pet/handlers/auth_handlers.py index f8fbcbde7..75a194923 100644 --- a/qiita_pet/handlers/auth_handlers.py +++ b/qiita_pet/handlers/auth_handlers.py @@ -6,8 +6,13 @@ # The full license is in the file LICENSE, distributed with this software. # ----------------------------------------------------------------------------- -from tornado.escape import url_escape, json_encode +import urllib.parse +import os + +from tornado.escape import url_escape, json_encode, json_decode from tornado.auth import OAuth2Mixin +from tornado.curl_httpclient import CurlAsyncHTTPClient +from tornado.web import HTTPError from qiita_pet.handlers.base_handlers import BaseHandler from qiita_core.qiita_settings import qiita_config, r_client @@ -18,47 +23,10 @@ from qiita_db.user import User from qiita_db.exceptions import (QiitaDBUnknownIDError, QiitaDBDuplicateError, QiitaDBError) +from qiita_db.logger import LogEntry # login code modified from https://gist.github.com/guillaumevincent/4771570 - - - -import os -import datetime -import inspect - -INDENT = 0 - -def inc_indent(): - global INDENT - INDENT += 1 -def dec_indent(): - global INDENT - INDENT -= 1 -def stefan(msg, inc=0, fn='oidc.log'): - global INDENT - if inc < 0: - dec_indent() - fp='/homes/sjanssen/bcf_qiita/Logs/%s' % fn - current_time = datetime.datetime.now() - pid = os.getpid() - FH = open(fp, 'a') - source = inspect.stack()[1] - FH.write("%stime=%s, pid=%s, fct=%s, file=%s, msg=%s\n" % ( - "~~" * INDENT, - current_time, - pid, - source.function, - source.filename, - msg)) - FH.close() - if inc > 0: - inc_indent() - - - - class AuthCreateHandler(BaseHandler): """User Creation""" def get(self): @@ -217,43 +185,176 @@ def get(self): class KeycloakMixin(OAuth2Mixin): - pass + config = dict() + + # environment variables that define proxies + vars_proxy = [env for env in os.environ if env.lower().endswith('_proxy')] + proxies = list(sorted({os.environ[var] for var in vars_proxy})) + if len(proxies) > 1: + msg = ("The OS serving Qiita defines multiple proxy servers via " + "environment variables, but with different values. Using the " + "first one:\n %s") % '\n '.join( + ['%s=%s' % (var, os.environ[var]) for var in vars_proxy]) + LogEntry.create('Runtime', msg) + try: + config['proxy_host'] = ':'.join(proxies[0].split(':')[:-1]) + config['proxy_port'] = int(proxies[0].split(':')[-1]) + except IndexError: + LogEntry.create('Runtime', ("Your proxy configuration doesn't seem to " + "follow the host:port pattern.")) + + def get_auth_http_client(self): + return CurlAsyncHTTPClient() + + async def get_authenticated_user(self, redirect_uri: str, code: str): + http = self.get_auth_http_client() + body = urllib.parse.urlencode( + { + "redirect_uri": redirect_uri, + "code": code, + "client_id": qiita_config.oidc[self.idp]['client_id'], + "client_secret": qiita_config.oidc[self.idp]['client_secret'], + "grant_type": "authorization_code" + } + ) + response = await http.fetch( + self._OAUTH_ACCESS_TOKEN_URL, + method="POST", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + body=body, + **self.config, + ) + return json_decode(response.body) class AuthLoginOIDCHandler(BaseHandler, KeycloakMixin): + idp = None + email_support = 'qiita.help@gmail.com' + async def post(self, login): code = self.get_argument('code', False) # Qiita might be configured for multiple identity providers. We learn # which one the user chose through different name attributes of the # html form button - idp = None msg = "" + self.idp = None if hasattr(self, 'path_args') and (len(self.path_args) > 0) and \ - (self.path_args[0] != None) and (self.path_args[0] != ""): - idp = self.path_args[0] + (self.path_args[0] is not None) and (self.path_args[0] != ""): + self.idp = self.path_args[0] else: msg = 'External Identity Provider not specified!' - if idp not in qiita_config.oidc.keys(): - msg = 'Unknown Identity Provider "%s", please check config file.' % idp + if self.idp not in qiita_config.oidc.keys(): + msg = ('Unknown Identity Provider "%s", ' + 'please check config file.') % self.idp - if idp is None: + if self.idp is None: self.render("index.html", message=msg, level='warning') - stefan("code is %s, idp = %s" % (code, idp), inc=+1) + self._OAUTH_AUTHORIZE_URL = qiita_config.oidc[self.idp][ + 'authorize_url'] + self._OAUTH_ACCESS_TOKEN_URL = qiita_config.oidc[self.idp][ + 'accesstoken_url'] + self._OAUTH_USERINFO_URL = qiita_config.oidc[self.idp][ + 'userinfo_url'] + if code: # step 2: we got a code and now want to exchange it for a user # access token - stefan("step2") - pass + access = await self.get_authenticated_user( + redirect_uri='%s%s' % ( + qiita_config.base_url, + qiita_config.oidc[self.idp]['redirect_endpoint']), + code=code) + access_token = access['access_token'] + if not access_token: + raise HTTPError(400, ( + "failed to exchange code for access token with " + "identity provider '%s'") % self.idp) + + # step 3: obtain user information (email, username, ...) from IdP + http = self.get_auth_http_client() + user_info_res = await http.fetch( + self._OAUTH_USERINFO_URL, + method="GET", + headers={"Accept": "application/json", + "Authorization": "Bearer {}".format(access_token)}, + **self.config, + ) + user_info = json_decode(user_info_res.body.decode( + 'utf8', 'replace')) + + if ('email' not in user_info.keys()) or \ + (user_info['email'] is None) or (user_info['email'] == ""): + raise HTTPError(400, ( + "Email address was not provided " + "from your identity provider '%s'") % self.idp) + + username = user_info['email'] + if not User.exists(username): + self.create_new_user(username) + else: + self.not_verified(username) + self.set_secure_cookie("user", username) + # self.set_secure_cookie("token", access_token) + + self.redirect('%s/' % qiita_config.base_url) else: # step 1: no code from IdP yet, thus retrieve one now - stefan("step1") self.authorize_redirect( - redirect_uri=qiita_config.base_url + qiita_config.oidc[idp]['redirect_endpoint'], - client_id=qiita_config.oidc[idp]['client_id'], - client_secret=qiita_config.oidc[idp]['client_secret'], + redirect_uri='%s%s' % ( + qiita_config.base_url, + qiita_config.oidc[self.idp]['redirect_endpoint']), + client_id=qiita_config.oidc[self.idp]['client_id'], + client_secret=qiita_config.oidc[self.idp]['client_secret'], response_type='code', scope=['openid'] ) - stefan("ende", inc=-1) + get = post # redirect will use GET method + + @execute_as_transaction + def create_new_user(self, username): + try: + created = User.create_oidc(username) + except QiitaDBDuplicateError: + msg = "Email already registered as a user" + if created: + try: + # qiita_config.base_url doesn't have a / at the end, but the + # qiita_config.portal_dir has it at the beginning but not at + # the end. This constructs the correct URL + msg = (("

User Successfully Registered!

Your Qiita " + "account has been successfully registered using '%s', " + "which was provided by the identity provider '%s'. " + "Your account is now awaiting authorization by a Qiita" + " admin.

If you have any questions regarding " + "the authorization process, please email us at %s.

") % ( + username, + qiita_config.oidc[self.idp]['label'], + self.email_support, + self.email_support)) + + self.redirect(u"%s/?level=success&message=%s" % ( + qiita_config.portal_dir, url_escape(msg))) + except Exception: + msg = (("Unable to create account. Please contact the qiita " + "developers at %s") % ( + self.email_support, self.email_support)) + self.redirect(u"%s/?level=danger&message=%s" % ( + qiita_config.portal_dir, url_escape(msg))) + return + else: + error_msg = u"?error=" + url_escape(msg) + self.redirect(u"%s/%s" % (qiita_config.portal_dir, error_msg)) + + def not_verified(self, username): + user = User(username) + if user.level == "unverified": + msg = (("You are not yet verified by an admin. Please wait or " + "contact the qiita developers at %s") % (self.email_support, self.email_support)) + self.redirect(u"%s/?level=danger&message=%s" % ( + qiita_config.portal_dir, url_escape(msg))) + else: + return From baf40df878bc16f70488ccf2ba140db038ffee13 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 21:20:55 +0100 Subject: [PATCH 10/61] add an admin page to activate users which requested authorization through OIDC --- qiita_pet/handlers/auth_handlers.py | 80 ++++++++++++++++++++++++++++- qiita_pet/webserver.py | 11 ++-- 2 files changed, 84 insertions(+), 7 deletions(-) diff --git a/qiita_pet/handlers/auth_handlers.py b/qiita_pet/handlers/auth_handlers.py index 75a194923..e9236d7a4 100644 --- a/qiita_pet/handlers/auth_handlers.py +++ b/qiita_pet/handlers/auth_handlers.py @@ -12,9 +12,10 @@ from tornado.escape import url_escape, json_encode, json_decode from tornado.auth import OAuth2Mixin from tornado.curl_httpclient import CurlAsyncHTTPClient -from tornado.web import HTTPError +from tornado.web import HTTPError, authenticated from qiita_pet.handlers.base_handlers import BaseHandler +from qiita_pet.handlers.portal import PortalEditBase from qiita_core.qiita_settings import qiita_config, r_client from qiita_core.util import execute_as_transaction from qiita_core.exceptions import (IncorrectPasswordError, IncorrectEmailError, @@ -24,6 +25,7 @@ from qiita_db.exceptions import (QiitaDBUnknownIDError, QiitaDBDuplicateError, QiitaDBError) from qiita_db.logger import LogEntry +import qiita_db as qdb # login code modified from https://gist.github.com/guillaumevincent/4771570 @@ -298,7 +300,7 @@ async def post(self, login): self.set_secure_cookie("user", username) # self.set_secure_cookie("token", access_token) - self.redirect('%s/' % qiita_config.base_url) + self.redirect("%s/" % qiita_config.portal_dir) else: # step 1: no code from IdP yet, thus retrieve one now self.authorize_redirect( @@ -358,3 +360,77 @@ def not_verified(self, username): qiita_config.portal_dir, url_escape(msg))) else: return + + +class AdminOIDCUserAuthorization(PortalEditBase): + """User Verification for Qiita-Account Creation following OIDC Login""" + @authenticated + @execute_as_transaction + def get(self): + # render page and transfer headers to be included for the table + self.check_admin() + headers=["email","name","affiliation","address", "phone"] + self.render('admin_user_authorization.html', headers=headers, + submit_url="/admin/user_authorization/") + + def post(self): + # check if logged in user is admin and fetch all checked boxes as well + # as the action + self.check_admin() + users = map(str, self.get_arguments('selected')) + action = self.get_argument('action') + # depending on the action either autorize (add) user or delete user + # from db (remove) + for user in users: + try: + with warnings.catch_warnings(record=True) as warns: + if action == "Add": + self.authorize_user(user) + elif action == "Remove": + user_to_delete = User(user) + user_to_delete.delete(user) + else: + raise HTTPError(400, + reason="Unknown action: %s" % action) + except QiitaDBError as e: + self.write(action.upper() + " ERROR:
" + str(e)) + return + msg = '; '.join([str(w.message) for w in warns]) + self.write(action + " completed successfully
" + msg) + + @authenticated + @execute_as_transaction + def authorize_user(self, user): + # authorize user by verifying login manually using tue standard Qiita + # verify function + self.check_admin() + User.verify_code(user, User(user).info['user_verify_code'], "create") + return + + +class AdminOIDCUserAuthorizationAjax(PortalEditBase): + @authenticated + @execute_as_transaction + def get(self): + # retrieving users with an unverified level + self.check_admin() + with qdb.sql_connection.TRN: + sql = """SELECT email,name,affiliation,address,phone + FROM qiita.qiita_user + WHERE user_level_id='5'""" + qdb.sql_connection.TRN.add(sql) + users = qdb.sql_connection.TRN.execute()[1:] + result = [] + # fetching information for each user + for list in users: + for user in list: + usermail = user[0] + user_unit = {} + user_unit['email'] = User(usermail).email + user_unit['name'] = User(usermail).info['name'] + user_unit['affiliation'] = User(usermail).info['affiliation'] + user_unit['address'] = User(usermail).info['address'] + user_unit['phone'] = User(usermail).info['phone'] + result.append(user_unit) + # returning information as JSON + self.write(dumps(result)) diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py index 27e4b0d37..d94c3901b 100644 --- a/qiita_pet/webserver.py +++ b/qiita_pet/webserver.py @@ -21,7 +21,8 @@ MainHandler, NoPageHandler, IFrame) from qiita_pet.handlers.auth_handlers import ( AuthCreateHandler, AuthLoginHandler, AuthLogoutHandler, AuthVerifyHandler, - AuthLoginOIDCHandler) + AuthLoginOIDCHandler, AdminOIDCUserAuthorization, + AdminOIDCUserAuthorizationAjax) from qiita_pet.handlers.user_handlers import ( ChangeForgotPasswordHandler, ForgotPasswordHandler, UserProfileHandler, UserMessagesHander, UserJobs) @@ -242,10 +243,10 @@ def __init__(self): # through the settings file if len(qiita_config.oidc) > 0: handlers.extend([ - (r"/auth/login_OIDC/(.*)", AuthLoginOIDCHandler) - #(r"/admin/user_authorization/", AdminOIDCUserAuthorization), - #(r"/admin/user_authorizationAjax/", - # AdminOIDCUserAuthorizationAjax), + (r"/auth/login_OIDC/(.*)", AuthLoginOIDCHandler), + (r"/admin/user_authorization/", AdminOIDCUserAuthorization), + (r"/admin/user_authorizationAjax/", + AdminOIDCUserAuthorizationAjax), ]) # rest endpoints From 670a55abb318dedfc03ad64241acb6dcde91a7bc Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 21:24:10 +0100 Subject: [PATCH 11/61] flake8 --- qiita_pet/handlers/auth_handlers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/qiita_pet/handlers/auth_handlers.py b/qiita_pet/handlers/auth_handlers.py index e9236d7a4..1a18df5d4 100644 --- a/qiita_pet/handlers/auth_handlers.py +++ b/qiita_pet/handlers/auth_handlers.py @@ -8,6 +8,7 @@ import urllib.parse import os +import warnings from tornado.escape import url_escape, json_encode, json_decode from tornado.auth import OAuth2Mixin @@ -369,7 +370,7 @@ class AdminOIDCUserAuthorization(PortalEditBase): def get(self): # render page and transfer headers to be included for the table self.check_admin() - headers=["email","name","affiliation","address", "phone"] + headers = ["email", "name", "affiliation", "address", "phone"] self.render('admin_user_authorization.html', headers=headers, submit_url="/admin/user_authorization/") @@ -419,7 +420,7 @@ def get(self): FROM qiita.qiita_user WHERE user_level_id='5'""" qdb.sql_connection.TRN.add(sql) - users = qdb.sql_connection.TRN.execute()[1:] + users = qdb.sql_connection.TRN.execute()[1:] result = [] # fetching information for each user for list in users: @@ -433,4 +434,4 @@ def get(self): user_unit['phone'] = User(usermail).info['phone'] result.append(user_unit) # returning information as JSON - self.write(dumps(result)) + self.write(json_encode(result)) From 091ffc684509b43722bd364ebbc78fb3c91ec6b5 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 21:34:06 +0100 Subject: [PATCH 12/61] adding menu entry for user authorization --- .../templates/admin_user_authorization.html | 122 ++++++++++++++++++ qiita_pet/templates/sitebase.html | 5 +- 2 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 qiita_pet/templates/admin_user_authorization.html diff --git a/qiita_pet/templates/admin_user_authorization.html b/qiita_pet/templates/admin_user_authorization.html new file mode 100644 index 000000000..865d0a859 --- /dev/null +++ b/qiita_pet/templates/admin_user_authorization.html @@ -0,0 +1,122 @@ +{% extends sitebase.html %} +{%block head%} + + +{%end%} +{%block content%} +
+
+ + + + +
+
+
+
+ | |
+ + + + + {% for head in headers %} + + {% end %} + + + + +
Select{{head}}
+
+
+{% end %} + diff --git a/qiita_pet/templates/sitebase.html b/qiita_pet/templates/sitebase.html index 65d8e6d68..0a6795aee 100644 --- a/qiita_pet/templates/sitebase.html +++ b/qiita_pet/templates/sitebase.html @@ -394,7 +394,10 @@ {% if user_level == 'admin' %}
  • View Errors
  • View Studies awaiting approval
    • -
  • Edit study portal connections
    • + {% if len(qiita_config.oidc) == 1 %} +
  • Edit study portal connections
    • + {% end %} +
  • View Users awaiting authorization
    • {% end %}
  • Sample Validation
  • Processing Jobs
    • From 1feefc0d752c33c428f98dffa1262c3151721b12 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 21 Mar 2024 09:11:18 +0100 Subject: [PATCH 13/61] do not expose traditional qiita internal user authentication, if OIDC is configured --- qiita_pet/webserver.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py index d94c3901b..d49d19827 100644 --- a/qiita_pet/webserver.py +++ b/qiita_pet/webserver.py @@ -106,13 +106,6 @@ class Application(tornado.web.Application): def __init__(self): handlers = [ (r"/", MainHandler), - (r"/auth/login/", AuthLoginHandler), - (r"/auth/logout/", AuthLogoutHandler), - (r"/auth/create/", AuthCreateHandler), - (r"/auth/verify/(.*)", AuthVerifyHandler), - (r"/auth/forgot/", ForgotPasswordHandler), - (r"/auth/reset/(.*)", ChangeForgotPasswordHandler), - (r"/profile/", UserProfileHandler), (r"/user/messages/", UserMessagesHander), (r"/user/jobs/", UserJobs), (r"/static/(.*)", tornado.web.StaticFileHandler, @@ -248,6 +241,17 @@ def __init__(self): (r"/admin/user_authorizationAjax/", AdminOIDCUserAuthorizationAjax), ]) + else: + # Qiita's traditional, internal user authentication + handlers.extend([ + (r"/auth/login/", AuthLoginHandler), + (r"/auth/logout/", AuthLogoutHandler), + (r"/auth/create/", AuthCreateHandler), + (r"/auth/verify/(.*)", AuthVerifyHandler), + (r"/auth/forgot/", ForgotPasswordHandler), + (r"/auth/reset/(.*)", ChangeForgotPasswordHandler), + (r"/profile/", UserProfileHandler) + ]) # rest endpoints handlers.extend(REST_ENDPOINTS) From 29ce7dd4213e73dd82770efb64fe0cfa7525d3ad Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 21 Mar 2024 11:14:19 +0100 Subject: [PATCH 14/61] use Qiita typical modal for OIDC login --- qiita_pet/templates/sitebase.html | 49 +++++++++++++------------------ 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/qiita_pet/templates/sitebase.html b/qiita_pet/templates/sitebase.html index 0a6795aee..9ace95f1d 100644 --- a/qiita_pet/templates/sitebase.html +++ b/qiita_pet/templates/sitebase.html @@ -160,18 +160,6 @@ // Based on http://codepen.io/willvincent/pen/LbeKKW // and https://datatables.net/examples/api/row_details.html - {% if len(qiita_config.oidc) > 1 %} - $('.modal').css({ - 'top': '30%', - 'margin-top': '-'+($('.modal').height() / 2)+'px', - 'margin-left': '+'+($('.modal').width() / 3)+'px'}); - $('#qiita_signin_modal').click(function(e){ - e.preventDefault(); - $('.qiita_auth_signin_options').modal('hide'); - window.location.href = '{% raw qiita_config.portal_dir %}'; - }); - {% end %} - Vue.component('data-table-processing-jobs', { template: '
    ', props: ['jobs'], @@ -468,22 +456,7 @@ {% elif len(qiita_config.oidc) > 1 %}
      - - + Sign In
    {% else %} @@ -666,5 +639,25 @@
    succesful jobs are not shown
    + + + From 2ca5bb870b5d810de50582e36eef26b1acc217b5 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 21 Mar 2024 11:16:39 +0100 Subject: [PATCH 15/61] wrong menu entrie affected --- qiita_pet/templates/sitebase.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qiita_pet/templates/sitebase.html b/qiita_pet/templates/sitebase.html index 9ace95f1d..12ceadefc 100644 --- a/qiita_pet/templates/sitebase.html +++ b/qiita_pet/templates/sitebase.html @@ -382,10 +382,10 @@ {% if user_level == 'admin' %}
  • View Errors
  • View Studies awaiting approval
    • - {% if len(qiita_config.oidc) == 1 %} -
  • Edit study portal connections
    • +
  • Edit study portal connections
    • + {% if len(qiita_config.oidc) > 0 %} +
  • View Users awaiting authorization
    • {% end %} -
  • View Users awaiting authorization
    • {% end %}
  • Sample Validation
  • Processing Jobs
    • From 1b787cb6e2b304b993da9e02e70821fc02824703 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 21 Mar 2024 11:17:39 +0100 Subject: [PATCH 16/61] always allow logout --- qiita_pet/webserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py index d49d19827..c426b43bf 100644 --- a/qiita_pet/webserver.py +++ b/qiita_pet/webserver.py @@ -106,6 +106,7 @@ class Application(tornado.web.Application): def __init__(self): handlers = [ (r"/", MainHandler), + (r"/auth/logout/", AuthLogoutHandler), (r"/user/messages/", UserMessagesHander), (r"/user/jobs/", UserJobs), (r"/static/(.*)", tornado.web.StaticFileHandler, @@ -245,7 +246,6 @@ def __init__(self): # Qiita's traditional, internal user authentication handlers.extend([ (r"/auth/login/", AuthLoginHandler), - (r"/auth/logout/", AuthLogoutHandler), (r"/auth/create/", AuthCreateHandler), (r"/auth/verify/(.*)", AuthVerifyHandler), (r"/auth/forgot/", ForgotPasswordHandler), From 88319b20cc4d4241f02ef855f7be6a2ffc3ef572 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 21 Mar 2024 13:08:27 +0100 Subject: [PATCH 17/61] improved error handling --- qiita_pet/handlers/auth_handlers.py | 88 ++++++++++++++++++----------- 1 file changed, 55 insertions(+), 33 deletions(-) diff --git a/qiita_pet/handlers/auth_handlers.py b/qiita_pet/handlers/auth_handlers.py index 1a18df5d4..f83126088 100644 --- a/qiita_pet/handlers/auth_handlers.py +++ b/qiita_pet/handlers/auth_handlers.py @@ -14,6 +14,7 @@ from tornado.auth import OAuth2Mixin from tornado.curl_httpclient import CurlAsyncHTTPClient from tornado.web import HTTPError, authenticated +from tornado.httpclient import HTTPClientError from qiita_pet.handlers.base_handlers import BaseHandler from qiita_pet.handlers.portal import PortalEditBase @@ -189,6 +190,7 @@ def get(self): class KeycloakMixin(OAuth2Mixin): config = dict() + email_support = 'qiita.help@gmail.com' # environment variables that define proxies vars_proxy = [env for env in os.environ if env.lower().endswith('_proxy')] @@ -220,14 +222,22 @@ async def get_authenticated_user(self, redirect_uri: str, code: str): "grant_type": "authorization_code" } ) - response = await http.fetch( - self._OAUTH_ACCESS_TOKEN_URL, - method="POST", - headers={"Content-Type": "application/x-www-form-urlencoded"}, - body=body, - **self.config, - ) - return json_decode(response.body) + try: + response = await http.fetch( + self._OAUTH_ACCESS_TOKEN_URL, + method="POST", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + body=body, + **self.config) + return json_decode(response.body) + except HTTPClientError as e: + msg = ("The external identity provider '%s' returns an '%s' error" + ", when sending a request against '%s'. Thus, we cannot log" + "you in. Please contact the Qiita support team at " + "%s") % ( + self.idp, str(e), self._OAUTH_ACCESS_TOKEN_URL, + self.email_support, self.email_support) + self.render("index.html", message=msg, level='danger') class AuthLoginOIDCHandler(BaseHandler, KeycloakMixin): @@ -250,6 +260,7 @@ async def post(self, login): if self.idp not in qiita_config.oidc.keys(): msg = ('Unknown Identity Provider "%s", ' 'please check config file.') % self.idp + self.idp = None if self.idp is None: self.render("index.html", message=msg, level='warning') @@ -277,31 +288,42 @@ async def post(self, login): # step 3: obtain user information (email, username, ...) from IdP http = self.get_auth_http_client() - user_info_res = await http.fetch( - self._OAUTH_USERINFO_URL, - method="GET", - headers={"Accept": "application/json", - "Authorization": "Bearer {}".format(access_token)}, - **self.config, - ) - user_info = json_decode(user_info_res.body.decode( - 'utf8', 'replace')) - - if ('email' not in user_info.keys()) or \ - (user_info['email'] is None) or (user_info['email'] == ""): - raise HTTPError(400, ( - "Email address was not provided " - "from your identity provider '%s'") % self.idp) - - username = user_info['email'] - if not User.exists(username): - self.create_new_user(username) - else: - self.not_verified(username) - self.set_secure_cookie("user", username) - # self.set_secure_cookie("token", access_token) - - self.redirect("%s/" % qiita_config.portal_dir) + try: + user_info_res = await http.fetch( + self._OAUTH_USERINFO_URL, + method="GET", + headers={ + "Accept": "application/json", + "Authorization": "Bearer {}".format(access_token)}, + **self.config, + ) + user_info = json_decode(user_info_res.body.decode( + 'utf8', 'replace')) + + if ('email' not in user_info.keys()) or \ + (user_info['email'] is None) or (user_info['email'] == ""): + raise HTTPError(400, ( + "Email address was not provided " + "from your identity provider '%s'") % self.idp) + + username = user_info['email'] + if not User.exists(username): + self.create_new_user(username) + else: + self.not_verified(username) + self.set_secure_cookie("user", username) + # self.set_secure_cookie("token", access_token) + + self.redirect("%s/" % qiita_config.portal_dir) + except HTTPClientError as e: + msg = ( + "The external identity provider '%s' returns an '%s' error" + ", when sending a request against '%s'. Thus, we cannot " + "log you in. Please contact the Qiita support team at " + "%s") % ( + self.idp, str(e), self._OAUTH_USERINFO_URL, + self.email_support, self.email_support) + self.render("index.html", message=msg, level='danger') else: # step 1: no code from IdP yet, thus retrieve one now self.authorize_redirect( From b1e1b6be471f35d305aafd8a2d150877c6b5327d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 22 Mar 2024 10:09:26 +0100 Subject: [PATCH 18/61] revert: let user change their profile, but not password - if provided through OIDC --- qiita_pet/templates/user_profile.html | 2 ++ qiita_pet/webserver.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/qiita_pet/templates/user_profile.html b/qiita_pet/templates/user_profile.html index 11eb1a53f..e8e582c5a 100644 --- a/qiita_pet/templates/user_profile.html +++ b/qiita_pet/templates/user_profile.html @@ -23,6 +23,7 @@

    User Information

    + {% if len(qiita_config.oidc) == 0 %}

    Change Password

    @@ -43,5 +44,6 @@

    Change Password

    + {% end %} {%end%} diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py index c426b43bf..374024b01 100644 --- a/qiita_pet/webserver.py +++ b/qiita_pet/webserver.py @@ -107,6 +107,7 @@ def __init__(self): handlers = [ (r"/", MainHandler), (r"/auth/logout/", AuthLogoutHandler), + (r"/profile/", UserProfileHandler), (r"/user/messages/", UserMessagesHander), (r"/user/jobs/", UserJobs), (r"/static/(.*)", tornado.web.StaticFileHandler, @@ -249,8 +250,7 @@ def __init__(self): (r"/auth/create/", AuthCreateHandler), (r"/auth/verify/(.*)", AuthVerifyHandler), (r"/auth/forgot/", ForgotPasswordHandler), - (r"/auth/reset/(.*)", ChangeForgotPasswordHandler), - (r"/profile/", UserProfileHandler) + (r"/auth/reset/(.*)", ChangeForgotPasswordHandler) ]) # rest endpoints From a7d3b847223947f5922d2b0f2c6e37c2257ac579 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 22 Mar 2024 10:10:06 +0100 Subject: [PATCH 19/61] speaking button names + move into correct div to always get displayed --- .../templates/admin_user_authorization.html | 40 +++++++++---------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/qiita_pet/templates/admin_user_authorization.html b/qiita_pet/templates/admin_user_authorization.html index 865d0a859..5bc4eea6c 100644 --- a/qiita_pet/templates/admin_user_authorization.html +++ b/qiita_pet/templates/admin_user_authorization.html @@ -13,7 +13,7 @@ {%end%} {%block content%} -
    -
    -
    - - -
    -
    -
    | |
    - - - - {% for head in headers %} - - {% end %} - - - - + + + + {% for head in headers %} + + {% end %} + + + +
    Select{{head}}
    Select{{head}}
    +
    + + + +
    {% end %} - From 125835afca4c79af02d58211939bfed21b167837 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 22 Mar 2024 10:11:31 +0100 Subject: [PATCH 20/61] use email from config + loop user_info from OIDC to fill DB --- qiita_pet/handlers/auth_handlers.py | 39 +++++++++++++---------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/qiita_pet/handlers/auth_handlers.py b/qiita_pet/handlers/auth_handlers.py index 74bb9445c..a5dc1eb67 100644 --- a/qiita_pet/handlers/auth_handlers.py +++ b/qiita_pet/handlers/auth_handlers.py @@ -192,7 +192,6 @@ def get(self): class KeycloakMixin(OAuth2Mixin): config = dict() - email_support = 'qiita.help@gmail.com' # environment variables that define proxies vars_proxy = [env for env in os.environ if env.lower().endswith('_proxy')] @@ -235,16 +234,15 @@ async def get_authenticated_user(self, redirect_uri: str, code: str): except HTTPClientError as e: msg = ("The external identity provider '%s' returns an '%s' error" ", when sending a request against '%s'. Thus, we cannot log" - "you in. Please contact the Qiita support team at " + " you in. Please contact the Qiita support team at " "%s") % ( self.idp, str(e), self._OAUTH_ACCESS_TOKEN_URL, - self.email_support, self.email_support) + qiita_config.help_email, qiita_config.help_email) self.render("index.html", message=msg, level='danger') class AuthLoginOIDCHandler(BaseHandler, KeycloakMixin): idp = None - email_support = 'qiita.help@gmail.com' async def post(self, login): code = self.get_argument('code', False) @@ -297,8 +295,7 @@ async def post(self, login): headers={ "Accept": "application/json", "Authorization": "Bearer {}".format(access_token)}, - **self.config, - ) + **self.config) user_info = json_decode(user_info_res.body.decode( 'utf8', 'replace')) @@ -310,13 +307,11 @@ async def post(self, login): username = user_info['email'] if not User.exists(username): - self.create_new_user(username) + self.create_new_user(username, user_info, self.idp) else: - self.not_verified(username) - self.set_secure_cookie("user", username) + self.check_verified(username) # self.set_secure_cookie("token", access_token) - self.redirect("%s/" % qiita_config.portal_dir) except HTTPClientError as e: msg = ( "The external identity provider '%s' returns an '%s' error" @@ -324,7 +319,7 @@ async def post(self, login): "log you in. Please contact the Qiita support team at " "%s") % ( self.idp, str(e), self._OAUTH_USERINFO_URL, - self.email_support, self.email_support) + qiita_config.help_email, qiita_config.help_email) self.render("index.html", message=msg, level='danger') else: # step 1: no code from IdP yet, thus retrieve one now @@ -340,9 +335,9 @@ async def post(self, login): get = post # redirect will use GET method @execute_as_transaction - def create_new_user(self, username): + def create_new_user(self, username, user_info, idp): try: - created = User.create_oidc(username) + created = User.create_oidc(username, user_info, idp) except QiitaDBDuplicateError: msg = "Email already registered as a user" if created: @@ -359,15 +354,15 @@ def create_new_user(self, username): "href=\"mailto:%s\">%s.

    ") % ( username, qiita_config.oidc[self.idp]['label'], - self.email_support, - self.email_support)) + qiita_config.help_email, + qiita_config.help_email)) self.redirect(u"%s/?level=success&message=%s" % ( qiita_config.portal_dir, url_escape(msg))) except Exception: msg = (("Unable to create account. Please contact the qiita " "developers at %s") % ( - self.email_support, self.email_support)) + qiita_config.help_email, qiita_config.help_email)) self.redirect(u"%s/?level=danger&message=%s" % ( qiita_config.portal_dir, url_escape(msg))) return @@ -375,16 +370,18 @@ def create_new_user(self, username): error_msg = u"?error=" + url_escape(msg) self.redirect(u"%s/%s" % (qiita_config.portal_dir, error_msg)) - def not_verified(self, username): + def check_verified(self, username): user = User(username) if user.level == "unverified": msg = (("You are not yet verified by an admin. Please wait or " "contact the qiita developers at %s") % (self.email_support, self.email_support)) + "'>%s") % (qiita_config.help_email, + qiita_config.help_email)) self.redirect(u"%s/?level=danger&message=%s" % ( qiita_config.portal_dir, url_escape(msg))) else: - return + self.set_secure_cookie("user", username) + self.redirect("%s/" % qiita_config.portal_dir) class AdminOIDCUserAuthorization(PortalEditBase): @@ -409,9 +406,9 @@ def post(self): for user in users: try: with warnings.catch_warnings(record=True) as warns: - if action == "Add": + if action == "Authorize_Users": self.authorize_user(user) - elif action == "Remove": + elif action == "Remove_Users": user_to_delete = User(user) user_to_delete.delete(user) else: From 5f280923939fdd15afd503b124f2d3bed660132f Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 22 Mar 2024 10:12:11 +0100 Subject: [PATCH 21/61] use OIDC info to prefil user information --- qiita_db/user.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/qiita_db/user.py b/qiita_db/user.py index df2028b66..3dd5090fb 100644 --- a/qiita_db/user.py +++ b/qiita_db/user.py @@ -245,7 +245,7 @@ def create(cls, email, password, info=None): return cls(email) @classmethod - def create_oidc(cls, email): + def create_oidc(cls, email, user_info, idp): """Creates a new user with information obtained from an external identity provider @@ -254,6 +254,10 @@ def create_oidc(cls, email): email : str The user's email fetched from the User Info of the Identity Provider upon successful authentication. + user_info : dict + User information provided by the external identity provider + idp : str + The label of the external identity provider as set in config file Raises ------ @@ -269,7 +273,11 @@ def create_oidc(cls, email): info['password'] = "not_necessary_due_to_OIDC" # verify code is necessary to manually authorize users on the admin # page - info['user_verify_code'] = "OIDC" + info['user_verify_code'] = idp + # check if we got useful information from the IdP + if 'name' in user_info.keys(): + info['name'] = user_info['name'] + qdb.util.check_table_cols(info, cls._table) columns = info.keys() values = [info[col] for col in columns] From 19b4d7b278cc71cbb95ea721dc5e85be28963e11 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 4 Apr 2024 15:07:07 +0200 Subject: [PATCH 22/61] drop admin user authorization --- qiita_pet/handlers/auth_handlers.py | 160 ++++-------------- .../templates/admin_user_authorization.html | 118 ------------- qiita_pet/templates/sitebase.html | 3 - qiita_pet/webserver.py | 8 +- 4 files changed, 39 insertions(+), 250 deletions(-) delete mode 100644 qiita_pet/templates/admin_user_authorization.html diff --git a/qiita_pet/handlers/auth_handlers.py b/qiita_pet/handlers/auth_handlers.py index a5dc1eb67..cd42d6763 100644 --- a/qiita_pet/handlers/auth_handlers.py +++ b/qiita_pet/handlers/auth_handlers.py @@ -202,12 +202,15 @@ class KeycloakMixin(OAuth2Mixin): "first one:\n %s") % '\n '.join( ['%s=%s' % (var, os.environ[var]) for var in vars_proxy]) LogEntry.create('Runtime', msg) - try: - config['proxy_host'] = ':'.join(proxies[0].split(':')[:-1]) - config['proxy_port'] = int(proxies[0].split(':')[-1]) - except IndexError: - LogEntry.create('Runtime', ("Your proxy configuration doesn't seem to " - "follow the host:port pattern.")) + elif len(proxies) == 1: + try: + config['proxy_host'] = ':'.join(proxies[0].split(':')[:-1]) + config['proxy_port'] = int(proxies[0].split(':')[-1]) + except IndexError: + LogEntry.create( + 'Runtime', + ("Your proxy configuration doesn't seem to " + "follow the host:port pattern.")) def get_auth_http_client(self): return CurlAsyncHTTPClient() @@ -309,8 +312,8 @@ async def post(self, login): if not User.exists(username): self.create_new_user(username, user_info, self.idp) else: - self.check_verified(username) - # self.set_secure_cookie("token", access_token) + self.set_secure_cookie("user", username) + self.redirect("%s/" % qiita_config.portal_dir) except HTTPClientError as e: msg = ( @@ -336,123 +339,34 @@ async def post(self, login): @execute_as_transaction def create_new_user(self, username, user_info, idp): + msg, msg_level = None, None # 'danger', 'success', 'info', 'warning' try: + # create user stub created = User.create_oidc(username, user_info, idp) - except QiitaDBDuplicateError: - msg = "Email already registered as a user" - if created: - try: - # qiita_config.base_url doesn't have a / at the end, but the - # qiita_config.portal_dir has it at the beginning but not at - # the end. This constructs the correct URL - msg = (("

    User Successfully Registered!

    Your Qiita " - "account has been successfully registered using '%s', " - "which was provided by the identity provider '%s'. " - "Your account is now awaiting authorization by a Qiita" - " admin.

    If you have any questions regarding " - "the authorization process, please email us at %s.

    ") % ( - username, - qiita_config.oidc[self.idp]['label'], - qiita_config.help_email, - qiita_config.help_email)) - - self.redirect(u"%s/?level=success&message=%s" % ( - qiita_config.portal_dir, url_escape(msg))) - except Exception: - msg = (("Unable to create account. Please contact the qiita " - "developers at %s") % ( - qiita_config.help_email, qiita_config.help_email)) - self.redirect(u"%s/?level=danger&message=%s" % ( - qiita_config.portal_dir, url_escape(msg))) - return - else: - error_msg = u"?error=" + url_escape(msg) - self.redirect(u"%s/%s" % (qiita_config.portal_dir, error_msg)) - - def check_verified(self, username): - user = User(username) - if user.level == "unverified": - msg = (("You are not yet verified by an admin. Please wait or " - "contact the qiita developers at %s") % (qiita_config.help_email, - qiita_config.help_email)) - self.redirect(u"%s/?level=danger&message=%s" % ( - qiita_config.portal_dir, url_escape(msg))) - else: - self.set_secure_cookie("user", username) - self.redirect("%s/" % qiita_config.portal_dir) - - -class AdminOIDCUserAuthorization(PortalEditBase): - """User Verification for Qiita-Account Creation following OIDC Login""" - @authenticated - @execute_as_transaction - def get(self): - # render page and transfer headers to be included for the table - self.check_admin() - headers = ["email", "name", "affiliation", "address", "phone"] - self.render('admin_user_authorization.html', headers=headers, - submit_url="/admin/user_authorization/") - - def post(self): - # check if logged in user is admin and fetch all checked boxes as well - # as the action - self.check_admin() - users = map(str, self.get_arguments('selected')) - action = self.get_argument('action') - # depending on the action either autorize (add) user or delete user - # from db (remove) - for user in users: - try: - with warnings.catch_warnings(record=True) as warns: - if action == "Authorize_Users": - self.authorize_user(user) - elif action == "Remove_Users": - user_to_delete = User(user) - user_to_delete.delete(user) - else: - raise HTTPError(400, - reason="Unknown action: %s" % action) - except QiitaDBError as e: - self.write(action.upper() + " ERROR:
    " + str(e)) - return - msg = '; '.join([str(w.message) for w in warns]) - self.write(action + " completed successfully
    " + msg) + if created: + msg, msg_level = (( + "

    User Successfully Registered!

    Your user '%s'," + " provided through '%s', has been successfully registered" + " and activated. Welcome to Qiita!

    " + "

    Please direct any upcoming questions to " + "%s

    ") % ( + username, qiita_config.oidc[idp]['label'], + qiita_config.help_email, + qiita_config.help_email)), 'success' + else: + msg, msg_level = ( + ("Unable to create account. Please contact the qiita " + "developers at %s") % ( + qiita_config.help_email, + qiita_config.help_email)), 'danger' - @authenticated - @execute_as_transaction - def authorize_user(self, user): - # authorize user by verifying login manually using tue standard Qiita - # verify function - self.check_admin() - User.verify_code(user, User(user).info['user_verify_code'], "create") - return + # activate user + User.verify_code( + username, User(username).info['user_verify_code'], "create") + self.set_secure_cookie("user", username) + except QiitaDBDuplicateError: + msg, msg_level = "Email already registered as a user", 'info' -class AdminOIDCUserAuthorizationAjax(PortalEditBase): - @authenticated - @execute_as_transaction - def get(self): - # retrieving users with an unverified level - self.check_admin() - with qdb.sql_connection.TRN: - sql = """SELECT email,name,affiliation,address,phone - FROM qiita.qiita_user - WHERE user_level_id='5'""" - qdb.sql_connection.TRN.add(sql) - users = qdb.sql_connection.TRN.execute()[1:] - result = [] - # fetching information for each user - for list in users: - for user in list: - usermail = user[0] - user_unit = {} - user_unit['email'] = User(usermail).email - user_unit['name'] = User(usermail).info['name'] - user_unit['affiliation'] = User(usermail).info['affiliation'] - user_unit['address'] = User(usermail).info['address'] - user_unit['phone'] = User(usermail).info['phone'] - result.append(user_unit) - # returning information as JSON - self.write(json_encode(result)) + self.redirect(u"%s/?level=%s&message=%s" % ( + qiita_config.portal_dir, msg_level, url_escape(msg))) diff --git a/qiita_pet/templates/admin_user_authorization.html b/qiita_pet/templates/admin_user_authorization.html deleted file mode 100644 index 5bc4eea6c..000000000 --- a/qiita_pet/templates/admin_user_authorization.html +++ /dev/null @@ -1,118 +0,0 @@ -{% extends sitebase.html %} -{%block head%} - - -{%end%} -{%block content%} -
    -
    - | |
    - - - - - {% for head in headers %} - - {% end %} - - - - -
    Select{{head}}
    -
    - - - -
    -
    -
    -{% end %} diff --git a/qiita_pet/templates/sitebase.html b/qiita_pet/templates/sitebase.html index 1ded06fe6..2b102594e 100644 --- a/qiita_pet/templates/sitebase.html +++ b/qiita_pet/templates/sitebase.html @@ -383,9 +383,6 @@
  • View Errors
  • View Studies awaiting approval
  • Edit study portal connections
    • - {% if len(qiita_config.oidc) > 0 %} -
  • View Users awaiting authorization
    • - {% end %} {% end %}
  • Sample Validation
  • Processing Jobs
    • diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py index 374024b01..ba309f590 100644 --- a/qiita_pet/webserver.py +++ b/qiita_pet/webserver.py @@ -21,8 +21,7 @@ MainHandler, NoPageHandler, IFrame) from qiita_pet.handlers.auth_handlers import ( AuthCreateHandler, AuthLoginHandler, AuthLogoutHandler, AuthVerifyHandler, - AuthLoginOIDCHandler, AdminOIDCUserAuthorization, - AdminOIDCUserAuthorizationAjax) + AuthLoginOIDCHandler) from qiita_pet.handlers.user_handlers import ( ChangeForgotPasswordHandler, ForgotPasswordHandler, UserProfileHandler, UserMessagesHander, UserJobs) @@ -238,10 +237,7 @@ def __init__(self): # through the settings file if len(qiita_config.oidc) > 0: handlers.extend([ - (r"/auth/login_OIDC/(.*)", AuthLoginOIDCHandler), - (r"/admin/user_authorization/", AdminOIDCUserAuthorization), - (r"/admin/user_authorizationAjax/", - AdminOIDCUserAuthorizationAjax), + (r"/auth/login_OIDC/(.*)", AuthLoginOIDCHandler) ]) else: # Qiita's traditional, internal user authentication From c9d413af8f74e836be55ecab4b54dd501b97bf38 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 5 Jun 2024 16:52:35 +0200 Subject: [PATCH 23/61] using the well-known json dict instead of manually providing multiple API endpoints through the config file --- qiita_core/configuration_manager.py | 32 +++++++++------- qiita_core/support_files/config_test.cfg | 31 +++++++++------- .../tests/test_configuration_manager.py | 37 +++++++++++++------ qiita_pet/handlers/auth_handlers.py | 17 +++++---- 4 files changed, 70 insertions(+), 47 deletions(-) diff --git a/qiita_core/configuration_manager.py b/qiita_core/configuration_manager.py index 52d018dc5..3a5c726a6 100644 --- a/qiita_core/configuration_manager.py +++ b/qiita_core/configuration_manager.py @@ -137,16 +137,18 @@ class ConfigurationManager(object): redirect_endpoint : str The internal Qiita endpoint the IdP shall redirect the user after logging in - authorize_url : str - The URL of the IdP to obtain a code (step 1) - accesstoken_url : str - The URL of the IdP to exchange the code from step 1 for an access - token (step 2) - userinfo_url : str - The URL of the IdP to obtain information about the user, like email, - username, ... + wellknown_uri : str + The URL of the well-known json document, specifying how API end points + like 'authorize', 'token' or 'userinfo' are defined. See e.g. + https://swagger.io/docs/specification/authentication/ + openid-connect-discovery/ label : str A speaking label for the Identity Provider + scope : str + The scope, i.e. fields about a user, which Qiita requests from the + Identity Provider, e.g. "profile email eduperson_orcid". + Will be automatically extended by the scope "openid", to enable the + "authorize_code" OIDC flow. Raises ------ @@ -436,14 +438,16 @@ def _get_oidc(self, config): if provider['redirect_endpoint'].endswith('/'): provider['redirect_endpoint'] = provider[ 'redirect_endpoint'][:-1] - provider['authorize_url'] = config.get( - section_name, 'AUTHORIZE_URL') - provider['accesstoken_url'] = config.get( - section_name, 'ACCESS_TOKEN_URL') - provider['userinfo_url'] = config.get( - section_name, 'USERINFO_URL') + provider['wellknown_uri'] = config.get( + section_name, 'WELLKNOWN_URI') provider['label'] = config.get(section_name, 'LABEL') if not provider['label']: # fallback, if no label is provided provider['label'] = section_name[len(PREFIX):] self.oidc[section_name[len(PREFIX):]] = provider + provider['scope'] = config.get( + section_name, 'SCOPE', fallback=None) + if not provider['scope']: + provider['scope'] = 'openid' + if 'openid' not in provider['scope']: + provider['scope'] = 'openid %s' % provider['scope'] diff --git a/qiita_core/support_files/config_test.cfg b/qiita_core/support_files/config_test.cfg index 3e125806a..0460cdc20 100644 --- a/qiita_core/support_files/config_test.cfg +++ b/qiita_core/support_files/config_test.cfg @@ -201,8 +201,8 @@ QIIMP = https://localhost:8898/ # --------------------- External Identity Provider settings -------------------- # user authentication happens per default within Qiita, i.e. when a user logs in, # the stored password hash and email address is compared against what a user -# just provided. You might however, use an external identity provider (IdP) to -# authenticate the user like +# just provided. You might however, use an external identity provider (IdP) to +# authenticate the user like # google: https://developers.google.com/identity/protocols/oauth2 or # github: https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/authorizing-oauth-apps or # self hosted keycloak: https://www.keycloak.org/ @@ -235,23 +235,26 @@ QIIMP = https://localhost:8898/ # ## client secret to verify Qiita as the correct client. Not all IdPs require ## a client secret! -#CLIENT_SECRET = 5M6zKl8SKrlnRP4tPgtrgZpCpcYCj7uK +#CLIENT_SECRET = verySecretString # -## redirect URL (end point in your Qiita instance), to which the IdP redirects -## after user types in his/her credentials. If you don't want to change code in +## redirect URL (end point in your Qiita instance), to which the IdP redirects +## after user types in his/her credentials. If you don't want to change code in ## qiita_pet/webserver.py the URL must follow the pattern: -## base_URL/auth/login_OIDC/foo where foo is the name of this config section +## base_URL/auth/login_OIDC/foo where foo is the name of this config section ## without the oidc_ prefix! #REDIRECT_ENDPOINT = /auth/login_OIDC/localkeycloak # -## URL for step 1: obtain code -#AUTHORIZE_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/auth -# -## URL for step 2: obtain user token -#ACCESS_TOKEN_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/token -# -## URL for step 3: obtain user infos -#USERINFO_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/userinfo +## The URL of the well-known json document, specifying how API end points +## like 'authorize', 'token' or 'userinfo' are defined. See e.g. +## https://swagger.io/docs/specification/authentication/ +## openid-connect-discovery/ +#WELLKNOWN_URI = https://keycloak.sso.gwdg.de/.well-known/openid-configuration # ## a speaking label for the Identity Provider. Section name is used if empty. #LABEL = GWDG Academic Cloud +# +## The scope, i.e. fields about a user, which Qiita requests from the +## Identity Provider, e.g. "profile email eduperson_orcid". +## Will be automatically extended by the scope "openid", to enable the +## "authorize_code" OIDC flow. +#SCOPE = openid diff --git a/qiita_core/tests/test_configuration_manager.py b/qiita_core/tests/test_configuration_manager.py index b1092def3..ee3c49d83 100644 --- a/qiita_core/tests/test_configuration_manager.py +++ b/qiita_core/tests/test_configuration_manager.py @@ -311,9 +311,7 @@ def test_get_oidc(self): obs._get_oidc(self.conf) self.assertEqual(obs.oidc['academicid']['client_id'], "foo") - self.assertTrue('gwdg.de' in obs.oidc['academicid']['authorize_url']) - self.assertTrue('gwdg.de' in obs.oidc['academicid']['accesstoken_url']) - self.assertTrue('gwdg.de' in obs.oidc['academicid']['userinfo_url']) + self.assertTrue('gwdg.de' in obs.oidc['academicid']['wellknown_uri']) self.assertEqual(obs.oidc['academicid']['label'], 'GWDG Academic Cloud') @@ -322,6 +320,18 @@ def test_get_oidc(self): obs._get_oidc(self.conf) self.assertEqual(obs.oidc['academicid']['label'], 'academicid') + self.assertEqual(obs.oidc['academicid']['scope'], 'openid') + print(obs.oidc['academicid']['scope']) + # test fallback, if no scope is provided + self.conf.set(SECTION_NAME, 'SCOPE', '') + obs._get_oidc(self.conf) + self.assertEqual(obs.oidc['academicid']['scope'], 'openid') + + # test if scope will be automatically extended with 'openid' + self.conf.set(SECTION_NAME, 'SCOPE', 'email affiliation') + obs._get_oidc(self.conf) + self.assertTrue('openid' in obs.oidc['academicid']['scope'].split()) + CONF = """ # ------------------------------ Main settings -------------------------------- [main] @@ -513,7 +523,7 @@ def test_get_oidc(self): # client secret to verify Qiita as the correct client. Not all IdPs require # a client secret. -CLIENT_SECRET = 5M6zKl8SKrlnRP4tPgtrgZpCpcYCj7uK +CLIENT_SECRET = verySecretString # redirect URL (end point in your Qiita instance), to which the IdP redirects # after user types in his/her credentials. If you don't want to change code in @@ -522,17 +532,20 @@ def test_get_oidc(self): # without the oidc_ prefix! REDIRECT_ENDPOINT = /auth/login_OIDC/academicid -# URL for step 1: obtain code -AUTHORIZE_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/auth - -# URL for step 2: obtain user token -ACCESS_TOKEN_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/token - -# URL for step 3: obtain user infos -USERINFO_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/userinfo +# The URL of the well-known json document, specifying how API end points +# like 'authorize', 'token' or 'userinfo' are defined. See e.g. +# https://swagger.io/docs/specification/authentication/ +# openid-connect-discovery/ +WELLKNOWN_URI = https://keycloak.sso.gwdg.de/.well-known/openid-configuration # a speaking label for the Identity Provider. Section name is used if empty. LABEL = GWDG Academic Cloud + +# The scope, i.e. fields about a user, which Qiita requests from the +# Identity Provider, e.g. "profile email eduperson_orcid". +# Will be automatically extended by the scope "openid", to enable the +# "authorize_code" OIDC flow. +SCOPE = openid """ if __name__ == '__main__': diff --git a/qiita_pet/handlers/auth_handlers.py b/qiita_pet/handlers/auth_handlers.py index cd42d6763..ce6446277 100644 --- a/qiita_pet/handlers/auth_handlers.py +++ b/qiita_pet/handlers/auth_handlers.py @@ -9,6 +9,7 @@ import urllib.parse import os import warnings +import requests from tornado.escape import url_escape, json_encode, json_decode from tornado.auth import OAuth2Mixin @@ -268,12 +269,14 @@ async def post(self, login): if self.idp is None: self.render("index.html", message=msg, level='warning') - self._OAUTH_AUTHORIZE_URL = qiita_config.oidc[self.idp][ - 'authorize_url'] - self._OAUTH_ACCESS_TOKEN_URL = qiita_config.oidc[self.idp][ - 'accesstoken_url'] - self._OAUTH_USERINFO_URL = qiita_config.oidc[self.idp][ - 'userinfo_url'] + idp_config = requests.get(qiita_config.oidc[self.idp]['wellknown_uri'], + proxies={env: os.environ[env] + for env in os.environ + if env.lower().endswith('_proxy')}).json() + + self._OAUTH_AUTHORIZE_URL = idp_config['authorization_endpoint'] + self._OAUTH_ACCESS_TOKEN_URL = idp_config['token_endpoint'] + self._OAUTH_USERINFO_URL = idp_config['userinfo_endpoint'] if code: # step 2: we got a code and now want to exchange it for a user @@ -333,7 +336,7 @@ async def post(self, login): client_id=qiita_config.oidc[self.idp]['client_id'], client_secret=qiita_config.oidc[self.idp]['client_secret'], response_type='code', - scope=['openid'] + scope=[qiita_config.oidc[self.idp]['scope']] ) get = post # redirect will use GET method From 9a5e7ccfa0046bc26a663366e4de29e4e59a26d7 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 5 Jun 2024 17:00:47 +0200 Subject: [PATCH 24/61] flake8 --- qiita_pet/handlers/auth_handlers.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/qiita_pet/handlers/auth_handlers.py b/qiita_pet/handlers/auth_handlers.py index ce6446277..77210cd14 100644 --- a/qiita_pet/handlers/auth_handlers.py +++ b/qiita_pet/handlers/auth_handlers.py @@ -8,17 +8,15 @@ import urllib.parse import os -import warnings import requests from tornado.escape import url_escape, json_encode, json_decode from tornado.auth import OAuth2Mixin from tornado.curl_httpclient import CurlAsyncHTTPClient -from tornado.web import HTTPError, authenticated +from tornado.web import HTTPError from tornado.httpclient import HTTPClientError from qiita_pet.handlers.base_handlers import BaseHandler -from qiita_pet.handlers.portal import PortalEditBase from qiita_core.qiita_settings import qiita_config, r_client from qiita_core.util import execute_as_transaction from qiita_core.exceptions import (IncorrectPasswordError, IncorrectEmailError, @@ -28,7 +26,6 @@ from qiita_db.exceptions import (QiitaDBUnknownIDError, QiitaDBDuplicateError, QiitaDBError) from qiita_db.logger import LogEntry -import qiita_db as qdb # login code modified from https://gist.github.com/guillaumevincent/4771570 @@ -269,7 +266,8 @@ async def post(self, login): if self.idp is None: self.render("index.html", message=msg, level='warning') - idp_config = requests.get(qiita_config.oidc[self.idp]['wellknown_uri'], + idp_config = requests.get( + qiita_config.oidc[self.idp]['wellknown_uri'], proxies={env: os.environ[env] for env in os.environ if env.lower().endswith('_proxy')}).json() From b2fc2798018724f2bdeaebca963c680d69b7eeef Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 5 Jun 2024 17:02:05 +0200 Subject: [PATCH 25/61] flake8 --- qiita_core/tests/test_configuration_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qiita_core/tests/test_configuration_manager.py b/qiita_core/tests/test_configuration_manager.py index 1b1694eac..b8f671074 100644 --- a/qiita_core/tests/test_configuration_manager.py +++ b/qiita_core/tests/test_configuration_manager.py @@ -332,6 +332,7 @@ def test_get_oidc(self): obs._get_oidc(self.conf) self.assertTrue('openid' in obs.oidc['academicid']['scope'].split()) + CONF = """ # ------------------------------ Main settings -------------------------------- [main] From 5cc08967b3f68f357b629c732d3a06e32b69a88b Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 5 Jun 2024 17:37:07 +0200 Subject: [PATCH 26/61] add ability to display OIDC logos --- qiita_core/configuration_manager.py | 6 ++++++ qiita_core/support_files/config_test.cfg | 4 ++++ qiita_core/tests/test_configuration_manager.py | 12 +++++++++++- qiita_pet/templates/sitebase.html | 6 +++++- 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/qiita_core/configuration_manager.py b/qiita_core/configuration_manager.py index 3a5c726a6..c10428542 100644 --- a/qiita_core/configuration_manager.py +++ b/qiita_core/configuration_manager.py @@ -149,6 +149,10 @@ class ConfigurationManager(object): Identity Provider, e.g. "profile email eduperson_orcid". Will be automatically extended by the scope "openid", to enable the "authorize_code" OIDC flow. + logo : str + Optional. Name of a file in qiita_pet/static/img that shall be + displayed for login through Service Provider, instead of a plain + button Raises ------ @@ -451,3 +455,5 @@ def _get_oidc(self, config): provider['scope'] = 'openid' if 'openid' not in provider['scope']: provider['scope'] = 'openid %s' % provider['scope'] + provider['logo'] = config.get( + section_name, 'LOGO', fallback=None) diff --git a/qiita_core/support_files/config_test.cfg b/qiita_core/support_files/config_test.cfg index f92e18ec4..f78fd1ac3 100644 --- a/qiita_core/support_files/config_test.cfg +++ b/qiita_core/support_files/config_test.cfg @@ -257,3 +257,7 @@ STATS_MAP_CENTER_LONGITUDE = ## Will be automatically extended by the scope "openid", to enable the ## "authorize_code" OIDC flow. #SCOPE = openid +# +##Optional. Name of a file in qiita_pet/static/img that shall be +##displayed for login through Service Provider, instead of a plain button +#LOGO = oidc_lifescienceAAI.png diff --git a/qiita_core/tests/test_configuration_manager.py b/qiita_core/tests/test_configuration_manager.py index b8f671074..d6d61845e 100644 --- a/qiita_core/tests/test_configuration_manager.py +++ b/qiita_core/tests/test_configuration_manager.py @@ -321,7 +321,6 @@ def test_get_oidc(self): self.assertEqual(obs.oidc['academicid']['label'], 'academicid') self.assertEqual(obs.oidc['academicid']['scope'], 'openid') - print(obs.oidc['academicid']['scope']) # test fallback, if no scope is provided self.conf.set(SECTION_NAME, 'SCOPE', '') obs._get_oidc(self.conf) @@ -332,6 +331,13 @@ def test_get_oidc(self): obs._get_oidc(self.conf) self.assertTrue('openid' in obs.oidc['academicid']['scope'].split()) + self.assertEqual(obs.oidc['academicid']['logo'], + 'oidc_lifescienceAAI.png') + # test fallback, if no scope is provided + self.conf.set(SECTION_NAME, 'logo', '') + obs._get_oidc(self.conf) + self.assertEqual(obs.oidc['academicid']['logo'], None) + CONF = """ # ------------------------------ Main settings -------------------------------- @@ -547,6 +553,10 @@ def test_get_oidc(self): # Will be automatically extended by the scope "openid", to enable the # "authorize_code" OIDC flow. SCOPE = openid + +# Optional. Name of a file in qiita_pet/static/img that shall be +# displayed for login through Service Provider, instead of a plain button +LOGO = oidc_lifescienceAAI.png """ if __name__ == '__main__': diff --git a/qiita_pet/templates/sitebase.html b/qiita_pet/templates/sitebase.html index 653eadfcb..532445035 100644 --- a/qiita_pet/templates/sitebase.html +++ b/qiita_pet/templates/sitebase.html @@ -642,7 +642,11 @@

    Login

    Please choose your login provider!

    {% for idp in qiita_config.oidc %} - {{qiita_config.oidc[idp]['label']}} + {% if qiita_config.oidc[idp]['logo'] %} + + {% else %} + {{qiita_config.oidc[idp]['label']}} + {% end %} {% end %}
    From 949084d21aab71ed134dbcee881e660a343f8707 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 5 Jun 2024 17:38:06 +0200 Subject: [PATCH 27/61] add OIDC logo --- qiita_pet/static/img/oidc_lifescienceAAI.png | Bin 0 -> 8560 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 qiita_pet/static/img/oidc_lifescienceAAI.png diff --git a/qiita_pet/static/img/oidc_lifescienceAAI.png b/qiita_pet/static/img/oidc_lifescienceAAI.png new file mode 100644 index 0000000000000000000000000000000000000000..ab72fa293def5199720b4a00bac0d824aad6d26a GIT binary patch literal 8560 zcmbW7+RhM|V;8cI5bA*F|Ic*gJb`wN~I zCw88-_O;L1Yk%$&^;KCG1C<074h{}O?u(Qf92|Tf?A{y&33kS3=6nOYAek!2O2Og6 z4Z__$z|k#`W0G2W2OwJU1D!F_;}lM>hP zUOCD3^473ec@^7N996`5Ga5!)G^>EkWU61qQ~$ldG=QzLrsGg=q50_0F33>(wk6u` z3?EP9_s{R2@eIQeix4Fg-W&us9qd275WSK6LZ4**hCAue&CpF8Kt`J<0q^G}wo_Z6=*<}8+Ctk_J`IG=kks^daq$4`?OS}8CL&K!g^*BzPpj|{1 z3Tns)6o`T+PZjarToY3J;s}ifsds4RrzM7^{=Ca|?7H-UAFwJC-e%mrUXuPRp4c&L z!}V>6|12X!kUS&=k$s?ML<#vIl|lp}o3@_$rvKASPck05{lNR0TCU($_5mdX+r?bA z-!0v@Uz!rG3_v!FL1vFbZa>pDOFx7I8y)H-7W)JThd)xjp{C<5f8e;_Vvu6^|Lc@5 z^YGhNie%vzyLsRDZ;7bI|NI+%l+vh$L7ogzs`qC9l3z;87s^d>#IfJzv(QNVYxe{D zpI;#;`LNUf5)zAjjPxi@!053j=46}}tv?&NH5_Lbm#;7a`&8 zQ^VbK^8?(vNURyM=yWI{)2OpNBoX$P8N}TnP~_uUSM0Cr?ohAGc@g#@M`!gCsi=CG zB~mCfF?#a@2to$-RWo`(zi3sH6L>6sM~@%9|0~@^cHh(Ve%cTq@5!qxo9Levw0S0g zl~Kih(9-njtwi?r!rx%~6(|97Cg?{8wv0DkvLPlLG_MZTzG8mIk3p&q0kdbjgI5ZE zeSkIOg9PK-pCLc%kfHNad2u=K0y~k#^kxm5Z|ifNnl6N&UH1aB#3GDD@7Kqf;d=pc0lx2JghB1UFGTBs( z6ceG2&AXp4n;Y?CX9uaNKuM1pIm-ucCE=pW`s2#!ORPNCA0JjK=XBd#@AJF2w;hBZ zmT+n5=)4YHQiUMfraC~%v9(Pn%TiQYA`??#gUmNp8KFU-OWW!sz5W4%G9&hB#K>SrH_#FI-R%B8IxMeO_n3>{X@L9DA|~X!d!G zja8J%_5Qd$tWqEwPcfOI3-qQs>wJ`HJTf&i^L~!QTJe~oxL?IZC;r$px&L%n$x&~+ zdplV}%4LdoyK(EB>djwNt2#h~t_?ai*`$;~OpT}wnCwOy#bA=@S+_XZz4kr#7LGIW z&yK?NZbLT$BO}(pNxxyd>|rRbyMSBEK=oRp9u{I{W!;?QH?mj-usLi1@;Gyea{~q% zEEMkgDIKYV9$WJO1in-VMNe$%qP zz{mB+BRgKqw^m&Ws`Cy4$`-ocP0bZJA4n?3@r1J>!BF3FCtMuxOXs4EF7B^R!}B)# zisy#JOdTSdF1Xomx^!k!VR^EQ)HD%ffca;r3SW)J9>m4;ZwS#>y5?=QPgn%o2lBv3fr_Da1wajcDn`)XK!WW>Cv%HScYP0k9t zvflN=S7x|lW#_^L$-&#@-M(+-#R#m5h{MQ`BRq@DwwY#7v{zMC1wEd1<2v7O=mvaI zmZPgLjXOLfkynUb1~}M#4$at`dQ4GQ7KnR!MsCsr6t$+MbBm(h0ggf0N;y1FyRd;l}rFh8_Q-A=(sE;CTV7$fW>Jp2WvGOwf;*{2Fsn?X|F zI7Sk$^#eTLJUMz&q|+%xg?E~P@9HG)Nbn5!OoAzESZOgO>L;88%2xFCeRzLVQBiRh zc<(jwa^Ys6Tnc^$-aoJiUr$`7DbjcEUoYs_Zn-v#j#yqG5`GNaYNMsUv9DM3@gXMV ztJ)S5tF?T7@UYHEO-r*bU`;WzeB|g#>m6mtjh?aog~|joPBvGmIWinAuNFU$aJ?%a zE-q60R(xLE#N((AKvIK(K=m{g6li*~!W%fQ!>YRGdSn<&5ui4{;bK6RRH|rk_d^v3 z?Q8sHyriGWneZO8^{9?52a!yRqa#;>_kk38E6>f8jjyilPb=Z6m@wPo3(PTb`+$ zkVepwXsOW%Cmfn541?OhwJ3bF+}tsdlz~}GOGK~3hM%UK)DM9^;JaI-Sees~*Zl8N z&@b?D78k$E*Zjbe!)q_bpck^#ak^x+FwQ1j8~$_K^s(-7bk^!5`q(c$XsuT?ESi3P zb{*T9R!a%;@f4${&bv#VBC_{cqiN_eAiui;Ih5wqUj4BL$!R zr~b%CtXF{b%#&p*hY_&n%(L8vIE&ZQe?U0iIN)ADIdX@}L8bHPg6wOdcgM)&#WTsH z0SVHFxdKKm*iDj&>YbgM^|IMeAWiM5oBH0ufn!*v zn&OkFc!U*yB>ors3%15_c+z!ed7&;B^t%$ZPo%L_T<|kg_6UZHqoem5PU%d9Mzi@k z`AMY)1MMDcw_=Vft-%);4Km_&>yG?lJ5^U5;Bs{iX-%48%Cd?I)0u*M#hj0t%%$F8 z!IxK}$)aNO7k}9a6_`byv?}Xsw6<4WvApcPX075<VgD{I`g4~8jlc(a4`orsCsg$aK})FZJfSaws5;P308(A{LdK2H9Wo3;AM*w zu$9+`@?H+n0f#y-Co0}~w#S}|3fCcGE3dyqp~zNAOO2)AkE?)Pz6ZbCn(V;-dflpp zx||=5^?YIiR-mlfy2W|9T|6rESf(~Xl@m{sMgKNO(y^$YP6I}X`ukXI$`nqe*m!O3 zH-1+H;DXf?5>Gas2Fa%#o^$bfWLfTl=O;%(JHy%7RkBjgNFB*-tNl5gyQX$KgI9o7 zgB&^uPgq)7+Rp%IMpPQvpEvrJk2lqEFB=&h^G^4W;rI)o6Y4(#uTM;q?6sAZCZ|9F zH7;os{r!mSy4F5YXj#PT>h{)pG{7%nJ#G?NyC%r2M2abFO!#gP-w3mbcBLHzbhV*y z_&LYs>}#-XVL{>T$`ADDNlm2wWuxgF^83~04*I^@J1lOB#%Xl{^MvEi!D@>bLlfs4 zuNR%y;{A;>S3D%{`mT=`IG(j-~QuHeCZ$cI7DUCdM#A1`;`Xo(SPZn|81` zF*30C(m*77jD1ouXHoH;s>!pkG{R!>XVWB#+ic}lC*-vzw62;+z4pEvf zqaKY74-bajG)}UGCDRVoS=zobGE1v34dXS|o^Ab*olK`LoN!#26)FnrORI-#j9)cM z&y{>ux4k!VZ7gk7Xsp|vfSr4|z2|&1UM1nma}!XV`Py4wwU(cYcN7ER+41a@V-n#-a$2C;#o!+BID*nDhb!ne)@O62) zjktiazrXOz3>$Lw91jOCZ!aG%|8C;JCk&rK}3C?(`Cp`B@UKdB`@C}Yu3uEi!BQNMF*D)@nS0b-0JR| zFu&P=Hl$3o%HChk&MFj&&lrwu-GpjcUOOwiOi*7}^UZPO8#W#rWqH<>$P} zg2$db)w?%wuw|iYW=85ws)4IOh3}5!{(0;DVWn_Ya4Z#W1q4;1|Otgqlky znLsktriz-gx8R0~<+KMq9nq5MqK+9>wf?sm;bTwhwIqwZ)Fjgp5#9&^1pEh5!M(PZ zMkx!2=Dk-0so;;Ppv=cL4MxbSleYyckRO|3GR(eGYK?H$5dg_$}6PsFJU@ zp`8Ouh#4O;KX&u{VbGP0gTslq_!Z^s2SGvCFzn5~7XJ3*;ec~ zhcnHiX|^wt!YWO^#5mO-TZEfjk-M}_dh&ad%1I=Z*guB+aeqAPJnaEGBX)t$EZnjQ z7N&FwoP$)-(lMrPBDlBn{LrGy;5)LRwVlvMHYg|<+NLbavP?Ch+WVXr)o=$ketr^& zTt98enz8=y>qorGx7oIR{*Kef-SMRsr`odY?*05qL+2FfK}$78m*zqrjYwS6Q1#&0 z_948?#8lL^wG>OLVgwpl&5(T=VEfeMH4{C`mMF^~M)~#N`1Ga@choGs&Cw@`w7{Md zzSdr(N;1wCvXqlZLuKP}H}!?ZdzlQ!GsN4{JNm`Qk11MVplwg&Ywzk}i;A;byXx1e zQ)_+h*U($=WX{uib;{9PDVB=JbG(@=2fa$y54spa-Z`sE?xUFWr~8`@sq+DvE3T1E zQ|s83E>Q`?{fnZ_03k(r&dom_S`;W4XI8q|QQ4j%e`$0pMSL?S4J)M=+z1h}%z{&A zH^=j#{>DU$g=Yb9ruQ3?kl0b!hS3{M$ZG!3-nN-<3vIqIz9j? zY|&aeZl??~kPC#9V3$}5Z)F<%VxUhhgb^`FoR)kijkTEOz~!_!-hE?yNMMI9WomS#6p|v9(R6cVySNn*CQC z9bQLYd#-{A&fQKr&(791)YmA#uSjkmcag1T*&{dTC7m$`1!4kYf`7l0DBSG@NHw+2`#z>UbR@|bE{W^rNG{29VLBa!_^^)T6 z-=uOl+Az))cD&|H^o2a-k?r>cBUB^Z*4(1tOXtgCy+X#&(9ozad;}kl&uM9wG7S0M zh8PNbsl(!YK1iFLmsukXm395#cu4H##zEhSZTiLC7cCijo*BZS(^8y~W_XydCc>wV zG?vWR#eVB*Q;_(!+Kt-m)aB4x@I_1zbLBBOwuX4^5fAr?j{H}q!HZXN9aY$);^Qsl zj#ofnZ%!VW)$M6^;G&hqwGEzBhjGyBsHfN23UT68;?1Y={6`VxRiZOvg`JZI+4frzMlw04W|sX?GpuAqoIRV=#DtBJdS&hhS#8^nN9V zhvIGVXr~5HOf~lH&{JN9v+I!eohKoqUvY#TEk5%z4lmbjyapoi3B(5@JmoId+P&aH z>vIE6rA1%N2`Ga;#=OI&4}bAHY2rSZXK7ycfNfd|{FM?0fXKv-uH3@K5FCsQMVl|Z zgR}2vV)d=|V<$-z8kEwmH_gpkX(mV9{MLkXe6RNTCz&r@)%cT5N;c7^Ba@L%L2k}g z5mBXT9z9Agj0lt(BzMJn{pI46F@5Hl!TI!FOCtQ>4u5eHOdmRek<-mjx##Clmnf&c zfaj#vwv>cy1upC9Z{0`y4l6&_9fxgCmi*Ir?E=rw()r$Zy;P$>ZXDV)w6w4(bFqCm zJPQ+TQVmBHUVX)lPPg78<>s^}(e+n*^Q5(oD%m7Z=dC4IZ6j_t%2qHGBe=bI+E}J& z`KH^Wz3|Uu?jljF?rv?MUy8pMo@hQdG8+5i)7b(-VA9ix{`L+!joIpY2?5)Qr?m+4 zJd7w33{oA_mKIxJt1aOxd=&Do} zJkap9x=u_5pqT6@6bYsC{)rY39_L$TLX{w)3axSTQK4}<7HJQGdZeeG%Cy$`IlE!J zbn5Ibi-^7la_Gj4?0+EnisyV78H6w18nAvY{n_>hcbM!$WyM%|Y0hcOL{_a8#I@OY za^L@1n#JfKfhNQ+bJFtPbGuh3bHyUt+eZ`=iWyN;4PBanbmn|QuC@;veDkMO86>=<%NkHVdD@Vvi!w|}=@Qv(7K zz8CI3z1x=Mw4TQHF^R$Ur0CQnti`x`x7noRDJ(xcvAnps<+ZN4!67JU1mT~T850)d zze3G{zPJzqHFqpw(NRrQ{8Q{rXQuz_v3t)AW}rHrZ+_J%O@b|^gOIu9P01_VYavhf z-4-{%e&bKcb4qgZ$AE{J!11fU8=*4{e7WTTn?1cnrS~d59nKKpz&jJo0pIY{7;Q$0 zrR6JCL!i)zL^B}>ymn6bnmad9^=Z>0v*i-(A2j9V;usQ4EAmO5o_#8mn2y2{tNCMF zW~sv3D`cI-DwWZ-|CSs*qq7~AnKW&xsAqa#U{)9s;h+jqQ zIt|%LDDypSh!PQg`sc+YlwqLaij#Sw7Wy&IYCroDsU^y|@&x$t1M1&aWI>NiWe|K^GEJe7#g$Fj zMTcX;`(3mGjIPfs#=m>+MLsl=KdvH7uwV1WQ(OQFHg_De{m2zkS1pmcYNBv9M;qMaR@F%3=fZ_BfE{@2%4TA}s~ z45iK+_v%k60++`w{w?Y6^6P z7V?r;B-*)(*q6%zEv?`C?y-X*Xiez*mq|jj4?jkvo0vNfl^;B(?-G4YQaCJHt9~+Bq50KK3 zE^aT$__bvUZ~{v>INTu~gT@XgnMz|#t`Et4bYU}{Jv}-md|1EDyD(Hv-()D^=G}4| z9|9$w-(B<2_!8Vj%S=alo z_duY-&aQ*qV*OCOSY)qc0|G(Fs5<*rrhqY$M&KeyxSPWKlj}hfi%#bQa!9(VQrK7P zu0AGqOAGzhz{*e-KA}77Iz)Za$4!`0iXN|-rcRNDP2v$}GQ-Se`_1St+CBS&9zOk0 zlkr=$YevVl!@QM3ZthiyZqE1OsjGt(nk-+GEKLtNEqihl~CZ5lzdD~nvj!nDY@Cs zh^oP}*kX$k3VUbB))sZ(%fT@Fz2o6LOLF5B#V|}5hS3F~?2U?;7G&K!vzjD3z^A(= z@*GGwA5Hw3r;Ca&&w>2WY9J3SXZ%DlQJU`ImXod}$*mNka+jm!3O%wavwwyImHInM zl{!F4#*|gy%IS%w=5B^vV2~6aRNuFc9ohe@>r&y|cj-qHX?*mi9v7|zQ>2)~BZal? z?{OY=>n@+76-6u&U$Rhxj^77Pbr&!i_*^$zTTkTPI!xb+nGP_Xwq6=d>;W(sxk=*I zJYmtF5y$KO?moy@wlv>c)2|gTX~FmkjCbdb8c8FnD-6Hp}_WoaluH0XBdElvp;R$ z>{i2uON8!y!otTx$j*0Hys^aMqKC>_L{`QHQ`6^O^<{%9$Gjx}D3gxh))c`{mGP<}BD0TK3dMCg6;>da

    Vdyem( zUWzT8AJVP3P5O$oWFrB|i8Qj8PnXO7mnYggJ5_mAO-*rAA;Bj75pJ`irJTN3qXR=j z5rfY^Vq&(R(C9CBkEeDK4PR$0$6d%OG7$^Sc~q_H2hFygFfbd;G`fh(!sgpJ%6KnI z4H$5@Fo4VYrxyB4SoWTe4{uuL#miSyeMI%bjVy`C5c*BO<-IYaLo<_;oUskFA~C}yYW@fRAo?}vlBdia`JI4 zYiUIVNz8o6yeS>MU_?`syYKCR!kK8m0XPV5W`^q`A(H994)1-R8FhpacdCQ?G`Qko z`2&I_+m&QOfT6=qFS#;EiSt4tLYNXpL36O>DIu!nb-jZV`2;rtyzHxo#X9d!jPh<- zFxHyaM~g+2s6$}&SgFH?Kcj<5@WS=ybd8AHDInaB8Wd}gtr{L&l_Fc$?YH7Tbj<4} zxW1K*PL@g*{FsyQIuy74-G|#bxn_PjhJpK5JnSoqXg+HQ+#iW7&e4u z|6|lrD2V^h1QkyB*F_Ka(I%RISof1svoazXTy#O*Djm{)v^G;Dt{Ig7=C$heDA9j< zfaM?Hrp8l>jJzaednkSLZQP|54!7qhz#SA%X!Z`qVH1ae@~u9lYOkdoFsf|;mc5gD zp6x!3=$w^e{`UwXKSSr+Q<<0?xKFe(05Vzl1f&F$1zq8{rWmH5U`|jym7n*8O(vWP z()#(059`e4onJ*I!E@Z9D#KJ#F#_1{0q6hz1wd;4Rb=oFkGt7+aysbKyuW(GF#qcE z1qIdyHMsseW?jCr$(p4N{eNUUwUeEwWx3P_c_z3XnK!$Nh&aju|f*_F(uM=!55P3xbHX7a&_7@3j?IDJSVcl?_{ ziLFKdarS)pSA7`JFY1 Date: Wed, 5 Jun 2024 20:56:30 +0200 Subject: [PATCH 28/61] revert to dev branch --- qiita_core/configuration_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_core/configuration_manager.py b/qiita_core/configuration_manager.py index c10428542..53d1c16c2 100644 --- a/qiita_core/configuration_manager.py +++ b/qiita_core/configuration_manager.py @@ -416,7 +416,7 @@ def _get_portal(self, config): raise ValueError(msg % (name, val, 'larger than 180°')) def _iframe(self, config): - self.iframe_qiimp = config.get('iframe', 'QIIMP') + self.iframe_qiimp = config.get('iframe', 'QIIMP', fallback=None) def _get_oidc(self, config): """Get the configuration of the open ID connect section(s) From d96bbae7d34a98a78fcb3878cc9c72509ca0ec45 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 5 Jun 2024 21:24:46 +0200 Subject: [PATCH 29/61] fixing config manager tests --- qiita_core/tests/test_configuration_manager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qiita_core/tests/test_configuration_manager.py b/qiita_core/tests/test_configuration_manager.py index d6d61845e..c092b4ef1 100644 --- a/qiita_core/tests/test_configuration_manager.py +++ b/qiita_core/tests/test_configuration_manager.py @@ -334,7 +334,7 @@ def test_get_oidc(self): self.assertEqual(obs.oidc['academicid']['logo'], 'oidc_lifescienceAAI.png') # test fallback, if no scope is provided - self.conf.set(SECTION_NAME, 'logo', '') + self.conf.remove_option(SECTION_NAME, 'LOGO') obs._get_oidc(self.conf) self.assertEqual(obs.oidc['academicid']['logo'], None) @@ -520,7 +520,6 @@ def test_get_oidc(self): # ----------------------------- iframes settings --------------------------- [iframe] -QIIMP = https://localhost:8898/ # ------------------- External Identity Provider settings ------------------ [oidc_academicid] From e0c40027f9cdef1d365499a54902178104062ac9 Mon Sep 17 00:00:00 2001 From: sjanssen2 Date: Thu, 20 Jun 2024 17:11:21 +0200 Subject: [PATCH 30/61] add missing template --- qiita_pet/templates/admin_purge_users.html | 123 +++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 qiita_pet/templates/admin_purge_users.html diff --git a/qiita_pet/templates/admin_purge_users.html b/qiita_pet/templates/admin_purge_users.html new file mode 100644 index 000000000..98bbc05a7 --- /dev/null +++ b/qiita_pet/templates/admin_purge_users.html @@ -0,0 +1,123 @@ +{% extends sitebase.html %} +{%block head%} + + +{%end%} +{%block content%} +Listing all users that are not yet validated but registered more than 30 days ago. +

    +
    + | |
    + + + + + {% for head in headers %} + + {% end %} + + + + +
    Select{{head}}
    +
    +
    +
    +
    +
    + + +
    +
    +
    +{% end %} From c0e715ba16569a768e04508f944aef458daf59d3 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Sat, 12 Oct 2024 07:56:48 -0600 Subject: [PATCH 31/61] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d29e6491d..33fb0303a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ Deployed on October 14th, 2024 * `Woltka v0.1.7, paired-end` superseded `Woltka v0.1.6` in `qp-woltka`; [more information](https://qiita.ucsd.edu/static/doc/html/processingdata/woltka_pairedend.html). Thank you to @qiyunzhu for the benchmarks! * Other general fixes, like [#3424](https://github.com/qiita-spots/qiita/pull/3424), [#3425](https://github.com/qiita-spots/qiita/pull/3425), [#3439](https://github.com/qiita-spots/qiita/pull/3439), [#3440](https://github.com/qiita-spots/qiita/pull/3440). * General SPP improvements, like: [NuQC modified to preserve metadata in fastq files](https://github.com/biocore/mg-scripts/pull/155), [use squeue instead of sacct](https://github.com/biocore/mg-scripts/pull/152), , [job aborts if Qiita study contains sample metadata columns reserved for prep-infos](https://github.com/biocore/mg-scripts/pull/151), [metapool generates OverrideCycles value](https://github.com/biocore/metagenomics_pooling_notebook/pull/225). +* We updated the available parameters for `Filter features against reference [filter_features]`, `Non V4 16S sequence assessment [non_v4_16s]` and all the phylogenetic analytical commands so they can use `Greengenes2 2024.09`. From 7693c5efcb93e06673ec10d62fa2156dd1d99686 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 08:26:34 +0100 Subject: [PATCH 32/61] extended configuration manager with optional OIDC sections --- qiita_core/configuration_manager.py | 51 ++++++++++++++++++ qiita_core/support_files/config_test.cfg | 54 +++++++++++++++++++ .../tests/test_configuration_manager.py | 51 ++++++++++++++++++ 3 files changed, 156 insertions(+) diff --git a/qiita_core/configuration_manager.py b/qiita_core/configuration_manager.py index 02fb555c4..738a8a5ba 100644 --- a/qiita_core/configuration_manager.py +++ b/qiita_core/configuration_manager.py @@ -127,6 +127,24 @@ class ConfigurationManager(object): The email address a user should write to when asking for help sysadmin_email : str The email address, Qiita sends internal notifications to a sys admin + None (=internal user authentication) or one or several 'oidc_' sections + to use external identity providers (IdP) with following values: + client_id : str + The name you registered Qiita with at the external IdP + client_secret : str + A secret string with which Qiita identifies at the external IdP (not + all IdPs need a secret) + redirect_endpoint : str + The internal Qiita endpoint the IdP shall redirect the user after + logging in + authorize_url : str + The URL of the IdP to obtain a code (step 1) + accesstoken_url : str + The URL of the IdP to exchange the code from step 1 for an access + token (step 2) + userinfo_url : str + The URL of the IdP to obtain information about the user, like email, + username, ... Raises ------ @@ -162,6 +180,7 @@ def __init__(self): self._get_vamps(config) self._get_portal(config) self._iframe(config) + self._get_oidc(config) def _get_main(self, config): """Get the configuration of the main section""" @@ -390,3 +409,35 @@ def _get_portal(self, config): def _iframe(self, config): self.iframe_qiimp = config.get('iframe', 'QIIMP', fallback=None) + + def _get_oidc(self, config): + """Get the configuration of the open ID connect section(s) + User can provide multiple sections with naming schema oidc_foo where + foo is the name of an Identity Provider - Qiita can handle multiple + Identity Providers simultaneously. + """ + PREFIX = 'oidc_' + self.oidc = dict() + for section_name in config.sections(): + if section_name.startswith(PREFIX): + provider = dict() + provider['client_id'] = config.get( + section_name, 'CLIENT_ID', fallback=None) + provider['client_secret'] = config.get( + section_name, 'CLIENT_SECRET', fallback=None) + provider['redirect_endpoint'] = config.get( + section_name, 'REDIRECT_ENDPOINT') + if provider['redirect_endpoint']: + if not provider['redirect_endpoint'].startswith('/'): + provider['redirect_endpoint'] = '/%s' % provider[ + 'redirect_endpoint'] + if provider['redirect_endpoint'].endswith('/'): + provider['redirect_endpoint'] = provider[ + 'redirect_endpoint'][:-1] + provider['authorize_url'] = config.get( + section_name, 'AUTHORIZE_URL') + provider['accesstoken_url'] = config.get( + section_name, 'ACCESS_TOKEN_URL') + provider['userinfo_url'] = config.get( + section_name, 'USERINFO_URL') + self.oidc[section_name[len(PREFIX):]] = provider diff --git a/qiita_core/support_files/config_test.cfg b/qiita_core/support_files/config_test.cfg index 917d49adf..721d6207d 100644 --- a/qiita_core/support_files/config_test.cfg +++ b/qiita_core/support_files/config_test.cfg @@ -196,3 +196,57 @@ STATS_MAP_CENTER_LONGITUDE = # On May 2024, we removed QIIMP from the code base but we will leave this # section in case we need to add access to another iframe in the future; note # that the qiita-terms are also accessed via iframe but this is internal + +# --------------------- External Identity Provider settings -------------------- +# user authentication happens per default within Qiita, i.e. when a user logs in, +# the stored password hash and email address is compared against what a user +# just provided. You might however, use an external identity provider (IdP) to +# authenticate the user like +# google: https://developers.google.com/identity/protocols/oauth2 or +# github: https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/authorizing-oauth-apps or +# self hosted keycloak: https://www.keycloak.org/ +# Thus, you don't have to deal with user verification, reset passwords, ... +# Authorization (i.e. if the authorized user is allowed to use Qiita or which +# user level he/she gets assigned is an independent process. You can even use +# multiple independent external identity providers! +# Qiita currently only support the "open ID connect" protocol with the implicit flow. +# Each identity provider comes as its own config section [oidc_foo] and needs +# to specify the following five fields: +# +# Typical identity provider manage multiple "realms" and specific "clients" per realm +# You need to contact your IdP and register Qiita as a new "client". The IdP will +# provide you with the correct values. +# +# The authorization protocol requires three steps to obtain user information: +# 1) you identify as the correct client and ask the IdP for a request code +# You have to forward the user to the login page of your IdP. To let the IdP +# know how to come back to Qiita, you need to provide a redirect URL +# 2) you exchange the code for a user token +# 3) you obtain information about the user for the obtaines user token +# Typically, each step is implemented as a separate URL endpoint +# +# To activate IdP: comment out the following config section + +#[oidc_academicid] +# +## client ID for Qiita as registered at your Identity Provider of choice +#CLIENT_ID = +# +## client secret to verify Qiita as the correct client. Not all IdPs require this +#CLIENT_SECRET = +# +## redirect URL (end point in your Qiita instance), to which the IdP redirects +## after user types in his/her credentials. If you don't want to change code in +## qiita_pet/webserver.py the URL must follow the pattern: +## base_URL/auth/login_OIDC/foo where foo is the name of this config section +## without the oidc_ prefix! +#REDIRECT_ENDPOINT = /auth/login_OIDC/localkeycloak +# +## URL for step 1: obtain code +#AUTHORIZE_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/auth +# +## URL for step 2: obtain user token +#ACCESS_TOKEN_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/token +# +## URL for step 3: obtain user infos +#USERINFO_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/userinfo diff --git a/qiita_core/tests/test_configuration_manager.py b/qiita_core/tests/test_configuration_manager.py index 92b27d969..2c9a2fd22 100644 --- a/qiita_core/tests/test_configuration_manager.py +++ b/qiita_core/tests/test_configuration_manager.py @@ -289,6 +289,32 @@ def test_get_portal_latlong(self): obs._get_portal(self.conf) self.assertEqual(obs.stats_map_center_longitude, -105.24827) + def test_get_postgres(self): + SECTION_NAME = 'oidc_academicid' + obs = ConfigurationManager() + self.assertTrue(len(obs.oidc), 1) + self.assertTrue(obs.oidc.keys(), [SECTION_NAME]) + + # assert endpoint starts with / + self.conf.set(SECTION_NAME, 'REDIRECT_ENDPOINT', 'auth/something') + obs._get_oidc(self.conf) + self.assertEqual(obs.oidc['academicid']['redirect_endpoint'], + '/auth/something') + + # assert endpoint does not end with / + self.conf.set(SECTION_NAME, 'REDIRECT_ENDPOINT', 'auth/something/') + obs._get_oidc(self.conf) + self.assertEqual(obs.oidc['academicid']['redirect_endpoint'], + '/auth/something') + + self.conf.set(SECTION_NAME, 'CLIENT_ID', 'foo') + obs._get_oidc(self.conf) + self.assertEqual(obs.oidc['academicid']['client_id'], "foo") + + self.assertTrue('gwdg.de' in obs.oidc['academicid']['authorize_url']) + self.assertTrue('gwdg.de' in obs.oidc['academicid']['accesstoken_url']) + self.assertTrue('gwdg.de' in obs.oidc['academicid']['userinfo_url']) + CONF = """ # ------------------------------ Main settings -------------------------------- @@ -471,6 +497,31 @@ def test_get_portal_latlong(self): # ----------------------------- iframes settings --------------------------- [iframe] + +# ------------------- External Identity Provider settings ------------------ +[oidc_academicid] + +# client ID for Qiita as registered at your Identity Provider of choice +CLIENT_ID = gi-qiita-prod + +# client secret to verify Qiita as the correct client. Not all IdPs require this +CLIENT_SECRET = + +# redirect URL (end point in your Qiita instance), to which the IdP redirects +# after user types in his/her credentials. If you don't want to change code in +# qiita_pet/webserver.py the URL must follow the pattern: +# base_URL/auth/login_OIDC/foo where foo is the name of this config section +# without the oidc_ prefix! +REDIRECT_ENDPOINT = /auth/login_OIDC/academicid + +# URL for step 1: obtain code +AUTHORIZE_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/auth + +# URL for step 2: obtain user token +ACCESS_TOKEN_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/token + +# URL for step 3: obtain user infos +USERINFO_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/userinfo """ if __name__ == '__main__': From b4ab605fd6b9873b4d47e05e8601beb9ae2bc361 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 4 Mar 2025 15:44:29 +0100 Subject: [PATCH 33/61] flake8 --- qiita_core/support_files/config_test.cfg | 29 +++++++++++++------ .../tests/test_configuration_manager.py | 29 +++++++++++++------ 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/qiita_core/support_files/config_test.cfg b/qiita_core/support_files/config_test.cfg index 721d6207d..f78fd1ac3 100644 --- a/qiita_core/support_files/config_test.cfg +++ b/qiita_core/support_files/config_test.cfg @@ -230,10 +230,11 @@ STATS_MAP_CENTER_LONGITUDE = #[oidc_academicid] # ## client ID for Qiita as registered at your Identity Provider of choice -#CLIENT_ID = +#CLIENT_ID = gi-qiita-prod # -## client secret to verify Qiita as the correct client. Not all IdPs require this -#CLIENT_SECRET = +## client secret to verify Qiita as the correct client. Not all IdPs require +## a client secret! +#CLIENT_SECRET = verySecretString # ## redirect URL (end point in your Qiita instance), to which the IdP redirects ## after user types in his/her credentials. If you don't want to change code in @@ -242,11 +243,21 @@ STATS_MAP_CENTER_LONGITUDE = ## without the oidc_ prefix! #REDIRECT_ENDPOINT = /auth/login_OIDC/localkeycloak # -## URL for step 1: obtain code -#AUTHORIZE_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/auth +## The URL of the well-known json document, specifying how API end points +## like 'authorize', 'token' or 'userinfo' are defined. See e.g. +## https://swagger.io/docs/specification/authentication/ +## openid-connect-discovery/ +#WELLKNOWN_URI = https://keycloak.sso.gwdg.de/.well-known/openid-configuration # -## URL for step 2: obtain user token -#ACCESS_TOKEN_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/token +## a speaking label for the Identity Provider. Section name is used if empty. +#LABEL = GWDG Academic Cloud # -## URL for step 3: obtain user infos -#USERINFO_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/userinfo +## The scope, i.e. fields about a user, which Qiita requests from the +## Identity Provider, e.g. "profile email eduperson_orcid". +## Will be automatically extended by the scope "openid", to enable the +## "authorize_code" OIDC flow. +#SCOPE = openid +# +##Optional. Name of a file in qiita_pet/static/img that shall be +##displayed for login through Service Provider, instead of a plain button +#LOGO = oidc_lifescienceAAI.png diff --git a/qiita_core/tests/test_configuration_manager.py b/qiita_core/tests/test_configuration_manager.py index 2c9a2fd22..82986e284 100644 --- a/qiita_core/tests/test_configuration_manager.py +++ b/qiita_core/tests/test_configuration_manager.py @@ -289,7 +289,7 @@ def test_get_portal_latlong(self): obs._get_portal(self.conf) self.assertEqual(obs.stats_map_center_longitude, -105.24827) - def test_get_postgres(self): + def test_get_oidc(self): SECTION_NAME = 'oidc_academicid' obs = ConfigurationManager() self.assertTrue(len(obs.oidc), 1) @@ -504,8 +504,9 @@ def test_get_postgres(self): # client ID for Qiita as registered at your Identity Provider of choice CLIENT_ID = gi-qiita-prod -# client secret to verify Qiita as the correct client. Not all IdPs require this -CLIENT_SECRET = +# client secret to verify Qiita as the correct client. Not all IdPs require +# a client secret. +CLIENT_SECRET = verySecretString # redirect URL (end point in your Qiita instance), to which the IdP redirects # after user types in his/her credentials. If you don't want to change code in @@ -514,14 +515,24 @@ def test_get_postgres(self): # without the oidc_ prefix! REDIRECT_ENDPOINT = /auth/login_OIDC/academicid -# URL for step 1: obtain code -AUTHORIZE_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/auth +# The URL of the well-known json document, specifying how API end points +# like 'authorize', 'token' or 'userinfo' are defined. See e.g. +# https://swagger.io/docs/specification/authentication/ +# openid-connect-discovery/ +WELLKNOWN_URI = https://keycloak.sso.gwdg.de/.well-known/openid-configuration + +# a speaking label for the Identity Provider. Section name is used if empty. +LABEL = GWDG Academic Cloud -# URL for step 2: obtain user token -ACCESS_TOKEN_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/token +# The scope, i.e. fields about a user, which Qiita requests from the +# Identity Provider, e.g. "profile email eduperson_orcid". +# Will be automatically extended by the scope "openid", to enable the +# "authorize_code" OIDC flow. +SCOPE = openid -# URL for step 3: obtain user infos -USERINFO_URL = https://keycloak.sso.gwdg.de/auth/realms/academiccloud/protocol/openid-connect/userinfo +# Optional. Name of a file in qiita_pet/static/img that shall be +# displayed for login through Service Provider, instead of a plain button +LOGO = oidc_lifescienceAAI.png """ if __name__ == '__main__': From baa7230c035c8c9902f8cb7ceba93282caaa3e45 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 12:12:45 +0100 Subject: [PATCH 34/61] also provide a label for a speaking name of the identity provider --- qiita_core/configuration_manager.py | 6 ++++++ qiita_core/tests/test_configuration_manager.py | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/qiita_core/configuration_manager.py b/qiita_core/configuration_manager.py index 738a8a5ba..c7fbf2e40 100644 --- a/qiita_core/configuration_manager.py +++ b/qiita_core/configuration_manager.py @@ -145,6 +145,8 @@ class ConfigurationManager(object): userinfo_url : str The URL of the IdP to obtain information about the user, like email, username, ... + label : str + A speaking label for the Identity Provider Raises ------ @@ -440,4 +442,8 @@ def _get_oidc(self, config): section_name, 'ACCESS_TOKEN_URL') provider['userinfo_url'] = config.get( section_name, 'USERINFO_URL') + provider['label'] = config.get(section_name, 'LABEL') + if not provider['label']: + # fallback, if no label is provided + provider['label'] = section_name[len(PREFIX):] self.oidc[section_name[len(PREFIX):]] = provider diff --git a/qiita_core/tests/test_configuration_manager.py b/qiita_core/tests/test_configuration_manager.py index 82986e284..3069e2982 100644 --- a/qiita_core/tests/test_configuration_manager.py +++ b/qiita_core/tests/test_configuration_manager.py @@ -315,6 +315,13 @@ def test_get_oidc(self): self.assertTrue('gwdg.de' in obs.oidc['academicid']['accesstoken_url']) self.assertTrue('gwdg.de' in obs.oidc['academicid']['userinfo_url']) + self.assertEqual(obs.oidc['academicid']['label'], + 'GWDG Academic Cloud') + # test fallback, if no label is provided + self.conf.set(SECTION_NAME, 'LABEL', '') + obs._get_oidc(self.conf) + self.assertEqual(obs.oidc['academicid']['label'], 'academicid') + CONF = """ # ------------------------------ Main settings -------------------------------- From 52e57ca42edaa3989b62fd94e097b3b71accfe24 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 12:13:20 +0100 Subject: [PATCH 35/61] start implementing the OIDC dance --- qiita_pet/handlers/auth_handlers.py | 19 ++++++++++++++ qiita_pet/templates/sitebase.html | 39 +++++++++++++++++++++++++++++ qiita_pet/webserver.py | 14 ++++++++++- 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/qiita_pet/handlers/auth_handlers.py b/qiita_pet/handlers/auth_handlers.py index 4c5e306b1..543c585d9 100644 --- a/qiita_pet/handlers/auth_handlers.py +++ b/qiita_pet/handlers/auth_handlers.py @@ -7,6 +7,7 @@ # ----------------------------------------------------------------------------- from tornado.escape import url_escape, json_encode +from tornado.auth import OAuth2Mixin from qiita_pet.handlers.base_handlers import BaseHandler from qiita_core.qiita_settings import qiita_config, r_client @@ -177,3 +178,21 @@ class AuthLogoutHandler(BaseHandler): def get(self): self.clear_cookie("user") self.redirect("%s/" % qiita_config.portal_dir) + + +class KeycloakMixin(OAuth2Mixin): + pass + + +class AuthLoginOIDCHandler(BaseHandler, KeycloakMixin): + async def get(self, login): + code = self.get_argument('code', False) + if code: + # step 2: we got a code and now want to exchange it for a user + # access token + print("step2") + pass + else: + # step 1: no code from IdP yet, thus retrieve one now + print("step1") + pass diff --git a/qiita_pet/templates/sitebase.html b/qiita_pet/templates/sitebase.html index 5e2e0633f..83b9a4208 100644 --- a/qiita_pet/templates/sitebase.html +++ b/qiita_pet/templates/sitebase.html @@ -160,6 +160,18 @@ // Based on http://codepen.io/willvincent/pen/LbeKKW // and https://datatables.net/examples/api/row_details.html + {% if len(qiita_config.oidc) > 0 %} + $('.modal').css({ + 'top': '30%', + 'margin-top': '-'+($('.modal').height() / 2)+'px', + 'margin-left': '+'+($('.modal').width() / 3)+'px'}); + $('#qiita_signin_modal').click(function(e){ + e.preventDefault(); + $('.qiita_auth_signin_options').modal('hide'); + window.location.href = '{% raw qiita_config.portal_dir %}'; + }); + {% end %} + Vue.component('data-table-processing-jobs', { template: '
    ', props: ['jobs'], @@ -445,6 +457,33 @@ + + {% elif len(qiita_config.oidc) == 1 %} + {% for idp in qiita_config.oidc %} +
    + +
    + {% end %} + + {% elif len(qiita_config.oidc) > 1 %} +
      + + +
    {% else %}
    diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py index 17fbf7af8..237b4971e 100644 --- a/qiita_pet/webserver.py +++ b/qiita_pet/webserver.py @@ -20,7 +20,8 @@ from qiita_pet.handlers.base_handlers import ( MainHandler, NoPageHandler, IFrame) from qiita_pet.handlers.auth_handlers import ( - AuthCreateHandler, AuthLoginHandler, AuthLogoutHandler, AuthVerifyHandler) + AuthCreateHandler, AuthLoginHandler, AuthLogoutHandler, AuthVerifyHandler, + AuthLoginOIDCHandler) from qiita_pet.handlers.user_handlers import ( ChangeForgotPasswordHandler, ForgotPasswordHandler, UserProfileHandler, UserMessagesHander, UserJobs, PurgeUsersAJAXHandler, PurgeUsersHandler) @@ -244,6 +245,16 @@ def __init__(self): (r"/qiita_db/studies/(.*)", APIStudiesListing) ] + # only expose open id connect endpoints iff at least one was configured + # through the settings file + if len(qiita_config.oidc) > 0: + handlers.extend([ + (r"/auth/login_OIDC/(.*)", AuthLoginOIDCHandler) + #(r"/admin/user_authorization/", AdminOIDCUserAuthorization), + #(r"/admin/user_authorizationAjax/", + # AdminOIDCUserAuthorizationAjax), + ]) + # rest endpoints handlers.extend(REST_ENDPOINTS) @@ -275,4 +286,5 @@ def __init__(self): "cookie_secret": qiita_config.cookie_secret, "login_url": "%s/auth/login/" % qiita_config.portal_dir, } + tornado.web.Application.__init__(self, handlers, **settings) From 4061373aedd67f5e9d8804240a98ed459e341cbf Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 12:57:55 +0100 Subject: [PATCH 36/61] modal not necessary, if only one provider was defined --- qiita_pet/templates/sitebase.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_pet/templates/sitebase.html b/qiita_pet/templates/sitebase.html index 83b9a4208..be161bdf3 100644 --- a/qiita_pet/templates/sitebase.html +++ b/qiita_pet/templates/sitebase.html @@ -160,7 +160,7 @@ // Based on http://codepen.io/willvincent/pen/LbeKKW // and https://datatables.net/examples/api/row_details.html - {% if len(qiita_config.oidc) > 0 %} + {% if len(qiita_config.oidc) > 1 %} $('.modal').css({ 'top': '30%', 'margin-top': '-'+($('.modal').height() / 2)+'px', From 51307d1f7f1ff5023b67fffd4dfa0b274e2cc87d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 12:58:19 +0100 Subject: [PATCH 37/61] error handling of provider not in config file --- qiita_pet/handlers/auth_handlers.py | 71 +++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/qiita_pet/handlers/auth_handlers.py b/qiita_pet/handlers/auth_handlers.py index 543c585d9..da3596114 100644 --- a/qiita_pet/handlers/auth_handlers.py +++ b/qiita_pet/handlers/auth_handlers.py @@ -21,6 +21,44 @@ # login code modified from https://gist.github.com/guillaumevincent/4771570 + + + +import os +import datetime +import inspect + +INDENT = 0 + +def inc_indent(): + global INDENT + INDENT += 1 +def dec_indent(): + global INDENT + INDENT -= 1 +def stefan(msg, inc=0, fn='oidc.log'): + global INDENT + if inc < 0: + dec_indent() + fp='/homes/sjanssen/bcf_qiita/Logs/%s' % fn + current_time = datetime.datetime.now() + pid = os.getpid() + FH = open(fp, 'a') + source = inspect.stack()[1] + FH.write("%stime=%s, pid=%s, fct=%s, file=%s, msg=%s\n" % ( + "~~" * INDENT, + current_time, + pid, + source.function, + source.filename, + msg)) + FH.close() + if inc > 0: + inc_indent() + + + + class AuthCreateHandler(BaseHandler): """User Creation""" def get(self): @@ -185,14 +223,39 @@ class KeycloakMixin(OAuth2Mixin): class AuthLoginOIDCHandler(BaseHandler, KeycloakMixin): - async def get(self, login): + async def post(self, login): code = self.get_argument('code', False) + + # Qiita might be configured for multiple identity providers. We learn + # which one the user chose through different name attributes of the + # html form button + idp = None + msg = "" + if hasattr(self, 'path_args') and (len(self.path_args) > 0) and \ + (self.path_args[0] != None) and (self.path_args[0] != ""): + idp = self.path_args[0] + else: + msg = 'External Identity Provider not specified!' + if idp not in qiita_config.oidc.keys(): + msg = 'Unknown Identity Provider "%s", please check config file.' % idp + + if idp is None: + self.render("index.html", message=msg, level='warning') + + stefan("code is %s, idp = %s" % (code, idp), inc=+1) if code: # step 2: we got a code and now want to exchange it for a user # access token - print("step2") + stefan("step2") pass else: # step 1: no code from IdP yet, thus retrieve one now - print("step1") - pass + stefan("step1") + self.authorize_redirect( + redirect_uri=qiita_config.base_url + qiita_config.oidc[idp]['redirect_endpoint'], + client_id=qiita_config.oidc[idp]['client_id'], + client_secret=qiita_config.oidc[idp]['client_secret'], + response_type='code', + scope=['openid'] + ) + stefan("ende", inc=-1) From 7a0ec9f742088b45efbc4fff99951f2aa6ef08d0 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 15:46:14 +0100 Subject: [PATCH 38/61] adding pycurl package to enable tornado curl_httpclients --- .github/workflows/qiita-ci.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index dd596af87..69ad24694 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -64,7 +64,7 @@ jobs: # Setting up main qiita conda environment conda config --add channels conda-forge conda deactivate - conda create --quiet --yes -n qiita python=3.9 pip libgfortran numpy nginx cython redis + conda create --quiet --yes -n qiita python=3.9 pip libgfortran numpy nginx cython redis pycurl conda env list conda activate qiita pip install -U pip diff --git a/setup.py b/setup.py index b07df6e5f..a66321a8d 100644 --- a/setup.py +++ b/setup.py @@ -110,7 +110,7 @@ 'openpyxl', 'sphinx-bootstrap-theme', 'Sphinx<3.0', 'gitpython', 'redbiom', 'pyzmq', 'sphinx_rtd_theme', 'paramiko', 'seaborn', 'matplotlib', 'scipy<=1.10.1', - 'nose', + 'nose', 'pycurl', 'flake8', 'six', 'qiita-files @ https://github.com/' 'qiita-spots/qiita-files/archive/master.zip', 'mock', 'python-jose', 'markdown2', 'iteration_utilities', From 0c365a18ac49a4c98a138d7f7fc6395e1d74bb0c Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 15:46:40 +0100 Subject: [PATCH 39/61] a new method to create a user, if information do not need to be entered manually, but are obtained from an external identity provider --- qiita_db/user.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/qiita_db/user.py b/qiita_db/user.py index b4beac184..59f13c78c 100644 --- a/qiita_db/user.py +++ b/qiita_db/user.py @@ -244,6 +244,45 @@ def create(cls, email, password, info=None): return cls(email) + @classmethod + def create_oidc(cls, email): + """Creates a new user with information obtained from an external + identity provider + + Parameters + ---------- + email : str + The user's email fetched from the User Info of the Identity Provider + upon successful authentication. + + Raises + ------ + IncorrectEmailError + Email string given is not a valid email + """ + if not validate_email(email): + raise IncorrectEmailError("Bad email given: %s" % email) + info = {} + # email and password are minimal needed information, password indicates + # OIDC user registration purely for admins + info['email'] = email + info['password'] = "not_necessary_due_to_OIDC" + #verify code is necessary to manually authorize users on the admin page + info['user_verify_code'] = "OIDC" + qdb.util.check_table_cols(info, cls._table) + columns = info.keys() + values = [info[col] for col in columns] + + # create user + sql = "INSERT INTO qiita.{0} ({1}) VALUES ({2})".format( + cls._table, ','.join(columns), ','.join(['%s'] * len(values))) + + qdb.sql_connection.TRN.add(sql, values) + + qdb.sql_connection.TRN.execute() + + return cls(email) + @classmethod def verify_code(cls, email, code, code_type): """Verify that a code and email match From e993a999f0a997e6b5b0458b1caa2e3b3f47c205 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 15:59:09 +0100 Subject: [PATCH 40/61] full OIDC dance implemented --- qiita_db/user.py | 7 +- qiita_pet/handlers/auth_handlers.py | 209 +++++++++++++++++++++------- 2 files changed, 159 insertions(+), 57 deletions(-) diff --git a/qiita_db/user.py b/qiita_db/user.py index 59f13c78c..df2028b66 100644 --- a/qiita_db/user.py +++ b/qiita_db/user.py @@ -252,8 +252,8 @@ def create_oidc(cls, email): Parameters ---------- email : str - The user's email fetched from the User Info of the Identity Provider - upon successful authentication. + The user's email fetched from the User Info of the Identity + Provider upon successful authentication. Raises ------ @@ -267,7 +267,8 @@ def create_oidc(cls, email): # OIDC user registration purely for admins info['email'] = email info['password'] = "not_necessary_due_to_OIDC" - #verify code is necessary to manually authorize users on the admin page + # verify code is necessary to manually authorize users on the admin + # page info['user_verify_code'] = "OIDC" qdb.util.check_table_cols(info, cls._table) columns = info.keys() diff --git a/qiita_pet/handlers/auth_handlers.py b/qiita_pet/handlers/auth_handlers.py index da3596114..186fbcabd 100644 --- a/qiita_pet/handlers/auth_handlers.py +++ b/qiita_pet/handlers/auth_handlers.py @@ -6,8 +6,13 @@ # The full license is in the file LICENSE, distributed with this software. # ----------------------------------------------------------------------------- -from tornado.escape import url_escape, json_encode +import urllib.parse +import os + +from tornado.escape import url_escape, json_encode, json_decode from tornado.auth import OAuth2Mixin +from tornado.curl_httpclient import CurlAsyncHTTPClient +from tornado.web import HTTPError from qiita_pet.handlers.base_handlers import BaseHandler from qiita_core.qiita_settings import qiita_config, r_client @@ -18,47 +23,10 @@ from qiita_db.user import User from qiita_db.exceptions import (QiitaDBUnknownIDError, QiitaDBDuplicateError, QiitaDBError) +from qiita_db.logger import LogEntry # login code modified from https://gist.github.com/guillaumevincent/4771570 - - - -import os -import datetime -import inspect - -INDENT = 0 - -def inc_indent(): - global INDENT - INDENT += 1 -def dec_indent(): - global INDENT - INDENT -= 1 -def stefan(msg, inc=0, fn='oidc.log'): - global INDENT - if inc < 0: - dec_indent() - fp='/homes/sjanssen/bcf_qiita/Logs/%s' % fn - current_time = datetime.datetime.now() - pid = os.getpid() - FH = open(fp, 'a') - source = inspect.stack()[1] - FH.write("%stime=%s, pid=%s, fct=%s, file=%s, msg=%s\n" % ( - "~~" * INDENT, - current_time, - pid, - source.function, - source.filename, - msg)) - FH.close() - if inc > 0: - inc_indent() - - - - class AuthCreateHandler(BaseHandler): """User Creation""" def get(self): @@ -219,43 +187,176 @@ def get(self): class KeycloakMixin(OAuth2Mixin): - pass + config = dict() + + # environment variables that define proxies + vars_proxy = [env for env in os.environ if env.lower().endswith('_proxy')] + proxies = list(sorted({os.environ[var] for var in vars_proxy})) + if len(proxies) > 1: + msg = ("The OS serving Qiita defines multiple proxy servers via " + "environment variables, but with different values. Using the " + "first one:\n %s") % '\n '.join( + ['%s=%s' % (var, os.environ[var]) for var in vars_proxy]) + LogEntry.create('Runtime', msg) + try: + config['proxy_host'] = ':'.join(proxies[0].split(':')[:-1]) + config['proxy_port'] = int(proxies[0].split(':')[-1]) + except IndexError: + LogEntry.create('Runtime', ("Your proxy configuration doesn't seem to " + "follow the host:port pattern.")) + + def get_auth_http_client(self): + return CurlAsyncHTTPClient() + + async def get_authenticated_user(self, redirect_uri: str, code: str): + http = self.get_auth_http_client() + body = urllib.parse.urlencode( + { + "redirect_uri": redirect_uri, + "code": code, + "client_id": qiita_config.oidc[self.idp]['client_id'], + "client_secret": qiita_config.oidc[self.idp]['client_secret'], + "grant_type": "authorization_code" + } + ) + response = await http.fetch( + self._OAUTH_ACCESS_TOKEN_URL, + method="POST", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + body=body, + **self.config, + ) + return json_decode(response.body) class AuthLoginOIDCHandler(BaseHandler, KeycloakMixin): + idp = None + email_support = 'qiita.help@gmail.com' + async def post(self, login): code = self.get_argument('code', False) # Qiita might be configured for multiple identity providers. We learn # which one the user chose through different name attributes of the # html form button - idp = None msg = "" + self.idp = None if hasattr(self, 'path_args') and (len(self.path_args) > 0) and \ - (self.path_args[0] != None) and (self.path_args[0] != ""): - idp = self.path_args[0] + (self.path_args[0] is not None) and (self.path_args[0] != ""): + self.idp = self.path_args[0] else: msg = 'External Identity Provider not specified!' - if idp not in qiita_config.oidc.keys(): - msg = 'Unknown Identity Provider "%s", please check config file.' % idp + if self.idp not in qiita_config.oidc.keys(): + msg = ('Unknown Identity Provider "%s", ' + 'please check config file.') % self.idp - if idp is None: + if self.idp is None: self.render("index.html", message=msg, level='warning') - stefan("code is %s, idp = %s" % (code, idp), inc=+1) + self._OAUTH_AUTHORIZE_URL = qiita_config.oidc[self.idp][ + 'authorize_url'] + self._OAUTH_ACCESS_TOKEN_URL = qiita_config.oidc[self.idp][ + 'accesstoken_url'] + self._OAUTH_USERINFO_URL = qiita_config.oidc[self.idp][ + 'userinfo_url'] + if code: # step 2: we got a code and now want to exchange it for a user # access token - stefan("step2") - pass + access = await self.get_authenticated_user( + redirect_uri='%s%s' % ( + qiita_config.base_url, + qiita_config.oidc[self.idp]['redirect_endpoint']), + code=code) + access_token = access['access_token'] + if not access_token: + raise HTTPError(400, ( + "failed to exchange code for access token with " + "identity provider '%s'") % self.idp) + + # step 3: obtain user information (email, username, ...) from IdP + http = self.get_auth_http_client() + user_info_res = await http.fetch( + self._OAUTH_USERINFO_URL, + method="GET", + headers={"Accept": "application/json", + "Authorization": "Bearer {}".format(access_token)}, + **self.config, + ) + user_info = json_decode(user_info_res.body.decode( + 'utf8', 'replace')) + + if ('email' not in user_info.keys()) or \ + (user_info['email'] is None) or (user_info['email'] == ""): + raise HTTPError(400, ( + "Email address was not provided " + "from your identity provider '%s'") % self.idp) + + username = user_info['email'] + if not User.exists(username): + self.create_new_user(username) + else: + self.not_verified(username) + self.set_secure_cookie("user", username) + # self.set_secure_cookie("token", access_token) + + self.redirect('%s/' % qiita_config.base_url) else: # step 1: no code from IdP yet, thus retrieve one now - stefan("step1") self.authorize_redirect( - redirect_uri=qiita_config.base_url + qiita_config.oidc[idp]['redirect_endpoint'], - client_id=qiita_config.oidc[idp]['client_id'], - client_secret=qiita_config.oidc[idp]['client_secret'], + redirect_uri='%s%s' % ( + qiita_config.base_url, + qiita_config.oidc[self.idp]['redirect_endpoint']), + client_id=qiita_config.oidc[self.idp]['client_id'], + client_secret=qiita_config.oidc[self.idp]['client_secret'], response_type='code', scope=['openid'] ) - stefan("ende", inc=-1) + get = post # redirect will use GET method + + @execute_as_transaction + def create_new_user(self, username): + try: + created = User.create_oidc(username) + except QiitaDBDuplicateError: + msg = "Email already registered as a user" + if created: + try: + # qiita_config.base_url doesn't have a / at the end, but the + # qiita_config.portal_dir has it at the beginning but not at + # the end. This constructs the correct URL + msg = (("

    User Successfully Registered!

    Your Qiita " + "account has been successfully registered using '%s', " + "which was provided by the identity provider '%s'. " + "Your account is now awaiting authorization by a Qiita" + " admin.

    If you have any questions regarding " + "the authorization process, please email us at %s.

    ") % ( + username, + qiita_config.oidc[self.idp]['label'], + self.email_support, + self.email_support)) + + self.redirect(u"%s/?level=success&message=%s" % ( + qiita_config.portal_dir, url_escape(msg))) + except Exception: + msg = (("Unable to create account. Please contact the qiita " + "developers at %s") % ( + self.email_support, self.email_support)) + self.redirect(u"%s/?level=danger&message=%s" % ( + qiita_config.portal_dir, url_escape(msg))) + return + else: + error_msg = u"?error=" + url_escape(msg) + self.redirect(u"%s/%s" % (qiita_config.portal_dir, error_msg)) + + def not_verified(self, username): + user = User(username) + if user.level == "unverified": + msg = (("You are not yet verified by an admin. Please wait or " + "contact the qiita developers at %s") % (self.email_support, self.email_support)) + self.redirect(u"%s/?level=danger&message=%s" % ( + qiita_config.portal_dir, url_escape(msg))) + else: + return From ca5f7f691ed2a2e1a66ff7f4897d90ed954d2f3c Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 21:20:55 +0100 Subject: [PATCH 41/61] add an admin page to activate users which requested authorization through OIDC --- qiita_pet/handlers/auth_handlers.py | 80 ++++++++++++++++++++++++++++- qiita_pet/webserver.py | 11 ++-- 2 files changed, 84 insertions(+), 7 deletions(-) diff --git a/qiita_pet/handlers/auth_handlers.py b/qiita_pet/handlers/auth_handlers.py index 186fbcabd..3bbd86781 100644 --- a/qiita_pet/handlers/auth_handlers.py +++ b/qiita_pet/handlers/auth_handlers.py @@ -12,9 +12,10 @@ from tornado.escape import url_escape, json_encode, json_decode from tornado.auth import OAuth2Mixin from tornado.curl_httpclient import CurlAsyncHTTPClient -from tornado.web import HTTPError +from tornado.web import HTTPError, authenticated from qiita_pet.handlers.base_handlers import BaseHandler +from qiita_pet.handlers.portal import PortalEditBase from qiita_core.qiita_settings import qiita_config, r_client from qiita_core.util import execute_as_transaction from qiita_core.exceptions import (IncorrectPasswordError, IncorrectEmailError, @@ -24,6 +25,7 @@ from qiita_db.exceptions import (QiitaDBUnknownIDError, QiitaDBDuplicateError, QiitaDBError) from qiita_db.logger import LogEntry +import qiita_db as qdb # login code modified from https://gist.github.com/guillaumevincent/4771570 @@ -300,7 +302,7 @@ async def post(self, login): self.set_secure_cookie("user", username) # self.set_secure_cookie("token", access_token) - self.redirect('%s/' % qiita_config.base_url) + self.redirect("%s/" % qiita_config.portal_dir) else: # step 1: no code from IdP yet, thus retrieve one now self.authorize_redirect( @@ -360,3 +362,77 @@ def not_verified(self, username): qiita_config.portal_dir, url_escape(msg))) else: return + + +class AdminOIDCUserAuthorization(PortalEditBase): + """User Verification for Qiita-Account Creation following OIDC Login""" + @authenticated + @execute_as_transaction + def get(self): + # render page and transfer headers to be included for the table + self.check_admin() + headers=["email","name","affiliation","address", "phone"] + self.render('admin_user_authorization.html', headers=headers, + submit_url="/admin/user_authorization/") + + def post(self): + # check if logged in user is admin and fetch all checked boxes as well + # as the action + self.check_admin() + users = map(str, self.get_arguments('selected')) + action = self.get_argument('action') + # depending on the action either autorize (add) user or delete user + # from db (remove) + for user in users: + try: + with warnings.catch_warnings(record=True) as warns: + if action == "Add": + self.authorize_user(user) + elif action == "Remove": + user_to_delete = User(user) + user_to_delete.delete(user) + else: + raise HTTPError(400, + reason="Unknown action: %s" % action) + except QiitaDBError as e: + self.write(action.upper() + " ERROR:
    " + str(e)) + return + msg = '; '.join([str(w.message) for w in warns]) + self.write(action + " completed successfully
    " + msg) + + @authenticated + @execute_as_transaction + def authorize_user(self, user): + # authorize user by verifying login manually using tue standard Qiita + # verify function + self.check_admin() + User.verify_code(user, User(user).info['user_verify_code'], "create") + return + + +class AdminOIDCUserAuthorizationAjax(PortalEditBase): + @authenticated + @execute_as_transaction + def get(self): + # retrieving users with an unverified level + self.check_admin() + with qdb.sql_connection.TRN: + sql = """SELECT email,name,affiliation,address,phone + FROM qiita.qiita_user + WHERE user_level_id='5'""" + qdb.sql_connection.TRN.add(sql) + users = qdb.sql_connection.TRN.execute()[1:] + result = [] + # fetching information for each user + for list in users: + for user in list: + usermail = user[0] + user_unit = {} + user_unit['email'] = User(usermail).email + user_unit['name'] = User(usermail).info['name'] + user_unit['affiliation'] = User(usermail).info['affiliation'] + user_unit['address'] = User(usermail).info['address'] + user_unit['phone'] = User(usermail).info['phone'] + result.append(user_unit) + # returning information as JSON + self.write(dumps(result)) diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py index 237b4971e..62b5e18f5 100644 --- a/qiita_pet/webserver.py +++ b/qiita_pet/webserver.py @@ -21,7 +21,8 @@ MainHandler, NoPageHandler, IFrame) from qiita_pet.handlers.auth_handlers import ( AuthCreateHandler, AuthLoginHandler, AuthLogoutHandler, AuthVerifyHandler, - AuthLoginOIDCHandler) + AuthLoginOIDCHandler, AdminOIDCUserAuthorization, + AdminOIDCUserAuthorizationAjax) from qiita_pet.handlers.user_handlers import ( ChangeForgotPasswordHandler, ForgotPasswordHandler, UserProfileHandler, UserMessagesHander, UserJobs, PurgeUsersAJAXHandler, PurgeUsersHandler) @@ -249,10 +250,10 @@ def __init__(self): # through the settings file if len(qiita_config.oidc) > 0: handlers.extend([ - (r"/auth/login_OIDC/(.*)", AuthLoginOIDCHandler) - #(r"/admin/user_authorization/", AdminOIDCUserAuthorization), - #(r"/admin/user_authorizationAjax/", - # AdminOIDCUserAuthorizationAjax), + (r"/auth/login_OIDC/(.*)", AuthLoginOIDCHandler), + (r"/admin/user_authorization/", AdminOIDCUserAuthorization), + (r"/admin/user_authorizationAjax/", + AdminOIDCUserAuthorizationAjax), ]) # rest endpoints From 4d5c6a2eb8b3925015797a75fd3e2f135dc74173 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 21:24:10 +0100 Subject: [PATCH 42/61] flake8 --- qiita_pet/handlers/auth_handlers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/qiita_pet/handlers/auth_handlers.py b/qiita_pet/handlers/auth_handlers.py index 3bbd86781..3bd2e6547 100644 --- a/qiita_pet/handlers/auth_handlers.py +++ b/qiita_pet/handlers/auth_handlers.py @@ -8,6 +8,7 @@ import urllib.parse import os +import warnings from tornado.escape import url_escape, json_encode, json_decode from tornado.auth import OAuth2Mixin @@ -371,7 +372,7 @@ class AdminOIDCUserAuthorization(PortalEditBase): def get(self): # render page and transfer headers to be included for the table self.check_admin() - headers=["email","name","affiliation","address", "phone"] + headers = ["email", "name", "affiliation", "address", "phone"] self.render('admin_user_authorization.html', headers=headers, submit_url="/admin/user_authorization/") @@ -421,7 +422,7 @@ def get(self): FROM qiita.qiita_user WHERE user_level_id='5'""" qdb.sql_connection.TRN.add(sql) - users = qdb.sql_connection.TRN.execute()[1:] + users = qdb.sql_connection.TRN.execute()[1:] result = [] # fetching information for each user for list in users: @@ -435,4 +436,4 @@ def get(self): user_unit['phone'] = User(usermail).info['phone'] result.append(user_unit) # returning information as JSON - self.write(dumps(result)) + self.write(json_encode(result)) From fd6d15e551c5bda7226f938dbc8f92a6fc8b3f03 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 20 Mar 2024 21:34:06 +0100 Subject: [PATCH 43/61] adding menu entry for user authorization --- .../templates/admin_user_authorization.html | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 qiita_pet/templates/admin_user_authorization.html diff --git a/qiita_pet/templates/admin_user_authorization.html b/qiita_pet/templates/admin_user_authorization.html new file mode 100644 index 000000000..865d0a859 --- /dev/null +++ b/qiita_pet/templates/admin_user_authorization.html @@ -0,0 +1,122 @@ +{% extends sitebase.html %} +{%block head%} + + +{%end%} +{%block content%} +
    +
    + + + + +
    +
    +
    +
    + | |
    + + + + + {% for head in headers %} + + {% end %} + + + + +
    Select{{head}}
    +
    +
    +{% end %} + From 9c8b824c5258de50924e3a718fb3d7e1b675f21d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 21 Mar 2024 09:11:18 +0100 Subject: [PATCH 44/61] do not expose traditional qiita internal user authentication, if OIDC is configured --- qiita_pet/webserver.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py index 62b5e18f5..fe461d811 100644 --- a/qiita_pet/webserver.py +++ b/qiita_pet/webserver.py @@ -108,13 +108,6 @@ class Application(tornado.web.Application): def __init__(self): handlers = [ (r"/", MainHandler), - (r"/auth/login/", AuthLoginHandler), - (r"/auth/logout/", AuthLogoutHandler), - (r"/auth/create/", AuthCreateHandler), - (r"/auth/verify/(.*)", AuthVerifyHandler), - (r"/auth/forgot/", ForgotPasswordHandler), - (r"/auth/reset/(.*)", ChangeForgotPasswordHandler), - (r"/profile/", UserProfileHandler), (r"/user/messages/", UserMessagesHander), (r"/user/jobs/", UserJobs), (r"/static/(.*)", tornado.web.StaticFileHandler, @@ -255,6 +248,17 @@ def __init__(self): (r"/admin/user_authorizationAjax/", AdminOIDCUserAuthorizationAjax), ]) + else: + # Qiita's traditional, internal user authentication + handlers.extend([ + (r"/auth/login/", AuthLoginHandler), + (r"/auth/logout/", AuthLogoutHandler), + (r"/auth/create/", AuthCreateHandler), + (r"/auth/verify/(.*)", AuthVerifyHandler), + (r"/auth/forgot/", ForgotPasswordHandler), + (r"/auth/reset/(.*)", ChangeForgotPasswordHandler), + (r"/profile/", UserProfileHandler) + ]) # rest endpoints handlers.extend(REST_ENDPOINTS) From a654e489255c97359c0eea58438c888093925924 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 21 Mar 2024 11:14:19 +0100 Subject: [PATCH 45/61] use Qiita typical modal for OIDC login --- qiita_pet/templates/sitebase.html | 49 +++++++++++++------------------ 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/qiita_pet/templates/sitebase.html b/qiita_pet/templates/sitebase.html index be161bdf3..4a36f8ea4 100644 --- a/qiita_pet/templates/sitebase.html +++ b/qiita_pet/templates/sitebase.html @@ -160,18 +160,6 @@ // Based on http://codepen.io/willvincent/pen/LbeKKW // and https://datatables.net/examples/api/row_details.html - {% if len(qiita_config.oidc) > 1 %} - $('.modal').css({ - 'top': '30%', - 'margin-top': '-'+($('.modal').height() / 2)+'px', - 'margin-left': '+'+($('.modal').width() / 3)+'px'}); - $('#qiita_signin_modal').click(function(e){ - e.preventDefault(); - $('.qiita_auth_signin_options').modal('hide'); - window.location.href = '{% raw qiita_config.portal_dir %}'; - }); - {% end %} - Vue.component('data-table-processing-jobs', { template: '
    ', props: ['jobs'], @@ -467,22 +455,7 @@ {% elif len(qiita_config.oidc) > 1 %}
      - - + Sign In
    {% else %} @@ -660,5 +633,25 @@
    succesful jobs are not shown
    + + + From 27f6d3543607d4a5309cd1cfcdf7d68911f40637 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 21 Mar 2024 11:17:39 +0100 Subject: [PATCH 46/61] always allow logout --- qiita_pet/webserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py index fe461d811..0e936285b 100644 --- a/qiita_pet/webserver.py +++ b/qiita_pet/webserver.py @@ -108,6 +108,7 @@ class Application(tornado.web.Application): def __init__(self): handlers = [ (r"/", MainHandler), + (r"/auth/logout/", AuthLogoutHandler), (r"/user/messages/", UserMessagesHander), (r"/user/jobs/", UserJobs), (r"/static/(.*)", tornado.web.StaticFileHandler, @@ -252,7 +253,6 @@ def __init__(self): # Qiita's traditional, internal user authentication handlers.extend([ (r"/auth/login/", AuthLoginHandler), - (r"/auth/logout/", AuthLogoutHandler), (r"/auth/create/", AuthCreateHandler), (r"/auth/verify/(.*)", AuthVerifyHandler), (r"/auth/forgot/", ForgotPasswordHandler), From 85bf1faeffc1479accf1aaa5846f8fe630a6c90a Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 21 Mar 2024 13:08:27 +0100 Subject: [PATCH 47/61] improved error handling --- qiita_pet/handlers/auth_handlers.py | 88 ++++++++++++++++++----------- 1 file changed, 55 insertions(+), 33 deletions(-) diff --git a/qiita_pet/handlers/auth_handlers.py b/qiita_pet/handlers/auth_handlers.py index 3bd2e6547..74bb9445c 100644 --- a/qiita_pet/handlers/auth_handlers.py +++ b/qiita_pet/handlers/auth_handlers.py @@ -14,6 +14,7 @@ from tornado.auth import OAuth2Mixin from tornado.curl_httpclient import CurlAsyncHTTPClient from tornado.web import HTTPError, authenticated +from tornado.httpclient import HTTPClientError from qiita_pet.handlers.base_handlers import BaseHandler from qiita_pet.handlers.portal import PortalEditBase @@ -191,6 +192,7 @@ def get(self): class KeycloakMixin(OAuth2Mixin): config = dict() + email_support = 'qiita.help@gmail.com' # environment variables that define proxies vars_proxy = [env for env in os.environ if env.lower().endswith('_proxy')] @@ -222,14 +224,22 @@ async def get_authenticated_user(self, redirect_uri: str, code: str): "grant_type": "authorization_code" } ) - response = await http.fetch( - self._OAUTH_ACCESS_TOKEN_URL, - method="POST", - headers={"Content-Type": "application/x-www-form-urlencoded"}, - body=body, - **self.config, - ) - return json_decode(response.body) + try: + response = await http.fetch( + self._OAUTH_ACCESS_TOKEN_URL, + method="POST", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + body=body, + **self.config) + return json_decode(response.body) + except HTTPClientError as e: + msg = ("The external identity provider '%s' returns an '%s' error" + ", when sending a request against '%s'. Thus, we cannot log" + "you in. Please contact the Qiita support team at " + "%s") % ( + self.idp, str(e), self._OAUTH_ACCESS_TOKEN_URL, + self.email_support, self.email_support) + self.render("index.html", message=msg, level='danger') class AuthLoginOIDCHandler(BaseHandler, KeycloakMixin): @@ -252,6 +262,7 @@ async def post(self, login): if self.idp not in qiita_config.oidc.keys(): msg = ('Unknown Identity Provider "%s", ' 'please check config file.') % self.idp + self.idp = None if self.idp is None: self.render("index.html", message=msg, level='warning') @@ -279,31 +290,42 @@ async def post(self, login): # step 3: obtain user information (email, username, ...) from IdP http = self.get_auth_http_client() - user_info_res = await http.fetch( - self._OAUTH_USERINFO_URL, - method="GET", - headers={"Accept": "application/json", - "Authorization": "Bearer {}".format(access_token)}, - **self.config, - ) - user_info = json_decode(user_info_res.body.decode( - 'utf8', 'replace')) - - if ('email' not in user_info.keys()) or \ - (user_info['email'] is None) or (user_info['email'] == ""): - raise HTTPError(400, ( - "Email address was not provided " - "from your identity provider '%s'") % self.idp) - - username = user_info['email'] - if not User.exists(username): - self.create_new_user(username) - else: - self.not_verified(username) - self.set_secure_cookie("user", username) - # self.set_secure_cookie("token", access_token) - - self.redirect("%s/" % qiita_config.portal_dir) + try: + user_info_res = await http.fetch( + self._OAUTH_USERINFO_URL, + method="GET", + headers={ + "Accept": "application/json", + "Authorization": "Bearer {}".format(access_token)}, + **self.config, + ) + user_info = json_decode(user_info_res.body.decode( + 'utf8', 'replace')) + + if ('email' not in user_info.keys()) or \ + (user_info['email'] is None) or (user_info['email'] == ""): + raise HTTPError(400, ( + "Email address was not provided " + "from your identity provider '%s'") % self.idp) + + username = user_info['email'] + if not User.exists(username): + self.create_new_user(username) + else: + self.not_verified(username) + self.set_secure_cookie("user", username) + # self.set_secure_cookie("token", access_token) + + self.redirect("%s/" % qiita_config.portal_dir) + except HTTPClientError as e: + msg = ( + "The external identity provider '%s' returns an '%s' error" + ", when sending a request against '%s'. Thus, we cannot " + "log you in. Please contact the Qiita support team at " + "%s") % ( + self.idp, str(e), self._OAUTH_USERINFO_URL, + self.email_support, self.email_support) + self.render("index.html", message=msg, level='danger') else: # step 1: no code from IdP yet, thus retrieve one now self.authorize_redirect( From 8a504cc0d0f71da8b51d60da6bf27324787ff10b Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 22 Mar 2024 10:09:26 +0100 Subject: [PATCH 48/61] revert: let user change their profile, but not password - if provided through OIDC --- qiita_pet/templates/user_profile.html | 2 ++ qiita_pet/webserver.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/qiita_pet/templates/user_profile.html b/qiita_pet/templates/user_profile.html index 5f5336d95..b8327eca3 100644 --- a/qiita_pet/templates/user_profile.html +++ b/qiita_pet/templates/user_profile.html @@ -34,6 +34,7 @@

    User Information

    + {% if len(qiita_config.oidc) == 0 %}

    Change Password

    @@ -54,5 +55,6 @@

    Change Password

    + {% end %} {%end%} diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py index 0e936285b..03c2a4fe7 100644 --- a/qiita_pet/webserver.py +++ b/qiita_pet/webserver.py @@ -109,6 +109,7 @@ def __init__(self): handlers = [ (r"/", MainHandler), (r"/auth/logout/", AuthLogoutHandler), + (r"/profile/", UserProfileHandler), (r"/user/messages/", UserMessagesHander), (r"/user/jobs/", UserJobs), (r"/static/(.*)", tornado.web.StaticFileHandler, @@ -256,8 +257,7 @@ def __init__(self): (r"/auth/create/", AuthCreateHandler), (r"/auth/verify/(.*)", AuthVerifyHandler), (r"/auth/forgot/", ForgotPasswordHandler), - (r"/auth/reset/(.*)", ChangeForgotPasswordHandler), - (r"/profile/", UserProfileHandler) + (r"/auth/reset/(.*)", ChangeForgotPasswordHandler) ]) # rest endpoints From ef05eeddb0f8c3d3f5be17ff40c260d650677365 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 22 Mar 2024 10:10:06 +0100 Subject: [PATCH 49/61] speaking button names + move into correct div to always get displayed --- .../templates/admin_user_authorization.html | 40 +++++++++---------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/qiita_pet/templates/admin_user_authorization.html b/qiita_pet/templates/admin_user_authorization.html index 865d0a859..5bc4eea6c 100644 --- a/qiita_pet/templates/admin_user_authorization.html +++ b/qiita_pet/templates/admin_user_authorization.html @@ -13,7 +13,7 @@ {%end%} {%block content%} -
    -
    -
    - - -
    -
    -
    | |
    - - - - {% for head in headers %} - - {% end %} - - - - + + + + {% for head in headers %} + + {% end %} + + + +
    Select{{head}}
    Select{{head}}
    +
    + + + +
    {% end %} - From a5270a074dbacfe56b60cfa157c10f5bf5b3c6b7 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 22 Mar 2024 10:11:31 +0100 Subject: [PATCH 50/61] use email from config + loop user_info from OIDC to fill DB --- qiita_pet/handlers/auth_handlers.py | 39 +++++++++++++---------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/qiita_pet/handlers/auth_handlers.py b/qiita_pet/handlers/auth_handlers.py index 74bb9445c..a5dc1eb67 100644 --- a/qiita_pet/handlers/auth_handlers.py +++ b/qiita_pet/handlers/auth_handlers.py @@ -192,7 +192,6 @@ def get(self): class KeycloakMixin(OAuth2Mixin): config = dict() - email_support = 'qiita.help@gmail.com' # environment variables that define proxies vars_proxy = [env for env in os.environ if env.lower().endswith('_proxy')] @@ -235,16 +234,15 @@ async def get_authenticated_user(self, redirect_uri: str, code: str): except HTTPClientError as e: msg = ("The external identity provider '%s' returns an '%s' error" ", when sending a request against '%s'. Thus, we cannot log" - "you in. Please contact the Qiita support team at " + " you in. Please contact the Qiita support team at " "%s") % ( self.idp, str(e), self._OAUTH_ACCESS_TOKEN_URL, - self.email_support, self.email_support) + qiita_config.help_email, qiita_config.help_email) self.render("index.html", message=msg, level='danger') class AuthLoginOIDCHandler(BaseHandler, KeycloakMixin): idp = None - email_support = 'qiita.help@gmail.com' async def post(self, login): code = self.get_argument('code', False) @@ -297,8 +295,7 @@ async def post(self, login): headers={ "Accept": "application/json", "Authorization": "Bearer {}".format(access_token)}, - **self.config, - ) + **self.config) user_info = json_decode(user_info_res.body.decode( 'utf8', 'replace')) @@ -310,13 +307,11 @@ async def post(self, login): username = user_info['email'] if not User.exists(username): - self.create_new_user(username) + self.create_new_user(username, user_info, self.idp) else: - self.not_verified(username) - self.set_secure_cookie("user", username) + self.check_verified(username) # self.set_secure_cookie("token", access_token) - self.redirect("%s/" % qiita_config.portal_dir) except HTTPClientError as e: msg = ( "The external identity provider '%s' returns an '%s' error" @@ -324,7 +319,7 @@ async def post(self, login): "log you in. Please contact the Qiita support team at " "%s") % ( self.idp, str(e), self._OAUTH_USERINFO_URL, - self.email_support, self.email_support) + qiita_config.help_email, qiita_config.help_email) self.render("index.html", message=msg, level='danger') else: # step 1: no code from IdP yet, thus retrieve one now @@ -340,9 +335,9 @@ async def post(self, login): get = post # redirect will use GET method @execute_as_transaction - def create_new_user(self, username): + def create_new_user(self, username, user_info, idp): try: - created = User.create_oidc(username) + created = User.create_oidc(username, user_info, idp) except QiitaDBDuplicateError: msg = "Email already registered as a user" if created: @@ -359,15 +354,15 @@ def create_new_user(self, username): "href=\"mailto:%s\">%s.

    ") % ( username, qiita_config.oidc[self.idp]['label'], - self.email_support, - self.email_support)) + qiita_config.help_email, + qiita_config.help_email)) self.redirect(u"%s/?level=success&message=%s" % ( qiita_config.portal_dir, url_escape(msg))) except Exception: msg = (("Unable to create account. Please contact the qiita " "developers at %s") % ( - self.email_support, self.email_support)) + qiita_config.help_email, qiita_config.help_email)) self.redirect(u"%s/?level=danger&message=%s" % ( qiita_config.portal_dir, url_escape(msg))) return @@ -375,16 +370,18 @@ def create_new_user(self, username): error_msg = u"?error=" + url_escape(msg) self.redirect(u"%s/%s" % (qiita_config.portal_dir, error_msg)) - def not_verified(self, username): + def check_verified(self, username): user = User(username) if user.level == "unverified": msg = (("You are not yet verified by an admin. Please wait or " "contact the qiita developers at %s") % (self.email_support, self.email_support)) + "'>%s") % (qiita_config.help_email, + qiita_config.help_email)) self.redirect(u"%s/?level=danger&message=%s" % ( qiita_config.portal_dir, url_escape(msg))) else: - return + self.set_secure_cookie("user", username) + self.redirect("%s/" % qiita_config.portal_dir) class AdminOIDCUserAuthorization(PortalEditBase): @@ -409,9 +406,9 @@ def post(self): for user in users: try: with warnings.catch_warnings(record=True) as warns: - if action == "Add": + if action == "Authorize_Users": self.authorize_user(user) - elif action == "Remove": + elif action == "Remove_Users": user_to_delete = User(user) user_to_delete.delete(user) else: From 2efb70ff1327f2ab6a8e08cedd3cd3447735fc3e Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 22 Mar 2024 10:12:11 +0100 Subject: [PATCH 51/61] use OIDC info to prefil user information --- qiita_db/user.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/qiita_db/user.py b/qiita_db/user.py index df2028b66..3dd5090fb 100644 --- a/qiita_db/user.py +++ b/qiita_db/user.py @@ -245,7 +245,7 @@ def create(cls, email, password, info=None): return cls(email) @classmethod - def create_oidc(cls, email): + def create_oidc(cls, email, user_info, idp): """Creates a new user with information obtained from an external identity provider @@ -254,6 +254,10 @@ def create_oidc(cls, email): email : str The user's email fetched from the User Info of the Identity Provider upon successful authentication. + user_info : dict + User information provided by the external identity provider + idp : str + The label of the external identity provider as set in config file Raises ------ @@ -269,7 +273,11 @@ def create_oidc(cls, email): info['password'] = "not_necessary_due_to_OIDC" # verify code is necessary to manually authorize users on the admin # page - info['user_verify_code'] = "OIDC" + info['user_verify_code'] = idp + # check if we got useful information from the IdP + if 'name' in user_info.keys(): + info['name'] = user_info['name'] + qdb.util.check_table_cols(info, cls._table) columns = info.keys() values = [info[col] for col in columns] From c8b1198f01fc4f0d575f9cf7488e093d55864141 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 4 Apr 2024 15:07:07 +0200 Subject: [PATCH 52/61] drop admin user authorization --- qiita_pet/handlers/auth_handlers.py | 160 ++++-------------- .../templates/admin_user_authorization.html | 118 ------------- qiita_pet/webserver.py | 8 +- 3 files changed, 39 insertions(+), 247 deletions(-) delete mode 100644 qiita_pet/templates/admin_user_authorization.html diff --git a/qiita_pet/handlers/auth_handlers.py b/qiita_pet/handlers/auth_handlers.py index a5dc1eb67..cd42d6763 100644 --- a/qiita_pet/handlers/auth_handlers.py +++ b/qiita_pet/handlers/auth_handlers.py @@ -202,12 +202,15 @@ class KeycloakMixin(OAuth2Mixin): "first one:\n %s") % '\n '.join( ['%s=%s' % (var, os.environ[var]) for var in vars_proxy]) LogEntry.create('Runtime', msg) - try: - config['proxy_host'] = ':'.join(proxies[0].split(':')[:-1]) - config['proxy_port'] = int(proxies[0].split(':')[-1]) - except IndexError: - LogEntry.create('Runtime', ("Your proxy configuration doesn't seem to " - "follow the host:port pattern.")) + elif len(proxies) == 1: + try: + config['proxy_host'] = ':'.join(proxies[0].split(':')[:-1]) + config['proxy_port'] = int(proxies[0].split(':')[-1]) + except IndexError: + LogEntry.create( + 'Runtime', + ("Your proxy configuration doesn't seem to " + "follow the host:port pattern.")) def get_auth_http_client(self): return CurlAsyncHTTPClient() @@ -309,8 +312,8 @@ async def post(self, login): if not User.exists(username): self.create_new_user(username, user_info, self.idp) else: - self.check_verified(username) - # self.set_secure_cookie("token", access_token) + self.set_secure_cookie("user", username) + self.redirect("%s/" % qiita_config.portal_dir) except HTTPClientError as e: msg = ( @@ -336,123 +339,34 @@ async def post(self, login): @execute_as_transaction def create_new_user(self, username, user_info, idp): + msg, msg_level = None, None # 'danger', 'success', 'info', 'warning' try: + # create user stub created = User.create_oidc(username, user_info, idp) - except QiitaDBDuplicateError: - msg = "Email already registered as a user" - if created: - try: - # qiita_config.base_url doesn't have a / at the end, but the - # qiita_config.portal_dir has it at the beginning but not at - # the end. This constructs the correct URL - msg = (("

    User Successfully Registered!

    Your Qiita " - "account has been successfully registered using '%s', " - "which was provided by the identity provider '%s'. " - "Your account is now awaiting authorization by a Qiita" - " admin.

    If you have any questions regarding " - "the authorization process, please email us at %s.

    ") % ( - username, - qiita_config.oidc[self.idp]['label'], - qiita_config.help_email, - qiita_config.help_email)) - - self.redirect(u"%s/?level=success&message=%s" % ( - qiita_config.portal_dir, url_escape(msg))) - except Exception: - msg = (("Unable to create account. Please contact the qiita " - "developers at %s") % ( - qiita_config.help_email, qiita_config.help_email)) - self.redirect(u"%s/?level=danger&message=%s" % ( - qiita_config.portal_dir, url_escape(msg))) - return - else: - error_msg = u"?error=" + url_escape(msg) - self.redirect(u"%s/%s" % (qiita_config.portal_dir, error_msg)) - - def check_verified(self, username): - user = User(username) - if user.level == "unverified": - msg = (("You are not yet verified by an admin. Please wait or " - "contact the qiita developers at %s") % (qiita_config.help_email, - qiita_config.help_email)) - self.redirect(u"%s/?level=danger&message=%s" % ( - qiita_config.portal_dir, url_escape(msg))) - else: - self.set_secure_cookie("user", username) - self.redirect("%s/" % qiita_config.portal_dir) - - -class AdminOIDCUserAuthorization(PortalEditBase): - """User Verification for Qiita-Account Creation following OIDC Login""" - @authenticated - @execute_as_transaction - def get(self): - # render page and transfer headers to be included for the table - self.check_admin() - headers = ["email", "name", "affiliation", "address", "phone"] - self.render('admin_user_authorization.html', headers=headers, - submit_url="/admin/user_authorization/") - - def post(self): - # check if logged in user is admin and fetch all checked boxes as well - # as the action - self.check_admin() - users = map(str, self.get_arguments('selected')) - action = self.get_argument('action') - # depending on the action either autorize (add) user or delete user - # from db (remove) - for user in users: - try: - with warnings.catch_warnings(record=True) as warns: - if action == "Authorize_Users": - self.authorize_user(user) - elif action == "Remove_Users": - user_to_delete = User(user) - user_to_delete.delete(user) - else: - raise HTTPError(400, - reason="Unknown action: %s" % action) - except QiitaDBError as e: - self.write(action.upper() + " ERROR:
    " + str(e)) - return - msg = '; '.join([str(w.message) for w in warns]) - self.write(action + " completed successfully
    " + msg) + if created: + msg, msg_level = (( + "

    User Successfully Registered!

    Your user '%s'," + " provided through '%s', has been successfully registered" + " and activated. Welcome to Qiita!

    " + "

    Please direct any upcoming questions to " + "%s

    ") % ( + username, qiita_config.oidc[idp]['label'], + qiita_config.help_email, + qiita_config.help_email)), 'success' + else: + msg, msg_level = ( + ("Unable to create account. Please contact the qiita " + "developers at %s") % ( + qiita_config.help_email, + qiita_config.help_email)), 'danger' - @authenticated - @execute_as_transaction - def authorize_user(self, user): - # authorize user by verifying login manually using tue standard Qiita - # verify function - self.check_admin() - User.verify_code(user, User(user).info['user_verify_code'], "create") - return + # activate user + User.verify_code( + username, User(username).info['user_verify_code'], "create") + self.set_secure_cookie("user", username) + except QiitaDBDuplicateError: + msg, msg_level = "Email already registered as a user", 'info' -class AdminOIDCUserAuthorizationAjax(PortalEditBase): - @authenticated - @execute_as_transaction - def get(self): - # retrieving users with an unverified level - self.check_admin() - with qdb.sql_connection.TRN: - sql = """SELECT email,name,affiliation,address,phone - FROM qiita.qiita_user - WHERE user_level_id='5'""" - qdb.sql_connection.TRN.add(sql) - users = qdb.sql_connection.TRN.execute()[1:] - result = [] - # fetching information for each user - for list in users: - for user in list: - usermail = user[0] - user_unit = {} - user_unit['email'] = User(usermail).email - user_unit['name'] = User(usermail).info['name'] - user_unit['affiliation'] = User(usermail).info['affiliation'] - user_unit['address'] = User(usermail).info['address'] - user_unit['phone'] = User(usermail).info['phone'] - result.append(user_unit) - # returning information as JSON - self.write(json_encode(result)) + self.redirect(u"%s/?level=%s&message=%s" % ( + qiita_config.portal_dir, msg_level, url_escape(msg))) diff --git a/qiita_pet/templates/admin_user_authorization.html b/qiita_pet/templates/admin_user_authorization.html deleted file mode 100644 index 5bc4eea6c..000000000 --- a/qiita_pet/templates/admin_user_authorization.html +++ /dev/null @@ -1,118 +0,0 @@ -{% extends sitebase.html %} -{%block head%} - - -{%end%} -{%block content%} -
    -
    - | |
    - - - - - {% for head in headers %} - - {% end %} - - - - -
    Select{{head}}
    -
    - - - -
    -
    -
    -{% end %} diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py index 03c2a4fe7..11f299d15 100644 --- a/qiita_pet/webserver.py +++ b/qiita_pet/webserver.py @@ -21,8 +21,7 @@ MainHandler, NoPageHandler, IFrame) from qiita_pet.handlers.auth_handlers import ( AuthCreateHandler, AuthLoginHandler, AuthLogoutHandler, AuthVerifyHandler, - AuthLoginOIDCHandler, AdminOIDCUserAuthorization, - AdminOIDCUserAuthorizationAjax) + AuthLoginOIDCHandler) from qiita_pet.handlers.user_handlers import ( ChangeForgotPasswordHandler, ForgotPasswordHandler, UserProfileHandler, UserMessagesHander, UserJobs, PurgeUsersAJAXHandler, PurgeUsersHandler) @@ -245,10 +244,7 @@ def __init__(self): # through the settings file if len(qiita_config.oidc) > 0: handlers.extend([ - (r"/auth/login_OIDC/(.*)", AuthLoginOIDCHandler), - (r"/admin/user_authorization/", AdminOIDCUserAuthorization), - (r"/admin/user_authorizationAjax/", - AdminOIDCUserAuthorizationAjax), + (r"/auth/login_OIDC/(.*)", AuthLoginOIDCHandler) ]) else: # Qiita's traditional, internal user authentication From 3d6f718d26c97ba95647b717464ef4ada6f2626a Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 5 Jun 2024 16:52:35 +0200 Subject: [PATCH 53/61] using the well-known json dict instead of manually providing multiple API endpoints through the config file --- qiita_core/configuration_manager.py | 32 +++++++++++-------- .../tests/test_configuration_manager.py | 18 +++++++++-- qiita_pet/handlers/auth_handlers.py | 17 ++++++---- 3 files changed, 43 insertions(+), 24 deletions(-) diff --git a/qiita_core/configuration_manager.py b/qiita_core/configuration_manager.py index c7fbf2e40..995d04388 100644 --- a/qiita_core/configuration_manager.py +++ b/qiita_core/configuration_manager.py @@ -137,16 +137,18 @@ class ConfigurationManager(object): redirect_endpoint : str The internal Qiita endpoint the IdP shall redirect the user after logging in - authorize_url : str - The URL of the IdP to obtain a code (step 1) - accesstoken_url : str - The URL of the IdP to exchange the code from step 1 for an access - token (step 2) - userinfo_url : str - The URL of the IdP to obtain information about the user, like email, - username, ... + wellknown_uri : str + The URL of the well-known json document, specifying how API end points + like 'authorize', 'token' or 'userinfo' are defined. See e.g. + https://swagger.io/docs/specification/authentication/ + openid-connect-discovery/ label : str A speaking label for the Identity Provider + scope : str + The scope, i.e. fields about a user, which Qiita requests from the + Identity Provider, e.g. "profile email eduperson_orcid". + Will be automatically extended by the scope "openid", to enable the + "authorize_code" OIDC flow. Raises ------ @@ -436,14 +438,16 @@ def _get_oidc(self, config): if provider['redirect_endpoint'].endswith('/'): provider['redirect_endpoint'] = provider[ 'redirect_endpoint'][:-1] - provider['authorize_url'] = config.get( - section_name, 'AUTHORIZE_URL') - provider['accesstoken_url'] = config.get( - section_name, 'ACCESS_TOKEN_URL') - provider['userinfo_url'] = config.get( - section_name, 'USERINFO_URL') + provider['wellknown_uri'] = config.get( + section_name, 'WELLKNOWN_URI') provider['label'] = config.get(section_name, 'LABEL') if not provider['label']: # fallback, if no label is provided provider['label'] = section_name[len(PREFIX):] self.oidc[section_name[len(PREFIX):]] = provider + provider['scope'] = config.get( + section_name, 'SCOPE', fallback=None) + if not provider['scope']: + provider['scope'] = 'openid' + if 'openid' not in provider['scope']: + provider['scope'] = 'openid %s' % provider['scope'] diff --git a/qiita_core/tests/test_configuration_manager.py b/qiita_core/tests/test_configuration_manager.py index 3069e2982..4c57e1ab4 100644 --- a/qiita_core/tests/test_configuration_manager.py +++ b/qiita_core/tests/test_configuration_manager.py @@ -311,9 +311,7 @@ def test_get_oidc(self): obs._get_oidc(self.conf) self.assertEqual(obs.oidc['academicid']['client_id'], "foo") - self.assertTrue('gwdg.de' in obs.oidc['academicid']['authorize_url']) - self.assertTrue('gwdg.de' in obs.oidc['academicid']['accesstoken_url']) - self.assertTrue('gwdg.de' in obs.oidc['academicid']['userinfo_url']) + self.assertTrue('gwdg.de' in obs.oidc['academicid']['wellknown_uri']) self.assertEqual(obs.oidc['academicid']['label'], 'GWDG Academic Cloud') @@ -322,6 +320,20 @@ def test_get_oidc(self): obs._get_oidc(self.conf) self.assertEqual(obs.oidc['academicid']['label'], 'academicid') +<<<<<<< HEAD +======= + self.assertEqual(obs.oidc['academicid']['scope'], 'openid') + print(obs.oidc['academicid']['scope']) + # test fallback, if no scope is provided + self.conf.set(SECTION_NAME, 'SCOPE', '') + obs._get_oidc(self.conf) + self.assertEqual(obs.oidc['academicid']['scope'], 'openid') + + # test if scope will be automatically extended with 'openid' + self.conf.set(SECTION_NAME, 'SCOPE', 'email affiliation') + obs._get_oidc(self.conf) + self.assertTrue('openid' in obs.oidc['academicid']['scope'].split()) +>>>>>>> c9d413af (using the well-known json dict instead of manually providing multiple API endpoints through the config file) CONF = """ # ------------------------------ Main settings -------------------------------- diff --git a/qiita_pet/handlers/auth_handlers.py b/qiita_pet/handlers/auth_handlers.py index cd42d6763..ce6446277 100644 --- a/qiita_pet/handlers/auth_handlers.py +++ b/qiita_pet/handlers/auth_handlers.py @@ -9,6 +9,7 @@ import urllib.parse import os import warnings +import requests from tornado.escape import url_escape, json_encode, json_decode from tornado.auth import OAuth2Mixin @@ -268,12 +269,14 @@ async def post(self, login): if self.idp is None: self.render("index.html", message=msg, level='warning') - self._OAUTH_AUTHORIZE_URL = qiita_config.oidc[self.idp][ - 'authorize_url'] - self._OAUTH_ACCESS_TOKEN_URL = qiita_config.oidc[self.idp][ - 'accesstoken_url'] - self._OAUTH_USERINFO_URL = qiita_config.oidc[self.idp][ - 'userinfo_url'] + idp_config = requests.get(qiita_config.oidc[self.idp]['wellknown_uri'], + proxies={env: os.environ[env] + for env in os.environ + if env.lower().endswith('_proxy')}).json() + + self._OAUTH_AUTHORIZE_URL = idp_config['authorization_endpoint'] + self._OAUTH_ACCESS_TOKEN_URL = idp_config['token_endpoint'] + self._OAUTH_USERINFO_URL = idp_config['userinfo_endpoint'] if code: # step 2: we got a code and now want to exchange it for a user @@ -333,7 +336,7 @@ async def post(self, login): client_id=qiita_config.oidc[self.idp]['client_id'], client_secret=qiita_config.oidc[self.idp]['client_secret'], response_type='code', - scope=['openid'] + scope=[qiita_config.oidc[self.idp]['scope']] ) get = post # redirect will use GET method From 39570300c199ad448cf197e8cddc87a5e8b07b33 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 5 Jun 2024 17:00:47 +0200 Subject: [PATCH 54/61] flake8 --- qiita_pet/handlers/auth_handlers.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/qiita_pet/handlers/auth_handlers.py b/qiita_pet/handlers/auth_handlers.py index ce6446277..77210cd14 100644 --- a/qiita_pet/handlers/auth_handlers.py +++ b/qiita_pet/handlers/auth_handlers.py @@ -8,17 +8,15 @@ import urllib.parse import os -import warnings import requests from tornado.escape import url_escape, json_encode, json_decode from tornado.auth import OAuth2Mixin from tornado.curl_httpclient import CurlAsyncHTTPClient -from tornado.web import HTTPError, authenticated +from tornado.web import HTTPError from tornado.httpclient import HTTPClientError from qiita_pet.handlers.base_handlers import BaseHandler -from qiita_pet.handlers.portal import PortalEditBase from qiita_core.qiita_settings import qiita_config, r_client from qiita_core.util import execute_as_transaction from qiita_core.exceptions import (IncorrectPasswordError, IncorrectEmailError, @@ -28,7 +26,6 @@ from qiita_db.exceptions import (QiitaDBUnknownIDError, QiitaDBDuplicateError, QiitaDBError) from qiita_db.logger import LogEntry -import qiita_db as qdb # login code modified from https://gist.github.com/guillaumevincent/4771570 @@ -269,7 +266,8 @@ async def post(self, login): if self.idp is None: self.render("index.html", message=msg, level='warning') - idp_config = requests.get(qiita_config.oidc[self.idp]['wellknown_uri'], + idp_config = requests.get( + qiita_config.oidc[self.idp]['wellknown_uri'], proxies={env: os.environ[env] for env in os.environ if env.lower().endswith('_proxy')}).json() From 648f2f91ce4361ca6817e4c18af911ab2e138310 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 5 Jun 2024 17:02:05 +0200 Subject: [PATCH 55/61] flake8 --- qiita_core/tests/test_configuration_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qiita_core/tests/test_configuration_manager.py b/qiita_core/tests/test_configuration_manager.py index 4c57e1ab4..5ea172ac2 100644 --- a/qiita_core/tests/test_configuration_manager.py +++ b/qiita_core/tests/test_configuration_manager.py @@ -335,6 +335,7 @@ def test_get_oidc(self): self.assertTrue('openid' in obs.oidc['academicid']['scope'].split()) >>>>>>> c9d413af (using the well-known json dict instead of manually providing multiple API endpoints through the config file) + CONF = """ # ------------------------------ Main settings -------------------------------- [main] From 73f92b9466faf45de1a2ab4e702b459892b5add1 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 5 Jun 2024 17:37:07 +0200 Subject: [PATCH 56/61] add ability to display OIDC logos --- qiita_core/configuration_manager.py | 6 ++++++ qiita_core/tests/test_configuration_manager.py | 8 +++++++- qiita_pet/templates/sitebase.html | 6 +++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/qiita_core/configuration_manager.py b/qiita_core/configuration_manager.py index 995d04388..53d1c16c2 100644 --- a/qiita_core/configuration_manager.py +++ b/qiita_core/configuration_manager.py @@ -149,6 +149,10 @@ class ConfigurationManager(object): Identity Provider, e.g. "profile email eduperson_orcid". Will be automatically extended by the scope "openid", to enable the "authorize_code" OIDC flow. + logo : str + Optional. Name of a file in qiita_pet/static/img that shall be + displayed for login through Service Provider, instead of a plain + button Raises ------ @@ -451,3 +455,5 @@ def _get_oidc(self, config): provider['scope'] = 'openid' if 'openid' not in provider['scope']: provider['scope'] = 'openid %s' % provider['scope'] + provider['logo'] = config.get( + section_name, 'LOGO', fallback=None) diff --git a/qiita_core/tests/test_configuration_manager.py b/qiita_core/tests/test_configuration_manager.py index 5ea172ac2..362df7343 100644 --- a/qiita_core/tests/test_configuration_manager.py +++ b/qiita_core/tests/test_configuration_manager.py @@ -323,7 +323,6 @@ def test_get_oidc(self): <<<<<<< HEAD ======= self.assertEqual(obs.oidc['academicid']['scope'], 'openid') - print(obs.oidc['academicid']['scope']) # test fallback, if no scope is provided self.conf.set(SECTION_NAME, 'SCOPE', '') obs._get_oidc(self.conf) @@ -335,6 +334,13 @@ def test_get_oidc(self): self.assertTrue('openid' in obs.oidc['academicid']['scope'].split()) >>>>>>> c9d413af (using the well-known json dict instead of manually providing multiple API endpoints through the config file) + self.assertEqual(obs.oidc['academicid']['logo'], + 'oidc_lifescienceAAI.png') + # test fallback, if no scope is provided + self.conf.set(SECTION_NAME, 'logo', '') + obs._get_oidc(self.conf) + self.assertEqual(obs.oidc['academicid']['logo'], None) + CONF = """ # ------------------------------ Main settings -------------------------------- diff --git a/qiita_pet/templates/sitebase.html b/qiita_pet/templates/sitebase.html index 4a36f8ea4..bec358f48 100644 --- a/qiita_pet/templates/sitebase.html +++ b/qiita_pet/templates/sitebase.html @@ -644,7 +644,11 @@

    Login

    Please choose your login provider!

    {% for idp in qiita_config.oidc %} - {{qiita_config.oidc[idp]['label']}} + {% if qiita_config.oidc[idp]['logo'] %} + + {% else %} + {{qiita_config.oidc[idp]['label']}} + {% end %} {% end %}
    From 0dc243d68f5140e413eb469a05c3f21493a2960b Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 5 Jun 2024 17:38:06 +0200 Subject: [PATCH 57/61] add OIDC logo --- qiita_pet/static/img/oidc_lifescienceAAI.png | Bin 0 -> 8560 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 qiita_pet/static/img/oidc_lifescienceAAI.png diff --git a/qiita_pet/static/img/oidc_lifescienceAAI.png b/qiita_pet/static/img/oidc_lifescienceAAI.png new file mode 100644 index 0000000000000000000000000000000000000000..ab72fa293def5199720b4a00bac0d824aad6d26a GIT binary patch literal 8560 zcmbW7+RhM|V;8cI5bA*F|Ic*gJb`wN~I zCw88-_O;L1Yk%$&^;KCG1C<074h{}O?u(Qf92|Tf?A{y&33kS3=6nOYAek!2O2Og6 z4Z__$z|k#`W0G2W2OwJU1D!F_;}lM>hP zUOCD3^473ec@^7N996`5Ga5!)G^>EkWU61qQ~$ldG=QzLrsGg=q50_0F33>(wk6u` z3?EP9_s{R2@eIQeix4Fg-W&us9qd275WSK6LZ4**hCAue&CpF8Kt`J<0q^G}wo_Z6=*<}8+Ctk_J`IG=kks^daq$4`?OS}8CL&K!g^*BzPpj|{1 z3Tns)6o`T+PZjarToY3J;s}ifsds4RrzM7^{=Ca|?7H-UAFwJC-e%mrUXuPRp4c&L z!}V>6|12X!kUS&=k$s?ML<#vIl|lp}o3@_$rvKASPck05{lNR0TCU($_5mdX+r?bA z-!0v@Uz!rG3_v!FL1vFbZa>pDOFx7I8y)H-7W)JThd)xjp{C<5f8e;_Vvu6^|Lc@5 z^YGhNie%vzyLsRDZ;7bI|NI+%l+vh$L7ogzs`qC9l3z;87s^d>#IfJzv(QNVYxe{D zpI;#;`LNUf5)zAjjPxi@!053j=46}}tv?&NH5_Lbm#;7a`&8 zQ^VbK^8?(vNURyM=yWI{)2OpNBoX$P8N}TnP~_uUSM0Cr?ohAGc@g#@M`!gCsi=CG zB~mCfF?#a@2to$-RWo`(zi3sH6L>6sM~@%9|0~@^cHh(Ve%cTq@5!qxo9Levw0S0g zl~Kih(9-njtwi?r!rx%~6(|97Cg?{8wv0DkvLPlLG_MZTzG8mIk3p&q0kdbjgI5ZE zeSkIOg9PK-pCLc%kfHNad2u=K0y~k#^kxm5Z|ifNnl6N&UH1aB#3GDD@7Kqf;d=pc0lx2JghB1UFGTBs( z6ceG2&AXp4n;Y?CX9uaNKuM1pIm-ucCE=pW`s2#!ORPNCA0JjK=XBd#@AJF2w;hBZ zmT+n5=)4YHQiUMfraC~%v9(Pn%TiQYA`??#gUmNp8KFU-OWW!sz5W4%G9&hB#K>SrH_#FI-R%B8IxMeO_n3>{X@L9DA|~X!d!G zja8J%_5Qd$tWqEwPcfOI3-qQs>wJ`HJTf&i^L~!QTJe~oxL?IZC;r$px&L%n$x&~+ zdplV}%4LdoyK(EB>djwNt2#h~t_?ai*`$;~OpT}wnCwOy#bA=@S+_XZz4kr#7LGIW z&yK?NZbLT$BO}(pNxxyd>|rRbyMSBEK=oRp9u{I{W!;?QH?mj-usLi1@;Gyea{~q% zEEMkgDIKYV9$WJO1in-VMNe$%qP zz{mB+BRgKqw^m&Ws`Cy4$`-ocP0bZJA4n?3@r1J>!BF3FCtMuxOXs4EF7B^R!}B)# zisy#JOdTSdF1Xomx^!k!VR^EQ)HD%ffca;r3SW)J9>m4;ZwS#>y5?=QPgn%o2lBv3fr_Da1wajcDn`)XK!WW>Cv%HScYP0k9t zvflN=S7x|lW#_^L$-&#@-M(+-#R#m5h{MQ`BRq@DwwY#7v{zMC1wEd1<2v7O=mvaI zmZPgLjXOLfkynUb1~}M#4$at`dQ4GQ7KnR!MsCsr6t$+MbBm(h0ggf0N;y1FyRd;l}rFh8_Q-A=(sE;CTV7$fW>Jp2WvGOwf;*{2Fsn?X|F zI7Sk$^#eTLJUMz&q|+%xg?E~P@9HG)Nbn5!OoAzESZOgO>L;88%2xFCeRzLVQBiRh zc<(jwa^Ys6Tnc^$-aoJiUr$`7DbjcEUoYs_Zn-v#j#yqG5`GNaYNMsUv9DM3@gXMV ztJ)S5tF?T7@UYHEO-r*bU`;WzeB|g#>m6mtjh?aog~|joPBvGmIWinAuNFU$aJ?%a zE-q60R(xLE#N((AKvIK(K=m{g6li*~!W%fQ!>YRGdSn<&5ui4{;bK6RRH|rk_d^v3 z?Q8sHyriGWneZO8^{9?52a!yRqa#;>_kk38E6>f8jjyilPb=Z6m@wPo3(PTb`+$ zkVepwXsOW%Cmfn541?OhwJ3bF+}tsdlz~}GOGK~3hM%UK)DM9^;JaI-Sees~*Zl8N z&@b?D78k$E*Zjbe!)q_bpck^#ak^x+FwQ1j8~$_K^s(-7bk^!5`q(c$XsuT?ESi3P zb{*T9R!a%;@f4${&bv#VBC_{cqiN_eAiui;Ih5wqUj4BL$!R zr~b%CtXF{b%#&p*hY_&n%(L8vIE&ZQe?U0iIN)ADIdX@}L8bHPg6wOdcgM)&#WTsH z0SVHFxdKKm*iDj&>YbgM^|IMeAWiM5oBH0ufn!*v zn&OkFc!U*yB>ors3%15_c+z!ed7&;B^t%$ZPo%L_T<|kg_6UZHqoem5PU%d9Mzi@k z`AMY)1MMDcw_=Vft-%);4Km_&>yG?lJ5^U5;Bs{iX-%48%Cd?I)0u*M#hj0t%%$F8 z!IxK}$)aNO7k}9a6_`byv?}Xsw6<4WvApcPX075<VgD{I`g4~8jlc(a4`orsCsg$aK})FZJfSaws5;P308(A{LdK2H9Wo3;AM*w zu$9+`@?H+n0f#y-Co0}~w#S}|3fCcGE3dyqp~zNAOO2)AkE?)Pz6ZbCn(V;-dflpp zx||=5^?YIiR-mlfy2W|9T|6rESf(~Xl@m{sMgKNO(y^$YP6I}X`ukXI$`nqe*m!O3 zH-1+H;DXf?5>Gas2Fa%#o^$bfWLfTl=O;%(JHy%7RkBjgNFB*-tNl5gyQX$KgI9o7 zgB&^uPgq)7+Rp%IMpPQvpEvrJk2lqEFB=&h^G^4W;rI)o6Y4(#uTM;q?6sAZCZ|9F zH7;os{r!mSy4F5YXj#PT>h{)pG{7%nJ#G?NyC%r2M2abFO!#gP-w3mbcBLHzbhV*y z_&LYs>}#-XVL{>T$`ADDNlm2wWuxgF^83~04*I^@J1lOB#%Xl{^MvEi!D@>bLlfs4 zuNR%y;{A;>S3D%{`mT=`IG(j-~QuHeCZ$cI7DUCdM#A1`;`Xo(SPZn|81` zF*30C(m*77jD1ouXHoH;s>!pkG{R!>XVWB#+ic}lC*-vzw62;+z4pEvf zqaKY74-bajG)}UGCDRVoS=zobGE1v34dXS|o^Ab*olK`LoN!#26)FnrORI-#j9)cM z&y{>ux4k!VZ7gk7Xsp|vfSr4|z2|&1UM1nma}!XV`Py4wwU(cYcN7ER+41a@V-n#-a$2C;#o!+BID*nDhb!ne)@O62) zjktiazrXOz3>$Lw91jOCZ!aG%|8C;JCk&rK}3C?(`Cp`B@UKdB`@C}Yu3uEi!BQNMF*D)@nS0b-0JR| zFu&P=Hl$3o%HChk&MFj&&lrwu-GpjcUOOwiOi*7}^UZPO8#W#rWqH<>$P} zg2$db)w?%wuw|iYW=85ws)4IOh3}5!{(0;DVWn_Ya4Z#W1q4;1|Otgqlky znLsktriz-gx8R0~<+KMq9nq5MqK+9>wf?sm;bTwhwIqwZ)Fjgp5#9&^1pEh5!M(PZ zMkx!2=Dk-0so;;Ppv=cL4MxbSleYyckRO|3GR(eGYK?H$5dg_$}6PsFJU@ zp`8Ouh#4O;KX&u{VbGP0gTslq_!Z^s2SGvCFzn5~7XJ3*;ec~ zhcnHiX|^wt!YWO^#5mO-TZEfjk-M}_dh&ad%1I=Z*guB+aeqAPJnaEGBX)t$EZnjQ z7N&FwoP$)-(lMrPBDlBn{LrGy;5)LRwVlvMHYg|<+NLbavP?Ch+WVXr)o=$ketr^& zTt98enz8=y>qorGx7oIR{*Kef-SMRsr`odY?*05qL+2FfK}$78m*zqrjYwS6Q1#&0 z_948?#8lL^wG>OLVgwpl&5(T=VEfeMH4{C`mMF^~M)~#N`1Ga@choGs&Cw@`w7{Md zzSdr(N;1wCvXqlZLuKP}H}!?ZdzlQ!GsN4{JNm`Qk11MVplwg&Ywzk}i;A;byXx1e zQ)_+h*U($=WX{uib;{9PDVB=JbG(@=2fa$y54spa-Z`sE?xUFWr~8`@sq+DvE3T1E zQ|s83E>Q`?{fnZ_03k(r&dom_S`;W4XI8q|QQ4j%e`$0pMSL?S4J)M=+z1h}%z{&A zH^=j#{>DU$g=Yb9ruQ3?kl0b!hS3{M$ZG!3-nN-<3vIqIz9j? zY|&aeZl??~kPC#9V3$}5Z)F<%VxUhhgb^`FoR)kijkTEOz~!_!-hE?yNMMI9WomS#6p|v9(R6cVySNn*CQC z9bQLYd#-{A&fQKr&(791)YmA#uSjkmcag1T*&{dTC7m$`1!4kYf`7l0DBSG@NHw+2`#z>UbR@|bE{W^rNG{29VLBa!_^^)T6 z-=uOl+Az))cD&|H^o2a-k?r>cBUB^Z*4(1tOXtgCy+X#&(9ozad;}kl&uM9wG7S0M zh8PNbsl(!YK1iFLmsukXm395#cu4H##zEhSZTiLC7cCijo*BZS(^8y~W_XydCc>wV zG?vWR#eVB*Q;_(!+Kt-m)aB4x@I_1zbLBBOwuX4^5fAr?j{H}q!HZXN9aY$);^Qsl zj#ofnZ%!VW)$M6^;G&hqwGEzBhjGyBsHfN23UT68;?1Y={6`VxRiZOvg`JZI+4frzMlw04W|sX?GpuAqoIRV=#DtBJdS&hhS#8^nN9V zhvIGVXr~5HOf~lH&{JN9v+I!eohKoqUvY#TEk5%z4lmbjyapoi3B(5@JmoId+P&aH z>vIE6rA1%N2`Ga;#=OI&4}bAHY2rSZXK7ycfNfd|{FM?0fXKv-uH3@K5FCsQMVl|Z zgR}2vV)d=|V<$-z8kEwmH_gpkX(mV9{MLkXe6RNTCz&r@)%cT5N;c7^Ba@L%L2k}g z5mBXT9z9Agj0lt(BzMJn{pI46F@5Hl!TI!FOCtQ>4u5eHOdmRek<-mjx##Clmnf&c zfaj#vwv>cy1upC9Z{0`y4l6&_9fxgCmi*Ir?E=rw()r$Zy;P$>ZXDV)w6w4(bFqCm zJPQ+TQVmBHUVX)lPPg78<>s^}(e+n*^Q5(oD%m7Z=dC4IZ6j_t%2qHGBe=bI+E}J& z`KH^Wz3|Uu?jljF?rv?MUy8pMo@hQdG8+5i)7b(-VA9ix{`L+!joIpY2?5)Qr?m+4 zJd7w33{oA_mKIxJt1aOxd=&Do} zJkap9x=u_5pqT6@6bYsC{)rY39_L$TLX{w)3axSTQK4}<7HJQGdZeeG%Cy$`IlE!J zbn5Ibi-^7la_Gj4?0+EnisyV78H6w18nAvY{n_>hcbM!$WyM%|Y0hcOL{_a8#I@OY za^L@1n#JfKfhNQ+bJFtPbGuh3bHyUt+eZ`=iWyN;4PBanbmn|QuC@;veDkMO86>=<%NkHVdD@Vvi!w|}=@Qv(7K zz8CI3z1x=Mw4TQHF^R$Ur0CQnti`x`x7noRDJ(xcvAnps<+ZN4!67JU1mT~T850)d zze3G{zPJzqHFqpw(NRrQ{8Q{rXQuz_v3t)AW}rHrZ+_J%O@b|^gOIu9P01_VYavhf z-4-{%e&bKcb4qgZ$AE{J!11fU8=*4{e7WTTn?1cnrS~d59nKKpz&jJo0pIY{7;Q$0 zrR6JCL!i)zL^B}>ymn6bnmad9^=Z>0v*i-(A2j9V;usQ4EAmO5o_#8mn2y2{tNCMF zW~sv3D`cI-DwWZ-|CSs*qq7~AnKW&xsAqa#U{)9s;h+jqQ zIt|%LDDypSh!PQg`sc+YlwqLaij#Sw7Wy&IYCroDsU^y|@&x$t1M1&aWI>NiWe|K^GEJe7#g$Fj zMTcX;`(3mGjIPfs#=m>+MLsl=KdvH7uwV1WQ(OQFHg_De{m2zkS1pmcYNBv9M;qMaR@F%3=fZ_BfE{@2%4TA}s~ z45iK+_v%k60++`w{w?Y6^6P z7V?r;B-*)(*q6%zEv?`C?y-X*Xiez*mq|jj4?jkvo0vNfl^;B(?-G4YQaCJHt9~+Bq50KK3 zE^aT$__bvUZ~{v>INTu~gT@XgnMz|#t`Et4bYU}{Jv}-md|1EDyD(Hv-()D^=G}4| z9|9$w-(B<2_!8Vj%S=alo z_duY-&aQ*qV*OCOSY)qc0|G(Fs5<*rrhqY$M&KeyxSPWKlj}hfi%#bQa!9(VQrK7P zu0AGqOAGzhz{*e-KA}77Iz)Za$4!`0iXN|-rcRNDP2v$}GQ-Se`_1St+CBS&9zOk0 zlkr=$YevVl!@QM3ZthiyZqE1OsjGt(nk-+GEKLtNEqihl~CZ5lzdD~nvj!nDY@Cs zh^oP}*kX$k3VUbB))sZ(%fT@Fz2o6LOLF5B#V|}5hS3F~?2U?;7G&K!vzjD3z^A(= z@*GGwA5Hw3r;Ca&&w>2WY9J3SXZ%DlQJU`ImXod}$*mNka+jm!3O%wavwwyImHInM zl{!F4#*|gy%IS%w=5B^vV2~6aRNuFc9ohe@>r&y|cj-qHX?*mi9v7|zQ>2)~BZal? z?{OY=>n@+76-6u&U$Rhxj^77Pbr&!i_*^$zTTkTPI!xb+nGP_Xwq6=d>;W(sxk=*I zJYmtF5y$KO?moy@wlv>c)2|gTX~FmkjCbdb8c8FnD-6Hp}_WoaluH0XBdElvp;R$ z>{i2uON8!y!otTx$j*0Hys^aMqKC>_L{`QHQ`6^O^<{%9$Gjx}D3gxh))c`{mGP<}BD0TK3dMCg6;>da

    Vdyem( zUWzT8AJVP3P5O$oWFrB|i8Qj8PnXO7mnYggJ5_mAO-*rAA;Bj75pJ`irJTN3qXR=j z5rfY^Vq&(R(C9CBkEeDK4PR$0$6d%OG7$^Sc~q_H2hFygFfbd;G`fh(!sgpJ%6KnI z4H$5@Fo4VYrxyB4SoWTe4{uuL#miSyeMI%bjVy`C5c*BO<-IYaLo<_;oUskFA~C}yYW@fRAo?}vlBdia`JI4 zYiUIVNz8o6yeS>MU_?`syYKCR!kK8m0XPV5W`^q`A(H994)1-R8FhpacdCQ?G`Qko z`2&I_+m&QOfT6=qFS#;EiSt4tLYNXpL36O>DIu!nb-jZV`2;rtyzHxo#X9d!jPh<- zFxHyaM~g+2s6$}&SgFH?Kcj<5@WS=ybd8AHDInaB8Wd}gtr{L&l_Fc$?YH7Tbj<4} zxW1K*PL@g*{FsyQIuy74-G|#bxn_PjhJpK5JnSoqXg+HQ+#iW7&e4u z|6|lrD2V^h1QkyB*F_Ka(I%RISof1svoazXTy#O*Djm{)v^G;Dt{Ig7=C$heDA9j< zfaM?Hrp8l>jJzaednkSLZQP|54!7qhz#SA%X!Z`qVH1ae@~u9lYOkdoFsf|;mc5gD zp6x!3=$w^e{`UwXKSSr+Q<<0?xKFe(05Vzl1f&F$1zq8{rWmH5U`|jym7n*8O(vWP z()#(059`e4onJ*I!E@Z9D#KJ#F#_1{0q6hz1wd;4Rb=oFkGt7+aysbKyuW(GF#qcE z1qIdyHMsseW?jCr$(p4N{eNUUwUeEwWx3P_c_z3XnK!$Nh&aju|f*_F(uM=!55P3xbHX7a&_7@3j?IDJSVcl?_{ ziLFKdarS)pSA7`JFY1 Date: Wed, 5 Jun 2024 21:24:46 +0200 Subject: [PATCH 58/61] fixing config manager tests --- qiita_core/tests/test_configuration_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_core/tests/test_configuration_manager.py b/qiita_core/tests/test_configuration_manager.py index 362df7343..5bb2fe7dd 100644 --- a/qiita_core/tests/test_configuration_manager.py +++ b/qiita_core/tests/test_configuration_manager.py @@ -337,7 +337,7 @@ def test_get_oidc(self): self.assertEqual(obs.oidc['academicid']['logo'], 'oidc_lifescienceAAI.png') # test fallback, if no scope is provided - self.conf.set(SECTION_NAME, 'logo', '') + self.conf.remove_option(SECTION_NAME, 'LOGO') obs._get_oidc(self.conf) self.assertEqual(obs.oidc['academicid']['logo'], None) From 0a29ac2ceba2e72ed833f65b2ac3bd46269bd363 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sun, 9 Mar 2025 16:31:00 +0100 Subject: [PATCH 59/61] create neccessary "mountpoints" (#3462) * add function that creates neccessary "mountpoints" * code style * addressing Antonio's suggestion to use get_mountpoint * raise error if src dir is not present at expected location * adding mountpoint directory FASTQ --- qiita_db/environment_manager.py | 34 +++++++++++++++++++ .../support_files/test_data/FASTQ/blank.txt | 1 + 2 files changed, 35 insertions(+) create mode 100644 qiita_db/support_files/test_data/FASTQ/blank.txt diff --git a/qiita_db/environment_manager.py b/qiita_db/environment_manager.py index 95fa8c468..f8b97aa0b 100644 --- a/qiita_db/environment_manager.py +++ b/qiita_db/environment_manager.py @@ -7,6 +7,7 @@ # ----------------------------------------------------------------------------- from os.path import abspath, dirname, join, exists, basename, splitext +from shutil import copytree from functools import partial from os import mkdir import gzip @@ -127,6 +128,36 @@ def _download_reference_files(): _insert_processed_params(ref) +def create_mountpoints(): + r"""In a fresh qiita setup, sub-directories under + qiita_config.base_data_dir might not yet exist. To avoid failing in + later steps, they are created here. + """ + with qdb.sql_connection.TRN: + sql = """SELECT DISTINCT mountpoint FROM qiita.data_directory + WHERE active = TRUE""" + qdb.sql_connection.TRN.add(sql) + created_subdirs = [] + for mountpoint in qdb.sql_connection.TRN.execute_fetchflatten(): + for (ddid, subdir) in qdb.util.get_mountpoint(mountpoint, + retrieve_all=True): + if not exists(join(qiita_config.base_data_dir, subdir)): + if qiita_config.test_environment: + # if in test mode, we want to potentially fill the + # new directory with according test data + copytree(get_support_file('test_data', subdir), + join(qiita_config.base_data_dir, subdir)) + else: + # in production mode, an empty directory is created + mkdir(join(qiita_config.base_data_dir, subdir)) + created_subdirs.append(subdir) + + if len(created_subdirs) > 0: + print("Created %i sub-directories as 'mount points':\n%s" + % (len(created_subdirs), + ''.join(map(lambda x: ' - %s\n' % x, created_subdirs)))) + + def make_environment(load_ontologies, download_reference, add_demo_user): r"""Creates the new environment specified in the configuration @@ -397,6 +428,9 @@ def patch(patches_dir=PATCHES_DIR, verbose=False, test=False): with qdb.sql_connection.TRN: _populate_test_db() + # create mountpoints as subdirectories in BASE_DATA_DIR + create_mountpoints() + patch_update_sql = "UPDATE settings SET current_patch = %s" for sql_patch_fp in sql_patch_files[next_patch_index:]: sql_patch_filename = basename(sql_patch_fp) diff --git a/qiita_db/support_files/test_data/FASTQ/blank.txt b/qiita_db/support_files/test_data/FASTQ/blank.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/qiita_db/support_files/test_data/FASTQ/blank.txt @@ -0,0 +1 @@ + From 5ab3ebbca79e66441f2df2b73d879ec136450048 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 11 Mar 2025 16:27:31 +0100 Subject: [PATCH 60/61] fix sub-directory path (#3464) When I executed this code, I found that get_mount_point() returns complete paths, e.g. `/root/data/job` instead of "just" the sub-directory, i.e. `job`. Thus `get_support_file` returns the dst directory, instead of the src directory :-/ --- qiita_db/environment_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_db/environment_manager.py b/qiita_db/environment_manager.py index f8b97aa0b..12226c0af 100644 --- a/qiita_db/environment_manager.py +++ b/qiita_db/environment_manager.py @@ -145,7 +145,7 @@ def create_mountpoints(): if qiita_config.test_environment: # if in test mode, we want to potentially fill the # new directory with according test data - copytree(get_support_file('test_data', subdir), + copytree(get_support_file('test_data', mountpoint), join(qiita_config.base_data_dir, subdir)) else: # in production mode, an empty directory is created From d0e03de39dba9998765dd115037675fa8225cacc Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 12 Mar 2025 17:30:44 +0100 Subject: [PATCH 61/61] Fix multiple validation jobs (#3465) * Update CHANGELOG.md * multiple validation jobs should be submitted as lead and dependent jobs, but the later must also made known by the DB --------- Co-authored-by: Antonio Gonzalez --- qiita_db/processing_job.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/qiita_db/processing_job.py b/qiita_db/processing_job.py index 27192bab7..c773ff927 100644 --- a/qiita_db/processing_job.py +++ b/qiita_db/processing_job.py @@ -1105,13 +1105,23 @@ def submit(self, parent_job_id=None, dependent_jobs_list=None): # organized into n 'queues' or 'chains', and # will all run simultaneously. for dependent in dependent_jobs_list: + # register dependent job as queued to make qiita + # aware of this child process + dependent._set_status('queued') + + dep_software = dependent.command.software + dep_job_dir = join(qdb.util.get_work_base_dir(), + dependent.id) p = Process(target=launcher['function'], - args=(plugin_env_script, - plugin_start_script, + args=(dep_software.environment_script, + dep_software.start_script, url, - self.id, - job_dir)) + dependent.id, + dep_job_dir)) p.start() + # assign the child process ID as external id to + # the dependent + dependent.external_id = p.pid else: error = ("execute_in_process must be defined", "as either true or false")