From 717870c7cc7ce5daf443ce7216a37903a3b20439 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Tue, 29 Oct 2024 15:48:51 +0000 Subject: [PATCH 1/5] Squashed all JWT auth Add aspell Enable jwt-cpp in fasttest Add test + some minor improvements reduce unneeded possible clash points fix parsing create user identified with jwt refactor + fix not lowercase update test fix typo in docs fix logical_error some refactor fix alg in jwks fix jwks fix user auth method not being checked update docs better exception on no sub claim throw exception if algo not specified in jwk Support access token authorization of existing users Also possible to filter users by e-mail using regex fix token accessstorage Add Azure token processor, move JWKS logic to separate file remove docs that will be obsolete in future remove redundant --- contrib/jwt-cpp-cmake/CMakeLists.txt | 7 +- .../external-authenticators/index.md | 3 +- .../operations/external-authenticators/jwt.md | 219 ++++++++++ src/Access/AccessControl.cpp | 18 + src/Access/AccessControl.h | 4 + src/Access/AccessTokenProcessor.cpp | 214 ++++++++++ src/Access/AccessTokenProcessor.h | 112 +++++ src/Access/Authentication.cpp | 18 +- src/Access/AuthenticationData.cpp | 22 +- src/Access/AuthenticationData.h | 4 + src/Access/Common/JWKSProvider.cpp | 92 ++++ src/Access/Common/JWKSProvider.h | 67 +++ src/Access/Credentials.cpp | 35 +- src/Access/Credentials.h | 39 ++ src/Access/ExternalAuthenticators.cpp | 138 +++++- src/Access/ExternalAuthenticators.h | 12 + src/Access/IAccessStorage.cpp | 1 + src/Access/JWTValidator.cpp | 369 ++++++++++++++++ src/Access/JWTValidator.h | 69 +++ src/Access/MultipleAccessStorage.cpp | 3 + src/Access/TokenAccessStorage.cpp | 398 ++++++++++++++++++ src/Access/TokenAccessStorage.h | 79 ++++ src/Access/UsersConfigAccessStorage.cpp | 12 +- src/CMakeLists.txt | 1 + src/Parsers/Access/ASTAuthenticationData.cpp | 7 +- src/Parsers/Access/ASTCreateUserQuery.h | 4 +- src/Parsers/Access/ParserCreateUserQuery.cpp | 15 + src/Parsers/Access/ParserCreateUserQuery.h | 4 +- src/Parsers/CommonParsers.h | 1 + src/Server/HTTP/authenticateUserByHTTP.cpp | 22 +- src/Server/TCPHandler.cpp | 19 + src/Server/TCPHandler.h | 1 + tests/docker_scripts/fasttest_runner.sh | 2 +- tests/integration/test_jwt_auth/__init__.py | 0 .../test_jwt_auth/configs/users.xml | 15 + .../test_jwt_auth/configs/validators.xml | 24 ++ .../helpers/generate_private_key.py | 21 + .../test_jwt_auth/helpers/jwt_jwk.py | 113 +++++ .../helpers/jwt_static_secret.py | 43 ++ .../test_jwt_auth/helpers/private_key_1 | 27 ++ .../test_jwt_auth/helpers/private_key_2 | 27 ++ .../test_jwt_auth/jwks_server/server.py | 33 ++ tests/integration/test_jwt_auth/test.py | 101 +++++ .../aspell-ignore/en/aspell-dict.txt | 7 + 44 files changed, 2399 insertions(+), 23 deletions(-) create mode 100644 docs/en/operations/external-authenticators/jwt.md create mode 100644 src/Access/AccessTokenProcessor.cpp create mode 100644 src/Access/AccessTokenProcessor.h create mode 100644 src/Access/Common/JWKSProvider.cpp create mode 100644 src/Access/Common/JWKSProvider.h create mode 100644 src/Access/JWTValidator.cpp create mode 100644 src/Access/JWTValidator.h create mode 100644 src/Access/TokenAccessStorage.cpp create mode 100644 src/Access/TokenAccessStorage.h create mode 100644 tests/integration/test_jwt_auth/__init__.py create mode 100644 tests/integration/test_jwt_auth/configs/users.xml create mode 100644 tests/integration/test_jwt_auth/configs/validators.xml create mode 100644 tests/integration/test_jwt_auth/helpers/generate_private_key.py create mode 100644 tests/integration/test_jwt_auth/helpers/jwt_jwk.py create mode 100644 tests/integration/test_jwt_auth/helpers/jwt_static_secret.py create mode 100644 tests/integration/test_jwt_auth/helpers/private_key_1 create mode 100644 tests/integration/test_jwt_auth/helpers/private_key_2 create mode 100644 tests/integration/test_jwt_auth/jwks_server/server.py create mode 100644 tests/integration/test_jwt_auth/test.py diff --git a/contrib/jwt-cpp-cmake/CMakeLists.txt b/contrib/jwt-cpp-cmake/CMakeLists.txt index 4cb8716bc68f..606c13d29de2 100644 --- a/contrib/jwt-cpp-cmake/CMakeLists.txt +++ b/contrib/jwt-cpp-cmake/CMakeLists.txt @@ -1,7 +1,4 @@ -set(ENABLE_JWT_CPP_DEFAULT OFF) -if(ENABLE_LIBRARIES AND CLICKHOUSE_CLOUD) - set(ENABLE_JWT_CPP_DEFAULT ON) -endif() +set(ENABLE_JWT_CPP_DEFAULT ON) option(ENABLE_JWT_CPP "Enable jwt-cpp library" ${ENABLE_JWT_CPP_DEFAULT}) @@ -20,4 +17,4 @@ set (JWT_CPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/jwt-cpp/include") add_library (_jwt-cpp INTERFACE) target_include_directories(_jwt-cpp SYSTEM BEFORE INTERFACE ${JWT_CPP_INCLUDE_DIR}) -add_library(ch_contrib::jwt-cpp ALIAS _jwt-cpp) +add_library(ch_contrib::jwt-cpp ALIAS _jwt-cpp) \ No newline at end of file diff --git a/docs/en/operations/external-authenticators/index.md b/docs/en/operations/external-authenticators/index.md index 568ecf0383fd..346eadd9a323 100644 --- a/docs/en/operations/external-authenticators/index.md +++ b/docs/en/operations/external-authenticators/index.md @@ -16,4 +16,5 @@ The following external authenticators and directories are supported: - [LDAP](./ldap.md#external-authenticators-ldap) [Authenticator](./ldap.md#ldap-external-authenticator) and [Directory](./ldap.md#ldap-external-user-directory) - Kerberos [Authenticator](./kerberos.md#external-authenticators-kerberos) - [SSL X.509 authentication](./ssl-x509.md#ssl-external-authentication) -- HTTP [Authenticator](./http.md) \ No newline at end of file +- HTTP [Authenticator](./http.md) +- JWT [Authenticator](./jwt.md) diff --git a/docs/en/operations/external-authenticators/jwt.md b/docs/en/operations/external-authenticators/jwt.md new file mode 100644 index 000000000000..fbc36f8399f7 --- /dev/null +++ b/docs/en/operations/external-authenticators/jwt.md @@ -0,0 +1,219 @@ +--- +slug: /en/operations/external-authenticators/jwt +--- +# JWT +import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_no_roadmap.md'; + + + +Existing and properly configured ClickHouse users can be authenticated via JWT. + +Currently, JWT can only be used as an external authenticator for existing users, which are defined in `users.xml` or in local access control paths. +The username will be extracted from the JWT after validating the token expiration and against the signature. Signature can be validated by: +- static public key +- static JWKS +- received from the JWKS servers + +It is mandatory for a JWT tot indicate the name of the ClickHouse user under `"sub"` claim, otherwise it will not be accepted. + +A JWT may additionally be verified by checking the JWT payload. +In this case, the occurrence of specified claims from the user settings in the JWT payload is checked. +See [Enabling JWT authentication in `users.xml`](#enabling-jwt-auth-in-users-xml) + +To use JWT authentication, JWT validators must be configured in ClickHouse config. + + +## Enabling JWT validators in ClickHouse {#enabling-jwt-validators-in-clickhouse} + +To enable JWT validators, add `token_validators` section in `config.xml`. This section may contain several JWT verifiers, minimum is 1. + +### Verifying JWT signature using static key {$verifying-jwt-signature-using-static-key} + +**Example** +```xml + + + + + HS256 + my_static_secret + + + +``` + +#### Parameters: + +- `algo` - Algorithm for validate signature. Supported: + + | HMAC | RSA | ECDSA | PSS | EdDSA | + |-------| ----- | ------ | ----- | ------- | + | HS256 | RS256 | ES256 | PS256 | Ed25519 | + | HS384 | RS384 | ES384 | PS384 | Ed448 | + | HS512 | RS512 | ES512 | PS512 | | + | | | ES256K | | | + Also support None. +- `static_key` - key for symmetric algorithms. Mandatory for `HS*` family algorithms. +- `static_key_in_base64` - indicates if the `static_key` key is base64-encoded. Optional, default: `False`. +- `public_key` - public key for asymmetric algorithms. Mandatory except for `HS*` family algorithms and `None`. +- `private_key` - private key for asymmetric algorithms. Optional. +- `public_key_password` - public key password. Optional. +- `private_key_password` - private key password. Optional. + +### Verifying JWT signature using static JWKS {$verifying-jwt-signature-using-static-jwks} + +:::note +Only RS* family algorithms are supported! +::: + +**Example** +```xml + + + + + {"keys": [{"kty": "RSA", "alg": "RS256", "kid": "mykid", "n": "_public_key_mod_", "e": "AQAB"}]} + + + +``` + +#### Parameters: +- `static_jwks` - content of JWKS in json +- `static_jwks_file` - path to file with JWKS + +:::note +Only one of `static_jwks` or `static_jwks_file` keys must be present in one verifier +::: + +### Verifying JWT signature using JWKS servers {$verifying-jwt-signature-using-static-jwks} + +**Example** +```xml + + + + + http://localhost:8000/.well-known/jwks.json + 1000 + 1000 + 1000 + 3 + 50 + 1000 + 300000 + + + +``` + +#### Parameters: + +- `uri` - JWKS endpoint. Mandatory. +- `refresh_ms` - Period for resend request for refreshing JWKS. Optional, default: 300000. + +Timeouts in milliseconds on the socket used for communicating with the server (optional): +- `connection_timeout_ms` - Default: 1000. +- `receive_timeout_ms` - Default: 1000. +- `send_timeout_ms` - Default: 1000. + +Retry parameters (optional): +- `max_tries` - The maximum number of attempts to make an authentication request. Default: 3. +- `retry_initial_backoff_ms` - The backoff initial interval on retry. Default: 50. +- `retry_max_backoff_ms` - The maximum backoff interval. Default: 1000. + +### Verifying access tokens {$verifying-access-tokens} + +Access tokens that are not JWT (and thus no data can be extracted from the token directly) need to be resolved by external providers. + +**Example** +```xml + + + + + google + + + +``` + +#### Parameters: + +- `provider` - name of provider that will be used for token processing. Mandatory parameter. Possible options: `google`. + + +### Enabling JWT authentication in `users.xml` {#enabling-jwt-auth-in-users-xml} + +In order to enable JWT authentication for the user, specify `jwt` section instead of `password` or other similar sections in the user definition. + +Parameters: +- `claims` - An optional string containing a json object that should be contained in the token payload. + +Example (goes into `users.xml`): +```xml + + + + + + {"resource_access":{"account": {"roles": ["view-profile"]}}} + + + +``` + +Here, the JWT payload must contain `["view-profile"]` on path `resource_access.account.roles`, otherwise authentication will not succeed even with a valid JWT. + +``` +{ +... + "resource_access": { + "account": { + "roles": ["view-profile"] + } + }, +... +} +``` + +:::note +JWT authentication cannot be used together with any other authentication method. The presence of any other sections like `password` alongside `jwt` will force ClickHouse to shut down. +::: + +### Enabling JWT authentication using SQL {#enabling-jwt-auth-using-sql} + +When [SQL-driven Access Control and Account Management](/docs/en/guides/sre/user-management/index.md#access-control) is enabled in ClickHouse, users identified by JWT authentication can also be created using SQL statements. + +```sql +CREATE USER my_user IDENTIFIED WITH jwt CLAIMS '{"resource_access":{"account": {"roles": ["view-profile"]}}}' +``` + +Or without additional JWT payload checks: + +```sql +CREATE USER my_user IDENTIFIED WITH jwt +``` + +## JWT authentication examples {#jwt-authentication-examples} + +#### Console client + +``` +clickhouse-client -jwt +``` + +#### HTTP requests + +``` +curl 'http://localhost:8080/?' \ + -H 'Authorization: Bearer ' \ + -H 'Content type: text/plain;charset=UTF-8' \ + --data-raw 'SELECT current_user()' +``` +:::note +ClickHouse will look for a JWT token in (by priority): +1. `X-ClickHouse-JWT-Token` header. +2. `Authorization` header. +3. `token` request parameter. In this case, the "Bearer" prefix should not exist. +::: diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index 432fe19634e6..d40ba7eee961 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -419,6 +420,12 @@ void AccessControl::addLDAPStorage(const String & storage_name_, const Poco::Uti LOG_DEBUG(getLogger(), "Added {} access storage '{}', LDAP server name: {}", String(new_storage->getStorageType()), new_storage->getStorageName(), new_storage->getLDAPServerName()); } +void AccessControl::addTokenStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_) +{ + auto new_storage = std::make_shared(storage_name_, *this, config_, prefix_); + addStorage(new_storage); + LOG_DEBUG(getLogger(), "Added {} access storage '{}'", String(new_storage->getStorageType()), new_storage->getStorageName()); +} void AccessControl::addStoragesFromUserDirectoriesConfig( const Poco::Util::AbstractConfiguration & config, @@ -444,6 +451,8 @@ void AccessControl::addStoragesFromUserDirectoriesConfig( type = DiskAccessStorage::STORAGE_TYPE; else if (type == "ldap") type = LDAPAccessStorage::STORAGE_TYPE; + else if (type == "token") + type = TokenAccessStorage::STORAGE_TYPE; String name = config.getString(prefix + ".name", type); @@ -477,6 +486,10 @@ void AccessControl::addStoragesFromUserDirectoriesConfig( bool allow_backup = config.getBool(prefix + ".allow_backup", true); addReplicatedStorage(name, zookeeper_path, get_zookeeper_function, allow_backup); } + else if (type == TokenAccessStorage::STORAGE_TYPE) + { + addTokenStorage(name, config, prefix); + } else throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "Unknown storage type '{}' at {} in config", type, prefix); } @@ -704,6 +717,11 @@ bool AccessControl::isNoPasswordAllowed() const return allow_no_password; } +bool AccessControl::isJWTEnabled() const +{ + return external_authenticators->isJWTAllowed(); +} + void AccessControl::setPlaintextPasswordAllowed(bool allow_plaintext_password_) { allow_plaintext_password = allow_plaintext_password_; diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h index ac64f89a7080..20a9e5818d66 100644 --- a/src/Access/AccessControl.h +++ b/src/Access/AccessControl.h @@ -93,6 +93,8 @@ class AccessControl : public MultipleAccessStorage /// Adds LDAPAccessStorage which allows querying remote LDAP server for user info. void addLDAPStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_); + void addTokenStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_); + void addReplicatedStorage(const String & storage_name, const String & zookeeper_path, const zkutil::GetZooKeeper & get_zookeeper_function, @@ -155,6 +157,8 @@ class AccessControl : public MultipleAccessStorage void setNoPasswordAllowed(bool allow_no_password_); bool isNoPasswordAllowed() const; + bool isJWTEnabled() const; + /// Allows users with plaintext password (by default it's allowed). void setPlaintextPasswordAllowed(bool allow_plaintext_password_); bool isPlaintextPasswordAllowed() const; diff --git a/src/Access/AccessTokenProcessor.cpp b/src/Access/AccessTokenProcessor.cpp new file mode 100644 index 000000000000..36c78f66588c --- /dev/null +++ b/src/Access/AccessTokenProcessor.cpp @@ -0,0 +1,214 @@ +#include +#include +#include + + +namespace DB +{ + +namespace +{ + /// The JSON reply from provider has only a few key-value pairs, so no need for SimdJSON/RapidJSON. + /// Reduce complexity by using picojson. + picojson::object parseJSON(const String & json_string) { + picojson::value jsonValue; + std::string err = picojson::parse(jsonValue, json_string); + + if (!err.empty()) { + throw std::runtime_error("JSON parsing error: " + err); + } + + if (!jsonValue.is()) { + throw std::runtime_error("JSON is not an object"); + } + + return jsonValue.get(); + } + + std::string getValueByKey(const picojson::object & jsonObject, const std::string & key) { + auto it = jsonObject.find(key); // Find the key in the object + if (it == jsonObject.end()) { + throw std::runtime_error("Key not found: " + key); + } + + const picojson::value &value = it->second; + if (!value.is()) { + throw std::runtime_error("Value for key '" + key + "' is not a string"); + } + + return value.get(); + } +} + + +const Poco::URI GoogleAccessTokenProcessor::token_info_uri = Poco::URI("https://www.googleapis.com/oauth2/v3/tokeninfo"); +const Poco::URI GoogleAccessTokenProcessor::user_info_uri = Poco::URI("https://www.googleapis.com/oauth2/v3/userinfo"); + +const Poco::URI AzureAccessTokenProcessor::user_info_uri = Poco::URI("https://graph.microsoft.com/v1.0/me"); + + +std::unique_ptr IAccessTokenProcessor::parseTokenProcessor( + const Poco::Util::AbstractConfiguration & config, + const String & prefix, + const String & name) +{ + if (config.hasProperty(prefix + ".provider")) + { + String provider = Poco::toLower(config.getString(prefix + ".provider")); + + String email_regex_str = config.hasProperty(prefix + ".email_filter") ? config.getString( + prefix + ".email_filter") : ""; + + if (provider == "google") + { + return std::make_unique(name, email_regex_str); + } + else if (provider == "azure") + { + if (!config.hasProperty(prefix + ".client_id")) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, + "Could not parse access token processor {}: client_id must be specified", name); + + if (!config.hasProperty(prefix + ".tenant_id")) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, + "Could not parse access token processor {}: tenant_id must be specified", name); + + String client_id_str = config.getString(prefix + ".client_id"); + String tenant_id_str = config.getString(prefix + ".tenant_id"); + String client_secret_str = config.hasProperty(prefix + ".client_secret") ? config.getString(prefix + ".client_secret") : ""; + + return std::make_unique(name, email_regex_str, client_id_str, tenant_id_str, client_secret_str); + } + else + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, + "Could not parse access token processor {}: unknown provider {}", name, provider); + } + + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, + "Could not parse access token processor {}: provider name must be specified", name); +} + + +bool GoogleAccessTokenProcessor::resolveAndValidate(const TokenCredentials & credentials) +{ + const String & token = credentials.getToken(); + + String user_name = tryGetUserName(token); + if (user_name.empty()) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to authenticate with access token"); + + auto user_info = getUserInfo(token); + + if (email_regex.ok()) + { + if (!user_info.contains("email")) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to authenticate user {}: e-mail address not found in user data.", user_name); + /// Additionally validate user email to match regex from config. + if (!RE2::FullMatch(user_info["email"], email_regex)) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to authenticate user {}: e-mail address is not permitted.", user_name); + } + /// Credentials are passed as const everywhere up the flow, so we have to comply, + /// in this case const_cast looks acceptable. + const_cast(credentials).setUserName(user_name); + const_cast(credentials).setGroups({}); + + return true; +} + +String GoogleAccessTokenProcessor::tryGetUserName(const String & token) const +{ + Poco::Net::HTTPSClientSession session(token_info_uri.getHost(), token_info_uri.getPort()); + + Poco::Net::HTTPRequest request{Poco::Net::HTTPRequest::HTTP_GET, token_info_uri.getPathAndQuery()}; + request.add("Authorization", "Bearer " + token); + session.sendRequest(request); + + Poco::Net::HTTPResponse response; + std::istream & responseStream = session.receiveResponse(response); + + if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to resolve access token, code: {}, reason: {}", response.getStatus(), response.getReason()); + + std::ostringstream responseString; + Poco::StreamCopier::copyStream(responseStream, responseString); + + try + { + picojson::object parsed_json = parseJSON(responseString.str()); + String username = getValueByKey(parsed_json, "sub"); + return username; + } + catch (const std::runtime_error &) + { + return ""; + } +} + +std::unordered_map GoogleAccessTokenProcessor::getUserInfo(const String & token) const +{ + std::unordered_map user_info; + + Poco::Net::HTTPSClientSession session(user_info_uri.getHost(), user_info_uri.getPort()); + + Poco::Net::HTTPRequest request{Poco::Net::HTTPRequest::HTTP_GET, user_info_uri.getPathAndQuery()}; + request.add("Authorization", "Bearer " + token); + session.sendRequest(request); + + Poco::Net::HTTPResponse response; + std::istream & responseStream = session.receiveResponse(response); + + if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to get user info by access token, code: {}, reason: {}", response.getStatus(), response.getReason()); + + std::ostringstream responseString; + Poco::StreamCopier::copyStream(responseStream, responseString); + + try + { + picojson::object parsed_json = parseJSON(responseString.str()); + user_info["email"] = getValueByKey(parsed_json, "email"); + user_info["sub"] = getValueByKey(parsed_json, "sub"); + return user_info; + } + catch (const std::runtime_error & e) + { + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to get user info by access token: {}", e.what()); + } +} + +bool AzureAccessTokenProcessor::resolveAndValidate(const TokenCredentials & credentials) +{ + /// Token is a JWT in this case, all we need is to decode it and verify against JWKS (similar to JWTValidator.h) + String user_name = credentials.getUserName(); + if (user_name.empty()) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to authenticate with access token: cannot extract username"); + + const String & token = credentials.getToken(); + + try + { + token_validator->validate("", token); + } + catch (...) + { + return false; + } + + const auto decoded_token = jwt::decode(token); + + if (email_regex.ok()) + { + if (!decoded_token.has_payload_claim("email")) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to authenticate user {}: e-mail address not found in user data.", user_name); + /// Additionally validate user email to match regex from config. + if (!RE2::FullMatch(decoded_token.get_payload_claim("email").as_string(), email_regex)) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to authenticate user {}: e-mail address is not permitted.", user_name); + } + /// Credentials are passed as const everywhere up the flow, so we have to comply, + /// in this case const_cast looks acceptable. + const_cast(credentials).setGroups({}); + + return true; +} + +} diff --git a/src/Access/AccessTokenProcessor.h b/src/Access/AccessTokenProcessor.h new file mode 100644 index 000000000000..a45f1bbac379 --- /dev/null +++ b/src/Access/AccessTokenProcessor.h @@ -0,0 +1,112 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int AUTHENTICATION_FAILED; + extern const int INVALID_CONFIG_PARAMETER; +} + +class GoogleAccessTokenProcessor; + +class IAccessTokenProcessor +{ +public: + IAccessTokenProcessor(const String & name_, const String & email_regex_str) : name(name_), email_regex(email_regex_str) + { + if (!email_regex_str.empty()) + { + /// Later, we will use .ok() to determine whether there was a regex specified in config or not. + if (!email_regex.ok()) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Invalid regex in definition of access token processor {}", name); + } + } + + String getName() + { + return name; + } + + virtual ~IAccessTokenProcessor() = default; + + virtual bool resolveAndValidate(const TokenCredentials & credentials) = 0; + + virtual std::set getGroups([[maybe_unused]] const TokenCredentials & credentials) + { + return {}; + } + + static std::unique_ptr parseTokenProcessor( + const Poco::Util::AbstractConfiguration & config, + const String & prefix, + const String & name); + +protected: + const String name; + re2::RE2 email_regex; +}; + + +class GoogleAccessTokenProcessor : public IAccessTokenProcessor +{ +public: + GoogleAccessTokenProcessor(const String & name_, const String & email_regex_str) : IAccessTokenProcessor(name_, email_regex_str) {} + + bool resolveAndValidate(const TokenCredentials & credentials) override; + +private: + static const Poco::URI token_info_uri; + static const Poco::URI user_info_uri; + + String tryGetUserName(const String & token) const; + + std::unordered_map getUserInfo(const String & token) const; +}; + + +class AzureAccessTokenProcessor : public IAccessTokenProcessor +{ +public: + AzureAccessTokenProcessor(const String & name_, + const String & email_regex_str, + const String & client_id_, + const String & tenant_id_, + const String & client_secret_, + const size_t jwks_refresh_interval = 300000) + : IAccessTokenProcessor(name_, email_regex_str), + client_id(client_id_), + tenant_id(tenant_id_), + client_secret(client_secret_), + jwks_uri_str("https://login.microsoftonline.com/" + tenant_id + "/discovery/v2.0/keys") + { + token_validator = std::make_unique(name + "_jwks_validator", std::make_unique(jwks_uri_str, jwks_refresh_interval)); + } + + bool resolveAndValidate(const TokenCredentials & credentials) override; +private: + static const Poco::URI user_info_uri; + + const String client_id; + const String tenant_id; + const String client_secret; + + const String jwks_uri_str; + + std::unique_ptr token_validator; +}; + +} diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 6aa5f3fdfff6..656284f34c01 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -235,7 +235,7 @@ bool Authentication::areCredentialsValid( const ExternalAuthenticators & external_authenticators, SettingsChanges & settings) { - if (!credentials.isReady()) + if (!typeid_cast(&credentials) && !credentials.isReady()) return false; if (const auto * gss_acceptor_context = typeid_cast(&credentials)) @@ -270,6 +270,22 @@ bool Authentication::areCredentialsValid( } #endif + if (const auto * token_credentials = typeid_cast(&credentials)) + { + if (authentication_method.getType() != AuthenticationType::JWT) + return false; + + if (token_credentials->isJWT()) + { + /// The token was parsed as JWT, no further action needed. + return external_authenticators.checkJWTCredentials(authentication_method.getJWTClaims(), *token_credentials); + } + else + { + return external_authenticators.checkAccessTokenCredentials(*token_credentials); + } + } + if ([[maybe_unused]] const auto * always_allow_credentials = typeid_cast(&credentials)) return true; diff --git a/src/Access/AuthenticationData.cpp b/src/Access/AuthenticationData.cpp index 99eed230db73..7bafbf65b267 100644 --- a/src/Access/AuthenticationData.cpp +++ b/src/Access/AuthenticationData.cpp @@ -15,7 +15,10 @@ #include #include #include +#include +#include +#include "Access/Common/AuthenticationType.h" #include #include "config.h" @@ -337,7 +340,10 @@ std::shared_ptr AuthenticationData::toAST() const } case AuthenticationType::JWT: { - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud"); + const auto & claims = getJWTClaims(); + if (!claims.empty()) + node->children.push_back(std::make_shared(claims)); + break; } case AuthenticationType::KERBEROS: { @@ -570,6 +576,20 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que auth_data.setHTTPAuthenticationServerName(server); auth_data.setHTTPAuthenticationScheme(scheme); } + else if (query.type == AuthenticationType::JWT) + { + if (!args.empty()) + { + String value = checkAndGetLiteralArgument(args[0], "claims"); + picojson::value json_obj; + auto error = picojson::parse(json_obj, value); + if (!error.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad JWT claims: {}", error); + if (!json_obj.is()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad JWT claims: is not an object"); + auth_data.setJWTClaims(value); + } + } else { throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected ASTAuthenticationData structure"); diff --git a/src/Access/AuthenticationData.h b/src/Access/AuthenticationData.h index 239a802edddc..7c0f432f6190 100644 --- a/src/Access/AuthenticationData.h +++ b/src/Access/AuthenticationData.h @@ -77,6 +77,9 @@ class AuthenticationData time_t getValidUntil() const { return valid_until; } void setValidUntil(time_t valid_until_) { valid_until = valid_until_; } + const String & getJWTClaims() const { return jwt_claims; } + void setJWTClaims(const String & jwt_claims_) { jwt_claims = jwt_claims_; } + friend bool operator ==(const AuthenticationData & lhs, const AuthenticationData & rhs); friend bool operator !=(const AuthenticationData & lhs, const AuthenticationData & rhs) { return !(lhs == rhs); } @@ -110,6 +113,7 @@ class AuthenticationData String http_auth_server_name; HTTPAuthenticationScheme http_auth_scheme = HTTPAuthenticationScheme::BASIC; time_t valid_until = 0; + String jwt_claims; }; } diff --git a/src/Access/Common/JWKSProvider.cpp b/src/Access/Common/JWKSProvider.cpp new file mode 100644 index 000000000000..1c306b232ba2 --- /dev/null +++ b/src/Access/Common/JWKSProvider.cpp @@ -0,0 +1,92 @@ +#include + +#include +#include +#include +#include + +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int AUTHENTICATION_FAILED; + extern const int INVALID_CONFIG_PARAMETER; +} + +jwt::jwks JWKSClient::getJWKS() +{ + std::shared_lock lock(mutex); + + auto now = std::chrono::high_resolution_clock::now(); + auto diff = std::chrono::duration(now - last_request_send).count(); + + if (diff < refresh_ms) { + jwt::jwks result(cached_jwks); + return result; + } + + Poco::Net::HTTPResponse response; + std::ostringstream responseString; + + Poco::Net::HTTPRequest request{Poco::Net::HTTPRequest::HTTP_GET, jwks_uri.getPathAndQuery()}; + + if (jwks_uri.getScheme() == "https") { + Poco::Net::HTTPSClientSession session = Poco::Net::HTTPSClientSession(jwks_uri.getHost(), jwks_uri.getPort()); + session.sendRequest(request); + std::istream & responseStream = session.receiveResponse(response); + if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK || !responseStream) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to get user info by access token, code: {}, reason: {}", response.getStatus(), response.getReason()); + Poco::StreamCopier::copyStream(responseStream, responseString); + } else { + Poco::Net::HTTPClientSession session = Poco::Net::HTTPClientSession(jwks_uri.getHost(), jwks_uri.getPort()); + session.sendRequest(request); + std::istream & responseStream = session.receiveResponse(response); + if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK || !responseStream) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to get user info by access token, code: {}, reason: {}", response.getStatus(), response.getReason()); + Poco::StreamCopier::copyStream(responseStream, responseString); + } + + last_request_send = std::chrono::high_resolution_clock::now(); + + jwt::jwks parsed_jwks; + + try { + parsed_jwks = jwt::parse_jwks(responseString.str()); + } + catch (const Exception & e) { + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to parse JWKS: {}", e.what()); + } + + cached_jwks = std::move(parsed_jwks); + return cached_jwks; +} + +StaticJWKSParams::StaticJWKSParams(const std::string &static_jwks_, const std::string &static_jwks_file_) +{ + if (static_jwks_.empty() && static_jwks_file_.empty()) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, + "JWT validator misconfigured: `static_jwks` or `static_jwks_file` keys must be present in static JWKS validator configuration"); + if (!static_jwks_.empty() && !static_jwks_file_.empty()) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, + "JWT validator misconfigured: `static_jwks` and `static_jwks_file` keys cannot both be present in static JWKS validator configuration"); + + static_jwks = static_jwks_; + static_jwks_file = static_jwks_file_; +} + +StaticJWKS::StaticJWKS(const StaticJWKSParams ¶ms) +{ + String content = String(params.static_jwks); + if (!params.static_jwks_file.empty()) { + std::ifstream ifs(params.static_jwks_file); + content = String((std::istreambuf_iterator(ifs)), (std::istreambuf_iterator())); + } + auto keys = jwt::parse_jwks(content); + jwks = std::move(keys); +} + +} diff --git a/src/Access/Common/JWKSProvider.h b/src/Access/Common/JWKSProvider.h new file mode 100644 index 000000000000..773208a138aa --- /dev/null +++ b/src/Access/Common/JWKSProvider.h @@ -0,0 +1,67 @@ +#include +#include +#include + +#include +#include +#include +#include +#include + + +namespace DB +{ + +class IJWKSProvider +{ +public: + virtual ~IJWKSProvider() = default; + + virtual jwt::jwks getJWKS() = 0; +}; + +class JWKSClient : public IJWKSProvider +{ +public: + explicit JWKSClient(const String & uri, const size_t refresh_ms_): refresh_ms(refresh_ms_), jwks_uri(uri) {} + + ~JWKSClient() override = default; + JWKSClient(const JWKSClient &) = delete; + JWKSClient(JWKSClient &&) = delete; + JWKSClient &operator=(const JWKSClient &) = delete; + JWKSClient &operator=(JWKSClient &&) = delete; + + jwt::jwks getJWKS() override; + +private: + size_t refresh_ms; + Poco::URI jwks_uri; + + std::shared_mutex mutex; + jwt::jwks cached_jwks; + std::chrono::time_point last_request_send; +}; + +struct StaticJWKSParams +{ + StaticJWKSParams(const std::string &static_jwks_, const std::string &static_jwks_file_); + + String static_jwks; + String static_jwks_file; +}; + +class StaticJWKS : public IJWKSProvider +{ +public: + explicit StaticJWKS(const StaticJWKSParams ¶ms); + +private: + jwt::jwks getJWKS() override + { + return jwks; + } + + jwt::jwks jwks; +}; + +} diff --git a/src/Access/Credentials.cpp b/src/Access/Credentials.cpp index f01700b6e461..796e3cc53b0c 100644 --- a/src/Access/Credentials.cpp +++ b/src/Access/Credentials.cpp @@ -1,6 +1,8 @@ #include -#include #include +#include + +#include namespace DB { @@ -8,6 +10,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int AUTHENTICATION_FAILED; } Credentials::Credentials(const String & user_name_) @@ -97,4 +100,34 @@ const String & BasicCredentials::getPassword() const return password; } +namespace +{ +String extractUsernameFromToken(const String & token) +{ + try + { + /// Attempt to handle token as JWT. + auto decoded_jwt = jwt::decode(token); + return decoded_jwt.get_subject(); + } + catch (...) + { + /// Token is not JWT, try to handle it as access token + return ""; + } +} +} + +TokenCredentials::TokenCredentials(const String & token_) + : Credentials(extractUsernameFromToken(token_)) + , token(token_) + { + // If username is empty, then the token is probably not JWT; + // we will try treating this token as an access token. + if (!user_name.empty()) + { + is_ready = true; + is_jwt = true; + } + } } diff --git a/src/Access/Credentials.h b/src/Access/Credentials.h index 52f33385a0e8..6f2a37149147 100644 --- a/src/Access/Credentials.h +++ b/src/Access/Credentials.h @@ -151,4 +151,43 @@ class SSHPTYCredentials : public Credentials #endif +class TokenCredentials : public Credentials +{ +public: + explicit TokenCredentials(const String & token_); + + const String & getToken() const + { + if (token.empty()) + { + throwNotReady(); + } + return token; + } + void setUserName(const String & user_name_) + { + user_name = user_name_; + if (!user_name.empty()) + { + is_ready = true; + } + } + bool isJWT() const + { + return is_jwt; + } + std::set getGroups() const + { + return groups; + } + void setGroups(const std::set & groups_) + { + groups = groups_; + } +private: + String token; + bool is_jwt = false; + std::set groups; +}; + } diff --git a/src/Access/ExternalAuthenticators.cpp b/src/Access/ExternalAuthenticators.cpp index 91d0ff7ff0f9..21772137c84f 100644 --- a/src/Access/ExternalAuthenticators.cpp +++ b/src/Access/ExternalAuthenticators.cpp @@ -2,14 +2,21 @@ #include #include #include +#include "Common/Logger.h" +#include "Common/logger_useful.h" #include #include #include #include +#include "Access/AccessControl.h" +#include "Access/Credentials.h" +#include #include #include +#include +#include #include #include @@ -253,7 +260,6 @@ HTTPAuthClientParams parseHTTPAuthParams(const Poco::Util::AbstractConfiguration return http_auth_params; } - } void parseLDAPRoleSearchParams(LDAPClient::RoleSearchParams & params, const Poco::Util::AbstractConfiguration & config, const String & prefix) @@ -271,6 +277,13 @@ void ExternalAuthenticators::resetImpl() ldap_client_params_blueprint.clear(); ldap_caches.clear(); kerberos_params.reset(); + jwt_validators.clear(); +} + +bool ExternalAuthenticators::isJWTAllowed() const +{ + std::lock_guard lock(mutex); + return !jwt_validators.empty(); } void ExternalAuthenticators::reset() @@ -279,6 +292,52 @@ void ExternalAuthenticators::reset() resetImpl(); } +void parseJWTValidators(std::unordered_map> & jwt_validators, + const Poco::Util::AbstractConfiguration & config, + const String & jwt_validators_config, + LoggerPtr log) +{ + Poco::Util::AbstractConfiguration::Keys jwt_validators_keys; + config.keys(jwt_validators_config, jwt_validators_keys); + jwt_validators.clear(); + for (const auto & jwt_validator : jwt_validators_keys) + { + if (jwt_validator == "settings_key") continue; + String prefix = fmt::format("{}.{}", jwt_validators_config, jwt_validator); + try + { + jwt_validators[jwt_validator] = IJWTValidator::parseJWTValidator(config, prefix, jwt_validator); + } + catch (...) + { + tryLogCurrentException(log, "Could not parse JWT validator" + backQuote(jwt_validator)); + } + } +} + +void parseAccessTokenProcessors(std::unordered_map> & access_token_processors, + const Poco::Util::AbstractConfiguration & config, + const String & access_token_processors_config, + LoggerPtr log) +{ + Poco::Util::AbstractConfiguration::Keys access_token_processors_keys; + config.keys(access_token_processors_config, access_token_processors_keys); + access_token_processors.clear(); + + for (const auto & processor : access_token_processors_keys) + { + String prefix = fmt::format("{}.{}", access_token_processors_config, processor); + try + { + access_token_processors[processor] = IAccessTokenProcessor::parseTokenProcessor(config, prefix, processor); + } + catch (...) + { + tryLogCurrentException(log, "Could not parse access token processor" + backQuote(processor)); + } + } +} + void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfiguration & config, LoggerPtr log) { std::lock_guard lock(mutex); @@ -290,8 +349,12 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur std::size_t ldap_servers_key_count = 0; std::size_t kerberos_keys_count = 0; std::size_t http_auth_server_keys_count = 0; + std::size_t jwt_validators_count = 0; + std::size_t access_token_processors_count = 0; const String http_auth_servers_config = "http_authentication_servers"; + const String jwt_validators_config = "jwt_validators"; + const String access_token_processors_config = "access_token_processors"; for (auto key : all_keys) { @@ -304,6 +367,8 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur ldap_servers_key_count += (key == "ldap_servers"); kerberos_keys_count += (key == "kerberos"); http_auth_server_keys_count += (key == http_auth_servers_config); + jwt_validators_count += (key == jwt_validators_config); + access_token_processors_count += (key == access_token_processors_config); } if (ldap_servers_key_count > 1) @@ -315,6 +380,12 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur if (http_auth_server_keys_count > 1) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Multiple http_authentication_servers sections are not allowed"); + if (jwt_validators_count > 1) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Multiple {} sections are not allowed", jwt_validators_config); + + if (access_token_processors_count > 1) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Multiple {} sections are not allowed", access_token_processors_config); + Poco::Util::AbstractConfiguration::Keys http_auth_server_names; config.keys(http_auth_servers_config, http_auth_server_names); http_auth_servers.clear(); @@ -369,6 +440,9 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur { tryLogCurrentException(log, "Could not parse Kerberos section"); } + + parseJWTValidators(jwt_validators, config, jwt_validators_config, log); + parseAccessTokenProcessors(access_token_processors, config, access_token_processors_config, log); } static UInt128 computeParamsHash(const LDAPClient::Params & params, const LDAPClient::RoleSearchParamsList * role_search_params) @@ -537,7 +611,7 @@ GSSAcceptorContext::Params ExternalAuthenticators::getKerberosParams() const return kerberos_params.value(); } -HTTPAuthClientParams ExternalAuthenticators::getHTTPAuthenticationParams(const String& server) const +HTTPAuthClientParams ExternalAuthenticators::getHTTPAuthenticationParams(const String & server) const { std::lock_guard lock{mutex}; @@ -547,6 +621,66 @@ HTTPAuthClientParams ExternalAuthenticators::getHTTPAuthenticationParams(const S return it->second; } +bool ExternalAuthenticators::checkJWTCredentials(const String & claims, const TokenCredentials & credentials) const +{ + std::lock_guard lock{mutex}; + + const auto token = String(credentials.getToken()); + const auto & user_name = credentials.getUserName(); + + if (jwt_validators.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "JWT authentication is not configured"); + + for (const auto & it : jwt_validators) + { + if (it.second->validate(claims, token)) + { + LOG_DEBUG(getLogger("JWTAuthentication"), "Authenticated with JWT for {} by {}", user_name, it.first); + return true; + } + LOG_TRACE(getLogger("JWTAuthentication"), "Failed authentication with JWT for {} by {}", user_name, it.first); + } + return false; +} + +bool ExternalAuthenticators::checkAccessTokenCredentials(const TokenCredentials & credentials) const +{ + std::lock_guard lock{mutex}; + + if (access_token_processors.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Access token authentication is not configured"); + + for (const auto & it : access_token_processors) + { + if (it.second->resolveAndValidate(credentials)) + { + LOG_DEBUG(getLogger("AccessTokenAuthentication"), "Authenticated user {} with access token by {}", credentials.getUserName(), it.first); + return true; + } + LOG_TRACE(getLogger("AccessTokenAuthentication"), "Failed authentication with access token by {}", it.first); + } + return false; +} + +bool ExternalAuthenticators::checkAccessTokenCredentialsByExactProcessor(const TokenCredentials & credentials, const String & name) const +{ + std::lock_guard lock{mutex}; + + if (access_token_processors.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Access token authentication is not configured"); + + for (const auto & it : access_token_processors) + { + if (name == it.second->getName() && it.second->resolveAndValidate(credentials)) + { + LOG_DEBUG(getLogger("AccessTokenAuthentication"), "Authenticated user {} with access token by {}", credentials.getUserName(), it.first); + return true; + } + } + LOG_TRACE(getLogger("AccessTokenAuthentication"), "Failed authentication with access token: no processor with name {}", name); + return false; +} + bool ExternalAuthenticators::checkHTTPBasicCredentials( const String & server, const BasicCredentials & credentials, SettingsChanges & settings) const { diff --git a/src/Access/ExternalAuthenticators.h b/src/Access/ExternalAuthenticators.h index 3a710e6df26a..b32c7aa5f95c 100644 --- a/src/Access/ExternalAuthenticators.h +++ b/src/Access/ExternalAuthenticators.h @@ -3,7 +3,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -12,6 +14,7 @@ #include #include +#include #include #include #include @@ -31,6 +34,7 @@ namespace DB { class SettingsChanges; +class AccessControl; class ExternalAuthenticators { @@ -43,9 +47,15 @@ class ExternalAuthenticators const LDAPClient::RoleSearchParamsList * role_search_params = nullptr, LDAPClient::SearchResultsList * role_search_results = nullptr) const; bool checkKerberosCredentials(const String & realm, const GSSAcceptorContext & credentials) const; bool checkHTTPBasicCredentials(const String & server, const BasicCredentials & credentials, SettingsChanges & settings) const; + bool checkJWTCredentials(const String & claims, const TokenCredentials & credentials) const; + + bool checkAccessTokenCredentials(const TokenCredentials & credentials) const; + bool checkAccessTokenCredentialsByExactProcessor(const TokenCredentials & credentials, const String & name) const; GSSAcceptorContext::Params getKerberosParams() const; + bool isJWTAllowed() const; + private: HTTPAuthClientParams getHTTPAuthenticationParams(const String& server) const; @@ -65,6 +75,8 @@ class ExternalAuthenticators mutable LDAPCaches ldap_caches TSA_GUARDED_BY(mutex) ; std::optional kerberos_params TSA_GUARDED_BY(mutex) ; std::unordered_map http_auth_servers TSA_GUARDED_BY(mutex) ; + std::unordered_map> jwt_validators TSA_GUARDED_BY(mutex) ; + std::unordered_map> access_token_processors TSA_GUARDED_BY(mutex) ; void resetImpl() TSA_REQUIRES(mutex); }; diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 72e0933e2142..020b9289d0d7 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -11,6 +11,7 @@ #include #include #include +#include "Access/Common/AuthenticationType.h" #include #include #include diff --git a/src/Access/JWTValidator.cpp b/src/Access/JWTValidator.cpp new file mode 100644 index 000000000000..a94334463ee3 --- /dev/null +++ b/src/Access/JWTValidator.cpp @@ -0,0 +1,369 @@ +#include "JWTValidator.h" + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int AUTHENTICATION_FAILED; + extern const int INVALID_CONFIG_PARAMETER; +} + +namespace +{ + +bool check_claims(const picojson::value & claims, const picojson::value & payload, const String & path); +bool check_claims(const picojson::value::object & claims, const picojson::value::object & payload, const String & path) +{ + for (const auto & it : claims) + { + const auto & payload_it = payload.find(it.first); + if (payload_it == payload.end()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "Key '{}.{}' not found in JWT payload", path, it.first); + return false; + } + if (!check_claims(it.second, payload_it->second, path + "." + it.first)) + { + return false; + } + } + return true; +} + +bool check_claims(const picojson::value::array & claims, const picojson::value::array & payload, const String & path) +{ + if (claims.size() > payload.size()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload too small for claims key '{}'", path); + return false; + } + for (size_t claims_i = 0; claims_i < claims.size(); ++claims_i) + { + bool found = false; + const auto & claims_val = claims.at(claims_i); + for (const auto & payload_val : payload) + { + if (!check_claims(claims_val, payload_val, path + "[" + std::to_string(claims_i) + "]")) + continue; + found = true; + } + if (!found) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not contain an object matching claims key '{}[{}]'", path, claims_i); + return false; + } + } + return true; +} + +bool check_claims(const picojson::value & claims, const picojson::value & payload, const String & path) +{ + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'array' in claims '{}'", path); + return false; + } + return check_claims(claims.get(), payload.get(), path); + } + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'object' in claims '{}'", path); + return false; + } + return check_claims(claims.get(), payload.get(), path); + } + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'bool' in claims '{}'", path); + return false; + } + if (claims.get() != payload.get()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match the value in the '{}' assertions. Expected '{}' but given '{}'", path, claims.get(), payload.get()); + return false; + } + return true; + } + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'double' in claims '{}'", path); + return false; + } + if (claims.get() != payload.get()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match the value in the '{}' assertions. Expected '{}' but given '{}'", path, claims.get(), payload.get()); + return false; + } + return true; + } + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'std::string' in claims '{}'", path); + return false; + } + if (claims.get() != payload.get()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match the value in the '{}' assertions. Expected '{}' but given '{}'", path, claims.get(), payload.get()); + return false; + } + return true; + } + #ifdef PICOJSON_USE_INT64 + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'int64_t' in claims '{}'", path); + return false; + } + if (claims.get() != payload.get()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match the value in claims '{}'. Expected '{}' but given '{}'", path, claims.get(), payload.get()); + return false; + } + return true; + } + #endif + LOG_ERROR(getLogger("JWTAuthentication"), "JWT claim '{}' does not match any known type", path); + return false; +} + +bool check_claims(const String & claims, const picojson::value::object & payload) +{ + if (claims.empty()) + return true; + picojson::value json; + auto errors = picojson::parse(json, claims); + if (!errors.empty()) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Bad JWT claims: {}", errors); + if (!json.is()) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Bad JWT claims: is not an object"); + return check_claims(json.get(), payload, ""); +} + +} + +bool IJWTValidator::validate(const String & claims, const String & token) const +{ + try + { + auto decoded_jwt = jwt::decode(token); + + validateImpl(decoded_jwt); + + if (!check_claims(claims, decoded_jwt.get_payload_json())) + return false; + + LOG_TRACE(getLogger("JWTAuthentication"), "{}: claims checked", name); + + return true; + } + catch (const std::exception & ex) + { + LOG_TRACE(getLogger("JWTAuthentication"), "{}: Failed to validate JWT: {}", name, ex.what()); + return false; + } +} + +void SimpleJWTValidatorParams::validate() const +{ + if (algo == "ps256" || + algo == "ps384" || + algo == "ps512" || + algo == "ed25519" || + algo == "ed448" || + algo == "rs256" || + algo == "rs384" || + algo == "rs512" || + algo == "es256" || + algo == "es256k" || + algo == "es384" || + algo == "es512" ) + { + if (public_key.empty()) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: `public_key` parameter required for {}", algo); + } + else if (algo == "hs256" || + algo == "hs384" || + algo == "hs512" ) + { + if (static_key.empty()) + throw DB::Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: `static_key` parameter required for {}", algo); + } + else if (algo != "none") + throw DB::Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: unknown algorithm {}", algo); +} + + +SimpleJWTValidator::SimpleJWTValidator(const String & name_, const SimpleJWTValidatorParams & params_) + : IJWTValidator(name_), verifier(jwt::verify()) +{ + auto algo = params_.algo; + + if (algo == "none") + verifier = verifier.allow_algorithm(jwt::algorithm::none()); + else if (algo == "ps256") + verifier = verifier.allow_algorithm(jwt::algorithm::ps256(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "ps384") + verifier = verifier.allow_algorithm(jwt::algorithm::ps384(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "ps512") + verifier = verifier.allow_algorithm(jwt::algorithm::ps512(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "ed25519") + verifier = verifier.allow_algorithm(jwt::algorithm::ed25519(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "ed448") + verifier = verifier.allow_algorithm(jwt::algorithm::ed448(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "rs256") + verifier = verifier.allow_algorithm(jwt::algorithm::rs256(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "rs384") + verifier = verifier.allow_algorithm(jwt::algorithm::rs384(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "rs512") + verifier = verifier.allow_algorithm(jwt::algorithm::rs512(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "es256") + verifier = verifier.allow_algorithm(jwt::algorithm::es256(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "es256k") + verifier = verifier.allow_algorithm(jwt::algorithm::es256k(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "es384") + verifier = verifier.allow_algorithm(jwt::algorithm::es384(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "es512") + verifier = verifier.allow_algorithm(jwt::algorithm::es512(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo.starts_with("hs")) + { + auto key = params_.static_key; + if (params_.static_key_in_base64) + key = base64Decode(key); + if (algo == "hs256") + verifier = verifier.allow_algorithm(jwt::algorithm::hs256(key)); + else if (algo == "hs384") + verifier = verifier.allow_algorithm(jwt::algorithm::hs384(key)); + else if (algo == "hs512") + verifier = verifier.allow_algorithm(jwt::algorithm::hs512(key)); + else + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: unknown algorithm {}", params_.algo); + } + else + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: unknown algorithm {}", params_.algo); +} + +void SimpleJWTValidator::validateImpl(const jwt::decoded_jwt & token) const +{ + verifier.verify(token); +} + +void JWKSValidator::validateImpl(const jwt::decoded_jwt & token) const +{ + auto jwk = provider->getJWKS().get_jwk(token.get_key_id()); + auto subject = token.get_subject(); + auto algo = Poco::toLower(token.get_algorithm()); + auto verifier = jwt::verify(); + String public_key; + + try + { + auto issuer = token.get_issuer(); + auto x5c = jwk.get_x5c_key_value(); + + if (!x5c.empty() && !issuer.empty()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "{}: Verifying {} with 'x5c' key", name, subject); + public_key = jwt::helper::convert_base64_der_to_pem(x5c); + } + } + catch (const jwt::error::claim_not_present_exception &) + { + LOG_TRACE(getLogger("JWTAuthentication"), "{}: issuer or x5c was not specified, skip verification against them", name); + } + catch (const std::bad_cast &) + { + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: invalid claim value type found, claims must be strings"); + } + + if (public_key.empty()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "{}: `issuer` or `x5c` not present, verifying {} with RSA components", name, subject); + const auto modulus = jwk.get_jwk_claim("n").as_string(); + const auto exponent = jwk.get_jwk_claim("e").as_string(); + public_key = jwt::helper::create_public_key_from_rsa_components(modulus, exponent); + } + + if (!jwk.has_algorithm()) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT validation error: missing `alg` in JWK"); + else if (Poco::toLower(jwk.get_algorithm()) != algo) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT validation error: `alg` in JWK does not match the algorithm used in JWT"); + + if (algo == "rs256") + verifier = verifier.allow_algorithm(jwt::algorithm::rs256(public_key, "", "", "")); + else if (algo == "rs384") + verifier = verifier.allow_algorithm(jwt::algorithm::rs384(public_key, "", "", "")); + else if (algo == "rs512") + verifier = verifier.allow_algorithm(jwt::algorithm::rs512(public_key, "", "", "")); + else + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: unknown algorithm {}", algo); + + verifier = verifier.leeway(60UL); + verifier.verify(token); +} + + +std::unique_ptr IJWTValidator::parseJWTValidator( + const Poco::Util::AbstractConfiguration & config, + const String & prefix, + const String & name) +{ + if (config.hasProperty(prefix + ".algo")) + { + SimpleJWTValidatorParams params = {}; + params.algo = Poco::toLower(config.getString(prefix + ".algo")); + params.static_key = config.getString(prefix + ".static_key", ""); + params.static_key_in_base64 = config.getBool(prefix + ".static_key_in_base64", false); + params.public_key = config.getString(prefix + ".public_key", ""); + params.private_key = config.getString(prefix + ".private_key", ""); + params.public_key_password = config.getString(prefix + ".public_key_password", ""); + params.private_key_password = config.getString(prefix + ".private_key_password", ""); + params.validate(); + return std::make_unique(name, params); + } + + std::shared_ptr provider; + if (config.hasProperty(prefix + ".uri")) + { + provider = std::make_shared(config.getString(prefix + ".uri"), config.getInt(prefix + ".refresh_ms", 300000)); + } + else if (config.hasProperty(prefix + ".static_jwks") || config.hasProperty(prefix + ".static_jwks_file")) + { + StaticJWKSParams params{ + config.getString(prefix + ".static_jwks", ""), + config.getString(prefix + ".static_jwks_file", "") + }; + provider = std::make_shared(params); + } + else + throw DB::Exception(ErrorCodes::BAD_ARGUMENTS, "Either JWKS or JWKS URI must be specified in configuration"); + + return std::make_unique(name, provider); +} + +} diff --git a/src/Access/JWTValidator.h b/src/Access/JWTValidator.h new file mode 100644 index 000000000000..9698e9570d6a --- /dev/null +++ b/src/Access/JWTValidator.h @@ -0,0 +1,69 @@ +#pragma once + +#include + +#include +#include +#include + +#include +#include + +#include "Access/HTTPAuthClient.h" + +#include +#include +#include + +namespace DB +{ + +class IJWTValidator +{ +public: + explicit IJWTValidator(const String & name_) : name(name_) {} + virtual bool validate(const String & claims, const String & token) const; + virtual ~IJWTValidator() = default; + + static std::unique_ptr parseJWTValidator( + const Poco::Util::AbstractConfiguration & config, + const String & prefix, + const String & name); + +protected: + virtual void validateImpl(const jwt::decoded_jwt & token) const = 0; + const String name; +}; + +struct SimpleJWTValidatorParams +{ + String algo; + String static_key; + bool static_key_in_base64; + String public_key; + String private_key; + String public_key_password; + String private_key_password; + void validate() const; +}; + +class SimpleJWTValidator : public IJWTValidator +{ +public: + explicit SimpleJWTValidator(const String & name_, const SimpleJWTValidatorParams & params_); +private: + void validateImpl(const jwt::decoded_jwt & token) const override; + jwt::verifier verifier; +}; + +class JWKSValidator : public IJWTValidator +{ +public: + explicit JWKSValidator(const String & name_, std::shared_ptr provider_) + : IJWTValidator(name_), provider(provider_) {} +private: + void validateImpl(const jwt::decoded_jwt & token) const override; + + std::shared_ptr provider; +}; +} diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index d51a7e005110..07e12064d4aa 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -9,6 +9,8 @@ #include #include +#include + namespace DB { @@ -453,6 +455,7 @@ MultipleAccessStorage::authenticateImpl(const Credentials & credentials, const P allow_no_password, allow_plaintext_password); if (auth_result) { + std::cerr << "\n\nAuth result in: \n\n" << storage->getStorageName(); std::lock_guard lock{mutex}; ids_cache.set(auth_result->user_id, storage); return auth_result; diff --git a/src/Access/TokenAccessStorage.cpp b/src/Access/TokenAccessStorage.cpp new file mode 100644 index 000000000000..a0bda64700e9 --- /dev/null +++ b/src/Access/TokenAccessStorage.cpp @@ -0,0 +1,398 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + +TokenAccessStorage::TokenAccessStorage(const String & storage_name_, AccessControl & access_control_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_) + : IAccessStorage(storage_name_), access_control(access_control_), config(config_), prefix(prefix_), + memory_storage(storage_name_, access_control.getChangesNotifier(), false) +{ + setConfiguration(); +} + +void TokenAccessStorage::setConfiguration() +{ + std::lock_guard lock(mutex); + + const String prefix_str = (prefix.empty() ? "" : prefix + "."); + + provider_name = config.getString(prefix_str + "processor"); + if (provider_name.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "'processor' must be specified for Token user directory"); + +// getTokenProcessorByName + + const bool has_roles = config.has(prefix_str + "roles"); + + std::set common_roles_cfg; + if (has_roles) + { + Poco::Util::AbstractConfiguration::Keys role_names; + config.keys(prefix_str + "roles", role_names); + + common_roles_cfg.insert(role_names.begin(), role_names.end()); + } + + common_role_names.swap(common_roles_cfg); + + external_role_hashes.clear(); + users_per_roles.clear(); + roles_per_users.clear(); + granted_role_names.clear(); + granted_role_ids.clear(); + + role_change_subscription = access_control.subscribeForChanges( + [this] (const UUID & id, const AccessEntityPtr & entity) + { + this->processRoleChange(id, entity); + } + ); +} + +void TokenAccessStorage::applyRoleChangeNoLock(bool grant, const UUID & role_id, const String & role_name) +{ + std::vector user_ids; + + // Build a list of ids of the relevant users. + if (common_role_names.contains(role_name)) + { + user_ids = memory_storage.findAll(); + } + else + { + const auto it = users_per_roles.find(role_name); + if (it != users_per_roles.end()) + { + const auto & user_names = it->second; + user_ids.reserve(user_names.size()); + + for (const auto & user_name : user_names) + { + if (const auto user_id = memory_storage.find(user_name)) + user_ids.emplace_back(*user_id); + } + } + } + + // Update the granted roles of the relevant users. + if (!user_ids.empty()) + { + auto update_func = [&role_id, &grant] (const AccessEntityPtr & entity_, const UUID &) -> AccessEntityPtr + { + if (auto user = typeid_cast>(entity_)) + { + auto changed_user = typeid_cast>(user->clone()); + if (grant) + changed_user->granted_roles.grant(role_id); + else + changed_user->granted_roles.revoke(role_id); + return changed_user; + } + return entity_; + }; + + memory_storage.update(user_ids, update_func); + } + + // Actualize granted_role_* mappings. + if (grant) + { + if (!user_ids.empty()) + { + granted_role_names.insert_or_assign(role_id, role_name); + granted_role_ids.insert_or_assign(role_name, role_id); + } + } + else + { + granted_role_ids.erase(role_name); + granted_role_names.erase(role_id); + } +} + +void TokenAccessStorage::processRoleChange(const UUID & id, const AccessEntityPtr & entity) +{ + std::lock_guard lock(mutex); + const auto role = typeid_cast>(entity); + const auto it = granted_role_names.find(id); + + if (role) // Added or renamed a role. + { + const auto & new_role_name = role->getName(); + if (it != granted_role_names.end()) // Renamed a granted role. + { + const auto & old_role_name = it->second; + if (new_role_name != old_role_name) + { + // Revoke the old role first, then grant the new role. + applyRoleChangeNoLock(false /* revoke */, id, old_role_name); + applyRoleChangeNoLock(true /* grant */, id, new_role_name); + } + } + else // Added a role. + { + applyRoleChangeNoLock(true /* grant */, id, new_role_name); + } + } + else // Removed a role. + { + if (it != granted_role_names.end()) // Removed a granted role. + { + const auto & old_role_name = it->second; + applyRoleChangeNoLock(false /* revoke */, id, old_role_name); + } + } +} + +const char * TokenAccessStorage::getStorageType() const +{ + return STORAGE_TYPE; +} + +bool TokenAccessStorage::exists(const UUID & id) const +{ + std::lock_guard lock(mutex); + return memory_storage.exists(id); +} + +String TokenAccessStorage::getStorageParamsJSON() const +{ + std::lock_guard lock(mutex); + Poco::JSON::Object params_json; + + params_json.set("processor", provider_name); + + Poco::JSON::Array common_role_names_json; + for (const auto & role : common_role_names) + { + common_role_names_json.add(role); + } + params_json.set("roles", common_role_names_json); + + std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + oss.exceptions(std::ios::failbit); + Poco::JSON::Stringifier::stringify(params_json, oss); + + return oss.str(); +} + +bool TokenAccessStorage::areTokenCredentialsValidNoLock(const User & user, const Credentials & credentials, const ExternalAuthenticators & external_authenticators) const +{ + if (!credentials.isReady()) + return false; + + if (credentials.getUserName() != user.getName()) + return false; + + if (const auto * token_credentials = dynamic_cast(&credentials)) + return external_authenticators.checkAccessTokenCredentials(*token_credentials); + + return false; +} + +std::optional TokenAccessStorage::findImpl(AccessEntityType type, const String & name) const +{ + std::lock_guard lock(mutex); + return memory_storage.find(type, name); +} + + +std::vector TokenAccessStorage::findAllImpl(AccessEntityType type) const +{ + std::lock_guard lock(mutex); + return memory_storage.findAll(type); +} + +AccessEntityPtr TokenAccessStorage::readImpl(const UUID & id, bool throw_if_not_exists) const +{ + std::lock_guard lock(mutex); + return memory_storage.read(id, throw_if_not_exists); +} + +std::optional> TokenAccessStorage::readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const +{ + std::lock_guard lock(mutex); + return memory_storage.readNameWithType(id, throw_if_not_exists); +} + +void TokenAccessStorage::assignRolesNoLock(User & user, const std::set & external_roles, std::size_t external_roles_hash) const +{ + const auto & user_name = user.getName(); + auto & granted_roles = user.granted_roles; + + auto grant_role = [this, &user_name, &granted_roles] (const String & role_name, const bool common) + { + auto it = granted_role_ids.find(role_name); + if (it == granted_role_ids.end()) + { + if (const auto role_id = access_control.find(role_name)) + { + granted_role_names.insert_or_assign(*role_id, role_name); + it = granted_role_ids.insert_or_assign(role_name, *role_id).first; + } + } + + if (it != granted_role_ids.end()) + { + const auto & role_id = it->second; + granted_roles.grant(role_id); + } + else + { + LOG_WARNING(getLogger(), "Unable to grant {} role '{}' to user '{}': role not found", (common ? "common" : "mapped"), role_name, user_name); + } + }; + + external_role_hashes.erase(user_name); + granted_roles = {}; + const auto old_role_names = std::move(roles_per_users[user_name]); + + // Grant the common roles first. + for (const auto & role_name : common_role_names) + { + grant_role(role_name, true /* common */); + } + + // Grant the mapped external roles and actualize users_per_roles mapping. + // external_roles allowed to overlap with common_role_names. + for (const auto & role_name : external_roles) + { + grant_role(role_name, false /* mapped */); + users_per_roles[role_name].insert(user_name); + } + + // Cleanup users_per_roles and granted_role_* mappings. + for (const auto & old_role_name : old_role_names) + { + if (external_roles.contains(old_role_name)) + continue; + + const auto rit = users_per_roles.find(old_role_name); + if (rit == users_per_roles.end()) + continue; + + auto & user_names = rit->second; + user_names.erase(user_name); + + if (!user_names.empty()) + continue; + + users_per_roles.erase(rit); + + if (common_role_names.contains(old_role_name)) + continue; + + const auto iit = granted_role_ids.find(old_role_name); + if (iit == granted_role_ids.end()) + continue; + + const auto old_role_id = iit->second; + granted_role_names.erase(old_role_id); + granted_role_ids.erase(iit); + } + + // Actualize roles_per_users mapping and external_role_hashes cache. + if (external_roles.empty()) + roles_per_users.erase(user_name); + else + roles_per_users[user_name] = std::move(external_roles); + + external_role_hashes[user_name] = external_roles_hash; +} + +void TokenAccessStorage::updateAssignedRolesNoLock(const UUID & id, const String & user_name, const std::set & external_roles) const +{ + // No need to include common_role_names in this hash each time, since they don't change. + const auto external_roles_hash = boost::hash>{}(external_roles); + + // Map and grant the roles from scratch only if the list of external role has changed. + const auto it = external_role_hashes.find(user_name); + if (it != external_role_hashes.end() && it->second == external_roles_hash) + return; + + auto update_func = [this, &external_roles, external_roles_hash] (const AccessEntityPtr & entity_, const UUID &) -> AccessEntityPtr + { + if (auto user = typeid_cast>(entity_)) + { + auto changed_user = typeid_cast>(user->clone()); + assignRolesNoLock(*changed_user, external_roles, external_roles_hash); + return changed_user; + } + return entity_; + }; + + memory_storage.update(id, update_func); +} + + +std::optional TokenAccessStorage::authenticateImpl( + const Credentials & credentials, + const Poco::Net::IPAddress & address, + [[maybe_unused]] const ExternalAuthenticators & external_authenticators, + [[maybe_unused]] bool throw_if_user_not_exists, + bool /* allow_no_password */, + bool /* allow_plaintext_password */) const +{ + std::lock_guard lock(mutex); + auto id = memory_storage.find(credentials.getUserName()); + UserPtr user = id ? memory_storage.read(*id) : nullptr; + + std::shared_ptr new_user; + if (!user) + { + // User does not exist, so we create one, and will add it if authentication is successful. + new_user = std::make_shared(); + new_user->setName(credentials.getUserName()); + new_user->authentication_methods.emplace_back(AuthenticationType::JWT); + user = new_user; + } + + if (!isAddressAllowed(*user, address)) + throwAddressNotAllowed(address); + + const auto & token_credentials = typeid_cast(credentials); + + if (!external_authenticators.checkAccessTokenCredentialsByExactProcessor(token_credentials, provider_name)) + { + // Even though token itself may be valid (especially in case of a jwt token), authentication has just failed. + if (throw_if_user_not_exists) + throwNotFound(AccessEntityType::USER, credentials.getUserName()); + else + return {}; + } + + std::set external_roles = token_credentials.getGroups(); + + if (new_user) + { + assignRolesNoLock(*new_user, external_roles, boost::hash>{}(external_roles)); + id = memory_storage.insert(new_user); + } + else + { + // Just in case external_roles are changed. + updateAssignedRolesNoLock(*id, user->getName(), external_roles); + } + + if (id) + return AuthResult{ .user_id = *id, .authentication_data = AuthenticationData(AuthenticationType::JWT) }; + return std::nullopt; +} + + +} diff --git a/src/Access/TokenAccessStorage.h b/src/Access/TokenAccessStorage.h new file mode 100644 index 000000000000..0908f30ea519 --- /dev/null +++ b/src/Access/TokenAccessStorage.h @@ -0,0 +1,79 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace Poco +{ + namespace Util + { + class AbstractConfiguration; + } +} + + +namespace DB +{ +class AccessControl; + +/// Implementation of IAccessStorage which allows to import user data from oauth server using access token. +/// Normally, this should be unified with LDAPAccessStorage, but not done to minimize changes to code that is common with upstream. +class TokenAccessStorage : public IAccessStorage +{ +public: + static constexpr char STORAGE_TYPE[] = "token"; + + explicit TokenAccessStorage(const String & storage_name_, AccessControl & access_control_, const Poco::Util::AbstractConfiguration & config, const String & prefix); + ~TokenAccessStorage() override = default; + + // IAccessStorage implementations. + const char * getStorageType() const override; + String getStorageParamsJSON() const override; + bool isReadOnly() const override { return true; } + bool exists(const UUID & id) const override; + +private: // IAccessStorage implementations. + + mutable std::recursive_mutex mutex; // Note: Reentrance possible by internal role lookup via access_control + AccessControl & access_control; + const Poco::Util::AbstractConfiguration & config; + const String & prefix; + + String provider_name; + + std::set common_role_names; // role name that should be granted to all users at all times + mutable std::map external_role_hashes; + mutable std::map> users_per_roles; // role name -> user names (...it should be granted to; may but don't have to exist for common roles) + mutable std::map> roles_per_users; // user name -> role names (...that should be granted to it; may but don't have to include common roles) + mutable std::map granted_role_names; // (currently granted) role id -> its name + mutable std::map granted_role_ids; // (currently granted) role name -> its id + scope_guard role_change_subscription; + mutable MemoryAccessStorage memory_storage; + + void setConfiguration(); + void processRoleChange(const UUID & id, const AccessEntityPtr & entity); + + bool areTokenCredentialsValidNoLock(const User & user, const Credentials & credentials, const ExternalAuthenticators & external_authenticators) const; + + std::optional findImpl(AccessEntityType type, const String & name) const override; + std::vector findAllImpl(AccessEntityType type) const override; + AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override; + std::optional> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override; + std::optional authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, + [[maybe_unused]] const ExternalAuthenticators & external_authenticators, + [[maybe_unused]] bool throw_if_user_not_exists, + bool allow_no_password, bool allow_plaintext_password) const override; + + + void applyRoleChangeNoLock(bool grant, const UUID & role_id, const String & role_name); + void assignRolesNoLock(User & user, const std::set & external_roles, std::size_t external_roles_hash) const; + void updateAssignedRolesNoLock(const UUID & id, const String & user_name, const std::set & external_roles) const; +}; +} diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 882563d1a337..81c9f5a30ac6 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -14,6 +14,7 @@ #include #include #include +#include "Access/Credentials.h" #include #include #include @@ -131,6 +132,7 @@ namespace bool has_password_double_sha1_hex = config.has(user_config + ".password_double_sha1_hex"); bool has_ldap = config.has(user_config + ".ldap"); bool has_kerberos = config.has(user_config + ".kerberos"); + bool has_jwt = config.has(user_config + ".jwt"); const auto certificates_config = user_config + ".ssl_certificates"; bool has_certificates = config.has(certificates_config); @@ -142,18 +144,18 @@ namespace bool has_http_auth = config.has(http_auth_config); size_t num_password_fields = has_no_password + has_password_plaintext + has_password_sha256_hex + has_password_double_sha1_hex - + has_ldap + has_kerberos + has_certificates + has_ssh_keys + has_http_auth; + + has_ldap + has_kerberos + has_certificates + has_ssh_keys + has_http_auth + has_jwt; if (num_password_fields > 1) throw Exception(ErrorCodes::BAD_ARGUMENTS, "More than one field of 'password', 'password_sha256_hex', " "'password_double_sha1_hex', 'no_password', 'ldap', 'kerberos', 'ssl_certificates', 'ssh_keys', " - "'http_authentication' are used to specify authentication info for user {}. " + "'http_authentication', 'jwt' are used to specify authentication info for user {}. " "Must be only one of them.", user_name); if (num_password_fields < 1) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Either 'password' or 'password_sha256_hex' " "or 'password_double_sha1_hex' or 'no_password' or 'ldap' or 'kerberos " - "or 'ssl_certificates' or 'ssh_keys' or 'http_authentication' must be specified for user {}.", user_name); + "or 'ssl_certificates' or 'ssh_keys' or 'http_authentication' or 'jwt' must be specified for user {}.", user_name); if (has_password_plaintext) { @@ -268,6 +270,10 @@ namespace auto scheme = config.getString(http_auth_config + ".scheme"); user->authentication_methods.back().setHTTPAuthenticationScheme(parseHTTPAuthenticationScheme(scheme)); } + else if (has_jwt) + { + user->authentication_methods.emplace_back(AuthenticationType::JWT); + } else { user->authentication_methods.emplace_back(); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 66d323d27349..e9d1d71edb34 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -393,6 +393,7 @@ target_link_libraries(clickhouse_common_io ch_contrib::zlib pcg_random Poco::Foundation + ch_contrib::jwt-cpp ) if (TARGET ch_contrib::libfiu) diff --git a/src/Parsers/Access/ASTAuthenticationData.cpp b/src/Parsers/Access/ASTAuthenticationData.cpp index 20655757499a..fb0667069373 100644 --- a/src/Parsers/Access/ASTAuthenticationData.cpp +++ b/src/Parsers/Access/ASTAuthenticationData.cpp @@ -106,8 +106,11 @@ void ASTAuthenticationData::formatImpl(WriteBuffer & ostr, const FormatSettings } case AuthenticationType::JWT: { - prefix = "CLAIMS"; - parameter = true; + if (!children.empty()) + { + prefix = "CLAIMS"; + parameter = true; + } break; } case AuthenticationType::LDAP: diff --git a/src/Parsers/Access/ASTCreateUserQuery.h b/src/Parsers/Access/ASTCreateUserQuery.h index 134224849381..7f9aeea5f021 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.h +++ b/src/Parsers/Access/ASTCreateUserQuery.h @@ -18,7 +18,7 @@ class ASTAuthenticationData; /** CREATE USER [IF NOT EXISTS | OR REPLACE] name - * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] + * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']|{WITH jwt [CLAIMS 'json_object']}}] * [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...]] * [DEFAULT DATABASE database | NONE] @@ -27,7 +27,7 @@ class ASTAuthenticationData; * * ALTER USER [IF EXISTS] name * [RENAME TO new_name] - * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] + * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']|{WITH jwt [CLAIMS 'json_object']}}] * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] * [DEFAULT DATABASE database | NONE] diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index 91e7f16606f5..6b5ae6a2325e 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -75,6 +75,7 @@ namespace bool expect_ssl_cert_subjects = false; bool expect_public_ssh_key = false; bool expect_http_auth_server = false; + bool expect_claims = false; auto parse_non_password_based_type = [&](auto check_type) { @@ -92,6 +93,8 @@ namespace expect_public_ssh_key = true; else if (check_type == AuthenticationType::HTTP) expect_http_auth_server = true; + else if (check_type == AuthenticationType::JWT) + expect_claims = true; else if (check_type != AuthenticationType::NO_PASSWORD) expect_password = true; @@ -147,6 +150,7 @@ namespace ASTPtr http_auth_scheme; ASTPtr ssl_cert_subjects; std::optional ssl_cert_subject_type; + ASTPtr jwt_claims; if (expect_password || expect_hash) { @@ -211,6 +215,14 @@ namespace return false; } } + else if (expect_claims) + { + if (ParserKeyword{Keyword::CLAIMS}.ignore(pos, expected)) + { + if (!ParserStringAndSubstitution{}.parse(pos, jwt_claims, expected)) + return false; + } + } auth_data = std::make_shared(); @@ -236,6 +248,9 @@ namespace if (http_auth_scheme) auth_data->children.push_back(std::move(http_auth_scheme)); + if (jwt_claims) + auth_data->children.push_back(std::move(jwt_claims)); + parseValidUntil(pos, expected, auth_data->valid_until); return true; diff --git a/src/Parsers/Access/ParserCreateUserQuery.h b/src/Parsers/Access/ParserCreateUserQuery.h index 4dfff8713d76..5f4cfcd6c45f 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.h +++ b/src/Parsers/Access/ParserCreateUserQuery.h @@ -7,7 +7,7 @@ namespace DB { /** Parses queries like * CREATE USER [IF NOT EXISTS | OR REPLACE] name - * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] + * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}|{WITH jwt}] * [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...]] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] @@ -15,7 +15,7 @@ namespace DB * * ALTER USER [IF EXISTS] name * [RENAME TO new_name] - * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] + * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}|{WITH jwt}] * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] * [ADD|MODIFY SETTINGS variable [=value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] [,...] ] diff --git a/src/Parsers/CommonParsers.h b/src/Parsers/CommonParsers.h index 62b5ca135cd8..99fea1f4afec 100644 --- a/src/Parsers/CommonParsers.h +++ b/src/Parsers/CommonParsers.h @@ -84,6 +84,7 @@ namespace DB MR_MACROS(CHECK_TABLE, "CHECK TABLE") \ MR_MACROS(CHECK_GRANT, "CHECK GRANT")\ MR_MACROS(CHECK, "CHECK") \ + MR_MACROS(CLAIMS, "CLAIMS") \ MR_MACROS(CLEANUP, "CLEANUP") \ MR_MACROS(CLEAR_COLUMN, "CLEAR COLUMN") \ MR_MACROS(CLEAR_INDEX, "CLEAR INDEX") \ diff --git a/src/Server/HTTP/authenticateUserByHTTP.cpp b/src/Server/HTTP/authenticateUserByHTTP.cpp index f96ffc2cb6a9..ac312d111c35 100644 --- a/src/Server/HTTP/authenticateUserByHTTP.cpp +++ b/src/Server/HTTP/authenticateUserByHTTP.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -19,7 +20,7 @@ #include #endif - +const String BEARER_PREFIX = "bearer "; namespace DB { @@ -80,6 +81,8 @@ bool authenticateUserByHTTP( bool has_http_credentials = request.hasCredentials() && request.get("Authorization") != "never"; bool has_credentials_in_query_params = params.has("user") || params.has("password"); + std::string jwt_token = request.get("X-ClickHouse-JWT-Token", request.get("Authorization", (params.has("token") ? BEARER_PREFIX + params.get("token") : ""))); + std::string spnego_challenge; SSLCertificateSubjects certificate_subjects; @@ -152,7 +155,7 @@ bool authenticateUserByHTTP( if (spnego_challenge.empty()) throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Invalid authentication: SPNEGO challenge is empty"); } - else + else if (Poco::icompare(scheme, "Bearer") < 0) { throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Invalid authentication: '{}' HTTP Authorization scheme is not supported", scheme); } @@ -208,6 +211,21 @@ bool authenticateUserByHTTP( { current_credentials = std::make_unique(*config_credentials); } + else if (!jwt_token.empty() && Poco::toLower(jwt_token).starts_with(BEARER_PREFIX)) + { + current_credentials = std::make_unique(jwt_token.substr(BEARER_PREFIX.length())); + + if (!static_cast(*current_credentials).isJWT()) + { + /// In case the token is an access token, we need to resolve it to get user name. + /// This is why (for now) the check is made twice: here and later in authentication. + global_context->getAccessControl().getExternalAuthenticators().checkAccessTokenCredentials( + static_cast(*current_credentials)); + } + if (!current_credentials->isReady()) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, + "Failed to authenticate with access token: token invalid or none of `access_token_processors` was able to resolve it."); + } else // I.e., now using user name and password strings ("Basic"). { if (!current_credentials) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index acf615f7704b..ddeb808a504f 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -1707,6 +1708,10 @@ void TCPHandler::receiveHello() if (is_ssh_based_auth) user.erase(0, std::string_view(EncodedUserInfo::SSH_KEY_AUTHENTICAION_MARKER).size()); + is_jwt_based_auth = user.starts_with(EncodedUserInfo::JWT_AUTHENTICAION_MARKER); + if (is_jwt_based_auth) + user.erase(0, std::string_view(EncodedUserInfo::JWT_AUTHENTICAION_MARKER).size()); + session = makeSession(); const auto & client_info = session->getClientInfo(); @@ -1794,6 +1799,20 @@ void TCPHandler::receiveHello() } #endif + if (is_jwt_based_auth) + { + auto credentials = TokenCredentials(password); + + if (!credentials.isJWT()) + { + /// In case the token is an access token, we need to resolve it to get user name. + /// This is why (for now) the check is made twice: here and later in authentication. + server.context()->getAccessControl().getExternalAuthenticators().checkAccessTokenCredentials(credentials); + } + session->authenticate(credentials, getClientAddress(client_info)); + return; + } + session->authenticate(user, password, getClientAddress(client_info)); } diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index 98170c140be2..aec62ee4ef15 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -230,6 +230,7 @@ class TCPHandler : public Poco::Net::TCPServerConnection String default_database; bool is_ssh_based_auth = false; /// authentication is via SSH pub-key challenge + bool is_jwt_based_auth = false; /// authentication is via JWT /// For inter-server secret (remote_server.*.secret) bool is_interserver_mode = false; bool is_interserver_authenticated = false; diff --git a/tests/docker_scripts/fasttest_runner.sh b/tests/docker_scripts/fasttest_runner.sh index 1dbbd49a28be..7bae54ade257 100755 --- a/tests/docker_scripts/fasttest_runner.sh +++ b/tests/docker_scripts/fasttest_runner.sh @@ -157,7 +157,7 @@ function clone_submodules contrib/libfiu contrib/incbin contrib/yaml-cpp - contrib/corrosion + contrib/jwt-cpp ) git submodule sync diff --git a/tests/integration/test_jwt_auth/__init__.py b/tests/integration/test_jwt_auth/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_jwt_auth/configs/users.xml b/tests/integration/test_jwt_auth/configs/users.xml new file mode 100644 index 000000000000..b3d3372ebaa9 --- /dev/null +++ b/tests/integration/test_jwt_auth/configs/users.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + default + default + + + diff --git a/tests/integration/test_jwt_auth/configs/validators.xml b/tests/integration/test_jwt_auth/configs/validators.xml new file mode 100644 index 000000000000..1522937629cd --- /dev/null +++ b/tests/integration/test_jwt_auth/configs/validators.xml @@ -0,0 +1,24 @@ + + + + + HS256 + my_secret + false + + + + hs256 + other_secret + false + + + + {"keys": [{"kty": "RSA", "alg": "rs256", "kid": "mykid", "n": "lICGC8S5pObyASih5qfmwuclG0oKsbzY2z9vgwqyhTYQOWcqYcTjVV4aQ30qb6E0-5W6rJ-jx9zx6GuAEGMiG_aWJEdbUAMGp-L1kz4lrw5U6GlwoZIvk4wqoRwsiyc-mnDMQAmiZLBNyt3wU6YnKgYmb4O1cSzcZ5HMbImJpj4tpYjqnIazvYMn_9Pxjkl0ezLCr52av0UkWHro1H4QMVfuEoNmHuWPww9jgHn-I-La0xdOhRpAa0XnJi65dXZd4330uWjeJwt413yz881uS4n1OLOGKG8ImDcNlwU_guyvk0n0aqT0zkOAPp9_yYo13MPWmiRCfOX8ozdN7VDIJw", "e": "AQAB"}]} + + + + http://resolver:8080/.well-known/jwks.json + + + diff --git a/tests/integration/test_jwt_auth/helpers/generate_private_key.py b/tests/integration/test_jwt_auth/helpers/generate_private_key.py new file mode 100644 index 000000000000..7b54fa63368b --- /dev/null +++ b/tests/integration/test_jwt_auth/helpers/generate_private_key.py @@ -0,0 +1,21 @@ +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.backends import default_backend + +# Generate RSA private key +private_key = rsa.generate_private_key( + public_exponent=65537, + key_size=2048, # Key size of 2048 bits + backend=default_backend() +) + +# Save the private key to a PEM file +pem_private_key = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() # You can add encryption if needed +) + +# Write the private key to a file +with open("new_private_key", "wb") as pem_file: + pem_file.write(pem_private_key) diff --git a/tests/integration/test_jwt_auth/helpers/jwt_jwk.py b/tests/integration/test_jwt_auth/helpers/jwt_jwk.py new file mode 100644 index 000000000000..265882efce76 --- /dev/null +++ b/tests/integration/test_jwt_auth/helpers/jwt_jwk.py @@ -0,0 +1,113 @@ +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization + +import base64 +import json +import jwt + + +""" +Only RS* family algorithms are supported!!! +""" +with open("./private_key_2", "rb") as key_file: + private_key = serialization.load_pem_private_key( + key_file.read(), + password=None, + ) + + +public_key = private_key.public_key() + + +def to_base64_url(data): + return base64.urlsafe_b64encode(data).decode("utf-8").rstrip("=") + + +def rsa_key_to_jwk(private_key=None, public_key=None): + if private_key: + # Convert the private key to its components + private_numbers = private_key.private_numbers() + public_numbers = private_key.public_key().public_numbers() + + jwk = { + "kty": "RSA", + "alg": "RS512", + "kid": "mykid", + "n": to_base64_url( + public_numbers.n.to_bytes( + (public_numbers.n.bit_length() + 7) // 8, byteorder="big" + ) + ), + "e": to_base64_url( + public_numbers.e.to_bytes( + (public_numbers.e.bit_length() + 7) // 8, byteorder="big" + ) + ), + "d": to_base64_url( + private_numbers.d.to_bytes( + (private_numbers.d.bit_length() + 7) // 8, byteorder="big" + ) + ), + "p": to_base64_url( + private_numbers.p.to_bytes( + (private_numbers.p.bit_length() + 7) // 8, byteorder="big" + ) + ), + "q": to_base64_url( + private_numbers.q.to_bytes( + (private_numbers.q.bit_length() + 7) // 8, byteorder="big" + ) + ), + "dp": to_base64_url( + private_numbers.dmp1.to_bytes( + (private_numbers.dmp1.bit_length() + 7) // 8, byteorder="big" + ) + ), + "dq": to_base64_url( + private_numbers.dmq1.to_bytes( + (private_numbers.dmq1.bit_length() + 7) // 8, byteorder="big" + ) + ), + "qi": to_base64_url( + private_numbers.iqmp.to_bytes( + (private_numbers.iqmp.bit_length() + 7) // 8, byteorder="big" + ) + ), + } + elif public_key: + # Convert the public key to its components + public_numbers = public_key.public_numbers() + + jwk = { + "kty": "RSA", + "alg": "RS512", + "kid": "mykid", + "n": to_base64_url( + public_numbers.n.to_bytes( + (public_numbers.n.bit_length() + 7) // 8, byteorder="big" + ) + ), + "e": to_base64_url( + public_numbers.e.to_bytes( + (public_numbers.e.bit_length() + 7) // 8, byteorder="big" + ) + ), + } + else: + raise ValueError("You must provide either a private or public key.") + + return jwk + + +# Convert to JWK +jwk_private = rsa_key_to_jwk(private_key=private_key) +jwk_public = rsa_key_to_jwk(public_key=public_key) + +print(f"Private JWK:\n{json.dumps(jwk_private)}\n") +print(f"Public JWK:\n{json.dumps(jwk_public)}\n") + +payload = {"sub": "jwt_user", "iss": "test_iss"} + +# Create a JWT +token = jwt.encode(payload, private_key, headers={"kid": "mykid"}, algorithm="RS512") +print(f"JWT:\n{token}") diff --git a/tests/integration/test_jwt_auth/helpers/jwt_static_secret.py b/tests/integration/test_jwt_auth/helpers/jwt_static_secret.py new file mode 100644 index 000000000000..5f1c7e0340af --- /dev/null +++ b/tests/integration/test_jwt_auth/helpers/jwt_static_secret.py @@ -0,0 +1,43 @@ +import jwt +import datetime + + +def create_jwt( + payload: dict, secret: str, algorithm: str = "HS256", expiration_minutes: int = None +) -> str: + """ + Create a JWT using a static secret and a specified encryption algorithm. + + :param payload: The payload to include in the JWT (as a dictionary). + :param secret: The secret key used to sign the JWT. + :param algorithm: The encryption algorithm to use (default is 'HS256'). + :param expiration_minutes: The time until the token expires (default is 60 minutes). + :return: The encoded JWT as a string. + """ + if expiration_minutes: + expiration = datetime.datetime.utcnow() + datetime.timedelta( + minutes=expiration_minutes + ) + payload["exp"] = expiration + + return jwt.encode(payload, secret, algorithm=algorithm) + + +if __name__ == "__main__": + secret = "my_secret" + payload = {"sub": "jwt_user"} # `sub` must contain user name + + """ + Supported algorithms: + | HMSC | RSA | ECDSA | PSS | EdDSA | + | ----- | ----- | ------ | ----- | ------- | + | HS256 | RS256 | ES256 | PS256 | Ed25519 | + | HS384 | RS384 | ES384 | PS384 | Ed448 | + | HS512 | RS512 | ES512 | PS512 | | + | | | ES256K | | | + And None + """ + algorithm = "HS256" + + token = create_jwt(payload, secret, algorithm) + print(f"Generated JWT: {token}") diff --git a/tests/integration/test_jwt_auth/helpers/private_key_1 b/tests/integration/test_jwt_auth/helpers/private_key_1 new file mode 100644 index 000000000000..a076a86e17a4 --- /dev/null +++ b/tests/integration/test_jwt_auth/helpers/private_key_1 @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAlICGC8S5pObyASih5qfmwuclG0oKsbzY2z9vgwqyhTYQOWcq +YcTjVV4aQ30qb6E0+5W6rJ+jx9zx6GuAEGMiG/aWJEdbUAMGp+L1kz4lrw5U6Glw +oZIvk4wqoRwsiyc+mnDMQAmiZLBNyt3wU6YnKgYmb4O1cSzcZ5HMbImJpj4tpYjq +nIazvYMn/9Pxjkl0ezLCr52av0UkWHro1H4QMVfuEoNmHuWPww9jgHn+I+La0xdO +hRpAa0XnJi65dXZd4330uWjeJwt413yz881uS4n1OLOGKG8ImDcNlwU/guyvk0n0 +aqT0zkOAPp9/yYo13MPWmiRCfOX8ozdN7VDIJwIDAQABAoIBADZfiLUuZrrWRK3f +7sfBmmCquY9wYNILT2uXooDcndjgnrgl6gK6UHKlbgBgB/WvlPK5NAyYtyMq5vgu +xEk7wvVyKC9IYUq+kOVP2JL9IlcibDxcvvypxfnETKeI5VZeHDH4MxEPdgJf+1vY +P3KhV52vestB8mFqB5l0bOEgyuGvO3/3D1JjOnFLS/K2vOj8D/KDRmwXRCcGHTxj +dj3wJH4UbCIsLgiaQBPkFmTteJDICb+7//6YQuB0t8sR/DZS9Z0GWcfy04Cp/m/E +4rRoTNz80MbbU9+k0Ly360SxPizcjpPYSRSD025i8Iqv8jvelq7Nzg69Kubc0KfN +mMrRdMECgYEAz4b7+OX+aO5o2ZQS+fHc8dyWc5umC+uT5xrUm22wZLYA5O8x0Rgj +vdO/Ho/XyN/GCyvNNV2rI2+CBTxez6NqesGDEmJ2n7TQ03xXLCVsnwVz694sPSMO +pzTbU6e42jvDo5DMPDv0Pg1CVQuM9ka6wb4DcolMyDql6QddY3iXHBkCgYEAtzAl +xEAABqdFAnCs3zRf9EZphGJiJ4gtoWmCxQs+IcrfyBNQCy6GqrzJOZ7fQiEoAeII +V0JmsNcnx3U1W0lp8N+1QNZoB4fOWXaX08BvOEe7gbJ6Xl5t52j792vQp1txpBhE +UDhz8m5R9i5qb3BzrYBiSTfak0Pq56Xw3jRDjj8CgYEAqX2QS07kQqT8gz85ZGOR +1QMY6aCks7WaXTR/kdW7K/Wts0xb/m7dugq3W+mVDh0c7UC/36b5v/4xTb9pm+HW +dB2ZxCkgwvz1VNSHiamjFhlo/Km+rcv1CsDTpHYmNi57cRowg71flFJV64l8fiN0 +IgnjXOcgC6RCnpiCQFxb5fkCgYB+Zq2YleSuspqOjXrrZPNU1YUXgN9jkbaSqwA9 +wH01ygvRvWm83XS0uSFMLhC1S7WUXwgMVdgP69YZ7glMHQMJ3wLtY0RS9eVvm8I1 +rZHQzsZWPvXqydOiGrHJzs4hvJpUdR4mEF4JCRBrAyoUDQ70yCKJjQ24EeQzxS/H +015N9wKBgB8DdFPvKXyygTMnBoZdpAhkE/x3TTi7DsLBxj7QxKmSHzlHGz0TubIB +m5/p9dGawQNzD4JwASuY5r4lKXmvYr+4TQPLq6c7EnoIZSwLdge+6PDhnDWJzvk1 +S/RuHWW4FKGzBStTmstG3m0xzxTMnQkV3kPimMim3I3VsxxeGEdq +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/test_jwt_auth/helpers/private_key_2 b/tests/integration/test_jwt_auth/helpers/private_key_2 new file mode 100644 index 000000000000..d0d1576f2017 --- /dev/null +++ b/tests/integration/test_jwt_auth/helpers/private_key_2 @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA0RRsKcZ5j9UckjioG4Phvav3dkg2sXP6tQ7ug0yowAo/u2Hf +fB+1OjKuhWTpA3E3YkMKj0RrT+tuUpmZEXqCAipEV7XcfCv3o7Poa7HTq1ti/abV +wT/KyfGjoNBBSJH4LTNAyo2J8ySKSDtpAEU52iL7s40Ra6I0vqp7/aRuPF5M4zcH +zN3zarG5EfSVSG1+gTkaRv8XJbra0IeIINmKv0F4++ww8ZxXTR6cvI+MsArUiAPw +zf7s5dMR4DNRG6YNTrPA0pTOqQE9sRPd62XsfU08plYm27naOUZO5avIPl1YO5I6 +Gi4kPdTvv3WFIy+QvoKoPhPCaD6EbdBpe8BbTQIDAQABAoIBABghJsCFfucKHdOE +RWZziHx22cblW6aML41wzTcLBFixdhx+lafCEwzF551OgZPbn5wwB4p0R3xAPAm9 +X0yEmnd8gEmtG+aavmg+rZ6sNbULhXenpvi4D4PR5uP61OX2rrEsvpgB0L9mYq0m +ah5VXvFdYzYcHDwTSsoMa+XgcbZ2qCW6Si3jnbBA1TPIJS5GjfPUQlu9g2FKQL5H +tlJ7L4Wq39zkueS6LH7kEXOoM+jHgA8F4f7MIrajmilYqnuXanVcMV3+K/6FvH2B +VBiLggG3CerhB3QyEvZBshvEvvcyRff2NK64CGr/xrAElj4cPHk/E499M1uvUXjE +boCrD+ECgYEA9LvLljf59h8WWF4bKQZGNKprgFdQIZ2iCEf+VGdGWt/mNg+LyXyn +3gS/vReON1eaMuEGklZM4Guh/ZPhsPaNmlu16PjmeYTIW1vQTHiO3KR7tAmWep70 +w+gVxDDzuRvBkuDF5oQsZnD3Ri9I7r+J5y9OhyZUsDXe/LJARivF3x0CgYEA2rRx +wl4mfuYmikvcO8I4vuKXcK1UyYmZQLhp6EHKfhSVgrt7XsstZX9AP2OxUUAocRks +e6vU/sKUSni7TQrZzAZHc8JXonDgmCqoMPBXIuUncvysGR1kmgVIbN8ISPKJuZoV +8Dbj3fQfHZ0g0R+mUcuZ+xBO5CKcjPWHZXZoxfECgYAQ/5o8bNbnyXD74k1wpAbs +UYn1+BqQuyot+RIpOqMgXLzYtGu5Kvdd7GaE88XlAiirsAWM1IGydMdjnYnniLh9 +KDGSZPddKWPhNJdbOGRz3tjYwHG7Qp8tnEkmv1+uU8c2NHaKdFPBKceDEHW4X4Vs +kVSa/oaTVqqOUrM0LIYp4QKBgQCW1aIriiGEnZhxAvbGJCJczAvkAzcZtBOFBmrM +ayuLnwiqXEEu1HPfr06RKWFuhxAdSF5cgNrqRSpe3jtXXCdvxdjbpmooNy8+4xSS +g/+kqmR1snvC6nmqnAAiTgP5w4RnBDUjMcggGLCpDOhIMkrT2Na+x7WRM6nCsceK +m4qREQKBgEWqdb/QkOMvvKAz2DPDeSrwlTyisrZu1G/86uE3ESb97DisPK+TF2Ts +r4RGUlKL79W3j5xjvIvqGEEDLC+8QKpay9OYXk3lbViPGB8akWMSP6Tw/8AedhVu +sjFqcBEFGOELwm7VjAcDeP6bXeXibFe+rysBrfFHUGllytCmNoAV +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/test_jwt_auth/jwks_server/server.py b/tests/integration/test_jwt_auth/jwks_server/server.py new file mode 100644 index 000000000000..96e07f02335e --- /dev/null +++ b/tests/integration/test_jwt_auth/jwks_server/server.py @@ -0,0 +1,33 @@ +import sys + +from bottle import response, route, run + + +@route("/.well-known/jwks.json") +def server(): + result = { + "keys": [ + { + "kty": "RSA", + "alg": "RS512", + "kid": "mykid", + "n": "0RRsKcZ5j9UckjioG4Phvav3dkg2sXP6tQ7ug0yowAo_u2HffB-1OjKuhWTpA3E3YkMKj0RrT-tuUpmZEXqCAipEV7XcfCv3o" + "7Poa7HTq1ti_abVwT_KyfGjoNBBSJH4LTNAyo2J8ySKSDtpAEU52iL7s40Ra6I0vqp7_aRuPF5M4zcHzN3zarG5EfSVSG1-gT" + "kaRv8XJbra0IeIINmKv0F4--ww8ZxXTR6cvI-MsArUiAPwzf7s5dMR4DNRG6YNTrPA0pTOqQE9sRPd62XsfU08plYm27naOUZ" + "O5avIPl1YO5I6Gi4kPdTvv3WFIy-QvoKoPhPCaD6EbdBpe8BbTQ", + "e": "AQAB"}, + ] + } + response.status = 200 + response.content_type = "application/json" + return result + + +@route("/") +def ping(): + response.content_type = "text/plain" + response.set_header("Content-Length", 2) + return "OK" + + +run(host="0.0.0.0", port=int(sys.argv[1])) diff --git a/tests/integration/test_jwt_auth/test.py b/tests/integration/test_jwt_auth/test.py new file mode 100644 index 000000000000..6a1e1fe68e72 --- /dev/null +++ b/tests/integration/test_jwt_auth/test.py @@ -0,0 +1,101 @@ +import os +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.mock_servers import start_mock_servers + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance( + "instance", + main_configs=["configs/validators.xml"], + user_configs=["configs/users.xml"], + with_minio=True, + # We actually don't need minio, but we need to run dummy resolver + # (a shortcut not to change cluster.py in a more unclear way, TBC later). +) +client = cluster.add_instance( + "client", +) + + +def run_jwks_server(): + script_dir = os.path.join(os.path.dirname(__file__), "jwks_server") + start_mock_servers( + cluster, + script_dir, + [ + ("server.py", "resolver", "8080"), + ], + ) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + run_jwks_server() + yield cluster + finally: + cluster.shutdown() + + +def curl_with_jwt(token, ip, https=False): + http_prefix = "https" if https else "http" + curl = f'curl -H "X-ClickHouse-JWT-Token: Bearer {token}" "{http_prefix}://{ip}:8123/?query=SELECT%20currentUser()"' + return curl + + +# See helpers/ directory if you need to re-create tokens (or understand how they are created) +def test_static_key(started_cluster): + res = client.exec_in_container( + [ + "bash", + "-c", + curl_with_jwt( + token="eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJqd3RfdXNlciJ9." + "kfivQ8qD_oY0UvihydeadD7xvuiO3zSmhFOc_SGbEPQ", + ip=cluster.get_instance_ip(instance.name), + ), + ] + ) + assert res == "jwt_user\n" + + +def test_static_jwks(started_cluster): + res = client.exec_in_container( + [ + "bash", + "-c", + curl_with_jwt( + token="eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Im15a2lkIn0." + "eyJzdWIiOiJqd3RfdXNlciIsImlzcyI6InRlc3RfaXNzIn0." + "CUioyRc_ms75YWkUwvPgLvaVk2Wmj8RzgqDALVd9LWUzCL5aU4yc_YaA3qnG_NoHd0uUF4FUjLxiocRoKNEgsE2jj7g_" + "wFMC5XHSHuFlfIZjovObXQEwGcKpXO2ser7ANu3k2jBC2FMpLfr_sZZ_GYSnqbp2WF6-l0uVQ0AHVwOy4x1Xkawiubkg" + "W2I2IosaEqT8QNuvvFWLWc1k-dgiNp8k6P-K4D4NBQub0rFlV0n7AEKNdV-_AEzaY_IqQT0sDeBSew_mdR0OH_N-6-" + "FmWWIroIn2DQ7pq93BkI7xdkqnxtt8RCWkCG8JLcoeJt8sHh7uTKi767loZJcPPNaxKA", + ip=cluster.get_instance_ip(instance.name), + ), + ] + ) + assert res == "jwt_user\n" + + +def test_jwks_server(started_cluster): + res = client.exec_in_container( + [ + "bash", + "-c", + curl_with_jwt( + token="eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzUxMiIsImtpZCI6Im15a2lkIn0." + "eyJzdWIiOiJqd3RfdXNlciIsImlzcyI6InRlc3RfaXNzIn0.MjegqrrVyrMMpkxIM-J_q-" + "Sw68Vk5xZuFpxecLLMFs5qzvnh0jslWtyRfi-ANJeJTONPZM5m0yP1ITt8BExoHWobkkR11bXz0ylYEIOgwxqw" + "36XhL2GkE17p-wMvfhCPhGOVL3b7msDRUKXNN48aAJA-NxRbQFhMr-eEx3HsrZXy17Qc7z-" + "0dINe355kzAInGp6gMk3uksAlJ3vMODK8jE-WYFqXusr5GFhXubZXdE2mK0mIbMUGisOZhZLc4QVwvUsYDLBCgJ2RHr5vm" + "jp17j_ZArIedUJkjeC4o72ZMC97kLVnVw94QJwNvd4YisxL6A_mWLTRq9FqNLD4HmbcOQ", + ip=cluster.get_instance_ip(instance.name), + ), + ] + ) + assert res == "jwt_user\n" diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index d6c785a7bc74..bacccfc93bae 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -277,6 +277,8 @@ DoubleDelta Doxygen Dresseler Durre +ECDSA +EdDSA ECMA ETag EachRow @@ -506,6 +508,8 @@ JoinAlgorithm JoinStrictness JumpConsistentHash Jupyter +jwks +JWKS KDevelop KafkaAssignedPartitions KafkaBackgroundReads @@ -3138,6 +3142,7 @@ uuid uuids uuidv vCPU +validators varPop varPopStable varSamp @@ -3155,6 +3160,8 @@ vectorscan vendoring verificationDepth verificationMode +verifier +verifiers versionedcollapsingmergetree vhost virtualized From 58b288d20a85d8bf2e8120464bd51a1fe5938706 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Fri, 7 Feb 2025 10:08:16 +0000 Subject: [PATCH 2/5] resolve tokenCredentials on creation --- src/Access/AccessTokenProcessor.cpp | 155 +++++++++++---------- src/Access/AccessTokenProcessor.h | 14 +- src/Access/Authentication.cpp | 13 +- src/Access/Common/JWKSProvider.cpp | 3 - src/Access/Credentials.cpp | 33 +---- src/Access/Credentials.h | 5 - src/Access/ExternalAuthenticators.cpp | 52 +++++-- src/Access/ExternalAuthenticators.h | 4 +- src/Access/IAccessStorage.cpp | 4 + src/Access/JWTValidator.cpp | 8 +- src/Access/JWTValidator.h | 2 +- src/Access/MultipleAccessStorage.cpp | 1 - src/Access/TokenAccessStorage.cpp | 24 ++-- src/Server/HTTP/authenticateUserByHTTP.cpp | 17 +-- src/Server/TCPHandler.cpp | 11 +- 15 files changed, 167 insertions(+), 179 deletions(-) diff --git a/src/Access/AccessTokenProcessor.cpp b/src/Access/AccessTokenProcessor.cpp index 36c78f66588c..3d7b04ebed6a 100644 --- a/src/Access/AccessTokenProcessor.cpp +++ b/src/Access/AccessTokenProcessor.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -38,13 +39,49 @@ namespace return value.get(); } + + picojson::object getObjectFromURI(const Poco::URI & uri, const String & token = "") + { + Poco::Net::HTTPResponse response; + std::ostringstream responseString; + + Poco::Net::HTTPRequest request{Poco::Net::HTTPRequest::HTTP_GET, uri.getPathAndQuery()}; + if (!token.empty()) + request.add("Authorization", "Bearer " + token); + + if (uri.getScheme() == "https") { + Poco::Net::HTTPSClientSession session(uri.getHost(), uri.getPort()); + session.sendRequest(request); + Poco::StreamCopier::copyStream(session.receiveResponse(response), responseString); + } + else + { + Poco::Net::HTTPClientSession session(uri.getHost(), uri.getPort()); + session.sendRequest(request); + Poco::StreamCopier::copyStream(session.receiveResponse(response), responseString); + } + + if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, + "Failed to get user info by access token, code: {}, reason: {}", response.getStatus(), + response.getReason()); + + try + { + return parseJSON(responseString.str()); + } + catch (const std::runtime_error & e) + { + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to parse server response: {}", e.what()); + } + } } -const Poco::URI GoogleAccessTokenProcessor::token_info_uri = Poco::URI("https://www.googleapis.com/oauth2/v3/tokeninfo"); +[[maybe_unused]] const Poco::URI GoogleAccessTokenProcessor::token_info_uri = Poco::URI("https://www.googleapis.com/oauth2/v3/tokeninfo"); const Poco::URI GoogleAccessTokenProcessor::user_info_uri = Poco::URI("https://www.googleapis.com/oauth2/v3/userinfo"); -const Poco::URI AzureAccessTokenProcessor::user_info_uri = Poco::URI("https://graph.microsoft.com/v1.0/me"); +const Poco::URI AzureAccessTokenProcessor::user_info_uri = Poco::URI("https://graph.microsoft.com/oidc/userinfo"); std::unique_ptr IAccessTokenProcessor::parseTokenProcessor( @@ -93,19 +130,24 @@ bool GoogleAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cre { const String & token = credentials.getToken(); - String user_name = tryGetUserName(token); - if (user_name.empty()) - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to authenticate with access token"); - auto user_info = getUserInfo(token); + String user_name = user_info["sub"]; if (email_regex.ok()) { if (!user_info.contains("email")) - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to authenticate user {}: e-mail address not found in user data.", user_name); + { + LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to validate {} by e-mail", name, user_name); + return false; + } + /// Additionally validate user email to match regex from config. if (!RE2::FullMatch(user_info["email"], email_regex)) - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to authenticate user {}: e-mail address is not permitted.", user_name); + { + LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to authenticate user {}: e-mail address is not permitted.", name, user_name); + return false; + } + } /// Credentials are passed as const everywhere up the flow, so we have to comply, /// in this case const_cast looks acceptable. @@ -115,100 +157,61 @@ bool GoogleAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cre return true; } -String GoogleAccessTokenProcessor::tryGetUserName(const String & token) const -{ - Poco::Net::HTTPSClientSession session(token_info_uri.getHost(), token_info_uri.getPort()); - - Poco::Net::HTTPRequest request{Poco::Net::HTTPRequest::HTTP_GET, token_info_uri.getPathAndQuery()}; - request.add("Authorization", "Bearer " + token); - session.sendRequest(request); - - Poco::Net::HTTPResponse response; - std::istream & responseStream = session.receiveResponse(response); - - if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK) - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to resolve access token, code: {}, reason: {}", response.getStatus(), response.getReason()); - - std::ostringstream responseString; - Poco::StreamCopier::copyStream(responseStream, responseString); - - try - { - picojson::object parsed_json = parseJSON(responseString.str()); - String username = getValueByKey(parsed_json, "sub"); - return username; - } - catch (const std::runtime_error &) - { - return ""; - } -} - std::unordered_map GoogleAccessTokenProcessor::getUserInfo(const String & token) const { - std::unordered_map user_info; - - Poco::Net::HTTPSClientSession session(user_info_uri.getHost(), user_info_uri.getPort()); - - Poco::Net::HTTPRequest request{Poco::Net::HTTPRequest::HTTP_GET, user_info_uri.getPathAndQuery()}; - request.add("Authorization", "Bearer " + token); - session.sendRequest(request); - - Poco::Net::HTTPResponse response; - std::istream & responseStream = session.receiveResponse(response); - - if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK) - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to get user info by access token, code: {}, reason: {}", response.getStatus(), response.getReason()); - - std::ostringstream responseString; - Poco::StreamCopier::copyStream(responseStream, responseString); + std::unordered_map user_info_map; + picojson::object user_info_json = getObjectFromURI(user_info_uri, token); try { - picojson::object parsed_json = parseJSON(responseString.str()); - user_info["email"] = getValueByKey(parsed_json, "email"); - user_info["sub"] = getValueByKey(parsed_json, "sub"); - return user_info; + user_info_map["email"] = getValueByKey(user_info_json, "email"); + user_info_map["sub"] = getValueByKey(user_info_json, "sub"); + return user_info_map; } - catch (const std::runtime_error & e) + catch (std::runtime_error & e) { - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to get user info by access token: {}", e.what()); + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "{}: Failed to get user info with token: {}", name, e.what()); } } + bool AzureAccessTokenProcessor::resolveAndValidate(const TokenCredentials & credentials) { - /// Token is a JWT in this case, all we need is to decode it and verify against JWKS (similar to JWTValidator.h) - String user_name = credentials.getUserName(); - if (user_name.empty()) - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to authenticate with access token: cannot extract username"); + /// Token is a JWT in this case, but we cannot directly verify it against Azure AD JWKS. We will not trust any data in this token. + /// e.g. see here: https://stackoverflow.com/questions/60778634/failing-signature-validation-of-jwt-tokens-from-azure-ad + /// Let Azure validate it: only valid tokens will be accepted. + /// Use GET https://graph.microsoft.com/oidc/userinfo to verify token and get sub at the same time const String & token = credentials.getToken(); try { - token_validator->validate("", token); + String username = validateTokenAndGetUsername(token); + if (!username.empty()) + { + /// Credentials are passed as const everywhere up the flow, so we have to comply, + /// in this case const_cast looks acceptable. + const_cast(credentials).setUserName(username); + } + else + LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to get username with token", name); + } catch (...) { return false; } - const auto decoded_token = jwt::decode(token); - - if (email_regex.ok()) - { - if (!decoded_token.has_payload_claim("email")) - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to authenticate user {}: e-mail address not found in user data.", user_name); - /// Additionally validate user email to match regex from config. - if (!RE2::FullMatch(decoded_token.get_payload_claim("email").as_string(), email_regex)) - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to authenticate user {}: e-mail address is not permitted.", user_name); - } - /// Credentials are passed as const everywhere up the flow, so we have to comply, - /// in this case const_cast looks acceptable. + /// TODO: do not store it in credentials. const_cast(credentials).setGroups({}); return true; } +String AzureAccessTokenProcessor::validateTokenAndGetUsername(const String & token) const +{ + picojson::object user_info_json = getObjectFromURI(user_info_uri, token); + return getValueByKey(user_info_json, "sub"); +} + } diff --git a/src/Access/AccessTokenProcessor.h b/src/Access/AccessTokenProcessor.h index a45f1bbac379..790553299afc 100644 --- a/src/Access/AccessTokenProcessor.h +++ b/src/Access/AccessTokenProcessor.h @@ -69,11 +69,9 @@ class GoogleAccessTokenProcessor : public IAccessTokenProcessor bool resolveAndValidate(const TokenCredentials & credentials) override; private: - static const Poco::URI token_info_uri; + [[maybe_unused]] static const Poco::URI token_info_uri; static const Poco::URI user_info_uri; - String tryGetUserName(const String & token) const; - std::unordered_map getUserInfo(const String & token) const; }; @@ -85,16 +83,12 @@ class AzureAccessTokenProcessor : public IAccessTokenProcessor const String & email_regex_str, const String & client_id_, const String & tenant_id_, - const String & client_secret_, - const size_t jwks_refresh_interval = 300000) + const String & client_secret_) : IAccessTokenProcessor(name_, email_regex_str), client_id(client_id_), tenant_id(tenant_id_), client_secret(client_secret_), - jwks_uri_str("https://login.microsoftonline.com/" + tenant_id + "/discovery/v2.0/keys") - { - token_validator = std::make_unique(name + "_jwks_validator", std::make_unique(jwks_uri_str, jwks_refresh_interval)); - } + jwks_uri_str("https://login.microsoftonline.com/" + tenant_id + "/discovery/v2.0/keys") {} bool resolveAndValidate(const TokenCredentials & credentials) override; private: @@ -106,7 +100,7 @@ class AzureAccessTokenProcessor : public IAccessTokenProcessor const String jwks_uri_str; - std::unique_ptr token_validator; + String validateTokenAndGetUsername(const String & token) const; }; } diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 656284f34c01..bbc90c4eee0e 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -275,15 +275,10 @@ bool Authentication::areCredentialsValid( if (authentication_method.getType() != AuthenticationType::JWT) return false; - if (token_credentials->isJWT()) - { - /// The token was parsed as JWT, no further action needed. - return external_authenticators.checkJWTCredentials(authentication_method.getJWTClaims(), *token_credentials); - } - else - { - return external_authenticators.checkAccessTokenCredentials(*token_credentials); - } + if (external_authenticators.checkJWTClaims(authentication_method.getJWTClaims(), *token_credentials)) + return true; + + return external_authenticators.checkAccessTokenCredentials(*token_credentials); } if ([[maybe_unused]] const auto * always_allow_credentials = typeid_cast(&credentials)) diff --git a/src/Access/Common/JWKSProvider.cpp b/src/Access/Common/JWKSProvider.cpp index 1c306b232ba2..94ee5e04cafe 100644 --- a/src/Access/Common/JWKSProvider.cpp +++ b/src/Access/Common/JWKSProvider.cpp @@ -1,12 +1,9 @@ #include #include -#include #include #include -#include - namespace DB { diff --git a/src/Access/Credentials.cpp b/src/Access/Credentials.cpp index 796e3cc53b0c..615643caab3f 100644 --- a/src/Access/Credentials.cpp +++ b/src/Access/Credentials.cpp @@ -100,34 +100,7 @@ const String & BasicCredentials::getPassword() const return password; } -namespace -{ -String extractUsernameFromToken(const String & token) -{ - try - { - /// Attempt to handle token as JWT. - auto decoded_jwt = jwt::decode(token); - return decoded_jwt.get_subject(); - } - catch (...) - { - /// Token is not JWT, try to handle it as access token - return ""; - } -} -} - -TokenCredentials::TokenCredentials(const String & token_) - : Credentials(extractUsernameFromToken(token_)) - , token(token_) - { - // If username is empty, then the token is probably not JWT; - // we will try treating this token as an access token. - if (!user_name.empty()) - { - is_ready = true; - is_jwt = true; - } - } +/// Unless the token is validated, we will not use any data from it, including username. +TokenCredentials::TokenCredentials(const String & token_) : Credentials(""), token(token_) {} + } diff --git a/src/Access/Credentials.h b/src/Access/Credentials.h index 6f2a37149147..bbd6416a3921 100644 --- a/src/Access/Credentials.h +++ b/src/Access/Credentials.h @@ -172,10 +172,6 @@ class TokenCredentials : public Credentials is_ready = true; } } - bool isJWT() const - { - return is_jwt; - } std::set getGroups() const { return groups; @@ -186,7 +182,6 @@ class TokenCredentials : public Credentials } private: String token; - bool is_jwt = false; std::set groups; }; diff --git a/src/Access/ExternalAuthenticators.cpp b/src/Access/ExternalAuthenticators.cpp index 21772137c84f..307d40367d13 100644 --- a/src/Access/ExternalAuthenticators.cpp +++ b/src/Access/ExternalAuthenticators.cpp @@ -621,24 +621,53 @@ HTTPAuthClientParams ExternalAuthenticators::getHTTPAuthenticationParams(const S return it->second; } -bool ExternalAuthenticators::checkJWTCredentials(const String & claims, const TokenCredentials & credentials) const +/// TODO: remove redundancy +bool ExternalAuthenticators::resolveJWTCredentials(const TokenCredentials & credentials, bool throw_not_configured = true) const +{ + std::lock_guard lock{mutex}; + + const auto token = String(credentials.getToken()); + + if (jwt_validators.empty() && throw_not_configured) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "JWT authentication is not configured"); + + for (const auto & it : jwt_validators) + { + String username; + if (it.second->validate("", token, username)) + { + /// Credentials are passed as const everywhere up the flow, so we have to comply, + /// in this case const_cast looks acceptable. + const_cast(credentials).setUserName(username); + LOG_TRACE(getLogger("JWTAuthentication"), "Extracted username {} from JWT by {}", username, it.first); + return true; + } + LOG_TRACE(getLogger("JWTAuthentication"), "Failed authentication with JWT by {}", it.first); + } + return false; +} + +bool ExternalAuthenticators::checkJWTClaims(const String & claims, const TokenCredentials & credentials) const { std::lock_guard lock{mutex}; const auto token = String(credentials.getToken()); - const auto & user_name = credentials.getUserName(); if (jwt_validators.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "JWT authentication is not configured"); for (const auto & it : jwt_validators) { - if (it.second->validate(claims, token)) + String username; + if (it.second->validate(claims, token, username)) { - LOG_DEBUG(getLogger("JWTAuthentication"), "Authenticated with JWT for {} by {}", user_name, it.first); + /// Credentials are passed as const everywhere up the flow, so we have to comply, + /// in this case const_cast looks acceptable. + const_cast(credentials).setUserName(username); + LOG_DEBUG(getLogger("JWTAuthentication"), "Authenticated with JWT for {} by {}", username, it.first); return true; } - LOG_TRACE(getLogger("JWTAuthentication"), "Failed authentication with JWT for {} by {}", user_name, it.first); + LOG_TRACE(getLogger("JWTAuthentication"), "Failed authentication with JWT by {}", it.first); } return false; } @@ -671,10 +700,17 @@ bool ExternalAuthenticators::checkAccessTokenCredentialsByExactProcessor(const T for (const auto & it : access_token_processors) { - if (name == it.second->getName() && it.second->resolveAndValidate(credentials)) + if (name == it.second->getName()) { - LOG_DEBUG(getLogger("AccessTokenAuthentication"), "Authenticated user {} with access token by {}", credentials.getUserName(), it.first); - return true; + if (it.second->resolveAndValidate(credentials)) { + LOG_DEBUG(getLogger("AccessTokenAuthentication"), "Authenticated user {} with access token by {}", + credentials.getUserName(), it.first); + return true; + } else + { + LOG_TRACE(getLogger("AccessTokenAuthentication"), "Failed authentication with access token by processor {}", name); + return false; + } } } LOG_TRACE(getLogger("AccessTokenAuthentication"), "Failed authentication with access token: no processor with name {}", name); diff --git a/src/Access/ExternalAuthenticators.h b/src/Access/ExternalAuthenticators.h index b32c7aa5f95c..33c251afd5e1 100644 --- a/src/Access/ExternalAuthenticators.h +++ b/src/Access/ExternalAuthenticators.h @@ -47,7 +47,9 @@ class ExternalAuthenticators const LDAPClient::RoleSearchParamsList * role_search_params = nullptr, LDAPClient::SearchResultsList * role_search_results = nullptr) const; bool checkKerberosCredentials(const String & realm, const GSSAcceptorContext & credentials) const; bool checkHTTPBasicCredentials(const String & server, const BasicCredentials & credentials, SettingsChanges & settings) const; - bool checkJWTCredentials(const String & claims, const TokenCredentials & credentials) const; + + bool resolveJWTCredentials(const TokenCredentials & credentials, bool throw_not_configured) const; + bool checkJWTClaims(const String & claims, const TokenCredentials & credentials) const; bool checkAccessTokenCredentials(const TokenCredentials & credentials) const; bool checkAccessTokenCredentialsByExactProcessor(const TokenCredentials & credentials, const String & name) const; diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 020b9289d0d7..83157552617d 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -33,6 +33,7 @@ namespace ErrorCodes extern const int ACCESS_ENTITY_NOT_FOUND; extern const int ACCESS_STORAGE_READONLY; extern const int ACCESS_STORAGE_DOESNT_ALLOW_BACKUP; + extern const int AUTHENTICATION_FAILED; extern const int WRONG_PASSWORD; extern const int IP_ADDRESS_NOT_ALLOWED; extern const int LOGICAL_ERROR; @@ -535,6 +536,9 @@ std::optional IAccessStorage::authenticateImpl( bool allow_no_password, bool allow_plaintext_password) const { + if (!typeid_cast(credentials).isReady()) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Could not resolve username from token"); + if (auto id = find(credentials.getUserName())) { if (auto user = tryRead(*id)) diff --git a/src/Access/JWTValidator.cpp b/src/Access/JWTValidator.cpp index a94334463ee3..3b9763448f05 100644 --- a/src/Access/JWTValidator.cpp +++ b/src/Access/JWTValidator.cpp @@ -167,7 +167,7 @@ bool check_claims(const String & claims, const picojson::value::object & payload } -bool IJWTValidator::validate(const String & claims, const String & token) const +bool IJWTValidator::validate(const String & claims, const String & token, String & username) { try { @@ -178,7 +178,7 @@ bool IJWTValidator::validate(const String & claims, const String & token) const if (!check_claims(claims, decoded_jwt.get_payload_json())) return false; - LOG_TRACE(getLogger("JWTAuthentication"), "{}: claims checked", name); + username = decoded_jwt.get_subject(); return true; } @@ -309,9 +309,7 @@ void JWKSValidator::validateImpl(const jwt::decoded_jwt parseJWTValidator( diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 07e12064d4aa..717450fb6b35 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -455,7 +455,6 @@ MultipleAccessStorage::authenticateImpl(const Credentials & credentials, const P allow_no_password, allow_plaintext_password); if (auth_result) { - std::cerr << "\n\nAuth result in: \n\n" << storage->getStorageName(); std::lock_guard lock{mutex}; ids_cache.set(auth_result->user_id, storage); return auth_result; diff --git a/src/Access/TokenAccessStorage.cpp b/src/Access/TokenAccessStorage.cpp index a0bda64700e9..7151338d16a4 100644 --- a/src/Access/TokenAccessStorage.cpp +++ b/src/Access/TokenAccessStorage.cpp @@ -35,8 +35,6 @@ void TokenAccessStorage::setConfiguration() if (provider_name.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "'processor' must be specified for Token user directory"); -// getTokenProcessorByName - const bool has_roles = config.has(prefix_str + "roles"); std::set common_roles_cfg; @@ -352,6 +350,17 @@ std::optional TokenAccessStorage::authenticateImpl( auto id = memory_storage.find(credentials.getUserName()); UserPtr user = id ? memory_storage.read(*id) : nullptr; + const auto & token_credentials = typeid_cast(credentials); + + if (!external_authenticators.checkAccessTokenCredentialsByExactProcessor(token_credentials, provider_name)) + { + // Even though token itself may be valid (especially in case of a jwt token), authentication has just failed. + if (throw_if_user_not_exists) + throwNotFound(AccessEntityType::USER, credentials.getUserName()); + else + return {}; + } + std::shared_ptr new_user; if (!user) { @@ -365,17 +374,6 @@ std::optional TokenAccessStorage::authenticateImpl( if (!isAddressAllowed(*user, address)) throwAddressNotAllowed(address); - const auto & token_credentials = typeid_cast(credentials); - - if (!external_authenticators.checkAccessTokenCredentialsByExactProcessor(token_credentials, provider_name)) - { - // Even though token itself may be valid (especially in case of a jwt token), authentication has just failed. - if (throw_if_user_not_exists) - throwNotFound(AccessEntityType::USER, credentials.getUserName()); - else - return {}; - } - std::set external_roles = token_credentials.getGroups(); if (new_user) diff --git a/src/Server/HTTP/authenticateUserByHTTP.cpp b/src/Server/HTTP/authenticateUserByHTTP.cpp index ac312d111c35..084aeb84fb2d 100644 --- a/src/Server/HTTP/authenticateUserByHTTP.cpp +++ b/src/Server/HTTP/authenticateUserByHTTP.cpp @@ -213,18 +213,13 @@ bool authenticateUserByHTTP( } else if (!jwt_token.empty() && Poco::toLower(jwt_token).starts_with(BEARER_PREFIX)) { - current_credentials = std::make_unique(jwt_token.substr(BEARER_PREFIX.length())); + const auto token_credentials = TokenCredentials(jwt_token.substr(BEARER_PREFIX.length())); + const auto & external_authenticators = global_context->getAccessControl().getExternalAuthenticators(); - if (!static_cast(*current_credentials).isJWT()) - { - /// In case the token is an access token, we need to resolve it to get user name. - /// This is why (for now) the check is made twice: here and later in authentication. - global_context->getAccessControl().getExternalAuthenticators().checkAccessTokenCredentials( - static_cast(*current_credentials)); - } - if (!current_credentials->isReady()) - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, - "Failed to authenticate with access token: token invalid or none of `access_token_processors` was able to resolve it."); + if (!external_authenticators.resolveJWTCredentials(token_credentials, false)) + external_authenticators.checkAccessTokenCredentials(token_credentials); + + current_credentials = std::make_unique(token_credentials); } else // I.e., now using user name and password strings ("Basic"). { diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index ddeb808a504f..4d4cf750a343 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1803,12 +1803,11 @@ void TCPHandler::receiveHello() { auto credentials = TokenCredentials(password); - if (!credentials.isJWT()) - { - /// In case the token is an access token, we need to resolve it to get user name. - /// This is why (for now) the check is made twice: here and later in authentication. - server.context()->getAccessControl().getExternalAuthenticators().checkAccessTokenCredentials(credentials); - } + const auto & external_authenticators = server.context()->getAccessControl().getExternalAuthenticators(); + + if (!external_authenticators.resolveJWTCredentials(credentials, false)) + external_authenticators.checkAccessTokenCredentials(credentials); + session->authenticate(credentials, getClientAddress(client_info)); return; } From 5f64da5f5871dda320dd399ef4a7948186217cda Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Fri, 7 Feb 2025 20:06:04 +0000 Subject: [PATCH 3/5] add basic docs for oauth --- .../external-authenticators/tokens.md | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 docs/en/operations/external-authenticators/tokens.md diff --git a/docs/en/operations/external-authenticators/tokens.md b/docs/en/operations/external-authenticators/tokens.md new file mode 100644 index 000000000000..19307751febb --- /dev/null +++ b/docs/en/operations/external-authenticators/tokens.md @@ -0,0 +1,101 @@ +--- +slug: /en/operations/external-authenticators/oauth +title: "OAuth 2.0" +--- +import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_no_roadmap.md'; + + + +OAuth 2.0 access tokens can be used to authenticate ClickHouse users. This works in two ways: + +- Existing users (defined in `users.xml` or in local access control paths) can be authenticated with access token if this user can be `IDENTIFIED WITH jwt`. +- Use Identity Provider (IdP) as an external user directory and allow locally undefined users to be authenticated with a token if it is valid and recognized by the provider. + +Though this authentication method is different from JWT authentication, it works under the same authentication method to maintain better compatibility. + +For both of these approaches a definition of `access_token_processors` is mandatory. + +## Access Token Processors + +To define an access token processor, add `access_token_processors` section to `config.xml`. Example: +```xml + + + + Google + ^[A-Za-z0-9._%+-]+@example\.com$ + + + azure + CLIENT_ID + TENANT_ID + + + +``` + +:::note +Different providers have different sets of parameters. +::: + +**Parameters** + +- `provider` -- name of identity provider. Mandatory, case-insensitive. Supported options: "Google", "Azure". +- `email_filter` -- Regex for validation of user emails. Optional parameter, only for Google IdP. +- `client_id` -- Azure AD (Entra ID) client ID. Optional parameter, only for Azure IdP. +- `tenant_id` -- Azure AD (Entra ID) tenant ID. Optional parameter, only for Azure IdP. + +## IdP as External Authenticator {#idp-external-authenticator} + +Locally defined users can be authenticated with an access token. To allow this, `jwt` must be specified as user's authentication method. Example: + +```xml + + + + + + + + + + +``` + +At each login attempt, ClickHouse will attempt to validate token and get user info against every defined access token provider. + +When SQL-driven [Access Control and Account Management](/docs/en/guides/sre/user-management/index.md#access-control) is enabled, users that are authenticated with tokens can also be created using the [CREATE USER](/docs/en/sql-reference/statements/create/user.md#create-user-statement) statement. + +Query: + +```sql +CREATE USER my_user IDENTIFIED WITH jwt; +``` + +## Identity Provider as an External User Directory {#idp-external-user-directory} + +If there is no suitable user pre-defined in ClickHouse, authentication is still possible: Identity Provider can be used as source of user information. +To allow this, add `token` section to the `users_directories` section of the `config.xml` file. + +At each login attempt, ClickHouse tries to find the user definition locally and authenticate it as usual. +If the user is not defined, ClickHouse will treat user as externally defined, and will try to validate the token and get user information from the specified processor. +If validated successfully, the user will be considered existing and authenticated. The user will be assigned roles from the list specified in the `roles` section. +All this implies that the SQL-driven [Access Control and Account Management](/docs/en/guides/sre/user-management/index.md#access-control) is enabled and roles are created using the [CREATE ROLE](/docs/en/sql-reference/statements/create/role.md#create-role-statement) statement. + +**Example** + +```xml + + + gogoogle + + + + + +``` + +**Parameters** + +- `server` — Name of one of processors defined in `access_token_processors` config section described above. This parameter is mandatory and cannot be empty. +- `roles` — Section with a list of locally defined roles that will be assigned to each user retrieved from the IdP. From ca229de340e3a173fadb51ad1ef98aa7629b31d3 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Sat, 15 Feb 2025 16:13:20 +0000 Subject: [PATCH 4/5] add caching, cleanup bs fix credentials cast + some better code --- .../external-authenticators/tokens.md | 7 + src/Access/AccessTokenProcessor.cpp | 136 +++++++++++++++--- src/Access/AccessTokenProcessor.h | 42 +++--- src/Access/Credentials.cpp | 2 +- src/Access/Credentials.h | 9 ++ src/Access/ExternalAuthenticators.cpp | 39 ++++- src/Access/ExternalAuthenticators.h | 10 ++ src/Access/IAccessStorage.cpp | 2 +- src/Access/TokenAccessStorage.cpp | 4 +- 9 files changed, 205 insertions(+), 46 deletions(-) diff --git a/docs/en/operations/external-authenticators/tokens.md b/docs/en/operations/external-authenticators/tokens.md index 19307751febb..6340e318237e 100644 --- a/docs/en/operations/external-authenticators/tokens.md +++ b/docs/en/operations/external-authenticators/tokens.md @@ -24,6 +24,7 @@ To define an access token processor, add `access_token_processors` section to `c Google ^[A-Za-z0-9._%+-]+@example\.com$ + 600 azure @@ -41,10 +42,16 @@ Different providers have different sets of parameters. **Parameters** - `provider` -- name of identity provider. Mandatory, case-insensitive. Supported options: "Google", "Azure". +- `cache_lifetime` -- maximum lifetime of cached token (in seconds). Optional, default: 3600. - `email_filter` -- Regex for validation of user emails. Optional parameter, only for Google IdP. - `client_id` -- Azure AD (Entra ID) client ID. Optional parameter, only for Azure IdP. - `tenant_id` -- Azure AD (Entra ID) tenant ID. Optional parameter, only for Azure IdP. +### Tokens cache +To reduce number of requests to IdP, tokens are cached internally for no longer then `cache_lifetime` seconds. +If token expires sooner than `cache_lifetime`, then cache entry for this token will only be valid while token is valid. +If token lifetime is longer than `cache_lifetime`, cache entry for this token will be valid for `cache_lifetime`. + ## IdP as External Authenticator {#idp-external-authenticator} Locally defined users can be authenticated with an access token. To allow this, `jwt` must be specified as user's authentication method. Example: diff --git a/src/Access/AccessTokenProcessor.cpp b/src/Access/AccessTokenProcessor.cpp index 3d7b04ebed6a..4398fe896527 100644 --- a/src/Access/AccessTokenProcessor.cpp +++ b/src/Access/AccessTokenProcessor.cpp @@ -9,7 +9,7 @@ namespace DB namespace { - /// The JSON reply from provider has only a few key-value pairs, so no need for SimdJSON/RapidJSON. + /// The JSON reply from provider has only a few key-value pairs, so no need for any advanced parsing. /// Reduce complexity by using picojson. picojson::object parseJSON(const String & json_string) { picojson::value jsonValue; @@ -26,18 +26,20 @@ namespace return jsonValue.get(); } - std::string getValueByKey(const picojson::object & jsonObject, const std::string & key) { + template + ValueType getValueByKey(const picojson::object & jsonObject, const std::string & key) { auto it = jsonObject.find(key); // Find the key in the object - if (it == jsonObject.end()) { + if (it == jsonObject.end()) + { throw std::runtime_error("Key not found: " + key); } - const picojson::value &value = it->second; - if (!value.is()) { - throw std::runtime_error("Value for key '" + key + "' is not a string"); + const picojson::value & value = it->second; + if (!value.is()) { + throw std::runtime_error("Value for key '" + key + "' has incorrect type."); } - return value.get(); + return value.get(); } picojson::object getObjectFromURI(const Poco::URI & uri, const String & token = "") @@ -96,9 +98,12 @@ std::unique_ptr IAccessTokenProcessor::parseTokenProcesso String email_regex_str = config.hasProperty(prefix + ".email_filter") ? config.getString( prefix + ".email_filter") : ""; + UInt64 cache_lifetime = config.hasProperty(prefix + ".cache_lifetime") ? config.getUInt64( + prefix + ".cache_lifetime") : 3600; + if (provider == "google") { - return std::make_unique(name, email_regex_str); + return std::make_unique(name, cache_lifetime, email_regex_str); } else if (provider == "azure") { @@ -110,11 +115,9 @@ std::unique_ptr IAccessTokenProcessor::parseTokenProcesso throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Could not parse access token processor {}: tenant_id must be specified", name); - String client_id_str = config.getString(prefix + ".client_id"); String tenant_id_str = config.getString(prefix + ".tenant_id"); - String client_secret_str = config.hasProperty(prefix + ".client_secret") ? config.getString(prefix + ".client_secret") : ""; - return std::make_unique(name, email_regex_str, client_id_str, tenant_id_str, client_secret_str); + return std::make_unique(name, cache_lifetime, email_regex_str, tenant_id_str); } else throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, @@ -132,10 +135,11 @@ bool GoogleAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cre auto user_info = getUserInfo(token); String user_name = user_info["sub"]; + bool has_email = user_info.contains("email"); if (email_regex.ok()) { - if (!user_info.contains("email")) + if (!has_email) { LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to validate {} by e-mail", name, user_name); return false; @@ -149,10 +153,59 @@ bool GoogleAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cre } } + /// Credentials are passed as const everywhere up the flow, so we have to comply, /// in this case const_cast looks acceptable. const_cast(credentials).setUserName(user_name); - const_cast(credentials).setGroups({}); + + auto token_info = getObjectFromURI(Poco::URI(token_info_uri), token); + if (token_info.contains("exp")) + const_cast(credentials).setExpiresAt(std::chrono::system_clock::from_time_t((getValueByKey(token_info, "exp")))); + + /// Groups info can only be retrieved if user email is known. + /// If no email found in user info, we skip this step and there are no external groups for the user. + if (has_email) + { + std::set external_groups_names; + const Poco::URI get_groups_uri = Poco::URI("https://cloudidentity.googleapis.com/v1/groups/-/memberships:searchDirectGroups?query=member_key_id==" + user_info["email"] + "'"); + + try + { + auto groups_response = getObjectFromURI(get_groups_uri, token); + + if (!groups_response.contains("memberships") || !groups_response["memberships"].is()) + { + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: Failed to get Google groups: invalid content in response from server", name); + return true; + } + + for (const auto & group: groups_response["memberships"].get()) + { + if (!group.is()) + { + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: Failed to get Google groups: invalid content in response from server", name); + continue; + } + + auto group_data = group.get(); + String group_name = getValueByKey(group_data["groupKey"].get(), "id"); + external_groups_names.insert(group_name); + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: User {}: new external group {}", name, user_name, group_name); + } + + const_cast(credentials).setGroups(external_groups_names); + } + catch (const Exception & e) + { + /// Could not get groups info. Log it and skip it. + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: Failed to get Google groups, no external roles will be mapped. reason: {}", name, e.what()); + return true; + } + } return true; } @@ -177,8 +230,9 @@ std::unordered_map GoogleAccessTokenProcessor::getUserInfo(const bool AzureAccessTokenProcessor::resolveAndValidate(const TokenCredentials & credentials) { - /// Token is a JWT in this case, but we cannot directly verify it against Azure AD JWKS. We will not trust any data in this token. - /// e.g. see here: https://stackoverflow.com/questions/60778634/failing-signature-validation-of-jwt-tokens-from-azure-ad + /// Token is a JWT in this case, but we cannot directly verify it against Azure AD JWKS. + /// We will not trust user data in this token except for 'exp' value to determine caching duration. + /// Explanation here: https://stackoverflow.com/questions/60778634/failing-signature-validation-of-jwt-tokens-from-azure-ad /// Let Azure validate it: only valid tokens will be accepted. /// Use GET https://graph.microsoft.com/oidc/userinfo to verify token and get sub at the same time @@ -202,8 +256,56 @@ bool AzureAccessTokenProcessor::resolveAndValidate(const TokenCredentials & cred return false; } - /// TODO: do not store it in credentials. - const_cast(credentials).setGroups({}); + try + { + const_cast(credentials).setExpiresAt(jwt::decode(token).get_expires_at()); + } + catch (...) { + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: No expiration data found in a valid token, will use default cache lifetime", name); + } + + std::set external_groups_names; + const Poco::URI get_groups_uri = Poco::URI("https://graph.microsoft.com/v1.0/me/memberOf"); + + try + { + auto groups_response = getObjectFromURI(get_groups_uri, token); + + if (!groups_response.contains("value") || !groups_response["value"].is()) + { + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: Failed to get Azure groups: invalid content in response from server", name); + return true; + } + + picojson::array groups_array = groups_response["value"].get(); + + for (const auto & group: groups_array) + { + /// Got some invalid response. Ignore this, log this. + if (!group.is()) + { + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: Failed to get Azure groups: invalid content in response from server", name); + continue; + } + + auto group_data = group.get(); + String group_name = getValueByKey(group_data, "id"); + external_groups_names.insert(group_name); + LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: User {}: new external group {}", name, credentials.getUserName(), group_name); + } + } + catch (const Exception & e) + { + /// Could not get groups info. Log it and skip it. + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: Failed to get Azure groups, no external roles will be mapped. reason: {}", name, e.what()); + return true; + } + + const_cast(credentials).setGroups(external_groups_names); return true; } diff --git a/src/Access/AccessTokenProcessor.h b/src/Access/AccessTokenProcessor.h index 790553299afc..75005873084d 100644 --- a/src/Access/AccessTokenProcessor.h +++ b/src/Access/AccessTokenProcessor.h @@ -26,7 +26,12 @@ class GoogleAccessTokenProcessor; class IAccessTokenProcessor { public: - IAccessTokenProcessor(const String & name_, const String & email_regex_str) : name(name_), email_regex(email_regex_str) + IAccessTokenProcessor(const String & name_, + const UInt64 cache_invalidation_interval_, + const String & email_regex_str) + : name(name_), + cache_invalidation_interval(cache_invalidation_interval_), + email_regex(email_regex_str) { if (!email_regex_str.empty()) { @@ -36,19 +41,12 @@ class IAccessTokenProcessor } } - String getName() - { - return name; - } - virtual ~IAccessTokenProcessor() = default; - virtual bool resolveAndValidate(const TokenCredentials & credentials) = 0; + String getName() { return name; } + UInt64 getCacheInvalidationInterval() { return cache_invalidation_interval; } - virtual std::set getGroups([[maybe_unused]] const TokenCredentials & credentials) - { - return {}; - } + virtual bool resolveAndValidate(const TokenCredentials & credentials) = 0; static std::unique_ptr parseTokenProcessor( const Poco::Util::AbstractConfiguration & config, @@ -57,6 +55,7 @@ class IAccessTokenProcessor protected: const String name; + const UInt64 cache_invalidation_interval; re2::RE2 email_regex; }; @@ -64,7 +63,10 @@ class IAccessTokenProcessor class GoogleAccessTokenProcessor : public IAccessTokenProcessor { public: - GoogleAccessTokenProcessor(const String & name_, const String & email_regex_str) : IAccessTokenProcessor(name_, email_regex_str) {} + GoogleAccessTokenProcessor(const String & name_, + const UInt64 cache_invalidation_interval_, + const String & email_regex_str) + : IAccessTokenProcessor(name_, cache_invalidation_interval_, email_regex_str) {} bool resolveAndValidate(const TokenCredentials & credentials) override; @@ -80,24 +82,16 @@ class AzureAccessTokenProcessor : public IAccessTokenProcessor { public: AzureAccessTokenProcessor(const String & name_, + const UInt64 cache_invalidation_interval_, const String & email_regex_str, - const String & client_id_, - const String & tenant_id_, - const String & client_secret_) - : IAccessTokenProcessor(name_, email_regex_str), - client_id(client_id_), - tenant_id(tenant_id_), - client_secret(client_secret_), - jwks_uri_str("https://login.microsoftonline.com/" + tenant_id + "/discovery/v2.0/keys") {} + const String & tenant_id_) + : IAccessTokenProcessor(name_, cache_invalidation_interval_, email_regex_str), + jwks_uri_str("https://login.microsoftonline.com/" + tenant_id_ + "/discovery/v2.0/keys") {} bool resolveAndValidate(const TokenCredentials & credentials) override; private: static const Poco::URI user_info_uri; - const String client_id; - const String tenant_id; - const String client_secret; - const String jwks_uri_str; String validateTokenAndGetUsername(const String & token) const; diff --git a/src/Access/Credentials.cpp b/src/Access/Credentials.cpp index 615643caab3f..da3adcfdab92 100644 --- a/src/Access/Credentials.cpp +++ b/src/Access/Credentials.cpp @@ -101,6 +101,6 @@ const String & BasicCredentials::getPassword() const } /// Unless the token is validated, we will not use any data from it, including username. -TokenCredentials::TokenCredentials(const String & token_) : Credentials(""), token(token_) {} +TokenCredentials::TokenCredentials(const String & token_) : Credentials(""), token(token_), expires_at(std::chrono::system_clock::now() + std::chrono::hours(1)) {} } diff --git a/src/Access/Credentials.h b/src/Access/Credentials.h index bbd6416a3921..6068e1cb563f 100644 --- a/src/Access/Credentials.h +++ b/src/Access/Credentials.h @@ -180,9 +180,18 @@ class TokenCredentials : public Credentials { groups = groups_; } + std::optional getExpiresAt() const + { + return expires_at; + } + void setExpiresAt(std::chrono::system_clock::time_point expires_at_) + { + expires_at = expires_at_; + } private: String token; std::set groups; + std::optional expires_at; }; } diff --git a/src/Access/ExternalAuthenticators.cpp b/src/Access/ExternalAuthenticators.cpp index 307d40367d13..680032a6b0de 100644 --- a/src/Access/ExternalAuthenticators.cpp +++ b/src/Access/ExternalAuthenticators.cpp @@ -621,7 +621,6 @@ HTTPAuthClientParams ExternalAuthenticators::getHTTPAuthenticationParams(const S return it->second; } -/// TODO: remove redundancy bool ExternalAuthenticators::resolveJWTCredentials(const TokenCredentials & credentials, bool throw_not_configured = true) const { std::lock_guard lock{mutex}; @@ -679,10 +678,48 @@ bool ExternalAuthenticators::checkAccessTokenCredentials(const TokenCredentials if (access_token_processors.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Access token authentication is not configured"); + /// lookup token in local cache if not expired. + auto cached_entry_iter = access_token_cache.find(credentials.getToken()); + if (cached_entry_iter != access_token_cache.end()) + { + if (cached_entry_iter->second.expires_at <= std::chrono::system_clock::now()) + { + LOG_TRACE(getLogger("AccessTokenAuthentication"), "Cache entry for user {} expired, removing", cached_entry_iter->second.user_name); + access_token_cache.erase(cached_entry_iter); + } + else + { + const auto & user_data = cached_entry_iter->second; + const_cast(credentials).setUserName(user_data.user_name); + const_cast(credentials).setGroups(user_data.external_roles); + LOG_TRACE(getLogger("AccessTokenAuthentication"), "Cache entry for user {} found, using it to authenticate", cached_entry_iter->second.user_name); + return true; + } + } + for (const auto & it : access_token_processors) { if (it.second->resolveAndValidate(credentials)) { + AccessTokenCacheEntry cache_entry; + cache_entry.user_name = credentials.getUserName(); + cache_entry.external_roles = credentials.getGroups(); + + auto default_expiration_ts = std::chrono::system_clock::now() + + std::chrono::minutes(it.second->getCacheInvalidationInterval()); + + if (credentials.getExpiresAt().has_value()) + { + if (credentials.getExpiresAt().value() < default_expiration_ts) + cache_entry.expires_at = credentials.getExpiresAt().value(); + } + else + { + cache_entry.expires_at = default_expiration_ts; + } + LOG_TRACE(getLogger("AccessTokenAuthentication"), "Cache entry for user {} added", cache_entry.user_name); + + access_token_cache[credentials.getToken()] = cache_entry; LOG_DEBUG(getLogger("AccessTokenAuthentication"), "Authenticated user {} with access token by {}", credentials.getUserName(), it.first); return true; } diff --git a/src/Access/ExternalAuthenticators.h b/src/Access/ExternalAuthenticators.h index 33c251afd5e1..5c7a344b2124 100644 --- a/src/Access/ExternalAuthenticators.h +++ b/src/Access/ExternalAuthenticators.h @@ -68,13 +68,23 @@ class ExternalAuthenticators LDAPClient::SearchResultsList last_successful_role_search_results; }; + struct AccessTokenCacheEntry + { + std::chrono::system_clock::time_point expires_at; + String user_name; + std::set external_roles; + }; + using LDAPCache = std::unordered_map; // user name -> cache entry using LDAPCaches = std::map; // server name -> cache using LDAPParams = std::map; // server name -> params + using AccessTokenCache = std::unordered_map; // Access token -> cache entry + mutable std::mutex mutex; LDAPParams ldap_client_params_blueprint TSA_GUARDED_BY(mutex) ; mutable LDAPCaches ldap_caches TSA_GUARDED_BY(mutex) ; + mutable AccessTokenCache access_token_cache TSA_GUARDED_BY(mutex) ; std::optional kerberos_params TSA_GUARDED_BY(mutex) ; std::unordered_map http_auth_servers TSA_GUARDED_BY(mutex) ; std::unordered_map> jwt_validators TSA_GUARDED_BY(mutex) ; diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 83157552617d..f501bf8a4c61 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -536,7 +536,7 @@ std::optional IAccessStorage::authenticateImpl( bool allow_no_password, bool allow_plaintext_password) const { - if (!typeid_cast(credentials).isReady()) + if (typeid_cast(&credentials) && !typeid_cast(&credentials)->isReady()) throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Could not resolve username from token"); if (auto id = find(credentials.getUserName())) diff --git a/src/Access/TokenAccessStorage.cpp b/src/Access/TokenAccessStorage.cpp index 7151338d16a4..baccf3489815 100644 --- a/src/Access/TokenAccessStorage.cpp +++ b/src/Access/TokenAccessStorage.cpp @@ -173,7 +173,7 @@ String TokenAccessStorage::getStorageParamsJSON() const std::lock_guard lock(mutex); Poco::JSON::Object params_json; - params_json.set("processor", provider_name); + params_json.set("provider", provider_name); Poco::JSON::Array common_role_names_json; for (const auto & role : common_role_names) @@ -252,7 +252,7 @@ void TokenAccessStorage::assignRolesNoLock(User & user, const std::set & } else { - LOG_WARNING(getLogger(), "Unable to grant {} role '{}' to user '{}': role not found", (common ? "common" : "mapped"), role_name, user_name); + LOG_TRACE(getLogger(), "Did not grant {} role '{}' to user '{}': role not found", (common ? "common" : "mapped"), role_name, user_name); } }; From b6b0bb212ae0fad51d16dbf476f3b62aa85fb9b1 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Wed, 16 Apr 2025 13:56:03 +0000 Subject: [PATCH 5/5] fix include --- src/Access/Credentials.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Access/Credentials.h b/src/Access/Credentials.h index 6068e1cb563f..c825c722f0b4 100644 --- a/src/Access/Credentials.h +++ b/src/Access/Credentials.h @@ -8,6 +8,8 @@ #include "config.h" +#include + namespace DB {