diff --git a/contrib/jwt-cpp-cmake/CMakeLists.txt b/contrib/jwt-cpp-cmake/CMakeLists.txt index 4cb8716bc68f..606c13d29de2 100644 --- a/contrib/jwt-cpp-cmake/CMakeLists.txt +++ b/contrib/jwt-cpp-cmake/CMakeLists.txt @@ -1,7 +1,4 @@ -set(ENABLE_JWT_CPP_DEFAULT OFF) -if(ENABLE_LIBRARIES AND CLICKHOUSE_CLOUD) - set(ENABLE_JWT_CPP_DEFAULT ON) -endif() +set(ENABLE_JWT_CPP_DEFAULT ON) option(ENABLE_JWT_CPP "Enable jwt-cpp library" ${ENABLE_JWT_CPP_DEFAULT}) @@ -20,4 +17,4 @@ set (JWT_CPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/jwt-cpp/include") add_library (_jwt-cpp INTERFACE) target_include_directories(_jwt-cpp SYSTEM BEFORE INTERFACE ${JWT_CPP_INCLUDE_DIR}) -add_library(ch_contrib::jwt-cpp ALIAS _jwt-cpp) +add_library(ch_contrib::jwt-cpp ALIAS _jwt-cpp) \ No newline at end of file diff --git a/docs/en/operations/external-authenticators/index.md b/docs/en/operations/external-authenticators/index.md index 568ecf0383fd..346eadd9a323 100644 --- a/docs/en/operations/external-authenticators/index.md +++ b/docs/en/operations/external-authenticators/index.md @@ -16,4 +16,5 @@ The following external authenticators and directories are supported: - [LDAP](./ldap.md#external-authenticators-ldap) [Authenticator](./ldap.md#ldap-external-authenticator) and [Directory](./ldap.md#ldap-external-user-directory) - Kerberos [Authenticator](./kerberos.md#external-authenticators-kerberos) - [SSL X.509 authentication](./ssl-x509.md#ssl-external-authentication) -- HTTP [Authenticator](./http.md) \ No newline at end of file +- HTTP [Authenticator](./http.md) +- JWT [Authenticator](./jwt.md) diff --git a/docs/en/operations/external-authenticators/jwt.md b/docs/en/operations/external-authenticators/jwt.md new file mode 100644 index 000000000000..fbc36f8399f7 --- /dev/null +++ b/docs/en/operations/external-authenticators/jwt.md @@ -0,0 +1,219 @@ +--- +slug: /en/operations/external-authenticators/jwt +--- +# JWT +import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_no_roadmap.md'; + + + +Existing and properly configured ClickHouse users can be authenticated via JWT. + +Currently, JWT can only be used as an external authenticator for existing users, which are defined in `users.xml` or in local access control paths. +The username will be extracted from the JWT after validating the token expiration and against the signature. Signature can be validated by: +- static public key +- static JWKS +- received from the JWKS servers + +It is mandatory for a JWT tot indicate the name of the ClickHouse user under `"sub"` claim, otherwise it will not be accepted. + +A JWT may additionally be verified by checking the JWT payload. +In this case, the occurrence of specified claims from the user settings in the JWT payload is checked. +See [Enabling JWT authentication in `users.xml`](#enabling-jwt-auth-in-users-xml) + +To use JWT authentication, JWT validators must be configured in ClickHouse config. + + +## Enabling JWT validators in ClickHouse {#enabling-jwt-validators-in-clickhouse} + +To enable JWT validators, add `token_validators` section in `config.xml`. This section may contain several JWT verifiers, minimum is 1. + +### Verifying JWT signature using static key {$verifying-jwt-signature-using-static-key} + +**Example** +```xml + + + + + HS256 + my_static_secret + + + +``` + +#### Parameters: + +- `algo` - Algorithm for validate signature. Supported: + + | HMAC | RSA | ECDSA | PSS | EdDSA | + |-------| ----- | ------ | ----- | ------- | + | HS256 | RS256 | ES256 | PS256 | Ed25519 | + | HS384 | RS384 | ES384 | PS384 | Ed448 | + | HS512 | RS512 | ES512 | PS512 | | + | | | ES256K | | | + Also support None. +- `static_key` - key for symmetric algorithms. Mandatory for `HS*` family algorithms. +- `static_key_in_base64` - indicates if the `static_key` key is base64-encoded. Optional, default: `False`. +- `public_key` - public key for asymmetric algorithms. Mandatory except for `HS*` family algorithms and `None`. +- `private_key` - private key for asymmetric algorithms. Optional. +- `public_key_password` - public key password. Optional. +- `private_key_password` - private key password. Optional. + +### Verifying JWT signature using static JWKS {$verifying-jwt-signature-using-static-jwks} + +:::note +Only RS* family algorithms are supported! +::: + +**Example** +```xml + + + + + {"keys": [{"kty": "RSA", "alg": "RS256", "kid": "mykid", "n": "_public_key_mod_", "e": "AQAB"}]} + + + +``` + +#### Parameters: +- `static_jwks` - content of JWKS in json +- `static_jwks_file` - path to file with JWKS + +:::note +Only one of `static_jwks` or `static_jwks_file` keys must be present in one verifier +::: + +### Verifying JWT signature using JWKS servers {$verifying-jwt-signature-using-static-jwks} + +**Example** +```xml + + + + + http://localhost:8000/.well-known/jwks.json + 1000 + 1000 + 1000 + 3 + 50 + 1000 + 300000 + + + +``` + +#### Parameters: + +- `uri` - JWKS endpoint. Mandatory. +- `refresh_ms` - Period for resend request for refreshing JWKS. Optional, default: 300000. + +Timeouts in milliseconds on the socket used for communicating with the server (optional): +- `connection_timeout_ms` - Default: 1000. +- `receive_timeout_ms` - Default: 1000. +- `send_timeout_ms` - Default: 1000. + +Retry parameters (optional): +- `max_tries` - The maximum number of attempts to make an authentication request. Default: 3. +- `retry_initial_backoff_ms` - The backoff initial interval on retry. Default: 50. +- `retry_max_backoff_ms` - The maximum backoff interval. Default: 1000. + +### Verifying access tokens {$verifying-access-tokens} + +Access tokens that are not JWT (and thus no data can be extracted from the token directly) need to be resolved by external providers. + +**Example** +```xml + + + + + google + + + +``` + +#### Parameters: + +- `provider` - name of provider that will be used for token processing. Mandatory parameter. Possible options: `google`. + + +### Enabling JWT authentication in `users.xml` {#enabling-jwt-auth-in-users-xml} + +In order to enable JWT authentication for the user, specify `jwt` section instead of `password` or other similar sections in the user definition. + +Parameters: +- `claims` - An optional string containing a json object that should be contained in the token payload. + +Example (goes into `users.xml`): +```xml + + + + + + {"resource_access":{"account": {"roles": ["view-profile"]}}} + + + +``` + +Here, the JWT payload must contain `["view-profile"]` on path `resource_access.account.roles`, otherwise authentication will not succeed even with a valid JWT. + +``` +{ +... + "resource_access": { + "account": { + "roles": ["view-profile"] + } + }, +... +} +``` + +:::note +JWT authentication cannot be used together with any other authentication method. The presence of any other sections like `password` alongside `jwt` will force ClickHouse to shut down. +::: + +### Enabling JWT authentication using SQL {#enabling-jwt-auth-using-sql} + +When [SQL-driven Access Control and Account Management](/docs/en/guides/sre/user-management/index.md#access-control) is enabled in ClickHouse, users identified by JWT authentication can also be created using SQL statements. + +```sql +CREATE USER my_user IDENTIFIED WITH jwt CLAIMS '{"resource_access":{"account": {"roles": ["view-profile"]}}}' +``` + +Or without additional JWT payload checks: + +```sql +CREATE USER my_user IDENTIFIED WITH jwt +``` + +## JWT authentication examples {#jwt-authentication-examples} + +#### Console client + +``` +clickhouse-client -jwt +``` + +#### HTTP requests + +``` +curl 'http://localhost:8080/?' \ + -H 'Authorization: Bearer ' \ + -H 'Content type: text/plain;charset=UTF-8' \ + --data-raw 'SELECT current_user()' +``` +:::note +ClickHouse will look for a JWT token in (by priority): +1. `X-ClickHouse-JWT-Token` header. +2. `Authorization` header. +3. `token` request parameter. In this case, the "Bearer" prefix should not exist. +::: diff --git a/docs/en/operations/external-authenticators/tokens.md b/docs/en/operations/external-authenticators/tokens.md new file mode 100644 index 000000000000..6340e318237e --- /dev/null +++ b/docs/en/operations/external-authenticators/tokens.md @@ -0,0 +1,108 @@ +--- +slug: /en/operations/external-authenticators/oauth +title: "OAuth 2.0" +--- +import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_no_roadmap.md'; + + + +OAuth 2.0 access tokens can be used to authenticate ClickHouse users. This works in two ways: + +- Existing users (defined in `users.xml` or in local access control paths) can be authenticated with access token if this user can be `IDENTIFIED WITH jwt`. +- Use Identity Provider (IdP) as an external user directory and allow locally undefined users to be authenticated with a token if it is valid and recognized by the provider. + +Though this authentication method is different from JWT authentication, it works under the same authentication method to maintain better compatibility. + +For both of these approaches a definition of `access_token_processors` is mandatory. + +## Access Token Processors + +To define an access token processor, add `access_token_processors` section to `config.xml`. Example: +```xml + + + + Google + ^[A-Za-z0-9._%+-]+@example\.com$ + 600 + + + azure + CLIENT_ID + TENANT_ID + + + +``` + +:::note +Different providers have different sets of parameters. +::: + +**Parameters** + +- `provider` -- name of identity provider. Mandatory, case-insensitive. Supported options: "Google", "Azure". +- `cache_lifetime` -- maximum lifetime of cached token (in seconds). Optional, default: 3600. +- `email_filter` -- Regex for validation of user emails. Optional parameter, only for Google IdP. +- `client_id` -- Azure AD (Entra ID) client ID. Optional parameter, only for Azure IdP. +- `tenant_id` -- Azure AD (Entra ID) tenant ID. Optional parameter, only for Azure IdP. + +### Tokens cache +To reduce number of requests to IdP, tokens are cached internally for no longer then `cache_lifetime` seconds. +If token expires sooner than `cache_lifetime`, then cache entry for this token will only be valid while token is valid. +If token lifetime is longer than `cache_lifetime`, cache entry for this token will be valid for `cache_lifetime`. + +## IdP as External Authenticator {#idp-external-authenticator} + +Locally defined users can be authenticated with an access token. To allow this, `jwt` must be specified as user's authentication method. Example: + +```xml + + + + + + + + + + +``` + +At each login attempt, ClickHouse will attempt to validate token and get user info against every defined access token provider. + +When SQL-driven [Access Control and Account Management](/docs/en/guides/sre/user-management/index.md#access-control) is enabled, users that are authenticated with tokens can also be created using the [CREATE USER](/docs/en/sql-reference/statements/create/user.md#create-user-statement) statement. + +Query: + +```sql +CREATE USER my_user IDENTIFIED WITH jwt; +``` + +## Identity Provider as an External User Directory {#idp-external-user-directory} + +If there is no suitable user pre-defined in ClickHouse, authentication is still possible: Identity Provider can be used as source of user information. +To allow this, add `token` section to the `users_directories` section of the `config.xml` file. + +At each login attempt, ClickHouse tries to find the user definition locally and authenticate it as usual. +If the user is not defined, ClickHouse will treat user as externally defined, and will try to validate the token and get user information from the specified processor. +If validated successfully, the user will be considered existing and authenticated. The user will be assigned roles from the list specified in the `roles` section. +All this implies that the SQL-driven [Access Control and Account Management](/docs/en/guides/sre/user-management/index.md#access-control) is enabled and roles are created using the [CREATE ROLE](/docs/en/sql-reference/statements/create/role.md#create-role-statement) statement. + +**Example** + +```xml + + + gogoogle + + + + + +``` + +**Parameters** + +- `server` — Name of one of processors defined in `access_token_processors` config section described above. This parameter is mandatory and cannot be empty. +- `roles` — Section with a list of locally defined roles that will be assigned to each user retrieved from the IdP. diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index 432fe19634e6..d40ba7eee961 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -419,6 +420,12 @@ void AccessControl::addLDAPStorage(const String & storage_name_, const Poco::Uti LOG_DEBUG(getLogger(), "Added {} access storage '{}', LDAP server name: {}", String(new_storage->getStorageType()), new_storage->getStorageName(), new_storage->getLDAPServerName()); } +void AccessControl::addTokenStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_) +{ + auto new_storage = std::make_shared(storage_name_, *this, config_, prefix_); + addStorage(new_storage); + LOG_DEBUG(getLogger(), "Added {} access storage '{}'", String(new_storage->getStorageType()), new_storage->getStorageName()); +} void AccessControl::addStoragesFromUserDirectoriesConfig( const Poco::Util::AbstractConfiguration & config, @@ -444,6 +451,8 @@ void AccessControl::addStoragesFromUserDirectoriesConfig( type = DiskAccessStorage::STORAGE_TYPE; else if (type == "ldap") type = LDAPAccessStorage::STORAGE_TYPE; + else if (type == "token") + type = TokenAccessStorage::STORAGE_TYPE; String name = config.getString(prefix + ".name", type); @@ -477,6 +486,10 @@ void AccessControl::addStoragesFromUserDirectoriesConfig( bool allow_backup = config.getBool(prefix + ".allow_backup", true); addReplicatedStorage(name, zookeeper_path, get_zookeeper_function, allow_backup); } + else if (type == TokenAccessStorage::STORAGE_TYPE) + { + addTokenStorage(name, config, prefix); + } else throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "Unknown storage type '{}' at {} in config", type, prefix); } @@ -704,6 +717,11 @@ bool AccessControl::isNoPasswordAllowed() const return allow_no_password; } +bool AccessControl::isJWTEnabled() const +{ + return external_authenticators->isJWTAllowed(); +} + void AccessControl::setPlaintextPasswordAllowed(bool allow_plaintext_password_) { allow_plaintext_password = allow_plaintext_password_; diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h index ac64f89a7080..20a9e5818d66 100644 --- a/src/Access/AccessControl.h +++ b/src/Access/AccessControl.h @@ -93,6 +93,8 @@ class AccessControl : public MultipleAccessStorage /// Adds LDAPAccessStorage which allows querying remote LDAP server for user info. void addLDAPStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_); + void addTokenStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_); + void addReplicatedStorage(const String & storage_name, const String & zookeeper_path, const zkutil::GetZooKeeper & get_zookeeper_function, @@ -155,6 +157,8 @@ class AccessControl : public MultipleAccessStorage void setNoPasswordAllowed(bool allow_no_password_); bool isNoPasswordAllowed() const; + bool isJWTEnabled() const; + /// Allows users with plaintext password (by default it's allowed). void setPlaintextPasswordAllowed(bool allow_plaintext_password_); bool isPlaintextPasswordAllowed() const; diff --git a/src/Access/AccessTokenProcessor.cpp b/src/Access/AccessTokenProcessor.cpp new file mode 100644 index 000000000000..4398fe896527 --- /dev/null +++ b/src/Access/AccessTokenProcessor.cpp @@ -0,0 +1,319 @@ +#include +#include +#include +#include + + +namespace DB +{ + +namespace +{ + /// The JSON reply from provider has only a few key-value pairs, so no need for any advanced parsing. + /// Reduce complexity by using picojson. + picojson::object parseJSON(const String & json_string) { + picojson::value jsonValue; + std::string err = picojson::parse(jsonValue, json_string); + + if (!err.empty()) { + throw std::runtime_error("JSON parsing error: " + err); + } + + if (!jsonValue.is()) { + throw std::runtime_error("JSON is not an object"); + } + + return jsonValue.get(); + } + + template + ValueType getValueByKey(const picojson::object & jsonObject, const std::string & key) { + auto it = jsonObject.find(key); // Find the key in the object + if (it == jsonObject.end()) + { + throw std::runtime_error("Key not found: " + key); + } + + const picojson::value & value = it->second; + if (!value.is()) { + throw std::runtime_error("Value for key '" + key + "' has incorrect type."); + } + + return value.get(); + } + + picojson::object getObjectFromURI(const Poco::URI & uri, const String & token = "") + { + Poco::Net::HTTPResponse response; + std::ostringstream responseString; + + Poco::Net::HTTPRequest request{Poco::Net::HTTPRequest::HTTP_GET, uri.getPathAndQuery()}; + if (!token.empty()) + request.add("Authorization", "Bearer " + token); + + if (uri.getScheme() == "https") { + Poco::Net::HTTPSClientSession session(uri.getHost(), uri.getPort()); + session.sendRequest(request); + Poco::StreamCopier::copyStream(session.receiveResponse(response), responseString); + } + else + { + Poco::Net::HTTPClientSession session(uri.getHost(), uri.getPort()); + session.sendRequest(request); + Poco::StreamCopier::copyStream(session.receiveResponse(response), responseString); + } + + if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, + "Failed to get user info by access token, code: {}, reason: {}", response.getStatus(), + response.getReason()); + + try + { + return parseJSON(responseString.str()); + } + catch (const std::runtime_error & e) + { + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to parse server response: {}", e.what()); + } + } +} + + +[[maybe_unused]] const Poco::URI GoogleAccessTokenProcessor::token_info_uri = Poco::URI("https://www.googleapis.com/oauth2/v3/tokeninfo"); +const Poco::URI GoogleAccessTokenProcessor::user_info_uri = Poco::URI("https://www.googleapis.com/oauth2/v3/userinfo"); + +const Poco::URI AzureAccessTokenProcessor::user_info_uri = Poco::URI("https://graph.microsoft.com/oidc/userinfo"); + + +std::unique_ptr IAccessTokenProcessor::parseTokenProcessor( + const Poco::Util::AbstractConfiguration & config, + const String & prefix, + const String & name) +{ + if (config.hasProperty(prefix + ".provider")) + { + String provider = Poco::toLower(config.getString(prefix + ".provider")); + + String email_regex_str = config.hasProperty(prefix + ".email_filter") ? config.getString( + prefix + ".email_filter") : ""; + + UInt64 cache_lifetime = config.hasProperty(prefix + ".cache_lifetime") ? config.getUInt64( + prefix + ".cache_lifetime") : 3600; + + if (provider == "google") + { + return std::make_unique(name, cache_lifetime, email_regex_str); + } + else if (provider == "azure") + { + if (!config.hasProperty(prefix + ".client_id")) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, + "Could not parse access token processor {}: client_id must be specified", name); + + if (!config.hasProperty(prefix + ".tenant_id")) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, + "Could not parse access token processor {}: tenant_id must be specified", name); + + String tenant_id_str = config.getString(prefix + ".tenant_id"); + + return std::make_unique(name, cache_lifetime, email_regex_str, tenant_id_str); + } + else + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, + "Could not parse access token processor {}: unknown provider {}", name, provider); + } + + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, + "Could not parse access token processor {}: provider name must be specified", name); +} + + +bool GoogleAccessTokenProcessor::resolveAndValidate(const TokenCredentials & credentials) +{ + const String & token = credentials.getToken(); + + auto user_info = getUserInfo(token); + String user_name = user_info["sub"]; + bool has_email = user_info.contains("email"); + + if (email_regex.ok()) + { + if (!has_email) + { + LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to validate {} by e-mail", name, user_name); + return false; + } + + /// Additionally validate user email to match regex from config. + if (!RE2::FullMatch(user_info["email"], email_regex)) + { + LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to authenticate user {}: e-mail address is not permitted.", name, user_name); + return false; + } + + } + + /// Credentials are passed as const everywhere up the flow, so we have to comply, + /// in this case const_cast looks acceptable. + const_cast(credentials).setUserName(user_name); + + auto token_info = getObjectFromURI(Poco::URI(token_info_uri), token); + if (token_info.contains("exp")) + const_cast(credentials).setExpiresAt(std::chrono::system_clock::from_time_t((getValueByKey(token_info, "exp")))); + + /// Groups info can only be retrieved if user email is known. + /// If no email found in user info, we skip this step and there are no external groups for the user. + if (has_email) + { + std::set external_groups_names; + const Poco::URI get_groups_uri = Poco::URI("https://cloudidentity.googleapis.com/v1/groups/-/memberships:searchDirectGroups?query=member_key_id==" + user_info["email"] + "'"); + + try + { + auto groups_response = getObjectFromURI(get_groups_uri, token); + + if (!groups_response.contains("memberships") || !groups_response["memberships"].is()) + { + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: Failed to get Google groups: invalid content in response from server", name); + return true; + } + + for (const auto & group: groups_response["memberships"].get()) + { + if (!group.is()) + { + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: Failed to get Google groups: invalid content in response from server", name); + continue; + } + + auto group_data = group.get(); + String group_name = getValueByKey(group_data["groupKey"].get(), "id"); + external_groups_names.insert(group_name); + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: User {}: new external group {}", name, user_name, group_name); + } + + const_cast(credentials).setGroups(external_groups_names); + } + catch (const Exception & e) + { + /// Could not get groups info. Log it and skip it. + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: Failed to get Google groups, no external roles will be mapped. reason: {}", name, e.what()); + return true; + } + } + + return true; +} + +std::unordered_map GoogleAccessTokenProcessor::getUserInfo(const String & token) const +{ + std::unordered_map user_info_map; + picojson::object user_info_json = getObjectFromURI(user_info_uri, token); + + try + { + user_info_map["email"] = getValueByKey(user_info_json, "email"); + user_info_map["sub"] = getValueByKey(user_info_json, "sub"); + return user_info_map; + } + catch (std::runtime_error & e) + { + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "{}: Failed to get user info with token: {}", name, e.what()); + } +} + + +bool AzureAccessTokenProcessor::resolveAndValidate(const TokenCredentials & credentials) +{ + /// Token is a JWT in this case, but we cannot directly verify it against Azure AD JWKS. + /// We will not trust user data in this token except for 'exp' value to determine caching duration. + /// Explanation here: https://stackoverflow.com/questions/60778634/failing-signature-validation-of-jwt-tokens-from-azure-ad + /// Let Azure validate it: only valid tokens will be accepted. + /// Use GET https://graph.microsoft.com/oidc/userinfo to verify token and get sub at the same time + + const String & token = credentials.getToken(); + + try + { + String username = validateTokenAndGetUsername(token); + if (!username.empty()) + { + /// Credentials are passed as const everywhere up the flow, so we have to comply, + /// in this case const_cast looks acceptable. + const_cast(credentials).setUserName(username); + } + else + LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: Failed to get username with token", name); + + } + catch (...) + { + return false; + } + + try + { + const_cast(credentials).setExpiresAt(jwt::decode(token).get_expires_at()); + } + catch (...) { + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: No expiration data found in a valid token, will use default cache lifetime", name); + } + + std::set external_groups_names; + const Poco::URI get_groups_uri = Poco::URI("https://graph.microsoft.com/v1.0/me/memberOf"); + + try + { + auto groups_response = getObjectFromURI(get_groups_uri, token); + + if (!groups_response.contains("value") || !groups_response["value"].is()) + { + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: Failed to get Azure groups: invalid content in response from server", name); + return true; + } + + picojson::array groups_array = groups_response["value"].get(); + + for (const auto & group: groups_array) + { + /// Got some invalid response. Ignore this, log this. + if (!group.is()) + { + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: Failed to get Azure groups: invalid content in response from server", name); + continue; + } + + auto group_data = group.get(); + String group_name = getValueByKey(group_data, "id"); + external_groups_names.insert(group_name); + LOG_TRACE(getLogger("AccessTokenProcessor"), "{}: User {}: new external group {}", name, credentials.getUserName(), group_name); + } + } + catch (const Exception & e) + { + /// Could not get groups info. Log it and skip it. + LOG_TRACE(getLogger("AccessTokenProcessor"), + "{}: Failed to get Azure groups, no external roles will be mapped. reason: {}", name, e.what()); + return true; + } + + const_cast(credentials).setGroups(external_groups_names); + + return true; +} + +String AzureAccessTokenProcessor::validateTokenAndGetUsername(const String & token) const +{ + picojson::object user_info_json = getObjectFromURI(user_info_uri, token); + return getValueByKey(user_info_json, "sub"); +} + +} diff --git a/src/Access/AccessTokenProcessor.h b/src/Access/AccessTokenProcessor.h new file mode 100644 index 000000000000..75005873084d --- /dev/null +++ b/src/Access/AccessTokenProcessor.h @@ -0,0 +1,100 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int AUTHENTICATION_FAILED; + extern const int INVALID_CONFIG_PARAMETER; +} + +class GoogleAccessTokenProcessor; + +class IAccessTokenProcessor +{ +public: + IAccessTokenProcessor(const String & name_, + const UInt64 cache_invalidation_interval_, + const String & email_regex_str) + : name(name_), + cache_invalidation_interval(cache_invalidation_interval_), + email_regex(email_regex_str) + { + if (!email_regex_str.empty()) + { + /// Later, we will use .ok() to determine whether there was a regex specified in config or not. + if (!email_regex.ok()) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Invalid regex in definition of access token processor {}", name); + } + } + + virtual ~IAccessTokenProcessor() = default; + + String getName() { return name; } + UInt64 getCacheInvalidationInterval() { return cache_invalidation_interval; } + + virtual bool resolveAndValidate(const TokenCredentials & credentials) = 0; + + static std::unique_ptr parseTokenProcessor( + const Poco::Util::AbstractConfiguration & config, + const String & prefix, + const String & name); + +protected: + const String name; + const UInt64 cache_invalidation_interval; + re2::RE2 email_regex; +}; + + +class GoogleAccessTokenProcessor : public IAccessTokenProcessor +{ +public: + GoogleAccessTokenProcessor(const String & name_, + const UInt64 cache_invalidation_interval_, + const String & email_regex_str) + : IAccessTokenProcessor(name_, cache_invalidation_interval_, email_regex_str) {} + + bool resolveAndValidate(const TokenCredentials & credentials) override; + +private: + [[maybe_unused]] static const Poco::URI token_info_uri; + static const Poco::URI user_info_uri; + + std::unordered_map getUserInfo(const String & token) const; +}; + + +class AzureAccessTokenProcessor : public IAccessTokenProcessor +{ +public: + AzureAccessTokenProcessor(const String & name_, + const UInt64 cache_invalidation_interval_, + const String & email_regex_str, + const String & tenant_id_) + : IAccessTokenProcessor(name_, cache_invalidation_interval_, email_regex_str), + jwks_uri_str("https://login.microsoftonline.com/" + tenant_id_ + "/discovery/v2.0/keys") {} + + bool resolveAndValidate(const TokenCredentials & credentials) override; +private: + static const Poco::URI user_info_uri; + + const String jwks_uri_str; + + String validateTokenAndGetUsername(const String & token) const; +}; + +} diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 6aa5f3fdfff6..bbc90c4eee0e 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -235,7 +235,7 @@ bool Authentication::areCredentialsValid( const ExternalAuthenticators & external_authenticators, SettingsChanges & settings) { - if (!credentials.isReady()) + if (!typeid_cast(&credentials) && !credentials.isReady()) return false; if (const auto * gss_acceptor_context = typeid_cast(&credentials)) @@ -270,6 +270,17 @@ bool Authentication::areCredentialsValid( } #endif + if (const auto * token_credentials = typeid_cast(&credentials)) + { + if (authentication_method.getType() != AuthenticationType::JWT) + return false; + + if (external_authenticators.checkJWTClaims(authentication_method.getJWTClaims(), *token_credentials)) + return true; + + return external_authenticators.checkAccessTokenCredentials(*token_credentials); + } + if ([[maybe_unused]] const auto * always_allow_credentials = typeid_cast(&credentials)) return true; diff --git a/src/Access/AuthenticationData.cpp b/src/Access/AuthenticationData.cpp index 99eed230db73..7bafbf65b267 100644 --- a/src/Access/AuthenticationData.cpp +++ b/src/Access/AuthenticationData.cpp @@ -15,7 +15,10 @@ #include #include #include +#include +#include +#include "Access/Common/AuthenticationType.h" #include #include "config.h" @@ -337,7 +340,10 @@ std::shared_ptr AuthenticationData::toAST() const } case AuthenticationType::JWT: { - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud"); + const auto & claims = getJWTClaims(); + if (!claims.empty()) + node->children.push_back(std::make_shared(claims)); + break; } case AuthenticationType::KERBEROS: { @@ -570,6 +576,20 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que auth_data.setHTTPAuthenticationServerName(server); auth_data.setHTTPAuthenticationScheme(scheme); } + else if (query.type == AuthenticationType::JWT) + { + if (!args.empty()) + { + String value = checkAndGetLiteralArgument(args[0], "claims"); + picojson::value json_obj; + auto error = picojson::parse(json_obj, value); + if (!error.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad JWT claims: {}", error); + if (!json_obj.is()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad JWT claims: is not an object"); + auth_data.setJWTClaims(value); + } + } else { throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected ASTAuthenticationData structure"); diff --git a/src/Access/AuthenticationData.h b/src/Access/AuthenticationData.h index 239a802edddc..7c0f432f6190 100644 --- a/src/Access/AuthenticationData.h +++ b/src/Access/AuthenticationData.h @@ -77,6 +77,9 @@ class AuthenticationData time_t getValidUntil() const { return valid_until; } void setValidUntil(time_t valid_until_) { valid_until = valid_until_; } + const String & getJWTClaims() const { return jwt_claims; } + void setJWTClaims(const String & jwt_claims_) { jwt_claims = jwt_claims_; } + friend bool operator ==(const AuthenticationData & lhs, const AuthenticationData & rhs); friend bool operator !=(const AuthenticationData & lhs, const AuthenticationData & rhs) { return !(lhs == rhs); } @@ -110,6 +113,7 @@ class AuthenticationData String http_auth_server_name; HTTPAuthenticationScheme http_auth_scheme = HTTPAuthenticationScheme::BASIC; time_t valid_until = 0; + String jwt_claims; }; } diff --git a/src/Access/Common/JWKSProvider.cpp b/src/Access/Common/JWKSProvider.cpp new file mode 100644 index 000000000000..94ee5e04cafe --- /dev/null +++ b/src/Access/Common/JWKSProvider.cpp @@ -0,0 +1,89 @@ +#include + +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int AUTHENTICATION_FAILED; + extern const int INVALID_CONFIG_PARAMETER; +} + +jwt::jwks JWKSClient::getJWKS() +{ + std::shared_lock lock(mutex); + + auto now = std::chrono::high_resolution_clock::now(); + auto diff = std::chrono::duration(now - last_request_send).count(); + + if (diff < refresh_ms) { + jwt::jwks result(cached_jwks); + return result; + } + + Poco::Net::HTTPResponse response; + std::ostringstream responseString; + + Poco::Net::HTTPRequest request{Poco::Net::HTTPRequest::HTTP_GET, jwks_uri.getPathAndQuery()}; + + if (jwks_uri.getScheme() == "https") { + Poco::Net::HTTPSClientSession session = Poco::Net::HTTPSClientSession(jwks_uri.getHost(), jwks_uri.getPort()); + session.sendRequest(request); + std::istream & responseStream = session.receiveResponse(response); + if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK || !responseStream) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to get user info by access token, code: {}, reason: {}", response.getStatus(), response.getReason()); + Poco::StreamCopier::copyStream(responseStream, responseString); + } else { + Poco::Net::HTTPClientSession session = Poco::Net::HTTPClientSession(jwks_uri.getHost(), jwks_uri.getPort()); + session.sendRequest(request); + std::istream & responseStream = session.receiveResponse(response); + if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK || !responseStream) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to get user info by access token, code: {}, reason: {}", response.getStatus(), response.getReason()); + Poco::StreamCopier::copyStream(responseStream, responseString); + } + + last_request_send = std::chrono::high_resolution_clock::now(); + + jwt::jwks parsed_jwks; + + try { + parsed_jwks = jwt::parse_jwks(responseString.str()); + } + catch (const Exception & e) { + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to parse JWKS: {}", e.what()); + } + + cached_jwks = std::move(parsed_jwks); + return cached_jwks; +} + +StaticJWKSParams::StaticJWKSParams(const std::string &static_jwks_, const std::string &static_jwks_file_) +{ + if (static_jwks_.empty() && static_jwks_file_.empty()) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, + "JWT validator misconfigured: `static_jwks` or `static_jwks_file` keys must be present in static JWKS validator configuration"); + if (!static_jwks_.empty() && !static_jwks_file_.empty()) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, + "JWT validator misconfigured: `static_jwks` and `static_jwks_file` keys cannot both be present in static JWKS validator configuration"); + + static_jwks = static_jwks_; + static_jwks_file = static_jwks_file_; +} + +StaticJWKS::StaticJWKS(const StaticJWKSParams ¶ms) +{ + String content = String(params.static_jwks); + if (!params.static_jwks_file.empty()) { + std::ifstream ifs(params.static_jwks_file); + content = String((std::istreambuf_iterator(ifs)), (std::istreambuf_iterator())); + } + auto keys = jwt::parse_jwks(content); + jwks = std::move(keys); +} + +} diff --git a/src/Access/Common/JWKSProvider.h b/src/Access/Common/JWKSProvider.h new file mode 100644 index 000000000000..773208a138aa --- /dev/null +++ b/src/Access/Common/JWKSProvider.h @@ -0,0 +1,67 @@ +#include +#include +#include + +#include +#include +#include +#include +#include + + +namespace DB +{ + +class IJWKSProvider +{ +public: + virtual ~IJWKSProvider() = default; + + virtual jwt::jwks getJWKS() = 0; +}; + +class JWKSClient : public IJWKSProvider +{ +public: + explicit JWKSClient(const String & uri, const size_t refresh_ms_): refresh_ms(refresh_ms_), jwks_uri(uri) {} + + ~JWKSClient() override = default; + JWKSClient(const JWKSClient &) = delete; + JWKSClient(JWKSClient &&) = delete; + JWKSClient &operator=(const JWKSClient &) = delete; + JWKSClient &operator=(JWKSClient &&) = delete; + + jwt::jwks getJWKS() override; + +private: + size_t refresh_ms; + Poco::URI jwks_uri; + + std::shared_mutex mutex; + jwt::jwks cached_jwks; + std::chrono::time_point last_request_send; +}; + +struct StaticJWKSParams +{ + StaticJWKSParams(const std::string &static_jwks_, const std::string &static_jwks_file_); + + String static_jwks; + String static_jwks_file; +}; + +class StaticJWKS : public IJWKSProvider +{ +public: + explicit StaticJWKS(const StaticJWKSParams ¶ms); + +private: + jwt::jwks getJWKS() override + { + return jwks; + } + + jwt::jwks jwks; +}; + +} diff --git a/src/Access/Credentials.cpp b/src/Access/Credentials.cpp index f01700b6e461..da3adcfdab92 100644 --- a/src/Access/Credentials.cpp +++ b/src/Access/Credentials.cpp @@ -1,6 +1,8 @@ #include -#include #include +#include + +#include namespace DB { @@ -8,6 +10,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int AUTHENTICATION_FAILED; } Credentials::Credentials(const String & user_name_) @@ -97,4 +100,7 @@ const String & BasicCredentials::getPassword() const return password; } +/// Unless the token is validated, we will not use any data from it, including username. +TokenCredentials::TokenCredentials(const String & token_) : Credentials(""), token(token_), expires_at(std::chrono::system_clock::now() + std::chrono::hours(1)) {} + } diff --git a/src/Access/Credentials.h b/src/Access/Credentials.h index 52f33385a0e8..c825c722f0b4 100644 --- a/src/Access/Credentials.h +++ b/src/Access/Credentials.h @@ -8,6 +8,8 @@ #include "config.h" +#include + namespace DB { @@ -151,4 +153,47 @@ class SSHPTYCredentials : public Credentials #endif +class TokenCredentials : public Credentials +{ +public: + explicit TokenCredentials(const String & token_); + + const String & getToken() const + { + if (token.empty()) + { + throwNotReady(); + } + return token; + } + void setUserName(const String & user_name_) + { + user_name = user_name_; + if (!user_name.empty()) + { + is_ready = true; + } + } + std::set getGroups() const + { + return groups; + } + void setGroups(const std::set & groups_) + { + groups = groups_; + } + std::optional getExpiresAt() const + { + return expires_at; + } + void setExpiresAt(std::chrono::system_clock::time_point expires_at_) + { + expires_at = expires_at_; + } +private: + String token; + std::set groups; + std::optional expires_at; +}; + } diff --git a/src/Access/ExternalAuthenticators.cpp b/src/Access/ExternalAuthenticators.cpp index 91d0ff7ff0f9..680032a6b0de 100644 --- a/src/Access/ExternalAuthenticators.cpp +++ b/src/Access/ExternalAuthenticators.cpp @@ -2,14 +2,21 @@ #include #include #include +#include "Common/Logger.h" +#include "Common/logger_useful.h" #include #include #include #include +#include "Access/AccessControl.h" +#include "Access/Credentials.h" +#include #include #include +#include +#include #include #include @@ -253,7 +260,6 @@ HTTPAuthClientParams parseHTTPAuthParams(const Poco::Util::AbstractConfiguration return http_auth_params; } - } void parseLDAPRoleSearchParams(LDAPClient::RoleSearchParams & params, const Poco::Util::AbstractConfiguration & config, const String & prefix) @@ -271,6 +277,13 @@ void ExternalAuthenticators::resetImpl() ldap_client_params_blueprint.clear(); ldap_caches.clear(); kerberos_params.reset(); + jwt_validators.clear(); +} + +bool ExternalAuthenticators::isJWTAllowed() const +{ + std::lock_guard lock(mutex); + return !jwt_validators.empty(); } void ExternalAuthenticators::reset() @@ -279,6 +292,52 @@ void ExternalAuthenticators::reset() resetImpl(); } +void parseJWTValidators(std::unordered_map> & jwt_validators, + const Poco::Util::AbstractConfiguration & config, + const String & jwt_validators_config, + LoggerPtr log) +{ + Poco::Util::AbstractConfiguration::Keys jwt_validators_keys; + config.keys(jwt_validators_config, jwt_validators_keys); + jwt_validators.clear(); + for (const auto & jwt_validator : jwt_validators_keys) + { + if (jwt_validator == "settings_key") continue; + String prefix = fmt::format("{}.{}", jwt_validators_config, jwt_validator); + try + { + jwt_validators[jwt_validator] = IJWTValidator::parseJWTValidator(config, prefix, jwt_validator); + } + catch (...) + { + tryLogCurrentException(log, "Could not parse JWT validator" + backQuote(jwt_validator)); + } + } +} + +void parseAccessTokenProcessors(std::unordered_map> & access_token_processors, + const Poco::Util::AbstractConfiguration & config, + const String & access_token_processors_config, + LoggerPtr log) +{ + Poco::Util::AbstractConfiguration::Keys access_token_processors_keys; + config.keys(access_token_processors_config, access_token_processors_keys); + access_token_processors.clear(); + + for (const auto & processor : access_token_processors_keys) + { + String prefix = fmt::format("{}.{}", access_token_processors_config, processor); + try + { + access_token_processors[processor] = IAccessTokenProcessor::parseTokenProcessor(config, prefix, processor); + } + catch (...) + { + tryLogCurrentException(log, "Could not parse access token processor" + backQuote(processor)); + } + } +} + void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfiguration & config, LoggerPtr log) { std::lock_guard lock(mutex); @@ -290,8 +349,12 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur std::size_t ldap_servers_key_count = 0; std::size_t kerberos_keys_count = 0; std::size_t http_auth_server_keys_count = 0; + std::size_t jwt_validators_count = 0; + std::size_t access_token_processors_count = 0; const String http_auth_servers_config = "http_authentication_servers"; + const String jwt_validators_config = "jwt_validators"; + const String access_token_processors_config = "access_token_processors"; for (auto key : all_keys) { @@ -304,6 +367,8 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur ldap_servers_key_count += (key == "ldap_servers"); kerberos_keys_count += (key == "kerberos"); http_auth_server_keys_count += (key == http_auth_servers_config); + jwt_validators_count += (key == jwt_validators_config); + access_token_processors_count += (key == access_token_processors_config); } if (ldap_servers_key_count > 1) @@ -315,6 +380,12 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur if (http_auth_server_keys_count > 1) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Multiple http_authentication_servers sections are not allowed"); + if (jwt_validators_count > 1) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Multiple {} sections are not allowed", jwt_validators_config); + + if (access_token_processors_count > 1) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Multiple {} sections are not allowed", access_token_processors_config); + Poco::Util::AbstractConfiguration::Keys http_auth_server_names; config.keys(http_auth_servers_config, http_auth_server_names); http_auth_servers.clear(); @@ -369,6 +440,9 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur { tryLogCurrentException(log, "Could not parse Kerberos section"); } + + parseJWTValidators(jwt_validators, config, jwt_validators_config, log); + parseAccessTokenProcessors(access_token_processors, config, access_token_processors_config, log); } static UInt128 computeParamsHash(const LDAPClient::Params & params, const LDAPClient::RoleSearchParamsList * role_search_params) @@ -537,7 +611,7 @@ GSSAcceptorContext::Params ExternalAuthenticators::getKerberosParams() const return kerberos_params.value(); } -HTTPAuthClientParams ExternalAuthenticators::getHTTPAuthenticationParams(const String& server) const +HTTPAuthClientParams ExternalAuthenticators::getHTTPAuthenticationParams(const String & server) const { std::lock_guard lock{mutex}; @@ -547,6 +621,139 @@ HTTPAuthClientParams ExternalAuthenticators::getHTTPAuthenticationParams(const S return it->second; } +bool ExternalAuthenticators::resolveJWTCredentials(const TokenCredentials & credentials, bool throw_not_configured = true) const +{ + std::lock_guard lock{mutex}; + + const auto token = String(credentials.getToken()); + + if (jwt_validators.empty() && throw_not_configured) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "JWT authentication is not configured"); + + for (const auto & it : jwt_validators) + { + String username; + if (it.second->validate("", token, username)) + { + /// Credentials are passed as const everywhere up the flow, so we have to comply, + /// in this case const_cast looks acceptable. + const_cast(credentials).setUserName(username); + LOG_TRACE(getLogger("JWTAuthentication"), "Extracted username {} from JWT by {}", username, it.first); + return true; + } + LOG_TRACE(getLogger("JWTAuthentication"), "Failed authentication with JWT by {}", it.first); + } + return false; +} + +bool ExternalAuthenticators::checkJWTClaims(const String & claims, const TokenCredentials & credentials) const +{ + std::lock_guard lock{mutex}; + + const auto token = String(credentials.getToken()); + + if (jwt_validators.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "JWT authentication is not configured"); + + for (const auto & it : jwt_validators) + { + String username; + if (it.second->validate(claims, token, username)) + { + /// Credentials are passed as const everywhere up the flow, so we have to comply, + /// in this case const_cast looks acceptable. + const_cast(credentials).setUserName(username); + LOG_DEBUG(getLogger("JWTAuthentication"), "Authenticated with JWT for {} by {}", username, it.first); + return true; + } + LOG_TRACE(getLogger("JWTAuthentication"), "Failed authentication with JWT by {}", it.first); + } + return false; +} + +bool ExternalAuthenticators::checkAccessTokenCredentials(const TokenCredentials & credentials) const +{ + std::lock_guard lock{mutex}; + + if (access_token_processors.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Access token authentication is not configured"); + + /// lookup token in local cache if not expired. + auto cached_entry_iter = access_token_cache.find(credentials.getToken()); + if (cached_entry_iter != access_token_cache.end()) + { + if (cached_entry_iter->second.expires_at <= std::chrono::system_clock::now()) + { + LOG_TRACE(getLogger("AccessTokenAuthentication"), "Cache entry for user {} expired, removing", cached_entry_iter->second.user_name); + access_token_cache.erase(cached_entry_iter); + } + else + { + const auto & user_data = cached_entry_iter->second; + const_cast(credentials).setUserName(user_data.user_name); + const_cast(credentials).setGroups(user_data.external_roles); + LOG_TRACE(getLogger("AccessTokenAuthentication"), "Cache entry for user {} found, using it to authenticate", cached_entry_iter->second.user_name); + return true; + } + } + + for (const auto & it : access_token_processors) + { + if (it.second->resolveAndValidate(credentials)) + { + AccessTokenCacheEntry cache_entry; + cache_entry.user_name = credentials.getUserName(); + cache_entry.external_roles = credentials.getGroups(); + + auto default_expiration_ts = std::chrono::system_clock::now() + + std::chrono::minutes(it.second->getCacheInvalidationInterval()); + + if (credentials.getExpiresAt().has_value()) + { + if (credentials.getExpiresAt().value() < default_expiration_ts) + cache_entry.expires_at = credentials.getExpiresAt().value(); + } + else + { + cache_entry.expires_at = default_expiration_ts; + } + LOG_TRACE(getLogger("AccessTokenAuthentication"), "Cache entry for user {} added", cache_entry.user_name); + + access_token_cache[credentials.getToken()] = cache_entry; + LOG_DEBUG(getLogger("AccessTokenAuthentication"), "Authenticated user {} with access token by {}", credentials.getUserName(), it.first); + return true; + } + LOG_TRACE(getLogger("AccessTokenAuthentication"), "Failed authentication with access token by {}", it.first); + } + return false; +} + +bool ExternalAuthenticators::checkAccessTokenCredentialsByExactProcessor(const TokenCredentials & credentials, const String & name) const +{ + std::lock_guard lock{mutex}; + + if (access_token_processors.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Access token authentication is not configured"); + + for (const auto & it : access_token_processors) + { + if (name == it.second->getName()) + { + if (it.second->resolveAndValidate(credentials)) { + LOG_DEBUG(getLogger("AccessTokenAuthentication"), "Authenticated user {} with access token by {}", + credentials.getUserName(), it.first); + return true; + } else + { + LOG_TRACE(getLogger("AccessTokenAuthentication"), "Failed authentication with access token by processor {}", name); + return false; + } + } + } + LOG_TRACE(getLogger("AccessTokenAuthentication"), "Failed authentication with access token: no processor with name {}", name); + return false; +} + bool ExternalAuthenticators::checkHTTPBasicCredentials( const String & server, const BasicCredentials & credentials, SettingsChanges & settings) const { diff --git a/src/Access/ExternalAuthenticators.h b/src/Access/ExternalAuthenticators.h index 3a710e6df26a..5c7a344b2124 100644 --- a/src/Access/ExternalAuthenticators.h +++ b/src/Access/ExternalAuthenticators.h @@ -3,7 +3,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -12,6 +14,7 @@ #include #include +#include #include #include #include @@ -31,6 +34,7 @@ namespace DB { class SettingsChanges; +class AccessControl; class ExternalAuthenticators { @@ -44,8 +48,16 @@ class ExternalAuthenticators bool checkKerberosCredentials(const String & realm, const GSSAcceptorContext & credentials) const; bool checkHTTPBasicCredentials(const String & server, const BasicCredentials & credentials, SettingsChanges & settings) const; + bool resolveJWTCredentials(const TokenCredentials & credentials, bool throw_not_configured) const; + bool checkJWTClaims(const String & claims, const TokenCredentials & credentials) const; + + bool checkAccessTokenCredentials(const TokenCredentials & credentials) const; + bool checkAccessTokenCredentialsByExactProcessor(const TokenCredentials & credentials, const String & name) const; + GSSAcceptorContext::Params getKerberosParams() const; + bool isJWTAllowed() const; + private: HTTPAuthClientParams getHTTPAuthenticationParams(const String& server) const; @@ -56,15 +68,27 @@ class ExternalAuthenticators LDAPClient::SearchResultsList last_successful_role_search_results; }; + struct AccessTokenCacheEntry + { + std::chrono::system_clock::time_point expires_at; + String user_name; + std::set external_roles; + }; + using LDAPCache = std::unordered_map; // user name -> cache entry using LDAPCaches = std::map; // server name -> cache using LDAPParams = std::map; // server name -> params + using AccessTokenCache = std::unordered_map; // Access token -> cache entry + mutable std::mutex mutex; LDAPParams ldap_client_params_blueprint TSA_GUARDED_BY(mutex) ; mutable LDAPCaches ldap_caches TSA_GUARDED_BY(mutex) ; + mutable AccessTokenCache access_token_cache TSA_GUARDED_BY(mutex) ; std::optional kerberos_params TSA_GUARDED_BY(mutex) ; std::unordered_map http_auth_servers TSA_GUARDED_BY(mutex) ; + std::unordered_map> jwt_validators TSA_GUARDED_BY(mutex) ; + std::unordered_map> access_token_processors TSA_GUARDED_BY(mutex) ; void resetImpl() TSA_REQUIRES(mutex); }; diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 72e0933e2142..f501bf8a4c61 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -11,6 +11,7 @@ #include #include #include +#include "Access/Common/AuthenticationType.h" #include #include #include @@ -32,6 +33,7 @@ namespace ErrorCodes extern const int ACCESS_ENTITY_NOT_FOUND; extern const int ACCESS_STORAGE_READONLY; extern const int ACCESS_STORAGE_DOESNT_ALLOW_BACKUP; + extern const int AUTHENTICATION_FAILED; extern const int WRONG_PASSWORD; extern const int IP_ADDRESS_NOT_ALLOWED; extern const int LOGICAL_ERROR; @@ -534,6 +536,9 @@ std::optional IAccessStorage::authenticateImpl( bool allow_no_password, bool allow_plaintext_password) const { + if (typeid_cast(&credentials) && !typeid_cast(&credentials)->isReady()) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Could not resolve username from token"); + if (auto id = find(credentials.getUserName())) { if (auto user = tryRead(*id)) diff --git a/src/Access/JWTValidator.cpp b/src/Access/JWTValidator.cpp new file mode 100644 index 000000000000..3b9763448f05 --- /dev/null +++ b/src/Access/JWTValidator.cpp @@ -0,0 +1,367 @@ +#include "JWTValidator.h" + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int AUTHENTICATION_FAILED; + extern const int INVALID_CONFIG_PARAMETER; +} + +namespace +{ + +bool check_claims(const picojson::value & claims, const picojson::value & payload, const String & path); +bool check_claims(const picojson::value::object & claims, const picojson::value::object & payload, const String & path) +{ + for (const auto & it : claims) + { + const auto & payload_it = payload.find(it.first); + if (payload_it == payload.end()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "Key '{}.{}' not found in JWT payload", path, it.first); + return false; + } + if (!check_claims(it.second, payload_it->second, path + "." + it.first)) + { + return false; + } + } + return true; +} + +bool check_claims(const picojson::value::array & claims, const picojson::value::array & payload, const String & path) +{ + if (claims.size() > payload.size()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload too small for claims key '{}'", path); + return false; + } + for (size_t claims_i = 0; claims_i < claims.size(); ++claims_i) + { + bool found = false; + const auto & claims_val = claims.at(claims_i); + for (const auto & payload_val : payload) + { + if (!check_claims(claims_val, payload_val, path + "[" + std::to_string(claims_i) + "]")) + continue; + found = true; + } + if (!found) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not contain an object matching claims key '{}[{}]'", path, claims_i); + return false; + } + } + return true; +} + +bool check_claims(const picojson::value & claims, const picojson::value & payload, const String & path) +{ + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'array' in claims '{}'", path); + return false; + } + return check_claims(claims.get(), payload.get(), path); + } + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'object' in claims '{}'", path); + return false; + } + return check_claims(claims.get(), payload.get(), path); + } + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'bool' in claims '{}'", path); + return false; + } + if (claims.get() != payload.get()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match the value in the '{}' assertions. Expected '{}' but given '{}'", path, claims.get(), payload.get()); + return false; + } + return true; + } + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'double' in claims '{}'", path); + return false; + } + if (claims.get() != payload.get()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match the value in the '{}' assertions. Expected '{}' but given '{}'", path, claims.get(), payload.get()); + return false; + } + return true; + } + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'std::string' in claims '{}'", path); + return false; + } + if (claims.get() != payload.get()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match the value in the '{}' assertions. Expected '{}' but given '{}'", path, claims.get(), payload.get()); + return false; + } + return true; + } + #ifdef PICOJSON_USE_INT64 + if (claims.is()) + { + if (!payload.is()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match key type 'int64_t' in claims '{}'", path); + return false; + } + if (claims.get() != payload.get()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "JWT payload does not match the value in claims '{}'. Expected '{}' but given '{}'", path, claims.get(), payload.get()); + return false; + } + return true; + } + #endif + LOG_ERROR(getLogger("JWTAuthentication"), "JWT claim '{}' does not match any known type", path); + return false; +} + +bool check_claims(const String & claims, const picojson::value::object & payload) +{ + if (claims.empty()) + return true; + picojson::value json; + auto errors = picojson::parse(json, claims); + if (!errors.empty()) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Bad JWT claims: {}", errors); + if (!json.is()) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Bad JWT claims: is not an object"); + return check_claims(json.get(), payload, ""); +} + +} + +bool IJWTValidator::validate(const String & claims, const String & token, String & username) +{ + try + { + auto decoded_jwt = jwt::decode(token); + + validateImpl(decoded_jwt); + + if (!check_claims(claims, decoded_jwt.get_payload_json())) + return false; + + username = decoded_jwt.get_subject(); + + return true; + } + catch (const std::exception & ex) + { + LOG_TRACE(getLogger("JWTAuthentication"), "{}: Failed to validate JWT: {}", name, ex.what()); + return false; + } +} + +void SimpleJWTValidatorParams::validate() const +{ + if (algo == "ps256" || + algo == "ps384" || + algo == "ps512" || + algo == "ed25519" || + algo == "ed448" || + algo == "rs256" || + algo == "rs384" || + algo == "rs512" || + algo == "es256" || + algo == "es256k" || + algo == "es384" || + algo == "es512" ) + { + if (public_key.empty()) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: `public_key` parameter required for {}", algo); + } + else if (algo == "hs256" || + algo == "hs384" || + algo == "hs512" ) + { + if (static_key.empty()) + throw DB::Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: `static_key` parameter required for {}", algo); + } + else if (algo != "none") + throw DB::Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: unknown algorithm {}", algo); +} + + +SimpleJWTValidator::SimpleJWTValidator(const String & name_, const SimpleJWTValidatorParams & params_) + : IJWTValidator(name_), verifier(jwt::verify()) +{ + auto algo = params_.algo; + + if (algo == "none") + verifier = verifier.allow_algorithm(jwt::algorithm::none()); + else if (algo == "ps256") + verifier = verifier.allow_algorithm(jwt::algorithm::ps256(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "ps384") + verifier = verifier.allow_algorithm(jwt::algorithm::ps384(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "ps512") + verifier = verifier.allow_algorithm(jwt::algorithm::ps512(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "ed25519") + verifier = verifier.allow_algorithm(jwt::algorithm::ed25519(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "ed448") + verifier = verifier.allow_algorithm(jwt::algorithm::ed448(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "rs256") + verifier = verifier.allow_algorithm(jwt::algorithm::rs256(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "rs384") + verifier = verifier.allow_algorithm(jwt::algorithm::rs384(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "rs512") + verifier = verifier.allow_algorithm(jwt::algorithm::rs512(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "es256") + verifier = verifier.allow_algorithm(jwt::algorithm::es256(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "es256k") + verifier = verifier.allow_algorithm(jwt::algorithm::es256k(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "es384") + verifier = verifier.allow_algorithm(jwt::algorithm::es384(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo == "es512") + verifier = verifier.allow_algorithm(jwt::algorithm::es512(params_.public_key, params_.private_key, params_.private_key_password, params_.private_key_password)); + else if (algo.starts_with("hs")) + { + auto key = params_.static_key; + if (params_.static_key_in_base64) + key = base64Decode(key); + if (algo == "hs256") + verifier = verifier.allow_algorithm(jwt::algorithm::hs256(key)); + else if (algo == "hs384") + verifier = verifier.allow_algorithm(jwt::algorithm::hs384(key)); + else if (algo == "hs512") + verifier = verifier.allow_algorithm(jwt::algorithm::hs512(key)); + else + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: unknown algorithm {}", params_.algo); + } + else + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: unknown algorithm {}", params_.algo); +} + +void SimpleJWTValidator::validateImpl(const jwt::decoded_jwt & token) const +{ + verifier.verify(token); +} + +void JWKSValidator::validateImpl(const jwt::decoded_jwt & token) const +{ + auto jwk = provider->getJWKS().get_jwk(token.get_key_id()); + auto subject = token.get_subject(); + auto algo = Poco::toLower(token.get_algorithm()); + auto verifier = jwt::verify(); + String public_key; + + try + { + auto issuer = token.get_issuer(); + auto x5c = jwk.get_x5c_key_value(); + + if (!x5c.empty() && !issuer.empty()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "{}: Verifying {} with 'x5c' key", name, subject); + public_key = jwt::helper::convert_base64_der_to_pem(x5c); + } + } + catch (const jwt::error::claim_not_present_exception &) + { + LOG_TRACE(getLogger("JWTAuthentication"), "{}: issuer or x5c was not specified, skip verification against them", name); + } + catch (const std::bad_cast &) + { + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: invalid claim value type found, claims must be strings"); + } + + if (public_key.empty()) + { + LOG_TRACE(getLogger("JWTAuthentication"), "{}: `issuer` or `x5c` not present, verifying {} with RSA components", name, subject); + const auto modulus = jwk.get_jwk_claim("n").as_string(); + const auto exponent = jwk.get_jwk_claim("e").as_string(); + public_key = jwt::helper::create_public_key_from_rsa_components(modulus, exponent); + } + + if (jwk.has_algorithm() && Poco::toLower(jwk.get_algorithm()) != algo) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT validation error: `alg` in JWK does not match the algorithm used in JWT"); + + if (algo == "rs256") + verifier = verifier.allow_algorithm(jwt::algorithm::rs256(public_key, "", "", "")); + else if (algo == "rs384") + verifier = verifier.allow_algorithm(jwt::algorithm::rs384(public_key, "", "", "")); + else if (algo == "rs512") + verifier = verifier.allow_algorithm(jwt::algorithm::rs512(public_key, "", "", "")); + else + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "JWT cannot be validated: unknown algorithm {}", algo); + + verifier = verifier.leeway(60UL); + verifier.verify(token); +} + + +std::unique_ptr IJWTValidator::parseJWTValidator( + const Poco::Util::AbstractConfiguration & config, + const String & prefix, + const String & name) +{ + if (config.hasProperty(prefix + ".algo")) + { + SimpleJWTValidatorParams params = {}; + params.algo = Poco::toLower(config.getString(prefix + ".algo")); + params.static_key = config.getString(prefix + ".static_key", ""); + params.static_key_in_base64 = config.getBool(prefix + ".static_key_in_base64", false); + params.public_key = config.getString(prefix + ".public_key", ""); + params.private_key = config.getString(prefix + ".private_key", ""); + params.public_key_password = config.getString(prefix + ".public_key_password", ""); + params.private_key_password = config.getString(prefix + ".private_key_password", ""); + params.validate(); + return std::make_unique(name, params); + } + + std::shared_ptr provider; + if (config.hasProperty(prefix + ".uri")) + { + provider = std::make_shared(config.getString(prefix + ".uri"), config.getInt(prefix + ".refresh_ms", 300000)); + } + else if (config.hasProperty(prefix + ".static_jwks") || config.hasProperty(prefix + ".static_jwks_file")) + { + StaticJWKSParams params{ + config.getString(prefix + ".static_jwks", ""), + config.getString(prefix + ".static_jwks_file", "") + }; + provider = std::make_shared(params); + } + else + throw DB::Exception(ErrorCodes::BAD_ARGUMENTS, "Either JWKS or JWKS URI must be specified in configuration"); + + return std::make_unique(name, provider); +} + +} diff --git a/src/Access/JWTValidator.h b/src/Access/JWTValidator.h new file mode 100644 index 000000000000..03bcdf4990d1 --- /dev/null +++ b/src/Access/JWTValidator.h @@ -0,0 +1,69 @@ +#pragma once + +#include + +#include +#include +#include + +#include +#include + +#include "Access/HTTPAuthClient.h" + +#include +#include +#include + +namespace DB +{ + +class IJWTValidator +{ +public: + explicit IJWTValidator(const String & name_) : name(name_) {} + virtual bool validate(const String & claims, const String & token, String & username); + virtual ~IJWTValidator() = default; + + static std::unique_ptr parseJWTValidator( + const Poco::Util::AbstractConfiguration & config, + const String & prefix, + const String & name); + +protected: + virtual void validateImpl(const jwt::decoded_jwt & token) const = 0; + const String name; +}; + +struct SimpleJWTValidatorParams +{ + String algo; + String static_key; + bool static_key_in_base64; + String public_key; + String private_key; + String public_key_password; + String private_key_password; + void validate() const; +}; + +class SimpleJWTValidator : public IJWTValidator +{ +public: + explicit SimpleJWTValidator(const String & name_, const SimpleJWTValidatorParams & params_); +private: + void validateImpl(const jwt::decoded_jwt & token) const override; + jwt::verifier verifier; +}; + +class JWKSValidator : public IJWTValidator +{ +public: + explicit JWKSValidator(const String & name_, std::shared_ptr provider_) + : IJWTValidator(name_), provider(provider_) {} +private: + void validateImpl(const jwt::decoded_jwt & token) const override; + + std::shared_ptr provider; +}; +} diff --git a/src/Access/TokenAccessStorage.cpp b/src/Access/TokenAccessStorage.cpp new file mode 100644 index 000000000000..baccf3489815 --- /dev/null +++ b/src/Access/TokenAccessStorage.cpp @@ -0,0 +1,396 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + +TokenAccessStorage::TokenAccessStorage(const String & storage_name_, AccessControl & access_control_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_) + : IAccessStorage(storage_name_), access_control(access_control_), config(config_), prefix(prefix_), + memory_storage(storage_name_, access_control.getChangesNotifier(), false) +{ + setConfiguration(); +} + +void TokenAccessStorage::setConfiguration() +{ + std::lock_guard lock(mutex); + + const String prefix_str = (prefix.empty() ? "" : prefix + "."); + + provider_name = config.getString(prefix_str + "processor"); + if (provider_name.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "'processor' must be specified for Token user directory"); + + const bool has_roles = config.has(prefix_str + "roles"); + + std::set common_roles_cfg; + if (has_roles) + { + Poco::Util::AbstractConfiguration::Keys role_names; + config.keys(prefix_str + "roles", role_names); + + common_roles_cfg.insert(role_names.begin(), role_names.end()); + } + + common_role_names.swap(common_roles_cfg); + + external_role_hashes.clear(); + users_per_roles.clear(); + roles_per_users.clear(); + granted_role_names.clear(); + granted_role_ids.clear(); + + role_change_subscription = access_control.subscribeForChanges( + [this] (const UUID & id, const AccessEntityPtr & entity) + { + this->processRoleChange(id, entity); + } + ); +} + +void TokenAccessStorage::applyRoleChangeNoLock(bool grant, const UUID & role_id, const String & role_name) +{ + std::vector user_ids; + + // Build a list of ids of the relevant users. + if (common_role_names.contains(role_name)) + { + user_ids = memory_storage.findAll(); + } + else + { + const auto it = users_per_roles.find(role_name); + if (it != users_per_roles.end()) + { + const auto & user_names = it->second; + user_ids.reserve(user_names.size()); + + for (const auto & user_name : user_names) + { + if (const auto user_id = memory_storage.find(user_name)) + user_ids.emplace_back(*user_id); + } + } + } + + // Update the granted roles of the relevant users. + if (!user_ids.empty()) + { + auto update_func = [&role_id, &grant] (const AccessEntityPtr & entity_, const UUID &) -> AccessEntityPtr + { + if (auto user = typeid_cast>(entity_)) + { + auto changed_user = typeid_cast>(user->clone()); + if (grant) + changed_user->granted_roles.grant(role_id); + else + changed_user->granted_roles.revoke(role_id); + return changed_user; + } + return entity_; + }; + + memory_storage.update(user_ids, update_func); + } + + // Actualize granted_role_* mappings. + if (grant) + { + if (!user_ids.empty()) + { + granted_role_names.insert_or_assign(role_id, role_name); + granted_role_ids.insert_or_assign(role_name, role_id); + } + } + else + { + granted_role_ids.erase(role_name); + granted_role_names.erase(role_id); + } +} + +void TokenAccessStorage::processRoleChange(const UUID & id, const AccessEntityPtr & entity) +{ + std::lock_guard lock(mutex); + const auto role = typeid_cast>(entity); + const auto it = granted_role_names.find(id); + + if (role) // Added or renamed a role. + { + const auto & new_role_name = role->getName(); + if (it != granted_role_names.end()) // Renamed a granted role. + { + const auto & old_role_name = it->second; + if (new_role_name != old_role_name) + { + // Revoke the old role first, then grant the new role. + applyRoleChangeNoLock(false /* revoke */, id, old_role_name); + applyRoleChangeNoLock(true /* grant */, id, new_role_name); + } + } + else // Added a role. + { + applyRoleChangeNoLock(true /* grant */, id, new_role_name); + } + } + else // Removed a role. + { + if (it != granted_role_names.end()) // Removed a granted role. + { + const auto & old_role_name = it->second; + applyRoleChangeNoLock(false /* revoke */, id, old_role_name); + } + } +} + +const char * TokenAccessStorage::getStorageType() const +{ + return STORAGE_TYPE; +} + +bool TokenAccessStorage::exists(const UUID & id) const +{ + std::lock_guard lock(mutex); + return memory_storage.exists(id); +} + +String TokenAccessStorage::getStorageParamsJSON() const +{ + std::lock_guard lock(mutex); + Poco::JSON::Object params_json; + + params_json.set("provider", provider_name); + + Poco::JSON::Array common_role_names_json; + for (const auto & role : common_role_names) + { + common_role_names_json.add(role); + } + params_json.set("roles", common_role_names_json); + + std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + oss.exceptions(std::ios::failbit); + Poco::JSON::Stringifier::stringify(params_json, oss); + + return oss.str(); +} + +bool TokenAccessStorage::areTokenCredentialsValidNoLock(const User & user, const Credentials & credentials, const ExternalAuthenticators & external_authenticators) const +{ + if (!credentials.isReady()) + return false; + + if (credentials.getUserName() != user.getName()) + return false; + + if (const auto * token_credentials = dynamic_cast(&credentials)) + return external_authenticators.checkAccessTokenCredentials(*token_credentials); + + return false; +} + +std::optional TokenAccessStorage::findImpl(AccessEntityType type, const String & name) const +{ + std::lock_guard lock(mutex); + return memory_storage.find(type, name); +} + + +std::vector TokenAccessStorage::findAllImpl(AccessEntityType type) const +{ + std::lock_guard lock(mutex); + return memory_storage.findAll(type); +} + +AccessEntityPtr TokenAccessStorage::readImpl(const UUID & id, bool throw_if_not_exists) const +{ + std::lock_guard lock(mutex); + return memory_storage.read(id, throw_if_not_exists); +} + +std::optional> TokenAccessStorage::readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const +{ + std::lock_guard lock(mutex); + return memory_storage.readNameWithType(id, throw_if_not_exists); +} + +void TokenAccessStorage::assignRolesNoLock(User & user, const std::set & external_roles, std::size_t external_roles_hash) const +{ + const auto & user_name = user.getName(); + auto & granted_roles = user.granted_roles; + + auto grant_role = [this, &user_name, &granted_roles] (const String & role_name, const bool common) + { + auto it = granted_role_ids.find(role_name); + if (it == granted_role_ids.end()) + { + if (const auto role_id = access_control.find(role_name)) + { + granted_role_names.insert_or_assign(*role_id, role_name); + it = granted_role_ids.insert_or_assign(role_name, *role_id).first; + } + } + + if (it != granted_role_ids.end()) + { + const auto & role_id = it->second; + granted_roles.grant(role_id); + } + else + { + LOG_TRACE(getLogger(), "Did not grant {} role '{}' to user '{}': role not found", (common ? "common" : "mapped"), role_name, user_name); + } + }; + + external_role_hashes.erase(user_name); + granted_roles = {}; + const auto old_role_names = std::move(roles_per_users[user_name]); + + // Grant the common roles first. + for (const auto & role_name : common_role_names) + { + grant_role(role_name, true /* common */); + } + + // Grant the mapped external roles and actualize users_per_roles mapping. + // external_roles allowed to overlap with common_role_names. + for (const auto & role_name : external_roles) + { + grant_role(role_name, false /* mapped */); + users_per_roles[role_name].insert(user_name); + } + + // Cleanup users_per_roles and granted_role_* mappings. + for (const auto & old_role_name : old_role_names) + { + if (external_roles.contains(old_role_name)) + continue; + + const auto rit = users_per_roles.find(old_role_name); + if (rit == users_per_roles.end()) + continue; + + auto & user_names = rit->second; + user_names.erase(user_name); + + if (!user_names.empty()) + continue; + + users_per_roles.erase(rit); + + if (common_role_names.contains(old_role_name)) + continue; + + const auto iit = granted_role_ids.find(old_role_name); + if (iit == granted_role_ids.end()) + continue; + + const auto old_role_id = iit->second; + granted_role_names.erase(old_role_id); + granted_role_ids.erase(iit); + } + + // Actualize roles_per_users mapping and external_role_hashes cache. + if (external_roles.empty()) + roles_per_users.erase(user_name); + else + roles_per_users[user_name] = std::move(external_roles); + + external_role_hashes[user_name] = external_roles_hash; +} + +void TokenAccessStorage::updateAssignedRolesNoLock(const UUID & id, const String & user_name, const std::set & external_roles) const +{ + // No need to include common_role_names in this hash each time, since they don't change. + const auto external_roles_hash = boost::hash>{}(external_roles); + + // Map and grant the roles from scratch only if the list of external role has changed. + const auto it = external_role_hashes.find(user_name); + if (it != external_role_hashes.end() && it->second == external_roles_hash) + return; + + auto update_func = [this, &external_roles, external_roles_hash] (const AccessEntityPtr & entity_, const UUID &) -> AccessEntityPtr + { + if (auto user = typeid_cast>(entity_)) + { + auto changed_user = typeid_cast>(user->clone()); + assignRolesNoLock(*changed_user, external_roles, external_roles_hash); + return changed_user; + } + return entity_; + }; + + memory_storage.update(id, update_func); +} + + +std::optional TokenAccessStorage::authenticateImpl( + const Credentials & credentials, + const Poco::Net::IPAddress & address, + [[maybe_unused]] const ExternalAuthenticators & external_authenticators, + [[maybe_unused]] bool throw_if_user_not_exists, + bool /* allow_no_password */, + bool /* allow_plaintext_password */) const +{ + std::lock_guard lock(mutex); + auto id = memory_storage.find(credentials.getUserName()); + UserPtr user = id ? memory_storage.read(*id) : nullptr; + + const auto & token_credentials = typeid_cast(credentials); + + if (!external_authenticators.checkAccessTokenCredentialsByExactProcessor(token_credentials, provider_name)) + { + // Even though token itself may be valid (especially in case of a jwt token), authentication has just failed. + if (throw_if_user_not_exists) + throwNotFound(AccessEntityType::USER, credentials.getUserName()); + else + return {}; + } + + std::shared_ptr new_user; + if (!user) + { + // User does not exist, so we create one, and will add it if authentication is successful. + new_user = std::make_shared(); + new_user->setName(credentials.getUserName()); + new_user->authentication_methods.emplace_back(AuthenticationType::JWT); + user = new_user; + } + + if (!isAddressAllowed(*user, address)) + throwAddressNotAllowed(address); + + std::set external_roles = token_credentials.getGroups(); + + if (new_user) + { + assignRolesNoLock(*new_user, external_roles, boost::hash>{}(external_roles)); + id = memory_storage.insert(new_user); + } + else + { + // Just in case external_roles are changed. + updateAssignedRolesNoLock(*id, user->getName(), external_roles); + } + + if (id) + return AuthResult{ .user_id = *id, .authentication_data = AuthenticationData(AuthenticationType::JWT) }; + return std::nullopt; +} + + +} diff --git a/src/Access/TokenAccessStorage.h b/src/Access/TokenAccessStorage.h new file mode 100644 index 000000000000..0908f30ea519 --- /dev/null +++ b/src/Access/TokenAccessStorage.h @@ -0,0 +1,79 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace Poco +{ + namespace Util + { + class AbstractConfiguration; + } +} + + +namespace DB +{ +class AccessControl; + +/// Implementation of IAccessStorage which allows to import user data from oauth server using access token. +/// Normally, this should be unified with LDAPAccessStorage, but not done to minimize changes to code that is common with upstream. +class TokenAccessStorage : public IAccessStorage +{ +public: + static constexpr char STORAGE_TYPE[] = "token"; + + explicit TokenAccessStorage(const String & storage_name_, AccessControl & access_control_, const Poco::Util::AbstractConfiguration & config, const String & prefix); + ~TokenAccessStorage() override = default; + + // IAccessStorage implementations. + const char * getStorageType() const override; + String getStorageParamsJSON() const override; + bool isReadOnly() const override { return true; } + bool exists(const UUID & id) const override; + +private: // IAccessStorage implementations. + + mutable std::recursive_mutex mutex; // Note: Reentrance possible by internal role lookup via access_control + AccessControl & access_control; + const Poco::Util::AbstractConfiguration & config; + const String & prefix; + + String provider_name; + + std::set common_role_names; // role name that should be granted to all users at all times + mutable std::map external_role_hashes; + mutable std::map> users_per_roles; // role name -> user names (...it should be granted to; may but don't have to exist for common roles) + mutable std::map> roles_per_users; // user name -> role names (...that should be granted to it; may but don't have to include common roles) + mutable std::map granted_role_names; // (currently granted) role id -> its name + mutable std::map granted_role_ids; // (currently granted) role name -> its id + scope_guard role_change_subscription; + mutable MemoryAccessStorage memory_storage; + + void setConfiguration(); + void processRoleChange(const UUID & id, const AccessEntityPtr & entity); + + bool areTokenCredentialsValidNoLock(const User & user, const Credentials & credentials, const ExternalAuthenticators & external_authenticators) const; + + std::optional findImpl(AccessEntityType type, const String & name) const override; + std::vector findAllImpl(AccessEntityType type) const override; + AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override; + std::optional> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override; + std::optional authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, + [[maybe_unused]] const ExternalAuthenticators & external_authenticators, + [[maybe_unused]] bool throw_if_user_not_exists, + bool allow_no_password, bool allow_plaintext_password) const override; + + + void applyRoleChangeNoLock(bool grant, const UUID & role_id, const String & role_name); + void assignRolesNoLock(User & user, const std::set & external_roles, std::size_t external_roles_hash) const; + void updateAssignedRolesNoLock(const UUID & id, const String & user_name, const std::set & external_roles) const; +}; +} diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 882563d1a337..81c9f5a30ac6 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -14,6 +14,7 @@ #include #include #include +#include "Access/Credentials.h" #include #include #include @@ -131,6 +132,7 @@ namespace bool has_password_double_sha1_hex = config.has(user_config + ".password_double_sha1_hex"); bool has_ldap = config.has(user_config + ".ldap"); bool has_kerberos = config.has(user_config + ".kerberos"); + bool has_jwt = config.has(user_config + ".jwt"); const auto certificates_config = user_config + ".ssl_certificates"; bool has_certificates = config.has(certificates_config); @@ -142,18 +144,18 @@ namespace bool has_http_auth = config.has(http_auth_config); size_t num_password_fields = has_no_password + has_password_plaintext + has_password_sha256_hex + has_password_double_sha1_hex - + has_ldap + has_kerberos + has_certificates + has_ssh_keys + has_http_auth; + + has_ldap + has_kerberos + has_certificates + has_ssh_keys + has_http_auth + has_jwt; if (num_password_fields > 1) throw Exception(ErrorCodes::BAD_ARGUMENTS, "More than one field of 'password', 'password_sha256_hex', " "'password_double_sha1_hex', 'no_password', 'ldap', 'kerberos', 'ssl_certificates', 'ssh_keys', " - "'http_authentication' are used to specify authentication info for user {}. " + "'http_authentication', 'jwt' are used to specify authentication info for user {}. " "Must be only one of them.", user_name); if (num_password_fields < 1) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Either 'password' or 'password_sha256_hex' " "or 'password_double_sha1_hex' or 'no_password' or 'ldap' or 'kerberos " - "or 'ssl_certificates' or 'ssh_keys' or 'http_authentication' must be specified for user {}.", user_name); + "or 'ssl_certificates' or 'ssh_keys' or 'http_authentication' or 'jwt' must be specified for user {}.", user_name); if (has_password_plaintext) { @@ -268,6 +270,10 @@ namespace auto scheme = config.getString(http_auth_config + ".scheme"); user->authentication_methods.back().setHTTPAuthenticationScheme(parseHTTPAuthenticationScheme(scheme)); } + else if (has_jwt) + { + user->authentication_methods.emplace_back(AuthenticationType::JWT); + } else { user->authentication_methods.emplace_back(); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 66d323d27349..e9d1d71edb34 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -393,6 +393,7 @@ target_link_libraries(clickhouse_common_io ch_contrib::zlib pcg_random Poco::Foundation + ch_contrib::jwt-cpp ) if (TARGET ch_contrib::libfiu) diff --git a/src/Parsers/Access/ASTAuthenticationData.cpp b/src/Parsers/Access/ASTAuthenticationData.cpp index 20655757499a..fb0667069373 100644 --- a/src/Parsers/Access/ASTAuthenticationData.cpp +++ b/src/Parsers/Access/ASTAuthenticationData.cpp @@ -106,8 +106,11 @@ void ASTAuthenticationData::formatImpl(WriteBuffer & ostr, const FormatSettings } case AuthenticationType::JWT: { - prefix = "CLAIMS"; - parameter = true; + if (!children.empty()) + { + prefix = "CLAIMS"; + parameter = true; + } break; } case AuthenticationType::LDAP: diff --git a/src/Parsers/Access/ASTCreateUserQuery.h b/src/Parsers/Access/ASTCreateUserQuery.h index 134224849381..7f9aeea5f021 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.h +++ b/src/Parsers/Access/ASTCreateUserQuery.h @@ -18,7 +18,7 @@ class ASTAuthenticationData; /** CREATE USER [IF NOT EXISTS | OR REPLACE] name - * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] + * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']|{WITH jwt [CLAIMS 'json_object']}}] * [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...]] * [DEFAULT DATABASE database | NONE] @@ -27,7 +27,7 @@ class ASTAuthenticationData; * * ALTER USER [IF EXISTS] name * [RENAME TO new_name] - * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] + * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']|{WITH jwt [CLAIMS 'json_object']}}] * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] * [DEFAULT DATABASE database | NONE] diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index 91e7f16606f5..6b5ae6a2325e 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -75,6 +75,7 @@ namespace bool expect_ssl_cert_subjects = false; bool expect_public_ssh_key = false; bool expect_http_auth_server = false; + bool expect_claims = false; auto parse_non_password_based_type = [&](auto check_type) { @@ -92,6 +93,8 @@ namespace expect_public_ssh_key = true; else if (check_type == AuthenticationType::HTTP) expect_http_auth_server = true; + else if (check_type == AuthenticationType::JWT) + expect_claims = true; else if (check_type != AuthenticationType::NO_PASSWORD) expect_password = true; @@ -147,6 +150,7 @@ namespace ASTPtr http_auth_scheme; ASTPtr ssl_cert_subjects; std::optional ssl_cert_subject_type; + ASTPtr jwt_claims; if (expect_password || expect_hash) { @@ -211,6 +215,14 @@ namespace return false; } } + else if (expect_claims) + { + if (ParserKeyword{Keyword::CLAIMS}.ignore(pos, expected)) + { + if (!ParserStringAndSubstitution{}.parse(pos, jwt_claims, expected)) + return false; + } + } auth_data = std::make_shared(); @@ -236,6 +248,9 @@ namespace if (http_auth_scheme) auth_data->children.push_back(std::move(http_auth_scheme)); + if (jwt_claims) + auth_data->children.push_back(std::move(jwt_claims)); + parseValidUntil(pos, expected, auth_data->valid_until); return true; diff --git a/src/Parsers/Access/ParserCreateUserQuery.h b/src/Parsers/Access/ParserCreateUserQuery.h index 4dfff8713d76..5f4cfcd6c45f 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.h +++ b/src/Parsers/Access/ParserCreateUserQuery.h @@ -7,7 +7,7 @@ namespace DB { /** Parses queries like * CREATE USER [IF NOT EXISTS | OR REPLACE] name - * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] + * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}|{WITH jwt}] * [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...]] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...] @@ -15,7 +15,7 @@ namespace DB * * ALTER USER [IF EXISTS] name * [RENAME TO new_name] - * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] + * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}|{WITH jwt}] * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] * [ADD|MODIFY SETTINGS variable [=value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] [,...] ] diff --git a/src/Parsers/CommonParsers.h b/src/Parsers/CommonParsers.h index 62b5ca135cd8..99fea1f4afec 100644 --- a/src/Parsers/CommonParsers.h +++ b/src/Parsers/CommonParsers.h @@ -84,6 +84,7 @@ namespace DB MR_MACROS(CHECK_TABLE, "CHECK TABLE") \ MR_MACROS(CHECK_GRANT, "CHECK GRANT")\ MR_MACROS(CHECK, "CHECK") \ + MR_MACROS(CLAIMS, "CLAIMS") \ MR_MACROS(CLEANUP, "CLEANUP") \ MR_MACROS(CLEAR_COLUMN, "CLEAR COLUMN") \ MR_MACROS(CLEAR_INDEX, "CLEAR INDEX") \ diff --git a/src/Server/HTTP/authenticateUserByHTTP.cpp b/src/Server/HTTP/authenticateUserByHTTP.cpp index f96ffc2cb6a9..084aeb84fb2d 100644 --- a/src/Server/HTTP/authenticateUserByHTTP.cpp +++ b/src/Server/HTTP/authenticateUserByHTTP.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -19,7 +20,7 @@ #include #endif - +const String BEARER_PREFIX = "bearer "; namespace DB { @@ -80,6 +81,8 @@ bool authenticateUserByHTTP( bool has_http_credentials = request.hasCredentials() && request.get("Authorization") != "never"; bool has_credentials_in_query_params = params.has("user") || params.has("password"); + std::string jwt_token = request.get("X-ClickHouse-JWT-Token", request.get("Authorization", (params.has("token") ? BEARER_PREFIX + params.get("token") : ""))); + std::string spnego_challenge; SSLCertificateSubjects certificate_subjects; @@ -152,7 +155,7 @@ bool authenticateUserByHTTP( if (spnego_challenge.empty()) throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Invalid authentication: SPNEGO challenge is empty"); } - else + else if (Poco::icompare(scheme, "Bearer") < 0) { throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Invalid authentication: '{}' HTTP Authorization scheme is not supported", scheme); } @@ -208,6 +211,16 @@ bool authenticateUserByHTTP( { current_credentials = std::make_unique(*config_credentials); } + else if (!jwt_token.empty() && Poco::toLower(jwt_token).starts_with(BEARER_PREFIX)) + { + const auto token_credentials = TokenCredentials(jwt_token.substr(BEARER_PREFIX.length())); + const auto & external_authenticators = global_context->getAccessControl().getExternalAuthenticators(); + + if (!external_authenticators.resolveJWTCredentials(token_credentials, false)) + external_authenticators.checkAccessTokenCredentials(token_credentials); + + current_credentials = std::make_unique(token_credentials); + } else // I.e., now using user name and password strings ("Basic"). { if (!current_credentials) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index acf615f7704b..4d4cf750a343 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -1707,6 +1708,10 @@ void TCPHandler::receiveHello() if (is_ssh_based_auth) user.erase(0, std::string_view(EncodedUserInfo::SSH_KEY_AUTHENTICAION_MARKER).size()); + is_jwt_based_auth = user.starts_with(EncodedUserInfo::JWT_AUTHENTICAION_MARKER); + if (is_jwt_based_auth) + user.erase(0, std::string_view(EncodedUserInfo::JWT_AUTHENTICAION_MARKER).size()); + session = makeSession(); const auto & client_info = session->getClientInfo(); @@ -1794,6 +1799,19 @@ void TCPHandler::receiveHello() } #endif + if (is_jwt_based_auth) + { + auto credentials = TokenCredentials(password); + + const auto & external_authenticators = server.context()->getAccessControl().getExternalAuthenticators(); + + if (!external_authenticators.resolveJWTCredentials(credentials, false)) + external_authenticators.checkAccessTokenCredentials(credentials); + + session->authenticate(credentials, getClientAddress(client_info)); + return; + } + session->authenticate(user, password, getClientAddress(client_info)); } diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index 98170c140be2..aec62ee4ef15 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -230,6 +230,7 @@ class TCPHandler : public Poco::Net::TCPServerConnection String default_database; bool is_ssh_based_auth = false; /// authentication is via SSH pub-key challenge + bool is_jwt_based_auth = false; /// authentication is via JWT /// For inter-server secret (remote_server.*.secret) bool is_interserver_mode = false; bool is_interserver_authenticated = false; diff --git a/tests/docker_scripts/fasttest_runner.sh b/tests/docker_scripts/fasttest_runner.sh index 1dbbd49a28be..7bae54ade257 100755 --- a/tests/docker_scripts/fasttest_runner.sh +++ b/tests/docker_scripts/fasttest_runner.sh @@ -157,7 +157,7 @@ function clone_submodules contrib/libfiu contrib/incbin contrib/yaml-cpp - contrib/corrosion + contrib/jwt-cpp ) git submodule sync diff --git a/tests/integration/test_jwt_auth/__init__.py b/tests/integration/test_jwt_auth/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_jwt_auth/configs/users.xml b/tests/integration/test_jwt_auth/configs/users.xml new file mode 100644 index 000000000000..b3d3372ebaa9 --- /dev/null +++ b/tests/integration/test_jwt_auth/configs/users.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + default + default + + + diff --git a/tests/integration/test_jwt_auth/configs/validators.xml b/tests/integration/test_jwt_auth/configs/validators.xml new file mode 100644 index 000000000000..1522937629cd --- /dev/null +++ b/tests/integration/test_jwt_auth/configs/validators.xml @@ -0,0 +1,24 @@ + + + + + HS256 + my_secret + false + + + + hs256 + other_secret + false + + + + {"keys": [{"kty": "RSA", "alg": "rs256", "kid": "mykid", "n": "lICGC8S5pObyASih5qfmwuclG0oKsbzY2z9vgwqyhTYQOWcqYcTjVV4aQ30qb6E0-5W6rJ-jx9zx6GuAEGMiG_aWJEdbUAMGp-L1kz4lrw5U6GlwoZIvk4wqoRwsiyc-mnDMQAmiZLBNyt3wU6YnKgYmb4O1cSzcZ5HMbImJpj4tpYjqnIazvYMn_9Pxjkl0ezLCr52av0UkWHro1H4QMVfuEoNmHuWPww9jgHn-I-La0xdOhRpAa0XnJi65dXZd4330uWjeJwt413yz881uS4n1OLOGKG8ImDcNlwU_guyvk0n0aqT0zkOAPp9_yYo13MPWmiRCfOX8ozdN7VDIJw", "e": "AQAB"}]} + + + + http://resolver:8080/.well-known/jwks.json + + + diff --git a/tests/integration/test_jwt_auth/helpers/generate_private_key.py b/tests/integration/test_jwt_auth/helpers/generate_private_key.py new file mode 100644 index 000000000000..7b54fa63368b --- /dev/null +++ b/tests/integration/test_jwt_auth/helpers/generate_private_key.py @@ -0,0 +1,21 @@ +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.backends import default_backend + +# Generate RSA private key +private_key = rsa.generate_private_key( + public_exponent=65537, + key_size=2048, # Key size of 2048 bits + backend=default_backend() +) + +# Save the private key to a PEM file +pem_private_key = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() # You can add encryption if needed +) + +# Write the private key to a file +with open("new_private_key", "wb") as pem_file: + pem_file.write(pem_private_key) diff --git a/tests/integration/test_jwt_auth/helpers/jwt_jwk.py b/tests/integration/test_jwt_auth/helpers/jwt_jwk.py new file mode 100644 index 000000000000..265882efce76 --- /dev/null +++ b/tests/integration/test_jwt_auth/helpers/jwt_jwk.py @@ -0,0 +1,113 @@ +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization + +import base64 +import json +import jwt + + +""" +Only RS* family algorithms are supported!!! +""" +with open("./private_key_2", "rb") as key_file: + private_key = serialization.load_pem_private_key( + key_file.read(), + password=None, + ) + + +public_key = private_key.public_key() + + +def to_base64_url(data): + return base64.urlsafe_b64encode(data).decode("utf-8").rstrip("=") + + +def rsa_key_to_jwk(private_key=None, public_key=None): + if private_key: + # Convert the private key to its components + private_numbers = private_key.private_numbers() + public_numbers = private_key.public_key().public_numbers() + + jwk = { + "kty": "RSA", + "alg": "RS512", + "kid": "mykid", + "n": to_base64_url( + public_numbers.n.to_bytes( + (public_numbers.n.bit_length() + 7) // 8, byteorder="big" + ) + ), + "e": to_base64_url( + public_numbers.e.to_bytes( + (public_numbers.e.bit_length() + 7) // 8, byteorder="big" + ) + ), + "d": to_base64_url( + private_numbers.d.to_bytes( + (private_numbers.d.bit_length() + 7) // 8, byteorder="big" + ) + ), + "p": to_base64_url( + private_numbers.p.to_bytes( + (private_numbers.p.bit_length() + 7) // 8, byteorder="big" + ) + ), + "q": to_base64_url( + private_numbers.q.to_bytes( + (private_numbers.q.bit_length() + 7) // 8, byteorder="big" + ) + ), + "dp": to_base64_url( + private_numbers.dmp1.to_bytes( + (private_numbers.dmp1.bit_length() + 7) // 8, byteorder="big" + ) + ), + "dq": to_base64_url( + private_numbers.dmq1.to_bytes( + (private_numbers.dmq1.bit_length() + 7) // 8, byteorder="big" + ) + ), + "qi": to_base64_url( + private_numbers.iqmp.to_bytes( + (private_numbers.iqmp.bit_length() + 7) // 8, byteorder="big" + ) + ), + } + elif public_key: + # Convert the public key to its components + public_numbers = public_key.public_numbers() + + jwk = { + "kty": "RSA", + "alg": "RS512", + "kid": "mykid", + "n": to_base64_url( + public_numbers.n.to_bytes( + (public_numbers.n.bit_length() + 7) // 8, byteorder="big" + ) + ), + "e": to_base64_url( + public_numbers.e.to_bytes( + (public_numbers.e.bit_length() + 7) // 8, byteorder="big" + ) + ), + } + else: + raise ValueError("You must provide either a private or public key.") + + return jwk + + +# Convert to JWK +jwk_private = rsa_key_to_jwk(private_key=private_key) +jwk_public = rsa_key_to_jwk(public_key=public_key) + +print(f"Private JWK:\n{json.dumps(jwk_private)}\n") +print(f"Public JWK:\n{json.dumps(jwk_public)}\n") + +payload = {"sub": "jwt_user", "iss": "test_iss"} + +# Create a JWT +token = jwt.encode(payload, private_key, headers={"kid": "mykid"}, algorithm="RS512") +print(f"JWT:\n{token}") diff --git a/tests/integration/test_jwt_auth/helpers/jwt_static_secret.py b/tests/integration/test_jwt_auth/helpers/jwt_static_secret.py new file mode 100644 index 000000000000..5f1c7e0340af --- /dev/null +++ b/tests/integration/test_jwt_auth/helpers/jwt_static_secret.py @@ -0,0 +1,43 @@ +import jwt +import datetime + + +def create_jwt( + payload: dict, secret: str, algorithm: str = "HS256", expiration_minutes: int = None +) -> str: + """ + Create a JWT using a static secret and a specified encryption algorithm. + + :param payload: The payload to include in the JWT (as a dictionary). + :param secret: The secret key used to sign the JWT. + :param algorithm: The encryption algorithm to use (default is 'HS256'). + :param expiration_minutes: The time until the token expires (default is 60 minutes). + :return: The encoded JWT as a string. + """ + if expiration_minutes: + expiration = datetime.datetime.utcnow() + datetime.timedelta( + minutes=expiration_minutes + ) + payload["exp"] = expiration + + return jwt.encode(payload, secret, algorithm=algorithm) + + +if __name__ == "__main__": + secret = "my_secret" + payload = {"sub": "jwt_user"} # `sub` must contain user name + + """ + Supported algorithms: + | HMSC | RSA | ECDSA | PSS | EdDSA | + | ----- | ----- | ------ | ----- | ------- | + | HS256 | RS256 | ES256 | PS256 | Ed25519 | + | HS384 | RS384 | ES384 | PS384 | Ed448 | + | HS512 | RS512 | ES512 | PS512 | | + | | | ES256K | | | + And None + """ + algorithm = "HS256" + + token = create_jwt(payload, secret, algorithm) + print(f"Generated JWT: {token}") diff --git a/tests/integration/test_jwt_auth/helpers/private_key_1 b/tests/integration/test_jwt_auth/helpers/private_key_1 new file mode 100644 index 000000000000..a076a86e17a4 --- /dev/null +++ b/tests/integration/test_jwt_auth/helpers/private_key_1 @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAlICGC8S5pObyASih5qfmwuclG0oKsbzY2z9vgwqyhTYQOWcq +YcTjVV4aQ30qb6E0+5W6rJ+jx9zx6GuAEGMiG/aWJEdbUAMGp+L1kz4lrw5U6Glw +oZIvk4wqoRwsiyc+mnDMQAmiZLBNyt3wU6YnKgYmb4O1cSzcZ5HMbImJpj4tpYjq +nIazvYMn/9Pxjkl0ezLCr52av0UkWHro1H4QMVfuEoNmHuWPww9jgHn+I+La0xdO +hRpAa0XnJi65dXZd4330uWjeJwt413yz881uS4n1OLOGKG8ImDcNlwU/guyvk0n0 +aqT0zkOAPp9/yYo13MPWmiRCfOX8ozdN7VDIJwIDAQABAoIBADZfiLUuZrrWRK3f +7sfBmmCquY9wYNILT2uXooDcndjgnrgl6gK6UHKlbgBgB/WvlPK5NAyYtyMq5vgu +xEk7wvVyKC9IYUq+kOVP2JL9IlcibDxcvvypxfnETKeI5VZeHDH4MxEPdgJf+1vY +P3KhV52vestB8mFqB5l0bOEgyuGvO3/3D1JjOnFLS/K2vOj8D/KDRmwXRCcGHTxj +dj3wJH4UbCIsLgiaQBPkFmTteJDICb+7//6YQuB0t8sR/DZS9Z0GWcfy04Cp/m/E +4rRoTNz80MbbU9+k0Ly360SxPizcjpPYSRSD025i8Iqv8jvelq7Nzg69Kubc0KfN +mMrRdMECgYEAz4b7+OX+aO5o2ZQS+fHc8dyWc5umC+uT5xrUm22wZLYA5O8x0Rgj +vdO/Ho/XyN/GCyvNNV2rI2+CBTxez6NqesGDEmJ2n7TQ03xXLCVsnwVz694sPSMO +pzTbU6e42jvDo5DMPDv0Pg1CVQuM9ka6wb4DcolMyDql6QddY3iXHBkCgYEAtzAl +xEAABqdFAnCs3zRf9EZphGJiJ4gtoWmCxQs+IcrfyBNQCy6GqrzJOZ7fQiEoAeII +V0JmsNcnx3U1W0lp8N+1QNZoB4fOWXaX08BvOEe7gbJ6Xl5t52j792vQp1txpBhE +UDhz8m5R9i5qb3BzrYBiSTfak0Pq56Xw3jRDjj8CgYEAqX2QS07kQqT8gz85ZGOR +1QMY6aCks7WaXTR/kdW7K/Wts0xb/m7dugq3W+mVDh0c7UC/36b5v/4xTb9pm+HW +dB2ZxCkgwvz1VNSHiamjFhlo/Km+rcv1CsDTpHYmNi57cRowg71flFJV64l8fiN0 +IgnjXOcgC6RCnpiCQFxb5fkCgYB+Zq2YleSuspqOjXrrZPNU1YUXgN9jkbaSqwA9 +wH01ygvRvWm83XS0uSFMLhC1S7WUXwgMVdgP69YZ7glMHQMJ3wLtY0RS9eVvm8I1 +rZHQzsZWPvXqydOiGrHJzs4hvJpUdR4mEF4JCRBrAyoUDQ70yCKJjQ24EeQzxS/H +015N9wKBgB8DdFPvKXyygTMnBoZdpAhkE/x3TTi7DsLBxj7QxKmSHzlHGz0TubIB +m5/p9dGawQNzD4JwASuY5r4lKXmvYr+4TQPLq6c7EnoIZSwLdge+6PDhnDWJzvk1 +S/RuHWW4FKGzBStTmstG3m0xzxTMnQkV3kPimMim3I3VsxxeGEdq +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/test_jwt_auth/helpers/private_key_2 b/tests/integration/test_jwt_auth/helpers/private_key_2 new file mode 100644 index 000000000000..d0d1576f2017 --- /dev/null +++ b/tests/integration/test_jwt_auth/helpers/private_key_2 @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA0RRsKcZ5j9UckjioG4Phvav3dkg2sXP6tQ7ug0yowAo/u2Hf +fB+1OjKuhWTpA3E3YkMKj0RrT+tuUpmZEXqCAipEV7XcfCv3o7Poa7HTq1ti/abV +wT/KyfGjoNBBSJH4LTNAyo2J8ySKSDtpAEU52iL7s40Ra6I0vqp7/aRuPF5M4zcH +zN3zarG5EfSVSG1+gTkaRv8XJbra0IeIINmKv0F4++ww8ZxXTR6cvI+MsArUiAPw +zf7s5dMR4DNRG6YNTrPA0pTOqQE9sRPd62XsfU08plYm27naOUZO5avIPl1YO5I6 +Gi4kPdTvv3WFIy+QvoKoPhPCaD6EbdBpe8BbTQIDAQABAoIBABghJsCFfucKHdOE +RWZziHx22cblW6aML41wzTcLBFixdhx+lafCEwzF551OgZPbn5wwB4p0R3xAPAm9 +X0yEmnd8gEmtG+aavmg+rZ6sNbULhXenpvi4D4PR5uP61OX2rrEsvpgB0L9mYq0m +ah5VXvFdYzYcHDwTSsoMa+XgcbZ2qCW6Si3jnbBA1TPIJS5GjfPUQlu9g2FKQL5H +tlJ7L4Wq39zkueS6LH7kEXOoM+jHgA8F4f7MIrajmilYqnuXanVcMV3+K/6FvH2B +VBiLggG3CerhB3QyEvZBshvEvvcyRff2NK64CGr/xrAElj4cPHk/E499M1uvUXjE +boCrD+ECgYEA9LvLljf59h8WWF4bKQZGNKprgFdQIZ2iCEf+VGdGWt/mNg+LyXyn +3gS/vReON1eaMuEGklZM4Guh/ZPhsPaNmlu16PjmeYTIW1vQTHiO3KR7tAmWep70 +w+gVxDDzuRvBkuDF5oQsZnD3Ri9I7r+J5y9OhyZUsDXe/LJARivF3x0CgYEA2rRx +wl4mfuYmikvcO8I4vuKXcK1UyYmZQLhp6EHKfhSVgrt7XsstZX9AP2OxUUAocRks +e6vU/sKUSni7TQrZzAZHc8JXonDgmCqoMPBXIuUncvysGR1kmgVIbN8ISPKJuZoV +8Dbj3fQfHZ0g0R+mUcuZ+xBO5CKcjPWHZXZoxfECgYAQ/5o8bNbnyXD74k1wpAbs +UYn1+BqQuyot+RIpOqMgXLzYtGu5Kvdd7GaE88XlAiirsAWM1IGydMdjnYnniLh9 +KDGSZPddKWPhNJdbOGRz3tjYwHG7Qp8tnEkmv1+uU8c2NHaKdFPBKceDEHW4X4Vs +kVSa/oaTVqqOUrM0LIYp4QKBgQCW1aIriiGEnZhxAvbGJCJczAvkAzcZtBOFBmrM +ayuLnwiqXEEu1HPfr06RKWFuhxAdSF5cgNrqRSpe3jtXXCdvxdjbpmooNy8+4xSS +g/+kqmR1snvC6nmqnAAiTgP5w4RnBDUjMcggGLCpDOhIMkrT2Na+x7WRM6nCsceK +m4qREQKBgEWqdb/QkOMvvKAz2DPDeSrwlTyisrZu1G/86uE3ESb97DisPK+TF2Ts +r4RGUlKL79W3j5xjvIvqGEEDLC+8QKpay9OYXk3lbViPGB8akWMSP6Tw/8AedhVu +sjFqcBEFGOELwm7VjAcDeP6bXeXibFe+rysBrfFHUGllytCmNoAV +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/test_jwt_auth/jwks_server/server.py b/tests/integration/test_jwt_auth/jwks_server/server.py new file mode 100644 index 000000000000..96e07f02335e --- /dev/null +++ b/tests/integration/test_jwt_auth/jwks_server/server.py @@ -0,0 +1,33 @@ +import sys + +from bottle import response, route, run + + +@route("/.well-known/jwks.json") +def server(): + result = { + "keys": [ + { + "kty": "RSA", + "alg": "RS512", + "kid": "mykid", + "n": "0RRsKcZ5j9UckjioG4Phvav3dkg2sXP6tQ7ug0yowAo_u2HffB-1OjKuhWTpA3E3YkMKj0RrT-tuUpmZEXqCAipEV7XcfCv3o" + "7Poa7HTq1ti_abVwT_KyfGjoNBBSJH4LTNAyo2J8ySKSDtpAEU52iL7s40Ra6I0vqp7_aRuPF5M4zcHzN3zarG5EfSVSG1-gT" + "kaRv8XJbra0IeIINmKv0F4--ww8ZxXTR6cvI-MsArUiAPwzf7s5dMR4DNRG6YNTrPA0pTOqQE9sRPd62XsfU08plYm27naOUZ" + "O5avIPl1YO5I6Gi4kPdTvv3WFIy-QvoKoPhPCaD6EbdBpe8BbTQ", + "e": "AQAB"}, + ] + } + response.status = 200 + response.content_type = "application/json" + return result + + +@route("/") +def ping(): + response.content_type = "text/plain" + response.set_header("Content-Length", 2) + return "OK" + + +run(host="0.0.0.0", port=int(sys.argv[1])) diff --git a/tests/integration/test_jwt_auth/test.py b/tests/integration/test_jwt_auth/test.py new file mode 100644 index 000000000000..6a1e1fe68e72 --- /dev/null +++ b/tests/integration/test_jwt_auth/test.py @@ -0,0 +1,101 @@ +import os +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.mock_servers import start_mock_servers + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance( + "instance", + main_configs=["configs/validators.xml"], + user_configs=["configs/users.xml"], + with_minio=True, + # We actually don't need minio, but we need to run dummy resolver + # (a shortcut not to change cluster.py in a more unclear way, TBC later). +) +client = cluster.add_instance( + "client", +) + + +def run_jwks_server(): + script_dir = os.path.join(os.path.dirname(__file__), "jwks_server") + start_mock_servers( + cluster, + script_dir, + [ + ("server.py", "resolver", "8080"), + ], + ) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + run_jwks_server() + yield cluster + finally: + cluster.shutdown() + + +def curl_with_jwt(token, ip, https=False): + http_prefix = "https" if https else "http" + curl = f'curl -H "X-ClickHouse-JWT-Token: Bearer {token}" "{http_prefix}://{ip}:8123/?query=SELECT%20currentUser()"' + return curl + + +# See helpers/ directory if you need to re-create tokens (or understand how they are created) +def test_static_key(started_cluster): + res = client.exec_in_container( + [ + "bash", + "-c", + curl_with_jwt( + token="eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJqd3RfdXNlciJ9." + "kfivQ8qD_oY0UvihydeadD7xvuiO3zSmhFOc_SGbEPQ", + ip=cluster.get_instance_ip(instance.name), + ), + ] + ) + assert res == "jwt_user\n" + + +def test_static_jwks(started_cluster): + res = client.exec_in_container( + [ + "bash", + "-c", + curl_with_jwt( + token="eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Im15a2lkIn0." + "eyJzdWIiOiJqd3RfdXNlciIsImlzcyI6InRlc3RfaXNzIn0." + "CUioyRc_ms75YWkUwvPgLvaVk2Wmj8RzgqDALVd9LWUzCL5aU4yc_YaA3qnG_NoHd0uUF4FUjLxiocRoKNEgsE2jj7g_" + "wFMC5XHSHuFlfIZjovObXQEwGcKpXO2ser7ANu3k2jBC2FMpLfr_sZZ_GYSnqbp2WF6-l0uVQ0AHVwOy4x1Xkawiubkg" + "W2I2IosaEqT8QNuvvFWLWc1k-dgiNp8k6P-K4D4NBQub0rFlV0n7AEKNdV-_AEzaY_IqQT0sDeBSew_mdR0OH_N-6-" + "FmWWIroIn2DQ7pq93BkI7xdkqnxtt8RCWkCG8JLcoeJt8sHh7uTKi767loZJcPPNaxKA", + ip=cluster.get_instance_ip(instance.name), + ), + ] + ) + assert res == "jwt_user\n" + + +def test_jwks_server(started_cluster): + res = client.exec_in_container( + [ + "bash", + "-c", + curl_with_jwt( + token="eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzUxMiIsImtpZCI6Im15a2lkIn0." + "eyJzdWIiOiJqd3RfdXNlciIsImlzcyI6InRlc3RfaXNzIn0.MjegqrrVyrMMpkxIM-J_q-" + "Sw68Vk5xZuFpxecLLMFs5qzvnh0jslWtyRfi-ANJeJTONPZM5m0yP1ITt8BExoHWobkkR11bXz0ylYEIOgwxqw" + "36XhL2GkE17p-wMvfhCPhGOVL3b7msDRUKXNN48aAJA-NxRbQFhMr-eEx3HsrZXy17Qc7z-" + "0dINe355kzAInGp6gMk3uksAlJ3vMODK8jE-WYFqXusr5GFhXubZXdE2mK0mIbMUGisOZhZLc4QVwvUsYDLBCgJ2RHr5vm" + "jp17j_ZArIedUJkjeC4o72ZMC97kLVnVw94QJwNvd4YisxL6A_mWLTRq9FqNLD4HmbcOQ", + ip=cluster.get_instance_ip(instance.name), + ), + ] + ) + assert res == "jwt_user\n" diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index d6c785a7bc74..bacccfc93bae 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -277,6 +277,8 @@ DoubleDelta Doxygen Dresseler Durre +ECDSA +EdDSA ECMA ETag EachRow @@ -506,6 +508,8 @@ JoinAlgorithm JoinStrictness JumpConsistentHash Jupyter +jwks +JWKS KDevelop KafkaAssignedPartitions KafkaBackgroundReads @@ -3138,6 +3142,7 @@ uuid uuids uuidv vCPU +validators varPop varPopStable varSamp @@ -3155,6 +3160,8 @@ vectorscan vendoring verificationDepth verificationMode +verifier +verifiers versionedcollapsingmergetree vhost virtualized