[DOP-21268] - integration with SSO (Keycloak)

.env.docker

@@ -4,17 +4,34 @@ ENV=LOCAL
 # Debug
 SYNCMASTER__SERVER__DEBUG=true
 
-# Logging Backend
-SYNCMASTER__SERVER__LOGGING__SETUP=True
-SYNCMASTER__SERVER__LOGGING__PRESET=colored
+# Logging
+SYNCMASTER__LOGGING__SETUP=True
+SYNCMASTER__LOGGING__PRESET=colored
+SYNCMASTER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}
 
-# Logging Worker
-SYNCMASTER__WORKER__LOGGING__SETUP=True
-SYNCMASTER__WORKER__LOGGING__PRESET=json
+# Session
+SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
+
+# Encrypt / Decrypt credentials data
+SYNCMASTER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
 
 # Postgres
 SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster
 
+# TODO: add to KeycloakAuthProvider documentation about creating new realms, add users, etc.
+# KEYCLOAK Auth
+SYNCMASTER__AUTH__SERVER_URL=http://keycloak:8080/
+SYNCMASTER__AUTH__REALM_NAME=manually_created
+SYNCMASTER__AUTH__CLIENT_ID=manually_created
+SYNCMASTER__AUTH__CLIENT_SECRET=generated_by_keycloak
+SYNCMASTER__AUTH__REDIRECT_URI=http://localhost:8000/v1/auth/callback
+SYNCMASTER__AUTH__SCOPE=email
+SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.keycloak_provider.KeycloakAuthProvider
+
+# Dummy Auth
+SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.dummy_provider.DummyAuthProvider
+SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=secret
+
 # RabbitMQ
 SYNCMASTER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/
 

.env.local

@@ -5,16 +5,23 @@ export ENV=LOCAL
 export SYNCMASTER__SERVER__DEBUG=true
 
 # Logging
-export SYNCMASTER__SERVER__LOGGING__SETUP=True
-export SYNCMASTER__SERVER__LOGGING__PRESET=colored
+export SYNCMASTER__LOGGING__SETUP=True
+export SYNCMASTER__LOGGING__PRESET=colored
+export SYNCMASTER__LOG_URL_TEMPLATE="https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}"
 
-# Logging Worker
-export SYNCMASTER__WORKER__LOGGING__SETUP=True
-export SYNCMASTER__WORKER__LOGGING__PRESET=json
+# Session
+export SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
+
+# Encrypt / Decrypt credentials data
+export SYNCMASTER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
 
 # Postgres
 export SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster
 
+# Auth
+export SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.dummy_provider.DummyAuthProvider
+export SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=secret
+
 # RabbitMQ
 export SYNCMASTER__BROKER__URL=amqp://guest:guest@localhost:5672/
 

.readthedocs.yaml

@@ -17,7 +17,7 @@ build:
       - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH python -m poetry install --no-root --all-extras --with docs --without dev,test
       - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH python -m poetry show -v
       - python -m pip list -v
-      - SYNCMASTER__DATABASE__URL=postgresql+psycopg://fake:[email protected]:5432/fake SYNCMASTER__BROKER__URL=amqp://fake:faket@fake:5672/ python -m syncmaster.backend.export_openapi_schema docs/_static/openapi.json
+      - SYNCMASTER__DATABASE__URL=postgresql+psycopg://fake:[email protected]:5432/fake SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key SYNCMASTER__BROKER__URL=amqp://fake:faket@fake:5672/ SYNCMASTER__CRYPTO_KEY=crypto_key SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=fakepython python -m syncmaster.backend.export_openapi_schema docs/_static/openapi.json
 
 sphinx:
   configuration: docs/conf.py

docker-compose.test.yml

@@ -40,6 +40,7 @@ services:
       - 8000:8000
     volumes:
       - ./syncmaster:/app/syncmaster
+      - ./docs/_static:/app/docs/_static
       - ./cached_jars:/root/.ivy2
       - ./reports:/app/reports
       - ./tests:/app/tests
@@ -156,6 +157,19 @@ services:
       retries: 3
     profiles: [hive, hdfs, all]
 
+  keycloak:
+    image: quay.io/keycloak/keycloak:latest
+    command: start-dev
+    restart: unless-stopped
+    environment:
+      KEYCLOAK_ADMIN: admin
+      KEYCLOAK_ADMIN_PASSWORD: admin
+    ports:
+      - 8080:8080
+    volumes:
+      - keycloak_data:/opt/keycloak/data
+    profiles: [keycloak, all]
+
   test-hive:
     image: mtsrus/hadoop:hadoop2.7.3-hive2.3.9
     restart: unless-stopped
@@ -184,3 +198,4 @@ services:
 volumes:
   postgres_test_data:
   rabbitmq_test_data:
+  keycloak_data:

docs/backend/auth/custom.rst

@@ -0,0 +1,10 @@
+.. _backend-auth-custom:
+
+Custom Auth provider
+====================
+
+You can implement custom auth provider by inheriting from class below and implementing necessary methods.
+
+.. autoclass:: syncmaster.backend.providers.auth.AuthProvider
+    :members:
+    :member-order: bysource

docs/backend/auth/dummy.rst

@@ -0,0 +1,82 @@
+.. _backend-auth-dummy:
+
+Dummy Auth provider
+===================
+
+Description
+-----------
+
+This auth provider allows to sign-in with any username and password, and and then issues an access token.
+
+After successful auth, username is saved to backend database. It is then used for creating audit records for any object change, see ``changed_by`` field.
+
+Interaction schema
+------------------
+
+.. dropdown:: Interaction schema
+
+    .. plantuml::
+
+        @startuml
+            title DummyAuthProvider
+            participant "Client"
+            participant "Backend"
+
+            == POST v1/auth/token ==
+
+            activate "Client"
+            alt Successful case
+                "Client" -> "Backend" ++ : login + password
+                "Backend" --> "Backend" : Password is completely ignored
+                "Backend" --> "Backend" : Check user in internal backend database
+                "Backend" -> "Backend" : Create user if not exist
+                "Backend" -[#green]> "Client" -- : Generate and return access_token
+
+            else User is blocked
+                "Client" -> "Backend" ++ : login + password
+                "Backend" --> "Backend" : Password is completely ignored
+                "Backend" --> "Backend" : Check user in internal backend database
+                "Backend" x-[#red]> "Client" -- : 401 Unauthorized
+
+            else User is deleted
+                "Client" -> "Backend" ++ : login + password
+                "Backend" --> "Backend" : Password is completely ignored
+                "Backend" --> "Backend" : Check user in internal backend database
+                "Backend" x-[#red]> "Client" -- : 404 Not found
+            end
+
+            == GET v1/namespaces ==
+
+            alt Successful case
+                "Client" -> "Backend" ++ : access_token
+                "Backend" --> "Backend" : Validate token
+                "Backend" --> "Backend" : Check user in internal backend database
+                "Backend" -> "Backend" : Get data
+                "Backend" -[#green]> "Client" -- : Return data
+
+            else Token is expired
+                "Client" -> "Backend" ++ : access_token
+                "Backend" --> "Backend" : Validate token
+                "Backend" x-[#red]> "Client" -- : 401 Unauthorized
+
+            else User is blocked
+                "Client" -> "Backend" ++ : access_token
+                "Backend" --> "Backend" : Validate token
+                "Backend" --> "Backend" : Check user in internal backend database
+                "Backend" x-[#red]> "Client" -- : 401 Unauthorized
+
+            else User is deleted
+                "Client" -> "Backend" ++ : access_token
+                "Backend" --> "Backend" : Validate token
+                "Backend" --> "Backend" : Check user in internal backend database
+                "Backend" x-[#red]> "Client" -- : 404 Not found
+            end
+
+            deactivate "Client"
+        @enduml
+
+Configuration
+-------------
+
+.. autopydantic_model:: syncmaster.backend.settings.auth.dummy.DummyAuthProviderSettings
+.. autopydantic_model:: syncmaster.backend.settings.auth.jwt.JWTSettings

docs/backend/auth/index.rst

@@ -0,0 +1,21 @@
+.. _backend-auth-providers:
+
+Auth Providers
+==============
+
+Syncmaster supports different auth provider implementations. You can change implementation via settings:
+
+.. autopydantic_model:: keycloak.backend.settings.auth.AuthSettings
+
+.. toctree::
+    :maxdepth: 2
+    :caption: Auth providers
+
+    dummy
+    keycloak
+
+.. toctree::
+    :maxdepth: 2
+    :caption: For developers
+
+    custom

docs/backend/auth/keycloak.rst

@@ -0,0 +1,81 @@
+.. _backend-auth-ldap:
+
+KeyCloak Auth provider
+==================
+
+Description
+-----------
+
+TODO:
+
+Strategies
+----------
+
+TODO:
+
+Interaction schema
+------------------
+
+.. dropdown:: Interaction schema
+
+    .. plantuml::
+
+        @startuml
+            title Keycloak Authorization Flow
+            participant "Client (User from Browser)" as Client
+            participant "Syncmaster"
+            participant "Keycloak"
+
+            == Client Authentication at Keycloak ==
+            Client -> Syncmaster : Request endpoint that requires authentication (/v1/users)
+
+            Syncmaster x-[#red]> Client : Redirect to Keycloak login URL (if no access token)
+
+            Client -> Keycloak : Callback redirect to Keycloak login page
+
+            alt Successful login
+                Client --> Keycloak : Log in with login and password
+            else Login failed
+                Keycloak x-[#red]> Client -- : Display error (401 Unauthorized)
+            end
+
+            Keycloak -> Client : Redirect to Syncmaster to callback endpoint with code
+            Client -> Syncmaster : Callback request to /v1/auth/callback with code
+            Syncmaster-> Keycloak : Exchange code for access token
+            Keycloak --> Syncmaster : Return JWT token
+            Syncmaster --> Client : Set JWT token in user's browser in cookies and redirect /v1/users
+
+            Client --> Syncmaster : Redirect to /v1/users
+            Syncmaster -> Syncmaster : Get user info from JWT token and check user in internal backend database
+            Syncmaster -> Syncmaster : Create user in internal backend database if not exist
+            Syncmaster -[#green]> Client -- : Return requested data
+
+
+
+            == GET v1/users ==
+            alt Successful case
+                Client -> Syncmaster : Request data with JWT token
+                Syncmaster --> Syncmaster : Get user info from JWT token and check user in internal backend database
+                Syncmaster -> Syncmaster : Create user in internal backend database if not exist
+                Syncmaster -[#green]> Client -- : Return requested data
+
+            else Access token is expired
+                Syncmaster -> Keycloak : Get new JWT token via refresh token
+                Keycloak --> Syncmaster : Return new JWT token
+                Syncmaster --> Syncmaster : Get user info from JWT token and check user in internal backend database
+                Syncmaster -> Syncmaster : Create user in internal backend database if not exist
+                Syncmaster -[#green]> Client -- : Return requested data and set new JWT token in user's browser in cookies
+
+            else Refresh token is expired
+                Syncmaster x-[#red]> Client -- : Redirect to Keycloak login URL
+            end
+
+            deactivate Client
+        @enduml
+
+Basic configuration
+-------------------
+
+.. autopydantic_model:: syncmaster.settings.auth.keycloak.KeycloakProviderSettings
+.. autopydantic_model:: syncmaster.settings.auth.jwt.JWTSettings
+

docs/backend/configuration/cors.rst

@@ -5,4 +5,4 @@ CORS settings
 
 These settings used to control `CORS <https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS>`_ options.
 
-.. autopydantic_model:: syncmaster.settings.server.cors.CORSSettings
+.. autopydantic_model:: syncmaster.backend.settings.server.cors.CORSSettings

docs/backend/configuration/index.rst

@@ -11,6 +11,7 @@ Configuration
     database
     broker
     logging
+    session
     cors
     debug
     monitoring

docs/backend/configuration/monitoring.rst

@@ -13,4 +13,4 @@ REST API server provides the following endpoints with Prometheus compatible metr
 
 These endpoints are enabled and configured using settings below:
 
-.. autopydantic_model:: syncmaster.settings.server.monitoring.MonitoringSettings
+.. autopydantic_model:: syncmaster.backend.settings.server.monitoring.MonitoringSettings

docs/backend/configuration/openapi.rst

@@ -5,8 +5,8 @@ OpenAPI settings
 
 These settings used to control exposing OpenAPI.json and SwaggerUI/ReDoc endpoints.
 
-.. autopydantic_model:: syncmaster.settings.server.openapi.OpenAPISettings
-.. autopydantic_model:: syncmaster.settings.server.openapi.SwaggerSettings
-.. autopydantic_model:: syncmaster.settings.server.openapi.RedocSettings
-.. autopydantic_model:: syncmaster.settings.server.openapi.LogoSettings
-.. autopydantic_model:: syncmaster.settings.server.openapi.FaviconSettings
+.. autopydantic_model:: syncmaster.backend.settings.server.openapi.OpenAPISettings
+.. autopydantic_model:: syncmaster.backend.settings.server.openapi.SwaggerSettings
+.. autopydantic_model:: syncmaster.backend.settings.server.openapi.RedocSettings
+.. autopydantic_model:: syncmaster.backend.settings.server.openapi.LogoSettings
+.. autopydantic_model:: syncmaster.backend.settings.server.openapi.FaviconSettings

docs/backend/configuration/session.rst

@@ -0,0 +1,8 @@
+.. _backend-configuration-server-session:
+
+Session settings
+================
+
+These settings used to control `Session <https://developer.mozilla.org/en-US/docs/Web/HTTP/Session>`_ options.
+
+.. autopydantic_model:: syncmaster.backend.settings.server.session.SessionSettings

docs/backend/configuration/static_files.rst

@@ -5,4 +5,4 @@ Serving static files
 
 These settings used to control serving static files by a server.
 
-.. autopydantic_model:: syncmaster.settings.server.static_files.StaticFilesSettings
+.. autopydantic_model:: syncmaster.backend.settings.server.static_files.StaticFilesSettings

docs/backend/install.rst

@@ -123,6 +123,6 @@ To start backend server you need to execute following command:
 
 .. code-block:: console
 
-    $ python -m horizon.backend --host 0.0.0.0 --port 8000
+    $ python -m syncmaster.backend --host 0.0.0.0 --port 8000
 
 After server is started and ready, open http://localhost:8000/docs.

docs/changelog/next_release/123.feature.rst

@@ -0,0 +1,13 @@
+- Implemented ``KeycloakAuthProvider`` for Single Sign-On (SSO) authentication.
+- Implemented ``DummyAuthProvider`` for development and testing environments.
+- Enabled dynamic selection of authentication provider via environment variable SYNCMASTER__AUTH__PROVIDER:
+
+    .. code::
+
+        # syncmaster.backend.providers.auth.keycloak_provider.KeycloakAuthProvider for Keycloak.
+        SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.keycloak_provider.KeycloakAuthProvider
+
+        # syncmaster.backend.providers.auth.dummy_provider.DummyAuthProvider for Dummy authentication.
+        SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.dummy_provider.DummyAuthProvider
+
+- Updated ``User`` model to include ``email``, ``first_name``, ``middle_name``, and ``last_name`` fields.

docs/index.rst

@@ -16,6 +16,7 @@
 
     backend/install
     backend/architecture
+    backend/auth/index
     backend/openapi
     backend/configuration/index