diff --git a/docs/deployment/configuration.mdx b/docs/deployment/configuration.mdx index fa884cb0b6..aad174ee19 100644 --- a/docs/deployment/configuration.mdx +++ b/docs/deployment/configuration.mdx @@ -63,6 +63,41 @@ Keep is highly configurable through environment variables. This allows you to cu | **DB_SERVICE_ACCOUNT** | Service account for database impersonation | No | None | Valid service account email | | **DB_IP_TYPE** | Specifies the Cloud SQL IP type | No | "public" | "public", "private" or "psc" | | **SKIP_DB_CREATION** | Skips database creation and migrations | No | "false" | "true" or "false" | +| **POSTGRES_SCHEMA** | PostgreSQL schema name for database objects | No | None | Valid schema name (e.g., "keep_prod") | + +#### PostgreSQL Schema Configuration + + + Keep supports organizing database objects within a specific PostgreSQL schema, which is useful for multi-tenant deployments or when sharing a database with other applications. + + +When using PostgreSQL, you can configure Keep to use a specific schema instead of the default 'public' schema: + +**Environment Variable Method:** +```bash +export POSTGRES_SCHEMA=keep_prod +export DATABASE_CONNECTION_STRING="postgresql+psycopg2://username:password@host:port/database" +``` + +The schema will be automatically set in the connection's search_path, allowing all Keep tables to be created and accessed within the specified schema. + +**Prerequisites:** +1. Create the schema in your PostgreSQL database: + ```sql + CREATE SCHEMA IF NOT EXISTS keep_prod; + ``` + +2. Grant necessary permissions to the Keep database user: + ```sql + GRANT USAGE, CREATE ON SCHEMA keep_prod TO keep_user; + -- After tables are created, you may also need: + GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA keep_prod TO keep_user; + GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA keep_prod TO keep_user; + ``` + + + When `POSTGRES_SCHEMA` is set, Keep will automatically configure the PostgreSQL search_path to use your schema with 'public' as a fallback for system extensions. All migrations and database operations will use the configured schema. + ### Resource Provisioning diff --git a/keep/api/core/db_utils.py b/keep/api/core/db_utils.py index 6d0841e7db..17588a9d04 100644 --- a/keep/api/core/db_utils.py +++ b/keep/api/core/db_utils.py @@ -134,6 +134,10 @@ def create_db_engine(): """ Creates a database engine based on the environment variables. """ + # Check for PostgreSQL schema configuration + postgres_schema = config("POSTGRES_SCHEMA", default=None) + connect_args = {} + if RUNNING_IN_CLOUD_RUN and not KEEP_FORCE_CONNECTION_STRING: engine = create_engine( "mysql+pymysql://", @@ -151,6 +155,11 @@ def create_db_engine(): json_serializer=dumps, ) elif DB_CONNECTION_STRING: + # Add PostgreSQL schema support + if postgres_schema and "postgresql" in DB_CONNECTION_STRING: + connect_args["options"] = f"-csearch_path={postgres_schema},public" + logger.info(f"PostgreSQL schema configured: {postgres_schema}") + try: logger.info(f"Creating a connection pool with size {DB_POOL_SIZE}") engine = create_engine( @@ -160,6 +169,7 @@ def create_db_engine(): json_serializer=dumps, echo=DB_ECHO, pool_pre_ping=True if KEEP_DB_PRE_PING_ENABLED else False, + connect_args=connect_args if connect_args else None, ) # SQLite does not support pool_size except TypeError: diff --git a/keep/api/models/db/migrations/versions/2025-04-08-10-43_59991b568c7d.py b/keep/api/models/db/migrations/versions/2025-04-08-10-43_59991b568c7d.py index b3e6e66755..0c12c2c26d 100644 --- a/keep/api/models/db/migrations/versions/2025-04-08-10-43_59991b568c7d.py +++ b/keep/api/models/db/migrations/versions/2025-04-08-10-43_59991b568c7d.py @@ -48,7 +48,7 @@ def upgrade() -> None: FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relname = 'idx_status_started' - AND n.nspname = 'public' + AND n.nspname = current_schema() ) THEN CREATE INDEX idx_status_started ON workflowexecution (status, started); diff --git a/tests/e2e_tests/README-schema-test.md b/tests/e2e_tests/README-schema-test.md new file mode 100644 index 0000000000..85513ed4c4 --- /dev/null +++ b/tests/e2e_tests/README-schema-test.md @@ -0,0 +1,94 @@ +# Testing PostgreSQL Custom Schema Configuration + +This directory contains test files to verify Keep's PostgreSQL custom schema functionality. + +## Files + +- `docker compose-test-schema.yml` - Docker Compose configuration with custom schema setup +- `docker-entrypoint-initdb.d/init-custom-schema.sql` - PostgreSQL initialization script +- `test-custom-schema.sh` - Automated test script +- `verify-schema.sql` - SQL queries to manually verify schema setup + +## Quick Test + +Run the automated test: +```bash +cd tests/e2e_tests +./test-custom-schema.sh +``` + +## Manual Testing + +1. Start the services: +```bash +# Set image environment variables (or use defaults) +export KEEPBACKEND_IMAGE="us-central1-docker.pkg.dev/keephq/keep/keep-api:latest" +export KEEPFRONTEND_IMAGE="us-central1-docker.pkg.dev/keephq/keep/keep-ui:latest" + +# Start services +docker compose -f docker compose-test-schema.yml up -d +``` + +2. Wait for initialization (about 30 seconds for migrations) + +3. Verify schema setup: +```bash +# Check if custom schema was created +docker compose -f docker compose-test-schema.yml exec postgres-custom-schema \ + psql -U keepuser -d keepdb -c "\dn keep_custom" + +# Run verification queries +docker compose -f docker compose-test-schema.yml exec postgres-custom-schema \ + psql -U keepuser -d keepdb -f /docker-entrypoint-initdb.d/verify-schema.sql + +# Check Keep logs +docker compose -f docker compose-test-schema.yml logs keep-backend-custom-schema | grep -i schema +``` + +4. Access Keep UI at http://localhost:3002 to verify functionality + +## What's Being Tested + +1. **Schema Creation**: Custom schema `keep_custom` is created with proper permissions +2. **Environment Variable**: `POSTGRES_SCHEMA=keep_custom` is properly handled +3. **Table Creation**: All Keep tables are created in the custom schema, not in public +4. **Migrations**: Alembic migrations run successfully in the custom schema +5. **Application Function**: Keep operates normally with the custom schema + +## Configuration Details + +The test uses: +- PostgreSQL 13 +- Custom schema name: `keep_custom` +- Database user: `keepuser` +- Database name: `keepdb` +- Keep backend port: 8082 +- Keep frontend port: 3002 +- PostgreSQL port: 5434 + +## Cleanup + +Remove all test containers and volumes: +```bash +docker compose -f docker compose-test-schema.yml down -v +``` + +## Troubleshooting + +If tests fail: + +1. Check Keep backend logs for migration errors: + ```bash + docker compose -f docker compose-test-schema.yml logs keep-backend-custom-schema + ``` + +2. Connect to PostgreSQL to inspect: + ```bash + docker compose -f docker compose-test-schema.yml exec postgres-custom-schema \ + psql -U keepuser -d keepdb + ``` + +3. Verify environment variables: + ```bash + docker compose -f docker compose-test-schema.yml exec keep-backend-custom-schema env | grep -E "(POSTGRES_SCHEMA|DATABASE_CONNECTION)" + ``` \ No newline at end of file diff --git a/tests/e2e_tests/docker-compose-test-schema.yml b/tests/e2e_tests/docker-compose-test-schema.yml new file mode 100644 index 0000000000..08bea44eca --- /dev/null +++ b/tests/e2e_tests/docker-compose-test-schema.yml @@ -0,0 +1,53 @@ +services: + # PostgreSQL with custom schema setup + postgres-custom-schema: + image: postgres:13 + environment: + POSTGRES_USER: keepuser + POSTGRES_PASSWORD: keeppassword + POSTGRES_DB: keepdb + ports: + - "5434:5432" + volumes: + - ./docker-entrypoint-initdb.d/init-custom-schema.sql:/docker-entrypoint-initdb.d/01-init-custom-schema.sql + - postgres-schema-test-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U keepuser -d keepdb"] + interval: 5s + timeout: 5s + retries: 5 + + # Keep backend with custom schema + keep-backend-custom-schema: + image: "us-central1-docker.pkg.dev/keephq/keep/keep-api:latest" + ports: + - "8082:8080" + environment: + - AUTH_TYPE=NO_AUTH + - DATABASE_CONNECTION_STRING=postgresql+psycopg2://keepuser:keeppassword@postgres-custom-schema:5432/keepdb + - POSTGRES_SCHEMA=keep_custom + - POSTHOG_DISABLED=true + - SECRET_MANAGER_DIRECTORY=/app + - SQLALCHEMY_WARN_20=1 + - DATABASE_ECHO=true # Enable to see SQL queries + depends_on: + postgres-custom-schema: + condition: service_healthy + + # Keep frontend for testing + keep-frontend-custom-schema: + image: "us-central1-docker.pkg.dev/keephq/keep/keep-ui:latest" + ports: + - "3002:3000" + environment: + - AUTH_TYPE=NO_AUTH + - NEXTAUTH_SECRET=secret + - NEXTAUTH_URL=http://localhost:3002 + - API_URL=http://keep-backend-custom-schema:8080 + - POSTHOG_DISABLED=true + - SENTRY_DISABLED=true + depends_on: + - keep-backend-custom-schema + +volumes: + postgres-schema-test-data: \ No newline at end of file diff --git a/tests/e2e_tests/docker-entrypoint-initdb.d/init-custom-schema.sql b/tests/e2e_tests/docker-entrypoint-initdb.d/init-custom-schema.sql new file mode 100644 index 0000000000..046b0b3c19 --- /dev/null +++ b/tests/e2e_tests/docker-entrypoint-initdb.d/init-custom-schema.sql @@ -0,0 +1,28 @@ +-- Create custom schema for Keep +CREATE SCHEMA IF NOT EXISTS keep_custom; + +-- Grant all necessary permissions to the keepuser +GRANT USAGE, CREATE ON SCHEMA keep_custom TO keepuser; + +-- Set search_path for the user (optional, as Keep will handle this) +ALTER USER keepuser SET search_path TO keep_custom, public; + +-- Create a test table to verify schema is working +CREATE TABLE keep_custom.schema_test ( + id SERIAL PRIMARY KEY, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + message TEXT DEFAULT 'Schema keep_custom is working!' +); + +-- Insert a test record +INSERT INTO keep_custom.schema_test (message) VALUES ('Custom schema initialized successfully'); + +-- Grant permissions on the test table +GRANT ALL PRIVILEGES ON TABLE keep_custom.schema_test TO keepuser; +GRANT USAGE, SELECT ON SEQUENCE keep_custom.schema_test_id_seq TO keepuser; + +-- Log the initialization +DO $$ +BEGIN + RAISE NOTICE 'Custom schema keep_custom created and configured for Keep'; +END $$; \ No newline at end of file diff --git a/tests/e2e_tests/test-custom-schema.sh b/tests/e2e_tests/test-custom-schema.sh new file mode 100755 index 0000000000..312d4fd4f7 --- /dev/null +++ b/tests/e2e_tests/test-custom-schema.sh @@ -0,0 +1,88 @@ +#!/bin/bash +set -e + +echo "=== Testing PostgreSQL Custom Schema Configuration ===" +echo + +# Colors for output +GREEN='\033[0;32m' +RED='\033[0;31m' +NC='\033[0m' # No Color + +# Function to check if command succeeded +check_result() { + if [ $? -eq 0 ]; then + echo -e "${GREEN}✓ $1${NC}" + else + echo -e "${RED}✗ $1${NC}" + exit 1 + fi +} + +# Note: Images are already set in docker-compose-test-schema.yml +# No need for sed replacements + +echo "1. Starting PostgreSQL with custom schema configuration..." +docker compose -f docker-compose-test-schema.yml up -d postgres-custom-schema +sleep 10 # Wait for PostgreSQL to initialize +check_result "PostgreSQL started" + +echo +echo "2. Verifying custom schema was created..." +docker compose -f docker-compose-test-schema.yml exec -T postgres-custom-schema psql -U keepuser -d keepdb -c "\dn keep_custom" | grep keep_custom +check_result "Custom schema 'keep_custom' exists" + +echo +echo "3. Checking schema permissions..." +docker compose -f docker-compose-test-schema.yml exec -T postgres-custom-schema psql -U keepuser -d keepdb -c "SELECT has_schema_privilege('keepuser', 'keep_custom', 'CREATE');" | grep -q 't' +check_result "User has CREATE permission on custom schema" + +echo +echo "4. Starting Keep backend with POSTGRES_SCHEMA=keep_custom..." +docker compose -f docker-compose-test-schema.yml up -d keep-backend-custom-schema +echo "Waiting for Keep backend to initialize and run migrations..." +sleep 30 # Wait for migrations to run +check_result "Keep backend started" + +echo +echo "5. Checking if Keep tables were created in custom schema..." +TABLES=$(docker compose -f docker-compose-test-schema.yml exec -T postgres-custom-schema psql -U keepuser -d keepdb -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'keep_custom' AND table_name NOT IN ('schema_test');" -t | tr -d ' ') +if [ "$TABLES" -gt "0" ]; then + echo -e "${GREEN}✓ Found $TABLES Keep tables in custom schema${NC}" +else + echo -e "${RED}✗ No Keep tables found in custom schema${NC}" + echo "Checking logs for errors..." + docker compose -f docker-compose-test-schema.yml logs keep-backend-custom-schema | tail -20 + exit 1 +fi + +echo +echo "6. Listing some Keep tables in custom schema..." +docker compose -f docker-compose-test-schema.yml exec -T postgres-custom-schema psql -U keepuser -d keepdb -c "SELECT table_name FROM information_schema.tables WHERE table_schema = 'keep_custom' AND table_name NOT IN ('schema_test') ORDER BY table_name LIMIT 10;" + +echo +echo "7. Verifying no Keep tables in public schema..." +PUBLIC_TABLES=$(docker compose -f docker-compose-test-schema.yml exec -T postgres-custom-schema psql -U keepuser -d keepdb -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public' AND (table_name LIKE 'keep%' OR table_name LIKE 'alert%' OR table_name LIKE 'workflow%');" -t | tr -d ' ') +if [ "$PUBLIC_TABLES" -eq "0" ]; then + echo -e "${GREEN}✓ No Keep tables found in public schema (as expected)${NC}" +else + echo -e "${RED}✗ Found $PUBLIC_TABLES Keep tables in public schema (unexpected)${NC}" +fi + +echo +echo "8. Testing API health endpoint..." +curl -s http://localhost:8082/healthcheck | grep -q "ok" +check_result "API health check passed" + +echo +echo "9. Checking current schema from Keep's perspective..." +docker compose -f docker-compose-test-schema.yml exec -T postgres-custom-schema psql -U keepuser -d keepdb -c "SHOW search_path;" + +echo +echo -e "${GREEN}=== All tests passed! ===${NC}" +echo "Keep is successfully using PostgreSQL with custom schema 'keep_custom'" +echo +echo "To view logs: docker compose -f docker-compose-test-schema.yml logs" +echo "To clean up: docker compose -f docker-compose-test-schema.yml down -v" + +# No backup file to restore since we skipped sed commands \ No newline at end of file diff --git a/tests/e2e_tests/verify-schema.sql b/tests/e2e_tests/verify-schema.sql new file mode 100644 index 0000000000..3c04b1d10a --- /dev/null +++ b/tests/e2e_tests/verify-schema.sql @@ -0,0 +1,35 @@ +-- Verification queries for custom schema setup + +-- 1. Check if custom schema exists +SELECT nspname AS schema_name, + pg_catalog.pg_get_userbyid(nspowner) AS owner +FROM pg_catalog.pg_namespace +WHERE nspname = 'keep_custom'; + +-- 2. Check current search_path +SHOW search_path; + +-- 3. List all Keep-related tables in custom schema +SELECT table_schema, table_name, table_type +FROM information_schema.tables +WHERE table_schema = 'keep_custom' +ORDER BY table_name; + +-- 4. Count tables in each schema +SELECT table_schema, COUNT(*) as table_count +FROM information_schema.tables +WHERE table_schema IN ('public', 'keep_custom') + AND table_type = 'BASE TABLE' +GROUP BY table_schema; + +-- 5. Check if alembic_version is in custom schema (migration tracking) +SELECT table_schema, table_name +FROM information_schema.tables +WHERE table_name = 'alembic_version'; + +-- 6. Verify the test migration with current_schema() worked +SELECT n.nspname as schema_name, + c.relname as index_name +FROM pg_class c +JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE c.relname = 'idx_status_started'; \ No newline at end of file