feat: Support for PostgreSQL custom schema configuration

docs/deployment/configuration.mdx

@@ -63,6 +63,41 @@ Keep is highly configurable through environment variables. This allows you to cu
 |     **DB_SERVICE_ACCOUNT**     |    Service account for database impersonation     |    No    |               None                |    Valid service account email     |
 |         **DB_IP_TYPE**         |          Specifies the Cloud SQL IP type          |    No    |             "public"              |    "public", "private" or "psc"    |
 |      **SKIP_DB_CREATION**      |      Skips database creation and migrations       |    No    |              "false"              |         "true" or "false"          |
+|      **POSTGRES_SCHEMA**       |     PostgreSQL schema name for database objects    |    No    |               None                |      Valid schema name (e.g., "keep_prod")         |
+
+#### PostgreSQL Schema Configuration
+
+<Note>
+  Keep supports organizing database objects within a specific PostgreSQL schema, which is useful for multi-tenant deployments or when sharing a database with other applications.
+</Note>
+
+When using PostgreSQL, you can configure Keep to use a specific schema instead of the default 'public' schema:
+
+**Environment Variable Method:**
+```bash
+export POSTGRES_SCHEMA=keep_prod
+export DATABASE_CONNECTION_STRING="postgresql+psycopg2://username:password@host:port/database"
+```
+
+The schema will be automatically set in the connection's search_path, allowing all Keep tables to be created and accessed within the specified schema.
+
+**Prerequisites:**
+1. Create the schema in your PostgreSQL database:
+   ```sql
+   CREATE SCHEMA IF NOT EXISTS keep_prod;
+   ```
+
+2. Grant necessary permissions to the Keep database user:
+   ```sql
+   GRANT USAGE, CREATE ON SCHEMA keep_prod TO keep_user;
+   -- After tables are created, you may also need:
+   GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA keep_prod TO keep_user;
+   GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA keep_prod TO keep_user;
+   ```
+
+<Info>
+  When `POSTGRES_SCHEMA` is set, Keep will automatically configure the PostgreSQL search_path to use your schema with 'public' as a fallback for system extensions. All migrations and database operations will use the configured schema.
+</Info>
 
 ### Resource Provisioning
 

keep/api/core/db_utils.py

@@ -134,6 +134,10 @@ def create_db_engine():
     """
     Creates a database engine based on the environment variables.
     """
+    # Check for PostgreSQL schema configuration
+    postgres_schema = config("POSTGRES_SCHEMA", default=None)
+    connect_args = {}
+
     if RUNNING_IN_CLOUD_RUN and not KEEP_FORCE_CONNECTION_STRING:
         engine = create_engine(
             "mysql+pymysql://",
@@ -151,6 +155,11 @@ def create_db_engine():
             json_serializer=dumps,
         )
     elif DB_CONNECTION_STRING:
+        # Add PostgreSQL schema support
+        if postgres_schema and "postgresql" in DB_CONNECTION_STRING:
+            connect_args["options"] = f"-csearch_path={postgres_schema},public"
+            logger.info(f"PostgreSQL schema configured: {postgres_schema}")
+
         try:
             logger.info(f"Creating a connection pool with size {DB_POOL_SIZE}")
             engine = create_engine(
@@ -160,6 +169,7 @@ def create_db_engine():
                 json_serializer=dumps,
                 echo=DB_ECHO,
                 pool_pre_ping=True if KEEP_DB_PRE_PING_ENABLED else False,
+                connect_args=connect_args if connect_args else None,
             )
         # SQLite does not support pool_size
         except TypeError:

keep/api/models/db/migrations/versions/2025-04-08-10-43_59991b568c7d.py

@@ -48,7 +48,7 @@ def upgrade() -> None:
                     FROM pg_class c
                     JOIN pg_namespace n ON n.oid = c.relnamespace
                     WHERE c.relname = 'idx_status_started'
-                    AND n.nspname = 'public'
+                    AND n.nspname = current_schema()
                 ) THEN
                     CREATE INDEX idx_status_started
                     ON workflowexecution (status, started);

tests/e2e_tests/README-schema-test.md

@@ -0,0 +1,94 @@
+# Testing PostgreSQL Custom Schema Configuration
+
+This directory contains test files to verify Keep's PostgreSQL custom schema functionality.
+
+## Files
+
+- `docker compose-test-schema.yml` - Docker Compose configuration with custom schema setup
+- `docker-entrypoint-initdb.d/init-custom-schema.sql` - PostgreSQL initialization script
+- `test-custom-schema.sh` - Automated test script
+- `verify-schema.sql` - SQL queries to manually verify schema setup
+
+## Quick Test
+
+Run the automated test:
+```bash
+cd tests/e2e_tests
+./test-custom-schema.sh
+```
+
+## Manual Testing
+
+1. Start the services:
+```bash
+# Set image environment variables (or use defaults)
+export KEEPBACKEND_IMAGE="us-central1-docker.pkg.dev/keephq/keep/keep-api:latest"
+export KEEPFRONTEND_IMAGE="us-central1-docker.pkg.dev/keephq/keep/keep-ui:latest"
+
+# Start services
+docker compose -f docker compose-test-schema.yml up -d
+```
+
+2. Wait for initialization (about 30 seconds for migrations)
+
+3. Verify schema setup:
+```bash
+# Check if custom schema was created
+docker compose -f docker compose-test-schema.yml exec postgres-custom-schema \
+  psql -U keepuser -d keepdb -c "\dn keep_custom"
+
+# Run verification queries
+docker compose -f docker compose-test-schema.yml exec postgres-custom-schema \
+  psql -U keepuser -d keepdb -f /docker-entrypoint-initdb.d/verify-schema.sql
+
+# Check Keep logs
+docker compose -f docker compose-test-schema.yml logs keep-backend-custom-schema | grep -i schema
+```
+
+4. Access Keep UI at http://localhost:3002 to verify functionality
+
+## What's Being Tested
+
+1. **Schema Creation**: Custom schema `keep_custom` is created with proper permissions
+2. **Environment Variable**: `POSTGRES_SCHEMA=keep_custom` is properly handled
+3. **Table Creation**: All Keep tables are created in the custom schema, not in public
+4. **Migrations**: Alembic migrations run successfully in the custom schema
+5. **Application Function**: Keep operates normally with the custom schema
+
+## Configuration Details
+
+The test uses:
+- PostgreSQL 13
+- Custom schema name: `keep_custom`
+- Database user: `keepuser`
+- Database name: `keepdb`
+- Keep backend port: 8082
+- Keep frontend port: 3002
+- PostgreSQL port: 5434
+
+## Cleanup
+
+Remove all test containers and volumes:
+```bash
+docker compose -f docker compose-test-schema.yml down -v
+```
+
+## Troubleshooting
+
+If tests fail:
+
+1. Check Keep backend logs for migration errors:
+   ```bash
+   docker compose -f docker compose-test-schema.yml logs keep-backend-custom-schema
+   ```
+
+2. Connect to PostgreSQL to inspect:
+   ```bash
+   docker compose -f docker compose-test-schema.yml exec postgres-custom-schema \
+     psql -U keepuser -d keepdb
+   ```
+
+3. Verify environment variables:
+   ```bash
+   docker compose -f docker compose-test-schema.yml exec keep-backend-custom-schema env | grep -E "(POSTGRES_SCHEMA|DATABASE_CONNECTION)"
+   ```

tests/e2e_tests/docker-compose-test-schema.yml

@@ -0,0 +1,53 @@
+services:
+  # PostgreSQL with custom schema setup
+  postgres-custom-schema:
+    image: postgres:13
+    environment:
+      POSTGRES_USER: keepuser
+      POSTGRES_PASSWORD: keeppassword
+      POSTGRES_DB: keepdb
+    ports:
+      - "5434:5432"
+    volumes:
+      - ./docker-entrypoint-initdb.d/init-custom-schema.sql:/docker-entrypoint-initdb.d/01-init-custom-schema.sql
+      - postgres-schema-test-data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U keepuser -d keepdb"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
+  # Keep backend with custom schema
+  keep-backend-custom-schema:
+    image: "us-central1-docker.pkg.dev/keephq/keep/keep-api:latest"
+    ports:
+      - "8082:8080"
+    environment:
+      - AUTH_TYPE=NO_AUTH
+      - DATABASE_CONNECTION_STRING=postgresql+psycopg2://keepuser:keeppassword@postgres-custom-schema:5432/keepdb
+      - POSTGRES_SCHEMA=keep_custom
+      - POSTHOG_DISABLED=true
+      - SECRET_MANAGER_DIRECTORY=/app
+      - SQLALCHEMY_WARN_20=1
+      - DATABASE_ECHO=true  # Enable to see SQL queries
+    depends_on:
+      postgres-custom-schema:
+        condition: service_healthy
+
+  # Keep frontend for testing
+  keep-frontend-custom-schema:
+    image: "us-central1-docker.pkg.dev/keephq/keep/keep-ui:latest"
+    ports:
+      - "3002:3000"
+    environment:
+      - AUTH_TYPE=NO_AUTH
+      - NEXTAUTH_SECRET=secret
+      - NEXTAUTH_URL=http://localhost:3002
+      - API_URL=http://keep-backend-custom-schema:8080
+      - POSTHOG_DISABLED=true
+      - SENTRY_DISABLED=true
+    depends_on:
+      - keep-backend-custom-schema
+
+volumes:
+  postgres-schema-test-data:

tests/e2e_tests/docker-entrypoint-initdb.d/init-custom-schema.sql

@@ -0,0 +1,28 @@
+-- Create custom schema for Keep
+CREATE SCHEMA IF NOT EXISTS keep_custom;
+
+-- Grant all necessary permissions to the keepuser
+GRANT USAGE, CREATE ON SCHEMA keep_custom TO keepuser;
+
+-- Set search_path for the user (optional, as Keep will handle this)
+ALTER USER keepuser SET search_path TO keep_custom, public;
+
+-- Create a test table to verify schema is working
+CREATE TABLE keep_custom.schema_test (
+    id SERIAL PRIMARY KEY,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    message TEXT DEFAULT 'Schema keep_custom is working!'
+);
+
+-- Insert a test record
+INSERT INTO keep_custom.schema_test (message) VALUES ('Custom schema initialized successfully');
+
+-- Grant permissions on the test table
+GRANT ALL PRIVILEGES ON TABLE keep_custom.schema_test TO keepuser;
+GRANT USAGE, SELECT ON SEQUENCE keep_custom.schema_test_id_seq TO keepuser;
+
+-- Log the initialization
+DO $$
+BEGIN
+    RAISE NOTICE 'Custom schema keep_custom created and configured for Keep';
+END $$;

tests/e2e_tests/test-custom-schema.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+set -e
+
+echo "=== Testing PostgreSQL Custom Schema Configuration ==="
+echo
+
+# Colors for output
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+NC='\033[0m' # No Color
+
+# Function to check if command succeeded
+check_result() {
+    if [ $? -eq 0 ]; then
+        echo -e "${GREEN}✓ $1${NC}"
+    else
+        echo -e "${RED}✗ $1${NC}"
+        exit 1
+    fi
+}
+
+# Note: Images are already set in docker-compose-test-schema.yml
+# No need for sed replacements
+
+echo "1. Starting PostgreSQL with custom schema configuration..."
+docker compose -f docker-compose-test-schema.yml up -d postgres-custom-schema
+sleep 10  # Wait for PostgreSQL to initialize
+check_result "PostgreSQL started"
+
+echo
+echo "2. Verifying custom schema was created..."
+docker compose -f docker-compose-test-schema.yml exec -T postgres-custom-schema psql -U keepuser -d keepdb -c "\dn keep_custom" | grep keep_custom
+check_result "Custom schema 'keep_custom' exists"
+
+echo
+echo "3. Checking schema permissions..."
+docker compose -f docker-compose-test-schema.yml exec -T postgres-custom-schema psql -U keepuser -d keepdb -c "SELECT has_schema_privilege('keepuser', 'keep_custom', 'CREATE');" | grep -q 't'
+check_result "User has CREATE permission on custom schema"
+
+echo
+echo "4. Starting Keep backend with POSTGRES_SCHEMA=keep_custom..."
+docker compose -f docker-compose-test-schema.yml up -d keep-backend-custom-schema
+echo "Waiting for Keep backend to initialize and run migrations..."
+sleep 30  # Wait for migrations to run
+check_result "Keep backend started"
+
+echo
+echo "5. Checking if Keep tables were created in custom schema..."
+TABLES=$(docker compose -f docker-compose-test-schema.yml exec -T postgres-custom-schema psql -U keepuser -d keepdb -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'keep_custom' AND table_name NOT IN ('schema_test');" -t | tr -d ' ')
+if [ "$TABLES" -gt "0" ]; then
+    echo -e "${GREEN}✓ Found $TABLES Keep tables in custom schema${NC}"
+else
+    echo -e "${RED}✗ No Keep tables found in custom schema${NC}"
+    echo "Checking logs for errors..."
+    docker compose -f docker-compose-test-schema.yml logs keep-backend-custom-schema | tail -20
+    exit 1
+fi
+
+echo
+echo "6. Listing some Keep tables in custom schema..."
+docker compose -f docker-compose-test-schema.yml exec -T postgres-custom-schema psql -U keepuser -d keepdb -c "SELECT table_name FROM information_schema.tables WHERE table_schema = 'keep_custom' AND table_name NOT IN ('schema_test') ORDER BY table_name LIMIT 10;"
+
+echo
+echo "7. Verifying no Keep tables in public schema..."
+PUBLIC_TABLES=$(docker compose -f docker-compose-test-schema.yml exec -T postgres-custom-schema psql -U keepuser -d keepdb -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public' AND (table_name LIKE 'keep%' OR table_name LIKE 'alert%' OR table_name LIKE 'workflow%');" -t | tr -d ' ')
+if [ "$PUBLIC_TABLES" -eq "0" ]; then
+    echo -e "${GREEN}✓ No Keep tables found in public schema (as expected)${NC}"
+else
+    echo -e "${RED}✗ Found $PUBLIC_TABLES Keep tables in public schema (unexpected)${NC}"
+fi
+
+echo
+echo "8. Testing API health endpoint..."
+curl -s http://localhost:8082/healthcheck | grep -q "ok"
+check_result "API health check passed"
+
+echo
+echo "9. Checking current schema from Keep's perspective..."
+docker compose -f docker-compose-test-schema.yml exec -T postgres-custom-schema psql -U keepuser -d keepdb -c "SHOW search_path;"
+
+echo
+echo -e "${GREEN}=== All tests passed! ===${NC}"
+echo "Keep is successfully using PostgreSQL with custom schema 'keep_custom'"
+echo
+echo "To view logs: docker compose -f docker-compose-test-schema.yml logs"
+echo "To clean up: docker compose -f docker-compose-test-schema.yml down -v"
+
+# No backup file to restore since we skipped sed commands

tests/e2e_tests/verify-schema.sql

@@ -0,0 +1,35 @@
+-- Verification queries for custom schema setup
+
+-- 1. Check if custom schema exists
+SELECT nspname AS schema_name, 
+       pg_catalog.pg_get_userbyid(nspowner) AS owner
+FROM pg_catalog.pg_namespace
+WHERE nspname = 'keep_custom';
+
+-- 2. Check current search_path
+SHOW search_path;
+
+-- 3. List all Keep-related tables in custom schema
+SELECT table_schema, table_name, table_type
+FROM information_schema.tables
+WHERE table_schema = 'keep_custom'
+ORDER BY table_name;
+
+-- 4. Count tables in each schema
+SELECT table_schema, COUNT(*) as table_count
+FROM information_schema.tables
+WHERE table_schema IN ('public', 'keep_custom')
+  AND table_type = 'BASE TABLE'
+GROUP BY table_schema;
+
+-- 5. Check if alembic_version is in custom schema (migration tracking)
+SELECT table_schema, table_name
+FROM information_schema.tables
+WHERE table_name = 'alembic_version';
+
+-- 6. Verify the test migration with current_schema() worked
+SELECT n.nspname as schema_name, 
+       c.relname as index_name
+FROM pg_class c
+JOIN pg_namespace n ON n.oid = c.relnamespace
+WHERE c.relname = 'idx_status_started';