compose.yaml

name: chess-crawl

x-python: &python
  image: chess-crawl:local
  build:
    context: .
  init: true
  user: "10001:10001"
  read_only: true
  tmpfs:
    # A tmpfs hides the image directory's owner; set it for the runtime user.
    - /tmp:uid=10001,gid=10001,mode=0700
  cap_drop:
    - ALL
  security_opt:
    - no-new-privileges:true

x-database-env: &database-env
  CHESS_CRAWL_DATABASE_MAX_CONNECTIONS: ${CHESS_CRAWL_DATABASE_MAX_CONNECTIONS:-32}
  CHESS_CRAWL_DATABASE_ADMISSION_TIMEOUT_S: ${CHESS_CRAWL_DATABASE_ADMISSION_TIMEOUT_S:-5.0}
  CHESS_CRAWL_DATABASE_URL: ${CHESS_CRAWL_DATABASE_URL:-postgresql://chess_crawl@postgres:5432/chess_crawl}
  CHESS_CRAWL_DATABASE_PASSWORD_FILE: /run/secrets/postgres_password
  # This explicit policy trusts only the bundled Compose service hostname.
  CHESS_CRAWL_DATABASE_TRANSPORT: local
  CHESS_CRAWL_DATABASE_TRUSTED_HOST: postgres
  CHESS_CRAWL_ARCHIVE_BACKEND: ${CHESS_CRAWL_ARCHIVE_BACKEND:-local}
  CHESS_CRAWL_ARCHIVE_DIRECTORY: /var/lib/chess-crawl/archive
  CHESS_CRAWL_ARCHIVE_S3_BUCKET: ${CHESS_CRAWL_ARCHIVE_S3_BUCKET:-}
  AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-us-east-1}
  CHESS_CRAWL_USAGE_LOG: ${CHESS_CRAWL_USAGE_LOG:-false}
  CHESS_CRAWL_EVENTS_ENABLED: ${CHESS_CRAWL_EVENTS_ENABLED:-true}

x-budget-env: &budget-env
  CHESS_CRAWL_JOB_MAX_GAMES: ${CHESS_CRAWL_JOB_MAX_GAMES:-100000}
  CHESS_CRAWL_JOB_MAX_NORMALIZATION_UNITS: ${CHESS_CRAWL_JOB_MAX_NORMALIZATION_UNITS:-100000}
  CHESS_CRAWL_JOB_MAX_REMOTE_BYTES: ${CHESS_CRAWL_JOB_MAX_REMOTE_BYTES:-268435456}
  CHESS_CRAWL_JOB_MAX_REMOTE_REQUESTS: ${CHESS_CRAWL_JOB_MAX_REMOTE_REQUESTS:-2000}
  CHESS_CRAWL_MAX_RESPONSE_BYTES: ${CHESS_CRAWL_MAX_RESPONSE_BYTES:-16777216}
  CHESS_CRAWL_WORKSPACE_MAX_GAMES: ${CHESS_CRAWL_WORKSPACE_MAX_GAMES:-1000000}
  CHESS_CRAWL_WORKSPACE_MAX_NORMALIZATION_UNITS: ${CHESS_CRAWL_WORKSPACE_MAX_NORMALIZATION_UNITS:-1000000}
  CHESS_CRAWL_WORKSPACE_MAX_REMOTE_BYTES: ${CHESS_CRAWL_WORKSPACE_MAX_REMOTE_BYTES:-2147483648}
  CHESS_CRAWL_WORKSPACE_MAX_REMOTE_REQUESTS: ${CHESS_CRAWL_WORKSPACE_MAX_REMOTE_REQUESTS:-10000}
  CHESS_CRAWL_WORKSPACE_MAX_ACTIVE_JOBS: ${CHESS_CRAWL_WORKSPACE_MAX_ACTIVE_JOBS:-2}
  CHESS_CRAWL_WORKSPACE_MAX_QUEUED_JOBS: ${CHESS_CRAWL_WORKSPACE_MAX_QUEUED_JOBS:-32}

x-worker-env: &worker-env
  <<: [*database-env, *budget-env]
  CHESS_CRAWL_CONTACT: ${CHESS_CRAWL_CONTACT:-set-me@example.invalid}
  CHESS_CRAWL_USER_AGENT: ${CHESS_CRAWL_USER_AGENT:-}
  CHESS_CRAWL_CHESSCOM_DELAY_S: ${CHESS_CRAWL_CHESSCOM_DELAY_S:-1}
  CHESS_CRAWL_LICHESS_DELAY_S: ${CHESS_CRAWL_LICHESS_DELAY_S:-1.5}
  CHESS_CRAWL_PROVIDER_MAX_RETRIES: ${CHESS_CRAWL_PROVIDER_MAX_RETRIES:-3}
  CHESS_CRAWL_POLL_INTERVAL: ${CHESS_CRAWL_POLL_INTERVAL:-1}
  CHESS_CRAWL_HEARTBEAT_INTERVAL: ${CHESS_CRAWL_HEARTBEAT_INTERVAL:-5}
  CHESS_CRAWL_HEARTBEAT_MAX_AGE: ${CHESS_CRAWL_HEARTBEAT_MAX_AGE:-20}
  CHESS_CRAWL_JOB_MAX_RETRIES: ${CHESS_CRAWL_JOB_MAX_RETRIES:-3}
  CHESS_CRAWL_RETRY_BASE: ${CHESS_CRAWL_RETRY_BASE:-30}
  CHESS_CRAWL_RETRY_MAX: ${CHESS_CRAWL_RETRY_MAX:-3600}
  CHESS_CRAWL_LICHESS_TOKEN: ${CHESS_CRAWL_LICHESS_TOKEN:-}
  CHESS_CRAWL_LICHESS_TOKEN_OWNER_SCOPE: ${CHESS_CRAWL_LICHESS_TOKEN_OWNER_SCOPE:-local}
  CHESS_CRAWL_LICHESS_CLOCKS: ${CHESS_CRAWL_LICHESS_CLOCKS:-true}
  CHESS_CRAWL_LICHESS_EVALS: ${CHESS_CRAWL_LICHESS_EVALS:-true}
  CHESS_CRAWL_LICHESS_ACCURACY: ${CHESS_CRAWL_LICHESS_ACCURACY:-true}
  CHESS_CRAWL_DISPATCH_RETENTION_SECONDS: ${CHESS_CRAWL_DISPATCH_RETENTION_SECONDS:-86400}
  CHESS_CRAWL_DISPATCH_CLEANUP_INTERVAL_SECONDS: ${CHESS_CRAWL_DISPATCH_CLEANUP_INTERVAL_SECONDS:-60}
  CHESS_CRAWL_DISPATCH_CLEANUP_BATCH_SIZE: ${CHESS_CRAWL_DISPATCH_CLEANUP_BATCH_SIZE:-256}

x-archive-jobs-env: &archive-jobs-env
  CHESS_CRAWL_ARCHIVE_JOBS_ENABLED: ${CHESS_CRAWL_ARCHIVE_JOBS_ENABLED:-false}
  CHESS_CRAWL_ARTIFACT_BACKEND: ${CHESS_CRAWL_ARTIFACT_BACKEND:-local}
  CHESS_CRAWL_ARTIFACT_DIRECTORY: ${CHESS_CRAWL_ARTIFACT_DIRECTORY:-/var/lib/chess-crawl/artifacts}
  CHESS_CRAWL_ARTIFACT_S3_BUCKET: ${CHESS_CRAWL_ARTIFACT_S3_BUCKET:-}
  CHESS_CRAWL_ASYNC_MAX_WORKING_SET_MEMBERS: ${CHESS_CRAWL_ASYNC_MAX_WORKING_SET_MEMBERS:-1000000}
  CHESS_CRAWL_ASYNC_EXPORT_MAX_ROWS: ${CHESS_CRAWL_ASYNC_EXPORT_MAX_ROWS:-1000000}
  CHESS_CRAWL_ASYNC_EXPORT_MAX_BYTES: ${CHESS_CRAWL_ASYNC_EXPORT_MAX_BYTES:-268435456}
  CHESS_CRAWL_ASYNC_EXPORT_PREPARE_SECONDS: ${CHESS_CRAWL_ASYNC_EXPORT_PREPARE_SECONDS:-600}
  CHESS_CRAWL_ARTIFACT_MAX_COUNT: ${CHESS_CRAWL_ARTIFACT_MAX_COUNT:-32}
  CHESS_CRAWL_ARTIFACT_MAX_BYTES: ${CHESS_CRAWL_ARTIFACT_MAX_BYTES:-1073741824}
  CHESS_CRAWL_ARTIFACT_TTL_SECONDS: ${CHESS_CRAWL_ARTIFACT_TTL_SECONDS:-86400}

x-source-ready: &source-ready
  init:
    condition: service_completed_successfully
  archive-init:
    condition: service_completed_successfully

x-artifacts-ready: &artifacts-ready
  <<: *source-ready
  artifacts-init:
    condition: service_completed_successfully

services:
  archive-init:
    <<: *python
    user: "0:0"
    cap_add: [CHOWN, FOWNER]
    command: ["python", "/app/docker/archive_init.py"]
    volumes:
      - archive_data:/var/lib/chess-crawl/archive
    restart: "no"

  artifacts-init:
    <<: *python
    user: "0:0"
    cap_add: [CHOWN, FOWNER]
    command: ["python", "/app/docker/archive_init.py", "${CHESS_CRAWL_ARTIFACT_DIRECTORY:-/var/lib/chess-crawl/artifacts}"]
    volumes:
      - artifacts_data:${CHESS_CRAWL_ARTIFACT_DIRECTORY:-/var/lib/chess-crawl/artifacts}
    restart: "no"

  postgres:
    image: postgres:18
    restart: unless-stopped
    environment:
      POSTGRES_USER: chess_crawl
      POSTGRES_DB: chess_crawl
      POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
    secrets:
      - postgres_password
    volumes:
      # PostgreSQL 18 places its versioned data directory beneath this mount.
      - postgres_data:/var/lib/postgresql
    healthcheck:
      test: ["CMD-SHELL", 'pg_isready -U "$$POSTGRES_USER" -d "$$POSTGRES_DB"']
      interval: 5s
      timeout: 5s
      retries: 5
      start_period: 10s
    stop_grace_period: 60s

  init:
    <<: *python
    command: ["python", "-m", "chess_crawl.operations", "migrate"]
    environment: *database-env
    secrets:
      - postgres_password
    depends_on:
      postgres:
        condition: service_healthy
    restart: "no"

  api:
    <<: *python
    restart: unless-stopped
    depends_on: *artifacts-ready
    volumes:
      - archive_data:/var/lib/chess-crawl/archive:ro
      - artifacts_data:${CHESS_CRAWL_ARTIFACT_DIRECTORY:-/var/lib/chess-crawl/artifacts}:ro
    environment:
      <<: [*database-env, *budget-env, *archive-jobs-env]
      CHESS_CRAWL_API_TOKEN_FILE: /run/secrets/api_token
      CHESS_CRAWL_MAX_GAMES: ${CHESS_CRAWL_MAX_GAMES:-1000}
      CHESS_CRAWL_MAX_DEPTH: ${CHESS_CRAWL_MAX_DEPTH:-2}
      CHESS_CRAWL_MAX_USERS: ${CHESS_CRAWL_MAX_USERS:-100}
      CHESS_CRAWL_MAX_JOBS: ${CHESS_CRAWL_MAX_JOBS:-200}
      CHESS_CRAWL_PAGE_SIZE: ${CHESS_CRAWL_PAGE_SIZE:-100}
      CHESS_CRAWL_MAX_DATE_SPAN_DAYS: ${CHESS_CRAWL_MAX_DATE_SPAN_DAYS:-366}
      CHESS_CRAWL_MAX_WORKING_SET_MEMBERS: ${CHESS_CRAWL_MAX_WORKING_SET_MEMBERS:-10000}
      CHESS_CRAWL_MAX_ANALYSIS_RESULTS: ${CHESS_CRAWL_MAX_ANALYSIS_RESULTS:-1000}
      CHESS_CRAWL_MAX_ANALYSIS_RESULT_BYTES: ${CHESS_CRAWL_MAX_ANALYSIS_RESULT_BYTES:-67108864}
      CHESS_CRAWL_EXPORT_MAX_ROWS: ${CHESS_CRAWL_EXPORT_MAX_ROWS:-100000}
      CHESS_CRAWL_EXPORT_MAX_BYTES: ${CHESS_CRAWL_EXPORT_MAX_BYTES:-67108864}
      CHESS_CRAWL_EXPORT_PREPARE_SECONDS: ${CHESS_CRAWL_EXPORT_PREPARE_SECONDS:-60}
      CHESS_CRAWL_EXPORT_DOWNLOAD_SECONDS: ${CHESS_CRAWL_EXPORT_DOWNLOAD_SECONDS:-300}
      CHESS_CRAWL_EXPORT_WORKSPACE_SLOTS: ${CHESS_CRAWL_EXPORT_WORKSPACE_SLOTS:-2}
      CHESS_CRAWL_EXPORT_OUTSTANDING_SPOOLS: ${CHESS_CRAWL_EXPORT_OUTSTANDING_SPOOLS:-4}
      CHESS_CRAWL_EXPORT_OUTSTANDING_BYTES: ${CHESS_CRAWL_EXPORT_OUTSTANDING_BYTES:-268435456}
      CHESS_CRAWL_EXPORT_WORKSPACE_OUTSTANDING_SPOOLS: ${CHESS_CRAWL_EXPORT_WORKSPACE_OUTSTANDING_SPOOLS:-2}
      CHESS_CRAWL_EXPORT_WORKSPACE_OUTSTANDING_BYTES: ${CHESS_CRAWL_EXPORT_WORKSPACE_OUTSTANDING_BYTES:-134217728}
    secrets:
      - postgres_password
      - api_token
    ports:
      - "${CHESS_CRAWL_BIND_ADDRESS:-127.0.0.1}:${CHESS_CRAWL_API_PORT:-8000}:8000"
    healthcheck:
      test: ["CMD", "python", "/app/docker/healthcheck.py", "api"]
      interval: 5s
      timeout: 5s
      retries: 5
      start_period: 10s

  worker: &worker
    <<: *python
    restart: unless-stopped
    depends_on: *artifacts-ready
    volumes:
      - archive_data:/var/lib/chess-crawl/archive
      - artifacts_data:${CHESS_CRAWL_ARTIFACT_DIRECTORY:-/var/lib/chess-crawl/artifacts}
    command: ["python", "-m", "chess_crawl.jobs.worker"]
    environment:
      <<: [*worker-env, *archive-jobs-env]
    secrets:
      - postgres_password
    stop_grace_period: 90s
    healthcheck:
      test: ["CMD", "python", "/app/docker/healthcheck.py", "worker"]
      interval: 5s
      timeout: 5s
      retries: 5
      start_period: 10s

  acquisition:
    <<: *worker
    profiles: [scalable]
    command: ["python", "-m", "chess_crawl.jobs.worker", "--stage", "acquisition"]
    depends_on: *source-ready
    volumes:
      - archive_data:/var/lib/chess-crawl/archive
    environment: *worker-env

  processing:
    <<: *worker
    profiles: [scalable]
    command: ["python", "-m", "chess_crawl.jobs.worker", "--stage", "processing"]
    volumes:
      - archive_data:/var/lib/chess-crawl/archive:ro
      - artifacts_data:${CHESS_CRAWL_ARTIFACT_DIRECTORY:-/var/lib/chess-crawl/artifacts}
    environment:
      <<: [*worker-env, *archive-jobs-env]
      CHESS_CRAWL_LICHESS_TOKEN: ""

  events:
    <<: *python
    restart: unless-stopped
    depends_on:
      init:
        condition: service_completed_successfully
      mercure:
        condition: service_healthy
    command: ["python", "-m", "chess_crawl.events.publisher"]
    environment:
      <<: *database-env
      CHESS_CRAWL_MERCURE_URL: http://mercure/.well-known/mercure
      CHESS_CRAWL_MERCURE_PUBLISHER_JWT_FILE: /run/secrets/mercure_publisher_jwt
      CHESS_CRAWL_MERCURE_TOPIC_PREFIX: ${CHESS_CRAWL_MERCURE_TOPIC_PREFIX:-https://chess-crawl.local}
      CHESS_CRAWL_EVENTS_RETENTION_SECONDS: ${CHESS_CRAWL_EVENTS_RETENTION_SECONDS:-86400}
      CHESS_CRAWL_EVENTS_CLEANUP_BATCH_SIZE: ${CHESS_CRAWL_EVENTS_CLEANUP_BATCH_SIZE:-256}
    secrets:
      - postgres_password
      - mercure_publisher_jwt
    stop_grace_period: 30s

  mercure:
    image: dunglas/mercure:v0.24.2
    restart: unless-stopped
    entrypoint: ["/bin/sh", "/etc/chess-crawl/mercure-entrypoint.sh"]
    environment:
      SERVER_NAME: ":80"
      MERCURE_PUBLISHER_JWT_ALG: HS256
      MERCURE_SUBSCRIBER_JWT_ALG: HS256
      MERCURE_EXTRA_DIRECTIVES: |
        transport bolt {
          path /data/mercure.db
          size ${CHESS_CRAWL_MERCURE_HISTORY_SIZE:-10000}
        }
    secrets:
      - mercure_signing_key
    volumes:
      - mercure_data:/data
      - mercure_config:/config
      - ./docker/mercure-entrypoint.sh:/etc/chess-crawl/mercure-entrypoint.sh:ro
    ports:
      - "${CHESS_CRAWL_BIND_ADDRESS:-127.0.0.1}:${CHESS_CRAWL_MERCURE_PORT:-3000}:80"
    healthcheck:
      test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:2019/mercure/health/ready"]
      interval: 5s
      timeout: 5s
      retries: 5
      start_period: 10s

volumes:
  archive_data:
  artifacts_data:
  postgres_data:
  mercure_data:
  mercure_config:

secrets:
  postgres_password:
    file: ${CHESS_CRAWL_SECRETS_DIR:-./data/dev-secrets}/postgres_password
  api_token:
    file: ${CHESS_CRAWL_SECRETS_DIR:-./data/dev-secrets}/api_token
  mercure_signing_key:
    file: ${CHESS_CRAWL_SECRETS_DIR:-./data/dev-secrets}/mercure_signing_key
  mercure_publisher_jwt:
    file: ${CHESS_CRAWL_SECRETS_DIR:-./data/dev-secrets}/mercure_publisher_jwt