diff --git a/.env.local.example b/.env.local.example deleted file mode 100644 index 1b2cb42..0000000 --- a/.env.local.example +++ /dev/null @@ -1,15 +0,0 @@ -# Local Development Overrides -# Copy this file to .env.local and customize for your local development environment -# This file should NOT be committed to git (.env.local is in .gitignore) - -# Generate a new secret key for your local development: -# python -c 'from django.core.management.utils import get_random_secret_key; print(get_random_secret_key())' -SECRET_KEY=your-secret-key-here - -# Development mode (usually True for local development) -DEBUG=True - -# Add any other local overrides here as needed -# For example: -# DB_PASSWORD=my-local-password -# ALLOWED_HOSTS=localhost,127.0.0.1,0.0.0.0,mylocal.dev diff --git a/.env.prod b/.env.prod index 7b23a50..1fe85a0 100644 --- a/.env.prod +++ b/.env.prod @@ -4,7 +4,7 @@ # CRITICAL: Strong cryptographically secure secret key for production # Generate with: python -c 'from django.core.management.utils import get_random_secret_key; print(get_random_secret_key())' -SECRET_KEY=django-insecure-CHANGE-THIS-IN-PRODUCTION-generate-new-key +# SECRET_KEY=django-insecure-CHANGE-THIS-IN-PRODUCTION-generate-new-key DEBUG=False # Production Database Configuration @@ -13,7 +13,8 @@ DB_USER=postgres DB_PASSWORD=postgres # Using default for now - change in real production! DB_HOST=db DB_PORT=5432 -DATABASE_URL=postgresql://postgres:postgres@db:5432/databus +REDIS_HOST=redis +REDIS_PORT=6379 # Production-specific Django settings ALLOWED_HOSTS=localhost,127.0.0.1,0.0.0.0 diff --git a/Dockerfile b/Dockerfile index 9d41307..34a19c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Multi-stage build for Django app with uv -FROM python:3.12-slim as base +FROM python:3.14-slim as base # Install system dependencies RUN apt-get update && apt-get install -y \ diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 906095d..783b34d 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -11,9 +11,9 @@ services: env_file: - .env - .env.dev - - .env.local environment: # DEBUG comes from .env.dev (.env sets False, .env.dev overrides to True) + - DEBUG=${DEBUG:-1} - SECRET_KEY=${SECRET_KEY:-dev-secret} - ALLOWED_HOSTS=${ALLOWED_HOSTS:-*} - DB_NAME=${DB_NAME:-realtime} @@ -23,6 +23,7 @@ services: - DB_PORT=5432 - REDIS_HOST=redis - REDIS_PORT=6379 + - RUN_MAKEMIGRATIONS=1 volumes: - .:/app depends_on: @@ -41,9 +42,9 @@ services: env_file: - .env - .env.dev - - .env.local environment: # DEBUG comes from .env.dev + - DEBUG=${DEBUG:-1} - SECRET_KEY=${SECRET_KEY:-dev-secret} - ALLOWED_HOSTS=${ALLOWED_HOSTS:-*} - DB_NAME=${DB_NAME:-realtime} @@ -53,6 +54,7 @@ services: - DB_PORT=5432 - REDIS_HOST=redis - REDIS_PORT=6379 + - MIGRATE_ON_CELERY=0 volumes: - .:/app depends_on: @@ -82,9 +84,9 @@ services: env_file: - .env - .env.dev - - .env.local environment: # DEBUG comes from .env.dev + - DEBUG=${DEBUG:-1} - SECRET_KEY=${SECRET_KEY:-dev-secret} - ALLOWED_HOSTS=${ALLOWED_HOSTS:-*} - DB_NAME=${DB_NAME:-realtime} @@ -94,6 +96,7 @@ services: - DB_PORT=5432 - REDIS_HOST=redis - REDIS_PORT=6379 + - MIGRATE_ON_CELERY=0 volumes: - .:/app depends_on: @@ -111,7 +114,7 @@ services: volumes: - postgres_data:/var/lib/postgresql/data healthcheck: - test: ["CMD", "pg_isready", "-U", "${DB_USER:-postgres}"] + test: ["CMD", "pg_isready", "-U", "${DB_USER:-postgres}", "-d", "${DB_NAME:-realtime}"] interval: 10s timeout: 5s retries: 5 diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 5303e59..97e1e8f 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -8,18 +8,6 @@ services: env_file: - .env - .env.prod - - .env.local - environment: - - DEBUG=0 - - SECRET_KEY=${SECRET_KEY} - - ALLOWED_HOSTS=${ALLOWED_HOSTS} - - DB_NAME=${DB_NAME} - - DB_USER=${DB_USER} - - DB_PASSWORD=${DB_PASSWORD} - - DB_HOST=db - - DB_PORT=5432 - - REDIS_HOST=redis - - REDIS_PORT=6379 depends_on: db: condition: service_healthy @@ -49,18 +37,6 @@ services: env_file: - .env - .env.prod - - .env.local - environment: - - DEBUG=0 - - SECRET_KEY=${SECRET_KEY} - - ALLOWED_HOSTS=${ALLOWED_HOSTS} - - DB_NAME=${DB_NAME} - - DB_USER=${DB_USER} - - DB_PASSWORD=${DB_PASSWORD} - - DB_HOST=db - - DB_PORT=5432 - - REDIS_HOST=redis - - REDIS_PORT=6379 depends_on: db: condition: service_healthy @@ -88,18 +64,6 @@ services: env_file: - .env - .env.prod - - .env.local - environment: - - DEBUG=0 - - SECRET_KEY=${SECRET_KEY} - - ALLOWED_HOSTS=${ALLOWED_HOSTS} - - DB_NAME=${DB_NAME} - - DB_USER=${DB_USER} - - DB_PASSWORD=${DB_PASSWORD} - - DB_HOST=db - - DB_PORT=5432 - - REDIS_HOST=redis - - REDIS_PORT=6379 depends_on: db: condition: service_healthy @@ -107,21 +71,25 @@ services: condition: service_healthy db: - image: postgis/postgis:15-3.3 + image: postgis/postgis:16-3.4 + env_file: + - .env + - .env.prod environment: + # Ensure the application database is created on first initialization - POSTGRES_DB=${DB_NAME} - POSTGRES_USER=${DB_USER} - POSTGRES_PASSWORD=${DB_PASSWORD} volumes: - postgres_data:/var/lib/postgresql/data healthcheck: - test: ["CMD", "pg_isready", "-U", "${DB_USER}"] + test: ["CMD", "pg_isready", "-U", "postgres"] interval: 10s timeout: 5s retries: 5 redis: - image: redis:7-alpine + image: redis:8-alpine command: ["redis-server", "--appendonly", "yes"] volumes: - redis_data:/data diff --git a/docker-compose.yml b/docker-compose.yml index 81ac97c..3c69bee 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,6 @@ services: env_file: - .env - .env.dev - - .env.local environment: # Fallbacks if not set in .env - DEBUG=${DEBUG:-1} @@ -23,6 +22,7 @@ services: - DB_PORT=5432 - REDIS_HOST=${REDIS_HOST:-redis} - REDIS_PORT=${REDIS_PORT:-6379} + - RUN_MAKEMIGRATIONS=0 depends_on: db: condition: service_healthy @@ -38,7 +38,6 @@ services: env_file: - .env - .env.dev - - .env.local environment: - DEBUG=${DEBUG:-1} - SECRET_KEY=${SECRET_KEY:-dev-secret-key} @@ -50,6 +49,7 @@ services: - DB_PORT=5432 - REDIS_HOST=${REDIS_HOST:-redis} - REDIS_PORT=${REDIS_PORT:-6379} + - MIGRATE_ON_CELERY=0 depends_on: db: condition: service_healthy @@ -76,7 +76,6 @@ services: env_file: - .env - .env.dev - - .env.local environment: - DEBUG=${DEBUG:-1} - SECRET_KEY=${SECRET_KEY:-dev-secret-key} @@ -88,6 +87,7 @@ services: - DB_PORT=5432 - REDIS_HOST=${REDIS_HOST:-redis} - REDIS_PORT=${REDIS_PORT:-6379} + - MIGRATE_ON_CELERY=0 depends_on: db: condition: service_healthy diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 89be166..6585d1a 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -25,7 +25,11 @@ fi # Build DATABASE_URL if missing (fallback) if [ -z "${DATABASE_URL:-}" ]; then if [[ -n "${DB_USER:-}" && -n "${DB_HOST:-}" && -n "${DB_NAME:-}" ]]; then - export DATABASE_URL="postgresql://${DB_USER}:${DB_PASSWORD:-}${DB_PASSWORD:+@}${DB_HOST}:${DB_PORT:-5432}/${DB_NAME}" + if [ -n "${DB_PASSWORD:-}" ]; then + export DATABASE_URL="postgresql://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:${DB_PORT:-5432}/${DB_NAME}" + else + export DATABASE_URL="postgresql://${DB_USER}@${DB_HOST}:${DB_PORT:-5432}/${DB_NAME}" + fi warn "DATABASE_URL not set; constructed: ${DATABASE_URL}" else warn "DATABASE_URL not set and insufficient components to construct it." @@ -53,13 +57,35 @@ if [ "$IS_CELERY" = true ]; then log "Database is ready!" # Optionally run migrations for beat/worker to ensure django_celery_beat tables exist - if [[ "${MIGRATE_ON_CELERY:-1}" == "1" || "${MIGRATE_ON_CELERY:-true}" == "true" ]]; then + # Default DISABLED to prevent concurrent migrations from multiple services + if [[ "${MIGRATE_ON_CELERY:-0}" == "1" || "${MIGRATE_ON_CELERY:-false}" == "true" ]]; then log "Applying pending migrations (Celery service)..." uv run python manage.py migrate --noinput || warn "Celery migration step failed (continuing)" else log "Skipping migrations in Celery service (MIGRATE_ON_CELERY=${MIGRATE_ON_CELERY})" fi + # If this is the beat process, ensure beat tables are present before starting + if [[ "${*:-}" == *" beat "* || "${*:-}" == *" beat" || "${*:-}" == *"beat "* ]]; then + log "Ensuring django_celery_beat tables exist before starting beat..." + until uv run python - <<'PY' +import os, psycopg2 +dsn=os.environ['DATABASE_URL'] +with psycopg2.connect(dsn) as conn: + with conn.cursor() as cur: + cur.execute("SELECT to_regclass('public.django_celery_beat_periodictask') IS NOT NULL;") + ok = cur.fetchone()[0] +import sys +print("OK" if ok else "WAIT") +sys.exit(0 if ok else 1) +PY + do + warn "django_celery_beat tables not ready - waiting" + sleep 2 + done + log "django_celery_beat tables ready." + fi + log "Starting Celery process..." else log "Starting Django application..." @@ -81,35 +107,50 @@ else log "Database is ready!" - # List of apps to make migrations for - APPS_TO_MIGRATE=("website" "gtfs" "feed" "alerts" "api") - - # Make migrations for the registered apps - log "Making migrations for apps: ${APPS_TO_MIGRATE[*]}" - uv run python manage.py makemigrations "${APPS_TO_MIGRATE[@]}" || warn "No changes detected for migrations" + # Optionally create new migration files (disabled by default to prevent conflicts) + if [[ "${RUN_MAKEMIGRATIONS:-0}" == "1" || "${RUN_MAKEMIGRATIONS:-false}" == "true" ]]; then + APPS_TO_MIGRATE=("website" "gtfs" "feed" "alerts" "api") + log "RUN_MAKEMIGRATIONS enabled. Creating migrations for: ${APPS_TO_MIGRATE[*]}" + uv run python manage.py makemigrations "${APPS_TO_MIGRATE[@]}" || warn "No changes detected for migrations" + else + log "Skipping makemigrations (RUN_MAKEMIGRATIONS=${RUN_MAKEMIGRATIONS:-0})" + fi # Run database migrations log "Running database migrations..." uv run python manage.py migrate --noinput - # Create superuser if it doesn't exist - if [[ "${DEBUG:-}" == "True" || "${DEBUG:-}" == "1" ]]; then - log "Ensuring development superuser 'admin' exists (DEBUG mode)" - uv run python manage.py shell -c "from django.contrib.auth import get_user_model; U=get_user_model();\nimport os;\nusername=os.environ.get('DJANGO_SUPERUSER_USERNAME','admin');\npassword=os.environ.get('DJANGO_SUPERUSER_PASSWORD','admin');\nemail=os.environ.get('DJANGO_SUPERUSER_EMAIL','admin@example.com');\nU.objects.filter(username=username).exists() or (U.objects.create_superuser(username, email, password) and print(f'Superuser created: {username}/{password}')) or print('Superuser already exists')" || warn "Superuser creation skipped" + # Create superuser if it doesn't exist using defaults in development mode + if [[ "${CREATE_SUPERUSER:-1}" == "1" && ( "${DEBUG:-}" == "True" || "${DEBUG:-}" == "1" ) ]]; then + # Provide defaults if not set for development + export DJANGO_SUPERUSER_USERNAME="${DJANGO_SUPERUSER_USERNAME:-admin}" + export DJANGO_SUPERUSER_PASSWORD="${DJANGO_SUPERUSER_PASSWORD:-admin}" + export DJANGO_SUPERUSER_EMAIL="${DJANGO_SUPERUSER_EMAIL:-admin@example.com}" + log "Ensuring development superuser '${DJANGO_SUPERUSER_USERNAME}' exists (DEBUG mode)" + # Run createsuperuser non-interactively, it will fail if user exists but the error is handled + set +e + uv run python manage.py createsuperuser --noinput + csu_exit=$? + set -e + if [ $csu_exit -eq 0 ]; then + log "Superuser created: ${DJANGO_SUPERUSER_USERNAME}/${DJANGO_SUPERUSER_PASSWORD}" else - log "DEBUG not true; skipping automatic superuser creation" + warn "Superuser creation skipped (maybe already exists)" fi + else + log "Skipping auto superuser creation (CREATE_SUPERUSER=${CREATE_SUPERUSER:-0} DEBUG=${DEBUG:-})" + fi # Collect static files log "Collecting static files..." uv run python manage.py collectstatic --noinput || warn "Static files collection skipped" - # Load initial data (if needed) - if [ -f bucr.json ]; then - log "Loading initial data fixture bucr.json" - uv run python manage.py loaddata bucr.json || warn "Initial data load failed" + # Load initial data (if needed) -> This should probably be looked at, there is no data when the container starts + if [ -f gtfs.json ]; then + log "Loading initial data fixture gtfs.json" + uv run python manage.py loaddata gtfs.json || warn "Initial data load failed" else - log "No optional initial data fixture bucr.json present" + log "No optional initial data fixture gtfs.json present" fi log "Django application setup complete!" diff --git a/scripts/dev.sh b/scripts/dev.sh index 1de1431..5d73a6f 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -1,5 +1,6 @@ #!/bin/bash # Development environment startup script for Databus (container) +# Use of emojis is intentional, requested by PO to make the script more engaging set -e # Exit on any error @@ -85,9 +86,7 @@ else fi -echo -e "${YELLOW}⚠️ Applying fake migration for django_celery_beat (only if necessary)...${NC}" -docker compose -f ${COMPOSE_FILE} exec web uv run python manage.py migrate django_celery_beat --fake || \ - echo -e "${YELLOW}⚠️ Fake migration failed or was not necessary.${NC}" +# Celery migrations are applied by the web service; worker/beat will skip. echo "" diff --git a/scripts/prod.sh b/scripts/prod.sh index 8dddbd6..8161753 100755 --- a/scripts/prod.sh +++ b/scripts/prod.sh @@ -51,4 +51,4 @@ echo " Exec shell: docker compose -f ${COMPOSE_FILE} exec app bash" echo " Migrations (re-run): docker compose -f ${COMPOSE_FILE} exec app uv run python manage.py migrate" echo " Create superuser: docker compose -f ${COMPOSE_FILE} exec app uv run python manage.py createsuperuser" echo "" -echo -e "${RED}🛑 To stop: docker compose -f ${COMPOSE_FILE} down${NC}" +echo -e "${BLUE}🛑 To stop: docker compose -f ${COMPOSE_FILE} down${NC}"