deploy-production.sh

#!/usr/bin/env bash
# Idempotent production deploy for single-VM Docker Compose stack.
# Requires: git, docker, flock. Run from repo root or set APP_DIR.
set -euo pipefail

# ---------------------------------------------------------------------------
# Configuration (all overridable via environment)
# ---------------------------------------------------------------------------
TARGET_BRANCH="${TARGET_BRANCH:-main}"
APP_DIR="${APP_DIR:-$HOME/web}"
ENV_FILE="${ENV_FILE:-.env.production}"
HOST_ENV_FILE="${HOST_ENV_FILE:-/var/lib/dabljaar/.env.production}"
BOOTSTRAP_MARKER="${BOOTSTRAP_MARKER:-/var/lib/vm-bootstrap.done}"
REQUIRE_BOOTSTRAP_MARKER="${REQUIRE_BOOTSTRAP_MARKER:-true}"
REPO_URL="${REPO_SSH_URL:-${REPO_FALLBACK:-}}"

case "$APP_DIR" in
  "~"|"~/"*) APP_DIR="${HOME}${APP_DIR#\~}" ;;
esac
export APP_DIR

if ! declare -f ensure_git_repository >/dev/null 2>&1; then
  _deploy_lib=""
  if [ -n "${BASH_SOURCE[0]:-}" ] && [ "${BASH_SOURCE[0]}" != "bash" ]; then
    _deploy_lib="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/git-sync.sh"
  fi
  if [ -z "$_deploy_lib" ] || [ ! -f "$_deploy_lib" ]; then
    _deploy_lib="${APP_DIR}/infra/scripts/lib/git-sync.sh"
  fi
  if [ -f "$_deploy_lib" ]; then
    # shellcheck source=lib/git-sync.sh
    source "$_deploy_lib"
  else
    echo "git-sync helper not loaded: $_deploy_lib" >&2
    exit 1
  fi
fi

normalize_app_dir

# Deploy lock
DEPLOY_LOCK="${DEPLOY_LOCK:-/var/lock/dabljaar-deploy.lock}"
if ! mkdir -p "$(dirname "$DEPLOY_LOCK")" 2>/dev/null || ! touch "$DEPLOY_LOCK" 2>/dev/null; then
  mkdir -p "$APP_DIR"
  DEPLOY_LOCK="$APP_DIR/.deploy.lock"
fi

# ---------------------------------------------------------------------------
# Deploy lock (with staleness guard — clears locks older than 3 hours)
# ---------------------------------------------------------------------------
if [ -f "$DEPLOY_LOCK" ]; then
  lock_age=$(( $(date +%s) - $(stat -c %Y "$DEPLOY_LOCK" 2>/dev/null || echo 0) ))
  if [ "$lock_age" -gt 10800 ]; then
    echo "Stale deploy lock detected (${lock_age}s old, >3h). Clearing."
    rm -f "$DEPLOY_LOCK"
  fi
fi

exec 3>"$DEPLOY_LOCK"
if ! flock -n 3; then
  echo "Another deploy is already running (lock: $DEPLOY_LOCK). Exiting."
  exit 1
fi

# ---------------------------------------------------------------------------
# Bootstrap checks (before deploy.log — repo may not exist yet)
# ---------------------------------------------------------------------------
if [ "${REQUIRE_BOOTSTRAP_MARKER}" = "true" ] && [ ! -f "$BOOTSTRAP_MARKER" ]; then
  echo "Required bootstrap marker is missing: $BOOTSTRAP_MARKER" >&2
  exit 1
fi

if [ "${REQUIRE_BOOTSTRAP_MARKER}" != "true" ] && [ ! -f "$BOOTSTRAP_MARKER" ]; then
  echo "Warning: bootstrap marker not found ($BOOTSTRAP_MARKER). Continuing because REQUIRE_BOOTSTRAP_MARKER=false."
fi

for cmd in git docker; do
  if ! command -v "$cmd" >/dev/null 2>&1; then
    echo "$cmd is not installed on the VM" >&2
    exit 1
  fi
done

install_env_file() {
  local app_env="$APP_DIR/$ENV_FILE"
  if [ -f "$HOST_ENV_FILE" ]; then
    install -m 600 "$HOST_ENV_FILE" "$app_env"
    echo "[$(date -Is)] Installed $app_env from $HOST_ENV_FILE"
  elif [ ! -r "$app_env" ]; then
    echo "Missing $HOST_ENV_FILE and $app_env" >&2
    exit 1
  fi
}

# ---------------------------------------------------------------------------
# git: sync repository to the exact commit SHA, then re-exec from checkout
# ---------------------------------------------------------------------------
if [ "${DEPLOY_SCRIPT_SYNCED:-}" != "1" ]; then
  echo ""
  echo "━━━ [$(date -Is)] PHASE: GIT SYNC ━━━"
  ensure_git_repository
  install_env_file
  echo "[$(date -Is)] Re-execing deploy script after git sync"
  export DEPLOY_SCRIPT_SYNCED=1
  exec bash "$APP_DIR/infra/scripts/deploy-production.sh"
fi

cd "$APP_DIR"

DEPLOY_LOG="$APP_DIR/deploy.log"
mkdir -p "$APP_DIR"

# ---------------------------------------------------------------------------
# Logging — tee all output to deploy.log (after git checkout exists)
# ---------------------------------------------------------------------------
exec > >(tee -a "$DEPLOY_LOG") 2>&1

phase() { echo ""; echo "━━━ [$(date -Is)] PHASE: $* ━━━"; }
log()   { echo "[$(date -Is)] $*"; }

phase "DEPLOY START  sha=${DEPLOY_SHA:-unknown}  reason=${DEPLOY_REASON:-unknown}"

# ---------------------------------------------------------------------------
# Env file validation
# ---------------------------------------------------------------------------
phase "ENV VALIDATION"

if [ ! -r "$ENV_FILE" ]; then
  log "Required env file is missing or unreadable: $APP_DIR/$ENV_FILE"
  exit 1
fi

require_env() {
  local key="$1"
  if ! grep -Eq "^[[:space:]]*${key}=" "$ENV_FILE"; then
    log "Missing required key in $ENV_FILE: $key"
    exit 1
  fi
}

require_env DOMAIN
require_env ACME_EMAIL
require_env SECRET_KEY
require_env POSTGRES_PASSWORD
require_env RABBITMQ_URL
require_env RABBITMQ_DEFAULT_PASS
require_env S3_ENDPOINT_URL
require_env S3_ACCESS_KEY_ID
require_env S3_SECRET_ACCESS_KEY
require_env S3_MEDIA_BUCKET
require_env S3_MODELS_BUCKET

DOMAIN_VALUE="$(grep -E '^[[:space:]]*DOMAIN=' "$ENV_FILE" | tail -n1 | cut -d= -f2- | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e 's/^"//' -e 's/"$//' -e "s/^'//" -e "s/'$//")"
if [ -z "$DOMAIN_VALUE" ]; then
  log "Unable to resolve DOMAIN from $ENV_FILE"
  exit 1
fi
log "Target domain: $DOMAIN_VALUE"

# ---------------------------------------------------------------------------
# Docker Compose selection (shared with ops helper scripts)
# ---------------------------------------------------------------------------
# shellcheck source=lib/compose-env.sh
source "$APP_DIR/infra/scripts/lib/compose-env.sh"

google_auth_enabled() {
  local val
  val="$(env_last_value GOOGLE_AUTH_ENABLED)"
  case "$(echo "$val" | tr '[:upper:]' '[:lower:]')" in
    true|1|yes) return 0 ;;
    *) return 1 ;;
  esac
}

if google_auth_enabled; then
  require_env GOOGLE_CLIENT_ID
  require_env VITE_GOOGLE_CLIENT_ID
  google_client_id="$(env_last_value GOOGLE_CLIENT_ID)"
  vite_google_client_id="$(env_last_value VITE_GOOGLE_CLIENT_ID)"
  if [ -z "$google_client_id" ] || [ -z "$vite_google_client_id" ]; then
    log "GOOGLE_AUTH_ENABLED is true but GOOGLE_CLIENT_ID or VITE_GOOGLE_CLIENT_ID is empty"
    exit 1
  fi
  if [ "$google_client_id" != "$vite_google_client_id" ]; then
    log "GOOGLE_CLIENT_ID and VITE_GOOGLE_CLIENT_ID must match when GOOGLE_AUTH_ENABLED=true"
    exit 1
  fi
  log "Google Sign-In enabled (client IDs match)"
fi

compose_profiles_enabled() {
  grep -Eq '^[[:space:]]*COMPOSE_PROFILES=.*observability' "$APP_DIR/$ENV_FILE" 2>/dev/null
}

if [ "${GPU_OVERLAY_ENABLED:-false}" = "true" ]; then
  log "GPU overlay enabled (COMPOSE_ENABLE_GPU set)"
  phase "GPU PREFLIGHT"
  if ! command -v nvidia-smi >/dev/null 2>&1; then
    log "COMPOSE_ENABLE_GPU=true but nvidia-smi not found on host"
    exit 1
  fi
  nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader
  if ! docker run --rm --gpus all nvidia/cuda:12.3.2-base-ubuntu22.04 nvidia-smi >/dev/null 2>&1; then
    log "Docker GPU passthrough failed — check NVIDIA Container Toolkit"
    exit 1
  fi
  log "GPU preflight passed"

  stt_device="$(env_last_value STT_DEVICE)"
  omnivoice_device="$(env_last_value OMNIVOICE_DEVICE)"
  if [ -z "$stt_device" ]; then
    log "Missing STT_DEVICE in $ENV_FILE (required when COMPOSE_ENABLE_GPU=true)"
    exit 1
  fi
  if [ -z "$omnivoice_device" ]; then
    log "Missing OMNIVOICE_DEVICE in $ENV_FILE (required when COMPOSE_ENABLE_GPU=true)"
    exit 1
  fi
  stt_lower="$(echo "$stt_device" | tr '[:upper:]' '[:lower:]')"
  omnivoice_lower="$(echo "$omnivoice_device" | tr '[:upper:]' '[:lower:]')"
  if [ "$stt_lower" = "cpu" ]; then
    log "Warning: effective STT_DEVICE=cpu while COMPOSE_ENABLE_GPU=true (last matching line in $ENV_FILE wins)"
  fi
  if [ "$omnivoice_lower" = "cpu" ]; then
    log "Warning: effective OMNIVOICE_DEVICE=cpu while COMPOSE_ENABLE_GPU=true (last matching line in $ENV_FILE wins)"
  fi
  stt_count="$(grep -cE '^[[:space:]]*STT_DEVICE=' "$APP_DIR/$ENV_FILE" 2>/dev/null || true)"
  if [ "$stt_count" -gt 1 ]; then
    log "Warning: STT_DEVICE appears ${stt_count} times in $ENV_FILE — effective value is ${stt_device}"
  fi
  omnivoice_count="$(grep -cE '^[[:space:]]*OMNIVOICE_DEVICE=' "$APP_DIR/$ENV_FILE" 2>/dev/null || true)"
  if [ "$omnivoice_count" -gt 1 ]; then
    log "Warning: OMNIVOICE_DEVICE appears ${omnivoice_count} times in $ENV_FILE — effective value is ${omnivoice_device}"
  fi
fi

# ---------------------------------------------------------------------------
# Diagnostics helper (called by on_exit trap on failure)
# ---------------------------------------------------------------------------
print_diagnostics() {
  echo ""
  echo "=== deploy diagnostics: compose ps -a ==="
  $COMPOSE ps -a || true
  echo "=== deploy diagnostics: recent service logs ==="
  $COMPOSE logs --tail=100 \
    backend caddy orchestrator rabbitmq \
    stt-service nmt-service tts-service media-service 2>/dev/null || true
  if compose_profiles_enabled; then
    echo "=== deploy diagnostics: observability service logs ==="
    $COMPOSE logs --tail=50 \
      loki grafana promtail victoriametrics tempo otel-collector 2>/dev/null || true
  fi
  if [ "${GPU_OVERLAY_ENABLED:-false}" = "true" ]; then
    echo "=== deploy diagnostics: nvidia-smi ==="
    nvidia-smi || true
  fi
  echo "=== deploy diagnostics: container health states ==="
  docker ps -aq | xargs -r docker inspect \
    --format '{{.Name}} {{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}} {{.State.Status}}' || true
}

on_exit() {
  local status=$?
  trap - EXIT
  if [ "$status" -ne 0 ]; then
    echo ""
    log "=== DEPLOY FAILED with exit code $status ==="
    print_diagnostics
  else
    log "=== DEPLOY SUCCEEDED ==="
  fi
  exit "$status"
}
trap on_exit EXIT

# ---------------------------------------------------------------------------
# Pre-build disk space guard (Docker builds can fill small VM disks)
# ---------------------------------------------------------------------------
phase "PRE-BUILD CHECKS"

check_disk_space() {
  local min_gb="${1:-5}"
  local available_kb
  available_kb=$(df -k "$APP_DIR" | awk 'NR==2 {print $4}')
  local available_gb=$(( available_kb / 1024 / 1024 ))
  if [ "$available_gb" -lt "$min_gb" ]; then
    log "Insufficient disk space: ${available_gb}GB available, ${min_gb}GB required."
    log "Run: docker system prune -af  to free space, then re-deploy."
    exit 1
  fi
  log "Disk space OK: ${available_gb}GB available on $(df -k "$APP_DIR" | awk 'NR==2 {print $1}')"
}

if [ "${GPU_OVERLAY_ENABLED:-false}" = "true" ] || compose_profiles_enabled; then
  check_disk_space 10
else
  check_disk_space 5
fi

# ---------------------------------------------------------------------------
# Validate Caddyfile before starting any containers
# ---------------------------------------------------------------------------
log "Validating Caddyfile..."
docker run --rm \
  --env-file "$APP_DIR/$ENV_FILE" \
  -v "$APP_DIR/Caddyfile.production:/etc/caddy/Caddyfile:ro" \
  caddy:2.10-alpine \
  caddy validate --config /etc/caddy/Caddyfile
log "Caddyfile is valid."

# ---------------------------------------------------------------------------
# Phase A: Build frontend (atomic swap) — done BEFORE containers start so
# Caddy always mounts a fresh dist/ when it is (re)created.
# ---------------------------------------------------------------------------
phase "FRONTEND BUILD"

fix_frontend_dir_permissions() {
  local dir="$1"
  if [ -d "$dir" ] && [ ! -w "$dir" ]; then
    if command -v sudo >/dev/null 2>&1; then
      sudo chown -R "$(id -u):$(id -g)" "$dir"
    else
      log "$dir is not writable and sudo is unavailable; fix ownership manually."
      exit 1
    fi
  fi
}

build_frontend_atomic() {
  local build_dir="$APP_DIR/frontend/dist.next"
  local dist_dir="$APP_DIR/frontend/dist"

  fix_frontend_dir_permissions "$APP_DIR/frontend/node_modules"
  rm -rf "$APP_DIR/frontend/node_modules"

  fix_frontend_dir_permissions "$build_dir"
  rm -rf "$build_dir"

  local vite_google_client_id
  vite_google_client_id="$(env_last_value VITE_GOOGLE_CLIENT_ID)"

  frontend_build_start="$(date +%s)"
  docker run --rm \
    --user "$(id -u):$(id -g)" \
    -v "$APP_DIR/frontend:/frontend" \
    -w /frontend \
    -e HOME=/tmp \
    -e npm_config_cache=/tmp/.npm \
    -e VITE_API_BASE_URL=/api \
    -e VITE_GOOGLE_CLIENT_ID="${vite_google_client_id}" \
    -e VITE_BUILD_OUTDIR=dist.next \
    node:24-alpine \
    sh -c "npm ci --legacy-peer-deps && npx tsc -b && npx vite build --outDir dist.next"
  frontend_build_end="$(date +%s)"
  frontend_build_seconds="$((frontend_build_end - frontend_build_start))"

  if [ ! -f "$build_dir/index.html" ]; then
    log "Frontend build did not produce frontend/dist.next/index.html"
    exit 1
  fi

  # Atomic swap: dist.next → dist (keep dist.prev as rollback for one cycle)
  if [ -d "$dist_dir" ]; then
    mv "$dist_dir" "$APP_DIR/frontend/dist.prev"
  fi
  mv "$build_dir" "$dist_dir"
  rm -rf "$APP_DIR/frontend/dist.prev"

  if [ ! -f "$dist_dir/index.html" ]; then
    log "Frontend atomic swap failed: $dist_dir/index.html missing"
    exit 1
  fi
  log "Frontend built in ${frontend_build_seconds}s → $dist_dir"
}

build_frontend_atomic

# ---------------------------------------------------------------------------
# Phase B: Infrastructure tier — postgres + rabbitmq
# Start these first and wait for them to be healthy before running migrations.
# ---------------------------------------------------------------------------
phase "INFRA TIER (postgres, rabbitmq)"

infra_up_start="$(date +%s)"
$COMPOSE up -d postgres rabbitmq

log "Waiting for postgres and rabbitmq to become healthy..."
wait_for_infra() {
  local max_attempts=30
  local attempt=1
  while [ "$attempt" -le "$max_attempts" ]; do
    local pg_healthy rmq_healthy
    pg_healthy=$($COMPOSE ps --format json postgres 2>/dev/null | grep -c '"Health":"healthy"' || true)
    rmq_healthy=$($COMPOSE ps --format json rabbitmq 2>/dev/null | grep -c '"Health":"healthy"' || true)

    # Fallback: try docker inspect if compose ps --format json isn't supported
    if [ "$pg_healthy" -eq 0 ]; then
      pg_healthy=$(docker inspect --format '{{.State.Health.Status}}' dabljaar_postgres 2>/dev/null | grep -c 'healthy' || true)
    fi
    if [ "$rmq_healthy" -eq 0 ]; then
      rmq_healthy=$(docker inspect --format '{{.State.Health.Status}}' dabljaar_rabbitmq 2>/dev/null | grep -c 'healthy' || true)
    fi

    if [ "$pg_healthy" -ge 1 ] && [ "$rmq_healthy" -ge 1 ]; then
      log "postgres and rabbitmq are healthy."
      return 0
    fi
    log "  attempt $attempt/$max_attempts: waiting for infra health..."
    sleep 5
    attempt=$((attempt + 1))
  done
  log "Timed out waiting for postgres/rabbitmq to become healthy."
  return 1
}
wait_for_infra
infra_up_end="$(date +%s)"
log "Infra tier up in $((infra_up_end - infra_up_start))s"

# ---------------------------------------------------------------------------
# Phase C: Database migrations
# Run in a one-off backend container with --no-deps (infra already running).
# ---------------------------------------------------------------------------
phase "DATABASE MIGRATIONS"

run_migrations() {
  local migrate_cmd='alembic upgrade head'
  local migrate_name="dabljaar_migrate_$(date +%s)"
  log "Building backend image for migrations at ${DEPLOY_SHA}"
  $COMPOSE build backend
  log "Running migrations via one-off backend container ($migrate_name)"
  if command -v timeout >/dev/null 2>&1; then
    timeout 600 $COMPOSE run --rm --no-deps --name "$migrate_name" \
      --entrypoint sh backend -lc "$migrate_cmd"
  else
    $COMPOSE run --rm --no-deps --name "$migrate_name" \
      --entrypoint sh backend -lc "$migrate_cmd"
  fi
}
run_migrations
log "Migrations complete."

# ---------------------------------------------------------------------------
# Phase D: AI services tier — stt, nmt, tts, media, orchestrator
# Start these in the background while backend builds. They self-warm models.
# ---------------------------------------------------------------------------
phase "AI SERVICES TIER (stt, nmt, tts, media, orchestrator)"

ai_up_start="$(date +%s)"
$COMPOSE up -d --build \
  stt-service nmt-service tts-service media-service orchestrator
log "AI services started (model pre-warming in background)."
ai_up_end="$(date +%s)"
log "AI services tier issued in $((ai_up_end - ai_up_start))s"

# ---------------------------------------------------------------------------
# Phase E: Backend
# ---------------------------------------------------------------------------
phase "BACKEND"

app_up_start="$(date +%s)"
$COMPOSE up -d --build backend
log "Backend container started."

# ---------------------------------------------------------------------------
# Phase E2: Caddy — force-recreate picks up fresh frontend/dist bind mount
# ---------------------------------------------------------------------------
phase "CADDY"

$COMPOSE up -d --force-recreate caddy
$COMPOSE up -d
log "Caddy container force-recreated (picks up fresh frontend/dist and Caddyfile.production)."
app_up_end="$(date +%s)"
log "Backend + Caddy tier up in $((app_up_end - app_up_start))s"

# ---------------------------------------------------------------------------
# Readiness gates — wait for each service to confirm it's accepting requests
# ---------------------------------------------------------------------------
phase "READINESS GATES"

wait_for_readiness() {
  local service="$1"
  local url="$2"
  local label="$3"
  local max_attempts="${4:-60}"
  local sleep_s="${5:-10}"
  local attempt=1
  while [ "$attempt" -le "$max_attempts" ]; do
    if $COMPOSE exec -T "$service" curl -fsS "$url" >/dev/null 2>&1; then
      log "$label readiness check passed (attempt $attempt)"
      return 0
    fi
    log "  $label: attempt $attempt/$max_attempts, retrying in ${sleep_s}s..."
    sleep "$sleep_s"
    attempt=$((attempt + 1))
  done
  log "FAIL: $label ($service) readiness check timed out after $((max_attempts * sleep_s))s"
  return 1
}

wait_for_backend_health() {
  local max_attempts="${1:-40}"
  local attempt=1
  while [ "$attempt" -le "$max_attempts" ]; do
    if $COMPOSE exec -T backend python -c \
        "import urllib.request; urllib.request.urlopen('http://localhost:8000/api/health').read()" \
        >/dev/null 2>&1; then
      log "backend health check passed (attempt $attempt)"
      return 0
    fi
    log "  backend: attempt $attempt/$max_attempts, retrying in 5s..."
    sleep 5
    attempt=$((attempt + 1))
  done
  log "FAIL: backend health check timed out"
  return 1
}

readiness_start="$(date +%s)"

wait_for_backend_health 40
wait_for_readiness orchestrator http://localhost:8081/readiness orchestrator 12 5
wait_for_readiness media-service http://localhost:8003/readiness media 12 10

# AI services: longer wait (model loading). These are non-blocking for
# user traffic — backend + caddy are already serving requests above.
wait_for_readiness stt-service http://localhost:8001/readiness stt 60 10
wait_for_readiness nmt-service http://localhost:8002/readiness nmt 60 10
if [ "${GPU_OVERLAY_ENABLED:-false}" = "true" ]; then
  wait_for_readiness tts-service http://localhost:8005/readiness tts 120 10
else
  wait_for_readiness tts-service http://localhost:8005/readiness tts 90 10
fi

readiness_end="$(date +%s)"
log "All readiness gates passed in $((readiness_end - readiness_start))s"

if [ "${GPU_OVERLAY_ENABLED:-false}" = "true" ]; then
  phase "GPU CONTAINER VALIDATION"
  bash "$APP_DIR/infra/scripts/validate-gpu-deploy.sh" \
    || log "WARN: GPU container validation reported issues (see above)"
fi

# Validate Caddy config inside the live container as a final sanity check.
$COMPOSE exec -T caddy caddy validate --config /etc/caddy/Caddyfile
log "Live Caddy config validated."

# ---------------------------------------------------------------------------
# Phase F: External health checks
# ---------------------------------------------------------------------------
phase "EXTERNAL HEALTH CHECKS"

health_checks_start="$(date +%s)"

# Edge API check — use the backend container directly to avoid the
# HTTP→HTTPS redirect that Caddy applies to plain-HTTP requests.
# (curl -f fails on 3xx, and Caddy redirects port-80 requests to HTTPS.)
edge_api_healthy=false
for _ in 1 2 3 4 5 6 7 8; do
  # Primary: check via backend container (bypasses Caddy redirect)
  if $COMPOSE exec -T backend python -c \
      "import urllib.request; urllib.request.urlopen('http://localhost:8000/api/health').read()" \
      >/dev/null 2>&1; then
    edge_api_healthy=true
    break
  fi
  # Secondary: HTTPS external (follows redirects, verifies TLS)
  if curl -fsS --max-time 10 "https://${DOMAIN_VALUE}/api/health" >/dev/null 2>&1; then
    edge_api_healthy=true
    break
  fi
  sleep 5
done

if [ "$edge_api_healthy" != "true" ]; then
  log "FAIL: API health check failed (both backend-direct and https://${DOMAIN_VALUE}/api/health)"
  exit 1
fi
log "Edge API health check passed."

# External HTTPS API check.
external_api_healthy=false
for _ in 1 2 3 4 5 6 7 8; do
  if curl -fsS --max-time 15 "https://${DOMAIN_VALUE}/api/health" >/dev/null 2>&1; then
    external_api_healthy=true
    break
  fi
  sleep 5
done

if [ "$external_api_healthy" != "true" ]; then
  log "FAIL: public API health check failed on https://${DOMAIN_VALUE}/api/health"
  exit 1
fi
log "External HTTPS API check passed."

# SPA root check — use --insecure only for the initial ACME cert provisioning
# window; normal deploys will have a valid cert.
spa_root_healthy=false
spa_headers_healthy=false
for _ in 1 2 3 4 5 6 7 8; do
  spa_headers="$(curl -fsS --max-time 15 --insecure -I "https://${DOMAIN_VALUE}/" 2>/dev/null || true)"
  if echo "$spa_headers" | grep -Eqi 'strict-transport-security'; then
    spa_headers_healthy=true
  fi
  if curl -fsS --max-time 15 --insecure "https://${DOMAIN_VALUE}/" | grep -Eqi '<!doctype html|<html'; then
    spa_root_healthy=true
  fi
  if [ "$spa_root_healthy" = "true" ] && [ "$spa_headers_healthy" = "true" ]; then
    break
  fi
  sleep 5
done

if [ "$spa_headers_healthy" != "true" ]; then
  log "FAIL: GET / missing Strict-Transport-Security — Caddy {$DOMAIN} site block not routing"
  exit 1
fi

if [ "$spa_root_healthy" != "true" ]; then
  log "FAIL: public SPA root check failed on https://${DOMAIN_VALUE}/"
  exit 1
fi
log "SPA root and security headers check passed."

if compose_profiles_enabled; then
  grafana_external_healthy=false
  for _ in 1 2 3 4 5 6 7 8; do
    if curl -fsS --max-time 15 "https://grafana.${DOMAIN_VALUE}/api/health" >/dev/null 2>&1; then
      grafana_external_healthy=true
      break
    fi
    sleep 5
  done
  if [ "$grafana_external_healthy" != "true" ]; then
    log "Warning: Grafana external health check failed on https://grafana.${DOMAIN_VALUE}/api/health (deploy continues)"
  else
    log "Grafana external health check passed (https://grafana.${DOMAIN_VALUE})."
  fi
fi

health_checks_end="$(date +%s)"
log "All external health checks passed in $((health_checks_end - health_checks_start))s"

# ---------------------------------------------------------------------------
# Post-deploy: prune old Docker images to reclaim disk space
# ---------------------------------------------------------------------------
phase "POST-DEPLOY CLEANUP"

log "Pruning dangling images and stopped containers..."
docker image prune -f >/dev/null 2>&1 || true
docker container prune -f >/dev/null 2>&1 || true
log "Cleanup done."

# ---------------------------------------------------------------------------
# Final summary
# ---------------------------------------------------------------------------
phase "DEPLOY COMPLETE"

DEPLOYED_SHA="$(git rev-parse --short HEAD)"
compose_up_seconds=$(( (app_up_end - infra_up_start) ))
total_seconds=$(( (health_checks_end - frontend_build_start) ))

echo ""
echo "deployment timing:"
echo "  frontend_build   = ${frontend_build_seconds}s"
echo "  infra_tier       = $((infra_up_end - infra_up_start))s"
echo "  ai_tier          = $((ai_up_end - ai_up_start))s (async — still warming)"
echo "  backend_caddy    = $((app_up_end - app_up_start))s"
echo "  readiness_gates  = $((readiness_end - readiness_start))s"
echo "  health_checks    = $((health_checks_end - health_checks_start))s"
echo "  ─────────────────────────────────────"
echo "  total            = ${total_seconds}s"
echo ""
echo "deployed $DEPLOYED_SHA on branch $(git branch --show-current) → https://${DOMAIN_VALUE}/"
if compose_profiles_enabled; then
  echo "observability: https://grafana.${DOMAIN_VALUE}/ (COMPOSE_PROFILES=observability)"
fi
$COMPOSE ps