Deploy to Production #1182
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ############################################################################## | |
| # Pipeline: Deploy to Production (with Preview) | |
| # - Runs on self-hosted runner (local.legal.org.ua) | |
| # - Phase 1: Deploy to inactive color, make preview available | |
| # - Phase 2: After approval, switch production traffic to new color | |
| # - Triggered by successful CI or manually via workflow_dispatch | |
| ############################################################################## | |
| name: 'Deploy to Production' | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| services: | |
| description: 'Services to deploy (comma-separated: backend,rada,openreyestr,frontend,platform,opendata-sync,monitoring). Leave empty to auto-detect from latest changes.' | |
| required: false | |
| default: '' | |
| type: string | |
| workflow_run: | |
| workflows: ["CI/CD: Local Build & Test"] | |
| types: [completed] | |
| branches: [main] | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| actions: read | |
| concurrency: | |
| group: deploy-prod | |
| cancel-in-progress: false | |
| jobs: | |
| # ─── Step 1: Detect what changed ───────────────────────────────────── | |
| detect-changes: | |
| name: Detect Changes | |
| runs-on: [self-hosted, local] | |
| if: | | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') | |
| outputs: | |
| shared: ${{ steps.final.outputs.shared }} | |
| backend: ${{ steps.final.outputs.backend }} | |
| rada: ${{ steps.final.outputs.rada }} | |
| openreyestr: ${{ steps.final.outputs.openreyestr }} | |
| frontend: ${{ steps.final.outputs.frontend }} | |
| deployment: ${{ steps.final.outputs.deployment }} | |
| any_backend: ${{ steps.final.outputs.any_backend }} | |
| services_to_build: ${{ steps.final.outputs.services_to_build }} | |
| services_to_deploy: ${{ steps.final.outputs.services_to_deploy }} | |
| monitoring: ${{ steps.final.outputs.monitoring }} | |
| platform: ${{ steps.final.outputs.platform }} | |
| opendata_sync: ${{ steps.final.outputs.opendata_sync }} | |
| has_changes: ${{ steps.final.outputs.has_changes }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Auto-detect changes (HEAD vs HEAD~1) | |
| id: changes | |
| uses: ./.github/actions/detect-changes | |
| with: | |
| baseline: deploy-prod | |
| - name: Resolve final change flags | |
| id: final | |
| run: | | |
| INPUT_SERVICES="${{ github.event.inputs.services }}" | |
| if [ -n "$INPUT_SERVICES" ]; then | |
| # Manual override: parse comma-separated service names | |
| echo "Using manual service selection: $INPUT_SERVICES" | |
| BACKEND=false RADA=false OPENREYESTR=false FRONTEND=false PLATFORM=false MONITORING=false OPENDATA_SYNC=false | |
| IFS=',' read -ra SVCS <<< "$INPUT_SERVICES" | |
| for svc in "${SVCS[@]}"; do | |
| svc=$(echo "$svc" | xargs) # trim whitespace | |
| case "$svc" in | |
| backend) BACKEND=true ;; | |
| rada) RADA=true ;; | |
| openreyestr) OPENREYESTR=true ;; | |
| frontend) FRONTEND=true ;; | |
| platform) PLATFORM=true ;; | |
| monitoring) MONITORING=true ;; | |
| opendata-sync) OPENDATA_SYNC=true ;; | |
| *) echo "::warning::Unknown service: $svc" ;; | |
| esac | |
| done | |
| echo "backend=$BACKEND" >> "$GITHUB_OUTPUT" | |
| echo "rada=$RADA" >> "$GITHUB_OUTPUT" | |
| echo "openreyestr=$OPENREYESTR" >> "$GITHUB_OUTPUT" | |
| echo "frontend=$FRONTEND" >> "$GITHUB_OUTPUT" | |
| echo "platform=$PLATFORM" >> "$GITHUB_OUTPUT" | |
| echo "opendata_sync=$OPENDATA_SYNC" >> "$GITHUB_OUTPUT" | |
| echo "monitoring=$MONITORING" >> "$GITHUB_OUTPUT" | |
| echo "shared=${{ steps.changes.outputs.shared }}" >> "$GITHUB_OUTPUT" | |
| echo "deployment=${{ steps.changes.outputs.deployment }}" >> "$GITHUB_OUTPUT" | |
| # Compute any_backend | |
| if [ "$BACKEND" = "true" ] || [ "$RADA" = "true" ] || [ "$OPENREYESTR" = "true" ]; then | |
| echo "any_backend=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "any_backend=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| # Build services lists | |
| SERVICES_LIST="" | |
| [ "$BACKEND" = "true" ] && SERVICES_LIST="${SERVICES_LIST:+$SERVICES_LIST,}\"backend\"" | |
| [ "$RADA" = "true" ] && SERVICES_LIST="${SERVICES_LIST:+$SERVICES_LIST,}\"rada\"" | |
| [ "$OPENREYESTR" = "true" ] && SERVICES_LIST="${SERVICES_LIST:+$SERVICES_LIST,}\"openreyestr\"" | |
| [ "$FRONTEND" = "true" ] && SERVICES_LIST="${SERVICES_LIST:+$SERVICES_LIST,}\"frontend\"" | |
| [ "$PLATFORM" = "true" ] && SERVICES_LIST="${SERVICES_LIST:+$SERVICES_LIST,}\"platform\"" | |
| [ "$OPENDATA_SYNC" = "true" ] && SERVICES_LIST="${SERVICES_LIST:+$SERVICES_LIST,}\"opendata-sync\"" | |
| echo "services_to_build=[${SERVICES_LIST}]" >> "$GITHUB_OUTPUT" | |
| echo "services_to_deploy=[${SERVICES_LIST}]" >> "$GITHUB_OUTPUT" | |
| if [ -z "$SERVICES_LIST" ] && [ "$MONITORING" != "true" ]; then | |
| echo "has_changes=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "has_changes=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| else | |
| # Auto-detect from latest changes | |
| echo "Auto-detecting changes from HEAD vs HEAD~1" | |
| echo "backend=${{ steps.changes.outputs.backend }}" >> "$GITHUB_OUTPUT" | |
| echo "rada=${{ steps.changes.outputs.rada }}" >> "$GITHUB_OUTPUT" | |
| echo "openreyestr=${{ steps.changes.outputs.openreyestr }}" >> "$GITHUB_OUTPUT" | |
| echo "frontend=${{ steps.changes.outputs.frontend }}" >> "$GITHUB_OUTPUT" | |
| echo "platform=${{ steps.changes.outputs.platform }}" >> "$GITHUB_OUTPUT" | |
| echo "opendata_sync=${{ steps.changes.outputs.opendata_sync }}" >> "$GITHUB_OUTPUT" | |
| echo "monitoring=${{ steps.changes.outputs.monitoring }}" >> "$GITHUB_OUTPUT" | |
| echo "shared=${{ steps.changes.outputs.shared }}" >> "$GITHUB_OUTPUT" | |
| echo "deployment=${{ steps.changes.outputs.deployment }}" >> "$GITHUB_OUTPUT" | |
| echo "any_backend=${{ steps.changes.outputs.any_backend }}" >> "$GITHUB_OUTPUT" | |
| echo "services_to_build=${{ steps.changes.outputs.services_to_build }}" >> "$GITHUB_OUTPUT" | |
| echo "services_to_deploy=${{ steps.changes.outputs.services_to_deploy }}" >> "$GITHUB_OUTPUT" | |
| SERVICES='${{ steps.changes.outputs.services_to_build }}' | |
| MONITORING='${{ steps.changes.outputs.monitoring }}' | |
| if [ "$SERVICES" = '[]' ] && [ "$MONITORING" != "true" ]; then | |
| echo "has_changes=false" >> "$GITHUB_OUTPUT" | |
| echo "No services changed — nothing to deploy" | |
| else | |
| echo "has_changes=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| fi | |
| # ─── Step 2: Pre-deploy tests ─────────────────────────────────────── | |
| pre-deploy-tests: | |
| name: Pre-Deploy Tests | |
| runs-on: [self-hosted, local] | |
| needs: [detect-changes] | |
| if: needs.detect-changes.outputs.has_changes == 'true' | |
| env: | |
| BACKEND_CHANGED: ${{ needs.detect-changes.outputs.backend }} | |
| FRONTEND_CHANGED: ${{ needs.detect-changes.outputs.frontend }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Ensure native build tools | |
| run: | | |
| if ! command -v make &>/dev/null; then | |
| sudo apt-get update -qq && sudo apt-get install -y -qq build-essential python3 | |
| fi | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| # Self-hosted runner: deployment/docker-compose.local.yml bind-mounts | |
| # packages/shared/dist and mcp_backend/dist into containers. When those host | |
| # dirs are absent, the Docker daemon (root) creates them root-owned, which | |
| # then blocks the next tsc build under the runner user (EACCES). Reset first. | |
| - name: Reset dist ownership (self-hosted runner) | |
| run: sudo chown -R "$(id -u):$(id -g)" packages/shared/dist mcp_backend/dist 2>/dev/null || true | |
| - name: Build shared package | |
| run: cd packages/shared && npm install --legacy-peer-deps && npm run build | |
| - name: Overlay proprietary source | |
| if: needs.detect-changes.outputs.backend == 'true' | |
| run: | | |
| CORE_TMP=$(mktemp -d) | |
| gh repo clone overthelex/secondlayer-core "$CORE_TMP" -- --depth 1 | |
| test -s "$CORE_TMP/src/services/chat-execution-loop.ts" && test -s "$CORE_TMP/src/prompts/chat-system-prompt.ts" \ | |
| || { echo "::error::core clone incomplete — key files missing in $CORE_TMP"; exit 1; } | |
| find "$CORE_TMP"/src/services -maxdepth 1 -name '*.ts' -exec cp -f {} mcp_backend/src/services/ \; | |
| cp -rf "$CORE_TMP"/src/prompts/* mcp_backend/src/prompts/ | |
| rm -rf "$CORE_TMP" | |
| test -s mcp_backend/src/services/chat-execution-loop.ts && test -s mcp_backend/src/prompts/chat-system-prompt.ts \ | |
| || { echo "::error::core overlay failed — key files missing after copy"; exit 1; } | |
| - name: Build & test backend | |
| if: needs.detect-changes.outputs.backend == 'true' | |
| run: | | |
| cd mcp_backend && npm install && npm run build | |
| npx jest --no-cache --forceExit \ | |
| src/controllers/__tests__/ \ | |
| src/middleware/__tests__/ \ | |
| src/adapters/__tests__/ \ | |
| src/services/__tests__/ | |
| - name: Install & test frontend | |
| if: needs.detect-changes.outputs.frontend == 'true' | |
| run: | | |
| cd lexwebapp | |
| npm install --legacy-peer-deps | |
| npm test 2>&1 | tee /tmp/vitest-output.txt || true | |
| if grep -q "Tests.*failed" /tmp/vitest-output.txt; then | |
| echo "::error::Some tests failed" | |
| exit 1 | |
| elif grep -q "Test Files.*passed" /tmp/vitest-output.txt; then | |
| echo "All tests passed" | |
| else | |
| echo "::error::Could not determine test results" | |
| exit 1 | |
| fi | |
| env: | |
| NODE_OPTIONS: --max-old-space-size=8192 | |
| - name: Build frontend | |
| if: needs.detect-changes.outputs.frontend == 'true' | |
| run: cd lexwebapp && npm run build | |
| env: | |
| VITE_API_URL: https://legal.org.ua | |
| NODE_OPTIONS: --max-old-space-size=8192 | |
| # ─── Step 2b: Self-heal test failures (DISABLED) ──────────────────── | |
| self-heal-tests: | |
| name: Self-Heal Test Failures | |
| runs-on: [self-hosted, local] | |
| needs: [detect-changes, pre-deploy-tests] | |
| if: false # temporarily disabled — not producing useful fixes | |
| env: | |
| CLAUDE_CODE_USE_BEDROCK: '1' | |
| CLAUDE_CODE_BEDROCK_MODEL: eu.anthropic.claude-sonnet-4-6 | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: eu-central-1 | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 10 | |
| token: ${{ github.token }} | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Guard against infinite loops | |
| id: guard | |
| run: | | |
| AUTOFIX_COUNT=$(git log -5 --format='%s' | grep -c '\[ci-autofix\]' || true) | |
| if [ "$AUTOFIX_COUNT" -ge 1 ]; then | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| echo "::warning::Found $AUTOFIX_COUNT autofix commit(s) in last 5 — skipping" | |
| else | |
| echo "skip=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Fetch test error logs | |
| if: steps.guard.outputs.skip == 'false' | |
| run: | | |
| gh run view "${{ github.run_id }}" --log-failed 2>&1 | tail -150 > /tmp/ci-errors.log | |
| echo "Captured $(wc -l < /tmp/ci-errors.log) lines of error logs" | |
| - name: Claude Code — diagnose and fix test failures | |
| if: steps.guard.outputs.skip == 'false' | |
| id: claude-fix | |
| continue-on-error: true | |
| run: | | |
| BRANCH_NAME="ci-autofix/tests-$(date +%Y%m%d-%H%M%S)-${GITHUB_SHA:0:7}" | |
| git checkout -b "$BRANCH_NAME" | |
| cat > /tmp/ci-prompt.txt <<'PROMPT_END' | |
| Pre-deploy tests failed. The error logs are in /tmp/ci-errors.log — read that file first. | |
| Your task: | |
| 1. Read /tmp/ci-errors.log to understand the failure | |
| 2. Determine if this is a build error or a test failure | |
| 3. Read the failing test file AND the source file it tests | |
| 4. Determine if the test is wrong (outdated assertions) or the source code has a bug | |
| 5. Fix whichever is actually incorrect | |
| 6. Run the specific test to verify: npx jest --no-cache <test-file> --forceExit | |
| 7. Stage ONLY the files you changed and commit with message: "fix: [ci-autofix] <description>" | |
| 8. Do NOT push — the pipeline will handle pushing and PR creation | |
| Rules: | |
| - Do NOT blindly change test expectations — understand WHY the test fails first | |
| - Do NOT modify CI workflow files | |
| - Do NOT install new dependencies unless absolutely necessary | |
| - Do NOT run git push | |
| - If you cannot fix the issue, create a GitHub issue describing the problem | |
| PROMPT_END | |
| npx -y @anthropic-ai/claude-code@latest \ | |
| -p "$(cat /tmp/ci-prompt.txt)" \ | |
| --allowedTools "Bash,Read,Edit,Write,Glob,Grep" \ | |
| --max-turns 25 | |
| timeout-minutes: 10 | |
| - name: Check if fix was produced | |
| if: steps.guard.outputs.skip == 'false' | |
| id: check-fix | |
| run: | | |
| CURRENT_BRANCH=$(git branch --show-current) | |
| if git log --oneline "main..$CURRENT_BRANCH" 2>/dev/null | grep -q .; then | |
| echo "has_fix=true" >> "$GITHUB_OUTPUT" | |
| echo "branch=$CURRENT_BRANCH" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "has_fix=false" >> "$GITHUB_OUTPUT" | |
| echo "::warning::Claude Code did not produce a fix" | |
| fi | |
| - name: Create issue if no fix produced | |
| if: steps.guard.outputs.skip == 'false' && steps.check-fix.outputs.has_fix == 'false' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| gh issue create \ | |
| --title "Pre-deploy test failure needs manual fix ($(date +%Y-%m-%d))" \ | |
| --body "$(cat <<EOF | |
| ## Pre-deploy test failure — auto-fix failed | |
| The self-heal agent could not produce a code fix for this test failure. | |
| **Failed run:** $RUN_URL | |
| **Commit:** ${GITHUB_SHA:0:7} | |
| Please investigate manually. | |
| EOF | |
| )" \ | |
| --label "bug,ci" | |
| - name: Push branch and create PR | |
| if: steps.guard.outputs.skip == 'false' && steps.check-fix.outputs.has_fix == 'true' | |
| env: | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| BRANCH="${{ steps.check-fix.outputs.branch }}" | |
| git push origin "$BRANCH" | |
| gh pr create \ | |
| --title "fix: [ci-autofix] Auto-fix for pre-deploy test failure" \ | |
| --body "$(cat <<EOF | |
| ## Auto-generated fix for test failure | |
| Claude Code analyzed the pre-deploy test failure and produced this fix automatically. | |
| **Source run:** $RUN_URL | |
| > Review carefully before merging — this is an automated fix. | |
| 🤖 Generated with [Claude Code](https://claude.com/claude-code) | |
| EOF | |
| )" \ | |
| --base main \ | |
| --head "$BRANCH" | |
| # ─── Step 3a: Deploy Preview (inactive color) ────────────────────── | |
| deploy-preview: | |
| name: Deploy Preview | |
| # Must run on the prod-reachable runner: this job SSHes to PROD_SERVER_IP. | |
| # The `local` label also matches local-runner (cthulhu), which cannot reach | |
| # prod's public IP → intermittent `ssh: connect … timed out`. prod-deploy is | |
| # a label unique to prod-runner. | |
| runs-on: [self-hosted, prod-deploy] | |
| needs: [detect-changes, pre-deploy-tests] | |
| if: | | |
| needs.detect-changes.outputs.has_changes == 'true' && | |
| needs.pre-deploy-tests.result == 'success' | |
| outputs: | |
| backend_version: ${{ steps.version.outputs.backend_version }} | |
| frontend_version: ${{ steps.version.outputs.frontend_version }} | |
| env: | |
| PROD_SERVER: ${{ secrets.PROD_SERVER_IP }} | |
| PROD_USER: ${{ secrets.PROD_USER || 'ubuntu' }} | |
| BACKEND_CHANGED: ${{ needs.detect-changes.outputs.backend }} | |
| RADA_CHANGED: ${{ needs.detect-changes.outputs.rada }} | |
| OPENREYESTR_CHANGED: ${{ needs.detect-changes.outputs.openreyestr }} | |
| FRONTEND_CHANGED: ${{ needs.detect-changes.outputs.frontend }} | |
| PLATFORM_CHANGED: ${{ needs.detect-changes.outputs.platform }} | |
| OPENDATA_SYNC_CHANGED: ${{ needs.detect-changes.outputs.opendata_sync }} | |
| MONITORING_CHANGED: ${{ needs.detect-changes.outputs.monitoring }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Compute next versions (backend + frontend) | |
| id: version | |
| run: | | |
| # --- Backend version (v*) --- | |
| LAST_BE_TAG=$(git tag -l "v*" --sort=-version:refname | head -1) | |
| if [ "$BACKEND_CHANGED" = "true" ] || [ "$RADA_CHANGED" = "true" ] || [ "$OPENREYESTR_CHANGED" = "true" ]; then | |
| if [ -z "$LAST_BE_TAG" ]; then | |
| BE_VERSION="v1.0.0" | |
| else | |
| CURRENT="${LAST_BE_TAG#v}" | |
| MAJOR=$(echo "$CURRENT" | cut -d. -f1) | |
| MINOR=$(echo "$CURRENT" | cut -d. -f2) | |
| PATCH=$(echo "$CURRENT" | cut -d. -f3) | |
| COMMITS=$(git log "${LAST_BE_TAG}..HEAD" --oneline --no-merges -- packages/shared/ mcp_backend/ mcp_rada/ mcp_openreyestr/) | |
| if echo "$COMMITS" | grep -qiE "^[a-f0-9]+ (feat|refactor)!:|BREAKING CHANGE"; then | |
| MAJOR=$((MAJOR + 1)); MINOR=0; PATCH=0 | |
| elif echo "$COMMITS" | grep -qiE "^[a-f0-9]+ feat(\(|:)"; then | |
| MINOR=$((MINOR + 1)); PATCH=0 | |
| else | |
| PATCH=$((PATCH + 1)) | |
| fi | |
| BE_VERSION="v${MAJOR}.${MINOR}.${PATCH}" | |
| fi | |
| else | |
| BE_VERSION="${LAST_BE_TAG:-v1.0.0}" | |
| fi | |
| echo "backend_version=${BE_VERSION}" >> "$GITHUB_OUTPUT" | |
| echo "Backend version: ${BE_VERSION}" | |
| # --- Frontend version (fe-v*) --- | |
| LAST_FE_TAG=$(git tag -l "fe-v*" --sort=-version:refname | head -1) | |
| if [ "$FRONTEND_CHANGED" = "true" ]; then | |
| if [ -z "$LAST_FE_TAG" ]; then | |
| FE_VERSION="fe-v1.0.0" | |
| else | |
| CURRENT="${LAST_FE_TAG#fe-v}" | |
| MAJOR=$(echo "$CURRENT" | cut -d. -f1) | |
| MINOR=$(echo "$CURRENT" | cut -d. -f2) | |
| PATCH=$(echo "$CURRENT" | cut -d. -f3) | |
| COMMITS=$(git log "${LAST_FE_TAG}..HEAD" --oneline --no-merges -- lexwebapp/) | |
| if echo "$COMMITS" | grep -qiE "^[a-f0-9]+ (feat|refactor)!:|BREAKING CHANGE"; then | |
| MAJOR=$((MAJOR + 1)); MINOR=0; PATCH=0 | |
| elif echo "$COMMITS" | grep -qiE "^[a-f0-9]+ feat(\(|:)"; then | |
| MINOR=$((MINOR + 1)); PATCH=0 | |
| else | |
| PATCH=$((PATCH + 1)) | |
| fi | |
| FE_VERSION="fe-v${MAJOR}.${MINOR}.${PATCH}" | |
| fi | |
| else | |
| FE_VERSION="${LAST_FE_TAG:-fe-v0.0.0}" | |
| fi | |
| echo "frontend_version=${FE_VERSION}" >> "$GITHUB_OUTPUT" | |
| echo "Frontend version: ${FE_VERSION}" | |
| - name: "Phase 1: Build, migrate, start inactive color, enable preview" | |
| env: | |
| PROD_SSH_KEY_PATH: ${{ secrets.PROD_SSH_KEY_PATH }} | |
| BACKEND_VERSION: ${{ steps.version.outputs.backend_version }} | |
| FRONTEND_VERSION: ${{ steps.version.outputs.frontend_version }} | |
| run: | | |
| if [ -z "$PROD_SSH_KEY_PATH" ]; then | |
| echo "::error::PROD_SSH_KEY_PATH secret is not set" | |
| exit 1 | |
| fi | |
| if [ -z "$PROD_SERVER" ]; then | |
| echo "::error::PROD_SERVER_IP secret is not set" | |
| exit 1 | |
| fi | |
| SSH_CMD="ssh -i $PROD_SSH_KEY_PATH -o StrictHostKeyChecking=no ${PROD_USER}@${PROD_SERVER}" | |
| REMOTE_REPO="/home/${PROD_USER}/SecondLayer" | |
| echo "=== Phase 1: Deploy preview (inactive color) ===" | |
| # Pull latest code and tags | |
| $SSH_CMD "git -C ${REMOTE_REPO} fetch origin main --tags && git -C ${REMOTE_REPO} reset --hard origin/main" | |
| # Materialize the (gitignored) Prometheus bearer-token file for the | |
| # bigbox EDRSR Qdrant scrape from .env.prod, so prometheus-prod always | |
| # starts cleanly — a missing credentials_file aborts its startup. | |
| $SSH_CMD "chmod +x ${REMOTE_REPO}/deployment/prometheus/ensure-qdrant-edrsr-key.sh && ${REMOTE_REPO}/deployment/prometheus/ensure-qdrant-edrsr-key.sh" | |
| # Write VERSION file with both versions | |
| $SSH_CMD "printf 'BACKEND_VERSION=%s\nFRONTEND_VERSION=%s\n' '${BACKEND_VERSION}' '${FRONTEND_VERSION}' > ${REMOTE_REPO}/VERSION" | |
| # Build dist on prod | |
| $SSH_CMD "cd ${REMOTE_REPO} && npm --prefix packages/shared install && npm --prefix packages/shared run build" | |
| if [ "$BACKEND_CHANGED" = "true" ]; then | |
| # Overlay proprietary source from private repo before building | |
| CORE_TMP=$(mktemp -d) | |
| gh repo clone overthelex/secondlayer-core "$CORE_TMP" -- --depth 1 | |
| test -s "$CORE_TMP/src/services/chat-execution-loop.ts" && test -s "$CORE_TMP/src/prompts/chat-system-prompt.ts" \ | |
| || { echo "::error::core clone incomplete — key files missing in $CORE_TMP"; exit 1; } | |
| SCP_CMD="scp -i $PROD_SSH_KEY_PATH -o StrictHostKeyChecking=no" | |
| $SCP_CMD -r "$CORE_TMP"/src/services/* "${PROD_USER}@${PROD_SERVER}:${REMOTE_REPO}/mcp_backend/src/services/" | |
| $SCP_CMD -r "$CORE_TMP"/src/prompts/* "${PROD_USER}@${PROD_SERVER}:${REMOTE_REPO}/mcp_backend/src/prompts/" | |
| rm -rf "$CORE_TMP" | |
| $SSH_CMD "test -s ${REMOTE_REPO}/mcp_backend/src/services/chat-execution-loop.ts && test -s ${REMOTE_REPO}/mcp_backend/src/prompts/chat-system-prompt.ts" \ | |
| || { echo "::error::core overlay missing on prod after scp"; exit 1; } | |
| $SSH_CMD "cd ${REMOTE_REPO} && npm --prefix mcp_backend install && npm --prefix mcp_backend run build" | |
| fi | |
| if [ "$RADA_CHANGED" = "true" ]; then | |
| $SSH_CMD "cd ${REMOTE_REPO} && npm --prefix mcp_rada install && npm --prefix mcp_rada run build" | |
| fi | |
| if [ "$OPENREYESTR_CHANGED" = "true" ]; then | |
| $SSH_CMD "cd ${REMOTE_REPO} && npm --prefix mcp_openreyestr install && npm --prefix mcp_openreyestr run build" | |
| fi | |
| # Phase 1: Build images, run migrations, start inactive color, write preview upstreams | |
| $SSH_CMD bash << DEPLOY_SCRIPT | |
| set -e | |
| cd ${REMOTE_REPO}/deployment | |
| export APP_VERSION=\$(grep 'BACKEND_VERSION=' ${REMOTE_REPO}/VERSION 2>/dev/null | cut -d= -f2 || echo 'dev') | |
| export APP_FRONTEND_VERSION=\$(grep 'FRONTEND_VERSION=' ${REMOTE_REPO}/VERSION 2>/dev/null | cut -d= -f2 || echo 'dev') | |
| DC="docker compose -f docker-compose.prod.yml --env-file .env.prod" | |
| # --- Determine active colors (per service group) --- | |
| BACKEND_COLOR=\$(grep '^backend=' .active-colors 2>/dev/null | cut -d= -f2 || echo "blue") | |
| FRONTEND_COLOR=\$(grep '^frontend=' .active-colors 2>/dev/null | cut -d= -f2 || echo "blue") | |
| [ "\$BACKEND_COLOR" = "green" ] && NEW_BACKEND="blue" || NEW_BACKEND="green" | |
| [ "\$FRONTEND_COLOR" = "green" ] && NEW_FRONTEND="blue" || NEW_FRONTEND="green" | |
| ANY_BACKEND=false | |
| if [ "${BACKEND_CHANGED}" = "true" ] || [ "${RADA_CHANGED}" = "true" ] || [ "${OPENREYESTR_CHANGED}" = "true" ]; then | |
| ANY_BACKEND=true | |
| fi | |
| echo "Backend: \$BACKEND_COLOR → \$NEW_BACKEND (changed: \$ANY_BACKEND)" | |
| echo "Frontend: \$FRONTEND_COLOR → \$NEW_FRONTEND (changed: ${FRONTEND_CHANGED})" | |
| # --- 1. Build images (old containers still serving traffic) --- | |
| BUILD="" | |
| [ "${BACKEND_CHANGED}" = "true" ] && BUILD="\$BUILD app-prod document-service-prod migrate-prod" | |
| [ "${RADA_CHANGED}" = "true" ] && BUILD="\$BUILD rada-mcp-app-prod rada-db-init-prod rada-migrate-prod" | |
| [ "${OPENREYESTR_CHANGED}" = "true" ] && BUILD="\$BUILD app-openreyestr-prod migrate-openreyestr-prod" | |
| [ "${FRONTEND_CHANGED}" = "true" ] && BUILD="\$BUILD lexwebapp-prod" | |
| [ "${PLATFORM_CHANGED}" = "true" ] && BUILD="\$BUILD platform-prod" | |
| [ "${OPENDATA_SYNC_CHANGED}" = "true" ] && BUILD="\$BUILD opendata-sync-prod" | |
| [ -n "\$BUILD" ] && \$DC build \$BUILD | |
| # --- 2. Run migrations (idempotent, safe while old containers serve) --- | |
| [ "${BACKEND_CHANGED}" = "true" ] && { \$DC up migrate-prod || true; } | |
| [ "${RADA_CHANGED}" = "true" ] && { \$DC up rada-db-init-prod || true; \$DC up rada-migrate-prod || true; } | |
| [ "${OPENREYESTR_CHANGED}" = "true" ] && { \$DC up migrate-openreyestr-prod || true; } | |
| # --- 3. Start new-color containers alongside old ones --- | |
| # Ensure infra services are running WITHOUT recreating them. | |
| # qdrant-prod was removed from the prod compose (LEXAI-1807 Phase B, | |
| # PR #2089) — vectors are served from bigbox via QDRANT_URL. | |
| \$DC up -d --no-recreate redis-prod pgbouncer-prod | |
| if [ "\$ANY_BACKEND" = "true" ]; then | |
| if [ "\$NEW_BACKEND" = "green" ]; then | |
| \$DC --profile green up -d --no-deps app-prod-green document-service-prod-green rada-mcp-app-prod-green app-openreyestr-prod-green | |
| else | |
| \$DC up -d --no-deps app-prod document-service-prod rada-mcp-app-prod app-openreyestr-prod | |
| fi | |
| fi | |
| if [ "${FRONTEND_CHANGED}" = "true" ]; then | |
| if [ "\$NEW_FRONTEND" = "green" ]; then | |
| \$DC --profile green up -d lexwebapp-prod-green | |
| else | |
| \$DC --profile blue up -d lexwebapp-prod-blue | |
| fi | |
| fi | |
| if [ "${PLATFORM_CHANGED}" = "true" ]; then | |
| \$DC up -d platform-prod | |
| fi | |
| if [ "${OPENDATA_SYNC_CHANGED}" = "true" ]; then | |
| \$DC up -d opendata-sync-prod | |
| fi | |
| # --- 4. Wait for new containers to be healthy --- | |
| wait_healthy() { | |
| local container=\$1 max_wait=\${2:-120} elapsed=0 | |
| echo "Waiting for \$container to be healthy..." | |
| while [ \$elapsed -lt \$max_wait ]; do | |
| local status | |
| status=\$(docker inspect --format='{{.State.Health.Status}}' "\$container" 2>/dev/null || echo "not_found") | |
| case "\$status" in | |
| healthy) echo "✓ \$container: docker healthy (\${elapsed}s)"; break ;; | |
| unhealthy) | |
| echo "✗ \$container: unhealthy after \${elapsed}s" | |
| docker logs --tail=20 "\$container" 2>&1 | |
| return 1 ;; | |
| *) sleep 5; elapsed=\$((elapsed + 5)) ;; | |
| esac | |
| done | |
| if [ \$elapsed -ge \$max_wait ]; then | |
| echo "✗ \$container: timed out (\${max_wait}s)" | |
| docker logs --tail=20 "\$container" 2>&1 | |
| return 1 | |
| fi | |
| # Application-level HTTP readiness check | |
| local port="" | |
| case "\$container" in | |
| *rada*) port=3001 ;; | |
| *openreyestr*) port=3005 ;; | |
| *document-service*) port=3002 ;; | |
| lexwebapp*|platform*) port=80 ;; | |
| secondlayer-app*|*app-prod*) port=3000 ;; | |
| *) echo "✓ \$container: no HTTP check needed"; return 0 ;; | |
| esac | |
| echo "Verifying HTTP readiness on port \$port..." | |
| local http_wait=0 | |
| while [ \$http_wait -lt 60 ]; do | |
| if docker exec "\$container" wget -q -O /dev/null --timeout=5 "http://127.0.0.1:\${port}/health" 2>/dev/null; then | |
| echo "✓ \$container: HTTP ready on :\$port (\${http_wait}s after docker healthy)" | |
| return 0 | |
| fi | |
| sleep 3; http_wait=\$((http_wait + 3)) | |
| done | |
| echo "✗ \$container: docker healthy but HTTP not responding on :\$port after 60s" | |
| docker logs --tail=30 "\$container" 2>&1 | |
| return 1 | |
| } | |
| HEALTH_OK=true | |
| if [ "\$ANY_BACKEND" = "true" ]; then | |
| if [ "\$NEW_BACKEND" = "green" ]; then | |
| wait_healthy "secondlayer-app-prod-green" 180 || HEALTH_OK=false | |
| wait_healthy "document-service-prod-green" 120 || HEALTH_OK=false | |
| wait_healthy "rada-mcp-app-prod-green" 120 || HEALTH_OK=false | |
| wait_healthy "openreyestr-app-prod-green" 120 || HEALTH_OK=false | |
| else | |
| wait_healthy "secondlayer-app-prod" 180 || HEALTH_OK=false | |
| wait_healthy "document-service-prod" 120 || HEALTH_OK=false | |
| wait_healthy "rada-mcp-app-prod" 120 || HEALTH_OK=false | |
| wait_healthy "openreyestr-app-prod" 120 || HEALTH_OK=false | |
| fi | |
| fi | |
| if [ "${FRONTEND_CHANGED}" = "true" ]; then | |
| if [ "\$NEW_FRONTEND" = "green" ]; then | |
| wait_healthy "lexwebapp-prod-green" 60 || HEALTH_OK=false | |
| else | |
| wait_healthy "lexwebapp-prod-blue" 60 || HEALTH_OK=false | |
| fi | |
| fi | |
| if [ "${PLATFORM_CHANGED}" = "true" ]; then | |
| wait_healthy "platform-prod" 60 || HEALTH_OK=false | |
| fi | |
| if [ "\$HEALTH_OK" != "true" ]; then | |
| echo "::error::New containers failed health check — aborting (old containers still serving)" | |
| # Stop failed new containers | |
| if [ "\$ANY_BACKEND" = "true" ]; then | |
| if [ "\$NEW_BACKEND" = "green" ]; then | |
| \$DC --profile green stop app-prod-green document-service-prod-green rada-mcp-app-prod-green app-openreyestr-prod-green 2>/dev/null || true | |
| \$DC --profile green rm -f app-prod-green document-service-prod-green rada-mcp-app-prod-green app-openreyestr-prod-green 2>/dev/null || true | |
| else | |
| \$DC stop app-prod document-service-prod rada-mcp-app-prod app-openreyestr-prod 2>/dev/null || true | |
| \$DC rm -f app-prod document-service-prod rada-mcp-app-prod app-openreyestr-prod 2>/dev/null || true | |
| fi | |
| fi | |
| if [ "${FRONTEND_CHANGED}" = "true" ]; then | |
| if [ "\$NEW_FRONTEND" = "green" ]; then | |
| \$DC --profile green stop lexwebapp-prod-green 2>/dev/null || true | |
| \$DC --profile green rm -f lexwebapp-prod-green 2>/dev/null || true | |
| else | |
| \$DC --profile blue stop lexwebapp-prod-blue 2>/dev/null || true | |
| \$DC --profile blue rm -f lexwebapp-prod-blue 2>/dev/null || true | |
| fi | |
| fi | |
| exit 1 | |
| fi | |
| # --- 5. Write preview upstreams pointing to new (inactive) color --- | |
| if [ "\$ANY_BACKEND" = "true" ]; then | |
| [ "\$NEW_BACKEND" = "green" ] && PREVIEW_BE_HOST="secondlayer-app-prod-green" || PREVIEW_BE_HOST="secondlayer-app-prod" | |
| else | |
| # No backend change — preview backend points to current prod backend | |
| [ "\$BACKEND_COLOR" = "green" ] && PREVIEW_BE_HOST="secondlayer-app-prod-green" || PREVIEW_BE_HOST="secondlayer-app-prod" | |
| fi | |
| if [ "${FRONTEND_CHANGED}" = "true" ]; then | |
| [ "\$NEW_FRONTEND" = "green" ] && PREVIEW_FE_HOST="lexwebapp-prod-green" || PREVIEW_FE_HOST="lexwebapp-prod-blue" | |
| else | |
| # No frontend change — preview frontend points to current prod frontend | |
| [ "\$FRONTEND_COLOR" = "green" ] && PREVIEW_FE_HOST="lexwebapp-prod-green" || PREVIEW_FE_HOST="lexwebapp-prod-blue" | |
| fi | |
| printf '# Preview upstreams — managed by deploy script\n# Points to inactive color containers for pre-production preview\n# DO NOT EDIT MANUALLY\n\nupstream preview_mcp_backend {\n server %s:3000;\n keepalive 128;\n}\n\nupstream preview_frontend {\n server %s:80;\n keepalive 32;\n}\n' "\$PREVIEW_BE_HOST" "\$PREVIEW_FE_HOST" > nginx/includes/preview-upstreams.conf | |
| # Recreate nginx to pick up preview upstreams (bind mount inode staleness) | |
| \$DC up -d nginx-prod --force-recreate | |
| echo "Preview enabled at preview.legal.org.ua" | |
| DEPLOY_SCRIPT | |
| - name: Verify preview health | |
| env: | |
| PROD_SSH_KEY_PATH: ${{ secrets.PROD_SSH_KEY_PATH }} | |
| run: | | |
| SSH_CMD="ssh -i $PROD_SSH_KEY_PATH -o StrictHostKeyChecking=no -o ConnectTimeout=10 ${PROD_USER}@${PROD_SERVER}" | |
| # Check preview backend health via nginx | |
| for i in 1 2 3 4 5; do | |
| HEALTH=$($SSH_CMD "docker exec nginx-prod wget -q -O- --timeout=10 'http://127.0.0.1:80' --header='Host: preview.legal.org.ua' 2>/dev/null | head -c 500") && { | |
| echo "✓ Preview frontend is responding" | |
| break | |
| } | |
| [ $i -eq 5 ] && echo "::warning::Preview frontend health check failed (non-blocking)" | |
| sleep 5 | |
| done | |
| echo "" | |
| echo "==========================================" | |
| echo " Preview is ready at:" | |
| echo " https://preview.legal.org.ua" | |
| echo "" | |
| echo " Waiting for approval to promote to prod" | |
| echo "==========================================" | |
| # ─── Step 3b: Promote to Production (requires approval) ─────────── | |
| promote-to-prod: | |
| name: Promote to Production | |
| # SSHes to PROD_SERVER_IP — pin to prod-runner (see Deploy Preview note). | |
| runs-on: [self-hosted, prod-deploy] | |
| needs: [detect-changes, deploy-preview] | |
| if: needs.deploy-preview.result == 'success' | |
| environment: production | |
| outputs: | |
| backend_version: ${{ needs.deploy-preview.outputs.backend_version }} | |
| frontend_version: ${{ needs.deploy-preview.outputs.frontend_version }} | |
| env: | |
| PROD_SERVER: ${{ secrets.PROD_SERVER_IP }} | |
| PROD_USER: ${{ secrets.PROD_USER || 'ubuntu' }} | |
| BACKEND_CHANGED: ${{ needs.detect-changes.outputs.backend }} | |
| RADA_CHANGED: ${{ needs.detect-changes.outputs.rada }} | |
| OPENREYESTR_CHANGED: ${{ needs.detect-changes.outputs.openreyestr }} | |
| FRONTEND_CHANGED: ${{ needs.detect-changes.outputs.frontend }} | |
| PLATFORM_CHANGED: ${{ needs.detect-changes.outputs.platform }} | |
| OPENDATA_SYNC_CHANGED: ${{ needs.detect-changes.outputs.opendata_sync }} | |
| MONITORING_CHANGED: ${{ needs.detect-changes.outputs.monitoring }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: "Phase 2: Switch production traffic to new color" | |
| env: | |
| PROD_SSH_KEY_PATH: ${{ secrets.PROD_SSH_KEY_PATH }} | |
| run: | | |
| SSH_CMD="ssh -i $PROD_SSH_KEY_PATH -o StrictHostKeyChecking=no ${PROD_USER}@${PROD_SERVER}" | |
| REMOTE_REPO="/home/${PROD_USER}/SecondLayer" | |
| echo "=== Phase 2: Promote preview to production ===" | |
| $SSH_CMD bash << PROMOTE_SCRIPT | |
| set -e | |
| cd ${REMOTE_REPO}/deployment | |
| DC="docker compose -f docker-compose.prod.yml --env-file .env.prod" | |
| # --- Determine colors (with reality check) --- | |
| BACKEND_COLOR=\$(grep '^backend=' .active-colors 2>/dev/null | cut -d= -f2 || echo "blue") | |
| FRONTEND_COLOR=\$(grep '^frontend=' .active-colors 2>/dev/null | cut -d= -f2 || echo "blue") | |
| # Safety: verify .active-colors matches actually running containers | |
| # If mismatch, trust docker over the file (handles partial deploy failures) | |
| if docker inspect secondlayer-app-prod-green --format '{{.State.Running}}' 2>/dev/null | grep -q true; then | |
| if ! docker inspect secondlayer-app-prod --format '{{.State.Running}}' 2>/dev/null | grep -q true; then | |
| BACKEND_COLOR="green" | |
| echo "WARNING: .active-colors disagrees with docker state, correcting backend=green" | |
| fi | |
| elif docker inspect secondlayer-app-prod --format '{{.State.Running}}' 2>/dev/null | grep -q true; then | |
| BACKEND_COLOR="blue" | |
| echo "WARNING: .active-colors disagrees with docker state, correcting backend=blue" | |
| fi | |
| if docker inspect lexwebapp-prod-green --format '{{.State.Running}}' 2>/dev/null | grep -q true; then | |
| if ! docker inspect lexwebapp-prod-blue --format '{{.State.Running}}' 2>/dev/null | grep -q true; then | |
| FRONTEND_COLOR="green" | |
| echo "WARNING: .active-colors disagrees with docker state, correcting frontend=green" | |
| fi | |
| elif docker inspect lexwebapp-prod-blue --format '{{.State.Running}}' 2>/dev/null | grep -q true; then | |
| FRONTEND_COLOR="blue" | |
| echo "WARNING: .active-colors disagrees with docker state, correcting frontend=blue" | |
| fi | |
| # Save corrected colors before proceeding | |
| printf 'backend=%s\nfrontend=%s\n' "\$BACKEND_COLOR" "\$FRONTEND_COLOR" > .active-colors | |
| [ "\$BACKEND_COLOR" = "green" ] && NEW_BACKEND="blue" || NEW_BACKEND="green" | |
| [ "\$FRONTEND_COLOR" = "green" ] && NEW_FRONTEND="blue" || NEW_FRONTEND="green" | |
| ANY_BACKEND=false | |
| if [ "${BACKEND_CHANGED}" = "true" ] || [ "${RADA_CHANGED}" = "true" ] || [ "${OPENREYESTR_CHANGED}" = "true" ]; then | |
| ANY_BACKEND=true | |
| fi | |
| echo "Promoting: Backend \$BACKEND_COLOR → \$NEW_BACKEND, Frontend \$FRONTEND_COLOR → \$NEW_FRONTEND" | |
| # --- 0. Verify new containers are still healthy before switching traffic --- | |
| if [ "\$ANY_BACKEND" = "true" ]; then | |
| if [ "\$NEW_BACKEND" = "green" ]; then | |
| BE_CONTAINER="secondlayer-app-prod-green" | |
| else | |
| BE_CONTAINER="secondlayer-app-prod" | |
| fi | |
| echo "Pre-switch health check: \$BE_CONTAINER" | |
| for i in 1 2 3 4 5 6; do | |
| if docker exec "\$BE_CONTAINER" wget -q -O /dev/null --timeout=5 "http://127.0.0.1:3000/health" 2>/dev/null; then | |
| echo "✓ \$BE_CONTAINER: HTTP ready" | |
| break | |
| fi | |
| if [ \$i -eq 6 ]; then | |
| echo "::error::\$BE_CONTAINER is not responding on :3000 — aborting promotion" | |
| exit 1 | |
| fi | |
| echo " Waiting... (\$i/6)" | |
| sleep 10 | |
| done | |
| fi | |
| # --- 1. Switch prod upstreams to new color --- | |
| if [ "\$ANY_BACKEND" = "true" ]; then | |
| [ "\$NEW_BACKEND" = "green" ] && BE_HOST="secondlayer-app-prod-green" || BE_HOST="secondlayer-app-prod" | |
| else | |
| [ "\$BACKEND_COLOR" = "green" ] && BE_HOST="secondlayer-app-prod-green" || BE_HOST="secondlayer-app-prod" | |
| fi | |
| if [ "${FRONTEND_CHANGED}" = "true" ]; then | |
| [ "\$NEW_FRONTEND" = "green" ] && FE_HOST="lexwebapp-prod-green" || FE_HOST="lexwebapp-prod-blue" | |
| else | |
| [ "\$FRONTEND_COLOR" = "green" ] && FE_HOST="lexwebapp-prod-green" || FE_HOST="lexwebapp-prod-blue" | |
| fi | |
| printf '# Active upstreams — managed by deploy script (blue-green switching)\n# DO NOT EDIT MANUALLY — overwritten during zero-downtime deploy\n\nupstream prod_mcp_backend {\n server %s:3000;\n keepalive 128;\n}\n\nupstream prod_frontend {\n server %s:80;\n keepalive 32;\n}\n' "\$BE_HOST" "\$FE_HOST" > nginx/includes/prod-upstreams.conf | |
| # --- 1b. Update dynamic DNS vars (resolver-based, no restart needed) --- | |
| printf '# Backend hostname variables for dynamic DNS resolution\n# Managed by deploy script (blue-green switching)\n\nset \$prod_backend_host %s;\nset \$prod_backend http://\$prod_backend_host:3000;\nset \$prod_frontend_host %s;\nset \$prod_frontend http://\$prod_frontend_host:80;\n' "\$BE_HOST" "\$FE_HOST" > nginx/includes/prod-backend-vars.conf | |
| # --- 2. Reset preview upstreams to placeholder --- | |
| printf '# Preview upstreams — managed by deploy script\n# Points to inactive color containers for pre-production preview\n# DO NOT EDIT MANUALLY\n\nupstream preview_mcp_backend {\n server 127.0.0.1:1; # placeholder — returns 502 when no preview is active\n}\n\nupstream preview_frontend {\n server 127.0.0.1:1; # placeholder — returns 502 when no preview is active\n}\n' > nginx/includes/preview-upstreams.conf | |
| # --- 3. Verify upstreams point to running containers --- | |
| UPSTREAM_BE=\$(grep 'server ' nginx/includes/prod-upstreams.conf | head -1 | awk '{print \$2}' | cut -d: -f1) | |
| if ! docker inspect "\$UPSTREAM_BE" --format '{{.State.Running}}' 2>/dev/null | grep -q true; then | |
| echo "ERROR: upstream \$UPSTREAM_BE is not running! Regenerating upstreams..." | |
| # Fallback: find whichever backend is actually running | |
| if docker inspect secondlayer-app-prod-green --format '{{.State.Running}}' 2>/dev/null | grep -q true; then | |
| BE_HOST="secondlayer-app-prod-green" | |
| else | |
| BE_HOST="secondlayer-app-prod" | |
| fi | |
| if docker inspect lexwebapp-prod-green --format '{{.State.Running}}' 2>/dev/null | grep -q true; then | |
| FE_HOST="lexwebapp-prod-green" | |
| else | |
| FE_HOST="lexwebapp-prod-blue" | |
| fi | |
| printf '# Active upstreams — managed by deploy script (blue-green switching)\n# DO NOT EDIT MANUALLY — overwritten during zero-downtime deploy\n\nupstream prod_mcp_backend {\n server %s:3000;\n keepalive 128;\n}\n\nupstream prod_frontend {\n server %s:80;\n keepalive 32;\n}\n' "\$BE_HOST" "\$FE_HOST" > nginx/includes/prod-upstreams.conf | |
| printf '# Backend hostname variables for dynamic DNS resolution\n# Managed by deploy script (blue-green switching)\n\nset \$prod_backend_host %s;\nset \$prod_backend http://\$prod_backend_host:3000;\nset \$prod_frontend_host %s;\nset \$prod_frontend http://\$prod_frontend_host:80;\n' "\$BE_HOST" "\$FE_HOST" > nginx/includes/prod-backend-vars.conf | |
| fi | |
| # Recreate nginx to pick up new upstreams | |
| \$DC up -d nginx-prod --force-recreate | |
| echo "Nginx recreated — production traffic switched to new containers" | |
| # Allow in-flight requests to complete | |
| sleep 5 | |
| # --- 4. Stop old-color containers --- | |
| if [ "\$ANY_BACKEND" = "true" ]; then | |
| if [ "\$BACKEND_COLOR" = "green" ]; then | |
| \$DC --profile green stop app-prod-green document-service-prod-green rada-mcp-app-prod-green app-openreyestr-prod-green 2>/dev/null || true | |
| \$DC --profile green rm -f app-prod-green document-service-prod-green rada-mcp-app-prod-green app-openreyestr-prod-green 2>/dev/null || true | |
| else | |
| \$DC stop app-prod document-service-prod rada-mcp-app-prod app-openreyestr-prod 2>/dev/null || true | |
| \$DC rm -f app-prod document-service-prod rada-mcp-app-prod app-openreyestr-prod 2>/dev/null || true | |
| fi | |
| BACKEND_COLOR="\$NEW_BACKEND" | |
| fi | |
| if [ "${FRONTEND_CHANGED}" = "true" ]; then | |
| if [ "\$FRONTEND_COLOR" = "green" ]; then | |
| \$DC --profile green stop lexwebapp-prod-green 2>/dev/null || true | |
| \$DC --profile green rm -f lexwebapp-prod-green 2>/dev/null || true | |
| else | |
| \$DC --profile blue stop lexwebapp-prod-blue 2>/dev/null || true | |
| \$DC --profile blue rm -f lexwebapp-prod-blue 2>/dev/null || true | |
| fi | |
| FRONTEND_COLOR="\$NEW_FRONTEND" | |
| fi | |
| # --- 5. Save active colors --- | |
| printf 'backend=%s\nfrontend=%s\n' "\$BACKEND_COLOR" "\$FRONTEND_COLOR" > .active-colors | |
| # --- 5a. Canonical DNS aliases --- | |
| # opendata-sync dials app-prod:3000, app-openreyestr-prod:3005 and | |
| # rada-mcp-app-prod:3001, while blue-green containers carry a colour | |
| # suffix. Those aliases now live in docker-compose.prod.yml under each | |
| # service's networks: block, so a container is created with them. | |
| # | |
| # They used to be attached here, to the already-running container, with | |
| # docker network disconnect + connect --alias. That re-attach hands the | |
| # container a NEW IP (measured 172.18.0.34 -> 172.18.0.4 about a minute | |
| # after start), and every TCP connection the process had already opened | |
| # keeps a source address that no longer exists on eth0. Nothing those | |
| # sockets send is delivered and no RST or FIN comes back, so node-redis, | |
| # which has no per-command timeout, never sees an error and never | |
| # reconnects: it writes into a black hole (13,728 bytes stuck in tx_queue | |
| # when this was caught). Every cache read then costs the full 2500ms | |
| # CacheAdapter guard, the rate limiter silently falls back to per-process | |
| # memory, and only the kernel's TCP keepalive ends it, about 15 minutes | |
| # later. That was LEXAI-1795, once per deploy, for every one of the three | |
| # services listed above. Do not re-attach a live container to a network. | |
| # Restart monitoring if needed | |
| if [ "${MONITORING_CHANGED}" = "true" ]; then | |
| echo "=== Restarting monitoring services ===" | |
| # Create the EDRSR metrics-cache sidecar if missing, and restart it | |
| # so it re-reads the bind-mounted script on config changes. | |
| \$DC up -d qdrant-edrsr-metrics-cache aws-cost-exporter 2>/dev/null || true | |
| \$DC restart qdrant-edrsr-metrics-cache aws-cost-exporter grafana-prod prometheus-prod 2>/dev/null || true | |
| fi | |
| docker image prune -f | |
| PROMOTE_SCRIPT | |
| - name: Prod health check | |
| env: | |
| PROD_SSH_KEY_PATH: ${{ secrets.PROD_SSH_KEY_PATH }} | |
| run: | | |
| SSH_CMD="ssh -i $PROD_SSH_KEY_PATH -o StrictHostKeyChecking=no -o ConnectTimeout=10 ${PROD_USER}@${PROD_SERVER}" | |
| # Determine active backend container name (blue or green) | |
| BACKEND_COLOR=$($SSH_CMD "grep '^backend=' /home/${PROD_USER}/SecondLayer/deployment/.active-colors 2>/dev/null | cut -d= -f2" || echo "blue") | |
| if [ "$BACKEND_COLOR" = "green" ]; then | |
| CONTAINER="secondlayer-app-prod-green" | |
| else | |
| CONTAINER="secondlayer-app-prod" | |
| fi | |
| echo "Checking container: $CONTAINER" | |
| # Health check via docker exec on prod (port 3000 is not exposed on host) | |
| for i in 1 2 3 4 5; do | |
| HEALTH=$($SSH_CMD "docker exec $CONTAINER wget -q -O- --timeout=10 http://127.0.0.1:3000/health/ready 2>/dev/null") && { | |
| echo "=== Production health (docker exec via SSH) ===" | |
| echo "$HEALTH" | jq . 2>/dev/null || echo "$HEALTH" | |
| STATUS=$(echo "$HEALTH" | jq -r '.status' 2>/dev/null) | |
| if [ "$STATUS" = "degraded" ]; then | |
| echo "::warning::Production is degraded" | |
| echo "$HEALTH" | jq -r '.checks | to_entries[] | select(.value.ok == false) | " ✗ \(.key): \(.value.error)"' 2>/dev/null | |
| else | |
| echo "✓ All checks passed" | |
| fi | |
| break | |
| } | |
| [ $i -eq 5 ] && echo "::error::Production health check failed" && exit 1 | |
| sleep 10 | |
| done | |
| # ─── Step 3c: Self-heal deploy failures (DISABLED) ───────────────── | |
| self-heal-deploy: | |
| name: Self-Heal Deploy Failures | |
| # SSHes to PROD_SERVER_IP — pin to prod-runner (see Deploy Preview note). | |
| runs-on: [self-hosted, prod-deploy] | |
| needs: [detect-changes, deploy-preview, promote-to-prod] | |
| if: false # temporarily disabled — not producing useful fixes | |
| env: | |
| CLAUDE_CODE_USE_BEDROCK: '1' | |
| CLAUDE_CODE_BEDROCK_MODEL: eu.anthropic.claude-sonnet-4-6 | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: eu-central-1 | |
| PROD_SERVER: ${{ secrets.PROD_SERVER_IP }} | |
| PROD_USER: ${{ secrets.PROD_USER || 'ubuntu' }} | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 10 | |
| token: ${{ github.token }} | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Guard against infinite loops | |
| id: guard | |
| run: | | |
| AUTOFIX_COUNT=$(git log -5 --format='%s' | grep -c '\[ci-autofix\]' || true) | |
| if [ "$AUTOFIX_COUNT" -ge 1 ]; then | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| echo "::warning::Found $AUTOFIX_COUNT autofix commit(s) in last 5 — skipping" | |
| else | |
| echo "skip=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Collect prod diagnostics | |
| if: steps.guard.outputs.skip == 'false' | |
| id: diag | |
| env: | |
| PROD_SSH_KEY_PATH: ${{ secrets.PROD_SSH_KEY_PATH }} | |
| run: | | |
| set +e | |
| SSH_CMD="ssh -i $PROD_SSH_KEY_PATH -o StrictHostKeyChecking=no -o ConnectTimeout=10 ${PROD_USER}@${PROD_SERVER}" | |
| REMOTE_REPO="/home/${PROD_USER}/SecondLayer" | |
| { | |
| # Test SSH connectivity first | |
| if ! $SSH_CMD "echo SSH_OK" 2>/dev/null; then | |
| echo "=== SSH to prod FAILED — server may be unreachable ===" | |
| echo "Skipping remote diagnostics. CI error log will be used instead." | |
| else | |
| echo "=== Docker container status ===" | |
| $SSH_CMD "cd ${REMOTE_REPO}/deployment && docker compose -f docker-compose.prod.yml --env-file .env.prod ps" 2>&1 || true | |
| echo "" | |
| echo "=== Last 30 lines from failing containers ===" | |
| for svc in app-prod rada-mcp-app-prod app-openreyestr-prod document-service-prod nginx-prod lexwebapp-prod-blue lexwebapp-prod-green; do | |
| echo "--- $svc ---" | |
| $SSH_CMD "cd ${REMOTE_REPO}/deployment && docker compose -f docker-compose.prod.yml --env-file .env.prod logs --tail=30 $svc" 2>&1 || true | |
| done | |
| echo "" | |
| echo "=== Health check responses ===" | |
| $SSH_CMD "docker exec secondlayer-app-prod curl -sf http://127.0.0.1:3000/health 2>&1 || echo 'Backend health: FAILED'" || true | |
| $SSH_CMD "docker exec rada-mcp-app-prod curl -sf http://127.0.0.1:3001/health 2>&1 || echo 'RADA health: FAILED'" || true | |
| $SSH_CMD "docker exec openreyestr-app-prod curl -sf http://127.0.0.1:3005/health 2>&1 || echo 'OpenReyestr health: FAILED'" || true | |
| fi | |
| } > /tmp/deploy-diag.log 2>&1 | |
| echo "Captured $(wc -l < /tmp/deploy-diag.log) lines of diagnostics" | |
| - name: Claude Code — diagnose and create fix PR | |
| if: steps.guard.outputs.skip == 'false' | |
| id: claude-fix | |
| continue-on-error: true | |
| run: | | |
| gh run view "${{ github.run_id }}" --log-failed 2>&1 | tail -80 > /tmp/ci-run-errors.log | |
| BRANCH_NAME="ci-autofix/deploy-$(date +%Y%m%d-%H%M%S)-${GITHUB_SHA:0:7}" | |
| git checkout -b "$BRANCH_NAME" | |
| cat > /tmp/ci-prompt.txt <<'PROMPT_END' | |
| Production deployment failed. Diagnostics are saved in files — read them first: | |
| 1. /tmp/ci-run-errors.log — CI run error output | |
| 2. /tmp/deploy-diag.log — production server diagnostics (container status, logs, health checks) | |
| Your task: | |
| 1. Read both diagnostic files to understand the failure | |
| 2. Analyze the failure — is it a code bug, config issue, Docker build error, or infrastructure problem? | |
| 3. If it's a code fix (typo, missing import, wrong env var, Docker config): | |
| - Fix the code locally | |
| - Stage ONLY the files you changed and commit with message: "fix: [ci-autofix] <description>" | |
| - Do NOT push — the pipeline will handle pushing and PR creation | |
| 4. If you cannot fix it with code changes, create a GitHub issue describing the problem | |
| Rules: | |
| - Do NOT SSH to production servers — no direct prod access | |
| - Do NOT run git push | |
| - Do NOT modify CI workflow files | |
| - Do NOT run git reset, git revert, or any destructive git commands | |
| - Focus on identifying and fixing the root cause in code | |
| PROMPT_END | |
| npx -y @anthropic-ai/claude-code@latest \ | |
| -p "$(cat /tmp/ci-prompt.txt)" \ | |
| --allowedTools "Bash,Read,Edit,Write,Glob,Grep" \ | |
| --max-turns 25 | |
| timeout-minutes: 10 | |
| - name: Check if fix was produced | |
| if: steps.guard.outputs.skip == 'false' | |
| id: check-fix | |
| run: | | |
| CURRENT_BRANCH=$(git branch --show-current) | |
| if git log --oneline "main..$CURRENT_BRANCH" 2>/dev/null | grep -q .; then | |
| echo "has_fix=true" >> "$GITHUB_OUTPUT" | |
| echo "branch=$CURRENT_BRANCH" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "has_fix=false" >> "$GITHUB_OUTPUT" | |
| echo "::warning::Claude Code did not produce a fix — manual intervention needed" | |
| fi | |
| - name: Create issue if no fix produced | |
| if: steps.guard.outputs.skip == 'false' && steps.check-fix.outputs.has_fix == 'false' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| gh issue create \ | |
| --title "Deploy failure needs manual fix ($(date +%Y-%m-%d))" \ | |
| --body "$(cat <<EOF | |
| ## Production deploy failure — auto-fix failed | |
| The self-heal agent could not produce a code fix for this deploy failure. | |
| **Failed run:** $RUN_URL | |
| **Commit:** ${GITHUB_SHA:0:7} | |
| > Production may be in a degraded state. Investigate promptly. | |
| EOF | |
| )" \ | |
| --label "bug,ci,urgent" | |
| - name: Push branch and create PR | |
| if: steps.guard.outputs.skip == 'false' && steps.check-fix.outputs.has_fix == 'true' | |
| env: | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| BRANCH="${{ steps.check-fix.outputs.branch }}" | |
| git push origin "$BRANCH" | |
| gh pr create \ | |
| --title "fix: [ci-autofix] Auto-fix for deploy failure" \ | |
| --body "$(cat <<EOF | |
| ## Auto-generated fix for deploy failure | |
| Claude Code analyzed the production deploy failure and produced this fix. | |
| **Source run:** $RUN_URL | |
| > Review and merge promptly, or rollback manually. | |
| 🤖 Generated with [Claude Code](https://claude.com/claude-code) | |
| EOF | |
| )" \ | |
| --base main \ | |
| --head "$BRANCH" | |
| # ─── Step 4: Create release ──────────────────────────────────────── | |
| create-release: | |
| name: Create Release | |
| runs-on: [self-hosted, local] | |
| needs: [detect-changes, deploy-preview, promote-to-prod] | |
| if: | | |
| always() && | |
| needs.promote-to-prod.result == 'success' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Create releases | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| SERVICES_TO_DEPLOY: ${{ needs.detect-changes.outputs.services_to_deploy }} | |
| BACKEND_VERSION: ${{ needs.deploy-preview.outputs.backend_version }} | |
| FRONTEND_VERSION: ${{ needs.deploy-preview.outputs.frontend_version }} | |
| BACKEND_CHANGED: ${{ needs.detect-changes.outputs.backend }} | |
| RADA_CHANGED: ${{ needs.detect-changes.outputs.rada }} | |
| OPENREYESTR_CHANGED: ${{ needs.detect-changes.outputs.openreyestr }} | |
| FRONTEND_CHANGED: ${{ needs.detect-changes.outputs.frontend }} | |
| run: | | |
| ANY_BACKEND=false | |
| if [ "$BACKEND_CHANGED" = "true" ] || [ "$RADA_CHANGED" = "true" ] || [ "$OPENREYESTR_CHANGED" = "true" ]; then | |
| ANY_BACKEND=true | |
| fi | |
| # --- Backend release --- | |
| if [ "$ANY_BACKEND" = "true" ]; then | |
| LAST_BE_TAG=$(git tag -l "v*" --sort=-version:refname | head -1) | |
| if [ -z "$LAST_BE_TAG" ]; then | |
| BE_COMMITS=$(git log --oneline --no-merges -50 -- packages/shared/ mcp_backend/ mcp_rada/ mcp_openreyestr/) | |
| else | |
| BE_COMMITS=$(git log "${LAST_BE_TAG}..HEAD" --oneline --no-merges -- packages/shared/ mcp_backend/ mcp_rada/ mcp_openreyestr/) | |
| fi | |
| BE_FEATURES=$(echo "$BE_COMMITS" | grep -iE "^[a-f0-9]+ feat(\(|:)" | sed 's/^[a-f0-9]* /- /' || true) | |
| BE_FIXES=$(echo "$BE_COMMITS" | grep -iE "^[a-f0-9]+ fix(\(|:)" | sed 's/^[a-f0-9]* /- /' || true) | |
| BE_OTHER=$(echo "$BE_COMMITS" | grep -viE "^[a-f0-9]+ (feat|fix)(\(|:)" | grep -v "^$" | sed 's/^[a-f0-9]* /- /' || true) | |
| BODY="## Backend Release\n" | |
| [ -n "$BE_FEATURES" ] && BODY="${BODY}\n### Features\n${BE_FEATURES}\n" | |
| [ -n "$BE_FIXES" ] && BODY="${BODY}\n### Fixes\n${BE_FIXES}\n" | |
| [ -n "$BE_OTHER" ] && BODY="${BODY}\n### Other\n${BE_OTHER}\n" | |
| BODY="${BODY}\n---\n**Commit:** ${GITHUB_SHA:0:7} | **Runner:** local" | |
| echo "Creating backend release ${BACKEND_VERSION}" | |
| git tag "${BACKEND_VERSION}" | |
| git push origin "${BACKEND_VERSION}" | |
| printf "%b" "$BODY" | gh release create "${BACKEND_VERSION}" \ | |
| --title "${BACKEND_VERSION} (Backend)" \ | |
| --notes-file - \ | |
| --latest | |
| fi | |
| # --- Frontend release --- | |
| if [ "$FRONTEND_CHANGED" = "true" ]; then | |
| LAST_FE_TAG=$(git tag -l "fe-v*" --sort=-version:refname | head -1) | |
| if [ -z "$LAST_FE_TAG" ]; then | |
| FE_COMMITS=$(git log --oneline --no-merges -50 -- lexwebapp/) | |
| else | |
| FE_COMMITS=$(git log "${LAST_FE_TAG}..HEAD" --oneline --no-merges -- lexwebapp/) | |
| fi | |
| FE_FEATURES=$(echo "$FE_COMMITS" | grep -iE "^[a-f0-9]+ feat(\(|:)" | sed 's/^[a-f0-9]* /- /' || true) | |
| FE_FIXES=$(echo "$FE_COMMITS" | grep -iE "^[a-f0-9]+ fix(\(|:)" | sed 's/^[a-f0-9]* /- /' || true) | |
| FE_OTHER=$(echo "$FE_COMMITS" | grep -viE "^[a-f0-9]+ (feat|fix)(\(|:)" | grep -v "^$" | sed 's/^[a-f0-9]* /- /' || true) | |
| BODY="## Frontend Release\n" | |
| [ -n "$FE_FEATURES" ] && BODY="${BODY}\n### Features\n${FE_FEATURES}\n" | |
| [ -n "$FE_FIXES" ] && BODY="${BODY}\n### Fixes\n${FE_FIXES}\n" | |
| [ -n "$FE_OTHER" ] && BODY="${BODY}\n### Other\n${FE_OTHER}\n" | |
| BODY="${BODY}\n---\n**Commit:** ${GITHUB_SHA:0:7} | **Runner:** local" | |
| echo "Creating frontend release ${FRONTEND_VERSION}" | |
| git tag "${FRONTEND_VERSION}" | |
| git push origin "${FRONTEND_VERSION}" | |
| # Frontend release is --latest only if no backend release was created | |
| LATEST_FLAG="" | |
| [ "$ANY_BACKEND" != "true" ] && LATEST_FLAG="--latest" | |
| printf "%b" "$BODY" | gh release create "${FRONTEND_VERSION}" \ | |
| --title "${FRONTEND_VERSION} (Frontend)" \ | |
| --notes-file - \ | |
| $LATEST_FLAG | |
| fi | |
| - name: Tag deploy marker | |
| run: | | |
| # Create a lightweight tag marking this commit as successfully deployed to prod. | |
| # detect-changes uses this to compare against the next deploy, ensuring no | |
| # changes are skipped between deploys. | |
| DEPLOY_TAG="deploy-prod-$(date -u +%Y%m%d-%H%M%S)" | |
| git tag "$DEPLOY_TAG" | |
| git push origin "$DEPLOY_TAG" | |
| echo "Created deploy marker tag: $DEPLOY_TAG" |