Deploy to Production #610
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ############################################################################## | |
| # Pipeline: Deploy to Production (with Preview) | |
| # - Runs on self-hosted runner (local.legal.org.ua) | |
| # - Phase 1: Deploy to inactive color, make preview available | |
| # - Phase 2: After approval, switch production traffic to new color | |
| # - Triggered by successful CI or manually via workflow_dispatch | |
| ############################################################################## | |
| name: 'Deploy to Production' | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| services: | |
| description: 'Services to deploy (comma-separated: backend,rada,openreyestr,frontend,platform,opendata-sync,monitoring). Leave empty to auto-detect from latest changes.' | |
| required: false | |
| default: '' | |
| type: string | |
| workflow_run: | |
| workflows: ["CI/CD: Local Build & Test"] | |
| types: [completed] | |
| branches: [main] | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| actions: read | |
| concurrency: | |
| group: deploy-prod | |
| cancel-in-progress: false | |
| jobs: | |
| # ─── Step 1: Detect what changed ───────────────────────────────────── | |
| detect-changes: | |
| name: Detect Changes | |
| runs-on: [self-hosted, local] | |
| if: | | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') | |
| outputs: | |
| shared: ${{ steps.final.outputs.shared }} | |
| backend: ${{ steps.final.outputs.backend }} | |
| rada: ${{ steps.final.outputs.rada }} | |
| openreyestr: ${{ steps.final.outputs.openreyestr }} | |
| frontend: ${{ steps.final.outputs.frontend }} | |
| deployment: ${{ steps.final.outputs.deployment }} | |
| any_backend: ${{ steps.final.outputs.any_backend }} | |
| services_to_build: ${{ steps.final.outputs.services_to_build }} | |
| services_to_deploy: ${{ steps.final.outputs.services_to_deploy }} | |
| monitoring: ${{ steps.final.outputs.monitoring }} | |
| platform: ${{ steps.final.outputs.platform }} | |
| opendata_sync: ${{ steps.final.outputs.opendata_sync }} | |
| has_changes: ${{ steps.final.outputs.has_changes }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Auto-detect changes (HEAD vs HEAD~1) | |
| id: changes | |
| uses: ./.github/actions/detect-changes | |
| with: | |
| baseline: deploy-prod | |
| - name: Resolve final change flags | |
| id: final | |
| run: | | |
| INPUT_SERVICES="${{ github.event.inputs.services }}" | |
| if [ -n "$INPUT_SERVICES" ]; then | |
| # Manual override: parse comma-separated service names | |
| echo "Using manual service selection: $INPUT_SERVICES" | |
| BACKEND=false RADA=false OPENREYESTR=false FRONTEND=false PLATFORM=false MONITORING=false OPENDATA_SYNC=false | |
| IFS=',' read -ra SVCS <<< "$INPUT_SERVICES" | |
| for svc in "${SVCS[@]}"; do | |
| svc=$(echo "$svc" | xargs) # trim whitespace | |
| case "$svc" in | |
| backend) BACKEND=true ;; | |
| rada) RADA=true ;; | |
| openreyestr) OPENREYESTR=true ;; | |
| frontend) FRONTEND=true ;; | |
| platform) PLATFORM=true ;; | |
| monitoring) MONITORING=true ;; | |
| opendata-sync) OPENDATA_SYNC=true ;; | |
| *) echo "::warning::Unknown service: $svc" ;; | |
| esac | |
| done | |
| echo "backend=$BACKEND" >> "$GITHUB_OUTPUT" | |
| echo "rada=$RADA" >> "$GITHUB_OUTPUT" | |
| echo "openreyestr=$OPENREYESTR" >> "$GITHUB_OUTPUT" | |
| echo "frontend=$FRONTEND" >> "$GITHUB_OUTPUT" | |
| echo "platform=$PLATFORM" >> "$GITHUB_OUTPUT" | |
| echo "opendata_sync=$OPENDATA_SYNC" >> "$GITHUB_OUTPUT" | |
| echo "monitoring=$MONITORING" >> "$GITHUB_OUTPUT" | |
| echo "shared=${{ steps.changes.outputs.shared }}" >> "$GITHUB_OUTPUT" | |
| echo "deployment=${{ steps.changes.outputs.deployment }}" >> "$GITHUB_OUTPUT" | |
| # Compute any_backend | |
| if [ "$BACKEND" = "true" ] || [ "$RADA" = "true" ] || [ "$OPENREYESTR" = "true" ]; then | |
| echo "any_backend=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "any_backend=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| # Build services lists | |
| SERVICES_LIST="" | |
| [ "$BACKEND" = "true" ] && SERVICES_LIST="${SERVICES_LIST:+$SERVICES_LIST,}\"backend\"" | |
| [ "$RADA" = "true" ] && SERVICES_LIST="${SERVICES_LIST:+$SERVICES_LIST,}\"rada\"" | |
| [ "$OPENREYESTR" = "true" ] && SERVICES_LIST="${SERVICES_LIST:+$SERVICES_LIST,}\"openreyestr\"" | |
| [ "$FRONTEND" = "true" ] && SERVICES_LIST="${SERVICES_LIST:+$SERVICES_LIST,}\"frontend\"" | |
| [ "$PLATFORM" = "true" ] && SERVICES_LIST="${SERVICES_LIST:+$SERVICES_LIST,}\"platform\"" | |
| [ "$OPENDATA_SYNC" = "true" ] && SERVICES_LIST="${SERVICES_LIST:+$SERVICES_LIST,}\"opendata-sync\"" | |
| echo "services_to_build=[${SERVICES_LIST}]" >> "$GITHUB_OUTPUT" | |
| echo "services_to_deploy=[${SERVICES_LIST}]" >> "$GITHUB_OUTPUT" | |
| if [ -z "$SERVICES_LIST" ] && [ "$MONITORING" != "true" ]; then | |
| echo "has_changes=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "has_changes=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| else | |
| # Auto-detect from latest changes | |
| echo "Auto-detecting changes from HEAD vs HEAD~1" | |
| echo "backend=${{ steps.changes.outputs.backend }}" >> "$GITHUB_OUTPUT" | |
| echo "rada=${{ steps.changes.outputs.rada }}" >> "$GITHUB_OUTPUT" | |
| echo "openreyestr=${{ steps.changes.outputs.openreyestr }}" >> "$GITHUB_OUTPUT" | |
| echo "frontend=${{ steps.changes.outputs.frontend }}" >> "$GITHUB_OUTPUT" | |
| echo "platform=${{ steps.changes.outputs.platform }}" >> "$GITHUB_OUTPUT" | |
| echo "opendata_sync=${{ steps.changes.outputs.opendata_sync }}" >> "$GITHUB_OUTPUT" | |
| echo "monitoring=${{ steps.changes.outputs.monitoring }}" >> "$GITHUB_OUTPUT" | |
| echo "shared=${{ steps.changes.outputs.shared }}" >> "$GITHUB_OUTPUT" | |
| echo "deployment=${{ steps.changes.outputs.deployment }}" >> "$GITHUB_OUTPUT" | |
| echo "any_backend=${{ steps.changes.outputs.any_backend }}" >> "$GITHUB_OUTPUT" | |
| echo "services_to_build=${{ steps.changes.outputs.services_to_build }}" >> "$GITHUB_OUTPUT" | |
| echo "services_to_deploy=${{ steps.changes.outputs.services_to_deploy }}" >> "$GITHUB_OUTPUT" | |
| SERVICES='${{ steps.changes.outputs.services_to_build }}' | |
| MONITORING='${{ steps.changes.outputs.monitoring }}' | |
| if [ "$SERVICES" = '[]' ] && [ "$MONITORING" != "true" ]; then | |
| echo "has_changes=false" >> "$GITHUB_OUTPUT" | |
| echo "No services changed — nothing to deploy" | |
| else | |
| echo "has_changes=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| fi | |
| # ─── Step 2: Pre-deploy tests ─────────────────────────────────────── | |
| pre-deploy-tests: | |
| name: Pre-Deploy Tests | |
| runs-on: [self-hosted, local] | |
| needs: [detect-changes] | |
| if: needs.detect-changes.outputs.has_changes == 'true' | |
| env: | |
| BACKEND_CHANGED: ${{ needs.detect-changes.outputs.backend }} | |
| FRONTEND_CHANGED: ${{ needs.detect-changes.outputs.frontend }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Ensure native build tools | |
| run: | | |
| if ! command -v make &>/dev/null; then | |
| sudo apt-get update -qq && sudo apt-get install -y -qq build-essential python3 | |
| fi | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Build shared package | |
| run: cd packages/shared && npm install --legacy-peer-deps && npm run build | |
| - name: Overlay proprietary source | |
| if: needs.detect-changes.outputs.backend == 'true' | |
| run: | | |
| CORE_TMP=$(mktemp -d) | |
| gh repo clone overthelex/secondlayer-core "$CORE_TMP" -- --depth 1 | |
| cp -f "$CORE_TMP"/src/services/* mcp_backend/src/services/ | |
| cp -rf "$CORE_TMP"/src/prompts/* mcp_backend/src/prompts/ | |
| rm -rf "$CORE_TMP" | |
| - name: Build & test backend | |
| if: needs.detect-changes.outputs.backend == 'true' | |
| run: | | |
| cd mcp_backend && npm install && npm run build | |
| npx jest --no-cache --forceExit \ | |
| src/controllers/__tests__/ \ | |
| src/middleware/__tests__/ \ | |
| src/adapters/__tests__/ \ | |
| src/services/__tests__/ | |
| - name: Install & test frontend | |
| if: needs.detect-changes.outputs.frontend == 'true' | |
| run: | | |
| cd lexwebapp | |
| npm install --legacy-peer-deps | |
| npm test 2>&1 | tee /tmp/vitest-output.txt || true | |
| if grep -q "Tests.*failed" /tmp/vitest-output.txt; then | |
| echo "::error::Some tests failed" | |
| exit 1 | |
| elif grep -q "Test Files.*passed" /tmp/vitest-output.txt; then | |
| echo "All tests passed" | |
| else | |
| echo "::error::Could not determine test results" | |
| exit 1 | |
| fi | |
| env: | |
| NODE_OPTIONS: --max-old-space-size=8192 | |
| - name: Build frontend | |
| if: needs.detect-changes.outputs.frontend == 'true' | |
| run: cd lexwebapp && npm run build | |
| env: | |
| VITE_API_URL: https://legal.org.ua | |
| NODE_OPTIONS: --max-old-space-size=8192 | |
| # ─── Step 2b: Self-heal test failures (DISABLED) ──────────────────── | |
| self-heal-tests: | |
| name: Self-Heal Test Failures | |
| runs-on: [self-hosted, local] | |
| needs: [detect-changes, pre-deploy-tests] | |
| if: false # temporarily disabled — not producing useful fixes | |
| env: | |
| CLAUDE_CODE_USE_BEDROCK: '1' | |
| CLAUDE_CODE_BEDROCK_MODEL: eu.anthropic.claude-sonnet-4-6 | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: eu-central-1 | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 10 | |
| token: ${{ github.token }} | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Guard against infinite loops | |
| id: guard | |
| run: | | |
| AUTOFIX_COUNT=$(git log -5 --format='%s' | grep -c '\[ci-autofix\]' || true) | |
| if [ "$AUTOFIX_COUNT" -ge 1 ]; then | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| echo "::warning::Found $AUTOFIX_COUNT autofix commit(s) in last 5 — skipping" | |
| else | |
| echo "skip=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Fetch test error logs | |
| if: steps.guard.outputs.skip == 'false' | |
| run: | | |
| gh run view "${{ github.run_id }}" --log-failed 2>&1 | tail -150 > /tmp/ci-errors.log | |
| echo "Captured $(wc -l < /tmp/ci-errors.log) lines of error logs" | |
| - name: Claude Code — diagnose and fix test failures | |
| if: steps.guard.outputs.skip == 'false' | |
| id: claude-fix | |
| continue-on-error: true | |
| run: | | |
| BRANCH_NAME="ci-autofix/tests-$(date +%Y%m%d-%H%M%S)-${GITHUB_SHA:0:7}" | |
| git checkout -b "$BRANCH_NAME" | |
| cat > /tmp/ci-prompt.txt <<'PROMPT_END' | |
| Pre-deploy tests failed. The error logs are in /tmp/ci-errors.log — read that file first. | |
| Your task: | |
| 1. Read /tmp/ci-errors.log to understand the failure | |
| 2. Determine if this is a build error or a test failure | |
| 3. Read the failing test file AND the source file it tests | |
| 4. Determine if the test is wrong (outdated assertions) or the source code has a bug | |
| 5. Fix whichever is actually incorrect | |
| 6. Run the specific test to verify: npx jest --no-cache <test-file> --forceExit | |
| 7. Stage ONLY the files you changed and commit with message: "fix: [ci-autofix] <description>" | |
| 8. Do NOT push — the pipeline will handle pushing and PR creation | |
| Rules: | |
| - Do NOT blindly change test expectations — understand WHY the test fails first | |
| - Do NOT modify CI workflow files | |
| - Do NOT install new dependencies unless absolutely necessary | |
| - Do NOT run git push | |
| - If you cannot fix the issue, create a GitHub issue describing the problem | |
| PROMPT_END | |
| npx -y @anthropic-ai/claude-code@latest \ | |
| -p "$(cat /tmp/ci-prompt.txt)" \ | |
| --allowedTools "Bash,Read,Edit,Write,Glob,Grep" \ | |
| --max-turns 25 | |
| timeout-minutes: 10 | |
| - name: Check if fix was produced | |
| if: steps.guard.outputs.skip == 'false' | |
| id: check-fix | |
| run: | | |
| CURRENT_BRANCH=$(git branch --show-current) | |
| if git log --oneline "main..$CURRENT_BRANCH" 2>/dev/null | grep -q .; then | |
| echo "has_fix=true" >> "$GITHUB_OUTPUT" | |
| echo "branch=$CURRENT_BRANCH" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "has_fix=false" >> "$GITHUB_OUTPUT" | |
| echo "::warning::Claude Code did not produce a fix" | |
| fi | |
| - name: Create issue if no fix produced | |
| if: steps.guard.outputs.skip == 'false' && steps.check-fix.outputs.has_fix == 'false' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| gh issue create \ | |
| --title "Pre-deploy test failure needs manual fix ($(date +%Y-%m-%d))" \ | |
| --body "$(cat <<EOF | |
| ## Pre-deploy test failure — auto-fix failed | |
| The self-heal agent could not produce a code fix for this test failure. | |
| **Failed run:** $RUN_URL | |
| **Commit:** ${GITHUB_SHA:0:7} | |
| Please investigate manually. | |
| EOF | |
| )" \ | |
| --label "bug,ci" | |
| - name: Push branch and create PR | |
| if: steps.guard.outputs.skip == 'false' && steps.check-fix.outputs.has_fix == 'true' | |
| env: | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| BRANCH="${{ steps.check-fix.outputs.branch }}" | |
| git push origin "$BRANCH" | |
| gh pr create \ | |
| --title "fix: [ci-autofix] Auto-fix for pre-deploy test failure" \ | |
| --body "$(cat <<EOF | |
| ## Auto-generated fix for test failure | |
| Claude Code analyzed the pre-deploy test failure and produced this fix automatically. | |
| **Source run:** $RUN_URL | |
| > Review carefully before merging — this is an automated fix. | |
| 🤖 Generated with [Claude Code](https://claude.com/claude-code) | |
| EOF | |
| )" \ | |
| --base main \ | |
| --head "$BRANCH" | |
| # ─── Step 3a: Deploy Preview (inactive color) ────────────────────── | |
| deploy-preview: | |
| name: Deploy Preview | |
| runs-on: [self-hosted, local] | |
| needs: [detect-changes, pre-deploy-tests] | |
| if: | | |
| needs.detect-changes.outputs.has_changes == 'true' && | |
| needs.pre-deploy-tests.result == 'success' | |
| outputs: | |
| backend_version: ${{ steps.version.outputs.backend_version }} | |
| frontend_version: ${{ steps.version.outputs.frontend_version }} | |
| env: | |
| PROD_SERVER: ${{ secrets.PROD_SERVER_IP }} | |
| PROD_USER: ${{ secrets.PROD_USER || 'ubuntu' }} | |
| BACKEND_CHANGED: ${{ needs.detect-changes.outputs.backend }} | |
| RADA_CHANGED: ${{ needs.detect-changes.outputs.rada }} | |
| OPENREYESTR_CHANGED: ${{ needs.detect-changes.outputs.openreyestr }} | |
| FRONTEND_CHANGED: ${{ needs.detect-changes.outputs.frontend }} | |
| PLATFORM_CHANGED: ${{ needs.detect-changes.outputs.platform }} | |
| OPENDATA_SYNC_CHANGED: ${{ needs.detect-changes.outputs.opendata_sync }} | |
| MONITORING_CHANGED: ${{ needs.detect-changes.outputs.monitoring }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Compute next versions (backend + frontend) | |
| id: version | |
| run: | | |
| # --- Backend version (v*) --- | |
| LAST_BE_TAG=$(git tag -l "v*" --sort=-version:refname | head -1) | |
| if [ "$BACKEND_CHANGED" = "true" ] || [ "$RADA_CHANGED" = "true" ] || [ "$OPENREYESTR_CHANGED" = "true" ]; then | |
| if [ -z "$LAST_BE_TAG" ]; then | |
| BE_VERSION="v1.0.0" | |
| else | |
| CURRENT="${LAST_BE_TAG#v}" | |
| MAJOR=$(echo "$CURRENT" | cut -d. -f1) | |
| MINOR=$(echo "$CURRENT" | cut -d. -f2) | |
| PATCH=$(echo "$CURRENT" | cut -d. -f3) | |
| COMMITS=$(git log "${LAST_BE_TAG}..HEAD" --oneline --no-merges -- packages/shared/ mcp_backend/ mcp_rada/ mcp_openreyestr/) | |
| if echo "$COMMITS" | grep -qiE "^[a-f0-9]+ (feat|refactor)!:|BREAKING CHANGE"; then | |
| MAJOR=$((MAJOR + 1)); MINOR=0; PATCH=0 | |
| elif echo "$COMMITS" | grep -qiE "^[a-f0-9]+ feat(\(|:)"; then | |
| MINOR=$((MINOR + 1)); PATCH=0 | |
| else | |
| PATCH=$((PATCH + 1)) | |
| fi | |
| BE_VERSION="v${MAJOR}.${MINOR}.${PATCH}" | |
| fi | |
| else | |
| BE_VERSION="${LAST_BE_TAG:-v1.0.0}" | |
| fi | |
| echo "backend_version=${BE_VERSION}" >> "$GITHUB_OUTPUT" | |
| echo "Backend version: ${BE_VERSION}" | |
| # --- Frontend version (fe-v*) --- | |
| LAST_FE_TAG=$(git tag -l "fe-v*" --sort=-version:refname | head -1) | |
| if [ "$FRONTEND_CHANGED" = "true" ]; then | |
| if [ -z "$LAST_FE_TAG" ]; then | |
| FE_VERSION="fe-v1.0.0" | |
| else | |
| CURRENT="${LAST_FE_TAG#fe-v}" | |
| MAJOR=$(echo "$CURRENT" | cut -d. -f1) | |
| MINOR=$(echo "$CURRENT" | cut -d. -f2) | |
| PATCH=$(echo "$CURRENT" | cut -d. -f3) | |
| COMMITS=$(git log "${LAST_FE_TAG}..HEAD" --oneline --no-merges -- lexwebapp/) | |
| if echo "$COMMITS" | grep -qiE "^[a-f0-9]+ (feat|refactor)!:|BREAKING CHANGE"; then | |
| MAJOR=$((MAJOR + 1)); MINOR=0; PATCH=0 | |
| elif echo "$COMMITS" | grep -qiE "^[a-f0-9]+ feat(\(|:)"; then | |
| MINOR=$((MINOR + 1)); PATCH=0 | |
| else | |
| PATCH=$((PATCH + 1)) | |
| fi | |
| FE_VERSION="fe-v${MAJOR}.${MINOR}.${PATCH}" | |
| fi | |
| else | |
| FE_VERSION="${LAST_FE_TAG:-fe-v0.0.0}" | |
| fi | |
| echo "frontend_version=${FE_VERSION}" >> "$GITHUB_OUTPUT" | |
| echo "Frontend version: ${FE_VERSION}" | |
| - name: "Phase 1: Build, migrate, start inactive color, enable preview" | |
| env: | |
| PROD_SSH_KEY_PATH: ${{ secrets.PROD_SSH_KEY_PATH }} | |
| BACKEND_VERSION: ${{ steps.version.outputs.backend_version }} | |
| FRONTEND_VERSION: ${{ steps.version.outputs.frontend_version }} | |
| run: | | |
| if [ -z "$PROD_SSH_KEY_PATH" ]; then | |
| echo "::error::PROD_SSH_KEY_PATH secret is not set" | |
| exit 1 | |
| fi | |
| if [ -z "$PROD_SERVER" ]; then | |
| echo "::error::PROD_SERVER_IP secret is not set" | |
| exit 1 | |
| fi | |
| SSH_CMD="ssh -i $PROD_SSH_KEY_PATH -o StrictHostKeyChecking=no ${PROD_USER}@${PROD_SERVER}" | |
| REMOTE_REPO="/home/${PROD_USER}/SecondLayer" | |
| echo "=== Phase 1: Deploy preview (inactive color) ===" | |
| # Pull latest code and tags | |
| $SSH_CMD "git -C ${REMOTE_REPO} fetch origin main --tags && git -C ${REMOTE_REPO} reset --hard origin/main" | |
| # Write VERSION file with both versions | |
| $SSH_CMD "printf 'BACKEND_VERSION=%s\nFRONTEND_VERSION=%s\n' '${BACKEND_VERSION}' '${FRONTEND_VERSION}' > ${REMOTE_REPO}/VERSION" | |
| # Build dist on prod | |
| $SSH_CMD "cd ${REMOTE_REPO} && npm --prefix packages/shared install && npm --prefix packages/shared run build" | |
| if [ "$BACKEND_CHANGED" = "true" ]; then | |
| # Overlay proprietary source from private repo before building | |
| CORE_TMP=$(mktemp -d) | |
| gh repo clone overthelex/secondlayer-core "$CORE_TMP" -- --depth 1 2>/dev/null | |
| SCP_CMD="scp -i $PROD_SSH_KEY_PATH -o StrictHostKeyChecking=no" | |
| $SCP_CMD -r "$CORE_TMP"/src/services/* "${PROD_USER}@${PROD_SERVER}:${REMOTE_REPO}/mcp_backend/src/services/" | |
| $SCP_CMD -r "$CORE_TMP"/src/prompts/* "${PROD_USER}@${PROD_SERVER}:${REMOTE_REPO}/mcp_backend/src/prompts/" | |
| rm -rf "$CORE_TMP" | |
| $SSH_CMD "cd ${REMOTE_REPO} && npm --prefix mcp_backend install && npm --prefix mcp_backend run build" | |
| fi | |
| if [ "$RADA_CHANGED" = "true" ]; then | |
| $SSH_CMD "cd ${REMOTE_REPO} && npm --prefix mcp_rada install && npm --prefix mcp_rada run build" | |
| fi | |
| if [ "$OPENREYESTR_CHANGED" = "true" ]; then | |
| $SSH_CMD "cd ${REMOTE_REPO} && npm --prefix mcp_openreyestr install && npm --prefix mcp_openreyestr run build" | |
| fi | |
| # Phase 1: Build images, run migrations, start inactive color, write preview upstreams | |
| $SSH_CMD bash << DEPLOY_SCRIPT | |
| set -e | |
| cd ${REMOTE_REPO}/deployment | |
| export APP_VERSION=\$(grep 'BACKEND_VERSION=' ${REMOTE_REPO}/VERSION 2>/dev/null | cut -d= -f2 || echo 'dev') | |
| export APP_FRONTEND_VERSION=\$(grep 'FRONTEND_VERSION=' ${REMOTE_REPO}/VERSION 2>/dev/null | cut -d= -f2 || echo 'dev') | |
| DC="docker compose -f docker-compose.prod.yml --env-file .env.prod" | |
| # --- Determine active colors (per service group) --- | |
| BACKEND_COLOR=\$(grep '^backend=' .active-colors 2>/dev/null | cut -d= -f2 || echo "blue") | |
| FRONTEND_COLOR=\$(grep '^frontend=' .active-colors 2>/dev/null | cut -d= -f2 || echo "blue") | |
| [ "\$BACKEND_COLOR" = "green" ] && NEW_BACKEND="blue" || NEW_BACKEND="green" | |
| [ "\$FRONTEND_COLOR" = "green" ] && NEW_FRONTEND="blue" || NEW_FRONTEND="green" | |
| ANY_BACKEND=false | |
| if [ "${BACKEND_CHANGED}" = "true" ] || [ "${RADA_CHANGED}" = "true" ] || [ "${OPENREYESTR_CHANGED}" = "true" ]; then | |
| ANY_BACKEND=true | |
| fi | |
| echo "Backend: \$BACKEND_COLOR → \$NEW_BACKEND (changed: \$ANY_BACKEND)" | |
| echo "Frontend: \$FRONTEND_COLOR → \$NEW_FRONTEND (changed: ${FRONTEND_CHANGED})" | |
| # --- 1. Build images (old containers still serving traffic) --- | |
| BUILD="" | |
| [ "${BACKEND_CHANGED}" = "true" ] && BUILD="\$BUILD app-prod document-service-prod migrate-prod" | |
| [ "${RADA_CHANGED}" = "true" ] && BUILD="\$BUILD rada-mcp-app-prod rada-db-init-prod rada-migrate-prod" | |
| [ "${OPENREYESTR_CHANGED}" = "true" ] && BUILD="\$BUILD app-openreyestr-prod migrate-openreyestr-prod" | |
| [ "${FRONTEND_CHANGED}" = "true" ] && BUILD="\$BUILD lexwebapp-prod" | |
| [ "${PLATFORM_CHANGED}" = "true" ] && BUILD="\$BUILD platform-prod" | |
| [ "${OPENDATA_SYNC_CHANGED}" = "true" ] && BUILD="\$BUILD opendata-sync-prod" | |
| [ -n "\$BUILD" ] && \$DC build \$BUILD | |
| # --- 2. Run migrations (idempotent, safe while old containers serve) --- | |
| [ "${BACKEND_CHANGED}" = "true" ] && { \$DC up migrate-prod || true; } | |
| [ "${RADA_CHANGED}" = "true" ] && { \$DC up rada-db-init-prod || true; \$DC up rada-migrate-prod || true; } | |
| [ "${OPENREYESTR_CHANGED}" = "true" ] && { \$DC up migrate-openreyestr-prod || true; } | |
| # --- 3. Start new-color containers alongside old ones --- | |
| # Ensure infra services are running WITHOUT recreating them. | |
| # Qdrant holds vector indices and may be running long embedding jobs; | |
| # recreating it interrupts indexing and takes 10+ min to recover. | |
| \$DC up -d --no-recreate qdrant-prod redis-prod pgbouncer-prod | |
| if [ "\$ANY_BACKEND" = "true" ]; then | |
| if [ "\$NEW_BACKEND" = "green" ]; then | |
| \$DC --profile green up -d --no-deps app-prod-green document-service-prod-green rada-mcp-app-prod-green app-openreyestr-prod-green | |
| else | |
| \$DC up -d --no-deps app-prod document-service-prod rada-mcp-app-prod app-openreyestr-prod | |
| fi | |
| fi | |
| if [ "${FRONTEND_CHANGED}" = "true" ]; then | |
| if [ "\$NEW_FRONTEND" = "green" ]; then | |
| \$DC --profile green up -d lexwebapp-prod-green | |
| else | |
| \$DC up -d lexwebapp-prod | |
| fi | |
| fi | |
| if [ "${PLATFORM_CHANGED}" = "true" ]; then | |
| \$DC up -d platform-prod | |
| fi | |
| if [ "${OPENDATA_SYNC_CHANGED}" = "true" ]; then | |
| \$DC up -d opendata-sync-prod | |
| fi | |
| # --- 4. Wait for new containers to be healthy --- | |
| wait_healthy() { | |
| local container=\$1 max_wait=\${2:-120} elapsed=0 | |
| echo "Waiting for \$container to be healthy..." | |
| while [ \$elapsed -lt \$max_wait ]; do | |
| local status | |
| status=\$(docker inspect --format='{{.State.Health.Status}}' "\$container" 2>/dev/null || echo "not_found") | |
| case "\$status" in | |
| healthy) echo "✓ \$container: healthy (\${elapsed}s)"; return 0 ;; | |
| unhealthy) | |
| echo "✗ \$container: unhealthy after \${elapsed}s" | |
| docker logs --tail=20 "\$container" 2>&1 | |
| return 1 ;; | |
| *) sleep 5; elapsed=\$((elapsed + 5)) ;; | |
| esac | |
| done | |
| echo "✗ \$container: timed out (\${max_wait}s)" | |
| docker logs --tail=20 "\$container" 2>&1 | |
| return 1 | |
| } | |
| HEALTH_OK=true | |
| if [ "\$ANY_BACKEND" = "true" ]; then | |
| if [ "\$NEW_BACKEND" = "green" ]; then | |
| wait_healthy "secondlayer-app-prod-green" 180 || HEALTH_OK=false | |
| wait_healthy "document-service-prod-green" 120 || HEALTH_OK=false | |
| wait_healthy "rada-mcp-app-prod-green" 120 || HEALTH_OK=false | |
| wait_healthy "openreyestr-app-prod-green" 120 || HEALTH_OK=false | |
| else | |
| wait_healthy "secondlayer-app-prod" 180 || HEALTH_OK=false | |
| wait_healthy "document-service-prod" 120 || HEALTH_OK=false | |
| wait_healthy "rada-mcp-app-prod" 120 || HEALTH_OK=false | |
| wait_healthy "openreyestr-app-prod" 120 || HEALTH_OK=false | |
| fi | |
| fi | |
| if [ "${FRONTEND_CHANGED}" = "true" ]; then | |
| if [ "\$NEW_FRONTEND" = "green" ]; then | |
| wait_healthy "lexwebapp-prod-green" 60 || HEALTH_OK=false | |
| else | |
| wait_healthy "lexwebapp-prod" 60 || HEALTH_OK=false | |
| fi | |
| fi | |
| if [ "${PLATFORM_CHANGED}" = "true" ]; then | |
| wait_healthy "platform-prod" 60 || HEALTH_OK=false | |
| fi | |
| if [ "\$HEALTH_OK" != "true" ]; then | |
| echo "::error::New containers failed health check — aborting (old containers still serving)" | |
| # Stop failed new containers | |
| if [ "\$ANY_BACKEND" = "true" ]; then | |
| if [ "\$NEW_BACKEND" = "green" ]; then | |
| \$DC --profile green stop app-prod-green document-service-prod-green rada-mcp-app-prod-green app-openreyestr-prod-green 2>/dev/null || true | |
| \$DC --profile green rm -f app-prod-green document-service-prod-green rada-mcp-app-prod-green app-openreyestr-prod-green 2>/dev/null || true | |
| else | |
| \$DC stop app-prod document-service-prod rada-mcp-app-prod app-openreyestr-prod 2>/dev/null || true | |
| \$DC rm -f app-prod document-service-prod rada-mcp-app-prod app-openreyestr-prod 2>/dev/null || true | |
| fi | |
| fi | |
| if [ "${FRONTEND_CHANGED}" = "true" ]; then | |
| if [ "\$NEW_FRONTEND" = "green" ]; then | |
| \$DC --profile green stop lexwebapp-prod-green 2>/dev/null || true | |
| \$DC --profile green rm -f lexwebapp-prod-green 2>/dev/null || true | |
| else | |
| \$DC stop lexwebapp-prod 2>/dev/null || true | |
| \$DC rm -f lexwebapp-prod 2>/dev/null || true | |
| fi | |
| fi | |
| exit 1 | |
| fi | |
| # --- 5. Write preview upstreams pointing to new (inactive) color --- | |
| if [ "\$ANY_BACKEND" = "true" ]; then | |
| [ "\$NEW_BACKEND" = "green" ] && PREVIEW_BE_HOST="secondlayer-app-prod-green" || PREVIEW_BE_HOST="secondlayer-app-prod" | |
| else | |
| # No backend change — preview backend points to current prod backend | |
| [ "\$BACKEND_COLOR" = "green" ] && PREVIEW_BE_HOST="secondlayer-app-prod-green" || PREVIEW_BE_HOST="secondlayer-app-prod" | |
| fi | |
| if [ "${FRONTEND_CHANGED}" = "true" ]; then | |
| [ "\$NEW_FRONTEND" = "green" ] && PREVIEW_FE_HOST="lexwebapp-prod-green" || PREVIEW_FE_HOST="lexwebapp-prod" | |
| else | |
| # No frontend change — preview frontend points to current prod frontend | |
| [ "\$FRONTEND_COLOR" = "green" ] && PREVIEW_FE_HOST="lexwebapp-prod-green" || PREVIEW_FE_HOST="lexwebapp-prod" | |
| fi | |
| printf '# Preview upstreams — managed by deploy script\n# Points to inactive color containers for pre-production preview\n# DO NOT EDIT MANUALLY\n\nupstream preview_mcp_backend {\n server %s:3000;\n keepalive 128;\n}\n\nupstream preview_frontend {\n server %s:80;\n keepalive 32;\n}\n' "\$PREVIEW_BE_HOST" "\$PREVIEW_FE_HOST" > nginx/includes/preview-upstreams.conf | |
| # Recreate nginx to pick up preview upstreams (bind mount inode staleness) | |
| \$DC up -d nginx-prod --force-recreate | |
| echo "Preview enabled at preview.legal.org.ua" | |
| DEPLOY_SCRIPT | |
| - name: Verify preview health | |
| env: | |
| PROD_SSH_KEY_PATH: ${{ secrets.PROD_SSH_KEY_PATH }} | |
| run: | | |
| SSH_CMD="ssh -i $PROD_SSH_KEY_PATH -o StrictHostKeyChecking=no -o ConnectTimeout=10 ${PROD_USER}@${PROD_SERVER}" | |
| # Check preview backend health via nginx | |
| for i in 1 2 3 4 5; do | |
| HEALTH=$($SSH_CMD "docker exec nginx-prod wget -q -O- --timeout=10 'http://127.0.0.1:80' --header='Host: preview.legal.org.ua' 2>/dev/null | head -c 500") && { | |
| echo "✓ Preview frontend is responding" | |
| break | |
| } | |
| [ $i -eq 5 ] && echo "::warning::Preview frontend health check failed (non-blocking)" | |
| sleep 5 | |
| done | |
| echo "" | |
| echo "==========================================" | |
| echo " Preview is ready at:" | |
| echo " https://preview.legal.org.ua" | |
| echo "" | |
| echo " Waiting for approval to promote to prod" | |
| echo "==========================================" | |
| # ─── Step 3b: Promote to Production (requires approval) ─────────── | |
| promote-to-prod: | |
| name: Promote to Production | |
| runs-on: [self-hosted, local] | |
| needs: [detect-changes, deploy-preview] | |
| if: needs.deploy-preview.result == 'success' | |
| environment: production | |
| outputs: | |
| backend_version: ${{ needs.deploy-preview.outputs.backend_version }} | |
| frontend_version: ${{ needs.deploy-preview.outputs.frontend_version }} | |
| env: | |
| PROD_SERVER: ${{ secrets.PROD_SERVER_IP }} | |
| PROD_USER: ${{ secrets.PROD_USER || 'ubuntu' }} | |
| BACKEND_CHANGED: ${{ needs.detect-changes.outputs.backend }} | |
| RADA_CHANGED: ${{ needs.detect-changes.outputs.rada }} | |
| OPENREYESTR_CHANGED: ${{ needs.detect-changes.outputs.openreyestr }} | |
| FRONTEND_CHANGED: ${{ needs.detect-changes.outputs.frontend }} | |
| PLATFORM_CHANGED: ${{ needs.detect-changes.outputs.platform }} | |
| OPENDATA_SYNC_CHANGED: ${{ needs.detect-changes.outputs.opendata_sync }} | |
| MONITORING_CHANGED: ${{ needs.detect-changes.outputs.monitoring }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: "Phase 2: Switch production traffic to new color" | |
| env: | |
| PROD_SSH_KEY_PATH: ${{ secrets.PROD_SSH_KEY_PATH }} | |
| run: | | |
| SSH_CMD="ssh -i $PROD_SSH_KEY_PATH -o StrictHostKeyChecking=no ${PROD_USER}@${PROD_SERVER}" | |
| REMOTE_REPO="/home/${PROD_USER}/SecondLayer" | |
| echo "=== Phase 2: Promote preview to production ===" | |
| $SSH_CMD bash << PROMOTE_SCRIPT | |
| set -e | |
| cd ${REMOTE_REPO}/deployment | |
| DC="docker compose -f docker-compose.prod.yml --env-file .env.prod" | |
| # --- Determine colors (with reality check) --- | |
| BACKEND_COLOR=\$(grep '^backend=' .active-colors 2>/dev/null | cut -d= -f2 || echo "blue") | |
| FRONTEND_COLOR=\$(grep '^frontend=' .active-colors 2>/dev/null | cut -d= -f2 || echo "blue") | |
| # Safety: verify .active-colors matches actually running containers | |
| # If mismatch, trust docker over the file (handles partial deploy failures) | |
| if docker inspect secondlayer-app-prod-green --format '{{.State.Running}}' 2>/dev/null | grep -q true; then | |
| if ! docker inspect secondlayer-app-prod --format '{{.State.Running}}' 2>/dev/null | grep -q true; then | |
| BACKEND_COLOR="green" | |
| echo "WARNING: .active-colors disagrees with docker state, correcting backend=green" | |
| fi | |
| elif docker inspect secondlayer-app-prod --format '{{.State.Running}}' 2>/dev/null | grep -q true; then | |
| BACKEND_COLOR="blue" | |
| echo "WARNING: .active-colors disagrees with docker state, correcting backend=blue" | |
| fi | |
| if docker inspect lexwebapp-prod-green --format '{{.State.Running}}' 2>/dev/null | grep -q true; then | |
| if ! docker inspect lexwebapp-prod --format '{{.State.Running}}' 2>/dev/null | grep -q true; then | |
| FRONTEND_COLOR="green" | |
| echo "WARNING: .active-colors disagrees with docker state, correcting frontend=green" | |
| fi | |
| elif docker inspect lexwebapp-prod --format '{{.State.Running}}' 2>/dev/null | grep -q true; then | |
| FRONTEND_COLOR="blue" | |
| echo "WARNING: .active-colors disagrees with docker state, correcting frontend=blue" | |
| fi | |
| # Save corrected colors before proceeding | |
| printf 'backend=%s\nfrontend=%s\n' "\$BACKEND_COLOR" "\$FRONTEND_COLOR" > .active-colors | |
| [ "\$BACKEND_COLOR" = "green" ] && NEW_BACKEND="blue" || NEW_BACKEND="green" | |
| [ "\$FRONTEND_COLOR" = "green" ] && NEW_FRONTEND="blue" || NEW_FRONTEND="green" | |
| ANY_BACKEND=false | |
| if [ "${BACKEND_CHANGED}" = "true" ] || [ "${RADA_CHANGED}" = "true" ] || [ "${OPENREYESTR_CHANGED}" = "true" ]; then | |
| ANY_BACKEND=true | |
| fi | |
| echo "Promoting: Backend \$BACKEND_COLOR → \$NEW_BACKEND, Frontend \$FRONTEND_COLOR → \$NEW_FRONTEND" | |
| # --- 1. Switch prod upstreams to new color --- | |
| if [ "\$ANY_BACKEND" = "true" ]; then | |
| [ "\$NEW_BACKEND" = "green" ] && BE_HOST="secondlayer-app-prod-green" || BE_HOST="secondlayer-app-prod" | |
| else | |
| [ "\$BACKEND_COLOR" = "green" ] && BE_HOST="secondlayer-app-prod-green" || BE_HOST="secondlayer-app-prod" | |
| fi | |
| if [ "${FRONTEND_CHANGED}" = "true" ]; then | |
| [ "\$NEW_FRONTEND" = "green" ] && FE_HOST="lexwebapp-prod-green" || FE_HOST="lexwebapp-prod" | |
| else | |
| [ "\$FRONTEND_COLOR" = "green" ] && FE_HOST="lexwebapp-prod-green" || FE_HOST="lexwebapp-prod" | |
| fi | |
| printf '# Active upstreams — managed by deploy script (blue-green switching)\n# DO NOT EDIT MANUALLY — overwritten during zero-downtime deploy\n\nupstream prod_mcp_backend {\n server %s:3000;\n keepalive 128;\n}\n\nupstream prod_frontend {\n server %s:80;\n keepalive 32;\n}\n' "\$BE_HOST" "\$FE_HOST" > nginx/includes/prod-upstreams.conf | |
| # --- 1b. Update dynamic DNS vars (resolver-based, no restart needed) --- | |
| printf '# Backend hostname variables for dynamic DNS resolution\n# Managed by deploy script (blue-green switching)\n\nset \$prod_backend_host %s;\nset \$prod_backend http://\$prod_backend_host:3000;\nset \$prod_frontend_host %s;\nset \$prod_frontend http://\$prod_frontend_host:80;\n' "\$BE_HOST" "\$FE_HOST" > nginx/includes/prod-backend-vars.conf | |
| # --- 2. Reset preview upstreams to placeholder --- | |
| printf '# Preview upstreams — managed by deploy script\n# Points to inactive color containers for pre-production preview\n# DO NOT EDIT MANUALLY\n\nupstream preview_mcp_backend {\n server 127.0.0.1:1; # placeholder — returns 502 when no preview is active\n}\n\nupstream preview_frontend {\n server 127.0.0.1:1; # placeholder — returns 502 when no preview is active\n}\n' > nginx/includes/preview-upstreams.conf | |
| # --- 3. Verify upstreams point to running containers --- | |
| UPSTREAM_BE=\$(grep 'server ' nginx/includes/prod-upstreams.conf | head -1 | awk '{print \$2}' | cut -d: -f1) | |
| if ! docker inspect "\$UPSTREAM_BE" --format '{{.State.Running}}' 2>/dev/null | grep -q true; then | |
| echo "ERROR: upstream \$UPSTREAM_BE is not running! Regenerating upstreams..." | |
| # Fallback: find whichever backend is actually running | |
| if docker inspect secondlayer-app-prod-green --format '{{.State.Running}}' 2>/dev/null | grep -q true; then | |
| BE_HOST="secondlayer-app-prod-green" | |
| else | |
| BE_HOST="secondlayer-app-prod" | |
| fi | |
| if docker inspect lexwebapp-prod-green --format '{{.State.Running}}' 2>/dev/null | grep -q true; then | |
| FE_HOST="lexwebapp-prod-green" | |
| else | |
| FE_HOST="lexwebapp-prod" | |
| fi | |
| printf '# Active upstreams — managed by deploy script (blue-green switching)\n# DO NOT EDIT MANUALLY — overwritten during zero-downtime deploy\n\nupstream prod_mcp_backend {\n server %s:3000;\n keepalive 128;\n}\n\nupstream prod_frontend {\n server %s:80;\n keepalive 32;\n}\n' "\$BE_HOST" "\$FE_HOST" > nginx/includes/prod-upstreams.conf | |
| printf '# Backend hostname variables for dynamic DNS resolution\n# Managed by deploy script (blue-green switching)\n\nset \$prod_backend_host %s;\nset \$prod_backend http://\$prod_backend_host:3000;\nset \$prod_frontend_host %s;\nset \$prod_frontend http://\$prod_frontend_host:80;\n' "\$BE_HOST" "\$FE_HOST" > nginx/includes/prod-backend-vars.conf | |
| fi | |
| # Recreate nginx to pick up new upstreams | |
| \$DC up -d nginx-prod --force-recreate | |
| echo "Nginx recreated — production traffic switched to new containers" | |
| # Allow in-flight requests to complete | |
| sleep 5 | |
| # --- 4. Stop old-color containers --- | |
| if [ "\$ANY_BACKEND" = "true" ]; then | |
| if [ "\$BACKEND_COLOR" = "green" ]; then | |
| \$DC --profile green stop app-prod-green document-service-prod-green rada-mcp-app-prod-green app-openreyestr-prod-green 2>/dev/null || true | |
| \$DC --profile green rm -f app-prod-green document-service-prod-green rada-mcp-app-prod-green app-openreyestr-prod-green 2>/dev/null || true | |
| else | |
| \$DC stop app-prod document-service-prod rada-mcp-app-prod app-openreyestr-prod 2>/dev/null || true | |
| \$DC rm -f app-prod document-service-prod rada-mcp-app-prod app-openreyestr-prod 2>/dev/null || true | |
| fi | |
| BACKEND_COLOR="\$NEW_BACKEND" | |
| fi | |
| if [ "${FRONTEND_CHANGED}" = "true" ]; then | |
| if [ "\$FRONTEND_COLOR" = "green" ]; then | |
| \$DC --profile green stop lexwebapp-prod-green 2>/dev/null || true | |
| \$DC --profile green rm -f lexwebapp-prod-green 2>/dev/null || true | |
| else | |
| \$DC stop lexwebapp-prod 2>/dev/null || true | |
| \$DC rm -f lexwebapp-prod 2>/dev/null || true | |
| fi | |
| FRONTEND_COLOR="\$NEW_FRONTEND" | |
| fi | |
| # --- 5. Save active colors --- | |
| printf 'backend=%s\nfrontend=%s\n' "\$BACKEND_COLOR" "\$FRONTEND_COLOR" > .active-colors | |
| # --- 5a. Add canonical DNS aliases for opendata-sync --- | |
| # opendata-sync connects to app-prod:3000 and app-openreyestr-prod:3005 | |
| # but blue-green containers have color suffixes — add aliases so DNS resolves | |
| NETWORK="deployment_secondlayer-prod-network" | |
| if [ "\$BACKEND_COLOR" = "green" ]; then | |
| BE_CONTAINER="secondlayer-app-prod-green" | |
| OR_CONTAINER="openreyestr-app-prod-green" | |
| else | |
| BE_CONTAINER="secondlayer-app-prod" | |
| OR_CONTAINER="openreyestr-app-prod" | |
| fi | |
| echo "Adding canonical DNS aliases (app-prod, app-openreyestr-prod) to \$BE_CONTAINER, \$OR_CONTAINER" | |
| docker network disconnect "\$NETWORK" "\$BE_CONTAINER" 2>/dev/null || true | |
| docker network connect --alias app-prod "\$NETWORK" "\$BE_CONTAINER" || true | |
| docker network disconnect "\$NETWORK" "\$OR_CONTAINER" 2>/dev/null || true | |
| docker network connect --alias app-openreyestr-prod "\$NETWORK" "\$OR_CONTAINER" || true | |
| # Restart monitoring if needed | |
| if [ "${MONITORING_CHANGED}" = "true" ]; then | |
| echo "=== Restarting monitoring services ===" | |
| \$DC restart grafana-prod prometheus-prod 2>/dev/null || true | |
| fi | |
| docker image prune -f | |
| PROMOTE_SCRIPT | |
| - name: Prod health check | |
| env: | |
| PROD_SSH_KEY_PATH: ${{ secrets.PROD_SSH_KEY_PATH }} | |
| run: | | |
| SSH_CMD="ssh -i $PROD_SSH_KEY_PATH -o StrictHostKeyChecking=no -o ConnectTimeout=10 ${PROD_USER}@${PROD_SERVER}" | |
| # Determine active backend container name (blue or green) | |
| BACKEND_COLOR=$($SSH_CMD "grep '^backend=' /home/${PROD_USER}/SecondLayer/deployment/.active-colors 2>/dev/null | cut -d= -f2" || echo "blue") | |
| if [ "$BACKEND_COLOR" = "green" ]; then | |
| CONTAINER="secondlayer-app-prod-green" | |
| else | |
| CONTAINER="secondlayer-app-prod" | |
| fi | |
| echo "Checking container: $CONTAINER" | |
| # Health check via docker exec on prod (port 3000 is not exposed on host) | |
| for i in 1 2 3 4 5; do | |
| HEALTH=$($SSH_CMD "docker exec $CONTAINER wget -q -O- --timeout=10 http://127.0.0.1:3000/health/ready 2>/dev/null") && { | |
| echo "=== Production health (docker exec via SSH) ===" | |
| echo "$HEALTH" | jq . 2>/dev/null || echo "$HEALTH" | |
| STATUS=$(echo "$HEALTH" | jq -r '.status' 2>/dev/null) | |
| if [ "$STATUS" = "degraded" ]; then | |
| echo "::warning::Production is degraded" | |
| echo "$HEALTH" | jq -r '.checks | to_entries[] | select(.value.ok == false) | " ✗ \(.key): \(.value.error)"' 2>/dev/null | |
| else | |
| echo "✓ All checks passed" | |
| fi | |
| break | |
| } | |
| [ $i -eq 5 ] && echo "::error::Production health check failed" && exit 1 | |
| sleep 10 | |
| done | |
| # ─── Step 3c: Self-heal deploy failures (DISABLED) ───────────────── | |
| self-heal-deploy: | |
| name: Self-Heal Deploy Failures | |
| runs-on: [self-hosted, local] | |
| needs: [detect-changes, deploy-preview, promote-to-prod] | |
| if: false # temporarily disabled — not producing useful fixes | |
| env: | |
| CLAUDE_CODE_USE_BEDROCK: '1' | |
| CLAUDE_CODE_BEDROCK_MODEL: eu.anthropic.claude-sonnet-4-6 | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: eu-central-1 | |
| PROD_SERVER: ${{ secrets.PROD_SERVER_IP }} | |
| PROD_USER: ${{ secrets.PROD_USER || 'ubuntu' }} | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 10 | |
| token: ${{ github.token }} | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Guard against infinite loops | |
| id: guard | |
| run: | | |
| AUTOFIX_COUNT=$(git log -5 --format='%s' | grep -c '\[ci-autofix\]' || true) | |
| if [ "$AUTOFIX_COUNT" -ge 1 ]; then | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| echo "::warning::Found $AUTOFIX_COUNT autofix commit(s) in last 5 — skipping" | |
| else | |
| echo "skip=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Collect prod diagnostics | |
| if: steps.guard.outputs.skip == 'false' | |
| id: diag | |
| env: | |
| PROD_SSH_KEY_PATH: ${{ secrets.PROD_SSH_KEY_PATH }} | |
| run: | | |
| set +e | |
| SSH_CMD="ssh -i $PROD_SSH_KEY_PATH -o StrictHostKeyChecking=no -o ConnectTimeout=10 ${PROD_USER}@${PROD_SERVER}" | |
| REMOTE_REPO="/home/${PROD_USER}/SecondLayer" | |
| { | |
| # Test SSH connectivity first | |
| if ! $SSH_CMD "echo SSH_OK" 2>/dev/null; then | |
| echo "=== SSH to prod FAILED — server may be unreachable ===" | |
| echo "Skipping remote diagnostics. CI error log will be used instead." | |
| else | |
| echo "=== Docker container status ===" | |
| $SSH_CMD "cd ${REMOTE_REPO}/deployment && docker compose -f docker-compose.prod.yml --env-file .env.prod ps" 2>&1 || true | |
| echo "" | |
| echo "=== Last 30 lines from failing containers ===" | |
| for svc in app-prod rada-mcp-app-prod app-openreyestr-prod document-service-prod nginx-prod lexwebapp-prod; do | |
| echo "--- $svc ---" | |
| $SSH_CMD "cd ${REMOTE_REPO}/deployment && docker compose -f docker-compose.prod.yml --env-file .env.prod logs --tail=30 $svc" 2>&1 || true | |
| done | |
| echo "" | |
| echo "=== Health check responses ===" | |
| $SSH_CMD "docker exec secondlayer-app-prod curl -sf http://127.0.0.1:3000/health 2>&1 || echo 'Backend health: FAILED'" || true | |
| $SSH_CMD "docker exec rada-mcp-app-prod curl -sf http://127.0.0.1:3001/health 2>&1 || echo 'RADA health: FAILED'" || true | |
| $SSH_CMD "docker exec openreyestr-app-prod curl -sf http://127.0.0.1:3005/health 2>&1 || echo 'OpenReyestr health: FAILED'" || true | |
| fi | |
| } > /tmp/deploy-diag.log 2>&1 | |
| echo "Captured $(wc -l < /tmp/deploy-diag.log) lines of diagnostics" | |
| - name: Claude Code — diagnose and create fix PR | |
| if: steps.guard.outputs.skip == 'false' | |
| id: claude-fix | |
| continue-on-error: true | |
| run: | | |
| gh run view "${{ github.run_id }}" --log-failed 2>&1 | tail -80 > /tmp/ci-run-errors.log | |
| BRANCH_NAME="ci-autofix/deploy-$(date +%Y%m%d-%H%M%S)-${GITHUB_SHA:0:7}" | |
| git checkout -b "$BRANCH_NAME" | |
| cat > /tmp/ci-prompt.txt <<'PROMPT_END' | |
| Production deployment failed. Diagnostics are saved in files — read them first: | |
| 1. /tmp/ci-run-errors.log — CI run error output | |
| 2. /tmp/deploy-diag.log — production server diagnostics (container status, logs, health checks) | |
| Your task: | |
| 1. Read both diagnostic files to understand the failure | |
| 2. Analyze the failure — is it a code bug, config issue, Docker build error, or infrastructure problem? | |
| 3. If it's a code fix (typo, missing import, wrong env var, Docker config): | |
| - Fix the code locally | |
| - Stage ONLY the files you changed and commit with message: "fix: [ci-autofix] <description>" | |
| - Do NOT push — the pipeline will handle pushing and PR creation | |
| 4. If you cannot fix it with code changes, create a GitHub issue describing the problem | |
| Rules: | |
| - Do NOT SSH to production servers — no direct prod access | |
| - Do NOT run git push | |
| - Do NOT modify CI workflow files | |
| - Do NOT run git reset, git revert, or any destructive git commands | |
| - Focus on identifying and fixing the root cause in code | |
| PROMPT_END | |
| npx -y @anthropic-ai/claude-code@latest \ | |
| -p "$(cat /tmp/ci-prompt.txt)" \ | |
| --allowedTools "Bash,Read,Edit,Write,Glob,Grep" \ | |
| --max-turns 25 | |
| timeout-minutes: 10 | |
| - name: Check if fix was produced | |
| if: steps.guard.outputs.skip == 'false' | |
| id: check-fix | |
| run: | | |
| CURRENT_BRANCH=$(git branch --show-current) | |
| if git log --oneline "main..$CURRENT_BRANCH" 2>/dev/null | grep -q .; then | |
| echo "has_fix=true" >> "$GITHUB_OUTPUT" | |
| echo "branch=$CURRENT_BRANCH" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "has_fix=false" >> "$GITHUB_OUTPUT" | |
| echo "::warning::Claude Code did not produce a fix — manual intervention needed" | |
| fi | |
| - name: Create issue if no fix produced | |
| if: steps.guard.outputs.skip == 'false' && steps.check-fix.outputs.has_fix == 'false' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| gh issue create \ | |
| --title "Deploy failure needs manual fix ($(date +%Y-%m-%d))" \ | |
| --body "$(cat <<EOF | |
| ## Production deploy failure — auto-fix failed | |
| The self-heal agent could not produce a code fix for this deploy failure. | |
| **Failed run:** $RUN_URL | |
| **Commit:** ${GITHUB_SHA:0:7} | |
| > Production may be in a degraded state. Investigate promptly. | |
| EOF | |
| )" \ | |
| --label "bug,ci,urgent" | |
| - name: Push branch and create PR | |
| if: steps.guard.outputs.skip == 'false' && steps.check-fix.outputs.has_fix == 'true' | |
| env: | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| BRANCH="${{ steps.check-fix.outputs.branch }}" | |
| git push origin "$BRANCH" | |
| gh pr create \ | |
| --title "fix: [ci-autofix] Auto-fix for deploy failure" \ | |
| --body "$(cat <<EOF | |
| ## Auto-generated fix for deploy failure | |
| Claude Code analyzed the production deploy failure and produced this fix. | |
| **Source run:** $RUN_URL | |
| > Review and merge promptly, or rollback manually. | |
| 🤖 Generated with [Claude Code](https://claude.com/claude-code) | |
| EOF | |
| )" \ | |
| --base main \ | |
| --head "$BRANCH" | |
| # ─── Step 4: Create release ──────────────────────────────────────── | |
| create-release: | |
| name: Create Release | |
| runs-on: [self-hosted, local] | |
| needs: [detect-changes, deploy-preview, promote-to-prod] | |
| if: | | |
| always() && | |
| needs.promote-to-prod.result == 'success' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Create releases | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| SERVICES_TO_DEPLOY: ${{ needs.detect-changes.outputs.services_to_deploy }} | |
| BACKEND_VERSION: ${{ needs.deploy-preview.outputs.backend_version }} | |
| FRONTEND_VERSION: ${{ needs.deploy-preview.outputs.frontend_version }} | |
| BACKEND_CHANGED: ${{ needs.detect-changes.outputs.backend }} | |
| RADA_CHANGED: ${{ needs.detect-changes.outputs.rada }} | |
| OPENREYESTR_CHANGED: ${{ needs.detect-changes.outputs.openreyestr }} | |
| FRONTEND_CHANGED: ${{ needs.detect-changes.outputs.frontend }} | |
| run: | | |
| ANY_BACKEND=false | |
| if [ "$BACKEND_CHANGED" = "true" ] || [ "$RADA_CHANGED" = "true" ] || [ "$OPENREYESTR_CHANGED" = "true" ]; then | |
| ANY_BACKEND=true | |
| fi | |
| # --- Backend release --- | |
| if [ "$ANY_BACKEND" = "true" ]; then | |
| LAST_BE_TAG=$(git tag -l "v*" --sort=-version:refname | head -1) | |
| if [ -z "$LAST_BE_TAG" ]; then | |
| BE_COMMITS=$(git log --oneline --no-merges -50 -- packages/shared/ mcp_backend/ mcp_rada/ mcp_openreyestr/) | |
| else | |
| BE_COMMITS=$(git log "${LAST_BE_TAG}..HEAD" --oneline --no-merges -- packages/shared/ mcp_backend/ mcp_rada/ mcp_openreyestr/) | |
| fi | |
| BE_FEATURES=$(echo "$BE_COMMITS" | grep -iE "^[a-f0-9]+ feat(\(|:)" | sed 's/^[a-f0-9]* /- /' || true) | |
| BE_FIXES=$(echo "$BE_COMMITS" | grep -iE "^[a-f0-9]+ fix(\(|:)" | sed 's/^[a-f0-9]* /- /' || true) | |
| BE_OTHER=$(echo "$BE_COMMITS" | grep -viE "^[a-f0-9]+ (feat|fix)(\(|:)" | grep -v "^$" | sed 's/^[a-f0-9]* /- /' || true) | |
| BODY="## Backend Release\n" | |
| [ -n "$BE_FEATURES" ] && BODY="${BODY}\n### Features\n${BE_FEATURES}\n" | |
| [ -n "$BE_FIXES" ] && BODY="${BODY}\n### Fixes\n${BE_FIXES}\n" | |
| [ -n "$BE_OTHER" ] && BODY="${BODY}\n### Other\n${BE_OTHER}\n" | |
| BODY="${BODY}\n---\n**Commit:** ${GITHUB_SHA:0:7} | **Runner:** local" | |
| echo "Creating backend release ${BACKEND_VERSION}" | |
| git tag "${BACKEND_VERSION}" | |
| git push origin "${BACKEND_VERSION}" | |
| printf "%b" "$BODY" | gh release create "${BACKEND_VERSION}" \ | |
| --title "${BACKEND_VERSION} (Backend)" \ | |
| --notes-file - \ | |
| --latest | |
| fi | |
| # --- Frontend release --- | |
| if [ "$FRONTEND_CHANGED" = "true" ]; then | |
| LAST_FE_TAG=$(git tag -l "fe-v*" --sort=-version:refname | head -1) | |
| if [ -z "$LAST_FE_TAG" ]; then | |
| FE_COMMITS=$(git log --oneline --no-merges -50 -- lexwebapp/) | |
| else | |
| FE_COMMITS=$(git log "${LAST_FE_TAG}..HEAD" --oneline --no-merges -- lexwebapp/) | |
| fi | |
| FE_FEATURES=$(echo "$FE_COMMITS" | grep -iE "^[a-f0-9]+ feat(\(|:)" | sed 's/^[a-f0-9]* /- /' || true) | |
| FE_FIXES=$(echo "$FE_COMMITS" | grep -iE "^[a-f0-9]+ fix(\(|:)" | sed 's/^[a-f0-9]* /- /' || true) | |
| FE_OTHER=$(echo "$FE_COMMITS" | grep -viE "^[a-f0-9]+ (feat|fix)(\(|:)" | grep -v "^$" | sed 's/^[a-f0-9]* /- /' || true) | |
| BODY="## Frontend Release\n" | |
| [ -n "$FE_FEATURES" ] && BODY="${BODY}\n### Features\n${FE_FEATURES}\n" | |
| [ -n "$FE_FIXES" ] && BODY="${BODY}\n### Fixes\n${FE_FIXES}\n" | |
| [ -n "$FE_OTHER" ] && BODY="${BODY}\n### Other\n${FE_OTHER}\n" | |
| BODY="${BODY}\n---\n**Commit:** ${GITHUB_SHA:0:7} | **Runner:** local" | |
| echo "Creating frontend release ${FRONTEND_VERSION}" | |
| git tag "${FRONTEND_VERSION}" | |
| git push origin "${FRONTEND_VERSION}" | |
| # Frontend release is --latest only if no backend release was created | |
| LATEST_FLAG="" | |
| [ "$ANY_BACKEND" != "true" ] && LATEST_FLAG="--latest" | |
| printf "%b" "$BODY" | gh release create "${FRONTEND_VERSION}" \ | |
| --title "${FRONTEND_VERSION} (Frontend)" \ | |
| --notes-file - \ | |
| $LATEST_FLAG | |
| fi | |
| - name: Tag deploy marker | |
| run: | | |
| # Create a lightweight tag marking this commit as successfully deployed to prod. | |
| # detect-changes uses this to compare against the next deploy, ensuring no | |
| # changes are skipped between deploys. | |
| DEPLOY_TAG="deploy-prod-$(date -u +%Y%m%d-%H%M%S)" | |
| git tag "$DEPLOY_TAG" | |
| git push origin "$DEPLOY_TAG" | |
| echo "Created deploy marker tag: $DEPLOY_TAG" |