ROSS staging release-train debug #20
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ROSS staging release-train debug | |
| on: | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: ross-staging-debug | |
| cancel-in-progress: false | |
| jobs: | |
| debug: | |
| outputs: | |
| artifact_upload: ${{ steps.debug_upload.outcome }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 180 | |
| environment: staging-debug | |
| env: | |
| FLY_API_TOKEN: ${{ secrets.STAGING_FLY_API_TOKEN }} | |
| FLY_ORG: ${{ vars.STAGING_FLY_ORG }} | |
| ROSS_STAGING_SUPABASE_URL: ${{ secrets.STAGING_SUPABASE_URL }} | |
| ROSS_STAGING_SUPABASE_PUBLISHABLE_KEY: ${{ secrets.STAGING_SUPABASE_PUBLISHABLE_KEY }} | |
| ROSS_STAGING_SUPABASE_SECRET_KEY: ${{ secrets.STAGING_SUPABASE_SECRET_KEY }} | |
| ROSS_STAGING_S3_ENDPOINT_URL: ${{ secrets.STAGING_S3_ENDPOINT_URL }} | |
| ROSS_STAGING_S3_REGION: ${{ secrets.STAGING_S3_REGION }} | |
| ROSS_STAGING_S3_ACCESS_KEY_ID: ${{ secrets.STAGING_S3_ACCESS_KEY_ID }} | |
| ROSS_STAGING_S3_SECRET_ACCESS_KEY: ${{ secrets.STAGING_S3_SECRET_ACCESS_KEY }} | |
| ROSS_STAGING_S3_BUCKET: ${{ vars.STAGING_S3_BUCKET }} | |
| ROSS_PRODUCTION_FLY_ORG: ${{ vars.PRODUCTION_FLY_ORG }} | |
| ROSS_PRODUCTION_API_APP: ${{ vars.PRODUCTION_API_APP }} | |
| ROSS_PRODUCTION_WEB_APP: ${{ vars.PRODUCTION_WEB_APP }} | |
| ROSS_PRODUCTION_WORKER_APP: ${{ vars.PRODUCTION_WORKER_APP }} | |
| ROSS_PRODUCTION_SUPABASE_URL: ${{ vars.PRODUCTION_SUPABASE_URL }} | |
| ROSS_PRODUCTION_S3_ENDPOINT_URL: ${{ vars.PRODUCTION_S3_ENDPOINT_URL }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: { fetch-depth: 0 } | |
| - uses: ./.github/actions/setup-ross-node | |
| - uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1 | |
| with: { version: 0.4.49 } | |
| - name: Establish isolated names and fail closed on production overlap | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| mkdir -p artifacts/staging-debug/{commands,diagnostics} | |
| node scripts/validate-staging-debug.mjs | tee artifacts/staging-debug/isolation.txt >> "$GITHUB_ENV" | |
| env | sed -nE '/^(API|WEB|WORKER)_APP=/p' >> artifacts/staging-debug/isolation.txt | |
| - name: Read-only staging dependency preflights | |
| run: node scripts/staging-debug-preflight.mjs | |
| - name: Run complete repository gate | |
| run: npm run install:all && npm run check | |
| - name: Provision ephemeral staging applications | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| for app in "$API_APP" "$WEB_APP" "$WORKER_APP"; do | |
| flyctl apps create "$app" --org "$FLY_ORG" --yes 2>&1 | tee "artifacts/staging-debug/commands/provision-${app}.log" | |
| done | |
| - name: Build candidate images in the isolated registry namespaces | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| export RELEASE_IMAGE_API_APP="$API_APP" RELEASE_IMAGE_WEB_APP="$WEB_APP" RELEASE_IMAGE_WORKER_APP="$WORKER_APP" | |
| export RELEASE_RUNTIME_API_APP="$API_APP" RELEASE_RUNTIME_WEB_APP="$WEB_APP" RELEASE_REHEARSAL_API_APP="$API_APP" | |
| export RELEASE_SIGNUPS_ENABLED=false | |
| export ROSS_SUPABASE_URL="$ROSS_STAGING_SUPABASE_URL" ROSS_SUPABASE_PUBLISHABLE_KEY="$ROSS_STAGING_SUPABASE_PUBLISHABLE_KEY" | |
| export ROSS_RELEASE_ID="staging-debug-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" | |
| echo "ROSS_STAGING_DEBUG_RELEASE_ID=$ROSS_RELEASE_ID" >> "$GITHUB_ENV" | |
| bash scripts/build-release-train-images.sh | |
| - name: Smoke exact candidate web image before deployment | |
| shell: bash | |
| run: bash scripts/smoke-staging-web-image.sh | |
| - name: Configure staging-only service dependencies | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| worker_secret="$(openssl rand -hex 32)"; echo "::add-mask::$worker_secret" | |
| signing_secret="$(openssl rand -hex 32)"; echo "::add-mask::$signing_secret" | |
| flyctl secrets set --stage --app "$WORKER_APP" "FILE_WORKER_SHARED_SECRET=$worker_secret" "FILE_WORKER_STORAGE_ORIGINS=$ROSS_STAGING_S3_ENDPOINT_URL" | |
| flyctl secrets set --stage --app "$API_APP" "SUPABASE_URL=$ROSS_STAGING_SUPABASE_URL" "SUPABASE_SECRET_KEY=$ROSS_STAGING_SUPABASE_SECRET_KEY" "R2_ENDPOINT_URL=$ROSS_STAGING_S3_ENDPOINT_URL" "R2_REGION=$ROSS_STAGING_S3_REGION" "R2_ACCESS_KEY_ID=$ROSS_STAGING_S3_ACCESS_KEY_ID" "R2_SECRET_ACCESS_KEY=$ROSS_STAGING_S3_SECRET_ACCESS_KEY" "R2_BUCKET_NAME=$ROSS_STAGING_S3_BUCKET" "FILE_WORKER_URL=http://${WORKER_APP}.flycast" "FILE_WORKER_SHARED_SECRET=$worker_secret" "DOWNLOAD_SIGNING_SECRET=$signing_secret" "USER_API_KEYS_ENCRYPTION_SECRET=$signing_secret" "MCP_CONNECTORS_ENCRYPTION_SECRET=$signing_secret" "ROSS_ENV=staging" "ROSS_HOSTED_MODE=controlled-beta" "HOSTED_MODEL_PROVIDERS=openai" "ROSS_RUNTIME_RELEASE_ID=$ROSS_STAGING_DEBUG_RELEASE_ID" "ROSS_DISABLE_DOCUMENT_SCAN_DISPATCHER=true" "ROSS_UPLOAD_SCAN_REQUIRED=false" "CORS_ALLOWED_ORIGINS=https://${WEB_APP}.fly.dev" "FRONTEND_URL=https://${WEB_APP}.fly.dev" "API_PUBLIC_URL=https://${API_APP}.fly.dev" | |
| flyctl secrets set --stage --app "$WEB_APP" "ROSS_RUNTIME_API_BASE_URL=https://${API_APP}.fly.dev" "ROSS_RUNTIME_APP_URL=https://${WEB_APP}.fly.dev" "ROSS_RUNTIME_SIGNUPS_ENABLED=false" "ROSS_RUNTIME_ENVIRONMENT=rehearsal" "ROSS_RUNTIME_RELEASE_ID=$ROSS_STAGING_DEBUG_RELEASE_ID" | |
| - name: Deploy and diagnose worker stage | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| bash scripts/fly-deploy-with-retry.sh . --config deploy/fly/rehearsal-file-worker.toml --app "$WORKER_APP" --image "$CANDIDATE_WORKER_IMAGE" --ha=false --yes --flycast --no-public-ips 2>&1 | tee artifacts/staging-debug/commands/worker.log | |
| flyctl status --app "$WORKER_APP" --json > artifacts/staging-debug/diagnostics/worker-candidate-status.json | |
| flyctl logs --app "$WORKER_APP" --no-tail > artifacts/staging-debug/diagnostics/worker-candidate.log 2>&1 || true | |
| - name: Deploy and diagnose API stage | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| bash scripts/fly-deploy-with-retry.sh . --config deploy/fly/rehearsal-api.toml --app "$API_APP" --image "$CANDIDATE_API_IMAGE" --ha=false --yes --flycast --no-public-ips 2>&1 | tee artifacts/staging-debug/commands/api.log | |
| flyctl status --app "$API_APP" --json > artifacts/staging-debug/diagnostics/api-candidate-status.json | |
| flyctl logs --app "$API_APP" --no-tail > artifacts/staging-debug/diagnostics/api-candidate.log 2>&1 || true | |
| - name: Deploy and diagnose web stage | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| bash scripts/fly-deploy-with-retry.sh . --config deploy/fly/rehearsal-frontend.toml --app "$WEB_APP" --image "$CANDIDATE_WEB_IMAGE" --ha=false --yes --flycast --no-public-ips 2>&1 | tee artifacts/staging-debug/commands/web.log | |
| flyctl status --app "$WEB_APP" --json > artifacts/staging-debug/diagnostics/web-candidate-status.json | |
| flyctl logs --app "$WEB_APP" --no-tail > artifacts/staging-debug/diagnostics/web-candidate.log 2>&1 || true | |
| - name: Run exact complete release-train integration probe | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| node scripts/run-staging-debug-probe.mjs rehearsal 2>&1 | tee artifacts/staging-debug/diagnostics/full-probe-before-failure.log | |
| printf '{"outcome":"passed","image":"%s","port":3000,"routes":["/login","/api/runtime-config"]}\n' "$CANDIDATE_WEB_IMAGE" > artifacts/staging-debug/candidate-deployment-result.json | |
| - name: Inject a genuine deployment failure and restore baseline | |
| shell: bash | |
| run: bash scripts/staging-debug-lifecycle.sh inject-failure-and-rollback | |
| - name: Verify full integration recovery after rollback | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| node scripts/run-staging-debug-probe.mjs rehearsal 2>&1 | tee artifacts/staging-debug/diagnostics/full-probe-after-rollback.log | |
| printf '{"outcome":"passed","runtimeEnvironment":"rehearsal"}\n' > artifacts/staging-debug/post-restoration-probe-result.json | |
| - name: Collect failure diagnostics | |
| if: failure() | |
| shell: bash | |
| run: | | |
| for app in "${API_APP:-}" "${WEB_APP:-}" "${WORKER_APP:-}"; do | |
| [ -n "$app" ] || continue | |
| flyctl status --app "$app" --json > "artifacts/staging-debug/diagnostics/${app}-failure-status.json" 2>&1 || true | |
| flyctl logs --app "$app" --no-tail > "artifacts/staging-debug/diagnostics/${app}-failure.log" 2>&1 || true | |
| done | |
| - name: Immediate defensive cleanup in approved staging context | |
| id: immediate_cleanup | |
| if: always() | |
| shell: bash | |
| env: | |
| ROSS_STAGING_DEBUG_CLEANUP_RESULT: artifacts/staging-debug/immediate-cleanup-result.json | |
| run: bash scripts/staging-debug-lifecycle.sh cleanup | |
| - name: Upload debug evidence for independent cleanup job | |
| id: debug_upload | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: ross-staging-debug-working-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: artifacts/staging-debug | |
| if-no-files-found: warn | |
| retention-days: 1 | |
| cleanup: | |
| name: Independent cleanup and final evidence | |
| needs: debug | |
| if: always() | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| env: | |
| # Repository secret: narrowly scoped to destroy staging-debug apps in the | |
| # dedicated staging organization. It must not require environment review. | |
| FLY_API_TOKEN: ${{ secrets.STAGING_FLY_CLEANUP_TOKEN }} | |
| API_APP: ross-api-debug-${{ github.run_id }}-${{ github.run_attempt }} | |
| WEB_APP: ross-web-debug-${{ github.run_id }}-${{ github.run_attempt }} | |
| WORKER_APP: ross-worker-debug-${{ github.run_id }}-${{ github.run_attempt }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1 | |
| with: { version: 0.4.49 } | |
| - name: Recover debug evidence | |
| id: download | |
| continue-on-error: true | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: ross-staging-debug-working-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: artifacts/staging-debug | |
| - name: Destroy deterministic ephemeral staging resources | |
| id: cleanup | |
| shell: bash | |
| run: bash scripts/staging-debug-lifecycle.sh cleanup | |
| - name: Record outcomes only after successful cleanup | |
| if: steps.cleanup.outcome == 'success' | |
| shell: bash | |
| run: | | |
| mkdir -p artifacts/staging-debug | |
| printf '{"outcome":"%s"}\n' "${{ needs.debug.result }}" > artifacts/staging-debug/debug-job-result.json | |
| printf '{"outcome":"%s"}\n' "${{ steps.download.outcome }}" > artifacts/staging-debug/evidence-download-result.json | |
| printf '{"outcome":"%s","productionPromotion":false,"stagingWorkflowRerun":false}\n' "${{ needs.debug.result }}" > artifacts/staging-debug/result.json | |
| printf '{"outcome":"%s"}\n' "${{ needs.debug.outputs.artifact_upload }}" > artifacts/staging-debug/debug-artifact-upload-result.json | |
| - name: Upload complete staging-debug evidence | |
| id: final_upload | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: ross-staging-debug-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: artifacts/staging-debug | |
| if-no-files-found: error | |
| retention-days: 30 | |
| - name: Record artifact upload outcome | |
| if: always() | |
| shell: bash | |
| run: | | |
| printf '### Staging-debug outcomes\n* Debug job: `%s`\n* Cleanup: `%s`\n* Artifact upload: `%s`\n' \ | |
| "${{ needs.debug.result }}" "${{ steps.cleanup.outcome }}" "${{ steps.final_upload.outcome }}" >> "$GITHUB_STEP_SUMMARY" | |
| test "${{ steps.cleanup.outcome }}" = success | |
| test "${{ steps.final_upload.outcome }}" = success | |
| test "${{ needs.debug.result }}" = success | |
| # Intentionally no production environment, production credentials, promotion input, or promotion job. |