From 16b86a075539d6371a9097a43d6dbac89f1051c1 Mon Sep 17 00:00:00 2001 From: Charles Nykamp <16085675+cqnykamp@users.noreply.github.com> Date: Fri, 17 Jul 2026 19:42:46 -0500 Subject: [PATCH 1/3] feat(ci): deploy to prod automatically on merge to main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prod previously waited for CI to re-run on main and then blocked on a manual approval in the prod environment. Branch protection requires PRs to be up to date with main, so the merged tree is content-identical to what PR checks already tested — the re-run and the approval added latency without adding safety. - Trigger prod-deploy directly on push to main - Drop the approve job; nothing references the prod environment's protection rules anymore - Queue concurrent deploys instead of canceling in-progress ones, so a second merge can never kill a live deploy mid-rollout (GitHub keeps only the newest pending run, which contains all earlier merges) - Record deployments on the prod GitHub Environment via the Deployments API, mirroring dev3: in_progress at start, success/failure rollup after all three deploys, best-effort so bookkeeping never blocks a deploy - Remove the tag-prod job; the deployment record supersedes the force-pushed prod tag Co-Authored-By: Claude Fable 5 --- .github/workflows/prod-deploy.yml | 105 ++++++++++++++++++------------ 1 file changed, 64 insertions(+), 41 deletions(-) diff --git a/.github/workflows/prod-deploy.yml b/.github/workflows/prod-deploy.yml index 5af7a42f6..e30f7dd14 100644 --- a/.github/workflows/prod-deploy.yml +++ b/.github/workflows/prod-deploy.yml @@ -1,37 +1,57 @@ name: Prod Deploy on: - workflow_run: - workflows: ["CI"] - types: [completed] + push: + branches: ["main"] permissions: contents: read id-token: write + deployments: write +# Never cancel a deploy that has started: killing a run mid-S3-sync or +# mid-ECS-rollout (or mid-rollback) can leave prod half-updated. Queued +# runs are safe to supersede — GitHub keeps only the newest pending run, +# and branch protection guarantees it contains every earlier merge. concurrency: group: prod-deploy - cancel-in-progress: true + cancel-in-progress: false jobs: - approve: - if: >- - ${{ - github.event.workflow_run.conclusion == 'success' && - github.event.workflow_run.head_branch == 'main' - }} + # Open a GitHub deployment so the repo's prod Environment always shows + # what is currently live. Uses the Deployments API rather than an + # `environment:` key so environment protection rules can never gate the + # deploy, and no deploy job depends on this one — bookkeeping must never + # block or skip an actual deploy. + create-deployment: runs-on: ubuntu-latest - environment: - name: prod + outputs: + deployment_id: ${{ steps.deployment.outputs.deployment_id }} steps: - - name: Queue prod deploy - run: echo "Preparing prod deploy for ${{ github.event.workflow_run.head_sha }}" + - name: Open GitHub deployment + id: deployment + env: + GH_TOKEN: ${{ github.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + payload=$(jq -n --arg ref "$GITHUB_SHA" \ + '{ref: $ref, environment: "prod", required_contexts: [], auto_merge: false, production_environment: true, description: "prod deploy"}') + id=$(gh api -X POST "repos/${GITHUB_REPOSITORY}/deployments" --input - -q .id <<<"$payload" 2>/dev/null || true) + if [[ -n "$id" ]]; then + echo "deployment_id=$id" >> "$GITHUB_OUTPUT" + gh api -X POST "repos/${GITHUB_REPOSITORY}/deployments/${id}/statuses" \ + -f state=in_progress \ + -f environment=prod \ + -f environment_url=https://doenet.org \ + -f log_url="$RUN_URL" >/dev/null 2>&1 || true + else + echo "::warning::could not create prod deployment record; deploy continues" + fi deploy-backend: - needs: approve uses: ./.github/workflows/reusable-deploy-backend.yml with: - deploy_ref: ${{ github.event.workflow_run.head_sha }} + deploy_ref: ${{ github.sha }} aws_region: us-east-2 ecr_repository: prod/doenet ecs_cluster: prod @@ -42,10 +62,9 @@ jobs: secrets: inherit deploy-frontend: - needs: approve uses: ./.github/workflows/reusable-deploy-frontend.yml with: - deploy_ref: ${{ github.event.workflow_run.head_sha }} + deploy_ref: ${{ github.sha }} aws_region: us-east-1 gha_role: arn:aws:iam::350646758180:role/github-actions-prod aws_account_id: "350646758180" @@ -54,10 +73,9 @@ jobs: secrets: inherit deploy-site: - needs: approve uses: ./.github/workflows/reusable-deploy-site.yml with: - deploy_ref: ${{ github.event.workflow_run.head_sha }} + deploy_ref: ${{ github.sha }} aws_region: us-east-1 gha_role: arn:aws:iam::350646758180:role/github-actions-prod aws_account_id: "350646758180" @@ -65,27 +83,32 @@ jobs: build_mode: prod secrets: inherit - tag-prod: - needs: - - approve - - deploy-backend - - deploy-frontend - - deploy-site + record-deployment: + needs: [create-deployment, deploy-backend, deploy-frontend, deploy-site] + if: always() && needs.create-deployment.outputs.deployment_id != '' runs-on: ubuntu-latest - permissions: - contents: write steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - fetch-depth: 0 - ref: ${{ github.event.workflow_run.head_sha }} - - - name: Update prod tag - shell: bash + - name: Finalize GitHub deployment status + env: + GH_TOKEN: ${{ github.token }} + ID: ${{ needs.create-deployment.outputs.deployment_id }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + B: ${{ needs.deploy-backend.result }} + F: ${{ needs.deploy-frontend.result }} + S: ${{ needs.deploy-site.result }} run: | - DEPLOY_REF="${{ github.event.workflow_run.head_sha }}" - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git tag -f prod "$DEPLOY_REF" - git push origin refs/tags/prod --force + if [[ "$B" == "success" && "$F" == "success" && "$S" == "success" ]]; then + state=success + desc="deployed to prod" + else + state=failure + desc="prod deploy failed (backend:$B frontend:$F site:$S)" + fi + # On a success status GitHub auto-inactivates the previous prod + # deployment, so the Environments tab reflects what is currently live. + gh api -X POST "repos/${GITHUB_REPOSITORY}/deployments/${ID}/statuses" \ + -f state="$state" \ + -f description="$desc" \ + -f environment=prod \ + -f environment_url=https://doenet.org \ + -f log_url="$RUN_URL" >/dev/null From fb46fc2f91982b9578cae203fc25ea290e8999ea Mon Sep 17 00:00:00 2001 From: Charles Nykamp <16085675+cqnykamp@users.noreply.github.com> Date: Fri, 17 Jul 2026 19:51:12 -0500 Subject: [PATCH 2/3] docs(infra): document prod rollback paths With deploys now automatic on merge, no human is in the loop when prod breaks. Write down the two rollback paths (revert PR, re-run last green Prod Deploy), the migration caveat that applies to both, and how to verify what prod is actually serving. Co-Authored-By: Claude Fable 5 --- infra/README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/infra/README.md b/infra/README.md index afaafb2d0..6f7a2e052 100644 --- a/infra/README.md +++ b/infra/README.md @@ -52,3 +52,26 @@ each is stamped with its ref and commit at build time: - `https://dev3.doenet.org/api/health` → `{ status, version: { ref, sha, builtAt } }` (backend) - `https://dev3.doenet.org/version.json` → `{ ref, sha, builtAt }` (frontend app) + +## Rolling back prod + +Every merge to `main` deploys to prod automatically, so `main` is the source of +truth for what prod should be running. There are two ways back: + +- **Revert the PR** — the real fix. Prod redeploys once the revert's checks + pass. Use this unless prod is badly broken right now. +- **Re-run the last good deploy** — the emergency stopgap while a revert's CI + runs. In Actions → `Prod Deploy`, find the last green run and click + "Re-run all jobs"; it redeploys that run's SHA. This does not change `main`, + so the next merge ships whatever is on `main` again — always follow up with + a revert. + +Neither path undoes database migrations: `prisma migrate deploy` runs at +container boot and is never rolled back, so after a rollback the old code runs +against the new schema. This is why migrations must be backward compatible +(expand/contract: add columns/tables in one release, remove in a later one). + +Backend deploys that fail to boot or pass health checks roll themselves back +(see `update-cluster-with-rollback`); manual rollback is for code that deploys +healthy but is functionally broken. Confirm what prod is actually running at +`https://doenet.org/api/health` and `https://doenet.org/version.json`. From e6487691f9e33eb9ec0ef73a227628e967f6c980 Mon Sep 17 00:00:00 2001 From: Charles Nykamp <16085675+cqnykamp@users.noreply.github.com> Date: Fri, 17 Jul 2026 20:02:01 -0500 Subject: [PATCH 3/3] refactor(ci): deduplicate deploy workflows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three deploy entry points (push to main -> dev3, /deploy-dev PR command, push to main -> prod) each repeated two things: the backend/frontend/site fan-out with its per-environment constants, and the GitHub deployment bookkeeping bash, which had already drifted between copies (lights-out handling, prod badge). - reusable-deploy-env.yml: single definition of a deploy — resolves per-environment constants (account, role, ECR repo, cluster, SNS topic, tag) from env_name and fans out to the three component workflows; exposes per-component results as outputs - actions/open-deployment: open a deployment record and mark it in_progress; best-effort, never fails the job - actions/finalize-deployment: set the final status from the three component results, with an optional lights-parameter input that records success-while-asleep as inactive (dev3 only) - callers shrink to trigger/gate logic plus one deploy job each No behavior change intended for any of the three entry points. Co-Authored-By: Claude Fable 5 --- .../actions/finalize-deployment/action.yml | 74 ++++++++++ .github/actions/open-deployment/action.yml | 58 ++++++++ .github/workflows/dev-deploy-pr.yml | 135 ++++++------------ .github/workflows/dev-deploy.yml | 128 +++++------------ .github/workflows/prod-deploy.yml | 103 ++++--------- .github/workflows/reusable-deploy-env.yml | 103 +++++++++++++ 6 files changed, 337 insertions(+), 264 deletions(-) create mode 100644 .github/actions/finalize-deployment/action.yml create mode 100644 .github/actions/open-deployment/action.yml create mode 100644 .github/workflows/reusable-deploy-env.yml diff --git a/.github/actions/finalize-deployment/action.yml b/.github/actions/finalize-deployment/action.yml new file mode 100644 index 000000000..41d777a74 --- /dev/null +++ b/.github/actions/finalize-deployment/action.yml @@ -0,0 +1,74 @@ +name: Finalize GitHub deployment +description: >- + Set the final status on a deployment record from the three component + deploy results. With lights-parameter set, a successful deploy while the + environment is scaled to zero is recorded as inactive instead of success. + +inputs: + deployment-id: + description: Deployment id returned by open-deployment + required: true + environment: + description: GitHub environment name (e.g. dev3, prod) + required: true + environment-url: + description: Public URL shown on the deployment record + required: true + backend-result: + description: Result of the backend deploy job + required: true + frontend-result: + description: Result of the frontend deploy job + required: true + site-result: + description: Result of the site deploy job + required: true + lights-parameter: + description: >- + SSM parameter holding the environment's lights state, for environments + that scale to zero. Requires AWS credentials to already be configured; + any read failure is treated as lights on. Empty disables the check. + required: false + default: "" + +runs: + using: composite + steps: + - name: Set final deployment status + shell: bash + env: + GH_TOKEN: ${{ github.token }} + DEPLOYMENT_ID: ${{ inputs.deployment-id }} + ENVIRONMENT: ${{ inputs.environment }} + ENVIRONMENT_URL: ${{ inputs.environment-url }} + LIGHTS_PARAMETER: ${{ inputs.lights-parameter }} + B: ${{ inputs.backend-result }} + F: ${{ inputs.frontend-result }} + S: ${{ inputs.site-result }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + if [[ "$B" == "success" && "$F" == "success" && "$S" == "success" ]]; then + lights=unknown + if [[ -n "$LIGHTS_PARAMETER" ]]; then + lights=$(aws ssm get-parameter --name "$LIGHTS_PARAMETER" \ + --query Parameter.Value --output text 2>/dev/null || echo unknown) + fi + if [[ "$lights" == "off" ]]; then + state=inactive + desc="$ENVIRONMENT is asleep (lights off); this ref serves on next wake" + else + state=success + desc="deployed to $ENVIRONMENT" + fi + else + state=failure + desc="$ENVIRONMENT deploy failed (backend:$B frontend:$F site:$S)" + fi + # On a success status GitHub auto-inactivates the previous deployment + # for this environment, so the Environments tab reflects what is live. + gh api -X POST "repos/${GITHUB_REPOSITORY}/deployments/${DEPLOYMENT_ID}/statuses" \ + -f state="$state" \ + -f description="$desc" \ + -f environment="$ENVIRONMENT" \ + -f environment_url="$ENVIRONMENT_URL" \ + -f log_url="$RUN_URL" >/dev/null diff --git a/.github/actions/open-deployment/action.yml b/.github/actions/open-deployment/action.yml new file mode 100644 index 000000000..7986182cf --- /dev/null +++ b/.github/actions/open-deployment/action.yml @@ -0,0 +1,58 @@ +name: Open GitHub deployment +description: >- + Open a GitHub deployment record for an environment and mark it + in_progress. Best-effort: never fails the job — an empty deployment_id + output means no record was created, and callers should skip finalizing. + +inputs: + environment: + description: GitHub environment name (e.g. dev3, prod) + required: true + environment-url: + description: Public URL shown on the deployment record + required: true + ref: + description: Git ref or SHA the deployment is attributed to + required: true + description: + description: Short description shown on the deployment record + required: true + production-environment: + description: Whether GitHub badges this environment as production (JSON boolean) + required: false + default: "false" + +outputs: + deployment_id: + description: The created deployment id, or empty if creation failed + value: ${{ steps.open.outputs.deployment_id }} + +runs: + using: composite + steps: + - name: Open deployment and mark in_progress + id: open + shell: bash + env: + GH_TOKEN: ${{ github.token }} + ENVIRONMENT: ${{ inputs.environment }} + ENVIRONMENT_URL: ${{ inputs.environment-url }} + REF: ${{ inputs.ref }} + DESCRIPTION: ${{ inputs.description }} + PRODUCTION_ENVIRONMENT: ${{ inputs.production-environment }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + payload=$(jq -n --arg ref "$REF" --arg env "$ENVIRONMENT" \ + --arg desc "$DESCRIPTION" --argjson prod "$PRODUCTION_ENVIRONMENT" \ + '{ref: $ref, environment: $env, required_contexts: [], auto_merge: false, production_environment: $prod, description: $desc}') + id=$(gh api -X POST "repos/${GITHUB_REPOSITORY}/deployments" --input - -q .id <<<"$payload" 2>/dev/null || true) + if [[ -n "$id" ]]; then + echo "deployment_id=$id" >> "$GITHUB_OUTPUT" + gh api -X POST "repos/${GITHUB_REPOSITORY}/deployments/${id}/statuses" \ + -f state=in_progress \ + -f environment="$ENVIRONMENT" \ + -f environment_url="$ENVIRONMENT_URL" \ + -f log_url="$RUN_URL" >/dev/null 2>&1 || true + else + echo "::warning::could not create $ENVIRONMENT deployment record; deploy continues" + fi diff --git a/.github/workflows/dev-deploy-pr.yml b/.github/workflows/dev-deploy-pr.yml index d0a0174a5..b73221620 100644 --- a/.github/workflows/dev-deploy-pr.yml +++ b/.github/workflows/dev-deploy-pr.yml @@ -144,85 +144,58 @@ jobs: echo "blocked=true" >> "$GITHUB_OUTPUT" fi - - name: Open GitHub deployment - id: deployment + - name: Resolve PR head commit + id: head if: steps.cmd.outputs.run == 'true' && steps.check.outputs.authorized == 'true' && steps.merge.outputs.blocked != 'true' env: GH_TOKEN: ${{ github.token }} PR: ${{ github.event.issue.number }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | # Attribute the deployment to the PR head commit so GitHub links it to # the PR (the built code may be refs/pull/N/merge, but the head SHA is # the real commit that shows up on the PR timeline). # Best-effort: deployment bookkeeping must never block an actual deploy. head_sha=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR}" -q .head.sha 2>/dev/null || true) - if [[ -n "$head_sha" ]]; then - payload=$(jq -n --arg ref "$head_sha" \ - '{ref: $ref, environment: "dev3", required_contexts: [], auto_merge: false, description: "dev3 deploy (PR command)"}') - id=$(gh api -X POST "repos/${GITHUB_REPOSITORY}/deployments" --input - -q .id <<<"$payload" 2>/dev/null || true) - fi - if [[ -n "${id:-}" ]]; then - echo "deployment_id=$id" >> "$GITHUB_OUTPUT" - gh api -X POST "repos/${GITHUB_REPOSITORY}/deployments/${id}/statuses" \ - -f state=in_progress \ - -f environment=dev3 \ - -f environment_url=https://dev3.doenet.org \ - -f log_url="$RUN_URL" >/dev/null 2>&1 || true - else - echo "::warning::could not create dev3 deployment record; deploy continues" + if [[ -z "$head_sha" ]]; then + echo "::warning::could not resolve PR head sha; deploying without a dev3 deployment record" fi + echo "sha=$head_sha" >> "$GITHUB_OUTPUT" - deploy-backend: - needs: gate - if: needs.gate.outputs.go == 'true' - uses: ./.github/workflows/reusable-deploy-backend.yml - with: - deploy_ref: ${{ needs.gate.outputs.deploy_ref }} - aws_region: us-east-2 - ecr_repository: dev3/doenet - ecs_cluster: dev3 - ecs_service: doenet-FARGATE - gha_role: arn:aws:iam::568298704244:role/github-actions-dev3 - sns_topic: arn:aws:sns:us-east-2:568298704244:dev3-critical-cloudwatch-notifications - tag_name: latest - secrets: inherit + # Local composite actions need the repo on disk. issue_comment events + # check out the default branch, so this is trusted code, not the PR's. + - name: Checkout repository + if: steps.head.outputs.sha != '' + uses: actions/checkout@v6 - deploy-frontend: - needs: gate - if: needs.gate.outputs.go == 'true' - uses: ./.github/workflows/reusable-deploy-frontend.yml - with: - deploy_ref: ${{ needs.gate.outputs.deploy_ref }} - aws_region: us-east-1 - gha_role: arn:aws:iam::568298704244:role/github-actions-dev3 - aws_account_id: "568298704244" - env_name: dev3 - build_mode: dev3 - secrets: inherit + - name: Open GitHub deployment + id: deployment + if: steps.head.outputs.sha != '' + uses: ./.github/actions/open-deployment + with: + environment: dev3 + environment-url: https://dev3.doenet.org + ref: ${{ steps.head.outputs.sha }} + description: dev3 deploy (PR command) - deploy-site: + deploy: needs: gate if: needs.gate.outputs.go == 'true' - uses: ./.github/workflows/reusable-deploy-site.yml + uses: ./.github/workflows/reusable-deploy-env.yml with: deploy_ref: ${{ needs.gate.outputs.deploy_ref }} - aws_region: us-east-1 - gha_role: arn:aws:iam::568298704244:role/github-actions-dev3 - aws_account_id: "568298704244" env_name: dev3 - build_mode: dev3 secrets: inherit notify: - needs: [gate, deploy-backend, deploy-frontend, deploy-site] + needs: [gate, deploy] if: always() && needs.gate.outputs.go == 'true' runs-on: ubuntu-latest steps: - # Read-only creds so we can ask whether dev3 is actually awake before - # claiming a deployment is live. Best-effort: never fails the job. + # Read-only creds so finalize-deployment can ask whether dev3 is + # actually awake before claiming a deployment is live. Best-effort: + # never fails the job; without creds the lights read falls back to + # treating dev3 as awake. - name: Configure AWS credentials - id: aws if: needs.gate.outputs.deployment_id != '' continue-on-error: true uses: aws-actions/configure-aws-credentials@v6 @@ -232,45 +205,21 @@ jobs: role-session-name: dev3-deploy-status aws-region: us-east-2 + - name: Checkout repository + if: needs.gate.outputs.deployment_id != '' + uses: actions/checkout@v6 + - name: Finalize GitHub deployment status if: needs.gate.outputs.deployment_id != '' - env: - GH_TOKEN: ${{ github.token }} - ID: ${{ needs.gate.outputs.deployment_id }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - HAVE_AWS: ${{ steps.aws.outcome }} - B: ${{ needs.deploy-backend.result }} - F: ${{ needs.deploy-frontend.result }} - S: ${{ needs.deploy-site.result }} - run: | - if [[ "$B" == "success" && "$F" == "success" && "$S" == "success" ]]; then - # dev3 scales to zero on the lights-out schedule (and the backend - # deploy is a no-op while it's asleep), so reflect whether it's - # actually serving rather than always claiming success. - lights=unknown - if [[ "$HAVE_AWS" == "success" ]]; then - lights=$(aws ssm get-parameter --name /dev3/doenet/lights \ - --query Parameter.Value --output text 2>/dev/null || echo unknown) - fi - if [[ "$lights" == "off" ]]; then - state=inactive - desc="dev3 is asleep (lights off); this ref serves on next wake" - else - state=success - desc="deployed to dev3" - fi - else - state=failure - desc="dev3 deploy failed (backend:$B frontend:$F site:$S)" - fi - # On a success status GitHub auto-inactivates the previous dev3 - # deployment, so the Environments tab reflects what is currently live. - gh api -X POST "repos/${GITHUB_REPOSITORY}/deployments/${ID}/statuses" \ - -f state="$state" \ - -f description="$desc" \ - -f environment=dev3 \ - -f environment_url=https://dev3.doenet.org \ - -f log_url="$RUN_URL" >/dev/null + uses: ./.github/actions/finalize-deployment + with: + deployment-id: ${{ needs.gate.outputs.deployment_id }} + environment: dev3 + environment-url: https://dev3.doenet.org + backend-result: ${{ needs.deploy.outputs.backend_result }} + frontend-result: ${{ needs.deploy.outputs.frontend_result }} + site-result: ${{ needs.deploy.outputs.site_result }} + lights-parameter: /dev3/doenet/lights - name: Comment result on the PR env: @@ -278,9 +227,9 @@ jobs: PR: ${{ github.event.issue.number }} REF: ${{ needs.gate.outputs.deploy_ref }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - B: ${{ needs.deploy-backend.result }} - F: ${{ needs.deploy-frontend.result }} - S: ${{ needs.deploy-site.result }} + B: ${{ needs.deploy.outputs.backend_result }} + F: ${{ needs.deploy.outputs.frontend_result }} + S: ${{ needs.deploy.outputs.site_result }} run: | if [[ "$B" == "success" && "$F" == "success" && "$S" == "success" ]]; then printf '%s\n' \ diff --git a/.github/workflows/dev-deploy.yml b/.github/workflows/dev-deploy.yml index c381ba108..4be262a6e 100644 --- a/.github/workflows/dev-deploy.yml +++ b/.github/workflows/dev-deploy.yml @@ -21,12 +21,13 @@ concurrency: jobs: # Resolve the ref once and open a GitHub deployment so the repo's dev3 - # Environment always shows what is currently live on dev3. + # Environment always shows what is currently live on dev3. Bookkeeping is + # best-effort and must never block an actual deploy. create-deployment: runs-on: ubuntu-latest outputs: deploy_ref: ${{ steps.ref.outputs.deploy_ref }} - deployment_id: ${{ steps.deployment.outputs.deployment_id }} + deployment_id: ${{ steps.open.outputs.deployment_id }} steps: - name: Resolve deploy ref id: ref @@ -41,75 +42,36 @@ jobs: echo "deploy_ref=$SHA" >> "$GITHUB_OUTPUT" fi - - name: Open GitHub deployment - id: deployment - env: - GH_TOKEN: ${{ github.token }} - REF: ${{ steps.ref.outputs.deploy_ref }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: | - # Best-effort: deployment bookkeeping must never block an actual deploy. - payload=$(jq -n --arg ref "$REF" \ - '{ref: $ref, environment: "dev3", required_contexts: [], auto_merge: false, description: "dev3 deploy"}') - id=$(gh api -X POST "repos/${GITHUB_REPOSITORY}/deployments" --input - -q .id <<<"$payload" 2>/dev/null || true) - if [[ -n "$id" ]]; then - echo "deployment_id=$id" >> "$GITHUB_OUTPUT" - gh api -X POST "repos/${GITHUB_REPOSITORY}/deployments/${id}/statuses" \ - -f state=in_progress \ - -f environment=dev3 \ - -f environment_url=https://dev3.doenet.org \ - -f log_url="$RUN_URL" >/dev/null 2>&1 || true - else - echo "::warning::could not create dev3 deployment record; deploy continues" - fi - - deploy-backend: - needs: create-deployment - uses: ./.github/workflows/reusable-deploy-backend.yml - with: - deploy_ref: ${{ needs.create-deployment.outputs.deploy_ref }} - aws_region: us-east-2 - ecr_repository: dev3/doenet - ecs_cluster: dev3 - ecs_service: doenet-FARGATE - gha_role: arn:aws:iam::568298704244:role/github-actions-dev3 - sns_topic: arn:aws:sns:us-east-2:568298704244:dev3-critical-cloudwatch-notifications - tag_name: latest - secrets: inherit + - name: Checkout repository + uses: actions/checkout@v6 - deploy-frontend: - needs: create-deployment - uses: ./.github/workflows/reusable-deploy-frontend.yml - with: - deploy_ref: ${{ needs.create-deployment.outputs.deploy_ref }} - aws_region: us-east-1 - gha_role: arn:aws:iam::568298704244:role/github-actions-dev3 - aws_account_id: "568298704244" - env_name: dev3 - build_mode: dev3 - secrets: inherit + - name: Open GitHub deployment + id: open + uses: ./.github/actions/open-deployment + with: + environment: dev3 + environment-url: https://dev3.doenet.org + ref: ${{ steps.ref.outputs.deploy_ref }} + description: dev3 deploy - deploy-site: + deploy: needs: create-deployment - uses: ./.github/workflows/reusable-deploy-site.yml + uses: ./.github/workflows/reusable-deploy-env.yml with: deploy_ref: ${{ needs.create-deployment.outputs.deploy_ref }} - aws_region: us-east-1 - gha_role: arn:aws:iam::568298704244:role/github-actions-dev3 - aws_account_id: "568298704244" env_name: dev3 - build_mode: dev3 secrets: inherit record-deployment: - needs: [create-deployment, deploy-backend, deploy-frontend, deploy-site] + needs: [create-deployment, deploy] if: always() && needs.create-deployment.outputs.deployment_id != '' runs-on: ubuntu-latest steps: - # Read-only creds so we can ask whether dev3 is actually awake before - # claiming a deployment is live. Best-effort: never fails the job. + # Read-only creds so finalize-deployment can ask whether dev3 is + # actually awake before claiming a deployment is live. Best-effort: + # never fails the job; without creds the lights read falls back to + # treating dev3 as awake. - name: Configure AWS credentials - id: aws continue-on-error: true uses: aws-actions/configure-aws-credentials@v6 with: @@ -118,42 +80,16 @@ jobs: role-session-name: dev3-deploy-status aws-region: us-east-2 + - name: Checkout repository + uses: actions/checkout@v6 + - name: Finalize GitHub deployment status - env: - GH_TOKEN: ${{ github.token }} - ID: ${{ needs.create-deployment.outputs.deployment_id }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - HAVE_AWS: ${{ steps.aws.outcome }} - B: ${{ needs.deploy-backend.result }} - F: ${{ needs.deploy-frontend.result }} - S: ${{ needs.deploy-site.result }} - run: | - if [[ "$B" == "success" && "$F" == "success" && "$S" == "success" ]]; then - # The pipeline succeeded, but dev3 scales to zero on the lights-out - # schedule (and the backend deploy is a no-op while it's asleep). - # Reflect whether it's actually serving so the tab can't claim a - # ref is live while the environment is off. - lights=unknown - if [[ "$HAVE_AWS" == "success" ]]; then - lights=$(aws ssm get-parameter --name /dev3/doenet/lights \ - --query Parameter.Value --output text 2>/dev/null || echo unknown) - fi - if [[ "$lights" == "off" ]]; then - state=inactive - desc="dev3 is asleep (lights off); this ref serves on next wake" - else - state=success - desc="deployed to dev3" - fi - else - state=failure - desc="dev3 deploy failed (backend:$B frontend:$F site:$S)" - fi - # On a success status GitHub auto-inactivates the previous dev3 - # deployment, so the Environments tab reflects what is currently live. - gh api -X POST "repos/${GITHUB_REPOSITORY}/deployments/${ID}/statuses" \ - -f state="$state" \ - -f description="$desc" \ - -f environment=dev3 \ - -f environment_url=https://dev3.doenet.org \ - -f log_url="$RUN_URL" >/dev/null + uses: ./.github/actions/finalize-deployment + with: + deployment-id: ${{ needs.create-deployment.outputs.deployment_id }} + environment: dev3 + environment-url: https://dev3.doenet.org + backend-result: ${{ needs.deploy.outputs.backend_result }} + frontend-result: ${{ needs.deploy.outputs.frontend_result }} + site-result: ${{ needs.deploy.outputs.site_result }} + lights-parameter: /dev3/doenet/lights diff --git a/.github/workflows/prod-deploy.yml b/.github/workflows/prod-deploy.yml index e30f7dd14..3fd92c334 100644 --- a/.github/workflows/prod-deploy.yml +++ b/.github/workflows/prod-deploy.yml @@ -21,94 +21,47 @@ jobs: # Open a GitHub deployment so the repo's prod Environment always shows # what is currently live. Uses the Deployments API rather than an # `environment:` key so environment protection rules can never gate the - # deploy, and no deploy job depends on this one — bookkeeping must never - # block or skip an actual deploy. + # deploy, and the deploy job does not depend on this one — bookkeeping + # must never block or skip an actual deploy. create-deployment: runs-on: ubuntu-latest outputs: - deployment_id: ${{ steps.deployment.outputs.deployment_id }} + deployment_id: ${{ steps.open.outputs.deployment_id }} steps: - - name: Open GitHub deployment - id: deployment - env: - GH_TOKEN: ${{ github.token }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: | - payload=$(jq -n --arg ref "$GITHUB_SHA" \ - '{ref: $ref, environment: "prod", required_contexts: [], auto_merge: false, production_environment: true, description: "prod deploy"}') - id=$(gh api -X POST "repos/${GITHUB_REPOSITORY}/deployments" --input - -q .id <<<"$payload" 2>/dev/null || true) - if [[ -n "$id" ]]; then - echo "deployment_id=$id" >> "$GITHUB_OUTPUT" - gh api -X POST "repos/${GITHUB_REPOSITORY}/deployments/${id}/statuses" \ - -f state=in_progress \ - -f environment=prod \ - -f environment_url=https://doenet.org \ - -f log_url="$RUN_URL" >/dev/null 2>&1 || true - else - echo "::warning::could not create prod deployment record; deploy continues" - fi - - deploy-backend: - uses: ./.github/workflows/reusable-deploy-backend.yml - with: - deploy_ref: ${{ github.sha }} - aws_region: us-east-2 - ecr_repository: prod/doenet - ecs_cluster: prod - ecs_service: doenet-FARGATE - gha_role: arn:aws:iam::350646758180:role/github-actions-prod - sns_topic: arn:aws:sns:us-east-2:350646758180:prod-critical-cloudwatch-notifications - tag_name: main - secrets: inherit + - name: Checkout repository + uses: actions/checkout@v6 - deploy-frontend: - uses: ./.github/workflows/reusable-deploy-frontend.yml - with: - deploy_ref: ${{ github.sha }} - aws_region: us-east-1 - gha_role: arn:aws:iam::350646758180:role/github-actions-prod - aws_account_id: "350646758180" - env_name: prod - build_mode: prod - secrets: inherit + - name: Open GitHub deployment + id: open + uses: ./.github/actions/open-deployment + with: + environment: prod + environment-url: https://doenet.org + ref: ${{ github.sha }} + description: prod deploy + production-environment: "true" - deploy-site: - uses: ./.github/workflows/reusable-deploy-site.yml + deploy: + uses: ./.github/workflows/reusable-deploy-env.yml with: deploy_ref: ${{ github.sha }} - aws_region: us-east-1 - gha_role: arn:aws:iam::350646758180:role/github-actions-prod - aws_account_id: "350646758180" env_name: prod - build_mode: prod secrets: inherit record-deployment: - needs: [create-deployment, deploy-backend, deploy-frontend, deploy-site] + needs: [create-deployment, deploy] if: always() && needs.create-deployment.outputs.deployment_id != '' runs-on: ubuntu-latest steps: + - name: Checkout repository + uses: actions/checkout@v6 + - name: Finalize GitHub deployment status - env: - GH_TOKEN: ${{ github.token }} - ID: ${{ needs.create-deployment.outputs.deployment_id }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - B: ${{ needs.deploy-backend.result }} - F: ${{ needs.deploy-frontend.result }} - S: ${{ needs.deploy-site.result }} - run: | - if [[ "$B" == "success" && "$F" == "success" && "$S" == "success" ]]; then - state=success - desc="deployed to prod" - else - state=failure - desc="prod deploy failed (backend:$B frontend:$F site:$S)" - fi - # On a success status GitHub auto-inactivates the previous prod - # deployment, so the Environments tab reflects what is currently live. - gh api -X POST "repos/${GITHUB_REPOSITORY}/deployments/${ID}/statuses" \ - -f state="$state" \ - -f description="$desc" \ - -f environment=prod \ - -f environment_url=https://doenet.org \ - -f log_url="$RUN_URL" >/dev/null + uses: ./.github/actions/finalize-deployment + with: + deployment-id: ${{ needs.create-deployment.outputs.deployment_id }} + environment: prod + environment-url: https://doenet.org + backend-result: ${{ needs.deploy.outputs.backend_result }} + frontend-result: ${{ needs.deploy.outputs.frontend_result }} + site-result: ${{ needs.deploy.outputs.site_result }} diff --git a/.github/workflows/reusable-deploy-env.yml b/.github/workflows/reusable-deploy-env.yml new file mode 100644 index 000000000..e49c578a2 --- /dev/null +++ b/.github/workflows/reusable-deploy-env.yml @@ -0,0 +1,103 @@ +name: Reusable - Deploy an environment (backend + frontend + site) + +# Single definition of what "a deploy" is: the per-environment constants and +# the fan-out to the three component workflows. Callers decide when and what +# ref to deploy; this workflow decides where and how. + +on: + workflow_call: + inputs: + deploy_ref: + description: Git ref or SHA to deploy + required: true + type: string + env_name: + description: "Target environment: dev3 or prod" + required: true + type: string + outputs: + backend_result: + description: Result of the backend deploy job + value: ${{ jobs.deploy-backend.result }} + frontend_result: + description: Result of the frontend deploy job + value: ${{ jobs.deploy-frontend.result }} + site_result: + description: Result of the site deploy job + value: ${{ jobs.deploy-site.result }} + +permissions: + contents: read + id-token: write + +jobs: + # Single source of truth for per-environment deploy constants. + config: + runs-on: ubuntu-latest + outputs: + aws_account_id: ${{ steps.env.outputs.aws_account_id }} + gha_role: ${{ steps.env.outputs.gha_role }} + ecr_repository: ${{ steps.env.outputs.ecr_repository }} + ecs_cluster: ${{ steps.env.outputs.ecs_cluster }} + sns_topic: ${{ steps.env.outputs.sns_topic }} + tag_name: ${{ steps.env.outputs.tag_name }} + steps: + - name: Resolve environment config + id: env + env: + ENV_NAME: ${{ inputs.env_name }} + run: | + case "$ENV_NAME" in + dev3) account=568298704244 tag_name=latest ;; + prod) account=350646758180 tag_name=main ;; + *) + echo "::error::unknown env_name '$ENV_NAME' (expected dev3 or prod)" + exit 1 + ;; + esac + { + echo "aws_account_id=$account" + echo "gha_role=arn:aws:iam::${account}:role/github-actions-${ENV_NAME}" + echo "ecr_repository=${ENV_NAME}/doenet" + echo "ecs_cluster=${ENV_NAME}" + echo "sns_topic=arn:aws:sns:us-east-2:${account}:${ENV_NAME}-critical-cloudwatch-notifications" + echo "tag_name=$tag_name" + } >> "$GITHUB_OUTPUT" + + deploy-backend: + needs: config + uses: ./.github/workflows/reusable-deploy-backend.yml + with: + deploy_ref: ${{ inputs.deploy_ref }} + aws_region: us-east-2 + ecr_repository: ${{ needs.config.outputs.ecr_repository }} + ecs_cluster: ${{ needs.config.outputs.ecs_cluster }} + ecs_service: doenet-FARGATE + gha_role: ${{ needs.config.outputs.gha_role }} + sns_topic: ${{ needs.config.outputs.sns_topic }} + tag_name: ${{ needs.config.outputs.tag_name }} + secrets: inherit + + deploy-frontend: + needs: config + uses: ./.github/workflows/reusable-deploy-frontend.yml + with: + deploy_ref: ${{ inputs.deploy_ref }} + aws_region: us-east-1 + gha_role: ${{ needs.config.outputs.gha_role }} + aws_account_id: ${{ needs.config.outputs.aws_account_id }} + env_name: ${{ inputs.env_name }} + build_mode: ${{ inputs.env_name }} + secrets: inherit + + deploy-site: + needs: config + uses: ./.github/workflows/reusable-deploy-site.yml + with: + deploy_ref: ${{ inputs.deploy_ref }} + aws_region: us-east-1 + gha_role: ${{ needs.config.outputs.gha_role }} + aws_account_id: ${{ needs.config.outputs.aws_account_id }} + env_name: ${{ inputs.env_name }} + build_mode: ${{ inputs.env_name }} + secrets: inherit