ci(images): add design-review to build/push matrix #2
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: deploy-prod | |
| # Promote digest-pinned images to prod droplets on signed version tags. | |
| # | |
| # Trigger: push tag v*.*.* (cut from main) or manual workflow_dispatch. | |
| # Triggers on version tags only — never on branch pushes. | |
| # | |
| # Steps: | |
| # 1. Preflight — CI green for the tag SHA; origin/main staging pins carry | |
| # that commit_sha (ladder tip may be a later pin-commit on main). | |
| # 2. Fail-closed Postgres backup to DO Spaces, then promote.sh --env prod | |
| # --confirm-prod for each pin service (staging.json → prod.json). | |
| # 3. Deploy master + validator with remote-deploy.sh --build-from registry | |
| # (GHCR digest pull + retag; no Rust compile on the droplet). | |
| # 4. Smoke /healthz (fail-closed). | |
| on: | |
| push: | |
| tags: | |
| - "v*.*.*" | |
| workflow_dispatch: | |
| inputs: | |
| commit_sha: | |
| description: "Commit SHA to deploy (must have passed staging)" | |
| required: false | |
| permissions: | |
| contents: write | |
| packages: read | |
| concurrency: | |
| group: deploy-prod | |
| cancel-in-progress: false | |
| jobs: | |
| preflight: | |
| name: preflight checks | |
| runs-on: ubuntu-latest | |
| outputs: | |
| commit_sha: ${{ steps.resolve.outputs.commit_sha }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Resolve commit SHA | |
| id: resolve | |
| run: | | |
| set -euo pipefail | |
| if [[ -n "${{ github.event.inputs.commit_sha }}" ]]; then | |
| SHA="${{ github.event.inputs.commit_sha }}" | |
| elif [[ "$GITHUB_REF" == refs/tags/* ]]; then | |
| SHA=$(git rev-list -n 1 "$GITHUB_REF") | |
| else | |
| SHA="$GITHUB_SHA" | |
| fi | |
| echo "commit_sha=$SHA" >> "$GITHUB_OUTPUT" | |
| echo "Resolved commit: $SHA" | |
| - name: Verify CI succeeded for this SHA | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| set -euo pipefail | |
| SHA="${{ steps.resolve.outputs.commit_sha }}" | |
| # Check that the ci workflow passed for this commit on main | |
| conclusion=$(gh run list \ | |
| --workflow ci.yml \ | |
| --branch main \ | |
| --commit "$SHA" \ | |
| --status completed \ | |
| --json conclusion \ | |
| --jq '.[0].conclusion // empty' 2>/dev/null || true) | |
| if [[ "$conclusion" != "success" ]]; then | |
| echo "CI has not passed for commit $SHA on main (conclusion: ${conclusion:-none})" | |
| echo "Failing — prod deploy requires a green CI run for this exact commit." | |
| exit 1 | |
| fi | |
| echo "CI passed for $SHA" | |
| - name: Verify staging pins exist for this SHA | |
| run: | | |
| set -euo pipefail | |
| SHA="${{ steps.resolve.outputs.commit_sha }}" | |
| # Pin commits land on origin/main after images.yml; the tag may point at | |
| # the image SHA while staging.json lives on a later pin commit. | |
| git fetch origin main | |
| if ! git show origin/main:deploy/pins/staging.json > /tmp/staging-pins.json 2>/dev/null; then | |
| echo "deploy/pins/staging.json not found on origin/main — staging pins must be committed first" | |
| exit 1 | |
| fi | |
| pin_sha=$(python3 -c \ | |
| "import json; print(json.load(open('/tmp/staging-pins.json')).get('commit_sha',''))") | |
| if [[ "$pin_sha" != "$SHA" ]]; then | |
| echo "Staging pins commit_sha=$pin_sha, expected $SHA" | |
| echo "Staging must record digests for this commit before promoting to prod." | |
| exit 1 | |
| fi | |
| echo "Staging pins on origin/main match commit $SHA" | |
| promote: | |
| name: promote staging → prod pins | |
| needs: preflight | |
| runs-on: ubuntu-latest | |
| environment: production | |
| outputs: | |
| pins_artifact: prod-pins-${{ needs.preflight.outputs.commit_sha }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.preflight.outputs.commit_sha }} | |
| fetch-depth: 0 | |
| - name: Load staging pins from origin/main | |
| run: | | |
| set -euo pipefail | |
| git fetch origin main | |
| git show origin/main:deploy/pins/staging.json > deploy/pins/staging.json | |
| # Digests manifest (optional pulls for prism/attest-helper) when present. | |
| SHA="${{ needs.preflight.outputs.commit_sha }}" | |
| if git show "origin/main:deploy/digests/${SHA}.json" > "deploy/digests/${SHA}.json" 2>/dev/null; then | |
| echo "Loaded deploy/digests/${SHA}.json from origin/main" | |
| else | |
| echo "WARNING: deploy/digests/${SHA}.json missing on origin/main" | |
| fi | |
| - name: Fail-closed — require Spaces backup credentials | |
| env: | |
| BASE_BACKUP_ENDPOINT: ${{ secrets.BASE_BACKUP_ENDPOINT }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.SPACES_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET_ACCESS_KEY || secrets.AWS_SECRET_ACCESS_KEY }} | |
| run: | | |
| set -euo pipefail | |
| missing=() | |
| [[ -n "${BASE_BACKUP_ENDPOINT:-}" ]] || missing+=("BASE_BACKUP_ENDPOINT") | |
| [[ -n "${AWS_ACCESS_KEY_ID:-}" ]] || missing+=("SPACES_ACCESS_KEY_ID (or AWS_ACCESS_KEY_ID)") | |
| [[ -n "${AWS_SECRET_ACCESS_KEY:-}" ]] || missing+=("SPACES_SECRET_ACCESS_KEY (or AWS_SECRET_ACCESS_KEY)") | |
| if [[ ${#missing[@]} -gt 0 ]]; then | |
| echo "Prod promote is fail-closed: missing GitHub secrets: ${missing[*]}" | |
| echo "Set DO Spaces credentials before cutting a prod tag." | |
| echo "See deploy/README.md § Promotion pipeline (BASE_BACKUP_ENDPOINT + Spaces keys)." | |
| echo "Postgres dump runs on the prod master host via SSH; Spaces upload runs here." | |
| exit 1 | |
| fi | |
| echo "Spaces backup credentials present" | |
| - name: Install SSH key | |
| run: | | |
| set -euo pipefail | |
| KEY="${{ secrets.PROD_SSH_KEY || secrets.STAGING_SSH_KEY }}" | |
| test -n "$KEY" || { echo "missing PROD_SSH_KEY (or STAGING_SSH_KEY fallback)"; exit 1; } | |
| mkdir -p ~/.ssh | |
| echo "$KEY" > ~/.ssh/deploy_ed25519 | |
| chmod 600 ~/.ssh/deploy_ed25519 | |
| echo "StrictHostKeyChecking accept-new" > ~/.ssh/config | |
| - name: Open firewall for this runner | |
| id: fw | |
| uses: ./.github/actions/do-firewall | |
| with: | |
| action: open | |
| token: ${{ secrets.DIGITALOCEAN_TOKEN }} | |
| - name: Backup prod Postgres (SSH) → Spaces | |
| id: backup | |
| env: | |
| BASE_BACKUP_ENDPOINT: ${{ secrets.BASE_BACKUP_ENDPOINT }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.SPACES_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET_ACCESS_KEY || secrets.AWS_SECRET_ACCESS_KEY }} | |
| BASE_BACKUP_BUCKET: ${{ secrets.BASE_BACKUP_BUCKET || 'base-backups' }} | |
| AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION || 'nyc3' }} | |
| run: | | |
| set -euo pipefail | |
| command -v aws >/dev/null || { sudo apt-get update -qq && sudo apt-get install -y -qq awscli; } | |
| HOST="root@${{ secrets.PROD_HOST }}" | |
| test -n "${{ secrets.PROD_HOST }}" || { echo "missing PROD_HOST"; exit 1; } | |
| export BASE_SSH_IDENTITY="$HOME/.ssh/deploy_ed25519" | |
| STAMP=$(date -u +%Y%m%dT%H%M%SZ) | |
| DUMP="/tmp/base-prod-${STAMP}.sql.gz" | |
| META="/tmp/base-prod-${STAMP}.meta.json" | |
| # Postgres is not published on the host; dump inside the container. | |
| ssh -i "$BASE_SSH_IDENTITY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new \ | |
| "$HOST" \ | |
| 'set -euo pipefail | |
| cd /opt/base | |
| cid=$(docker compose -f docker-compose.yml -f deploy/compose/role-master.yml -f deploy/compose/env-prod.yml ps -q postgres) | |
| test -n "$cid" | |
| docker exec "$cid" sh -c "pg_dump -U \"\$POSTGRES_USER\" -d \"\$POSTGRES_DB\" --format=plain --no-owner --no-acl"' \ | |
| | gzip -n > "$DUMP" | |
| BYTES=$(wc -c < "$DUMP") | |
| SUM=$(sha256sum "$DUMP" | awk '{print $1}') | |
| test "$BYTES" -gt 32 | |
| python3 - "$META" "$STAMP" "$SUM" "$BYTES" <<'PY' | |
| import json, sys | |
| path, stamp, digest, nbytes = sys.argv[1:5] | |
| json.dump({ | |
| "ok": True, | |
| "env": "prod", | |
| "stamp": stamp, | |
| "sha256": digest, | |
| "bytes": int(nbytes), | |
| "source": "ssh+docker-exec", | |
| }, open(path, "w"), indent=2, sort_keys=True) | |
| open(path, "a").write("\n") | |
| PY | |
| URI="s3://${BASE_BACKUP_BUCKET}/pg/prod/base-prod-${STAMP}.sql.gz" | |
| aws --endpoint-url "$BASE_BACKUP_ENDPOINT" s3 cp "$DUMP" "$URI" --only-show-errors | |
| aws --endpoint-url "$BASE_BACKUP_ENDPOINT" s3 cp "$META" "${URI%.sql.gz}.meta.json" --only-show-errors | |
| echo "backup_uri=$URI" >> "$GITHUB_OUTPUT" | |
| echo "Backup ok: $URI ($BYTES bytes, sha256=$SUM)" | |
| - name: Close firewall for promote runner | |
| if: always() && steps.fw.outputs.ip != '' | |
| uses: ./.github/actions/do-firewall | |
| with: | |
| action: close | |
| token: ${{ secrets.DIGITALOCEAN_TOKEN }} | |
| ip: ${{ steps.fw.outputs.ip }} | |
| firewall-id: ${{ steps.fw.outputs.firewall-id }} | |
| - name: Promote each service staging → prod | |
| run: | | |
| set -euo pipefail | |
| SHA="${{ needs.preflight.outputs.commit_sha }}" | |
| chmod +x deploy/scripts/promote.sh | |
| # Backup already succeeded above (fail-closed). promote.sh --skip-backup | |
| # is used only because Postgres is not reachable as PGHOST from the runner; | |
| # we never skip the Spaces backup step itself. | |
| mapfile -t SERVICES < <(python3 -c ' | |
| import json | |
| for s in sorted(json.load(open("deploy/pins/staging.json"))["services"]): | |
| print(s) | |
| ') | |
| for svc in "${SERVICES[@]}"; do | |
| IMAGE=$(python3 -c ' | |
| import json, sys | |
| meta = json.load(open("deploy/pins/staging.json"))["services"][sys.argv[1]] | |
| img = meta.get("image") or "" | |
| digest = meta["digest"] | |
| if "/" in img and "@sha256:" in img: | |
| print(img) | |
| else: | |
| print(f"ghcr.io/baseintelligence/base/{sys.argv[1]}@{digest}") | |
| ' "$svc") | |
| echo "Promoting $svc → $IMAGE" | |
| ./deploy/scripts/promote.sh \ | |
| --env prod \ | |
| --service "$svc" \ | |
| --image "$IMAGE" \ | |
| --confirm-prod \ | |
| --commit "$SHA" \ | |
| --skip-backup | |
| done | |
| python3 -c 'import json; p=json.load(open("deploy/pins/prod.json")); assert p["commit_sha"]; print(p["commit_sha"], list(p["services"]))' | |
| - name: Stage pins artifact | |
| run: | | |
| set -euo pipefail | |
| SHA="${{ needs.preflight.outputs.commit_sha }}" | |
| STAGE="/tmp/prod-pins-artifact" | |
| mkdir -p "$STAGE/deploy/pins" "$STAGE/deploy/digests" | |
| cp deploy/pins/prod.json "$STAGE/deploy/pins/prod.json" | |
| cp deploy/pins/staging.json "$STAGE/deploy/pins/staging.json" | |
| if [[ -f "deploy/digests/${SHA}.json" ]]; then | |
| cp "deploy/digests/${SHA}.json" "$STAGE/deploy/digests/${SHA}.json" | |
| fi | |
| - name: Upload promoted pins artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: prod-pins-${{ needs.preflight.outputs.commit_sha }} | |
| path: /tmp/prod-pins-artifact | |
| if-no-files-found: error | |
| - name: Commit prod pins to origin/main | |
| run: | | |
| set -euo pipefail | |
| SHA="${{ needs.preflight.outputs.commit_sha }}" | |
| mkdir -p /tmp/prod-pin-commit | |
| cp deploy/pins/prod.json /tmp/prod-pin-commit/prod.json | |
| if [[ -f "deploy/digests/${SHA}.json" ]]; then | |
| cp "deploy/digests/${SHA}.json" "/tmp/prod-pin-commit/${SHA}.json" | |
| fi | |
| git fetch origin main | |
| git checkout -B main origin/main | |
| cp /tmp/prod-pin-commit/prod.json deploy/pins/prod.json | |
| if [[ -f "/tmp/prod-pin-commit/${SHA}.json" ]]; then | |
| mkdir -p deploy/digests | |
| cp "/tmp/prod-pin-commit/${SHA}.json" "deploy/digests/${SHA}.json" | |
| fi | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git add deploy/pins/prod.json | |
| if [[ -f "deploy/digests/${SHA}.json" ]]; then | |
| git add "deploy/digests/${SHA}.json" | |
| fi | |
| if git diff --cached --quiet; then | |
| echo "No pin changes to commit" | |
| exit 0 | |
| fi | |
| git commit -m "deploy: promote prod pins for ${SHA}" | |
| git push origin main | |
| deploy: | |
| name: deploy prod ${{ matrix.role }} | |
| needs: [preflight, promote] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| environment: production | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - role: master | |
| host_secret: PROD_HOST | |
| - role: validator | |
| host_secret: PROD_VALIDATOR_HOST | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.preflight.outputs.commit_sha }} | |
| - name: Download promoted pins | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: prod-pins-${{ needs.preflight.outputs.commit_sha }} | |
| path: /tmp/prod-pins | |
| - name: Install promoted pins into tree | |
| run: | | |
| set -euo pipefail | |
| SHA="${{ needs.preflight.outputs.commit_sha }}" | |
| if [[ -f /tmp/prod-pins/deploy/pins/prod.json ]]; then | |
| BASE=/tmp/prod-pins/deploy | |
| elif [[ -f /tmp/prod-pins/pins/prod.json ]]; then | |
| BASE=/tmp/prod-pins | |
| else | |
| echo "unexpected artifact layout:"; find /tmp/prod-pins -type f | head -50 | |
| exit 1 | |
| fi | |
| mkdir -p deploy/pins deploy/digests | |
| cp "$BASE/pins/prod.json" deploy/pins/prod.json | |
| cp "$BASE/pins/staging.json" deploy/pins/staging.json | |
| if [[ -f "$BASE/digests/${SHA}.json" ]]; then | |
| cp "$BASE/digests/${SHA}.json" "deploy/digests/${SHA}.json" | |
| fi | |
| python3 -c ' | |
| import json, re | |
| p = json.load(open("deploy/pins/prod.json")) | |
| d = p["services"]["validator"]["digest"] | |
| h = d.split(":", 1)[-1].lower() | |
| assert not re.fullmatch(r"0+", h) and not re.fullmatch(r"0+1", h), d | |
| print("prod pins ready", p["commit_sha"]) | |
| ' | |
| - name: Install SSH key | |
| run: | | |
| set -euo pipefail | |
| KEY="${{ secrets.PROD_SSH_KEY || secrets.STAGING_SSH_KEY }}" | |
| test -n "$KEY" || { echo "missing PROD_SSH_KEY (or STAGING_SSH_KEY fallback)"; exit 1; } | |
| mkdir -p ~/.ssh | |
| echo "$KEY" > ~/.ssh/deploy_ed25519 | |
| chmod 600 ~/.ssh/deploy_ed25519 | |
| echo "StrictHostKeyChecking accept-new" > ~/.ssh/config | |
| - name: Resolve host | |
| id: host | |
| run: | | |
| set -euo pipefail | |
| if [[ "${{ matrix.role }}" == "master" ]]; then | |
| h="${{ secrets.PROD_HOST }}" | |
| else | |
| h="${{ secrets.PROD_VALIDATOR_HOST }}" | |
| fi | |
| test -n "$h" || { echo "missing host secret for ${{ matrix.role }}"; exit 1; } | |
| echo "value=$h" >> "$GITHUB_OUTPUT" | |
| - name: Open firewall for this runner | |
| id: fw | |
| uses: ./.github/actions/do-firewall | |
| with: | |
| action: open | |
| token: ${{ secrets.DIGITALOCEAN_TOKEN }} | |
| - name: Deploy (registry digests) | |
| run: | | |
| set -euo pipefail | |
| chmod +x deploy/scripts/remote-deploy.sh | |
| export BASE_SSH_IDENTITY="$HOME/.ssh/deploy_ed25519" | |
| EXTRA=() | |
| if [[ "${{ matrix.role }}" == "validator" ]]; then | |
| if [[ -n "${{ secrets.PROD_MASTER_GATEWAY_URL }}" ]]; then | |
| EXTRA+=(--gateway-endpoint "${{ secrets.PROD_MASTER_GATEWAY_URL }}") | |
| fi | |
| fi | |
| ./deploy/scripts/remote-deploy.sh \ | |
| --host "root@${{ steps.host.outputs.value }}" \ | |
| --role "${{ matrix.role }}" \ | |
| --env prod \ | |
| --build-from registry \ | |
| "${EXTRA[@]}" | |
| - name: Smoke health (fail-closed) | |
| run: | | |
| set -euo pipefail | |
| export BASE_SSH_IDENTITY="$HOME/.ssh/deploy_ed25519" | |
| HOST="root@${{ steps.host.outputs.value }}" | |
| ssh -i "$BASE_SSH_IDENTITY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new \ | |
| "$HOST" \ | |
| 'cd /opt/base && docker compose -f docker-compose.yml -f deploy/compose/role-${{ matrix.role }}.yml -f deploy/compose/env-prod.yml ps --format "table {{.Service}}\t{{.Status}}"' | |
| # Health probe | |
| ssh -i "$BASE_SSH_IDENTITY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new \ | |
| "$HOST" \ | |
| 'for i in $(seq 1 12); do \ | |
| if docker exec $(docker ps -q --filter name=validator) curl -fsS -m 5 http://127.0.0.1:8080/healthz 2>/dev/null; then \ | |
| echo "validator health: ok"; exit 0; \ | |
| fi; \ | |
| sleep 5; \ | |
| done; \ | |
| echo "validator health: FAILED"; exit 1' | |
| - name: Close firewall for this runner | |
| if: always() && steps.fw.outputs.ip != '' | |
| uses: ./.github/actions/do-firewall | |
| with: | |
| action: close | |
| token: ${{ secrets.DIGITALOCEAN_TOKEN }} | |
| ip: ${{ steps.fw.outputs.ip }} | |
| firewall-id: ${{ steps.fw.outputs.firewall-id }} |