snapshot-restore-drill #63
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: snapshot-restore-drill | |
| on: | |
| schedule: | |
| - cron: "30 4 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| project_id: | |
| description: "Scaleway Project ID" | |
| required: true | |
| default: "4ca4f592-005a-45dc-a016-91bd11dd5348" | |
| target_names: | |
| description: "Restore targets: all or comma-separated keys (node1-main,jenkins,observability,rocket,vpn-control)" | |
| required: true | |
| default: "all" | |
| tfstate_bucket: | |
| description: "Object Storage bucket for Terraform state" | |
| required: true | |
| default: "terra-snapshots-state" | |
| tfstate_region: | |
| description: "Object Storage region" | |
| required: true | |
| default: "fr-par" | |
| tfstate_endpoint: | |
| description: "Object Storage endpoint" | |
| required: true | |
| default: "https://s3.fr-par.scw.cloud" | |
| tfstate_key: | |
| description: "Legacy snapshot state key or per-target state prefix" | |
| required: true | |
| default: "terraform/snapshots/terraform.tfstate" | |
| snapshot_source: | |
| description: "Snapshot source to restore: auto or manual" | |
| required: false | |
| default: "auto" | |
| manual_snapshot_name: | |
| description: "Manual snapshot key to restore when snapshot_source=manual" | |
| required: false | |
| default: "" | |
| expected_rocket_message: | |
| description: "Optional Rocket.Chat #general message text that must exist in restored data." | |
| required: false | |
| default: "" | |
| concurrency: | |
| group: snapshot-restore-drill-${{ github.ref_name }} | |
| cancel-in-progress: false | |
| permissions: | |
| contents: read | |
| actions: read | |
| jobs: | |
| janitor: | |
| if: github.ref_name == 'env/prod' | |
| runs-on: ubuntu-latest | |
| environment: prod | |
| outputs: | |
| summary: ${{ steps.janitor.outputs.summary }} | |
| status: ${{ steps.janitor.outputs.status }} | |
| env: | |
| TF_IN_AUTOMATION: "true" | |
| TF_INPUT: "false" | |
| SCW_ACCESS_KEY: ${{ secrets.SNAPSHOTS_SCW_ACCESS_KEY }} | |
| SCW_SECRET_KEY: ${{ secrets.SNAPSHOTS_SCW_SECRET_KEY }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.SNAPSHOTS_SCW_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.SNAPSHOTS_SCW_SECRET_KEY }} | |
| SNAPSHOTS_TFSTATE_BUCKET: ${{ github.event.inputs.tfstate_bucket || 'terra-snapshots-state' }} | |
| SNAPSHOTS_TFSTATE_ENDPOINT: ${{ github.event.inputs.tfstate_endpoint || 'https://s3.fr-par.scw.cloud' }} | |
| SNAPSHOTS_TFSTATE_REGION: ${{ github.event.inputs.tfstate_region || 'fr-par' }} | |
| TF_VAR_project_id: ${{ github.event.inputs.project_id || '4ca4f592-005a-45dc-a016-91bd11dd5348' }} | |
| RESTORE_DRILL_TARGET_KEYS: jenkins,node1-main,observability,rocket,vpn-control | |
| RESTORE_DRILL_CURRENT_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }} | |
| RESTORE_DRILL_JANITOR_MIN_AGE_HOURS: "6" | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| submodules: false | |
| fetch-depth: 1 | |
| - uses: hashicorp/setup-terraform@v4 | |
| with: | |
| terraform_version: 1.6.6 | |
| terraform_wrapper: false | |
| - name: Mask sensitive values | |
| run: | | |
| mask() { | |
| if [[ -n "${1:-}" ]]; then | |
| echo "::add-mask::$1" | |
| fi | |
| } | |
| mask "${SCW_ACCESS_KEY:-}" | |
| mask "${SCW_SECRET_KEY:-}" | |
| mask "${AWS_ACCESS_KEY_ID:-}" | |
| mask "${AWS_SECRET_ACCESS_KEY:-}" | |
| - name: Restore drill janitor | |
| id: janitor | |
| run: bash .github/scripts/restore-drill-janitor.sh | |
| - name: Upload janitor result | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: restore-janitor-result | |
| path: restore-janitor-result.json | |
| if-no-files-found: warn | |
| retention-days: 7 | |
| resolve: | |
| needs: janitor | |
| if: github.ref_name == 'env/prod' | |
| runs-on: ubuntu-latest | |
| environment: prod | |
| outputs: | |
| matrix: ${{ steps.matrix.outputs.matrix }} | |
| env: | |
| SNAPSHOTS_TARGET_NAMES: ${{ github.event.inputs.target_names || 'all' }} | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| submodules: false | |
| fetch-depth: 1 | |
| - name: Resolve restore matrix | |
| id: matrix | |
| run: bash .github/scripts/resolve-restore-matrix.sh | |
| drill: | |
| needs: resolve | |
| if: github.ref_name == 'env/prod' | |
| runs-on: ubuntu-latest | |
| environment: prod | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 4 | |
| matrix: ${{ fromJson(needs.resolve.outputs.matrix) }} | |
| env: | |
| TF_IN_AUTOMATION: "true" | |
| TF_INPUT: "false" | |
| SCW_ACCESS_KEY: ${{ secrets.SNAPSHOTS_SCW_ACCESS_KEY }} | |
| SCW_SECRET_KEY: ${{ secrets.SNAPSHOTS_SCW_SECRET_KEY }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.SNAPSHOTS_SCW_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.SNAPSHOTS_SCW_SECRET_KEY }} | |
| SNAPSHOTS_TFSTATE_BUCKET: ${{ github.event.inputs.tfstate_bucket || 'terra-snapshots-state' }} | |
| SNAPSHOTS_TFSTATE_ENDPOINT: ${{ github.event.inputs.tfstate_endpoint || 'https://s3.fr-par.scw.cloud' }} | |
| SNAPSHOTS_TFSTATE_REGION: ${{ github.event.inputs.tfstate_region || 'fr-par' }} | |
| SNAPSHOTS_TFSTATE_KEY: ${{ github.event.inputs.tfstate_key || 'terraform/snapshots/terraform.tfstate' }} | |
| RESTORE_SNAPSHOT_SOURCE: ${{ github.event.inputs.snapshot_source || 'auto' }} | |
| RESTORE_MANUAL_SNAPSHOT_NAME: ${{ github.event.inputs.manual_snapshot_name || '' }} | |
| TF_VAR_project_id: ${{ github.event.inputs.project_id || '4ca4f592-005a-45dc-a016-91bd11dd5348' }} | |
| TF_VAR_target_key: ${{ matrix.target_key }} | |
| TF_VAR_instance_type: ${{ matrix.instance_type }} | |
| TF_VAR_run_id: ${{ github.run_id }}-${{ github.run_attempt }} | |
| TF_VAR_luks_key_access_key: ${{ secrets.SNAPSHOTS_SCW_ACCESS_KEY }} | |
| TF_VAR_luks_key_secret_key: ${{ secrets.SNAPSHOTS_SCW_SECRET_KEY }} | |
| TF_VAR_result_bucket_name: ${{ github.event.inputs.tfstate_bucket || 'terra-snapshots-state' }} | |
| TF_VAR_result_object_key: terraform/snapshot-restore-drill/results/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.target_key }}.json | |
| TF_VAR_result_region: ${{ github.event.inputs.tfstate_region || 'fr-par' }} | |
| TF_VAR_result_endpoint: ${{ github.event.inputs.tfstate_endpoint || 'https://s3.fr-par.scw.cloud' }} | |
| TF_VAR_rocket_restore_expected_message: ${{ github.event.inputs.expected_rocket_message || '' }} | |
| defaults: | |
| run: | |
| shell: bash | |
| working-directory: terraform/snapshot-restore-drill | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| submodules: false | |
| fetch-depth: 1 | |
| - uses: hashicorp/setup-terraform@v4 | |
| with: | |
| terraform_version: 1.6.6 | |
| terraform_wrapper: false | |
| - name: Mask sensitive values | |
| run: | | |
| mask() { | |
| if [[ -n "${1:-}" ]]; then | |
| echo "::add-mask::$1" | |
| fi | |
| } | |
| mask "${SCW_ACCESS_KEY:-}" | |
| mask "${SCW_SECRET_KEY:-}" | |
| mask "${AWS_ACCESS_KEY_ID:-}" | |
| mask "${AWS_SECRET_ACCESS_KEY:-}" | |
| mask "${TF_VAR_luks_key_access_key:-}" | |
| mask "${TF_VAR_luks_key_secret_key:-}" | |
| mask "${TF_VAR_luks_key_url:-}" | |
| - name: Resolve snapshot input | |
| working-directory: terraform/snapshots | |
| run: | | |
| set -euo pipefail | |
| state_prefix="${SNAPSHOTS_TFSTATE_KEY}" | |
| if [[ "${state_prefix}" == *.tfstate ]]; then | |
| state_prefix="$(dirname "${state_prefix}")" | |
| fi | |
| state_prefix="${state_prefix%/}" | |
| if [[ -z "${state_prefix}" || "${state_prefix}" == "." ]]; then | |
| state_prefix="terraform/snapshots" | |
| fi | |
| terraform init -input=false \ | |
| -backend-config="bucket=${SNAPSHOTS_TFSTATE_BUCKET}" \ | |
| -backend-config="key=${state_prefix}/${{ matrix.target_key }}.tfstate" \ | |
| -backend-config="region=${SNAPSHOTS_TFSTATE_REGION}" \ | |
| -backend-config="endpoint=${SNAPSHOTS_TFSTATE_ENDPOINT}" \ | |
| -backend-config="skip_requesting_account_id=true" \ | |
| -backend-config="skip_credentials_validation=true" \ | |
| -backend-config="skip_metadata_api_check=true" \ | |
| -backend-config="skip_region_validation=true" \ | |
| -backend-config="force_path_style=true" | |
| snapshot_source="${RESTORE_SNAPSHOT_SOURCE:-auto}" | |
| if [[ "${snapshot_source}" != "auto" && "${snapshot_source}" != "manual" ]]; then | |
| echo "snapshot_source must be auto or manual" >&2 | |
| exit 1 | |
| fi | |
| if [[ "${snapshot_source}" == "manual" ]]; then | |
| manual_name="${RESTORE_MANUAL_SNAPSHOT_NAME:-}" | |
| if [[ -z "${manual_name}" ]]; then | |
| echo "manual_snapshot_name is required when snapshot_source=manual" >&2 | |
| exit 1 | |
| fi | |
| if [[ "${manual_name}" == *"/"* || "${manual_name}" == *".."* ]]; then | |
| echo "manual_snapshot_name must be a snapshot key without path separators" >&2 | |
| exit 1 | |
| fi | |
| manual_key="${{ matrix.target_key }}/${manual_name}" | |
| snapshot_id="$(terraform output -json manual_snapshot_ids 2>/dev/null | jq -r --arg key "${manual_key}" '.[$key] // empty')" | |
| else | |
| snapshot_id="$(terraform output -json latest_snapshot_ids 2>/dev/null | jq -r --arg target "${{ matrix.target_key }}" '.[$target] // empty')" | |
| fi | |
| if [[ -z "${snapshot_id}" ]]; then | |
| echo "No ${snapshot_source} snapshot found for target ${{ matrix.target_key }}" >&2 | |
| exit 1 | |
| fi | |
| echo "::add-mask::${snapshot_id}" | |
| echo "TF_VAR_snapshot_id=${snapshot_id}" >> "${GITHUB_ENV}" | |
| - name: Resolve runner SSH CIDR | |
| run: | | |
| set -euo pipefail | |
| runner_ip="$(curl -fsS https://checkip.amazonaws.com | tr -d '[:space:]')" | |
| if ! [[ "${runner_ip}" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then | |
| echo "Could not resolve a valid runner IPv4 address" >&2 | |
| exit 1 | |
| fi | |
| echo "::add-mask::${runner_ip}" | |
| echo "TF_VAR_ssh_cidr=${runner_ip}/32" >> "${GITHUB_ENV}" | |
| - name: Generate ephemeral SSH key | |
| id: ssh | |
| run: | | |
| set -euo pipefail | |
| key_path="${RUNNER_TEMP}/restore-drill-${{ matrix.target_key }}" | |
| ssh-keygen -t ed25519 -N "" -f "${key_path}" -C "restore-drill-${GITHUB_RUN_ID}-${{ matrix.target_key }}" >/dev/null | |
| public_key="$(cat "${key_path}.pub")" | |
| echo "::add-mask::${public_key}" | |
| echo "key_path=${key_path}" >> "${GITHUB_OUTPUT}" | |
| echo "TF_VAR_ssh_public_key=${public_key}" >> "${GITHUB_ENV}" | |
| - name: Terraform init (restore drill state) | |
| id: restore_init | |
| run: | | |
| terraform init -input=false \ | |
| -backend-config="bucket=${SNAPSHOTS_TFSTATE_BUCKET}" \ | |
| -backend-config="key=terraform/snapshot-restore-drill/${{ matrix.target_key }}.tfstate" \ | |
| -backend-config="region=${SNAPSHOTS_TFSTATE_REGION}" \ | |
| -backend-config="endpoint=${SNAPSHOTS_TFSTATE_ENDPOINT}" \ | |
| -backend-config="skip_requesting_account_id=true" \ | |
| -backend-config="skip_credentials_validation=true" \ | |
| -backend-config="skip_metadata_api_check=true" \ | |
| -backend-config="skip_region_validation=true" \ | |
| -backend-config="force_path_style=true" | |
| - name: Prepare restore drill metadata | |
| run: echo "TF_VAR_created_at=$(date -u +%Y%m%dT%H%M%SZ)" >> "${GITHUB_ENV}" | |
| - name: Terraform apply | |
| id: apply | |
| run: | | |
| set -euo pipefail | |
| redact_tf_output() { | |
| sed -E \ | |
| -e 's/[a-z]{2}-[a-z]+-[0-9]\/[0-9a-fA-F-]{36}/<scw-id>/g' \ | |
| -e 's/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/<uuid>/g' | |
| } | |
| terraform apply -auto-approve -no-color -parallelism=1 2>&1 | redact_tf_output | |
| - name: Wait for restore drill | |
| id: wait | |
| run: | | |
| set -euo pipefail | |
| if ! command -v aws >/dev/null 2>&1; then | |
| sudo apt-get update -y | |
| sudo apt-get install -y awscli | |
| fi | |
| key_path="${{ steps.ssh.outputs.key_path }}" | |
| result_uri="s3://${TF_VAR_result_bucket_name}/${TF_VAR_result_object_key}" | |
| tail_remote_logs() { | |
| local ip | |
| ip="$(terraform output -raw server_ip 2>/dev/null || true)" | |
| if [[ -z "${ip}" || ! -f "${key_path}" ]]; then | |
| return | |
| fi | |
| echo "::add-mask::${ip}" | |
| ssh -i "${key_path}" \ | |
| -o StrictHostKeyChecking=no \ | |
| -o UserKnownHostsFile=/dev/null \ | |
| -o ConnectTimeout=10 \ | |
| -o ServerAliveInterval=10 \ | |
| -o LogLevel=ERROR \ | |
| "root@${ip}" 'tail -n 200 /var/log/terra-restore-drill.log || true' || true | |
| } | |
| deadline=$((SECONDS + 2700)) | |
| last_status="" | |
| while [ "${SECONDS}" -lt "${deadline}" ]; do | |
| if aws --endpoint-url="${SNAPSHOTS_TFSTATE_ENDPOINT}" s3 cp "${result_uri}" status.json >/dev/null 2>&1 && jq -e . status.json >/dev/null 2>&1; then | |
| status="$(jq -r '.status // "unknown"' status.json)" | |
| phase="$(jq -r '.phase // "unknown"' status.json)" | |
| message="$(jq -r '.message // ""' status.json)" | |
| current="${status}:${phase}:${message}" | |
| if [[ "${current}" != "${last_status}" ]]; then | |
| echo "${current}" | |
| last_status="${current}" | |
| fi | |
| if [[ "${status}" == "success" ]]; then | |
| cat status.json | |
| exit 0 | |
| fi | |
| if [[ "${status}" == "failed" ]]; then | |
| cat status.json | |
| tail_remote_logs | |
| exit 1 | |
| fi | |
| fi | |
| sleep 15 | |
| done | |
| echo "Restore drill timed out for ${{ matrix.target_key }}" >&2 | |
| tail_remote_logs | |
| exit 1 | |
| - name: Collect restore drill logs | |
| if: always() | |
| run: | | |
| set +e | |
| ip="$(terraform output -raw server_ip 2>/dev/null)" | |
| key_path="${{ steps.ssh.outputs.key_path }}" | |
| if [[ -n "${ip}" && -f "${key_path}" ]]; then | |
| echo "::add-mask::${ip}" | |
| ssh -i "${key_path}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -o LogLevel=ERROR "root@${ip}" \ | |
| 'echo "--- status ---"; cat /var/lib/terra-restore-drill/status.json 2>/dev/null || true; echo "--- restore log ---"; tail -n 200 /var/log/terra-restore-drill.log 2>/dev/null || true' || true | |
| fi | |
| - name: Terraform destroy | |
| id: destroy | |
| if: always() | |
| run: | | |
| set +e | |
| redact_tf_output() { | |
| sed -E \ | |
| -e 's/[a-z]{2}-[a-z]+-[0-9]\/[0-9a-fA-F-]{36}/<scw-id>/g' \ | |
| -e 's/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/<uuid>/g' | |
| } | |
| terraform destroy -auto-approve -no-color -parallelism=1 2>&1 | redact_tf_output | |
| rc=${PIPESTATUS[0]} | |
| if [[ "${rc}" -eq 0 ]]; then | |
| echo "success" > destroy-status.txt | |
| else | |
| echo "failed" > destroy-status.txt | |
| fi | |
| exit "${rc}" | |
| - name: Verify restore drill cleanup | |
| id: cleanup_verify | |
| if: always() | |
| env: | |
| RESTORE_INIT_OUTCOME: ${{ steps.restore_init.outcome }} | |
| DESTROY_OUTCOME: ${{ steps.destroy.outcome }} | |
| run: | | |
| set +e | |
| if [[ "${RESTORE_INIT_OUTCOME:-}" != "success" ]]; then | |
| echo "not-run" > cleanup-verify-status.txt | |
| exit 0 | |
| fi | |
| if [[ "${DESTROY_OUTCOME:-}" != "success" ]]; then | |
| echo "failed" > cleanup-verify-status.txt | |
| echo "Terraform destroy did not complete successfully." >&2 | |
| exit 1 | |
| fi | |
| remaining="$(terraform state list 2>/dev/null | sed '/^[[:space:]]*$/d' | wc -l | tr -d '[:space:]')" | |
| if [[ "${remaining}" == "0" ]]; then | |
| echo "success" > cleanup-verify-status.txt | |
| exit 0 | |
| fi | |
| echo "failed" > cleanup-verify-status.txt | |
| echo "Terraform restore-drill state still contains ${remaining} managed resource(s)." >&2 | |
| exit 1 | |
| - name: Write restore drill result | |
| if: always() | |
| env: | |
| TARGET_KEY: ${{ matrix.target_key }} | |
| RESTORE_INIT_OUTCOME: ${{ steps.restore_init.outcome }} | |
| APPLY_OUTCOME: ${{ steps.apply.outcome }} | |
| WAIT_OUTCOME: ${{ steps.wait.outcome }} | |
| DESTROY_OUTCOME: ${{ steps.destroy.outcome }} | |
| run: | | |
| set -euo pipefail | |
| cleanup="$(cat cleanup-verify-status.txt 2>/dev/null || true)" | |
| if [[ -z "${cleanup}" ]]; then | |
| cleanup="$(cat destroy-status.txt 2>/dev/null || true)" | |
| fi | |
| if [[ -z "${cleanup}" ]]; then | |
| case "${DESTROY_OUTCOME:-}" in | |
| success) cleanup="success" ;; | |
| failure) cleanup="failed" ;; | |
| *) cleanup="not-run" ;; | |
| esac | |
| fi | |
| if [[ -s status.json ]] && jq -e . status.json >/dev/null 2>&1; then | |
| jq \ | |
| --arg target "${TARGET_KEY}" \ | |
| --arg cleanup "${cleanup}" \ | |
| '.target = $target | .cleanup = $cleanup | .checks = (.checks // [])' \ | |
| status.json > "restore-result-${TARGET_KEY}.json" | |
| else | |
| phase="unknown" | |
| message="restore drill did not produce a verifier status file" | |
| if [[ "${RESTORE_INIT_OUTCOME:-}" == "failure" ]]; then | |
| phase="terraform-init" | |
| message="Terraform backend initialization failed before verifier startup" | |
| elif [[ "${APPLY_OUTCOME:-}" == "failure" ]]; then | |
| phase="terraform-apply" | |
| message="Terraform apply failed before verifier health checks completed" | |
| elif [[ "${WAIT_OUTCOME:-}" == "failure" ]]; then | |
| phase="wait-for-health-checks" | |
| message="Verifier health checks did not complete successfully" | |
| fi | |
| jq -n \ | |
| --arg target "${TARGET_KEY}" \ | |
| --arg phase "${phase}" \ | |
| --arg status "failed" \ | |
| --arg message "${message}" \ | |
| --arg cleanup "${cleanup}" \ | |
| --arg timestamp "$(date -u +%FT%TZ)" \ | |
| '{target: $target, phase: $phase, status: $status, message: $message, cleanup: $cleanup, timestamp: $timestamp, checks: []}' \ | |
| > "restore-result-${TARGET_KEY}.json" | |
| fi | |
| jq -r '"Restore result: \(.target) status=\(.status) phase=\(.phase) cleanup=\(.cleanup)"' "restore-result-${TARGET_KEY}.json" | |
| - name: Upload restore drill result | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: restore-result-${{ matrix.target_key }} | |
| path: terraform/snapshot-restore-drill/restore-result-${{ matrix.target_key }}.json | |
| if-no-files-found: error | |
| retention-days: 7 | |
| - name: Remove remote restore drill result | |
| if: always() | |
| run: | | |
| set +e | |
| if command -v aws >/dev/null 2>&1; then | |
| aws --endpoint-url="${SNAPSHOTS_TFSTATE_ENDPOINT}" \ | |
| s3 rm "s3://${TF_VAR_result_bucket_name}/${TF_VAR_result_object_key}" \ | |
| >/dev/null 2>&1 || true | |
| fi | |
| notify: | |
| needs: | |
| - janitor | |
| - resolve | |
| - drill | |
| if: always() && github.ref_name == 'env/prod' | |
| runs-on: ubuntu-latest | |
| environment: prod | |
| env: | |
| SNAPSHOTS_DISCORD_WEBHOOK_URL: ${{ secrets.SNAPSHOTS_DISCORD_WEBHOOK_URL }} | |
| steps: | |
| - name: Mask sensitive values | |
| run: | | |
| if [[ -n "${SNAPSHOTS_DISCORD_WEBHOOK_URL:-}" ]]; then | |
| echo "::add-mask::${SNAPSHOTS_DISCORD_WEBHOOK_URL}" | |
| fi | |
| - name: Download restore drill results | |
| if: env.SNAPSHOTS_DISCORD_WEBHOOK_URL != '' | |
| uses: actions/download-artifact@v8 | |
| continue-on-error: true | |
| with: | |
| pattern: restore-result-* | |
| path: restore-results | |
| merge-multiple: true | |
| - name: Notify Discord (restore drill status) | |
| if: env.SNAPSHOTS_DISCORD_WEBHOOK_URL != '' | |
| env: | |
| JANITOR_RESULT: ${{ needs.janitor.result }} | |
| JANITOR_SUMMARY: ${{ needs.janitor.outputs.summary }} | |
| RESOLVE_RESULT: ${{ needs.resolve.result }} | |
| DRILL_RESULT: ${{ needs.drill.result }} | |
| run: | | |
| set -euo pipefail | |
| mkdir -p restore-results | |
| mapfile -t result_files < <(find restore-results -type f -name '*.json' | sort) | |
| if [[ "${#result_files[@]}" -gt 0 ]]; then | |
| results="$(jq -s '.' "${result_files[@]}")" | |
| else | |
| results="[]" | |
| fi | |
| run_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| trigger="${{ github.event_name }}" | |
| schedule="${{ github.event.schedule }}" | |
| branch="${{ github.ref_name }}" | |
| actor="${{ github.actor }}" | |
| ts="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" | |
| janitor_summary="${JANITOR_SUMMARY:-No janitor details were reported.}" | |
| trigger_label="${trigger}" | |
| if [[ "${trigger}" == "schedule" && -n "${schedule}" ]]; then | |
| trigger_label="cron (${schedule})" | |
| elif [[ "${trigger}" == "workflow_dispatch" ]]; then | |
| trigger_label="dispatch" | |
| fi | |
| status="success" | |
| title="Snapshot restore drill passed" | |
| color=3066993 | |
| if [[ "${JANITOR_RESULT}" != "success" || "${RESOLVE_RESULT}" != "success" || "${DRILL_RESULT}" != "success" ]]; then | |
| status="failed" | |
| title="Snapshot restore drill failed" | |
| color=15158332 | |
| fi | |
| target_lines="$(jq -r ' | |
| def value_or_unknown($v): if ($v == null or $v == "") then "unknown" else $v end; | |
| def truncate($max): if length > $max then .[0:$max] + "..." else . end; | |
| def total_checks: ((.checks // []) | length); | |
| def passed_checks: ((.checks // []) | map(select(.status == "passed")) | length); | |
| def failed_checks: ((.checks // []) | map(select(.status != "passed")) | map(.name) | join(", ")); | |
| def target_line: | |
| (if (.status == "success" and .phase == "complete" and .cleanup == "success") then "[ok]" else "[fail]" end) | |
| + " `" + .target + "` " | |
| + "health checks " + (passed_checks | tostring) + " passed of " + (total_checks | tostring) | |
| + ", Terraform cleanup " + value_or_unknown(.cleanup) | |
| + (if (.status == "success" and .phase == "complete" and .cleanup == "success") then | |
| "" | |
| else | |
| ", phase " + value_or_unknown(.phase) | |
| + ", " + ((.message // "restore drill failed") | truncate(120)) | |
| + (if failed_checks == "" then "" else ", failed: " + failed_checks end) | |
| end); | |
| if length == 0 then | |
| "No restore result artifacts were available." | |
| else | |
| map(target_line) | join("\n") | |
| end | |
| ' <<< "${results}")" | |
| payload="$(jq -n \ | |
| --arg title "${title}" \ | |
| --arg status "${status}" \ | |
| --arg trigger "${trigger_label}" \ | |
| --arg branch "${branch}" \ | |
| --arg actor "${actor}" \ | |
| --arg run_url "${run_url}" \ | |
| --arg target_lines "${target_lines}" \ | |
| --arg timestamp "${ts}" \ | |
| --arg janitor_result "${JANITOR_RESULT}" \ | |
| --arg janitor_summary "${janitor_summary}" \ | |
| --arg resolve_result "${RESOLVE_RESULT}" \ | |
| --arg drill_result "${DRILL_RESULT}" \ | |
| --argjson color "${color}" \ | |
| '{ | |
| username: "TerraInfra", | |
| embeds: [ | |
| { | |
| title: $title, | |
| color: $color, | |
| fields: ([ | |
| {name: "Status", value: $status, inline: true}, | |
| {name: "Trigger", value: $trigger, inline: true}, | |
| {name: "Branch", value: $branch, inline: true}, | |
| {name: "Janitor", value: ($janitor_result + "\n" + $janitor_summary), inline: false}, | |
| {name: "Resolve job", value: $resolve_result, inline: true}, | |
| {name: "Drill jobs", value: $drill_result, inline: true}, | |
| {name: "GitHub run", value: ("[Open run](" + $run_url + ")"), inline: true}, | |
| {name: "Actor", value: $actor, inline: true}, | |
| {name: "Targets", value: (if ($target_lines | length) > 0 then $target_lines else "n/a" end), inline: false} | |
| ]), | |
| timestamp: $timestamp | |
| } | |
| ] | |
| }')" | |
| { | |
| echo "## Snapshot restore drill" | |
| echo | |
| echo "- Status: ${status}" | |
| echo "- Janitor job: ${JANITOR_RESULT}" | |
| echo "- Janitor sweep: ${janitor_summary}" | |
| echo "- Resolve job: ${RESOLVE_RESULT}" | |
| echo "- Drill jobs: ${DRILL_RESULT}" | |
| echo | |
| echo "| Target | Restore result | Health checks | Terraform cleanup | Phase | Message |" | |
| echo "| --- | --- | ---: | --- | --- | --- |" | |
| jq -r ' | |
| def cell($v): (($v // "-") | tostring | gsub("[|]"; "/") | gsub("\n"; " ") | if length > 120 then .[0:117] + "..." else . end); | |
| def total_checks: ((.checks // []) | length); | |
| def passed_checks: ((.checks // []) | map(select(.status == "passed")) | length); | |
| def check_summary: | |
| if total_checks == 0 then | |
| "no health checks reported" | |
| else | |
| (passed_checks | tostring) + " passed of " + (total_checks | tostring) | |
| end; | |
| if length == 0 then | |
| "| n/a | failed | no result artifact | unknown | unknown | No restore result artifacts were available. |" | |
| else | |
| .[] | |
| | "| `" + .target + "` | " + cell(.status) + " | " + check_summary + " | " + cell(.cleanup) + " | " + cell(.phase) + " | " + cell(.message) + " |" | |
| end | |
| ' <<< "${results}" | |
| } >> "${GITHUB_STEP_SUMMARY}" | |
| curl -fsS -H 'Content-Type: application/json' -d "${payload}" "${SNAPSHOTS_DISCORD_WEBHOOK_URL}" || echo "Discord webhook notification failed" >&2 |