Skip to content

terraform-snapshots #241

terraform-snapshots

terraform-snapshots #241

name: terraform-snapshots
on:
schedule:
- cron: "0 3 * * *"
workflow_dispatch:
inputs:
project_id:
description: "Scaleway Project ID"
required: true
default: "4ca4f592-005a-45dc-a016-91bd11dd5348"
target_names:
description: "Snapshot targets: all or comma-separated keys (node1-main,jenkins,observability,rocket,vpn-control)"
required: true
default: "all"
manual_ttl_hours:
description: "TTL (hours) for manual snapshots (default 24)"
required: false
default: "24"
manual_snapshot_name:
description: "Optional manual snapshot key (auto-generated when empty)"
required: false
default: ""
tfstate_bucket:
description: "Object Storage bucket for Terraform state"
required: true
default: "terra-snapshots-state"
tfstate_key:
description: "Legacy state key or per-target state prefix"
required: true
default: "terraform/snapshots/terraform.tfstate"
tfstate_region:
description: "Object Storage region"
required: true
default: "fr-par"
tfstate_endpoint:
description: "Object Storage endpoint"
required: true
default: "https://s3.fr-par.scw.cloud"
concurrency:
group: terraform-snapshots-${{ github.ref_name }}
cancel-in-progress: false
permissions:
contents: read
actions: read
jobs:
resolve:
if: github.ref_name == 'env/prod'
runs-on: ubuntu-latest
environment: prod
outputs:
matrix: ${{ steps.matrix.outputs.matrix }}
target_keys: ${{ steps.matrix.outputs.target_keys }}
snapshot_kind: ${{ steps.matrix.outputs.snapshot_kind }}
requested_manual_snapshot_name: ${{ steps.matrix.outputs.requested_manual_snapshot_name }}
retention: ${{ steps.matrix.outputs.retention }}
env:
SNAPSHOTS_TARGET_NAMES: ${{ github.event.inputs.target_names || 'all' }}
SNAPSHOTS_MANUAL_TTL_HOURS: ${{ github.event.inputs.manual_ttl_hours || '24' }}
SNAPSHOTS_MANUAL_SNAPSHOT_NAME: ${{ github.event.inputs.manual_snapshot_name || '' }}
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v6
with:
submodules: false
fetch-depth: 1
- name: Resolve snapshot matrix
id: matrix
run: bash .github/scripts/resolve-snapshot-matrix.sh
snapshots:
needs: resolve
if: github.ref_name == 'env/prod'
runs-on: ubuntu-latest
environment: prod
strategy:
fail-fast: false
max-parallel: 4
matrix: ${{ fromJson(needs.resolve.outputs.matrix) }}
env:
TF_IN_AUTOMATION: "true"
TF_INPUT: "false"
SCW_ACCESS_KEY: ${{ secrets.SNAPSHOTS_SCW_ACCESS_KEY }}
SCW_SECRET_KEY: ${{ secrets.SNAPSHOTS_SCW_SECRET_KEY }}
AWS_ACCESS_KEY_ID: ${{ secrets.SNAPSHOTS_SCW_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.SNAPSHOTS_SCW_SECRET_KEY }}
SNAPSHOTS_TFSTATE_BUCKET: ${{ github.event.inputs.tfstate_bucket || 'terra-snapshots-state' }}
SNAPSHOTS_TFSTATE_ENDPOINT: ${{ github.event.inputs.tfstate_endpoint || 'https://s3.fr-par.scw.cloud' }}
SNAPSHOTS_TFSTATE_KEY: ${{ github.event.inputs.tfstate_key || 'terraform/snapshots/terraform.tfstate' }}
SNAPSHOTS_TFSTATE_REGION: ${{ github.event.inputs.tfstate_region || 'fr-par' }}
SNAPSHOTS_MANUAL_TTL_HOURS: ${{ github.event.inputs.manual_ttl_hours || '24' }}
TF_VAR_project_id: ${{ github.event.inputs.project_id || '4ca4f592-005a-45dc-a016-91bd11dd5348' }}
TF_VAR_env_name: prod
defaults:
run:
shell: bash
working-directory: terraform/snapshots
steps:
- uses: actions/checkout@v6
with:
submodules: false
fetch-depth: 1
- uses: hashicorp/setup-terraform@v4
with:
terraform_version: 1.6.6
terraform_wrapper: false
- name: Mask sensitive values
run: |
mask() {
if [[ -n "${1:-}" ]]; then
echo "::add-mask::$1"
fi
}
mask "${SCW_ACCESS_KEY:-}"
mask "${SCW_SECRET_KEY:-}"
mask "${AWS_ACCESS_KEY_ID:-}"
mask "${AWS_SECRET_ACCESS_KEY:-}"
- name: Resolve target state key
run: |
set -euo pipefail
state_prefix="${SNAPSHOTS_TFSTATE_KEY}"
if [[ "${state_prefix}" == *.tfstate ]]; then
state_prefix="$(dirname "${state_prefix}")"
fi
state_prefix="${state_prefix%/}"
if [[ -z "${state_prefix}" || "${state_prefix}" == "." ]]; then
state_prefix="terraform/snapshots"
fi
echo "SNAPSHOT_STATE_KEY=${state_prefix}/${{ matrix.target_key }}.tfstate" >> "${GITHUB_ENV}"
- name: Terraform init (remote state)
id: init
run: |
terraform init -input=false \
-backend-config="bucket=${SNAPSHOTS_TFSTATE_BUCKET}" \
-backend-config="key=${SNAPSHOT_STATE_KEY}" \
-backend-config="region=${SNAPSHOTS_TFSTATE_REGION}" \
-backend-config="endpoint=${SNAPSHOTS_TFSTATE_ENDPOINT}" \
-backend-config="skip_requesting_account_id=true" \
-backend-config="skip_credentials_validation=true" \
-backend-config="skip_metadata_api_check=true" \
-backend-config="skip_region_validation=true" \
-backend-config="force_path_style=true"
- name: Load Terraform state inputs
id: state
run: |
set -euo pipefail
manual_snapshots="{}"
if raw="$(terraform output -json manual_snapshots 2>/dev/null)"; then
if parsed="$(echo "${raw}" | jq -c . 2>/dev/null)"; then
if [[ "${parsed}" != "null" && -n "${parsed}" ]]; then
manual_snapshots="${parsed}"
fi
fi
fi
auto_snapshot_trigger=""
if out="$(terraform output -raw auto_snapshot_trigger 2>/dev/null)"; then
auto_snapshot_trigger="${out}"
fi
if [[ -z "${auto_snapshot_trigger}" ]]; then
auto_snapshot_trigger="$(terraform state pull 2>/dev/null | jq -r '
.resources[]
| select(.type == "time_static" and .name == "snapshot_trigger")
| .instances[0].attributes.triggers.bucket
// empty
' 2>/dev/null || true)"
fi
if [[ "${auto_snapshot_trigger}" == "null" ]]; then
auto_snapshot_trigger=""
fi
if [[ -n "${auto_snapshot_trigger}" ]] && ! [[ "${auto_snapshot_trigger}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then
auto_snapshot_trigger=""
fi
echo "manual_snapshots=${manual_snapshots}" >> "${GITHUB_OUTPUT}"
echo "auto_snapshot_trigger=${auto_snapshot_trigger}" >> "${GITHUB_OUTPUT}"
- name: Prepare Terraform variables
id: vars
env:
SNAPSHOT_KIND: ${{ needs.resolve.outputs.snapshot_kind }}
REQUESTED_MANUAL_SNAPSHOT_NAME: ${{ needs.resolve.outputs.requested_manual_snapshot_name }}
run: |
set -euo pipefail
manual_snapshots='${{ steps.state.outputs.manual_snapshots }}'
auto_snapshot_trigger_state='${{ steps.state.outputs.auto_snapshot_trigger }}'
target_key='${{ matrix.target_key }}'
target_tags='${{ toJson(matrix.tags) }}'
snapshot_targets="$(jq -cn \
--arg key "${target_key}" \
--arg volume_name '${{ matrix.volume_name }}' \
--arg name_prefix '${{ matrix.name_prefix }}' \
--argjson tags "${target_tags}" \
'{($key): {volume_name: $volume_name, name_prefix: $name_prefix, tags: $tags}}')"
selected_target_keys="$(jq -cn --arg key "${target_key}" '[$key]')"
auto_snapshot_trigger_refresh="${auto_snapshot_trigger_state}"
if [[ -z "${auto_snapshot_trigger_refresh}" || "${auto_snapshot_trigger_refresh}" == "null" ]] || ! [[ "${auto_snapshot_trigger_refresh}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then
auto_snapshot_trigger_refresh="$(date -u +%Y-%m-%d)"
fi
if terraform state pull >/dev/null 2>&1; then
redact_tf_output() {
sed -E \
-e 's/[a-z]{2}-[a-z]+-[0-9]\/[0-9a-fA-F-]{36}/<scw-id>/g' \
-e 's/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/<uuid>/g'
}
jq -n \
--arg auto_snapshot_trigger "${auto_snapshot_trigger_refresh}" \
--arg requested_manual_snapshot_name "" \
--argjson manual_snapshots "${manual_snapshots}" \
--argjson snapshot_targets "${snapshot_targets}" \
'{auto_snapshot_trigger: $auto_snapshot_trigger, requested_manual_snapshot_name: $requested_manual_snapshot_name, manual_snapshots: $manual_snapshots, snapshot_targets: $snapshot_targets}' \
> refresh.auto.tfvars.json
terraform apply -refresh-only -auto-approve -no-color -var-file=refresh.auto.tfvars.json 2>&1 | redact_tf_output
keep_targets="$(terraform output -json manual_snapshot_request_targets 2>/dev/null | jq -c . 2>/dev/null || echo '{}')"
if [[ "${keep_targets}" == "{}" ]]; then
keep_keys="$(terraform output -json manual_snapshot_request_keys 2>/dev/null | jq -c . 2>/dev/null || echo '[]')"
keep_targets="$(jq -cn --argjson keep "${keep_keys}" --argjson targets "${selected_target_keys}" '$keep | map({(.): $targets}) | add // {}')"
fi
manual_snapshots="$(echo "${manual_snapshots}" | jq -c --argjson keep_targets "${keep_targets}" '
with_entries(
select($keep_targets[.key] != null)
| .value.targets = $keep_targets[.key]
)
')"
else
echo "No Terraform state yet; skipping refresh-only manual snapshot pruning."
fi
auto_snapshot_trigger="${auto_snapshot_trigger_state}"
if [[ "${{ github.event_name }}" == "schedule" ]]; then
auto_snapshot_trigger="$(date -u +%Y-%m-%d)"
fi
if [[ -z "${auto_snapshot_trigger}" || "${auto_snapshot_trigger}" == "null" ]]; then
auto_snapshot_trigger="$(date -u +%Y-%m-%d)"
fi
if ! [[ "${auto_snapshot_trigger}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then
auto_snapshot_trigger="$(date -u +%Y-%m-%d)"
fi
if [[ "${SNAPSHOT_KIND}" == "manual" ]]; then
created_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
manual_snapshots="$(echo "${manual_snapshots}" | jq -c \
--arg name "${REQUESTED_MANUAL_SNAPSHOT_NAME}" \
--arg created_at "${created_at}" \
--argjson ttl_hours "${SNAPSHOTS_MANUAL_TTL_HOURS}" \
--argjson targets "${selected_target_keys}" \
'. + {($name): {created_at: $created_at, ttl_hours: $ttl_hours, targets: $targets}}')"
fi
jq -n \
--arg auto_snapshot_trigger "${auto_snapshot_trigger}" \
--arg requested_manual_snapshot_name "${REQUESTED_MANUAL_SNAPSHOT_NAME}" \
--argjson manual_snapshots "${manual_snapshots}" \
--argjson snapshot_targets "${snapshot_targets}" \
'{auto_snapshot_trigger: $auto_snapshot_trigger, requested_manual_snapshot_name: $requested_manual_snapshot_name, manual_snapshots: $manual_snapshots, snapshot_targets: $snapshot_targets}' \
> ci.auto.tfvars.json
echo "auto_snapshot_trigger=${auto_snapshot_trigger}" >> "${GITHUB_OUTPUT}"
- name: Terraform apply
id: apply
run: |
set -euo pipefail
redact_tf_output() {
sed -E \
-e 's/[a-z]{2}-[a-z]+-[0-9]\/[0-9a-fA-F-]{36}/<scw-id>/g' \
-e 's/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/<uuid>/g'
}
terraform apply -auto-approve -no-color -var-file=ci.auto.tfvars.json 2>&1 | redact_tf_output | tee terraform-apply.log
- name: Capture snapshot info
id: snapshot
env:
SNAPSHOT_KIND: ${{ needs.resolve.outputs.snapshot_kind }}
RETENTION: ${{ needs.resolve.outputs.retention }}
run: |
set -euo pipefail
target_key='${{ matrix.target_key }}'
if [[ "${SNAPSHOT_KIND}" == "auto" ]]; then
snapshot_id="$(terraform output -json latest_snapshot_ids 2>/dev/null | jq -r --arg target "${target_key}" '.[$target] // empty')"
snapshot_name="$(terraform output -json latest_snapshot_names 2>/dev/null | jq -r --arg target "${target_key}" '.[$target] // empty')"
else
snapshot_id="$(terraform output -json requested_manual_snapshot_ids 2>/dev/null | jq -r --arg target "${target_key}" '.[$target] // empty')"
snapshot_name="$(terraform output -json requested_manual_snapshot_names 2>/dev/null | jq -r --arg target "${target_key}" '.[$target] // empty')"
fi
if [[ -z "${snapshot_id}" || "${snapshot_id}" == "null" ]]; then
echo "Snapshot ID is empty for target ${target_key} (kind=${SNAPSHOT_KIND})" >&2
exit 1
fi
echo "::add-mask::${snapshot_id}"
jq -n \
--arg target "${target_key}" \
--arg status "success" \
--arg kind "${SNAPSHOT_KIND}" \
--arg snapshot_name "${snapshot_name}" \
--arg retention "${RETENTION}" \
--arg phase "complete" \
--arg timestamp "$(date -u +%FT%TZ)" \
'{target: $target, status: $status, kind: $kind, snapshot_name: $snapshot_name, retention: $retention, phase: $phase, timestamp: $timestamp}' \
> "snapshot-result-${target_key}.json"
- name: Write snapshot result
if: always()
env:
TARGET_KEY: ${{ matrix.target_key }}
SNAPSHOT_KIND: ${{ needs.resolve.outputs.snapshot_kind }}
RETENTION: ${{ needs.resolve.outputs.retention }}
INIT_OUTCOME: ${{ steps.init.outcome }}
APPLY_OUTCOME: ${{ steps.apply.outcome }}
CAPTURE_OUTCOME: ${{ steps.snapshot.outcome }}
run: |
set -euo pipefail
result_file="snapshot-result-${TARGET_KEY}.json"
if [[ -s "${result_file}" ]] && jq -e . "${result_file}" >/dev/null 2>&1; then
exit 0
fi
phase="unknown"
message="snapshot job did not produce a result file"
if [[ "${INIT_OUTCOME:-}" == "failure" ]]; then
phase="terraform-init"
message="Terraform backend initialization failed"
elif [[ "${APPLY_OUTCOME:-}" == "failure" ]]; then
phase="terraform-apply"
message="Terraform apply failed"
elif [[ "${CAPTURE_OUTCOME:-}" == "failure" ]]; then
phase="capture-snapshot-info"
message="Snapshot output capture failed"
fi
jq -n \
--arg target "${TARGET_KEY}" \
--arg status "failed" \
--arg kind "${SNAPSHOT_KIND}" \
--arg retention "${RETENTION}" \
--arg phase "${phase}" \
--arg message "${message}" \
--arg timestamp "$(date -u +%FT%TZ)" \
'{target: $target, status: $status, kind: $kind, retention: $retention, phase: $phase, message: $message, timestamp: $timestamp}' \
> "${result_file}"
- name: Upload snapshot result
if: always()
uses: actions/upload-artifact@v7
with:
name: snapshot-result-${{ matrix.target_key }}
path: terraform/snapshots/snapshot-result-${{ matrix.target_key }}.json
if-no-files-found: error
retention-days: 7
notify:
needs:
- resolve
- snapshots
if: always() && github.ref_name == 'env/prod'
runs-on: ubuntu-latest
environment: prod
env:
SNAPSHOTS_DISCORD_WEBHOOK_URL: ${{ secrets.SNAPSHOTS_DISCORD_WEBHOOK_URL }}
steps:
- name: Mask sensitive values
run: |
if [[ -n "${SNAPSHOTS_DISCORD_WEBHOOK_URL:-}" ]]; then
echo "::add-mask::${SNAPSHOTS_DISCORD_WEBHOOK_URL}"
fi
- name: Download snapshot results
if: env.SNAPSHOTS_DISCORD_WEBHOOK_URL != ''
uses: actions/download-artifact@v8
continue-on-error: true
with:
pattern: snapshot-result-*
path: snapshot-results
merge-multiple: true
- name: Notify Discord (snapshot status)
if: env.SNAPSHOTS_DISCORD_WEBHOOK_URL != ''
env:
RESOLVE_RESULT: ${{ needs.resolve.result }}
SNAPSHOTS_RESULT: ${{ needs.snapshots.result }}
SNAPSHOT_KIND: ${{ needs.resolve.outputs.snapshot_kind }}
TARGET_KEYS: ${{ needs.resolve.outputs.target_keys }}
RETENTION: ${{ needs.resolve.outputs.retention }}
run: |
set -euo pipefail
mkdir -p snapshot-results
mapfile -t result_files < <(find snapshot-results -type f -name '*.json' | sort)
if [[ "${#result_files[@]}" -gt 0 ]]; then
results="$(jq -s '.' "${result_files[@]}")"
else
results="[]"
fi
run_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
trigger="${{ github.event_name }}"
schedule="${{ github.event.schedule }}"
branch="${{ github.ref_name }}"
actor="${{ github.actor }}"
ts="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
trigger_label="${trigger}"
if [[ "${trigger}" == "schedule" && -n "${schedule}" ]]; then
trigger_label="cron (${schedule})"
elif [[ "${trigger}" == "workflow_dispatch" ]]; then
trigger_label="dispatch"
fi
status="success"
title="Scaleway snapshot created (${SNAPSHOT_KIND})"
color=3066993
if [[ "${SNAPSHOT_KIND}" == "auto" ]]; then
color=3447003
fi
if [[ "${RESOLVE_RESULT}" != "success" || "${SNAPSHOTS_RESULT}" != "success" ]]; then
status="failed"
title="Scaleway snapshot failed (${SNAPSHOT_KIND})"
color=15158332
fi
snapshot_lines="$(jq -r '
def value_or_unknown($v): if ($v == null or $v == "") then "unknown" else $v end;
def truncate($max): if length > $max then .[0:$max] + "..." else . end;
def target_line:
(if (.status == "success" and .phase == "complete") then "[ok]" else "[fail]" end)
+ " `" + .target + "` "
+ "snapshot `" + value_or_unknown(.snapshot_name) + "`"
+ ", phase " + value_or_unknown(.phase)
+ (if (.status == "success" and .phase == "complete") then
""
else
", " + ((.message // "snapshot job failed") | truncate(120))
end);
if length == 0 then
"No snapshot result artifacts were available."
else
map(target_line) | join("\n")
end
' <<< "${results}")"
payload="$(jq -n \
--arg title "${title}" \
--arg status "${status}" \
--arg kind "${SNAPSHOT_KIND}" \
--arg trigger "${trigger_label}" \
--arg branch "${branch}" \
--arg actor "${actor}" \
--arg targets "${TARGET_KEYS}" \
--arg snapshots "${snapshot_lines}" \
--arg retention "${RETENTION}" \
--arg run_url "${run_url}" \
--arg timestamp "${ts}" \
--arg resolve_result "${RESOLVE_RESULT}" \
--arg snapshots_result "${SNAPSHOTS_RESULT}" \
--argjson color "${color}" \
'{
username: "TerraInfra",
embeds: [
{
title: $title,
color: $color,
fields: [
{name: "Status", value: $status, inline: true},
{name: "Type", value: $kind, inline: true},
{name: "Trigger", value: $trigger, inline: true},
{name: "Branch", value: $branch, inline: true},
{name: "Selected targets", value: (if ($targets | length) > 0 then ("`" + $targets + "`") else "n/a" end), inline: false},
{name: "Results", value: (if ($snapshots | length) > 0 then $snapshots else "n/a" end), inline: false},
{name: "Retention", value: $retention, inline: true},
{name: "Resolve job", value: $resolve_result, inline: true},
{name: "Snapshot jobs", value: $snapshots_result, inline: true},
{name: "GitHub run", value: ("[Open run](" + $run_url + ")"), inline: true},
{name: "Actor", value: $actor, inline: true}
],
timestamp: $timestamp
}
]
}')"
{
echo "## Scaleway snapshots"
echo
echo "- Status: ${status}"
echo "- Resolve job: ${RESOLVE_RESULT}"
echo "- Snapshot jobs: ${SNAPSHOTS_RESULT}"
echo "- Retention: ${RETENTION}"
echo
echo "| Target | Result | Snapshot | Retention | Phase | Message |"
echo "| --- | --- | --- | --- | --- | --- |"
jq -r '
def cell($v): (($v // "-") | tostring | gsub("[|]"; "/") | gsub("\n"; " ") | if length > 120 then .[0:117] + "..." else . end);
if length == 0 then
"| n/a | failed | n/a | n/a | unknown | No snapshot result artifacts were available. |"
else
.[]
| "| `" + .target + "` | " + cell(.status) + " | `" + cell(.snapshot_name) + "` | " + cell(.retention) + " | " + cell(.phase) + " | " + cell(.message) + " |"
end
' <<< "${results}"
} >> "${GITHUB_STEP_SUMMARY}"
curl -fsS -H 'Content-Type: application/json' -d "${payload}" "${SNAPSHOTS_DISCORD_WEBHOOK_URL}" || echo "Discord webhook notification failed" >&2