Daily OA.Report Parsing (daily snapshots only) #430
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Daily OA.Report Parsing (daily snapshots only) | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' # every day at midnight UTC | |
| workflow_dispatch: {} # allows manual runs from GitHub UI | |
| concurrency: | |
| group: "${{ github.workflow }}-${{ github.run_id }}" | |
| cancel-in-progress: false | |
| jobs: | |
| daily-snapshots: | |
| runs-on: ubuntu-latest | |
| env: | |
| TZ: Europe/London | |
| steps: | |
| - name: Checkout main | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: main | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| - name: Install Google Chrome (for Selenium) | |
| uses: browser-actions/setup-chrome@v1 | |
| with: | |
| chrome-version: stable | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: Write credentials & settings (daily snapshots) | |
| env: | |
| GOOGLE_CREDS_JSON: ${{ secrets.GOOGLE_CREDS_JSON }} | |
| SETTINGS_YAML: ${{ secrets.SETTINGS_YAML }} | |
| run: | | |
| mkdir -p config | |
| printf '%s' "$GOOGLE_CREDS_JSON" > config/google_creds.json | |
| printf '%s' "$SETTINGS_YAML" > config/settings.yaml | |
| # Run all three extractors for staging + dev, as well as up to three retries | |
| - name: Run extractors with retries (daily snapshots) | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| run_with_retry() { | |
| local attempts=3 | |
| local delay=20 | |
| local cmd="$*" | |
| for n in $(seq 1 "$attempts"); do | |
| echo "::group::Attempt $n: $cmd" | |
| if eval "$cmd"; then | |
| echo "✔ Succeeded on attempt $n" | |
| echo "::endgroup::" | |
| return 0 | |
| fi | |
| status=$? | |
| echo "✖ Failed (exit $status). Retrying in ${delay}s..." | |
| echo "::endgroup::" | |
| sleep "$delay" | |
| delay=$((delay*2)) | |
| done | |
| echo "✖ Failed after $attempts attempts: $cmd" | |
| return 1 | |
| } | |
| EXTRACTORS=(insights actions explore) | |
| ENVS=(staging dev) | |
| for env in "${ENVS[@]}"; do | |
| for ex in "${EXTRACTORS[@]}"; do | |
| run_with_retry python -m "extractors.${ex}" --env "${env}" | |
| done | |
| done |