fetch-stats #7085
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: fetch-stats | |
| on: | |
| schedule: | |
| # - cron: '0 10 * * *' # runs at 10:00 UTC everyday | |
| - cron: '5,35 6,7,8,9 * * 1-5' # runs at 5 and 35 past the hour UTC, Monday-Friday | |
| # this should catch ONS publications at 7am or 9:30am GMT or BST | |
| # Allow manual triggering of workflow | |
| workflow_dispatch: | |
| jobs: | |
| fetch: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false # if a scraper errors, the rest will keep running | |
| matrix: | |
| include: | |
| - country: uk | |
| index: cpi | |
| - country: uk | |
| index: cpih | |
| - country: uk | |
| index: rpi | |
| - country: ie | |
| index: cpi | |
| # - country: ar | |
| # index: cpi | |
| - country: za | |
| index: cpi | |
| - country: za | |
| index: ppi | |
| - country: ng | |
| index: cpi | |
| # - country: mx | |
| # index: cpi | |
| - country: jp | |
| index: cpi | |
| steps: | |
| - name: checkout repo content | |
| uses: actions/checkout@v3 | |
| - uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| - name: setup python | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: '3.11' | |
| - name: Cache dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: install python packages | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: execute py script | |
| run: | | |
| python src/nsofetch/fetch_${{ matrix.country }}.py ${{ matrix.index }} | |
| env: | |
| PYTHONPATH: src | |
| PROXY_URL: ${{ secrets.PROXY_URL }} | |
| - name: Save scraper output | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.country }}-${{ matrix.index }} | |
| path: data/${{ matrix.country }}_inflation_${{ matrix.index }}.csv | |
| deploy: | |
| needs: [fetch] # wait for the previous job to complete | |
| if: ${{ always() }} # run even if the previous job fails | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: checkout repo content | |
| uses: actions/checkout@v3 | |
| - name: Load scraper output | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts | |
| - name: Move scraper output into place | |
| run: mv artifacts/*/* data | |
| - name: Add last updated timestamp | |
| run: | | |
| echo "Last updated: `date -u +"%Y-%m-%d %H:%M:%S"`" > status.txt | |
| - name: Commit files | |
| run: | | |
| git config --local user.email "action@github.com" | |
| git config --local user.name "GitHub Action" | |
| git add -vA | |
| git commit -vm "update stats data" -a | |
| - name: Push changes | |
| uses: ad-m/github-push-action@master | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| branch: ${{ github.ref }} | |
| - name: Clean up artifacts | |
| uses: geekyeggo/delete-artifact@v2 | |
| with: | |
| name: '*' | |
| failOnError: false | |
| # notify_on_failure: | |
| # needs: [deploy] | |
| # if: ${{ failure() }} # run if anything before this point fails | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - name: Send a message to Slack if something went wrong | |
| # uses: slackapi/slack-github-action@v2 | |
| # with: | |
| # method: chat.postMessage | |
| # token: ${{ secrets.SLACK_BOT_TOKEN }} | |
| # payload: | | |
| # channel: "${{ env.SLACK_CHANNEL_ID }}" | |
| # username: "NSO stats fetcher" | |
| # icon_emoji: ":chart_with_downwards_trend:" | |
| # text: ":warning: Uh oh – the NSO stats fetcher has errored :warning:" |