Run collection #2
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run collection | |
| on: | |
| schedule: | |
| - cron: 0 0 * * * | |
| workflow_dispatch: null | |
| env: | |
| DLB_BOT_EMAIL: ${{ secrets.DLB_BOT_EMAIL }} | |
| DLB_BOT_TOKEN: ${{ secrets.DLB_BOT_TOKEN }} | |
| DLB_BOT_USERNAME: ${{ secrets.DLB_BOT_USERNAME }} | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Free up disk space | |
| run: | | |
| df -h | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /opt/ghc | |
| echo | |
| df -h | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.8 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{secrets.AWS_S3_ACCESS_KEY_ID}} | |
| aws-secret-access-key: ${{secrets.AWS_S3_SECRET_ACCESS_KEY}} | |
| aws-region: eu-west-2 | |
| - name: Configure git | |
| run: | | |
| git config user.email "${DLB_BOT_EMAIL}" | |
| git config user.name "${DLB_BOT_USERNAME}" | |
| git remote set-url origin https://${DLB_BOT_USERNAME}:${DLB_BOT_TOKEN}@github.com/${GITHUB_REPOSITORY}.git | |
| git checkout ${GITHUB_REF##*/} | |
| - name: Update makerules | |
| run: make makerules | |
| - name: Commit updated makerules | |
| run: make commit-makerules | |
| - name: Install dependencies | |
| run: make init | |
| - name: Run the collector | |
| run: make collect | |
| - name: Commit collection logs | |
| run: make commit-collection | |
| - name: Save collected resources to S3 | |
| run: make save-resources | |
| - name: Build the collection database | |
| run: make collection | |
| - name: Push collection database to S3 | |
| run: make save-collection | |
| - name: transform collected files | |
| run: make transformed | |
| - name: Save transformed files to S3 | |
| run: make save-transformed | |
| - name: Build datasets from the transformed files | |
| run: make dataset | |
| - name: Save datasets to S3 | |
| run: make save-dataset |