Add Mekari & LookerStudio dashboard information #144
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Daily update to Google Sheets | |
| on: | |
| push: | |
| branches: [master] | |
| pull_request: | |
| branches: [master] | |
| schedule: | |
| - cron: "0 0 * * *" | |
| jobs: | |
| daily-update: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # Summon our code from the depths of GitHub! | |
| - uses: actions/checkout@v4 | |
| # Create a new development branch for PRs | |
| - name: Create development branch for PRs | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| git checkout -b development-${{ github.event.pull_request.number }} | |
| git push -u origin development-${{ github.event.pull_request.number }} | |
| # Prepare the ultimate Python environment for our Explosion magic! | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| cache: "pip" | |
| # Verify our Python version, for only the mightiest can cast Explosion! | |
| - name: Verify Python version | |
| run: python --version | |
| # Gather the ingredients for our Explosion spell! | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| # Summon our trusty Playwright familiars! | |
| - name: Install Playwright browsers | |
| run: playwright install --with-deps chromium firefox webkit | |
| # Ensure our secret incantations are in place! | |
| - name: Verify environment variables | |
| run: | | |
| if [ -z "${{ secrets.GCP_JSON }}" ]; then echo "GCP_JSON is not set! Our Explosion will fizzle!"; exit 1; fi | |
| if [ -z "${{ secrets.GOOGLE_SHEETS_ID }}" ]; then echo "GOOGLE_SHEETS_ID is not set! We have no target for our Explosion!"; exit 1; fi | |
| # Unleash our web-crawling Explosion upon the job sites! | |
| - name: Run scraping | |
| env: | |
| PYTHONUNBUFFERED: 1 | |
| run: | | |
| chmod +x ./pipeline/scrape.sh | |
| ./pipeline/scrape.sh | |
| # Channel our scraped data into Google Sheets with explosive force! | |
| - name: Upload to Google Sheets | |
| env: | |
| GCP_JSON: ${{ secrets.GCP_JSON }} | |
| GOOGLE_SHEETS_ID: ${{ secrets.GOOGLE_SHEETS_ID }} | |
| PYTHONUNBUFFERED: 1 | |
| run: PYTHONPATH="${PYTHONPATH:-$(pwd)}" python -m pipeline.upload_to_sheets | |
| # Fine-tune our spell for maximum visual impact! | |
| - name: Adjust Column Widths, Apply and Filter View Formatting | |
| env: | |
| GCP_JSON: ${{ secrets.GCP_JSON }} | |
| GOOGLE_SHEETS_ID: ${{ secrets.GOOGLE_SHEETS_ID }} | |
| PYTHONUNBUFFERED: 1 | |
| run: PYTHONPATH="${PYTHONPATH:-$(pwd)}" python -m pipeline.sheet_updater | |
| # Vanish without a trace, like a true Crimson Demon after casting Explosion! | |
| - name: Cleanup | |
| if: always() | |
| run: rm -rf output | |
| # Proclaim our success to the world! | |
| - name: Declare victory | |
| run: echo "Explosion magic successfully cast! The job data has been gloriously updated!" | |
| # Commit changes to the development branch | |
| - name: Commit changes | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| git config --local user.email "[email protected]" | |
| git config --local user.name "GitHub Action" | |
| git add . | |
| git commit -m "💥 Explosive update from PR #${{ github.event.pull_request.number }}" || echo "No changes to commit" | |
| git push origin development-${{ github.event.pull_request.number }} |