Add collector for Norwich County Council #583
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Implement New Collector | |
| env: | |
| DOTNET_VERSION: "10.0.x" | |
| on: | |
| issue_comment: | |
| types: [created] | |
| permissions: | |
| contents: write | |
| issues: write | |
| pull-requests: write | |
| jobs: | |
| check_trigger: | |
| name: Check Trigger | |
| runs-on: ubuntu-latest | |
| if: | | |
| github.event.issue.pull_request == null && | |
| contains(github.event.issue.labels.*.name, 'New Collector') && | |
| (github.event.comment.body == '/implement' || github.event.comment.body == '/implement-collector') | |
| outputs: | |
| should_run: ${{ steps.check.outputs.should_run }} | |
| issue_number: ${{ steps.get_issue.outputs.issue_number }} | |
| issue_body: ${{ steps.get_issue.outputs.issue_body }} | |
| council_name: ${{ steps.get_issue.outputs.council_name }} | |
| council_name_pascal: ${{ steps.get_issue.outputs.council_name_pascal }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Check if comment author has write access | |
| id: check | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const script = require('./.github/scripts/implement-collector/check-user-permission.js'); | |
| await script({ github, context, core }); | |
| - name: Get issue details | |
| id: get_issue | |
| if: steps.check.outputs.should_run == 'true' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const script = require('./.github/scripts/implement-collector/parse-issue-details.js'); | |
| await script({ github, context, core }); | |
| - name: Add reaction to comment | |
| if: steps.check.outputs.should_run == 'true' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| await github.rest.reactions.createForIssueComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: context.payload.comment.id, | |
| content: 'rocket' | |
| }); | |
| implement_collector: | |
| name: Implement Collector | |
| needs: check_trigger | |
| if: needs.check_trigger.outputs.should_run == 'true' | |
| runs-on: self-hosted | |
| timeout-minutes: 180 | |
| outputs: | |
| failure_summary: ${{ steps.failure_summary.outputs.summary }} | |
| env: | |
| AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| steps: | |
| - name: Generate GitHub App token | |
| id: generate-token | |
| uses: actions/create-github-app-token@v1 | |
| with: | |
| app-id: ${{ secrets.GH_APP_ID }} | |
| private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup .NET | |
| uses: actions/setup-dotnet@v4 | |
| with: | |
| dotnet-version: ${{ env.DOTNET_VERSION }} | |
| - name: Setup Dart SDK | |
| uses: dart-lang/setup-dart@v1 | |
| - name: Restore dependencies | |
| run: dotnet restore | |
| - name: Install AI Tools | |
| uses: ./.github/actions/install-ai-tools | |
| with: | |
| codex_config: ${{ secrets.CODEX_CONFIG }} | |
| playwright_config: | | |
| { | |
| "browser": { | |
| "browserName": "firefox", | |
| "launchOptions": { | |
| "headless": true | |
| }, | |
| "contextOptions": { | |
| "recordHar": { | |
| "path": "./.agent/playwright/out/requests.har", | |
| "content": "embed" | |
| } | |
| } | |
| }, | |
| "outputDir": "./.agent/playwright/out", | |
| "saveTrace": true, | |
| "isolated": true | |
| } | |
| - name: Create output directory | |
| run: mkdir -p .agent/playwright/out | |
| - name: Install AWS CLI | |
| run: | | |
| if ! command -v aws &> /dev/null; then | |
| echo "Installing AWS CLI..." | |
| curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" | |
| unzip -q awscliv2.zip | |
| sudo ./aws/install | |
| rm -rf aws awscliv2.zip | |
| else | |
| echo "AWS CLI already installed: $(aws --version)" | |
| fi | |
| - name: Write issue body to file | |
| run: | | |
| cat << 'ISSUE_EOF' > .agent/playwright/out/issue_body.txt | |
| ${{ needs.check_trigger.outputs.issue_body }} | |
| ISSUE_EOF | |
| - name: Setup PATH for Codex subshells | |
| run: | | |
| # Ensure dotnet is available in login shells spawned by Codex | |
| DOTNET_PATH=$(dirname "$(which dotnet)") | |
| echo "export PATH=\"$DOTNET_PATH:\$PATH\"" >> ~/.bashrc | |
| # Verify it works in a login shell | |
| bash -lc 'dotnet --version' | |
| - name: Implement collector with Codex (with retries) | |
| env: | |
| ISSUE_TITLE: ${{ needs.check_trigger.outputs.council_name }} | |
| ISSUE_NUMBER: ${{ needs.check_trigger.outputs.issue_number }} | |
| COLLECTOR_NAME: ${{ needs.check_trigger.outputs.council_name_pascal }} | |
| BINDAYS_ENABLE_HTTP_LOGGING: "true" | |
| MAX_ATTEMPTS: "5" | |
| run: ./.github/scripts/implement-collector/implement-with-retries.sh | |
| - name: Run final integration tests | |
| run: ./.github/scripts/implement-collector/run-collector-tests.sh | |
| - name: Upload artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: collector-data | |
| path: | | |
| .agent/playwright/out/*.json | |
| .agent/playwright/out/*.har | |
| .agent/playwright/out/*.zip | |
| .agent/playwright/out/*.png | |
| retention-days: 7 | |
| if-no-files-found: ignore | |
| - name: Upload screenshots to Cloudflare R2 | |
| id: upload_screenshot | |
| if: always() | |
| continue-on-error: true | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} | |
| AWS_ENDPOINT_URL: ${{ secrets.CLOUDFLARE_R2_ENDPOINT }} | |
| COLLECTOR_NAME: ${{ needs.check_trigger.outputs.council_name_pascal }} | |
| ISSUE_NUMBER: ${{ needs.check_trigger.outputs.issue_number }} | |
| run: | | |
| set +e | |
| # Find screenshot file using glob pattern (name may include suffix like "DistrictCouncil") | |
| SCREENSHOT_FILE=$(find .agent/playwright/out -name "*-screenshot.png" -type f | head -n 1) | |
| if [ -z "$SCREENSHOT_FILE" ] || [ ! -f "$SCREENSHOT_FILE" ]; then | |
| echo "⚠️ Screenshot not found in .agent/playwright/out/" | |
| echo "screenshot_url=" >> $GITHUB_OUTPUT | |
| exit 0 | |
| fi | |
| echo "📤 Uploading screenshot: $SCREENSHOT_FILE" | |
| SCREENSHOT_BASENAME=$(basename "$SCREENSHOT_FILE") | |
| R2_KEY="screenshots/${ISSUE_NUMBER}/${SCREENSHOT_BASENAME}" | |
| BUCKET="${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }}" | |
| if aws s3 cp "$SCREENSHOT_FILE" "s3://${BUCKET}/${R2_KEY}" \ | |
| --endpoint-url "${AWS_ENDPOINT_URL}" \ | |
| --acl public-read \ | |
| --content-type image/png; then | |
| if [ -n "${{ secrets.CLOUDFLARE_R2_CUSTOM_DOMAIN }}" ]; then | |
| SCREENSHOT_URL="https://${{ secrets.CLOUDFLARE_R2_CUSTOM_DOMAIN }}/${BUCKET}/${R2_KEY}" | |
| else | |
| ACCOUNT_ID="${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }}" | |
| SCREENSHOT_URL="https://${BUCKET}.${ACCOUNT_ID}.r2.cloudflarestorage.com/${R2_KEY}" | |
| fi | |
| echo "✅ Screenshot uploaded successfully!" | |
| echo "🔗 Public URL: $SCREENSHOT_URL" | |
| echo "screenshot_url=$SCREENSHOT_URL" >> $GITHUB_OUTPUT | |
| else | |
| echo "❌ Failed to upload screenshot to R2" | |
| echo "screenshot_url=" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Create branch and commit | |
| id: commit | |
| env: | |
| ISSUE_NUMBER: ${{ needs.check_trigger.outputs.issue_number }} | |
| run: ./.github/scripts/implement-collector/create-branch-and-commit.sh | |
| - name: Refresh GitHub App token | |
| id: refresh-token | |
| uses: actions/create-github-app-token@v1 | |
| with: | |
| app-id: ${{ secrets.GH_APP_ID }} | |
| private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} | |
| - name: Create Pull Request | |
| uses: actions/github-script@v7 | |
| env: | |
| COUNCIL_NAME: ${{ needs.check_trigger.outputs.council_name }} | |
| COUNCIL_NAME_PASCAL: ${{ env.COLLECTOR_NAME }} | |
| BRANCH_NAME: ${{ steps.commit.outputs.branch_name }} | |
| ISSUE_NUMBER: ${{ needs.check_trigger.outputs.issue_number }} | |
| SCREENSHOT_URL: ${{ steps.upload_screenshot.outputs.screenshot_url }} | |
| with: | |
| github-token: ${{ steps.refresh-token.outputs.token }} | |
| script: | | |
| const script = require('./.github/scripts/implement-collector/create-pr.js'); | |
| await script({ github, context, core }); | |
| - name: Summarise failure | |
| id: failure_summary | |
| if: failure() | |
| env: | |
| COLLECTOR_NAME: ${{ needs.check_trigger.outputs.council_name_pascal }} | |
| run: ./.github/scripts/implement-collector/summarise-failure.sh | |
| report_failure: | |
| name: Report Failure | |
| needs: [check_trigger, implement_collector] | |
| if: failure() && needs.check_trigger.outputs.should_run == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Comment on issue about failure | |
| uses: actions/github-script@v7 | |
| env: | |
| ISSUE_NUMBER: ${{ needs.check_trigger.outputs.issue_number }} | |
| FAILURE_SUMMARY: ${{ needs.implement_collector.outputs.failure_summary }} | |
| with: | |
| script: | | |
| const script = require('./.github/scripts/implement-collector/report-failure.js'); | |
| await script({ github, context, core }); |