From 8e2b4868ed5243e86e9532ffc3950d1fcce213c2 Mon Sep 17 00:00:00 2001 From: Daniel Alkurdi Date: Sat, 8 Aug 2026 16:10:15 +1000 Subject: [PATCH] chore(ci): retire the Actions collection workflow The scheduled run on the maintainer's server has produced a clean cycle (healthy, 30/30 due sources), so the fallback workflow, its README badge and its doc references go. The workflow's write deploy key and the COLLECTOR_DEPLOY_KEY secret were removed from the repository settings in the same pass: with the workflow gone they were an unused write credential. --- .github/workflows/collect.yml | 122 ---------------------------------- AGENTS.md | 4 +- README.md | 3 +- docs/collector.md | 17 +++-- 4 files changed, 14 insertions(+), 132 deletions(-) delete mode 100644 .github/workflows/collect.yml diff --git a/.github/workflows/collect.yml b/.github/workflows/collect.yml deleted file mode 100644 index 8b855aa..0000000 --- a/.github/workflows/collect.yml +++ /dev/null @@ -1,122 +0,0 @@ -name: Collect AI policy developments - -# Scheduling moved to Argus server (systemd timer policai-collect.timer); -# this workflow remains as a manual fallback. -on: - workflow_dispatch: - inputs: - source: - description: "Run a single source id (optional)" - required: false - type: string - -permissions: - contents: write - issues: write - -concurrency: - group: collect - cancel-in-progress: false - -jobs: - collect: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v5 - with: - # Fetch and push over SSH as the collector deploy key, which is a - # bypass actor on the "Protect main" ruleset. The default - # GITHUB_TOKEN cannot push to main — see docs/collector.md. - ssh-key: ${{ secrets.COLLECTOR_DEPLOY_KEY }} - - - name: Setup Node.js - uses: actions/setup-node@v5 - with: - node-version: 20.19.0 - cache: npm - - - name: Install dependencies - run: npm ci - - - name: Cache Playwright Chromium - uses: actions/cache@v4 - with: - path: ~/.cache/ms-playwright - key: playwright-chromium-${{ runner.os }}-${{ hashFiles('package-lock.json') }} - - - name: Install headless Chromium for browser retrieval - run: npx playwright-core install --with-deps chromium - - - name: Run collector - id: collector - continue-on-error: true - env: - SOURCE_ID: ${{ inputs.source }} - run: | - if [ -n "$SOURCE_ID" ]; then - npm run collect -- --source="$SOURCE_ID" - else - npm run collect - fi - - - name: Validate data - run: npm run validate:data - - - name: Guard the curated registry - run: | - # The collector must never modify the curated policy registry. - if ! git diff --quiet -- data/policies.json; then - echo "::error::Collector modified data/policies.json — aborting." - git diff -- data/policies.json - exit 1 - fi - - - name: Commit and push changes - run: | - if git diff --quiet; then - echo "No changes to commit." - exit 0 - fi - git config user.name "policai-collector[bot]" - git config user.email "policai-collector[bot]@users.noreply.github.com" - git add data/developments.json public/data/meta.json data/watch-state.json data/source-reviews.json - git commit -m "chore(data): daily collection $(date -u +%F)" - # Back-to-back dispatched runs can race this push; rebase the data - # commit onto whatever landed in the meantime and retry. - for attempt in 1 2 3; do - if git push; then - exit 0 - fi - echo "Push rejected (attempt ${attempt}); rebasing onto latest main." - git pull --rebase origin main - done - git push - - - name: Fail after preserving unhealthy collection state - if: steps.collector.outcome == 'failure' - run: | - echo "::error::Collector reported failed coverage or a fatal error; operational state was preserved before failing." - exit 1 - - - name: Alert on failure - if: failure() - env: - GH_TOKEN: ${{ github.token }} - run: | - # Open (or comment on) a tracking issue so scheduled failures are - # never silent. Deduplicates into one open issue at a time. - gh label create collector-failure \ - --description "Automated alert from collect.yml" \ - --color D93F0B || true - run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" - existing=$(gh issue list --label collector-failure --state open --json number --jq '.[0].number') - if [ -n "$existing" ]; then - gh issue comment "$existing" --body "Still failing: $run_url" - else - gh issue create \ - --title "Daily collection workflow is failing" \ - --label collector-failure \ - --body "$(printf 'The scheduled collector run failed.\n\nLatest failed run: %s\n\nOpened automatically by collect.yml; further failures are added as comments while this issue stays open. Close it once the workflow is green.' "$run_url")" - fi diff --git a/AGENTS.md b/AGENTS.md index 58c41a2..6e683df 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ Policai is an Australian AI policy tracker. It maintains a curated register of AI policy, regulation, governance and court guidance across federal and state/territory jurisdictions, plus an automated "developments" feed of newly detected policy activity. -**Git is the database.** All canonical data is JSON committed to this repository (`public/data/`, `data/`). The deployed site only reads that data. It is self-hosted at [policai.org](https://policai.org) behind a Cloudflare tunnel. Collection runs daily on the maintainer's server, in a checkout separate from the one that serves the site, and pushes new detections to GitHub; the serving checkout pulls those commits and serves them through ISR without a rebuild. GitHub Actions no longer schedules collection; `.github/workflows/collect.yml` is kept as a manual fallback. There is no runtime database, no auth, and no admin dashboard. +**Git is the database.** All canonical data is JSON committed to this repository (`public/data/`, `data/`). The deployed site only reads that data. It is self-hosted at [policai.org](https://policai.org) behind a Cloudflare tunnel. Collection runs daily on the maintainer's server, in a checkout separate from the one that serves the site, and pushes new detections to GitHub; the serving checkout pulls those commits and serves them through ISR without a rebuild. There is no runtime database, no auth, and no admin dashboard. ## Tech Stack @@ -18,7 +18,7 @@ Policai is an Australian AI policy tracker. It maintains a curated register of A - **Analysis:** keyword heuristic by default; Claude, an Anthropic model, batched through the Claude Code CLI on the collection host when `USE_CLAUDE_CLASSIFIER` is set — both paths cap stored/displayed confidence at 0.65 (`MACHINE_CONFIDENCE_CAP`) - **Scraping:** Cheerio - **Testing:** Vitest (+ Testing Library) -- **Automation:** a daily scheduled run on the maintainer's server; GitHub Actions (`.github/workflows/collect.yml`) kept as a manual fallback +- **Automation:** a daily scheduled run on the maintainer's server ## Commands diff --git a/README.md b/README.md index 13f1513..d52eeb4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ # Policai -[![Daily collection](https://github.com/l0cka/policai/actions/workflows/collect.yml/badge.svg)](https://github.com/l0cka/policai/actions/workflows/collect.yml) Policai is an Australian AI policy tracker. It maintains a curated register of AI policy, regulation, governance and court guidance across federal and state/territory jurisdictions, and automatically detects new developments from official government sources every day. @@ -28,7 +27,7 @@ Product surface: - `data/source-reviews.json` — detections staged for curated review - `data/source-monitoring.json` — the manual-source review ledger -The maintainer's server runs the collector daily, from its own checkout, over the official sources that reliably permit machine retrieval. The GitHub Actions workflow ([collect.yml](.github/workflows/collect.yml)) no longer schedules collection and is kept as a manual fallback. Sources protected by browser challenges are kept in the same source catalogue but reviewed through the manual coverage ledger. Candidate pages from browser-only sources are retrieved through a self-hosted Firecrawl instance, falling back to headless Chromium when Firecrawl is unavailable. New items are classified by keyword heuristic by default, or by Claude, an Anthropic model, in batches when the collector's Claude classifier is enabled (it is enabled in production). Either path caps stored confidence at 0.65, so an automated detection never reads as more certain than an editor's review. Detections are validated and committed. The site reads that data from disk and revalidates hourly; there is no runtime database. +The maintainer's server runs the collector daily, from its own checkout, over the official sources that reliably permit machine retrieval. Sources protected by browser challenges are kept in the same source catalogue but reviewed through the manual coverage ledger. Candidate pages from browser-only sources are retrieved through a self-hosted Firecrawl instance, falling back to headless Chromium when Firecrawl is unavailable. New items are classified by keyword heuristic by default, or by Claude, an Anthropic model, in batches when the collector's Claude classifier is enabled (it is enabled in production). Either path caps stored confidence at 0.65, so an automated detection never reads as more certain than an editor's review. Detections are validated and committed. The site reads that data from disk and revalidates hourly; there is no runtime database. High-confidence detections are staged in `data/source-reviews.json`; a reviewer uses the local stage → approve → publish workflow before they enter the register. Public register and timeline reads only expose verified records. The collector never writes to `policies.json` directly, and CI enforces that. diff --git a/docs/collector.md b/docs/collector.md index 16ab46d..4178fed 100644 --- a/docs/collector.md +++ b/docs/collector.md @@ -228,11 +228,11 @@ the site. Each run: 5. If collection reported failed coverage, fail the run only after that operational state is preserved -[`.github/workflows/collect.yml`](../.github/workflows/collect.yml) no longer -schedules collection; it is kept as a manual fallback -(Actions → "Collect AI policy developments" → Run workflow, optionally with a -single source id) and runs the same steps, including opening or commenting on -an issue labelled `collector-failure` on any failure. +The GitHub Actions collection workflow has been retired after the scheduled +run on the maintainer's server produced a clean cycle. Failures still open or +comment on an issue labelled `collector-failure`, now raised by the scheduled +run itself. A manual pass can be run from any checkout with +`npm run collect -- --source=`. The site's own checkout pulls the push on its timer. Because pages read the JSON from disk at request time and revalidate hourly, the new data appears @@ -245,7 +245,12 @@ version cannot be lost merely because its state write completed first. **Repository configuration:** -- `COLLECTOR_DEPLOY_KEY` secret — private half of the repo's write deploy key ("collector (collect.yml push)"), used only by the GitHub Actions fallback run. Checkout uses it (`ssh-key:`) so the push authenticates as the deploy key, and the "Protect main" ruleset lists **Deploy keys** as a bypass actor (`bypass_mode: always`). Without this pair the fallback push is rejected with `GH013: Repository rule violations` — the default `GITHUB_TOKEN` cannot be a bypass actor on a user-owned repo. The scheduled collection run pushes over its own SSH-authenticated git remote and does not use this secret. The safety story does not depend on the ruleset here: the registry-guard step runs on both paths and enforces that automation never touches `policies.json`. +- The retired Actions workflow used a write deploy key (`COLLECTOR_DEPLOY_KEY` + secret) as a ruleset bypass actor; both the key and the secret can be + deleted now that the workflow is gone. The scheduled collection run pushes + over its own SSH-authenticated git remote. The safety story does not depend + on the ruleset: the registry guard enforces that automation never touches + `policies.json`. ## Reviewing detections into the register