Repository navigation
Sync discovered catalog #92
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Required repository secrets: | |
| # - CLOUDFLARE_API_TOKEN with Workers KV Storage: Read | |
| # - CLOUDFLARE_ACCOUNT_ID for the Cloudflare account that owns DISCOVERY | |
| name: Sync discovered catalog | |
| on: | |
| schedule: | |
| - cron: "37 9 * * *" | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| concurrency: | |
| group: sync-discovered | |
| cancel-in-progress: false | |
| jobs: | |
| sync: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout main | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: main | |
| - name: Setup Bun | |
| uses: oven-sh/setup-bun@v2 | |
| - name: Install dependencies | |
| run: bun install --frozen-lockfile | |
| - name: Count existing discovered domains | |
| id: before | |
| run: | | |
| count=$(find domains -mindepth 2 -maxdepth 2 -type f -name integrations.json 2>/dev/null | wc -l | tr -d ' ') | |
| echo "count=$count" >> "$GITHUB_OUTPUT" | |
| # sync:discovered no longer imports the generated output/catalog-seeds.json | |
| # (scripts/batch/args.ts explains why), so it runs straight from a clean | |
| # checkout with no normalize step in front of it. That import is what made | |
| # every nightly run since July fail with | |
| # `Cannot find module '../../output/catalog-seeds.json'`. | |
| - name: Sync from KV | |
| env: | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| run: bun run sync:discovered | |
| - name: Guard against large catalog deletion | |
| run: | | |
| before="${{ steps.before.outputs.count }}" | |
| after=$(find domains -mindepth 2 -maxdepth 2 -type f -name integrations.json 2>/dev/null | wc -l | tr -d ' ') | |
| min=$(( before - before / 10 )) | |
| echo "domains before=$before after=$after min=$min" | |
| # The denylist legitimately removes a handful of files; only a large | |
| # loss (>10%) means the sync went wrong. | |
| if [ "$after" -lt "$min" ]; then | |
| echo "Refusing to commit: domains/ lost more than 10% of domain files." | |
| exit 1 | |
| fi | |
| - name: Refresh GitHub stars seed | |
| continue-on-error: true | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| stars=$(gh api repos/UsefulSoftwareCo/integrations --jq .stargazers_count) | |
| case "$stars" in | |
| ''|*[!0-9]*) | |
| echo "Invalid GitHub star count: $stars" | |
| exit 1 | |
| ;; | |
| esac | |
| fetched_at=$(($(date +%s) * 1000)) | |
| mkdir -p src/data | |
| tmp=$(mktemp) | |
| printf '{ "stars": %s, "fetchedAt": %s }\n' "$stars" "$fetched_at" > "$tmp" | |
| mv "$tmp" src/data/github-stars.json | |
| # Probe the MCP endpoints before the build reads their verdicts. normalize | |
| # writes output/mcp.json, which the probe reads alongside the domains/ | |
| # tree; the build then re-runs normalize so the fresh verdicts reach the | |
| # published pages in the same commit. | |
| - name: Normalize catalog for probing | |
| run: bun run normalize | |
| - name: Verify MCP endpoints | |
| continue-on-error: true | |
| run: bun scripts/verify-mcp-endpoints.ts --stale 7 | |
| - name: Build | |
| run: bun run build | |
| - name: Commit and push changes | |
| run: | | |
| # git diff misses untracked files — new domain files are the common case. | |
| if [ -z "$(git status --porcelain -- domains/ output/mcp-endpoints.json src/data/github-stars.json)" ]; then | |
| echo "No discovered catalog, endpoint verdict or GitHub stars changes." | |
| exit 0 | |
| fi | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git add -A domains/ output/mcp-endpoints.json src/data/github-stars.json | |
| git commit -m "sync discovered catalog, endpoint verdicts and GitHub stars" | |
| # A human push to main between checkout and here would make this | |
| # non-fast-forward; rebase our one commit on top instead of failing. | |
| git pull --rebase origin main | |
| git push origin HEAD:main |