Ingest #41
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Ingest | |
| on: | |
| schedule: | |
| # Offset from Regenerate (06:20) so that job's PR has typically merged and | |
| # the source: generated records this job must not overwrite are on main. | |
| - cron: "20 8 * * *" | |
| workflow_dispatch: | |
| defaults: | |
| run: | |
| shell: bash -xeuo pipefail {0} | |
| concurrency: | |
| group: ingest | |
| env: | |
| HOMEBREW_DEVELOPER: 1 | |
| HOMEBREW_NO_AUTO_UPDATE: 1 | |
| HOMEBREW_NO_ENV_HINTS: 1 | |
| permissions: {} | |
| jobs: | |
| match: | |
| name: Match advisory candidates | |
| if: github.repository_owner == 'Homebrew' && github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write # Push the bot-owned candidate branch. | |
| pull-requests: write # Open a pull request for that branch. | |
| # Baseline local Aug 2026 measurement with --no-history and the trimmed | |
| # data/repology.json (select_ecosystems + overrides.yml skip applied): | |
| # 90820 records in 126.4 min, peak RSS 1.68 GB. | |
| timeout-minutes: 180 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Homebrew | |
| uses: Homebrew/actions/setup-homebrew@a657b8b0cd35d0f65cce41fce9b24cf054b49869 # 2026.08.24.1 | |
| with: | |
| core: true | |
| # --new-history is an authoring capability from brew main. Keeping | |
| # the explicit flag makes older clients fail fast instead of walking | |
| # history for every existing record. | |
| stable: false | |
| - name: Set up Ruby | |
| uses: Homebrew/actions/setup-ruby@a657b8b0cd35d0f65cce41fce9b24cf054b49869 # 2026.08.24.1 | |
| with: | |
| bundler-cache: true | |
| portable-ruby: true | |
| - name: Runner diagnostics | |
| run: free -m && df -h | |
| - name: Build Repology index | |
| # Built fresh for each sweep and read via --repology; the committed | |
| # data/repology.json is a frozen fallback for Repology API outages. | |
| run: | | |
| bundle exec rake repology:build || | |
| echo "::warning::Repology build failed; using the committed index." | |
| - name: Match advisories | |
| # Step-scoped: setup-homebrew unsets this so brew install can use the API. | |
| env: | |
| HOMEBREW_NO_INSTALL_FROM_API: 1 | |
| # Run one bulk sweep to avoid extra OSV traffic and live Repology fallbacks. | |
| # Split its output into formula-stable branches below. | |
| # Existing records keep their reviewed ranges; new fixed candidates get | |
| # the first Homebrew version that shipped the fix. The matcher reports | |
| # the residual history-walk count so full-sweep cost remains visible. | |
| run: > | |
| brew advisory-match --all --verbose --output advisories --new-history | |
| --repology data/repology.json | |
| --overrides data/overrides.yml | |
| - name: Drop uncomparable and rejected candidates | |
| # Uncomparable candidates read as "every version affected" in raw OSV | |
| # (`{introduced: "0"}`, no `fixed`), so they are discarded until the | |
| # matcher can emit a non-asserting shape. Rejected ids | |
| # (data/rejected-candidates.txt) would otherwise be re-proposed daily. | |
| run: | | |
| git ls-files --others --exclude-standard -z -- advisories/ | | |
| bundle exec rake advisories:filter | |
| - name: Configure git | |
| uses: Homebrew/actions/git-user-config@a657b8b0cd35d0f65cce41fce9b24cf054b49869 # 2026.08.24.1 | |
| with: | |
| username: BrewTestBot | |
| - name: Set up commit signing | |
| uses: Homebrew/actions/setup-commit-signing@a657b8b0cd35d0f65cce41fce9b24cf054b49869 # 2026.08.24.1 | |
| with: | |
| signing_key: ${{ secrets.BREWTESTBOT_SSH_SIGNING_KEY }} | |
| - name: Snapshot matched advisories | |
| id: snapshot | |
| run: | | |
| # Local-only commit used to build each shard. | |
| git switch --detach "${GITHUB_SHA}" | |
| git add advisories/ | |
| if git diff --cached --quiet; then | |
| echo "No changes." | |
| exit 0 | |
| fi | |
| git commit -m "Matched advisory candidates (snapshot)" | |
| # Preserve the complete reviewed delta so a later publication failure | |
| # does not require another multi-hour match before it can be inspected. | |
| git format-patch --stdout --full-index --binary "${GITHUB_SHA}..HEAD" > "${RUNNER_TEMP}/matched-advisories-snapshot.patch" | |
| { | |
| echo "diff_base=${GITHUB_SHA}" | |
| echo "commit=$(git rev-parse HEAD)" | |
| echo "created=true" | |
| } >> "${GITHUB_OUTPUT}" | |
| - name: Partition matched advisories | |
| if: steps.snapshot.outputs.created == 'true' | |
| env: | |
| DIFF_BASE_SHA: ${{ steps.snapshot.outputs.diff_base }} | |
| SNAPSHOT_SHA: ${{ steps.snapshot.outputs.commit }} | |
| SHARD_OUTPUT: ${{ runner.temp }}/advisory-shards | |
| # Deltas at or below this many files publish as one standing PR; | |
| # anything larger (backfills, matcher changes) falls back to | |
| # stable formula shards. | |
| SINGLE_PR_LIMIT: "500" | |
| run: | | |
| mkdir -p "${SHARD_OUTPUT}" | |
| changed="${RUNNER_TEMP}/changed-advisories" | |
| git diff --name-only -z --no-renames --diff-filter=AM \ | |
| "${DIFF_BASE_SHA}" "${SNAPSHOT_SHA}" -- advisories/ > "${changed}" | |
| count="$(tr -dc '\0' < "${changed}" | wc -c)" | |
| if (( count <= SINGLE_PR_LIMIT )); then | |
| echo "Publishing ${count} paths as a single pull request." | |
| cp "${changed}" "${SHARD_OUTPUT}/all" | |
| else | |
| bundle exec rake advisories:shard < "${changed}" | |
| fi | |
| - name: Configure push credentials | |
| if: steps.snapshot.outputs.created == 'true' | |
| # Authenticate only; the shard loop pushes each branch. The machine | |
| # token (when provisioned) authors PRs whose CI runs without manual | |
| # approval; github.token PRs hold their runs at action_required. | |
| uses: Homebrew/actions/git-try-push@a657b8b0cd35d0f65cce41fce9b24cf054b49869 # 2026.08.24.1 | |
| with: | |
| token: ${{ secrets.HOMEBREW_GITHUB_PUBLIC_REPO_TOKEN || github.token }} | |
| tries: 0 | |
| - name: Commit, push and open sharded pull requests | |
| if: steps.snapshot.outputs.created == 'true' | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.HOMEBREW_GITHUB_PUBLIC_REPO_TOKEN || github.token }} | |
| DIFF_BASE_SHA: ${{ steps.snapshot.outputs.diff_base }} | |
| SNAPSHOT_SHA: ${{ steps.snapshot.outputs.commit }} | |
| SHARD_OUTPUT: ${{ runner.temp }}/advisory-shards | |
| run: | | |
| shopt -s nullglob | |
| manifests=("${SHARD_OUTPUT}"/*) | |
| # The matcher only writes records, so no manifests indicates a bug. | |
| if (( ${#manifests[@]} == 0 )); then | |
| echo "::error::No advisory shards were produced." | |
| exit 1 | |
| fi | |
| # The matcher runs for over an hour, so main may advance after checkout. | |
| # Base every published branch on main as it exists at publication time. | |
| git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main | |
| PUBLISH_BASE_SHA="$(git rev-parse origin/main)" | |
| merge_status=0 | |
| PUBLISH_SNAPSHOT_TREE="$( | |
| git merge-tree --write-tree --no-messages --merge-base="${DIFF_BASE_SHA}" \ | |
| "${PUBLISH_BASE_SHA}" "${SNAPSHOT_SHA}" | |
| )" || merge_status=$? | |
| if (( merge_status != 0 )); then | |
| if (( merge_status == 1 )); then | |
| echo "::error::Matched advisory changes conflict with main at publication time." | |
| else | |
| echo "::error::git merge-tree failed with exit status ${merge_status}." | |
| fi | |
| echo "${PUBLISH_SNAPSHOT_TREE}" | |
| exit "${merge_status}" | |
| fi | |
| readonly PUSH_RETRIES_EXHAUSTED=75 | |
| push_branch() { | |
| local branch="$1" | |
| local attempt delay | |
| for attempt in 1 2 3 4 5; do | |
| if git push --force origin "HEAD:refs/heads/${branch}"; then | |
| return 0 | |
| fi | |
| if (( attempt == 5 )); then | |
| echo "::error::Push failed after ${attempt} attempts for ${branch}." | |
| return "${PUSH_RETRIES_EXHAUSTED}" | |
| fi | |
| delay=$(( attempt * 15 )) | |
| echo "::warning::Push attempt ${attempt} failed for ${branch}; retrying in ${delay}s." | |
| sleep "${delay}" | |
| done | |
| return "${PUSH_RETRIES_EXHAUSTED}" | |
| } | |
| # Commands must use `|| return` because this function runs in an `if`. | |
| publish_shard() { | |
| local manifest="$1" | |
| local shard="${manifest##*/}" | |
| local branch title body open_pr_number | |
| if [[ "${shard}" == "all" ]]; then | |
| # The standing PR: force-pushed in place each run until merged. | |
| branch="matched-advisories" | |
| title="Matched advisory candidates" | |
| body="Automated candidates via \`brew advisory-match --all\`. See [CONTRIBUTING.md](https://github.com/${GITHUB_REPOSITORY}/blob/HEAD/CONTRIBUTING.md#reviewing-matched-candidates) for the review checklist." | |
| else | |
| branch="matched-advisories-${shard}" | |
| title="Matched advisory candidates (shard ${shard})" | |
| body="Automated candidates via \`brew advisory-match --all\`, sharded by formula (SHA-256 bucket \`${shard}\`). See [CONTRIBUTING.md](https://github.com/${GITHUB_REPOSITORY}/blob/HEAD/CONTRIBUTING.md#reviewing-matched-candidates) for the review checklist." | |
| fi | |
| git switch --discard-changes --force-create "${branch}" "${PUBLISH_BASE_SHA}" || return | |
| git clean --force -d -- advisories/ || return | |
| git restore --source="${PUBLISH_SNAPSHOT_TREE}" --pathspec-from-file="${manifest}" \ | |
| --pathspec-file-nul || return | |
| # Regenerate rebuilds data/advisories.json separately. | |
| git add advisories/ || return | |
| if git diff --cached --quiet; then | |
| echo "Shard ${shard} is already on main; nothing to publish." | |
| return 0 | |
| fi | |
| git commit -m "${title}" \ | |
| -m "Base: ${PUBLISH_BASE_SHA}" || return | |
| # No lease: every branch is rebuilt from main on each run. | |
| push_branch "${branch}" || return | |
| open_pr_number="$(gh pr list --head "${branch}" --state open \ | |
| --json number,isCrossRepository \ | |
| --jq '.[] | select(.isCrossRepository | not) | .number')" || return | |
| if [[ -n "${open_pr_number}" ]]; then | |
| echo "PR #${open_pr_number} already open for ${branch}." | |
| else | |
| gh pr create --base main --head "${branch}" \ | |
| --title "${title}" \ | |
| --body "${body}" || return | |
| fi | |
| } | |
| failed=0 | |
| for manifest in "${manifests[@]}"; do | |
| if publish_shard "${manifest}"; then | |
| echo "Published shard ${manifest##*/}." | |
| else | |
| publish_status=$? | |
| echo "::error::Failed to publish shard ${manifest##*/}." | |
| failed=1 | |
| if (( publish_status == PUSH_RETRIES_EXHAUSTED )); then | |
| # A systemic remote failure must not consume the remaining job | |
| # timeout by retrying every shard. | |
| break | |
| fi | |
| fi | |
| done | |
| # Reconcile only after full publication: a partial failure must not | |
| # delete the previous run's complete review branches. The successful | |
| # retry performs the cleanup. | |
| if (( failed != 0 )); then | |
| exit "${failed}" | |
| fi | |
| # A mode switch (standing <-> sharded) leaves the other mode's PRs | |
| # open with superseded content; close them and delete their branches. | |
| # Every successful manifest counts as published, including those already on main. | |
| published=" " | |
| for manifest in "${manifests[@]}"; do | |
| shard="${manifest##*/}" | |
| if [[ "${shard}" == "all" ]]; then | |
| published+="matched-advisories " | |
| else | |
| published+="matched-advisories-${shard} " | |
| fi | |
| done | |
| open_prs="${RUNNER_TEMP}/open-candidate-prs" | |
| gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls?state=open&per_page=100" \ | |
| --jq '.[] | select(((.head.repo.full_name // "") == .base.repo.full_name) and (.head.ref | test("^matched-advisories(-[0-9a-f]{2})?$"))) | [.number, .head.ref] | @tsv' \ | |
| > "${open_prs}" | |
| while IFS=$'\t' read -r number head; do | |
| if [[ "${published}" != *" ${head} "* ]]; then | |
| gh pr close "${number}" --delete-branch \ | |
| --comment "Superseded: this run published the current candidates elsewhere." || { | |
| echo "::error::Failed to close superseded PR #${number}." | |
| failed=1 | |
| } | |
| fi | |
| done < "${open_prs}" | |
| exit "${failed}" | |
| - name: Upload failed matched snapshot | |
| if: failure() && steps.snapshot.outputs.created == 'true' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: matched-advisories-snapshot-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: ${{ runner.temp }}/matched-advisories-snapshot.patch | |
| if-no-files-found: error | |
| retention-days: 7 |