ingest.yml

name: Ingest

on:
  schedule:
    # Offset from Regenerate (06:20) so that job's PR has typically merged and
    # the source: generated records this job must not overwrite are on main.
    - cron: "20 8 * * *"
  workflow_dispatch:
    inputs:
      reconcile_history:
        description: Reconcile matched terminal ranges (requires reviewed boundary pins)
        type: boolean
        default: false

defaults:
  run:
    shell: bash -xeuo pipefail {0}

concurrency:
  group: ingest

env:
  HOMEBREW_DEVELOPER: 1
  HOMEBREW_NO_AUTO_UPDATE: 1
  HOMEBREW_NO_ENV_HINTS: 1

permissions: {}

jobs:
  match:
    name: Match advisory candidates
    if: github.repository_owner == 'Homebrew' && github.ref == 'refs/heads/main'
    runs-on: ubuntu-latest
    permissions:
      contents: write # Push the bot-owned candidate branch.
      pull-requests: write # Open a pull request for that branch.
    # Baseline local Aug 2026 measurement with --no-history and the trimmed
    # data/repology.json (select_ecosystems + overrides.yml skip applied):
    # 90820 records in 126.4 min, peak RSS 1.68 GB.
    timeout-minutes: 180
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Set up Homebrew
        uses: Homebrew/actions/setup-homebrew@f8d4222eb633e65c2a0157383cf80861fc7fae7f # 2026.09.07.1
        with:
          core: true
          # --new-history is an authoring capability from brew main. Keeping
          # the explicit flag makes older clients fail fast instead of walking
          # history for every existing record.
          stable: false

      - name: Build core history commit graph
        run: git -C "$(brew --repository homebrew/core)" commit-graph write --reachable --changed-paths

      - name: Set up Ruby
        uses: Homebrew/actions/setup-ruby@f8d4222eb633e65c2a0157383cf80861fc7fae7f # 2026.09.07.1
        with:
          bundler-cache: true
          portable-ruby: true

      - name: Runner diagnostics
        run: free -m && df -h

      - name: Build Repology index
        # Built fresh for each sweep and read via --repology; the committed
        # data/repology.json is a frozen fallback for Repology API outages.
        run: |
          bundle exec rake repology:build ||
            echo "::warning::Repology build failed; using the committed index."

      - name: Match advisories
        # Step-scoped: setup-homebrew unsets this so brew install can use the API.
        env:
          HOMEBREW_NO_INSTALL_FROM_API: 1
          RECONCILE_HISTORY: ${{ inputs.reconcile_history && 'true' || 'false' }}
        # Run one bulk sweep to avoid extra OSV traffic and live Repology fallbacks.
        # Split its output into formula-stable branches below.
        # Scheduled runs preserve terminal ranges. A manual reconciliation must
        # be requested after adding pins for hand-reviewed Homebrew boundaries.
        run: |
          history_args=(--new-history)
          if [[ "${RECONCILE_HISTORY}" == "true" ]]; then
            history_args=(--reconcile-history)
          fi
          brew advisory-match --all --verbose --output advisories "${history_args[@]}" \
            --repology data/repology.json \
            --overrides data/overrides.yml

      - name: Drop uncomparable and rejected candidates
        # Uncomparable candidates read as "every version affected" in raw OSV
        # (`{introduced: "0"}`, no `fixed`), so they are discarded until the
        # matcher can emit a non-asserting shape. Rejected ids
        # (data/rejected-candidates.txt) would otherwise be re-proposed daily.
        run: |
          git ls-files --others --exclude-standard -z -- advisories/ |
            bundle exec rake advisories:filter

      - name: Configure git
        uses: Homebrew/actions/git-user-config@f8d4222eb633e65c2a0157383cf80861fc7fae7f # 2026.09.07.1
        with:
          username: BrewTestBot

      - name: Set up commit signing
        uses: Homebrew/actions/setup-commit-signing@f8d4222eb633e65c2a0157383cf80861fc7fae7f # 2026.09.07.1
        with:
          signing_key: ${{ secrets.BREWTESTBOT_SSH_SIGNING_KEY }}

      - name: Snapshot matched advisories
        id: snapshot
        run: |
          # Local-only commit used to build each shard.
          git switch --detach "${GITHUB_SHA}"
          git add advisories/
          if git diff --cached --quiet; then
            echo "No changes."
            exit 0
          fi
          git commit -m "Matched advisory candidates (snapshot)"
          # Preserve the complete reviewed delta so a later publication failure
          # does not require another multi-hour match before it can be inspected.
          git format-patch --stdout --full-index --binary "${GITHUB_SHA}..HEAD" > "${RUNNER_TEMP}/matched-advisories-snapshot.patch"
          {
            echo "diff_base=${GITHUB_SHA}"
            echo "commit=$(git rev-parse HEAD)"
            echo "created=true"
          } >> "${GITHUB_OUTPUT}"

      - name: Partition matched advisories
        if: steps.snapshot.outputs.created == 'true'
        env:
          DIFF_BASE_SHA: ${{ steps.snapshot.outputs.diff_base }}
          SNAPSHOT_SHA: ${{ steps.snapshot.outputs.commit }}
          SHARD_OUTPUT: ${{ runner.temp }}/advisory-shards
          # Deltas at or below this many files publish as one standing PR;
          # anything larger (backfills, matcher changes) falls back to
          # stable formula shards.
          SINGLE_PR_LIMIT: "500"
        run: |
          mkdir -p "${SHARD_OUTPUT}"
          changed="${RUNNER_TEMP}/changed-advisories"
          git diff --name-only -z --no-renames --diff-filter=AMD \
            "${DIFF_BASE_SHA}" "${SNAPSHOT_SHA}" -- advisories/ > "${changed}"
          count="$(tr -dc '\0' < "${changed}" | wc -c)"
          if (( count <= SINGLE_PR_LIMIT )); then
            echo "Publishing ${count} paths as a single pull request."
            cp "${changed}" "${SHARD_OUTPUT}/all"
          else
            bundle exec rake advisories:shard < "${changed}"
          fi

      - name: Configure push credentials
        if: steps.snapshot.outputs.created == 'true'
        # Authenticate only; the shard loop pushes each branch. The machine
        # token (when provisioned) authors PRs whose CI runs without manual
        # approval; github.token PRs hold their runs at action_required.
        uses: Homebrew/actions/git-try-push@f8d4222eb633e65c2a0157383cf80861fc7fae7f # 2026.09.07.1
        with:
          token: ${{ secrets.HOMEBREW_GITHUB_PUBLIC_REPO_TOKEN || github.token }}
          tries: 0

      - name: Commit, push and open sharded pull requests
        if: steps.snapshot.outputs.created == 'true'
        env:
          RECONCILE_HISTORY: ${{ inputs.reconcile_history && 'true' || 'false' }}
          GITHUB_TOKEN: ${{ secrets.HOMEBREW_GITHUB_PUBLIC_REPO_TOKEN || github.token }}
          DIFF_BASE_SHA: ${{ steps.snapshot.outputs.diff_base }}
          SNAPSHOT_SHA: ${{ steps.snapshot.outputs.commit }}
          SHARD_OUTPUT: ${{ runner.temp }}/advisory-shards
        run: |
          # Daily ingest must not replace or close a pending reconciliation review.
          branch_prefix="matched-advisories"
          title_prefix="Matched advisory candidates"
          history_option="--new-history"
          if [[ "${RECONCILE_HISTORY}" == "true" ]]; then
            branch_prefix="reconciled-advisories"
            title_prefix="Reconciled advisory history"
            history_option="--reconcile-history"
          fi
          shopt -s nullglob
          manifests=("${SHARD_OUTPUT}"/*)
          # A non-empty snapshot must produce at least one manifest, including deletions.
          if (( ${#manifests[@]} == 0 )); then
            echo "::error::No advisory shards were produced."
            exit 1
          fi

          # The matcher runs for over an hour, so main may advance after checkout.
          # Base every published branch on main as it exists at publication time.
          git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main
          PUBLISH_BASE_SHA="$(git rev-parse origin/main)"
          merge_status=0
          PUBLISH_SNAPSHOT_TREE="$(
            git merge-tree --write-tree --no-messages --merge-base="${DIFF_BASE_SHA}" \
              "${PUBLISH_BASE_SHA}" "${SNAPSHOT_SHA}"
          )" || merge_status=$?
          if (( merge_status != 0 )); then
            if (( merge_status == 1 )); then
              echo "::error::Matched advisory changes conflict with main at publication time."
            else
              echo "::error::git merge-tree failed with exit status ${merge_status}."
            fi
            echo "${PUBLISH_SNAPSHOT_TREE}"
            exit "${merge_status}"
          fi

          readonly PUSH_RETRIES_EXHAUSTED=75
          push_branch() {
            local branch="$1"
            local attempt delay
            for attempt in 1 2 3 4 5; do
              if git push --force origin "HEAD:refs/heads/${branch}"; then
                return 0
              fi
              if (( attempt == 5 )); then
                echo "::error::Push failed after ${attempt} attempts for ${branch}."
                return "${PUSH_RETRIES_EXHAUSTED}"
              fi
              delay=$(( attempt * 15 ))
              echo "::warning::Push attempt ${attempt} failed for ${branch}; retrying in ${delay}s."
              sleep "${delay}"
            done
            return "${PUSH_RETRIES_EXHAUSTED}"
          }

          # Commands must use `|| return` because this function runs in an `if`.
          publish_shard() {
            local manifest="$1"
            local shard="${manifest##*/}"
            local branch title body open_pr_number
            if [[ "${shard}" == "all" ]]; then
              # The standing PR: force-pushed in place each run until merged.
              branch="${branch_prefix}"
              title="${title_prefix}"
              body="Automated candidates via \`brew advisory-match --all ${history_option}\`. See [CONTRIBUTING.md](https://github.com/${GITHUB_REPOSITORY}/blob/HEAD/CONTRIBUTING.md#reviewing-matched-candidates) for the review checklist."
            else
              branch="${branch_prefix}-${shard}"
              title="${title_prefix} (shard ${shard})"
              body="Automated candidates via \`brew advisory-match --all ${history_option}\`, sharded by formula (SHA-256 bucket \`${shard}\`). See [CONTRIBUTING.md](https://github.com/${GITHUB_REPOSITORY}/blob/HEAD/CONTRIBUTING.md#reviewing-matched-candidates) for the review checklist."
            fi

            git switch --discard-changes --force-create "${branch}" "${PUBLISH_BASE_SHA}" || return
            git clean --force -d -- advisories/ || return
            # Stage only this shard, including deletions. Regenerate rebuilds the index separately.
            PUBLISH_BASE_SHA="${PUBLISH_BASE_SHA}" PUBLISH_SNAPSHOT_TREE="${PUBLISH_SNAPSHOT_TREE}" \
              bundle exec rake advisories:restore < "${manifest}" || return
            if git diff --cached --quiet; then
              echo "Shard ${shard} is already on main; nothing to publish."
              return 0
            fi
            git commit -m "${title}" \
              -m "Base: ${PUBLISH_BASE_SHA}" || return
            # No lease: every branch is rebuilt from main on each run.
            push_branch "${branch}" || return

            open_pr_number="$(gh pr list --head "${branch}" --state open \
              --json number,isCrossRepository \
              --jq '.[] | select(.isCrossRepository | not) | .number')" || return
            if [[ -n "${open_pr_number}" ]]; then
              echo "PR #${open_pr_number} already open for ${branch}."
            else
              gh pr create --base main --head "${branch}" \
                --title "${title}" \
                --body "${body}" || return
            fi
          }

          failed=0
          for manifest in "${manifests[@]}"; do
            if publish_shard "${manifest}"; then
              echo "Published shard ${manifest##*/}."
            else
              publish_status=$?
              echo "::error::Failed to publish shard ${manifest##*/}."
              failed=1
              if (( publish_status == PUSH_RETRIES_EXHAUSTED )); then
                # A systemic remote failure must not consume the remaining job
                # timeout by retrying every shard.
                break
              fi
            fi
          done

          # Reconcile only after full publication: a partial failure must not
          # delete the previous run's complete review branches. The successful
          # retry performs the cleanup.
          if (( failed != 0 )); then
            exit "${failed}"
          fi

          # A mode switch (standing <-> sharded) leaves the other mode's PRs
          # open with superseded content; close them and delete their branches.
          # Every successful manifest counts as published, including those already on main.
          published=" "
          for manifest in "${manifests[@]}"; do
            shard="${manifest##*/}"
            if [[ "${shard}" == "all" ]]; then
              published+="${branch_prefix} "
            else
              published+="${branch_prefix}-${shard} "
            fi
          done
          open_prs="${RUNNER_TEMP}/open-candidate-prs"
          gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls?state=open&per_page=100" \
            --jq '.[] | select(((.head.repo.full_name // "") == .base.repo.full_name) and (.head.ref | test("^(matched|reconciled)-advisories(-[0-9a-f]{2})?$"))) | [.number, .head.ref] | @tsv' \
            > "${open_prs}"
          while IFS=$'\t' read -r number head; do
            if [[ "${head}" != "${branch_prefix}" && "${head}" != "${branch_prefix}-"* ]]; then
              continue
            fi
            if [[ "${published}" != *" ${head} "* ]]; then
              gh pr close "${number}" --delete-branch \
                --comment "Superseded: this run published the current candidates elsewhere." || {
                echo "::error::Failed to close superseded PR #${number}."
                failed=1
              }
            fi
          done < "${open_prs}"

          exit "${failed}"

      - name: Upload failed matched snapshot
        if: failure() && steps.snapshot.outputs.created == 'true'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: matched-advisories-snapshot-${{ github.run_id }}-${{ github.run_attempt }}
          path: ${{ runner.temp }}/matched-advisories-snapshot.patch
          if-no-files-found: error
          retention-days: 7