ci: Fix sccache discarding writes, gate large cache saves to main, fix grype

.github/workflows/build-boot-artifacts.yml

@@ -155,9 +155,13 @@ jobs:
       # Persist mkosi's downloaded distro packages between runs. Restore-keys let
       # a run start from the most recent cache for this arch even when the
       # profile set has changed, which is the common case.
+      #
+      # restore/save rather than actions/cache so the save can be limited to main;
+      # see "Save mkosi package cache" below for why.
       - name: Restore mkosi package cache
+        id: mkosi-cache
         if: inputs.build_type == 'ephemeral'
-        uses: actions/cache@v4
+        uses: actions/cache/restore@v4
         with:
           path: pxe/.mkosi-package-cache
           key: mkosi-pkgs-${{ inputs.arch }}-${{ hashFiles('pxe/mkosi.conf', 'pxe/mkosi.profiles/**/mkosi.conf') }}
@@ -469,6 +473,23 @@ jobs:
 
           echo "Build completed!"
 
+      # Only main publishes this cache. Actions cache entries are ref-scoped: a
+      # pull request that saves writes a copy nothing else can read, so a herd of
+      # concurrent pull requests starting from a cold cache each stores its own
+      # byte-identical archive. Nine copies of this one 1.3 GB entry were live at
+      # once, 10.7 GB of duplication in the same 50 GB pool that has to hold
+      # main's sccache. Pull requests keep restoring main's copy through the
+      # restore-keys prefix above, so they lose nothing.
+      #
+      # Saved on failure as well, because the packages mkosi already downloaded
+      # are valid regardless of whether the image build that followed succeeded.
+      - name: Save mkosi package cache
+        if: ${{ !cancelled() && inputs.build_type == 'ephemeral' && github.ref == 'refs/heads/main' && steps.mkosi-cache.outputs.cache-hit != 'true' }}
+        uses: actions/cache/save@v4
+        with:
+          path: pxe/.mkosi-package-cache
+          key: ${{ steps.mkosi-cache.outputs.cache-primary-key }}
+
       # Activate the runtime prepared above only after both build paths finish.
       # This exports the matching Python library directory beside its PATH.
       - name: Activate Python for output validation

.github/workflows/ci.yaml

@@ -979,8 +979,16 @@ jobs:
       # eight minutes spent before the first test starts. Cargo.lock keys it, so
       # a run that changes no dependencies hits the primary key and skips the
       # upload rather than rewriting an equivalent entry.
+      #
+      # Only main saves. This entry is ~5 GB, and the restore-keys prefix above
+      # means any pull request whose Cargo.lock differs from main's takes a
+      # prefix match rather than an exact one, so cache-hit is false and the
+      # save fires, storing a ref-scoped 5 GB copy nothing else can read. Three
+      # such copies were live at once, 15 GB of a 50 GB pool, which evicted the
+      # sccache objects the image builds read down to 3.5 GB. Pull requests keep
+      # restoring main's entry through that same prefix, so they still start warm.
       - name: Save sccache
-        if: ${{ !cancelled() && steps.sccache.outputs.cache-hit != 'true' }}
+        if: ${{ !cancelled() && github.ref == 'refs/heads/main' && steps.sccache.outputs.cache-hit != 'true' }}
         uses: actions/cache/save@v4
         with:
           path: .sccache

.github/workflows/docker-build.yml

@@ -76,7 +76,7 @@ on:
         default: ''
         type: string
       scan:
-        description: 'Run a Grype vulnerability scan on main after the built image has been pushed.'
+        description: "Have main's pushed image scanned for vulnerabilities. Records a scan target; image-scan.yml scans it after this workflow finishes. Requires push."
         required: false
         default: false
         type: boolean
@@ -269,11 +269,13 @@ jobs:
 
       # Loading means exporting the whole image to the daemon, so it is only
       # worth doing when a later step reads it back: the deferred push on
-      # mirrors, or the main-branch Grype scan, which inspects the local
-      # daemon. A pull request runs neither, yet the 8 GB boot-artifacts
-      # carrier still spent a measured 5m20s on the export before every step
-      # that could have used it was skipped. Decided once here so the build and
-      # the summary cannot disagree about what happened.
+      # mirrors. A pull request never does, yet the 8 GB boot-artifacts carrier
+      # still spent a measured 5m20s on the export before every step that could
+      # have used it was skipped. Decided once here so the build and the summary
+      # cannot disagree about what happened.
+      #
+      # The scan clause is inert now the scan reads the registry, but is the only
+      # clause consulting inputs.load.
       - name: Decide whether to load the image locally
         id: loadgate
         env:
@@ -344,43 +346,34 @@ jobs:
       # Scan only the image pushed by the main branch build. This avoids a
       # vulnerability scan on every pull request while retaining the mainline
       # security signal.
-      - name: Compute Grype scan key
-        id: scankey
-        if: ${{ inputs.scan && github.ref == 'refs/heads/main' && steps.build.outcome == 'success' }}
+      #
+      # image-scan.yml does the scanning once this workflow has finished, so it
+      # no longer holds a build job open. Requires push: a later scan can only
+      # fetch an image that reached the registry.
+      - name: Record the pushed image as a scan target
+        id: scantarget
+        if: ${{ inputs.scan && inputs.push && github.ref == 'refs/heads/main' && steps.build.outcome == 'success' }}
         env:
           IMAGE_NAME: ${{ inputs.image_name }}
-        run: |
-          key="$(basename -- "$IMAGE_NAME")"
-          printf 'key=%s\n' "$key" >> "$GITHUB_OUTPUT"
-
-      # security-container-scan inspects the local Docker daemon and never pulls
-      # for itself. On this repo main pushes straight from buildx and loads
-      # nothing, so the scan's precheck reported "local docker image not found"
-      # and both the SBOM and Grype steps recorded outcome=skipped -- invisible,
-      # because continue-on-error and fail-build=false turn that into a green
-      # check. Fetching the image we just pushed is what makes the mainline scan
-      # actually scan. A warning rather than a failure: this must not be able to
-      # break publishing, but it must stop being silent.
-      - name: Fetch pushed image for the scan
-        if: ${{ inputs.scan && github.ref == 'refs/heads/main' && steps.build.outcome == 'success' && steps.loadgate.outputs.load != 'true' }}
-        env:
           IMAGE: ${{ steps.tag_list.outputs.primary }}
+          RUNNER_LABEL: ${{ inputs.runner }}
         run: |
-          if ! docker pull "${IMAGE}"; then
-            echo "::warning::could not pull ${IMAGE}; the Grype scan will find no local image and skip"
-          fi
+          set -euo pipefail
+          key="$(basename -- "${IMAGE_NAME}")"
+          printf 'key=%s\n' "${key}" >> "$GITHUB_OUTPUT"
+          # The runner label travels with the target: a single-platform manifest
+          # can only be pulled by the architecture that built it.
+          jq -n --arg key "${key}" --arg image "${IMAGE}" --arg runner "${RUNNER_LABEL}" \
+            '{key: $key, image: $image, runner: $runner}' | tee scan-target.json
 
-      - name: Grype vulnerability scan
-        if: ${{ inputs.scan && github.ref == 'refs/heads/main' && steps.build.outcome == 'success' }}
-        continue-on-error: true
-        uses: NVIDIA/dsx-github-actions/.github/actions/security-container-scan@aa4e470cc53f3886545c7d72a984eb81f1d203e2 # v1.16.2
+      - name: Upload the scan target
+        if: ${{ inputs.scan && inputs.push && github.ref == 'refs/heads/main' && steps.build.outcome == 'success' }}
+        uses: actions/upload-artifact@v4
         with:
-          image: ${{ inputs.image_name }}:${{ inputs.image_tag }}
-          fail-on: critical
-          fail-build: 'false'
-          write-summary: 'false'
-          artifact-name: grype-${{ steps.scankey.outputs.key }}-${{ github.run_id }}-${{ github.run_attempt }}
-          sbom-artifact-name: sbom-${{ steps.scankey.outputs.key }}-${{ github.run_id }}-${{ github.run_attempt }}
+          name: scan-target-${{ steps.scantarget.outputs.key }}
+          path: scan-target.json
+          # Consumed by image-scan.yml minutes later, never wanted again.
+          retention-days: 1
 
       - name: Display build summary
         run: |

.github/workflows/image-scan.yml

@@ -0,0 +1,198 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Grype vulnerability scans of the images main publishes.
+#
+# These ran inside the build job in docker-build.yml. They are advisory
+# (fail-build is false and nothing consumes their result), but they sat on Core
+# CI's critical path, where the last job a run waits on spent 24 of its 38
+# minutes fetching an 8 GB image back out of the registry and scanning it.
+# Running them after the workflow finishes covers the same images without Core CI
+# waiting on them.
+#
+# The contract: docker-build.yml's "Record the pushed image as a scan target"
+# step uploads one scan-target-<image> artifact per image, holding the image
+# reference and the runner label that built it. This workflow makes those its
+# matrix, so scan: true at the ci.yaml call site remains the only place deciding
+# which images are covered. Only main records targets, so only main scans.
+#
+# Editing note: workflow_run triggers are read from the default branch, so
+# changes here take effect only once merged and cannot be exercised by a pull
+# request.
+
+name: NICo Image Scan
+
+on:
+  workflow_run:
+    workflows: ["NICo Core CI"]
+    types: [completed]
+
+permissions:
+  contents: read
+  # Required to list and download the triggering run's artifacts.
+  actions: read
+
+jobs:
+  collect:
+    # workflow_run fires for every ref Core CI ran on, including pull requests,
+    # and those record no targets. Checked here so the common case costs one
+    # skipped job rather than an artifact listing.
+    if: ${{ github.event.workflow_run.head_branch == 'main' }}
+    runs-on: ubuntu-latest
+    outputs:
+      targets: ${{ steps.matrix.outputs.targets }}
+    steps:
+      # Deliberately not gated on the triggering run's conclusion. A target only
+      # exists for an image that built and pushed successfully, so one unrelated
+      # failing job elsewhere in Core CI is no reason to stop scanning the images
+      # that did publish.
+      - name: Look for scan targets
+        id: present
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          RUN_ID: ${{ github.event.workflow_run.id }}
+        run: |
+          set -euo pipefail
+          count=$(gh api --paginate "repos/${REPO}/actions/runs/${RUN_ID}/artifacts" \
+            --jq '[.artifacts[] | select(.name | startswith("scan-target-"))] | length' \
+            | jq -s 'add // 0')
+          printf 'count=%s\n' "${count}" >> "$GITHUB_OUTPUT"
+          if [ "${count}" -eq 0 ]; then
+            echo "::notice::Core CI run ${RUN_ID} recorded no scan targets; nothing to scan."
+          fi
+
+      # Skipped when there is nothing to fetch: download-artifact fails outright
+      # on a pattern that matches no artifact.
+      - name: Download the scan targets
+        if: ${{ steps.present.outputs.count != '0' }}
+        uses: actions/download-artifact@v4
+        with:
+          pattern: scan-target-*
+          path: targets
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build the scan matrix
+        id: matrix
+        env:
+          COUNT: ${{ steps.present.outputs.count }}
+        run: |
+          set -euo pipefail
+          if [ "${COUNT}" = '0' ]; then
+            printf 'targets=[]\n' >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          # Each artifact expands to its own directory holding one
+          # scan-target.json; -s folds the objects into the array the matrix
+          # takes. Sorted so the job list is stable between runs.
+          targets=$(find targets -name scan-target.json -exec cat {} + | jq -sc 'sort_by(.key)')
+          printf 'targets=%s\n' "${targets}" >> "$GITHUB_OUTPUT"
+          jq -r '.[] | "\(.key)\t\(.image)\t\(.runner)"' <<<"${targets}"
+
+  scan:
+    needs: collect
+    if: ${{ needs.collect.outputs.targets != '[]' }}
+    strategy:
+      # The images are independent, and a scan that fails should not hide the
+      # results for the rest.
+      fail-fast: false
+      matrix:
+        target: ${{ fromJson(needs.collect.outputs.targets) }}
+    # The label the image was built on. A single-platform manifest can only be
+    # pulled by a host of that architecture.
+    runs-on: ${{ matrix.target.runner }}
+    # Generous against the largest image: the boot-artifacts carrier has taken
+    # 24 minutes to fetch and scan. Present so a hung scan cannot occupy a
+    # self-hosted runner for the six-hour default.
+    timeout-minutes: 60
+    steps:
+      - name: Resolve the registry host
+        id: registry
+        env:
+          IMAGE: ${{ matrix.target.image }}
+        run: printf 'host=%s\n' "${IMAGE%%/*}" >> "$GITHUB_OUTPUT"
+
+      # Same credentials and fallback order docker-build.yml pushed with, so
+      # anything it could publish, this can read back.
+      - name: Log in to the registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ steps.registry.outputs.host }}
+          username: ${{ secrets.NICO_TARGET_REGISTRY_USERNAME || secrets.NVCR_USERNAME }}
+          password: ${{ secrets.NICO_TARGET_REGISTRY_TOKEN || secrets.NVCR_TOKEN }}
+
+      # security-container-scan inspects the local Docker daemon and never pulls
+      # for itself; without this the scan reports "local docker image not found"
+      # and silently skips.
+      - name: Pull the image
+        env:
+          IMAGE: ${{ matrix.target.image }}
+        run: docker pull "${IMAGE}"
+
+      - name: Grype vulnerability scan
+        id: scan
+        continue-on-error: true
+        uses: NVIDIA/dsx-github-actions/.github/actions/security-container-scan@aa4e470cc53f3886545c7d72a984eb81f1d203e2 # v1.16.2
+        with:
+          image: ${{ matrix.target.image }}
+          fail-on: critical
+          fail-build: 'false'
+          write-summary: 'false'
+          # Named after the Core CI run rather than this one, which is what makes
+          # a report traceable back to the build that produced the image.
+          artifact-name: grype-${{ matrix.target.key }}-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}
+          sbom-artifact-name: sbom-${{ matrix.target.key }}-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}
+
+      # fail-build stays false, preserving the policy the scan already had: this
+      # change is about when the scan runs, not about converting pre-existing
+      # findings into red checks. Since nothing downstream waits on this
+      # workflow any more, tightening that is now a decision on its own.
+      # Meanwhile the status is at least stated, rather than being knowable only
+      # by opening the artifact.
+      - name: Report the scan status
+        if: ${{ !cancelled() }}
+        env:
+          KEY: ${{ matrix.target.key }}
+          IMAGE: ${{ matrix.target.image }}
+          STATUS: ${{ steps.scan.outputs.status }}
+          DETAIL: ${{ steps.scan.outputs.detail }}
+          SHA: ${{ github.event.workflow_run.head_sha }}
+        run: |
+          set -euo pipefail
+          status="${STATUS:-unknown}"
+          # Counts and statuses only. This summary is public on a public
+          # repository, so which CVE affects which shipped image stays in the
+          # artifact rather than being published as a checklist.
+          {
+            printf '### %s\n\n' "${KEY}"
+            printf -- '- Image: %s\n' "${IMAGE}"
+            printf -- '- Commit: %s\n' "${SHA}"
+            printf -- '- Status: %s\n' "${status}"
+          } >>"$GITHUB_STEP_SUMMARY"
+          if [ "${status}" != 'ok' ]; then
+            echo "::warning::${KEY}: Grype status ${status} (${DETAIL:-no detail reported})"
+          fi
+
+      # These runners are self-hosted, so the daemon keeps whatever a job leaves
+      # behind. Nothing here reuses the image, and the boot-artifacts carrier
+      # alone is 8 GB, so every scan would otherwise add to the disk until it
+      # ran out. Untagging is enough: the layers become unreferenced and the
+      # daemon reclaims them.
+      - name: Remove the pulled image
+        if: ${{ always() }}
+        env:
+          IMAGE: ${{ matrix.target.image }}
+        run: docker image rm "${IMAGE}" || true