ci: sign release artifacts with cosign and attach SPDX SBOMs

.github/workflows/release.yml

@@ -75,11 +75,15 @@ jobs:
     permissions:
       contents: read
       packages: write
+      # Keyless signing exchanges this workflow's OIDC identity for a
+      # short-lived Fulcio certificate; there is no long-lived key to store.
+      id-token: write
     env:
       REGISTRY: ${{ needs.setup.outputs.registry }}
       TAG: ${{ needs.setup.outputs.tag }}
       VERSION: ${{ needs.setup.outputs.version }}
       PRERELEASE: ${{ needs.setup.outputs.prerelease }}
+      IMAGE_METADATA_DIR: _image-metadata
     steps:
       - name: Checkout code
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
@@ -93,26 +97,36 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Install cosign
+        uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
+        with:
+          cosign-release: v3.1.3
+
       # A single multi-arch `docker buildx build` per image publishes the full
       # tag (e.g. "v0.4.0") and, for stable releases, the short alias
       # (e.g. "0.4.0") in the same push - no separate retag step. RC images are
       # published under the long form ("v0.4.0-rc.1") for testers only: the
       # short-tag namespace is deliberately reserved for stable releases that
       # consumers can safely pin to, so RC tags get no short alias.
+      #
+      # IMAGE_SBOM attaches a per-platform SPDX SBOM to the image index;
+      # IMAGE_METADATA_DIR captures the digest buildx actually pushed, which is
+      # what gets signed below.
       - name: Build and push images with tag ${{ needs.setup.outputs.tag }}
         run: |
           set -euo pipefail
-          if [ "${PRERELEASE}" = "true" ]; then
-            make push
-          else
-            make push IMAGE_EXTRA_TAG="${VERSION}"
+          mkdir -p "${IMAGE_METADATA_DIR}"
+          extra_tag=""
+          if [ "${PRERELEASE}" != "true" ]; then
+            extra_tag="${VERSION}"
           fi
+          make push IMAGE_SBOM=true IMAGE_METADATA_DIR="${IMAGE_METADATA_DIR}" IMAGE_EXTRA_TAG="${extra_tag}"
 
       # Confirm every published tag is a real multi-arch manifest list rather
       # than a silently degraded single-arch image: a build that pushed only one
       # architecture would otherwise go unnoticed until a consumer on the other
       # architecture failed to pull. Stable releases also carry the short alias.
-      - name: Verify images are multi-arch
+      - name: Verify published image manifests and SBOMs
         run: |
           set -euo pipefail
           tags="${TAG}"
@@ -130,8 +144,21 @@ jobs:
                   || { echo "::error::${ref} is missing platform ${platform}"; exit 1; }
               done
             done
+            # Once per image: every tag resolves to the same index. IMAGE_SBOM is
+            # matched as an exact string in the Makefile, so a future "TRUE" or
+            # "1" would quietly produce no SBOM; assert the attestation is really
+            # attached rather than trusting the flag was spelled as make expects.
+            sbom="$(docker buildx imagetools inspect "${REGISTRY}/${IMAGE}:${TAG}" \
+              --format '{{ json (index .SBOM "linux/amd64").SPDX }}' 2>/dev/null || true)"
+            if [ -z "${sbom}" ] || [ "${sbom}" = "null" ]; then
+              echo "::error::${REGISTRY}/${IMAGE}:${TAG} has no SPDX SBOM attached"
+              exit 1
+            fi
           done
 
+      - name: Sign the published images
+        run: ./hack/release/sign-images.sh
+
   # Publish the raw CRDs as a standalone release asset so consumers can install
   # them without pulling a Helm chart. The bundle carries the unmodified CRDs the
   # charts install (no downstream-specific labels), split into crds/hub and
@@ -141,6 +168,7 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write
+      id-token: write
     env:
       TAG: ${{ needs.setup.outputs.tag }}
     steps:
@@ -149,9 +177,17 @@ jobs:
         with:
           ref: ${{ needs.setup.outputs.tag }}
 
+      - name: Install cosign
+        uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
+        with:
+          cosign-release: v3.1.3
+
       - name: Package CRDs
         run: make crd-package TAG="${TAG}"
 
+      - name: Sign the CRD bundle checksum
+        run: ./hack/release/sign-crd-bundle.sh
+
       # --clobber makes the upload idempotent so a re-run replaces the asset
       # rather than failing on a name collision. The token is scoped to this
       # step so it is not in the environment of the packaging step above.
@@ -163,6 +199,7 @@ jobs:
           gh release upload "${TAG}" \
             "_crd-package/kubefleet-crds-${TAG}.tgz" \
             "_crd-package/kubefleet-crds-${TAG}.tgz.sha256" \
+            "_crd-package/kubefleet-crds-${TAG}.tgz.sha256.bundle" \
             --clobber
 
   # Charts are published only for stable releases: an RC must be installable by

.github/workflows/workflow-lint.yml

@@ -49,7 +49,7 @@ jobs:
       # The release scripts live outside the workflow files, so actionlint's
       # embedded-shell checking does not reach them.
       - name: Shellcheck the release scripts
-        run: shellcheck hack/release/*.sh hack/release/testdata/gh
+        run: shellcheck hack/release/*.sh hack/release/testdata/*
 
       - name: Test the release scripts
         run: ./hack/release/test-release-scripts.sh

.gitignore

@@ -51,3 +51,4 @@ _crd-package/
 # Squad: SubSquad activation file (local to this machine)
 .squad-workstream
 .squad/.cache/
+_image-metadata/

Makefile

@@ -265,6 +265,24 @@ BINFMT_IMAGE ?= mcr.microsoft.com/mirror/docker/tonistiigi/binfmt:$(BINFMT_VERSI
 PLATFORMS ?= $(TARGET_OS)/$(TARGET_ARCH)
 RELEASE_PLATFORMS ?= linux/amd64,linux/arm64
 
+# Attach an SPDX SBOM to the image index. BuildKit generates one per platform
+# and stores it alongside the image in the registry, so consumers can read what
+# is in an image without unpacking it. Off by default because it slows every
+# local build; the release path turns it on.
+IMAGE_SBOM ?= false
+
+# When set, buildx writes each image's build metadata (including
+# "containerimage.digest") to $(IMAGE_METADATA_DIR)/<image>.json. The release
+# workflow signs those digests: signing a tag would sign whatever the tag points
+# at when cosign runs, not what this build actually pushed.
+IMAGE_METADATA_DIR ?=
+
+# Expanded into every docker-build-* target. Kept here so the three recipes stay
+# identical to each other.
+image_build_flags = \
+	$(if $(filter true,$(IMAGE_SBOM)),--sbom=true) \
+	$(if $(IMAGE_METADATA_DIR),--metadata-file $(IMAGE_METADATA_DIR)/$(1).json)
+
 .PHONY: push
 push: ## Build and push all Docker images as multi-arch manifests
 	$(MAKE) OUTPUT_TYPE="type=registry" PLATFORMS="$(RELEASE_PLATFORMS)" docker-build-hub-agent docker-build-member-agent docker-build-refresh-token
@@ -366,6 +384,7 @@ docker-build-hub-agent: docker-buildx-builder ## Build hub-agent image
 		--pull \
 		--tag $(REGISTRY)/$(HUB_AGENT_IMAGE_NAME):$(HUB_AGENT_IMAGE_VERSION) \
 		$(if $(IMAGE_EXTRA_TAG),--tag $(REGISTRY)/$(HUB_AGENT_IMAGE_NAME):$(IMAGE_EXTRA_TAG)) \
+		$(call image_build_flags,$(HUB_AGENT_IMAGE_NAME)) \
 		--progress=$(BUILDKIT_PROGRESS_TYPE) .
 
 .PHONY: docker-build-member-agent
@@ -377,6 +396,7 @@ docker-build-member-agent: docker-buildx-builder ## Build member-agent image
 		--pull \
 		--tag $(REGISTRY)/$(MEMBER_AGENT_IMAGE_NAME):$(MEMBER_AGENT_IMAGE_VERSION) \
 		$(if $(IMAGE_EXTRA_TAG),--tag $(REGISTRY)/$(MEMBER_AGENT_IMAGE_NAME):$(IMAGE_EXTRA_TAG)) \
+		$(call image_build_flags,$(MEMBER_AGENT_IMAGE_NAME)) \
 		--progress=$(BUILDKIT_PROGRESS_TYPE) .
 
 .PHONY: docker-build-refresh-token
@@ -388,6 +408,7 @@ docker-build-refresh-token: docker-buildx-builder ## Build refresh-token image
 		--pull \
 		--tag $(REGISTRY)/$(REFRESH_TOKEN_IMAGE_NAME):$(REFRESH_TOKEN_IMAGE_VERSION) \
 		$(if $(IMAGE_EXTRA_TAG),--tag $(REGISTRY)/$(REFRESH_TOKEN_IMAGE_NAME):$(IMAGE_EXTRA_TAG)) \
+		$(call image_build_flags,$(REFRESH_TOKEN_IMAGE_NAME)) \
 		--progress=$(BUILDKIT_PROGRESS_TYPE) .
 
 ## -----------------------------------

RELEASING.md

@@ -11,7 +11,8 @@ supported are covered in [VERSIONING.md](VERSIONING.md) and
 | Artifact | Location | Stable (`v0.4.0`) | Release candidate (`v0.4.0-rc.1`) |
 | --- | --- | --- | --- |
 | Agent images (`hub-agent`, `member-agent`, `refresh-token`) | `ghcr.io/kubefleet-dev/kubefleet/<image>` | `:v0.4.0` and `:0.4.0` | `:v0.4.0-rc.1` only |
-| CRD bundle (`kubefleet-crds-<tag>.tgz` + `.sha256`) | GitHub Release asset | Yes | Yes |
+| Image signatures + SPDX SBOMs | Alongside each image in the registry | Yes | Yes |
+| CRD bundle (`kubefleet-crds-<tag>.tgz`, `.sha256`, `.sha256.bundle`) | GitHub Release asset | Yes | Yes |
 | Helm charts (OCI) | `oci://ghcr.io/kubefleet-dev/kubefleet/charts/<chart>` | Yes | No |
 | Helm charts (index) | `https://kubefleet-dev.github.io/kubefleet/charts` | Yes | No |
 | GitHub Release | Releases page | Published | Published, flagged pre-release |
@@ -89,6 +90,64 @@ The two jobs with real branching logic — `create-draft-release` and
 in the workflow, and are covered by `hack/release/test-release-scripts.sh`,
 which CI runs on every change to either.
 
+## Verifying a release
+
+The container images and the CRD bundle are signed with
+[cosign](https://docs.sigstore.dev/) in keyless mode: the workflow's GitHub OIDC
+identity is exchanged for a short-lived Fulcio certificate and the signature is
+recorded in Rekor. There is no long-lived signing key to hold, and nothing to
+rotate.
+
+The Helm charts are **not** signed yet, on either the OCI or the index channel.
+Signing the OCI charts is the same keyless flow used for images and is tracked
+as a follow-up; the classic-repo `.prov` mechanism needs a long-lived GPG key
+this project has no custody story for.
+
+Images are signed by digest rather than by tag, so a signature is bound to the
+exact bytes the release built. Verify one with:
+
+```bash
+cosign verify \
+  --certificate-oidc-issuer https://token.actions.githubusercontent.com \
+  --certificate-identity-regexp '^https://github\.com/kubefleet-dev/kubefleet/\.github/workflows/release\.yml@' \
+  ghcr.io/kubefleet-dev/kubefleet/hub-agent:v0.4.0
+```
+
+Each image also carries a per-platform SPDX SBOM attached to its index, readable
+without pulling the image:
+
+```bash
+docker buildx imagetools inspect ghcr.io/kubefleet-dev/kubefleet/hub-agent:v0.4.0 \
+  --format '{{ json (index .SBOM "linux/amd64").SPDX }}'
+```
+
+`.SBOM` is keyed by platform because every published image is a multi-platform
+index; there is one SBOM per architecture.
+
+The CRD bundle's checksum file is signed as a blob; verifying it and then
+checking the tarball against it covers the tarball:
+
+```bash
+cosign verify-blob \
+  --bundle kubefleet-crds-v0.4.0.tgz.sha256.bundle \
+  --certificate-oidc-issuer https://token.actions.githubusercontent.com \
+  --certificate-identity-regexp '^https://github\.com/kubefleet-dev/kubefleet/\.github/workflows/release\.yml@' \
+  kubefleet-crds-v0.4.0.tgz.sha256
+sha256sum -c kubefleet-crds-v0.4.0.tgz.sha256
+```
+
+Verification needs Sigstore's trust root (the Fulcio and Rekor keys), which
+cosign fetches once and caches under `~/.sigstore`. On a machine with no network
+access, prime that cache first or pass `--trusted-root`; otherwise the commands
+above fail for a reason that has nothing to do with the signature.
+
+The release workflow runs these same verifications immediately after signing, so
+a signature that cannot be verified fails the release instead of shipping.
+`release.yml` is the only workflow holding `id-token: write`. Any OIDC trust
+policy added later (cloud role assumption, trusted publishing) must be scoped to
+that workflow's `job_workflow_ref`, never to `repo:kubefleet-dev/kubefleet:*`,
+or it would be assumable from any workflow in the repository.
+
 ## Recovering from a failed run
 
 The normal recovery is **Re-run failed jobs** on the workflow run. Jobs that
@@ -107,8 +166,9 @@ is harmless while the release is still a draft — nothing has been announced ye
 | `create-draft-release` | Nothing | See [Re-releasing an existing tag](#re-releasing-an-existing-tag) if it refused because the release is already published. |
 | `publish-images` | Any images pushed before the failure (`make push` builds hub-agent, member-agent, then refresh-token in order) | Fix, then re-run failed jobs. |
 | `publish-crds` | Possibly the images — it runs in parallel with `publish-images`, not after it | Fix, then re-run failed jobs. |
+| Either signing step | Whatever that job published before signing | Usually a Sigstore or registry transient rather than a code fault — re-run failed jobs first. A signature that will not verify fails the job by design, so nothing unverifiable ships. |
 | `publish-charts-oci` / `publish-charts-pages` | Images; CRD bundle is attached to the still-hidden draft | Fix, then re-run failed jobs. The release stays a draft until the charts land. |
-| `publish-release` | Images, charts | The asset check found the draft incomplete or its bundle failed its own checksum. Inspect `gh release view <tag>`, re-upload, re-run failed jobs. |
+| `publish-release` | Images, charts | The asset check found the draft incomplete or its bundle failed its own checksum. Re-run `publish-crds` rather than hand-uploading: it regenerates the tarball, checksum, and signature together, and a hand-replaced checksum would no longer match its signature. |
 
 `publish-charts-pages` serializes across *all* releases, because the action it
 uses rewrites the whole `gh-pages` branch. GitHub keeps at most one pending

hack/release/publish-release.sh

@@ -19,14 +19,15 @@ set -euo pipefail
 
 bundle="kubefleet-crds-${TAG}.tgz"
 checksum="${bundle}.sha256"
+signature="${checksum}.bundle"
 
 # Only assets GitHub finished receiving count. An upload interrupted mid-stream
 # leaves an asset row with the right name in a non-"uploaded" state, which a
 # name-only check would accept.
 assets="$(gh release view "${TAG}" --json assets \
   --jq '.assets[] | select(.state == "uploaded" and .size > 0) | .name')"
 
-for want in "${bundle}" "${checksum}"; do
+for want in "${bundle}" "${checksum}" "${signature}"; do
   if ! grep -qxF -- "${want}" <<<"${assets}"; then
     echo "::error::Release ${TAG} is missing fully-uploaded asset ${want}; leaving it as a draft."
     exit 1

hack/release/sign-crd-bundle.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# Sign the CRD bundle's checksum file with cosign, keyless.
+#
+# The checksum is what gets signed rather than the tarball itself: it already
+# binds the tarball's bytes, it is the file a consumer checks the download
+# against, and it keeps the signature bundle small. Verifying the signature and
+# then running `sha256sum -c` covers the tarball transitively.
+#
+# Environment:
+#   TAG               release tag, e.g. v0.4.0
+#   CRD_PACKAGE_DIR   directory holding the packaged bundle (default: _crd-package)
+#   GITHUB_REPOSITORY owner/repo, used to bound the identity on verification
+
+set -euo pipefail
+
+: "${TAG:?TAG must be set}"
+: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY must be set}"
+
+package_dir="${CRD_PACKAGE_DIR:-_crd-package}"
+checksum="${package_dir}/kubefleet-crds-${TAG}.tgz.sha256"
+bundle="${checksum}.bundle"
+
+if [ ! -f "${checksum}" ]; then
+  echo "::error::No checksum file at ${checksum}; run 'make crd-package' first."
+  exit 1
+fi
+
+OIDC_ISSUER="https://token.actions.githubusercontent.com"
+# See sign-images.sh for why this is bound to the workflow file and why the
+# repository name has its dots escaped before going into a regexp.
+repo_pattern="${GITHUB_REPOSITORY//./\\.}"
+IDENTITY_PATTERN="^https://github\.com/${repo_pattern}/\.github/workflows/release\.yml@"
+
+echo "Signing ${checksum}"
+cosign sign-blob --yes --bundle "${bundle}" "${checksum}"
+
+# Same reasoning as sign-images.sh: verify with the identity consumers will use,
+# so a signature that cannot be verified fails the release rather than shipping.
+cosign verify-blob \
+  --bundle "${bundle}" \
+  --certificate-oidc-issuer "${OIDC_ISSUER}" \
+  --certificate-identity-regexp "${IDENTITY_PATTERN}" \
+  "${checksum}" >/dev/null
+
+echo "✅ ${checksum} signed and verified; bundle at ${bundle}"