kubefleet-dev · Yetkin Timocin (ytimocin) · Aug 13, 2026 · Aug 18, 2026
@@ -66,6 +66,13 @@ jobs:
         with:
           ref: ${{ needs.setup.outputs.tag }}
 
+      # This job gates every publishing job, so failing here is the last point
+      # at which a release can be abandoned for free. Signing runs long after
+      # the images are public, so a run started from a ref that cannot produce
+      # a verifiable signature has to be stopped now or not at all.
+      - name: Check this ref can sign a release
+        run: ./hack/release/check-signing-ref.sh
+
       - name: Create or reuse the draft release
         run: ./hack/release/create-draft-release.sh
 
@@ -75,11 +82,15 @@ jobs:
     permissions:
       contents: read
       packages: write
+      # Keyless signing exchanges this workflow's OIDC identity for a
+      # short-lived Fulcio certificate; there is no long-lived key to store.
+      id-token: write
     env:
       REGISTRY: ${{ needs.setup.outputs.registry }}
       TAG: ${{ needs.setup.outputs.tag }}
       VERSION: ${{ needs.setup.outputs.version }}
       PRERELEASE: ${{ needs.setup.outputs.prerelease }}
+      IMAGE_METADATA_DIR: _image-metadata
     steps:
       - name: Checkout code
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
@@ -93,45 +104,96 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Install cosign
+        uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
+        with:
+          cosign-release: v3.1.3
+
       # A single multi-arch `docker buildx build` per image publishes the full
       # tag (e.g. "v0.4.0") and, for stable releases, the short alias
       # (e.g. "0.4.0") in the same push - no separate retag step. RC images are
       # published under the long form ("v0.4.0-rc.1") for testers only: the
       # short-tag namespace is deliberately reserved for stable releases that
       # consumers can safely pin to, so RC tags get no short alias.
+      #
+      # IMAGE_SBOM attaches a per-platform SPDX SBOM to the image index;
+      # IMAGE_METADATA_DIR captures the digest buildx actually pushed, which is
+      # what gets signed below.
       - name: Build and push images with tag ${{ needs.setup.outputs.tag }}
         run: |
           set -euo pipefail
-          if [ "${PRERELEASE}" = "true" ]; then
-            make push
-          else
-            make push IMAGE_EXTRA_TAG="${VERSION}"
+          mkdir -p "${IMAGE_METADATA_DIR}"
+          extra_tag=""
+          if [ "${PRERELEASE}" != "true" ]; then
+            extra_tag="${VERSION}"
           fi
+          make push IMAGE_SBOM=true IMAGE_METADATA_DIR="${IMAGE_METADATA_DIR}" IMAGE_EXTRA_TAG="${extra_tag}"
 
       # Confirm every published tag is a real multi-arch manifest list rather
       # than a silently degraded single-arch image: a build that pushed only one
       # architecture would otherwise go unnoticed until a consumer on the other
       # architecture failed to pull. Stable releases also carry the short alias.
-      - name: Verify images are multi-arch
+      - name: Verify published image manifests and SBOMs
         run: |
           set -euo pipefail
           tags="${TAG}"
           if [ "${PRERELEASE}" != "true" ]; then
             tags="${tags} ${VERSION}"
           fi
-          echo "✅ Verifying published images:"
+          # An independent statement of what a release must contain. Deriving
+          # this from RELEASE_PLATFORMS would make the build config both the
+          # input and the oracle: dropping a platform there - or overriding it
+          # in the job env - would silently narrow the check to match, and a
+          # single-arch release would ship green. Cross-checked against the
+          # Makefile so the two cannot disagree without someone noticing.
+          expected_platforms="linux/amd64 linux/arm64"
+          # Compared as sets: this is about which platforms are built, and
+          # failing a release over a reordered RELEASE_PLATFORMS would be noise.
+          built_sorted="$(make -s print-release-platforms | sort | xargs)"
+          expected_sorted="$(tr ' ' '\n' <<<"${expected_platforms}" | sort | xargs)"
+          if [ "${built_sorted}" != "${expected_sorted}" ]; then
+            echo "::error::RELEASE_PLATFORMS builds '${built_sorted}' but this release is verified for '${expected_sorted}'. Update both deliberately."
+            exit 1
+          fi
+          platforms="${expected_platforms}"
+          echo "✅ Verifying published images for: ${platforms}"
           for IMAGE in "${HUB_AGENT_IMAGE_NAME}" "${MEMBER_AGENT_IMAGE_NAME}" "${REFRESH_TOKEN_IMAGE_NAME}"; do
             for tag in ${tags}; do
               ref="${REGISTRY}/${IMAGE}:${tag}"
               echo "  - ${ref}"
               manifest="$(docker buildx imagetools inspect "${ref}")"
-              for platform in linux/amd64 linux/arm64; do
+              for platform in ${platforms}; do
                 grep -q "Platform:.*${platform}" <<<"${manifest}" \
                   || { echo "::error::${ref} is missing platform ${platform}"; exit 1; }
               done
             done
+            # Once per tag would be redundant - every tag resolves to the same
+            # index - but once per platform is not: BuildKit emits one SBOM per
+            # platform, and an image whose arm64 SBOM was missing would
+            # otherwise publish behind a green amd64 check. IMAGE_SBOM is
+            # matched as an exact string in the Makefile, so a future "TRUE" or
+            # "1" would quietly produce no SBOM; assert the attestation is really
+            # attached rather than trusting the flag was spelled as make expects.
+            for platform in ${platforms}; do
+              sbom_err="$(mktemp)"
+              sbom="$(docker buildx imagetools inspect "${REGISTRY}/${IMAGE}:${TAG}" \
+                --format "{{ json (index .SBOM \"${platform}\").SPDX }}" 2>"${sbom_err}" || true)"
+              if [ -z "${sbom}" ] || [ "${sbom}" = "null" ]; then
+                # Print what buildx said. A missing attestation and a registry
+                # or template error are different faults with different fixes,
+                # and discarding stderr makes every one of them look like the
+                # first.
+                echo "::error::${REGISTRY}/${IMAGE}:${TAG} has no SPDX SBOM for ${platform}: $(tr '\n' ' ' <"${sbom_err}")"
+                rm -f "${sbom_err}"
+                exit 1
+              fi
+              rm -f "${sbom_err}"
+            done
           done
 
+      - name: Sign the published images
+        run: ./hack/release/sign-images.sh
+
   # Publish the raw CRDs as a standalone release asset so consumers can install
   # them without pulling a Helm chart. The bundle carries the unmodified CRDs the
   # charts install (no downstream-specific labels), split into crds/hub and
@@ -141,6 +203,7 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write
+      id-token: write
     env:
       TAG: ${{ needs.setup.outputs.tag }}
     steps:
@@ -149,9 +212,17 @@ jobs:
         with:
           ref: ${{ needs.setup.outputs.tag }}
 
+      - name: Install cosign
+        uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
+        with:
+          cosign-release: v3.1.3
+
       - name: Package CRDs
         run: make crd-package TAG="${TAG}"
 
+      - name: Sign the CRD bundle checksum
+        run: ./hack/release/sign-crd-bundle.sh
+
       # --clobber makes the upload idempotent so a re-run replaces the asset
       # rather than failing on a name collision. The token is scoped to this
       # step so it is not in the environment of the packaging step above.
@@ -163,6 +234,7 @@ jobs:
           gh release upload "${TAG}" \
             "_crd-package/kubefleet-crds-${TAG}.tgz" \
             "_crd-package/kubefleet-crds-${TAG}.tgz.sha256" \
+            "_crd-package/kubefleet-crds-${TAG}.tgz.sha256.bundle" \
             --clobber
 
   # Charts are published only for stable releases: an RC must be installable by

@@ -49,7 +49,7 @@ jobs:
       # The release scripts live outside the workflow files, so actionlint's
       # embedded-shell checking does not reach them.
       - name: Shellcheck the release scripts
-        run: shellcheck hack/release/*.sh hack/release/testdata/gh
+        run: shellcheck hack/release/*.sh hack/release/testdata/*
 
       - name: Test the release scripts
         run: ./hack/release/test-release-scripts.sh
@@ -51,3 +51,4 @@ _crd-package/
 # Squad: SubSquad activation file (local to this machine)
 .squad-workstream
 .squad/.cache/
+_image-metadata/
@@ -265,6 +265,28 @@ BINFMT_IMAGE ?= mcr.microsoft.com/mirror/docker/tonistiigi/binfmt:$(BINFMT_VERSI
 PLATFORMS ?= $(TARGET_OS)/$(TARGET_ARCH)
 RELEASE_PLATFORMS ?= linux/amd64,linux/arm64
 
+.PHONY: print-release-platforms
+print-release-platforms: ## Print the platforms a release publishes, one per line
+	@echo "$(RELEASE_PLATFORMS)" | tr ',' '\n'
+
+# Attach an SPDX SBOM to the image index. BuildKit generates one per platform
+# and stores it alongside the image in the registry, so consumers can read what
+# is in an image without unpacking it. Off by default because it slows every
+# local build; the release path turns it on.
+IMAGE_SBOM ?= false
+
+# When set, buildx writes each image's build metadata (including
+# "containerimage.digest") to $(IMAGE_METADATA_DIR)/<image>.json. The release
+# workflow signs those digests: signing a tag would sign whatever the tag points
+# at when cosign runs, not what this build actually pushed.
+IMAGE_METADATA_DIR ?=
+
+# Expanded into every docker-build-* target. Kept here so the three recipes stay
+# identical to each other.
+image_build_flags = \
+	$(if $(filter true,$(IMAGE_SBOM)),--sbom=true) \
+	$(if $(IMAGE_METADATA_DIR),--metadata-file $(IMAGE_METADATA_DIR)/$(1).json)
+
 .PHONY: push
 push: ## Build and push all Docker images as multi-arch manifests
 	$(MAKE) OUTPUT_TYPE="type=registry" PLATFORMS="$(RELEASE_PLATFORMS)" docker-build-hub-agent docker-build-member-agent docker-build-refresh-token
@@ -366,6 +388,7 @@ docker-build-hub-agent: docker-buildx-builder ## Build hub-agent image
 		--pull \
 		--tag $(REGISTRY)/$(HUB_AGENT_IMAGE_NAME):$(HUB_AGENT_IMAGE_VERSION) \
 		$(if $(IMAGE_EXTRA_TAG),--tag $(REGISTRY)/$(HUB_AGENT_IMAGE_NAME):$(IMAGE_EXTRA_TAG)) \
+		$(call image_build_flags,$(HUB_AGENT_IMAGE_NAME)) \
 		--progress=$(BUILDKIT_PROGRESS_TYPE) .
 
 .PHONY: docker-build-member-agent
@@ -377,6 +400,7 @@ docker-build-member-agent: docker-buildx-builder ## Build member-agent image
 		--pull \
 		--tag $(REGISTRY)/$(MEMBER_AGENT_IMAGE_NAME):$(MEMBER_AGENT_IMAGE_VERSION) \
 		$(if $(IMAGE_EXTRA_TAG),--tag $(REGISTRY)/$(MEMBER_AGENT_IMAGE_NAME):$(IMAGE_EXTRA_TAG)) \
+		$(call image_build_flags,$(MEMBER_AGENT_IMAGE_NAME)) \
 		--progress=$(BUILDKIT_PROGRESS_TYPE) .
 
 .PHONY: docker-build-refresh-token
@@ -388,6 +412,7 @@ docker-build-refresh-token: docker-buildx-builder ## Build refresh-token image
 		--pull \
 		--tag $(REGISTRY)/$(REFRESH_TOKEN_IMAGE_NAME):$(REFRESH_TOKEN_IMAGE_VERSION) \
 		$(if $(IMAGE_EXTRA_TAG),--tag $(REGISTRY)/$(REFRESH_TOKEN_IMAGE_NAME):$(IMAGE_EXTRA_TAG)) \
+		$(call image_build_flags,$(REFRESH_TOKEN_IMAGE_NAME)) \
 		--progress=$(BUILDKIT_PROGRESS_TYPE) .
 
 ## -----------------------------------

@@ -11,7 +11,8 @@ supported are covered in [VERSIONING.md](VERSIONING.md) and
 | Artifact | Location | Stable (`v0.4.0`) | Release candidate (`v0.4.0-rc.1`) |
 | --- | --- | --- | --- |
 | Agent images (`hub-agent`, `member-agent`, `refresh-token`) | `ghcr.io/kubefleet-dev/kubefleet/<image>` | `:v0.4.0` and `:0.4.0` | `:v0.4.0-rc.1` only |
-| CRD bundle (`kubefleet-crds-<tag>.tgz` + `.sha256`) | GitHub Release asset | Yes | Yes |
+| Image signatures + SPDX SBOMs | Alongside each image in the registry | Yes | Yes |
+| CRD bundle (`kubefleet-crds-<tag>.tgz`, `.sha256`, `.sha256.bundle`) | GitHub Release asset | Yes | Yes |
 | Helm charts (OCI) | `oci://ghcr.io/kubefleet-dev/kubefleet/charts/<chart>` | Yes | No |
 | Helm charts (index) | `https://kubefleet-dev.github.io/kubefleet/charts` | Yes | No |
 | GitHub Release | Releases page | Published | Published, flagged pre-release |
@@ -89,6 +90,92 @@ The two jobs with real branching logic — `create-draft-release` and
 in the workflow, and are covered by `hack/release/test-release-scripts.sh`,
 which CI runs on every change to either.
 
+## Verifying a release
+
+The container images and the CRD bundle are signed with
+[cosign](https://docs.sigstore.dev/) in keyless mode: the workflow's GitHub OIDC
+identity is exchanged for a short-lived Fulcio certificate and the signature is
+recorded in Rekor. There is no long-lived signing key to hold, and nothing to
+rotate.
+
+The Helm charts are **not** signed yet, on either the OCI or the index channel.
+Signing the OCI charts is the same keyless flow used for images and is tracked
+as a follow-up; the classic-repo `.prov` mechanism needs a long-lived GPG key
+this project has no custody story for.
+
+Releases are signed with cosign v3 (the workflow pins the exact version). Verify
+with cosign v3.0 or later — the `verify-blob --bundle` format below changed
+between v2 and v3, and an older client reports it as a bad signature rather than
+as a version mismatch.
+
+Images are signed by digest rather than by tag, so a signature is bound to the
+exact bytes the release built. Verify one with:
+
+```bash
+cosign verify \
+  --certificate-oidc-issuer https://token.actions.githubusercontent.com \
+  --certificate-identity-regexp '^https://github\.com/kubefleet-dev/kubefleet/\.github/workflows/release\.yml@refs/(tags/v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?|heads/(main|release-[0-9]+\.[0-9]+))$' \
+  ghcr.io/kubefleet-dev/kubefleet/hub-agent:v0.4.0
+```
+
+Each image also carries a per-platform SPDX SBOM attached to its index, readable
+without pulling the image:
+
+```bash
+docker buildx imagetools inspect ghcr.io/kubefleet-dev/kubefleet/hub-agent:v0.4.0 \
+  --format '{{ json (index .SBOM "linux/amd64").SPDX }}'
+```
+
+`.SBOM` is keyed by platform because every published image is a multi-platform
+index; there is one SBOM per architecture.
+
+The CRD bundle's checksum file is signed as a blob; verifying it and then
+checking the tarball against it covers the tarball:
+
+```bash
+gh release download v0.4.0 --pattern 'kubefleet-crds-*'
+cosign verify-blob \
+  --bundle kubefleet-crds-v0.4.0.tgz.sha256.bundle \
+  --certificate-oidc-issuer https://token.actions.githubusercontent.com \
+  --certificate-identity-regexp '^https://github\.com/kubefleet-dev/kubefleet/\.github/workflows/release\.yml@refs/(tags/v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?|heads/(main|release-[0-9]+\.[0-9]+))$' \
+  kubefleet-crds-v0.4.0.tgz.sha256
+sha256sum -c kubefleet-crds-v0.4.0.tgz.sha256
+```
+
+Verification needs Sigstore's trust root (the Fulcio and Rekor keys), which
+cosign fetches once and caches under `~/.sigstore`. On a machine with no network
+access, prime that cache first or pass `--trusted-root`; otherwise the commands
+above fail for a reason that has nothing to do with the signature.
+
+The identity pattern is deliberately narrow in two ways, and both matter to
+whoever copies these commands:
+
+- **It names `release.yml`, not just the repository.** `release.yml` is not the
+  only workflow holding `id-token: write` — `squad-docs.yml` has it for GitHub
+  Pages — so a repository-wide pattern would accept a signature from a workflow
+  that has nothing to do with releases.
+- **It names the refs a release can come from:** a `vX.Y.Z` or `vX.Y.Z-rc.N` tag,
+  or the `main` / `release-X.Y` branches. A `workflow_dispatch` runs the workflow
+  *definition* from the ref it was started on, so without this anyone able to
+  push a branch and dispatch it could sign under an identity consumers trust.
+  Start a dispatched release from `main`, a release branch, or the tag itself;
+  from anywhere else `create-draft-release` fails before creating the draft, so
+  no draft and no image exists.
+
+What this does not buy: it is a bound on which workflow *definitions* can sign,
+not an access control. Someone who can already push to this repository can push
+a `v*` tag at a commit carrying a modified `release.yml`, and the tag arm accepts
+it. Closing that needs a GitHub ruleset restricting who may create `v*` tags and
+push to `main` / `release-*`; the repository has no such ruleset today, and
+adding one is worth doing independently of this workflow.
+
+The release workflow runs these same verifications immediately after signing, so
+a signature that cannot be verified fails the release instead of shipping. Any
+OIDC trust policy added later (cloud role assumption, trusted publishing) must be
+scoped to that workflow's `job_workflow_ref`, never to
+`repo:kubefleet-dev/kubefleet:*`, or it would be assumable from any workflow in
+the repository.
+
 ## Recovering from a failed run
 
 The normal recovery is **Re-run failed jobs** on the workflow run. Jobs that
@@ -104,11 +191,12 @@ is harmless while the release is still a draft — nothing has been announced ye
 | Where it failed | What is already public | What to do |
 | --- | --- | --- |
 | `setup` | Nothing | The tag is malformed. Delete it, fix, re-tag. |
-| `create-draft-release` | Nothing | See [Re-releasing an existing tag](#re-releasing-an-existing-tag) if it refused because the release is already published. |
+| `create-draft-release` | Nothing | Two causes. If it failed on **Check this ref can sign a release**, the run was started from a ref a release cannot sign under — re-dispatch from `main`, a `release-X.Y` branch, or the tag itself; nothing was published, so there is nothing to clean up. If it refused because the release is already published, see [Re-releasing an existing tag](#re-releasing-an-existing-tag). |
 | `publish-images` | Any images pushed before the failure (`make push` builds hub-agent, member-agent, then refresh-token in order) | Fix, then re-run failed jobs. |
 | `publish-crds` | Possibly the images — it runs in parallel with `publish-images`, not after it | Fix, then re-run failed jobs. |
+| Either signing step | Whatever that job published before signing | Usually a Sigstore or registry transient rather than a code fault — re-run failed jobs first. A signature that will not verify fails the job by design, so nothing unverifiable ships. |
 | `publish-charts-oci` / `publish-charts-pages` | Images; CRD bundle is attached to the still-hidden draft | Fix, then re-run failed jobs. The release stays a draft until the charts land. |
-| `publish-release` | Images, charts | The asset check found the draft incomplete or its bundle failed its own checksum. Inspect `gh release view <tag>`, re-upload, re-run failed jobs. |
+| `publish-release` | Images, charts | The asset check found the draft incomplete or its bundle failed its own checksum. Re-run `publish-crds` rather than hand-uploading: it regenerates the tarball, checksum, and signature together, and a hand-replaced checksum would no longer match its signature. |
 
 `publish-charts-pages` serializes across *all* releases, because the action it
 uses rewrites the whole `gh-pages` branch. GitHub keeps at most one pending