From e048e5c6979454e00a84cf52acc83f06ec69d738 Mon Sep 17 00:00:00 2001 From: Yetkin Timocin Date: Thu, 13 Aug 2026 13:12:42 -0700 Subject: [PATCH 1/2] ci: sign release artifacts with cosign and attach SPDX SBOMs Images and the CRD bundle now carry Sigstore signatures, made keyless with the release workflow's GitHub OIDC identity, so there is no long-lived signing key to hold or rotate. Each image also carries a per-platform SPDX SBOM attached to its index, readable without pulling the image. Images are signed by digest, not by tag. buildx records the digest it pushed via --metadata-file and the signing script reads it from there: signing a tag would sign whatever that tag resolves to when cosign runs, which is not necessarily what the run built. --recursive covers the per-platform manifests inside the index as well, so a verifier that has already resolved to a platform-specific digest still finds a signature. The signing script derives its work list from the build metadata directory rather than from a list of image names. A hand-maintained list can drift from what is actually built and publish an unsigned image while the job still exits 0 - the same failure shape as a hardcoded chart version. The complementary invariant, that everything a release should contain was in fact built, stays with the verification step, which now also asserts the SBOM is really attached and runs before anything is signed. Both signing steps verify what they just produced and fail the release if it does not check out: a signature nobody can verify is worse than no signature, because RELEASING.md tells users to rely on it. The identity those checks pin is bound to this workflow file rather than to the repository, so another workflow that later gains id-token: write cannot mint signatures that pass, and it carries no ref constraint so both tag pushes and workflow_dispatch runs verify. RELEASING.md documents the byte-identical pattern for consumers. The CRD bundle's checksum is signed as a blob and its signature bundle uploaded as a third release asset, which publish-release now requires before it will publish. Part of #693. Signed-off-by: Yetkin Timocin --- .github/workflows/release.yml | 47 ++++++++- .github/workflows/workflow-lint.yml | 2 +- .gitignore | 1 + Makefile | 21 ++++ RELEASING.md | 64 +++++++++++- hack/release/publish-release.sh | 3 +- hack/release/sign-crd-bundle.sh | 45 ++++++++ hack/release/sign-images.sh | 94 +++++++++++++++++ hack/release/test-release-scripts.sh | 149 +++++++++++++++++++++++++-- hack/release/testdata/cosign | 33 ++++++ 10 files changed, 444 insertions(+), 15 deletions(-) create mode 100755 hack/release/sign-crd-bundle.sh create mode 100755 hack/release/sign-images.sh create mode 100755 hack/release/testdata/cosign diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d7f0e9d31..ae2918843 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -75,11 +75,15 @@ jobs: permissions: contents: read packages: write + # Keyless signing exchanges this workflow's OIDC identity for a + # short-lived Fulcio certificate; there is no long-lived key to store. + id-token: write env: REGISTRY: ${{ needs.setup.outputs.registry }} TAG: ${{ needs.setup.outputs.tag }} VERSION: ${{ needs.setup.outputs.version }} PRERELEASE: ${{ needs.setup.outputs.prerelease }} + IMAGE_METADATA_DIR: _image-metadata steps: - name: Checkout code uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -93,26 +97,36 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Install cosign + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + with: + cosign-release: v3.1.3 + # A single multi-arch `docker buildx build` per image publishes the full # tag (e.g. "v0.4.0") and, for stable releases, the short alias # (e.g. "0.4.0") in the same push - no separate retag step. RC images are # published under the long form ("v0.4.0-rc.1") for testers only: the # short-tag namespace is deliberately reserved for stable releases that # consumers can safely pin to, so RC tags get no short alias. + # + # IMAGE_SBOM attaches a per-platform SPDX SBOM to the image index; + # IMAGE_METADATA_DIR captures the digest buildx actually pushed, which is + # what gets signed below. - name: Build and push images with tag ${{ needs.setup.outputs.tag }} run: | set -euo pipefail - if [ "${PRERELEASE}" = "true" ]; then - make push - else - make push IMAGE_EXTRA_TAG="${VERSION}" + mkdir -p "${IMAGE_METADATA_DIR}" + extra_tag="" + if [ "${PRERELEASE}" != "true" ]; then + extra_tag="${VERSION}" fi + make push IMAGE_SBOM=true IMAGE_METADATA_DIR="${IMAGE_METADATA_DIR}" IMAGE_EXTRA_TAG="${extra_tag}" # Confirm every published tag is a real multi-arch manifest list rather # than a silently degraded single-arch image: a build that pushed only one # architecture would otherwise go unnoticed until a consumer on the other # architecture failed to pull. Stable releases also carry the short alias. - - name: Verify images are multi-arch + - name: Verify published image manifests and SBOMs run: | set -euo pipefail tags="${TAG}" @@ -130,8 +144,21 @@ jobs: || { echo "::error::${ref} is missing platform ${platform}"; exit 1; } done done + # Once per image: every tag resolves to the same index. IMAGE_SBOM is + # matched as an exact string in the Makefile, so a future "TRUE" or + # "1" would quietly produce no SBOM; assert the attestation is really + # attached rather than trusting the flag was spelled as make expects. + sbom="$(docker buildx imagetools inspect "${REGISTRY}/${IMAGE}:${TAG}" \ + --format '{{ json (index .SBOM "linux/amd64").SPDX }}' 2>/dev/null || true)" + if [ -z "${sbom}" ] || [ "${sbom}" = "null" ]; then + echo "::error::${REGISTRY}/${IMAGE}:${TAG} has no SPDX SBOM attached" + exit 1 + fi done + - name: Sign the published images + run: ./hack/release/sign-images.sh + # Publish the raw CRDs as a standalone release asset so consumers can install # them without pulling a Helm chart. The bundle carries the unmodified CRDs the # charts install (no downstream-specific labels), split into crds/hub and @@ -141,6 +168,7 @@ jobs: runs-on: ubuntu-latest permissions: contents: write + id-token: write env: TAG: ${{ needs.setup.outputs.tag }} steps: @@ -149,9 +177,17 @@ jobs: with: ref: ${{ needs.setup.outputs.tag }} + - name: Install cosign + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + with: + cosign-release: v3.1.3 + - name: Package CRDs run: make crd-package TAG="${TAG}" + - name: Sign the CRD bundle checksum + run: ./hack/release/sign-crd-bundle.sh + # --clobber makes the upload idempotent so a re-run replaces the asset # rather than failing on a name collision. The token is scoped to this # step so it is not in the environment of the packaging step above. @@ -163,6 +199,7 @@ jobs: gh release upload "${TAG}" \ "_crd-package/kubefleet-crds-${TAG}.tgz" \ "_crd-package/kubefleet-crds-${TAG}.tgz.sha256" \ + "_crd-package/kubefleet-crds-${TAG}.tgz.sha256.bundle" \ --clobber # Charts are published only for stable releases: an RC must be installable by diff --git a/.github/workflows/workflow-lint.yml b/.github/workflows/workflow-lint.yml index 1d0b133f1..24a787dcf 100644 --- a/.github/workflows/workflow-lint.yml +++ b/.github/workflows/workflow-lint.yml @@ -49,7 +49,7 @@ jobs: # The release scripts live outside the workflow files, so actionlint's # embedded-shell checking does not reach them. - name: Shellcheck the release scripts - run: shellcheck hack/release/*.sh hack/release/testdata/gh + run: shellcheck hack/release/*.sh hack/release/testdata/* - name: Test the release scripts run: ./hack/release/test-release-scripts.sh diff --git a/.gitignore b/.gitignore index 38af08316..7b4b6c49b 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,4 @@ _crd-package/ # Squad: SubSquad activation file (local to this machine) .squad-workstream .squad/.cache/ +_image-metadata/ diff --git a/Makefile b/Makefile index 392ff6179..6c8c08f48 100644 --- a/Makefile +++ b/Makefile @@ -265,6 +265,24 @@ BINFMT_IMAGE ?= mcr.microsoft.com/mirror/docker/tonistiigi/binfmt:$(BINFMT_VERSI PLATFORMS ?= $(TARGET_OS)/$(TARGET_ARCH) RELEASE_PLATFORMS ?= linux/amd64,linux/arm64 +# Attach an SPDX SBOM to the image index. BuildKit generates one per platform +# and stores it alongside the image in the registry, so consumers can read what +# is in an image without unpacking it. Off by default because it slows every +# local build; the release path turns it on. +IMAGE_SBOM ?= false + +# When set, buildx writes each image's build metadata (including +# "containerimage.digest") to $(IMAGE_METADATA_DIR)/.json. The release +# workflow signs those digests: signing a tag would sign whatever the tag points +# at when cosign runs, not what this build actually pushed. +IMAGE_METADATA_DIR ?= + +# Expanded into every docker-build-* target. Kept here so the three recipes stay +# identical to each other. +image_build_flags = \ + $(if $(filter true,$(IMAGE_SBOM)),--sbom=true) \ + $(if $(IMAGE_METADATA_DIR),--metadata-file $(IMAGE_METADATA_DIR)/$(1).json) + .PHONY: push push: ## Build and push all Docker images as multi-arch manifests $(MAKE) OUTPUT_TYPE="type=registry" PLATFORMS="$(RELEASE_PLATFORMS)" docker-build-hub-agent docker-build-member-agent docker-build-refresh-token @@ -366,6 +384,7 @@ docker-build-hub-agent: docker-buildx-builder ## Build hub-agent image --pull \ --tag $(REGISTRY)/$(HUB_AGENT_IMAGE_NAME):$(HUB_AGENT_IMAGE_VERSION) \ $(if $(IMAGE_EXTRA_TAG),--tag $(REGISTRY)/$(HUB_AGENT_IMAGE_NAME):$(IMAGE_EXTRA_TAG)) \ + $(call image_build_flags,$(HUB_AGENT_IMAGE_NAME)) \ --progress=$(BUILDKIT_PROGRESS_TYPE) . .PHONY: docker-build-member-agent @@ -377,6 +396,7 @@ docker-build-member-agent: docker-buildx-builder ## Build member-agent image --pull \ --tag $(REGISTRY)/$(MEMBER_AGENT_IMAGE_NAME):$(MEMBER_AGENT_IMAGE_VERSION) \ $(if $(IMAGE_EXTRA_TAG),--tag $(REGISTRY)/$(MEMBER_AGENT_IMAGE_NAME):$(IMAGE_EXTRA_TAG)) \ + $(call image_build_flags,$(MEMBER_AGENT_IMAGE_NAME)) \ --progress=$(BUILDKIT_PROGRESS_TYPE) . .PHONY: docker-build-refresh-token @@ -388,6 +408,7 @@ docker-build-refresh-token: docker-buildx-builder ## Build refresh-token image --pull \ --tag $(REGISTRY)/$(REFRESH_TOKEN_IMAGE_NAME):$(REFRESH_TOKEN_IMAGE_VERSION) \ $(if $(IMAGE_EXTRA_TAG),--tag $(REGISTRY)/$(REFRESH_TOKEN_IMAGE_NAME):$(IMAGE_EXTRA_TAG)) \ + $(call image_build_flags,$(REFRESH_TOKEN_IMAGE_NAME)) \ --progress=$(BUILDKIT_PROGRESS_TYPE) . ## ----------------------------------- diff --git a/RELEASING.md b/RELEASING.md index 5bab9bb4f..1b6a34da8 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -11,7 +11,8 @@ supported are covered in [VERSIONING.md](VERSIONING.md) and | Artifact | Location | Stable (`v0.4.0`) | Release candidate (`v0.4.0-rc.1`) | | --- | --- | --- | --- | | Agent images (`hub-agent`, `member-agent`, `refresh-token`) | `ghcr.io/kubefleet-dev/kubefleet/` | `:v0.4.0` and `:0.4.0` | `:v0.4.0-rc.1` only | -| CRD bundle (`kubefleet-crds-.tgz` + `.sha256`) | GitHub Release asset | Yes | Yes | +| Image signatures + SPDX SBOMs | Alongside each image in the registry | Yes | Yes | +| CRD bundle (`kubefleet-crds-.tgz`, `.sha256`, `.sha256.bundle`) | GitHub Release asset | Yes | Yes | | Helm charts (OCI) | `oci://ghcr.io/kubefleet-dev/kubefleet/charts/` | Yes | No | | Helm charts (index) | `https://kubefleet-dev.github.io/kubefleet/charts` | Yes | No | | GitHub Release | Releases page | Published | Published, flagged pre-release | @@ -89,6 +90,64 @@ The two jobs with real branching logic — `create-draft-release` and in the workflow, and are covered by `hack/release/test-release-scripts.sh`, which CI runs on every change to either. +## Verifying a release + +The container images and the CRD bundle are signed with +[cosign](https://docs.sigstore.dev/) in keyless mode: the workflow's GitHub OIDC +identity is exchanged for a short-lived Fulcio certificate and the signature is +recorded in Rekor. There is no long-lived signing key to hold, and nothing to +rotate. + +The Helm charts are **not** signed yet, on either the OCI or the index channel. +Signing the OCI charts is the same keyless flow used for images and is tracked +as a follow-up; the classic-repo `.prov` mechanism needs a long-lived GPG key +this project has no custody story for. + +Images are signed by digest rather than by tag, so a signature is bound to the +exact bytes the release built. Verify one with: + +```bash +cosign verify \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate-identity-regexp '^https://github\.com/kubefleet-dev/kubefleet/\.github/workflows/release\.yml@' \ + ghcr.io/kubefleet-dev/kubefleet/hub-agent:v0.4.0 +``` + +Each image also carries a per-platform SPDX SBOM attached to its index, readable +without pulling the image: + +```bash +docker buildx imagetools inspect ghcr.io/kubefleet-dev/kubefleet/hub-agent:v0.4.0 \ + --format '{{ json (index .SBOM "linux/amd64").SPDX }}' +``` + +`.SBOM` is keyed by platform because every published image is a multi-platform +index; there is one SBOM per architecture. + +The CRD bundle's checksum file is signed as a blob; verifying it and then +checking the tarball against it covers the tarball: + +```bash +cosign verify-blob \ + --bundle kubefleet-crds-v0.4.0.tgz.sha256.bundle \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate-identity-regexp '^https://github\.com/kubefleet-dev/kubefleet/\.github/workflows/release\.yml@' \ + kubefleet-crds-v0.4.0.tgz.sha256 +sha256sum -c kubefleet-crds-v0.4.0.tgz.sha256 +``` + +Verification needs Sigstore's trust root (the Fulcio and Rekor keys), which +cosign fetches once and caches under `~/.sigstore`. On a machine with no network +access, prime that cache first or pass `--trusted-root`; otherwise the commands +above fail for a reason that has nothing to do with the signature. + +The release workflow runs these same verifications immediately after signing, so +a signature that cannot be verified fails the release instead of shipping. +`release.yml` is the only workflow holding `id-token: write`. Any OIDC trust +policy added later (cloud role assumption, trusted publishing) must be scoped to +that workflow's `job_workflow_ref`, never to `repo:kubefleet-dev/kubefleet:*`, +or it would be assumable from any workflow in the repository. + ## Recovering from a failed run The normal recovery is **Re-run failed jobs** on the workflow run. Jobs that @@ -107,8 +166,9 @@ is harmless while the release is still a draft — nothing has been announced ye | `create-draft-release` | Nothing | See [Re-releasing an existing tag](#re-releasing-an-existing-tag) if it refused because the release is already published. | | `publish-images` | Any images pushed before the failure (`make push` builds hub-agent, member-agent, then refresh-token in order) | Fix, then re-run failed jobs. | | `publish-crds` | Possibly the images — it runs in parallel with `publish-images`, not after it | Fix, then re-run failed jobs. | +| Either signing step | Whatever that job published before signing | Usually a Sigstore or registry transient rather than a code fault — re-run failed jobs first. A signature that will not verify fails the job by design, so nothing unverifiable ships. | | `publish-charts-oci` / `publish-charts-pages` | Images; CRD bundle is attached to the still-hidden draft | Fix, then re-run failed jobs. The release stays a draft until the charts land. | -| `publish-release` | Images, charts | The asset check found the draft incomplete or its bundle failed its own checksum. Inspect `gh release view `, re-upload, re-run failed jobs. | +| `publish-release` | Images, charts | The asset check found the draft incomplete or its bundle failed its own checksum. Re-run `publish-crds` rather than hand-uploading: it regenerates the tarball, checksum, and signature together, and a hand-replaced checksum would no longer match its signature. | `publish-charts-pages` serializes across *all* releases, because the action it uses rewrites the whole `gh-pages` branch. GitHub keeps at most one pending diff --git a/hack/release/publish-release.sh b/hack/release/publish-release.sh index ed7d91d03..9d4a32664 100755 --- a/hack/release/publish-release.sh +++ b/hack/release/publish-release.sh @@ -19,6 +19,7 @@ set -euo pipefail bundle="kubefleet-crds-${TAG}.tgz" checksum="${bundle}.sha256" +signature="${checksum}.bundle" # Only assets GitHub finished receiving count. An upload interrupted mid-stream # leaves an asset row with the right name in a non-"uploaded" state, which a @@ -26,7 +27,7 @@ checksum="${bundle}.sha256" assets="$(gh release view "${TAG}" --json assets \ --jq '.assets[] | select(.state == "uploaded" and .size > 0) | .name')" -for want in "${bundle}" "${checksum}"; do +for want in "${bundle}" "${checksum}" "${signature}"; do if ! grep -qxF -- "${want}" <<<"${assets}"; then echo "::error::Release ${TAG} is missing fully-uploaded asset ${want}; leaving it as a draft." exit 1 diff --git a/hack/release/sign-crd-bundle.sh b/hack/release/sign-crd-bundle.sh new file mode 100755 index 000000000..efbf05f67 --- /dev/null +++ b/hack/release/sign-crd-bundle.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Sign the CRD bundle's checksum file with cosign, keyless. +# +# The checksum is what gets signed rather than the tarball itself: it already +# binds the tarball's bytes, it is the file a consumer checks the download +# against, and it keeps the signature bundle small. Verifying the signature and +# then running `sha256sum -c` covers the tarball transitively. +# +# Environment: +# TAG release tag, e.g. v0.4.0 +# CRD_PACKAGE_DIR directory holding the packaged bundle (default: _crd-package) +# GITHUB_REPOSITORY owner/repo, used to bound the identity on verification + +set -euo pipefail + +: "${TAG:?TAG must be set}" +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY must be set}" + +package_dir="${CRD_PACKAGE_DIR:-_crd-package}" +checksum="${package_dir}/kubefleet-crds-${TAG}.tgz.sha256" +bundle="${checksum}.bundle" + +if [ ! -f "${checksum}" ]; then + echo "::error::No checksum file at ${checksum}; run 'make crd-package' first." + exit 1 +fi + +OIDC_ISSUER="https://token.actions.githubusercontent.com" +# See sign-images.sh for why this is bound to the workflow file and why the +# repository name has its dots escaped before going into a regexp. +repo_pattern="${GITHUB_REPOSITORY//./\\.}" +IDENTITY_PATTERN="^https://github\.com/${repo_pattern}/\.github/workflows/release\.yml@" + +echo "Signing ${checksum}" +cosign sign-blob --yes --bundle "${bundle}" "${checksum}" + +# Same reasoning as sign-images.sh: verify with the identity consumers will use, +# so a signature that cannot be verified fails the release rather than shipping. +cosign verify-blob \ + --bundle "${bundle}" \ + --certificate-oidc-issuer "${OIDC_ISSUER}" \ + --certificate-identity-regexp "${IDENTITY_PATTERN}" \ + "${checksum}" >/dev/null + +echo "✅ ${checksum} signed and verified; bundle at ${bundle}" diff --git a/hack/release/sign-images.sh b/hack/release/sign-images.sh new file mode 100755 index 000000000..76f3a5ef0 --- /dev/null +++ b/hack/release/sign-images.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# Sign each published image with cosign, keyless, using the workflow's GitHub +# OIDC identity. +# +# Images are signed by *digest*, read from the build metadata buildx wrote +# during this run. Signing a tag would sign whatever that tag resolves to when +# cosign runs, which is not necessarily what this run pushed - the whole point +# of a signature is to bind it to specific bytes. +# +# The set of images comes from the metadata directory rather than from a list +# passed in, so "everything this run built gets signed" is structurally true. A +# hand-maintained list could drop an image while the job still exited 0, +# publishing an unsigned image that the documentation tells users to verify. +# +# The complementary invariant - that everything a release *should* contain was +# built - is asserted by the multi-arch verification step that runs immediately +# before this script. Keep that ordering: it is what stops a dropped build +# target from being silently absent here. +# +# Environment: +# REGISTRY image repository prefix, e.g. ghcr.io/kubefleet-dev/kubefleet +# IMAGE_METADATA_DIR directory holding .json from `buildx --metadata-file` +# GITHUB_REPOSITORY owner/repo, used to bound the identity on verification + +set -euo pipefail +shopt -s nullglob + +: "${REGISTRY:?REGISTRY must be set}" +: "${IMAGE_METADATA_DIR:?IMAGE_METADATA_DIR must be set}" +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY must be set}" + +OIDC_ISSUER="https://token.actions.githubusercontent.com" +# GITHUB_REPOSITORY is interpolated into a regexp, and repository names may +# contain dots, which would otherwise match any character. +repo_pattern="${GITHUB_REPOSITORY//./\\.}" +# Bound to this workflow file, not merely to the repository: any other workflow +# that ever gains id-token: write could otherwise mint signatures that pass. No +# ref constraint - a workflow_dispatch release signs as @refs/heads/ +# while a tag push signs as @refs/tags/. +IDENTITY_PATTERN="^https://github\.com/${repo_pattern}/\.github/workflows/release\.yml@" + +metadata_files=("${IMAGE_METADATA_DIR}"/*.json) +if [ "${#metadata_files[@]}" -eq 0 ]; then + echo "::error::No build metadata in ${IMAGE_METADATA_DIR}; there is nothing to sign, which means the build did not publish what it should have." + exit 1 +fi + +for metadata in "${metadata_files[@]}"; do + image="$(basename "${metadata}" .json)" + + digest="$(jq -r '."containerimage.digest" // empty' "${metadata}")" + case "${digest}" in + sha256:*) ;; + *) + echo "::error::${metadata} has no usable containerimage.digest (got '${digest}')." + exit 1 + ;; + esac + + ref="${REGISTRY}/${image}@${digest}" + echo "Signing ${ref}" + # --recursive also signs the per-platform manifests inside the index. Without + # it, anything that has already resolved to a platform-specific digest - some + # admission controllers, per-arch mirroring - finds no signature. + cosign sign --recursive --yes "${ref}" + + # Verify what was just signed. This is not ceremony: it is the only check that + # the signature resolves against the identity consumers will verify with. A + # signature nobody can verify is worse than no signature, because the + # documentation tells users to rely on it. + # + # Retried because the realistic failure is a registry read-after-write race, + # and re-running this job to recover would rebuild and re-push every image + # under a new digest. + verify_log="$(mktemp)" + for attempt in 1 2 3; do + if cosign verify \ + --certificate-oidc-issuer "${OIDC_ISSUER}" \ + --certificate-identity-regexp "${IDENTITY_PATTERN}" \ + "${ref}" >/dev/null 2>"${verify_log}"; then + break + fi + if [ "${attempt}" = 3 ]; then + # Print what cosign actually said: a certificate-identity mismatch is a + # configuration fault, not a transient, and the two need different fixes. + echo "::error::Signed ${ref} but the signature did not verify after 3 attempts: $(tr '\n' ' ' <"${verify_log}")" + rm -f "${verify_log}" + exit 1 + fi + sleep 2 + done + rm -f "${verify_log}" + echo "✅ ${ref} signed and verified" +done diff --git a/hack/release/test-release-scripts.sh b/hack/release/test-release-scripts.sh index a7eedda69..a831564d7 100755 --- a/hack/release/test-release-scripts.sh +++ b/hack/release/test-release-scripts.sh @@ -16,6 +16,8 @@ export PATH="${here}/testdata:${PATH}" create_script="${here}/create-draft-release.sh" publish_script="${here}/publish-release.sh" +sign_images_script="${here}/sign-images.sh" +sign_bundle_script="${here}/sign-crd-bundle.sh" passed=0 failed=0 @@ -91,16 +93,16 @@ expect_no_gh "release create" "API error: creates nothing" echo "== publish-release.sh ==" -both_uploaded="$(printf 'kubefleet-crds-v0.4.0.tgz;uploaded;4096\nkubefleet-crds-v0.4.0.tgz.sha256;uploaded;98')" +all_uploaded="$(printf 'kubefleet-crds-v0.4.0.tgz;uploaded;4096\nkubefleet-crds-v0.4.0.tgz.sha256;uploaded;98\nkubefleet-crds-v0.4.0.tgz.sha256.bundle;uploaded;2048')" -run_case FAKE_GH_STATE=draft FAKE_GH_ASSETS="${both_uploaded}" TAG=v0.4.0 PRERELEASE=false \ +run_case FAKE_GH_STATE=draft FAKE_GH_ASSETS="${all_uploaded}" TAG=v0.4.0 PRERELEASE=false \ bash "${publish_script}" expect_rc 0 "complete draft, stable: publishes" expect_gh "gh release edit v0.4.0 --draft=false --prerelease=false" \ "stable release is published without the pre-release flag" run_case FAKE_GH_STATE=draft \ - FAKE_GH_ASSETS="$(printf 'kubefleet-crds-v0.4.0-rc.1.tgz;uploaded;4096\nkubefleet-crds-v0.4.0-rc.1.tgz.sha256;uploaded;98')" \ + FAKE_GH_ASSETS="$(printf 'kubefleet-crds-v0.4.0-rc.1.tgz;uploaded;4096\nkubefleet-crds-v0.4.0-rc.1.tgz.sha256;uploaded;98\nkubefleet-crds-v0.4.0-rc.1.tgz.sha256.bundle;uploaded;2048')" \ TAG=v0.4.0-rc.1 PRERELEASE=true bash "${publish_script}" expect_rc 0 "complete draft, RC: publishes" # A draft created by hand defaults to prerelease=false, so the flag has to be @@ -118,18 +120,18 @@ expect_rc 1 "no assets at all: refuses to publish" # GitHub keeps an asset row for an upload that never finished; it is present by # name but not in the "uploaded" state. run_case FAKE_GH_STATE=draft \ - FAKE_GH_ASSETS="$(printf 'kubefleet-crds-v0.4.0.tgz;new;0\nkubefleet-crds-v0.4.0.tgz.sha256;uploaded;98')" \ + FAKE_GH_ASSETS="$(printf 'kubefleet-crds-v0.4.0.tgz;new;0\nkubefleet-crds-v0.4.0.tgz.sha256;uploaded;98\nkubefleet-crds-v0.4.0.tgz.sha256.bundle;uploaded;2048')" \ TAG=v0.4.0 PRERELEASE=false bash "${publish_script}" expect_rc 1 "interrupted upload (state != uploaded): refuses to publish" expect_no_gh "release edit" "interrupted upload: release stays a draft" run_case FAKE_GH_STATE=draft \ - FAKE_GH_ASSETS="$(printf 'kubefleet-crds-v0.4.0.tgz;uploaded;0\nkubefleet-crds-v0.4.0.tgz.sha256;uploaded;98')" \ + FAKE_GH_ASSETS="$(printf 'kubefleet-crds-v0.4.0.tgz;uploaded;0\nkubefleet-crds-v0.4.0.tgz.sha256;uploaded;98\nkubefleet-crds-v0.4.0.tgz.sha256.bundle;uploaded;2048')" \ TAG=v0.4.0 PRERELEASE=false bash "${publish_script}" expect_rc 1 "zero-byte asset: refuses to publish" # Names alone are not proof; the bundle ships a checksum, so it gets checked. -run_case FAKE_GH_STATE=draft FAKE_GH_ASSETS="${both_uploaded}" FAKE_GH_DOWNLOAD=corrupt \ +run_case FAKE_GH_STATE=draft FAKE_GH_ASSETS="${all_uploaded}" FAKE_GH_DOWNLOAD=corrupt \ TAG=v0.4.0 PRERELEASE=false bash "${publish_script}" expect_rc 1 "bundle that fails its own checksum: refuses to publish" expect_no_gh "release edit" "failed checksum: release stays a draft" @@ -140,6 +142,141 @@ run_case FAKE_GH_STATE=draft \ TAG=v0.4.0 PRERELEASE=false bash "${publish_script}" expect_rc 1 "similar-but-wrong asset names: refuses to publish" +echo "== sign-images.sh ==" + +# buildx writes one metadata file per image; "containerimage.digest" in it is +# the only record of what this run actually pushed. +write_metadata() { # write_metadata + mkdir -p "$1" + if [ -n "$3" ]; then + printf '{"containerimage.digest":"%s","image.name":"ghcr.io/x/%s"}\n' "$3" "$2" >"$1/$2.json" + else + printf '{"image.name":"ghcr.io/x/%s"}\n' "$2" >"$1/$2.json" + fi +} + +hub_digest="sha256:1111111111111111111111111111111111111111111111111111111111111111" +member_digest="sha256:2222222222222222222222222222222222222222222222222222222222222222" +token_digest="sha256:3333333333333333333333333333333333333333333333333333333333333333" + +meta_dir="$(mktemp -d)" +write_metadata "${meta_dir}" hub-agent "${hub_digest}" +write_metadata "${meta_dir}" member-agent "${member_digest}" +write_metadata "${meta_dir}" refresh-token "${token_digest}" +run_case REGISTRY=ghcr.io/kubefleet-dev/kubefleet IMAGE_METADATA_DIR="${meta_dir}" \ + GITHUB_REPOSITORY=kubefleet-dev/kubefleet bash "${sign_images_script}" +expect_rc 0 "metadata for every built image: signs successfully" +expect_gh "cosign sign --recursive --yes ghcr.io/kubefleet-dev/kubefleet/hub-agent@${hub_digest}" \ + "signs the hub-agent digest this run pushed" +expect_gh "cosign sign --recursive --yes ghcr.io/kubefleet-dev/kubefleet/member-agent@${member_digest}" \ + "signs the member-agent digest this run pushed" +# Driving the loop off the metadata directory is what makes "everything built +# gets signed" structural: a hand-maintained list could drop this one silently. +expect_gh "cosign sign --recursive --yes ghcr.io/kubefleet-dev/kubefleet/refresh-token@${token_digest}" \ + "signs every image present in the metadata directory" +# Signing a tag would sign whatever it resolves to at signing time, not the +# bytes this run built. +expect_no_gh "ghcr.io/kubefleet-dev/kubefleet/hub-agent:" "never signs by tag" +expect_gh "cosign verify" "verifies each signature it creates" +rm -rf "${meta_dir}" + +meta_dir="$(mktemp -d)" +run_case REGISTRY=ghcr.io/kubefleet-dev/kubefleet IMAGE_METADATA_DIR="${meta_dir}" \ + GITHUB_REPOSITORY=kubefleet-dev/kubefleet bash "${sign_images_script}" +expect_rc 1 "no metadata at all: fails rather than signing nothing" +expect_no_gh "cosign sign" "no metadata: signs nothing" +rm -rf "${meta_dir}" + +meta_dir="$(mktemp -d)" +write_metadata "${meta_dir}" hub-agent "" +run_case REGISTRY=ghcr.io/kubefleet-dev/kubefleet IMAGE_METADATA_DIR="${meta_dir}" \ + GITHUB_REPOSITORY=kubefleet-dev/kubefleet bash "${sign_images_script}" +expect_rc 1 "metadata without a digest: fails rather than signing something else" +expect_no_gh "cosign sign" "no digest: signs nothing" +rm -rf "${meta_dir}" + +# An unverifiable signature is worse than none, because the docs tell users to +# rely on it - so a failed verification has to fail the release. +meta_dir="$(mktemp -d)" +write_metadata "${meta_dir}" hub-agent "${hub_digest}" +run_case REGISTRY=ghcr.io/kubefleet-dev/kubefleet IMAGE_METADATA_DIR="${meta_dir}" \ + GITHUB_REPOSITORY=kubefleet-dev/kubefleet FAKE_COSIGN_FAIL=verify \ + bash "${sign_images_script}" +expect_rc 1 "signature that will not verify: fails the release" +rm -rf "${meta_dir}" + +echo "== signing identity pattern ==" + +# The identity regexp is the whole security value of keyless signing: it is what +# a consumer pins to. These assert the exact string both scripts build. +identity_pattern_for() { # identity_pattern_for + GITHUB_REPOSITORY="$1" bash -c \ + 'repo_pattern="${GITHUB_REPOSITORY//./\\.}"; echo "^https://github\.com/${repo_pattern}/\.github/workflows/release\.yml@"' +} + +pattern="$(identity_pattern_for kubefleet-dev/kubefleet)" +check_identity() { # check_identity + if grep -qE "${pattern}" <<<"$2"; then result=accept; else result=reject; fi + if [ "${result}" = "$1" ]; then ok "$3"; else + rc="n/a"; output="${result} for $2"; log=""; bad "$3" + fi +} + +check_identity accept \ + "https://github.com/kubefleet-dev/kubefleet/.github/workflows/release.yml@refs/tags/v0.4.0" \ + "accepts a tag-triggered release signature" +# A workflow_dispatch release signs under the branch ref, so the pattern must +# not pin refs/tags. +check_identity accept \ + "https://github.com/kubefleet-dev/kubefleet/.github/workflows/release.yml@refs/heads/main" \ + "accepts a dispatch-triggered release signature" +check_identity reject \ + "https://github.com/kubefleet-dev/kubefleet/.github/workflows/squad-release.yml@refs/tags/v0.4.0" \ + "rejects another workflow in the same repository" +check_identity reject \ + "https://github.com/kubefleet-dev/kubefleet/.github/workflows/release.yml.bak@refs/tags/v0.4.0" \ + "rejects a filename that merely starts with release.yml" +check_identity reject \ + "https://github.com/evil/kubefleet/.github/workflows/release.yml@refs/tags/v0.4.0" \ + "rejects the same workflow path in a different repository" +# Repository names may contain dots; unescaped they would match any character. +if [ "$(identity_pattern_for org/weird.name)" = '^https://github\.com/org/weird\.name/\.github/workflows/release\.yml@' ]; then + ok "escapes dots in the repository name" +else + rc="n/a"; output="$(identity_pattern_for org/weird.name)"; log=""; bad "escapes dots in the repository name" +fi + +echo "== sign-crd-bundle.sh ==" + +pkg_dir="$(mktemp -d)" +printf 'abc123 kubefleet-crds-v0.4.0.tgz\n' >"${pkg_dir}/kubefleet-crds-v0.4.0.tgz.sha256" +run_case TAG=v0.4.0 CRD_PACKAGE_DIR="${pkg_dir}" GITHUB_REPOSITORY=kubefleet-dev/kubefleet \ + bash "${sign_bundle_script}" +expect_rc 0 "checksum present: signs successfully" +expect_gh "cosign sign-blob --yes --bundle ${pkg_dir}/kubefleet-crds-v0.4.0.tgz.sha256.bundle" \ + "writes the signature bundle next to the checksum" +expect_gh "cosign verify-blob" "verifies the blob signature it just created" +if [ -f "${pkg_dir}/kubefleet-crds-v0.4.0.tgz.sha256.bundle" ]; then + ok "signature bundle exists for upload" +else + bad "signature bundle was not produced" +fi +rm -rf "${pkg_dir}" + +pkg_dir="$(mktemp -d)" +run_case TAG=v0.4.0 CRD_PACKAGE_DIR="${pkg_dir}" GITHUB_REPOSITORY=kubefleet-dev/kubefleet \ + bash "${sign_bundle_script}" +expect_rc 1 "checksum missing: fails instead of signing nothing" +expect_no_gh "cosign" "checksum missing: does not invoke cosign at all" +rm -rf "${pkg_dir}" + +pkg_dir="$(mktemp -d)" +printf 'abc123 kubefleet-crds-v0.4.0.tgz\n' >"${pkg_dir}/kubefleet-crds-v0.4.0.tgz.sha256" +run_case TAG=v0.4.0 CRD_PACKAGE_DIR="${pkg_dir}" GITHUB_REPOSITORY=kubefleet-dev/kubefleet \ + FAKE_COSIGN_FAIL=verify-blob bash "${sign_bundle_script}" +expect_rc 1 "blob signature that will not verify: fails the release" +rm -rf "${pkg_dir}" + echo echo "passed=${passed} failed=${failed}" [ "${failed}" -eq 0 ] diff --git a/hack/release/testdata/cosign b/hack/release/testdata/cosign new file mode 100755 index 000000000..58207efd4 --- /dev/null +++ b/hack/release/testdata/cosign @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Stand-in for cosign, used by test-release-scripts.sh. Keyless signing needs a +# real OIDC identity, so the signing scripts can only be exercised against a +# stub; what these tests cover is the logic around cosign - which digest gets +# signed, and what happens when a step fails - not cosign itself. +# +# Environment: +# FAKE_COSIGN_FAIL subcommand that should fail: sign | verify | sign-blob | +# verify-blob (default: none) +# FAKE_GH_LOG file every invocation is appended to (shared with the gh stub) + +set -uo pipefail + +echo "cosign $*" >>"${FAKE_GH_LOG}" + +subcommand="${1:-}" + +if [ "${subcommand}" = "${FAKE_COSIGN_FAIL:-}" ]; then + echo "cosign ${subcommand}: simulated failure" >&2 + exit 1 +fi + +# sign-blob is expected to leave a signature bundle behind. +if [ "${subcommand}" = "sign-blob" ]; then + args=("$@") + for i in "${!args[@]}"; do + if [ "${args[$i]}" = "--bundle" ]; then + printf '{"fake":"bundle"}\n' >"${args[$((i + 1))]}" + fi + done +fi + +exit 0 From d0ae274425df9273c249d8ef4a6cf919c9b6f76f Mon Sep 17 00:00:00 2001 From: Yetkin Timocin Date: Tue, 18 Aug 2026 12:46:56 -0700 Subject: [PATCH 2/2] ci: bind release signatures to trusted refs and verify SBOMs per platform The cosign identity pattern ended immediately after "@", so it accepted release.yml run from any ref. A workflow_dispatch runs the workflow definition from the ref it was started on, which made "any branch in the repository" part of the trusted set. Constrain it to release tags and the main / release-X.Y branches. Move the pattern and the OIDC issuer into hack/release/identity.sh, read by both signing scripts and by their tests. The tests previously reimplemented the pattern, so they asserted a copy and would have kept passing after the scripts drifted. Be explicit about what this does not buy: it bounds which workflow definitions can sign, not who can sign. Someone with write access can still push a v* tag at a commit carrying a modified release.yml, and the tag arm accepts it. Closing that needs a ruleset restricting who may create v* tags, which the repository does not have today. Signing runs after the images are pushed, so a run from an untrusted ref would have published three images, signed them, written a Rekor entry, and only then failed verification. Add check-signing-ref.sh as the first step of create-draft-release, which gates every publishing job: the same identity is now tested before anything exists to clean up. The SBOM gate inspected only linux/amd64 while the release promises one per platform, so an image missing its arm64 SBOM would have published behind a green check. Iterate the platforms instead, cross-checked against the Makefile as a set so a narrowed RELEASE_PLATFORMS fails loudly rather than narrowing the check to match. Capture buildx's stderr too - a registry error and a missing attestation need different fixes and were reported identically. RELEASING.md claimed release.yml was the only workflow holding id-token: write; squad-docs.yml has it as well. Tests: 49 -> 76. Mutating the issuer, dropping the pattern's anchor, widening setup-release.yml's tag grammar, or leaving one of the two documented patterns stale each now fail the suite; before, all four passed. Signed-off-by: Yetkin Timocin --- .github/workflows/release.yml | 53 +++++-- Makefile | 4 + RELEASING.md | 44 ++++-- hack/release/check-signing-ref.sh | 44 ++++++ hack/release/identity.sh | 63 +++++++++ hack/release/sign-crd-bundle.sh | 11 +- hack/release/sign-images.sh | 15 +- hack/release/test-release-scripts.sh | 200 +++++++++++++++++++++++---- 8 files changed, 379 insertions(+), 55 deletions(-) create mode 100755 hack/release/check-signing-ref.sh create mode 100644 hack/release/identity.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ae2918843..93cb156c6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -66,6 +66,13 @@ jobs: with: ref: ${{ needs.setup.outputs.tag }} + # This job gates every publishing job, so failing here is the last point + # at which a release can be abandoned for free. Signing runs long after + # the images are public, so a run started from a ref that cannot produce + # a verifiable signature has to be stopped now or not at all. + - name: Check this ref can sign a release + run: ./hack/release/check-signing-ref.sh + - name: Create or reuse the draft release run: ./hack/release/create-draft-release.sh @@ -133,27 +140,55 @@ jobs: if [ "${PRERELEASE}" != "true" ]; then tags="${tags} ${VERSION}" fi - echo "✅ Verifying published images:" + # An independent statement of what a release must contain. Deriving + # this from RELEASE_PLATFORMS would make the build config both the + # input and the oracle: dropping a platform there - or overriding it + # in the job env - would silently narrow the check to match, and a + # single-arch release would ship green. Cross-checked against the + # Makefile so the two cannot disagree without someone noticing. + expected_platforms="linux/amd64 linux/arm64" + # Compared as sets: this is about which platforms are built, and + # failing a release over a reordered RELEASE_PLATFORMS would be noise. + built_sorted="$(make -s print-release-platforms | sort | xargs)" + expected_sorted="$(tr ' ' '\n' <<<"${expected_platforms}" | sort | xargs)" + if [ "${built_sorted}" != "${expected_sorted}" ]; then + echo "::error::RELEASE_PLATFORMS builds '${built_sorted}' but this release is verified for '${expected_sorted}'. Update both deliberately." + exit 1 + fi + platforms="${expected_platforms}" + echo "✅ Verifying published images for: ${platforms}" for IMAGE in "${HUB_AGENT_IMAGE_NAME}" "${MEMBER_AGENT_IMAGE_NAME}" "${REFRESH_TOKEN_IMAGE_NAME}"; do for tag in ${tags}; do ref="${REGISTRY}/${IMAGE}:${tag}" echo " - ${ref}" manifest="$(docker buildx imagetools inspect "${ref}")" - for platform in linux/amd64 linux/arm64; do + for platform in ${platforms}; do grep -q "Platform:.*${platform}" <<<"${manifest}" \ || { echo "::error::${ref} is missing platform ${platform}"; exit 1; } done done - # Once per image: every tag resolves to the same index. IMAGE_SBOM is + # Once per tag would be redundant - every tag resolves to the same + # index - but once per platform is not: BuildKit emits one SBOM per + # platform, and an image whose arm64 SBOM was missing would + # otherwise publish behind a green amd64 check. IMAGE_SBOM is # matched as an exact string in the Makefile, so a future "TRUE" or # "1" would quietly produce no SBOM; assert the attestation is really # attached rather than trusting the flag was spelled as make expects. - sbom="$(docker buildx imagetools inspect "${REGISTRY}/${IMAGE}:${TAG}" \ - --format '{{ json (index .SBOM "linux/amd64").SPDX }}' 2>/dev/null || true)" - if [ -z "${sbom}" ] || [ "${sbom}" = "null" ]; then - echo "::error::${REGISTRY}/${IMAGE}:${TAG} has no SPDX SBOM attached" - exit 1 - fi + for platform in ${platforms}; do + sbom_err="$(mktemp)" + sbom="$(docker buildx imagetools inspect "${REGISTRY}/${IMAGE}:${TAG}" \ + --format "{{ json (index .SBOM \"${platform}\").SPDX }}" 2>"${sbom_err}" || true)" + if [ -z "${sbom}" ] || [ "${sbom}" = "null" ]; then + # Print what buildx said. A missing attestation and a registry + # or template error are different faults with different fixes, + # and discarding stderr makes every one of them look like the + # first. + echo "::error::${REGISTRY}/${IMAGE}:${TAG} has no SPDX SBOM for ${platform}: $(tr '\n' ' ' <"${sbom_err}")" + rm -f "${sbom_err}" + exit 1 + fi + rm -f "${sbom_err}" + done done - name: Sign the published images diff --git a/Makefile b/Makefile index 6c8c08f48..4867e3205 100644 --- a/Makefile +++ b/Makefile @@ -265,6 +265,10 @@ BINFMT_IMAGE ?= mcr.microsoft.com/mirror/docker/tonistiigi/binfmt:$(BINFMT_VERSI PLATFORMS ?= $(TARGET_OS)/$(TARGET_ARCH) RELEASE_PLATFORMS ?= linux/amd64,linux/arm64 +.PHONY: print-release-platforms +print-release-platforms: ## Print the platforms a release publishes, one per line + @echo "$(RELEASE_PLATFORMS)" | tr ',' '\n' + # Attach an SPDX SBOM to the image index. BuildKit generates one per platform # and stores it alongside the image in the registry, so consumers can read what # is in an image without unpacking it. Off by default because it slows every diff --git a/RELEASING.md b/RELEASING.md index 1b6a34da8..de0e117fd 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -103,13 +103,18 @@ Signing the OCI charts is the same keyless flow used for images and is tracked as a follow-up; the classic-repo `.prov` mechanism needs a long-lived GPG key this project has no custody story for. +Releases are signed with cosign v3 (the workflow pins the exact version). Verify +with cosign v3.0 or later — the `verify-blob --bundle` format below changed +between v2 and v3, and an older client reports it as a bad signature rather than +as a version mismatch. + Images are signed by digest rather than by tag, so a signature is bound to the exact bytes the release built. Verify one with: ```bash cosign verify \ --certificate-oidc-issuer https://token.actions.githubusercontent.com \ - --certificate-identity-regexp '^https://github\.com/kubefleet-dev/kubefleet/\.github/workflows/release\.yml@' \ + --certificate-identity-regexp '^https://github\.com/kubefleet-dev/kubefleet/\.github/workflows/release\.yml@refs/(tags/v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?|heads/(main|release-[0-9]+\.[0-9]+))$' \ ghcr.io/kubefleet-dev/kubefleet/hub-agent:v0.4.0 ``` @@ -128,10 +133,11 @@ The CRD bundle's checksum file is signed as a blob; verifying it and then checking the tarball against it covers the tarball: ```bash +gh release download v0.4.0 --pattern 'kubefleet-crds-*' cosign verify-blob \ --bundle kubefleet-crds-v0.4.0.tgz.sha256.bundle \ --certificate-oidc-issuer https://token.actions.githubusercontent.com \ - --certificate-identity-regexp '^https://github\.com/kubefleet-dev/kubefleet/\.github/workflows/release\.yml@' \ + --certificate-identity-regexp '^https://github\.com/kubefleet-dev/kubefleet/\.github/workflows/release\.yml@refs/(tags/v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?|heads/(main|release-[0-9]+\.[0-9]+))$' \ kubefleet-crds-v0.4.0.tgz.sha256 sha256sum -c kubefleet-crds-v0.4.0.tgz.sha256 ``` @@ -141,12 +147,34 @@ cosign fetches once and caches under `~/.sigstore`. On a machine with no network access, prime that cache first or pass `--trusted-root`; otherwise the commands above fail for a reason that has nothing to do with the signature. +The identity pattern is deliberately narrow in two ways, and both matter to +whoever copies these commands: + +- **It names `release.yml`, not just the repository.** `release.yml` is not the + only workflow holding `id-token: write` — `squad-docs.yml` has it for GitHub + Pages — so a repository-wide pattern would accept a signature from a workflow + that has nothing to do with releases. +- **It names the refs a release can come from:** a `vX.Y.Z` or `vX.Y.Z-rc.N` tag, + or the `main` / `release-X.Y` branches. A `workflow_dispatch` runs the workflow + *definition* from the ref it was started on, so without this anyone able to + push a branch and dispatch it could sign under an identity consumers trust. + Start a dispatched release from `main`, a release branch, or the tag itself; + from anywhere else `create-draft-release` fails before creating the draft, so + no draft and no image exists. + +What this does not buy: it is a bound on which workflow *definitions* can sign, +not an access control. Someone who can already push to this repository can push +a `v*` tag at a commit carrying a modified `release.yml`, and the tag arm accepts +it. Closing that needs a GitHub ruleset restricting who may create `v*` tags and +push to `main` / `release-*`; the repository has no such ruleset today, and +adding one is worth doing independently of this workflow. + The release workflow runs these same verifications immediately after signing, so -a signature that cannot be verified fails the release instead of shipping. -`release.yml` is the only workflow holding `id-token: write`. Any OIDC trust -policy added later (cloud role assumption, trusted publishing) must be scoped to -that workflow's `job_workflow_ref`, never to `repo:kubefleet-dev/kubefleet:*`, -or it would be assumable from any workflow in the repository. +a signature that cannot be verified fails the release instead of shipping. Any +OIDC trust policy added later (cloud role assumption, trusted publishing) must be +scoped to that workflow's `job_workflow_ref`, never to +`repo:kubefleet-dev/kubefleet:*`, or it would be assumable from any workflow in +the repository. ## Recovering from a failed run @@ -163,7 +191,7 @@ is harmless while the release is still a draft — nothing has been announced ye | Where it failed | What is already public | What to do | | --- | --- | --- | | `setup` | Nothing | The tag is malformed. Delete it, fix, re-tag. | -| `create-draft-release` | Nothing | See [Re-releasing an existing tag](#re-releasing-an-existing-tag) if it refused because the release is already published. | +| `create-draft-release` | Nothing | Two causes. If it failed on **Check this ref can sign a release**, the run was started from a ref a release cannot sign under — re-dispatch from `main`, a `release-X.Y` branch, or the tag itself; nothing was published, so there is nothing to clean up. If it refused because the release is already published, see [Re-releasing an existing tag](#re-releasing-an-existing-tag). | | `publish-images` | Any images pushed before the failure (`make push` builds hub-agent, member-agent, then refresh-token in order) | Fix, then re-run failed jobs. | | `publish-crds` | Possibly the images — it runs in parallel with `publish-images`, not after it | Fix, then re-run failed jobs. | | Either signing step | Whatever that job published before signing | Usually a Sigstore or registry transient rather than a code fault — re-run failed jobs first. A signature that will not verify fails the job by design, so nothing unverifiable ships. | diff --git a/hack/release/check-signing-ref.sh b/hack/release/check-signing-ref.sh new file mode 100755 index 000000000..7190e51ea --- /dev/null +++ b/hack/release/check-signing-ref.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Fail before anything is published if this run cannot produce a signature +# consumers will accept. +# +# Signing happens after the images are pushed, so without this check a release +# started from the wrong ref publishes three images, signs them, writes an +# immutable Rekor entry, and only then fails on verification - leaving public +# artifacts behind that no release references and a tag that can no longer be +# safely reused. +# +# What Fulcio actually puts in the SAN is job_workflow_ref: the ref of the +# workflow file defining the *job* that signs. That equals GITHUB_WORKFLOW_REF +# only while the signing jobs are defined inline in release.yml, which they are +# today. Move them into a reusable workflow - release.yml already calls one for +# setup - and cosign would see that file's ref instead, breaking verification +# while this check still passed. Anyone doing that has to revisit both this +# script and the pattern in identity.sh. +# +# Environment: +# GITHUB_REPOSITORY owner/repo +# GITHUB_WORKFLOW_REF owner/repo/.github/workflows/release.yml@refs/... + +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=hack/release/identity.sh +. "${here}/identity.sh" + +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY must be set}" +: "${GITHUB_WORKFLOW_REF:?GITHUB_WORKFLOW_REF must be set}" + +pattern="$(release_identity_pattern "${GITHUB_REPOSITORY}")" +identity="https://github.com/${GITHUB_WORKFLOW_REF}" + +# grep's ERE is marginally more permissive than the RE2 cosign will use - it +# anchors per line where Go anchors at end of text - so in principle this could +# accept something cosign later rejects. Git refnames cannot contain newlines, +# which is the only construct where the two differ here. +if ! grep -qE "${pattern}" <<<"${identity}"; then + echo "::error::This run would sign as '${identity}', which is not an identity a release signs under. Push a vX.Y.Z tag, or dispatch from main, a release-X.Y branch, or the tag itself. Nothing has been published." + exit 1 +fi + +echo "✅ ${identity} is a trusted release identity" diff --git a/hack/release/identity.sh b/hack/release/identity.sh new file mode 100644 index 000000000..cc39185eb --- /dev/null +++ b/hack/release/identity.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# The cosign identity a genuine KubeFleet release signs under: OIDC issuer plus +# certificate-identity pattern. +# +# Sourced by the signing scripts and by their tests so there is exactly one +# definition. A second copy would be worse than none: the tests would assert +# their own reimplementation and keep passing after the scripts had drifted. +# +# This file is meant to be sourced, not executed. + +# The issuer half is not decoration. The identity below is only a string; any +# OIDC provider could mint a certificate carrying it. Pinning the issuer is what +# makes the pattern mean "GitHub Actions said so". +# shellcheck disable=SC2034 # read by the scripts that source this file +RELEASE_OIDC_ISSUER="https://token.actions.githubusercontent.com" + +# The tag shapes a release can carry, as an ERE fragment. Named here so the +# tests can check it against setup-release.yml by literal comparison instead of +# carrying a hand-escaped third copy - which is the very thing this file exists +# to avoid. +# shellcheck disable=SC2034 # read by the scripts that source this file +RELEASE_TAG_GRAMMAR='v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?' + +# release_identity_pattern +# +# Prints the --certificate-identity-regexp that matches a release signature and +# nothing else. Two things are bound: +# +# The workflow file. Any other workflow that gains id-token: write could +# otherwise mint signatures that pass; the repository already has one such +# workflow (squad-docs.yml, for GitHub Pages). +# +# The ref. workflow_dispatch runs the workflow *definition* from the ref it +# was started on, so without this anyone able to push a branch and dispatch it +# could run a modified release.yml and sign under an identity consumers trust. +# +# What this does and does not buy, stated plainly: it removes "any branch in the +# repository" from the trusted set, which is the cheap and large win. It does +# not make the identity unforgeable by someone who already has write access - +# they can still push a `v*` tag at a commit carrying a modified release.yml, +# and the tag arm accepts it. Closing that needs a GitHub ruleset restricting +# who may create `v*` tags and push to `main`/`release-*`, which is repository +# configuration rather than anything this file can enforce. Treat the pattern as +# a bound on which *workflow definitions* can sign, not as an access control. +# +# The pattern is anchored at both ends because cosign matches it as a search, +# not as a full match: unanchored, a trusted identity appearing anywhere in an +# attacker-chosen string would satisfy it. +release_identity_pattern() { + local repo="$1" + # The repository name is interpolated into a regexp and may contain dots, + # which would otherwise match any character. + local repo_pattern="${repo//./\\.}" + # Asserted against setup-release.yml by test-release-scripts.sh: if the tag + # grammar there widens and this does not, signing breaks after images are + # already published. + local tag_ref="tags/${RELEASE_TAG_GRAMMAR}" + local branch_ref='heads/(main|release-[0-9]+\.[0-9]+)' + # printf rather than echo: this string is mostly backslashes, and echo's + # handling of them is implementation-defined. + printf '^https://github\\.com/%s/\\.github/workflows/release\\.yml@refs/(%s|%s)$\n' \ + "${repo_pattern}" "${tag_ref}" "${branch_ref}" +} diff --git a/hack/release/sign-crd-bundle.sh b/hack/release/sign-crd-bundle.sh index efbf05f67..4db221080 100755 --- a/hack/release/sign-crd-bundle.sh +++ b/hack/release/sign-crd-bundle.sh @@ -13,6 +13,10 @@ set -euo pipefail +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=hack/release/identity.sh +. "${here}/identity.sh" + : "${TAG:?TAG must be set}" : "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY must be set}" @@ -25,11 +29,8 @@ if [ ! -f "${checksum}" ]; then exit 1 fi -OIDC_ISSUER="https://token.actions.githubusercontent.com" -# See sign-images.sh for why this is bound to the workflow file and why the -# repository name has its dots escaped before going into a regexp. -repo_pattern="${GITHUB_REPOSITORY//./\\.}" -IDENTITY_PATTERN="^https://github\.com/${repo_pattern}/\.github/workflows/release\.yml@" +OIDC_ISSUER="${RELEASE_OIDC_ISSUER}" +IDENTITY_PATTERN="$(release_identity_pattern "${GITHUB_REPOSITORY}")" echo "Signing ${checksum}" cosign sign-blob --yes --bundle "${bundle}" "${checksum}" diff --git a/hack/release/sign-images.sh b/hack/release/sign-images.sh index 76f3a5ef0..779f32ee2 100755 --- a/hack/release/sign-images.sh +++ b/hack/release/sign-images.sh @@ -25,19 +25,16 @@ set -euo pipefail shopt -s nullglob +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=hack/release/identity.sh +. "${here}/identity.sh" + : "${REGISTRY:?REGISTRY must be set}" : "${IMAGE_METADATA_DIR:?IMAGE_METADATA_DIR must be set}" : "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY must be set}" -OIDC_ISSUER="https://token.actions.githubusercontent.com" -# GITHUB_REPOSITORY is interpolated into a regexp, and repository names may -# contain dots, which would otherwise match any character. -repo_pattern="${GITHUB_REPOSITORY//./\\.}" -# Bound to this workflow file, not merely to the repository: any other workflow -# that ever gains id-token: write could otherwise mint signatures that pass. No -# ref constraint - a workflow_dispatch release signs as @refs/heads/ -# while a tag push signs as @refs/tags/. -IDENTITY_PATTERN="^https://github\.com/${repo_pattern}/\.github/workflows/release\.yml@" +OIDC_ISSUER="${RELEASE_OIDC_ISSUER}" +IDENTITY_PATTERN="$(release_identity_pattern "${GITHUB_REPOSITORY}")" metadata_files=("${IMAGE_METADATA_DIR}"/*.json) if [ "${#metadata_files[@]}" -eq 0 ]; then diff --git a/hack/release/test-release-scripts.sh b/hack/release/test-release-scripts.sh index a831564d7..c84ce487a 100755 --- a/hack/release/test-release-scripts.sh +++ b/hack/release/test-release-scripts.sh @@ -19,6 +19,19 @@ publish_script="${here}/publish-release.sh" sign_images_script="${here}/sign-images.sh" sign_bundle_script="${here}/sign-crd-bundle.sh" +# The same definition the signing scripts use, so the assertions below are +# about the scripts' real behaviour rather than about a copy of it. +# shellcheck source=hack/release/identity.sh +. "${here}/identity.sh" +identity_regexp="$(release_identity_pattern kubefleet-dev/kubefleet)" + +# The issuer, unlike the pattern, is written out here as a literal rather than +# read from identity.sh. Sourcing it would make this agree with whatever the +# scripts say, including a swap to an attacker's issuer - the assertion would +# move with the thing it is supposed to pin. GitHub's issuer is a fixed public +# constant, so hard-coding it is what gives the check its value. +github_oidc_issuer="https://token.actions.githubusercontent.com" + passed=0 failed=0 @@ -29,7 +42,10 @@ run_case() { export FAKE_GH_LOG output="" rc=0 - output="$(env "$@" 2>&1)" || rc=$? + # stdin from /dev/null: these scripts are meant to run unattended, and a + # command that unexpectedly reads stdin should fail the case rather than + # block forever on the terminal of whoever ran the suite by hand. + output="$(env "$@" &1)" || rc=$? log="$(cat "${FAKE_GH_LOG}")" rm -f "${FAKE_GH_LOG}" } @@ -63,6 +79,44 @@ expect_output() { # expect_output if grep -qF -- "$1" <<<"${output}"; then ok "$2"; else bad "$2 - output lacks '$1'"; fi } +echo "== signing identity constants ==" + +if [ "${RELEASE_OIDC_ISSUER}" = "${github_oidc_issuer}" ]; then + ok "identity.sh pins GitHub's OIDC issuer" +else + rc="n/a"; output="identity.sh has RELEASE_OIDC_ISSUER=${RELEASE_OIDC_ISSUER}"; log="" + bad "identity.sh pins GitHub's OIDC issuer" +fi + +echo "== check-signing-ref.sh ==" + +# This runs before anything is published, so its whole value is that it agrees +# with what cosign will decide much later. Same pattern, opposite consequence: +# here a mismatch costs nothing, at signing time it costs a published image. +check_ref_script="${here}/check-signing-ref.sh" +check_ref() { # check_ref + run_case GITHUB_REPOSITORY=kubefleet-dev/kubefleet \ + GITHUB_WORKFLOW_REF="kubefleet-dev/kubefleet/.github/workflows/release.yml@$2" \ + bash "${check_ref_script}" + expect_rc "$1" "$3" +} + +check_ref 0 refs/tags/v0.4.0 "a tag push can sign" +check_ref 0 refs/tags/v0.4.0-rc.1 "a release-candidate tag can sign" +check_ref 0 refs/heads/main "a dispatch from the default branch can sign" +check_ref 0 refs/heads/release-0.4 "a dispatch from a release branch can sign" +check_ref 1 refs/heads/some-feature-branch "a dispatch from an arbitrary branch is refused" +check_ref 1 refs/tags/nonsense "a non-semver tag is refused" +expect_output "Nothing has been published" "refusal says nothing was published" + +# -u because env does not clear what it inherits, and GITHUB_WORKFLOW_REF is +# always set inside Actions: without it this case would take the "present but +# untrusted" branch in CI and never reach the guard it is meant to cover. +run_case -u GITHUB_WORKFLOW_REF GITHUB_REPOSITORY=kubefleet-dev/kubefleet \ + bash "${check_ref_script}" +expect_rc 1 "missing GITHUB_WORKFLOW_REF: fails rather than assuming trusted" +expect_output "GITHUB_WORKFLOW_REF" "missing GITHUB_WORKFLOW_REF: names the variable" + echo "== create-draft-release.sh ==" run_case FAKE_GH_STATE=absent TAG=v0.4.0 PRERELEASE=false bash "${create_script}" @@ -178,6 +232,15 @@ expect_gh "cosign sign --recursive --yes ghcr.io/kubefleet-dev/kubefleet/refresh # bytes this run built. expect_no_gh "ghcr.io/kubefleet-dev/kubefleet/hub-agent:" "never signs by tag" expect_gh "cosign verify" "verifies each signature it creates" +# Which identity it verifies against is the point. Without this, every +# assertion above would still pass against a script that accepted any signer. +expect_gh "--certificate-identity-regexp ${identity_regexp}" \ + "verifies images against the shared ref-constrained identity" +# The identity is only a string; any OIDC provider could mint a certificate +# carrying it. Pinning the issuer is what makes the pattern mean anything, so +# an unpinned issuer has to fail the suite rather than pass unnoticed. +expect_gh "--certificate-oidc-issuer ${github_oidc_issuer}" \ + "binds image verification to GitHub's OIDC issuer" rm -rf "${meta_dir}" meta_dir="$(mktemp -d)" @@ -207,14 +270,18 @@ rm -rf "${meta_dir}" echo "== signing identity pattern ==" -# The identity regexp is the whole security value of keyless signing: it is what -# a consumer pins to. These assert the exact string both scripts build. -identity_pattern_for() { # identity_pattern_for - GITHUB_REPOSITORY="$1" bash -c \ - 'repo_pattern="${GITHUB_REPOSITORY//./\\.}"; echo "^https://github\.com/${repo_pattern}/\.github/workflows/release\.yml@"' -} - -pattern="$(identity_pattern_for kubefleet-dev/kubefleet)" +# The identity regexp is half the security value of keyless signing - the issuer +# asserted above is the other half - and it is what a consumer pins to. These +# exercise the definition the signing scripts actually use (sourced at the top of +# this file) rather than a copy of it: a copy would keep passing after the +# scripts had drifted, which is the one failure this suite most needs to catch. +# +# cosign matches with Go's RE2; this uses ERE. They are not equivalent: grep +# anchors per line where Go anchors at start and end of text, so grep is the +# more permissive of the two. Every `reject` below therefore holds a fortiori +# under RE2, and every `accept` is valid because these inputs are single-line. +# Anything relying on a construct where the two differ needs checking against Go. +pattern="${identity_regexp}" check_identity() { # check_identity if grep -qE "${pattern}" <<<"$2"; then result=accept; else result=reject; fi if [ "${result}" = "$1" ]; then ok "$3"; else @@ -222,28 +289,109 @@ check_identity() { # check_identity fi } -check_identity accept \ - "https://github.com/kubefleet-dev/kubefleet/.github/workflows/release.yml@refs/tags/v0.4.0" \ - "accepts a tag-triggered release signature" -# A workflow_dispatch release signs under the branch ref, so the pattern must -# not pin refs/tags. -check_identity accept \ - "https://github.com/kubefleet-dev/kubefleet/.github/workflows/release.yml@refs/heads/main" \ - "accepts a dispatch-triggered release signature" +workflow="https://github.com/kubefleet-dev/kubefleet/.github/workflows/release.yml" + +check_identity accept "${workflow}@refs/tags/v0.4.0" \ + "accepts a stable tag-triggered release signature" +check_identity accept "${workflow}@refs/tags/v0.4.0-rc.1" \ + "accepts a release-candidate tag" +check_identity accept "${workflow}@refs/tags/v10.20.30" \ + "accepts multi-digit version components" +# A workflow_dispatch runs the workflow definition from the ref it was started +# on, and RELEASING.md documents dispatching from the default branch. +check_identity accept "${workflow}@refs/heads/main" \ + "accepts a dispatch from the default branch" +check_identity accept "${workflow}@refs/heads/release-0.4" \ + "accepts a dispatch from a release branch" + +# The reason the ref is constrained at all: a dispatch from an unprotected +# branch would otherwise run a modified release.yml under a trusted identity. +check_identity reject "${workflow}@refs/heads/attacker-branch" \ + "rejects a dispatch from an arbitrary branch" +check_identity reject "${workflow}@refs/heads/main-evil" \ + "rejects a branch that merely starts with the default branch name" +check_identity reject "${workflow}@refs/heads/release-0.4-evil" \ + "rejects a release branch with a suffix" +check_identity reject "${workflow}@refs/tags/v0.4.0-evil" \ + "rejects a tag with a non-release-candidate suffix" +check_identity reject "${workflow}@refs/tags/notatag" \ + "rejects a tag that is not semver-shaped" +check_identity reject "${workflow}@refs/tags/v0.4.0/extra" \ + "rejects anything appended after a valid tag" +check_identity reject "prefix-${workflow}@refs/tags/v0.4.0" \ + "rejects a trusted identity embedded in a longer string" check_identity reject \ - "https://github.com/kubefleet-dev/kubefleet/.github/workflows/squad-release.yml@refs/tags/v0.4.0" \ + "https://github.com/kubefleet-dev/kubefleet/.github/workflows/squad-docs.yml@refs/heads/main" \ "rejects another workflow in the same repository" -check_identity reject \ - "https://github.com/kubefleet-dev/kubefleet/.github/workflows/release.yml.bak@refs/tags/v0.4.0" \ +check_identity reject "${workflow}.bak@refs/tags/v0.4.0" \ "rejects a filename that merely starts with release.yml" check_identity reject \ "https://github.com/evil/kubefleet/.github/workflows/release.yml@refs/tags/v0.4.0" \ "rejects the same workflow path in a different repository" -# Repository names may contain dots; unescaped they would match any character. -if [ "$(identity_pattern_for org/weird.name)" = '^https://github\.com/org/weird\.name/\.github/workflows/release\.yml@' ]; then - ok "escapes dots in the repository name" + +# RELEASING.md hands consumers a verification command containing this pattern +# verbatim. If it drifts, users pin an identity the release no longer signs +# under and every verification fails - the kind of break that only shows up in +# someone else's terminal, long after the release shipped. +# +# Counted, not grepped: the pattern appears in more than one code block, and a +# first-match test would pass while a maintainer updated one and missed the +# other - which is the realistic way this drifts. +releasing_md="${here}/../../RELEASING.md" +if [ ! -f "${releasing_md}" ]; then + rc="n/a"; output="no RELEASING.md at ${releasing_md}"; log="" + bad "RELEASING.md documents the pattern the scripts use" +else + current="$(grep -cF -- "${identity_regexp}" "${releasing_md}")" + total="$(grep -c -- '--certificate-identity-regexp' "${releasing_md}")" + if [ "${current}" -gt 0 ] && [ "${current}" = "${total}" ]; then + ok "every documented identity in RELEASING.md is the one the scripts use (${current})" + else + rc="n/a"; log="" + output="${current} of ${total} --certificate-identity-regexp occurrences match ${identity_regexp}" + bad "every documented identity in RELEASING.md is the one the scripts use" + fi +fi + +# The tag grammar is spelled independently in setup-release.yml, which gates the +# whole release. Deliberately one-way: the dangerous direction is that file +# widening - say to accept -beta.N - while identity.sh does not, because then a +# tag passes validation, gets built and pushed, and only fails at signing. The +# reverse is harmless, since setup-release.yml rejects the tag before anything +# is published. Compared literally against the grammar identity.sh exports, so +# there is no hand-escaped third copy to re-escape on a legitimate change. +# +# A literal comparison only catches widenings that disturb the mirrored +# fragment. Appending a second optional group would leave it intact and pass +# here; check-signing-ref.sh is the backstop for that, and it fails the release +# at the gate rather than after publishing. +setup_yml="${here}/../../.github/workflows/setup-release.yml" +if [ ! -f "${setup_yml}" ]; then + rc="n/a"; output="no setup-release.yml at ${setup_yml}"; log="" + bad "setup-release.yml validates no tag the signing identity would reject" +elif grep -qF -- "${RELEASE_TAG_GRAMMAR}" "${setup_yml}"; then + ok "setup-release.yml validates no tag the signing identity would reject" +else + rc="n/a"; log="" + output="setup-release.yml no longer spells the grammar identity.sh exports: ${RELEASE_TAG_GRAMMAR}" + bad "setup-release.yml validates no tag the signing identity would reject" +fi + +# Repository names may contain dots; unescaped they would match any character, +# so a lookalike repository would verify. Asserted behaviourally rather than by +# inspecting the string: what matters is what the pattern accepts. +dotted_pattern="$(release_identity_pattern org/weird.name)" +dotted_workflow="https://github.com/org/weird.name/.github/workflows/release.yml" +if grep -qE "${dotted_pattern}" <<<"${dotted_workflow}@refs/tags/v0.4.0"; then + ok "a dotted repository name still matches itself" +else + rc="n/a"; output="${dotted_pattern}"; log=""; bad "a dotted repository name still matches itself" +fi +if grep -qE "${dotted_pattern}" <<<"https://github.com/org/weirdXname/.github/workflows/release.yml@refs/tags/v0.4.0"; then + rc="n/a"; output="${dotted_pattern}"; log="" + bad "an unescaped dot would match a lookalike repository" else - rc="n/a"; output="$(identity_pattern_for org/weird.name)"; log=""; bad "escapes dots in the repository name" + ok "escapes dots so a lookalike repository does not match" fi echo "== sign-crd-bundle.sh ==" @@ -256,6 +404,10 @@ expect_rc 0 "checksum present: signs successfully" expect_gh "cosign sign-blob --yes --bundle ${pkg_dir}/kubefleet-crds-v0.4.0.tgz.sha256.bundle" \ "writes the signature bundle next to the checksum" expect_gh "cosign verify-blob" "verifies the blob signature it just created" +expect_gh "--certificate-identity-regexp ${identity_regexp}" \ + "verifies the CRD bundle against the shared ref-constrained identity" +expect_gh "--certificate-oidc-issuer ${github_oidc_issuer}" \ + "binds CRD bundle verification to GitHub's OIDC issuer" if [ -f "${pkg_dir}/kubefleet-crds-v0.4.0.tgz.sha256.bundle" ]; then ok "signature bundle exists for upload" else