Merge pull request #859 from durable-workflow/fix/2116-published-verification-continuation

Continue release checks without rebuilding immutable images

Author: rmcdaniel

.github/workflows/phpunit-feature.yml

@@ -179,7 +179,7 @@ jobs:
 
       - name: Key the source-free PHP tool images
         id: php_tools_key
-        run: echo "key=server-php-tools-v1-${{ runner.os }}-${{ runner.arch }}-$(date -u +%Y%m%d)-${{ hashFiles('Dockerfile', 'Dockerfile.dockerignore', '.dockerignore', 'docker/**') }}" >> "$GITHUB_OUTPUT"
+        run: echo "key=server-php-tools-v2-${{ runner.os }}-${{ runner.arch }}-$(date -u +%Y%m%d)-${{ hashFiles('Dockerfile', 'Dockerfile.dockerignore', '.dockerignore', 'docker/**') }}" >> "$GITHUB_OUTPUT"
 
       - name: Restore the source-free PHP tool images
         id: php_tools_cache
@@ -201,7 +201,7 @@ jobs:
             docker build -t durable-workflow-server-corpus-validator - <<'DOCKERFILE'
           FROM durable-workflow-server-phpunit-base
           RUN apt-get update \
-              && apt-get install -y --no-install-recommends git \
+              && apt-get install -y --no-install-recommends git jq \
               && rm -rf /var/lib/apt/lists/* \
               && server-verify-trust final
           DOCKERFILE
@@ -215,6 +215,8 @@ jobs:
           docker run --rm --network none --entrypoint server-verify-trust durable-workflow-server-phpunit-base
           docker run --rm --network none --entrypoint server-verify-trust durable-workflow-server-corpus-validator final
           docker run --rm --network none --entrypoint server-verify-brotli durable-workflow-server-corpus-validator
+          docker run --rm --network none --entrypoint sh durable-workflow-server-corpus-validator \
+            -c 'git --version && jq --version'
 
       # Cache writes precede the dependency checkout and all application code.
       # These images contain only toolchains, qualified by the checks above.
@@ -353,7 +355,7 @@ jobs:
                 -w /app \
                 --entrypoint sh \
                 durable-workflow-server-corpus-validator \
-                -lc 'set -eu; tar --no-same-owner -xf - -C /app; cp -a /app/workflow-package /workflow; php scripts/ci/prepare-release-workflow-composer-metadata.php; composer update durable-workflow/workflow --with-dependencies --minimal-changes --no-install --no-scripts --no-autoloader --no-interaction --no-progress --prefer-dist; composer install --no-interaction --no-progress --prefer-dist; git --version; vendor/bin/phpunit --version; git rev-parse --verify "$CORPUS_BASE_REF^{commit}"; php scripts/ci/check-worker-openapi-evolution.php "$CORPUS_BASE_REF"; vendor/bin/phpunit tests/Feature tests/Unit/NexusContractTest.php tests/Unit/CodecRegressionCorpusTest.php tests/Unit/PhpunitFeatureWorkflowContractTest.php tests/Unit/OpenApiDocumentEvolutionTest.php tests/Unit/NamespaceConformanceRunnerContractTest.php tests/Unit/NamespaceRuntimeContractTest.php tests/Unit/SchedulesConformanceRunnerContractTest.php tests/Unit/SchedulesRuntimeContractTest.php tests/Unit/ServerParitySdkDirectionResponseSummaryTest.php tests/Unit/ServerParitySdkQueryTransportTest.php tests/Unit/ServerParitySdkUpdateTransportTest.php tests/Unit/ServerParityLockPressureRetryTest.php tests/Unit/ServerParityManyUpdatesRetryTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImageImmutabilityGuardTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImagePublishWorkflowContractTest.php --filter test_php_security_audits_fail_before_release_publication --no-progress --colors=never; python3 scripts/ci/validate-regression-corpus.py --base-ref "$CORPUS_BASE_REF" --verify-counterfactual'
+                -lc 'set -eu; tar --no-same-owner -xf - -C /app; cp -a /app/workflow-package /workflow; php scripts/ci/prepare-release-workflow-composer-metadata.php; composer update durable-workflow/workflow --with-dependencies --minimal-changes --no-install --no-scripts --no-autoloader --no-interaction --no-progress --prefer-dist; composer install --no-interaction --no-progress --prefer-dist; git --version; vendor/bin/phpunit --version; git rev-parse --verify "$CORPUS_BASE_REF^{commit}"; php scripts/ci/check-worker-openapi-evolution.php "$CORPUS_BASE_REF"; vendor/bin/phpunit tests/Feature tests/Unit/NexusContractTest.php tests/Unit/CodecRegressionCorpusTest.php tests/Unit/PhpunitFeatureWorkflowContractTest.php tests/Unit/OpenApiDocumentEvolutionTest.php tests/Unit/NamespaceConformanceRunnerContractTest.php tests/Unit/NamespaceRuntimeContractTest.php tests/Unit/SchedulesConformanceRunnerContractTest.php tests/Unit/SchedulesRuntimeContractTest.php tests/Unit/ServerParitySdkDirectionResponseSummaryTest.php tests/Unit/ServerParitySdkQueryTransportTest.php tests/Unit/ServerParitySdkUpdateTransportTest.php tests/Unit/ServerParityLockPressureRetryTest.php tests/Unit/ServerParityManyUpdatesRetryTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImageContinuationTest.php tests/Unit/ReleaseImageImmutabilityGuardTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImagePublishWorkflowContractTest.php --filter test_php_security_audits_fail_before_release_publication --no-progress --colors=never; python3 scripts/ci/validate-regression-corpus.py --base-ref "$CORPUS_BASE_REF" --verify-counterfactual'
 
       - name: Select production image qualification for changed build inputs
         id: image_inputs

.github/workflows/release.yml

@@ -14,6 +14,11 @@ on:
       release_commit:
         description: 'Full source commit for the existing immutable release tag'
         required: true
+      existing_image_digest:
+        description: 'Continue verification of this existing sha256 index without rebuilding images'
+        required: false
+        default: ''
+        type: string
 permissions:
   contents: read
 
@@ -104,6 +109,8 @@ jobs:
         id: release_publish
         env:
           INPUT_TAG: ${{ github.event.inputs.tag }}
+          INPUT_RELEASE_COMMIT: ${{ inputs.release_commit }}
+          INPUT_EXISTING_IMAGE_DIGEST: ${{ inputs.existing_image_digest }}
           SOURCE_RELEASE_VERSION: ${{ steps.source_release.outputs.version }}
           DOCKERHUB_IMAGE: ${{ env.DOCKERHUB_IMAGE }}
           GHCR_IMAGE: ${{ env.GHCR_IMAGE }}
@@ -179,6 +186,7 @@ jobs:
 
       - name: Resolve shared release image cache
         id: cache
+        if: ${{ steps.release_publish.outputs.verification_only != 'true' }}
         env:
           RELEASE_IMAGE_CACHE_ROOT: release-source
           RELEASE_SOURCE_COMMIT: ${{ steps.release_source.outputs.commit }}
@@ -188,16 +196,19 @@ jobs:
         run: node scripts/ci/resolve-release-image-cache.mjs
 
       - name: Refuse replacement of published release images
+        if: ${{ steps.release_publish.outputs.verification_only != 'true' }}
         env:
           RELEASE_TAG: ${{ steps.release_publish.outputs.tag }}
         run: sh scripts/ci/guard-unpublished-release-images.sh
 
       - name: Start exact image build timer
         id: build_started
+        if: ${{ steps.release_publish.outputs.verification_only != 'true' }}
         run: printf 'epoch_seconds=%s\n' "$(date +%s)" >> "$GITHUB_OUTPUT"
 
       - name: Build and push exact image tags
         id: build
+        if: ${{ steps.release_publish.outputs.verification_only != 'true' }}
         # Bound cold multi-architecture extension and application builds.
         timeout-minutes: 120
         continue-on-error: true
@@ -239,9 +250,11 @@ jobs:
           DOCKER_BUILD_OUTCOME: ${{ steps.build.outcome || 'skipped' }}
           BUILT_IMAGE_DIGEST: ${{ steps.build.outputs.digest }}
           BUILT_IMAGE_METADATA: ${{ steps.build.outputs.metadata }}
+          RELEASE_IMAGE_VERIFICATION_ONLY: ${{ steps.release_publish.outputs.verification_only }}
+          EXPECTED_IMAGE_DIGEST: ${{ steps.release_publish.outputs.existing_image_digest }}
           RELEASE_COMMIT: ${{ steps.release_source.outputs.commit }}
-          RELEASE_RUN_ID: ${{ github.run_id }}
-          RELEASE_RUN_ATTEMPT: ${{ github.run_attempt }}
+          RELEASE_RUN_ID: ${{ steps.release_publish.outputs.verification_only != 'true' && github.run_id || '' }}
+          RELEASE_RUN_ATTEMPT: ${{ steps.release_publish.outputs.verification_only != 'true' && github.run_attempt || '' }}
           WORKFLOW_PACKAGE_SOURCE: ${{ steps.workflow.outputs.source }}
           WORKFLOW_PACKAGE_REF: ${{ steps.workflow.outputs.ref }}
           WORKFLOW_PACKAGE_COMMIT: ${{ steps.workflow.outputs.commit }}
@@ -250,23 +263,27 @@ jobs:
       - name: Verify published image trust stores
         id: trust_stores
         if: ${{ always() && success() && steps.exact.outputs.exact_publish_outcome == 'success' }}
+        timeout-minutes: 10
         env:
-          RELEASE_IMAGE: ${{ env.DOCKERHUB_IMAGE }}:${{ steps.release_publish.outputs.tag }}
+          RELEASE_IMAGE: ${{ env.DOCKERHUB_IMAGE }}@${{ steps.exact.outputs.image_digest }}
+          DW_PULL_ATTEMPTS: "3"
+          DW_PULL_BACKOFF_SECONDS: "2"
         run: |
           set -euo pipefail
           # GHSA-6q48-6c7w-9gw2: every published architecture's system, PHP,
           # Composer, Node and venv stores, checked offline in the image.
           IFS=',' read -ra platforms <<<"$RELEASE_IMAGE_PLATFORMS"
           for platform in "${platforms[@]}"; do
-            docker run --rm --pull always --platform "$platform" --network none \
+            DOCKER_DEFAULT_PLATFORM="$platform" bash scripts/ci/pull-images.sh "$RELEASE_IMAGE"
+            docker run --rm --pull never --platform "$platform" --network none \
               --entrypoint server-verify-trust "$RELEASE_IMAGE"
           done
 
       - name: Verify bare image first-run readiness
         id: first_run_readiness
         if: ${{ always() && success() && steps.exact.outputs.exact_publish_outcome == 'success' }}
         env:
-          DW_SERVER_IMAGE: ${{ env.DOCKERHUB_IMAGE }}:${{ steps.release_publish.outputs.tag }}
+          DW_SERVER_IMAGE: ${{ env.DOCKERHUB_IMAGE }}@${{ steps.exact.outputs.image_digest }}
           DW_FIRST_RUN_SCOPE: release-first-run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}
         working-directory: release-source
         run: scripts/smoke-published-first-run.sh
@@ -275,7 +292,7 @@ jobs:
         id: source_free_compose
         if: ${{ always() && success() && steps.first_run_readiness.outcome == 'success' }}
         env:
-          DW_SERVER_IMAGE: ${{ env.DOCKERHUB_IMAGE }}:${{ steps.release_publish.outputs.tag }}
+          DW_SERVER_IMAGE: ${{ env.DOCKERHUB_IMAGE }}@${{ steps.exact.outputs.image_digest }}
           DW_SERVER_PLATFORM: linux/amd64
           DW_PUBLISHED_COMPOSE_PROFILE: local
           DW_PUBLISHED_COMPOSE_PROJECT: release-compose-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}
@@ -337,7 +354,7 @@ jobs:
         if: ${{ always() && success() && steps.exact.outputs.exact_publish_outcome == 'success' }}
         env:
           RELEASE_TAG: ${{ steps.release_publish.outputs.tag }}
-          SERVER_IMAGE: ${{ env.DOCKERHUB_IMAGE }}:${{ steps.release_publish.outputs.tag }}
+          SERVER_IMAGE: ${{ env.DOCKERHUB_IMAGE }}@${{ steps.exact.outputs.image_digest }}
           WORKFLOW_PACKAGE_SOURCE: ${{ steps.workflow.outputs.source }}
           WORKFLOW_PACKAGE_REF: ${{ steps.workflow.outputs.ref }}
           WORKFLOW_PACKAGE_COMMIT: ${{ steps.workflow.outputs.commit }}
@@ -357,6 +374,25 @@ jobs:
           docker volume rm -f "${RELEASE_PROTOCOL_CATALOG_SCOPE}-database" \
             >/dev/null 2>&1 || true
 
+      - name: Recheck immutable release identities before promotion
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RELEASE_TAG: ${{ steps.release_publish.outputs.tag }}
+          RELEASE_COMMIT: ${{ steps.release_source.outputs.commit }}
+          RELEASE_IMAGE_VERIFICATION_ONLY: "true"
+          EXPECTED_IMAGE_DIGEST: ${{ steps.exact.outputs.image_digest }}
+          DOCKER_BUILD_OUTCOME: skipped
+          DOCKERHUB_IMAGE: ${{ env.DOCKERHUB_IMAGE }}
+          GHCR_IMAGE: ${{ env.GHCR_IMAGE }}
+          RELEASE_RUN_ID: ${{ steps.release_publish.outputs.verification_only != 'true' && github.run_id || '' }}
+          RELEASE_RUN_ATTEMPT: ${{ steps.release_publish.outputs.verification_only != 'true' && github.run_attempt || '' }}
+          WORKFLOW_PACKAGE_SOURCE: ${{ steps.workflow.outputs.source }}
+          WORKFLOW_PACKAGE_REF: ${{ steps.workflow.outputs.ref }}
+          WORKFLOW_PACKAGE_COMMIT: ${{ steps.workflow.outputs.commit }}
+        run: |
+          scripts/ci/verify-release-tag-source.sh
+          scripts/ci/verify-release-exact-images.sh
+
       - name: Resolve rolling image aliases
         id: rolling
         if: >-
@@ -380,6 +416,7 @@ jobs:
           DOCKERHUB_IMAGE: ${{ env.DOCKERHUB_IMAGE }}
           GHCR_IMAGE: ${{ env.GHCR_IMAGE }}
           ROLLING_SHOULD_PROMOTE: ${{ steps.rolling.outputs.rolling_should_promote }}
+          VERIFIED_IMAGE_DIGEST: ${{ steps.exact.outputs.image_digest }}
           RELEASE_IMAGE_TAG_SOURCE_URL: ${{ github.server_url }}/${{ github.repository }}.git
         run: scripts/ci/promote-release-image-rolling-tags.sh
 
@@ -445,6 +482,7 @@ jobs:
     needs: publish
     if: >-
       always() &&
+      needs.publish.result == 'success' &&
       needs.publish.outputs.exact_publish_outcome == 'success' &&
       needs.publish.outputs.published_database_tls_outcome == 'success' &&
       github.repository == 'durable-workflow/server' &&

docs/helm-validation.md

@@ -29,6 +29,15 @@ TLS probes on the verified digest. It does not rebuild or replace an image.
 Both release paths require the TLS gate to pass before publishing the matching
 chart.
 
+If exact images were uploaded but verification failed before the GitHub Release
+was created, dispatch the protected `Release` workflow from `main` with `tag`,
+the full `release_commit`, and `existing_image_digest` from the original upload.
+This explicit continuation skips image build and push. It verifies both
+registries and architectures against that index and source, then runs all normal
+trust-store, first-run, Compose, database TLS and protocol checks before the
+existing alias, GitHub Release and chart steps. The images retain their original
+publisher provenance. Do not rerun the image producer or replace immutable tags.
+
 For a local repeat, set `TLS_TEST_MODE=published`, `TLS_TEST_IMAGE` to an
 immutable `repository@sha256:...` reference, and unique `TLS_TEST_PROJECT` and
 `TLS_TEST_ROOT` values, then run `scripts/ci/qualify-database-tls.sh`. The

scripts/ci/promote-release-image-rolling-tags.sh

@@ -7,6 +7,7 @@ dockerhub_image="${DOCKERHUB_IMAGE:-durableworkflow/server}"
 ghcr_image="${GHCR_IMAGE:-ghcr.io/durable-workflow/server}"
 docker_bin="${DOCKER:-docker}"
 tag_source_url="${RELEASE_IMAGE_TAG_SOURCE_URL:-}"
+verified_image_digest="${VERIFIED_IMAGE_DIGEST:-}"
 
 fail() {
     printf '::error title=%s::%s\n' "$1" "$2" >&2
@@ -82,6 +83,10 @@ if ! is_stable_semver_tag "$release_tag"; then
     fail "Stable semver tag required" "Rolling image aliases can only be promoted from stable semver tags; got '${release_tag}'."
 fi
 
+if [ -n "$verified_image_digest" ] && ! printf '%s' "$verified_image_digest" | grep -Eq '^sha256:[0-9a-f]{64}$'; then
+    fail "Invalid verified image digest" "Rolling aliases require an exact lowercase sha256 digest when VERIFIED_IMAGE_DIGEST is supplied."
+fi
+
 major="${release_tag%%.*}"
 minor_patch="${release_tag#*.}"
 minor="${minor_patch%%.*}"
@@ -102,12 +107,16 @@ fi
 
 promote_image() {
     image="$1"
+    source_ref="${image}:${release_tag}"
+    if [ -n "$verified_image_digest" ]; then
+        source_ref="${image}@${verified_image_digest}"
+    fi
 
     "$docker_bin" buildx imagetools create \
         --tag "${image}:${minor_alias}" \
         --tag "${image}:${major_alias}" \
         --tag "${image}:latest" \
-        "${image}:${release_tag}"
+        "$source_ref"
 }
 
 promote_image "$dockerhub_image"

scripts/ci/validate-release-image-publish.sh

@@ -6,6 +6,8 @@ dockerhub_image="${DOCKERHUB_IMAGE:-durableworkflow/server}"
 ghcr_image="${GHCR_IMAGE:-ghcr.io/durable-workflow/server}"
 release_tag=""
 missing_credentials=""
+existing_image_digest="${INPUT_EXISTING_IMAGE_DIGEST:-}"
+verification_only=false
 
 fail() {
     title="$1"
@@ -79,6 +81,19 @@ if [ -n "$source_release_version" ]; then
     fi
 fi
 
+if [ -n "$existing_image_digest" ]; then
+    if [ "${GITHUB_EVENT_NAME:-}" != workflow_dispatch ] || [ "${GITHUB_REF:-}" != refs/heads/main ]; then
+        fail "Invalid release continuation context" "Existing image verification requires workflow_dispatch from main."
+    fi
+    if ! printf '%s' "$existing_image_digest" | grep -Eq '^sha256:[0-9a-f]{64}$'; then
+        fail "Invalid existing image digest" "Existing image verification requires an exact lowercase sha256 index digest."
+    fi
+    if ! printf '%s' "${INPUT_RELEASE_COMMIT:-}" | grep -Eq '^[0-9a-f]{40}$'; then
+        fail "Invalid release continuation source" "Existing image verification requires the full lowercase source commit for the immutable tag."
+    fi
+    verification_only=true
+fi
+
 if [ -z "${DOCKERHUB_USERNAME:-}" ]; then
     append_missing_credential "DOCKERHUB_USERNAME"
 fi
@@ -98,6 +113,8 @@ fi
 if [ -n "${GITHUB_OUTPUT:-}" ]; then
     {
         printf 'tag=%s\n' "$release_tag"
+        printf 'verification_only=%s\n' "$verification_only"
+        printf 'existing_image_digest=%s\n' "$existing_image_digest"
         if is_semver_tag "$release_tag"; then
             printf 'is_semver=true\n'
         else

scripts/ci/verify-release-exact-images.sh

@@ -9,6 +9,7 @@ docker_bin="${DOCKER:-docker}"
 build_outcome="${DOCKER_BUILD_OUTCOME:-}"
 built_image_digest="${BUILT_IMAGE_DIGEST:-}"
 built_image_metadata="${BUILT_IMAGE_METADATA:-}"
+requested_image_digest="${EXPECTED_IMAGE_DIGEST:-}"
 release_commit="${RELEASE_COMMIT:-}"
 run_id="${RELEASE_RUN_ID:-}"
 run_attempt="${RELEASE_RUN_ATTEMPT:-}"
@@ -136,6 +137,19 @@ case "$verification_only" in
         ;;
 esac
 
+if [ -n "$requested_image_digest" ]; then
+    if [ "$verification_only" != true ] || ! printf '%s' "$requested_image_digest" | grep -Eq '^sha256:[0-9a-f]{64}$'; then
+        fail "Invalid release continuation digest" "An expected existing-image digest requires verification-only mode and an exact lowercase sha256 index." "exact_continuation_digest_invalid"
+    fi
+    if [ "${build_outcome:-skipped}" != skipped ] || [ -n "$built_image_digest" ] || [ -n "$built_image_metadata" ]; then
+        fail "Release continuation must not build images" "Existing immutable-image continuation requires the producer to be skipped with no new build metadata." "exact_continuation_producer_ran"
+    fi
+    if [ -z "$workflow_package_source" ] || [ -z "$workflow_package_ref" ] \
+        || ! printf '%s' "$workflow_package_commit" | grep -Eq '^[0-9a-f]{40}$'; then
+        fail "Release continuation package authority missing" "Existing immutable-image continuation requires the complete locked Workflow source, version and commit." "exact_continuation_package_authority_missing"
+    fi
+fi
+
 if ! printf '%s' "$release_tag" | grep -Eq '^[0-9A-Za-z._-]+$'; then
     fail "Invalid exact release image tag" "Release image tag '${release_tag}' contains characters that are not safe for Docker tags." "exact_release_tag_invalid"
 fi
@@ -190,6 +204,9 @@ if [ -n "$direct_build_digest" ]; then
 else
     expected_image_digest="$metadata_build_digest"
 fi
+if [ -n "$requested_image_digest" ]; then
+    expected_image_digest="$requested_image_digest"
+fi
 
 if [ -z "$expected_image_digest" ]; then
     if [ "$verify_published_metadata" != "true" ]; then
@@ -234,6 +251,9 @@ inspect_ref() {
     fi
 
     if [ -n "$expected_image_digest" ] && [ "$expected_image_digest" != "$digest" ]; then
+        if [ -n "$requested_image_digest" ]; then
+            fail "Exact release continuation digest mismatch" "Published exact image manifest ${ref} has digest ${digest}, but continuation requires ${expected_image_digest}." "exact_continuation_digest_mismatch"
+        fi
         fail "Exact release image build digest mismatch" "Published exact image manifest ${ref} has digest ${digest}, but this release build produced ${expected_image_digest}." "exact_manifest_build_digest_mismatch"
     fi