From c63480fd551488f2af1195be40020097a6c6f93a Mon Sep 17 00:00:00 2001
From: Benjamin Sznajder <benjams@il.ibm.com>
Date: Thu, 3 Sep 2026 13:20:19 +0300
Subject: [PATCH 1/6] feat(ci): pin a single torch-spyre commit across all jobs
 in a run

torch-spyre is baked into each runner pod image, so matrix jobs landed
on pods at different HEADs and one run mixed several torch-spyre commits,
making results incomparable (#449). Resolve origin/main to one SHA per
run, check that exact commit out before uv add in every job, assert each
job's HEAD matches, and report the pinned + hf-adapters SHAs in the run
summary.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Benjamin Sznajder <benjams@il.ibm.com>
---
 .github/actions/build-hf-adapters/action.yaml |  42 ++-
 .github/actions/gather-runner-info/action.yml |  38 +++
 .github/workflows/_test_matrix.yaml           | 293 ++++++++++++++++--
 .github/workflows/push-to-clickhouse.yaml     | 115 ++++++-
 4 files changed, 452 insertions(+), 36 deletions(-)

diff --git a/.github/actions/build-hf-adapters/action.yaml b/.github/actions/build-hf-adapters/action.yaml
index 5b03eddb..39ce4cfa 100644
--- a/.github/actions/build-hf-adapters/action.yaml
+++ b/.github/actions/build-hf-adapters/action.yaml
@@ -21,6 +21,19 @@ inputs:
       ($GITHUB_SERVER_URL/$GITHUB_REPOSITORY). Set by the Test-With: flow.
     required: false
     default: ''
+  torch_spyre_sha:
+    description: >
+      Full 40-char torch-spyre commit SHA to pin (issue #449). When set, the
+      baked /home/senuser/torch-spyre checkout is fetched + checked out at
+      exactly this commit before `uv add`, so every job in a run installs the
+      identical torch-spyre instead of whatever HEAD each pod's image happened
+      to bake in. Empty (default) keeps today's behaviour, so callers that omit
+      it (e.g. push-to-clickhouse's scan jobs) are unaffected. Named
+      torch_spyre_sha -- NOT torch_spyre_ref (an unrelated, unused dispatch
+      input elsewhere) and NOT the Checkout step's INPUT_REF (= inputs.ref, the
+      hf-adapters ref) -- so the three never get confused.
+    required: false
+    default: ''
 outputs:
   target_sha:
     description: The resolved commit SHA that was checked out
@@ -94,7 +107,11 @@ runs:
       working-directory: /home/senuser
       env:
         UV_GROUPS: "${{ inputs.uv_groups }}"
-        INPUT_REF: ""
+        # Pinned torch-spyre SHA (issue #449). Empty = today's behaviour (install
+        # whatever HEAD this pod's image baked in). Renamed from the previously
+        # hardcoded, dead `INPUT_REF: ""` here so it is never confused with the
+        # Checkout step's INPUT_REF (= inputs.ref, the hf-adapters ref).
+        TORCH_SPYRE_SHA: ${{ inputs.torch_spyre_sha }}
       run: |
         source "$HOME/.bashrc"
         source /etc/profile.d/ibm-aiu-setup.sh
@@ -129,21 +146,28 @@ runs:
           GROUP_FLAGS="$GROUP_FLAGS --group $group"
         done
 
-        if [[ -n "$INPUT_REF" ]]; then
+        if [[ -n "$TORCH_SPYRE_SHA" ]]; then
+          # Pinned mode (issue #449): install the exact torch-spyre commit the
+          # resolve-torch-spyre job picked, NOT whatever HEAD this pod's image
+          # baked in. `git fetch origin <sha>` retrieves that specific object even
+          # when this pod's checkout predates it (with a full-fetch fallback);
+          # then check it out and re-sync submodules so the working tree matches
+          # the pinned commit exactly before `uv add` builds from it.
           cd /home/senuser/torch-spyre
-          TARGET_SHA="$INPUT_REF"
-          TARGET_REF="$INPUT_REF"
-          echo "TARGET_REF=${TARGET_REF}"
-          echo "TARGET_SHA=${TARGET_SHA}"
-          git fetch --tags --force origin "+${TARGET_SHA}" || \
+          echo "Pinning torch-spyre to ${TORCH_SPYRE_SHA}"
+          git fetch --tags --force origin "${TORCH_SPYRE_SHA}" || \
             git fetch --tags --force origin
-          git checkout --force "${TARGET_SHA}"
+          git checkout --force "${TORCH_SPYRE_SHA}"
           git submodule sync --recursive
           git submodule update --init --recursive --force
           git log -1 --oneline
           cd "$CLONED_HF_ADAPTERS_DIR"
+          # Same install tail as the unpinned branch below -- the ONLY difference
+          # is the fetch/checkout preamble above, so a pinned run behaves exactly
+          # like today's install, just at a fixed SHA.
           uv add torch-spyre /home/senuser/torch-spyre
-          uv sync --verbose --refresh $GROUP_FLAGS
+          uv lock --upgrade-package torch-spyre
+          uv sync --frozen --verbose --refresh $GROUP_FLAGS
         else
           cd "$CLONED_HF_ADAPTERS_DIR"
           # though we pin to main
diff --git a/.github/actions/gather-runner-info/action.yml b/.github/actions/gather-runner-info/action.yml
index d5da05a4..96873e42 100644
--- a/.github/actions/gather-runner-info/action.yml
+++ b/.github/actions/gather-runner-info/action.yml
@@ -25,6 +25,21 @@ inputs:
       printed.
     required: false
     default: ''
+  pinned_torch_spyre_sha:
+    description: |
+      The torch-spyre commit SHA this run pinned (from the resolve-torch-spyre
+      job; issue #449). When set, this action asserts that the checked-out
+      torch-spyre HEAD equals it and fails the job on mismatch -- the per-shard
+      proof that the pin actually took. Empty = unpinned (no assertion).
+    required: false
+    default: ''
+  hf_adapters_sha:
+    description: |
+      The hf-adapters commit SHA build-hf-adapters resolved for this job (its
+      `target_sha` output). Reported alongside the pinned torch-spyre SHA in the
+      run summary for reproducibility. Optional.
+    required: false
+    default: ''
 
 runs:
   using: composite
@@ -43,6 +58,9 @@ runs:
         RUNNER_NAME: ${{ runner.name }}
         RUNNER_OS: ${{ runner.os }}
         RUNNER_ARCH: ${{ runner.arch }}
+        # Pin reporting/assertion (issue #449). Empty = unpinned, no assertion.
+        PINNED_TORCH_SPYRE_SHA: ${{ inputs.pinned_torch_spyre_sha }}
+        HF_ADAPTERS_SHA: ${{ inputs.hf_adapters_sha }}
       run: |
         source "$HOME/.bashrc"
         source /etc/profile.d/ibm-aiu-setup.sh
@@ -87,4 +105,24 @@ runs:
             git -C "$TORCH_SPYRE_DIR" log -n 1 --oneline || true
           fi
         fi
+        # torch-spyre pin check (issue #449). Runs regardless of VERBOSE -- it is
+        # a correctness gate, not diagnostics: when a pin is in effect, this job's
+        # checked-out torch-spyre MUST match the SHA the run resolved once up
+        # front. A mismatch means some job installed a different commit (the exact
+        # inconsistency #449 fixes), so fail loudly rather than report a false
+        # green. Empty PINNED_TORCH_SPYRE_SHA (unpinned callers) skips the check.
+        if [[ -n "$PINNED_TORCH_SPYRE_SHA" ]]; then
+          echo "--- torch-spyre pin check ---"
+          ACTUAL_TORCH_SPYRE_SHA="$(git -C "$TORCH_SPYRE_DIR" rev-parse HEAD)"
+          echo "pinned:  $PINNED_TORCH_SPYRE_SHA"
+          echo "checked: $ACTUAL_TORCH_SPYRE_SHA"
+          if [[ "$ACTUAL_TORCH_SPYRE_SHA" != "$PINNED_TORCH_SPYRE_SHA" ]]; then
+            echo "::error::torch-spyre HEAD ($ACTUAL_TORCH_SPYRE_SHA) != pinned SHA ($PINNED_TORCH_SPYRE_SHA) for job ${GITHUB_JOB}"
+            exit 1
+          fi
+          # One row per job in the run summary: which hf-adapters + torch-spyre
+          # commit this job actually ran. Rows append concurrently/unordered
+          # under the header the resolve-torch-spyre job writes -- that's fine.
+          echo "| ${GITHUB_JOB} | ${HF_ADAPTERS_SHA:-<none>} | ${ACTUAL_TORCH_SPYRE_SHA} |" >> "$GITHUB_STEP_SUMMARY"
+        fi
         echo "================================="
diff --git a/.github/workflows/_test_matrix.yaml b/.github/workflows/_test_matrix.yaml
index fc0597c0..5da51032 100644
--- a/.github/workflows/_test_matrix.yaml
+++ b/.github/workflows/_test_matrix.yaml
@@ -176,6 +176,67 @@ jobs:
           RESOLVED="$(scripts/resolve_test_type.sh ${RAW_TEST_TYPE})"
           echo "test_type=${RESOLVED}" >> "${GITHUB_OUTPUT}"
 
+  # ---------------------------------------------------------------------------
+  # Resolve the torch-spyre revision ONCE per run and pin it (issue #449).
+  #
+  # torch-spyre lives as a git checkout baked into each runner POD IMAGE at
+  # /home/senuser/torch-spyre, and every matrix job below lands on a different
+  # pod whose image was built at a different time -- so without this, one run
+  # installs several different torch-spyre commits and its results aren't
+  # comparable. This job resolves origin/main to a single full SHA that every
+  # suite/retry job then checks out (via build-hf-adapters' torch_spyre_sha
+  # input) before `uv add`, so all jobs install the identical commit.
+  #
+  # Must run on a spyre runner that HAS the baked checkout (the image_label
+  # set), not ubuntu-latest: the torch-spyre origin remote lives inside that
+  # checkout's .git/config -- there is no torch-spyre URL anywhere in this repo.
+  # `git fetch origin` moves refs/remotes/origin/main to what origin advertises
+  # NOW; `git rev-parse origin/main` reads that ref, NOT this pod's stale HEAD,
+  # which is what makes the pin immune to per-pod image drift.
+  # ---------------------------------------------------------------------------
+  resolve-torch-spyre:
+    name: Resolve torch-spyre revision
+    if: ${{ !inputs.skip_tests }}
+    runs-on:
+      - x86_64
+      - spyre_pf_x1
+      - linux
+      - ${{ inputs.image_label }}
+    timeout-minutes: 15
+    outputs:
+      torch_spyre_sha: ${{ steps.resolve.outputs.torch_spyre_sha }}
+    steps:
+      - name: Resolve origin/main to a pinned SHA
+        id: resolve
+        working-directory: /home/senuser/torch-spyre
+        run: |
+          source "$HOME/.bashrc"
+          source /etc/profile.d/ibm-aiu-setup.sh
+          git config --global --add safe.directory '*'
+          git fetch --force origin
+          SHA="$(git rev-parse origin/main)"
+          if [[ ! "$SHA" =~ ^[0-9a-f]{40}$ ]]; then
+            echo "resolve-torch-spyre: origin/main did not resolve to a 40-char SHA (got '$SHA')" >&2
+            exit 1
+          fi
+          echo "Resolved torch-spyre origin/main -> ${SHA}"
+          echo "torch_spyre_sha=${SHA}" >> "$GITHUB_OUTPUT"
+
+      - name: Record pinned SHA in the run summary
+        run: |
+          {
+            echo "## Pinned torch-spyre revision"
+            echo ""
+            echo "All jobs in this run install torch-spyre at this exact commit:"
+            echo ""
+            echo '```'
+            echo "${{ steps.resolve.outputs.torch_spyre_sha }}"
+            echo '```'
+            echo ""
+            echo "| job | hf-adapters SHA | torch-spyre SHA |"
+            echo "|---|---|---|"
+          } >> "$GITHUB_STEP_SUMMARY"
+
   pre-commit:
     name: pre-commit
     runs-on: ubuntu-latest
@@ -398,6 +459,10 @@ jobs:
       - name: Run mypy
         run: mypy --config-file=pyproject.toml
 
+# NOTE (issue #449): if re-enabling cpu-tests, add `resolve-torch-spyre` to its
+# `needs:` and pass `torch_spyre_sha:
+# ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}` to build-hf-adapters
+# so it pins the same torch-spyre commit as the rest of the run.
 #  cpu-tests:
 #    name: CPU tests (${{ matrix.test_file }})
 #    runs-on:
@@ -446,7 +511,7 @@ jobs:
 
   spyre-load-tests:
     name: Spyre load (${{ matrix.model_key }})
-    needs: [generate-matrix, resolve-test-type]
+    needs: [generate-matrix, resolve-test-type, resolve-torch-spyre]
     # Suite key: "load". Runs on the "regression"/"trunk" or "unit" coarse
     # tiers (unit = every suite except smoke), or when "load" appears as a
     # space-separated word in test_type (e.g. "smoke load") for fine-grained
@@ -476,10 +541,13 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -487,6 +555,10 @@ jobs:
           verbose: 'true'
           # Echo the exact runner-label selectors this job's runs-on used.
           runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }}
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_load_spyre.py for ${{ matrix.model_key }}
         env:
@@ -517,7 +589,7 @@ jobs:
 
   spyre-smoke-tests:
     name: Spyre smoke (${{ matrix.model_key }})
-    needs: [generate-matrix, resolve-test-type]
+    needs: [generate-matrix, resolve-test-type, resolve-torch-spyre]
     # Suite key: "smoke" (used directly, e.g. TEST_TYPE=smoke, not as a tier
     # name), also the target of the "integration" coarse tier (this is the
     # only job that tier runs). Same allowlist mechanism as spyre-load-tests
@@ -547,10 +619,13 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -558,6 +633,10 @@ jobs:
           verbose: 'true'
           # Echo the exact runner-label selectors this job's runs-on used.
           runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }}
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_e2e_smoke_spyre.py for ${{ matrix.model_key }}
         env:
@@ -588,7 +667,7 @@ jobs:
 
   spyre-token-compare-tests:
     name: Spyre token compare (${{ matrix.model_key }})
-    needs: [generate-matrix, resolve-test-type]
+    needs: [generate-matrix, resolve-test-type, resolve-torch-spyre]
     # Suite key: "token_compare". Part of the "unit" coarse tier.
     if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'integration' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' token_compare ')) }}
     # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-token-compare-tests-retry) gets a shot on a fresh pod.
@@ -614,10 +693,13 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -625,6 +707,10 @@ jobs:
           verbose: 'true'
           # Echo the exact runner-label selectors this job's runs-on used.
           runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }}
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_e2e_token_compare_spyre.py for ${{ matrix.model_key }}
         env:
@@ -655,7 +741,7 @@ jobs:
 
   spyre-embed-compare-tests:
     name: Spyre embed compare (${{ matrix.model_key }})
-    needs: [generate-matrix, resolve-test-type]
+    needs: [generate-matrix, resolve-test-type, resolve-torch-spyre]
     # Suite key: "embed_compare". Part of the "unit" coarse tier.
     if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' embed_compare ')) }}
     # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-embed-compare-tests-retry) gets a shot on a fresh pod.
@@ -681,10 +767,13 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -692,6 +781,10 @@ jobs:
           verbose: 'true'
           # Echo the exact runner-label selectors this job's runs-on used.
           runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }}
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_e2e_embed_compare_spyre.py for ${{ matrix.model_key }}
         env:
@@ -722,7 +815,7 @@ jobs:
 
   spyre-vlm-e2e-tests:
     name: Spyre VLM e2e (${{ matrix.model_key }})
-    needs: [generate-matrix, resolve-test-type]
+    needs: [generate-matrix, resolve-test-type, resolve-torch-spyre]
     # Suite key: "vlm". Part of the "unit" coarse tier.
     if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' vlm ')) }}
     # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-vlm-e2e-tests-retry) gets a shot on a fresh pod.
@@ -749,10 +842,13 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -760,6 +856,10 @@ jobs:
           verbose: 'true'
           # Echo the exact runner-label selectors this job's runs-on used.
           runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }}
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_vlm_e2e_spyre.py for ${{ matrix.model_key }}
         env:
@@ -801,6 +901,11 @@ jobs:
   # previously documented to OOM -- watch that job if re-enabling this one too),
   # and the spyre_pf_x2 labels (actionlint.yaml) + extra_test_flags_multi_spyre
   # (callers) remain in place so re-enabling is just an uncomment.
+  # NOTE (issue #449): when re-enabling, also add `resolve-torch-spyre` to this
+  # job's `needs:` and pass `torch_spyre_sha:
+  # ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}` to build-hf-adapters
+  # (plus pinned_torch_spyre_sha/hf_adapters_sha to gather-runner-info), so it
+  # installs the same pinned torch-spyre commit as every other job.
   # spyre-vlm-e2e-tests-x2:
   #   name: Spyre VLM e2e x2 (${{ matrix.model_key }})
   #   needs: [generate-matrix, resolve-test-type]
@@ -848,7 +953,7 @@ jobs:
 
   spyre-reranker-compare-tests:
     name: Spyre reranker compare (${{ matrix.model_key }})
-    needs: [generate-matrix, resolve-test-type]
+    needs: [generate-matrix, resolve-test-type, resolve-torch-spyre]
     # Suite key: "reranker_compare". Part of the "unit" coarse tier.
     if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' reranker_compare ')) }}
     # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-reranker-compare-tests-retry) gets a shot on a fresh pod.
@@ -873,15 +978,22 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_e2e_reranker_compare_spyre.py for ${{ matrix.model_key }}
         env:
@@ -912,7 +1024,7 @@ jobs:
 
   spyre-masked-lm-compare-tests:
     name: Spyre masked-LM compare (${{ matrix.model_key }})
-    needs: [generate-matrix, resolve-test-type]
+    needs: [generate-matrix, resolve-test-type, resolve-torch-spyre]
     # Suite key: "masked_lm_compare". Part of the "unit" coarse tier.
     if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' masked_lm_compare ')) }}
     # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-masked-lm-compare-tests-retry) gets a shot on a fresh pod.
@@ -937,15 +1049,22 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_e2e_masked_lm_compare_spyre.py for ${{ matrix.model_key }}
         env:
@@ -976,7 +1095,7 @@ jobs:
 
   spyre-question-answering-compare-tests:
     name: Spyre question-answering compare (${{ matrix.model_key }})
-    needs: [generate-matrix, resolve-test-type]
+    needs: [generate-matrix, resolve-test-type, resolve-torch-spyre]
     # Suite key: "question_answering_compare". Part of the "unit" coarse tier.
     if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' question_answering_compare ')) }}
     # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-question-answering-compare-tests-retry) gets a shot on a fresh pod.
@@ -1001,15 +1120,22 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_e2e_question_answering_compare_spyre.py for ${{ matrix.model_key }}
         env:
@@ -1040,7 +1166,7 @@ jobs:
 
   spyre-seq-classification-compare-tests:
     name: Spyre seq-classification compare (${{ matrix.model_key }})
-    needs: [generate-matrix, resolve-test-type]
+    needs: [generate-matrix, resolve-test-type, resolve-torch-spyre]
     # Suite key: "seq_classification_compare". Part of the "unit" coarse tier.
     if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' seq_classification_compare ')) }}
     # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-seq-classification-compare-tests-retry) gets a shot on a fresh pod.
@@ -1065,15 +1191,22 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_e2e_seq_classification_compare_spyre.py for ${{ matrix.model_key }}
         env:
@@ -1104,7 +1237,7 @@ jobs:
 
   spyre-token-classification-compare-tests:
     name: Spyre token-classification compare (${{ matrix.model_key }})
-    needs: [generate-matrix, resolve-test-type]
+    needs: [generate-matrix, resolve-test-type, resolve-torch-spyre]
     # Suite key: "token_classification_compare". Part of the "unit" coarse tier.
     if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' token_classification_compare ')) }}
     # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-token-classification-compare-tests-retry) gets a shot on a fresh pod.
@@ -1129,15 +1262,22 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_e2e_token_classification_compare_spyre.py for ${{ matrix.model_key }}
         env:
@@ -1182,7 +1322,7 @@ jobs:
   # ---------------------------------------------------------------------------
   spyre-edge-cases-tests:
     name: Spyre edge case ${{ matrix.test_file }} (${{ matrix.model_key }})
-    needs: [generate-matrix]
+    needs: [generate-matrix, resolve-torch-spyre]
     if: ${{ !inputs.skip_tests && inputs.edge_cases_only }}
     # Don't let one failed (model, test_file) leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-edge-cases-tests-retry) gets a shot on a fresh pod.
     continue-on-error: true
@@ -1215,15 +1355,22 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run ${{ matrix.test_file }} for ${{ matrix.model_key }}
         env:
@@ -1261,7 +1408,7 @@ jobs:
 
   spyre-model-module-tests:
     name: Spyre model-module tests (${{ matrix.config }})
-    needs: [resolve-test-type, generate-module-config-matrix]
+    needs: [resolve-test-type, generate-module-config-matrix, resolve-torch-spyre]
     # Suite key: "model_module". Part of the "unit" coarse tier.
     if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' model_module ')) }}
     # Don't let one failed config leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-model-module-tests-retry) gets a shot on a fresh pod.
@@ -1288,10 +1435,13 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test oot"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -1299,6 +1449,10 @@ jobs:
           verbose: 'true'
           # Echo the exact runner-label selectors this job's runs-on used.
           runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }}
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run module tests for ${{ matrix.config }}
         env:
@@ -1444,7 +1598,7 @@ jobs:
   # `!cancelled() &&` below is required on every *-retry job, not cosmetic: without a status-check function, GitHub applies an implicit success() gate on top of the condition and can skip the job outright.
   spyre-load-tests-retry:
     name: Spyre load (${{ matrix.model_key }}) (pod-level retry)
-    needs: [collect-failed-suites]
+    needs: [collect-failed-suites, resolve-torch-spyre]
     if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_load == 'true' }}
     runs-on:
       - x86_64
@@ -1464,16 +1618,23 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
           runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }}
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_load_spyre.py for ${{ matrix.model_key }} (pod-level retry)
         env:
@@ -1499,7 +1660,7 @@ jobs:
 
   spyre-smoke-tests-retry:
     name: Spyre smoke (${{ matrix.model_key }}) (pod-level retry)
-    needs: [collect-failed-suites]
+    needs: [collect-failed-suites, resolve-torch-spyre]
     if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_smoke == 'true' }}
     runs-on:
       - x86_64
@@ -1519,16 +1680,23 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
           runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }}
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_e2e_smoke_spyre.py for ${{ matrix.model_key }} (pod-level retry)
         env:
@@ -1553,7 +1721,7 @@ jobs:
 
   spyre-token-compare-tests-retry:
     name: Spyre token compare (${{ matrix.model_key }}) (pod-level retry)
-    needs: [collect-failed-suites]
+    needs: [collect-failed-suites, resolve-torch-spyre]
     if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_token_compare == 'true' }}
     runs-on:
       - x86_64
@@ -1573,16 +1741,23 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
           runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }}
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_e2e_token_compare_spyre.py for ${{ matrix.model_key }} (pod-level retry)
         env:
@@ -1607,7 +1782,7 @@ jobs:
 
   spyre-embed-compare-tests-retry:
     name: Spyre embed compare (${{ matrix.model_key }}) (pod-level retry)
-    needs: [collect-failed-suites]
+    needs: [collect-failed-suites, resolve-torch-spyre]
     if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_embed_compare == 'true' }}
     runs-on:
       - x86_64
@@ -1627,16 +1802,23 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
           runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }}
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_e2e_embed_compare_spyre.py for ${{ matrix.model_key }} (pod-level retry)
         env:
@@ -1661,7 +1843,7 @@ jobs:
 
   spyre-vlm-e2e-tests-retry:
     name: Spyre VLM e2e (${{ matrix.model_key }}) (pod-level retry)
-    needs: [collect-failed-suites]
+    needs: [collect-failed-suites, resolve-torch-spyre]
     if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_vlm == 'true' }}
     runs-on:
       - x86_64
@@ -1681,16 +1863,23 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
           runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }}
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_vlm_e2e_spyre.py for ${{ matrix.model_key }} (pod-level retry)
         env:
@@ -1715,7 +1904,7 @@ jobs:
 
   spyre-reranker-compare-tests-retry:
     name: Spyre reranker compare (${{ matrix.model_key }}) (pod-level retry)
-    needs: [collect-failed-suites]
+    needs: [collect-failed-suites, resolve-torch-spyre]
     if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_reranker == 'true' }}
     runs-on:
       - x86_64
@@ -1735,15 +1924,22 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_e2e_reranker_compare_spyre.py for ${{ matrix.model_key }} (pod-level retry)
         env:
@@ -1768,7 +1964,7 @@ jobs:
 
   spyre-masked-lm-compare-tests-retry:
     name: Spyre masked-LM compare (${{ matrix.model_key }}) (pod-level retry)
-    needs: [collect-failed-suites]
+    needs: [collect-failed-suites, resolve-torch-spyre]
     if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_masked_lm == 'true' }}
     runs-on:
       - x86_64
@@ -1788,15 +1984,22 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_e2e_masked_lm_compare_spyre.py for ${{ matrix.model_key }} (pod-level retry)
         env:
@@ -1821,7 +2024,7 @@ jobs:
 
   spyre-question-answering-compare-tests-retry:
     name: Spyre question-answering compare (${{ matrix.model_key }}) (pod-level retry)
-    needs: [collect-failed-suites]
+    needs: [collect-failed-suites, resolve-torch-spyre]
     if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_qa == 'true' }}
     runs-on:
       - x86_64
@@ -1841,15 +2044,22 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_e2e_question_answering_compare_spyre.py for ${{ matrix.model_key }} (pod-level retry)
         env:
@@ -1874,7 +2084,7 @@ jobs:
 
   spyre-seq-classification-compare-tests-retry:
     name: Spyre seq-classification compare (${{ matrix.model_key }}) (pod-level retry)
-    needs: [collect-failed-suites]
+    needs: [collect-failed-suites, resolve-torch-spyre]
     if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_seq_classification == 'true' }}
     runs-on:
       - x86_64
@@ -1894,15 +2104,22 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_e2e_seq_classification_compare_spyre.py for ${{ matrix.model_key }} (pod-level retry)
         env:
@@ -1927,7 +2144,7 @@ jobs:
 
   spyre-token-classification-compare-tests-retry:
     name: Spyre token-classification compare (${{ matrix.model_key }}) (pod-level retry)
-    needs: [collect-failed-suites]
+    needs: [collect-failed-suites, resolve-torch-spyre]
     if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_token_classification == 'true' }}
     runs-on:
       - x86_64
@@ -1947,15 +2164,22 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_e2e_token_classification_compare_spyre.py for ${{ matrix.model_key }} (pod-level retry)
         env:
@@ -1980,7 +2204,7 @@ jobs:
 
   spyre-edge-cases-tests-retry:
     name: Spyre edge case ${{ matrix.test_file }} (${{ matrix.model_key }}) (pod-level retry)
-    needs: [collect-failed-suites]
+    needs: [collect-failed-suites, resolve-torch-spyre]
     if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_edge_cases == 'true' }}
     runs-on:
       - x86_64
@@ -2000,15 +2224,22 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run ${{ matrix.test_file }} for ${{ matrix.model_key }} (pod-level retry)
         env:
@@ -2040,7 +2271,7 @@ jobs:
 
   spyre-model-module-tests-retry:
     name: Spyre model-module tests (${{ matrix.config }}) (pod-level retry)
-    needs: [collect-failed-suites]
+    needs: [collect-failed-suites, resolve-torch-spyre]
     if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_model_module == 'true' }}
     runs-on:
       - x86_64
@@ -2060,16 +2291,23 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test oot"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
           runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }}
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run module tests for ${{ matrix.config }} (pod-level retry)
         env:
@@ -2158,6 +2396,12 @@ jobs:
       - adapter-coverage
       - check-uv-lock
 #      - cpu-tests
+      # resolve-torch-spyre gates the whole matrix (every suite/retry job needs
+      # it; issue #449). If it fails, those jobs skip, collect-failed-suites finds
+      # no failed descriptors and spyre-tests-result would pass -- a false green
+      # over a matrix that never ran. Listing it here (and in the predicate below)
+      # turns the required check red instead, same guard pattern as generate-matrix.
+      - resolve-torch-spyre
       # Depends on the aggregator, not the individual spyre-*-tests jobs, since those are continue-on-error and always report "success"; spyre-tests-result still transitively waits for the whole matrix via collect-failed-suites and the *-retry jobs.
       - spyre-tests-result
     if: always()
@@ -2175,6 +2419,7 @@ jobs:
                  contains(needs.generate-module-config-matrix.result, 'failure') || contains(needs.generate-module-config-matrix.result, 'cancelled') ||
                  contains(needs.adapter-coverage.result, 'failure') || contains(needs.adapter-coverage.result, 'cancelled') ||
                  contains(needs.check-uv-lock.result, 'failure') || contains(needs.check-uv-lock.result, 'cancelled') ||
+                 contains(needs.resolve-torch-spyre.result, 'failure') || contains(needs.resolve-torch-spyre.result, 'cancelled') ||
                  contains(needs.spyre-tests-result.result, 'failure') || contains(needs.spyre-tests-result.result, 'cancelled') }}; then
             echo "One or more test suites failed."
             exit 1
diff --git a/.github/workflows/push-to-clickhouse.yaml b/.github/workflows/push-to-clickhouse.yaml
index d7e749da..18d31eb9 100644
--- a/.github/workflows/push-to-clickhouse.yaml
+++ b/.github/workflows/push-to-clickhouse.yaml
@@ -303,6 +303,64 @@ jobs:
           path: shards/
           retention-days: 7
 
+  # ---------------------------------------------------------------------------
+  # Resolve the torch-spyre revision ONCE and pin it for the whole scan (#449).
+  #
+  # torch-spyre is a git checkout baked into each runner POD IMAGE at
+  # /home/senuser/torch-spyre, and the weekly scan fans out over dozens of shards
+  # that each land on a different pod -- so without this, one scan mixed several
+  # torch-spyre commits (issue #449's example: 75 jobs across 6 commits) and its
+  # ClickHouse rows weren't comparable. This job resolves origin/main to a single
+  # SHA that every weekly-model-scan-* job checks out (via build-hf-adapters'
+  # torch_spyre_sha input) before `uv add`.
+  #
+  # It CANNOT live in generate-matrix: that job runs on spyre_pf_x0 /
+  # image_spyre_backend with working-directory `.` and has no
+  # /home/senuser/torch-spyre (see the comment on that job). This job runs on the
+  # scan runners' combo (spyre_pf_x1 / image_torch_spyre), where the baked
+  # checkout -- and its origin remote, the only place the torch-spyre URL exists
+  # -- is present. It runs in parallel with generate-matrix and needs no
+  # HF/ClickHouse credentials.
+  # ---------------------------------------------------------------------------
+  resolve-torch-spyre:
+    name: Resolve torch-spyre revision
+    runs-on: [x86_64, spyre_pf_x1, linux, image_torch_spyre]
+    timeout-minutes: 15
+    outputs:
+      torch_spyre_sha: ${{ steps.resolve.outputs.torch_spyre_sha }}
+    steps:
+      - name: Resolve origin/main to a pinned SHA
+        id: resolve
+        working-directory: /home/senuser/torch-spyre
+        run: |
+          source "$HOME/.bashrc"
+          source /etc/profile.d/ibm-aiu-setup.sh
+          git config --global --add safe.directory '*'
+          # `git fetch origin` updates refs/remotes/origin/main to what origin
+          # advertises NOW; `git rev-parse origin/main` reads that ref, NOT this
+          # pod's stale checked-out HEAD -- which is what makes the pin immune to
+          # per-pod image drift.
+          git fetch --force origin
+          SHA="$(git rev-parse origin/main)"
+          if [[ ! "$SHA" =~ ^[0-9a-f]{40}$ ]]; then
+            echo "resolve-torch-spyre: origin/main did not resolve to a 40-char SHA (got '$SHA')" >&2
+            exit 1
+          fi
+          echo "Resolved torch-spyre origin/main -> ${SHA}"
+          echo "torch_spyre_sha=${SHA}" >> "$GITHUB_OUTPUT"
+
+      - name: Record pinned SHA in the run summary
+        run: |
+          {
+            echo "## Pinned torch-spyre revision (weekly scan)"
+            echo ""
+            echo "Every model-scan job installs torch-spyre at this exact commit:"
+            echo ""
+            echo '```'
+            echo "${{ steps.resolve.outputs.torch_spyre_sha }}"
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
   # Split by runner tier so max-parallel caps total cards held at once. The cap
   # depends on who dispatched the run:
   #   - manual workflow_dispatch shares the fleet with PR/daily CI, so it stays
@@ -323,7 +381,7 @@ jobs:
   # This showed up in testing as matrix_x4=[].
   weekly-model-scan-x1:
     name: Weekly model scan x1 (${{ matrix.mode }} shard ${{ matrix.shard_index }})
-    needs: generate-matrix
+    needs: [generate-matrix, resolve-torch-spyre]
     if: needs.generate-matrix.outputs.matrix_x1 != '[]'
     runs-on: [x86_64, spyre_pf_x1, linux, image_torch_spyre]
     timeout-minutes: 4320
@@ -349,6 +407,23 @@ jobs:
       - uses: ./.github/actions/build-hf-adapters
         with:
           uv_groups: "dev spyre test models-ops"
+          # Pin torch-spyre to the SHA resolved once for this scan (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+
+      # Assert this shard's checked-out torch-spyre HEAD == the run's pinned SHA
+      # (issue #449). gather-runner-info runs BEFORE build-hf-adapters in these
+      # jobs, so the pin check rides on this dedicated post-build step instead.
+      - name: Verify torch-spyre pin
+        run: |
+          ACTUAL="$(git -C /home/senuser/torch-spyre rev-parse HEAD)"
+          PINNED="${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}"
+          echo "pinned:  $PINNED"
+          echo "checked: $ACTUAL"
+          if [[ "$ACTUAL" != "$PINNED" ]]; then
+            echo "::error::torch-spyre HEAD ($ACTUAL) != pinned SHA ($PINNED)"
+            exit 1
+          fi
+          echo "| ${{ matrix.mode }} shard ${{ matrix.shard_index }} | ${ACTUAL} |" >> "$GITHUB_STEP_SUMMARY"
 
       - name: Download shard
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
@@ -412,7 +487,7 @@ jobs:
 
   weekly-model-scan-x2:
     name: Weekly model scan x2 (${{ matrix.mode }} shard ${{ matrix.shard_index }})
-    needs: generate-matrix
+    needs: [generate-matrix, resolve-torch-spyre]
     if: needs.generate-matrix.outputs.matrix_x2 != '[]'
     runs-on: [x86_64, spyre_pf_x2, linux, image_torch_spyre]
     timeout-minutes: 4320
@@ -438,6 +513,23 @@ jobs:
       - uses: ./.github/actions/build-hf-adapters
         with:
           uv_groups: "dev spyre test models-ops"
+          # Pin torch-spyre to the SHA resolved once for this scan (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+
+      # Assert this shard's checked-out torch-spyre HEAD == the run's pinned SHA
+      # (issue #449). gather-runner-info runs BEFORE build-hf-adapters in these
+      # jobs, so the pin check rides on this dedicated post-build step instead.
+      - name: Verify torch-spyre pin
+        run: |
+          ACTUAL="$(git -C /home/senuser/torch-spyre rev-parse HEAD)"
+          PINNED="${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}"
+          echo "pinned:  $PINNED"
+          echo "checked: $ACTUAL"
+          if [[ "$ACTUAL" != "$PINNED" ]]; then
+            echo "::error::torch-spyre HEAD ($ACTUAL) != pinned SHA ($PINNED)"
+            exit 1
+          fi
+          echo "| ${{ matrix.mode }} shard ${{ matrix.shard_index }} | ${ACTUAL} |" >> "$GITHUB_STEP_SUMMARY"
 
       - name: Download shard
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
@@ -501,7 +593,7 @@ jobs:
 
   weekly-model-scan-x4:
     name: Weekly model scan x4 (${{ matrix.mode }} shard ${{ matrix.shard_index }})
-    needs: generate-matrix
+    needs: [generate-matrix, resolve-torch-spyre]
     if: needs.generate-matrix.outputs.matrix_x4 != '[]'
     runs-on: [x86_64, spyre_pf_x4, linux, image_torch_spyre]
     timeout-minutes: 4320
@@ -527,6 +619,23 @@ jobs:
       - uses: ./.github/actions/build-hf-adapters
         with:
           uv_groups: "dev spyre test models-ops"
+          # Pin torch-spyre to the SHA resolved once for this scan (issue #449).
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+
+      # Assert this shard's checked-out torch-spyre HEAD == the run's pinned SHA
+      # (issue #449). gather-runner-info runs BEFORE build-hf-adapters in these
+      # jobs, so the pin check rides on this dedicated post-build step instead.
+      - name: Verify torch-spyre pin
+        run: |
+          ACTUAL="$(git -C /home/senuser/torch-spyre rev-parse HEAD)"
+          PINNED="${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}"
+          echo "pinned:  $PINNED"
+          echo "checked: $ACTUAL"
+          if [[ "$ACTUAL" != "$PINNED" ]]; then
+            echo "::error::torch-spyre HEAD ($ACTUAL) != pinned SHA ($PINNED)"
+            exit 1
+          fi
+          echo "| ${{ matrix.mode }} shard ${{ matrix.shard_index }} | ${ACTUAL} |" >> "$GITHUB_STEP_SUMMARY"
 
       - name: Download shard
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0

From e480eaa1ff8fb9afe00f1c5156334a143cd37f92 Mon Sep 17 00:00:00 2001
From: Benjamin Sznajder <benjams@il.ibm.com>
Date: Thu, 3 Sep 2026 16:18:01 +0300
Subject: [PATCH 2/6] fix docstring

Signed-off-by: Benjamin Sznajder <benjams@il.ibm.com>
---
 .github/actions/build-hf-adapters/action.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/build-hf-adapters/action.yaml b/.github/actions/build-hf-adapters/action.yaml
index 39ce4cfa..31841af0 100644
--- a/.github/actions/build-hf-adapters/action.yaml
+++ b/.github/actions/build-hf-adapters/action.yaml
@@ -28,7 +28,7 @@ inputs:
       exactly this commit before `uv add`, so every job in a run installs the
       identical torch-spyre instead of whatever HEAD each pod's image happened
       to bake in. Empty (default) keeps today's behaviour, so callers that omit
-      it (e.g. push-to-clickhouse's scan jobs) are unaffected. Named
+      it are unaffected. Named
       torch_spyre_sha -- NOT torch_spyre_ref (an unrelated, unused dispatch
       input elsewhere) and NOT the Checkout step's INPUT_REF (= inputs.ref, the
       hf-adapters ref) -- so the three never get confused.

From 08401d55791fdc2f6171241e9b1f0d959d316577 Mon Sep 17 00:00:00 2001
From: Benjamin Sznajder <benjams@il.ibm.com>
Date: Tue, 8 Sep 2026 15:25:42 +0300
Subject: [PATCH 3/6] feat(ci): skip torch-spyre pin/install on prebaked
 integration image

The issue #449 pin unconditionally checked out origin/main into the baked
torch-spyre, clobbering the version the integration image was built to
test. Add a prebaked_image opt-out: integration-tests passes it true, which
skips resolve-torch-spyre, the per-job checkout, and the torch-spyre
reinstall entirely, so the run uses the image's baked-in venv as-is.
PR/daily/weekly keep pinning origin/main (default false).

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
Signed-off-by: Benjamin Sznajder <benjams@il.ibm.com>
---
 .github/actions/build-hf-adapters/action.yaml |  67 +++++-
 .github/workflows/_test_matrix.yaml           | 197 ++++++++++++++++--
 .github/workflows/integration-tests.yaml      |   7 +
 3 files changed, 251 insertions(+), 20 deletions(-)

diff --git a/.github/actions/build-hf-adapters/action.yaml b/.github/actions/build-hf-adapters/action.yaml
index 31841af0..3430713f 100644
--- a/.github/actions/build-hf-adapters/action.yaml
+++ b/.github/actions/build-hf-adapters/action.yaml
@@ -32,8 +32,25 @@ inputs:
       torch_spyre_sha -- NOT torch_spyre_ref (an unrelated, unused dispatch
       input elsewhere) and NOT the Checkout step's INPUT_REF (= inputs.ref, the
       hf-adapters ref) -- so the three never get confused.
+      Ignored when prebaked_image is 'true' (that path installs no torch-spyre
+      at all); a caller should never set both.
     required: false
     default: ''
+  prebaked_image:
+    description: >
+      'true' on the integration-tests path (issue #449 review): the runner pod
+      image is purpose-built with torch-spyre already installed in the project
+      venv ($VIRTUAL_ENV) -- e.g. the exact torch-spyre of the PR that triggered
+      this run -- and the suites run against that venv via
+      `uv run --active --no-sync` (see the Makefile). So this action must NOT
+      install, checkout, or re-resolve torch-spyre at all: doing so would either
+      overwrite the image's torch-spyre with a different version (the review's
+      concern) or fail outright, since the lockfile pins a +cpu torch with no
+      ppc64le wheel and any on-device resolve errors. 'false' (default) keeps the
+      pin/unpinned install paths below that DO install torch-spyre from the baked
+      /home/senuser/torch-spyre source.
+    required: false
+    default: 'false'
 outputs:
   target_sha:
     description: The resolved commit SHA that was checked out
@@ -112,6 +129,9 @@ runs:
         # hardcoded, dead `INPUT_REF: ""` here so it is never confused with the
         # Checkout step's INPUT_REF (= inputs.ref, the hf-adapters ref).
         TORCH_SPYRE_SHA: ${{ inputs.torch_spyre_sha }}
+        # 'true' = trust the image's pre-installed torch-spyre; install nothing
+        # for it here (issue #449 review). See the input description above.
+        PREBAKED_IMAGE: ${{ inputs.prebaked_image }}
       run: |
         source "$HOME/.bashrc"
         source /etc/profile.d/ibm-aiu-setup.sh
@@ -146,7 +166,39 @@ runs:
           GROUP_FLAGS="$GROUP_FLAGS --group $group"
         done
 
-        if [[ -n "$TORCH_SPYRE_SHA" ]]; then
+        if [[ "$PREBAKED_IMAGE" == "true" ]]; then
+          # -----------------------------------------------------------------
+          # Prebaked-image mode (integration-tests; issue #449 review).
+          #
+          # FULLY SKIP torch-spyre install. The runner pod image was purpose-
+          # built with the exact torch-spyre under test already installed in the
+          # project venv ($VIRTUAL_ENV), and the suites run against that venv via
+          # `uv run --active --no-sync` (see the Makefile). We therefore do NOT
+          # `git checkout` /home/senuser/torch-spyre and do NOT `uv add` /
+          # `uv lock` / `uv sync` it here -- either would defeat the purpose:
+          #   * `uv add torch-spyre /home/senuser/torch-spyre` would rebuild from
+          #     the baked SOURCE checkout, whose HEAD is NOT guaranteed to equal
+          #     the torch-spyre already installed in the image venv -- exactly the
+          #     "overrides it with a different torch-spyre version" breakage the
+          #     review flagged; and
+          #   * any `uv sync` re-resolves the lockfile, which pins a +cpu torch
+          #     with no ppc64le wheel, so the resolve errors out on device (the
+          #     same reason the Makefile runs tests with --no-sync).
+          #
+          # ASSUMPTION (documented deliberately): the prebaked image's venv
+          # already contains torch-spyre AND every hf-adapters dependency this
+          # run needs. This action does not install hf-adapters' own dep groups
+          # here on the prebaked path; if a future image stops pre-installing a
+          # required group, add an install that targets the active venv WITHOUT
+          # re-resolving or reinstalling torch-spyre (e.g.
+          # `uv sync --active --inexact --no-install-package torch-spyre ...`),
+          # never a plain `uv add torch-spyre <path>` / `uv sync`.
+          echo "Prebaked image: using the torch-spyre already installed in the image venv; skipping all torch-spyre install/checkout."
+          if [[ -n "$TORCH_SPYRE_SHA" ]]; then
+            echo "::warning::prebaked_image=true ignores torch_spyre_sha=${TORCH_SPYRE_SHA}; a caller should not set both."
+          fi
+          git -C /home/senuser/torch-spyre log -1 --oneline || true
+        elif [[ -n "$TORCH_SPYRE_SHA" ]]; then
           # Pinned mode (issue #449): install the exact torch-spyre commit the
           # resolve-torch-spyre job picked, NOT whatever HEAD this pod's image
           # baked in. `git fetch origin <sha>` retrieves that specific object even
@@ -178,5 +230,14 @@ runs:
           uv lock --upgrade-package torch-spyre
           uv sync --frozen --verbose --refresh $GROUP_FLAGS
         fi
-        uv pip show torch
-        uv pip freeze
+        # Post-install diagnostics. On the prebaked path there is no project
+        # .venv (we ran no sync) -- the relevant environment is the image's
+        # active venv, so target it with --active there. Non-fatal (|| true): a
+        # failed probe must never fail the build.
+        if [[ "$PREBAKED_IMAGE" == "true" ]]; then
+          uv pip show --active torch || true
+          uv pip freeze --active || true
+        else
+          uv pip show torch
+          uv pip freeze
+        fi
diff --git a/.github/workflows/_test_matrix.yaml b/.github/workflows/_test_matrix.yaml
index 4868ab6a..1651516d 100644
--- a/.github/workflows/_test_matrix.yaml
+++ b/.github/workflows/_test_matrix.yaml
@@ -119,6 +119,23 @@ on:
         required: false
         type: boolean
         default: false
+      prebaked_image:
+        description: >-
+          When true, run every suite on the torch-spyre that is ALREADY baked
+          into the runner pod image, and skip the resolve-torch-spyre job + the
+          per-job pin/checkout entirely. This is the integration-tests contract:
+          an upstream torch-spyre PR triggers hf-adapters on an ephemeral image
+          built FROM that exact torch-spyre (selected via image_label), so the
+          run MUST test that version -- pinning origin/main here would
+          `git checkout --force` it away and silently test the wrong commit
+          (see issue #449 review). Default false keeps issue #449's behaviour:
+          resolve origin/main once and pin it across every job, for the PR /
+          daily / weekly matrices whose pods bake in whatever HEAD happened to
+          be current when each image was built. Named to match the sibling
+          repos' torch-spyre / spyre-inference integration workflows.
+        required: false
+        type: boolean
+        default: false
 
 permissions:
   # Needed by collect-failed-suites to list/download failed-suite-* artifacts via the REST API; reusable workflows can only narrow the caller's permissions, so callers must grant this too.
@@ -195,10 +212,19 @@ jobs:
   # `git fetch origin` moves refs/remotes/origin/main to what origin advertises
   # NOW; `git rev-parse origin/main` reads that ref, NOT this pod's stale HEAD,
   # which is what makes the pin immune to per-pod image drift.
+  #
+  # SKIPPED when prebaked_image is true (the integration-tests contract): there
+  # the pod image already carries the exact torch-spyre under test, so pinning
+  # origin/main would clobber it. With this job skipped its torch_spyre_sha
+  # output is '', so every suite/retry job passes an empty torch_spyre_sha to
+  # build-hf-adapters (-> its unpinned branch installs the baked-in torch-spyre)
+  # and an empty pinned_torch_spyre_sha to gather-runner-info (-> its pin
+  # assertion no-ops). A skipped job is not a failure, so the spyre-tests-result
+  # gate stays green (it only reds on failure/cancelled).
   # ---------------------------------------------------------------------------
   resolve-torch-spyre:
     name: Resolve torch-spyre revision
-    if: ${{ !inputs.skip_tests }}
+    if: ${{ !inputs.skip_tests && !inputs.prebaked_image }}
     runs-on:
       - x86_64
       - spyre_pf_x1
@@ -519,7 +545,13 @@ jobs:
     # space-separated word in test_type (e.g. "smoke load") for fine-grained
     # selection.
     # See spyre-smoke-tests for the other keys.
-    if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' load ')) }}
+    # The resolve-torch-spyre guard lets this job run when the pin succeeded
+    # (pinned mode) OR was skipped (prebaked_image mode) but NOT when it failed
+    # -- a failed pin must skip the matrix, not silently run on baked-in
+    # torch-spyre (issue #449). Referencing needs.resolve-torch-spyre.result also
+    # lifts the implicit success() gate that would otherwise cascade-skip this
+    # job when resolve-torch-spyre is skipped.
+    if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' load ')) }}
     # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-load-tests-retry) gets a shot on a fresh pod.
     continue-on-error: true
     runs-on:
@@ -549,7 +581,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -597,7 +633,9 @@ jobs:
     # only job that tier runs). Same allowlist mechanism as spyre-load-tests
     # above -- to add a new suite later, give the new job its own key and
     # copy this `if:` line with that key; no existing job needs to change.
-    if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' smoke ')) }}
+    # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked),
+    # not pin-failure (issue #449); see spyre-load-tests for the full rationale.
+    if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' smoke ')) }}
     # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-smoke-tests-retry) gets a shot on a fresh pod.
     continue-on-error: true
     runs-on:
@@ -627,7 +665,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -671,7 +713,9 @@ jobs:
     name: Spyre token compare (${{ matrix.model_key }})
     needs: [generate-matrix, resolve-test-type, resolve-torch-spyre]
     # Suite key: "token_compare". Part of the "unit" coarse tier.
-    if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'integration' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' token_compare ')) }}
+    # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked),
+    # not pin-failure (issue #449); see spyre-load-tests for the full rationale.
+    if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'integration' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' token_compare ')) }}
     # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-token-compare-tests-retry) gets a shot on a fresh pod.
     continue-on-error: true
     runs-on:
@@ -701,7 +745,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -767,6 +815,10 @@ jobs:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # This job does not pin torch-spyre (it runs on the image's baked-in
+          # torch-spyre), but on the integration path it must still fully skip
+          # the torch-spyre install like every other suite (issue #449 review).
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -801,7 +853,9 @@ jobs:
     name: Spyre embed compare (${{ matrix.model_key }})
     needs: [generate-matrix, resolve-test-type, resolve-torch-spyre]
     # Suite key: "embed_compare". Part of the "unit" coarse tier.
-    if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' embed_compare ')) }}
+    # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked),
+    # not pin-failure (issue #449); see spyre-load-tests for the full rationale.
+    if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' embed_compare ')) }}
     # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-embed-compare-tests-retry) gets a shot on a fresh pod.
     continue-on-error: true
     runs-on:
@@ -831,7 +885,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -875,7 +933,9 @@ jobs:
     name: Spyre VLM e2e (${{ matrix.model_key }})
     needs: [generate-matrix, resolve-test-type, resolve-torch-spyre]
     # Suite key: "vlm". Part of the "unit" coarse tier.
-    if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' vlm ')) }}
+    # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked),
+    # not pin-failure (issue #449); see spyre-load-tests for the full rationale.
+    if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' vlm ')) }}
     # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-vlm-e2e-tests-retry) gets a shot on a fresh pod.
     continue-on-error: true
     runs-on:
@@ -906,7 +966,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -1013,7 +1077,9 @@ jobs:
     name: Spyre reranker compare (${{ matrix.model_key }})
     needs: [generate-matrix, resolve-test-type, resolve-torch-spyre]
     # Suite key: "reranker_compare". Part of the "unit" coarse tier.
-    if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' reranker_compare ')) }}
+    # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked),
+    # not pin-failure (issue #449); see spyre-load-tests for the full rationale.
+    if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' reranker_compare ')) }}
     # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-reranker-compare-tests-retry) gets a shot on a fresh pod.
     continue-on-error: true
     runs-on:
@@ -1042,7 +1108,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -1084,7 +1154,9 @@ jobs:
     name: Spyre masked-LM compare (${{ matrix.model_key }})
     needs: [generate-matrix, resolve-test-type, resolve-torch-spyre]
     # Suite key: "masked_lm_compare". Part of the "unit" coarse tier.
-    if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' masked_lm_compare ')) }}
+    # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked),
+    # not pin-failure (issue #449); see spyre-load-tests for the full rationale.
+    if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' masked_lm_compare ')) }}
     # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-masked-lm-compare-tests-retry) gets a shot on a fresh pod.
     continue-on-error: true
     runs-on:
@@ -1113,7 +1185,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -1155,7 +1231,9 @@ jobs:
     name: Spyre question-answering compare (${{ matrix.model_key }})
     needs: [generate-matrix, resolve-test-type, resolve-torch-spyre]
     # Suite key: "question_answering_compare". Part of the "unit" coarse tier.
-    if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' question_answering_compare ')) }}
+    # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked),
+    # not pin-failure (issue #449); see spyre-load-tests for the full rationale.
+    if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' question_answering_compare ')) }}
     # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-question-answering-compare-tests-retry) gets a shot on a fresh pod.
     continue-on-error: true
     runs-on:
@@ -1184,7 +1262,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -1226,7 +1308,9 @@ jobs:
     name: Spyre seq-classification compare (${{ matrix.model_key }})
     needs: [generate-matrix, resolve-test-type, resolve-torch-spyre]
     # Suite key: "seq_classification_compare". Part of the "unit" coarse tier.
-    if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' seq_classification_compare ')) }}
+    # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked),
+    # not pin-failure (issue #449); see spyre-load-tests for the full rationale.
+    if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' seq_classification_compare ')) }}
     # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-seq-classification-compare-tests-retry) gets a shot on a fresh pod.
     continue-on-error: true
     runs-on:
@@ -1255,7 +1339,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -1297,7 +1385,9 @@ jobs:
     name: Spyre token-classification compare (${{ matrix.model_key }})
     needs: [generate-matrix, resolve-test-type, resolve-torch-spyre]
     # Suite key: "token_classification_compare". Part of the "unit" coarse tier.
-    if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' token_classification_compare ')) }}
+    # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked),
+    # not pin-failure (issue #449); see spyre-load-tests for the full rationale.
+    if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' token_classification_compare ')) }}
     # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-token-classification-compare-tests-retry) gets a shot on a fresh pod.
     continue-on-error: true
     runs-on:
@@ -1326,7 +1416,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -1381,7 +1475,9 @@ jobs:
   spyre-edge-cases-tests:
     name: Spyre edge case ${{ matrix.test_file }} (${{ matrix.model_key }})
     needs: [generate-matrix, resolve-torch-spyre]
-    if: ${{ !inputs.skip_tests && inputs.edge_cases_only }}
+    # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked),
+    # not pin-failure (issue #449); see spyre-load-tests for the full rationale.
+    if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && inputs.edge_cases_only }}
     # Don't let one failed (model, test_file) leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-edge-cases-tests-retry) gets a shot on a fresh pod.
     continue-on-error: true
     runs-on:
@@ -1418,7 +1514,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -1467,7 +1567,9 @@ jobs:
     name: Spyre model-module tests (${{ matrix.config }})
     needs: [resolve-test-type, generate-module-config-matrix, resolve-torch-spyre]
     # Suite key: "model_module". Part of the "unit" coarse tier.
-    if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' model_module ')) }}
+    # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked),
+    # not pin-failure (issue #449); see spyre-load-tests for the full rationale.
+    if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' model_module ')) }}
     # Don't let one failed config leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-model-module-tests-retry) gets a shot on a fresh pod.
     continue-on-error: true
     runs-on:
@@ -1498,7 +1600,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -1684,7 +1790,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -1746,7 +1856,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -1807,7 +1921,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -1866,6 +1984,10 @@ jobs:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # This job does not pin torch-spyre (it runs on the image's baked-in
+          # torch-spyre), but on the integration path it must still fully skip
+          # the torch-spyre install like every other suite (issue #449 review).
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -1918,7 +2040,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -1979,7 +2105,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -2040,7 +2170,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -2100,7 +2234,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -2160,7 +2298,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -2220,7 +2362,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -2280,7 +2426,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -2340,7 +2490,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -2407,7 +2561,11 @@ jobs:
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
           # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
           torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
@@ -2509,10 +2667,15 @@ jobs:
       - check-uv-lock
 #      - cpu-tests
       # resolve-torch-spyre gates the whole matrix (every suite/retry job needs
-      # it; issue #449). If it fails, those jobs skip, collect-failed-suites finds
-      # no failed descriptors and spyre-tests-result would pass -- a false green
-      # over a matrix that never ran. Listing it here (and in the predicate below)
-      # turns the required check red instead, same guard pattern as generate-matrix.
+      # it; issue #449). When it FAILS, each suite job's result-guard excludes
+      # 'failure' so they skip, collect-failed-suites finds no failed descriptors
+      # and spyre-tests-result would pass -- a false green over a matrix that
+      # never ran. Listing it here (and in the predicate below) turns the required
+      # check red instead, same guard pattern as generate-matrix. When it is
+      # SKIPPED (prebaked_image=true, the integration path), that is intentional,
+      # not a failure: skipped != failure/cancelled here, so the gate stays green
+      # and the suite jobs (whose guard also admits 'skipped') run on the image's
+      # baked-in torch-spyre.
       - resolve-torch-spyre
       # Depends on the aggregator, not the individual spyre-*-tests jobs, since those are continue-on-error and always report "success"; spyre-tests-result still transitively waits for the whole matrix via collect-failed-suites and the *-retry jobs.
       - spyre-tests-result
diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml
index 5b0ee728..079df26d 100644
--- a/.github/workflows/integration-tests.yaml
+++ b/.github/workflows/integration-tests.yaml
@@ -222,6 +222,13 @@ jobs:
       repository: ${{ inputs.repository || '' }}
       test_type: ${{ inputs.test_type || 'integration' }}
       models: ${{ (inputs.models != '' && inputs.models) || needs.resolve-models.outputs.models || '' }}
+      # Run on the torch-spyre baked into the (possibly ephemeral, per-PR) image
+      # this run was dispatched onto -- do NOT resolve/pin origin/main, which
+      # would `git checkout --force` away the exact torch-spyre version this
+      # integration run exists to test (issue #449 review). Every other caller
+      # (test_pull_request / test_daily / push-to-clickhouse) omits this, keeping
+      # the default false = pin origin/main once across the run.
+      prebaked_image: true
 
   # ----------------------------------------------------------------------------------
   # Fan-in gate job so callers can poll a single stable job name for pass/fail.

From a217e4eaea7583db31d62f5cd296904845c029a5 Mon Sep 17 00:00:00 2001
From: BenjSz <benjams@il.ibm.com>
Date: Tue, 8 Sep 2026 16:23:35 +0300
Subject: [PATCH 4/6] Update pip commands for active environment check

remove useless --active in uv pip freeze | show
---
 .github/actions/build-hf-adapters/action.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/actions/build-hf-adapters/action.yaml b/.github/actions/build-hf-adapters/action.yaml
index 3430713f..d9988510 100644
--- a/.github/actions/build-hf-adapters/action.yaml
+++ b/.github/actions/build-hf-adapters/action.yaml
@@ -235,8 +235,8 @@ runs:
         # active venv, so target it with --active there. Non-fatal (|| true): a
         # failed probe must never fail the build.
         if [[ "$PREBAKED_IMAGE" == "true" ]]; then
-          uv pip show --active torch || true
-          uv pip freeze --active || true
+          uv pip show torch || true
+          uv pip freeze || true
         else
           uv pip show torch
           uv pip freeze

From 129a3f42f8f5e9e0df55ebfc70089c001f3108c3 Mon Sep 17 00:00:00 2001
From: Benjamin Sznajder <benjams@il.ibm.com>
Date: Thu, 10 Sep 2026 13:07:31 +0300
Subject: [PATCH 5/6] feat(ci): pin torch-spyre for model-components suite;
 document retry fail-closed

Address PR #463 review (issue #449):

1. spyre-model-components-tests (+ -retry) was the one suite left unpinned
   on the default path -- it ran on its pod's baked-in torch-spyre HEAD while
   every sibling suite installed the resolved SHA, the exact within-run
   inconsistency #449 removes. The suite exercises torch-spyre (moves tensors
   to Spyre, requires_spyre), so there is no reason to exempt it. Pin it like
   the others: add resolve-torch-spyre to needs, gate on its result, pass
   torch_spyre_sha to build, and add the pinned_torch_spyre_sha/hf_adapters_sha
   pin-verify to gather-runner-info. On the prebaked integration path
   resolve-torch-spyre is skipped, torch_spyre_sha is '' and the verify no-ops,
   matching token_compare/embed_compare exactly.

2. Document why the *-retry jobs use !cancelled() rather than the primary
   jobs' result == guard: fail-closed on a pin failure is emergent (a failed
   pin skips every primary, so nothing runs, so has_failed_* stays false and
   the retry never fires). Full note on spyre-load-tests-retry; short pointer
   on spyre-model-components-tests-retry.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .github/workflows/_test_matrix.yaml | 45 +++++++++++++++++++++++------
 1 file changed, 36 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/_test_matrix.yaml b/.github/workflows/_test_matrix.yaml
index e3791341..bd9e8bb7 100644
--- a/.github/workflows/_test_matrix.yaml
+++ b/.github/workflows/_test_matrix.yaml
@@ -860,9 +860,11 @@ jobs:
 
   spyre-model-components-tests:
     name: Spyre model component tests
-    needs: [resolve-test-type]
+    needs: [resolve-test-type, resolve-torch-spyre]
     # Suite key: "model_components". Part of the "unit" and "integration" coarse tiers.
-    if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'integration' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' model_components ')) }}
+    # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked),
+    # not pin-failure (issue #449); see spyre-load-tests for the full rationale.
+    if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'integration' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' model_components ')) }}
     # Don't let a failed run fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry gets a shot on a fresh pod.
     continue-on-error: true
     runs-on:
@@ -880,13 +882,16 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
-          # This job does not pin torch-spyre (it runs on the image's baked-in
-          # torch-spyre), but on the integration path it must still fully skip
-          # the torch-spyre install like every other suite (issue #449 review).
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
           prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
@@ -894,6 +899,10 @@ jobs:
         with:
           verbose: 'true'
           runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }}
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_model_components_spyre.py
         run: |
@@ -1905,6 +1914,14 @@ jobs:
   spyre-load-tests-retry:
     name: Spyre load (${{ matrix.model_key }}) (pod-level retry)
     needs: [collect-failed-suites, resolve-torch-spyre]
+    # Unlike the primary suites, the retries do not gate on
+    # needs.resolve-torch-spyre.result. They do not need to: fail-closed on a pin
+    # failure is emergent here. If resolve-torch-spyre failed, every primary suite
+    # skipped (their result == guard), so nothing ran, so no suite failed, so
+    # collect-failed-suites reports has_failed_* == 'false' and this retry never
+    # fires. The !cancelled() gate only keeps the retry alive across the expected
+    # failure() of its primary; it never resurrects a run that skipped on pin
+    # failure. All *-retry jobs share this rationale.
     if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_load == 'true' }}
     runs-on:
       - x86_64
@@ -2154,7 +2171,10 @@ jobs:
 
   spyre-model-components-tests-retry:
     name: Spyre model component tests (pod-level retry)
-    needs: [collect-failed-suites]
+    needs: [collect-failed-suites, resolve-torch-spyre]
+    # Fail-closed on pin failure is emergent, not gated here: if resolve-torch-spyre
+    # failed, every primary suite skipped, so nothing ran, so has_failed_model_components
+    # is false and this retry never fires. See spyre-load-tests-retry for the full note.
     if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_model_components == 'true' }}
     runs-on:
       - x86_64
@@ -2174,13 +2194,16 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
-          # This job does not pin torch-spyre (it runs on the image's baked-in
-          # torch-spyre), but on the integration path it must still fully skip
-          # the torch-spyre install like every other suite (issue #449 review).
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
           prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
@@ -2188,6 +2211,10 @@ jobs:
         with:
           verbose: 'true'
           runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }}
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_model_components_spyre.py (pod-level retry)
         run: |

From db0d481dd441aade30bd426b76c3fdabdd9d530c Mon Sep 17 00:00:00 2001
From: Benjamin Sznajder <benjams@il.ibm.com>
Date: Thu, 10 Sep 2026 13:21:59 +0300
Subject: [PATCH 6/6] conitnue the fix on new merged

Signed-off-by: Benjamin Sznajder <benjams@il.ibm.com>
---
 .github/workflows/_test_matrix.yaml | 66 ++++++++++++++++++++++++++---
 1 file changed, 60 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/_test_matrix.yaml b/.github/workflows/_test_matrix.yaml
index bd9e8bb7..ebc72b70 100644
--- a/.github/workflows/_test_matrix.yaml
+++ b/.github/workflows/_test_matrix.yaml
@@ -713,10 +713,12 @@ jobs:
 
   spyre-multicard-smoke-tests:
     name: Spyre multicard smoke x2 (${{ matrix.model_key }})
-    needs: [generate-matrix, resolve-test-type]
+    needs: [generate-matrix, resolve-test-type, resolve-torch-spyre]
     # Suite key: "multicard_smoke". Runs in the unit, regression, and trunk
     # tiers or when selected explicitly.
-    if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && needs.generate-matrix.outputs.multicard_smoke_matrix != '[]' && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' multicard_smoke ')) }}
+    # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked),
+    # not pin-failure (issue #449); see spyre-load-tests for the full rationale.
+    if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && needs.generate-matrix.outputs.multicard_smoke_matrix != '[]' && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' multicard_smoke ')) }}
     # Don't let the initial pod failure fail the workflow outright -- the
     # fresh-pod retry and spyre-tests-result decide the authoritative result.
     continue-on-error: true
@@ -739,16 +741,27 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
           runner_labels: ${{ format('["x86_64","spyre_pf_x2","linux","{0}"]', inputs.image_label) }}
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run multicard smoke for ${{ matrix.model_key }}
         env:
@@ -1090,9 +1103,11 @@ jobs:
 
   spyre-clip-e2e-tests:
     name: Spyre CLIP e2e (${{ matrix.model_key }})
-    needs: [generate-matrix, resolve-test-type]
+    needs: [generate-matrix, resolve-test-type, resolve-torch-spyre]
     # Suite key: "clip". Part of the "unit" coarse tier.
-    if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' clip ')) }}
+    # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked),
+    # not pin-failure (issue #449); see spyre-load-tests for the full rationale.
+    if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' clip ')) }}
     continue-on-error: true
     runs-on:
       - x86_64
@@ -1114,16 +1129,27 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
           runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }}
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_e2e_clip_compare_spyre.py for ${{ matrix.model_key }}
         env:
@@ -2052,7 +2078,10 @@ jobs:
 
   spyre-multicard-smoke-tests-retry:
     name: Spyre multicard smoke x2 (${{ matrix.model_key }}) (pod-level retry)
-    needs: [collect-failed-suites]
+    needs: [collect-failed-suites, resolve-torch-spyre]
+    # Fail-closed on pin failure is emergent, not gated here: if resolve-torch-spyre
+    # failed, every primary suite skipped, so nothing ran, so has_failed_multicard_smoke
+    # is false and this retry never fires. See spyre-load-tests-retry for the full note.
     if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_multicard_smoke == 'true' }}
     runs-on:
       - x86_64
@@ -2072,16 +2101,27 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
           runner_labels: ${{ format('["x86_64","spyre_pf_x2","linux","{0}"]', inputs.image_label) }}
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run multicard smoke for ${{ matrix.model_key }} (pod-level retry)
         env:
@@ -2365,7 +2405,10 @@ jobs:
 
   spyre-clip-e2e-tests-retry:
     name: Spyre CLIP e2e (${{ matrix.model_key }}) (pod-level retry)
-    needs: [collect-failed-suites]
+    needs: [collect-failed-suites, resolve-torch-spyre]
+    # Fail-closed on pin failure is emergent, not gated here: if resolve-torch-spyre
+    # failed, every primary suite skipped, so nothing ran, so has_failed_clip
+    # is false and this retry never fires. See spyre-load-tests-retry for the full note.
     if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_clip == 'true' }}
     runs-on:
       - x86_64
@@ -2385,16 +2428,27 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - uses: ./.github/actions/build-hf-adapters
+        id: build
         with:
           uv_groups: "dev spyre test"
           ref: ${{ inputs.ref }}
           repository: ${{ inputs.repository }}
+          # Pin torch-spyre to the SHA resolved once for this run (issue #449).
+          # On the integration path (prebaked_image=true) this is empty and
+          # ignored: build-hf-adapters installs no torch-spyre and uses the one
+          # baked into the image venv instead.
+          torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          prebaked_image: ${{ inputs.prebaked_image }}
 
       - name: Gather runner info
         uses: ./.github/actions/gather-runner-info
         with:
           verbose: 'true'
           runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }}
+          # Assert this job's torch-spyre HEAD == the run's pinned SHA and report
+          # both SHAs in the run summary (issue #449).
+          pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}
+          hf_adapters_sha: ${{ steps.build.outputs.target_sha }}
 
       - name: Run test_e2e_clip_compare_spyre.py for ${{ matrix.model_key }} (pod-level retry)
         env: