diff --git a/.github/actions/build-hf-adapters/action.yaml b/.github/actions/build-hf-adapters/action.yaml index 5b03eddb..d9988510 100644 --- a/.github/actions/build-hf-adapters/action.yaml +++ b/.github/actions/build-hf-adapters/action.yaml @@ -21,6 +21,36 @@ inputs: ($GITHUB_SERVER_URL/$GITHUB_REPOSITORY). Set by the Test-With: flow. required: false default: '' + torch_spyre_sha: + description: > + Full 40-char torch-spyre commit SHA to pin (issue #449). When set, the + baked /home/senuser/torch-spyre checkout is fetched + checked out at + exactly this commit before `uv add`, so every job in a run installs the + identical torch-spyre instead of whatever HEAD each pod's image happened + to bake in. Empty (default) keeps today's behaviour, so callers that omit + it are unaffected. Named + torch_spyre_sha -- NOT torch_spyre_ref (an unrelated, unused dispatch + input elsewhere) and NOT the Checkout step's INPUT_REF (= inputs.ref, the + hf-adapters ref) -- so the three never get confused. + Ignored when prebaked_image is 'true' (that path installs no torch-spyre + at all); a caller should never set both. + required: false + default: '' + prebaked_image: + description: > + 'true' on the integration-tests path (issue #449 review): the runner pod + image is purpose-built with torch-spyre already installed in the project + venv ($VIRTUAL_ENV) -- e.g. the exact torch-spyre of the PR that triggered + this run -- and the suites run against that venv via + `uv run --active --no-sync` (see the Makefile). So this action must NOT + install, checkout, or re-resolve torch-spyre at all: doing so would either + overwrite the image's torch-spyre with a different version (the review's + concern) or fail outright, since the lockfile pins a +cpu torch with no + ppc64le wheel and any on-device resolve errors. 'false' (default) keeps the + pin/unpinned install paths below that DO install torch-spyre from the baked + /home/senuser/torch-spyre source. + required: false + default: 'false' outputs: target_sha: description: The resolved commit SHA that was checked out @@ -94,7 +124,14 @@ runs: working-directory: /home/senuser env: UV_GROUPS: "${{ inputs.uv_groups }}" - INPUT_REF: "" + # Pinned torch-spyre SHA (issue #449). Empty = today's behaviour (install + # whatever HEAD this pod's image baked in). Renamed from the previously + # hardcoded, dead `INPUT_REF: ""` here so it is never confused with the + # Checkout step's INPUT_REF (= inputs.ref, the hf-adapters ref). + TORCH_SPYRE_SHA: ${{ inputs.torch_spyre_sha }} + # 'true' = trust the image's pre-installed torch-spyre; install nothing + # for it here (issue #449 review). See the input description above. + PREBAKED_IMAGE: ${{ inputs.prebaked_image }} run: | source "$HOME/.bashrc" source /etc/profile.d/ibm-aiu-setup.sh @@ -129,21 +166,60 @@ runs: GROUP_FLAGS="$GROUP_FLAGS --group $group" done - if [[ -n "$INPUT_REF" ]]; then + if [[ "$PREBAKED_IMAGE" == "true" ]]; then + # ----------------------------------------------------------------- + # Prebaked-image mode (integration-tests; issue #449 review). + # + # FULLY SKIP torch-spyre install. The runner pod image was purpose- + # built with the exact torch-spyre under test already installed in the + # project venv ($VIRTUAL_ENV), and the suites run against that venv via + # `uv run --active --no-sync` (see the Makefile). We therefore do NOT + # `git checkout` /home/senuser/torch-spyre and do NOT `uv add` / + # `uv lock` / `uv sync` it here -- either would defeat the purpose: + # * `uv add torch-spyre /home/senuser/torch-spyre` would rebuild from + # the baked SOURCE checkout, whose HEAD is NOT guaranteed to equal + # the torch-spyre already installed in the image venv -- exactly the + # "overrides it with a different torch-spyre version" breakage the + # review flagged; and + # * any `uv sync` re-resolves the lockfile, which pins a +cpu torch + # with no ppc64le wheel, so the resolve errors out on device (the + # same reason the Makefile runs tests with --no-sync). + # + # ASSUMPTION (documented deliberately): the prebaked image's venv + # already contains torch-spyre AND every hf-adapters dependency this + # run needs. This action does not install hf-adapters' own dep groups + # here on the prebaked path; if a future image stops pre-installing a + # required group, add an install that targets the active venv WITHOUT + # re-resolving or reinstalling torch-spyre (e.g. + # `uv sync --active --inexact --no-install-package torch-spyre ...`), + # never a plain `uv add torch-spyre ` / `uv sync`. + echo "Prebaked image: using the torch-spyre already installed in the image venv; skipping all torch-spyre install/checkout." + if [[ -n "$TORCH_SPYRE_SHA" ]]; then + echo "::warning::prebaked_image=true ignores torch_spyre_sha=${TORCH_SPYRE_SHA}; a caller should not set both." + fi + git -C /home/senuser/torch-spyre log -1 --oneline || true + elif [[ -n "$TORCH_SPYRE_SHA" ]]; then + # Pinned mode (issue #449): install the exact torch-spyre commit the + # resolve-torch-spyre job picked, NOT whatever HEAD this pod's image + # baked in. `git fetch origin ` retrieves that specific object even + # when this pod's checkout predates it (with a full-fetch fallback); + # then check it out and re-sync submodules so the working tree matches + # the pinned commit exactly before `uv add` builds from it. cd /home/senuser/torch-spyre - TARGET_SHA="$INPUT_REF" - TARGET_REF="$INPUT_REF" - echo "TARGET_REF=${TARGET_REF}" - echo "TARGET_SHA=${TARGET_SHA}" - git fetch --tags --force origin "+${TARGET_SHA}" || \ + echo "Pinning torch-spyre to ${TORCH_SPYRE_SHA}" + git fetch --tags --force origin "${TORCH_SPYRE_SHA}" || \ git fetch --tags --force origin - git checkout --force "${TARGET_SHA}" + git checkout --force "${TORCH_SPYRE_SHA}" git submodule sync --recursive git submodule update --init --recursive --force git log -1 --oneline cd "$CLONED_HF_ADAPTERS_DIR" + # Same install tail as the unpinned branch below -- the ONLY difference + # is the fetch/checkout preamble above, so a pinned run behaves exactly + # like today's install, just at a fixed SHA. uv add torch-spyre /home/senuser/torch-spyre - uv sync --verbose --refresh $GROUP_FLAGS + uv lock --upgrade-package torch-spyre + uv sync --frozen --verbose --refresh $GROUP_FLAGS else cd "$CLONED_HF_ADAPTERS_DIR" # though we pin to main @@ -154,5 +230,14 @@ runs: uv lock --upgrade-package torch-spyre uv sync --frozen --verbose --refresh $GROUP_FLAGS fi - uv pip show torch - uv pip freeze + # Post-install diagnostics. On the prebaked path there is no project + # .venv (we ran no sync) -- the relevant environment is the image's + # active venv, so target it with --active there. Non-fatal (|| true): a + # failed probe must never fail the build. + if [[ "$PREBAKED_IMAGE" == "true" ]]; then + uv pip show torch || true + uv pip freeze || true + else + uv pip show torch + uv pip freeze + fi diff --git a/.github/actions/gather-runner-info/action.yml b/.github/actions/gather-runner-info/action.yml index d5da05a4..96873e42 100644 --- a/.github/actions/gather-runner-info/action.yml +++ b/.github/actions/gather-runner-info/action.yml @@ -25,6 +25,21 @@ inputs: printed. required: false default: '' + pinned_torch_spyre_sha: + description: | + The torch-spyre commit SHA this run pinned (from the resolve-torch-spyre + job; issue #449). When set, this action asserts that the checked-out + torch-spyre HEAD equals it and fails the job on mismatch -- the per-shard + proof that the pin actually took. Empty = unpinned (no assertion). + required: false + default: '' + hf_adapters_sha: + description: | + The hf-adapters commit SHA build-hf-adapters resolved for this job (its + `target_sha` output). Reported alongside the pinned torch-spyre SHA in the + run summary for reproducibility. Optional. + required: false + default: '' runs: using: composite @@ -43,6 +58,9 @@ runs: RUNNER_NAME: ${{ runner.name }} RUNNER_OS: ${{ runner.os }} RUNNER_ARCH: ${{ runner.arch }} + # Pin reporting/assertion (issue #449). Empty = unpinned, no assertion. + PINNED_TORCH_SPYRE_SHA: ${{ inputs.pinned_torch_spyre_sha }} + HF_ADAPTERS_SHA: ${{ inputs.hf_adapters_sha }} run: | source "$HOME/.bashrc" source /etc/profile.d/ibm-aiu-setup.sh @@ -87,4 +105,24 @@ runs: git -C "$TORCH_SPYRE_DIR" log -n 1 --oneline || true fi fi + # torch-spyre pin check (issue #449). Runs regardless of VERBOSE -- it is + # a correctness gate, not diagnostics: when a pin is in effect, this job's + # checked-out torch-spyre MUST match the SHA the run resolved once up + # front. A mismatch means some job installed a different commit (the exact + # inconsistency #449 fixes), so fail loudly rather than report a false + # green. Empty PINNED_TORCH_SPYRE_SHA (unpinned callers) skips the check. + if [[ -n "$PINNED_TORCH_SPYRE_SHA" ]]; then + echo "--- torch-spyre pin check ---" + ACTUAL_TORCH_SPYRE_SHA="$(git -C "$TORCH_SPYRE_DIR" rev-parse HEAD)" + echo "pinned: $PINNED_TORCH_SPYRE_SHA" + echo "checked: $ACTUAL_TORCH_SPYRE_SHA" + if [[ "$ACTUAL_TORCH_SPYRE_SHA" != "$PINNED_TORCH_SPYRE_SHA" ]]; then + echo "::error::torch-spyre HEAD ($ACTUAL_TORCH_SPYRE_SHA) != pinned SHA ($PINNED_TORCH_SPYRE_SHA) for job ${GITHUB_JOB}" + exit 1 + fi + # One row per job in the run summary: which hf-adapters + torch-spyre + # commit this job actually ran. Rows append concurrently/unordered + # under the header the resolve-torch-spyre job writes -- that's fine. + echo "| ${GITHUB_JOB} | ${HF_ADAPTERS_SHA:-} | ${ACTUAL_TORCH_SPYRE_SHA} |" >> "$GITHUB_STEP_SUMMARY" + fi echo "=================================" diff --git a/.github/workflows/_test_matrix.yaml b/.github/workflows/_test_matrix.yaml index fe5cfedc..ebc72b70 100644 --- a/.github/workflows/_test_matrix.yaml +++ b/.github/workflows/_test_matrix.yaml @@ -119,6 +119,23 @@ on: required: false type: boolean default: false + prebaked_image: + description: >- + When true, run every suite on the torch-spyre that is ALREADY baked + into the runner pod image, and skip the resolve-torch-spyre job + the + per-job pin/checkout entirely. This is the integration-tests contract: + an upstream torch-spyre PR triggers hf-adapters on an ephemeral image + built FROM that exact torch-spyre (selected via image_label), so the + run MUST test that version -- pinning origin/main here would + `git checkout --force` it away and silently test the wrong commit + (see issue #449 review). Default false keeps issue #449's behaviour: + resolve origin/main once and pin it across every job, for the PR / + daily / weekly matrices whose pods bake in whatever HEAD happened to + be current when each image was built. Named to match the sibling + repos' torch-spyre / spyre-inference integration workflows. + required: false + type: boolean + default: false permissions: # Needed by collect-failed-suites to list/download failed-suite-* artifacts via the REST API; reusable workflows can only narrow the caller's permissions, so callers must grant this too. @@ -178,6 +195,76 @@ jobs: RESOLVED="$(scripts/resolve_test_type.sh ${RAW_TEST_TYPE})" echo "test_type=${RESOLVED}" >> "${GITHUB_OUTPUT}" + # --------------------------------------------------------------------------- + # Resolve the torch-spyre revision ONCE per run and pin it (issue #449). + # + # torch-spyre lives as a git checkout baked into each runner POD IMAGE at + # /home/senuser/torch-spyre, and every matrix job below lands on a different + # pod whose image was built at a different time -- so without this, one run + # installs several different torch-spyre commits and its results aren't + # comparable. This job resolves origin/main to a single full SHA that every + # suite/retry job then checks out (via build-hf-adapters' torch_spyre_sha + # input) before `uv add`, so all jobs install the identical commit. + # + # Must run on a spyre runner that HAS the baked checkout (the image_label + # set), not ubuntu-latest: the torch-spyre origin remote lives inside that + # checkout's .git/config -- there is no torch-spyre URL anywhere in this repo. + # `git fetch origin` moves refs/remotes/origin/main to what origin advertises + # NOW; `git rev-parse origin/main` reads that ref, NOT this pod's stale HEAD, + # which is what makes the pin immune to per-pod image drift. + # + # SKIPPED when prebaked_image is true (the integration-tests contract): there + # the pod image already carries the exact torch-spyre under test, so pinning + # origin/main would clobber it. With this job skipped its torch_spyre_sha + # output is '', so every suite/retry job passes an empty torch_spyre_sha to + # build-hf-adapters (-> its unpinned branch installs the baked-in torch-spyre) + # and an empty pinned_torch_spyre_sha to gather-runner-info (-> its pin + # assertion no-ops). A skipped job is not a failure, so the spyre-tests-result + # gate stays green (it only reds on failure/cancelled). + # --------------------------------------------------------------------------- + resolve-torch-spyre: + name: Resolve torch-spyre revision + if: ${{ !inputs.skip_tests && !inputs.prebaked_image }} + runs-on: + - x86_64 + - spyre_pf_x1 + - linux + - ${{ inputs.image_label }} + timeout-minutes: 15 + outputs: + torch_spyre_sha: ${{ steps.resolve.outputs.torch_spyre_sha }} + steps: + - name: Resolve origin/main to a pinned SHA + id: resolve + working-directory: /home/senuser/torch-spyre + run: | + source "$HOME/.bashrc" + source /etc/profile.d/ibm-aiu-setup.sh + git config --global --add safe.directory '*' + git fetch --force origin + SHA="$(git rev-parse origin/main)" + if [[ ! "$SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "resolve-torch-spyre: origin/main did not resolve to a 40-char SHA (got '$SHA')" >&2 + exit 1 + fi + echo "Resolved torch-spyre origin/main -> ${SHA}" + echo "torch_spyre_sha=${SHA}" >> "$GITHUB_OUTPUT" + + - name: Record pinned SHA in the run summary + run: | + { + echo "## Pinned torch-spyre revision" + echo "" + echo "All jobs in this run install torch-spyre at this exact commit:" + echo "" + echo '```' + echo "${{ steps.resolve.outputs.torch_spyre_sha }}" + echo '```' + echo "" + echo "| job | hf-adapters SHA | torch-spyre SHA |" + echo "|---|---|---|" + } >> "$GITHUB_STEP_SUMMARY" + pre-commit: name: pre-commit runs-on: ubuntu-latest @@ -402,6 +489,10 @@ jobs: - name: Run mypy run: mypy --config-file=pyproject.toml +# NOTE (issue #449): if re-enabling cpu-tests, add `resolve-torch-spyre` to its +# `needs:` and pass `torch_spyre_sha: +# ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}` to build-hf-adapters +# so it pins the same torch-spyre commit as the rest of the run. # cpu-tests: # name: CPU tests (${{ matrix.test_file }}) # runs-on: @@ -450,13 +541,19 @@ jobs: spyre-load-tests: name: Spyre load (${{ matrix.model_key }}) - needs: [generate-matrix, resolve-test-type] + needs: [generate-matrix, resolve-test-type, resolve-torch-spyre] # Suite key: "load". Runs on the "regression"/"trunk" or "unit" coarse # tiers (unit = every suite except smoke), or when "load" appears as a # space-separated word in test_type (e.g. "smoke load") for fine-grained # selection. # See spyre-smoke-tests for the other keys. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' load ')) }} + # The resolve-torch-spyre guard lets this job run when the pin succeeded + # (pinned mode) OR was skipped (prebaked_image mode) but NOT when it failed + # -- a failed pin must skip the matrix, not silently run on baked-in + # torch-spyre (issue #449). Referencing needs.resolve-torch-spyre.result also + # lifts the implicit success() gate that would otherwise cascade-skip this + # job when resolve-torch-spyre is skipped. + if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' load ')) }} # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-load-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -480,10 +577,17 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info @@ -491,6 +595,10 @@ jobs: verbose: 'true' # Echo the exact runner-label selectors this job's runs-on used. runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }} + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_load_spyre.py for ${{ matrix.model_key }} env: @@ -521,13 +629,15 @@ jobs: spyre-smoke-tests: name: Spyre smoke (${{ matrix.model_key }}) - needs: [generate-matrix, resolve-test-type] + needs: [generate-matrix, resolve-test-type, resolve-torch-spyre] # Suite key: "smoke" (used directly, e.g. TEST_TYPE=smoke, not as a tier # name), also the target of the "integration" coarse tier (this is the # only job that tier runs). Same allowlist mechanism as spyre-load-tests # above -- to add a new suite later, give the new job its own key and # copy this `if:` line with that key; no existing job needs to change. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' smoke ')) }} + # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked), + # not pin-failure (issue #449); see spyre-load-tests for the full rationale. + if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' smoke ')) }} # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-smoke-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -551,10 +661,17 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info @@ -562,6 +679,10 @@ jobs: verbose: 'true' # Echo the exact runner-label selectors this job's runs-on used. runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }} + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_e2e_smoke_spyre.py for ${{ matrix.model_key }} env: @@ -592,10 +713,12 @@ jobs: spyre-multicard-smoke-tests: name: Spyre multicard smoke x2 (${{ matrix.model_key }}) - needs: [generate-matrix, resolve-test-type] + needs: [generate-matrix, resolve-test-type, resolve-torch-spyre] # Suite key: "multicard_smoke". Runs in the unit, regression, and trunk # tiers or when selected explicitly. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && needs.generate-matrix.outputs.multicard_smoke_matrix != '[]' && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' multicard_smoke ')) }} + # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked), + # not pin-failure (issue #449); see spyre-load-tests for the full rationale. + if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && needs.generate-matrix.outputs.multicard_smoke_matrix != '[]' && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' multicard_smoke ')) }} # Don't let the initial pod failure fail the workflow outright -- the # fresh-pod retry and spyre-tests-result decide the authoritative result. continue-on-error: true @@ -618,16 +741,27 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' runner_labels: ${{ format('["x86_64","spyre_pf_x2","linux","{0}"]', inputs.image_label) }} + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run multicard smoke for ${{ matrix.model_key }} env: @@ -659,9 +793,11 @@ jobs: spyre-token-compare-tests: name: Spyre token compare (${{ matrix.model_key }}) - needs: [generate-matrix, resolve-test-type] + needs: [generate-matrix, resolve-test-type, resolve-torch-spyre] # Suite key: "token_compare". Part of the "unit" coarse tier. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'integration' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' token_compare ')) }} + # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked), + # not pin-failure (issue #449); see spyre-load-tests for the full rationale. + if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'integration' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' token_compare ')) }} # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-token-compare-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -685,10 +821,17 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info @@ -696,6 +839,10 @@ jobs: verbose: 'true' # Echo the exact runner-label selectors this job's runs-on used. runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }} + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_e2e_token_compare_spyre.py for ${{ matrix.model_key }} env: @@ -726,9 +873,11 @@ jobs: spyre-model-components-tests: name: Spyre model component tests - needs: [resolve-test-type] + needs: [resolve-test-type, resolve-torch-spyre] # Suite key: "model_components". Part of the "unit" and "integration" coarse tiers. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'integration' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' model_components ')) }} + # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked), + # not pin-failure (issue #449); see spyre-load-tests for the full rationale. + if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'integration' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' model_components ')) }} # Don't let a failed run fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -746,16 +895,27 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }} + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_model_components_spyre.py run: | @@ -782,9 +942,11 @@ jobs: spyre-embed-compare-tests: name: Spyre embed compare (${{ matrix.model_key }}) - needs: [generate-matrix, resolve-test-type] + needs: [generate-matrix, resolve-test-type, resolve-torch-spyre] # Suite key: "embed_compare". Part of the "unit" coarse tier. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' embed_compare ')) }} + # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked), + # not pin-failure (issue #449); see spyre-load-tests for the full rationale. + if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' embed_compare ')) }} # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-embed-compare-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -808,10 +970,17 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info @@ -819,6 +988,10 @@ jobs: verbose: 'true' # Echo the exact runner-label selectors this job's runs-on used. runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }} + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_e2e_embed_compare_spyre.py for ${{ matrix.model_key }} env: @@ -849,9 +1022,11 @@ jobs: spyre-vlm-e2e-tests: name: Spyre VLM e2e (${{ matrix.model_key }}) - needs: [generate-matrix, resolve-test-type] + needs: [generate-matrix, resolve-test-type, resolve-torch-spyre] # Suite key: "vlm". Part of the "unit" coarse tier. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' vlm ')) }} + # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked), + # not pin-failure (issue #449); see spyre-load-tests for the full rationale. + if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' vlm ')) }} # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-vlm-e2e-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -876,10 +1051,17 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info @@ -887,6 +1069,10 @@ jobs: verbose: 'true' # Echo the exact runner-label selectors this job's runs-on used. runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }} + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_vlm_e2e_spyre.py for ${{ matrix.model_key }} env: @@ -917,9 +1103,11 @@ jobs: spyre-clip-e2e-tests: name: Spyre CLIP e2e (${{ matrix.model_key }}) - needs: [generate-matrix, resolve-test-type] + needs: [generate-matrix, resolve-test-type, resolve-torch-spyre] # Suite key: "clip". Part of the "unit" coarse tier. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' clip ')) }} + # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked), + # not pin-failure (issue #449); see spyre-load-tests for the full rationale. + if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' clip ')) }} continue-on-error: true runs-on: - x86_64 @@ -941,16 +1129,27 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }} + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_e2e_clip_compare_spyre.py for ${{ matrix.model_key }} env: @@ -992,6 +1191,11 @@ jobs: # previously documented to OOM -- watch that job if re-enabling this one too), # and the spyre_pf_x2 labels (actionlint.yaml) + extra_test_flags_multi_spyre # (callers) remain in place so re-enabling is just an uncomment. + # NOTE (issue #449): when re-enabling, also add `resolve-torch-spyre` to this + # job's `needs:` and pass `torch_spyre_sha: + # ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}` to build-hf-adapters + # (plus pinned_torch_spyre_sha/hf_adapters_sha to gather-runner-info), so it + # installs the same pinned torch-spyre commit as every other job. # spyre-vlm-e2e-tests-x2: # name: Spyre VLM e2e x2 (${{ matrix.model_key }}) # needs: [generate-matrix, resolve-test-type] @@ -1039,9 +1243,11 @@ jobs: spyre-reranker-compare-tests: name: Spyre reranker compare (${{ matrix.model_key }}) - needs: [generate-matrix, resolve-test-type] + needs: [generate-matrix, resolve-test-type, resolve-torch-spyre] # Suite key: "reranker_compare". Part of the "unit" coarse tier. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' reranker_compare ')) }} + # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked), + # not pin-failure (issue #449); see spyre-load-tests for the full rationale. + if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' reranker_compare ')) }} # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-reranker-compare-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -1064,15 +1270,26 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_e2e_reranker_compare_spyre.py for ${{ matrix.model_key }} env: @@ -1103,9 +1320,11 @@ jobs: spyre-masked-lm-compare-tests: name: Spyre masked-LM compare (${{ matrix.model_key }}) - needs: [generate-matrix, resolve-test-type] + needs: [generate-matrix, resolve-test-type, resolve-torch-spyre] # Suite key: "masked_lm_compare". Part of the "unit" coarse tier. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' masked_lm_compare ')) }} + # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked), + # not pin-failure (issue #449); see spyre-load-tests for the full rationale. + if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' masked_lm_compare ')) }} # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-masked-lm-compare-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -1128,15 +1347,26 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_e2e_masked_lm_compare_spyre.py for ${{ matrix.model_key }} env: @@ -1167,9 +1397,11 @@ jobs: spyre-question-answering-compare-tests: name: Spyre question-answering compare (${{ matrix.model_key }}) - needs: [generate-matrix, resolve-test-type] + needs: [generate-matrix, resolve-test-type, resolve-torch-spyre] # Suite key: "question_answering_compare". Part of the "unit" coarse tier. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' question_answering_compare ')) }} + # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked), + # not pin-failure (issue #449); see spyre-load-tests for the full rationale. + if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' question_answering_compare ')) }} # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-question-answering-compare-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -1192,15 +1424,26 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_e2e_question_answering_compare_spyre.py for ${{ matrix.model_key }} env: @@ -1231,9 +1474,11 @@ jobs: spyre-seq-classification-compare-tests: name: Spyre seq-classification compare (${{ matrix.model_key }}) - needs: [generate-matrix, resolve-test-type] + needs: [generate-matrix, resolve-test-type, resolve-torch-spyre] # Suite key: "seq_classification_compare". Part of the "unit" coarse tier. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' seq_classification_compare ')) }} + # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked), + # not pin-failure (issue #449); see spyre-load-tests for the full rationale. + if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' seq_classification_compare ')) }} # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-seq-classification-compare-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -1256,15 +1501,26 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_e2e_seq_classification_compare_spyre.py for ${{ matrix.model_key }} env: @@ -1295,9 +1551,11 @@ jobs: spyre-token-classification-compare-tests: name: Spyre token-classification compare (${{ matrix.model_key }}) - needs: [generate-matrix, resolve-test-type] + needs: [generate-matrix, resolve-test-type, resolve-torch-spyre] # Suite key: "token_classification_compare". Part of the "unit" coarse tier. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' token_classification_compare ')) }} + # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked), + # not pin-failure (issue #449); see spyre-load-tests for the full rationale. + if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' token_classification_compare ')) }} # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-token-classification-compare-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -1320,15 +1578,26 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_e2e_token_classification_compare_spyre.py for ${{ matrix.model_key }} env: @@ -1373,8 +1642,10 @@ jobs: # --------------------------------------------------------------------------- spyre-edge-cases-tests: name: Spyre edge case ${{ matrix.test_file }} (${{ matrix.model_key }}) - needs: [generate-matrix] - if: ${{ !inputs.skip_tests && inputs.edge_cases_only }} + needs: [generate-matrix, resolve-torch-spyre] + # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked), + # not pin-failure (issue #449); see spyre-load-tests for the full rationale. + if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && inputs.edge_cases_only }} # Don't let one failed (model, test_file) leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-edge-cases-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -1405,15 +1676,26 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run ${{ matrix.test_file }} for ${{ matrix.model_key }} env: @@ -1451,9 +1733,11 @@ jobs: spyre-model-module-tests: name: Spyre model-module tests (${{ matrix.config }}) - needs: [resolve-test-type, generate-module-config-matrix] + needs: [resolve-test-type, generate-module-config-matrix, resolve-torch-spyre] # Suite key: "model_module". Part of the "unit" coarse tier. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' model_module ')) }} + # resolve-torch-spyre guard: run on pin-success OR pin-skipped (prebaked), + # not pin-failure (issue #449); see spyre-load-tests for the full rationale. + if: ${{ (needs.resolve-torch-spyre.result == 'success' || needs.resolve-torch-spyre.result == 'skipped') && !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' model_module ')) }} # Don't let one failed config leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-model-module-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -1478,10 +1762,17 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test oot" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info @@ -1489,6 +1780,10 @@ jobs: verbose: 'true' # Echo the exact runner-label selectors this job's runs-on used. runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }} + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run module tests for ${{ matrix.config }} env: @@ -1644,7 +1939,15 @@ jobs: # `!cancelled() &&` below is required on every *-retry job, not cosmetic: without a status-check function, GitHub applies an implicit success() gate on top of the condition and can skip the job outright. spyre-load-tests-retry: name: Spyre load (${{ matrix.model_key }}) (pod-level retry) - needs: [collect-failed-suites] + needs: [collect-failed-suites, resolve-torch-spyre] + # Unlike the primary suites, the retries do not gate on + # needs.resolve-torch-spyre.result. They do not need to: fail-closed on a pin + # failure is emergent here. If resolve-torch-spyre failed, every primary suite + # skipped (their result == guard), so nothing ran, so no suite failed, so + # collect-failed-suites reports has_failed_* == 'false' and this retry never + # fires. The !cancelled() gate only keeps the retry alive across the expected + # failure() of its primary; it never resurrects a run that skipped on pin + # failure. All *-retry jobs share this rationale. if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_load == 'true' }} runs-on: - x86_64 @@ -1664,16 +1967,27 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }} + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_load_spyre.py for ${{ matrix.model_key }} (pod-level retry) env: @@ -1699,7 +2013,7 @@ jobs: spyre-smoke-tests-retry: name: Spyre smoke (${{ matrix.model_key }}) (pod-level retry) - needs: [collect-failed-suites] + needs: [collect-failed-suites, resolve-torch-spyre] if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_smoke == 'true' }} runs-on: - x86_64 @@ -1719,16 +2033,27 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }} + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_e2e_smoke_spyre.py for ${{ matrix.model_key }} (pod-level retry) env: @@ -1753,7 +2078,10 @@ jobs: spyre-multicard-smoke-tests-retry: name: Spyre multicard smoke x2 (${{ matrix.model_key }}) (pod-level retry) - needs: [collect-failed-suites] + needs: [collect-failed-suites, resolve-torch-spyre] + # Fail-closed on pin failure is emergent, not gated here: if resolve-torch-spyre + # failed, every primary suite skipped, so nothing ran, so has_failed_multicard_smoke + # is false and this retry never fires. See spyre-load-tests-retry for the full note. if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_multicard_smoke == 'true' }} runs-on: - x86_64 @@ -1773,16 +2101,27 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' runner_labels: ${{ format('["x86_64","spyre_pf_x2","linux","{0}"]', inputs.image_label) }} + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run multicard smoke for ${{ matrix.model_key }} (pod-level retry) env: @@ -1807,7 +2146,7 @@ jobs: spyre-token-compare-tests-retry: name: Spyre token compare (${{ matrix.model_key }}) (pod-level retry) - needs: [collect-failed-suites] + needs: [collect-failed-suites, resolve-torch-spyre] if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_token_compare == 'true' }} runs-on: - x86_64 @@ -1827,16 +2166,27 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }} + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_e2e_token_compare_spyre.py for ${{ matrix.model_key }} (pod-level retry) env: @@ -1861,7 +2211,10 @@ jobs: spyre-model-components-tests-retry: name: Spyre model component tests (pod-level retry) - needs: [collect-failed-suites] + needs: [collect-failed-suites, resolve-torch-spyre] + # Fail-closed on pin failure is emergent, not gated here: if resolve-torch-spyre + # failed, every primary suite skipped, so nothing ran, so has_failed_model_components + # is false and this retry never fires. See spyre-load-tests-retry for the full note. if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_model_components == 'true' }} runs-on: - x86_64 @@ -1881,16 +2234,27 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }} + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_model_components_spyre.py (pod-level retry) run: | @@ -1911,7 +2275,7 @@ jobs: spyre-embed-compare-tests-retry: name: Spyre embed compare (${{ matrix.model_key }}) (pod-level retry) - needs: [collect-failed-suites] + needs: [collect-failed-suites, resolve-torch-spyre] if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_embed_compare == 'true' }} runs-on: - x86_64 @@ -1931,16 +2295,27 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }} + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_e2e_embed_compare_spyre.py for ${{ matrix.model_key }} (pod-level retry) env: @@ -1965,7 +2340,7 @@ jobs: spyre-vlm-e2e-tests-retry: name: Spyre VLM e2e (${{ matrix.model_key }}) (pod-level retry) - needs: [collect-failed-suites] + needs: [collect-failed-suites, resolve-torch-spyre] if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_vlm == 'true' }} runs-on: - x86_64 @@ -1985,16 +2360,27 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }} + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_vlm_e2e_spyre.py for ${{ matrix.model_key }} (pod-level retry) env: @@ -2019,7 +2405,10 @@ jobs: spyre-clip-e2e-tests-retry: name: Spyre CLIP e2e (${{ matrix.model_key }}) (pod-level retry) - needs: [collect-failed-suites] + needs: [collect-failed-suites, resolve-torch-spyre] + # Fail-closed on pin failure is emergent, not gated here: if resolve-torch-spyre + # failed, every primary suite skipped, so nothing ran, so has_failed_clip + # is false and this retry never fires. See spyre-load-tests-retry for the full note. if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_clip == 'true' }} runs-on: - x86_64 @@ -2039,16 +2428,27 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }} + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_e2e_clip_compare_spyre.py for ${{ matrix.model_key }} (pod-level retry) env: @@ -2073,7 +2473,7 @@ jobs: spyre-reranker-compare-tests-retry: name: Spyre reranker compare (${{ matrix.model_key }}) (pod-level retry) - needs: [collect-failed-suites] + needs: [collect-failed-suites, resolve-torch-spyre] if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_reranker == 'true' }} runs-on: - x86_64 @@ -2093,15 +2493,26 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_e2e_reranker_compare_spyre.py for ${{ matrix.model_key }} (pod-level retry) env: @@ -2126,7 +2537,7 @@ jobs: spyre-masked-lm-compare-tests-retry: name: Spyre masked-LM compare (${{ matrix.model_key }}) (pod-level retry) - needs: [collect-failed-suites] + needs: [collect-failed-suites, resolve-torch-spyre] if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_masked_lm == 'true' }} runs-on: - x86_64 @@ -2146,15 +2557,26 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_e2e_masked_lm_compare_spyre.py for ${{ matrix.model_key }} (pod-level retry) env: @@ -2179,7 +2601,7 @@ jobs: spyre-question-answering-compare-tests-retry: name: Spyre question-answering compare (${{ matrix.model_key }}) (pod-level retry) - needs: [collect-failed-suites] + needs: [collect-failed-suites, resolve-torch-spyre] if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_qa == 'true' }} runs-on: - x86_64 @@ -2199,15 +2621,26 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_e2e_question_answering_compare_spyre.py for ${{ matrix.model_key }} (pod-level retry) env: @@ -2232,7 +2665,7 @@ jobs: spyre-seq-classification-compare-tests-retry: name: Spyre seq-classification compare (${{ matrix.model_key }}) (pod-level retry) - needs: [collect-failed-suites] + needs: [collect-failed-suites, resolve-torch-spyre] if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_seq_classification == 'true' }} runs-on: - x86_64 @@ -2252,15 +2685,26 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_e2e_seq_classification_compare_spyre.py for ${{ matrix.model_key }} (pod-level retry) env: @@ -2285,7 +2729,7 @@ jobs: spyre-token-classification-compare-tests-retry: name: Spyre token-classification compare (${{ matrix.model_key }}) (pod-level retry) - needs: [collect-failed-suites] + needs: [collect-failed-suites, resolve-torch-spyre] if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_token_classification == 'true' }} runs-on: - x86_64 @@ -2305,15 +2749,26 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run test_e2e_token_classification_compare_spyre.py for ${{ matrix.model_key }} (pod-level retry) env: @@ -2338,7 +2793,7 @@ jobs: spyre-edge-cases-tests-retry: name: Spyre edge case ${{ matrix.test_file }} (${{ matrix.model_key }}) (pod-level retry) - needs: [collect-failed-suites] + needs: [collect-failed-suites, resolve-torch-spyre] if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_edge_cases == 'true' }} runs-on: - x86_64 @@ -2358,15 +2813,26 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run ${{ matrix.test_file }} for ${{ matrix.model_key }} (pod-level retry) env: @@ -2398,7 +2864,7 @@ jobs: spyre-model-module-tests-retry: name: Spyre model-module tests (${{ matrix.config }}) (pod-level retry) - needs: [collect-failed-suites] + needs: [collect-failed-suites, resolve-torch-spyre] if: ${{ !cancelled() && needs.collect-failed-suites.outputs.has_failed_model_module == 'true' }} runs-on: - x86_64 @@ -2418,16 +2884,27 @@ jobs: sparse-checkout-cone-mode: false - uses: ./.github/actions/build-hf-adapters + id: build with: uv_groups: "dev spyre test oot" ref: ${{ inputs.ref }} repository: ${{ inputs.repository }} + # Pin torch-spyre to the SHA resolved once for this run (issue #449). + # On the integration path (prebaked_image=true) this is empty and + # ignored: build-hf-adapters installs no torch-spyre and uses the one + # baked into the image venv instead. + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + prebaked_image: ${{ inputs.prebaked_image }} - name: Gather runner info uses: ./.github/actions/gather-runner-info with: verbose: 'true' runner_labels: ${{ format('["x86_64","spyre_pf_x1","linux","{0}"]', inputs.image_label) }} + # Assert this job's torch-spyre HEAD == the run's pinned SHA and report + # both SHAs in the run summary (issue #449). + pinned_torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + hf_adapters_sha: ${{ steps.build.outputs.target_sha }} - name: Run module tests for ${{ matrix.config }} (pod-level retry) env: @@ -2522,6 +2999,17 @@ jobs: - adapter-coverage - check-uv-lock # - cpu-tests + # resolve-torch-spyre gates the whole matrix (every suite/retry job needs + # it; issue #449). When it FAILS, each suite job's result-guard excludes + # 'failure' so they skip, collect-failed-suites finds no failed descriptors + # and spyre-tests-result would pass -- a false green over a matrix that + # never ran. Listing it here (and in the predicate below) turns the required + # check red instead, same guard pattern as generate-matrix. When it is + # SKIPPED (prebaked_image=true, the integration path), that is intentional, + # not a failure: skipped != failure/cancelled here, so the gate stays green + # and the suite jobs (whose guard also admits 'skipped') run on the image's + # baked-in torch-spyre. + - resolve-torch-spyre # Depends on the aggregator, not the individual spyre-*-tests jobs, since those are continue-on-error and always report "success"; spyre-tests-result still transitively waits for the whole matrix via collect-failed-suites and the *-retry jobs. - spyre-tests-result if: always() @@ -2539,6 +3027,7 @@ jobs: contains(needs.generate-module-config-matrix.result, 'failure') || contains(needs.generate-module-config-matrix.result, 'cancelled') || contains(needs.adapter-coverage.result, 'failure') || contains(needs.adapter-coverage.result, 'cancelled') || contains(needs.check-uv-lock.result, 'failure') || contains(needs.check-uv-lock.result, 'cancelled') || + contains(needs.resolve-torch-spyre.result, 'failure') || contains(needs.resolve-torch-spyre.result, 'cancelled') || contains(needs.spyre-tests-result.result, 'failure') || contains(needs.spyre-tests-result.result, 'cancelled') }}; then echo "One or more test suites failed." exit 1 diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 5b0ee728..079df26d 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -222,6 +222,13 @@ jobs: repository: ${{ inputs.repository || '' }} test_type: ${{ inputs.test_type || 'integration' }} models: ${{ (inputs.models != '' && inputs.models) || needs.resolve-models.outputs.models || '' }} + # Run on the torch-spyre baked into the (possibly ephemeral, per-PR) image + # this run was dispatched onto -- do NOT resolve/pin origin/main, which + # would `git checkout --force` away the exact torch-spyre version this + # integration run exists to test (issue #449 review). Every other caller + # (test_pull_request / test_daily / push-to-clickhouse) omits this, keeping + # the default false = pin origin/main once across the run. + prebaked_image: true # ---------------------------------------------------------------------------------- # Fan-in gate job so callers can poll a single stable job name for pass/fail. diff --git a/.github/workflows/push-to-clickhouse.yaml b/.github/workflows/push-to-clickhouse.yaml index d7e749da..18d31eb9 100644 --- a/.github/workflows/push-to-clickhouse.yaml +++ b/.github/workflows/push-to-clickhouse.yaml @@ -303,6 +303,64 @@ jobs: path: shards/ retention-days: 7 + # --------------------------------------------------------------------------- + # Resolve the torch-spyre revision ONCE and pin it for the whole scan (#449). + # + # torch-spyre is a git checkout baked into each runner POD IMAGE at + # /home/senuser/torch-spyre, and the weekly scan fans out over dozens of shards + # that each land on a different pod -- so without this, one scan mixed several + # torch-spyre commits (issue #449's example: 75 jobs across 6 commits) and its + # ClickHouse rows weren't comparable. This job resolves origin/main to a single + # SHA that every weekly-model-scan-* job checks out (via build-hf-adapters' + # torch_spyre_sha input) before `uv add`. + # + # It CANNOT live in generate-matrix: that job runs on spyre_pf_x0 / + # image_spyre_backend with working-directory `.` and has no + # /home/senuser/torch-spyre (see the comment on that job). This job runs on the + # scan runners' combo (spyre_pf_x1 / image_torch_spyre), where the baked + # checkout -- and its origin remote, the only place the torch-spyre URL exists + # -- is present. It runs in parallel with generate-matrix and needs no + # HF/ClickHouse credentials. + # --------------------------------------------------------------------------- + resolve-torch-spyre: + name: Resolve torch-spyre revision + runs-on: [x86_64, spyre_pf_x1, linux, image_torch_spyre] + timeout-minutes: 15 + outputs: + torch_spyre_sha: ${{ steps.resolve.outputs.torch_spyre_sha }} + steps: + - name: Resolve origin/main to a pinned SHA + id: resolve + working-directory: /home/senuser/torch-spyre + run: | + source "$HOME/.bashrc" + source /etc/profile.d/ibm-aiu-setup.sh + git config --global --add safe.directory '*' + # `git fetch origin` updates refs/remotes/origin/main to what origin + # advertises NOW; `git rev-parse origin/main` reads that ref, NOT this + # pod's stale checked-out HEAD -- which is what makes the pin immune to + # per-pod image drift. + git fetch --force origin + SHA="$(git rev-parse origin/main)" + if [[ ! "$SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "resolve-torch-spyre: origin/main did not resolve to a 40-char SHA (got '$SHA')" >&2 + exit 1 + fi + echo "Resolved torch-spyre origin/main -> ${SHA}" + echo "torch_spyre_sha=${SHA}" >> "$GITHUB_OUTPUT" + + - name: Record pinned SHA in the run summary + run: | + { + echo "## Pinned torch-spyre revision (weekly scan)" + echo "" + echo "Every model-scan job installs torch-spyre at this exact commit:" + echo "" + echo '```' + echo "${{ steps.resolve.outputs.torch_spyre_sha }}" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + # Split by runner tier so max-parallel caps total cards held at once. The cap # depends on who dispatched the run: # - manual workflow_dispatch shares the fleet with PR/daily CI, so it stays @@ -323,7 +381,7 @@ jobs: # This showed up in testing as matrix_x4=[]. weekly-model-scan-x1: name: Weekly model scan x1 (${{ matrix.mode }} shard ${{ matrix.shard_index }}) - needs: generate-matrix + needs: [generate-matrix, resolve-torch-spyre] if: needs.generate-matrix.outputs.matrix_x1 != '[]' runs-on: [x86_64, spyre_pf_x1, linux, image_torch_spyre] timeout-minutes: 4320 @@ -349,6 +407,23 @@ jobs: - uses: ./.github/actions/build-hf-adapters with: uv_groups: "dev spyre test models-ops" + # Pin torch-spyre to the SHA resolved once for this scan (issue #449). + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + + # Assert this shard's checked-out torch-spyre HEAD == the run's pinned SHA + # (issue #449). gather-runner-info runs BEFORE build-hf-adapters in these + # jobs, so the pin check rides on this dedicated post-build step instead. + - name: Verify torch-spyre pin + run: | + ACTUAL="$(git -C /home/senuser/torch-spyre rev-parse HEAD)" + PINNED="${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}" + echo "pinned: $PINNED" + echo "checked: $ACTUAL" + if [[ "$ACTUAL" != "$PINNED" ]]; then + echo "::error::torch-spyre HEAD ($ACTUAL) != pinned SHA ($PINNED)" + exit 1 + fi + echo "| ${{ matrix.mode }} shard ${{ matrix.shard_index }} | ${ACTUAL} |" >> "$GITHUB_STEP_SUMMARY" - name: Download shard uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 @@ -412,7 +487,7 @@ jobs: weekly-model-scan-x2: name: Weekly model scan x2 (${{ matrix.mode }} shard ${{ matrix.shard_index }}) - needs: generate-matrix + needs: [generate-matrix, resolve-torch-spyre] if: needs.generate-matrix.outputs.matrix_x2 != '[]' runs-on: [x86_64, spyre_pf_x2, linux, image_torch_spyre] timeout-minutes: 4320 @@ -438,6 +513,23 @@ jobs: - uses: ./.github/actions/build-hf-adapters with: uv_groups: "dev spyre test models-ops" + # Pin torch-spyre to the SHA resolved once for this scan (issue #449). + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + + # Assert this shard's checked-out torch-spyre HEAD == the run's pinned SHA + # (issue #449). gather-runner-info runs BEFORE build-hf-adapters in these + # jobs, so the pin check rides on this dedicated post-build step instead. + - name: Verify torch-spyre pin + run: | + ACTUAL="$(git -C /home/senuser/torch-spyre rev-parse HEAD)" + PINNED="${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}" + echo "pinned: $PINNED" + echo "checked: $ACTUAL" + if [[ "$ACTUAL" != "$PINNED" ]]; then + echo "::error::torch-spyre HEAD ($ACTUAL) != pinned SHA ($PINNED)" + exit 1 + fi + echo "| ${{ matrix.mode }} shard ${{ matrix.shard_index }} | ${ACTUAL} |" >> "$GITHUB_STEP_SUMMARY" - name: Download shard uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 @@ -501,7 +593,7 @@ jobs: weekly-model-scan-x4: name: Weekly model scan x4 (${{ matrix.mode }} shard ${{ matrix.shard_index }}) - needs: generate-matrix + needs: [generate-matrix, resolve-torch-spyre] if: needs.generate-matrix.outputs.matrix_x4 != '[]' runs-on: [x86_64, spyre_pf_x4, linux, image_torch_spyre] timeout-minutes: 4320 @@ -527,6 +619,23 @@ jobs: - uses: ./.github/actions/build-hf-adapters with: uv_groups: "dev spyre test models-ops" + # Pin torch-spyre to the SHA resolved once for this scan (issue #449). + torch_spyre_sha: ${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }} + + # Assert this shard's checked-out torch-spyre HEAD == the run's pinned SHA + # (issue #449). gather-runner-info runs BEFORE build-hf-adapters in these + # jobs, so the pin check rides on this dedicated post-build step instead. + - name: Verify torch-spyre pin + run: | + ACTUAL="$(git -C /home/senuser/torch-spyre rev-parse HEAD)" + PINNED="${{ needs.resolve-torch-spyre.outputs.torch_spyre_sha }}" + echo "pinned: $PINNED" + echo "checked: $ACTUAL" + if [[ "$ACTUAL" != "$PINNED" ]]; then + echo "::error::torch-spyre HEAD ($ACTUAL) != pinned SHA ($PINNED)" + exit 1 + fi + echo "| ${{ matrix.mode }} shard ${{ matrix.shard_index }} | ${ACTUAL} |" >> "$GITHUB_STEP_SUMMARY" - name: Download shard uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0