smoke-tests.yml

name: Smoke Tests (reusable)

# Heart-owned reusable smoke-test workflow for the PyAuto workspaces.
# Each workspace's .github/workflows/smoke_tests.yml is a thin caller:
#
#   jobs:
#     smoke:
#       uses: PyAutoLabs/PyAutoHeart/.github/workflows/smoke-tests.yml@main
#       with: { chain: "PyAutoNerves PyAutoFit PyAutoArray PyAutoGalaxy PyAutoLens" }
#       secrets: inherit
#
# The workflow owns the ceremony (dependency-chain checkout at the matching
# branch, python setup, cache dirs, running the workspace's own
# .github/scripts/run_smoke.py, Slack on failure); everything that differs
# per workspace stays IN the workspace: the install epilogue
# (.github/scripts/smoke_install.sh, receiving PYTHON_VERSION — extras,
# pins, version conditionals) and the smoke runner itself.
#
# One deliberate exception: `arcticpy: true`. arcticpy is a stack-level system
# dependency (source-only C++ sdist, GSL headers, a numpy-downgrade trap) whose
# recipe had drifted into four separate copies across the CTI repos. That is
# not workspace-specific variation, it is one recipe with no owner — so Heart
# owns it, in .github/actions/install-arcticpy, and the CTI workspace asks for
# it with a flag instead of carrying a copy in its epilogue.
#
# `runner` lets a caller point that same ceremony at a DIFFERENT workspace
# script — a re-timing harness, a one-off diagnostic sweep — instead of
# copying the chain-checkout/install steps into a second workflow. The copy is
# the thing to avoid: run_smoke.py's own docstring records what happened the
# last time this ceremony was duplicated, when a local copy of the env resolver
# silently drifted and left the PR gate unable to read the release profile.
# Callers reach it with their own workflow_dispatch and pass `runner:` through;
# the default keeps every existing caller byte-identical. The dependency
# chain is an input and checkouts use the calling owner, so forks of the
# organism reuse this workflow without modification.
#
# The workflow filename is intentionally stable for downstream callers.
#
# THREE CACHES ride this workflow, all added as non-fatal steps and all keyed
# on a manual epoch salt (PYAUTO_CACHE_EPOCH) so all of them can be invalidated
# at once. The JAX persistent compilation cache is restored and saved per
# (runner OS x python leg x jaxlib): the runner copies the job env into every
# script subprocess, so every script's JAX writes to and reads from the one
# directory named below. The numba on-disk function cache is restored and saved
# per (runner OS x full python version); numba stamps every entry with its
# source file's (mtime, size) — which a fresh `pip install` changes — so a
# `Stamp source mtimes from content` step rewrites every installed .py's mtime
# as a hash of its CONTENT, and identical content then hits while changed
# content misses. The workspace's simulated `dataset/` tree is restored
# and saved per (simulator-script hash x dependency-chain SHAs), with NO
# broader fallback — a different simulator or library commit must miss rather
# than restore FITS this run would not have written. All three are measured
# before and after the run into `test-results/cache_state.json`, the sidecar
# that rides the smoke-timings artifact: every per-script timing row the Heart
# ingests can then say whether that script ran hot or cold, instead of reading
# a first cache hit as a 4x speedup and its first invalidation as a 4x
# regression.
#
# A FOURTH cache, and the same rules: pip's own wheel cache under ~/.cache/pip.
# It is what the FIXED overhead of this gate is made of. Measured over the six
# most recent smoke runs of one `_test` workspace (12 legs), the seconds spent
# before the first script runs are 104 s mean = 22.6% of the job, and inside
# that: install 83.6 s (80%), the dependency-chain clone 16.5 s (16%),
# everything else ~4 s. So the clones below are `--depth 1` and the wheels the
# install downloads are cached across runs. The PyAuto libraries themselves
# stay SOURCE installs from those clones — the wheels being cached are the
# third-party ones underneath them (jax, numpy, scipy, matplotlib, astropy).
#
# That overhead is now DATA, not a thing to re-measure by hand: `setup_s` in
# the sidecar is job start -> first script, stamped by the two `Mark ...` steps
# below and differenced by the recorder. `null` when either mark is missing —
# an unknown, never a fabricated zero.

on:
  workflow_call:
    inputs:
      chain:
        description: "Space-separated dependency libraries in install order, e.g. 'PyAutoNerves PyAutoFit'"
        required: true
        type: string
      python-versions:
        description: "JSON list of python versions to matrix over"
        required: false
        type: string
        default: '["3.12", "3.13"]'
      slack-channel-id:
        description: "Slack channel for failure notification"
        required: false
        type: string
        default: "C03S98FEDK2"
      runner:
        description: "Workspace-relative runner script to execute (default: the smoke runner)"
        required: false
        type: string
        default: ".github/scripts/run_smoke.py"
      runner-args:
        description: "Extra arguments appended to the runner invocation (word-split)"
        required: false
        type: string
        default: ""
      script-timeout:
        description: "Per-script wall-clock cap in seconds (BUILD_SCRIPT_TIMEOUT). Empty leaves it unset, so the runner's own default and any profile override apply unchanged."
        required: false
        type: string
        default: ""
      arcticpy:
        description: >-
          Install arcticpy (the C++ arctic clocking code `import autocti`
          requires) before the workspace epilogue. CTI workspaces only.
        required: false
        type: boolean
        default: false

jobs:
  # Two reasons to skip the matrix, classified from ONE diff.
  #
  #   docs_only                 — the diff is nothing but prose.
  #   no_smoke_relevant_changes — the diff touches nothing any smoke script
  #                               could be built from or selected by.
  #
  # They are independent and ORed: a docs-only diff satisfies both, which is
  # fine. Both are computed from the same `$files` list — one diff, two
  # verdicts — so neither can disagree with the other about what changed.
  #
  # FAIL CLOSED — any doubt (no base sha, the all-zero sha, unfetchable base,
  # empty diff, a single unclassifiable file) runs the matrix; only an explicit
  # every-file-matches verdict skips. Both flags are therefore initialised
  # `false` BEFORE the guard, so every early exit out of it runs everything.
  # The docs allowlist is deliberately tiny: dotfiles and path-keyed sidecars
  # count as code. The two-dot diff against the base TIP (not the merge-base)
  # means upstream drift on the base branch shows up as extra files and
  # conservatively runs the matrix. A skipped matrix job reports "skipped",
  # which satisfies required-check semantics — that is the whole reason this
  # gate lives INSIDE the workflow instead of in `on.pull_request.paths`, where
  # a filtered-out job reports no conclusion at all and a required check sits
  # pending forever.
  changes:
    runs-on: ubuntu-latest
    outputs:
      docs_only: ${{ steps.diff.outputs.docs_only }}
      no_smoke_relevant_changes: ${{ steps.diff.outputs.no_smoke_relevant_changes }}
    steps:
      - name: Checkout workspace (shallow)
        uses: actions/checkout@v4
      - name: Classify the diff
        id: diff
        env:
          BASE: ${{ github.event.pull_request.base.sha || github.event.before }}
          EVENT_NAME: ${{ github.event_name }}
        run: |
          docs_only=false
          no_smoke_relevant_changes=false
          files=""
          if [ -n "$BASE" ] \
             && [ "$BASE" != "0000000000000000000000000000000000000000" ] \
             && git fetch --quiet --depth=1 origin "$BASE" 2>/dev/null; then
            files="$(git diff --name-only "$BASE" HEAD 2>/dev/null || true)"
            if [ -n "$files" ]; then
              docs_only=true
              while IFS= read -r f; do
                case "$f" in
                  *.md|docs/*|LICENSE|runtime.txt) ;;
                  *) docs_only=false; break ;;
                esac
              done <<< "$files"
              # Relevance gate. A smoke entry is a path in `smoke_tests.txt` /
              # `smoke_notebooks.txt`, resolved under `scripts/`, run with the
              # workspace's `config/` and this file's own ceremony — so a diff
              # touching none of `scripts/`, `config/`, the two entry lists or
              # `.github/` (which holds the vendored runner and the caller
              # workflow) cannot change what any smoke script does.
              #
              # PULL_REQUEST ONLY, and this is the load-bearing half of the
              # condition. `config/repos.yaml` lists "Smoke Tests" under
              # `required_workflows` for the workspace groups; `ci_status`
              # reads that workflow's conclusion on the `main` HEAD commit and
              # only an explicit `success` counts as green (`skipped` is not in
              # FAILURE_CONCLUSIONS so it does not RED, but it is not `success`
              # either, so the rollup returns an unknown and readiness can
              # never go GREEN — and `cancelled` DOES red). Narrowing the PR
              # side is free because Heart never reads it; narrowing the `main`
              # side would cost the readiness gate its signal. So a push to
              # `main` keeps running the full matrix, always.
              if [ "$EVENT_NAME" = "pull_request" ]; then
                no_smoke_relevant_changes=true
                while IFS= read -r f; do
                  case "$f" in
                    scripts/*|config/*|smoke_tests.txt|smoke_notebooks.txt|.github/*)
                      no_smoke_relevant_changes=false; break ;;
                  esac
                done <<< "$files"
              fi
            fi
          fi
          echo "docs_only=$docs_only" >> "$GITHUB_OUTPUT"
          echo "no_smoke_relevant_changes=$no_smoke_relevant_changes" >> "$GITHUB_OUTPUT"
          # One reason per skipped run, so the summary reads as an explanation
          # rather than a list of coincidences; docs-only is the narrower one
          # and wins when a diff satisfies both.
          if [ "$docs_only" = true ]; then
            {
              echo "### Smoke matrix skipped — docs/metadata-only change"
              printf -- '- `%s`\n' $files
            } >> "$GITHUB_STEP_SUMMARY"
          elif [ "$no_smoke_relevant_changes" = true ]; then
            {
              echo "### Smoke matrix skipped — no smoke-relevant change"
              echo
              echo "No changed path is under \`scripts/\`, \`config/\`, \`.github/\`,"
              echo "or is \`smoke_tests.txt\` / \`smoke_notebooks.txt\`, so no smoke"
              echo "script this gate runs can be affected. Changed files:"
              echo
              printf -- '- `%s`\n' $files
            } >> "$GITHUB_STEP_SUMMARY"
          fi

  smoke:
    needs: changes
    # Both skip reasons, ORed. Fail-closed reads the same either way: the
    # matrix runs unless the `changes` job said "true" out loud.
    if: >-
      needs.changes.outputs.docs_only != 'true'
      && needs.changes.outputs.no_smoke_relevant_changes != 'true'
    runs-on: ubuntu-latest
    # The manual salt carried by every cache key below. Bump it to invalidate
    # every cache at once — a runner-image change, a cache-format change — in
    # one edit instead of hand-editing each key and hoping none was missed.
    env:
      PYAUTO_CACHE_EPOCH: "1"
    strategy:
      fail-fast: false
      matrix:
        python-version: ${{ fromJSON(inputs.python-versions) }}
    steps:
      # The first half of `setup_s`: the wall clock at the top of the job,
      # before anything this workflow does. Differenced against the second mark
      # (immediately before the runner) by the recorder below, so how long this
      # gate spends getting ready is a measured field in the sidecar rather
      # than something a human re-derives from the Actions UI every time.
      #
      # Non-fatal and env-free, like the other measurement steps: a mark we
      # could not write costs one `null` in the sidecar, never the gate.
      - name: Mark job start
        continue-on-error: true
        shell: bash
        run: date +%s > "$RUNNER_TEMP/job_start"

      - name: Checkout workspace (repo under test, at the PR ref)
        uses: actions/checkout@v4
        with:
          path: workspace

      - name: Clone dependency chain + PyAutoHands (matching branch if it exists)
        # DEPTH 1, on the clone and on the matching-branch fetch alike: nothing
        # downstream reads this history. The libraries are `pip install`ed from
        # these trees by the workspace's own epilogue, their version comes from
        # `setup.py`'s VERSION env default rather than from `git describe`, and
        # the only other reader is `Resolve cache keys`, whose
        # `git -C "$dep" rev-parse HEAD` answers on a shallow clone exactly as
        # it does on a full one.
        #
        # The matching-branch fallback is UNCHANGED in meaning — a library PR
        # must still be tested against its workspace — but a shallow clone is
        # single-branch, so there is no `origin/$BRANCH` for `git checkout` to
        # DWIM from and `checkout "$BRANCH"` fails with `did not match any
        # file(s) known to git`. `--depth 1` fetch + `checkout -B "$BRANCH"
        # FETCH_HEAD` is the sequence that works, verified against a scratch
        # shallow clone.
        shell: bash
        run: |
          set -e
          BRANCH="${{ github.head_ref || github.ref_name }}"
          OWNER="${{ github.repository_owner }}"
          for dep in ${{ inputs.chain }} PyAutoHands; do
            git clone --depth 1 "https://github.com/$OWNER/$dep" "$dep"
            if git -C "$dep" ls-remote --exit-code --heads origin "$BRANCH" >/dev/null 2>&1; then
              echo "Branch $BRANCH exists in $dep — checking it out"
              git -C "$dep" fetch --depth 1 origin "$BRANCH"
              git -C "$dep" checkout -B "$BRANCH" FETCH_HEAD
            fi
          done

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Restore the pip wheel cache
        # The biggest single piece of this gate's fixed cost: the install step
        # below is ~84 s per leg and almost all of it is PyPI download +
        # wheel-build for the third-party stack under the PyAuto libraries
        # (jax, numpy, scipy, matplotlib, astropy...). `~/.cache/pip` is where
        # pip keeps those, so restoring it turns most of that step into a local
        # copy. The PyAuto libraries themselves are unaffected: the epilogue
        # `pip install ./<lib>`s them from the clones above, which pip does not
        # cache, so a source install stays a source install.
        #
        # `actions/cache@v4` rather than `setup-python`'s own `cache: pip`, for
        # three reasons, all of them properties this file already holds:
        #   1. NON-FATAL. Eleven required checks ride this workflow and every
        #      cache step here is `continue-on-error: true`. A `cache:` inside
        #      setup-python cannot be, and a `cache-dependency-path` that
        #      matches NO file is a hard error in that action ("No file in ...
        #      matched to [...]") — while the chain is a caller-supplied input,
        #      so the set of files that will exist is not this file's to know.
        #   2. The EPOCH SALT. Every cache here is keyed on
        #      PYAUTO_CACHE_EPOCH so all of them invalidate in one edit;
        #      setup-python's key is its own and cannot carry it.
        #   3. One key shape, read the same way as the other three.
        #
        # Keyed on the CONTENT of the dependency declarations (the libraries
        # ship their dependencies in `pyproject.toml`, not in a requirements
        # file — the globs below name both shapes plus any the workspace
        # carries), so a dependency change misses and everything else hits.
        # `hashFiles()` returns "" when nothing matches, which is a constant
        # key rather than an error; the prefix fallback then carries the run.
        # This step is AFTER the clone above, so the chain's files exist when
        # the expression resolves.
        uses: actions/cache@v4
        id: pip-cache
        continue-on-error: true
        with:
          path: ~/.cache/pip
          key: pyauto-pip-${{ runner.os }}-py${{ matrix.python-version }}-e${{ env.PYAUTO_CACHE_EPOCH }}-${{ hashFiles('*/pyproject.toml', '*/setup.py', 'workspace/requirements*.txt', 'workspace/**/requirements*.txt') }}
          restore-keys: |
            pyauto-pip-${{ runner.os }}-py${{ matrix.python-version }}-e${{ env.PYAUTO_CACHE_EPOCH }}-

      # arcticpy is a stack-level system dependency with a fiddly, easy-to-drift
      # build recipe — it belongs to Heart, not to each workspace's epilogue.
      # Gated the same way lib-tests.yml gates its own arcticpy step, so every
      # non-CTI caller is unaffected. This runs BEFORE the epilogue: the
      # epilogue installs autocti, and `import autocti` needs arcticpy present.
      - name: Install arcticpy (CTI workspaces only — source-only C++ sdist)
        if: ${{ inputs.arcticpy }}
        uses: PyAutoLabs/PyAutoHeart/.github/actions/install-arcticpy@main

      - name: Install (base + the workspace's own epilogue)
        env:
          PYTHON_VERSION: ${{ matrix.python-version }}
        run: |
          pip install --upgrade pip setuptools wheel
          pip install pyyaml
          bash workspace/.github/scripts/smoke_install.sh

      - name: Resolve cache keys
        # Four components, each one a thing that MUST invalidate a cache:
        #   jaxlib — a cached compilation is a jaxlib-specific artefact, so a
        #     jaxlib bump must never read the previous wheel's entries;
        #   pyfull — the FULL python version, because numba keys every cache
        #     entry on the directory its source was found in and the
        #     site-packages path carries the patch version;
        #   chain  — the dependency-chain commit SHAs, because a library commit
        #     can change what a simulator writes into dataset/;
        #   sims   — the hash of every simulator script in the workspace,
        #     because an edit to one changes the datasets it produces.
        # Non-fatal like every step this workflow gained: an unresolvable key
        # degrades to a cache miss, never to a red gate.
        id: keys
        continue-on-error: true
        env:
          # Via env, never interpolated into the script body — the clone step
          # above records why: `chain` is caller-supplied text.
          CHAIN: ${{ inputs.chain }}
          # hashFiles() is an expression, not a shell command, so it resolves
          # here and the script reads it back as a plain variable.
          SIMS_HASH: ${{ hashFiles('workspace/**/simulator*.py', 'workspace/**/simulator/**', 'workspace/**/simulators/**') }}
        run: |
          jaxlib="$(python -c 'import jaxlib; print(jaxlib.__version__)' 2>/dev/null || echo nojax)"
          pyfull="$(python -c 'import sys; print(sys.version.split()[0])')"
          chain="$(for dep in $CHAIN; do git -C "$dep" rev-parse HEAD; done | sha256sum | cut -c1-16)"
          {
            echo "jaxlib=${jaxlib:-nojax}"
            echo "pyfull=$pyfull"
            echo "chain=$chain"
            echo "sims=$SIMS_HASH"
          } >> "$GITHUB_OUTPUT"

      - name: Restore the JAX compile cache
        # The run-id suffix makes every save a fresh SUPERSET of what was
        # restored (the accumulating-cache pattern): the exact key never hits,
        # the restore-keys prefix takes the newest previous entry, and the save
        # writes the merged directory back under a new key. Traced code that
        # changed simply misses its own entries and recompiles them, so this
        # cache needs no invalidation rule of its own. And a PR run reads the
        # default branch's caches under GitHub's cache scoping — which is
        # exactly what a contributor wants: a first push that is already warm.
        uses: actions/cache/restore@v4
        id: jax-cache
        continue-on-error: true
        with:
          path: ${{ github.workspace }}/.pyauto_jax_cache
          key: pyauto-jax-${{ runner.os }}-py${{ matrix.python-version }}-jaxlib${{ steps.keys.outputs.jaxlib }}-e${{ env.PYAUTO_CACHE_EPOCH }}-${{ github.run_id }}
          restore-keys: |
            pyauto-jax-${{ runner.os }}-py${{ matrix.python-version }}-jaxlib${{ steps.keys.outputs.jaxlib }}-e${{ env.PYAUTO_CACHE_EPOCH }}-

      - name: Restore simulated datasets
        # ONE fallback line and no broader one, deliberately: a different
        # simulator script or a different library commit MUST miss, because the
        # FITS it would restore are not the FITS this run would have written.
        # Correctness over hit rate.
        #
        # Not keyed on the python leg: simulated FITS do not depend on the
        # interpreter's minor version, so both legs share one entry.
        #
        # Restoring is safe whatever lands here — `should_simulate` keeps a
        # restored dataset only while its resolution regime matches the one in
        # force, and deletes and re-simulates it when it does not.
        uses: actions/cache/restore@v4
        id: dataset-cache
        continue-on-error: true
        with:
          path: workspace/dataset
          key: pyauto-datasets-${{ runner.os }}-e${{ env.PYAUTO_CACHE_EPOCH }}-sim${{ steps.keys.outputs.sims }}-chain${{ steps.keys.outputs.chain }}-${{ github.run_id }}
          restore-keys: |
            pyauto-datasets-${{ runner.os }}-e${{ env.PYAUTO_CACHE_EPOCH }}-sim${{ steps.keys.outputs.sims }}-chain${{ steps.keys.outputs.chain }}-

      - name: Restore the numba cache
        # Same accumulating pattern as the compile cache above, and the prefix
        # fallback is SAFE for the reason the stamp step below exists: a numba
        # entry carries its source file's (mtime, size), the stamp makes that
        # mtime a function of the file's content, so a changed file misses its
        # own entry however old the restored directory is. That is why this key
        # names no library commit — and a numba version bump invalidates its own
        # index format on its own terms.
        uses: actions/cache/restore@v4
        id: numba-cache
        continue-on-error: true
        with:
          path: ${{ github.workspace }}/.numba_cache
          key: pyauto-numba-${{ runner.os }}-py${{ steps.keys.outputs.pyfull }}-e${{ env.PYAUTO_CACHE_EPOCH }}-${{ github.run_id }}
          restore-keys: |
            pyauto-numba-${{ runner.os }}-py${{ steps.keys.outputs.pyfull }}-e${{ env.PYAUTO_CACHE_EPOCH }}-

      - name: Measure the caches before the run
        # The "before" half of the sidecar: what a restore actually put on disk,
        # as against what the run leaves behind. A missing path measures as
        # zero rather than failing — a cold cache is a measurement, not a fault.
        id: cache-before
        continue-on-error: true
        env:
          JAX_PATH: ${{ github.workspace }}/.pyauto_jax_cache
          NUMBA_PATH: ${{ github.workspace }}/.numba_cache
          DS_PATH: workspace/dataset
        run: |
          measure() {
            mb="$(du -sm "$1" 2>/dev/null | cut -f1)"
            n="$(find "$1" -type f 2>/dev/null | wc -l)"
            echo "${mb:-0} ${n:-0}"
          }
          set -- $(measure "$JAX_PATH")
          jax_mb="$1"; jax_n="$2"
          set -- $(measure "$NUMBA_PATH")
          numba_mb="$1"; numba_n="$2"
          set -- $(measure "$DS_PATH")
          ds_mb="$1"; ds_n="$2"
          {
            echo "jax_mb=$jax_mb"
            echo "jax_n=$jax_n"
            echo "numba_mb=$numba_mb"
            echo "numba_n=$numba_n"
            echo "ds_mb=$ds_mb"
            echo "ds_n=$ds_n"
          } >> "$GITHUB_OUTPUT"

      - name: Stamp source mtimes from content
        # numba stamps every cache entry with its source file's
        # (st_mtime, st_size) and stores it under a directory named for the
        # source's own directory. The workspace's own `smoke_install.sh` does a
        # `pip install ./<lib>` per library, which gives every installed .py a
        # brand-new mtime, so a restored numba cache would miss EVERY entry it
        # holds — it would cost upload time and buy nothing. This step rewrites
        # each mtime as a function of the file's CONTENT, so an unchanged file
        # stamps identically on every runner and a changed one stamps
        # differently: exactly the invalidation numba wants, expressed in the
        # one field it already checks.
        #
        # Non-fatal like the rest: a file we could not stat or touch costs one
        # cache entry, never the gate.
        id: stamp
        continue-on-error: true
        env:
          # Nothing here but the installed packages, found below: the chain is
          # `pip install`ed by the workspace's epilogue and the runner's own
          # PYTHONPATH names only PyAutoHands, so the clones are not what the
          # scripts import. Via env, never interpolated into the script body —
          # the same house rule the key resolver and the recorder follow.
          STAMP_DIRS: ""
        run: |
          python - <<'PY'
          import hashlib
          import importlib.util
          import os
          import pathlib

          roots = [pathlib.Path(d) for d in os.environ.get("STAMP_DIRS", "").split()]
          # The INSTALLED copies, which is what every script imports.
          # `find_spec` on a top-level name resolves it WITHOUT importing it, so
          # a package this workspace does not install simply drops out instead
          # of raising.
          for pkg in ("autonerves", "autofit", "autoarray", "autogalaxy",
                      "autolens", "autocti", "autoreduce"):
              try:
                  spec = importlib.util.find_spec(pkg)
              except (ImportError, ValueError):
                  continue
              if spec is None:
                  continue
              roots.extend(pathlib.Path(p)
                           for p in (spec.submodule_search_locations or []))

          stamped = 0
          for root in roots:
              if not root.is_dir():
                  continue
              for path in root.rglob("*.py"):
                  try:
                      digest = hashlib.sha1(path.read_bytes()).digest()
                      mtime = 1_000_000_000 + int.from_bytes(digest[:4], "big")
                      os.utime(path, (mtime, mtime))
                  except OSError:
                      continue
                  stamped += 1
          print(f"stamped {stamped} source files under {len(roots)} roots")
          PY

      - name: Prepare cache dirs
        # numba's directory is no longer among these: it is the restored and
        # saved cache above, under the workspace, and the cache action creates
        # it. Matplotlib's stays — it is scratch, wanted only so the backend
        # does not try to write into a read-only home.
        run: mkdir -p /tmp/matplotlib

      - name: Validate env profiles (strict — PyAutoHands#161 step 5)
        # Binding PR-time config gate for every workspace that carries a smoke
        # profile: the profile must parse, no dead patterns, no legacy names.
        # Config-only errors fail here in seconds instead of in the next
        # nightly. The release-only strict flags — no PYAUTO_DISABLE_JAX
        # enumeration creeping back (strict-derivation), every script's
        # resolved backend matching its JAX name marker (strict-markers) —
        # apply only where a canonical profile_release.yaml exists (the
        # *_workspace_test repos).
        run: |
          # Fire if the workspace carries a smoke profile: the canonical
          # profile_smoke.yaml (PyAutoHands#161/#181 step 6 rename).
          # --strict-declarations (PyAutoHands#187): pattern-overrides that are
          # expressible as in-file '# ENV:' declarations are errors — the
          # migration landed in every workspace, so new ones must not creep in.
          if [ -f workspace/config/build/profile_smoke.yaml ]; then
            if [ -f workspace/config/build/profile_release.yaml ]; then
              python PyAutoHands/autohands/validate_env_profiles.py workspace \
                --strict-derivation --strict-markers --strict-declarations
            else
              python PyAutoHands/autohands/validate_env_profiles.py workspace \
                --strict-declarations
            fi
          else
            echo "No config/build/profile_smoke.yaml — profile validation skipped."
          fi

      # The PR gate's copy of workspace-validation.yml's guard: the runner
      # below writes into test-results/ and the upload step globs it (plus
      # every smoke_timings.json), so a result file the workspace repo
      # tracks would ride into the timings artifact here exactly as it rode
      # into the release report there. Same step text, kept in step by
      # tests/test_workflow_wiring.py.
      - name: Discard result files the workspace checkout carries
        # A workspace repo must never ship its own test-results/ — but one did:
        # autolens_workspace_test#311 committed a local smoke run's
        # `*__script.json` / `.md` / `smoke_timings.json` (three stale failures,
        # `/home/<user>` paths). The shard uploaded the whole directory, the
        # analyze stage merged those failures into the report, and two nightly
        # releases (2026-09-09, 2026-09-10) stopped at Stage 3 while every
        # script that actually ran passed. The runner below writes its own
        # reports into test-results/; anything already there arrived from git
        # and is not evidence about this run, so it goes — loudly, one
        # ::warning:: per file, so a leaking repo is visible in the run log
        # rather than silently laundered. cache_state.json is kept: it is this
        # run's sidecar (written by the cache steps), not a result. The shell
        # is self-contained (no `${{ }}`) so tests/test_workflow_wiring.py can
        # execute it verbatim against a planted stale file.
        shell: bash
        run: |
          set -u
          ws="${WORKSPACE_DIR:-workspace}"
          [ -d "$ws" ] || { echo "no $ws checkout — nothing to discard"; exit 0; }
          found=0
          while IFS= read -r f; do
            found=1
            echo "::warning title=Stale result file in the workspace checkout::$f was committed to the workspace repo and is not a result of this run — discarded"
            rm -f "$f"
          done < <(
            { [ -d "$ws/test-results" ] && find "$ws/test-results" -type f \
                \( -name '*__script.json' -o -name '*__script.md' \
                   -o -name 'smoke_timings.json' -o -name 'report.json' \) \
                ! -name 'cache_state.json'; } ;
            find "$ws" -path "$ws/.git" -prune -o -path "$ws/test-results" -prune \
                -o -type f -name 'smoke_timings.json' -print
          )
          [ "$found" -eq 0 ] && echo "no stale result files under $ws"
          exit 0

      # The second half of `setup_s`. Placed IMMEDIATELY before the runner and
      # nowhere else: the runner step's own command is the one thing in this
      # workflow that must stay byte-identical (a workspace owns its runner),
      # so the mark cannot live inside it and every step between the two marks
      # is honestly part of the setup being measured.
      - name: Mark scripts start
        continue-on-error: true
        shell: bash
        run: date +%s > "$RUNNER_TEMP/scripts_start"

      - name: Run the workspace runner (the smoke runner by default)
        env:
          JAX_ENABLE_X64: "True"
          # Same reason as workspace-validation.yml's workflow-level env: JAX's
          # default traceback filter strips the frames that name the exception,
          # so a failing script reports the placeholder instead of the failure.
          # This runner is the other surface that executes workspace scripts
          # under JAX, so it needs the same treatment.
          JAX_TRACEBACK_FILTERING: "off"
          # The restored-and-saved directory, not a per-run temporary: the
          # runner copies this job env into every script subprocess, so all
          # of them share the one numba cache the steps above and below
          # carry between runs.
          NUMBA_CACHE_DIR: ${{ github.workspace }}/.numba_cache
          MPLCONFIGDIR: /tmp/matplotlib
          PYTHONPATH: ${{ github.workspace }}/PyAutoHands/autohands
          # autonerves defaults this to ~/.cache/pyauto_jax when it is unset; an
          # explicit workspace path is the one the cache steps above and below
          # can name. The runner copies this job env into every script
          # subprocess, so each script's JAX writes to and reads from the same
          # compilation cache instead of one per-process temporary.
          JAX_COMPILATION_CACHE_DIR: ${{ github.workspace }}/.pyauto_jax_cache
          # Persist EVERY compile, not only those over JAX's 1.0s default
          # (autonerves setdefaults the same 1s whenever the cache dir is set;
          # a preset value wins). The smoke scripts are hundreds of sub-second
          # compiles each, so under the 1s regime a "hit" cache held 1-4
          # entries and every run re-paid the compile: measured 2->513,
          # 4->299 and 1->~490 entries on the three slowest JAX scripts, and
          # 23-31s -> 10-18s per script, output byte-identical. The price is
          # cache-artefact size (~500 entries / ~3 MB per JAX script, per
          # python leg); the way out is this one line, or an epoch bump to
          # drop the fat caches.
          JAX_PERSISTENT_CACHE_MIN_COMPILE_TIME_SECS: "0"
          # Via env, never interpolated into the script body: these inputs reach
          # here from a caller's workflow_dispatch, and `${{ }}` inside `run:`
          # would splice their text straight into the shell.
          RUNNER: ${{ inputs.runner }}
          RUNNER_ARGS: ${{ inputs.runner-args }}
          SCRIPT_TIMEOUT: ${{ inputs.script-timeout }}
        run: |
          cd workspace
          # Exported only when the caller asked for a cap. Setting it
          # unconditionally would hand run_smoke.py an empty string to int(),
          # and would also state a process-wide cap where today there is none.
          if [ -n "$SCRIPT_TIMEOUT" ]; then
            export BUILD_SCRIPT_TIMEOUT="$SCRIPT_TIMEOUT"
          fi
          python "$RUNNER" $RUNNER_ARGS

      - name: Record cache state (rides the smoke-timings artifact)
        # This file is the unit of measurement for the whole cache story: every
        # per-script timing row the Heart ingests from the artifact below can
        # then say whether that script ran hot or cold. Without it a timing
        # dataset reads a cache landing as a 4x speedup and its invalidation as
        # a 4x regression, and neither is a change anybody made.
        #
        # if: always() — a red run's cache state is exactly as interesting as a
        # green one's, and the artifact is uploaded either way.
        #
        # It also carries `setup_s`: the seconds from job start to the first
        # script, differenced from the two `Mark ...` marks. The marks are
        # non-fatal, so either may be absent; `null` then says so, because a
        # fabricated 0 would read as a gate with no fixed cost at all.
        id: cache-state
        if: always()
        continue-on-error: true
        env:
          EPOCH: ${{ env.PYAUTO_CACHE_EPOCH }}
          JAX_PATH: ${{ github.workspace }}/.pyauto_jax_cache
          JAX_KEY_RESTORED: ${{ steps.jax-cache.outputs.cache-matched-key }}
          JAX_EXACT: ${{ steps.jax-cache.outputs.cache-hit }}
          JAX_MB_BEFORE: ${{ steps.cache-before.outputs.jax_mb }}
          JAX_N_BEFORE: ${{ steps.cache-before.outputs.jax_n }}
          NUMBA_PATH: ${{ github.workspace }}/.numba_cache
          NUMBA_KEY_RESTORED: ${{ steps.numba-cache.outputs.cache-matched-key }}
          NUMBA_EXACT: ${{ steps.numba-cache.outputs.cache-hit }}
          NUMBA_MB_BEFORE: ${{ steps.cache-before.outputs.numba_mb }}
          NUMBA_N_BEFORE: ${{ steps.cache-before.outputs.numba_n }}
          DS_PATH: workspace/dataset
          DS_KEY_RESTORED: ${{ steps.dataset-cache.outputs.cache-matched-key }}
          DS_EXACT: ${{ steps.dataset-cache.outputs.cache-hit }}
          DS_MB_BEFORE: ${{ steps.cache-before.outputs.ds_mb }}
          DS_N_BEFORE: ${{ steps.cache-before.outputs.ds_n }}
        run: |
          python - <<'PY'
          import json
          import os
          import subprocess
          from pathlib import Path


          def as_int(text):
              try:
                  return int(str(text).strip())
              except (TypeError, ValueError):
                  return 0


          def measure(path):
              """(size in MiB, file count) for a path that may not exist."""
              if not path or not os.path.isdir(path):
                  return 0, 0
              proc = subprocess.run(["du", "-sm", path], capture_output=True,
                                    text=True)
              head = proc.stdout.split()
              size = as_int(head[0]) if head else 0
              return size, sum(len(files) for _r, _d, files in os.walk(path))


          def section(path, key, exact, mb_before, n_before):
              mb_after, n_after = measure(path)
              mb_before, n_before = as_int(mb_before), as_int(n_before)
              return {
                  "path": path,
                  # A restored key at all is a hit; the EXACT key is the
                  # accumulating pattern's never-case, kept separate so a
                  # reader is not left guessing which one "hit" meant.
                  "restored_key": key,
                  "hit": bool(key),
                  "exact": exact == "true",
                  "size_mb_before": mb_before,
                  "size_mb_after": mb_after,
                  "entries_before": n_before,
                  "entries_after": n_after,
              }, (mb_after != mb_before or n_after != n_before)


          def setup_seconds():
              """Job start -> first script, in whole seconds, or None.

              Both marks are written by non-fatal steps, so a missing or
              unreadable one is an unknown rather than a zero. A negative or
              absurd difference is discarded for the same reason: it would be
              a clock we do not trust, not a measurement.
              """
              temp = os.environ.get("RUNNER_TEMP") or ""
              if not temp:
                  return None
              stamps = []
              for name in ("job_start", "scripts_start"):
                  try:
                      stamps.append(int(Path(temp, name).read_text().strip()))
                  except (OSError, TypeError, ValueError):
                      return None
              delta = stamps[1] - stamps[0]
              return delta if delta >= 0 else None


          env = os.environ.get
          jax, jax_changed = section(
              env("JAX_PATH", ""), env("JAX_KEY_RESTORED", ""),
              env("JAX_EXACT", ""), env("JAX_MB_BEFORE", ""),
              env("JAX_N_BEFORE", ""),
          )
          numba, numba_changed = section(
              env("NUMBA_PATH", ""), env("NUMBA_KEY_RESTORED", ""),
              env("NUMBA_EXACT", ""), env("NUMBA_MB_BEFORE", ""),
              env("NUMBA_N_BEFORE", ""),
          )
          datasets, ds_changed = section(
              env("DS_PATH", ""), env("DS_KEY_RESTORED", ""),
              env("DS_EXACT", ""), env("DS_MB_BEFORE", ""),
              env("DS_N_BEFORE", ""),
          )
          # The schema string does not move: a `numba` section is ADDITIVE, and
          # an ingest that predates it reads the two sections it knows and the
          # absence of the third honestly (see heart/checks/smoke_timings.py).
          payload = {
              "schema": "cache_state/1",
              "epoch": env("EPOCH", ""),
              "jax": jax,
              "numba": numba,
              "datasets": datasets,
              # Additive like `numba` was: the schema string does not move, and
              # an ingest that predates this field reads its absence honestly.
              "setup_s": setup_seconds(),
          }
          out = Path("workspace/test-results")
          out.mkdir(parents=True, exist_ok=True)
          (out / "cache_state.json").write_text(
              json.dumps(payload, sort_keys=True, indent=2) + "\n")
          print(json.dumps(payload, sort_keys=True))

          github_output = env("GITHUB_OUTPUT")
          if github_output:
              with open(github_output, "a", encoding="utf-8") as handle:
                  handle.write("jax_changed=%s\n"
                               % ("true" if jax_changed else "false"))
                  handle.write("numba_changed=%s\n"
                               % ("true" if numba_changed else "false"))
                  handle.write("ds_changed=%s\n"
                               % ("true" if ds_changed else "false"))
          PY

      - name: Upload the smoke report dir (per-script timings)
        # PyAutoHands#264. The runner writes a consolidated smoke_timings.json
        # into its report dir on every report write — one row per executed
        # script/notebook with the runner's OWN measured duration, the cap in
        # force, and the exit code. In the job log that data evaporates with
        # the ~90-day log retention and is only recovered by hand-scraping;
        # as an artifact it becomes a standing dataset the Heart board can
        # ingest, for every gate run across all ten workspaces at once.
        #
        # WHY A GLOB and not one literal path: this workflow does not pass
        # `--report-dir` — each workspace's own `.github/scripts/run_smoke.py`
        # does, and the runner script is deliberately left in the workspace
        # (see the header). `test-results/` is the house name, used by
        # workspace-validation.yml's own run_python.py invocations; the second
        # pattern catches a workspace that named its report dir something else,
        # so no repo needs editing for its timings to land here.
        #
        # No retention-days: the dataset is the point, so it keeps the repo's
        # full default artifact retention rather than a shortened window.
        #
        # if: always() + if-no-files-found: ignore are both load-bearing. A
        # docs-only change skips the matrix entirely; a crash before the first
        # report write (a failed install, a missing script list) leaves no
        # report dir at all. Neither is a reason to fail a gate — and an upload
        # that failed closed would turn "we collected no timings" into "the PR
        # is red", which is exactly backwards.
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: smoke-timings-${{ matrix.python-version }}
          path: |
            workspace/test-results/
            workspace/**/smoke_timings.json
          if-no-files-found: ignore

      - name: Save the JAX compile cache
        # Saved even on a RED run: a compilation-cache entry is complete or
        # absent, never half-written, so the compiling a failing run did is
        # still worth keeping for the next attempt. The `jax_changed` guard is
        # the no-op case — an unchanged cache is not re-uploaded under a new key.
        if: always() && steps.cache-state.outputs.jax_changed == 'true'
        continue-on-error: true
        uses: actions/cache/save@v4
        with:
          path: ${{ github.workspace }}/.pyauto_jax_cache
          key: pyauto-jax-${{ runner.os }}-py${{ matrix.python-version }}-jaxlib${{ steps.keys.outputs.jaxlib }}-e${{ env.PYAUTO_CACHE_EPOCH }}-${{ github.run_id }}

      - name: Save the numba cache
        # Saved on a red run for the same reason as the compile cache: a numba
        # cache entry is an index line plus a complete compiled artefact, or it
        # is absent. Same no-op guard as above.
        if: always() && steps.cache-state.outputs.numba_changed == 'true'
        continue-on-error: true
        uses: actions/cache/save@v4
        with:
          path: ${{ github.workspace }}/.numba_cache
          key: pyauto-numba-${{ runner.os }}-py${{ steps.keys.outputs.pyfull }}-e${{ env.PYAUTO_CACHE_EPOCH }}-${{ github.run_id }}

      - name: Save simulated datasets
        # ONLY on a green run, unlike the compile cache above: a simulator
        # killed mid-write leaves a truncated FITS on disk, `should_simulate`
        # would keep it (its regime matches), and caching it would poison every
        # later run that restored it. Same no-op guard as above.
        if: success() && steps.cache-state.outputs.ds_changed == 'true'
        continue-on-error: true
        uses: actions/cache/save@v4
        with:
          path: workspace/dataset
          key: pyauto-datasets-${{ runner.os }}-e${{ env.PYAUTO_CACHE_EPOCH }}-sim${{ steps.keys.outputs.sims }}-chain${{ steps.keys.outputs.chain }}-${{ github.run_id }}

      - name: Slack notify on failure
        if: ${{ failure() }}
        uses: slackapi/slack-github-action@v1.21.0
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
        with:
          channel-id: ${{ inputs.slack-channel-id }}
          payload: |
            {
              "text": "${{ github.repository }}/${{ github.ref_name }} smoke tests (Python ${{ matrix.python-version }}) ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
            }