fix(tests): tag every tier a test belongs to, not just the invoked one

.github/actions/run-matrix-config/action.yml

@@ -20,8 +20,18 @@ inputs:
     default: ''
   test_type:
     description: >-
-      Raw test tier; resolved via `make print-test-type` and exported as
-      SPYRE_TEST_TIER for the testtype__<tier> tag. Empty = Makefile default.
+      Raw test tier this run was INVOKED as; resolved via `make print-test-type` and
+      exported as SPYRE_TEST_TIER for the invoked_as__<tier> tag. Empty = Makefile
+      default.
+    required: false
+    default: ''
+  test_types:
+    description: >-
+      Every tier this leg's tests BELONG to, whitespace-separated (the matrix entry's
+      own `test_types`). Exported as SPYRE_TEST_TIERS, one testtype__<tier> tag each.
+      This set is what makes coverage reusable -- the invoked tier alone under-reports,
+      so a later run of another tier re-executes identical work. Empty falls back to
+      test_type.
     required: false
     default: ''
   runner_labels:
@@ -204,6 +214,7 @@ runs:
       shell: bash
       env:
         RAW_TEST_TYPE: ${{ inputs.test_type }}
+        RAW_TEST_TYPES: ${{ inputs.test_types }}
       run: |
         echo "Running tests for config: ${{ inputs.cfg }}${{ inputs.retry == 'true' && ' (pod-level retry)' || '' }}..."
         # An empty TEST_TYPE fails the Makefile's validation, so omit the arg and
@@ -215,6 +226,12 @@ runs:
         fi
         export SPYRE_TEST_TIER
         echo "Resolved SPYRE_TEST_TIER=${SPYRE_TEST_TIER}"
+        # Passed through verbatim, NOT through print-test-type: this is the matrix
+        # entry's declared membership set, already valid tier names, and resolving it
+        # would collapse the set to one value. Unset => tags.test_tiers() falls back to
+        # the invoked tier.
+        export SPYRE_TEST_TIERS="${RAW_TEST_TYPES}"
+        echo "Declared SPYRE_TEST_TIERS=${SPYRE_TEST_TIERS:-<unset, falling back to ${SPYRE_TEST_TIER}>}"
         timeout --signal=TERM --kill-after=1m 90m \
           make ${{ inputs.test_target }} COVERAGE=1 JUNIT_XML=junit-${{ inputs.test_target }}.xml
         echo "tests done"

.github/workflows/_test_matrix.yaml

@@ -599,6 +599,10 @@ jobs:
           test_target: ${{ matrix.test_target }}
           durations_run_id: ${{ needs.resolve_durations.outputs.durations_run_id }}
           test_type: ${{ inputs.test_type }}
+          # The matrix entry's own declared membership set, not the invoked tier: one
+          # testtype__ tag per member, which is what lets a later run of another tier
+          # reuse this leg's coverage instead of re-running it.
+          test_types: ${{ matrix.test_types }}
           # Echo the exact runner-label selectors this job's runs-on used (matrix.runs_on plus the per-PR runner_label override, or the standing matrix.image_label).
           runner_labels: ${{ format('["{0}","{1}"]', join(matrix.runs_on, '","'), inputs.runner_label || matrix.image_label) }}
           job_slot: ${{ strategy.job-index }}
@@ -683,6 +687,9 @@ jobs:
           test_target: ${{ matrix.test_target }}
           durations_run_id: ${{ needs.resolve_durations.outputs.durations_run_id }}
           test_type: ${{ inputs.test_type }}
+          # Survives the retry hop: the failed-suite descriptor is toJSON(matrix), so
+          # the whole entry -- test_types included -- is rebuilt by collect_failed_suites.
+          test_types: ${{ matrix.test_types }}
           runner_labels: ${{ format('["{0}","{1}"]', join(matrix.runs_on, '","'), inputs.runner_label || matrix.image_label) }}
           job_slot: retry-${{ strategy.job-index }}
           retry: 'true'

tests/plugin/spyre_testing_plugin/tags.py

@@ -44,12 +44,49 @@ def model_from_params(params):
 
 
 def test_tier():
-    """Suite tier from SPYRE_TEST_TIER (exported by CI's run-matrix-config; empty
-    on a local run). Read live rather than at import so tests that monkeypatch the
-    env still see it."""
+    """How this run was INVOKED, from SPYRE_TEST_TIER (exported by CI's
+    run-matrix-config; empty on a local run). Read live rather than at import so tests
+    that monkeypatch the env still see it.
+
+    One value, and NOT the same fact as test_tiers() below: a leg invoked as
+    `regression` may hold tests that also belong to `unit` and `integration`.
+
+    Deliberately NOT emitted as a case tag. The invocation is a property of the RUN,
+    while case tags are hashed into test_case_id (see ingest_xml_si.v2_test_case_id) --
+    tagging it would give one test three identities depending on which tier happened to
+    launch it, which is the fragmentation the derived identity exists to prevent. It
+    belongs on the run row instead, where si_test_runs.test_type already records it.
+    """
     return os.environ.get("SPYRE_TEST_TIER", "")
 
 
+def test_tiers():
+    """Every tier this leg's tests BELONG to, from SPYRE_TEST_TIERS.
+
+    Declared per matrix entry as `test_types` in _test_matrix.yaml and threaded
+    through run-matrix-config; whitespace-separated, e.g.
+    "unit integration regression trunk".
+
+    This is the set that makes coverage reusable. The invocation tier alone cannot:
+    a `regression` run of a leg whose tests are also `unit` and `integration` members
+    would record only `regression`, so a later `integration` run finds no coverage and
+    re-executes identical work.
+
+    Membership is read from the declaration and never inferred from a tier ladder.
+    Measured on this repo's 32 matrix entries: 13 declare `unit regression trunk`
+    WITHOUT `integration`, so a ladder ("in unit => in everything above") would claim
+    coverage for 13 legs that never ran integration and silently skip real tests.
+
+    Falls back to the single invocation tier when unset, so a local `make test` and any
+    caller not yet passing test_types keep emitting a tag rather than none.
+    """
+    raw = os.environ.get("SPYRE_TEST_TIERS", "")
+    if not raw.strip():
+        tier = test_tier()
+        return [tier] if tier else []
+    return sorted(set(raw.split()))
+
+
 def result_tags(params):
     """The (name, value) JUnit property pairs for these params; empty when no
     model param is recognized and no tier is set, so callers append
@@ -59,7 +96,6 @@ def result_tags(params):
     model = model_from_params(params)
     if model:
         tags.append(("tag", f"model__{model}"))
-    tier = test_tier()
-    if tier:
+    for tier in test_tiers():
         tags.append(("tag", f"testtype__{tier}"))
     return tags

tests/test_tags.py

@@ -0,0 +1,202 @@
+# Copyright 2026 The Spyre-Inference Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""CPU-only tests for the JUnit result tags (no hardware needed).
+
+The invariant that matters: `testtype__` carries the tiers a test BELONGS to, read
+from the matrix entry's declared set, and is never inferred from a tier ladder. A
+ladder over-claims coverage, and over-claimed coverage silently skips real tests --
+so the ladder tests below are the ones that must not regress.
+"""
+
+import re
+from pathlib import Path
+
+import yaml
+
+# Imported as a MODULE, deliberately: `test_tier`/`test_tiers` start with `test_`, so
+# importing them by name makes pytest collect the production functions themselves as
+# (vacuously passing) tests.
+from spyre_testing_plugin import tags
+
+_REPO_ROOT = Path(__file__).resolve().parents[1]
+_MATRIX = _REPO_ROOT / ".github" / "workflows" / "_test_matrix.yaml"
+
+
+def _tags(pairs):
+    """Just the tag values; every pair's name is the literal 'tag'."""
+    assert {n for n, _ in pairs} <= {"tag"}
+    return [v for _, v in pairs]
+
+
+# ── the declared set ────────────────────────────────────────────────────────────
+
+
+def test_tiers_reads_the_declared_set(monkeypatch):
+    monkeypatch.setenv("SPYRE_TEST_TIERS", "unit integration regression trunk")
+    assert tags.test_tiers() == ["integration", "regression", "trunk", "unit"]
+
+
+def test_tiers_dedups_and_sorts(monkeypatch):
+    # Sorted and deduped so two writers agree on the same set regardless of the
+    # order it was declared in -- the tags feed a content hash downstream.
+    monkeypatch.setenv("SPYRE_TEST_TIERS", "trunk unit trunk  unit")
+    assert tags.test_tiers() == ["trunk", "unit"]
+
+
+def test_every_declared_tier_becomes_a_tag(monkeypatch):
+    monkeypatch.setenv("SPYRE_TEST_TIERS", "unit integration regression trunk")
+    monkeypatch.delenv("SPYRE_TEST_TIER", raising=False)
+    assert _tags(tags.result_tags({})) == [
+        "testtype__integration",
+        "testtype__regression",
+        "testtype__trunk",
+        "testtype__unit",
+    ]
+
+
+# ── the fallback: a caller that does not pass the set still tags ────────────────
+
+
+def test_falls_back_to_the_invoked_tier(monkeypatch):
+    """A local `make test` and any un-updated caller keep emitting a tag."""
+    monkeypatch.delenv("SPYRE_TEST_TIERS", raising=False)
+    monkeypatch.setenv("SPYRE_TEST_TIER", "regression")
+    assert tags.test_tiers() == ["regression"]
+
+
+def test_blank_tiers_falls_back_too(monkeypatch):
+    # The action exports "" when the input is unset, so whitespace must not read as
+    # a one-element set containing the empty string.
+    monkeypatch.setenv("SPYRE_TEST_TIERS", "   ")
+    monkeypatch.setenv("SPYRE_TEST_TIER", "unit")
+    assert tags.test_tiers() == ["unit"]
+
+
+def test_no_tier_anywhere_emits_no_tier_tag(monkeypatch):
+    monkeypatch.delenv("SPYRE_TEST_TIERS", raising=False)
+    monkeypatch.delenv("SPYRE_TEST_TIER", raising=False)
+    assert tags.test_tiers() == []
+    assert _tags(tags.result_tags({})) == []
+
+
+# ── membership vs invocation are different facts ────────────────────────────────
+
+
+def test_the_invocation_is_never_a_case_tag(monkeypatch):
+    """The invocation must NOT reach the tags, because case tags are hashed into
+    test_case_id: tagging it would give one test a different identity per invoking
+    tier. It is a property of the run (si_test_runs.test_type), not of the test."""
+    monkeypatch.setenv("SPYRE_TEST_TIERS", "unit regression trunk")
+    monkeypatch.setenv("SPYRE_TEST_TIER", "regression")
+    values = _tags(tags.result_tags({}))
+    assert not any("invoked_as" in v for v in values)
+    assert values == [
+        "testtype__regression",
+        "testtype__trunk",
+        "testtype__unit",
+    ]
+    # Still readable for whoever records the run row.
+    assert tags.test_tier() == "regression"
+
+
+def test_identity_does_not_vary_by_invoking_tier(monkeypatch):
+    """The concrete regression: the same test, same declared membership, invoked as
+    two different tiers, must produce the SAME tag list -- otherwise its hashed
+    test_case_id splits and every trend query fragments."""
+    monkeypatch.setenv("SPYRE_TEST_TIERS", "unit regression trunk")
+    monkeypatch.setenv("SPYRE_TEST_TIER", "regression")
+    as_regression = _tags(tags.result_tags({}))
+    monkeypatch.setenv("SPYRE_TEST_TIER", "unit")
+    as_unit = _tags(tags.result_tags({}))
+    assert as_regression == as_unit
+
+
+def test_model_tag_still_rides_along(monkeypatch):
+    monkeypatch.setenv("SPYRE_TEST_TIERS", "unit")
+    monkeypatch.delenv("SPYRE_TEST_TIER", raising=False)
+    assert _tags(tags.result_tags({"model": "ibm/granite"})) == [
+        "model__ibm/granite",
+        "testtype__unit",
+    ]
+
+
+# ── the ladder must never be inferred ──────────────────────────────────────────
+
+# Ordered weakest-to-strongest. Only used to PROVE the code does not apply it.
+_LADDER = ("unit", "integration", "regression", "trunk")
+
+
+def _declared_sets():
+    """Every `test_types` set declared in the real matrix."""
+    doc = yaml.safe_load(_MATRIX.read_text(encoding="utf-8"))
+    jobs = doc["jobs"]
+    include = jobs["test"]["strategy"]["matrix"]["include"]
+    return [entry["test_types"].split() for entry in include if entry.get("test_types")]
+
+
+def test_the_matrix_still_declares_membership_sets():
+    """Guards the source of truth itself: if `test_types` ever disappears from the
+    matrix, the tags silently degrade to the invoked tier via the fallback."""
+    sets = _declared_sets()
+    assert len(sets) >= 20, f"expected the full shard matrix, got {len(sets)} entries"
+    assert any(len(s) > 1 for s in sets), "no multi-tier entry left to reuse"
+
+
+def test_a_ladder_would_over_claim_this_repos_matrix():
+    """The reason membership is read and not inferred, re-derived from the matrix.
+
+    If this ever finds zero over-claims the ladder has become safe FOR THESE FILES,
+    which is a property of the files and not a rule -- do not start inferring it.
+    """
+    over = []
+    for declared in _declared_sets():
+        idx = [_LADDER.index(t) for t in declared if t in _LADDER]
+        if not idx:
+            continue
+        implied = set(_LADDER[min(idx) :])
+        missing = implied - set(declared)
+        if missing:
+            over.append((sorted(declared), sorted(missing)))
+    assert over, (
+        "a tier ladder no longer over-claims any matrix entry; this is a property of "
+        "the current files, not a licence to infer the ladder"
+    )
+
+
+def test_tags_are_exactly_the_declared_set_not_the_ladder_closure(monkeypatch):
+    """The end-to-end assertion: a leg declaring unit/regression/trunk must NOT be
+    tagged integration, even though integration sits between them in the ladder."""
+    monkeypatch.setenv("SPYRE_TEST_TIERS", "unit regression trunk")
+    monkeypatch.delenv("SPYRE_TEST_TIER", raising=False)
+    values = _tags(tags.result_tags({}))
+    assert "testtype__integration" not in values
+    assert sorted(values) == [
+        "testtype__regression",
+        "testtype__trunk",
+        "testtype__unit",
+    ]
+
+
+# ── the tag shape the ingest parses ────────────────────────────────────────────
+
+
+def test_tag_values_match_the_ingest_namespace_form(monkeypatch):
+    """`namespace__value`, which is what ingest_xml_si.v2_tags_for_case reads off
+    `<property name="tag" value="..."/>`."""
+    monkeypatch.setenv("SPYRE_TEST_TIERS", "unit trunk")
+    monkeypatch.setenv("SPYRE_TEST_TIER", "unit")
+    for name, value in tags.result_tags({"model": "ibm/granite"}):
+        assert name == "tag"
+        assert re.fullmatch(r"[a-z_]+__\S+", value), value