diff --git a/.github/actions/run-matrix-config/action.yml b/.github/actions/run-matrix-config/action.yml index 4b827d28e..c95c5f7f6 100644 --- a/.github/actions/run-matrix-config/action.yml +++ b/.github/actions/run-matrix-config/action.yml @@ -20,8 +20,18 @@ inputs: default: '' test_type: description: >- - Raw test tier; resolved via `make print-test-type` and exported as - SPYRE_TEST_TIER for the testtype__ tag. Empty = Makefile default. + Raw test tier this run was INVOKED as; resolved via `make print-test-type` and + exported as SPYRE_TEST_TIER for the invoked_as__ tag. Empty = Makefile + default. + required: false + default: '' + test_types: + description: >- + Every tier this leg's tests BELONG to, whitespace-separated (the matrix entry's + own `test_types`). Exported as SPYRE_TEST_TIERS, one testtype__ tag each. + This set is what makes coverage reusable -- the invoked tier alone under-reports, + so a later run of another tier re-executes identical work. Empty falls back to + test_type. required: false default: '' runner_labels: @@ -204,6 +214,7 @@ runs: shell: bash env: RAW_TEST_TYPE: ${{ inputs.test_type }} + RAW_TEST_TYPES: ${{ inputs.test_types }} run: | echo "Running tests for config: ${{ inputs.cfg }}${{ inputs.retry == 'true' && ' (pod-level retry)' || '' }}..." # An empty TEST_TYPE fails the Makefile's validation, so omit the arg and @@ -215,6 +226,12 @@ runs: fi export SPYRE_TEST_TIER echo "Resolved SPYRE_TEST_TIER=${SPYRE_TEST_TIER}" + # Passed through verbatim, NOT through print-test-type: this is the matrix + # entry's declared membership set, already valid tier names, and resolving it + # would collapse the set to one value. Unset => tags.test_tiers() falls back to + # the invoked tier. + export SPYRE_TEST_TIERS="${RAW_TEST_TYPES}" + echo "Declared SPYRE_TEST_TIERS=${SPYRE_TEST_TIERS:-}" timeout --signal=TERM --kill-after=1m 90m \ make ${{ inputs.test_target }} COVERAGE=1 JUNIT_XML=junit-${{ inputs.test_target }}.xml echo "tests done" diff --git a/.github/workflows/_test_matrix.yaml b/.github/workflows/_test_matrix.yaml index 3b48e9220..9aa1e3277 100644 --- a/.github/workflows/_test_matrix.yaml +++ b/.github/workflows/_test_matrix.yaml @@ -599,6 +599,10 @@ jobs: test_target: ${{ matrix.test_target }} durations_run_id: ${{ needs.resolve_durations.outputs.durations_run_id }} test_type: ${{ inputs.test_type }} + # The matrix entry's own declared membership set, not the invoked tier: one + # testtype__ tag per member, which is what lets a later run of another tier + # reuse this leg's coverage instead of re-running it. + test_types: ${{ matrix.test_types }} # Echo the exact runner-label selectors this job's runs-on used (matrix.runs_on plus the per-PR runner_label override, or the standing matrix.image_label). runner_labels: ${{ format('["{0}","{1}"]', join(matrix.runs_on, '","'), inputs.runner_label || matrix.image_label) }} job_slot: ${{ strategy.job-index }} @@ -683,6 +687,9 @@ jobs: test_target: ${{ matrix.test_target }} durations_run_id: ${{ needs.resolve_durations.outputs.durations_run_id }} test_type: ${{ inputs.test_type }} + # Survives the retry hop: the failed-suite descriptor is toJSON(matrix), so + # the whole entry -- test_types included -- is rebuilt by collect_failed_suites. + test_types: ${{ matrix.test_types }} runner_labels: ${{ format('["{0}","{1}"]', join(matrix.runs_on, '","'), inputs.runner_label || matrix.image_label) }} job_slot: retry-${{ strategy.job-index }} retry: 'true' diff --git a/tests/plugin/spyre_testing_plugin/tags.py b/tests/plugin/spyre_testing_plugin/tags.py index 5f5a357b7..1d40f73f6 100644 --- a/tests/plugin/spyre_testing_plugin/tags.py +++ b/tests/plugin/spyre_testing_plugin/tags.py @@ -44,12 +44,49 @@ def model_from_params(params): def test_tier(): - """Suite tier from SPYRE_TEST_TIER (exported by CI's run-matrix-config; empty - on a local run). Read live rather than at import so tests that monkeypatch the - env still see it.""" + """How this run was INVOKED, from SPYRE_TEST_TIER (exported by CI's + run-matrix-config; empty on a local run). Read live rather than at import so tests + that monkeypatch the env still see it. + + One value, and NOT the same fact as test_tiers() below: a leg invoked as + `regression` may hold tests that also belong to `unit` and `integration`. + + Deliberately NOT emitted as a case tag. The invocation is a property of the RUN, + while case tags are hashed into test_case_id (see ingest_xml_si.v2_test_case_id) -- + tagging it would give one test three identities depending on which tier happened to + launch it, which is the fragmentation the derived identity exists to prevent. It + belongs on the run row instead, where si_test_runs.test_type already records it. + """ return os.environ.get("SPYRE_TEST_TIER", "") +def test_tiers(): + """Every tier this leg's tests BELONG to, from SPYRE_TEST_TIERS. + + Declared per matrix entry as `test_types` in _test_matrix.yaml and threaded + through run-matrix-config; whitespace-separated, e.g. + "unit integration regression trunk". + + This is the set that makes coverage reusable. The invocation tier alone cannot: + a `regression` run of a leg whose tests are also `unit` and `integration` members + would record only `regression`, so a later `integration` run finds no coverage and + re-executes identical work. + + Membership is read from the declaration and never inferred from a tier ladder. + Measured on this repo's 32 matrix entries: 13 declare `unit regression trunk` + WITHOUT `integration`, so a ladder ("in unit => in everything above") would claim + coverage for 13 legs that never ran integration and silently skip real tests. + + Falls back to the single invocation tier when unset, so a local `make test` and any + caller not yet passing test_types keep emitting a tag rather than none. + """ + raw = os.environ.get("SPYRE_TEST_TIERS", "") + if not raw.strip(): + tier = test_tier() + return [tier] if tier else [] + return sorted(set(raw.split())) + + def result_tags(params): """The (name, value) JUnit property pairs for these params; empty when no model param is recognized and no tier is set, so callers append @@ -59,7 +96,6 @@ def result_tags(params): model = model_from_params(params) if model: tags.append(("tag", f"model__{model}")) - tier = test_tier() - if tier: + for tier in test_tiers(): tags.append(("tag", f"testtype__{tier}")) return tags diff --git a/tests/test_tags.py b/tests/test_tags.py new file mode 100644 index 000000000..38d1f08d3 --- /dev/null +++ b/tests/test_tags.py @@ -0,0 +1,202 @@ +# Copyright 2026 The Spyre-Inference Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CPU-only tests for the JUnit result tags (no hardware needed). + +The invariant that matters: `testtype__` carries the tiers a test BELONGS to, read +from the matrix entry's declared set, and is never inferred from a tier ladder. A +ladder over-claims coverage, and over-claimed coverage silently skips real tests -- +so the ladder tests below are the ones that must not regress. +""" + +import re +from pathlib import Path + +import yaml + +# Imported as a MODULE, deliberately: `test_tier`/`test_tiers` start with `test_`, so +# importing them by name makes pytest collect the production functions themselves as +# (vacuously passing) tests. +from spyre_testing_plugin import tags + +_REPO_ROOT = Path(__file__).resolve().parents[1] +_MATRIX = _REPO_ROOT / ".github" / "workflows" / "_test_matrix.yaml" + + +def _tags(pairs): + """Just the tag values; every pair's name is the literal 'tag'.""" + assert {n for n, _ in pairs} <= {"tag"} + return [v for _, v in pairs] + + +# ── the declared set ──────────────────────────────────────────────────────────── + + +def test_tiers_reads_the_declared_set(monkeypatch): + monkeypatch.setenv("SPYRE_TEST_TIERS", "unit integration regression trunk") + assert tags.test_tiers() == ["integration", "regression", "trunk", "unit"] + + +def test_tiers_dedups_and_sorts(monkeypatch): + # Sorted and deduped so two writers agree on the same set regardless of the + # order it was declared in -- the tags feed a content hash downstream. + monkeypatch.setenv("SPYRE_TEST_TIERS", "trunk unit trunk unit") + assert tags.test_tiers() == ["trunk", "unit"] + + +def test_every_declared_tier_becomes_a_tag(monkeypatch): + monkeypatch.setenv("SPYRE_TEST_TIERS", "unit integration regression trunk") + monkeypatch.delenv("SPYRE_TEST_TIER", raising=False) + assert _tags(tags.result_tags({})) == [ + "testtype__integration", + "testtype__regression", + "testtype__trunk", + "testtype__unit", + ] + + +# ── the fallback: a caller that does not pass the set still tags ──────────────── + + +def test_falls_back_to_the_invoked_tier(monkeypatch): + """A local `make test` and any un-updated caller keep emitting a tag.""" + monkeypatch.delenv("SPYRE_TEST_TIERS", raising=False) + monkeypatch.setenv("SPYRE_TEST_TIER", "regression") + assert tags.test_tiers() == ["regression"] + + +def test_blank_tiers_falls_back_too(monkeypatch): + # The action exports "" when the input is unset, so whitespace must not read as + # a one-element set containing the empty string. + monkeypatch.setenv("SPYRE_TEST_TIERS", " ") + monkeypatch.setenv("SPYRE_TEST_TIER", "unit") + assert tags.test_tiers() == ["unit"] + + +def test_no_tier_anywhere_emits_no_tier_tag(monkeypatch): + monkeypatch.delenv("SPYRE_TEST_TIERS", raising=False) + monkeypatch.delenv("SPYRE_TEST_TIER", raising=False) + assert tags.test_tiers() == [] + assert _tags(tags.result_tags({})) == [] + + +# ── membership vs invocation are different facts ──────────────────────────────── + + +def test_the_invocation_is_never_a_case_tag(monkeypatch): + """The invocation must NOT reach the tags, because case tags are hashed into + test_case_id: tagging it would give one test a different identity per invoking + tier. It is a property of the run (si_test_runs.test_type), not of the test.""" + monkeypatch.setenv("SPYRE_TEST_TIERS", "unit regression trunk") + monkeypatch.setenv("SPYRE_TEST_TIER", "regression") + values = _tags(tags.result_tags({})) + assert not any("invoked_as" in v for v in values) + assert values == [ + "testtype__regression", + "testtype__trunk", + "testtype__unit", + ] + # Still readable for whoever records the run row. + assert tags.test_tier() == "regression" + + +def test_identity_does_not_vary_by_invoking_tier(monkeypatch): + """The concrete regression: the same test, same declared membership, invoked as + two different tiers, must produce the SAME tag list -- otherwise its hashed + test_case_id splits and every trend query fragments.""" + monkeypatch.setenv("SPYRE_TEST_TIERS", "unit regression trunk") + monkeypatch.setenv("SPYRE_TEST_TIER", "regression") + as_regression = _tags(tags.result_tags({})) + monkeypatch.setenv("SPYRE_TEST_TIER", "unit") + as_unit = _tags(tags.result_tags({})) + assert as_regression == as_unit + + +def test_model_tag_still_rides_along(monkeypatch): + monkeypatch.setenv("SPYRE_TEST_TIERS", "unit") + monkeypatch.delenv("SPYRE_TEST_TIER", raising=False) + assert _tags(tags.result_tags({"model": "ibm/granite"})) == [ + "model__ibm/granite", + "testtype__unit", + ] + + +# ── the ladder must never be inferred ────────────────────────────────────────── + +# Ordered weakest-to-strongest. Only used to PROVE the code does not apply it. +_LADDER = ("unit", "integration", "regression", "trunk") + + +def _declared_sets(): + """Every `test_types` set declared in the real matrix.""" + doc = yaml.safe_load(_MATRIX.read_text(encoding="utf-8")) + jobs = doc["jobs"] + include = jobs["test"]["strategy"]["matrix"]["include"] + return [entry["test_types"].split() for entry in include if entry.get("test_types")] + + +def test_the_matrix_still_declares_membership_sets(): + """Guards the source of truth itself: if `test_types` ever disappears from the + matrix, the tags silently degrade to the invoked tier via the fallback.""" + sets = _declared_sets() + assert len(sets) >= 20, f"expected the full shard matrix, got {len(sets)} entries" + assert any(len(s) > 1 for s in sets), "no multi-tier entry left to reuse" + + +def test_a_ladder_would_over_claim_this_repos_matrix(): + """The reason membership is read and not inferred, re-derived from the matrix. + + If this ever finds zero over-claims the ladder has become safe FOR THESE FILES, + which is a property of the files and not a rule -- do not start inferring it. + """ + over = [] + for declared in _declared_sets(): + idx = [_LADDER.index(t) for t in declared if t in _LADDER] + if not idx: + continue + implied = set(_LADDER[min(idx) :]) + missing = implied - set(declared) + if missing: + over.append((sorted(declared), sorted(missing))) + assert over, ( + "a tier ladder no longer over-claims any matrix entry; this is a property of " + "the current files, not a licence to infer the ladder" + ) + + +def test_tags_are_exactly_the_declared_set_not_the_ladder_closure(monkeypatch): + """The end-to-end assertion: a leg declaring unit/regression/trunk must NOT be + tagged integration, even though integration sits between them in the ladder.""" + monkeypatch.setenv("SPYRE_TEST_TIERS", "unit regression trunk") + monkeypatch.delenv("SPYRE_TEST_TIER", raising=False) + values = _tags(tags.result_tags({})) + assert "testtype__integration" not in values + assert sorted(values) == [ + "testtype__regression", + "testtype__trunk", + "testtype__unit", + ] + + +# ── the tag shape the ingest parses ──────────────────────────────────────────── + + +def test_tag_values_match_the_ingest_namespace_form(monkeypatch): + """`namespace__value`, which is what ingest_xml_si.v2_tags_for_case reads off + ``.""" + monkeypatch.setenv("SPYRE_TEST_TIERS", "unit trunk") + monkeypatch.setenv("SPYRE_TEST_TIER", "unit") + for name, value in tags.result_tags({"model": "ibm/granite"}): + assert name == "tag" + assert re.fullmatch(r"[a-z_]+__\S+", value), value