torch-spyre · ashokponkumar · Sep 10, 2026 · Sep 10, 2026
@@ -85,7 +85,7 @@ adapter-coverage-tests: ## Run adapter registry coverage check (suite key: adapt
 	$(PYTEST) -v --noconftest tests/test_adapter_coverage.py $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
 
 smoke-tests: ## Run e2e smoke tests (suite key: smoke)
-	$(PYTEST) $(PYTEST_ARGS) tests/spyre/test_e2e_smoke_spyre.py $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
+	$(PYTEST) $(PYTEST_ARGS) --suite smoke tests/spyre/test_e2e_smoke_spyre.py $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
 
 load-tests: ## Run load tests (suite key: load)
 	# test_load_spyre.py is the one suite file with FOUR model_path-parametrized
@@ -97,44 +97,44 @@ load-tests: ## Run load tests (suite key: load)
 	# selects the one matching parametrization, same as every other suite target
 	# gets from --model-path (safe here because MODEL_PATH is always one exact
 	# registry path, never an attacker-controlled or ambiguous substring).
-	$(PYTEST) $(PYTEST_ARGS) tests/spyre/test_load_spyre.py $(if $(MODEL_PATH),-k "$(MODEL_PATH)",$(K_ARGS)) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
+	$(PYTEST) $(PYTEST_ARGS) --suite load tests/spyre/test_load_spyre.py $(if $(MODEL_PATH),-k "$(MODEL_PATH)",$(K_ARGS)) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
 
 token-compare-tests: ## Run token-compare tests (suite key: token_compare)
-	$(PYTEST) $(PYTEST_ARGS) tests/spyre/test_e2e_token_compare_spyre.py $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
+	$(PYTEST) $(PYTEST_ARGS) --suite token_compare tests/spyre/test_e2e_token_compare_spyre.py $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
 
 model-components-tests: ## Run model component tests (suite key: model_components)
-	$(PYTEST) $(PYTEST_ARGS) tests/spyre/test_model_components_spyre.py $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
+	$(PYTEST) $(PYTEST_ARGS) --suite model_components tests/spyre/test_model_components_spyre.py $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
 
 embed-compare-tests: ## Run embed-compare tests (suite key: embed_compare)
-	$(PYTEST) $(PYTEST_ARGS) tests/spyre/test_e2e_embed_compare_spyre.py $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
+	$(PYTEST) $(PYTEST_ARGS) --suite embed_compare tests/spyre/test_e2e_embed_compare_spyre.py $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
 
 clip-tests: ## Run CLIP e2e tests (suite key: clip)
-	$(PYTEST) $(PYTEST_ARGS) tests/spyre/test_e2e_clip_compare_spyre.py $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
+	$(PYTEST) $(PYTEST_ARGS) --suite clip tests/spyre/test_e2e_clip_compare_spyre.py $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
 
 vlm-tests: ## Run VLM e2e tests (suite key: vlm)
-	$(PYTEST) $(PYTEST_ARGS) tests/spyre/test_vlm_e2e_spyre.py $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
+	$(PYTEST) $(PYTEST_ARGS) --suite vlm tests/spyre/test_vlm_e2e_spyre.py $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
 
 reranker-tests: ## Run reranker compare tests (suite key: reranker_compare)
-	$(PYTEST) $(PYTEST_ARGS) tests/spyre/test_e2e_reranker_compare_spyre.py $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
+	$(PYTEST) $(PYTEST_ARGS) --suite reranker_compare tests/spyre/test_e2e_reranker_compare_spyre.py $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
 
 masked-lm-compare-tests: ## Run masked-LM compare tests (suite key: masked_lm_compare)
-	$(PYTEST) $(PYTEST_ARGS) tests/spyre/test_e2e_masked_lm_compare_spyre.py $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
+	$(PYTEST) $(PYTEST_ARGS) --suite masked_lm_compare tests/spyre/test_e2e_masked_lm_compare_spyre.py $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
 
 question-answering-compare-tests: ## Run question-answering compare tests (suite key: question_answering_compare)
-	$(PYTEST) $(PYTEST_ARGS) tests/spyre/test_e2e_question_answering_compare_spyre.py $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
+	$(PYTEST) $(PYTEST_ARGS) --suite question_answering_compare tests/spyre/test_e2e_question_answering_compare_spyre.py $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
 
 seq-classification-compare-tests: ## Run seq-classification compare tests (suite key: seq_classification_compare)
-	$(PYTEST) $(PYTEST_ARGS) tests/spyre/test_e2e_seq_classification_compare_spyre.py $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
+	$(PYTEST) $(PYTEST_ARGS) --suite seq_classification_compare tests/spyre/test_e2e_seq_classification_compare_spyre.py $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
 
 token-classification-compare-tests: ## Run token-classification compare tests (suite key: token_classification_compare)
-	$(PYTEST) $(PYTEST_ARGS) tests/spyre/test_e2e_token_classification_compare_spyre.py $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
+	$(PYTEST) $(PYTEST_ARGS) --suite token_classification_compare tests/spyre/test_e2e_token_classification_compare_spyre.py $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
 
 # EDGE_CASE_FILE narrows edge-cases-tests to one file under tests/spyre/edge_cases/
 # (matrix-style per-file CI jobs pass this); empty = run every file in the directory.
 # All edge-case tests are @pytest.mark.slow, so --run-slow is passed unconditionally.
 EDGE_CASE_FILE ?=
 edge-cases-tests: ## Run edge-case tests (suite key: edge_cases; EDGE_CASE_FILE=<file>.py narrows to one)
-	$(PYTEST) $(PYTEST_ARGS) --run-slow $(if $(EDGE_CASE_FILE),tests/spyre/edge_cases/$(EDGE_CASE_FILE),tests/spyre/edge_cases/) $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
+	$(PYTEST) $(PYTEST_ARGS) --suite edge_cases --run-slow $(if $(EDGE_CASE_FILE),tests/spyre/edge_cases/$(EDGE_CASE_FILE),tests/spyre/edge_cases/) $(K_ARGS) $(MODEL_PATH_ARGS) $(if $(JUNIT_XML),--junitxml=$(JUNIT_XML))
 
 # MODULE_CONFIG narrows model-module-tests to one YAML config (matrix-style
 # per-config CI jobs pass this); empty = run every config in tests/configs/module_tests.

@@ -0,0 +1,121 @@
+# Copyright 2026 The Torch-Spyre Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Which test tiers each suite belongs to, for the JUnit `testtype__<tier>` tags.
+
+WHY THE TAGS EXIST. The CI/CD warehouse answers "has this artifact already been tested
+at tier X" by reading these tags off the ingested JUnit XML. A run therefore has to record
+every tier its tests BELONG to, not the one tier that happened to invoke it: a `regression`
+run of a suite that is also a `unit` and `integration` member must say so, or a later
+`integration` run finds no coverage and re-executes identical work.
+
+WHY A TABLE HERE. A suite target (`make smoke-tests`) does not know its tiers -- the
+mapping lives in the CI job gates, and CI calls the per-suite targets directly, never
+`make tests`. So the tier cannot be recovered at test time from TEST_TYPE; it has to be
+declared. `--suite <key>` names the suite, and this table turns that into the tier set.
+
+MEMBERSHIP IS DECLARED, NEVER INFERRED FROM A LADDER. It is tempting to treat the tiers as
+nested (unit < integration < regression < trunk) and expand upward. Do not: 11 of the 15
+suites below are `[regression, trunk, unit]` with NO `integration`, so closing the ladder
+would claim integration coverage for suites that never ran under it and silently skip real
+tests. torch-spyre learned the same lesson -- see the "never inferred from a ladder" note
+in its filter_configs.py.
+
+SOURCE OF TRUTH is each suite job's `if:` gate in .github/workflows/_test_matrix.yaml,
+because that is what actually decides whether a suite runs. The Makefile's `tests` target
+carries a second, HAND-MAINTAINED copy in its `case` block, and the two have already
+drifted (see tests/test_tier_tags.py, which pins the difference): the Makefile omits `clip`
+from every suites= list, has no `multicard_smoke` target, and puts `model_components` only
+in regression/trunk even though Makefile:8 documents it as an integration suite. Keep this
+table in step with the WORKFLOW; the test reports drift rather than guessing.
+"""
+
+from __future__ import annotations
+
+# suite key -> the tiers whose runs include that suite.
+# Mirrors the `if:` gate of each suite job in .github/workflows/_test_matrix.yaml.
+# Five suites are deliberately absent. tier_tags() returns [] for any of them, so their
+# cases still carry a model tag -- they simply contribute no tier-coverage data.
+#   perf             -- a scaffold that echoes an empty JUnit file with no <testcase>
+#                       elements (Makefile `tests` target): nothing to tag.
+#   edge_cases       -- gated on `inputs.edge_cases_only`, never on a tier, and absent
+#                       from every suites= list. Its target still passes --suite so the
+#                       key is declared in one place if it ever joins a tier.
+#   multicard_smoke  -- CI runs scripts/run_multicard_smoke.py under torchrun, not
+#                       pytest, so it emits no JUnit XML at all.
+#   adapter_coverage -- runs with --noconftest (Makefile), so the autouse fixture never
+#                       binds; passing --suite there would be an unknown-option error.
+#   model_module     -- delegates to the oot_framework run_test.sh, which does its own
+#                       marker-based tagging (torch-spyre's mechanism), not this one.
+SUITE_TIERS: dict[str, tuple[str, ...]] = {
+    "clip": ("regression", "trunk", "unit"),
+    "embed_compare": ("regression", "trunk", "unit"),
+    "load": ("regression", "trunk", "unit"),
+    "masked_lm_compare": ("regression", "trunk", "unit"),
+    "model_components": ("integration", "regression", "trunk", "unit"),
+    "question_answering_compare": ("regression", "trunk", "unit"),
+    "reranker_compare": ("regression", "trunk", "unit"),
+    "seq_classification_compare": ("regression", "trunk", "unit"),
+    "smoke": ("regression", "trunk"),
+    "token_classification_compare": ("regression", "trunk", "unit"),
+    "token_compare": ("integration", "regression", "trunk", "unit"),
+    "vlm": ("regression", "trunk", "unit"),
+}
+
+# Parametrize argnames whose value names the model under test. `model_path` covers almost
+# every parametrized test here (tests/conftest.py's pytest_generate_tests owns that axis
+# and can rewrite it from --model-path); the rest are for the few suites using other names.
+MODEL_PARAM_NAMES = ("model_path", "model", "model_key", "model_info")
+
+
+def tier_tags(suite: str) -> list[str]:
+    """`testtype__<tier>` for every tier `suite` belongs to; empty for an unknown suite.
+
+    Empty rather than raising: an untagged case is a gap in reuse data, while a raise
+    would fail a test run over a reporting concern.
+    """
+    return [f"testtype__{tier}" for tier in SUITE_TIERS.get(suite, ())]
+
+
+def model_tag(params) -> str | None:
+    """`model__<id>` from a test's parametrization, or None when no model param is bound."""
+    for name in MODEL_PARAM_NAMES:
+        if name not in params:
+            continue
+        value = params[name]
+        if value is None:
+            continue
+        # (model_id, ...) tuple: the id is the first element.
+        if isinstance(value, (tuple, list)) and value:
+            value = value[0]
+        # vLLM-style model-info objects carry the id on .name.
+        name_attr = getattr(value, "name", None)
+        text = str(name_attr if name_attr is not None else value).strip()
+        return f"model__{text}" if text else None
+    return None
+
+
+def result_tags(suite: str, params) -> list[tuple[str, str]]:
+    """The (name, value) JUnit property pairs for one test case.
+
+    Emitted as `<property name="tag" value="namespace__value"/>`, the shape the ClickHouse
+    ingest reads (see .github/scripts/ingest_xml_hf_adapters.py extract_properties).
+    """
+    tags: list[tuple[str, str]] = []
+    model = model_tag(params)
+    if model:
+        tags.append(("tag", model))
+    for tag in tier_tags(suite):
+        tags.append(("tag", tag))
+    return tags
@@ -45,6 +45,7 @@
 import os
 import sys
 import types
+import warnings
 from typing import Any, Union
 
 import pytest
@@ -182,6 +183,49 @@ def pytest_addoption(parser: Parser) -> None:
             "in the test decorators are ignored."
         ),
     )
+    parser.addoption(
+        "--suite",
+        default="",
+        help=(
+            "Suite key this run belongs to (e.g. ``smoke``, ``token_compare``), used "
+            "only to stamp the JUnit ``testtype__<tier>`` tags the CI/CD warehouse "
+            "reads. Each Makefile suite target passes its own key; see "
+            "tests/_tier_tags.py for the suite -> tier table. Unset means no tier "
+            "tag, which costs reuse data but never fails a run."
+        ),
+    )
+
+
+@pytest.fixture(autouse=True)
+def _emit_result_tags(request, record_property):
+    """Stamp each test's ``testtype__<tier>`` / ``model__<id>`` JUnit tags.
+
+    The CI/CD warehouse reads these to decide whether a tier's coverage already exists
+    for an artifact, so a run must record every tier its suite BELONGS to (from
+    ``--suite`` via tests/_tier_tags.py), not the one that invoked it. Autouse because
+    the tags describe every case, not an opt-in subset.
+
+    Imported inside the fixture and wrapped: tagging is REPORTING, so it must never be
+    able to fail a test run. At conftest module scope a bad import aborts collection for
+    the whole suite, and pytest then reports the first failing import in the chain --
+    which can look like an unrelated dependency error rather than this one.
+    """
+    suite = request.config.getoption("--suite")
+    if not suite:
+        return
+    try:
+        from tests._tier_tags import result_tags
+    except Exception as exc:  # pragma: no cover - defensive
+        warnings.warn(
+            f"result tags unavailable, tests run untagged: {exc!r}", stacklevel=1
+        )
+        return
+    params = getattr(getattr(request.node, "callspec", None), "params", {})
+    try:
+        for name, value in result_tags(suite, params):
+            record_property(name, value)
+    except Exception as exc:  # pragma: no cover - defensive
+        warnings.warn(f"could not stamp result tags: {exc!r}", stacklevel=1)
 
 
 def pytest_generate_tests(metafunc: Metafunc) -> None: