tests: Enable a subset of GSM8k upstream vllm tests

.github/workflows/_test_matrix.yaml

@@ -492,6 +492,35 @@ jobs:
               - linux
             image_label: image_torch_spyre
             test_target: test-upstream-shard-6
+          # GSM8K accuracy gate, one server-based eval per config sharded across
+          # parallel 1-card jobs (one config per shard). Heavy (full vLLM server +
+          # batched eval per config), so regression/trunk only -- excluded from
+          # integration/unit like the upstream suites. Adding/removing a shard =
+          # add/remove an entry here and bump QUALITY_SHARDS (Makefile default).
+          - cfg: Quality accuracy gate (shard 1/3)
+            test_types: regression trunk
+            runs_on:
+              - x86_64
+              - spyre_pf_x1
+              - linux
+            image_label: image_torch_spyre
+            test_target: test-quality-shard-0
+          - cfg: Quality accuracy gate (shard 2/3)
+            test_types: regression trunk
+            runs_on:
+              - x86_64
+              - spyre_pf_x1
+              - linux
+            image_label: image_torch_spyre
+            test_target: test-quality-shard-1
+          - cfg: Quality accuracy gate (shard 3/3)
+            test_types: regression trunk
+            runs_on:
+              - x86_64
+              - spyre_pf_x1
+              - linux
+            image_label: image_torch_spyre
+            test_target: test-quality-shard-2
           - cfg: Upstream distributed vLLM tests
             test_types: regression trunk
             runs_on:

Makefile

@@ -121,7 +121,8 @@ RESULTS_DIR ?= .
 .PHONY: help test tests run-one aiu-setup perf-tests coverage print-test-type \
         test-smoke test-smoke-shard test-probes test-probes-shard test-attention test-attention-shard \
         test-distributed test-distributed-shard test-upstream test-upstream-shard \
-        test-upstream-distributed tests-single-card tests-multi-card
+        test-upstream-distributed test-quality test-quality-shard \
+        tests-single-card tests-multi-card
 
 help: ## Show this help message
 	@awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -256,8 +257,10 @@ test-distributed-shard: ## Run one distributed shard (DIST_SHARDS=N DIST_SHARD_I
 test-distributed-shard-%:
 	$(MAKE) test-distributed-shard DIST_SHARD_ID=$* JUNIT_XML=$(JUNIT_XML)
 
+# `not gsm8k` carves the GSM8K accuracy gate out of the upstream suite: it also
+# carries the `upstream` marker but is its own sharded suite (test-quality below).
 test-upstream: ## Run the upstream (non-distributed) marker combo, unsharded (local full run).
-	$(MAKE) run-one MARK_OVERRIDE='upstream and not distributed' JUNIT_XML=$(JUNIT_XML)
+	$(MAKE) run-one MARK_OVERRIDE='upstream and not distributed and not gsm8k' JUNIT_XML=$(JUNIT_XML)
 
 # Non-distributed upstream tests are sharded across parallel CI jobs. The heavy
 # model tests (under a models/ path) used to be a separate test-upstream-model
@@ -267,7 +270,7 @@ test-upstream: ## Run the upstream (non-distributed) marker combo, unsharded (lo
 UPSTREAM_SHARDS ?= 7
 UPSTREAM_SHARD_ID ?= 0
 test-upstream-shard: ## Run one non-distributed upstream shard (UPSTREAM_SHARDS=N UPSTREAM_SHARD_ID=i).
-	$(MAKE) run-one MARK_OVERRIDE='upstream and not distributed' \
+	$(MAKE) run-one MARK_OVERRIDE='upstream and not distributed and not gsm8k' \
 	  PYTEST_ARGS='$(PYTEST_ARGS) --upstream-shards=$(UPSTREAM_SHARDS) --upstream-shard-id=$(UPSTREAM_SHARD_ID)' \
 	  JUNIT_XML=$(JUNIT_XML)
 
@@ -280,9 +283,33 @@ test-upstream-shard-%:
 test-upstream-distributed: ## Run the upstream+distributed marker combo.
 	$(MAKE) run-one MARK_OVERRIDE='upstream and distributed' OMP_THREADS=1 JUNIT_XML=$(JUNIT_XML)
 
+# The GSM8K accuracy gate: server-based correctness evals kept in their own suite
+# (not the upstream per-op suite) because each one starts a vLLM server and runs a
+# batched eval -- minutes per config, single card. `gsm8k and upstream` names the
+# `upstream` marker so the eval file is collected from the cached vLLM checkout
+# (as test-upstream does); no --upstream flag needed.
+test-quality: ## Run the GSM8K accuracy gate (server-based evals), unsharded (local full run). Needs 1 card.
+	$(MAKE) run-one MARK_OVERRIDE='gsm8k and upstream' JUNIT_XML=$(JUNIT_XML)
+
+# One eval per config (gsm8k_configs/models-spyre.txt) sharded across parallel
+# 1-card CI jobs, one config per shard. The plugin owns the partition
+# (--quality-shards), balancing by recorded runtime when a durations file is
+# present, else evenly. QUALITY_SHARDS is the single source of the count.
+QUALITY_SHARDS ?= 3
+QUALITY_SHARD_ID ?= 0
+test-quality-shard: ## Run one GSM8K quality shard (QUALITY_SHARDS=N QUALITY_SHARD_ID=i). Needs 1 card.
+	$(MAKE) run-one MARK_OVERRIDE='gsm8k and upstream' \
+	  PYTEST_ARGS='$(PYTEST_ARGS) --quality-shards=$(QUALITY_SHARDS) --quality-shard-id=$(QUALITY_SHARD_ID)' \
+	  JUNIT_XML=$(JUNIT_XML)
+
+# CI runs one matrix job per shard as `test-quality-shard-<i>` so each JUnit
+# artifact name is unique; the pattern maps <i> to QUALITY_SHARD_ID.
+test-quality-shard-%:
+	$(MAKE) test-quality-shard QUALITY_SHARD_ID=$* JUNIT_XML=$(JUNIT_XML)
+
 # Single-card / multi-card split, grouping the 6 marker combos above by how many cards they need.
 # Each suite gets its own junit-<target>/junit-<target>.xml subdir, matching GHA's artifact-name/file-name layout (_test_matrix.yaml) so a Jenkins run's JUnit paths line up 1:1 with a GHA run's.
-tests-single-card: ## Run the 1-card marker combos (smoke shards / attention shards / encoder-attention / upstream shards). Needs 1 card.
+tests-single-card: ## Run the 1-card marker combos (smoke shards / attention shards / encoder-attention / upstream shards / quality shards). Needs 1 card.
 	mkdir -p "$(RESULTS_DIR)"; \
 	rc=0; \
 	for i in $$(seq 0 $$(( $(SMOKE_SHARDS) - 1 ))); do \
@@ -295,6 +322,9 @@ tests-single-card: ## Run the 1-card marker combos (smoke shards / attention sha
 	for i in $$(seq 0 $$(( $(UPSTREAM_SHARDS) - 1 ))); do \
 	  mkdir -p "$(RESULTS_DIR)/junit-test-upstream-shard-$$i" && $(MAKE) test-upstream-shard UPSTREAM_SHARD_ID=$$i JUNIT_XML="$(RESULTS_DIR)/junit-test-upstream-shard-$$i/junit-test-upstream-shard-$$i.xml" || rc=1; \
 	done; \
+	for i in $$(seq 0 $$(( $(QUALITY_SHARDS) - 1 ))); do \
+	  mkdir -p "$(RESULTS_DIR)/junit-test-quality-shard-$$i" && $(MAKE) test-quality-shard QUALITY_SHARD_ID=$$i JUNIT_XML="$(RESULTS_DIR)/junit-test-quality-shard-$$i/junit-test-quality-shard-$$i.xml" || rc=1; \
+	done; \
 	exit $$rc
 
 tests-multi-card: ## Run the 2-card marker combos (distributed shards/upstream-distributed/probes). Needs 2 cards.

pyproject.toml

@@ -280,6 +280,7 @@ markers = [
     "uses_subprocess: tests that spawn a subprocess engine; the plugin runs them first, before the main process initializes the Spyre device",
     "rotary: Spyre RoPE tests (tests/custom_ops/test_rotary_embedding.py, tests/multimodal/test_pixtral.py)",
     "fp8: FP8 quantization tests (tests/custom_ops/test_fp8.py)",
+    "gsm8k: server-based GSM8K accuracy eval (upstream tests/evals/gsm8k); slow, runs in its own CI job",
     "conv: Spyre Conv2d custom-op tests (tests/custom_ops/test_conv.py)",
     "pixtral: Pixtral vision-tower patch tests (tests/multimodal/test_pixtral.py)",
     "multimodal: VLM tests. Local (tests/e2e/test_multimodal.py) is slow and needs a card; the upstream ones are host-side processor tests and need neither",

tests/plugin/spyre_testing_plugin/gsm8k_configs/granite-4.1-3b.yaml

@@ -0,0 +1,11 @@
+# GSM8K accuracy gate for granite-4.1-3b on Spyre (fp16, no quantized variant).
+# accuracy_threshold is the measured on-device baseline; num_questions is small to gate cheaply.
+# request_timeout_seconds is a *total* per-request budget (incl. queue-wait), so it must exceed the whole batched run's wall-clock.
+model_name: "ibm-granite/granite-4.1-3b"
+accuracy_threshold: 0.625
+num_questions: 64
+num_fewshot: 5
+max_concurrency: 4
+server_args: "--max-model-len 4096"
+request_timeout_seconds: 2400
+startup_max_wait_seconds: 1800

tests/plugin/spyre_testing_plugin/gsm8k_configs/mistral-7b-instruct-v0.3.yaml

@@ -0,0 +1,11 @@
+# GSM8K accuracy gate for Mistral-7B-Instruct-v0.3 on Spyre (fp16, no quantized variant).
+# accuracy_threshold is the measured on-device baseline; num_questions is small to gate cheaply.
+# request_timeout_seconds is a *total* per-request budget (incl. queue-wait), so it must exceed the whole batched run's wall-clock.
+model_name: "mistralai/Mistral-7B-Instruct-v0.3"
+accuracy_threshold: 0.375
+num_questions: 64
+num_fewshot: 5
+max_concurrency: 4
+server_args: "--max-model-len 4096"
+request_timeout_seconds: 2400
+startup_max_wait_seconds: 1800

tests/plugin/spyre_testing_plugin/gsm8k_configs/models-spyre.txt

@@ -0,0 +1,6 @@
+# GSM8K eval configs runnable on Spyre (fp16, no quantization).
+# Consumed by upstream tests/evals/gsm8k/test_gsm8k_correctness.py; selected via
+# the `config_list` field on that file's entry in upstream_tests.yaml.
+qwen2.5-0.5b-instruct.yaml
+granite-4.1-3b.yaml
+mistral-7b-instruct-v0.3.yaml

tests/plugin/spyre_testing_plugin/gsm8k_configs/qwen2.5-0.5b-instruct.yaml

@@ -0,0 +1,11 @@
+# GSM8K accuracy gate for Qwen2.5-0.5B-Instruct on Spyre (fp16, no quantized variant).
+# accuracy_threshold is the measured on-device baseline; num_questions is small to gate cheaply.
+# request_timeout_seconds is a *total* per-request budget (incl. queue-wait), so it must exceed the whole batched run's wall-clock.
+model_name: "Qwen/Qwen2.5-0.5B-Instruct"
+accuracy_threshold: 0.3125
+num_questions: 64
+num_fewshot: 5
+max_concurrency: 4
+server_args: "--max-model-len 4096"
+request_timeout_seconds: 2400
+startup_max_wait_seconds: 1800

tests/plugin/spyre_testing_plugin/models.py

@@ -99,15 +99,21 @@ class FileConfig:
     """Filter configuration for a single upstream test file.
 
     Attributes:
-        rel_path:   Path relative to upstream repo root
-                    (e.g. "tests/kernels/core/test_layernorm.py").
-        allow_list: Tests allowed to run from this file.
-        block_list: Tests blocked from running (takes precedence over allow_list).
+        rel_path:    Path relative to upstream repo root
+                     (e.g. "tests/kernels/core/test_layernorm.py").
+        allow_list:  Tests allowed to run from this file.
+        block_list:  Tests blocked from running (takes precedence over allow_list).
+        config_list: Optional path (relative to this plugin package) to a config-list
+                     file for tests parametrized by an upstream `config_filename`
+                     fixture (e.g. tests/evals/gsm8k). When set, it overrides the
+                     upstream conftest's `--config-list-file` so its own
+                     `pytest_generate_tests` parametrizes from Spyre-owned configs.
     """
 
     rel_path: str
     allow_list: tuple[AllowEntry, ...] = ()
     block_list: tuple[BlockEntry, ...] = ()
+    config_list: str | None = None
 
 
 @dataclass(frozen=True)

tests/plugin/spyre_testing_plugin/pytest_plugin.py

@@ -215,6 +215,7 @@ def _parse_config(raw_tests: dict) -> UpstreamTestConfig:
                 rel_path=file_entry["rel_path"],
                 allow_list=tuple(allow_list),
                 block_list=tuple(block_list),
+                config_list=file_entry.get("config_list"),
             )
         )
     return UpstreamTestConfig(files=tuple(files))
@@ -772,6 +773,15 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
     if not fc:
         return
 
+    # Tests parametrized by an upstream `config_filename` fixture (e.g. gsm8k evals)
+    # aren't reachable via param_overrides, which only rewrites the test's own
+    # parametrize markers. Instead, point the upstream conftest's --config-list-file at
+    # a Spyre-owned list; its own (later-running) pytest_generate_tests then parametrizes
+    # config_filename from our configs. tryfirst here guarantees we set it first.
+    if fc.config_list and "config_filename" in metafunc.fixturenames:
+        list_path = (_YAML_PATH.parent / fc.config_list).resolve()
+        metafunc.config.option.config_list_file = str(list_path)
+
     test_name = metafunc.definition.originalname or metafunc.definition.name
     allow_entry = _find_allow_entry(test_name, fc.allow_list)
     if not allow_entry or not allow_entry.param_overrides:

tests/plugin/spyre_testing_plugin/sharding.py

@@ -14,7 +14,7 @@
 
 """Duration-weighted test sharding for CI fan-out.
 
-Each suite (attn/smoke/upstream/dist/probe) is split across N parallel jobs; a job
+Each suite (attn/smoke/upstream/dist/probe/quality) is split across N parallel jobs; a job
 keeps only its shard's slice. Every shard job computes the same weighted greedy
 longest-processing-time partition, so no cross-job coordination is needed and
 the union of all shards is the full selection exactly once (guarded by
@@ -60,7 +60,7 @@ def _will_skip(item: pytest.Item) -> bool:
 def add_shard_options(parser) -> None:
     """Register one --<suite>-shards / --<suite>-shard-id pair per CI suite."""
     group = parser.getgroup("spyre-test-sharding")
-    for suite in ("attn", "smoke", "upstream", "dist", "probe"):
+    for suite in ("attn", "smoke", "upstream", "dist", "probe", "quality"):
         group.addoption(
             f"--{suite}-shards",
             type=int,
@@ -93,6 +93,7 @@ def apply_shards(config: pytest.Config, items: list[pytest.Item]) -> None:
     _apply_upstream_shard(config, items)
     _apply_distributed_shard(config, items)
     _apply_probe_shard(config, items)
+    _apply_quality_shard(config, items)
 
 
 def _load_durations(config: pytest.Config) -> dict[str, float]:
@@ -297,12 +298,14 @@ def _apply_upstream_shard(config: pytest.Config, items: list[pytest.Item]) -> No
     def weight(item: pytest.Item) -> int:
         return 8 if "models/" in item.nodeid else 1
 
+    # The GSM8K gate also carries the `upstream` marker but is its own sharded suite
+    # (_apply_quality_shard); excluded here so its server-based evals stay out of this partition.
     _apply_shard(
         config,
         items,
         num_shards=config.getoption("--upstream-shards"),
         shard_id=config.getoption("--upstream-shard-id"),
-        select=lambda item: True,
+        select=lambda item: not item.get_closest_marker("gsm8k"),
         weight=weight,
         label="upstream",
         durations=_load_durations(config),
@@ -352,6 +355,23 @@ def select(item: pytest.Item) -> bool:
     )
 
 
+def _apply_quality_shard(config: pytest.Config, items: list[pytest.Item]) -> None:
+    # The GSM8K accuracy gate (Makefile test-quality: `gsm8k and upstream`), one
+    # server-based eval per config in gsm8k_configs/models-spyre.txt. Each starts a
+    # vLLM server and runs a batched eval -- heavy and roughly per-config uniform, so
+    # a shard per config; durations refine that. Single card each (one server at a time).
+    _apply_shard(
+        config,
+        items,
+        num_shards=config.getoption("--quality-shards"),
+        shard_id=config.getoption("--quality-shard-id"),
+        select=lambda item: bool(item.get_closest_marker("gsm8k")),
+        weight=lambda item: 1,
+        label="quality",
+        durations=_load_durations(config),
+    )
+
+
 # Per-nodeid wall time this session, written out when SPYRE_TEST_DURATIONS_OUT is
 # set (CI). A later run pins the merged file and feeds it back as
 # SPYRE_TEST_DURATIONS so _apply_shard balances shards by measured runtime. Keyed

tests/plugin/spyre_testing_plugin/upstream_tests.yaml

@@ -130,6 +130,19 @@ tests:
           mode: skip # model_info0 (Qwen3-0.6B): RAS::VFIO::MapDMAFailed; model_info1 (Qwen3-0.6B-FP8): KeyError PlatformEnum.OOT - FP8 quant not supported on Spyre; model_info2 (Qwen3.5-0.8B): Qwen3_5ForConditionalGeneration not in vLLM registry
           tags: [model, generation, upstream, uses_subprocess]
 
+    # Server-based GSM8K eval: launches `vllm serve` (RemoteOpenAIServer) and hits
+    # the OpenAI completions endpoint. Runs compiled (server_args omit --enforce-eager)
+    # and batched (max_concurrency matches the server's batch area) to reflect
+    # real-world serving. The upstream `config_filename` fixture is steered to the
+    # Spyre configs (one per model) via `config_list`; thresholds are measured
+    # on-device baselines.
+    - rel_path: tests/evals/gsm8k/test_gsm8k_correctness.py
+      config_list: gsm8k_configs/models-spyre.txt
+      allow_list:
+        - test: "test_gsm8k_correctness"
+          mode: mandatory_pass
+          tags: [gsm8k, upstream, uses_subprocess]
+
     - rel_path: tests/models/language/pooling/test_embedding.py
       allow_list:
         - test: "test_models"

tests/test_sharding.py

@@ -461,6 +461,7 @@ def _makefile_shard_counts() -> dict[str, int]:
         ("upstream", "UPSTREAM_SHARDS"),
         ("distributed", "DIST_SHARDS"),
         ("probes", "PROBE_SHARDS"),
+        ("quality", "QUALITY_SHARDS"),
     ):
         m = re.search(rf"^{var}\s*\?=\s*(\d+)", text, re.MULTILINE)
         assert m, f"{var} not found in Makefile"
@@ -472,7 +473,7 @@ def _matrix_shard_ids() -> dict[str, list[int]]:
     text = (_REPO_ROOT / ".github/workflows/_test_matrix.yaml").read_text()
     return {
         suite: sorted({int(n) for n in re.findall(rf"test-{suite}-shard-(\d+)\b", text)})
-        for suite in ("smoke", "attention", "upstream", "distributed", "probes")
+        for suite in ("smoke", "attention", "upstream", "distributed", "probes", "quality")
     }
 
 

tests/test_upstream_gating.py

@@ -53,16 +53,21 @@
     assert not _markexpr_selects_upstream(markexpr)
 
 
+# Target families whose suites live in the upstream vLLM tree and so must trigger a clone.
+# The gsm8k accuracy gate (test-quality*) is an upstream suite too, despite its name.
+UPSTREAM_TARGET_PREFIXES = ("test-upstream", "test-quality")
+
+
 def test_makefile_upstream_targets_are_classified_correctly(pytestconfig):
     """The Makefile's combos all mention `upstream`, most negatively; only the
-    test-upstream* targets should trigger a clone.
+    test-upstream* and test-quality* targets should trigger a clone.
     """
     makefile = (Path(pytestconfig.rootpath) / "Makefile").read_text()
     overrides = dict(re.findall(r"^(test-[\w-]+):.*\n\t.*MARK_OVERRIDE='([^']*)'", makefile, re.M))
     assert len(overrides) >= 8, f"failed to parse MARK_OVERRIDEs out of the Makefile: {overrides}"
 
     for target, markexpr in overrides.items():
-        assert _markexpr_selects_upstream(markexpr) == target.startswith("test-upstream"), (
+        assert _markexpr_selects_upstream(markexpr) == target.startswith(UPSTREAM_TARGET_PREFIXES), (
             f"{target} ({markexpr!r}) is on the wrong side of the upstream gate"
         )