diff --git a/.github/workflows/_test_matrix.yaml b/.github/workflows/_test_matrix.yaml index 3b48e9220..30af68804 100644 --- a/.github/workflows/_test_matrix.yaml +++ b/.github/workflows/_test_matrix.yaml @@ -492,6 +492,35 @@ jobs: - linux image_label: image_torch_spyre test_target: test-upstream-shard-6 + # GSM8K accuracy gate, one server-based eval per config sharded across + # parallel 1-card jobs (one config per shard). Heavy (full vLLM server + + # batched eval per config), so regression/trunk only -- excluded from + # integration/unit like the upstream suites. Adding/removing a shard = + # add/remove an entry here and bump QUALITY_SHARDS (Makefile default). + - cfg: Quality accuracy gate (shard 1/3) + test_types: regression trunk + runs_on: + - x86_64 + - spyre_pf_x1 + - linux + image_label: image_torch_spyre + test_target: test-quality-shard-0 + - cfg: Quality accuracy gate (shard 2/3) + test_types: regression trunk + runs_on: + - x86_64 + - spyre_pf_x1 + - linux + image_label: image_torch_spyre + test_target: test-quality-shard-1 + - cfg: Quality accuracy gate (shard 3/3) + test_types: regression trunk + runs_on: + - x86_64 + - spyre_pf_x1 + - linux + image_label: image_torch_spyre + test_target: test-quality-shard-2 - cfg: Upstream distributed vLLM tests test_types: regression trunk runs_on: diff --git a/Makefile b/Makefile index 5716524a3..4c353872b 100644 --- a/Makefile +++ b/Makefile @@ -121,7 +121,8 @@ RESULTS_DIR ?= . .PHONY: help test tests run-one aiu-setup perf-tests coverage print-test-type \ test-smoke test-smoke-shard test-probes test-probes-shard test-attention test-attention-shard \ test-distributed test-distributed-shard test-upstream test-upstream-shard \ - test-upstream-distributed tests-single-card tests-multi-card + test-upstream-distributed test-quality test-quality-shard \ + tests-single-card tests-multi-card help: ## Show this help message @awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) @@ -256,8 +257,10 @@ test-distributed-shard: ## Run one distributed shard (DIST_SHARDS=N DIST_SHARD_I test-distributed-shard-%: $(MAKE) test-distributed-shard DIST_SHARD_ID=$* JUNIT_XML=$(JUNIT_XML) +# `not gsm8k` carves the GSM8K accuracy gate out of the upstream suite: it also +# carries the `upstream` marker but is its own sharded suite (test-quality below). test-upstream: ## Run the upstream (non-distributed) marker combo, unsharded (local full run). - $(MAKE) run-one MARK_OVERRIDE='upstream and not distributed' JUNIT_XML=$(JUNIT_XML) + $(MAKE) run-one MARK_OVERRIDE='upstream and not distributed and not gsm8k' JUNIT_XML=$(JUNIT_XML) # Non-distributed upstream tests are sharded across parallel CI jobs. The heavy # model tests (under a models/ path) used to be a separate test-upstream-model @@ -267,7 +270,7 @@ test-upstream: ## Run the upstream (non-distributed) marker combo, unsharded (lo UPSTREAM_SHARDS ?= 7 UPSTREAM_SHARD_ID ?= 0 test-upstream-shard: ## Run one non-distributed upstream shard (UPSTREAM_SHARDS=N UPSTREAM_SHARD_ID=i). - $(MAKE) run-one MARK_OVERRIDE='upstream and not distributed' \ + $(MAKE) run-one MARK_OVERRIDE='upstream and not distributed and not gsm8k' \ PYTEST_ARGS='$(PYTEST_ARGS) --upstream-shards=$(UPSTREAM_SHARDS) --upstream-shard-id=$(UPSTREAM_SHARD_ID)' \ JUNIT_XML=$(JUNIT_XML) @@ -280,9 +283,33 @@ test-upstream-shard-%: test-upstream-distributed: ## Run the upstream+distributed marker combo. $(MAKE) run-one MARK_OVERRIDE='upstream and distributed' OMP_THREADS=1 JUNIT_XML=$(JUNIT_XML) +# The GSM8K accuracy gate: server-based correctness evals kept in their own suite +# (not the upstream per-op suite) because each one starts a vLLM server and runs a +# batched eval -- minutes per config, single card. `gsm8k and upstream` names the +# `upstream` marker so the eval file is collected from the cached vLLM checkout +# (as test-upstream does); no --upstream flag needed. +test-quality: ## Run the GSM8K accuracy gate (server-based evals), unsharded (local full run). Needs 1 card. + $(MAKE) run-one MARK_OVERRIDE='gsm8k and upstream' JUNIT_XML=$(JUNIT_XML) + +# One eval per config (gsm8k_configs/models-spyre.txt) sharded across parallel +# 1-card CI jobs, one config per shard. The plugin owns the partition +# (--quality-shards), balancing by recorded runtime when a durations file is +# present, else evenly. QUALITY_SHARDS is the single source of the count. +QUALITY_SHARDS ?= 3 +QUALITY_SHARD_ID ?= 0 +test-quality-shard: ## Run one GSM8K quality shard (QUALITY_SHARDS=N QUALITY_SHARD_ID=i). Needs 1 card. + $(MAKE) run-one MARK_OVERRIDE='gsm8k and upstream' \ + PYTEST_ARGS='$(PYTEST_ARGS) --quality-shards=$(QUALITY_SHARDS) --quality-shard-id=$(QUALITY_SHARD_ID)' \ + JUNIT_XML=$(JUNIT_XML) + +# CI runs one matrix job per shard as `test-quality-shard-` so each JUnit +# artifact name is unique; the pattern maps to QUALITY_SHARD_ID. +test-quality-shard-%: + $(MAKE) test-quality-shard QUALITY_SHARD_ID=$* JUNIT_XML=$(JUNIT_XML) + # Single-card / multi-card split, grouping the 6 marker combos above by how many cards they need. # Each suite gets its own junit-/junit-.xml subdir, matching GHA's artifact-name/file-name layout (_test_matrix.yaml) so a Jenkins run's JUnit paths line up 1:1 with a GHA run's. -tests-single-card: ## Run the 1-card marker combos (smoke shards / attention shards / encoder-attention / upstream shards). Needs 1 card. +tests-single-card: ## Run the 1-card marker combos (smoke shards / attention shards / encoder-attention / upstream shards / quality shards). Needs 1 card. mkdir -p "$(RESULTS_DIR)"; \ rc=0; \ for i in $$(seq 0 $$(( $(SMOKE_SHARDS) - 1 ))); do \ @@ -295,6 +322,9 @@ tests-single-card: ## Run the 1-card marker combos (smoke shards / attention sha for i in $$(seq 0 $$(( $(UPSTREAM_SHARDS) - 1 ))); do \ mkdir -p "$(RESULTS_DIR)/junit-test-upstream-shard-$$i" && $(MAKE) test-upstream-shard UPSTREAM_SHARD_ID=$$i JUNIT_XML="$(RESULTS_DIR)/junit-test-upstream-shard-$$i/junit-test-upstream-shard-$$i.xml" || rc=1; \ done; \ + for i in $$(seq 0 $$(( $(QUALITY_SHARDS) - 1 ))); do \ + mkdir -p "$(RESULTS_DIR)/junit-test-quality-shard-$$i" && $(MAKE) test-quality-shard QUALITY_SHARD_ID=$$i JUNIT_XML="$(RESULTS_DIR)/junit-test-quality-shard-$$i/junit-test-quality-shard-$$i.xml" || rc=1; \ + done; \ exit $$rc tests-multi-card: ## Run the 2-card marker combos (distributed shards/upstream-distributed/probes). Needs 2 cards. diff --git a/pyproject.toml b/pyproject.toml index ddd329326..370031453 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -280,6 +280,7 @@ markers = [ "uses_subprocess: tests that spawn a subprocess engine; the plugin runs them first, before the main process initializes the Spyre device", "rotary: Spyre RoPE tests (tests/custom_ops/test_rotary_embedding.py, tests/multimodal/test_pixtral.py)", "fp8: FP8 quantization tests (tests/custom_ops/test_fp8.py)", + "gsm8k: server-based GSM8K accuracy eval (upstream tests/evals/gsm8k); slow, runs in its own CI job", "conv: Spyre Conv2d custom-op tests (tests/custom_ops/test_conv.py)", "pixtral: Pixtral vision-tower patch tests (tests/multimodal/test_pixtral.py)", "multimodal: VLM tests. Local (tests/e2e/test_multimodal.py) is slow and needs a card; the upstream ones are host-side processor tests and need neither", diff --git a/tests/plugin/spyre_testing_plugin/gsm8k_configs/granite-4.1-3b.yaml b/tests/plugin/spyre_testing_plugin/gsm8k_configs/granite-4.1-3b.yaml new file mode 100644 index 000000000..cb76131cd --- /dev/null +++ b/tests/plugin/spyre_testing_plugin/gsm8k_configs/granite-4.1-3b.yaml @@ -0,0 +1,11 @@ +# GSM8K accuracy gate for granite-4.1-3b on Spyre (fp16, no quantized variant). +# accuracy_threshold is the measured on-device baseline; num_questions is small to gate cheaply. +# request_timeout_seconds is a *total* per-request budget (incl. queue-wait), so it must exceed the whole batched run's wall-clock. +model_name: "ibm-granite/granite-4.1-3b" +accuracy_threshold: 0.625 +num_questions: 64 +num_fewshot: 5 +max_concurrency: 4 +server_args: "--max-model-len 4096" +request_timeout_seconds: 2400 +startup_max_wait_seconds: 1800 diff --git a/tests/plugin/spyre_testing_plugin/gsm8k_configs/mistral-7b-instruct-v0.3.yaml b/tests/plugin/spyre_testing_plugin/gsm8k_configs/mistral-7b-instruct-v0.3.yaml new file mode 100644 index 000000000..16e78e2ee --- /dev/null +++ b/tests/plugin/spyre_testing_plugin/gsm8k_configs/mistral-7b-instruct-v0.3.yaml @@ -0,0 +1,11 @@ +# GSM8K accuracy gate for Mistral-7B-Instruct-v0.3 on Spyre (fp16, no quantized variant). +# accuracy_threshold is the measured on-device baseline; num_questions is small to gate cheaply. +# request_timeout_seconds is a *total* per-request budget (incl. queue-wait), so it must exceed the whole batched run's wall-clock. +model_name: "mistralai/Mistral-7B-Instruct-v0.3" +accuracy_threshold: 0.375 +num_questions: 64 +num_fewshot: 5 +max_concurrency: 4 +server_args: "--max-model-len 4096" +request_timeout_seconds: 2400 +startup_max_wait_seconds: 1800 diff --git a/tests/plugin/spyre_testing_plugin/gsm8k_configs/models-spyre.txt b/tests/plugin/spyre_testing_plugin/gsm8k_configs/models-spyre.txt new file mode 100644 index 000000000..b7cf31c88 --- /dev/null +++ b/tests/plugin/spyre_testing_plugin/gsm8k_configs/models-spyre.txt @@ -0,0 +1,6 @@ +# GSM8K eval configs runnable on Spyre (fp16, no quantization). +# Consumed by upstream tests/evals/gsm8k/test_gsm8k_correctness.py; selected via +# the `config_list` field on that file's entry in upstream_tests.yaml. +qwen2.5-0.5b-instruct.yaml +granite-4.1-3b.yaml +mistral-7b-instruct-v0.3.yaml diff --git a/tests/plugin/spyre_testing_plugin/gsm8k_configs/qwen2.5-0.5b-instruct.yaml b/tests/plugin/spyre_testing_plugin/gsm8k_configs/qwen2.5-0.5b-instruct.yaml new file mode 100644 index 000000000..487165bb0 --- /dev/null +++ b/tests/plugin/spyre_testing_plugin/gsm8k_configs/qwen2.5-0.5b-instruct.yaml @@ -0,0 +1,11 @@ +# GSM8K accuracy gate for Qwen2.5-0.5B-Instruct on Spyre (fp16, no quantized variant). +# accuracy_threshold is the measured on-device baseline; num_questions is small to gate cheaply. +# request_timeout_seconds is a *total* per-request budget (incl. queue-wait), so it must exceed the whole batched run's wall-clock. +model_name: "Qwen/Qwen2.5-0.5B-Instruct" +accuracy_threshold: 0.3125 +num_questions: 64 +num_fewshot: 5 +max_concurrency: 4 +server_args: "--max-model-len 4096" +request_timeout_seconds: 2400 +startup_max_wait_seconds: 1800 diff --git a/tests/plugin/spyre_testing_plugin/models.py b/tests/plugin/spyre_testing_plugin/models.py index 5624ff828..f9ca51999 100644 --- a/tests/plugin/spyre_testing_plugin/models.py +++ b/tests/plugin/spyre_testing_plugin/models.py @@ -99,15 +99,21 @@ class FileConfig: """Filter configuration for a single upstream test file. Attributes: - rel_path: Path relative to upstream repo root - (e.g. "tests/kernels/core/test_layernorm.py"). - allow_list: Tests allowed to run from this file. - block_list: Tests blocked from running (takes precedence over allow_list). + rel_path: Path relative to upstream repo root + (e.g. "tests/kernels/core/test_layernorm.py"). + allow_list: Tests allowed to run from this file. + block_list: Tests blocked from running (takes precedence over allow_list). + config_list: Optional path (relative to this plugin package) to a config-list + file for tests parametrized by an upstream `config_filename` + fixture (e.g. tests/evals/gsm8k). When set, it overrides the + upstream conftest's `--config-list-file` so its own + `pytest_generate_tests` parametrizes from Spyre-owned configs. """ rel_path: str allow_list: tuple[AllowEntry, ...] = () block_list: tuple[BlockEntry, ...] = () + config_list: str | None = None @dataclass(frozen=True) diff --git a/tests/plugin/spyre_testing_plugin/pytest_plugin.py b/tests/plugin/spyre_testing_plugin/pytest_plugin.py index c487434a2..3cb185403 100644 --- a/tests/plugin/spyre_testing_plugin/pytest_plugin.py +++ b/tests/plugin/spyre_testing_plugin/pytest_plugin.py @@ -215,6 +215,7 @@ def _parse_config(raw_tests: dict) -> UpstreamTestConfig: rel_path=file_entry["rel_path"], allow_list=tuple(allow_list), block_list=tuple(block_list), + config_list=file_entry.get("config_list"), ) ) return UpstreamTestConfig(files=tuple(files)) @@ -772,6 +773,15 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: if not fc: return + # Tests parametrized by an upstream `config_filename` fixture (e.g. gsm8k evals) + # aren't reachable via param_overrides, which only rewrites the test's own + # parametrize markers. Instead, point the upstream conftest's --config-list-file at + # a Spyre-owned list; its own (later-running) pytest_generate_tests then parametrizes + # config_filename from our configs. tryfirst here guarantees we set it first. + if fc.config_list and "config_filename" in metafunc.fixturenames: + list_path = (_YAML_PATH.parent / fc.config_list).resolve() + metafunc.config.option.config_list_file = str(list_path) + test_name = metafunc.definition.originalname or metafunc.definition.name allow_entry = _find_allow_entry(test_name, fc.allow_list) if not allow_entry or not allow_entry.param_overrides: diff --git a/tests/plugin/spyre_testing_plugin/sharding.py b/tests/plugin/spyre_testing_plugin/sharding.py index d85bbbe8b..831aef37a 100644 --- a/tests/plugin/spyre_testing_plugin/sharding.py +++ b/tests/plugin/spyre_testing_plugin/sharding.py @@ -14,7 +14,7 @@ """Duration-weighted test sharding for CI fan-out. -Each suite (attn/smoke/upstream/dist/probe) is split across N parallel jobs; a job +Each suite (attn/smoke/upstream/dist/probe/quality) is split across N parallel jobs; a job keeps only its shard's slice. Every shard job computes the same weighted greedy longest-processing-time partition, so no cross-job coordination is needed and the union of all shards is the full selection exactly once (guarded by @@ -60,7 +60,7 @@ def _will_skip(item: pytest.Item) -> bool: def add_shard_options(parser) -> None: """Register one ---shards / ---shard-id pair per CI suite.""" group = parser.getgroup("spyre-test-sharding") - for suite in ("attn", "smoke", "upstream", "dist", "probe"): + for suite in ("attn", "smoke", "upstream", "dist", "probe", "quality"): group.addoption( f"--{suite}-shards", type=int, @@ -93,6 +93,7 @@ def apply_shards(config: pytest.Config, items: list[pytest.Item]) -> None: _apply_upstream_shard(config, items) _apply_distributed_shard(config, items) _apply_probe_shard(config, items) + _apply_quality_shard(config, items) def _load_durations(config: pytest.Config) -> dict[str, float]: @@ -297,12 +298,14 @@ def _apply_upstream_shard(config: pytest.Config, items: list[pytest.Item]) -> No def weight(item: pytest.Item) -> int: return 8 if "models/" in item.nodeid else 1 + # The GSM8K gate also carries the `upstream` marker but is its own sharded suite + # (_apply_quality_shard); excluded here so its server-based evals stay out of this partition. _apply_shard( config, items, num_shards=config.getoption("--upstream-shards"), shard_id=config.getoption("--upstream-shard-id"), - select=lambda item: True, + select=lambda item: not item.get_closest_marker("gsm8k"), weight=weight, label="upstream", durations=_load_durations(config), @@ -352,6 +355,23 @@ def select(item: pytest.Item) -> bool: ) +def _apply_quality_shard(config: pytest.Config, items: list[pytest.Item]) -> None: + # The GSM8K accuracy gate (Makefile test-quality: `gsm8k and upstream`), one + # server-based eval per config in gsm8k_configs/models-spyre.txt. Each starts a + # vLLM server and runs a batched eval -- heavy and roughly per-config uniform, so + # a shard per config; durations refine that. Single card each (one server at a time). + _apply_shard( + config, + items, + num_shards=config.getoption("--quality-shards"), + shard_id=config.getoption("--quality-shard-id"), + select=lambda item: bool(item.get_closest_marker("gsm8k")), + weight=lambda item: 1, + label="quality", + durations=_load_durations(config), + ) + + # Per-nodeid wall time this session, written out when SPYRE_TEST_DURATIONS_OUT is # set (CI). A later run pins the merged file and feeds it back as # SPYRE_TEST_DURATIONS so _apply_shard balances shards by measured runtime. Keyed diff --git a/tests/plugin/spyre_testing_plugin/upstream_tests.yaml b/tests/plugin/spyre_testing_plugin/upstream_tests.yaml index 0cdd519cb..8bab566d5 100644 --- a/tests/plugin/spyre_testing_plugin/upstream_tests.yaml +++ b/tests/plugin/spyre_testing_plugin/upstream_tests.yaml @@ -130,6 +130,19 @@ tests: mode: skip # model_info0 (Qwen3-0.6B): RAS::VFIO::MapDMAFailed; model_info1 (Qwen3-0.6B-FP8): KeyError PlatformEnum.OOT - FP8 quant not supported on Spyre; model_info2 (Qwen3.5-0.8B): Qwen3_5ForConditionalGeneration not in vLLM registry tags: [model, generation, upstream, uses_subprocess] + # Server-based GSM8K eval: launches `vllm serve` (RemoteOpenAIServer) and hits + # the OpenAI completions endpoint. Runs compiled (server_args omit --enforce-eager) + # and batched (max_concurrency matches the server's batch area) to reflect + # real-world serving. The upstream `config_filename` fixture is steered to the + # Spyre configs (one per model) via `config_list`; thresholds are measured + # on-device baselines. + - rel_path: tests/evals/gsm8k/test_gsm8k_correctness.py + config_list: gsm8k_configs/models-spyre.txt + allow_list: + - test: "test_gsm8k_correctness" + mode: mandatory_pass + tags: [gsm8k, upstream, uses_subprocess] + - rel_path: tests/models/language/pooling/test_embedding.py allow_list: - test: "test_models" diff --git a/tests/test_sharding.py b/tests/test_sharding.py index 0a8da53e1..295fe3723 100644 --- a/tests/test_sharding.py +++ b/tests/test_sharding.py @@ -461,6 +461,7 @@ def _makefile_shard_counts() -> dict[str, int]: ("upstream", "UPSTREAM_SHARDS"), ("distributed", "DIST_SHARDS"), ("probes", "PROBE_SHARDS"), + ("quality", "QUALITY_SHARDS"), ): m = re.search(rf"^{var}\s*\?=\s*(\d+)", text, re.MULTILINE) assert m, f"{var} not found in Makefile" @@ -472,7 +473,7 @@ def _matrix_shard_ids() -> dict[str, list[int]]: text = (_REPO_ROOT / ".github/workflows/_test_matrix.yaml").read_text() return { suite: sorted({int(n) for n in re.findall(rf"test-{suite}-shard-(\d+)\b", text)}) - for suite in ("smoke", "attention", "upstream", "distributed", "probes") + for suite in ("smoke", "attention", "upstream", "distributed", "probes", "quality") } diff --git a/tests/test_upstream_gating.py b/tests/test_upstream_gating.py index d819fd4ff..9b0eb92ea 100644 --- a/tests/test_upstream_gating.py +++ b/tests/test_upstream_gating.py @@ -53,16 +53,21 @@ def test_markexpr_does_not_request_upstream(markexpr): assert not _markexpr_selects_upstream(markexpr) +# Target families whose suites live in the upstream vLLM tree and so must trigger a clone. +# The gsm8k accuracy gate (test-quality*) is an upstream suite too, despite its name. +UPSTREAM_TARGET_PREFIXES = ("test-upstream", "test-quality") + + def test_makefile_upstream_targets_are_classified_correctly(pytestconfig): """The Makefile's combos all mention `upstream`, most negatively; only the - test-upstream* targets should trigger a clone. + test-upstream* and test-quality* targets should trigger a clone. """ makefile = (Path(pytestconfig.rootpath) / "Makefile").read_text() overrides = dict(re.findall(r"^(test-[\w-]+):.*\n\t.*MARK_OVERRIDE='([^']*)'", makefile, re.M)) assert len(overrides) >= 8, f"failed to parse MARK_OVERRIDEs out of the Makefile: {overrides}" for target, markexpr in overrides.items(): - assert _markexpr_selects_upstream(markexpr) == target.startswith("test-upstream"), ( + assert _markexpr_selects_upstream(markexpr) == target.startswith(UPSTREAM_TARGET_PREFIXES), ( f"{target} ({markexpr!r}) is on the wrong side of the upstream gate" )