diff --git a/.github/scripts/README.md b/.github/scripts/README.md index 534e5c92..d940c88e 100644 --- a/.github/scripts/README.md +++ b/.github/scripts/README.md @@ -59,4 +59,19 @@ No workflow changes needed! - transformers - torch +### Optional PR impact filtering + +The reusable test workflow first writes the complete generated matrices to one +JSON file. When `filter_models_by_changes` is enabled for a pull request, +`filter_test_matrix.py` atomically replaces that file with matrices restricted +to adapters affected by the changed Python modules. This narrowing is attempted +only when every changed file is an `hf_adapters/hf_*.py` file, or when every +change is documentation under `docs/` or one of the recognized root docs. +Mixed or unrecognized changes retain the complete matrices. A final publisher +exposes the file through the usual individual GitHub Actions outputs. If +filtering is skipped or fails, the original complete file remains available. + +`test_matrix_config.py` is the shared source of truth for matrix keys, model +registries, representative paths, and complete path lists. + These are installed in the `generate-matrix` job before running the script. diff --git a/.github/scripts/filter_test_matrix.py b/.github/scripts/filter_test_matrix.py new file mode 100644 index 00000000..db60eacd --- /dev/null +++ b/.github/scripts/filter_test_matrix.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""Filter generated CI model matrices using the changed-file import impact. + +This is deliberately a second pass: ``generate_test_matrix.py`` remains the +source of the complete matrices, and disabling this filter restores them +without changing model-selection behavior. +""" + +from __future__ import annotations + +import argparse +import ast +import json +import os +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from test_matrix_config import MATRIX_CONFIG, MatrixKey # noqa: E402 + +DOC_PREFIXES = ("docs/",) +ROOT_DOC_PATHS = {"README.md", "ARCHITECTURE.md", "ONBOARDING.md", "CLAUDE.md"} + + +def _is_documentation_path(path: str) -> bool: + return path.startswith(DOC_PREFIXES) or path in ROOT_DOC_PATHS + + +def _is_hf_adapter_path(path: str) -> bool: + candidate = Path(path) + return ( + candidate.suffix == ".py" + and len(candidate.parts) > 1 + and candidate.parts[0] == "hf_adapters" + and candidate.name.startswith("hf_") + ) + + +def _module_for_path(path: str) -> str | None: + candidate = Path(path) + if ( + candidate.suffix != ".py" + or not candidate.parts + or candidate.parts[0] != "hf_adapters" + ): + return None + parts = list(candidate.with_suffix("").parts) + if parts[-1] == "__init__": + parts.pop() + return ".".join(parts) + + +def _local_imports(path: Path, module: str) -> set[str]: + """Return imported ``hf_adapters`` modules, without importing the code.""" + try: + tree = ast.parse(path.read_text()) + except (OSError, SyntaxError): + return set() + imports: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imports.update( + alias.name + for alias in node.names + if alias.name.startswith("hf_adapters.") + ) + elif isinstance(node, ast.ImportFrom): + imported_module = node.module or "" + if node.level: + package = module.split(".")[:-1] + keep = len(package) - (node.level - 1) + imported_module = ".".join( + [*package[: max(keep, 0)], *imported_module.split(".")] + ).rstrip(".") + if imported_module == "hf_adapters": + imports.update(f"hf_adapters.{alias.name}" for alias in node.names) + elif imported_module.startswith("hf_adapters."): + imports.add(imported_module) + return imports + + +def dependency_graph(root: Path = ROOT) -> dict[str, set[str]]: + graph: dict[str, set[str]] = {} + for path in (root / "hf_adapters").rglob("*.py"): + module = _module_for_path(path.relative_to(root).as_posix()) + if module: + graph[module] = _local_imports(path, module) + return graph + + +def _depends_on(module: str, changed: set[str], graph: dict[str, set[str]]) -> bool: + pending = [module] + seen: set[str] = set() + while pending: + current = pending.pop() + if current in changed: + return True + if current in seen: + continue + seen.add(current) + pending.extend(graph.get(current, ())) + return False + + +def affected_adapters(changed_files: list[str], root: Path = ROOT) -> set[str] | None: + """Return adapter filenames, or ``None`` when all models must be retained.""" + normalized = [path.removeprefix("./") for path in changed_files if path.strip()] + if not normalized: + return None + + # Limit tests only for two deliberately narrow PR shapes. In particular, + # mixing documentation with adapter changes retains the complete matrices. + if all(_is_documentation_path(path) for path in normalized): + return set() + if not all(_is_hf_adapter_path(path) for path in normalized): + return None + + changed_modules = { + module for path in normalized if (module := _module_for_path(path)) is not None + } + + graph = dependency_graph(root) + adapter_names = { + info["adapter"] + for models, _, _ in MATRIX_CONFIG.values() + for info in models.values() + } + affected = { + adapter + for adapter in adapter_names + if _depends_on(f"hf_adapters.{Path(adapter).stem}", changed_modules, graph) + } + # A changed production module with no known consumers is ambiguous. Keep all. + return affected or None + + +def filter_matrices( + matrices: dict[str, list[str]], adapters: set[str] | None +) -> dict[str, list[str]]: + if adapters is None: + return matrices + paths_by_adapter: dict[str, set[str]] = {} + for models, _, _ in MATRIX_CONFIG.values(): + for info in models.values(): + paths_by_adapter.setdefault(info["adapter"], set()).add(info["path"]) + keep = ( + set().union(*(paths_by_adapter.get(adapter, set()) for adapter in adapters)) + if adapters + else set() + ) + filtered = { + key.value: [path for path in matrices[key.value] if path in keep] + for key in MatrixKey + if key is not MatrixKey.COMBINED + } + filtered[MatrixKey.COMBINED.value] = ( + filtered[MatrixKey.CAUSAL.value] + + filtered[MatrixKey.EMBED.value] + + filtered[MatrixKey.MASKED_LM.value] + + filtered[MatrixKey.QUESTION_ANSWERING.value] + + filtered[MatrixKey.TOKEN_CLASSIFICATION.value] + ) + return filtered + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--changed-files", required=True, help="newline-delimited changed-files file" + ) + parser.add_argument("--matrix-file", required=True, type=Path) + args = parser.parse_args() + + # obtain the list of changed files + changed_files = Path(args.changed_files).read_text().splitlines() + print("Changed files:", ", ".join(changed_files) or "(none)") + + # figure out the list of affected adapters (directly or indirectly) + adapters = affected_adapters(changed_files) + print( + "Affected adapters:", + ( + "ALL (conservative fallback)" + if adapters is None + else ", ".join(sorted(adapters)) or "none" + ), + ) + + matrices = json.loads(args.matrix_file.read_text()) + filtered = filter_matrices(matrices, adapters) + print("Filtered matrices size:") + for key, paths in filtered.items(): + print(f" {key}: {len(paths)} model(s)") + + # Atomic replacement leaves the complete matrix file untouched if filtering + # fails before a valid filtered document has been fully written. + with tempfile.NamedTemporaryFile( + mode="w", dir=args.matrix_file.parent, delete=False + ) as stream: + json.dump(filtered, stream) + temporary_path = Path(stream.name) + os.replace(temporary_path, args.matrix_file) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/generate_test_matrix.py b/.github/scripts/generate_test_matrix.py index 66e9704a..6551fd00 100644 --- a/.github/scripts/generate_test_matrix.py +++ b/.github/scripts/generate_test_matrix.py @@ -23,7 +23,7 @@ project_root = Path(__file__).parent.parent.parent sys.path.insert(0, str(project_root)) -import tests.model_registry # noqa: E402 +from test_matrix_config import MATRIX_CONFIG, MatrixKey # noqa: E402 def generate_matrices(exclude_models=None, only_models=None): @@ -46,50 +46,23 @@ def generate_matrices(exclude_models=None, only_models=None): # --only so a caller can target a non-representative checkpoint, e.g. a # larger model sharing an adapter with a smaller default). Adding a new # category later just means adding a row here. - registry = tests.model_registry - categories = { - "causal": (registry.CAUSAL_PATHS, registry.ALL_CAUSAL_PATHS), - "embed": (registry.EMBED_PATHS, registry.ALL_EMBED_PATHS), - "vision": (registry.VISION_PATHS, registry.ALL_VISION_PATHS), - "masked_lm": (registry.MASKED_LM_PATHS, registry.ALL_MASKED_LM_PATHS), - "question_answering": ( - registry.QUESTION_ANSWERING_PATHS, - registry.ALL_QUESTION_ANSWERING_PATHS, - ), - "reranker": (registry.RERANKER_PATHS, registry.ALL_RERANKER_PATHS), - "token_classification": ( - registry.TOKEN_CLASSIFICATION_PATHS, - registry.ALL_TOKEN_CLASSIFICATION_PATHS, - ), - } - paths = {} - for name, (representative_paths, all_paths) in categories.items(): + for name, (_, representative_paths, all_paths) in MATRIX_CONFIG.items(): source = all_paths if only_models else representative_paths selected = [p for p in source if p not in exclude_models] if only_models: selected = [p for p in selected if p in only_models] paths[name] = selected - # Feeds spyre-load-tests' matrix: test_load_spyre.py's five model_path suites. - combined_paths = ( - paths["causal"] - + paths["embed"] - + paths["masked_lm"] - + paths["question_answering"] - + paths["token_classification"] + # Feeds test_load_spyre.py's five model_path-parametrized suites. + paths[MatrixKey.COMBINED] = ( + paths[MatrixKey.CAUSAL] + + paths[MatrixKey.EMBED] + + paths[MatrixKey.MASKED_LM] + + paths[MatrixKey.QUESTION_ANSWERING] + + paths[MatrixKey.TOKEN_CLASSIFICATION] ) - - return { - "causal": paths["causal"], - "embed": paths["embed"], - "vision": paths["vision"], - "masked_lm": paths["masked_lm"], - "question_answering": paths["question_answering"], - "combined": combined_paths, - "reranker": paths["reranker"], - "token_classification": paths["token_classification"], - } + return paths def format_for_github_actions(matrices): @@ -102,16 +75,7 @@ def format_for_github_actions(matrices): Returns: dict: Dictionary with JSON-stringified matrices """ - return { - "causal_matrix": json.dumps(matrices["causal"]), - "embed_matrix": json.dumps(matrices["embed"]), - "vision_matrix": json.dumps(matrices["vision"]), - "masked_lm_matrix": json.dumps(matrices["masked_lm"]), - "question_answering_matrix": json.dumps(matrices["question_answering"]), - "combined_matrix": json.dumps(matrices["combined"]), - "reranker_matrix": json.dumps(matrices["reranker"]), - "token_classification_matrix": json.dumps(matrices["token_classification"]), - } + return {key.value: json.dumps(matrices[key]) for key in MatrixKey} def write_github_output(outputs): @@ -154,40 +118,28 @@ def main(): help="If given, restrict all matrices to just these model paths " "(e.g., Qwen/Qwen3-0.6B ministral/Ministral-3B-Instruct)", ) + parser.add_argument("--output-file", type=Path) + parser.add_argument("--publish-file", type=Path) args = parser.parse_args() + if args.publish_file: + stored = json.loads(args.publish_file.read_text()) + missing = {key.value for key in MatrixKey} - set(stored) + if missing: + parser.error(f"matrix file is missing keys: {', '.join(sorted(missing))}") + write_github_output( + {key.value: json.dumps(stored[key.value]) for key in MatrixKey} + ) + return + # Generate matrices matrices = generate_matrices(exclude_models=args.exclude, only_models=args.only) # Print summary for workflow logs print("Generated test matrices:") - print( - f" Causal models ({len(matrices['causal'])}): {', '.join(matrices['causal'])}" - ) - print( - f" Embedding models ({len(matrices['embed'])}): {', '.join(matrices['embed'])}" - ) - print( - f" Vision models ({len(matrices['vision'])}): {', '.join(matrices['vision'])}" - ) - print( - f" Masked-LM models ({len(matrices['masked_lm'])}): {', '.join(matrices['masked_lm'])}" - ) - print( - f" Question-answering models ({len(matrices['question_answering'])}): " - f"{', '.join(matrices['question_answering'])}" - ) - print( - f" Combined ({len(matrices['combined'])}): {', '.join(matrices['combined'])}" - ) - print( - f" Reranker models ({len(matrices['reranker'])}): {', '.join(matrices['reranker'])}" - ) - print( - f" Token-classification models ({len(matrices['token_classification'])}): " - f"{', '.join(matrices['token_classification'])}" - ) + for key in MatrixKey: + print(f" {key.value} ({len(matrices[key])}): {', '.join(matrices[key])}") if args.exclude: print(f"\nExcluded models: {', '.join(args.exclude)}") @@ -196,6 +148,10 @@ def main(): # Format for GitHub Actions outputs = format_for_github_actions(matrices) + if args.output_file: + args.output_file.write_text( + json.dumps({key.value: matrices[key] for key in MatrixKey}) + ) # Write to GitHub Actions output write_github_output(outputs) diff --git a/.github/scripts/test_matrix_config.py b/.github/scripts/test_matrix_config.py new file mode 100644 index 00000000..08df1c29 --- /dev/null +++ b/.github/scripts/test_matrix_config.py @@ -0,0 +1,87 @@ +"""Single source of truth for generated CI model matrices.""" + +from enum import StrEnum + +import tests.model_registry as registry + + +class MatrixKey(StrEnum): + CAUSAL = "causal_matrix" + EMBED = "embed_matrix" + VISION = "vision_matrix" + MASKED_LM = "masked_lm_matrix" + QUESTION_ANSWERING = "question_answering_matrix" + TOKEN_CLASSIFICATION = "token_classification_matrix" + RERANKER = "reranker_matrix" + COMBINED = "combined_matrix" + + +# Combined is derived from causal + embed, so it intentionally has no entry. +MATRIX_CONFIG = { + MatrixKey.CAUSAL: ( + registry.CAUSAL_LM_MODELS, + registry.CAUSAL_PATHS, + registry.ALL_CAUSAL_PATHS, + ), + MatrixKey.EMBED: ( + registry.EMBEDDING_MODELS, + registry.EMBED_PATHS, + registry.ALL_EMBED_PATHS, + ), + MatrixKey.VISION: ( + registry.VISION_MODELS, + registry.VISION_PATHS, + registry.ALL_VISION_PATHS, + ), + MatrixKey.MASKED_LM: ( + registry.MASKED_LM_MODELS, + registry.MASKED_LM_PATHS, + registry.ALL_MASKED_LM_PATHS, + ), + MatrixKey.QUESTION_ANSWERING: ( + registry.QUESTION_ANSWERING_MODELS, + registry.QUESTION_ANSWERING_PATHS, + registry.ALL_QUESTION_ANSWERING_PATHS, + ), + MatrixKey.TOKEN_CLASSIFICATION: ( + registry.TOKEN_CLASSIFICATION_MODELS, + registry.TOKEN_CLASSIFICATION_PATHS, + registry.ALL_TOKEN_CLASSIFICATION_PATHS, + ), + MatrixKey.RERANKER: ( + registry.RERANKER_MODELS, + registry.RERANKER_PATHS, + registry.ALL_RERANKER_PATHS, + ), +} + + +def _validate_matrix_config() -> None: + expected_keys = set(MatrixKey) - {MatrixKey.COMBINED} + if set(MATRIX_CONFIG) != expected_keys: + raise RuntimeError("every non-derived MatrixKey must have MATRIX_CONFIG") + + configured_registries = {id(models) for models, _, _ in MATRIX_CONFIG.values()} + discovered_registries = { + id(value): name + for name, value in vars(registry).items() + if name.endswith("_MODELS") + and isinstance(value, dict) + and value + and all( + isinstance(info, dict) and {"path", "adapter", "size"} <= info.keys() + for info in value.values() + ) + } + missing = { + name + for identity, name in discovered_registries.items() + if identity not in configured_registries + } + if missing: + raise RuntimeError( + f"model registries missing from MATRIX_CONFIG: {', '.join(sorted(missing))}" + ) + + +_validate_matrix_config() diff --git a/.github/workflows/_test_matrix.yaml b/.github/workflows/_test_matrix.yaml index 3bfd4cff..fb32d78b 100644 --- a/.github/workflows/_test_matrix.yaml +++ b/.github/workflows/_test_matrix.yaml @@ -117,6 +117,13 @@ on: required: false type: boolean default: false + filter_models_by_changes: + description: >- + Filter generated model matrices to adapters affected by the PR's + changed files. Conservative fallbacks keep the full matrices. + required: false + type: boolean + default: false permissions: # Needed by collect-failed-suites to list/download failed-suite-* artifacts via the REST API; reusable workflows can only narrow the caller's permissions, so callers must grant this too. @@ -239,8 +246,7 @@ jobs: python -m pip install --upgrade pip pip install transformers torch pytest - - name: Generate matrices - id: generate + - name: Generate complete matrices env: # Piped through env (not interpolated directly into the script) since # this is caller-supplied workflow_call input. @@ -253,7 +259,30 @@ jobs: read -r -a ONLY_ARGS <<< "$ONLY_MODELS" ONLY_ARGS=(--only "${ONLY_ARGS[@]}") fi - python .github/scripts/generate_test_matrix.py "${ONLY_ARGS[@]}" + python .github/scripts/generate_test_matrix.py \ + "${ONLY_ARGS[@]}" \ + --output-file "$RUNNER_TEMP/test-matrices.json" + + - name: Filter matrices by changed adapters + if: ${{ inputs.filter_models_by_changes && github.event_name == 'pull_request' }} + continue-on-error: true + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + # Keep checkout shallow and fetch only the two commits needed for the diff. + git fetch --no-tags --depth=1 origin "$BASE_SHA" "$HEAD_SHA" + git diff --name-only "$BASE_SHA" "$HEAD_SHA" > "$RUNNER_TEMP/changed-files.txt" + python .github/scripts/filter_test_matrix.py \ + --changed-files "$RUNNER_TEMP/changed-files.txt" \ + --matrix-file "$RUNNER_TEMP/test-matrices.json" + + - name: Publish matrices + id: generate + if: ${{ always() }} + run: | + python .github/scripts/generate_test_matrix.py \ + --publish-file "$RUNNER_TEMP/test-matrices.json" # =========================================================================== # Generates spyre-model-module-tests' matrix from the tests/configs/module_tests/ @@ -451,7 +480,7 @@ jobs: # space-separated word in test_type (e.g. "smoke load") for fine-grained # selection. # See spyre-smoke-tests for the other keys. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' load ')) }} + if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && needs.generate-matrix.outputs.combined_matrix != '[]' && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' load ')) }} # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-load-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -522,7 +551,7 @@ jobs: # only job that tier runs). Same allowlist mechanism as spyre-load-tests # above -- to add a new suite later, give the new job its own key and # copy this `if:` line with that key; no existing job needs to change. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' smoke ')) }} + if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && needs.generate-matrix.outputs.causal_matrix != '[]' && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' smoke ')) }} # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-smoke-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -589,7 +618,7 @@ jobs: name: Spyre token compare (${{ matrix.model_key }}) needs: [generate-matrix, resolve-test-type] # Suite key: "token_compare". Part of the "unit" coarse tier. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'integration' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' token_compare ')) }} + if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && needs.generate-matrix.outputs.causal_matrix != '[]' && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'integration' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' token_compare ')) }} # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-token-compare-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -656,7 +685,7 @@ jobs: name: Spyre embed compare (${{ matrix.model_key }}) needs: [generate-matrix, resolve-test-type] # Suite key: "embed_compare". Part of the "unit" coarse tier. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' embed_compare ')) }} + if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && needs.generate-matrix.outputs.embed_matrix != '[]' && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' embed_compare ')) }} # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-embed-compare-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -723,7 +752,7 @@ jobs: name: Spyre VLM e2e (${{ matrix.model_key }}) needs: [generate-matrix, resolve-test-type] # Suite key: "vlm". Part of the "unit" coarse tier. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' vlm ')) }} + if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && needs.generate-matrix.outputs.vision_matrix != '[]' && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' vlm ')) }} # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-vlm-e2e-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -849,7 +878,7 @@ jobs: name: Spyre reranker compare (${{ matrix.model_key }}) needs: [generate-matrix, resolve-test-type] # Suite key: "reranker_compare". Part of the "unit" coarse tier. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' reranker_compare ')) }} + if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && needs.generate-matrix.outputs.reranker_matrix != '[]' && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' reranker_compare ')) }} # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-reranker-compare-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -913,7 +942,7 @@ jobs: name: Spyre masked-LM compare (${{ matrix.model_key }}) needs: [generate-matrix, resolve-test-type] # Suite key: "masked_lm_compare". Part of the "unit" coarse tier. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' masked_lm_compare ')) }} + if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && needs.generate-matrix.outputs.masked_lm_matrix != '[]' && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' masked_lm_compare ')) }} # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-masked-lm-compare-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -977,7 +1006,7 @@ jobs: name: Spyre question-answering compare (${{ matrix.model_key }}) needs: [generate-matrix, resolve-test-type] # Suite key: "question_answering_compare". Part of the "unit" coarse tier. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' question_answering_compare ')) }} + if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && needs.generate-matrix.outputs.question_answering_matrix != '[]' && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' question_answering_compare ')) }} # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-question-answering-compare-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -1041,7 +1070,7 @@ jobs: name: Spyre token-classification compare (${{ matrix.model_key }}) needs: [generate-matrix, resolve-test-type] # Suite key: "token_classification_compare". Part of the "unit" coarse tier. - if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' token_classification_compare ')) }} + if: ${{ !inputs.skip_tests && !inputs.edge_cases_only && needs.generate-matrix.outputs.token_classification_matrix != '[]' && (needs.resolve-test-type.outputs.test_type == 'regression' || needs.resolve-test-type.outputs.test_type == 'trunk' || needs.resolve-test-type.outputs.test_type == 'unit' || contains(format(' {0} ', needs.resolve-test-type.outputs.test_type), ' token_classification_compare ')) }} # Don't let one failed model leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-token-classification-compare-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: @@ -1118,7 +1147,7 @@ jobs: spyre-edge-cases-tests: name: Spyre edge case ${{ matrix.test_file }} (${{ matrix.model_key }}) needs: [generate-matrix] - if: ${{ !inputs.skip_tests && inputs.edge_cases_only }} + if: ${{ !inputs.skip_tests && inputs.edge_cases_only && needs.generate-matrix.outputs.causal_matrix != '[]' }} # Don't let one failed (model, test_file) leg fail this job outright -- spyre-tests-result decides pass/fail after the pod-level retry (spyre-edge-cases-tests-retry) gets a shot on a fresh pod. continue-on-error: true runs-on: diff --git a/.github/workflows/test_pull_request.yaml b/.github/workflows/test_pull_request.yaml index 76cb0f07..71e0904a 100644 --- a/.github/workflows/test_pull_request.yaml +++ b/.github/workflows/test_pull_request.yaml @@ -136,6 +136,8 @@ jobs: ref: ${{ inputs.ref || '' }} repository: ${{ inputs.repository || '' }} skip_tests: ${{ needs.detect-label.outputs.labeled == 'true' }} + # One-line kill switch: set false to restore the complete model matrices. + filter_models_by_changes: true # ---------------------------------------------------------------------------------- # Required check gate -- fans in detect-label and the delegated suite run so a diff --git a/tests/test_filter_test_matrix.py b/tests/test_filter_test_matrix.py new file mode 100644 index 00000000..00d5a8f2 --- /dev/null +++ b/tests/test_filter_test_matrix.py @@ -0,0 +1,200 @@ +import importlib.util +import json +import os +import subprocess +import sys +from pathlib import Path + +SCRIPT = Path(__file__).parents[1] / ".github/scripts/filter_test_matrix.py" +GENERATOR = Path(__file__).parents[1] / ".github/scripts/generate_test_matrix.py" +SPEC = importlib.util.spec_from_file_location("filter_test_matrix", SCRIPT) +filter_test_matrix = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(filter_test_matrix) +test_matrix_config = importlib.import_module("test_matrix_config") + + +def test_direct_adapter_change_selects_that_adapter(): + assert filter_test_matrix.affected_adapters(["hf_adapters/hf_llama.py"]) == { + "hf_llama.py" + } + + +def test_common_change_selects_all_dependent_registered_adapters(): + affected = filter_test_matrix.affected_adapters(["hf_adapters/hf_common.py"]) + assert affected is not None + assert "hf_llama.py" in affected + assert "hf_qwen3.py" in affected + + +def test_transitive_dependency_is_followed(): + affected = filter_test_matrix.affected_adapters(["hf_adapters/hf_mistral.py"]) + assert affected is not None + assert "hf_mistral.py" in affected + assert "hf_mistral3.py" in affected + + +def test_dependency_graph_resolves_relative_imports(tmp_path): + package = tmp_path / "hf_adapters" + package.mkdir() + (package / "hf_example.py").write_text("from .hf_helper import value\n") + (package / "hf_helper.py").write_text("value = 1\n") + + graph = filter_test_matrix.dependency_graph(tmp_path) + + assert graph["hf_adapters.hf_example"] == {"hf_adapters.hf_helper"} + + +def test_docs_only_change_selects_no_adapters(): + assert ( + filter_test_matrix.affected_adapters(["README.md", "docs/design.md"]) == set() + ) + + +def test_docs_mixed_with_adapter_change_keeps_everything(): + assert ( + filter_test_matrix.affected_adapters( + ["hf_adapters/hf_llama.py", "docs/design.md"] + ) + is None + ) + + +def test_non_hf_prefixed_file_in_adapter_directory_keeps_everything(): + assert filter_test_matrix.affected_adapters(["hf_adapters/st_backend.py"]) is None + assert ( + filter_test_matrix.affected_adapters(["hf_adapters/auto_spyre_model.py"]) + is None + ) + + +def test_empty_changed_file_list_keeps_everything(): + assert filter_test_matrix.affected_adapters([]) is None + + +def test_unknown_or_ci_change_keeps_everything(): + assert filter_test_matrix.affected_adapters(["setup.cfg"]) is None + assert filter_test_matrix.affected_adapters(["requirements.txt"]) is None + assert ( + filter_test_matrix.affected_adapters( + [".github/workflows/test_pull_request.yaml"] + ) + is None + ) + + +def test_filter_preserves_generated_order_and_rebuilds_combined(): + matrices = { + "causal_matrix": ["gpt2", "Qwen/Qwen3-0.6B"], + "embed_matrix": ["Qwen/Qwen3-Embedding-0.6B"], + "vision_matrix": [], + "masked_lm_matrix": [], + "question_answering_matrix": [], + "token_classification_matrix": [], + "reranker_matrix": [], + "combined_matrix": ["stale"], + } + result = filter_test_matrix.filter_matrices(matrices, {"hf_qwen3.py"}) + assert result["causal_matrix"] == ["Qwen/Qwen3-0.6B"] + assert result["embed_matrix"] == ["Qwen/Qwen3-Embedding-0.6B"] + assert result["combined_matrix"] == result["causal_matrix"] + result["embed_matrix"] + + +def test_matrix_config_covers_every_non_derived_key(): + assert set(filter_test_matrix.MATRIX_CONFIG) == set( + filter_test_matrix.MatrixKey + ) - {filter_test_matrix.MatrixKey.COMBINED} + + +def test_matrix_config_covers_every_model_registry(): + test_matrix_config._validate_matrix_config() + + +def test_main_filters_the_durable_matrix_file(monkeypatch, tmp_path): + matrices = { + key.value: ["gpt2"] if key is filter_test_matrix.MatrixKey.CAUSAL else [] + for key in filter_test_matrix.MatrixKey + } + matrices[filter_test_matrix.MatrixKey.COMBINED.value] = ["gpt2"] + matrix_file = tmp_path / "matrices.json" + matrix_file.write_text(json.dumps(matrices)) + changed_files = tmp_path / "changed-files.txt" + changed_files.write_text("README.md\n") + monkeypatch.setattr( + sys, + "argv", + [ + "filter_test_matrix.py", + "--changed-files", + str(changed_files), + "--matrix-file", + str(matrix_file), + ], + ) + + filter_test_matrix.main() + + assert json.loads(matrix_file.read_text()) == { + key.value: [] for key in filter_test_matrix.MatrixKey + } + + +def test_failed_filter_leaves_durable_matrix_file_unchanged(monkeypatch, tmp_path): + matrix_file = tmp_path / "matrices.json" + original = '{"causal_matrix": ["gpt2"]}' + matrix_file.write_text(original) + changed_files = tmp_path / "changed-files.txt" + changed_files.write_text("README.md\n") + monkeypatch.setattr( + sys, + "argv", + [ + "filter_test_matrix.py", + "--changed-files", + str(changed_files), + "--matrix-file", + str(matrix_file), + ], + ) + + try: + filter_test_matrix.main() + except KeyError: + pass + else: + raise AssertionError("incomplete matrix should fail filtering") + + assert matrix_file.read_text() == original + + +def test_generate_file_can_be_published_as_github_outputs(tmp_path): + matrix_file = tmp_path / "matrices.json" + github_output = tmp_path / "github-output.txt" + subprocess.run( + [ + sys.executable, + str(GENERATOR), + "--only", + "Qwen/Qwen3-0.6B", + "--output-file", + str(matrix_file), + ], + check=True, + capture_output=True, + text=True, + ) + env = os.environ.copy() + env["GITHUB_OUTPUT"] = str(github_output) + subprocess.run( + [sys.executable, str(GENERATOR), "--publish-file", str(matrix_file)], + check=True, + capture_output=True, + text=True, + env=env, + ) + + published = dict( + line.split("=", 1) for line in github_output.read_text().splitlines() + ) + assert json.loads(published["causal_matrix"]) == ["Qwen/Qwen3-0.6B"] + assert set(published) == {key.value for key in filter_test_matrix.MatrixKey}