diff --git a/.github/scripts/run_smoke.py b/.github/scripts/run_smoke.py index 3822a454..b53f9e04 100644 --- a/.github/scripts/run_smoke.py +++ b/.github/scripts/run_smoke.py @@ -1,10 +1,20 @@ """ Run the workspace smoke test suite. -Reads `smoke_tests.txt` from the workspace root and `config/build/env_vars.yaml` -for per-script env var overrides, then runs each listed script with the -appropriate environment. Continues through failures and exits non-zero -if any script failed. +Reads `smoke_tests.txt` (Python scripts) and `smoke_notebooks.txt` +(Jupyter notebooks) from the workspace root, plus +`config/build/env_vars.yaml` for per-entry env var overrides, then +runs each listed entry with the appropriate environment. Continues +through failures and exits non-zero if any entry failed. + +Notebook execution uses `jupyter nbconvert --to notebook --execute`. +On failure the runner regenerates the single failing notebook from its +source `.py` script via PyAutoBuild's `py_to_notebook` and retries +once — this catches stale notebooks where the script has moved on but +the on-disk `.ipynb` wasn't refreshed by `/pre_build`'s +`generate.py`. Whole-workspace regeneration stays the responsibility +of `generate.py`; smoke only regenerates the single notebook in front +of it so the recovery is cheap. Mirrors the logic of the `/smoke-test` skill so CI and local runs stay in sync. @@ -13,8 +23,10 @@ from __future__ import annotations import os +import shutil import subprocess import sys +import tempfile import time from pathlib import Path @@ -23,18 +35,22 @@ WORKSPACE = Path(__file__).resolve().parents[2] SMOKE_FILE = WORKSPACE / "smoke_tests.txt" +NOTEBOOK_FILE = WORKSPACE / "smoke_notebooks.txt" ENV_VARS_FILE = WORKSPACE / "config" / "build" / "env_vars.yaml" SCRIPTS_DIR = WORKSPACE / "scripts" +NOTEBOOKS_DIR = WORKSPACE / "notebooks" -def load_smoke_scripts() -> list[str]: - scripts: list[str] = [] - for line in SMOKE_FILE.read_text().splitlines(): +def load_lines(path: Path) -> list[str]: + if not path.exists(): + return [] + out: list[str] = [] + for line in path.read_text().splitlines(): line = line.strip() if not line or line.startswith("#"): continue - scripts.append(line) - return scripts + out.append(line) + return out def load_env_config() -> dict: @@ -43,18 +59,18 @@ def load_env_config() -> dict: return yaml.safe_load(ENV_VARS_FILE.read_text()) or {} -def pattern_matches(pattern: str, script_path: str) -> bool: +def pattern_matches(pattern: str, rel_path: str) -> bool: if "/" in pattern: - return pattern in script_path - return Path(script_path).stem == pattern + return pattern in rel_path + return Path(rel_path).stem == pattern -def build_env(script_rel: str, cfg: dict) -> dict: +def build_env(rel_path: str, cfg: dict) -> dict: env = os.environ.copy() defaults = cfg.get("defaults") or {} env.update({k: str(v) for k, v in defaults.items()}) for override in cfg.get("overrides") or []: - if pattern_matches(override["pattern"], script_rel): + if pattern_matches(override["pattern"], rel_path): for key in override.get("unset", []): env.pop(key, None) for key, val in (override.get("set") or {}).items(): @@ -62,7 +78,7 @@ def build_env(script_rel: str, cfg: dict) -> dict: return env -def run_one(script_rel: str, cfg: dict) -> tuple[str, int, float, str]: +def run_script(script_rel: str, cfg: dict) -> tuple[str, int, float, str]: env = build_env(script_rel, cfg) script_path = SCRIPTS_DIR / script_rel t0 = time.time() @@ -73,34 +89,116 @@ def run_one(script_rel: str, cfg: dict) -> tuple[str, int, float, str]: capture_output=True, text=True, ) - elapsed = time.time() - t0 - output = result.stdout + result.stderr - return script_rel, result.returncode, elapsed, output + return script_rel, result.returncode, time.time() - t0, result.stdout + result.stderr + + +def execute_notebook(nb_path: Path, env: dict) -> tuple[int, str]: + # Write the executed copy to a throwaway path so the on-disk notebook + # under notebooks/ is never modified — checked-in notebooks stay clean. + tmp_dir = Path(tempfile.mkdtemp(prefix="smoke_nb_")) + try: + result = subprocess.run( + [ + "jupyter", "nbconvert", "--to", "notebook", "--execute", + "--output-dir", str(tmp_dir), + "--output", nb_path.name, + str(nb_path), + ], + cwd=str(WORKSPACE), + env=env, + capture_output=True, + text=True, + ) + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + return result.returncode, result.stdout + result.stderr + + +def regenerate_notebook(nb_rel: str) -> Path: + """Regenerate a notebook from its source `.py` into a temp dir. + + The regenerated copy lives in /tmp; the on-disk `notebooks/` tree is + never modified, so a smoke run leaves the worktree clean. + """ + from build_util import py_to_notebook # PyAutoBuild/autobuild on PYTHONPATH + + script_path = SCRIPTS_DIR / Path(nb_rel).with_suffix(".py") + if not script_path.exists(): + raise FileNotFoundError(f"No source script at {script_path}") + tmp_dir = Path(tempfile.mkdtemp(prefix="smoke_regen_")) + tmp_script = tmp_dir / script_path.name + shutil.copy(script_path, tmp_script) + old_cwd = os.getcwd() + try: + os.chdir(tmp_dir) + generated = py_to_notebook(tmp_script) + finally: + os.chdir(old_cwd) + return generated + + +def run_notebook(nb_rel: str, cfg: dict) -> tuple[str, int, float, str]: + env = build_env(nb_rel, cfg) + nb_path = NOTEBOOKS_DIR / nb_rel + t0 = time.time() + + if not nb_path.exists(): + return nb_rel, 1, 0.0, f"Notebook not found: {nb_path}\n" + + rc, output = execute_notebook(nb_path, env) + if rc == 0: + return nb_rel, 0, time.time() - t0, output + + print(" notebook failed; regenerating from source script and retrying...") + try: + nb_path = regenerate_notebook(nb_rel) + except Exception as exc: + output += f"\n[regenerate_notebook] {exc}\n" + return nb_rel, rc, time.time() - t0, output + + rc2, output2 = execute_notebook(nb_path, env) + output += "\n--- regenerated from script and retried ---\n" + output2 + return nb_rel, rc2, time.time() - t0, output def main() -> int: - if not SMOKE_FILE.exists(): - print(f"ERROR: no smoke_tests.txt at {SMOKE_FILE}", file=sys.stderr) - return 1 - scripts = load_smoke_scripts() - if not scripts: - print("No smoke test scripts listed.") - return 0 cfg = load_env_config() + scripts = load_lines(SMOKE_FILE) + notebooks = load_lines(NOTEBOOK_FILE) + + if not scripts and not notebooks: + print("No smoke tests listed.") + return 0 - print(f"Running {len(scripts)} smoke test script(s) from {SMOKE_FILE.name}\n") failures: list[tuple[str, int, str]] = [] - for script_rel in scripts: - print(f"::group::{script_rel}") - name, rc, elapsed, output = run_one(script_rel, cfg) - print(output, end="") - status = "PASS" if rc == 0 else f"FAIL (exit {rc})" - print(f"\n[{status}] {name} — {elapsed:.1f}s") - print("::endgroup::") - if rc != 0: - failures.append((name, rc, output)) - - total = len(scripts) + total = 0 + + if scripts: + print(f"Running {len(scripts)} script smoke test(s) from {SMOKE_FILE.name}\n") + for rel in scripts: + print(f"::group::script: {rel}") + name, rc, elapsed, output = run_script(rel, cfg) + print(output, end="") + status = "PASS" if rc == 0 else f"FAIL (exit {rc})" + print(f"\n[{status}] {name} — {elapsed:.1f}s") + print("::endgroup::") + total += 1 + if rc != 0: + failures.append((f"script: {name}", rc, output)) + + if notebooks: + print(f"\nRunning {len(notebooks)} notebook smoke test(s) from {NOTEBOOK_FILE.name}\n") + for rel in notebooks: + print(f"::group::notebook: {rel}") + name, rc, elapsed, output = run_notebook(rel, cfg) + print(output, end="") + status = "PASS" if rc == 0 else f"FAIL (exit {rc})" + print(f"\n[{status}] {name} — {elapsed:.1f}s") + print("::endgroup::") + total += 1 + if rc != 0: + failures.append((f"notebook: {name}", rc, output)) + passed = total - len(failures) print(f"\n=== Smoke test summary: {passed}/{total} passed ===") for name, rc, _ in failures: diff --git a/.github/workflows/smoke_tests.yml b/.github/workflows/smoke_tests.yml index 81550484..1b296626 100644 --- a/.github/workflows/smoke_tests.yml +++ b/.github/workflows/smoke_tests.yml @@ -20,6 +20,11 @@ jobs: with: repository: PyAutoLabs/PyAutoFit path: PyAutoFit + - name: Checkout PyAutoBuild + uses: actions/checkout@v4 + with: + repository: PyAutoLabs/PyAutoBuild + path: PyAutoBuild - name: Checkout workspace uses: actions/checkout@v4 with: @@ -59,6 +64,7 @@ jobs: pip install ./PyAutoConf ./PyAutoFit fi pip install nautilus-sampler + pip install jupyter nbconvert ipynb-py-convert - name: Prepare cache dirs run: | mkdir -p /tmp/numba_cache /tmp/matplotlib @@ -67,6 +73,7 @@ jobs: JAX_ENABLE_X64: "True" NUMBA_CACHE_DIR: /tmp/numba_cache MPLCONFIGDIR: /tmp/matplotlib + PYTHONPATH: ${{ github.workspace }}/PyAutoBuild/autobuild run: | cd workspace python .github/scripts/run_smoke.py diff --git a/smoke_notebooks.txt b/smoke_notebooks.txt new file mode 100644 index 00000000..6a801510 --- /dev/null +++ b/smoke_notebooks.txt @@ -0,0 +1,2 @@ +overview/overview_1_the_basics.ipynb +searches/mcmc.ipynb