diff --git a/autonerves/fitsable.py b/autonerves/fitsable.py index 16876eb..a1ae1f8 100644 --- a/autonerves/fitsable.py +++ b/autonerves/fitsable.py @@ -14,6 +14,77 @@ from typing import Dict, Optional, Union, List +# Exactly 8 characters, and that ceiling is load-bearing rather than stylistic. +# A 9-character keyword does not raise here: astropy silently promotes it to a +# HIERARCH card, which `header.get("...")` by the short name then misses, +# returning None. The reader in ``autoarray.util.dataset_util`` treats None as +# "unknown regime" and falls back to its shape heuristic -- so an over-long key +# would not fail loudly, it would quietly un-fix the interferometer case this +# stamp exists for. Any rename must stay within 8 chars, and is a wire-format +# change (the reader duplicates this literal), not a refactor. +SMALL_DATASETS_HEADER_KEY = "SMALLDAT" +SMALL_DATASETS_HEADER_COMMENT = "PYAUTO_SMALL_DATASETS active at write time" + + +def stamp_small_datasets_regime(header): + """ + Record the small-datasets regime in a FITS ``header``, in place. + + ``PYAUTO_SMALL_DATASETS=1`` caps simulated datasets to a reduced + resolution. Nothing else on disk records that fact, so a dataset written + by a capped run can survive into a later full-resolution run and be loaded + silently -- the root cause of autolens_workspace_test#260. + + Writing the regime *here*, in the same call that writes the data, makes the + stamp **truthful by construction**: it cannot disagree with the file it + sits in, and there is no stamped-but-empty-directory failure mode. That is + the property a marker file written before simulation cannot have. + + Unlike a shape heuristic it also does not depend on the data looking + different, which is what makes it the only discriminant able to catch + capped **interferometer** datasets: their visibility count is fixed by the + committed uv file while the real-space grid behind it is capped, so a + capped run writes identical ``NAXIS`` with different values and trips no + assertion at all. + + The card is written in both regimes on every FITS written through this + module -- which is every FITS the PyAuto libraries author, because all 18 + library write sites build their HDUList here even when they call + ``hdu_list.writeto`` themselves. It is **not** universal, and readers must + not assume it is: PyAutoFit's aggregator assembles some HDULists by hand + (``autofit/aggregator/summary/aggregate_fits.py``), and workspace scripts + that use raw astropy directly (e.g. the lenstool converter in + autolens_workspace) write unstamped files. Those read as absent, i.e. + unknown -- which is the safe direction, and exactly why absence must never + be read as "full resolution". + + The three states: + + - ``SMALLDAT = T`` -- written under ``PYAUTO_SMALL_DATASETS=1`` (capped). + - ``SMALLDAT = F`` -- written at full resolution. + - **absent** -- unknown. Written before this stamp existed, or by another + tool. Readers must fall back to their legacy heuristic and, above all, + must never read absence as "full resolution". + + Recording ``F`` explicitly rather than relying on absence is the whole + point of the always-write: it lets a reader distinguish "known full" from + "no idea", and only the first of those is safe to act on. See + ``autoarray.util.dataset_util.should_simulate``, whose predicate ends in + ``shutil.rmtree``. + + Assignment (not ``append``) is deliberate -- it is idempotent, so the two + call sites below can both stamp the same header without duplicating the + card. + """ + from autonerves.test_mode import small_datasets + + header[SMALL_DATASETS_HEADER_KEY] = ( + small_datasets(), + SMALL_DATASETS_HEADER_COMMENT, + ) + return header + + def hdu_list_for_output_from( values_list: List[np.ndarray], header_dict: Optional[dict] = None, @@ -68,6 +139,8 @@ def hdu_list_for_output_from( except ValueError: header.append((key_str, float(value), [""])) + stamp_small_datasets_regime(header) + for i, values in enumerate(values_list): if ext_name_list is not None: @@ -136,6 +209,9 @@ def write_hdu_list(hdu_list, file_path, overwrite=False): overwrite : bool If ``True`` an existing file is replaced. """ + if len(hdu_list) > 0: + stamp_small_datasets_regime(hdu_list[0].header) + file_path = Path(file_path) file_path.parent.mkdir(parents=True, exist_ok=True) if overwrite and file_path.is_file(): diff --git a/autonerves/test_mode.py b/autonerves/test_mode.py index 030a011..e0474b6 100644 --- a/autonerves/test_mode.py +++ b/autonerves/test_mode.py @@ -214,3 +214,23 @@ def with_test_mode_segment(base: Path) -> Path: PosixPath('output/test_mode/results_folder') """ return base / "test_mode" if is_test_mode() else base + + +def small_datasets(): + """ + Return True if the small-datasets regime is active. + + ``PYAUTO_SMALL_DATASETS=1`` caps simulated datasets to a reduced + resolution (``SMALL_DATASETS_SHAPE_NATIVE = (16, 16)`` in + ``autoarray.util.dataset_util``) so smoke runs stay fast. The cap changes + what a simulator writes to disk, which makes it a *provenance* fact about + every file written while it is active -- see + :func:`autonerves.fitsable.stamp_small_datasets_regime`, which records it + in the FITS header of every array the stack writes. + + The env var is deliberately compared against the exact string ``"1"``, + matching every other ``PYAUTO_*`` switch in this module and the readers in + ``autoarray.util.dataset_util``. ``"0"``, ``"true"`` and the unset case all + mean "full resolution". + """ + return os.environ.get("PYAUTO_SMALL_DATASETS") == "1" diff --git a/test_autonerves/conftest.py b/test_autonerves/conftest.py index b52a728..816a492 100644 --- a/test_autonerves/conftest.py +++ b/test_autonerves/conftest.py @@ -24,3 +24,26 @@ def make_config(files_directory): files_directory / "config", files_directory / "default", ) + + +@pytest.fixture(autouse=True) +def _regime_independent_test_output(monkeypatch): + """ + Clear ``PYAUTO_SMALL_DATASETS`` for every test unless the test sets it. + + Since PyAutoNerves#153 every FITS the stack writes carries a ``SMALLDAT`` + card whose value tracks this env var at write time. Several tests write into + **tracked** fixture paths (14 of them across PyAutoArray and this repo -- + a pre-existing pattern), so without this the bytes those tests produce + depend on the ambient environment: run the suite in a shell exporting + ``PYAUTO_SMALL_DATASETS=1`` -- which ``should_simulate``'s own docstring calls + the default for most harness runs -- and the suite passes but leaves the + working tree dirty. + + Pinning it here restores the property the stamp took away, that test output + is a function of the test and not of the shell, and does so in one place + rather than by rewriting every fixture-writing test. Tests that need a + regime set it with ``monkeypatch.setenv`` in their body, which runs after + this fixture and wins. + """ + monkeypatch.delenv("PYAUTO_SMALL_DATASETS", raising=False) diff --git a/test_autonerves/files/array_out.fits b/test_autonerves/files/array_out.fits index f5e4cb4..85cddd5 100644 Binary files a/test_autonerves/files/array_out.fits and b/test_autonerves/files/array_out.fits differ diff --git a/test_autonerves/test_fitsable.py b/test_autonerves/test_fitsable.py index e243d53..27f78f7 100644 --- a/test_autonerves/test_fitsable.py +++ b/test_autonerves/test_fitsable.py @@ -93,3 +93,132 @@ def test__header_obj_from(): assert isinstance(header_obj, fits.header.Header) assert header_obj["BITPIX"] == -64 + + +""" +__small-datasets regime stamp (PyAutoNerves#153)__ + +`PYAUTO_SMALL_DATASETS=1` caps simulated datasets to a reduced resolution. +Nothing else on disk records that, so a capped dataset can survive into a later +full-resolution run and be loaded silently (autolens_workspace_test#260). The +stamp records the regime in the same call that writes the data, which makes it +truthful by construction and -- unlike a shape heuristic -- able to catch +corruption that leaves the shape unchanged. + +Both funnels are covered below because they are genuinely separate paths: +`output_to_fits` builds and writes in one call, while the multi-HDU dataset +writers in PyAutoArray build via `hdu_list_for_output_from` and write via +`write_hdu_list`, never touching `output_to_fits`. +""" + +KEY = fitsable.SMALL_DATASETS_HEADER_KEY + + +def _stamp(file_path, hdu=0): + with fits.open(file_path) as hdu_list: + return hdu_list[hdu].header.get(KEY) + + +def test__output_to_fits__stamps_the_regime(tmp_path, monkeypatch): + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + fitsable.output_to_fits(np.ones((4, 4)), file_path=tmp_path / "small.fits") + assert _stamp(tmp_path / "small.fits") is True + + monkeypatch.delenv("PYAUTO_SMALL_DATASETS", raising=False) + fitsable.output_to_fits(np.ones((4, 4)), file_path=tmp_path / "full.fits") + assert _stamp(tmp_path / "full.fits") is False + + +def test__stamp_is_written_in_both_regimes__absence_is_never_full( + tmp_path, monkeypatch +): + # The full regime is recorded EXPLICITLY as F rather than by omission. + # That is the whole point: a reader can then tell "known full" from "no + # idea", and only the first is safe to act on. If F were encoded as + # absence, every legacy dataset would masquerade as full resolution and + # the original bug would come straight back. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "0") + fitsable.output_to_fits(np.ones((4, 4)), file_path=tmp_path / "zero.fits") + + assert KEY in fits.open(tmp_path / "zero.fits")[0].header + assert _stamp(tmp_path / "zero.fits") is False + + +def test__stamp_round_trips_as_a_fits_boolean_not_a_string(tmp_path, monkeypatch): + # Readers distinguish True/False/absent and must never coerce: bool("F") + # is True. Pin the on-disk type so a future change to the header-writing + # path cannot silently downgrade the card to a string or a float. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + fitsable.output_to_fits(np.ones((4, 4)), file_path=tmp_path / "t.fits") + + value = fits.open(tmp_path / "t.fits")[0].header[KEY] + assert isinstance(value, bool) + assert "SMALLDAT= T" in str( + fits.open(tmp_path / "t.fits")[0].header.cards[KEY] + ) + + +def test__multi_hdu_funnel__stamps_every_hdu(tmp_path, monkeypatch): + # The path PyAutoArray's fits_imaging/fits_interferometer take when given a + # single `file_path` -- it bypasses output_to_fits entirely. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + + hdu_list = fitsable.hdu_list_for_output_from( + values_list=[np.ones((4, 4)), np.zeros((4, 4))], + ext_name_list=["data", "noise_map"], + ) + fitsable.write_hdu_list(hdu_list, file_path=tmp_path / "dataset.fits") + + assert _stamp(tmp_path / "dataset.fits", hdu=0) is True + assert _stamp(tmp_path / "dataset.fits", hdu=1) is True + + +def test__write_hdu_list__stamps_an_hdu_list_built_elsewhere(tmp_path, monkeypatch): + # write_hdu_list is the terminal write, so it must stamp even an HDUList + # this module did not construct -- `hdu_list_for_output_from` is publicly + # re-exported as `aa.hdu_list_for_output_from`, so callers can and do build + # their own. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + + hdu_list = fits.HDUList([fits.PrimaryHDU(np.ones((4, 4)))]) + fitsable.write_hdu_list(hdu_list, file_path=tmp_path / "raw.fits") + + assert _stamp(tmp_path / "raw.fits") is True + + +def test__stamp_does_not_disturb_header_dict_entries(tmp_path, monkeypatch): + # The stamp rides alongside the mask's PIXSCAY/PIXSCAX/ORIGINY/ORIGINX + # cards; it must not displace or overwrite any of them. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + + fitsable.output_to_fits( + np.ones((4, 4)), + file_path=tmp_path / "h.fits", + header_dict={"PIXSCAY": 0.5, "PIXSCAX": 0.5, "ORIGINY": 0.0, "ORIGINX": 0.0}, + ) + + header = fits.open(tmp_path / "h.fits")[0].header + assert header["PIXSCAY"] == 0.5 + assert header["PIXSCAX"] == 0.5 + assert header[KEY] is True + + +def test__stamp_is_idempotent_across_both_funnels(tmp_path, monkeypatch): + # hdu_list_for_output_from and write_hdu_list BOTH stamp, and output_to_fits + # goes through both. Assignment (not append) keeps that from duplicating the + # card -- a duplicate would make header[KEY] ambiguous. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + fitsable.output_to_fits(np.ones((4, 4)), file_path=tmp_path / "once.fits") + + header = fits.open(tmp_path / "once.fits")[0].header + assert len([c for c in header.cards if c.keyword == KEY]) == 1 + + +def test__stamp_key_stays_within_the_fits_standard_card_limit(): + # Not stylistic. A 9-char keyword is silently promoted to a HIERARCH card + # by astropy rather than raising, and header.get() by the short name then + # returns None -- which readers treat as "unknown regime" and fall back to + # the shape heuristic. An over-long key would therefore not fail loudly, it + # would quietly un-fix the interferometer case. Pin the ceiling. + assert len(KEY) <= 8 + assert KEY == KEY.upper()