fix: stamp the small-datasets regime into FITS headers at the writer funnel

autonerves/fitsable.py

@@ -14,6 +14,77 @@
 from typing import Dict, Optional, Union, List
 
 
+# Exactly 8 characters, and that ceiling is load-bearing rather than stylistic.
+# A 9-character keyword does not raise here: astropy silently promotes it to a
+# HIERARCH card, which `header.get("...")` by the short name then misses,
+# returning None. The reader in ``autoarray.util.dataset_util`` treats None as
+# "unknown regime" and falls back to its shape heuristic -- so an over-long key
+# would not fail loudly, it would quietly un-fix the interferometer case this
+# stamp exists for. Any rename must stay within 8 chars, and is a wire-format
+# change (the reader duplicates this literal), not a refactor.
+SMALL_DATASETS_HEADER_KEY = "SMALLDAT"
+SMALL_DATASETS_HEADER_COMMENT = "PYAUTO_SMALL_DATASETS active at write time"
+
+
+def stamp_small_datasets_regime(header):
+    """
+    Record the small-datasets regime in a FITS ``header``, in place.
+
+    ``PYAUTO_SMALL_DATASETS=1`` caps simulated datasets to a reduced
+    resolution. Nothing else on disk records that fact, so a dataset written
+    by a capped run can survive into a later full-resolution run and be loaded
+    silently -- the root cause of autolens_workspace_test#260.
+
+    Writing the regime *here*, in the same call that writes the data, makes the
+    stamp **truthful by construction**: it cannot disagree with the file it
+    sits in, and there is no stamped-but-empty-directory failure mode. That is
+    the property a marker file written before simulation cannot have.
+
+    Unlike a shape heuristic it also does not depend on the data looking
+    different, which is what makes it the only discriminant able to catch
+    capped **interferometer** datasets: their visibility count is fixed by the
+    committed uv file while the real-space grid behind it is capped, so a
+    capped run writes identical ``NAXIS`` with different values and trips no
+    assertion at all.
+
+    The card is written in both regimes on every FITS written through this
+    module -- which is every FITS the PyAuto libraries author, because all 18
+    library write sites build their HDUList here even when they call
+    ``hdu_list.writeto`` themselves. It is **not** universal, and readers must
+    not assume it is: PyAutoFit's aggregator assembles some HDULists by hand
+    (``autofit/aggregator/summary/aggregate_fits.py``), and workspace scripts
+    that use raw astropy directly (e.g. the lenstool converter in
+    autolens_workspace) write unstamped files. Those read as absent, i.e.
+    unknown -- which is the safe direction, and exactly why absence must never
+    be read as "full resolution".
+
+    The three states:
+
+    - ``SMALLDAT = T`` -- written under ``PYAUTO_SMALL_DATASETS=1`` (capped).
+    - ``SMALLDAT = F`` -- written at full resolution.
+    - **absent** -- unknown. Written before this stamp existed, or by another
+      tool. Readers must fall back to their legacy heuristic and, above all,
+      must never read absence as "full resolution".
+
+    Recording ``F`` explicitly rather than relying on absence is the whole
+    point of the always-write: it lets a reader distinguish "known full" from
+    "no idea", and only the first of those is safe to act on. See
+    ``autoarray.util.dataset_util.should_simulate``, whose predicate ends in
+    ``shutil.rmtree``.
+
+    Assignment (not ``append``) is deliberate -- it is idempotent, so the two
+    call sites below can both stamp the same header without duplicating the
+    card.
+    """
+    from autonerves.test_mode import small_datasets
+
+    header[SMALL_DATASETS_HEADER_KEY] = (
+        small_datasets(),
+        SMALL_DATASETS_HEADER_COMMENT,
+    )
+    return header
+
+
 def hdu_list_for_output_from(
     values_list: List[np.ndarray],
     header_dict: Optional[dict] = None,
@@ -68,6 +139,8 @@ def hdu_list_for_output_from(
             except ValueError:
                 header.append((key_str, float(value), [""]))
 
+    stamp_small_datasets_regime(header)
+
     for i, values in enumerate(values_list):
 
         if ext_name_list is not None:
@@ -136,6 +209,9 @@ def write_hdu_list(hdu_list, file_path, overwrite=False):
     overwrite : bool
         If ``True`` an existing file is replaced.
     """
+    if len(hdu_list) > 0:
+        stamp_small_datasets_regime(hdu_list[0].header)
+
     file_path = Path(file_path)
     file_path.parent.mkdir(parents=True, exist_ok=True)
     if overwrite and file_path.is_file():

autonerves/test_mode.py

@@ -214,3 +214,23 @@ def with_test_mode_segment(base: Path) -> Path:
     PosixPath('output/test_mode/results_folder')
     """
     return base / "test_mode" if is_test_mode() else base
+
+
+def small_datasets():
+    """
+    Return True if the small-datasets regime is active.
+
+    ``PYAUTO_SMALL_DATASETS=1`` caps simulated datasets to a reduced
+    resolution (``SMALL_DATASETS_SHAPE_NATIVE = (16, 16)`` in
+    ``autoarray.util.dataset_util``) so smoke runs stay fast. The cap changes
+    what a simulator writes to disk, which makes it a *provenance* fact about
+    every file written while it is active -- see
+    :func:`autonerves.fitsable.stamp_small_datasets_regime`, which records it
+    in the FITS header of every array the stack writes.
+
+    The env var is deliberately compared against the exact string ``"1"``,
+    matching every other ``PYAUTO_*`` switch in this module and the readers in
+    ``autoarray.util.dataset_util``. ``"0"``, ``"true"`` and the unset case all
+    mean "full resolution".
+    """
+    return os.environ.get("PYAUTO_SMALL_DATASETS") == "1"

test_autonerves/conftest.py

@@ -24,3 +24,26 @@ def make_config(files_directory):
         files_directory / "config",
         files_directory / "default",
     )
+
+
+@pytest.fixture(autouse=True)
+def _regime_independent_test_output(monkeypatch):
+    """
+    Clear ``PYAUTO_SMALL_DATASETS`` for every test unless the test sets it.
+
+    Since PyAutoNerves#153 every FITS the stack writes carries a ``SMALLDAT``
+    card whose value tracks this env var at write time. Several tests write into
+    **tracked** fixture paths (14 of them across PyAutoArray and this repo --
+    a pre-existing pattern), so without this the bytes those tests produce
+    depend on the ambient environment: run the suite in a shell exporting
+    ``PYAUTO_SMALL_DATASETS=1`` -- which ``should_simulate``'s own docstring calls
+    the default for most harness runs -- and the suite passes but leaves the
+    working tree dirty.
+
+    Pinning it here restores the property the stamp took away, that test output
+    is a function of the test and not of the shell, and does so in one place
+    rather than by rewriting every fixture-writing test. Tests that need a
+    regime set it with ``monkeypatch.setenv`` in their body, which runs after
+    this fixture and wins.
+    """
+    monkeypatch.delenv("PYAUTO_SMALL_DATASETS", raising=False)

test_autonerves/test_fitsable.py

@@ -93,3 +93,132 @@ def test__header_obj_from():
 
     assert isinstance(header_obj, fits.header.Header)
     assert header_obj["BITPIX"] == -64
+
+
+"""
+__small-datasets regime stamp (PyAutoNerves#153)__
+
+`PYAUTO_SMALL_DATASETS=1` caps simulated datasets to a reduced resolution.
+Nothing else on disk records that, so a capped dataset can survive into a later
+full-resolution run and be loaded silently (autolens_workspace_test#260). The
+stamp records the regime in the same call that writes the data, which makes it
+truthful by construction and -- unlike a shape heuristic -- able to catch
+corruption that leaves the shape unchanged.
+
+Both funnels are covered below because they are genuinely separate paths:
+`output_to_fits` builds and writes in one call, while the multi-HDU dataset
+writers in PyAutoArray build via `hdu_list_for_output_from` and write via
+`write_hdu_list`, never touching `output_to_fits`.
+"""
+
+KEY = fitsable.SMALL_DATASETS_HEADER_KEY
+
+
+def _stamp(file_path, hdu=0):
+    with fits.open(file_path) as hdu_list:
+        return hdu_list[hdu].header.get(KEY)
+
+
+def test__output_to_fits__stamps_the_regime(tmp_path, monkeypatch):
+    monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1")
+    fitsable.output_to_fits(np.ones((4, 4)), file_path=tmp_path / "small.fits")
+    assert _stamp(tmp_path / "small.fits") is True
+
+    monkeypatch.delenv("PYAUTO_SMALL_DATASETS", raising=False)
+    fitsable.output_to_fits(np.ones((4, 4)), file_path=tmp_path / "full.fits")
+    assert _stamp(tmp_path / "full.fits") is False
+
+
+def test__stamp_is_written_in_both_regimes__absence_is_never_full(
+    tmp_path, monkeypatch
+):
+    # The full regime is recorded EXPLICITLY as F rather than by omission.
+    # That is the whole point: a reader can then tell "known full" from "no
+    # idea", and only the first is safe to act on. If F were encoded as
+    # absence, every legacy dataset would masquerade as full resolution and
+    # the original bug would come straight back.
+    monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "0")
+    fitsable.output_to_fits(np.ones((4, 4)), file_path=tmp_path / "zero.fits")
+
+    assert KEY in fits.open(tmp_path / "zero.fits")[0].header
+    assert _stamp(tmp_path / "zero.fits") is False
+
+
+def test__stamp_round_trips_as_a_fits_boolean_not_a_string(tmp_path, monkeypatch):
+    # Readers distinguish True/False/absent and must never coerce: bool("F")
+    # is True. Pin the on-disk type so a future change to the header-writing
+    # path cannot silently downgrade the card to a string or a float.
+    monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1")
+    fitsable.output_to_fits(np.ones((4, 4)), file_path=tmp_path / "t.fits")
+
+    value = fits.open(tmp_path / "t.fits")[0].header[KEY]
+    assert isinstance(value, bool)
+    assert "SMALLDAT=                    T" in str(
+        fits.open(tmp_path / "t.fits")[0].header.cards[KEY]
+    )
+
+
+def test__multi_hdu_funnel__stamps_every_hdu(tmp_path, monkeypatch):
+    # The path PyAutoArray's fits_imaging/fits_interferometer take when given a
+    # single `file_path` -- it bypasses output_to_fits entirely.
+    monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1")
+
+    hdu_list = fitsable.hdu_list_for_output_from(
+        values_list=[np.ones((4, 4)), np.zeros((4, 4))],
+        ext_name_list=["data", "noise_map"],
+    )
+    fitsable.write_hdu_list(hdu_list, file_path=tmp_path / "dataset.fits")
+
+    assert _stamp(tmp_path / "dataset.fits", hdu=0) is True
+    assert _stamp(tmp_path / "dataset.fits", hdu=1) is True
+
+
+def test__write_hdu_list__stamps_an_hdu_list_built_elsewhere(tmp_path, monkeypatch):
+    # write_hdu_list is the terminal write, so it must stamp even an HDUList
+    # this module did not construct -- `hdu_list_for_output_from` is publicly
+    # re-exported as `aa.hdu_list_for_output_from`, so callers can and do build
+    # their own.
+    monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1")
+
+    hdu_list = fits.HDUList([fits.PrimaryHDU(np.ones((4, 4)))])
+    fitsable.write_hdu_list(hdu_list, file_path=tmp_path / "raw.fits")
+
+    assert _stamp(tmp_path / "raw.fits") is True
+
+
+def test__stamp_does_not_disturb_header_dict_entries(tmp_path, monkeypatch):
+    # The stamp rides alongside the mask's PIXSCAY/PIXSCAX/ORIGINY/ORIGINX
+    # cards; it must not displace or overwrite any of them.
+    monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1")
+
+    fitsable.output_to_fits(
+        np.ones((4, 4)),
+        file_path=tmp_path / "h.fits",
+        header_dict={"PIXSCAY": 0.5, "PIXSCAX": 0.5, "ORIGINY": 0.0, "ORIGINX": 0.0},
+    )
+
+    header = fits.open(tmp_path / "h.fits")[0].header
+    assert header["PIXSCAY"] == 0.5
+    assert header["PIXSCAX"] == 0.5
+    assert header[KEY] is True
+
+
+def test__stamp_is_idempotent_across_both_funnels(tmp_path, monkeypatch):
+    # hdu_list_for_output_from and write_hdu_list BOTH stamp, and output_to_fits
+    # goes through both. Assignment (not append) keeps that from duplicating the
+    # card -- a duplicate would make header[KEY] ambiguous.
+    monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1")
+    fitsable.output_to_fits(np.ones((4, 4)), file_path=tmp_path / "once.fits")
+
+    header = fits.open(tmp_path / "once.fits")[0].header
+    assert len([c for c in header.cards if c.keyword == KEY]) == 1
+
+
+def test__stamp_key_stays_within_the_fits_standard_card_limit():
+    # Not stylistic. A 9-char keyword is silently promoted to a HIERARCH card
+    # by astropy rather than raising, and header.get() by the short name then
+    # returns None -- which readers treat as "unknown regime" and fall back to
+    # the shape heuristic. An over-long key would therefore not fail loudly, it
+    # would quietly un-fix the interferometer case. Pin the ceiling.
+    assert len(KEY) <= 8
+    assert KEY == KEY.upper()