Merge pull request #399 from PyAutoLabs/feature/adapt-image-cache-mask-validation

fix: raise a clear exception when adapt data and the data mask disagree

Author: Jammy2211

autoarray/inversion/mappers/abstract.py

@@ -5,6 +5,7 @@
 from autonerves import conf
 from autonerves import cached_property
 
+from autoarray import exc
 from autoarray.inversion.linear_obj.linear_obj import LinearObj
 from autoarray.inversion.linear_obj.func_list import UniqueMappings
 from autoarray.inversion.linear_obj.neighbors import Neighbors
@@ -464,7 +465,30 @@ def pixel_signals_from(self, signal_scale: float, xp=np) -> np.ndarray:
         signal_scale
             A factor which controls how rapidly the smoothness of regularization varies from high signal regions to
             low signal regions.
-        """
+
+        Raises
+        ------
+        exc.InversionException
+            If the `adapt_data` is not defined on the same mask as the data being fitted, which would otherwise
+            surface as a bare `IndexError` from the slim-index lookup inside `adaptive_pixel_signals_from`.
+        """
+        adapt_data = self.adapt_data.array
+
+        data_pixels = self.over_sampler.mask.pixels_in_mask
+
+        if adapt_data.shape[0] != data_pixels:
+            raise exc.InversionException(
+                f"The adapt image passed to the mapper has {adapt_data.shape[0]} pixels, but the data being "
+                f"fitted has {data_pixels} pixels, so the adapt image cannot be indexed by the data's pixels.\n\n"
+                "The adapt image is therefore defined on a different mask to the dataset. The usual cause is a "
+                "stale adapt-image cache: the per-galaxy adapt images written to a previous search's "
+                "`files/galaxy_images_*.fits` are keyed by the search identifier, which encodes the model and "
+                "the search but not the dataset. Re-running with a changed mask, image resolution or "
+                "`PYAUTO_SMALL_DATASETS` setting, but an unchanged model, reuses that search's output "
+                "directory and its now-stale cache.\n\n"
+                "Delete the affected search's output directory (or the `galaxy_images_*.fits` files within it) "
+                "and re-run so the adapt images are recomputed on the current mask."
+            )
 
         return mapper_util.adaptive_pixel_signals_from(
             pixels=self.pixels,
@@ -473,7 +497,7 @@ def pixel_signals_from(self, signal_scale: float, xp=np) -> np.ndarray:
             pix_indexes_for_sub_slim_index=self.pix_indexes_for_sub_slim_index,
             pix_size_for_sub_slim_index=self.pix_sizes_for_sub_slim_index,
             slim_index_for_sub_slim_index=self.over_sampler.slim_for_sub_slim,
-            adapt_data=self.adapt_data.array,
+            adapt_data=adapt_data,
             xp=xp,
         )
 

test_autoarray/inversion/pixelization/mappers/test_abstract.py

@@ -98,7 +98,7 @@ def test__data_weight_total_for_pix_from__multi_pixel_mappings__sums_weights_per
     assert data_weight_total_for_pix == pytest.approx([1.2, 0.7, 0.3, 1.4, 4.4], 1.0e-4)
 
 
-def test__adaptive_pixel_signals_from___matches_util(grid_2d_7x7, image_7x7):
+def test__adaptive_pixel_signals_from___matches_util(grid_2d_7x7):
     pixels = 6
     signal_scale = 2.0
     interpolator = aa.m.MockInterpolator(
@@ -110,11 +110,15 @@ def test__adaptive_pixel_signals_from___matches_util(grid_2d_7x7, image_7x7):
 
     over_sampler = aa.OverSampler(mask=grid_2d_7x7.mask, sub_size=1)
 
+    # The adapt image is defined on the data's own mask, as it is for a real fit.
+
+    adapt_data = aa.Array2D(values=np.ones(9), mask=grid_2d_7x7.mask)
+
     mapper = aa.m.MockMapper(
         source_plane_data_grid=grid_2d_7x7,
         over_sampler=over_sampler,
         interpolator=interpolator,
-        adapt_data=image_7x7,
+        adapt_data=adapt_data,
         parameters=pixels,
     )
 
@@ -127,12 +131,56 @@ def test__adaptive_pixel_signals_from___matches_util(grid_2d_7x7, image_7x7):
         pix_indexes_for_sub_slim_index=interpolator.mappings,
         pix_size_for_sub_slim_index=interpolator.sizes,
         slim_index_for_sub_slim_index=over_sampler.slim_for_sub_slim,
-        adapt_data=np.array(image_7x7),
+        adapt_data=np.array(adapt_data),
     )
 
     assert (pixel_signals == pixel_signals_util).all()
 
 
+def test__pixel_signals_from__adapt_data_on_a_different_mask__raises_clear_exception(
+    grid_2d_7x7,
+):
+    """
+    An adapt image defined on a different mask to the data cannot be indexed by the data's slim indexes.
+
+    The usual cause is a stale adapt-image cache being reused after the dataset's mask changed, and before
+    this guard it surfaced as a bare `IndexError` several frames deep in `adaptive_pixel_signals_from`,
+    which reads as an off-by-one rather than a mask mismatch (PyAutoGalaxy#516).
+    """
+    interpolator = aa.m.MockInterpolator(
+        mappings=np.array([[1], [1], [4], [0], [0], [3], [0], [0], [3]]),
+        sizes=np.array([1, 1, 1, 1, 1, 1, 1, 1, 1]),
+        weights=np.ones(9),
+    )
+
+    over_sampler = aa.OverSampler(mask=grid_2d_7x7.mask, sub_size=1)
+
+    # The data has 9 unmasked pixels, the adapt image only 8.
+
+    adapt_mask = np.full(fill_value=True, shape=(7, 7))
+    adapt_mask[2:5, 2:5] = False
+    adapt_mask[2, 2] = True
+
+    adapt_data = aa.Array2D(
+        values=np.ones(8),
+        mask=aa.Mask2D(mask=adapt_mask, pixel_scales=1.0),
+    )
+
+    mapper = aa.m.MockMapper(
+        source_plane_data_grid=grid_2d_7x7,
+        over_sampler=over_sampler,
+        interpolator=interpolator,
+        adapt_data=adapt_data,
+        parameters=6,
+    )
+
+    with pytest.raises(aa.exc.InversionException) as e:
+        mapper.pixel_signals_from(signal_scale=2.0)
+
+    assert "8 pixels" in str(e.value)
+    assert "9 pixels" in str(e.value)
+
+
 def test__mapped_to_source_from__delaunay_mapper__matches_mapping_matrix_util(
     grid_2d_7x7,
 ):

test_autoarray/inversion/pixelization/mappers/test_rectangular.py

@@ -1,3 +1,5 @@
+import numpy as np
+
 import autoarray as aa
 
 from autoarray.inversion.mesh.mesh.rectangular_adapt_density import (
@@ -52,7 +54,7 @@ def test__pix_indexes_for_sub_slim_index__rectangular_uniform_mesh__matches_util
 
 
 def test__pixel_signals_from__rectangular_adapt_density_mesh__matches_util(
-    grid_2d_sub_1_7x7, image_7x7
+    grid_2d_sub_1_7x7,
 ):
 
     mesh_grid = overlay_grid_from(
@@ -61,6 +63,10 @@ def test__pixel_signals_from__rectangular_adapt_density_mesh__matches_util(
 
     mesh = aa.mesh.RectangularAdaptDensity(shape=(3, 3))
 
+    # The adapt image is defined on the data's own mask, as it is for a real fit.
+
+    image_7x7 = aa.Array2D(values=np.ones(9), mask=grid_2d_sub_1_7x7.mask)
+
     interpolator = mesh.interpolator_from(
         source_plane_data_grid=grid_2d_sub_1_7x7,
         source_plane_mesh_grid=mesh_grid,