diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..f4ae08ee --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,112 @@ +# PyAutoCTI — Agent Instructions + +Canonical, agent-agnostic instructions for this repo. `CLAUDE.md` imports this +file; any tool that does not process `@`-imports should read this directly. + +## What this repo is + +**PyAutoCTI** (package `autocti`) is a Bayesian library for calibrating and +modelling Charge Transfer Inefficiency (CTI) in CCD imaging: charge-injection +imaging (`ImagingCI`) and 1D datasets (`Dataset1D`), trap/CCD models clocked +through the C++ **arctic** code (`Clocker1D`/`Clocker2D` wrapping `arcticpy`), +FPR/EPER extraction (`autocti/extract/`), and per-dataset `Fit*`/`Analysis*` +classes. Heritage: Euclid VIS CTI calibration; also HST ACS +(`autocti/instruments/acs`). + +Dependency direction: autocti may import **autoarray** (data structures), +**autofit** (model-fitting), and **autoconf** (config). Nothing in the PyAuto +stack imports autocti — it is a leaf like PyAutoLens. + +## Resurrection status (2026-07) + +This repo was unmaintained for ~2 years and is being brought back into the +ecosystem via the CTI resurrection epic +([PyAutoCTI#82](https://github.com/PyAutoLabs/PyAutoCTI/issues/82)). Phase 0 +(importable + unit tests green on the current stack) is complete. **The +visualization layer (`autocti/plot/`, `*/plot/*_plotters.py`, +`*/model/plotter_interface.py`) is quarantined**: it still targets the removed +autoarray Plotter API and is rewritten on the matplotlib function API +(mirroring PyAutoGalaxy) in Phase 1. Until then `autocti.plot` is not +importable, `Analysis` visualization no-ops with a logged warning, and the +plot tests are skipped via `test_autocti/conftest.py`. + +## arcticpy (read before installing) + +`import autocti` requires **arcticpy** (pinned 2.6), which is deliberately not +a pip dependency: + +- Its PyPI sdist is **source-only C++** — it needs `libgsl-dev` headers and a + toolchain to build. +- Its own requirements **downgrade numpy below 2.0**, breaking a modern stack. + +Install it after numpy is in place: + +```bash +pip install arcticpy==2.6 --no-build-isolation --no-deps +``` + +If GSL headers are missing and you lack root, extract them locally +(`apt-get download libgsl-dev && dpkg -x ...`) and point `CPPFLAGS`/`LDFLAGS` +at them. + +## Quick commands + +```bash +pip install -e ".[dev]" # install with dev/test extras +python -m pytest test_autocti/ # full test suite +python -m pytest test_autocti/extract/ # one focused directory +``` + +In a sandboxed / restricted environment, point numba and matplotlib at +writable caches: + +```bash +NUMBA_CACHE_DIR=/tmp/numba_cache MPLCONFIGDIR=/tmp/matplotlib python -m pytest test_autocti/ +``` + +## Related repos + +- **Source siblings:** PyAutoConf, PyAutoArray, PyAutoFit (upstream). +- **autocti_workspace** — runnable examples/tutorials (updated in epic Phase 4). +- **autocti_workspace_test** — regression scripts + Euclid tvac/temporal + heritage (rebuilt in epic Phase 5). +- **Science context:** `PyAutoMemory/wiki/cti/` (trap physics, arctic + algorithm, Euclid VIS / HST ACS heritage). + +## Public API + +The public surface is defined authoritatively in `autocti/__init__.py` — read +it rather than trusting a hand-maintained table. Canonical import: + +```python +import autocti as ac +``` + +## Key rules / footguns + +- Import direction: autoarray / autofit / autoconf only — never autogalaxy or + autolens. +- Unit tests are numpy-only; there is no JAX in this library (arctic is C++). +- Slicing an autoarray `Mask2D` returns a plain ndarray — rebuild a `Mask2D` + with the parent's `pixel_scales` before constructing an `Array2D` from it + (see `autocti/extract/two_d/abstract.py`). +- Fits I/O goes through `autoconf.fitsable` (`ndarray_via_fits_from`, + `output_to_fits`, `hdu_list_for_output_from`) — instance `.output_to_fits` + methods no longer exist on autoarray structures. +- All files use Unix line endings (LF, `\n`) — never `\r\n`. + +## Working on issues + +1. Read the issue description and any linked plan. +2. Identify affected files and make the change. +3. Run the full suite: `python -m pytest test_autocti/`. +4. If you changed public API, say so explicitly — autocti_workspace may need + updates. +5. Ensure all tests pass before opening a PR. + +## Never rewrite history + +Never rewrite pushed history on any repo with a remote — no `git init` over a +tracked repo, no force-push to `main`, no fresh-start "Initial commit", no +`filter-repo` / `filter-branch` / `rebase -i` on pushed branches. To get a +clean tree: `git fetch origin && git reset --hard origin/main && git clean -fd`. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..b9e475a0 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,5 @@ +# PyAutoCTI — agent instructions +The canonical, agent-agnostic instructions live in `AGENTS.md`. Claude Code loads them +via the import below; if your tool does not process `@`-imports, open `AGENTS.md` in +this directory and read it directly. +@AGENTS.md diff --git a/autocti/__init__.py b/autocti/__init__.py index c76067de..946ddf58 100644 --- a/autocti/__init__.py +++ b/autocti/__init__.py @@ -71,7 +71,10 @@ from .clocker.two_d import Clocker2D from . import aggregator as agg from . import util -from . import plot + +# `from . import plot` is quarantined: the Plotter object stack targets the +# removed autoarray Plotter API and is rewritten on the new matplotlib function +# API in Phase 1 of the CTI resurrection epic (PyAutoCTI#82). from . import mock as m # noqa from autoconf import conf diff --git a/autocti/aggregator/dataset_1d.py b/autocti/aggregator/dataset_1d.py index e06deb5a..e4c35386 100644 --- a/autocti/aggregator/dataset_1d.py +++ b/autocti/aggregator/dataset_1d.py @@ -3,6 +3,7 @@ import autofit as af import autoarray as aa +from autoconf.fitsable import ndarray_via_hdu_from from autocti.dataset_1d.dataset_1d.dataset_1d import Dataset1D @@ -54,23 +55,28 @@ def _dataset_1d_list_from( for fit in fit_list: layout = fit.value(name=f"{folder}.layout") - data = aa.Array1D.from_primary_hdu(primary_hdu=fit.value(name=f"{folder}.data")) - noise_map = aa.Array1D.from_primary_hdu( - primary_hdu=fit.value(name=f"{folder}.noise_map") - ) - pre_cti_data = aa.Array1D.from_primary_hdu( - primary_hdu=fit.value(name=f"{folder}.pre_cti_data") + hdu_list = fit.value(name=f"{folder}.dataset") + + pixel_scales = hdu_list[0].header["PIXSCA"] + + mask = aa.Mask1D( + mask=ndarray_via_hdu_from(hdu_list[0]).astype("bool"), + pixel_scales=pixel_scales, ) + def values_from(hdu: int) -> aa.Array1D: + return aa.Array1D.no_mask( + values=ndarray_via_hdu_from(hdu_list[hdu]), + pixel_scales=pixel_scales, + ) + dataset = Dataset1D( - data=data, - noise_map=noise_map, - pre_cti_data=pre_cti_data, + data=values_from(hdu=1), + noise_map=values_from(hdu=2), + pre_cti_data=values_from(hdu=3), layout=layout, ) - mask = aa.Mask1D.from_primary_hdu(primary_hdu=fit.value(name=f"{folder}.mask")) - dataset_list.append(dataset.apply_mask(mask=mask)) return dataset_list diff --git a/autocti/aggregator/imaging_ci.py b/autocti/aggregator/imaging_ci.py index 37d7c9e5..1bb1b8fe 100644 --- a/autocti/aggregator/imaging_ci.py +++ b/autocti/aggregator/imaging_ci.py @@ -1,6 +1,7 @@ from functools import partial import autoarray as aa +from autoconf.fitsable import ndarray_via_hdu_from import autofit as af from autocti.charge_injection.imaging.imaging import ImagingCI @@ -56,33 +57,40 @@ def _imaging_ci_list_from(fit: af.Fit, use_dataset_full: bool = False): for fit in fit_list: layout = fit.value(name=f"{folder}.layout") - data = aa.Array2D.from_primary_hdu(primary_hdu=fit.value(name=f"{folder}.data")) - noise_map = aa.Array2D.from_primary_hdu( - primary_hdu=fit.value(name=f"{folder}.noise_map") + hdu_list = fit.value(name=f"{folder}.dataset") + + pixel_scales = ( + hdu_list[0].header["PIXSCAY"], + hdu_list[0].header["PIXSCAX"], ) - pre_cti_data = aa.Array2D.from_primary_hdu( - primary_hdu=fit.value(name=f"{folder}.pre_cti_data") + + mask = aa.Mask2D( + mask=ndarray_via_hdu_from(hdu_list[0]).astype("bool"), + pixel_scales=pixel_scales, ) - try: - cosmic_ray_map = aa.Array2D.from_primary_hdu( - primary_hdu=fit.value(name=f"{folder}.cosmic_ray_map") + + def values_from(hdu: int) -> aa.Array2D: + return aa.Array2D.no_mask( + values=ndarray_via_hdu_from(hdu_list[hdu]), + pixel_scales=pixel_scales, ) - except AttributeError: + + try: + cosmic_ray_map = values_from(hdu=4) + except IndexError: cosmic_ray_map = None settings_dict = fit.value(name="dataset.settings_dict") dataset = ImagingCI( - data=data, - noise_map=noise_map, - pre_cti_data=pre_cti_data, + data=values_from(hdu=1), + noise_map=values_from(hdu=2), + pre_cti_data=values_from(hdu=3), cosmic_ray_map=cosmic_ray_map, settings_dict=settings_dict, layout=layout, ) - mask = aa.Mask2D.from_primary_hdu(primary_hdu=fit.value(name=f"{folder}.mask")) - dataset_list.append(dataset.apply_mask(mask=mask)) return dataset_list diff --git a/autocti/charge_injection/imaging/imaging.py b/autocti/charge_injection/imaging/imaging.py index 01e8d70e..3c0b78db 100644 --- a/autocti/charge_injection/imaging/imaging.py +++ b/autocti/charge_injection/imaging/imaging.py @@ -1,333 +1,344 @@ -import numpy as np -from pathlib import Path -from typing import Optional, List, Dict, Union - -import autoarray as aa - -from autocti.charge_injection.imaging.settings import SettingsImagingCI -from autocti.charge_injection.layout import Layout2DCI -from autocti.extract.settings import SettingsExtract -from autocti.mask import mask_2d -from autocti import exc - - -class ImagingCI(aa.Imaging): - def __init__( - self, - data: aa.Array2D, - noise_map: aa.Array2D, - pre_cti_data: aa.Array2D, - layout: Layout2DCI, - cosmic_ray_map: Optional[aa.Array2D] = None, - mask_persistence=None, - noise_scaling_map_dict: Optional[Dict] = None, - fpr_value: Optional[float] = None, - settings_dict: Optional[Dict] = None, - ): - super().__init__(data=data, noise_map=noise_map) - - self.data = self.data.native - self.noise_map = self.noise_map.native - self.pre_cti_data = pre_cti_data.native - - if cosmic_ray_map is not None: - cosmic_ray_map = cosmic_ray_map.native - - self.cosmic_ray_map = cosmic_ray_map - self.mask_persistence = mask_persistence - - if noise_scaling_map_dict is not None: - noise_scaling_map_dict = { - key: noise_scaling_map.native - for key, noise_scaling_map in noise_scaling_map_dict.items() - } - - self.noise_scaling_map_dict = noise_scaling_map_dict - - self.layout = layout - - if fpr_value is None: - fpr_value = np.round( - np.mean( - self.layout.extract.parallel_fpr.median_list_from( - array=self.data, - settings=SettingsExtract( - pixels_from_end=min( - 10, self.layout.smallest_parallel_rows_within_ci_regions - ) - ), - ) - ), - 2, - ) - - self.fpr_value = fpr_value - self.settings_dict = settings_dict - - @property - def mask(self): - return self.data.mask - - @property - def region_list(self): - return self.layout.region_list - - @property - def norm_columns_list(self) -> List: - """ - The `layout` describes the 2D regions on the data containing charge whose input signal properties are know - beforehand (e.g. charge injection imaging). - - However, the exact values may not be known and therefore need to be estimated from the image. - - This function estimates the normalization of every column of data in the 2D regions, by taking the median - of each column. If a mask is applied (e.g. to remove cosmic rays) these pixels are omitted from the median. - - Returns - ------- - A list of the normalization of every column of the charge regions - """ - masked_image = np.ma.array(data=self.data, mask=self.data.mask) - - return [ - np.ma.median(masked_image[region.y0 : region.y1, column_index]) - for region in self.region_list - for column_index in range(region.x0, region.x1) - ] - - @property - def pre_cti_data_residual_map(self) -> aa.Array2D: - """ - The residuals of the data and the pre CTI data. - - This is used to assess whether the pre CTI data has been estimated accurately (e.g. from the FPR of the - data) and includes e specific set of visualization functions. - - Returns - ------- - The residual map of the data and pre CTI data. - """ - return self.data - self.pre_cti_data - - def apply_mask(self, mask: mask_2d.Mask2D) -> "ImagingCI": - image = aa.Array2D(values=self.data.native, mask=mask) - noise_map = aa.Array2D(values=self.noise_map.native, mask=mask) - - if self.cosmic_ray_map is not None: - cosmic_ray_map = aa.Array2D(values=self.cosmic_ray_map.native, mask=mask) - - else: - cosmic_ray_map = None - - if self.noise_scaling_map_dict is not None: - noise_scaling_map_dict = { - key: aa.Array2D(values=noise_scaling_map.native, mask=mask) - for key, noise_scaling_map in self.noise_scaling_map_dict.items() - } - - else: - noise_scaling_map_dict = None - - return ImagingCI( - data=image, - noise_map=noise_map, - pre_cti_data=self.pre_cti_data.native, - layout=self.layout, - cosmic_ray_map=cosmic_ray_map, - mask_persistence=self.mask_persistence, - noise_scaling_map_dict=noise_scaling_map_dict, - fpr_value=self.fpr_value, - settings_dict=self.settings_dict, - ) - - def apply_settings(self, settings: SettingsImagingCI): - if settings.parallel_pixels is not None: - dataset = self.layout.extract.parallel_calibration.imaging_ci_from( - dataset=self, columns=settings.parallel_pixels - ) - - mask = self.layout.extract.parallel_calibration.mask_2d_from( - mask=self.mask, columns=settings.parallel_pixels - ) - - elif settings.serial_pixels is not None: - dataset = self.layout.extract.serial_calibration.imaging_ci_from( - dataset=self, rows=settings.serial_pixels - ) - - mask = self.layout.extract.serial_calibration.mask_2d_from( - mask=self.mask, rows=settings.serial_pixels - ) - - else: - return self - - dataset = dataset.apply_mask(mask=mask) - - return dataset - - def set_noise_scaling_map_dict(self, noise_scaling_map_dict: Dict): - self.noise_scaling_map_dict = { - key: noise_scaling_map.native - for key, noise_scaling_map in noise_scaling_map_dict.items() - } - - @classmethod - def from_fits( - cls, - pixel_scales: aa.type.PixelScales, - layout: Layout2DCI, - data_path: Optional[Union[Path, str]] = None, - data_hdu: int = 0, - data: aa.Array2D = None, - noise_map_path: Optional[Union[Path, str]] = None, - noise_map_hdu: int = 0, - noise_map_from_single_value: float = None, - pre_cti_data_path: Optional[Union[Path, str]] = None, - pre_cti_data_hdu: int = 0, - pre_cti_data: aa.Array2D = None, - cosmic_ray_map_path: Optional[Union[Path, str]] = None, - cosmic_ray_map_hdu: int = 0, - settings_dict: Optional[Dict] = None, - ) -> "ImagingCI": - """ - Load charge injection imaging from multiple .fits file. - - For each attribute of the charge injection data (e.g. `data`, `noise_map`, `pre_cti_data`) the path to - the .fits and the `hdu` containing the data can be specified. - - The `noise_map` assumes the noise value in each `data` value are independent, where these values are the - RMS standard deviation error in each pixel. - - If the dataset has a mask associated with it (e.g. in a `mask.fits` file) the file must be loaded separately - via the `Mask2D` object and applied to the imaging after loading via fits using the `from_fits` method. - - Parameters - ---------- - pixel_scales - The (y,x) arcsecond-to-pixel units conversion factor of every pixel. If this is input as a `float`, - it is converted to a (float, float). - layout - The layout of the charge injection, containing information like where the parallel and serial FPR and - EPER are located. - data_path - The path to the data .fits file containing the image data (e.g. '/path/to/data.fits'). - data_hdu - The hdu the image data is contained in the .fits file specified by `data_path`. - data - Manually input the data as an `Array2D` instead of loading it via a .fits file. - noise_map_path - The path to the noise_map .fits file containing the noise_map (e.g. '/path/to/noise_map.fits'). - noise_map_hdu - The hdu the noise map is contained in the .fits file specified by `noise_map_path`. - noise_map_from_single_value - Creates a `noise_map` of constant values if this is input instead of loading via .fits. - pre_cti_data_path - The path to the pre CTI data .fits file containing the image data (e.g. '/path/to/pre_cti_data.fits'). - pre_cti_data_hdu - The hdu the pre cti data is contained in the .fits file specified by `pre_cti_data_path`. - pre_cti_data - Manually input the pre CTI data as an `Array2D` instead of loading it via a .fits file. - cosmic_ray_map_path - The path to the cosmic ray map .fits file containing the map of cosmic - rays (e.g. '/path/to/cosmic_ray_map.fits'). - cosmic_ray_map_hdu - The hdu the cosmic ray data is contained in the .fits file specified by `cosmic_ray_map_path`. - settings_dict - A dictionary of settings associated with the charge injeciton imaging (e.g. voltage settings) which is - used for visualization. - """ - if data_path is not None and data is None: - data = aa.Array2D.from_fits( - file_path=data_path, hdu=data_hdu, pixel_scales=pixel_scales - ) - - if noise_map_path is not None: - noise_map = aa.util.array_2d.numpy_array_2d_via_fits_from( - file_path=noise_map_path, hdu=noise_map_hdu - ) - else: - noise_map = np.ones(data.shape_native) * noise_map_from_single_value - - noise_map = aa.Array2D.no_mask(values=noise_map, pixel_scales=pixel_scales) - - if pre_cti_data_path is not None and pre_cti_data is None: - pre_cti_data = aa.Array2D.from_fits( - file_path=pre_cti_data_path, - hdu=pre_cti_data_hdu, - pixel_scales=pixel_scales, - ) - elif pre_cti_data is None: - raise exc.ImagingCIException( - "Cannot load pre_cti_data from .fits and pass explicit pre_cti_data." - ) - - pre_cti_data = aa.Array2D.no_mask( - values=pre_cti_data.native, pixel_scales=pixel_scales - ) - - if cosmic_ray_map_path is not None: - cosmic_ray_map = aa.Array2D.from_fits( - file_path=cosmic_ray_map_path, - hdu=cosmic_ray_map_hdu, - pixel_scales=pixel_scales, - ) - - else: - cosmic_ray_map = None - - return ImagingCI( - data=data, - noise_map=noise_map, - pre_cti_data=pre_cti_data, - cosmic_ray_map=cosmic_ray_map, - layout=layout, - settings_dict=settings_dict, - ) - - def output_to_fits( - self, - data_path: Union[Path, str], - noise_map_path: Optional[Union[Path, str]] = None, - pre_cti_data_path: Optional[Union[Path, str]] = None, - cosmic_ray_map_path: Optional[Union[Path, str]] = None, - overwrite: bool = False, - ): - """ - Output the charge injection imaging dataset to multiple .fits file. - - For each attribute of the charge injection imaging data (e.g. `data`, `noise_map`, `pre_cti_data`) the path to - the .fits can be specified, with `hdu=0` assumed automatically. - - If the `data` has been masked, the masked data is output to .fits files. A mask can be separately output to - a file `mask.fits` via the `Mask` objects `output_to_fits` method. - - Parameters - ---------- - data_path - The path to the data .fits file where the image data is output (e.g. '/path/to/data.fits'). - noise_map_path - The path to the noise_map .fits where the noise_map is output (e.g. '/path/to/noise_map.fits'). - pre_cti_data_path - The path to the pre CTI data .fits file where the pre CTI data is output (e.g. '/path/to/pre_cti_data.fits'). - cosmic_ray_map_path - The path to the cosmic ray map .fits file where the cosmic ray map is - output (e.g. '/path/to/cosmic_ray_map.fits'). - overwrite - If `True`, the .fits files are overwritten if they already exist, if `False` they are not and an - exception is raised. - """ - self.data.output_to_fits(file_path=data_path, overwrite=overwrite) - - if noise_map_path is not None: - self.noise_map.output_to_fits(file_path=noise_map_path, overwrite=overwrite) - - if pre_cti_data_path is not None: - self.pre_cti_data.output_to_fits( - file_path=pre_cti_data_path, overwrite=overwrite - ) - - if self.cosmic_ray_map is not None and cosmic_ray_map_path is not None: - self.cosmic_ray_map.output_to_fits( - file_path=cosmic_ray_map_path, overwrite=overwrite - ) +import numpy as np +from pathlib import Path +from typing import Optional, List, Dict, Union + +import autoarray as aa +from autoconf import fitsable + +from autocti.charge_injection.imaging.settings import SettingsImagingCI +from autocti.charge_injection.layout import Layout2DCI +from autocti.extract.settings import SettingsExtract +from autocti.mask import mask_2d +from autocti import exc + + +class ImagingCI(aa.Imaging): + def __init__( + self, + data: aa.Array2D, + noise_map: aa.Array2D, + pre_cti_data: aa.Array2D, + layout: Layout2DCI, + cosmic_ray_map: Optional[aa.Array2D] = None, + mask_persistence=None, + noise_scaling_map_dict: Optional[Dict] = None, + fpr_value: Optional[float] = None, + settings_dict: Optional[Dict] = None, + ): + super().__init__(data=data, noise_map=noise_map) + + self.data = self.data.native + self.noise_map = self.noise_map.native + self.pre_cti_data = pre_cti_data.native + + if cosmic_ray_map is not None: + cosmic_ray_map = cosmic_ray_map.native + + self.cosmic_ray_map = cosmic_ray_map + self.mask_persistence = mask_persistence + + if noise_scaling_map_dict is not None: + noise_scaling_map_dict = { + key: noise_scaling_map.native + for key, noise_scaling_map in noise_scaling_map_dict.items() + } + + self.noise_scaling_map_dict = noise_scaling_map_dict + + self.layout = layout + + if fpr_value is None: + fpr_value = np.round( + np.mean( + self.layout.extract.parallel_fpr.median_list_from( + array=self.data, + settings=SettingsExtract( + pixels_from_end=min( + 10, self.layout.smallest_parallel_rows_within_ci_regions + ) + ), + ) + ), + 2, + ) + + self.fpr_value = fpr_value + self.settings_dict = settings_dict + + @property + def mask(self): + return self.data.mask + + @property + def region_list(self): + return self.layout.region_list + + @property + def norm_columns_list(self) -> List: + """ + The `layout` describes the 2D regions on the data containing charge whose input signal properties are know + beforehand (e.g. charge injection imaging). + + However, the exact values may not be known and therefore need to be estimated from the image. + + This function estimates the normalization of every column of data in the 2D regions, by taking the median + of each column. If a mask is applied (e.g. to remove cosmic rays) these pixels are omitted from the median. + + Returns + ------- + A list of the normalization of every column of the charge regions + """ + masked_image = np.ma.array(data=self.data, mask=self.data.mask) + + return [ + np.ma.median(masked_image[region.y0 : region.y1, column_index]) + for region in self.region_list + for column_index in range(region.x0, region.x1) + ] + + @property + def pre_cti_data_residual_map(self) -> aa.Array2D: + """ + The residuals of the data and the pre CTI data. + + This is used to assess whether the pre CTI data has been estimated accurately (e.g. from the FPR of the + data) and includes e specific set of visualization functions. + + Returns + ------- + The residual map of the data and pre CTI data. + """ + return self.data - self.pre_cti_data + + def apply_mask(self, mask: mask_2d.Mask2D) -> "ImagingCI": + image = aa.Array2D(values=self.data.native, mask=mask) + noise_map = aa.Array2D(values=self.noise_map.native, mask=mask) + + if self.cosmic_ray_map is not None: + cosmic_ray_map = aa.Array2D(values=self.cosmic_ray_map.native, mask=mask) + + else: + cosmic_ray_map = None + + if self.noise_scaling_map_dict is not None: + noise_scaling_map_dict = { + key: aa.Array2D(values=noise_scaling_map.native, mask=mask) + for key, noise_scaling_map in self.noise_scaling_map_dict.items() + } + + else: + noise_scaling_map_dict = None + + return ImagingCI( + data=image, + noise_map=noise_map, + pre_cti_data=self.pre_cti_data.native, + layout=self.layout, + cosmic_ray_map=cosmic_ray_map, + mask_persistence=self.mask_persistence, + noise_scaling_map_dict=noise_scaling_map_dict, + fpr_value=self.fpr_value, + settings_dict=self.settings_dict, + ) + + def apply_settings(self, settings: SettingsImagingCI): + if settings.parallel_pixels is not None: + dataset = self.layout.extract.parallel_calibration.imaging_ci_from( + dataset=self, columns=settings.parallel_pixels + ) + + mask = self.layout.extract.parallel_calibration.mask_2d_from( + mask=self.mask, columns=settings.parallel_pixels + ) + + elif settings.serial_pixels is not None: + dataset = self.layout.extract.serial_calibration.imaging_ci_from( + dataset=self, rows=settings.serial_pixels + ) + + mask = self.layout.extract.serial_calibration.mask_2d_from( + mask=self.mask, rows=settings.serial_pixels + ) + + else: + return self + + dataset = dataset.apply_mask(mask=mask) + + return dataset + + def set_noise_scaling_map_dict(self, noise_scaling_map_dict: Dict): + self.noise_scaling_map_dict = { + key: noise_scaling_map.native + for key, noise_scaling_map in noise_scaling_map_dict.items() + } + + @classmethod + def from_fits( + cls, + pixel_scales: aa.type.PixelScales, + layout: Layout2DCI, + data_path: Optional[Union[Path, str]] = None, + data_hdu: int = 0, + data: aa.Array2D = None, + noise_map_path: Optional[Union[Path, str]] = None, + noise_map_hdu: int = 0, + noise_map_from_single_value: float = None, + pre_cti_data_path: Optional[Union[Path, str]] = None, + pre_cti_data_hdu: int = 0, + pre_cti_data: aa.Array2D = None, + cosmic_ray_map_path: Optional[Union[Path, str]] = None, + cosmic_ray_map_hdu: int = 0, + settings_dict: Optional[Dict] = None, + ) -> "ImagingCI": + """ + Load charge injection imaging from multiple .fits file. + + For each attribute of the charge injection data (e.g. `data`, `noise_map`, `pre_cti_data`) the path to + the .fits and the `hdu` containing the data can be specified. + + The `noise_map` assumes the noise value in each `data` value are independent, where these values are the + RMS standard deviation error in each pixel. + + If the dataset has a mask associated with it (e.g. in a `mask.fits` file) the file must be loaded separately + via the `Mask2D` object and applied to the imaging after loading via fits using the `from_fits` method. + + Parameters + ---------- + pixel_scales + The (y,x) arcsecond-to-pixel units conversion factor of every pixel. If this is input as a `float`, + it is converted to a (float, float). + layout + The layout of the charge injection, containing information like where the parallel and serial FPR and + EPER are located. + data_path + The path to the data .fits file containing the image data (e.g. '/path/to/data.fits'). + data_hdu + The hdu the image data is contained in the .fits file specified by `data_path`. + data + Manually input the data as an `Array2D` instead of loading it via a .fits file. + noise_map_path + The path to the noise_map .fits file containing the noise_map (e.g. '/path/to/noise_map.fits'). + noise_map_hdu + The hdu the noise map is contained in the .fits file specified by `noise_map_path`. + noise_map_from_single_value + Creates a `noise_map` of constant values if this is input instead of loading via .fits. + pre_cti_data_path + The path to the pre CTI data .fits file containing the image data (e.g. '/path/to/pre_cti_data.fits'). + pre_cti_data_hdu + The hdu the pre cti data is contained in the .fits file specified by `pre_cti_data_path`. + pre_cti_data + Manually input the pre CTI data as an `Array2D` instead of loading it via a .fits file. + cosmic_ray_map_path + The path to the cosmic ray map .fits file containing the map of cosmic + rays (e.g. '/path/to/cosmic_ray_map.fits'). + cosmic_ray_map_hdu + The hdu the cosmic ray data is contained in the .fits file specified by `cosmic_ray_map_path`. + settings_dict + A dictionary of settings associated with the charge injeciton imaging (e.g. voltage settings) which is + used for visualization. + """ + if data_path is not None and data is None: + data = aa.Array2D.from_fits( + file_path=data_path, hdu=data_hdu, pixel_scales=pixel_scales + ) + + if noise_map_path is not None: + noise_map = fitsable.ndarray_via_fits_from( + file_path=noise_map_path, hdu=noise_map_hdu + ) + else: + noise_map = np.ones(data.shape_native) * noise_map_from_single_value + + noise_map = aa.Array2D.no_mask(values=noise_map, pixel_scales=pixel_scales) + + if pre_cti_data_path is not None and pre_cti_data is None: + pre_cti_data = aa.Array2D.from_fits( + file_path=pre_cti_data_path, + hdu=pre_cti_data_hdu, + pixel_scales=pixel_scales, + ) + elif pre_cti_data is None: + raise exc.ImagingCIException( + "Cannot load pre_cti_data from .fits and pass explicit pre_cti_data." + ) + + pre_cti_data = aa.Array2D.no_mask( + values=pre_cti_data.native, pixel_scales=pixel_scales + ) + + if cosmic_ray_map_path is not None: + cosmic_ray_map = aa.Array2D.from_fits( + file_path=cosmic_ray_map_path, + hdu=cosmic_ray_map_hdu, + pixel_scales=pixel_scales, + ) + + else: + cosmic_ray_map = None + + return ImagingCI( + data=data, + noise_map=noise_map, + pre_cti_data=pre_cti_data, + cosmic_ray_map=cosmic_ray_map, + layout=layout, + settings_dict=settings_dict, + ) + + def output_to_fits( + self, + data_path: Union[Path, str], + noise_map_path: Optional[Union[Path, str]] = None, + pre_cti_data_path: Optional[Union[Path, str]] = None, + cosmic_ray_map_path: Optional[Union[Path, str]] = None, + overwrite: bool = False, + ): + """ + Output the charge injection imaging dataset to multiple .fits file. + + For each attribute of the charge injection imaging data (e.g. `data`, `noise_map`, `pre_cti_data`) the path to + the .fits can be specified, with `hdu=0` assumed automatically. + + If the `data` has been masked, the masked data is output to .fits files. A mask can be separately output to + a file `mask.fits` via the `Mask` objects `output_to_fits` method. + + Parameters + ---------- + data_path + The path to the data .fits file where the image data is output (e.g. '/path/to/data.fits'). + noise_map_path + The path to the noise_map .fits where the noise_map is output (e.g. '/path/to/noise_map.fits'). + pre_cti_data_path + The path to the pre CTI data .fits file where the pre CTI data is output (e.g. '/path/to/pre_cti_data.fits'). + cosmic_ray_map_path + The path to the cosmic ray map .fits file where the cosmic ray map is + output (e.g. '/path/to/cosmic_ray_map.fits'). + overwrite + If `True`, the .fits files are overwritten if they already exist, if `False` they are not and an + exception is raised. + """ + fitsable.output_to_fits( + values=np.asarray(self.data.native), file_path=data_path, overwrite=overwrite + ) + + if noise_map_path is not None: + fitsable.output_to_fits( + values=np.asarray(self.noise_map.native), + file_path=noise_map_path, + overwrite=overwrite, + ) + + if pre_cti_data_path is not None: + fitsable.output_to_fits( + values=np.asarray(self.pre_cti_data.native), + file_path=pre_cti_data_path, + overwrite=overwrite, + ) + + if self.cosmic_ray_map is not None and cosmic_ray_map_path is not None: + fitsable.output_to_fits( + values=np.asarray(self.cosmic_ray_map.native), + file_path=cosmic_ray_map_path, + overwrite=overwrite, + ) diff --git a/autocti/charge_injection/model/analysis.py b/autocti/charge_injection/model/analysis.py index ccffc540..6e1e9b4e 100644 --- a/autocti/charge_injection/model/analysis.py +++ b/autocti/charge_injection/model/analysis.py @@ -1,300 +1,303 @@ -import logging -from typing import List, Optional - -from autoconf import conf -from autoconf.dictable import to_dict - -import autoarray as aa -import autofit as af - -from autocti.charge_injection.imaging.imaging import ImagingCI -from autocti.charge_injection.fit import FitImagingCI -from autocti.charge_injection.model.visualizer import VisualizerImagingCI -from autocti.charge_injection.model.result import ResultImagingCI -from autocti.clocker.two_d import Clocker2D -from autocti.charge_injection.hyper import HyperCINoiseCollection -from autocti.model.analysis import AnalysisCTI -from autocti.model.settings import SettingsCTI2D -from autocti.preloads import Preloads - -from autocti import exc - -logger = logging.getLogger(__name__) - -logger.setLevel(level="INFO") - - -class AnalysisImagingCI(AnalysisCTI): - Result = ResultImagingCI - Visualizer = VisualizerImagingCI - - def __init__( - self, - dataset: ImagingCI, - clocker: Clocker2D, - settings_cti: SettingsCTI2D = SettingsCTI2D(), - dataset_full: Optional[ImagingCI] = None, - ): - """ - Fits a CTI model to a charge injection imaging dataset via a non-linear search. - - The `Analysis` class defines the `log_likelihood_function` which fits the model to the dataset and returns the - log likelihood value defining how well the model fitted the data. - - It handles many other tasks, such as visualization, outputting results to hard-disk and storing results in - a format that can be loaded after the model-fit is complete. - - This class is used for model-fits which fit a CTI model via a `CTI2D` object to a charge injection - imaging dataset. - - Parameters - ---------- - dataset - The charge injection dataset that the model is fitted to. - clocker - The CTI arctic clocker used by the non-linear search and model-fit. - settings_cti - The settings controlling aspects of the CTI model in this model-fit. - dataset_full - The full dataset, which is visualized separate from the `dataset` that is fitted, which for example may - not have the FPR masked and thus enable visualization of the FPR. - """ - super().__init__( - dataset=dataset, - clocker=clocker, - settings_cti=settings_cti, - dataset_full=dataset_full, - ) - - self.preloads = Preloads() - - parallel_fast_index_list = None - parallel_fast_column_lists = None - - serial_fast_index_list = None - serial_fast_row_lists = None - - if self.clocker.parallel_fast_mode and not self.clocker.serial_fast_mode: - ( - parallel_fast_index_list, - parallel_fast_column_lists, - ) = clocker.fast_indexes_from(data=dataset.pre_cti_data, for_parallel=True) - - elif not self.clocker.parallel_fast_mode and self.clocker.serial_fast_mode: - serial_fast_index_list, serial_fast_row_lists = clocker.fast_indexes_from( - data=dataset.pre_cti_data, for_parallel=False - ) - - elif self.clocker.parallel_fast_mode and self.clocker.serial_fast_mode: - raise exc.ClockerException( - "Both parallel fast model and serial fast mode cannot be turned on.\n" - "Only switch on parallel fast mode for parallel + serial clocking." - ) - - self.preloads = Preloads( - parallel_fast_index_list=parallel_fast_index_list, - parallel_fast_column_lists=parallel_fast_column_lists, - serial_fast_index_list=serial_fast_index_list, - serial_fast_row_lists=serial_fast_row_lists, - ) - - def region_list_from(self, model: af.Collection) -> List: - """ - Inspects the CTI model and determines which regions are fitted for and therefore should be visualized. - - For example, if the model only includes parallel CTI, the serial regions are not fitted for and thus are not - visualized. - - Parameters - ---------- - model - The CTI model, composed via PyAutoFit, which represents the parallel and serial CTI model compoenents - fitted for by the non-linear search. - - Returns - ------- - A list of the regions fitted for by the model and therefore visualized. - - """ - if model.cti.parallel_ccd is not None and model.cti.serial_ccd is None: - return ["parallel_fpr", "parallel_eper"] - elif model.cti.serial_ccd is not None and model.cti.parallel_ccd is None: - return ["serial_fpr", "serial_eper"] - elif model.cti.pixel_bounce_list is not None: - return ["serial_fpr", "serial_eper"] - return ["parallel_fpr", "parallel_eper", "serial_fpr", "serial_eper"] - - def modify_before_fit(self, paths: af.DirectoryPaths, model: af.Collection): - """ - This function is called immediately before the non-linear search begins and performs final tasks and checks - before it begins. - - This function: - - 1) Visualizes the charge injection imaging dataset, which does not change during the analysis and thus can be - done once. - - 2) Checks if the noise-map is fixed (it is not if hyper functionality is on), and if it is fixed it - sets the noise-normalization to the preloads for computational speed. - - Parameters - ---------- - paths - The paths object which manages all paths, e.g. where the non-linear search outputs are stored, - visualization and the pickled objects used by the aggregator output by this function. - model - The model object, which includes model components representing the galaxies that are fitted to - the imaging data. - """ - - if paths.is_complete: - return self - - if not model.has(HyperCINoiseCollection): - noise_normalization = aa.util.fit.noise_normalization_with_mask_from( - noise_map=self.dataset.noise_map, mask=self.dataset.mask - ) - - self.preloads.noise_normalization = noise_normalization - - logger.info( - "PRELOADS - Noise Normalization preloaded for model-fit (noise-map is fixed)." - ) - - return self - - def log_likelihood_function(self, instance: af.ModelInstance) -> float: - """ - Determine the fitness of a particular model - - Parameters - ---------- - instance - - Returns - ------- - fit: Fit - How fit the model is and the model - """ - - self.settings_cti.check_total_density_within_range( - parallel_traps=instance.cti.parallel_trap_list, - serial_traps=instance.cti.serial_trap_list, - ) - - fit = self.fit_via_instance_and_dataset_from( - instance=instance, dataset=self.dataset, hyper_noise_scale=True - ) - - return fit.figure_of_merit - - def fit_via_instance_and_dataset_from( - self, - instance: af.ModelInstance, - dataset: ImagingCI, - hyper_noise_scale: bool = True, - ) -> FitImagingCI: - hyper_noise_scalar_dict = None - - if hyper_noise_scale and hasattr(instance, "hyper_noise"): - hyper_noise_scalar_dict = instance.hyper_noise.as_dict - - post_cti_data = self.clocker.add_cti( - data=dataset.pre_cti_data, - cti=instance.cti, - preloads=self.preloads, - ) - - return FitImagingCI( - dataset=dataset, - post_cti_data=post_cti_data, - hyper_noise_scalar_dict=hyper_noise_scalar_dict, - preloads=self.preloads, - ) - - def fit_via_instance_from( - self, instance: af.ModelInstance, hyper_noise_scale: bool = True - ) -> FitImagingCI: - return self.fit_via_instance_and_dataset_from( - instance=instance, - dataset=self.dataset, - hyper_noise_scale=hyper_noise_scale, - ) - - def save_attributes(self, paths: af.DirectoryPaths): - """ - Before the model-fit via the non-linear search begins, this routine saves attributes of the `Analysis` object - to the `files` folder such that they can be loaded after the analysis using PyAutoFit's database and - aggregator tools. - - For this analysis the following are output: - - - The charge injection dataset (data / noise-map / pre cti data / cosmic ray map / layout / settings etc.). - - The mask applied to the dataset. - - The clocker used for modeling / clocking CTI. - - The settings used for modeling / clocking CTI. - - The full 1D dataset (e.g. unmasked, used for visualizariton). - - It is common for these attributes to be loaded by many of the template aggregator functions given in the - `aggregator` modules. For example, when using the database tools to reperform a fit, this will by default - load the dataset, settings and other attributes necessary to perform a fit using the attributes output by - this function. - - Parameters - ---------- - paths - The paths object which manages all paths, e.g. where the non-linear search outputs are stored, - visualization,and the pickled objects used by the aggregator output by this function. - """ - - paths.save_json( - name="clocker", - object_dict=to_dict(self.clocker), - ) - - paths.save_json( - name="settings_cti", - object_dict=to_dict(self.settings_cti), - ) - - if conf.instance["visualize"]["plots"]["combined_only"]: - return - - def output_dataset(dataset, prefix): - paths.save_fits( - name="data", - hdu=dataset.data.hdu_for_output, - prefix=prefix, - ) - paths.save_fits( - name="noise_map", - hdu=dataset.noise_map.hdu_for_output, - prefix=prefix, - ) - paths.save_fits( - name="pre_cti_data", - hdu=dataset.pre_cti_data.hdu_for_output, - prefix=prefix, - ) - paths.save_json( - name="layout", - object_dict=to_dict(dataset.layout), - prefix=prefix, - ) - paths.save_fits( - name="mask", - hdu=dataset.mask.hdu_for_output, - prefix=prefix, - ) - - if self.dataset.settings_dict is not None: - paths.save_json( - name="settings_dict", - object_dict=self.dataset.settings_dict, - prefix="dataset", - ) - - output_dataset(dataset=self.dataset, prefix="dataset") - - if self.dataset_full is not None: - output_dataset(dataset=self.dataset_full, prefix="dataset_full") +import numpy as np +import logging +from typing import List, Optional + +from autoconf import conf +from autoconf.dictable import to_dict + +import autoarray as aa +import autofit as af +from autoconf.fitsable import hdu_list_for_output_from + +from autocti.charge_injection.imaging.imaging import ImagingCI +from autocti.charge_injection.fit import FitImagingCI +from autocti.charge_injection.model.visualizer import VisualizerImagingCI +from autocti.charge_injection.model.result import ResultImagingCI +from autocti.clocker.two_d import Clocker2D +from autocti.charge_injection.hyper import HyperCINoiseCollection +from autocti.model.analysis import AnalysisCTI +from autocti.model.settings import SettingsCTI2D +from autocti.preloads import Preloads + +from autocti import exc + +logger = logging.getLogger(__name__) + +logger.setLevel(level="INFO") + + +class AnalysisImagingCI(AnalysisCTI): + Result = ResultImagingCI + Visualizer = VisualizerImagingCI + + def __init__( + self, + dataset: ImagingCI, + clocker: Clocker2D, + settings_cti: SettingsCTI2D = SettingsCTI2D(), + dataset_full: Optional[ImagingCI] = None, + ): + """ + Fits a CTI model to a charge injection imaging dataset via a non-linear search. + + The `Analysis` class defines the `log_likelihood_function` which fits the model to the dataset and returns the + log likelihood value defining how well the model fitted the data. + + It handles many other tasks, such as visualization, outputting results to hard-disk and storing results in + a format that can be loaded after the model-fit is complete. + + This class is used for model-fits which fit a CTI model via a `CTI2D` object to a charge injection + imaging dataset. + + Parameters + ---------- + dataset + The charge injection dataset that the model is fitted to. + clocker + The CTI arctic clocker used by the non-linear search and model-fit. + settings_cti + The settings controlling aspects of the CTI model in this model-fit. + dataset_full + The full dataset, which is visualized separate from the `dataset` that is fitted, which for example may + not have the FPR masked and thus enable visualization of the FPR. + """ + super().__init__( + dataset=dataset, + clocker=clocker, + settings_cti=settings_cti, + dataset_full=dataset_full, + ) + + self.preloads = Preloads() + + parallel_fast_index_list = None + parallel_fast_column_lists = None + + serial_fast_index_list = None + serial_fast_row_lists = None + + if self.clocker.parallel_fast_mode and not self.clocker.serial_fast_mode: + ( + parallel_fast_index_list, + parallel_fast_column_lists, + ) = clocker.fast_indexes_from(data=dataset.pre_cti_data, for_parallel=True) + + elif not self.clocker.parallel_fast_mode and self.clocker.serial_fast_mode: + serial_fast_index_list, serial_fast_row_lists = clocker.fast_indexes_from( + data=dataset.pre_cti_data, for_parallel=False + ) + + elif self.clocker.parallel_fast_mode and self.clocker.serial_fast_mode: + raise exc.ClockerException( + "Both parallel fast model and serial fast mode cannot be turned on.\n" + "Only switch on parallel fast mode for parallel + serial clocking." + ) + + self.preloads = Preloads( + parallel_fast_index_list=parallel_fast_index_list, + parallel_fast_column_lists=parallel_fast_column_lists, + serial_fast_index_list=serial_fast_index_list, + serial_fast_row_lists=serial_fast_row_lists, + ) + + def region_list_from(self, model: af.Collection) -> List: + """ + Inspects the CTI model and determines which regions are fitted for and therefore should be visualized. + + For example, if the model only includes parallel CTI, the serial regions are not fitted for and thus are not + visualized. + + Parameters + ---------- + model + The CTI model, composed via PyAutoFit, which represents the parallel and serial CTI model compoenents + fitted for by the non-linear search. + + Returns + ------- + A list of the regions fitted for by the model and therefore visualized. + + """ + if model.cti.parallel_ccd is not None and model.cti.serial_ccd is None: + return ["parallel_fpr", "parallel_eper"] + elif model.cti.serial_ccd is not None and model.cti.parallel_ccd is None: + return ["serial_fpr", "serial_eper"] + elif model.cti.pixel_bounce_list is not None: + return ["serial_fpr", "serial_eper"] + return ["parallel_fpr", "parallel_eper", "serial_fpr", "serial_eper"] + + def modify_before_fit(self, paths: af.DirectoryPaths, model: af.Collection): + """ + This function is called immediately before the non-linear search begins and performs final tasks and checks + before it begins. + + This function: + + 1) Visualizes the charge injection imaging dataset, which does not change during the analysis and thus can be + done once. + + 2) Checks if the noise-map is fixed (it is not if hyper functionality is on), and if it is fixed it + sets the noise-normalization to the preloads for computational speed. + + Parameters + ---------- + paths + The paths object which manages all paths, e.g. where the non-linear search outputs are stored, + visualization and the pickled objects used by the aggregator output by this function. + model + The model object, which includes model components representing the galaxies that are fitted to + the imaging data. + """ + + if paths.is_complete: + return self + + if not model.has(HyperCINoiseCollection): + noise_normalization = aa.util.fit.noise_normalization_with_mask_from( + noise_map=self.dataset.noise_map, mask=self.dataset.mask + ) + + self.preloads.noise_normalization = noise_normalization + + logger.info( + "PRELOADS - Noise Normalization preloaded for model-fit (noise-map is fixed)." + ) + + return self + + def log_likelihood_function(self, instance: af.ModelInstance) -> float: + """ + Determine the fitness of a particular model + + Parameters + ---------- + instance + + Returns + ------- + fit: Fit + How fit the model is and the model + """ + + self.settings_cti.check_total_density_within_range( + parallel_traps=instance.cti.parallel_trap_list, + serial_traps=instance.cti.serial_trap_list, + ) + + fit = self.fit_via_instance_and_dataset_from( + instance=instance, dataset=self.dataset, hyper_noise_scale=True + ) + + return fit.figure_of_merit + + def fit_via_instance_and_dataset_from( + self, + instance: af.ModelInstance, + dataset: ImagingCI, + hyper_noise_scale: bool = True, + ) -> FitImagingCI: + hyper_noise_scalar_dict = None + + if hyper_noise_scale and hasattr(instance, "hyper_noise"): + hyper_noise_scalar_dict = instance.hyper_noise.as_dict + + post_cti_data = self.clocker.add_cti( + data=dataset.pre_cti_data, + cti=instance.cti, + preloads=self.preloads, + ) + + return FitImagingCI( + dataset=dataset, + post_cti_data=post_cti_data, + hyper_noise_scalar_dict=hyper_noise_scalar_dict, + preloads=self.preloads, + ) + + def fit_via_instance_from( + self, instance: af.ModelInstance, hyper_noise_scale: bool = True + ) -> FitImagingCI: + return self.fit_via_instance_and_dataset_from( + instance=instance, + dataset=self.dataset, + hyper_noise_scale=hyper_noise_scale, + ) + + def save_attributes(self, paths: af.DirectoryPaths): + """ + Before the model-fit via the non-linear search begins, this routine saves attributes of the `Analysis` object + to the `files` folder such that they can be loaded after the analysis using PyAutoFit's database and + aggregator tools. + + For this analysis the following are output: + + - The charge injection dataset (data / noise-map / pre cti data / cosmic ray map / layout / settings etc.). + - The mask applied to the dataset. + - The clocker used for modeling / clocking CTI. + - The settings used for modeling / clocking CTI. + - The full 1D dataset (e.g. unmasked, used for visualizariton). + + It is common for these attributes to be loaded by many of the template aggregator functions given in the + `aggregator` modules. For example, when using the database tools to reperform a fit, this will by default + load the dataset, settings and other attributes necessary to perform a fit using the attributes output by + this function. + + Parameters + ---------- + paths + The paths object which manages all paths, e.g. where the non-linear search outputs are stored, + visualization,and the pickled objects used by the aggregator output by this function. + """ + + paths.save_json( + name="clocker", + object_dict=to_dict(self.clocker), + ) + + paths.save_json( + name="settings_cti", + object_dict=to_dict(self.settings_cti), + ) + + if conf.instance["visualize"]["plots"]["combined_only"]: + return + + def output_dataset(dataset, prefix): + values_list = [ + np.asarray(dataset.mask).astype("float"), + np.asarray(dataset.data.native), + np.asarray(dataset.noise_map.native), + np.asarray(dataset.pre_cti_data.native), + ] + ext_name_list = ["mask", "data", "noise_map", "pre_cti_data"] + + if dataset.cosmic_ray_map is not None: + values_list.append(np.asarray(dataset.cosmic_ray_map.native)) + ext_name_list.append("cosmic_ray_map") + + paths.save_fits( + name="dataset", + fits=hdu_list_for_output_from( + values_list=values_list, + ext_name_list=ext_name_list, + header_dict=dataset.mask.header_dict, + ), + prefix=prefix, + ) + paths.save_json( + name="layout", + object_dict=to_dict(dataset.layout), + prefix=prefix, + ) + + if self.dataset.settings_dict is not None: + paths.save_json( + name="settings_dict", + object_dict=self.dataset.settings_dict, + prefix="dataset", + ) + + output_dataset(dataset=self.dataset, prefix="dataset") + + if self.dataset_full is not None: + output_dataset(dataset=self.dataset_full, prefix="dataset_full") diff --git a/autocti/charge_injection/model/visualizer.py b/autocti/charge_injection/model/visualizer.py index 39be7fc6..0cee02e1 100644 --- a/autocti/charge_injection/model/visualizer.py +++ b/autocti/charge_injection/model/visualizer.py @@ -1,212 +1,270 @@ -from autoconf import conf - -import autofit as af - -from autocti.charge_injection.model.plotter_interface import PlotterInterfaceImagingCI - - -class VisualizerImagingCI(af.Visualizer): - @staticmethod - def visualize_before_fit( - analysis, - paths: af.AbstractPaths, - model: af.AbstractPriorModel, - ): - """ - PyAutoFit calls this function immediately before the non-linear search begins. - - It visualizes objects which do not change throughout the model fit like the dataset. - - Parameters - ---------- - paths - The paths object which manages all paths, e.g. where the non-linear search outputs are stored, - visualization and the pickled objects used by the aggregator output by this function. - model - The model object, which includes model components representing the galaxies that are fitted to - the imaging data. - """ - - if conf.instance["visualize"]["plots"]["combined_only"]: - return - - visualizer = PlotterInterfaceImagingCI(image_path=paths.image_path) - - region_list = analysis.region_list_from(model=model) - - if conf.instance["visualize"]["plots"]["dataset"]["fpr_non_uniformity"]: - region_list += ["fpr_non_uniformity"] - - visualizer.dataset(dataset=analysis.dataset) - visualizer.dataset_regions(dataset=analysis.dataset, region_list=region_list) - - if analysis.dataset_full is not None: - visualizer.dataset(dataset=analysis.dataset_full, folder_suffix="_full") - visualizer.dataset_regions( - dataset=analysis.dataset_full, - region_list=region_list, - folder_suffix="_full", - ) - - @staticmethod - def visualize_before_fit_combined( - analyses, - paths: af.AbstractPaths, - model: af.AbstractPriorModel, - ): - if analyses is None: - return - - visualizer = PlotterInterfaceImagingCI(image_path=paths.image_path) - - region_list = analyses[0].region_list_from(model=model) - - if conf.instance["visualize"]["plots"]["dataset"]["fpr_non_uniformity"]: - region_list += ["fpr_non_uniformity"] - - dataset_list = [analysis.dataset for analysis in analyses] - fpr_value_list = [dataset.fpr_value for dataset in dataset_list] - - dataset_list = analyses[0].in_ascending_fpr_order_from( - quantity_list=dataset_list, - fpr_value_list=fpr_value_list, - ) - - visualizer.dataset_combined( - dataset_list=dataset_list, - ) - - visualizer.dataset_regions_combined( - dataset_list=dataset_list, - region_list=region_list, - ) - - if analyses[0].dataset_full is not None: - dataset_full_list = [analysis.dataset_full for analysis in analyses] - - dataset_full_list = analyses[0].in_ascending_fpr_order_from( - quantity_list=dataset_full_list, - fpr_value_list=fpr_value_list, - ) - - visualizer.dataset_combined( - dataset_list=dataset_full_list, - folder_suffix="_full", - filename_suffix="_full", - ) - visualizer.dataset_regions_combined( - dataset_list=dataset_full_list, - region_list=region_list, - folder_suffix="_full", - filename_suffix="_full", - ) - - @staticmethod - def visualize( - analysis, - paths: af.DirectoryPaths, - instance: af.ModelInstance, - during_analysis: bool, - ): - """ - Output images of the maximum log likelihood model inferred by the model-fit. This function is called throughout - the non-linear search at regular intervals, and therefore provides on-the-fly visualization of how well the - model-fit is going. - - The images output by this function are customized using the file `config/visualize/plots.yaml`. - - Parameters - ---------- - paths - The paths object which manages all paths, e.g. where the non-linear search outputs are stored, - visualization, and the pickled objects used by the aggregator output by this function. - instance - An instance of the model that is being fitted to the data by this analysis (whose parameters have been set - via a non-linear search). - during_analysis - If True the visualization is being performed midway through the non-linear search before it is finished, - which may change which images are output. - """ - - if conf.instance["visualize"]["plots"]["combined_only"]: - return - - fit = analysis.fit_via_instance_from(instance=instance) - region_list = analysis.region_list_from(model=instance) - - visualizer = PlotterInterfaceImagingCI(image_path=paths.image_path) - visualizer.fit(fit=fit, during_analysis=during_analysis) - visualizer.fit_1d_regions( - fit=fit, during_analysis=during_analysis, region_list=region_list - ) - - if analysis.dataset_full is not None: - fit_full = analysis.fit_via_instance_and_dataset_from( - instance=instance, dataset=analysis.dataset_full - ) - - visualizer.fit( - fit=fit_full, during_analysis=during_analysis, folder_suffix="_full" - ) - visualizer.fit_1d_regions( - fit=fit_full, - during_analysis=during_analysis, - region_list=region_list, - folder_suffix="_full", - ) - - @staticmethod - def visualize_combined( - analyses, - paths: af.DirectoryPaths, - instance: af.ModelInstance, - during_analysis: bool, - ): - if analyses is None: - return - - fit_list = [ - analysis.fit_via_instance_from(instance=instance) for analysis in analyses - ] - - fpr_value_list = [fit.dataset.fpr_value for fit in fit_list] - - fit_list = analyses[0].in_ascending_fpr_order_from( - quantity_list=fit_list, - fpr_value_list=fpr_value_list, - ) - - region_list = analyses[0].region_list_from(model=instance) - - visualizer = PlotterInterfaceImagingCI(image_path=paths.image_path) - visualizer.fit_combined(fit_list=fit_list, during_analysis=during_analysis) - visualizer.fit_1d_regions_combined( - fit_list=fit_list, - region_list=region_list, - during_analysis=during_analysis, - ) - - if analyses[0].dataset_full is not None: - fit_full_list = [ - analysis.fit_via_instance_and_dataset_from( - instance=instance, dataset=analysis.dataset_full - ) - for analysis in analyses - ] - - fit_full_list = analyses[0].in_ascending_fpr_order_from( - quantity_list=fit_full_list, - fpr_value_list=fpr_value_list, - ) - - visualizer.fit_combined( - fit_list=fit_full_list, - during_analysis=during_analysis, - folder_suffix="_full", - ) - visualizer.fit_1d_regions_combined( - fit_list=fit_full_list, - region_list=region_list, - during_analysis=during_analysis, - folder_suffix="_full", - ) +from autoconf import conf + +import logging + +import autofit as af + +logger = logging.getLogger(__name__) + + +class VisualizerImagingCI(af.Visualizer): + @staticmethod + def visualize_before_fit( + analysis, + paths: af.AbstractPaths, + model: af.AbstractPriorModel, + ): + """ + PyAutoFit calls this function immediately before the non-linear search begins. + + It visualizes objects which do not change throughout the model fit like the dataset. + + Parameters + ---------- + paths + The paths object which manages all paths, e.g. where the non-linear search outputs are stored, + visualization and the pickled objects used by the aggregator output by this function. + model + The model object, which includes model components representing the galaxies that are fitted to + the imaging data. + """ + # Imported lazily: the PlotterInterface stack still targets the removed + # autoarray Plotter API and is rewritten on the new matplotlib function + # API in Phase 1 of the CTI resurrection epic (PyAutoCTI#82). + try: + from autocti.charge_injection.model.plotter_interface import ( + PlotterInterfaceImagingCI, + ) + except ImportError: + logger.warning( + "PyAutoCTI visualization is disabled until the Phase 1 " + "Plotter->matplotlib migration (PyAutoCTI#82)." + ) + return + + + if conf.instance["visualize"]["plots"]["combined_only"]: + return + + visualizer = PlotterInterfaceImagingCI(image_path=paths.image_path) + + region_list = analysis.region_list_from(model=model) + + if conf.instance["visualize"]["plots"]["dataset"]["fpr_non_uniformity"]: + region_list += ["fpr_non_uniformity"] + + visualizer.dataset(dataset=analysis.dataset) + visualizer.dataset_regions(dataset=analysis.dataset, region_list=region_list) + + if analysis.dataset_full is not None: + visualizer.dataset(dataset=analysis.dataset_full, folder_suffix="_full") + visualizer.dataset_regions( + dataset=analysis.dataset_full, + region_list=region_list, + folder_suffix="_full", + ) + + @staticmethod + def visualize_before_fit_combined( + analyses, + paths: af.AbstractPaths, + model: af.AbstractPriorModel, + ): + # Imported lazily: the PlotterInterface stack still targets the removed + # autoarray Plotter API and is rewritten on the new matplotlib function + # API in Phase 1 of the CTI resurrection epic (PyAutoCTI#82). + try: + from autocti.charge_injection.model.plotter_interface import ( + PlotterInterfaceImagingCI, + ) + except ImportError: + logger.warning( + "PyAutoCTI visualization is disabled until the Phase 1 " + "Plotter->matplotlib migration (PyAutoCTI#82)." + ) + return + + if analyses is None: + return + + visualizer = PlotterInterfaceImagingCI(image_path=paths.image_path) + + region_list = analyses[0].region_list_from(model=model) + + if conf.instance["visualize"]["plots"]["dataset"]["fpr_non_uniformity"]: + region_list += ["fpr_non_uniformity"] + + dataset_list = [analysis.dataset for analysis in analyses] + fpr_value_list = [dataset.fpr_value for dataset in dataset_list] + + dataset_list = analyses[0].in_ascending_fpr_order_from( + quantity_list=dataset_list, + fpr_value_list=fpr_value_list, + ) + + visualizer.dataset_combined( + dataset_list=dataset_list, + ) + + visualizer.dataset_regions_combined( + dataset_list=dataset_list, + region_list=region_list, + ) + + if analyses[0].dataset_full is not None: + dataset_full_list = [analysis.dataset_full for analysis in analyses] + + dataset_full_list = analyses[0].in_ascending_fpr_order_from( + quantity_list=dataset_full_list, + fpr_value_list=fpr_value_list, + ) + + visualizer.dataset_combined( + dataset_list=dataset_full_list, + folder_suffix="_full", + filename_suffix="_full", + ) + visualizer.dataset_regions_combined( + dataset_list=dataset_full_list, + region_list=region_list, + folder_suffix="_full", + filename_suffix="_full", + ) + + @staticmethod + def visualize( + analysis, + paths: af.DirectoryPaths, + instance: af.ModelInstance, + during_analysis: bool, + ): + """ + Output images of the maximum log likelihood model inferred by the model-fit. This function is called throughout + the non-linear search at regular intervals, and therefore provides on-the-fly visualization of how well the + model-fit is going. + + The images output by this function are customized using the file `config/visualize/plots.yaml`. + + Parameters + ---------- + paths + The paths object which manages all paths, e.g. where the non-linear search outputs are stored, + visualization, and the pickled objects used by the aggregator output by this function. + instance + An instance of the model that is being fitted to the data by this analysis (whose parameters have been set + via a non-linear search). + during_analysis + If True the visualization is being performed midway through the non-linear search before it is finished, + which may change which images are output. + """ + # Imported lazily: the PlotterInterface stack still targets the removed + # autoarray Plotter API and is rewritten on the new matplotlib function + # API in Phase 1 of the CTI resurrection epic (PyAutoCTI#82). + try: + from autocti.charge_injection.model.plotter_interface import ( + PlotterInterfaceImagingCI, + ) + except ImportError: + logger.warning( + "PyAutoCTI visualization is disabled until the Phase 1 " + "Plotter->matplotlib migration (PyAutoCTI#82)." + ) + return + + + if conf.instance["visualize"]["plots"]["combined_only"]: + return + + fit = analysis.fit_via_instance_from(instance=instance) + region_list = analysis.region_list_from(model=instance) + + visualizer = PlotterInterfaceImagingCI(image_path=paths.image_path) + visualizer.fit(fit=fit, during_analysis=during_analysis) + visualizer.fit_1d_regions( + fit=fit, during_analysis=during_analysis, region_list=region_list + ) + + if analysis.dataset_full is not None: + fit_full = analysis.fit_via_instance_and_dataset_from( + instance=instance, dataset=analysis.dataset_full + ) + + visualizer.fit( + fit=fit_full, during_analysis=during_analysis, folder_suffix="_full" + ) + visualizer.fit_1d_regions( + fit=fit_full, + during_analysis=during_analysis, + region_list=region_list, + folder_suffix="_full", + ) + + @staticmethod + def visualize_combined( + analyses, + paths: af.DirectoryPaths, + instance: af.ModelInstance, + during_analysis: bool, + ): + # Imported lazily: the PlotterInterface stack still targets the removed + # autoarray Plotter API and is rewritten on the new matplotlib function + # API in Phase 1 of the CTI resurrection epic (PyAutoCTI#82). + try: + from autocti.charge_injection.model.plotter_interface import ( + PlotterInterfaceImagingCI, + ) + except ImportError: + logger.warning( + "PyAutoCTI visualization is disabled until the Phase 1 " + "Plotter->matplotlib migration (PyAutoCTI#82)." + ) + return + + if analyses is None: + return + + fit_list = [ + analysis.fit_via_instance_from(instance=instance) for analysis in analyses + ] + + fpr_value_list = [fit.dataset.fpr_value for fit in fit_list] + + fit_list = analyses[0].in_ascending_fpr_order_from( + quantity_list=fit_list, + fpr_value_list=fpr_value_list, + ) + + region_list = analyses[0].region_list_from(model=instance) + + visualizer = PlotterInterfaceImagingCI(image_path=paths.image_path) + visualizer.fit_combined(fit_list=fit_list, during_analysis=during_analysis) + visualizer.fit_1d_regions_combined( + fit_list=fit_list, + region_list=region_list, + during_analysis=during_analysis, + ) + + if analyses[0].dataset_full is not None: + fit_full_list = [ + analysis.fit_via_instance_and_dataset_from( + instance=instance, dataset=analysis.dataset_full + ) + for analysis in analyses + ] + + fit_full_list = analyses[0].in_ascending_fpr_order_from( + quantity_list=fit_full_list, + fpr_value_list=fpr_value_list, + ) + + visualizer.fit_combined( + fit_list=fit_full_list, + during_analysis=during_analysis, + folder_suffix="_full", + ) + visualizer.fit_1d_regions_combined( + fit_list=fit_full_list, + region_list=region_list, + during_analysis=during_analysis, + folder_suffix="_full", + ) diff --git a/autocti/charge_injection/ou_sim_ci.py b/autocti/charge_injection/ou_sim_ci.py index 497ba2f1..25299989 100644 --- a/autocti/charge_injection/ou_sim_ci.py +++ b/autocti/charge_injection/ou_sim_ci.py @@ -1,267 +1,285 @@ -import math -import numpy as np -from typing import List, Union - -from arcticpy import CCDPhase -from arcticpy import TrapInstantCapture - -from autocti.instruments.euclid import euclid_util - -from autoarray.layout import layout_util -from autoarray.structures.arrays.uniform_2d import Array2D - -from autocti.clocker.two_d import Clocker2D -from autocti.model.model_util import CTI2D - -from autocti.charge_injection.layout import Layout2DCI -from autocti.charge_injection.imaging.simulator import SimulatorImagingCI - -from autocti.charge_injection import ci_util - -""" -Note on the rotations of arrays: - -The function 'non_uniform_array_for_ou_sim' returns an array that is rotated according to the iquad parameter -(which ELVIS uses to define which quadrant the data corresponds to). This uniquely defines the orientation of the -ndarray necessary to add parallel and serial CTI in the correct direction. These are defined accoridng to: - - http://euclid.esac.esa.int/dm/dpdd/latest/le1dpd/dpcards/le1_visrawframe.html - -This function will return a array, which OU-Sim will then add the following effects too: - - - Add cosmic rays. - - Bias. - - Non-linearity. - - Crosstalk? - -The addition of CTI can then be performed using either the 'add_cti_to_array_for_ou_sim' function, or the standard -function OU-Sim use to add CTI (I guess they ultimately both flow through arctic in an identical way, albeit our -code uses SWIG and omits the need to define arCTIc parameter files. - -Due to the rotations performed above, this means all images produced will be ndarrays oriented in the same way. I -am not clear on how standard ELViS data products are oriented, but it may be that we require rotations before writing -them to fits to oriented them in the way they are observed (VIS_CTI has tools for this, as I'm sure ELViS does too). -""" - - -def quadrant_id_from(iquad: int) -> str: - """ - The ELVIS simulator uses the `iquad` parameter to determine how images are rotated before clocking via arctic. - - This script converts this parameter to the `quadrant_id` used by PyAutoCTI, which in turn gives the appropriate - `roe_corner` for rotation. - - The mapping of `iquad` to `quadrant_id` does not depend on the CCD id, because ELVIS has already performed - extractions / rotations on the quadrant data beforehand. - - Parameters - ---------- - iquad - The ELVIS parameter defining the quadrant of the data and therefore the rotateion before arctic clocking. - - Returns - ------- - str - The quadrant ID string, which is either E, G, H or G - """ - - if iquad == 0: - return "E" - elif iquad == 1: - return "F" - elif iquad == 2: - return "H" - elif iquad == 3: - return "G" - - -def injection_total_from( - injection_start: int, - injection_end: int, - injection_on: int, - injection_off: int, -): - """ - The total number of charge injection regions for these electronics settings. - """ - - injection_range = injection_end - injection_start - - for injection_total in range(100): - total_pixels = math.floor( - (injection_total + 1) * (injection_on) + injection_total * injection_off - ) - - if total_pixels > injection_range: - return injection_total - - -def charge_injection_array_from( - ccd_id: str, - quadrant_id: str, - injection_norm: float, - injection_start: int = 16, - injection_end: int = 2066, - injection_on: int = 200, - injection_off: int = 200, - parallel_size: int = 2086, - serial_size: int = 2128, - serial_prescan_size: int = 51, - serial_overscan_size: int = 29, - pixel_scales=0.1, - use_non_uniform_pattern: bool = True, - ci_seed: int = -1, -) -> Union[np.ndarray, Array2D]: - """ - Returns a charge injection line image suitable for OU-SIM to run through the ElVIS simulator. - - By default, this array has dimensions (2086, 2128), representing a Euclid quadrant with a serial prescan of - size 51 pixels, a serial overscan with 29 pixels and parallel overscan with 20 pixels. - - The charge injection line pattern is simulated using the VIS_CTI Processing element, and includes - effects such as a non-uniform charge injection pattern. The charge injection is simulated in 3 distinct - regions on the quadrant. - - This function assumes the same orientation for the charge injection line image, irrespective of the Euclid - CCDPhase ID and Quadrant ID. The orientation that it assumes has arctic clock the ndarray towards [0, 0]. However, - based on an input CCDPhase ID and Quadrant ID, the array is rotated to match Euclid clocking. - - Parameters - ---------- - ccd_id - The CCDPhase ID of Euclid (runs 1 through 6) - quadrant_id - The quadrant id (E, F, G, H) - injection_norm - The normalization of the charge injection region. - parallel_size - The size of the image in the parallel clocking direction (e.g. number of rows). - serial_size - The size of the image in the serial clocking direction (e.g. number of columns). - serial_overscan_size - The size of the serial overscan - pixel_scales - The arc-second to pixel scale conversion factor. - - Returns - ------- - ndarray - The charge injection line image oriented to match a given Euclid quadrant. - """ - shape_native = (parallel_size, serial_size) - - injection_total = injection_total_from( - injection_start=injection_start, - injection_end=injection_end, - injection_on=injection_on, - injection_off=injection_off, - ) - - """ - Specify the charge injection regions on the CCDPhase, which in this case is 5 equally spaced rectangular blocks. - - At the end of this function the ndarray containing the charge injection data is rotated based on the quadrant_id. - We therefore do not need rotated `regions_ci`'s from the function below, and input `roe_corner=(1,0)`, which - corresponds to quadrant E which is never rotated. - """ - - regions_ci = ci_util.region_list_ci_via_electronics_from( - injection_start=injection_start, - injection_on=injection_on, - injection_off=injection_off, - injection_total=injection_total, - parallel_size=parallel_size, - serial_size=serial_size, - serial_prescan_size=serial_prescan_size, - serial_overscan_size=serial_overscan_size, - roe_corner=(1, 0), - ) - - """ - Use the charge injection normalization_list and regions to create `Layout2DCI` of every image we'll simulate. - """ - layout = Layout2DCI(shape_2d=shape_native, region_list=regions_ci) - - """ - The simulator object creates simulations of charge injeciton imaging. - """ - """ - Create every pre-cti charge injection image using each `Layout2DCI` - """ - if use_non_uniform_pattern: - simulator = SimulatorImagingCI( - pixel_scales=pixel_scales, - norm=injection_norm, - row_slope=0.0, - column_sigma=100.0, - max_norm=200000, - ci_seed=ci_seed, - ) - - pre_cti_data = simulator.pre_cti_data_non_uniform_from(layout=layout) - else: - simulator = SimulatorImagingCI(pixel_scales=pixel_scales, norm=injection_norm) - - pre_cti_data = simulator.pre_cti_data_uniform_from(layout=layout) - - """ - The OU-SIM parameter iquad defines the quadrant_id of the data (e.g. "E", "F", "G" or "H"). - """ - # quadrant_id = quadrant_id_from(iquad=iquad) - - roe_corner = euclid_util.roe_corner_from(ccd_id=ccd_id, quadrant_id=quadrant_id) - - """ - The array is rotated back to its original reference frame via the roe_corner, so other OU-Sim processing - works correctly. - """ - return layout_util.rotate_array_via_roe_corner_from( - array=pre_cti_data.native, roe_corner=roe_corner - ) - - -def add_cti_to_pre_cti_data( - pre_cti_data: Union[np.ndarray, Array2D], - ccd_id: str, - quadrant_id: str, - clocker: Clocker2D, - parallel_trap_list: List[TrapInstantCapture], - parallel_ccd: CCDPhase, - serial_trap_list: List[TrapInstantCapture], - serial_ccd: CCDPhase, -) -> Union[np.ndarray, Array2D]: - # quadrant_id = quadrant_id_from(iquad=iquad) - - roe_corner = euclid_util.roe_corner_from(ccd_id=ccd_id, quadrant_id=quadrant_id) - - pre_cti_data = layout_util.rotate_array_via_roe_corner_from( - array=pre_cti_data, roe_corner=roe_corner - ) - - """ - The `Clocker` models the CCDPhase read-out, including CTI. - - For parallel clocking, we use 'charge injection mode' which transfers the charge of every pixel over the full CCDPhase. - """ - - """ - The CTI model used by arCTIc to add CTI to the input image in the parallel direction, which contains: - - - 2 `TrapInstantCapture` species in the parallel direction. - - A simple CCDPhase volume beta parametrization. - - 3 `TrapInstantCapture` species in the serial direction. - - A simple CCDPhase volume beta parametrization. - """ - - cti = CTI2D( - parallel_trap_list=parallel_trap_list, - parallel_ccd=parallel_ccd, - serial_trap_list=serial_trap_list, - serial_ccd=serial_ccd, - ) - - post_cti_data = clocker.add_cti(data=pre_cti_data, cti=cti) - - return layout_util.rotate_array_via_roe_corner_from( - array=post_cti_data, roe_corner=roe_corner - ) +import math +import numpy as np +from typing import List, Union + +from arcticpy import CCDPhase +from arcticpy import TrapInstantCapture + +from autocti.instruments.euclid import euclid_util + +from autoarray.layout import layout_util +from autoarray.structures.arrays.uniform_2d import Array2D + +from autocti.clocker.two_d import Clocker2D +from autocti.model.model_util import CTI2D + +from autocti.charge_injection.layout import Layout2DCI +from autocti.charge_injection.imaging.simulator import SimulatorImagingCI + +from autocti.charge_injection import ci_util + +""" +Note on the rotations of arrays: + +The function 'non_uniform_array_for_ou_sim' returns an array that is rotated according to the iquad parameter +(which ELVIS uses to define which quadrant the data corresponds to). This uniquely defines the orientation of the +ndarray necessary to add parallel and serial CTI in the correct direction. These are defined accoridng to: + + http://euclid.esac.esa.int/dm/dpdd/latest/le1dpd/dpcards/le1_visrawframe.html + +This function will return a array, which OU-Sim will then add the following effects too: + + - Add cosmic rays. + - Bias. + - Non-linearity. + - Crosstalk? + +The addition of CTI can then be performed using either the 'add_cti_to_array_for_ou_sim' function, or the standard +function OU-Sim use to add CTI (I guess they ultimately both flow through arctic in an identical way, albeit our +code uses SWIG and omits the need to define arCTIc parameter files. + +Due to the rotations performed above, this means all images produced will be ndarrays oriented in the same way. I +am not clear on how standard ELViS data products are oriented, but it may be that we require rotations before writing +them to fits to oriented them in the way they are observed (VIS_CTI has tools for this, as I'm sure ELViS does too). +""" + + +def quadrant_id_from(iquad: int) -> str: + """ + The ELVIS simulator uses the `iquad` parameter to determine how images are rotated before clocking via arctic. + + This script converts this parameter to the `quadrant_id` used by PyAutoCTI, which in turn gives the appropriate + `roe_corner` for rotation. + + The mapping of `iquad` to `quadrant_id` does not depend on the CCD id, because ELVIS has already performed + extractions / rotations on the quadrant data beforehand. + + Parameters + ---------- + iquad + The ELVIS parameter defining the quadrant of the data and therefore the rotateion before arctic clocking. + + Returns + ------- + str + The quadrant ID string, which is either E, G, H or G + """ + + if iquad == 0: + return "E" + elif iquad == 1: + return "F" + elif iquad == 2: + return "H" + elif iquad == 3: + return "G" + + +def injection_total_from( + injection_start: int, + injection_end: int, + injection_on: int, + injection_off: int, +): + """ + The total number of charge injection regions for these electronics settings. + """ + + injection_range = injection_end - injection_start + + for injection_total in range(100): + total_pixels = math.floor( + (injection_total + 1) * (injection_on) + injection_total * injection_off + ) + + if total_pixels > injection_range: + return injection_total + + +def charge_injection_array_from( + ccd_id: str, + quadrant_id: str, + injection_norm: float, + injection_start: int = 16, + injection_end: int = 2066, + injection_on: int = 200, + injection_off: int = 200, + parallel_size: int = 2086, + serial_size: int = 2128, + serial_prescan_size: int = 51, + serial_overscan_size: int = 29, + pixel_scales=0.1, + use_non_uniform_pattern: bool = True, + ci_seed: int = -1, +) -> Union[np.ndarray, Array2D]: + """ + Returns a charge injection line image suitable for OU-SIM to run through the ElVIS simulator. + + By default, this array has dimensions (2086, 2128), representing a Euclid quadrant with a serial prescan of + size 51 pixels, a serial overscan with 29 pixels and parallel overscan with 20 pixels. + + The charge injection line pattern is simulated using the VIS_CTI Processing element, and includes + effects such as a non-uniform charge injection pattern. The charge injection is simulated in 3 distinct + regions on the quadrant. + + This function assumes the same orientation for the charge injection line image, irrespective of the Euclid + CCDPhase ID and Quadrant ID. The orientation that it assumes has arctic clock the ndarray towards [0, 0]. However, + based on an input CCDPhase ID and Quadrant ID, the array is rotated to match Euclid clocking. + + Parameters + ---------- + ccd_id + The CCDPhase ID of Euclid (runs 1 through 6) + quadrant_id + The quadrant id (E, F, G, H) + injection_norm + The normalization of the charge injection region. + parallel_size + The size of the image in the parallel clocking direction (e.g. number of rows). + serial_size + The size of the image in the serial clocking direction (e.g. number of columns). + serial_overscan_size + The size of the serial overscan + pixel_scales + The arc-second to pixel scale conversion factor. + + Returns + ------- + ndarray + The charge injection line image oriented to match a given Euclid quadrant. + """ + shape_native = (parallel_size, serial_size) + + injection_total = injection_total_from( + injection_start=injection_start, + injection_end=injection_end, + injection_on=injection_on, + injection_off=injection_off, + ) + + """ + Specify the charge injection regions on the CCDPhase, which in this case is 5 equally spaced rectangular blocks. + + At the end of this function the ndarray containing the charge injection data is rotated based on the quadrant_id. + We therefore do not need rotated `regions_ci`'s from the function below, and input `roe_corner=(1,0)`, which + corresponds to quadrant E which is never rotated. + """ + + regions_ci = ci_util.region_list_ci_via_electronics_from( + injection_start=injection_start, + injection_on=injection_on, + injection_off=injection_off, + injection_total=injection_total, + parallel_size=parallel_size, + serial_size=serial_size, + serial_prescan_size=serial_prescan_size, + serial_overscan_size=serial_overscan_size, + roe_corner=(1, 0), + ) + + """ + Use the charge injection normalization_list and regions to create `Layout2DCI` of every image we'll simulate. + """ + layout = Layout2DCI(shape_2d=shape_native, region_list=regions_ci) + + """ + The simulator object creates simulations of charge injeciton imaging. + """ + """ + Create every pre-cti charge injection image using each `Layout2DCI` + """ + if use_non_uniform_pattern: + simulator = SimulatorImagingCI( + pixel_scales=pixel_scales, + norm=injection_norm, + row_slope=0.0, + column_sigma=100.0, + max_norm=200000, + ci_seed=ci_seed, + ) + + pre_cti_data = simulator.pre_cti_data_non_uniform_from(layout=layout) + else: + simulator = SimulatorImagingCI(pixel_scales=pixel_scales, norm=injection_norm) + + pre_cti_data = simulator.pre_cti_data_uniform_from(layout=layout) + + """ + The OU-SIM parameter iquad defines the quadrant_id of the data (e.g. "E", "F", "G" or "H"). + """ + # quadrant_id = quadrant_id_from(iquad=iquad) + + roe_corner = euclid_util.roe_corner_from(ccd_id=ccd_id, quadrant_id=quadrant_id) + + """ + The array is rotated back to its original reference frame via the roe_corner, so other OU-Sim processing + works correctly. + """ + # rotate_array_via_roe_corner_from returns a plain ndarray, so the result is + # wrapped back into an Array2D for downstream use. + return Array2D.no_mask( + values=layout_util.rotate_array_via_roe_corner_from( + array=pre_cti_data.native, roe_corner=roe_corner + ), + pixel_scales=pixel_scales, + ).native + + +def add_cti_to_pre_cti_data( + pre_cti_data: Union[np.ndarray, Array2D], + ccd_id: str, + quadrant_id: str, + clocker: Clocker2D, + parallel_trap_list: List[TrapInstantCapture], + parallel_ccd: CCDPhase, + serial_trap_list: List[TrapInstantCapture], + serial_ccd: CCDPhase, +) -> Union[np.ndarray, Array2D]: + # quadrant_id = quadrant_id_from(iquad=iquad) + + roe_corner = euclid_util.roe_corner_from(ccd_id=ccd_id, quadrant_id=quadrant_id) + + pixel_scales = getattr(pre_cti_data, "pixel_scales", 0.1) + + # The rotation utility operates on and returns a plain 2D ndarray, but the + # clocker requires an `Array2D`, so the result is wrapped back up. + pre_cti_data = Array2D.no_mask( + values=layout_util.rotate_array_via_roe_corner_from( + array=np.asarray( + pre_cti_data.native if hasattr(pre_cti_data, "native") else pre_cti_data + ), + roe_corner=roe_corner, + ), + pixel_scales=pixel_scales, + ).native + + """ + The `Clocker` models the CCDPhase read-out, including CTI. + + For parallel clocking, we use 'charge injection mode' which transfers the charge of every pixel over the full CCDPhase. + """ + + """ + The CTI model used by arCTIc to add CTI to the input image in the parallel direction, which contains: + + - 2 `TrapInstantCapture` species in the parallel direction. + - A simple CCDPhase volume beta parametrization. + - 3 `TrapInstantCapture` species in the serial direction. + - A simple CCDPhase volume beta parametrization. + """ + + cti = CTI2D( + parallel_trap_list=parallel_trap_list, + parallel_ccd=parallel_ccd, + serial_trap_list=serial_trap_list, + serial_ccd=serial_ccd, + ) + + post_cti_data = clocker.add_cti(data=pre_cti_data, cti=cti) + + return Array2D.no_mask( + values=layout_util.rotate_array_via_roe_corner_from( + array=np.asarray(post_cti_data), roe_corner=roe_corner + ), + pixel_scales=pixel_scales, + ).native diff --git a/autocti/config/priors/ccd.yaml b/autocti/config/priors/ccd.yaml index 594dedb0..93f65e3d 100644 --- a/autocti/config/priors/ccd.yaml +++ b/autocti/config/priors/ccd.yaml @@ -6,7 +6,7 @@ CCDPhase: width_modifier: type: Absolute value: 0.2 - gaussian_limits: + limits: lower: 0.0 upper: 1.0 well_fill_power: @@ -16,7 +16,7 @@ CCDPhase: width_modifier: type: Absolute value: 0.2 - gaussian_limits: + limits: lower: 0.0 upper: 1.0 well_notch_depth: @@ -26,7 +26,7 @@ CCDPhase: width_modifier: type: Absolute value: 0.2 - gaussian_limits: + limits: lower: 0.0 upper: 1.0 first_electron_fill: diff --git a/autocti/config/priors/hyper.yaml b/autocti/config/priors/hyper.yaml index 474c81d6..11972c9e 100644 --- a/autocti/config/priors/hyper.yaml +++ b/autocti/config/priors/hyper.yaml @@ -6,6 +6,6 @@ HyperCINoiseScalar: width_modifier: type: Relative value: 0.5 - gaussian_limits: + limits: lower: 0.0 upper: inf diff --git a/autocti/config/priors/traps.yaml b/autocti/config/priors/traps.yaml index 4b574b3a..63916fb4 100644 --- a/autocti/config/priors/traps.yaml +++ b/autocti/config/priors/traps.yaml @@ -6,7 +6,7 @@ TrapInstantCapture: width_modifier: type: Relative value: 0.5 - gaussian_limits: + limits: lower: 0.0 upper: inf release_timescale: @@ -16,7 +16,7 @@ TrapInstantCapture: width_modifier: type: Relative value: 0.5 - gaussian_limits: + limits: lower: 0.0 upper: inf fractional_volume_full_exposed: @@ -33,7 +33,7 @@ TrapInstantCaptureContinuum: width_modifier: type: Relative value: 0.5 - gaussian_limits: + limits: lower: 0.0 upper: inf release_timescale: @@ -43,7 +43,7 @@ TrapInstantCaptureContinuum: width_modifier: type: Relative value: 0.5 - gaussian_limits: + limits: lower: 0.0 upper: inf release_timescale_sigma: @@ -53,6 +53,6 @@ TrapInstantCaptureContinuum: width_modifier: type: Relative value: 0.5 - gaussian_limits: + limits: lower: 0.0 upper: inf diff --git a/autocti/dataset_1d/dataset_1d/dataset_1d.py b/autocti/dataset_1d/dataset_1d/dataset_1d.py index 4b687541..5729e3d6 100644 --- a/autocti/dataset_1d/dataset_1d/dataset_1d.py +++ b/autocti/dataset_1d/dataset_1d/dataset_1d.py @@ -1,263 +1,276 @@ -import numpy as np -from pathlib import Path -from typing import Optional, Dict, Union - -import autoarray as aa - -from autocti import exc -from autocti.extract.settings import SettingsExtract -from autocti.layout.one_d import Layout1D - - -class Dataset1D(aa.AbstractDataset): - def __init__( - self, - data: aa.Array1D, - noise_map: aa.Array1D, - pre_cti_data: aa.Array1D, - layout: Layout1D, - fpr_value: Optional[float] = None, - settings_dict: Optional[Dict] = None, - ): - super().__init__(data=data, noise_map=noise_map) - - self.data = data - self.noise_map = noise_map - self.pre_cti_data = pre_cti_data - self.layout = layout - - if fpr_value is None: - fpr_value = np.round( - np.median( - self.layout.extract.fpr.stacked_array_1d_from( - array=self.data, - settings=SettingsExtract( - pixels_from_end=min( - 5, self.layout.extract.fpr.total_pixels_min - ) - ), - ) - ), - 2, - ) - - self.fpr_value = fpr_value - - self.settings_dict = settings_dict - - def apply_mask(self, mask: aa.Mask1D) -> "Dataset1D": - data = aa.Array1D(values=self.data, mask=mask).native - noise_map = aa.Array1D(values=self.noise_map.astype("float"), mask=mask).native - - return Dataset1D( - data=data, - noise_map=noise_map, - pre_cti_data=self.pre_cti_data, - layout=self.layout, - fpr_value=self.fpr_value, - settings_dict=self.settings_dict, - ) - - @classmethod - def from_fits( - cls, - pixel_scales: aa.type.PixelScales, - layout: Layout1D, - data_path: Optional[Union[Path, str]] = None, - data_hdu: int = 0, - noise_map_path: Optional[Union[Path, str]] = None, - noise_map_hdu: int = 0, - noise_map_from_single_value: float = None, - pre_cti_data_path: Optional[Union[Path, str]] = None, - pre_cti_data_hdu: int = 0, - pre_cti_data: aa.Array1D = None, - settings_dict: Optional[Dict] = None, - ): - """ - Load 1D dataset from multiple .fits file. - - For each attribute of the 1D dataset (e.g. `data`, `noise_map`, `pre_cti_data`) the path to the .fits and - the `hdu` containing the data can be specified. - - The `noise_map` assumes the noise value in each `data` value are independent, where these values are the - RMS standard deviation error in each pixel. - - If the dataset has a mask associated with it (e.g. in a `mask.fits` file) the file must be loaded separately - via the `Mask1D` object and applied to the imaging after loading via fits using the `from_fits` method. - - Parameters - ---------- - pixel_scales - The (y,x) arcsecond-to-pixel units conversion factor of every pixel. If this is input as a `float`, - it is converted to a (float, float). - layout - The layout of the 1D dataset, containing information like where the FPR and EPER are located. - data_path - The path to the data .fits file containing the data (e.g. '/path/to/data.fits'). - data_hdu - The hdu the image data is contained in the .fits file specified by `data_path`. - noise_map_path - The path to the noise_map .fits file containing the noise_map (e.g. '/path/to/noise_map.fits'). - noise_map_hdu - The hdu the noise map is contained in the .fits file specified by `noise_map_path`. - noise_map_from_single_value - Creates a `noise_map` of constant values if this is input instead of loading via .fits. - pre_cti_data_path - The path to the pre CTI data .fits file containing the image data (e.g. '/path/to/pre_cti_data.fits'). - pre_cti_data_hdu - The hdu the pre cti data is contained in the .fits file specified by `pre_cti_data_path`. - pre_cti_data - Manually input the pre CTI data as an `Array1D` instead of loading it via a .fits file. - settings_dict - A dictionary of settings associated with the charge injeciton imaging (e.g. voltage settings) which is - used for visualization. - """ - data = aa.Array1D.from_fits( - file_path=data_path, hdu=data_hdu, pixel_scales=pixel_scales - ) - - if noise_map_path is not None: - noise_map = aa.util.array_1d.numpy_array_1d_via_fits_from( - file_path=noise_map_path, hdu=noise_map_hdu - ).astype("float") - else: - noise_map = np.ones(data.shape_native) * noise_map_from_single_value - - noise_map = aa.Array1D.no_mask(values=noise_map, pixel_scales=pixel_scales) - - if pre_cti_data_path is not None and pre_cti_data is None: - pre_cti_data = aa.Array1D.from_fits( - file_path=pre_cti_data_path, - hdu=pre_cti_data_hdu, - pixel_scales=pixel_scales, - ) - else: - raise exc.LayoutException( - "Cannot estimate pre_cti_data data from non-uniform charge injectiono pattern" - ) - - pre_cti_data = aa.Array1D.no_mask( - values=pre_cti_data.native, pixel_scales=pixel_scales - ) - - return Dataset1D( - data=data, - noise_map=noise_map, - pre_cti_data=pre_cti_data, - layout=layout, - settings_dict=settings_dict, - ) - - def output_to_fits( - self, - data_path: Union[Path, str], - noise_map_path: Optional[Union[Path, str]] = None, - pre_cti_data_path: Optional[Union[Path, str]] = None, - overwrite: bool = False, - ): - """ - Output the 1D dataset to multiple .fits file. - - For each attribute of the 1D dataset data (e.g. `data`, `noise_map`, `pre_cti_data`) the path to - the .fits can be specified, with `hdu=0` assumed automatically. - - If the `data` has been masked, the masked data is output to .fits files. A mask can be separately output to - a file `mask.fits` via the `Mask` objects `output_to_fits` method. - - Parameters - ---------- - data_path - The path to the data .fits file where the image data is output (e.g. '/path/to/data.fits'). - noise_map_path - The path to the noise_map .fits where the noise_map is output (e.g. '/path/to/noise_map.fits'). - pre_cti_data_path - The path to the pre CTI data .fits file where the pre CTI data is output (e.g. '/path/to/pre_cti_data.fits'). - overwrite - If `True`, the .fits files are overwritten if they already exist, if `False` they are not and an - exception is raised. - """ - self.data.output_to_fits(file_path=data_path, overwrite=overwrite) - self.noise_map.output_to_fits(file_path=noise_map_path, overwrite=overwrite) - self.pre_cti_data.output_to_fits( - file_path=pre_cti_data_path, overwrite=overwrite - ) - - @classmethod - def from_pixel_line_dict( - cls, - pixel_line_dict: dict, - size: int, - ) -> "Dataset1D": - """ - Parse a pixel line output from the warm-pixels script. - - Pixel lines are individual or averaged lines found by searching for - warm pixels or consistent warm pixels in CCD data. The warm pixel - and its are extracted and saved as a JSON which can then be loaded - and fit as part of autocti. - - Parameters - ---------- - pixel_line_dict - A dictionary describing a pixel line collection. - - e.g. - { - "location": [ - 2, - 4, - ], - "flux": 1234., - "data": [ - 5.0, - 3.0, - 2.0, - 1.0, - ], - "noise": [ - 1.0, - 1.0, - 1.0, - 1.0, - ] - } - - location - The location of the warm pixel in (row, column) where row is the - distance to the serial register - 1 - flux - The computed flux of the warm pixel prior to CTI - data - The extracted pixel line. A 1D array where the first entry is - the warm pixel (FPR) and the remaining entries are the trail - (EPER) - noise - The noise map for the pixel line. - size - The size of the CCD. That is, the number of pixels in the parallel - direction. - - Returns - ------- - A Dataset1D initialised to represent the pixel line. The pixel line - and noise are embedded in Array1Ds of the same size as the array in - the parallel direction - """ - serial_distance, _ = map(int, pixel_line_dict["location"]) - - def make_array(data): - array = np.zeros(size) - array[serial_distance : serial_distance + len(data)] = data - return aa.Array1D.no_mask(array, pixel_scales=0.1) - - return Dataset1D( - data=make_array(pixel_line_dict["data"]), - noise_map=make_array(pixel_line_dict["noise"]), - pre_cti_data=make_array(np.array([pixel_line_dict["flux"]])), - layout=Layout1D( - shape_1d=(size,), - region_list=[ - aa.Region1D(region=(serial_distance, serial_distance + 1)) - ], - ), - ) +import numpy as np +from pathlib import Path +from typing import Optional, Dict, Union + +import autoarray as aa +from autoconf import fitsable + +from autocti import exc +from autocti.extract.settings import SettingsExtract +from autocti.layout.one_d import Layout1D + + +class Dataset1D(aa.AbstractDataset): + def __init__( + self, + data: aa.Array1D, + noise_map: aa.Array1D, + pre_cti_data: aa.Array1D, + layout: Layout1D, + fpr_value: Optional[float] = None, + settings_dict: Optional[Dict] = None, + ): + super().__init__(data=data, noise_map=noise_map) + + self.data = data + self.noise_map = noise_map + self.pre_cti_data = pre_cti_data + self.layout = layout + + # CTI datasets perform no grid calculations; `FitDataset` accesses + # `dataset.grids.*` unconditionally, so an empty interface is attached. + self.grids = aa.GridsInterface() + + if fpr_value is None: + fpr_value = np.round( + np.median( + self.layout.extract.fpr.stacked_array_1d_from( + array=self.data, + settings=SettingsExtract( + pixels_from_end=min( + 5, self.layout.extract.fpr.total_pixels_min + ) + ), + ) + ), + 2, + ) + + self.fpr_value = fpr_value + + self.settings_dict = settings_dict + + def apply_mask(self, mask: aa.Mask1D) -> "Dataset1D": + data = aa.Array1D(values=self.data, mask=mask).native + noise_map = aa.Array1D(values=self.noise_map.astype("float"), mask=mask).native + + return Dataset1D( + data=data, + noise_map=noise_map, + pre_cti_data=self.pre_cti_data, + layout=self.layout, + fpr_value=self.fpr_value, + settings_dict=self.settings_dict, + ) + + @classmethod + def from_fits( + cls, + pixel_scales: aa.type.PixelScales, + layout: Layout1D, + data_path: Optional[Union[Path, str]] = None, + data_hdu: int = 0, + noise_map_path: Optional[Union[Path, str]] = None, + noise_map_hdu: int = 0, + noise_map_from_single_value: float = None, + pre_cti_data_path: Optional[Union[Path, str]] = None, + pre_cti_data_hdu: int = 0, + pre_cti_data: aa.Array1D = None, + settings_dict: Optional[Dict] = None, + ): + """ + Load 1D dataset from multiple .fits file. + + For each attribute of the 1D dataset (e.g. `data`, `noise_map`, `pre_cti_data`) the path to the .fits and + the `hdu` containing the data can be specified. + + The `noise_map` assumes the noise value in each `data` value are independent, where these values are the + RMS standard deviation error in each pixel. + + If the dataset has a mask associated with it (e.g. in a `mask.fits` file) the file must be loaded separately + via the `Mask1D` object and applied to the imaging after loading via fits using the `from_fits` method. + + Parameters + ---------- + pixel_scales + The (y,x) arcsecond-to-pixel units conversion factor of every pixel. If this is input as a `float`, + it is converted to a (float, float). + layout + The layout of the 1D dataset, containing information like where the FPR and EPER are located. + data_path + The path to the data .fits file containing the data (e.g. '/path/to/data.fits'). + data_hdu + The hdu the image data is contained in the .fits file specified by `data_path`. + noise_map_path + The path to the noise_map .fits file containing the noise_map (e.g. '/path/to/noise_map.fits'). + noise_map_hdu + The hdu the noise map is contained in the .fits file specified by `noise_map_path`. + noise_map_from_single_value + Creates a `noise_map` of constant values if this is input instead of loading via .fits. + pre_cti_data_path + The path to the pre CTI data .fits file containing the image data (e.g. '/path/to/pre_cti_data.fits'). + pre_cti_data_hdu + The hdu the pre cti data is contained in the .fits file specified by `pre_cti_data_path`. + pre_cti_data + Manually input the pre CTI data as an `Array1D` instead of loading it via a .fits file. + settings_dict + A dictionary of settings associated with the charge injeciton imaging (e.g. voltage settings) which is + used for visualization. + """ + data = aa.Array1D.from_fits( + file_path=data_path, hdu=data_hdu, pixel_scales=pixel_scales + ) + + if noise_map_path is not None: + noise_map = fitsable.ndarray_via_fits_from( + file_path=noise_map_path, hdu=noise_map_hdu + ).astype("float") + else: + noise_map = np.ones(data.shape_native) * noise_map_from_single_value + + noise_map = aa.Array1D.no_mask(values=noise_map, pixel_scales=pixel_scales) + + if pre_cti_data_path is not None and pre_cti_data is None: + pre_cti_data = aa.Array1D.from_fits( + file_path=pre_cti_data_path, + hdu=pre_cti_data_hdu, + pixel_scales=pixel_scales, + ) + else: + raise exc.LayoutException( + "Cannot estimate pre_cti_data data from non-uniform charge injectiono pattern" + ) + + pre_cti_data = aa.Array1D.no_mask( + values=pre_cti_data.native, pixel_scales=pixel_scales + ) + + return Dataset1D( + data=data, + noise_map=noise_map, + pre_cti_data=pre_cti_data, + layout=layout, + settings_dict=settings_dict, + ) + + def output_to_fits( + self, + data_path: Union[Path, str], + noise_map_path: Optional[Union[Path, str]] = None, + pre_cti_data_path: Optional[Union[Path, str]] = None, + overwrite: bool = False, + ): + """ + Output the 1D dataset to multiple .fits file. + + For each attribute of the 1D dataset data (e.g. `data`, `noise_map`, `pre_cti_data`) the path to + the .fits can be specified, with `hdu=0` assumed automatically. + + If the `data` has been masked, the masked data is output to .fits files. A mask can be separately output to + a file `mask.fits` via the `Mask` objects `output_to_fits` method. + + Parameters + ---------- + data_path + The path to the data .fits file where the image data is output (e.g. '/path/to/data.fits'). + noise_map_path + The path to the noise_map .fits where the noise_map is output (e.g. '/path/to/noise_map.fits'). + pre_cti_data_path + The path to the pre CTI data .fits file where the pre CTI data is output (e.g. '/path/to/pre_cti_data.fits'). + overwrite + If `True`, the .fits files are overwritten if they already exist, if `False` they are not and an + exception is raised. + """ + fitsable.output_to_fits( + values=np.asarray(self.data.native), file_path=data_path, overwrite=overwrite + ) + fitsable.output_to_fits( + values=np.asarray(self.noise_map.native), + file_path=noise_map_path, + overwrite=overwrite, + ) + fitsable.output_to_fits( + values=np.asarray(self.pre_cti_data.native), + file_path=pre_cti_data_path, + overwrite=overwrite, + ) + + @classmethod + def from_pixel_line_dict( + cls, + pixel_line_dict: dict, + size: int, + ) -> "Dataset1D": + """ + Parse a pixel line output from the warm-pixels script. + + Pixel lines are individual or averaged lines found by searching for + warm pixels or consistent warm pixels in CCD data. The warm pixel + and its are extracted and saved as a JSON which can then be loaded + and fit as part of autocti. + + Parameters + ---------- + pixel_line_dict + A dictionary describing a pixel line collection. + + e.g. + { + "location": [ + 2, + 4, + ], + "flux": 1234., + "data": [ + 5.0, + 3.0, + 2.0, + 1.0, + ], + "noise": [ + 1.0, + 1.0, + 1.0, + 1.0, + ] + } + + location + The location of the warm pixel in (row, column) where row is the + distance to the serial register - 1 + flux + The computed flux of the warm pixel prior to CTI + data + The extracted pixel line. A 1D array where the first entry is + the warm pixel (FPR) and the remaining entries are the trail + (EPER) + noise + The noise map for the pixel line. + size + The size of the CCD. That is, the number of pixels in the parallel + direction. + + Returns + ------- + A Dataset1D initialised to represent the pixel line. The pixel line + and noise are embedded in Array1Ds of the same size as the array in + the parallel direction + """ + serial_distance, _ = map(int, pixel_line_dict["location"]) + + def make_array(data): + array = np.zeros(size) + array[serial_distance : serial_distance + len(data)] = data + return aa.Array1D.no_mask(array, pixel_scales=0.1) + + return Dataset1D( + data=make_array(pixel_line_dict["data"]), + noise_map=make_array(pixel_line_dict["noise"]), + pre_cti_data=make_array(np.array([pixel_line_dict["flux"]])), + layout=Layout1D( + shape_1d=(size,), + region_list=[ + aa.Region1D(region=(serial_distance, serial_distance + 1)) + ], + ), + ) diff --git a/autocti/dataset_1d/dataset_1d/simulator.py b/autocti/dataset_1d/dataset_1d/simulator.py index 697e0ed2..f437a329 100644 --- a/autocti/dataset_1d/dataset_1d/simulator.py +++ b/autocti/dataset_1d/dataset_1d/simulator.py @@ -19,7 +19,7 @@ def __init__( pixel_scales: aa.type.PixelScales, norm: float, read_noise: Optional[float] = None, - add_poisson_noise: bool = False, + add_poisson_noise_to_data: bool = False, charge_noise: Optional[float] = None, noise_if_add_noise_false: float = 0.1, noise_seed: int = -1, @@ -36,7 +36,7 @@ def __init__( super().__init__( exposure_time=1.0, - add_poisson_noise=add_poisson_noise, + add_poisson_noise_to_data=add_poisson_noise_to_data, noise_if_add_noise_false=noise_if_add_noise_false, noise_seed=noise_seed, ) diff --git a/autocti/dataset_1d/model/analysis.py b/autocti/dataset_1d/model/analysis.py index 1818e6e2..40c3e081 100644 --- a/autocti/dataset_1d/model/analysis.py +++ b/autocti/dataset_1d/model/analysis.py @@ -1,185 +1,181 @@ -from typing import List, Optional - -from autoconf.dictable import to_dict - -import autofit as af - -from autocti.dataset_1d.dataset_1d.dataset_1d import Dataset1D -from autocti.dataset_1d.fit import FitDataset1D -from autocti.dataset_1d.model.visualizer import VisualizerDataset1D -from autocti.dataset_1d.model.result import ResultDataset1D -from autocti.model.analysis import AnalysisCTI -from autocti.model.settings import SettingsCTI1D -from autocti.clocker.one_d import Clocker1D - - -class AnalysisDataset1D(AnalysisCTI): - Result = ResultDataset1D - Visualizer = VisualizerDataset1D - - def __init__( - self, - dataset: Dataset1D, - clocker: Clocker1D, - settings_cti: SettingsCTI1D = SettingsCTI1D(), - dataset_full: Optional[Dataset1D] = None, - ): - """ - Fits a CTI model to a 1D CTI dataset via a non-linear search. - - The `Analysis` class defines the `log_likelihood_function` which fits the model to the dataset and returns the - log likelihood value defining how well the model fitted the data. - - It handles many other tasks, such as visualization, outputting results to hard-disk and storing results in - a format that can be loaded after the model-fit is complete. - - This class is used for model-fits which fit a CTI model via a `CTI1D` object to a charge injection - imaging dataset. - - Parameters - ---------- - dataset - The 1D CTI dataset that the model is fitted to. - clocker - The CTI arctic clocker used by the non-linear search and model-fit. - settings_cti - The settings controlling aspects of the CTI model in this model-fit. - dataset_full - The full dataset, which is visualized separate from the `dataset` that is fitted, which for example may - not have the FPR masked and thus enable visualization of the FPR. - """ - super().__init__( - dataset=dataset, - clocker=clocker, - settings_cti=settings_cti, - dataset_full=dataset_full, - ) - - def region_list_from(self) -> List: - return ["fpr", "eper"] - - def modify_before_fit(self, paths: af.DirectoryPaths, model: af.Collection): - """ - This function is called immediately before the non-linear search begins and performs final tasks and checks - before it begins. - - This function: - - 1) Visualizes the 1D dataset, which does not change during the analysis and thus can be done once. - - Parameters - ---------- - paths - The paths object which manages all paths, e.g. where the non-linear search outputs are stored, - visualization and the pickled objects used by the aggregator output by this function. - model - The model object, which includes model components representing the galaxies that are fitted to - the imaging data. - """ - - if paths.is_complete: - return self - - return self - - def log_likelihood_function(self, instance: af.ModelInstance) -> float: - """ - Determine the fitness of a particular model - - Parameters - ---------- - instance - - Returns - ------- - fit: Fit - How fit the model is and the model - """ - - self.settings_cti.check_total_density_within_range(traps=instance.cti.trap_list) - - fit = self.fit_via_instance_from(instance=instance) - - return fit.log_likelihood - - def fit_via_instance_and_dataset_from( - self, instance: af.ModelInstance, dataset: Dataset1D - ) -> FitDataset1D: - post_cti_data = self.clocker.add_cti( - data=dataset.pre_cti_data, cti=instance.cti - ) - - return FitDataset1D(dataset=dataset, post_cti_data=post_cti_data) - - def fit_via_instance_from(self, instance: af.ModelInstance) -> FitDataset1D: - return self.fit_via_instance_and_dataset_from( - instance=instance, dataset=self.dataset - ) - - def save_attributes(self, paths: af.DirectoryPaths): - """ - Before the model-fit via the non-linear search begins, this routine saves attributes of the `Analysis` object - to the `files` folder such that they can be loaded after the analysis using PyAutoFit's database and - aggregator tools. - - For this analysis the following are output: - - - The 1D dataset (data / noise-map / pre cti data / layout / settings etc.). - - The mask applied to the dataset. - - The clocker used for modeling / clocking CTI. - - The settings used for modeling / clocking CTI. - - The full 1D dataset (e.g. unmasked, used for visualizariton). - - It is common for these attributes to be loaded by many of the template aggregator functions given in the - `aggregator` modules. For example, when using the database tools to reperform a fit, this will by default - load the dataset, settings and other attributes necessary to perform a fit using the attributes output by - this function. - - Parameters - ---------- - paths - The paths object which manages all paths, e.g. where the non-linear search outputs are stored, - visualization,and the pickled objects used by the aggregator output by this function. - """ - - def output_dataset(dataset, prefix): - paths.save_fits( - name="data", - hdu=dataset.data.hdu_for_output, - prefix=prefix, - ) - paths.save_fits( - name="noise_map", - hdu=dataset.noise_map.hdu_for_output, - prefix=prefix, - ) - paths.save_fits( - name="pre_cti_data", - hdu=dataset.pre_cti_data.hdu_for_output, - prefix=prefix, - ) - paths.save_fits( - name="mask", - hdu=dataset.mask.hdu_for_output, - prefix=prefix, - ) - paths.save_json( - name="layout", - object_dict=to_dict(dataset.layout), - prefix=prefix, - ) - - output_dataset(dataset=self.dataset, prefix="dataset") - - if self.dataset_full is not None: - output_dataset(dataset=self.dataset_full, prefix="dataset_full") - - paths.save_json( - name="clocker", - object_dict=to_dict(self.clocker), - ) - - paths.save_json( - name="settings_cti", - object_dict=to_dict(self.settings_cti), - ) +import numpy as np +from typing import List, Optional + +from autoconf.dictable import to_dict + +import autofit as af +from autoconf.fitsable import hdu_list_for_output_from + +from autocti.dataset_1d.dataset_1d.dataset_1d import Dataset1D +from autocti.dataset_1d.fit import FitDataset1D +from autocti.dataset_1d.model.visualizer import VisualizerDataset1D +from autocti.dataset_1d.model.result import ResultDataset1D +from autocti.model.analysis import AnalysisCTI +from autocti.model.settings import SettingsCTI1D +from autocti.clocker.one_d import Clocker1D + + +class AnalysisDataset1D(AnalysisCTI): + Result = ResultDataset1D + Visualizer = VisualizerDataset1D + + def __init__( + self, + dataset: Dataset1D, + clocker: Clocker1D, + settings_cti: SettingsCTI1D = SettingsCTI1D(), + dataset_full: Optional[Dataset1D] = None, + ): + """ + Fits a CTI model to a 1D CTI dataset via a non-linear search. + + The `Analysis` class defines the `log_likelihood_function` which fits the model to the dataset and returns the + log likelihood value defining how well the model fitted the data. + + It handles many other tasks, such as visualization, outputting results to hard-disk and storing results in + a format that can be loaded after the model-fit is complete. + + This class is used for model-fits which fit a CTI model via a `CTI1D` object to a charge injection + imaging dataset. + + Parameters + ---------- + dataset + The 1D CTI dataset that the model is fitted to. + clocker + The CTI arctic clocker used by the non-linear search and model-fit. + settings_cti + The settings controlling aspects of the CTI model in this model-fit. + dataset_full + The full dataset, which is visualized separate from the `dataset` that is fitted, which for example may + not have the FPR masked and thus enable visualization of the FPR. + """ + super().__init__( + dataset=dataset, + clocker=clocker, + settings_cti=settings_cti, + dataset_full=dataset_full, + ) + + def region_list_from(self) -> List: + return ["fpr", "eper"] + + def modify_before_fit(self, paths: af.DirectoryPaths, model: af.Collection): + """ + This function is called immediately before the non-linear search begins and performs final tasks and checks + before it begins. + + This function: + + 1) Visualizes the 1D dataset, which does not change during the analysis and thus can be done once. + + Parameters + ---------- + paths + The paths object which manages all paths, e.g. where the non-linear search outputs are stored, + visualization and the pickled objects used by the aggregator output by this function. + model + The model object, which includes model components representing the galaxies that are fitted to + the imaging data. + """ + + if paths.is_complete: + return self + + return self + + def log_likelihood_function(self, instance: af.ModelInstance) -> float: + """ + Determine the fitness of a particular model + + Parameters + ---------- + instance + + Returns + ------- + fit: Fit + How fit the model is and the model + """ + + self.settings_cti.check_total_density_within_range(traps=instance.cti.trap_list) + + fit = self.fit_via_instance_from(instance=instance) + + return fit.log_likelihood + + def fit_via_instance_and_dataset_from( + self, instance: af.ModelInstance, dataset: Dataset1D + ) -> FitDataset1D: + post_cti_data = self.clocker.add_cti( + data=dataset.pre_cti_data, cti=instance.cti + ) + + return FitDataset1D(dataset=dataset, post_cti_data=post_cti_data) + + def fit_via_instance_from(self, instance: af.ModelInstance) -> FitDataset1D: + return self.fit_via_instance_and_dataset_from( + instance=instance, dataset=self.dataset + ) + + def save_attributes(self, paths: af.DirectoryPaths): + """ + Before the model-fit via the non-linear search begins, this routine saves attributes of the `Analysis` object + to the `files` folder such that they can be loaded after the analysis using PyAutoFit's database and + aggregator tools. + + For this analysis the following are output: + + - The 1D dataset (data / noise-map / pre cti data / layout / settings etc.). + - The mask applied to the dataset. + - The clocker used for modeling / clocking CTI. + - The settings used for modeling / clocking CTI. + - The full 1D dataset (e.g. unmasked, used for visualizariton). + + It is common for these attributes to be loaded by many of the template aggregator functions given in the + `aggregator` modules. For example, when using the database tools to reperform a fit, this will by default + load the dataset, settings and other attributes necessary to perform a fit using the attributes output by + this function. + + Parameters + ---------- + paths + The paths object which manages all paths, e.g. where the non-linear search outputs are stored, + visualization,and the pickled objects used by the aggregator output by this function. + """ + + def output_dataset(dataset, prefix): + paths.save_fits( + name="dataset", + fits=hdu_list_for_output_from( + values_list=[ + np.asarray(dataset.mask).astype("float"), + np.asarray(dataset.data.native), + np.asarray(dataset.noise_map.native), + np.asarray(dataset.pre_cti_data.native), + ], + ext_name_list=["mask", "data", "noise_map", "pre_cti_data"], + header_dict=dataset.mask.header_dict, + ), + prefix=prefix, + ) + paths.save_json( + name="layout", + object_dict=to_dict(dataset.layout), + prefix=prefix, + ) + + output_dataset(dataset=self.dataset, prefix="dataset") + + if self.dataset_full is not None: + output_dataset(dataset=self.dataset_full, prefix="dataset_full") + + paths.save_json( + name="clocker", + object_dict=to_dict(self.clocker), + ) + + paths.save_json( + name="settings_cti", + object_dict=to_dict(self.settings_cti), + ) diff --git a/autocti/dataset_1d/model/visualizer.py b/autocti/dataset_1d/model/visualizer.py index 65d21b3f..996d7519 100644 --- a/autocti/dataset_1d/model/visualizer.py +++ b/autocti/dataset_1d/model/visualizer.py @@ -1,185 +1,232 @@ -import autofit as af - -from autocti.dataset_1d.model.plotter_interface import PlotterInterfaceDataset1D - - -class VisualizerDataset1D(af.Visualizer): - @staticmethod - def visualize_before_fit( - analysis, - paths: af.AbstractPaths, - model: af.AbstractPriorModel, - ): - """ - PyAutoFit calls this function immediately before the non-linear search begins. - - It visualizes objects which do not change throughout the model fit like the dataset. - - Parameters - ---------- - paths - The paths object which manages all paths, e.g. where the non-linear search outputs are stored, - visualization and the pickled objects used by the aggregator output by this function. - model - The model object, which includes model components representing the galaxies that are fitted to - the imaging data. - """ - - region_list = analysis.region_list_from() - - visualizer = PlotterInterfaceDataset1D(image_path=paths.image_path) - visualizer.dataset(dataset=analysis.dataset) - visualizer.dataset_regions(dataset=analysis.dataset, region_list=region_list) - - if analysis.dataset_full is not None: - visualizer.dataset(dataset=analysis.dataset_full, folder_suffix="_full") - visualizer.dataset_regions( - dataset=analysis.dataset_full, - region_list=region_list, - folder_suffix="_full", - ) - - @staticmethod - def visualize_before_fit_combined( - analyses, - paths: af.AbstractPaths, - model: af.AbstractPriorModel, - ): - if analyses is None: - return - - plotter = PlotterInterfaceDataset1D(image_path=paths.image_path) - - region_list = analyses[0].region_list_from() - - dataset_list = [analysis.dataset for analysis in analyses] - fpr_value_list = [dataset.fpr_value for dataset in dataset_list] - - dataset_list = analyses[0].in_ascending_fpr_order_from( - quantity_list=dataset_list, - fpr_value_list=fpr_value_list, - ) - - plotter.dataset_combined( - dataset_list=dataset_list, - ) - plotter.dataset_regions_combined( - dataset_list=dataset_list, - region_list=region_list, - ) - - if analyses[0].dataset_full is not None: - dataset_full_list = [analysis.dataset_full for analysis in analyses] - - dataset_full_list = analyses[0].in_ascending_fpr_order_from( - quantity_list=dataset_full_list, - fpr_value_list=fpr_value_list, - ) - - plotter.dataset_combined( - dataset_list=dataset_full_list, folder_suffix="_full" - ) - plotter.dataset_regions_combined( - dataset_list=dataset_full_list, - region_list=region_list, - folder_suffix="_full", - ) - - @staticmethod - def visualize( - analysis, - paths: af.DirectoryPaths, - instance: af.ModelInstance, - during_analysis: bool, - ): - """ - Output images of the maximum log likelihood model inferred by the model-fit. This function is called throughout - the non-linear search at regular intervals, and therefore provides on-the-fly visualization of how well the - model-fit is going. - - The images output by this function are customized using the file `config/visualize/plots.yaml`. - - Parameters - ---------- - paths - The paths object which manages all paths, e.g. where the non-linear search outputs are stored, - visualization, and the pickled objects used by the aggregator output by this function. - instance - An instance of the model that is being fitted to the data by this analysis (whose parameters have been set - via a non-linear search). - during_analysis - If True the visualization is being performed midway through the non-linear search before it is finished, - which may change which images are output. - """ - - region_list = analysis.region_list_from() - - visualizer = PlotterInterfaceDataset1D(image_path=paths.image_path) - - fit = analysis.fit_via_instance_from(instance=instance) - visualizer.fit(fit=fit, during_analysis=during_analysis) - visualizer.fit_regions( - fit=fit, region_list=region_list, during_analysis=during_analysis - ) - - if analysis.dataset_full is not None: - fit = analysis.fit_via_instance_and_dataset_from( - instance=instance, dataset=analysis.dataset_full - ) - visualizer.fit(fit=fit, during_analysis=during_analysis) - visualizer.fit_regions( - fit=fit, region_list=region_list, during_analysis=during_analysis - ) - - @staticmethod - def visualize_combined( - analyses, - paths: af.DirectoryPaths, - instance: af.ModelInstance, - during_analysis: bool, - ): - if analyses is None: - return - - fit_list = [ - analysis.fit_via_instance_from(instance=instance) for analysis in analyses - ] - - fpr_value_list = [fit.dataset.fpr_value for fit in fit_list] - - fit_list = analyses[0].in_ascending_fpr_order_from( - quantity_list=fit_list, - fpr_value_list=fpr_value_list, - ) - - region_list = analyses[0].region_list_from() - - visualizer = PlotterInterfaceDataset1D(image_path=paths.image_path) - visualizer.fit_combined(fit_list=fit_list, during_analysis=during_analysis) - visualizer.fit_region_combined( - fit_list=fit_list, - region_list=region_list, - during_analysis=during_analysis, - ) - - if analyses[0].dataset_full is not None: - fit_full_list = [ - analysis.fit_via_instance_and_dataset_from( - instance=instance, dataset=analysis.dataset_full - ) - for analysis in analyses - ] - - fit_full_list = analyses[0].in_ascending_fpr_order_from( - quantity_list=fit_full_list, - fpr_value_list=fpr_value_list, - ) - - visualizer.fit_combined( - fit_list=fit_full_list, during_analysis=during_analysis - ) - visualizer.fit_region_combined( - fit_list=fit_full_list, - region_list=region_list, - during_analysis=during_analysis, - ) +import logging + +import autofit as af + +logger = logging.getLogger(__name__) + + +class VisualizerDataset1D(af.Visualizer): + @staticmethod + def visualize_before_fit( + analysis, + paths: af.AbstractPaths, + model: af.AbstractPriorModel, + ): + """ + PyAutoFit calls this function immediately before the non-linear search begins. + + It visualizes objects which do not change throughout the model fit like the dataset. + + Parameters + ---------- + paths + The paths object which manages all paths, e.g. where the non-linear search outputs are stored, + visualization and the pickled objects used by the aggregator output by this function. + model + The model object, which includes model components representing the galaxies that are fitted to + the imaging data. + """ + # Imported lazily: the PlotterInterface stack still targets the removed + # autoarray Plotter API and is rewritten on the new matplotlib function + # API in Phase 1 of the CTI resurrection epic (PyAutoCTI#82). + try: + from autocti.dataset_1d.model.plotter_interface import ( + PlotterInterfaceDataset1D, + ) + except ImportError: + logger.warning( + "PyAutoCTI visualization is disabled until the Phase 1 " + "Plotter->matplotlib migration (PyAutoCTI#82)." + ) + return + + region_list = analysis.region_list_from() + + visualizer = PlotterInterfaceDataset1D(image_path=paths.image_path) + visualizer.dataset(dataset=analysis.dataset) + visualizer.dataset_regions(dataset=analysis.dataset, region_list=region_list) + + if analysis.dataset_full is not None: + visualizer.dataset(dataset=analysis.dataset_full, folder_suffix="_full") + visualizer.dataset_regions( + dataset=analysis.dataset_full, + region_list=region_list, + folder_suffix="_full", + ) + + @staticmethod + def visualize_before_fit_combined( + analyses, + paths: af.AbstractPaths, + model: af.AbstractPriorModel, + ): + if analyses is None: + return + + try: + from autocti.dataset_1d.model.plotter_interface import ( + PlotterInterfaceDataset1D, + ) + except ImportError: + logger.warning( + "PyAutoCTI visualization is disabled until the Phase 1 " + "Plotter->matplotlib migration (PyAutoCTI#82)." + ) + return + + plotter = PlotterInterfaceDataset1D(image_path=paths.image_path) + + region_list = analyses[0].region_list_from() + + dataset_list = [analysis.dataset for analysis in analyses] + fpr_value_list = [dataset.fpr_value for dataset in dataset_list] + + dataset_list = analyses[0].in_ascending_fpr_order_from( + quantity_list=dataset_list, + fpr_value_list=fpr_value_list, + ) + + plotter.dataset_combined( + dataset_list=dataset_list, + ) + plotter.dataset_regions_combined( + dataset_list=dataset_list, + region_list=region_list, + ) + + if analyses[0].dataset_full is not None: + dataset_full_list = [analysis.dataset_full for analysis in analyses] + + dataset_full_list = analyses[0].in_ascending_fpr_order_from( + quantity_list=dataset_full_list, + fpr_value_list=fpr_value_list, + ) + + plotter.dataset_combined( + dataset_list=dataset_full_list, folder_suffix="_full" + ) + plotter.dataset_regions_combined( + dataset_list=dataset_full_list, + region_list=region_list, + folder_suffix="_full", + ) + + @staticmethod + def visualize( + analysis, + paths: af.DirectoryPaths, + instance: af.ModelInstance, + during_analysis: bool, + ): + """ + Output images of the maximum log likelihood model inferred by the model-fit. This function is called throughout + the non-linear search at regular intervals, and therefore provides on-the-fly visualization of how well the + model-fit is going. + + The images output by this function are customized using the file `config/visualize/plots.yaml`. + + Parameters + ---------- + paths + The paths object which manages all paths, e.g. where the non-linear search outputs are stored, + visualization, and the pickled objects used by the aggregator output by this function. + instance + An instance of the model that is being fitted to the data by this analysis (whose parameters have been set + via a non-linear search). + during_analysis + If True the visualization is being performed midway through the non-linear search before it is finished, + which may change which images are output. + """ + try: + from autocti.dataset_1d.model.plotter_interface import ( + PlotterInterfaceDataset1D, + ) + except ImportError: + logger.warning( + "PyAutoCTI visualization is disabled until the Phase 1 " + "Plotter->matplotlib migration (PyAutoCTI#82)." + ) + return + + region_list = analysis.region_list_from() + + visualizer = PlotterInterfaceDataset1D(image_path=paths.image_path) + + fit = analysis.fit_via_instance_from(instance=instance) + visualizer.fit(fit=fit, during_analysis=during_analysis) + visualizer.fit_regions( + fit=fit, region_list=region_list, during_analysis=during_analysis + ) + + if analysis.dataset_full is not None: + fit = analysis.fit_via_instance_and_dataset_from( + instance=instance, dataset=analysis.dataset_full + ) + visualizer.fit(fit=fit, during_analysis=during_analysis) + visualizer.fit_regions( + fit=fit, region_list=region_list, during_analysis=during_analysis + ) + + @staticmethod + def visualize_combined( + analyses, + paths: af.DirectoryPaths, + instance: af.ModelInstance, + during_analysis: bool, + ): + if analyses is None: + return + + try: + from autocti.dataset_1d.model.plotter_interface import ( + PlotterInterfaceDataset1D, + ) + except ImportError: + logger.warning( + "PyAutoCTI visualization is disabled until the Phase 1 " + "Plotter->matplotlib migration (PyAutoCTI#82)." + ) + return + + fit_list = [ + analysis.fit_via_instance_from(instance=instance) for analysis in analyses + ] + + fpr_value_list = [fit.dataset.fpr_value for fit in fit_list] + + fit_list = analyses[0].in_ascending_fpr_order_from( + quantity_list=fit_list, + fpr_value_list=fpr_value_list, + ) + + region_list = analyses[0].region_list_from() + + visualizer = PlotterInterfaceDataset1D(image_path=paths.image_path) + visualizer.fit_combined(fit_list=fit_list, during_analysis=during_analysis) + visualizer.fit_region_combined( + fit_list=fit_list, + region_list=region_list, + during_analysis=during_analysis, + ) + + if analyses[0].dataset_full is not None: + fit_full_list = [ + analysis.fit_via_instance_and_dataset_from( + instance=instance, dataset=analysis.dataset_full + ) + for analysis in analyses + ] + + fit_full_list = analyses[0].in_ascending_fpr_order_from( + quantity_list=fit_full_list, + fpr_value_list=fpr_value_list, + ) + + visualizer.fit_combined( + fit_list=fit_full_list, during_analysis=during_analysis + ) + visualizer.fit_region_combined( + fit_list=fit_full_list, + region_list=region_list, + during_analysis=during_analysis, + ) diff --git a/autocti/extract/two_d/abstract.py b/autocti/extract/two_d/abstract.py index f96e0d51..815b9647 100644 --- a/autocti/extract/two_d/abstract.py +++ b/autocti/extract/two_d/abstract.py @@ -225,7 +225,16 @@ def array_2d_list_from( region_list = settings.region_list_from(region_list=region_list) arr_list = [array.native[region.slice] for region in region_list] - mask_2d_list = [array.mask[region.slice] for region in region_list] + + # Slicing a Mask2D returns a raw ndarray, so a Mask2D is rebuilt from it + # with the parent array's pixel scales. + mask_2d_list = [ + aa.Mask2D( + mask=np.asarray(array.mask)[region.slice], + pixel_scales=array.pixel_scales, + ) + for region in region_list + ] return [ aa.Array2D(values=arr, mask=mask_2d).native diff --git a/autocti/instruments/acs/array_2d.py b/autocti/instruments/acs/array_2d.py index 8e804ff8..e9909435 100644 --- a/autocti/instruments/acs/array_2d.py +++ b/autocti/instruments/acs/array_2d.py @@ -1,326 +1,327 @@ -from astropy.io import fits -import logging -import numpy as np -import os -import shutil - -from autoarray.structures.arrays.uniform_2d import Array2D - -from autoarray import exc -from autoarray.structures.arrays import array_2d_util -from autoarray.layout import layout_util - -from autocti.instruments.acs import acs_util - -logging.basicConfig() -logger = logging.getLogger() -logger.setLevel("INFO") - - -class Array2DACS(Array2D): - """ - An ACS array consists of four quadrants ('A', 'B', 'C', 'D') which have the following layout (which are described - at the following STScI - link https://github.com/spacetelescope/hstcal/blob/main/pkg/acs/calacs/acscte/dopcte-gen2.c#L418). - - <--------S----------- ---------S-----------> - [] [========= 2 =========] [========= 3 =========] [] /\ - / [xxxxxxxxxxxxxxxxxxxxx] [xxxxxxxxxxxxxxxxxxxxx] / | - | [xxxxxxxxxxxxxxxxxxxxx] [xxxxxxxxxxxxxxxxxxxxx] | | Direction arctic - P [xxxxxxxxx B/C xxxxxxx] [xxxxxxxxx A/D xxxxxxx] P | clocks an image - | [xxxxxxxxxxxxxxxxxxxxx] [xxxxxxxxxxxxxxxxxxxxx] | | without any rotation - \/ [xxxxxxxxxxxxxxxxxxxxx] [xxxxxxxxxxxxxxxxxxxxx] \/ | (e.g. towards row 0 - | of the NumPy arrays) - - For a ACS .fits file: - - - The images contained in hdu 1 correspond to quadrants B (left) and A (right). - - The images contained in hdu 4 correspond to quadrants C (left) and D (right). - """ - - @classmethod - def from_fits(cls, file_path, quadrant_letter): - """ - Use the input .fits file and quadrant letter to extract the quadrant from the full CCD, perform - the rotations required to give correct arctic clocking and convert the image from units of COUNTS / CPS to - ELECTRONS. - - See the docstring of the `FrameACS` class for a complete description of the HST FPA, quadrants and - rotations. - - Also see https://github.com/spacetelescope/hstcal/blob/main/pkg/acs/calacs/acscte/dopcte-gen2.c#L418 - """ - - hdu = acs_util.fits_hdu_via_quadrant_letter_from( - quadrant_letter=quadrant_letter - ) - - array = array_2d_util.numpy_array_2d_via_fits_from(file_path=file_path, hdu=hdu) - - return cls.from_ccd(array_electrons=array, quadrant_letter=quadrant_letter) - - @classmethod - def from_ccd( - cls, - array_electrons, - quadrant_letter, - header=None, - bias_subtract_via_prescan=False, - bias=None, - ): - """ - Using an input array of both quadrants in electrons, use the quadrant letter to extract the quadrant from the - full CCD and perform the rotations required to give correct arctic. - - See the docstring of the `FrameACS` class for a complete description of the HST FPA, quadrants and - rotations. - - Also see https://github.com/spacetelescope/hstcal/blob/main/pkg/acs/calacs/acscte/dopcte-gen2.c#L418 - """ - if quadrant_letter == "A": - array_electrons = array_electrons[0:2068, 0:2072] - roe_corner = (1, 0) - use_flipud = True - - if bias is not None: - bias = bias[0:2068, 0:2072] - - elif quadrant_letter == "B": - array_electrons = array_electrons[0:2068, 2072:4144] - roe_corner = (1, 1) - use_flipud = True - - if bias is not None: - bias = bias[0:2068, 2072:4144] - - elif quadrant_letter == "C": - array_electrons = array_electrons[0:2068, 0:2072] - - roe_corner = (1, 0) - use_flipud = False - - if bias is not None: - bias = bias[0:2068, 0:2072] - - elif quadrant_letter == "D": - array_electrons = array_electrons[0:2068, 2072:4144] - - roe_corner = (1, 1) - use_flipud = False - - if bias is not None: - bias = bias[0:2068, 2072:4144] - - else: - raise exc.ArrayException( - "Quadrant letter for FrameACS must be A, B, C or D." - ) - - return cls.quadrant_a( - array_electrons=array_electrons, - header=header, - roe_corner=roe_corner, - use_flipud=use_flipud, - bias_subtract_via_prescan=bias_subtract_via_prescan, - bias=bias, - ) - - @classmethod - def quadrant_a( - cls, - array_electrons, - roe_corner, - use_flipud, - header=None, - bias_subtract_via_prescan=False, - bias=None, - ): - """ - Use an input array of the left quadrant in electrons and perform the rotations required to give correct - arctic clocking. - - See the docstring of the `FrameACS` class for a complete description of the HST FPA, quadrants and - rotations. - - Also see https://github.com/spacetelescope/hstcal/blob/main/pkg/acs/calacs/acscte/dopcte-gen2.c#L418 - """ - - array_electrons = layout_util.rotate_array_via_roe_corner_from( - array=array_electrons, roe_corner=roe_corner - ) - - if use_flipud: - array_electrons = np.flipud(array_electrons) - - if bias_subtract_via_prescan: - bias_serial_prescan_value = acs_util.prescan_fitted_bias_column( - array_electrons[:, 18:24] - ) - - array_electrons -= bias_serial_prescan_value - - header.bias_serial_prescan_column = bias_serial_prescan_value - - if bias is not None: - bias = layout_util.rotate_array_via_roe_corner_from( - array=bias, roe_corner=roe_corner - ) - - if use_flipud: - bias = np.flipud(bias) - - array_electrons -= bias - - header.bias = Array2DACS.no_mask(values=bias, pixel_scales=0.05) - - return cls.no_mask(values=array_electrons, header=header, pixel_scales=0.05) - - @classmethod - def quadrant_b( - cls, array_electrons, header=None, bias_subtract_via_prescan=False, bias=None - ): - """ - Use an input array of the right quadrant in electrons and perform the rotations required to give correct - arctic clocking. - - See the docstring of the `FrameACS` class for a complete description of the HST FPA, quadrants and - rotations. - - Also see https://github.com/spacetelescope/hstcal/blob/main/pkg/acs/calacs/acscte/dopcte-gen2.c#L418 - """ - - array_electrons = layout_util.rotate_array_via_roe_corner_from( - array=array_electrons, roe_corner=(1, 1) - ) - - array_electrons = np.flipud(array_electrons) - - if bias_subtract_via_prescan: - bias_serial_prescan_value = acs_util.prescan_fitted_bias_column( - array_electrons[:, 18:24] - ) - - array_electrons -= bias_serial_prescan_value - - header.bias_serial_prescan_column = bias_serial_prescan_value - - if bias is not None: - bias = layout_util.rotate_array_via_roe_corner_from( - array=bias, roe_corner=(1, 1) - ) - - bias = np.flipud(bias) - - array_electrons -= bias - - header.bias = Array2DACS.no_mask(values=bias, pixel_scales=0.05) - - return cls.no_mask(values=array_electrons, header=header, pixel_scales=0.05) - - @classmethod - def quadrant_c( - cls, array_electrons, header=None, bias_subtract_via_prescan=False, bias=None - ): - """ - Use an input array of the left quadrant in electrons and perform the rotations required to give correct - arctic clocking. - - See the docstring of the `FrameACS` class for a complete description of the HST FPA, quadrants and - rotations. - - Also see https://github.com/spacetelescope/hstcal/blob/main/pkg/acs/calacs/acscte/dopcte-gen2.c#L418 - """ - - array_electrons = layout_util.rotate_array_via_roe_corner_from( - array=array_electrons, roe_corner=(1, 0) - ) - - if bias_subtract_via_prescan: - bias_serial_prescan_value = acs_util.prescan_fitted_bias_column( - array_electrons[:, 18:24] - ) - - array_electrons -= bias_serial_prescan_value - - header.bias_serial_prescan_column = bias_serial_prescan_value - - if bias is not None: - bias = layout_util.rotate_array_via_roe_corner_from( - array=bias, roe_corner=(1, 0) - ) - - array_electrons -= bias - - header.bias = Array2DACS.no_mask(values=bias, pixel_scales=0.05) - - return cls.no_mask(values=array_electrons, header=header, pixel_scales=0.05) - - @classmethod - def quadrant_d( - cls, array_electrons, header=None, bias_subtract_via_prescan=False, bias=None - ): - """ - Use an input array of the right quadrant in electrons and perform the rotations required to give correct - arctic clocking. - - See the docstring of the `FrameACS` class for a complete description of the HST FPA, quadrants and - rotations. - - Also see https://github.com/spacetelescope/hstcal/blob/main/pkg/acs/calacs/acscte/dopcte-gen2.c#L418 - """ - - array_electrons = layout_util.rotate_array_via_roe_corner_from( - array=array_electrons, roe_corner=(1, 1) - ) - - if bias_subtract_via_prescan: - bias_serial_prescan_value = acs_util.prescan_fitted_bias_column( - array_electrons[:, 18:24] - ) - - array_electrons -= bias_serial_prescan_value - - header.bias_serial_prescan_column = bias_serial_prescan_value - - if bias is not None: - bias = layout_util.rotate_array_via_roe_corner_from( - array=bias, roe_corner=(1, 1) - ) - - array_electrons -= bias - - header.bias = Array2DACS.no_mask(values=bias, pixel_scales=0.05) - - return cls.no_mask(values=array_electrons, header=header, pixel_scales=0.05) - - def update_fits(self, original_file_path, new_file_path): - """ - Output the array to a .fits file. - - Parameters - ---------- - file_path - The path the file is output to, including the filename and the ``.fits`` extension, - e.g. '/path/to/filename.fits' - """ - - new_file_dir = os.path.split(new_file_path)[0] - - if not os.path.exists(new_file_dir): - os.makedirs(new_file_dir) - - if not os.path.exists(new_file_path): - shutil.copy(original_file_path, new_file_path) - - hdulist = fits.open(new_file_path) - - hdulist[self.header.hdu].data = self.layout_2d.original_orientation_from( - array=self - ) - - ext_header = hdulist[4].header - bscale = ext_header["BSCALE"] - - os.remove(new_file_path) - - hdulist.writeto(new_file_path) +from astropy.io import fits +import logging +import numpy as np +import os +import shutil + +from autoarray.structures.arrays.uniform_2d import Array2D + +from autoarray import exc +from autoarray.structures.arrays import array_2d_util +from autoarray.layout import layout_util + +from autocti.instruments.acs import acs_util +from autoconf import fitsable + +logging.basicConfig() +logger = logging.getLogger() +logger.setLevel("INFO") + + +class Array2DACS(Array2D): + """ + An ACS array consists of four quadrants ('A', 'B', 'C', 'D') which have the following layout (which are described + at the following STScI + link https://github.com/spacetelescope/hstcal/blob/main/pkg/acs/calacs/acscte/dopcte-gen2.c#L418). + + <--------S----------- ---------S-----------> + [] [========= 2 =========] [========= 3 =========] [] /\ + / [xxxxxxxxxxxxxxxxxxxxx] [xxxxxxxxxxxxxxxxxxxxx] / | + | [xxxxxxxxxxxxxxxxxxxxx] [xxxxxxxxxxxxxxxxxxxxx] | | Direction arctic + P [xxxxxxxxx B/C xxxxxxx] [xxxxxxxxx A/D xxxxxxx] P | clocks an image + | [xxxxxxxxxxxxxxxxxxxxx] [xxxxxxxxxxxxxxxxxxxxx] | | without any rotation + \/ [xxxxxxxxxxxxxxxxxxxxx] [xxxxxxxxxxxxxxxxxxxxx] \/ | (e.g. towards row 0 + | of the NumPy arrays) + + For a ACS .fits file: + + - The images contained in hdu 1 correspond to quadrants B (left) and A (right). + - The images contained in hdu 4 correspond to quadrants C (left) and D (right). + """ + + @classmethod + def from_fits(cls, file_path, quadrant_letter): + """ + Use the input .fits file and quadrant letter to extract the quadrant from the full CCD, perform + the rotations required to give correct arctic clocking and convert the image from units of COUNTS / CPS to + ELECTRONS. + + See the docstring of the `FrameACS` class for a complete description of the HST FPA, quadrants and + rotations. + + Also see https://github.com/spacetelescope/hstcal/blob/main/pkg/acs/calacs/acscte/dopcte-gen2.c#L418 + """ + + hdu = acs_util.fits_hdu_via_quadrant_letter_from( + quadrant_letter=quadrant_letter + ) + + array = fitsable.ndarray_via_fits_from(file_path=file_path, hdu=hdu) + + return cls.from_ccd(array_electrons=array, quadrant_letter=quadrant_letter) + + @classmethod + def from_ccd( + cls, + array_electrons, + quadrant_letter, + header=None, + bias_subtract_via_prescan=False, + bias=None, + ): + """ + Using an input array of both quadrants in electrons, use the quadrant letter to extract the quadrant from the + full CCD and perform the rotations required to give correct arctic. + + See the docstring of the `FrameACS` class for a complete description of the HST FPA, quadrants and + rotations. + + Also see https://github.com/spacetelescope/hstcal/blob/main/pkg/acs/calacs/acscte/dopcte-gen2.c#L418 + """ + if quadrant_letter == "A": + array_electrons = array_electrons[0:2068, 0:2072] + roe_corner = (1, 0) + use_flipud = True + + if bias is not None: + bias = bias[0:2068, 0:2072] + + elif quadrant_letter == "B": + array_electrons = array_electrons[0:2068, 2072:4144] + roe_corner = (1, 1) + use_flipud = True + + if bias is not None: + bias = bias[0:2068, 2072:4144] + + elif quadrant_letter == "C": + array_electrons = array_electrons[0:2068, 0:2072] + + roe_corner = (1, 0) + use_flipud = False + + if bias is not None: + bias = bias[0:2068, 0:2072] + + elif quadrant_letter == "D": + array_electrons = array_electrons[0:2068, 2072:4144] + + roe_corner = (1, 1) + use_flipud = False + + if bias is not None: + bias = bias[0:2068, 2072:4144] + + else: + raise exc.ArrayException( + "Quadrant letter for FrameACS must be A, B, C or D." + ) + + return cls.quadrant_a( + array_electrons=array_electrons, + header=header, + roe_corner=roe_corner, + use_flipud=use_flipud, + bias_subtract_via_prescan=bias_subtract_via_prescan, + bias=bias, + ) + + @classmethod + def quadrant_a( + cls, + array_electrons, + roe_corner, + use_flipud, + header=None, + bias_subtract_via_prescan=False, + bias=None, + ): + """ + Use an input array of the left quadrant in electrons and perform the rotations required to give correct + arctic clocking. + + See the docstring of the `FrameACS` class for a complete description of the HST FPA, quadrants and + rotations. + + Also see https://github.com/spacetelescope/hstcal/blob/main/pkg/acs/calacs/acscte/dopcte-gen2.c#L418 + """ + + array_electrons = layout_util.rotate_array_via_roe_corner_from( + array=array_electrons, roe_corner=roe_corner + ) + + if use_flipud: + array_electrons = np.flipud(array_electrons) + + if bias_subtract_via_prescan: + bias_serial_prescan_value = acs_util.prescan_fitted_bias_column( + array_electrons[:, 18:24] + ) + + array_electrons -= bias_serial_prescan_value + + header.bias_serial_prescan_column = bias_serial_prescan_value + + if bias is not None: + bias = layout_util.rotate_array_via_roe_corner_from( + array=bias, roe_corner=roe_corner + ) + + if use_flipud: + bias = np.flipud(bias) + + array_electrons -= bias + + header.bias = Array2DACS.no_mask(values=bias, pixel_scales=0.05) + + return cls.no_mask(values=array_electrons, header=header, pixel_scales=0.05) + + @classmethod + def quadrant_b( + cls, array_electrons, header=None, bias_subtract_via_prescan=False, bias=None + ): + """ + Use an input array of the right quadrant in electrons and perform the rotations required to give correct + arctic clocking. + + See the docstring of the `FrameACS` class for a complete description of the HST FPA, quadrants and + rotations. + + Also see https://github.com/spacetelescope/hstcal/blob/main/pkg/acs/calacs/acscte/dopcte-gen2.c#L418 + """ + + array_electrons = layout_util.rotate_array_via_roe_corner_from( + array=array_electrons, roe_corner=(1, 1) + ) + + array_electrons = np.flipud(array_electrons) + + if bias_subtract_via_prescan: + bias_serial_prescan_value = acs_util.prescan_fitted_bias_column( + array_electrons[:, 18:24] + ) + + array_electrons -= bias_serial_prescan_value + + header.bias_serial_prescan_column = bias_serial_prescan_value + + if bias is not None: + bias = layout_util.rotate_array_via_roe_corner_from( + array=bias, roe_corner=(1, 1) + ) + + bias = np.flipud(bias) + + array_electrons -= bias + + header.bias = Array2DACS.no_mask(values=bias, pixel_scales=0.05) + + return cls.no_mask(values=array_electrons, header=header, pixel_scales=0.05) + + @classmethod + def quadrant_c( + cls, array_electrons, header=None, bias_subtract_via_prescan=False, bias=None + ): + """ + Use an input array of the left quadrant in electrons and perform the rotations required to give correct + arctic clocking. + + See the docstring of the `FrameACS` class for a complete description of the HST FPA, quadrants and + rotations. + + Also see https://github.com/spacetelescope/hstcal/blob/main/pkg/acs/calacs/acscte/dopcte-gen2.c#L418 + """ + + array_electrons = layout_util.rotate_array_via_roe_corner_from( + array=array_electrons, roe_corner=(1, 0) + ) + + if bias_subtract_via_prescan: + bias_serial_prescan_value = acs_util.prescan_fitted_bias_column( + array_electrons[:, 18:24] + ) + + array_electrons -= bias_serial_prescan_value + + header.bias_serial_prescan_column = bias_serial_prescan_value + + if bias is not None: + bias = layout_util.rotate_array_via_roe_corner_from( + array=bias, roe_corner=(1, 0) + ) + + array_electrons -= bias + + header.bias = Array2DACS.no_mask(values=bias, pixel_scales=0.05) + + return cls.no_mask(values=array_electrons, header=header, pixel_scales=0.05) + + @classmethod + def quadrant_d( + cls, array_electrons, header=None, bias_subtract_via_prescan=False, bias=None + ): + """ + Use an input array of the right quadrant in electrons and perform the rotations required to give correct + arctic clocking. + + See the docstring of the `FrameACS` class for a complete description of the HST FPA, quadrants and + rotations. + + Also see https://github.com/spacetelescope/hstcal/blob/main/pkg/acs/calacs/acscte/dopcte-gen2.c#L418 + """ + + array_electrons = layout_util.rotate_array_via_roe_corner_from( + array=array_electrons, roe_corner=(1, 1) + ) + + if bias_subtract_via_prescan: + bias_serial_prescan_value = acs_util.prescan_fitted_bias_column( + array_electrons[:, 18:24] + ) + + array_electrons -= bias_serial_prescan_value + + header.bias_serial_prescan_column = bias_serial_prescan_value + + if bias is not None: + bias = layout_util.rotate_array_via_roe_corner_from( + array=bias, roe_corner=(1, 1) + ) + + array_electrons -= bias + + header.bias = Array2DACS.no_mask(values=bias, pixel_scales=0.05) + + return cls.no_mask(values=array_electrons, header=header, pixel_scales=0.05) + + def update_fits(self, original_file_path, new_file_path): + """ + Output the array to a .fits file. + + Parameters + ---------- + file_path + The path the file is output to, including the filename and the ``.fits`` extension, + e.g. '/path/to/filename.fits' + """ + + new_file_dir = os.path.split(new_file_path)[0] + + if not os.path.exists(new_file_dir): + os.makedirs(new_file_dir) + + if not os.path.exists(new_file_path): + shutil.copy(original_file_path, new_file_path) + + hdulist = fits.open(new_file_path) + + hdulist[self.header.hdu].data = self.layout_2d.original_orientation_from( + array=self + ) + + ext_header = hdulist[4].header + bscale = ext_header["BSCALE"] + + os.remove(new_file_path) + + hdulist.writeto(new_file_path) diff --git a/autocti/instruments/acs/image.py b/autocti/instruments/acs/image.py index 72023819..9eec7a91 100644 --- a/autocti/instruments/acs/image.py +++ b/autocti/instruments/acs/image.py @@ -1,132 +1,133 @@ -import logging -import os -from os import path - -from autoarray import exc -from autoarray.structures.arrays import array_2d_util - -from autocti.instruments.acs.array_2d import Array2DACS -from autocti.instruments.acs.header import HeaderACS - -from autocti.instruments.acs import acs_util - -logging.basicConfig() -logger = logging.getLogger() -logger.setLevel("INFO") - - -class ImageACS(Array2DACS): - """ - The layout of an ACS array and image is given in `FrameACS`. - - This class handles specifically the image of an ACS observation, assuming that it contains specific - header info. - """ - - @classmethod - def from_fits( - cls, - file_path, - quadrant_letter, - bias_subtract_via_bias_file=False, - bias_subtract_via_prescan=False, - bias_file_path=None, - use_calibrated_gain=True, - ): - """ - Use the input .fits file and quadrant letter to extract the quadrant from the full CCD, perform - the rotations required to give correct arctic clocking and convert the image from units of COUNTS / CPS to - ELECTRONS. - - See the docstring of the `FrameACS` class for a complete description of the HST FPA, quadrants and - rotations. - - Also see https://github.com/spacetelescope/hstcal/blob/main/pkg/acs/calacs/acscte/dopcte-gen2.c#L418 - - Parameters - ---------- - file_path - The full path of the file that the image is loaded from, including the file name and ``.fits`` extension. - quadrant_letter - The letter of the ACS quadrant the image is extracted from and loaded. - bias_subtract_via_bias_file - If True, the corresponding bias file of the image is loaded (via the name of the file in the fits header). - bias_subtract_via_prescan - If True, the prescan on the image is used to estimate a component of bias that is subtracted from the image. - bias_file_path - If `bias_subtract_via_bias_file=True`, this overwrites the path to the bias file instead of the default - behaviour of using the .fits header. - use_calibrated_gain - If True, the calibrated gain values are used to convert from COUNTS to ELECTRONS. - """ - - hdu = acs_util.fits_hdu_via_quadrant_letter_from( - quadrant_letter=quadrant_letter - ) - - header_sci_obj = array_2d_util.header_obj_from(file_path=file_path, hdu=0) - header_hdu_obj = array_2d_util.header_obj_from(file_path=file_path, hdu=hdu) - - header = HeaderACS( - header_sci_obj=header_sci_obj, - header_hdu_obj=header_hdu_obj, - hdu=hdu, - quadrant_letter=quadrant_letter, - ) - - if header.header_sci_obj["TELESCOP"] != "HST": - raise exc.ArrayException( - f"The file {file_path} does not point to a valid HST ACS dataset." - ) - - if header.header_sci_obj["INSTRUME"] != "ACS": - raise exc.ArrayException( - f"The file {file_path} does not point to a valid HST ACS dataset." - ) - - array = array_2d_util.numpy_array_2d_via_fits_from( - file_path=file_path, hdu=hdu, do_not_scale_image_data=True - ) - - array = header.array_original_to_electrons( - array=array, use_calibrated_gain=use_calibrated_gain - ) - - if bias_subtract_via_bias_file: - if bias_file_path is None: - file_dir = os.path.split(file_path)[0] - bias_file_path = path.join(file_dir, header.bias_file) - - bias = array_2d_util.numpy_array_2d_via_fits_from( - file_path=bias_file_path, hdu=hdu, do_not_scale_image_data=True - ) - - header_sci_obj = array_2d_util.header_obj_from( - file_path=bias_file_path, hdu=0 - ) - header_hdu_obj = array_2d_util.header_obj_from( - file_path=bias_file_path, hdu=hdu - ) - - bias_header = HeaderACS( - header_sci_obj=header_sci_obj, - header_hdu_obj=header_hdu_obj, - hdu=hdu, - quadrant_letter=quadrant_letter, - ) - - if bias_header.original_units != "COUNTS": - raise exc.ArrayException("Cannot use bias frame not in counts.") - - bias = bias * bias_header.calibrated_gain - - else: - bias = None - - return cls.from_ccd( - array_electrons=array, - quadrant_letter=quadrant_letter, - header=header, - bias_subtract_via_prescan=bias_subtract_via_prescan, - bias=bias, - ) +import logging +import os +from os import path + +from autoarray import exc +from autoarray.structures.arrays import array_2d_util + +from autocti.instruments.acs.array_2d import Array2DACS +from autocti.instruments.acs.header import HeaderACS + +from autocti.instruments.acs import acs_util +from autoconf import fitsable + +logging.basicConfig() +logger = logging.getLogger() +logger.setLevel("INFO") + + +class ImageACS(Array2DACS): + """ + The layout of an ACS array and image is given in `FrameACS`. + + This class handles specifically the image of an ACS observation, assuming that it contains specific + header info. + """ + + @classmethod + def from_fits( + cls, + file_path, + quadrant_letter, + bias_subtract_via_bias_file=False, + bias_subtract_via_prescan=False, + bias_file_path=None, + use_calibrated_gain=True, + ): + """ + Use the input .fits file and quadrant letter to extract the quadrant from the full CCD, perform + the rotations required to give correct arctic clocking and convert the image from units of COUNTS / CPS to + ELECTRONS. + + See the docstring of the `FrameACS` class for a complete description of the HST FPA, quadrants and + rotations. + + Also see https://github.com/spacetelescope/hstcal/blob/main/pkg/acs/calacs/acscte/dopcte-gen2.c#L418 + + Parameters + ---------- + file_path + The full path of the file that the image is loaded from, including the file name and ``.fits`` extension. + quadrant_letter + The letter of the ACS quadrant the image is extracted from and loaded. + bias_subtract_via_bias_file + If True, the corresponding bias file of the image is loaded (via the name of the file in the fits header). + bias_subtract_via_prescan + If True, the prescan on the image is used to estimate a component of bias that is subtracted from the image. + bias_file_path + If `bias_subtract_via_bias_file=True`, this overwrites the path to the bias file instead of the default + behaviour of using the .fits header. + use_calibrated_gain + If True, the calibrated gain values are used to convert from COUNTS to ELECTRONS. + """ + + hdu = acs_util.fits_hdu_via_quadrant_letter_from( + quadrant_letter=quadrant_letter + ) + + header_sci_obj = fitsable.header_obj_from(file_path=file_path, hdu=0) + header_hdu_obj = fitsable.header_obj_from(file_path=file_path, hdu=hdu) + + header = HeaderACS( + header_sci_obj=header_sci_obj, + header_hdu_obj=header_hdu_obj, + hdu=hdu, + quadrant_letter=quadrant_letter, + ) + + if header.header_sci_obj["TELESCOP"] != "HST": + raise exc.ArrayException( + f"The file {file_path} does not point to a valid HST ACS dataset." + ) + + if header.header_sci_obj["INSTRUME"] != "ACS": + raise exc.ArrayException( + f"The file {file_path} does not point to a valid HST ACS dataset." + ) + + array = fitsable.ndarray_via_fits_from( + file_path=file_path, hdu=hdu, do_not_scale_image_data=True + ) + + array = header.array_original_to_electrons( + array=array, use_calibrated_gain=use_calibrated_gain + ) + + if bias_subtract_via_bias_file: + if bias_file_path is None: + file_dir = os.path.split(file_path)[0] + bias_file_path = path.join(file_dir, header.bias_file) + + bias = fitsable.ndarray_via_fits_from( + file_path=bias_file_path, hdu=hdu, do_not_scale_image_data=True + ) + + header_sci_obj = fitsable.header_obj_from( + file_path=bias_file_path, hdu=0 + ) + header_hdu_obj = fitsable.header_obj_from( + file_path=bias_file_path, hdu=hdu + ) + + bias_header = HeaderACS( + header_sci_obj=header_sci_obj, + header_hdu_obj=header_hdu_obj, + hdu=hdu, + quadrant_letter=quadrant_letter, + ) + + if bias_header.original_units != "COUNTS": + raise exc.ArrayException("Cannot use bias frame not in counts.") + + bias = bias * bias_header.calibrated_gain + + else: + bias = None + + return cls.from_ccd( + array_electrons=array, + quadrant_letter=quadrant_letter, + header=header, + bias_subtract_via_prescan=bias_subtract_via_prescan, + bias=bias, + ) diff --git a/autocti/mask/mask_2d.py b/autocti/mask/mask_2d.py index f77ee082..eb2804d0 100644 --- a/autocti/mask/mask_2d.py +++ b/autocti/mask/mask_2d.py @@ -1,427 +1,428 @@ -import numpy as np -from typing import List, Tuple - -import autoarray as aa - -from autoarray import exc - -from autocti.extract.settings import SettingsExtract -from autocti.layout.two_d import Layout2D - - -class SettingsMask2D: - def __init__( - self, - parallel_fpr_pixels: Tuple[int, int] = None, - parallel_eper_pixels: Tuple[int, int] = None, - serial_fpr_pixels: Tuple[int, int] = None, - serial_eper_pixels: Tuple[int, int] = None, - cosmic_ray_parallel_buffer: int = 10, - cosmic_ray_serial_buffer: int = 10, - cosmic_ray_diagonal_buffer: int = 3, - readout_persistence_infront_buffer: int = 0, - readout_persistence_behind_buffer: int = 0, - ): - """ - Settings which customize how the mask is created. - - There are three features whose masking can be customized: - - 1) The FPR / EPER masking: the extent of masks on these specific regions of the data (e.g. the - length of the FPR mask in pixels). - - 2) Cosmic ray masking: buffers around cosmic rays in the parallel and serial directions which mask the CTI - trails of these cosmic rays. - - 3) Read noise persistence masking: buffers around read noise persistence rows in front and behind the - flagged rows containing read noise persistence. - - Parameters - ---------- - parallel_fpr_pixels - The integer range of pixels masked in each parallel FPR region, for example `parallel_fpr_pixels=(1,2)` - masks just the second row of parallel FPR pixels. - parallel_eper_pixels - The integer range of pixels masked in each parallel EPER region, for example `parallel_eper_pixels=(1,2)` - masks just the second row of parallel EPER pixels. - serial_fpr_pixels - The integer range of pixels masked in each serial FPR region, for example `serial_fpr_pixels=(1,2)` - masks just the second column of serial FPR pixels. - serial_eper_pixels - The integer range of pixels masked in each serial EPER region, for example `serial_eper_pixels=(1,2)` - masks just the second column of serial EPER pixels. - cosmic_ray_parallel_buffer - The number of pixels masked in the parallel direction behind each cosmic ray, to mask the CTI trail. - cosmic_ray_serial_buffer - The number of pixels masked in the serial direction behind each cosmic ray, to mask the CTI trail. - cosmic_ray_diagonal_buffer - The number of pixels masked in the parallel and serial direction behind each cosmic ray, to mask the - serial CTI trail or the parallel CTI trail. - readout_persistence_infront_buffer - The number of rows masked in front of each read noise persistence region. - readout_persistence_behind_buffer - The number of rows masked behind each read noise persistence region. - """ - self.parallel_fpr_pixels = parallel_fpr_pixels - self.parallel_eper_pixels = parallel_eper_pixels - self.serial_fpr_pixels = serial_fpr_pixels - self.serial_eper_pixels = serial_eper_pixels - - self.cosmic_ray_parallel_buffer = cosmic_ray_parallel_buffer - self.cosmic_ray_serial_buffer = cosmic_ray_serial_buffer - self.cosmic_ray_diagonal_buffer = cosmic_ray_diagonal_buffer - - self.readout_persistence_infront_buffer = readout_persistence_infront_buffer - self.readout_persistence_behind_buffer = readout_persistence_behind_buffer - - -class Mask2D(aa.Mask2D): - @classmethod - def manual(cls, mask, pixel_scales, origin=(0.0, 0.0), invert=False): - """ - Create a Mask2D (see *Mask2D.__new__*) by inputting the array values in 2D, for example: - - mask=np.array([[False, False], - [True, False]]) - - mask=[[False, False], - [True, False]] - - Parameters - ---------- - mask - The bool values of the mask input as an ndarray of shape [total_y_pixels, total_x_pixels ]or a list of - lists. - pixel_scales - The (y,x) arcsecond-to-pixel units conversion factor of every pixel. If this is input as a `float`, - it is converted to a (float, float). - origin : (float, float) - The (y,x) scaled units origin of the mask's coordinate system. - invert - If `True`, the ``bool``'s of the input ``mask`` are inverted, for example `False`'s become `True` - and visa versa. - """ - if type(mask) is list: - mask = np.asarray(mask).astype("bool") - - if invert: - mask = np.invert(mask) - - pixel_scales = aa.util.geometry.convert_pixel_scales_2d( - pixel_scales=pixel_scales - ) - - if len(mask.shape) != 2: - raise exc.MaskException("The input mask is not a two dimensional array") - - return cls(mask=mask, pixel_scales=pixel_scales, origin=origin) - - @classmethod - def all_false(cls, shape_native, pixel_scales, origin=(0.0, 0.0), invert=False): - """Create a mask where all pixels are `False` and therefore unmasked. - - Parameters - ---------- - mask - The bool values of the mask input as an ndarray of shape [total_y_pixels, total_x_pixels ]or a list of - lists. - pixel_scales - The (y,x) arcsecond-to-pixel units conversion factor of every pixel. If this is input as a `float`, - it is converted to a (float, float). - origin : (float, float) - The (y,x) scaled units origin of the mask's coordinate system. - invert - If `True`, the ``bool``'s of the input ``mask`` are inverted, for example `False`'s become `True` - and visa versa. - """ - return cls.manual( - mask=np.full(shape=shape_native, fill_value=False), - pixel_scales=pixel_scales, - origin=origin, - invert=invert, - ) - - @classmethod - def from_masked_regions(cls, shape_native, pixel_scales, masked_regions): - mask = cls.all_false(shape_native=shape_native, pixel_scales=pixel_scales) - masked_regions = list( - map(lambda region: aa.Region2D(region=region), masked_regions) - ) - for region in masked_regions: - mask[region.y0 : region.y1, region.x0 : region.x1] = True - - return mask - - @classmethod - def from_cosmic_ray_map_buffed(cls, cosmic_ray_map, settings=SettingsMask2D()): - """ - Returns the mask used for CTI Calibration, which is all `False` unless specific regions are input for masking. - - Parameters - ---------- - cosmic_ray_map : array_2d.Array2D - 2D arrays flagging where cosmic rays on the image. - cosmic_ray_parallel_buffer - The number of pixels from each ray pixels are masked in the parallel direction. - cosmic_ray_serial_buffer - The number of pixels from each ray pixels are masked in the serial direction. - cosmic_ray_diagonal_buffer - The number of pixels from each ray pixels are masked in the digonal up from the parallel + serial direction. - """ - mask = cls.all_false( - shape_native=cosmic_ray_map.shape_native, - pixel_scales=cosmic_ray_map.pixel_scales, - ) - - cosmic_ray_mask = (cosmic_ray_map.native > 0.0).astype("bool") - - for y in range(mask.shape[0]): - for x in range(mask.shape[1]): - if cosmic_ray_mask[y, x]: - x0 = int(x) - - y0 = y - y1 = y + 1 + settings.cosmic_ray_parallel_buffer - - y1 = mask.shape[0] if y1 > mask.shape[0] else y1 - - mask[y0:y1, x] = True - - x1 = int(x + 1 + settings.cosmic_ray_serial_buffer) - - x1 = mask.shape[1] if x1 > mask.shape[1] else x1 - - mask[y, x0:x1] = True - - y0 = y - y1 = y + 1 + settings.cosmic_ray_diagonal_buffer - x1 = int(x + 1 + settings.cosmic_ray_diagonal_buffer) - - y1 = mask.shape[0] if y1 > mask.shape[0] else y1 - x1 = mask.shape[1] if x1 > mask.shape[1] else x1 - - mask[y0:y1, x0:x1] = True - - return mask - - @classmethod - def from_fits( - cls, file_path, pixel_scales, hdu=0, origin=(0.0, 0.0), resized_mask_shape=None - ): - """ - Loads the image from a .fits file. - - Parameters - ---------- - file_path - The full path of the fits file. - hdu - The HDU number in the fits file containing the image image. - pixel_scales or (float, float) - The arc-second to pixel conversion factor of each pixel. - """ - - if type(pixel_scales) is not tuple: - if type(pixel_scales) is float or int: - pixel_scales = (float(pixel_scales), float(pixel_scales)) - - mask = cls.manual( - mask=aa.util.array_2d.numpy_array_2d_via_fits_from( - file_path=file_path, hdu=hdu - ), - pixel_scales=pixel_scales, - origin=origin, - ) - - if resized_mask_shape is not None: - mask = mask.derive_mask.resized_from(new_shape=resized_mask_shape) - - return mask - - @classmethod - def masked_fpr_and_eper_from( - cls, - mask: "Mask2D", - layout: Layout2D, - settings: "SettingsMask2D", - pixel_scales: aa.type.PixelScales, - ) -> "Mask2D": - if settings.parallel_fpr_pixels is not None: - parallel_fpr_mask = cls.masked_parallel_fpr_from( - layout=layout, settings=settings, pixel_scales=pixel_scales - ) - - mask = mask + parallel_fpr_mask - - if settings.parallel_eper_pixels is not None: - parallel_eper_mask = cls.masked_parallel_eper_from( - layout=layout, settings=settings, pixel_scales=pixel_scales - ) - - mask = mask + parallel_eper_mask - - if settings.serial_fpr_pixels is not None: - serial_fpr_mask = cls.masked_serial_fpr_from( - layout=layout, settings=settings, pixel_scales=pixel_scales - ) - - mask = mask + serial_fpr_mask - - if settings.serial_eper_pixels is not None: - serial_eper_mask = cls.masked_serial_eper_from( - layout=layout, settings=settings, pixel_scales=pixel_scales - ) - - mask = mask + serial_eper_mask - - return mask - - @classmethod - def masked_parallel_fpr_from( - cls, - layout: Layout2D, - settings: "SettingsMask2D", - pixel_scales: aa.type.PixelScales, - invert: bool = False, - ) -> "Mask2D": - fpr_regions = layout.extract.parallel_fpr.region_list_from( - settings=SettingsExtract(pixels=settings.parallel_fpr_pixels) - ) - mask = np.full(layout.shape_2d, False) - - for region in fpr_regions: - mask[region.y0 : region.y1, region.x0 : region.x1] = True - - if invert: - mask = np.invert(mask) - - return Mask2D(mask=mask.astype("bool"), pixel_scales=pixel_scales) - - @classmethod - def masked_parallel_eper_from( - cls, - layout: Layout2D, - settings: "SettingsMask2D", - pixel_scales: aa.type.PixelScales, - invert: bool = False, - ) -> "Mask2D": - eper_regions = layout.extract.parallel_eper.region_list_from( - settings=SettingsExtract(pixels=settings.parallel_eper_pixels) - ) - - mask = np.full(layout.shape_2d, False) - - for region in eper_regions: - mask[region.y0 : region.y1, region.x0 : region.x1] = True - - if invert: - mask = np.invert(mask) - - return Mask2D(mask=mask.astype("bool"), pixel_scales=pixel_scales) - - @classmethod - def masked_serial_fpr_from( - cls, - layout: Layout2D, - settings: "SettingsMask2D", - pixel_scales: aa.type.PixelScales, - invert: bool = False, - ) -> "Mask2D": - fpr_regions = layout.extract.serial_fpr.region_list_from( - settings=SettingsExtract(pixels=settings.serial_fpr_pixels) - ) - mask = np.full(layout.shape_2d, False) - - for region in fpr_regions: - mask[region.y0 : region.y1, region.x0 : region.x1] = True - - if invert: - mask = np.invert(mask) - - return Mask2D(mask=mask.astype("bool"), pixel_scales=pixel_scales) - - @classmethod - def masked_serial_eper_from( - cls, - layout: Layout2D, - settings: "SettingsMask2D", - pixel_scales: aa.type.PixelScales, - invert: bool = False, - ) -> "Mask2D": - eper_regions = layout.extract.serial_eper.region_list_from( - settings=SettingsExtract(pixels=settings.serial_eper_pixels) - ) - mask = np.full(layout.shape_2d, False) - - for region in eper_regions: - mask[region.y0 : region.y1, region.x0 : region.x1] = True - - if invert: - mask = np.invert(mask) - - return Mask2D(mask=mask.astype("bool"), pixel_scales=pixel_scales) - - @classmethod - def masked_readout_persistence_from( - cls, - layout: Layout2D, - row_value_list: List[float], - readout_persistence_threshold: float, - settings: "SettingsMask2D", - pixel_scales: aa.type.PixelScales, - invert: bool = False, - ) -> "Mask2D": - """ - Read noise persistence is a feature of CCDs whereby the signal from a high signal pixel (e.g. cosmic ray) can - persist into the signal of subsequent rows of pixels. - - This leads to a 'streak' of signal values in the x direction, which typically need to be masked out. - - This function produces a read noise persistence mask from a list of row values, where the values are the - average signal in each row of the image after other features (e.g. the charge injection) have been removed. - - All rows with a signal above an input `readout_persistence_threshold` are masked out, where this threshold - should be estimated from the data itself or based on the CCD's properties. - - Parameters - ---------- - layout - The layout of the CCD (where the parallel overscan begins and ends, where the charge injection - regions are, etc.). - row_value_list - The average signal in each row of the image after other features (e.g. the charge injection) have been - removed. - readout_persistence_threshold - The threshold above which a row is masked out, assuming that this threshold means that a signal is - so bright that it must be due to read noise persistence. - settings - The settings of the mask (e.g. the number of pixels to mask out). - pixel_scales - The pixel scales of the CCD in arc-seconds per pixel, which is passed to the mask. - invert - If `True`, the mask is inverted such that all pixels that are masked are unmasked and visa versa. - - Returns - ------- - The read noise persistence mask. - """ - mask_row = [ - row_value > readout_persistence_threshold for row_value in row_value_list - ] - - mask = np.full(layout.shape_2d, False) - - for y in range(layout.shape_2d[0]): - if mask_row[y]: - ylow = max(y - settings.readout_persistence_infront_buffer, 0) - yhigh = min( - y + settings.readout_persistence_behind_buffer + 1, - layout.shape_2d[0], - ) - - mask[ylow:yhigh, :] = True - - if invert: - mask = np.invert(mask) - - return Mask2D(mask=mask.astype("bool"), pixel_scales=pixel_scales) +import numpy as np +from typing import List, Tuple + +import autoarray as aa +from autoconf import fitsable + +from autoarray import exc + +from autocti.extract.settings import SettingsExtract +from autocti.layout.two_d import Layout2D + + +class SettingsMask2D: + def __init__( + self, + parallel_fpr_pixels: Tuple[int, int] = None, + parallel_eper_pixels: Tuple[int, int] = None, + serial_fpr_pixels: Tuple[int, int] = None, + serial_eper_pixels: Tuple[int, int] = None, + cosmic_ray_parallel_buffer: int = 10, + cosmic_ray_serial_buffer: int = 10, + cosmic_ray_diagonal_buffer: int = 3, + readout_persistence_infront_buffer: int = 0, + readout_persistence_behind_buffer: int = 0, + ): + """ + Settings which customize how the mask is created. + + There are three features whose masking can be customized: + + 1) The FPR / EPER masking: the extent of masks on these specific regions of the data (e.g. the + length of the FPR mask in pixels). + + 2) Cosmic ray masking: buffers around cosmic rays in the parallel and serial directions which mask the CTI + trails of these cosmic rays. + + 3) Read noise persistence masking: buffers around read noise persistence rows in front and behind the + flagged rows containing read noise persistence. + + Parameters + ---------- + parallel_fpr_pixels + The integer range of pixels masked in each parallel FPR region, for example `parallel_fpr_pixels=(1,2)` + masks just the second row of parallel FPR pixels. + parallel_eper_pixels + The integer range of pixels masked in each parallel EPER region, for example `parallel_eper_pixels=(1,2)` + masks just the second row of parallel EPER pixels. + serial_fpr_pixels + The integer range of pixels masked in each serial FPR region, for example `serial_fpr_pixels=(1,2)` + masks just the second column of serial FPR pixels. + serial_eper_pixels + The integer range of pixels masked in each serial EPER region, for example `serial_eper_pixels=(1,2)` + masks just the second column of serial EPER pixels. + cosmic_ray_parallel_buffer + The number of pixels masked in the parallel direction behind each cosmic ray, to mask the CTI trail. + cosmic_ray_serial_buffer + The number of pixels masked in the serial direction behind each cosmic ray, to mask the CTI trail. + cosmic_ray_diagonal_buffer + The number of pixels masked in the parallel and serial direction behind each cosmic ray, to mask the + serial CTI trail or the parallel CTI trail. + readout_persistence_infront_buffer + The number of rows masked in front of each read noise persistence region. + readout_persistence_behind_buffer + The number of rows masked behind each read noise persistence region. + """ + self.parallel_fpr_pixels = parallel_fpr_pixels + self.parallel_eper_pixels = parallel_eper_pixels + self.serial_fpr_pixels = serial_fpr_pixels + self.serial_eper_pixels = serial_eper_pixels + + self.cosmic_ray_parallel_buffer = cosmic_ray_parallel_buffer + self.cosmic_ray_serial_buffer = cosmic_ray_serial_buffer + self.cosmic_ray_diagonal_buffer = cosmic_ray_diagonal_buffer + + self.readout_persistence_infront_buffer = readout_persistence_infront_buffer + self.readout_persistence_behind_buffer = readout_persistence_behind_buffer + + +class Mask2D(aa.Mask2D): + @classmethod + def manual(cls, mask, pixel_scales, origin=(0.0, 0.0), invert=False): + """ + Create a Mask2D (see *Mask2D.__new__*) by inputting the array values in 2D, for example: + + mask=np.array([[False, False], + [True, False]]) + + mask=[[False, False], + [True, False]] + + Parameters + ---------- + mask + The bool values of the mask input as an ndarray of shape [total_y_pixels, total_x_pixels ]or a list of + lists. + pixel_scales + The (y,x) arcsecond-to-pixel units conversion factor of every pixel. If this is input as a `float`, + it is converted to a (float, float). + origin : (float, float) + The (y,x) scaled units origin of the mask's coordinate system. + invert + If `True`, the ``bool``'s of the input ``mask`` are inverted, for example `False`'s become `True` + and visa versa. + """ + if type(mask) is list: + mask = np.asarray(mask).astype("bool") + + if invert: + mask = np.invert(mask) + + pixel_scales = aa.util.geometry.convert_pixel_scales_2d( + pixel_scales=pixel_scales + ) + + if len(mask.shape) != 2: + raise exc.MaskException("The input mask is not a two dimensional array") + + return cls(mask=mask, pixel_scales=pixel_scales, origin=origin) + + @classmethod + def all_false(cls, shape_native, pixel_scales, origin=(0.0, 0.0), invert=False): + """Create a mask where all pixels are `False` and therefore unmasked. + + Parameters + ---------- + mask + The bool values of the mask input as an ndarray of shape [total_y_pixels, total_x_pixels ]or a list of + lists. + pixel_scales + The (y,x) arcsecond-to-pixel units conversion factor of every pixel. If this is input as a `float`, + it is converted to a (float, float). + origin : (float, float) + The (y,x) scaled units origin of the mask's coordinate system. + invert + If `True`, the ``bool``'s of the input ``mask`` are inverted, for example `False`'s become `True` + and visa versa. + """ + return cls.manual( + mask=np.full(shape=shape_native, fill_value=False), + pixel_scales=pixel_scales, + origin=origin, + invert=invert, + ) + + @classmethod + def from_masked_regions(cls, shape_native, pixel_scales, masked_regions): + mask = cls.all_false(shape_native=shape_native, pixel_scales=pixel_scales) + masked_regions = list( + map(lambda region: aa.Region2D(region=region), masked_regions) + ) + for region in masked_regions: + mask[region.y0 : region.y1, region.x0 : region.x1] = True + + return mask + + @classmethod + def from_cosmic_ray_map_buffed(cls, cosmic_ray_map, settings=SettingsMask2D()): + """ + Returns the mask used for CTI Calibration, which is all `False` unless specific regions are input for masking. + + Parameters + ---------- + cosmic_ray_map : array_2d.Array2D + 2D arrays flagging where cosmic rays on the image. + cosmic_ray_parallel_buffer + The number of pixels from each ray pixels are masked in the parallel direction. + cosmic_ray_serial_buffer + The number of pixels from each ray pixels are masked in the serial direction. + cosmic_ray_diagonal_buffer + The number of pixels from each ray pixels are masked in the digonal up from the parallel + serial direction. + """ + mask = cls.all_false( + shape_native=cosmic_ray_map.shape_native, + pixel_scales=cosmic_ray_map.pixel_scales, + ) + + cosmic_ray_mask = (cosmic_ray_map.native > 0.0).astype("bool") + + for y in range(mask.shape[0]): + for x in range(mask.shape[1]): + if cosmic_ray_mask[y, x]: + x0 = int(x) + + y0 = y + y1 = y + 1 + settings.cosmic_ray_parallel_buffer + + y1 = mask.shape[0] if y1 > mask.shape[0] else y1 + + mask[y0:y1, x] = True + + x1 = int(x + 1 + settings.cosmic_ray_serial_buffer) + + x1 = mask.shape[1] if x1 > mask.shape[1] else x1 + + mask[y, x0:x1] = True + + y0 = y + y1 = y + 1 + settings.cosmic_ray_diagonal_buffer + x1 = int(x + 1 + settings.cosmic_ray_diagonal_buffer) + + y1 = mask.shape[0] if y1 > mask.shape[0] else y1 + x1 = mask.shape[1] if x1 > mask.shape[1] else x1 + + mask[y0:y1, x0:x1] = True + + return mask + + @classmethod + def from_fits( + cls, file_path, pixel_scales, hdu=0, origin=(0.0, 0.0), resized_mask_shape=None + ): + """ + Loads the image from a .fits file. + + Parameters + ---------- + file_path + The full path of the fits file. + hdu + The HDU number in the fits file containing the image image. + pixel_scales or (float, float) + The arc-second to pixel conversion factor of each pixel. + """ + + if type(pixel_scales) is not tuple: + if type(pixel_scales) is float or int: + pixel_scales = (float(pixel_scales), float(pixel_scales)) + + mask = cls.manual( + mask=fitsable.ndarray_via_fits_from( + file_path=file_path, hdu=hdu + ), + pixel_scales=pixel_scales, + origin=origin, + ) + + if resized_mask_shape is not None: + mask = mask.derive_mask.resized_from(new_shape=resized_mask_shape) + + return mask + + @classmethod + def masked_fpr_and_eper_from( + cls, + mask: "Mask2D", + layout: Layout2D, + settings: "SettingsMask2D", + pixel_scales: aa.type.PixelScales, + ) -> "Mask2D": + if settings.parallel_fpr_pixels is not None: + parallel_fpr_mask = cls.masked_parallel_fpr_from( + layout=layout, settings=settings, pixel_scales=pixel_scales + ) + + mask = mask + parallel_fpr_mask + + if settings.parallel_eper_pixels is not None: + parallel_eper_mask = cls.masked_parallel_eper_from( + layout=layout, settings=settings, pixel_scales=pixel_scales + ) + + mask = mask + parallel_eper_mask + + if settings.serial_fpr_pixels is not None: + serial_fpr_mask = cls.masked_serial_fpr_from( + layout=layout, settings=settings, pixel_scales=pixel_scales + ) + + mask = mask + serial_fpr_mask + + if settings.serial_eper_pixels is not None: + serial_eper_mask = cls.masked_serial_eper_from( + layout=layout, settings=settings, pixel_scales=pixel_scales + ) + + mask = mask + serial_eper_mask + + return mask + + @classmethod + def masked_parallel_fpr_from( + cls, + layout: Layout2D, + settings: "SettingsMask2D", + pixel_scales: aa.type.PixelScales, + invert: bool = False, + ) -> "Mask2D": + fpr_regions = layout.extract.parallel_fpr.region_list_from( + settings=SettingsExtract(pixels=settings.parallel_fpr_pixels) + ) + mask = np.full(layout.shape_2d, False) + + for region in fpr_regions: + mask[region.y0 : region.y1, region.x0 : region.x1] = True + + if invert: + mask = np.invert(mask) + + return Mask2D(mask=mask.astype("bool"), pixel_scales=pixel_scales) + + @classmethod + def masked_parallel_eper_from( + cls, + layout: Layout2D, + settings: "SettingsMask2D", + pixel_scales: aa.type.PixelScales, + invert: bool = False, + ) -> "Mask2D": + eper_regions = layout.extract.parallel_eper.region_list_from( + settings=SettingsExtract(pixels=settings.parallel_eper_pixels) + ) + + mask = np.full(layout.shape_2d, False) + + for region in eper_regions: + mask[region.y0 : region.y1, region.x0 : region.x1] = True + + if invert: + mask = np.invert(mask) + + return Mask2D(mask=mask.astype("bool"), pixel_scales=pixel_scales) + + @classmethod + def masked_serial_fpr_from( + cls, + layout: Layout2D, + settings: "SettingsMask2D", + pixel_scales: aa.type.PixelScales, + invert: bool = False, + ) -> "Mask2D": + fpr_regions = layout.extract.serial_fpr.region_list_from( + settings=SettingsExtract(pixels=settings.serial_fpr_pixels) + ) + mask = np.full(layout.shape_2d, False) + + for region in fpr_regions: + mask[region.y0 : region.y1, region.x0 : region.x1] = True + + if invert: + mask = np.invert(mask) + + return Mask2D(mask=mask.astype("bool"), pixel_scales=pixel_scales) + + @classmethod + def masked_serial_eper_from( + cls, + layout: Layout2D, + settings: "SettingsMask2D", + pixel_scales: aa.type.PixelScales, + invert: bool = False, + ) -> "Mask2D": + eper_regions = layout.extract.serial_eper.region_list_from( + settings=SettingsExtract(pixels=settings.serial_eper_pixels) + ) + mask = np.full(layout.shape_2d, False) + + for region in eper_regions: + mask[region.y0 : region.y1, region.x0 : region.x1] = True + + if invert: + mask = np.invert(mask) + + return Mask2D(mask=mask.astype("bool"), pixel_scales=pixel_scales) + + @classmethod + def masked_readout_persistence_from( + cls, + layout: Layout2D, + row_value_list: List[float], + readout_persistence_threshold: float, + settings: "SettingsMask2D", + pixel_scales: aa.type.PixelScales, + invert: bool = False, + ) -> "Mask2D": + """ + Read noise persistence is a feature of CCDs whereby the signal from a high signal pixel (e.g. cosmic ray) can + persist into the signal of subsequent rows of pixels. + + This leads to a 'streak' of signal values in the x direction, which typically need to be masked out. + + This function produces a read noise persistence mask from a list of row values, where the values are the + average signal in each row of the image after other features (e.g. the charge injection) have been removed. + + All rows with a signal above an input `readout_persistence_threshold` are masked out, where this threshold + should be estimated from the data itself or based on the CCD's properties. + + Parameters + ---------- + layout + The layout of the CCD (where the parallel overscan begins and ends, where the charge injection + regions are, etc.). + row_value_list + The average signal in each row of the image after other features (e.g. the charge injection) have been + removed. + readout_persistence_threshold + The threshold above which a row is masked out, assuming that this threshold means that a signal is + so bright that it must be due to read noise persistence. + settings + The settings of the mask (e.g. the number of pixels to mask out). + pixel_scales + The pixel scales of the CCD in arc-seconds per pixel, which is passed to the mask. + invert + If `True`, the mask is inverted such that all pixels that are masked are unmasked and visa versa. + + Returns + ------- + The read noise persistence mask. + """ + mask_row = [ + row_value > readout_persistence_threshold for row_value in row_value_list + ] + + mask = np.full(layout.shape_2d, False) + + for y in range(layout.shape_2d[0]): + if mask_row[y]: + ylow = max(y - settings.readout_persistence_infront_buffer, 0) + yhigh = min( + y + settings.readout_persistence_behind_buffer + 1, + layout.shape_2d[0], + ) + + mask[ylow:yhigh, :] = True + + if invert: + mask = np.invert(mask) + + return Mask2D(mask=mask.astype("bool"), pixel_scales=pixel_scales) diff --git a/optional_requirements.txt b/optional_requirements.txt deleted file mode 100644 index 94e37990..00000000 --- a/optional_requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -numba -pyyaml -arcticpy==2.6 -ultranest==3.6.2 - diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..deca1a48 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,74 @@ +[build-system] +requires = ["setuptools>=79.0", "setuptools-scm", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "autocti" +dynamic = ["version"] +description = "PyAutoCTI: Charge Transfer Inefficiency Modeling" +readme = { file = "README.rst", content-type = "text/x-rst" } +license = { text = "MIT" } +requires-python = ">=3.9" +authors = [ + { name = "James Nightingale", email = "James.Nightingale@newcastle.ac.uk" }, + { name = "Richard Massey", email = "r.j.massey@durham.ac.uk" }, + { name = "Jacob Kegerreis" }, + { name = "Richard Hayes", email = "richard@rghsoftware.co.uk" }, +] +classifiers = [ + "Intended Audience :: Science/Research", + "Topic :: Scientific/Engineering :: Physics", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13" +] +keywords = ["cli"] +# arcticpy (the C++ arctic clocking code) is a hard import of autocti but is +# deliberately NOT a pip dependency: its sdist is source-only (needs libgsl-dev +# and a C++ toolchain) and its own requirements downgrade numpy below 2.0. +# Install it separately with `pip install arcticpy==2.6 --no-build-isolation +# --no-deps` after installing numpy. +dependencies = [ + "autofit", + "autoarray", + "nautilus-sampler==1.0.5" +] + +[project.urls] +Homepage = "https://github.com/PyAutoLabs/PyAutoCTI" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +exclude = ["docs", "test_autocti", "test_autocti*"] + +[tool.setuptools_scm] +version_scheme = "post-release" +local_scheme = "no-local-version" + +[project.optional-dependencies] +optional = [ + "numba", +] +docs = [ + "sphinx", + "furo", + "myst-parser", + "sphinx_copybutton", + "sphinx_design", + "sphinx_inline_tabs", + "sphinx_autodoc_typehints" +] +test = ["pytest"] +dev = ["pytest", "black"] + +[tool.setuptools.package-data] +"autocti.config" = ["*"] + +[tool.pytest.ini_options] +testpaths = ["test_autocti"] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 4c240661..00000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -nautilus-sampler==1.0.4 \ No newline at end of file diff --git a/setup.py b/setup.py index 549d09d7..1c6dbc09 100644 --- a/setup.py +++ b/setup.py @@ -1,45 +1,8 @@ -from codecs import open -from os.path import abspath, dirname, join -from os import environ - -from setuptools import find_packages, setup - -this_dir = abspath(dirname(__file__)) -with open(join(this_dir, "README.rst"), encoding="utf-8") as file: - long_description = file.read() - -with open(join(this_dir, "requirements.txt")) as f: - requirements = f.read().split("\n") - -version = environ.get("VERSION", "1.0.dev0") -requirements.extend( - [f"autoconf=={version}", f"autofit=={version}", f"autoarray=={version}"] -) - -setup( - name="autocti", - version=version, - description="PyAutoCTI: Charge Transfer Inefficiency Modeling", - long_description=long_description, - long_description_content_type="text/x-rst", - url="https://github.com/jammy2211/PyAutoCTI", - author="James Nightingale, Richard Massey, Jacob Kegerreis and Richard Hayes", - author_email="james.w.nightingale@durham.ac.uk", - include_package_data=True, - license="MIT License", - classifiers=[ - "Intended Audience :: Science/Research", - "Topic :: Scientific/Engineering :: Physics", - "License :: OSI Approved :: MIT License", - "Natural Language :: English", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - ], - python_requires=">=3.7", - keywords="cli", - packages=find_packages(exclude=["docs", "test_autocti", "test_autocti*"]), - install_requires=requirements, - setup_requires=["pytest-runner"], - tests_require=["pytest"], -) +import os +from setuptools import setup + +version = os.environ.get("VERSION", "1.0.dev0") + +setup( + version=version, +) diff --git a/test_autocti/aggregator/test_aggregator_dataset_1d.py b/test_autocti/aggregator/test_aggregator_dataset_1d.py index 1e33f080..539d888f 100644 --- a/test_autocti/aggregator/test_aggregator_dataset_1d.py +++ b/test_autocti/aggregator/test_aggregator_dataset_1d.py @@ -1,93 +1,103 @@ -import copy -import pytest - -import autocti as ac - -from test_autocti.aggregator.conftest import clean, aggregator_from - -database_file = "db_dataset_1d" - - -def test__dataset_gen_from__analysis_has_single_dataset( - dataset_1d_7, clocker_1d, samples_1d, model_1d -): - analysis = ac.AnalysisDataset1D(dataset=dataset_1d_7, clocker=clocker_1d) - - agg = aggregator_from( - database_file=database_file, - analysis=analysis, - model=model_1d, - samples=samples_1d, - ) - - dataset_agg = ac.agg.Dataset1DAgg(aggregator=agg) - dataset_gen = dataset_agg.dataset_list_gen_from() - - for dataset in dataset_gen: - assert (dataset[0].data == dataset_1d_7.data).all() - assert dataset[0].layout.prescan[1] == pytest.approx( - dataset_1d_7.layout.prescan[1], 1.0e-4 - ) - - clean(database_file=database_file) - - -def test__dataset_gen_from__analysis_has_multi_dataset( - dataset_1d_7, clocker_1d, samples_1d, model_1d -): - analysis = ac.AnalysisDataset1D(dataset=dataset_1d_7, clocker=clocker_1d) - - agg = aggregator_from( - database_file=database_file, - analysis=analysis + analysis, - model=model_1d, - samples=samples_1d, - ) - - dataset_agg = ac.agg.Dataset1DAgg(aggregator=agg, use_dataset_full=False) - dataset_gen = dataset_agg.dataset_list_gen_from() - - for dataset_list in dataset_gen: - assert (dataset_list[0].data == dataset_1d_7.data).all() - assert (dataset_list[1].data == dataset_1d_7.data).all() - assert dataset_list[0].layout.prescan[1] == pytest.approx( - dataset_1d_7.layout.prescan[1], 1.0e-4 - ) - assert dataset_list[1].layout.prescan[1] == pytest.approx( - dataset_1d_7.layout.prescan[1], 1.0e-4 - ) - - clean(database_file=database_file) - - -def test__dataset_gen_from__analysis_use_dataset_full( - dataset_1d_7, clocker_1d, samples_1d, model_1d -): - dataset_1d_7_full = copy.copy(dataset_1d_7) - dataset_1d_7_full.data[0] = 100.0 - - analysis = ac.AnalysisDataset1D( - dataset=dataset_1d_7, clocker=clocker_1d, dataset_full=dataset_1d_7_full - ) - - agg = aggregator_from( - database_file=database_file, - analysis=analysis + analysis, - model=model_1d, - samples=samples_1d, - ) - - dataset_agg = ac.agg.Dataset1DAgg(aggregator=agg, use_dataset_full=True) - dataset_gen = dataset_agg.dataset_list_gen_from() - - for dataset_list in dataset_gen: - assert dataset_list[0].data[0] == pytest.approx(100.0, 1.0e-4) - assert dataset_list[1].data[0] == pytest.approx(100.0, 1.0e-4) - assert dataset_list[0].layout.prescan[1] == pytest.approx( - dataset_1d_7.layout.prescan[1], 1.0e-4 - ) - assert dataset_list[1].layout.prescan[1] == pytest.approx( - dataset_1d_7.layout.prescan[1], 1.0e-4 - ) - - clean(database_file=database_file) +import copy +import pytest + +import autocti as ac + +from test_autocti.aggregator.conftest import clean, aggregator_from + +database_file = "db_dataset_1d" + + +def test__dataset_gen_from__analysis_has_single_dataset( + dataset_1d_7, clocker_1d, samples_1d, model_1d +): + analysis = ac.AnalysisDataset1D(dataset=dataset_1d_7, clocker=clocker_1d) + + agg = aggregator_from( + database_file=database_file, + analysis=analysis, + model=model_1d, + samples=samples_1d, + ) + + dataset_agg = ac.agg.Dataset1DAgg(aggregator=agg) + dataset_gen = dataset_agg.dataset_list_gen_from() + + for dataset in dataset_gen: + assert (dataset[0].data == dataset_1d_7.data).all() + assert dataset[0].layout.prescan[1] == pytest.approx( + dataset_1d_7.layout.prescan[1], 1.0e-4 + ) + + clean(database_file=database_file) + + +@pytest.mark.skip( + reason="Analysis summing (analysis + analysis) was removed from PyAutoFit in favour " + "of AnalysisFactor/FactorGraphModel; these tests are ported in Phase 2 of the CTI " + "resurrection epic (PyAutoCTI#82)." +) +def test__dataset_gen_from__analysis_has_multi_dataset( + dataset_1d_7, clocker_1d, samples_1d, model_1d +): + analysis = ac.AnalysisDataset1D(dataset=dataset_1d_7, clocker=clocker_1d) + + agg = aggregator_from( + database_file=database_file, + analysis=analysis + analysis, + model=model_1d, + samples=samples_1d, + ) + + dataset_agg = ac.agg.Dataset1DAgg(aggregator=agg, use_dataset_full=False) + dataset_gen = dataset_agg.dataset_list_gen_from() + + for dataset_list in dataset_gen: + assert (dataset_list[0].data == dataset_1d_7.data).all() + assert (dataset_list[1].data == dataset_1d_7.data).all() + assert dataset_list[0].layout.prescan[1] == pytest.approx( + dataset_1d_7.layout.prescan[1], 1.0e-4 + ) + assert dataset_list[1].layout.prescan[1] == pytest.approx( + dataset_1d_7.layout.prescan[1], 1.0e-4 + ) + + clean(database_file=database_file) + + +@pytest.mark.skip( + reason="Analysis summing (analysis + analysis) was removed from PyAutoFit in favour " + "of AnalysisFactor/FactorGraphModel; these tests are ported in Phase 2 of the CTI " + "resurrection epic (PyAutoCTI#82)." +) +def test__dataset_gen_from__analysis_use_dataset_full( + dataset_1d_7, clocker_1d, samples_1d, model_1d +): + dataset_1d_7_full = copy.copy(dataset_1d_7) + dataset_1d_7_full.data[0] = 100.0 + + analysis = ac.AnalysisDataset1D( + dataset=dataset_1d_7, clocker=clocker_1d, dataset_full=dataset_1d_7_full + ) + + agg = aggregator_from( + database_file=database_file, + analysis=analysis + analysis, + model=model_1d, + samples=samples_1d, + ) + + dataset_agg = ac.agg.Dataset1DAgg(aggregator=agg, use_dataset_full=True) + dataset_gen = dataset_agg.dataset_list_gen_from() + + for dataset_list in dataset_gen: + assert dataset_list[0].data[0] == pytest.approx(100.0, 1.0e-4) + assert dataset_list[1].data[0] == pytest.approx(100.0, 1.0e-4) + assert dataset_list[0].layout.prescan[1] == pytest.approx( + dataset_1d_7.layout.prescan[1], 1.0e-4 + ) + assert dataset_list[1].layout.prescan[1] == pytest.approx( + dataset_1d_7.layout.prescan[1], 1.0e-4 + ) + + clean(database_file=database_file) diff --git a/test_autocti/aggregator/test_aggregator_fit_imaging_ci.py b/test_autocti/aggregator/test_aggregator_fit_imaging_ci.py index e447ef4e..5b46881a 100644 --- a/test_autocti/aggregator/test_aggregator_fit_imaging_ci.py +++ b/test_autocti/aggregator/test_aggregator_fit_imaging_ci.py @@ -1,90 +1,96 @@ -import autocti as ac - -from test_autocti.aggregator.conftest import clean, aggregator_from - -database_file = "db_fit_imaging_ci" - - -def test__fit_imaging_ci_randomly_drawn_via_pdf_gen_from( - imaging_ci_7x7, parallel_clocker_2d, samples_2d, model_2d -): - analysis = ac.AnalysisImagingCI(dataset=imaging_ci_7x7, clocker=parallel_clocker_2d) - - agg = aggregator_from( - database_file=database_file, - analysis=analysis, - model=model_2d, - samples=samples_2d, - ) - - fit_agg = ac.agg.FitImagingCIAgg(aggregator=agg) - fit_pdf_gen = fit_agg.randomly_drawn_via_pdf_gen_from(total_samples=2) - - i = 0 - - for fit_gen in fit_pdf_gen: - for fit_list in fit_gen: - i += 1 - - assert fit_list[0].post_cti_data[0] is not None - - assert i == 2 - - clean(database_file=database_file) - - -def test__fit_imaging_ci_randomly_drawn_via_pdf_gen_from__multi_analysis( - imaging_ci_7x7, parallel_clocker_2d, samples_2d, model_2d -): - analysis = ac.AnalysisImagingCI(dataset=imaging_ci_7x7, clocker=parallel_clocker_2d) - - agg = aggregator_from( - database_file=database_file, - analysis=analysis + analysis, - model=model_2d, - samples=samples_2d, - ) - - fit_agg = ac.agg.FitImagingCIAgg(aggregator=agg) - fit_pdf_gen = fit_agg.randomly_drawn_via_pdf_gen_from(total_samples=2) - - i = 0 - - for fit_gen in fit_pdf_gen: - for fit_list in fit_gen: - i += 1 - - assert fit_list[0].post_cti_data[0] is not None - assert fit_list[1].post_cti_data[0] is not None - - assert i == 2 - - clean(database_file=database_file) - - -def test__fit_imaging_ci_all_above_weight_gen( - imaging_ci_7x7, parallel_clocker_2d, samples_2d, model_2d -): - analysis = ac.AnalysisImagingCI(dataset=imaging_ci_7x7, clocker=parallel_clocker_2d) - - agg = aggregator_from( - database_file=database_file, - analysis=analysis, - model=model_2d, - samples=samples_2d, - ) - - fit_agg = ac.agg.FitImagingCIAgg(aggregator=agg) - fit_pdf_gen = fit_agg.all_above_weight_gen_from(minimum_weight=-1.0) - - i = 0 - - for fit_gen in fit_pdf_gen: - for fit_list in fit_gen: - i += 1 - - assert fit_list[0].post_cti_data[0] is not None - - assert i == 2 - - clean(database_file=database_file) +import pytest +import autocti as ac + +from test_autocti.aggregator.conftest import clean, aggregator_from + +database_file = "db_fit_imaging_ci" + + +def test__fit_imaging_ci_randomly_drawn_via_pdf_gen_from( + imaging_ci_7x7, parallel_clocker_2d, samples_2d, model_2d +): + analysis = ac.AnalysisImagingCI(dataset=imaging_ci_7x7, clocker=parallel_clocker_2d) + + agg = aggregator_from( + database_file=database_file, + analysis=analysis, + model=model_2d, + samples=samples_2d, + ) + + fit_agg = ac.agg.FitImagingCIAgg(aggregator=agg) + fit_pdf_gen = fit_agg.randomly_drawn_via_pdf_gen_from(total_samples=2) + + i = 0 + + for fit_gen in fit_pdf_gen: + for fit_list in fit_gen: + i += 1 + + assert fit_list[0].post_cti_data[0] is not None + + assert i == 2 + + clean(database_file=database_file) + + +@pytest.mark.skip( + reason="Analysis summing (analysis + analysis) was removed from PyAutoFit in favour " + "of AnalysisFactor/FactorGraphModel; these tests are ported in Phase 2 of the CTI " + "resurrection epic (PyAutoCTI#82)." +) +def test__fit_imaging_ci_randomly_drawn_via_pdf_gen_from__multi_analysis( + imaging_ci_7x7, parallel_clocker_2d, samples_2d, model_2d +): + analysis = ac.AnalysisImagingCI(dataset=imaging_ci_7x7, clocker=parallel_clocker_2d) + + agg = aggregator_from( + database_file=database_file, + analysis=analysis + analysis, + model=model_2d, + samples=samples_2d, + ) + + fit_agg = ac.agg.FitImagingCIAgg(aggregator=agg) + fit_pdf_gen = fit_agg.randomly_drawn_via_pdf_gen_from(total_samples=2) + + i = 0 + + for fit_gen in fit_pdf_gen: + for fit_list in fit_gen: + i += 1 + + assert fit_list[0].post_cti_data[0] is not None + assert fit_list[1].post_cti_data[0] is not None + + assert i == 2 + + clean(database_file=database_file) + + +def test__fit_imaging_ci_all_above_weight_gen( + imaging_ci_7x7, parallel_clocker_2d, samples_2d, model_2d +): + analysis = ac.AnalysisImagingCI(dataset=imaging_ci_7x7, clocker=parallel_clocker_2d) + + agg = aggregator_from( + database_file=database_file, + analysis=analysis, + model=model_2d, + samples=samples_2d, + ) + + fit_agg = ac.agg.FitImagingCIAgg(aggregator=agg) + fit_pdf_gen = fit_agg.all_above_weight_gen_from(minimum_weight=-1.0) + + i = 0 + + for fit_gen in fit_pdf_gen: + for fit_list in fit_gen: + i += 1 + + assert fit_list[0].post_cti_data[0] is not None + + assert i == 2 + + clean(database_file=database_file) diff --git a/test_autocti/aggregator/test_aggregator_imaging_ci.py b/test_autocti/aggregator/test_aggregator_imaging_ci.py index 1e330e96..8c8454c1 100644 --- a/test_autocti/aggregator/test_aggregator_imaging_ci.py +++ b/test_autocti/aggregator/test_aggregator_imaging_ci.py @@ -1,100 +1,110 @@ -import copy -import pytest - -import autocti as ac - -from test_autocti.aggregator.conftest import clean, aggregator_from - -database_file = "db_imaging_ci" - - -def test__dataset_gen_from__analysis_has_single_dataset( - imaging_ci_7x7, parallel_clocker_2d, samples_2d, model_2d -): - analysis = ac.AnalysisImagingCI(dataset=imaging_ci_7x7, clocker=parallel_clocker_2d) - - agg = aggregator_from( - database_file=database_file, - analysis=analysis, - model=model_2d, - samples=samples_2d, - ) - - dataset_agg = ac.agg.ImagingCIAgg(aggregator=agg) - dataset_gen = dataset_agg.dataset_list_gen_from() - - for dataset_list in dataset_gen: - assert (dataset_list[0].data == imaging_ci_7x7.data).all() - assert dataset_list[0].layout.parallel_overscan[1] == pytest.approx( - imaging_ci_7x7.layout.parallel_overscan[1], 1.0e-4 - ) - - clean(database_file=database_file) - - -def test__dataset_gen_from__analysis_has_multi_dataset( - imaging_ci_7x7, parallel_clocker_2d, samples_2d, model_2d -): - analysis = ac.AnalysisImagingCI(dataset=imaging_ci_7x7, clocker=parallel_clocker_2d) - - agg = aggregator_from( - database_file=database_file, - analysis=analysis + analysis, - model=model_2d, - samples=samples_2d, - ) - - dataset_agg = ac.agg.ImagingCIAgg(aggregator=agg) - dataset_gen = dataset_agg.dataset_list_gen_from() - - for dataset_list in dataset_gen: - assert (dataset_list[0].data == imaging_ci_7x7.data).all() - assert (dataset_list[1].data == imaging_ci_7x7.data).all() - - assert dataset_list[0].layout.parallel_overscan[1] == pytest.approx( - imaging_ci_7x7.layout.parallel_overscan[1], 1.0e-4 - ) - assert dataset_list[1].layout.parallel_overscan[1] == pytest.approx( - imaging_ci_7x7.layout.parallel_overscan[1], 1.0e-4 - ) - - clean(database_file=database_file) - - -def test__dataset_gen_from__analysis_use_dataset_full( - imaging_ci_7x7, parallel_clocker_2d, samples_2d, model_2d -): - imaging_ci_7x7_full = copy.copy(imaging_ci_7x7) - imaging_ci_7x7_full.data[0] = 100.0 - - analysis = ac.AnalysisImagingCI( - dataset=imaging_ci_7x7, - clocker=parallel_clocker_2d, - dataset_full=imaging_ci_7x7_full, - ) - - agg = aggregator_from( - database_file=database_file, - analysis=analysis + analysis, - model=model_2d, - samples=samples_2d, - ) - - dataset_agg = ac.agg.ImagingCIAgg(aggregator=agg, use_dataset_full=True) - dataset_gen = dataset_agg.dataset_list_gen_from() - - for dataset_list in dataset_gen: - assert dataset_list[0].data[0] == pytest.approx(100.0, 1.0e-4) - assert dataset_list[1].data[0] == pytest.approx(100.0, 1.0e-4) - - assert (dataset_list[0].data == imaging_ci_7x7.data).all() - assert (dataset_list[1].data == imaging_ci_7x7.data).all() - - assert dataset_list[0].layout.parallel_overscan[1] == pytest.approx( - imaging_ci_7x7.layout.parallel_overscan[1], 1.0e-4 - ) - assert dataset_list[1].layout.parallel_overscan[1] == pytest.approx( - imaging_ci_7x7.layout.parallel_overscan[1], 1.0e-4 - ) - - clean(database_file=database_file) +import copy +import pytest + +import autocti as ac + +from test_autocti.aggregator.conftest import clean, aggregator_from + +database_file = "db_imaging_ci" + + +def test__dataset_gen_from__analysis_has_single_dataset( + imaging_ci_7x7, parallel_clocker_2d, samples_2d, model_2d +): + analysis = ac.AnalysisImagingCI(dataset=imaging_ci_7x7, clocker=parallel_clocker_2d) + + agg = aggregator_from( + database_file=database_file, + analysis=analysis, + model=model_2d, + samples=samples_2d, + ) + + dataset_agg = ac.agg.ImagingCIAgg(aggregator=agg) + dataset_gen = dataset_agg.dataset_list_gen_from() + + for dataset_list in dataset_gen: + assert (dataset_list[0].data == imaging_ci_7x7.data).all() + assert dataset_list[0].layout.parallel_overscan[1] == pytest.approx( + imaging_ci_7x7.layout.parallel_overscan[1], 1.0e-4 + ) + + clean(database_file=database_file) + + +@pytest.mark.skip( + reason="Analysis summing (analysis + analysis) was removed from PyAutoFit in favour " + "of AnalysisFactor/FactorGraphModel; these tests are ported in Phase 2 of the CTI " + "resurrection epic (PyAutoCTI#82)." +) +def test__dataset_gen_from__analysis_has_multi_dataset( + imaging_ci_7x7, parallel_clocker_2d, samples_2d, model_2d +): + analysis = ac.AnalysisImagingCI(dataset=imaging_ci_7x7, clocker=parallel_clocker_2d) + + agg = aggregator_from( + database_file=database_file, + analysis=analysis + analysis, + model=model_2d, + samples=samples_2d, + ) + + dataset_agg = ac.agg.ImagingCIAgg(aggregator=agg) + dataset_gen = dataset_agg.dataset_list_gen_from() + + for dataset_list in dataset_gen: + assert (dataset_list[0].data == imaging_ci_7x7.data).all() + assert (dataset_list[1].data == imaging_ci_7x7.data).all() + + assert dataset_list[0].layout.parallel_overscan[1] == pytest.approx( + imaging_ci_7x7.layout.parallel_overscan[1], 1.0e-4 + ) + assert dataset_list[1].layout.parallel_overscan[1] == pytest.approx( + imaging_ci_7x7.layout.parallel_overscan[1], 1.0e-4 + ) + + clean(database_file=database_file) + + +@pytest.mark.skip( + reason="Analysis summing (analysis + analysis) was removed from PyAutoFit in favour " + "of AnalysisFactor/FactorGraphModel; these tests are ported in Phase 2 of the CTI " + "resurrection epic (PyAutoCTI#82)." +) +def test__dataset_gen_from__analysis_use_dataset_full( + imaging_ci_7x7, parallel_clocker_2d, samples_2d, model_2d +): + imaging_ci_7x7_full = copy.copy(imaging_ci_7x7) + imaging_ci_7x7_full.data[0] = 100.0 + + analysis = ac.AnalysisImagingCI( + dataset=imaging_ci_7x7, + clocker=parallel_clocker_2d, + dataset_full=imaging_ci_7x7_full, + ) + + agg = aggregator_from( + database_file=database_file, + analysis=analysis + analysis, + model=model_2d, + samples=samples_2d, + ) + + dataset_agg = ac.agg.ImagingCIAgg(aggregator=agg, use_dataset_full=True) + dataset_gen = dataset_agg.dataset_list_gen_from() + + for dataset_list in dataset_gen: + assert dataset_list[0].data[0] == pytest.approx(100.0, 1.0e-4) + assert dataset_list[1].data[0] == pytest.approx(100.0, 1.0e-4) + + assert (dataset_list[0].data == imaging_ci_7x7.data).all() + assert (dataset_list[1].data == imaging_ci_7x7.data).all() + + assert dataset_list[0].layout.parallel_overscan[1] == pytest.approx( + imaging_ci_7x7.layout.parallel_overscan[1], 1.0e-4 + ) + assert dataset_list[1].layout.parallel_overscan[1] == pytest.approx( + imaging_ci_7x7.layout.parallel_overscan[1], 1.0e-4 + ) + + clean(database_file=database_file) diff --git a/test_autocti/charge_injection/test_ou_sim_ci.py b/test_autocti/charge_injection/test_ou_sim_ci.py index 3f0f2272..8c0e505f 100644 --- a/test_autocti/charge_injection/test_ou_sim_ci.py +++ b/test_autocti/charge_injection/test_ou_sim_ci.py @@ -1,429 +1,437 @@ -import numpy as np - -import autocti as ac - -from autocti.charge_injection import ou_sim_ci - - -def test__non_uniform_array_is_correct_with_rotation(): - # bottom left - - array = ou_sim_ci.charge_injection_array_from( - ccd_id="123", - quadrant_id="E", - injection_start=0, - injection_end=2000, - injection_on=200, - injection_off=200, - injection_norm=50000.0, - ) - - assert array.shape_native == (2086, 2128) - assert array.native[0, 50] == 0 - assert array.native[0, 2099] == 0 - assert (array.native[0:200, 51:2099] > 0).all() - assert 49000.0 < np.mean(array.native[0:200, 51:2099]) < 51000.0 - - # top left - - array = ou_sim_ci.charge_injection_array_from( - ccd_id="123", - quadrant_id="H", - injection_start=0, - injection_end=2000, - injection_on=200, - injection_off=200, - injection_norm=50000.0, - ) - - assert array.native[1938, 50] == 0 - assert array.native[1938, 2099] == 0 - assert (array.native[1928:2128, 51:2099] > 0).all() - assert 49000.0 < np.mean(array.native[1928:2128, 51:2099]) < 51000.0 - - # bottom right - - array = ou_sim_ci.charge_injection_array_from( - ccd_id="123", - quadrant_id="F", - injection_start=0, - injection_end=2000, - injection_on=200, - injection_off=200, - injection_norm=50000.0, - ) - - assert array.shape_native == (2086, 2128) - assert array.native[0, 28] == 0 - assert array.native[0, 2077] == 0 - assert (array.native[0:200, 29:2077] > 0).all() - assert 49000.0 < np.mean(array.native[0:200, 51:2099]) < 51000.0 - - # top right - - array = ou_sim_ci.charge_injection_array_from( - ccd_id="123", - quadrant_id="G", - injection_start=0, - injection_end=2000, - injection_on=200, - injection_off=200, - injection_norm=50000.0, - ) - - assert array.shape_native == (2086, 2128) - assert array.native[1938, 28] == 0 - assert array.native[1938, 2077] == 0 - assert (array.native[1928:2128, 29:2077] > 0).all() - assert 49000.0 < np.mean(array.native[1928:2128, 29:2077]) < 51000.0 - - -def test__add_cti_to_pre_cti_data(): - clocker = ac.Clocker2D(parallel_express=2, serial_express=2) - - parallel_trap_list = [ac.TrapInstantCapture(density=0.13, release_timescale=1.25)] - parallel_ccd = ac.CCDPhase( - well_fill_power=0.8, well_notch_depth=0.0, full_well_depth=84700.0 - ) - serial_trap_list = [ac.TrapInstantCapture(density=0.0442, release_timescale=0.8)] - serial_ccd = ac.CCDPhase( - well_fill_power=0.8, well_notch_depth=0.0, full_well_depth=84700.0 - ) - - # bottom left - - array = ou_sim_ci.charge_injection_array_from( - ccd_id="123", - quadrant_id="E", - injection_start=0, - injection_end=2000, - injection_on=200, - injection_off=200, - injection_norm=50000.0, - ) - - assert array.native[199, 100] > 0.0 - assert array.native[200, 100] == 0.0 - - pre_cti_data = array.native[:, 100:101] - pre_cti_data.mask = pre_cti_data.mask[:, 100:101] - - post_cti_data = ou_sim_ci.add_cti_to_pre_cti_data( - pre_cti_data=pre_cti_data, - ccd_id="123", - quadrant_id="E", - clocker=clocker, - parallel_trap_list=parallel_trap_list, - parallel_ccd=parallel_ccd, - serial_trap_list=serial_trap_list, - serial_ccd=serial_ccd, - ) - - assert post_cti_data[200, 0] > 0.0 - - # top left - - array = ou_sim_ci.charge_injection_array_from( - ccd_id="123", - quadrant_id="H", - injection_start=0, - injection_end=2000, - injection_on=200, - injection_off=200, - injection_norm=50000.0, - ) - - assert array.native[1886, 100] > 0.0 - assert array.native[1885, 100] == 0.0 - - pre_cti_data = array.native[:, 100:101] - pre_cti_data.mask = pre_cti_data.mask[:, 100:101] - - post_cti_data = ou_sim_ci.add_cti_to_pre_cti_data( - pre_cti_data=pre_cti_data, - ccd_id="123", - quadrant_id="H", - clocker=clocker, - parallel_trap_list=parallel_trap_list, - parallel_ccd=parallel_ccd, - serial_trap_list=serial_trap_list, - serial_ccd=serial_ccd, - ) - - assert post_cti_data[1885, 0] > 0.0 - - # bottom right - - array = ou_sim_ci.charge_injection_array_from( - ccd_id="123", - quadrant_id="F", - injection_start=0, - injection_end=2000, - injection_on=200, - injection_off=200, - injection_norm=50000.0, - ) - - assert array.native[199, 100] > 0.0 - assert array.native[200, 100] == 0.0 - - pre_cti_data = array.native[:, 100:101] - pre_cti_data.mask = pre_cti_data.mask[:, 100:101] - - post_cti_data = ou_sim_ci.add_cti_to_pre_cti_data( - pre_cti_data=pre_cti_data, - ccd_id="123", - quadrant_id="F", - clocker=clocker, - parallel_trap_list=parallel_trap_list, - parallel_ccd=parallel_ccd, - serial_trap_list=serial_trap_list, - serial_ccd=serial_ccd, - ) - - assert post_cti_data[200, 0] > 0.0 - - # top right - - array = ou_sim_ci.charge_injection_array_from( - ccd_id="123", - quadrant_id="G", - injection_start=0, - injection_end=2000, - injection_on=200, - injection_off=200, - injection_norm=50000.0, - ) - - assert array.native[1886, 100] > 0.0 - assert array.native[1885, 100] == 0.0 - - pre_cti_data = array.native[:, 100:101] - pre_cti_data.mask = pre_cti_data.mask[:, 100:101] - - post_cti_data = ou_sim_ci.add_cti_to_pre_cti_data( - pre_cti_data=pre_cti_data, - ccd_id="123", - quadrant_id="G", - clocker=clocker, - parallel_trap_list=parallel_trap_list, - parallel_ccd=parallel_ccd, - serial_trap_list=serial_trap_list, - serial_ccd=serial_ccd, - ) - - assert post_cti_data[1885, 0] > 0.0 - - -def test__tvac_values(): - array = ou_sim_ci.charge_injection_array_from( - # iquad=0, - ccd_id="123", - quadrant_id="E", - injection_start=16, - injection_end=2086, - injection_on=420, - injection_off=100, - injection_norm=50000.0, - ) - - tvac_region_1 = ac.Region2D(region=[16, 436, 51, 2099]) - - assert (array[tvac_region_1.slice] > 10000).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - - tvac_region_1 = ac.Region2D(region=[536, 956, 51, 2099]) - - assert (array[tvac_region_1.slice] > 10000).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - - array = ou_sim_ci.charge_injection_array_from( - # iquad=1, - ccd_id="123", - quadrant_id="F", - injection_start=16, - injection_end=2086, - injection_on=420, - injection_off=100, - injection_norm=50000.0, - ) - - tvac_region_1 = ac.Region2D(region=[16, 436, 29, 2077]) - - assert (array[tvac_region_1.slice] > 10000).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - - tvac_region_1 = ac.Region2D(region=[536, 956, 29, 2077]) - - assert (array[tvac_region_1.slice] > 10000).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - - array = ou_sim_ci.charge_injection_array_from( - # iquad=2, - ccd_id="123", - quadrant_id="H", - injection_start=16, - injection_end=2086, - injection_on=420, - injection_off=100, - injection_norm=50000.0, - ) - - tvac_region_1 = ac.Region2D(region=[1650, 2070, 51, 2099]) - - assert (array[tvac_region_1.slice] > 10000).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - - tvac_region_1 = ac.Region2D(region=[1130, 1550, 51, 2099]) - - assert (array[tvac_region_1.slice] > 10000).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - - array = ou_sim_ci.charge_injection_array_from( - # iquad=3, - ccd_id="123", - quadrant_id="G", - injection_start=16, - injection_end=2086, - injection_on=420, - injection_off=100, - injection_norm=50000.0, - ) - - tvac_region_1 = ac.Region2D(region=[1650, 2070, 29, 2077]) - - assert (array[tvac_region_1.slice] > 10000).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - - tvac_region_1 = ac.Region2D(region=[1130, 1550, 29, 2077]) - - assert (array[tvac_region_1.slice] > 10000).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - - array = ou_sim_ci.charge_injection_array_from( - ccd_id="456", - quadrant_id="E", - injection_start=16, - injection_end=2086, - injection_on=420, - injection_off=100, - injection_norm=50000.0, - ) - - tvac_region_1 = ac.Region2D(region=[1650, 2070, 29, 2077]) - - assert (array[tvac_region_1.slice] > 10000).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - - tvac_region_1 = ac.Region2D(region=[1130, 1550, 29, 2077]) - - assert (array[tvac_region_1.slice] > 10000).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - - array = ou_sim_ci.charge_injection_array_from( - ccd_id="456", - quadrant_id="F", - injection_start=16, - injection_end=2086, - injection_on=420, - injection_off=100, - injection_norm=50000.0, - ) - - tvac_region_1 = ac.Region2D(region=[1650, 2070, 51, 2099]) - - assert (array[tvac_region_1.slice] > 10000).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - - tvac_region_1 = ac.Region2D(region=[1130, 1550, 51, 2099]) - - assert (array[tvac_region_1.slice] > 10000).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - - array = ou_sim_ci.charge_injection_array_from( - ccd_id="456", - quadrant_id="G", - injection_start=16, - injection_end=2086, - injection_on=420, - injection_off=100, - injection_norm=50000.0, - ) - - tvac_region_1 = ac.Region2D(region=[16, 436, 51, 2099]) - - assert (array[tvac_region_1.slice] > 10000).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - - tvac_region_1 = ac.Region2D(region=[536, 956, 51, 2099]) - - assert (array[tvac_region_1.slice] > 10000).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - - array = ou_sim_ci.charge_injection_array_from( - ccd_id="456", - quadrant_id="H", - injection_start=16, - injection_end=2086, - injection_on=420, - injection_off=100, - injection_norm=50000.0, - ) - - tvac_region_1 = ac.Region2D(region=[16, 436, 29, 2077]) - - assert (array[tvac_region_1.slice] > 10000).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - - tvac_region_1 = ac.Region2D(region=[536, 956, 29, 2077]) - - assert (array[tvac_region_1.slice] > 10000).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() - assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() - assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() +import numpy as np + +import autocti as ac + +from autocti.charge_injection import ou_sim_ci + + +def test__non_uniform_array_is_correct_with_rotation(): + # bottom left + + array = ou_sim_ci.charge_injection_array_from( + ccd_id="123", + quadrant_id="E", + injection_start=0, + injection_end=2000, + injection_on=200, + injection_off=200, + injection_norm=50000.0, + ) + + assert array.shape_native == (2086, 2128) + assert array.native[0, 50] == 0 + assert array.native[0, 2099] == 0 + assert (array.native[0:200, 51:2099] > 0).all() + assert 49000.0 < np.mean(array.native[0:200, 51:2099]) < 51000.0 + + # top left + + array = ou_sim_ci.charge_injection_array_from( + ccd_id="123", + quadrant_id="H", + injection_start=0, + injection_end=2000, + injection_on=200, + injection_off=200, + injection_norm=50000.0, + ) + + assert array.native[1938, 50] == 0 + assert array.native[1938, 2099] == 0 + assert (array.native[1928:2128, 51:2099] > 0).all() + assert 49000.0 < np.mean(array.native[1928:2128, 51:2099]) < 51000.0 + + # bottom right + + array = ou_sim_ci.charge_injection_array_from( + ccd_id="123", + quadrant_id="F", + injection_start=0, + injection_end=2000, + injection_on=200, + injection_off=200, + injection_norm=50000.0, + ) + + assert array.shape_native == (2086, 2128) + assert array.native[0, 28] == 0 + assert array.native[0, 2077] == 0 + assert (array.native[0:200, 29:2077] > 0).all() + assert 49000.0 < np.mean(array.native[0:200, 51:2099]) < 51000.0 + + # top right + + array = ou_sim_ci.charge_injection_array_from( + ccd_id="123", + quadrant_id="G", + injection_start=0, + injection_end=2000, + injection_on=200, + injection_off=200, + injection_norm=50000.0, + ) + + assert array.shape_native == (2086, 2128) + assert array.native[1938, 28] == 0 + assert array.native[1938, 2077] == 0 + assert (array.native[1928:2128, 29:2077] > 0).all() + assert 49000.0 < np.mean(array.native[1928:2128, 29:2077]) < 51000.0 + + +def test__add_cti_to_pre_cti_data(): + clocker = ac.Clocker2D(parallel_express=2, serial_express=2) + + parallel_trap_list = [ac.TrapInstantCapture(density=0.13, release_timescale=1.25)] + parallel_ccd = ac.CCDPhase( + well_fill_power=0.8, well_notch_depth=0.0, full_well_depth=84700.0 + ) + serial_trap_list = [ac.TrapInstantCapture(density=0.0442, release_timescale=0.8)] + serial_ccd = ac.CCDPhase( + well_fill_power=0.8, well_notch_depth=0.0, full_well_depth=84700.0 + ) + + # bottom left + + array = ou_sim_ci.charge_injection_array_from( + ccd_id="123", + quadrant_id="E", + injection_start=0, + injection_end=2000, + injection_on=200, + injection_off=200, + injection_norm=50000.0, + ) + + assert array.native[199, 100] > 0.0 + assert array.native[200, 100] == 0.0 + + pre_cti_data = ac.Array2D.no_mask( + values=np.asarray(array.native)[:, 100:101], + pixel_scales=array.pixel_scales, + ) + + post_cti_data = ou_sim_ci.add_cti_to_pre_cti_data( + pre_cti_data=pre_cti_data, + ccd_id="123", + quadrant_id="E", + clocker=clocker, + parallel_trap_list=parallel_trap_list, + parallel_ccd=parallel_ccd, + serial_trap_list=serial_trap_list, + serial_ccd=serial_ccd, + ) + + assert post_cti_data[200, 0] > 0.0 + + # top left + + array = ou_sim_ci.charge_injection_array_from( + ccd_id="123", + quadrant_id="H", + injection_start=0, + injection_end=2000, + injection_on=200, + injection_off=200, + injection_norm=50000.0, + ) + + assert array.native[1886, 100] > 0.0 + assert array.native[1885, 100] == 0.0 + + pre_cti_data = ac.Array2D.no_mask( + values=np.asarray(array.native)[:, 100:101], + pixel_scales=array.pixel_scales, + ) + + post_cti_data = ou_sim_ci.add_cti_to_pre_cti_data( + pre_cti_data=pre_cti_data, + ccd_id="123", + quadrant_id="H", + clocker=clocker, + parallel_trap_list=parallel_trap_list, + parallel_ccd=parallel_ccd, + serial_trap_list=serial_trap_list, + serial_ccd=serial_ccd, + ) + + assert post_cti_data[1885, 0] > 0.0 + + # bottom right + + array = ou_sim_ci.charge_injection_array_from( + ccd_id="123", + quadrant_id="F", + injection_start=0, + injection_end=2000, + injection_on=200, + injection_off=200, + injection_norm=50000.0, + ) + + assert array.native[199, 100] > 0.0 + assert array.native[200, 100] == 0.0 + + pre_cti_data = ac.Array2D.no_mask( + values=np.asarray(array.native)[:, 100:101], + pixel_scales=array.pixel_scales, + ) + + post_cti_data = ou_sim_ci.add_cti_to_pre_cti_data( + pre_cti_data=pre_cti_data, + ccd_id="123", + quadrant_id="F", + clocker=clocker, + parallel_trap_list=parallel_trap_list, + parallel_ccd=parallel_ccd, + serial_trap_list=serial_trap_list, + serial_ccd=serial_ccd, + ) + + assert post_cti_data[200, 0] > 0.0 + + # top right + + array = ou_sim_ci.charge_injection_array_from( + ccd_id="123", + quadrant_id="G", + injection_start=0, + injection_end=2000, + injection_on=200, + injection_off=200, + injection_norm=50000.0, + ) + + assert array.native[1886, 100] > 0.0 + assert array.native[1885, 100] == 0.0 + + pre_cti_data = ac.Array2D.no_mask( + values=np.asarray(array.native)[:, 100:101], + pixel_scales=array.pixel_scales, + ) + + post_cti_data = ou_sim_ci.add_cti_to_pre_cti_data( + pre_cti_data=pre_cti_data, + ccd_id="123", + quadrant_id="G", + clocker=clocker, + parallel_trap_list=parallel_trap_list, + parallel_ccd=parallel_ccd, + serial_trap_list=serial_trap_list, + serial_ccd=serial_ccd, + ) + + assert post_cti_data[1885, 0] > 0.0 + + +def test__tvac_values(): + array = ou_sim_ci.charge_injection_array_from( + # iquad=0, + ccd_id="123", + quadrant_id="E", + injection_start=16, + injection_end=2086, + injection_on=420, + injection_off=100, + injection_norm=50000.0, + ) + + tvac_region_1 = ac.Region2D(region=[16, 436, 51, 2099]) + + assert (array[tvac_region_1.slice] > 10000).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + + tvac_region_1 = ac.Region2D(region=[536, 956, 51, 2099]) + + assert (array[tvac_region_1.slice] > 10000).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + + array = ou_sim_ci.charge_injection_array_from( + # iquad=1, + ccd_id="123", + quadrant_id="F", + injection_start=16, + injection_end=2086, + injection_on=420, + injection_off=100, + injection_norm=50000.0, + ) + + tvac_region_1 = ac.Region2D(region=[16, 436, 29, 2077]) + + assert (array[tvac_region_1.slice] > 10000).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + + tvac_region_1 = ac.Region2D(region=[536, 956, 29, 2077]) + + assert (array[tvac_region_1.slice] > 10000).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + + array = ou_sim_ci.charge_injection_array_from( + # iquad=2, + ccd_id="123", + quadrant_id="H", + injection_start=16, + injection_end=2086, + injection_on=420, + injection_off=100, + injection_norm=50000.0, + ) + + tvac_region_1 = ac.Region2D(region=[1650, 2070, 51, 2099]) + + assert (array[tvac_region_1.slice] > 10000).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + + tvac_region_1 = ac.Region2D(region=[1130, 1550, 51, 2099]) + + assert (array[tvac_region_1.slice] > 10000).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + + array = ou_sim_ci.charge_injection_array_from( + # iquad=3, + ccd_id="123", + quadrant_id="G", + injection_start=16, + injection_end=2086, + injection_on=420, + injection_off=100, + injection_norm=50000.0, + ) + + tvac_region_1 = ac.Region2D(region=[1650, 2070, 29, 2077]) + + assert (array[tvac_region_1.slice] > 10000).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + + tvac_region_1 = ac.Region2D(region=[1130, 1550, 29, 2077]) + + assert (array[tvac_region_1.slice] > 10000).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + + array = ou_sim_ci.charge_injection_array_from( + ccd_id="456", + quadrant_id="E", + injection_start=16, + injection_end=2086, + injection_on=420, + injection_off=100, + injection_norm=50000.0, + ) + + tvac_region_1 = ac.Region2D(region=[1650, 2070, 29, 2077]) + + assert (array[tvac_region_1.slice] > 10000).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + + tvac_region_1 = ac.Region2D(region=[1130, 1550, 29, 2077]) + + assert (array[tvac_region_1.slice] > 10000).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + + array = ou_sim_ci.charge_injection_array_from( + ccd_id="456", + quadrant_id="F", + injection_start=16, + injection_end=2086, + injection_on=420, + injection_off=100, + injection_norm=50000.0, + ) + + tvac_region_1 = ac.Region2D(region=[1650, 2070, 51, 2099]) + + assert (array[tvac_region_1.slice] > 10000).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + + tvac_region_1 = ac.Region2D(region=[1130, 1550, 51, 2099]) + + assert (array[tvac_region_1.slice] > 10000).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + + array = ou_sim_ci.charge_injection_array_from( + ccd_id="456", + quadrant_id="G", + injection_start=16, + injection_end=2086, + injection_on=420, + injection_off=100, + injection_norm=50000.0, + ) + + tvac_region_1 = ac.Region2D(region=[16, 436, 51, 2099]) + + assert (array[tvac_region_1.slice] > 10000).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + + tvac_region_1 = ac.Region2D(region=[536, 956, 51, 2099]) + + assert (array[tvac_region_1.slice] > 10000).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + + array = ou_sim_ci.charge_injection_array_from( + ccd_id="456", + quadrant_id="H", + injection_start=16, + injection_end=2086, + injection_on=420, + injection_off=100, + injection_norm=50000.0, + ) + + tvac_region_1 = ac.Region2D(region=[16, 436, 29, 2077]) + + assert (array[tvac_region_1.slice] > 10000).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + + tvac_region_1 = ac.Region2D(region=[536, 956, 29, 2077]) + + assert (array[tvac_region_1.slice] > 10000).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x0 - 1] == 0).all() + assert (array[tvac_region_1.y0 : tvac_region_1.y1, tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y0 - 1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() + assert (array[tvac_region_1.y1, tvac_region_1.x0 : tvac_region_1.x1] == 0).all() diff --git a/test_autocti/config/priors/ccd.yaml b/test_autocti/config/priors/ccd.yaml index 0d6f1738..c20a5e3b 100644 --- a/test_autocti/config/priors/ccd.yaml +++ b/test_autocti/config/priors/ccd.yaml @@ -1,6 +1,6 @@ CCDPhase: full_well_depth: - gaussian_limits: + limits: lower: 0.0 upper: 1.0 lower_limit: 0.0 @@ -10,7 +10,7 @@ CCDPhase: type: Absolute value: 0.2 well_fill_power: - gaussian_limits: + limits: lower: 0.0 upper: 1.0 lower_limit: 0.0 @@ -20,7 +20,7 @@ CCDPhase: type: Absolute value: 0.2 well_notch_depth: - gaussian_limits: + limits: lower: 0.0 upper: 1.0 lower_limit: 0.0 diff --git a/test_autocti/config/priors/hyper.yaml b/test_autocti/config/priors/hyper.yaml index 2f043c8f..ac115621 100644 --- a/test_autocti/config/priors/hyper.yaml +++ b/test_autocti/config/priors/hyper.yaml @@ -1,6 +1,6 @@ HyperCINoiseScalar: scale_factor: - gaussian_limits: + limits: lower: 0.0 upper: inf lower_limit: 0.0 diff --git a/test_autocti/config/priors/traps.yaml b/test_autocti/config/priors/traps.yaml index 2287d8da..6727f20e 100644 --- a/test_autocti/config/priors/traps.yaml +++ b/test_autocti/config/priors/traps.yaml @@ -1,6 +1,6 @@ TrapInstantCapture: density: - gaussian_limits: + limits: lower: 0.0 upper: inf lower_limit: 0.0 @@ -10,7 +10,7 @@ TrapInstantCapture: type: Relative value: 0.5 release_timescale: - gaussian_limits: + limits: lower: 0.0 upper: inf lower_limit: 0.0 diff --git a/test_autocti/conftest.py b/test_autocti/conftest.py index 977f9ccb..26967589 100644 --- a/test_autocti/conftest.py +++ b/test_autocti/conftest.py @@ -7,6 +7,18 @@ from autofit import conf from autocti import fixtures +# The Plotter object stack targets the removed autoarray Plotter API and is +# rewritten on the new matplotlib function API in Phase 1 of the CTI +# resurrection epic (PyAutoCTI#82); its tests are quarantined until then. +collect_ignore_glob = [ + "plot/*", + "*/plot/*", +] +collect_ignore = [ + path.join("dataset_1d", "model", "test_plotter_interface_1d.py"), + path.join("charge_injection", "model", "test_plotter_interface_ci.py"), +] + class PlotPatch: def __init__(self): diff --git a/test_autocti/dataset_1d/dataset_1d/test_simulator.py b/test_autocti/dataset_1d/dataset_1d/test_simulator.py index a3448758..ab0682cf 100644 --- a/test_autocti/dataset_1d/dataset_1d/test_simulator.py +++ b/test_autocti/dataset_1d/dataset_1d/test_simulator.py @@ -8,7 +8,7 @@ def test__no_instrumental_effects_input__only_cti_simulated(clocker_1d, traps_x2 layout = ac.Layout1D(shape_1d=(5,), region_list=[(0, 5)]) simulator = ac.SimulatorDataset1D( - pixel_scales=1.0, norm=10.0, add_poisson_noise=False + pixel_scales=1.0, norm=10.0, add_poisson_noise_to_data=False ) cti = ac.CTI1D(trap_list=traps_x2, ccd=ccd) @@ -26,7 +26,7 @@ def test__include_charge_noise__is_added_before_cti(clocker_1d, traps_x2, ccd): pixel_scales=1.0, norm=10.0, charge_noise=1.0, - add_poisson_noise=False, + add_poisson_noise_to_data=False, noise_seed=1, ) @@ -49,7 +49,7 @@ def test__include_read_noise__is_added_after_cti(clocker_1d, traps_x2, ccd): pixel_scales=1.0, norm=10.0, read_noise=1.0, - add_poisson_noise=False, + add_poisson_noise_to_data=False, noise_seed=1, ) @@ -82,7 +82,7 @@ def test__via_pre_cti_data(clocker_1d, traps_x2, ccd): pixel_scales=1.0, norm=10.0, read_noise=4.0, - add_poisson_noise=False, + add_poisson_noise_to_data=False, noise_seed=1, ) @@ -107,7 +107,7 @@ def test__via_post_cti_data(clocker_1d, traps_x2, ccd): pixel_scales=1.0, norm=10.0, read_noise=4.0, - add_poisson_noise=False, + add_poisson_noise_to_data=False, noise_seed=1, ) diff --git a/test_autocti/instruments/acs/test_image.py b/test_autocti/instruments/acs/test_image.py index d79621ab..6eadb73d 100644 --- a/test_autocti/instruments/acs/test_image.py +++ b/test_autocti/instruments/acs/test_image.py @@ -1,455 +1,456 @@ -import numpy as np -from astropy.io import fits -import copy -import shutil -import os -from os import path -import pytest - -import autocti as ac - - -def create_acs_fits( - fits_path, acs_ccd, acs_ccd_0, acs_ccd_1, units, bias_file_path=None -): - if path.exists(fits_path): - shutil.rmtree(fits_path) - - os.makedirs(fits_path) - - hdu_list = fits.HDUList() - - hdu_list.append(fits.ImageHDU(acs_ccd)) - hdu_list.append(fits.ImageHDU(acs_ccd_0)) - hdu_list.append(fits.ImageHDU(acs_ccd)) - hdu_list.append(fits.ImageHDU(acs_ccd)) - hdu_list.append(fits.ImageHDU(acs_ccd_1)) - hdu_list.append(fits.ImageHDU(acs_ccd)) - - hdu_list[0].header.set("CCDGAIN", 1.0, "Instrument GAIN") - hdu_list[0].header.set("TELESCOP", "HST", "Telescope Name") - hdu_list[0].header.set("INSTRUME", "ACS", "Instrument Name") - hdu_list[0].header.set("EXPTIME", 1000.0, "exposure duration (seconds)--calculated") - hdu_list[0].header.set( - "DATE-OBS", "2000-01-01", "UT date of start of observation (yyyy-mm-dd)" - ) - hdu_list[0].header.set( - "TIME-OBS", "00:00:00", "UT time of start of observation (hh:mm:ss)" - ) - hdu_list[0].header.set("BIASFILE", f"jref${bias_file_path}", "Bias file name") - - if units in "COUNTS": - hdu_list[1].header.set("BUNIT", "COUNTS", "brightness units") - hdu_list[4].header.set("BUNIT", "COUNTS", "brightness units") - elif units in "CPS": - hdu_list[1].header.set("BUNIT", "CPS", "brightness units") - hdu_list[4].header.set("BUNIT", "CPS", "brightness units") - - hdu_list[1].header.set( - "BSCALE", 2.0, "scale factor for array value to physical value" - ) - hdu_list[1].header.set("BZERO", 10.0, "physical value for an array value of zero") - - hdu_list[4].header.set( - "BSCALE", 2.0, "scale factor for array value to physical value" - ) - hdu_list[4].header.set("BZERO", 10.0, "physical value for an array value of zero") - - hdu_list.writeto(path.join(fits_path, "acs_ccd.fits")) - - -def create_acs_bias_fits(fits_path, bias_ccd, bias_ccd_0, bias_ccd_1): - hdu_list = fits.HDUList() - - hdu_list.append(fits.ImageHDU(bias_ccd)) - hdu_list.append(fits.ImageHDU(bias_ccd_0)) - hdu_list.append(fits.ImageHDU(bias_ccd)) - hdu_list.append(fits.ImageHDU(bias_ccd)) - hdu_list.append(fits.ImageHDU(bias_ccd_1)) - hdu_list.append(fits.ImageHDU(bias_ccd)) - - hdu_list[0].header.set("CCDGAIN", 1.0, "Instrument GAIN") - hdu_list[1].header.set("BUNIT", "COUNTS", "brightness units") - hdu_list[4].header.set("BUNIT", "COUNTS", "brightness units") - - hdu_list.writeto(path.join(fits_path, "acs_bias_ccd.fits")) - - -def test__from_fits__reads_header_from_header_correctly(acs_ccd): - fits_path = path.join( - "{}".format(path.dirname(path.realpath(__file__))), "files", "acs" - ) - - create_acs_fits( - fits_path=fits_path, - acs_ccd=acs_ccd, - acs_ccd_0=acs_ccd, - acs_ccd_1=acs_ccd, - units="COUNTS", - ) - - file_path = path.join(fits_path, "acs_ccd.fits") - - array = ac.acs.ImageACS.from_fits(file_path=file_path, quadrant_letter="B") - - assert array.header.exposure_time == 1000.0 - assert array.header.date_of_observation == "2000-01-01" - assert array.header.time_of_observation == "00:00:00" - assert array.header.modified_julian_date == 51544.0 - - array = ac.acs.ImageACS.from_fits(file_path=file_path, quadrant_letter="C") - - assert array.header.exposure_time == 1000.0 - assert array.header.date_of_observation == "2000-01-01" - assert array.header.time_of_observation == "00:00:00" - assert array.header.modified_julian_date == 51544.0 - - -def test__from_fits__in_counts__uses_fits_header_correctly_converts_and_picks_correct_quadrant( - acs_ccd, -): - fits_path = path.join( - "{}".format(path.dirname(path.realpath(__file__))), "files", "acs" - ) - - file_path = path.join(fits_path, "acs_ccd.fits") - - acs_ccd_0 = copy.copy(acs_ccd) - acs_ccd_0[0, 0] = 10.0 - acs_ccd_0[0, -1] = 20.0 - - acs_ccd_1 = copy.copy(acs_ccd) - acs_ccd_1[-1, 0] = 30.0 - acs_ccd_1[-1, -1] = 40.0 - - create_acs_fits( - fits_path=fits_path, - acs_ccd=acs_ccd, - acs_ccd_0=acs_ccd_0, - acs_ccd_1=acs_ccd_1, - units="COUNTS", - ) - - array = ac.acs.ImageACS.from_fits( - file_path=file_path, quadrant_letter="A", use_calibrated_gain=False - ) - - assert array.native[0, 0] == (30.0 * 2.0) + 10.0 - assert array.in_counts.native[0, 0] == 30.0 - assert array.shape_native == (2068, 2072) - - array_original = array.header.array_electrons_to_original( - array=array, use_calibrated_gain=False - ) - - assert array_original.native[0, 0] == 30.0 - - array = ac.acs.ImageACS.from_fits( - file_path=file_path, quadrant_letter="B", use_calibrated_gain=False - ) - - assert array.native[0, 0] == (40.0 * 2.0) + 10.0 - assert array.shape_native == (2068, 2072) - - array_original = array.header.array_electrons_to_original( - array=array, use_calibrated_gain=False - ) - - assert array_original.native[0, 0] == 40.0 - - array = ac.acs.ImageACS.from_fits( - file_path=file_path, quadrant_letter="C", use_calibrated_gain=False - ) - - assert array.native[0, 0] == (10.0 * 2.0) + 10.0 - assert array.shape_native == (2068, 2072) - - array_original = array.header.array_electrons_to_original( - array=array, use_calibrated_gain=False - ) - - assert array_original.native[0, 0] == 10.0 - - array = ac.acs.ImageACS.from_fits( - file_path=file_path, quadrant_letter="D", use_calibrated_gain=False - ) - - assert array.native[0, 0] == (20.0 * 2.0) + 10.0 - assert array.shape_native == (2068, 2072) - - array_original = array.header.array_electrons_to_original( - array=array, use_calibrated_gain=False - ) - - assert array_original.native[0, 0] == 20.0 - - -def test__from_fits__in_counts_per_second__uses_fits_header_correctly_converts_and_picks_correct_quadrant( - acs_ccd, -): - fits_path = path.join( - "{}".format(path.dirname(path.realpath(__file__))), "files", "acs" - ) - - file_path = path.join(fits_path, "acs_ccd.fits") - - acs_ccd_0 = copy.copy(acs_ccd) - acs_ccd_0[0, 0] = 10.0 - acs_ccd_0[0, -1] = 20.0 - - acs_ccd_1 = copy.copy(acs_ccd) - acs_ccd_1[-1, 0] = 30.0 - acs_ccd_1[-1, -1] = 40.0 - - create_acs_fits( - fits_path=fits_path, - acs_ccd=acs_ccd, - acs_ccd_0=acs_ccd_0, - acs_ccd_1=acs_ccd_1, - units="CPS", - ) - - array = ac.acs.ImageACS.from_fits( - file_path=file_path, quadrant_letter="A", use_calibrated_gain=False - ) - - assert array.native[0, 0] == (30.0 * 1000.0 * 2.0) + 10.0 - assert array.shape_native == (2068, 2072) - - array_original = array.header.array_electrons_to_original( - array=array, use_calibrated_gain=False - ) - - assert array_original.native[0, 0] == 30.0 - - array = ac.acs.ImageACS.from_fits( - file_path=file_path, quadrant_letter="B", use_calibrated_gain=False - ) - - assert array.native[0, 0] == (40.0 * 1000.0 * 2.0) + 10.0 - assert array.shape_native == (2068, 2072) - - array_original = array.header.array_electrons_to_original( - array=array, use_calibrated_gain=False - ) - - assert array_original.native[0, 0] == 40.0 - - array = ac.acs.ImageACS.from_fits( - file_path=file_path, quadrant_letter="C", use_calibrated_gain=False - ) - - assert array.native[0, 0] == (10.0 * 1000.0 * 2.0) + 10.0 - assert array.shape_native == (2068, 2072) - - array_original = array.header.array_electrons_to_original( - array=array, use_calibrated_gain=False - ) - - assert array_original.native[0, 0] == 10.0 - - array = ac.acs.ImageACS.from_fits( - file_path=file_path, quadrant_letter="D", use_calibrated_gain=False - ) - - assert array.native[0, 0] == (20.0 * 1000.0 * 2.0) + 10.0 - assert array.shape_native == (2068, 2072) - - array_original = array.header.array_electrons_to_original( - array=array, use_calibrated_gain=False - ) - - assert array_original.native[0, 0] == 20.0 - - -def test__from_fits__in_counts__uses_bias_prescan_correctly(acs_ccd): - fits_path = path.join( - "{}".format(path.dirname(path.realpath(__file__))), "files", "acs" - ) - - file_path = path.join(fits_path, "acs_ccd.fits") - - acs_ccd_0 = copy.copy(acs_ccd) - acs_ccd_0[0, 0] = 10.0 - acs_ccd_0[0, -1] = 20.0 - - acs_ccd_1 = copy.copy(acs_ccd) - acs_ccd_1[-1, 0] = 30.0 - acs_ccd_1[-1, -1] = 40.0 - - create_acs_fits( - fits_path=fits_path, - acs_ccd=acs_ccd, - acs_ccd_0=acs_ccd_0, - acs_ccd_1=acs_ccd_1, - units="COUNTS", - ) - - array = ac.acs.ImageACS.from_fits( - file_path=file_path, - quadrant_letter="A", - bias_subtract_via_prescan=True, - use_calibrated_gain=False, - ) - - assert array.native[0, 0] == pytest.approx(10.0, (30.0 * 2.0) + 10.0 - 10.0, 1.0e-4) - assert array.header.bias_serial_prescan_column[0][0] == pytest.approx(10.0, 1.0e-4) - - array = ac.acs.ImageACS.from_fits( - file_path=file_path, - quadrant_letter="B", - bias_subtract_via_prescan=True, - use_calibrated_gain=False, - ) - - assert array.native[0, 0] == pytest.approx(10.0, (40.0 * 2.0) + 10.0 - 10.0, 1.0e-4) - assert array.header.bias_serial_prescan_column[0][0] == pytest.approx(10.0, 1.0e-4) - - array = ac.acs.ImageACS.from_fits( - file_path=file_path, - quadrant_letter="C", - bias_subtract_via_prescan=True, - use_calibrated_gain=False, - ) - - assert array.native[0, 0] == pytest.approx(10.0, (10.0 * 2.0) + 10.0 - 10.0, 1.0e-4) - assert array.header.bias_serial_prescan_column[0][0] == pytest.approx(10.0, 1.0e-4) - - array = ac.acs.ImageACS.from_fits( - file_path=file_path, - quadrant_letter="D", - bias_subtract_via_prescan=True, - use_calibrated_gain=False, - ) - - assert array.native[0, 0] == pytest.approx(10.0, (20.0 * 2.0) + 10.0 - 10.0, 1.0e-4) - assert array.header.bias_serial_prescan_column[0][0] == pytest.approx(10.0, 1.0e-4) - - -def test__from_fits__in_counts__uses_bias_file_subtraction_correctly(acs_ccd): - fits_path = path.join( - "{}".format(path.dirname(path.realpath(__file__))), "files", "acs" - ) - - acs_ccd_0 = copy.copy(acs_ccd) - acs_ccd_0[0, 0] = 10.0 - acs_ccd_0[0, -1] = 20.0 - - acs_ccd_1 = copy.copy(acs_ccd) - acs_ccd_1[-1, 0] = 30.0 - acs_ccd_1[-1, -1] = 40.0 - - create_acs_fits( - fits_path=fits_path, - acs_ccd=acs_ccd, - acs_ccd_0=acs_ccd_0, - acs_ccd_1=acs_ccd_1, - units="COUNTS", - bias_file_path=path.join(fits_path, "acs_bias_ccd.fits"), - ) - - create_acs_bias_fits( - fits_path=fits_path, - bias_ccd=np.zeros((2068, 4144)), - bias_ccd_0=np.ones((2068, 4144)), - bias_ccd_1=2.0 * np.ones((2068, 4144)), - ) - - file_path = path.join(fits_path, "acs_ccd.fits") - - array = ac.acs.ImageACS.from_fits( - file_path=file_path, - quadrant_letter="A", - bias_subtract_via_bias_file=True, - use_calibrated_gain=False, - ) - - assert array.native[0, 0] == pytest.approx(10.0, (30.0 * 2.0) + 10.0 - 2.0, 1.0e-4) - assert array.header.bias[0] == pytest.approx(2.0, 1.0e-1) - - array = ac.acs.ImageACS.from_fits( - file_path=file_path, - quadrant_letter="B", - bias_subtract_via_bias_file=True, - use_calibrated_gain=False, - ) - - assert array.native[0, 0] == pytest.approx(10.0, (40.0 * 2.0) + 10.0 - 2.0, 1.0e-4) - assert array.header.bias[0] == pytest.approx(2.0, 1.0e-1) - - array = ac.acs.ImageACS.from_fits( - file_path=file_path, - quadrant_letter="C", - bias_subtract_via_bias_file=True, - use_calibrated_gain=False, - ) - - assert array.native[0, 0] == pytest.approx(10.0, (10.0 * 2.0) + 10.0 - 1.0, 1.0e-4) - assert array.header.bias[0] == pytest.approx(1.0, 1.0e-1) - - array = ac.acs.ImageACS.from_fits( - file_path=file_path, - quadrant_letter="D", - bias_subtract_via_bias_file=True, - use_calibrated_gain=False, - bias_file_path=path.join(fits_path, "acs_bias_ccd.fits"), - ) - - assert array.native[0, 0] == pytest.approx(10.0, (20.0 * 2.0) + 10.0 - 1.0, 1.0e-4) - assert array.header.bias[0] == pytest.approx(1.0, 1.0e-1) - - -def test__output_quadrants_to_fits(acs_ccd): - fits_path = path.join( - "{}".format(path.dirname(path.realpath(__file__))), "files", "acs" - ) - - file_path = path.join(fits_path, "acs_ccd.fits") - - acs_ccd_0 = copy.copy(acs_ccd) - acs_ccd_0[0, 0] = 10.0 - acs_ccd_0[0, -1] = 20.0 - - acs_ccd_1 = copy.copy(acs_ccd) - acs_ccd_1[-1, 0] = 30.0 - acs_ccd_1[-1, -1] = 40.0 - - create_acs_fits( - fits_path=fits_path, - acs_ccd=acs_ccd, - acs_ccd_0=acs_ccd_0, - acs_ccd_1=acs_ccd_1, - units="COUNTS", - ) - - quadrant_a = ac.acs.ImageACS.from_fits(file_path=file_path, quadrant_letter="A") - quadrant_b = ac.acs.ImageACS.from_fits(file_path=file_path, quadrant_letter="B") - quadrant_c = ac.acs.ImageACS.from_fits(file_path=file_path, quadrant_letter="C") - quadrant_d = ac.acs.ImageACS.from_fits(file_path=file_path, quadrant_letter="D") - - file_path = path.join( - "{}".format(path.dirname(path.realpath(__file__))), "files", "output.fits" - ) - - ac.acs.acs_util.output_quadrants_to_fits( - quadrant_a=quadrant_a, - quadrant_b=quadrant_b, - quadrant_c=quadrant_c, - quadrant_d=quadrant_d, - file_path=file_path, - overwrite=True, - ) - - acs_ccd_output = ac.util.array_2d.numpy_array_2d_via_fits_from( - file_path=file_path, hdu=1, do_not_scale_image_data=True - ) - - assert acs_ccd_output[0, 0] == 10.0 - assert acs_ccd_output[0, -1] == 20.0 - - acs_ccd_output = ac.util.array_2d.numpy_array_2d_via_fits_from( - file_path=file_path, hdu=4, do_not_scale_image_data=True - ) - - assert acs_ccd_output[-1, 0] == 30.0 - assert acs_ccd_output[-1, -1] == 40.0 +import numpy as np +from autoconf import fitsable +from astropy.io import fits +import copy +import shutil +import os +from os import path +import pytest + +import autocti as ac + + +def create_acs_fits( + fits_path, acs_ccd, acs_ccd_0, acs_ccd_1, units, bias_file_path=None +): + if path.exists(fits_path): + shutil.rmtree(fits_path) + + os.makedirs(fits_path) + + hdu_list = fits.HDUList() + + hdu_list.append(fits.ImageHDU(acs_ccd)) + hdu_list.append(fits.ImageHDU(acs_ccd_0)) + hdu_list.append(fits.ImageHDU(acs_ccd)) + hdu_list.append(fits.ImageHDU(acs_ccd)) + hdu_list.append(fits.ImageHDU(acs_ccd_1)) + hdu_list.append(fits.ImageHDU(acs_ccd)) + + hdu_list[0].header.set("CCDGAIN", 1.0, "Instrument GAIN") + hdu_list[0].header.set("TELESCOP", "HST", "Telescope Name") + hdu_list[0].header.set("INSTRUME", "ACS", "Instrument Name") + hdu_list[0].header.set("EXPTIME", 1000.0, "exposure duration (seconds)--calculated") + hdu_list[0].header.set( + "DATE-OBS", "2000-01-01", "UT date of start of observation (yyyy-mm-dd)" + ) + hdu_list[0].header.set( + "TIME-OBS", "00:00:00", "UT time of start of observation (hh:mm:ss)" + ) + hdu_list[0].header.set("BIASFILE", f"jref${bias_file_path}", "Bias file name") + + if units in "COUNTS": + hdu_list[1].header.set("BUNIT", "COUNTS", "brightness units") + hdu_list[4].header.set("BUNIT", "COUNTS", "brightness units") + elif units in "CPS": + hdu_list[1].header.set("BUNIT", "CPS", "brightness units") + hdu_list[4].header.set("BUNIT", "CPS", "brightness units") + + hdu_list[1].header.set( + "BSCALE", 2.0, "scale factor for array value to physical value" + ) + hdu_list[1].header.set("BZERO", 10.0, "physical value for an array value of zero") + + hdu_list[4].header.set( + "BSCALE", 2.0, "scale factor for array value to physical value" + ) + hdu_list[4].header.set("BZERO", 10.0, "physical value for an array value of zero") + + hdu_list.writeto(path.join(fits_path, "acs_ccd.fits")) + + +def create_acs_bias_fits(fits_path, bias_ccd, bias_ccd_0, bias_ccd_1): + hdu_list = fits.HDUList() + + hdu_list.append(fits.ImageHDU(bias_ccd)) + hdu_list.append(fits.ImageHDU(bias_ccd_0)) + hdu_list.append(fits.ImageHDU(bias_ccd)) + hdu_list.append(fits.ImageHDU(bias_ccd)) + hdu_list.append(fits.ImageHDU(bias_ccd_1)) + hdu_list.append(fits.ImageHDU(bias_ccd)) + + hdu_list[0].header.set("CCDGAIN", 1.0, "Instrument GAIN") + hdu_list[1].header.set("BUNIT", "COUNTS", "brightness units") + hdu_list[4].header.set("BUNIT", "COUNTS", "brightness units") + + hdu_list.writeto(path.join(fits_path, "acs_bias_ccd.fits")) + + +def test__from_fits__reads_header_from_header_correctly(acs_ccd): + fits_path = path.join( + "{}".format(path.dirname(path.realpath(__file__))), "files", "acs" + ) + + create_acs_fits( + fits_path=fits_path, + acs_ccd=acs_ccd, + acs_ccd_0=acs_ccd, + acs_ccd_1=acs_ccd, + units="COUNTS", + ) + + file_path = path.join(fits_path, "acs_ccd.fits") + + array = ac.acs.ImageACS.from_fits(file_path=file_path, quadrant_letter="B") + + assert array.header.exposure_time == 1000.0 + assert array.header.date_of_observation == "2000-01-01" + assert array.header.time_of_observation == "00:00:00" + assert array.header.modified_julian_date == 51544.0 + + array = ac.acs.ImageACS.from_fits(file_path=file_path, quadrant_letter="C") + + assert array.header.exposure_time == 1000.0 + assert array.header.date_of_observation == "2000-01-01" + assert array.header.time_of_observation == "00:00:00" + assert array.header.modified_julian_date == 51544.0 + + +def test__from_fits__in_counts__uses_fits_header_correctly_converts_and_picks_correct_quadrant( + acs_ccd, +): + fits_path = path.join( + "{}".format(path.dirname(path.realpath(__file__))), "files", "acs" + ) + + file_path = path.join(fits_path, "acs_ccd.fits") + + acs_ccd_0 = copy.copy(acs_ccd) + acs_ccd_0[0, 0] = 10.0 + acs_ccd_0[0, -1] = 20.0 + + acs_ccd_1 = copy.copy(acs_ccd) + acs_ccd_1[-1, 0] = 30.0 + acs_ccd_1[-1, -1] = 40.0 + + create_acs_fits( + fits_path=fits_path, + acs_ccd=acs_ccd, + acs_ccd_0=acs_ccd_0, + acs_ccd_1=acs_ccd_1, + units="COUNTS", + ) + + array = ac.acs.ImageACS.from_fits( + file_path=file_path, quadrant_letter="A", use_calibrated_gain=False + ) + + assert array.native[0, 0] == (30.0 * 2.0) + 10.0 + assert array.in_counts.native[0, 0] == 30.0 + assert array.shape_native == (2068, 2072) + + array_original = array.header.array_electrons_to_original( + array=array, use_calibrated_gain=False + ) + + assert array_original.native[0, 0] == 30.0 + + array = ac.acs.ImageACS.from_fits( + file_path=file_path, quadrant_letter="B", use_calibrated_gain=False + ) + + assert array.native[0, 0] == (40.0 * 2.0) + 10.0 + assert array.shape_native == (2068, 2072) + + array_original = array.header.array_electrons_to_original( + array=array, use_calibrated_gain=False + ) + + assert array_original.native[0, 0] == 40.0 + + array = ac.acs.ImageACS.from_fits( + file_path=file_path, quadrant_letter="C", use_calibrated_gain=False + ) + + assert array.native[0, 0] == (10.0 * 2.0) + 10.0 + assert array.shape_native == (2068, 2072) + + array_original = array.header.array_electrons_to_original( + array=array, use_calibrated_gain=False + ) + + assert array_original.native[0, 0] == 10.0 + + array = ac.acs.ImageACS.from_fits( + file_path=file_path, quadrant_letter="D", use_calibrated_gain=False + ) + + assert array.native[0, 0] == (20.0 * 2.0) + 10.0 + assert array.shape_native == (2068, 2072) + + array_original = array.header.array_electrons_to_original( + array=array, use_calibrated_gain=False + ) + + assert array_original.native[0, 0] == 20.0 + + +def test__from_fits__in_counts_per_second__uses_fits_header_correctly_converts_and_picks_correct_quadrant( + acs_ccd, +): + fits_path = path.join( + "{}".format(path.dirname(path.realpath(__file__))), "files", "acs" + ) + + file_path = path.join(fits_path, "acs_ccd.fits") + + acs_ccd_0 = copy.copy(acs_ccd) + acs_ccd_0[0, 0] = 10.0 + acs_ccd_0[0, -1] = 20.0 + + acs_ccd_1 = copy.copy(acs_ccd) + acs_ccd_1[-1, 0] = 30.0 + acs_ccd_1[-1, -1] = 40.0 + + create_acs_fits( + fits_path=fits_path, + acs_ccd=acs_ccd, + acs_ccd_0=acs_ccd_0, + acs_ccd_1=acs_ccd_1, + units="CPS", + ) + + array = ac.acs.ImageACS.from_fits( + file_path=file_path, quadrant_letter="A", use_calibrated_gain=False + ) + + assert array.native[0, 0] == (30.0 * 1000.0 * 2.0) + 10.0 + assert array.shape_native == (2068, 2072) + + array_original = array.header.array_electrons_to_original( + array=array, use_calibrated_gain=False + ) + + assert array_original.native[0, 0] == 30.0 + + array = ac.acs.ImageACS.from_fits( + file_path=file_path, quadrant_letter="B", use_calibrated_gain=False + ) + + assert array.native[0, 0] == (40.0 * 1000.0 * 2.0) + 10.0 + assert array.shape_native == (2068, 2072) + + array_original = array.header.array_electrons_to_original( + array=array, use_calibrated_gain=False + ) + + assert array_original.native[0, 0] == 40.0 + + array = ac.acs.ImageACS.from_fits( + file_path=file_path, quadrant_letter="C", use_calibrated_gain=False + ) + + assert array.native[0, 0] == (10.0 * 1000.0 * 2.0) + 10.0 + assert array.shape_native == (2068, 2072) + + array_original = array.header.array_electrons_to_original( + array=array, use_calibrated_gain=False + ) + + assert array_original.native[0, 0] == 10.0 + + array = ac.acs.ImageACS.from_fits( + file_path=file_path, quadrant_letter="D", use_calibrated_gain=False + ) + + assert array.native[0, 0] == (20.0 * 1000.0 * 2.0) + 10.0 + assert array.shape_native == (2068, 2072) + + array_original = array.header.array_electrons_to_original( + array=array, use_calibrated_gain=False + ) + + assert array_original.native[0, 0] == 20.0 + + +def test__from_fits__in_counts__uses_bias_prescan_correctly(acs_ccd): + fits_path = path.join( + "{}".format(path.dirname(path.realpath(__file__))), "files", "acs" + ) + + file_path = path.join(fits_path, "acs_ccd.fits") + + acs_ccd_0 = copy.copy(acs_ccd) + acs_ccd_0[0, 0] = 10.0 + acs_ccd_0[0, -1] = 20.0 + + acs_ccd_1 = copy.copy(acs_ccd) + acs_ccd_1[-1, 0] = 30.0 + acs_ccd_1[-1, -1] = 40.0 + + create_acs_fits( + fits_path=fits_path, + acs_ccd=acs_ccd, + acs_ccd_0=acs_ccd_0, + acs_ccd_1=acs_ccd_1, + units="COUNTS", + ) + + array = ac.acs.ImageACS.from_fits( + file_path=file_path, + quadrant_letter="A", + bias_subtract_via_prescan=True, + use_calibrated_gain=False, + ) + + assert array.native[0, 0] == pytest.approx(10.0, (30.0 * 2.0) + 10.0 - 10.0, 1.0e-4) + assert array.header.bias_serial_prescan_column[0][0] == pytest.approx(10.0, 1.0e-4) + + array = ac.acs.ImageACS.from_fits( + file_path=file_path, + quadrant_letter="B", + bias_subtract_via_prescan=True, + use_calibrated_gain=False, + ) + + assert array.native[0, 0] == pytest.approx(10.0, (40.0 * 2.0) + 10.0 - 10.0, 1.0e-4) + assert array.header.bias_serial_prescan_column[0][0] == pytest.approx(10.0, 1.0e-4) + + array = ac.acs.ImageACS.from_fits( + file_path=file_path, + quadrant_letter="C", + bias_subtract_via_prescan=True, + use_calibrated_gain=False, + ) + + assert array.native[0, 0] == pytest.approx(10.0, (10.0 * 2.0) + 10.0 - 10.0, 1.0e-4) + assert array.header.bias_serial_prescan_column[0][0] == pytest.approx(10.0, 1.0e-4) + + array = ac.acs.ImageACS.from_fits( + file_path=file_path, + quadrant_letter="D", + bias_subtract_via_prescan=True, + use_calibrated_gain=False, + ) + + assert array.native[0, 0] == pytest.approx(10.0, (20.0 * 2.0) + 10.0 - 10.0, 1.0e-4) + assert array.header.bias_serial_prescan_column[0][0] == pytest.approx(10.0, 1.0e-4) + + +def test__from_fits__in_counts__uses_bias_file_subtraction_correctly(acs_ccd): + fits_path = path.join( + "{}".format(path.dirname(path.realpath(__file__))), "files", "acs" + ) + + acs_ccd_0 = copy.copy(acs_ccd) + acs_ccd_0[0, 0] = 10.0 + acs_ccd_0[0, -1] = 20.0 + + acs_ccd_1 = copy.copy(acs_ccd) + acs_ccd_1[-1, 0] = 30.0 + acs_ccd_1[-1, -1] = 40.0 + + create_acs_fits( + fits_path=fits_path, + acs_ccd=acs_ccd, + acs_ccd_0=acs_ccd_0, + acs_ccd_1=acs_ccd_1, + units="COUNTS", + bias_file_path=path.join(fits_path, "acs_bias_ccd.fits"), + ) + + create_acs_bias_fits( + fits_path=fits_path, + bias_ccd=np.zeros((2068, 4144)), + bias_ccd_0=np.ones((2068, 4144)), + bias_ccd_1=2.0 * np.ones((2068, 4144)), + ) + + file_path = path.join(fits_path, "acs_ccd.fits") + + array = ac.acs.ImageACS.from_fits( + file_path=file_path, + quadrant_letter="A", + bias_subtract_via_bias_file=True, + use_calibrated_gain=False, + ) + + assert array.native[0, 0] == pytest.approx(10.0, (30.0 * 2.0) + 10.0 - 2.0, 1.0e-4) + assert array.header.bias[0] == pytest.approx(2.0, 1.0e-1) + + array = ac.acs.ImageACS.from_fits( + file_path=file_path, + quadrant_letter="B", + bias_subtract_via_bias_file=True, + use_calibrated_gain=False, + ) + + assert array.native[0, 0] == pytest.approx(10.0, (40.0 * 2.0) + 10.0 - 2.0, 1.0e-4) + assert array.header.bias[0] == pytest.approx(2.0, 1.0e-1) + + array = ac.acs.ImageACS.from_fits( + file_path=file_path, + quadrant_letter="C", + bias_subtract_via_bias_file=True, + use_calibrated_gain=False, + ) + + assert array.native[0, 0] == pytest.approx(10.0, (10.0 * 2.0) + 10.0 - 1.0, 1.0e-4) + assert array.header.bias[0] == pytest.approx(1.0, 1.0e-1) + + array = ac.acs.ImageACS.from_fits( + file_path=file_path, + quadrant_letter="D", + bias_subtract_via_bias_file=True, + use_calibrated_gain=False, + bias_file_path=path.join(fits_path, "acs_bias_ccd.fits"), + ) + + assert array.native[0, 0] == pytest.approx(10.0, (20.0 * 2.0) + 10.0 - 1.0, 1.0e-4) + assert array.header.bias[0] == pytest.approx(1.0, 1.0e-1) + + +def test__output_quadrants_to_fits(acs_ccd): + fits_path = path.join( + "{}".format(path.dirname(path.realpath(__file__))), "files", "acs" + ) + + file_path = path.join(fits_path, "acs_ccd.fits") + + acs_ccd_0 = copy.copy(acs_ccd) + acs_ccd_0[0, 0] = 10.0 + acs_ccd_0[0, -1] = 20.0 + + acs_ccd_1 = copy.copy(acs_ccd) + acs_ccd_1[-1, 0] = 30.0 + acs_ccd_1[-1, -1] = 40.0 + + create_acs_fits( + fits_path=fits_path, + acs_ccd=acs_ccd, + acs_ccd_0=acs_ccd_0, + acs_ccd_1=acs_ccd_1, + units="COUNTS", + ) + + quadrant_a = ac.acs.ImageACS.from_fits(file_path=file_path, quadrant_letter="A") + quadrant_b = ac.acs.ImageACS.from_fits(file_path=file_path, quadrant_letter="B") + quadrant_c = ac.acs.ImageACS.from_fits(file_path=file_path, quadrant_letter="C") + quadrant_d = ac.acs.ImageACS.from_fits(file_path=file_path, quadrant_letter="D") + + file_path = path.join( + "{}".format(path.dirname(path.realpath(__file__))), "files", "output.fits" + ) + + ac.acs.acs_util.output_quadrants_to_fits( + quadrant_a=quadrant_a, + quadrant_b=quadrant_b, + quadrant_c=quadrant_c, + quadrant_d=quadrant_d, + file_path=file_path, + overwrite=True, + ) + + acs_ccd_output = fitsable.ndarray_via_fits_from( + file_path=file_path, hdu=1, do_not_scale_image_data=True + ) + + assert acs_ccd_output[0, 0] == 10.0 + assert acs_ccd_output[0, -1] == 20.0 + + acs_ccd_output = fitsable.ndarray_via_fits_from( + file_path=file_path, hdu=4, do_not_scale_image_data=True + ) + + assert acs_ccd_output[-1, 0] == 30.0 + assert acs_ccd_output[-1, -1] == 40.0 diff --git a/test_autocti/mask/test_mask_2d.py b/test_autocti/mask/test_mask_2d.py index 24950111..aca4756f 100644 --- a/test_autocti/mask/test_mask_2d.py +++ b/test_autocti/mask/test_mask_2d.py @@ -1,733 +1,737 @@ -import os -from os import path -import shutil - -import numpy as np -import pytest -import autocti as ac -from autocti import exc - -test_data_path = path.join( - "{}".format(path.dirname(path.realpath(__file__))), "files", "array" -) - - -def test__manual(): - mask = ac.Mask2D(mask=[[False, False], [True, True]], pixel_scales=1.0) - - assert type(mask) == ac.Mask2D - assert (mask == np.array([[False, False], [True, True]])).all() - assert mask.pixel_scales == (1.0, 1.0) - assert mask.origin == (0.0, 0.0) - - mask = ac.Mask2D( - mask=[[False, False, True], [True, True, False]], - pixel_scales=(2.0, 3.0), - origin=(0.0, 1.0), - ) - - assert type(mask) == ac.Mask2D - assert (mask == np.array([[False, False, True], [True, True, False]])).all() - assert mask.pixel_scales == (2.0, 3.0) - assert mask.origin == (0.0, 1.0) - - mask = ac.Mask2D( - mask=[[False, False, True], [True, True, False]], pixel_scales=1.0, invert=True - ) - - assert type(mask) == ac.Mask2D - assert (mask == np.array([[True, True, False], [False, False, True]])).all() - - -def test__mask__input_is_1d_mask__no_shape_native__raises_exception(): - with pytest.raises(exc.MaskException): - ac.Mask2D(mask=[False, False, True], pixel_scales=1.0) - - with pytest.raises(exc.MaskException): - ac.Mask2D(mask=[False, False, True], pixel_scales=False) - - with pytest.raises(exc.MaskException): - ac.Mask2D(mask=[False, False, True], pixel_scales=1.0) - - with pytest.raises(exc.MaskException): - ac.Mask2D(mask=[False, False, True], pixel_scales=False) - - -def test__unmasked(): - mask = ac.Mask2D.all_false(shape_native=(5, 5), pixel_scales=1.0, invert=False) - - assert mask.shape == (5, 5) - assert ( - mask - == np.array( - [ - [False, False, False, False, False], - [False, False, False, False, False], - [False, False, False, False, False], - [False, False, False, False, False], - [False, False, False, False, False], - ] - ) - ).all() - - mask = ac.Mask2D.all_false( - shape_native=(3, 3), pixel_scales=(1.5, 1.5), invert=False - ) - - assert mask.shape == (3, 3) - assert ( - mask - == np.array( - [[False, False, False], [False, False, False], [False, False, False]] - ) - ).all() - - assert mask.pixel_scales == (1.5, 1.5) - assert mask.origin == (0.0, 0.0) - - mask = ac.Mask2D.all_false( - shape_native=(3, 3), pixel_scales=(2.0, 2.5), invert=True, origin=(1.0, 2.0) - ) - - assert mask.shape == (3, 3) - assert ( - mask == np.array([[True, True, True], [True, True, True], [True, True, True]]) - ).all() - - assert mask.pixel_scales == (2.0, 2.5) - assert mask.origin == (1.0, 2.0) - - -def test__from_masked_regions(): - mask = ac.Mask2D.from_masked_regions( - shape_native=(3, 3), masked_regions=[(0, 3, 2, 3)], pixel_scales=1.0 - ) - - assert ( - mask - == np.array([[False, False, True], [False, False, True], [False, False, True]]) - ).all() - - mask = ac.Mask2D.from_masked_regions( - shape_native=(3, 3), - masked_regions=[(0, 3, 2, 3), (0, 2, 0, 2)], - pixel_scales=1.0, - ) - - assert ( - mask == np.array([[True, True, True], [True, True, True], [False, False, True]]) - ).all() - - -def test__cosmic_ray_mask_included_in_total_mask(): - cosmic_ray_map = ac.Array2D.no_mask( - values=np.array( - [[False, False, False], [False, True, False], [False, False, False]] - ), - pixel_scales=1.0, - ) - - mask = ac.Mask2D.from_cosmic_ray_map_buffed( - cosmic_ray_map=cosmic_ray_map, - settings=ac.SettingsMask2D( - cosmic_ray_parallel_buffer=0, - cosmic_ray_serial_buffer=0, - cosmic_ray_diagonal_buffer=0, - ), - ) - - assert ( - mask - == np.array( - [[False, False, False], [False, True, False], [False, False, False]] - ) - ).all() - - cosmic_ray_map = ac.Array2D.no_mask( - values=[[False, True, False], [False, False, False], [False, False, False]], - pixel_scales=1.0, - ) - - mask = ac.Mask2D.from_cosmic_ray_map_buffed( - cosmic_ray_map=cosmic_ray_map, - settings=ac.SettingsMask2D( - cosmic_ray_parallel_buffer=2, - cosmic_ray_serial_buffer=0, - cosmic_ray_diagonal_buffer=0, - ), - ) - - assert ( - mask - == np.array([[False, True, False], [False, True, False], [False, True, False]]) - ).all() - - cosmic_ray_map = ac.Array2D.no_mask( - values=[[False, False, False], [True, False, False], [False, False, False]], - pixel_scales=1.0, - ) - - mask = ac.Mask2D.from_cosmic_ray_map_buffed( - cosmic_ray_map=cosmic_ray_map, - settings=ac.SettingsMask2D( - cosmic_ray_parallel_buffer=0, - cosmic_ray_serial_buffer=2, - cosmic_ray_diagonal_buffer=0, - ), - ) - - assert ( - mask - == np.array([[False, False, False], [True, True, True], [False, False, False]]) - ).all() - - cosmic_ray_map = ac.Array2D.no_mask( - values=[ - [False, False, False, False], - [False, True, False, False], - [False, False, False, False], - [False, False, False, False], - ], - pixel_scales=1.0, - ) - - mask = ac.Mask2D.from_cosmic_ray_map_buffed( - cosmic_ray_map=cosmic_ray_map, - settings=ac.SettingsMask2D( - cosmic_ray_parallel_buffer=0, - cosmic_ray_serial_buffer=0, - cosmic_ray_diagonal_buffer=2, - ), - ) - - assert ( - mask - == np.array( - [ - [False, False, False, False], - [False, True, True, True], - [False, True, True, True], - [False, True, True, True], - ] - ) - ).all() - - -def test__load_and_output_mask_to_fits(): - mask = ac.Mask2D.from_fits( - file_path=path.join(test_data_path, "3x3_ones.fits"), - hdu=0, - pixel_scales=(1.0, 1.0), - ) - - output_data_dir = path.join(test_data_path, "output_test") - - if path.exists(output_data_dir): - shutil.rmtree(output_data_dir) - - os.makedirs(output_data_dir) - - mask.output_to_fits(file_path=path.join(output_data_dir, "mask.fits")) - - mask = ac.Mask2D.from_fits( - file_path=path.join(output_data_dir, "mask.fits"), - hdu=0, - pixel_scales=(1.0, 1.0), - origin=(2.0, 2.0), - ) - - assert (mask == np.ones((3, 3))).all() - assert mask.pixel_scales == (1.0, 1.0) - assert mask.origin == (2.0, 2.0) - - -def test__masked_parallel_fpr_from(): - layout = ac.Layout2DCI(shape_2d=(10, 3), region_list=[(1, 4, 0, 3)]) - - mask = ac.Mask2D.masked_parallel_fpr_from( - layout=layout, - settings=ac.SettingsMask2D(parallel_fpr_pixels=(0, 2)), - pixel_scales=0.1, - ) - - assert type(mask) == ac.Mask2D - - assert ( - mask - == np.array( - [ - [False, False, False], - [True, True, True], - [True, True, True], - [False, False, False], - [False, False, False], - [False, False, False], - [False, False, False], - [False, False, False], - [False, False, False], - [False, False, False], - ] - ) - ).all() - - layout = ac.Layout2DCI(shape_2d=(10, 3), region_list=[(1, 4, 0, 3)]) - - mask = ac.Mask2D.masked_parallel_fpr_from( - layout=layout, - settings=ac.SettingsMask2D(parallel_fpr_pixels=(0, 2)), - pixel_scales=0.1, - invert=True, - ) - - assert type(mask) == ac.Mask2D - - assert ( - mask - == np.array( - [ - [True, True, True], - [False, False, False], - [False, False, False], - [True, True, True], - [True, True, True], - [True, True, True], - [True, True, True], - [True, True, True], - [True, True, True], - [True, True, True], - ] - ) - ).all() - - layout = ac.Layout2DCI(shape_2d=(10, 3), region_list=[(1, 4, 0, 1), (1, 4, 2, 3)]) - - mask = ac.Mask2D.masked_parallel_fpr_from( - layout=layout, - settings=ac.SettingsMask2D(parallel_fpr_pixels=(0, 2)), - pixel_scales=0.1, - ) - - assert type(mask) == ac.Mask2D - - assert ( - mask - == np.array( - [ - [False, False, False], - [True, False, True], - [True, False, True], - [False, False, False], - [False, False, False], - [False, False, False], - [False, False, False], - [False, False, False], - [False, False, False], - [False, False, False], - ] - ) - ).all() - - -def test__masked_parallel_eper_from(): - layout = ac.Layout2DCI(shape_2d=(10, 3), region_list=[(1, 4, 0, 3)]) - - mask = ac.Mask2D.masked_parallel_eper_from( - layout=layout, - settings=ac.SettingsMask2D(parallel_eper_pixels=(0, 4)), - pixel_scales=0.1, - ) - - assert type(mask) == ac.Mask2D - - assert ( - mask - == np.array( - [ - [False, False, False], - [False, False, False], - [False, False, False], - [False, False, False], - [True, True, True], - [True, True, True], - [True, True, True], - [True, True, True], - [False, False, False], - [False, False, False], - ] - ) - ).all() - - layout = ac.Layout2DCI(shape_2d=(10, 3), region_list=[(1, 4, 0, 3)]) - - mask = ac.Mask2D.masked_parallel_eper_from( - layout=layout, - settings=ac.SettingsMask2D(parallel_eper_pixels=(0, 4)), - pixel_scales=0.1, - invert=True, - ) - - assert type(mask) == ac.Mask2D - - assert ( - mask - == np.array( - [ - [True, True, True], - [True, True, True], - [True, True, True], - [True, True, True], - [False, False, False], - [False, False, False], - [False, False, False], - [False, False, False], - [True, True, True], - [True, True, True], - ] - ) - ).all() - - layout = ac.Layout2DCI(shape_2d=(10, 3), region_list=[(1, 4, 0, 1), (1, 4, 2, 3)]) - - mask = ac.Mask2D.masked_parallel_eper_from( - layout=layout, - settings=ac.SettingsMask2D(parallel_eper_pixels=(0, 4)), - pixel_scales=0.1, - ) - - assert type(mask) == ac.Mask2D - - assert ( - mask - == np.array( - [ - [False, False, False], - [False, False, False], - [False, False, False], - [False, False, False], - [True, False, True], - [True, False, True], - [True, False, True], - [True, False, True], - [False, False, False], - [False, False, False], - ] - ) - ).all() - - -def test__masked_serial_fpr_from(): - layout = ac.Layout2DCI(shape_2d=(3, 10), region_list=[(0, 3, 1, 4)]) - - mask = ac.Mask2D.masked_serial_fpr_from( - layout=layout, - settings=ac.SettingsMask2D(serial_fpr_pixels=(0, 2)), - pixel_scales=0.1, - ) - - assert type(mask) == ac.Mask2D - - assert ( - mask - == np.array( - [ - [False, True, True, False, False, False, False, False, False, False], - [False, True, True, False, False, False, False, False, False, False], - [False, True, True, False, False, False, False, False, False, False], - ] - ) - ).all() - - layout = ac.Layout2DCI(shape_2d=(3, 10), region_list=[(0, 3, 1, 4)]) - - mask = ac.Mask2D.masked_serial_fpr_from( - layout=layout, - settings=ac.SettingsMask2D(serial_fpr_pixels=(0, 2)), - pixel_scales=0.1, - invert=True, - ) - - assert type(mask) == ac.Mask2D - - assert ( - mask - == np.array( - [ - [True, False, False, True, True, True, True, True, True, True], - [True, False, False, True, True, True, True, True, True, True], - [True, False, False, True, True, True, True, True, True, True], - ] - ) - ).all() - - layout = ac.Layout2DCI(shape_2d=(3, 10), region_list=[(0, 1, 1, 4), (2, 3, 1, 4)]) - - mask = ac.Mask2D.masked_serial_fpr_from( - layout=layout, - settings=ac.SettingsMask2D(serial_fpr_pixels=(0, 3)), - pixel_scales=0.1, - ) - - assert type(mask) == ac.Mask2D - - assert ( - mask - == np.array( - [ - [False, True, True, True, False, False, False, False, False, False], - [False, False, False, False, False, False, False, False, False, False], - [False, True, True, True, False, False, False, False, False, False], - ] - ) - ).all() - - -def test__masked_serial_eper_from(): - layout = ac.Layout2DCI(shape_2d=(3, 10), region_list=[(0, 3, 1, 4)]) - - mask = ac.Mask2D.masked_serial_eper_from( - layout=layout, - settings=ac.SettingsMask2D(serial_eper_pixels=(0, 6)), - pixel_scales=0.1, - ) - - assert type(mask) == ac.Mask2D - - assert ( - mask - == np.array( - [ - [False, False, False, False, True, True, True, True, True, True], - [False, False, False, False, True, True, True, True, True, True], - [False, False, False, False, True, True, True, True, True, True], - ] - ) - ).all() - - mask = ac.Mask2D.masked_serial_eper_from( - layout=layout, - settings=ac.SettingsMask2D(serial_eper_pixels=(0, 6)), - pixel_scales=0.1, - invert=True, - ) - - assert type(mask) == ac.Mask2D - - assert ( - mask - == np.array( - [ - [True, True, True, True, False, False, False, False, False, False], - [True, True, True, True, False, False, False, False, False, False], - [True, True, True, True, False, False, False, False, False, False], - ] - ) - ).all() - - layout = ac.Layout2DCI(shape_2d=(3, 10), region_list=[(0, 1, 1, 4), (2, 3, 1, 4)]) - - mask = ac.Mask2D.masked_serial_eper_from( - layout=layout, - settings=ac.SettingsMask2D(serial_eper_pixels=(0, 6)), - pixel_scales=0.1, - ) - - assert type(mask) == ac.Mask2D - - assert ( - mask - == np.array( - [ - [False, False, False, False, True, True, True, True, True, True], - [False, False, False, False, False, False, False, False, False, False], - [False, False, False, False, True, True, True, True, True, True], - ] - ) - ).all() - - -def test__masked_fpr_and_eper_from(imaging_ci_7x7): - unmasked = ac.Mask2D.all_false( - shape_native=imaging_ci_7x7.shape_native, pixel_scales=1.0 - ) - - layout = ac.Layout2DCI(shape_2d=(7, 7), region_list=[(1, 5, 1, 5)]) - - mask = ac.Mask2D.masked_fpr_and_eper_from( - layout=layout, - mask=unmasked, - settings=ac.SettingsMask2D(parallel_fpr_pixels=(0, 1)), - pixel_scales=0.1, - ) - - assert ( - mask - == np.array( - [ - [False, False, False, False, False, False, False], - [False, True, True, True, True, False, False], - [False, False, False, False, False, False, False], - [False, False, False, False, False, False, False], - [False, False, False, False, False, False, False], - [False, False, False, False, False, False, False], - [False, False, False, False, False, False, False], - ] - ) - ).all() - - mask = ac.Mask2D.masked_fpr_and_eper_from( - layout=layout, - mask=unmasked, - settings=ac.SettingsMask2D(parallel_eper_pixels=(0, 1)), - pixel_scales=0.1, - ) - - assert ( - mask - == np.array( - [ - [False, False, False, False, False, False, False], - [False, False, False, False, False, False, False], - [False, False, False, False, False, False, False], - [False, False, False, False, False, False, False], - [False, False, False, False, False, False, False], - [False, True, True, True, True, False, False], - [False, False, False, False, False, False, False], - ] - ) - ).all() - - mask = ac.Mask2D.masked_fpr_and_eper_from( - layout=layout, - mask=unmasked, - settings=ac.SettingsMask2D(serial_fpr_pixels=(0, 1)), - pixel_scales=0.1, - ) - - assert ( - mask - == np.array( - [ - [False, False, False, False, False, False, False], - [False, True, False, False, False, False, False], - [False, True, False, False, False, False, False], - [False, True, False, False, False, False, False], - [False, True, False, False, False, False, False], - [False, False, False, False, False, False, False], - [False, False, False, False, False, False, False], - ] - ) - ).all() - - mask = ac.Mask2D.masked_fpr_and_eper_from( - layout=layout, - mask=unmasked, - settings=ac.SettingsMask2D(serial_eper_pixels=(0, 1)), - pixel_scales=0.1, - ) - - assert ( - mask - == np.array( - [ - [False, False, False, False, False, False, False], - [False, False, False, False, False, True, False], - [False, False, False, False, False, True, False], - [False, False, False, False, False, True, False], - [False, False, False, False, False, True, False], - [False, False, False, False, False, False, False], - [False, False, False, False, False, False, False], - ] - ) - ).all() - - -def test__masked_readout_persistence_from(): - layout = ac.Layout2DCI(shape_2d=(4, 3), region_list=[(1, 4, 0, 3)]) - - mask = ac.Mask2D.masked_readout_persistence_from( - layout=layout, - row_value_list=[1.0, 2.0, 3.0, 4.0], - readout_persistence_threshold=1.5, - settings=ac.SettingsMask2D(), - pixel_scales=0.1, - ) - - assert ( - mask - == np.array( - [ - [False, False, False], - [True, True, True], - [True, True, True], - [True, True, True], - ] - ) - ).all() - - layout = ac.Layout2DCI(shape_2d=(6, 2), region_list=[(1, 2, 0, 2)]) - - mask = ac.Mask2D.masked_readout_persistence_from( - layout=layout, - row_value_list=[1.0, 2.0, 3.0, 4.0, 2.0, 3.5], - readout_persistence_threshold=3.1, - settings=ac.SettingsMask2D(), - pixel_scales=0.1, - ) - - assert ( - mask - == np.array( - [ - [False, False], - [False, False], - [False, False], - [True, True], - [False, False], - [True, True], - ] - ) - ).all() - - mask = ac.Mask2D.masked_readout_persistence_from( - layout=layout, - row_value_list=[1.0, 2.0, 3.0, 4.0, 2.0, 3.5], - readout_persistence_threshold=3.1, - settings=ac.SettingsMask2D(readout_persistence_infront_buffer=1), - pixel_scales=0.1, - ) - - assert ( - mask - == np.array( - [ - [False, False], - [False, False], - [True, True], - [True, True], - [True, True], - [True, True], - ] - ) - ).all() - - mask = ac.Mask2D.masked_readout_persistence_from( - layout=layout, - row_value_list=[1.0, 2.0, 3.0, 4.0, 2.0, 3.5], - readout_persistence_threshold=3.1, - settings=ac.SettingsMask2D(readout_persistence_behind_buffer=1), - pixel_scales=0.1, - ) - - assert ( - mask - == np.array( - [ - [False, False], - [False, False], - [False, False], - [True, True], - [True, True], - [True, True], - ] - ) - ).all() +import os +from os import path +import shutil + +import numpy as np +from autoconf import fitsable +import pytest +import autocti as ac +from autocti import exc + +test_data_path = path.join( + "{}".format(path.dirname(path.realpath(__file__))), "files", "array" +) + + +def test__manual(): + mask = ac.Mask2D(mask=[[False, False], [True, True]], pixel_scales=1.0) + + assert type(mask) == ac.Mask2D + assert (mask == np.array([[False, False], [True, True]])).all() + assert mask.pixel_scales == (1.0, 1.0) + assert mask.origin == (0.0, 0.0) + + mask = ac.Mask2D( + mask=[[False, False, True], [True, True, False]], + pixel_scales=(2.0, 3.0), + origin=(0.0, 1.0), + ) + + assert type(mask) == ac.Mask2D + assert (mask == np.array([[False, False, True], [True, True, False]])).all() + assert mask.pixel_scales == (2.0, 3.0) + assert mask.origin == (0.0, 1.0) + + mask = ac.Mask2D( + mask=[[False, False, True], [True, True, False]], pixel_scales=1.0, invert=True + ) + + assert type(mask) == ac.Mask2D + assert (mask == np.array([[True, True, False], [False, False, True]])).all() + + +def test__mask__input_is_1d_mask__no_shape_native__raises_exception(): + with pytest.raises(exc.MaskException): + ac.Mask2D(mask=[False, False, True], pixel_scales=1.0) + + with pytest.raises(exc.MaskException): + ac.Mask2D(mask=[False, False, True], pixel_scales=False) + + with pytest.raises(exc.MaskException): + ac.Mask2D(mask=[False, False, True], pixel_scales=1.0) + + with pytest.raises(exc.MaskException): + ac.Mask2D(mask=[False, False, True], pixel_scales=False) + + +def test__unmasked(): + mask = ac.Mask2D.all_false(shape_native=(5, 5), pixel_scales=1.0, invert=False) + + assert mask.shape == (5, 5) + assert ( + mask + == np.array( + [ + [False, False, False, False, False], + [False, False, False, False, False], + [False, False, False, False, False], + [False, False, False, False, False], + [False, False, False, False, False], + ] + ) + ).all() + + mask = ac.Mask2D.all_false( + shape_native=(3, 3), pixel_scales=(1.5, 1.5), invert=False + ) + + assert mask.shape == (3, 3) + assert ( + mask + == np.array( + [[False, False, False], [False, False, False], [False, False, False]] + ) + ).all() + + assert mask.pixel_scales == (1.5, 1.5) + assert mask.origin == (0.0, 0.0) + + mask = ac.Mask2D.all_false( + shape_native=(3, 3), pixel_scales=(2.0, 2.5), invert=True, origin=(1.0, 2.0) + ) + + assert mask.shape == (3, 3) + assert ( + mask == np.array([[True, True, True], [True, True, True], [True, True, True]]) + ).all() + + assert mask.pixel_scales == (2.0, 2.5) + assert mask.origin == (1.0, 2.0) + + +def test__from_masked_regions(): + mask = ac.Mask2D.from_masked_regions( + shape_native=(3, 3), masked_regions=[(0, 3, 2, 3)], pixel_scales=1.0 + ) + + assert ( + mask + == np.array([[False, False, True], [False, False, True], [False, False, True]]) + ).all() + + mask = ac.Mask2D.from_masked_regions( + shape_native=(3, 3), + masked_regions=[(0, 3, 2, 3), (0, 2, 0, 2)], + pixel_scales=1.0, + ) + + assert ( + mask == np.array([[True, True, True], [True, True, True], [False, False, True]]) + ).all() + + +def test__cosmic_ray_mask_included_in_total_mask(): + cosmic_ray_map = ac.Array2D.no_mask( + values=np.array( + [[False, False, False], [False, True, False], [False, False, False]] + ), + pixel_scales=1.0, + ) + + mask = ac.Mask2D.from_cosmic_ray_map_buffed( + cosmic_ray_map=cosmic_ray_map, + settings=ac.SettingsMask2D( + cosmic_ray_parallel_buffer=0, + cosmic_ray_serial_buffer=0, + cosmic_ray_diagonal_buffer=0, + ), + ) + + assert ( + mask + == np.array( + [[False, False, False], [False, True, False], [False, False, False]] + ) + ).all() + + cosmic_ray_map = ac.Array2D.no_mask( + values=[[False, True, False], [False, False, False], [False, False, False]], + pixel_scales=1.0, + ) + + mask = ac.Mask2D.from_cosmic_ray_map_buffed( + cosmic_ray_map=cosmic_ray_map, + settings=ac.SettingsMask2D( + cosmic_ray_parallel_buffer=2, + cosmic_ray_serial_buffer=0, + cosmic_ray_diagonal_buffer=0, + ), + ) + + assert ( + mask + == np.array([[False, True, False], [False, True, False], [False, True, False]]) + ).all() + + cosmic_ray_map = ac.Array2D.no_mask( + values=[[False, False, False], [True, False, False], [False, False, False]], + pixel_scales=1.0, + ) + + mask = ac.Mask2D.from_cosmic_ray_map_buffed( + cosmic_ray_map=cosmic_ray_map, + settings=ac.SettingsMask2D( + cosmic_ray_parallel_buffer=0, + cosmic_ray_serial_buffer=2, + cosmic_ray_diagonal_buffer=0, + ), + ) + + assert ( + mask + == np.array([[False, False, False], [True, True, True], [False, False, False]]) + ).all() + + cosmic_ray_map = ac.Array2D.no_mask( + values=[ + [False, False, False, False], + [False, True, False, False], + [False, False, False, False], + [False, False, False, False], + ], + pixel_scales=1.0, + ) + + mask = ac.Mask2D.from_cosmic_ray_map_buffed( + cosmic_ray_map=cosmic_ray_map, + settings=ac.SettingsMask2D( + cosmic_ray_parallel_buffer=0, + cosmic_ray_serial_buffer=0, + cosmic_ray_diagonal_buffer=2, + ), + ) + + assert ( + mask + == np.array( + [ + [False, False, False, False], + [False, True, True, True], + [False, True, True, True], + [False, True, True, True], + ] + ) + ).all() + + +def test__load_and_output_mask_to_fits(): + mask = ac.Mask2D.from_fits( + file_path=path.join(test_data_path, "3x3_ones.fits"), + hdu=0, + pixel_scales=(1.0, 1.0), + ) + + output_data_dir = path.join(test_data_path, "output_test") + + if path.exists(output_data_dir): + shutil.rmtree(output_data_dir) + + os.makedirs(output_data_dir) + + fitsable.output_to_fits( + values=np.asarray(mask).astype("float"), + file_path=path.join(output_data_dir, "mask.fits"), + ) + + mask = ac.Mask2D.from_fits( + file_path=path.join(output_data_dir, "mask.fits"), + hdu=0, + pixel_scales=(1.0, 1.0), + origin=(2.0, 2.0), + ) + + assert (mask == np.ones((3, 3))).all() + assert mask.pixel_scales == (1.0, 1.0) + assert mask.origin == (2.0, 2.0) + + +def test__masked_parallel_fpr_from(): + layout = ac.Layout2DCI(shape_2d=(10, 3), region_list=[(1, 4, 0, 3)]) + + mask = ac.Mask2D.masked_parallel_fpr_from( + layout=layout, + settings=ac.SettingsMask2D(parallel_fpr_pixels=(0, 2)), + pixel_scales=0.1, + ) + + assert type(mask) == ac.Mask2D + + assert ( + mask + == np.array( + [ + [False, False, False], + [True, True, True], + [True, True, True], + [False, False, False], + [False, False, False], + [False, False, False], + [False, False, False], + [False, False, False], + [False, False, False], + [False, False, False], + ] + ) + ).all() + + layout = ac.Layout2DCI(shape_2d=(10, 3), region_list=[(1, 4, 0, 3)]) + + mask = ac.Mask2D.masked_parallel_fpr_from( + layout=layout, + settings=ac.SettingsMask2D(parallel_fpr_pixels=(0, 2)), + pixel_scales=0.1, + invert=True, + ) + + assert type(mask) == ac.Mask2D + + assert ( + mask + == np.array( + [ + [True, True, True], + [False, False, False], + [False, False, False], + [True, True, True], + [True, True, True], + [True, True, True], + [True, True, True], + [True, True, True], + [True, True, True], + [True, True, True], + ] + ) + ).all() + + layout = ac.Layout2DCI(shape_2d=(10, 3), region_list=[(1, 4, 0, 1), (1, 4, 2, 3)]) + + mask = ac.Mask2D.masked_parallel_fpr_from( + layout=layout, + settings=ac.SettingsMask2D(parallel_fpr_pixels=(0, 2)), + pixel_scales=0.1, + ) + + assert type(mask) == ac.Mask2D + + assert ( + mask + == np.array( + [ + [False, False, False], + [True, False, True], + [True, False, True], + [False, False, False], + [False, False, False], + [False, False, False], + [False, False, False], + [False, False, False], + [False, False, False], + [False, False, False], + ] + ) + ).all() + + +def test__masked_parallel_eper_from(): + layout = ac.Layout2DCI(shape_2d=(10, 3), region_list=[(1, 4, 0, 3)]) + + mask = ac.Mask2D.masked_parallel_eper_from( + layout=layout, + settings=ac.SettingsMask2D(parallel_eper_pixels=(0, 4)), + pixel_scales=0.1, + ) + + assert type(mask) == ac.Mask2D + + assert ( + mask + == np.array( + [ + [False, False, False], + [False, False, False], + [False, False, False], + [False, False, False], + [True, True, True], + [True, True, True], + [True, True, True], + [True, True, True], + [False, False, False], + [False, False, False], + ] + ) + ).all() + + layout = ac.Layout2DCI(shape_2d=(10, 3), region_list=[(1, 4, 0, 3)]) + + mask = ac.Mask2D.masked_parallel_eper_from( + layout=layout, + settings=ac.SettingsMask2D(parallel_eper_pixels=(0, 4)), + pixel_scales=0.1, + invert=True, + ) + + assert type(mask) == ac.Mask2D + + assert ( + mask + == np.array( + [ + [True, True, True], + [True, True, True], + [True, True, True], + [True, True, True], + [False, False, False], + [False, False, False], + [False, False, False], + [False, False, False], + [True, True, True], + [True, True, True], + ] + ) + ).all() + + layout = ac.Layout2DCI(shape_2d=(10, 3), region_list=[(1, 4, 0, 1), (1, 4, 2, 3)]) + + mask = ac.Mask2D.masked_parallel_eper_from( + layout=layout, + settings=ac.SettingsMask2D(parallel_eper_pixels=(0, 4)), + pixel_scales=0.1, + ) + + assert type(mask) == ac.Mask2D + + assert ( + mask + == np.array( + [ + [False, False, False], + [False, False, False], + [False, False, False], + [False, False, False], + [True, False, True], + [True, False, True], + [True, False, True], + [True, False, True], + [False, False, False], + [False, False, False], + ] + ) + ).all() + + +def test__masked_serial_fpr_from(): + layout = ac.Layout2DCI(shape_2d=(3, 10), region_list=[(0, 3, 1, 4)]) + + mask = ac.Mask2D.masked_serial_fpr_from( + layout=layout, + settings=ac.SettingsMask2D(serial_fpr_pixels=(0, 2)), + pixel_scales=0.1, + ) + + assert type(mask) == ac.Mask2D + + assert ( + mask + == np.array( + [ + [False, True, True, False, False, False, False, False, False, False], + [False, True, True, False, False, False, False, False, False, False], + [False, True, True, False, False, False, False, False, False, False], + ] + ) + ).all() + + layout = ac.Layout2DCI(shape_2d=(3, 10), region_list=[(0, 3, 1, 4)]) + + mask = ac.Mask2D.masked_serial_fpr_from( + layout=layout, + settings=ac.SettingsMask2D(serial_fpr_pixels=(0, 2)), + pixel_scales=0.1, + invert=True, + ) + + assert type(mask) == ac.Mask2D + + assert ( + mask + == np.array( + [ + [True, False, False, True, True, True, True, True, True, True], + [True, False, False, True, True, True, True, True, True, True], + [True, False, False, True, True, True, True, True, True, True], + ] + ) + ).all() + + layout = ac.Layout2DCI(shape_2d=(3, 10), region_list=[(0, 1, 1, 4), (2, 3, 1, 4)]) + + mask = ac.Mask2D.masked_serial_fpr_from( + layout=layout, + settings=ac.SettingsMask2D(serial_fpr_pixels=(0, 3)), + pixel_scales=0.1, + ) + + assert type(mask) == ac.Mask2D + + assert ( + mask + == np.array( + [ + [False, True, True, True, False, False, False, False, False, False], + [False, False, False, False, False, False, False, False, False, False], + [False, True, True, True, False, False, False, False, False, False], + ] + ) + ).all() + + +def test__masked_serial_eper_from(): + layout = ac.Layout2DCI(shape_2d=(3, 10), region_list=[(0, 3, 1, 4)]) + + mask = ac.Mask2D.masked_serial_eper_from( + layout=layout, + settings=ac.SettingsMask2D(serial_eper_pixels=(0, 6)), + pixel_scales=0.1, + ) + + assert type(mask) == ac.Mask2D + + assert ( + mask + == np.array( + [ + [False, False, False, False, True, True, True, True, True, True], + [False, False, False, False, True, True, True, True, True, True], + [False, False, False, False, True, True, True, True, True, True], + ] + ) + ).all() + + mask = ac.Mask2D.masked_serial_eper_from( + layout=layout, + settings=ac.SettingsMask2D(serial_eper_pixels=(0, 6)), + pixel_scales=0.1, + invert=True, + ) + + assert type(mask) == ac.Mask2D + + assert ( + mask + == np.array( + [ + [True, True, True, True, False, False, False, False, False, False], + [True, True, True, True, False, False, False, False, False, False], + [True, True, True, True, False, False, False, False, False, False], + ] + ) + ).all() + + layout = ac.Layout2DCI(shape_2d=(3, 10), region_list=[(0, 1, 1, 4), (2, 3, 1, 4)]) + + mask = ac.Mask2D.masked_serial_eper_from( + layout=layout, + settings=ac.SettingsMask2D(serial_eper_pixels=(0, 6)), + pixel_scales=0.1, + ) + + assert type(mask) == ac.Mask2D + + assert ( + mask + == np.array( + [ + [False, False, False, False, True, True, True, True, True, True], + [False, False, False, False, False, False, False, False, False, False], + [False, False, False, False, True, True, True, True, True, True], + ] + ) + ).all() + + +def test__masked_fpr_and_eper_from(imaging_ci_7x7): + unmasked = ac.Mask2D.all_false( + shape_native=imaging_ci_7x7.shape_native, pixel_scales=1.0 + ) + + layout = ac.Layout2DCI(shape_2d=(7, 7), region_list=[(1, 5, 1, 5)]) + + mask = ac.Mask2D.masked_fpr_and_eper_from( + layout=layout, + mask=unmasked, + settings=ac.SettingsMask2D(parallel_fpr_pixels=(0, 1)), + pixel_scales=0.1, + ) + + assert ( + mask + == np.array( + [ + [False, False, False, False, False, False, False], + [False, True, True, True, True, False, False], + [False, False, False, False, False, False, False], + [False, False, False, False, False, False, False], + [False, False, False, False, False, False, False], + [False, False, False, False, False, False, False], + [False, False, False, False, False, False, False], + ] + ) + ).all() + + mask = ac.Mask2D.masked_fpr_and_eper_from( + layout=layout, + mask=unmasked, + settings=ac.SettingsMask2D(parallel_eper_pixels=(0, 1)), + pixel_scales=0.1, + ) + + assert ( + mask + == np.array( + [ + [False, False, False, False, False, False, False], + [False, False, False, False, False, False, False], + [False, False, False, False, False, False, False], + [False, False, False, False, False, False, False], + [False, False, False, False, False, False, False], + [False, True, True, True, True, False, False], + [False, False, False, False, False, False, False], + ] + ) + ).all() + + mask = ac.Mask2D.masked_fpr_and_eper_from( + layout=layout, + mask=unmasked, + settings=ac.SettingsMask2D(serial_fpr_pixels=(0, 1)), + pixel_scales=0.1, + ) + + assert ( + mask + == np.array( + [ + [False, False, False, False, False, False, False], + [False, True, False, False, False, False, False], + [False, True, False, False, False, False, False], + [False, True, False, False, False, False, False], + [False, True, False, False, False, False, False], + [False, False, False, False, False, False, False], + [False, False, False, False, False, False, False], + ] + ) + ).all() + + mask = ac.Mask2D.masked_fpr_and_eper_from( + layout=layout, + mask=unmasked, + settings=ac.SettingsMask2D(serial_eper_pixels=(0, 1)), + pixel_scales=0.1, + ) + + assert ( + mask + == np.array( + [ + [False, False, False, False, False, False, False], + [False, False, False, False, False, True, False], + [False, False, False, False, False, True, False], + [False, False, False, False, False, True, False], + [False, False, False, False, False, True, False], + [False, False, False, False, False, False, False], + [False, False, False, False, False, False, False], + ] + ) + ).all() + + +def test__masked_readout_persistence_from(): + layout = ac.Layout2DCI(shape_2d=(4, 3), region_list=[(1, 4, 0, 3)]) + + mask = ac.Mask2D.masked_readout_persistence_from( + layout=layout, + row_value_list=[1.0, 2.0, 3.0, 4.0], + readout_persistence_threshold=1.5, + settings=ac.SettingsMask2D(), + pixel_scales=0.1, + ) + + assert ( + mask + == np.array( + [ + [False, False, False], + [True, True, True], + [True, True, True], + [True, True, True], + ] + ) + ).all() + + layout = ac.Layout2DCI(shape_2d=(6, 2), region_list=[(1, 2, 0, 2)]) + + mask = ac.Mask2D.masked_readout_persistence_from( + layout=layout, + row_value_list=[1.0, 2.0, 3.0, 4.0, 2.0, 3.5], + readout_persistence_threshold=3.1, + settings=ac.SettingsMask2D(), + pixel_scales=0.1, + ) + + assert ( + mask + == np.array( + [ + [False, False], + [False, False], + [False, False], + [True, True], + [False, False], + [True, True], + ] + ) + ).all() + + mask = ac.Mask2D.masked_readout_persistence_from( + layout=layout, + row_value_list=[1.0, 2.0, 3.0, 4.0, 2.0, 3.5], + readout_persistence_threshold=3.1, + settings=ac.SettingsMask2D(readout_persistence_infront_buffer=1), + pixel_scales=0.1, + ) + + assert ( + mask + == np.array( + [ + [False, False], + [False, False], + [True, True], + [True, True], + [True, True], + [True, True], + ] + ) + ).all() + + mask = ac.Mask2D.masked_readout_persistence_from( + layout=layout, + row_value_list=[1.0, 2.0, 3.0, 4.0, 2.0, 3.5], + readout_persistence_threshold=3.1, + settings=ac.SettingsMask2D(readout_persistence_behind_buffer=1), + pixel_scales=0.1, + ) + + assert ( + mask + == np.array( + [ + [False, False], + [False, False], + [False, False], + [True, True], + [True, True], + [True, True], + ] + ) + ).all()