Skip to content

Commit 12a3eca

Browse files
Jammy2211Jammy2211
authored
Merge pull request #87 from PyAutoLabs/feature/cti-resurrection-phase2
CTI resurrection Phase 2: autofit sync — factor-graph aggregator port
2 parents 0aad137 + 1b79264 commit 12a3eca

7 files changed

Lines changed: 517 additions & 468 deletions

File tree

‎AGENTS.md‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,10 @@ ecosystem via the CTI resurrection epic
2626
visualization layer rewritten on the matplotlib **function API**, mirroring
2727
PyAutoGalaxy: per-domain `plot/*_plots.py` function modules, config-gated
2828
`model/plotter.py` orchestrators, `autocti/util/plot_utils.py` helpers) are
29-
complete. Remaining: Phase 2 autofit sync (5 aggregator tests skipped pending
30-
the `AnalysisFactor`/`FactorGraphModel` port), Phase 3 CI + ecosystem
31-
plumbing, Phase 4 workspace update, Phase 5 workspace_test rebuild + release.
29+
complete, as is Phase 2 (autofit sync: multi-dataset fits and the aggregator
30+
run through `af.AnalysisFactor`/`af.FactorGraphModel`; the test suite has no
31+
skips). Remaining: Phase 3 CI + ecosystem plumbing, Phase 4 workspace update,
32+
Phase 5 workspace_test rebuild + release.
3233

3334
## arcticpy (read before installing)
3435

‎autocti/aggregator/fit_dataset_1d.py‎

Lines changed: 183 additions & 159 deletions
Original file line numberDiff line numberDiff line change
@@ -1,159 +1,183 @@
1-
from __future__ import annotations
2-
from typing import TYPE_CHECKING, List, Optional
3-
4-
from autocti.aggregator.abstract import AggBase
5-
6-
if TYPE_CHECKING:
7-
from autocti.clocker.abstract import AbstractClocker
8-
from autocti.dataset_1d.fit import FitDataset1D
9-
10-
import autofit as af
11-
12-
from autocti.aggregator.dataset_1d import _dataset_1d_list_from
13-
14-
15-
def _fit_dataset_1d_list_from(
16-
fit: af.Fit,
17-
instance: Optional[af.ModelInstance] = None,
18-
use_dataset_full: bool = False,
19-
clocker_list: Optional[AbstractClocker] = None,
20-
) -> List[FitDataset1D]:
21-
"""
22-
Returns a list of `FitDataset1D` object from a `PyAutoFit` sqlite database `Fit` object.
23-
24-
The results of a model-fit can be stored in a sqlite database, including the following attributes of the fit:
25-
26-
- The masked dataset (e.g. data / noise map / pre cti data) as .fits files (contained in `dataset` folder).
27-
- The clocker used to add CTI in the fit (`dataset/clocker.json`).
28-
- The settings used for clocking CIT (contained in `dataset/settings_cti.json`).
29-
30-
Each individual attribute can be loaded from the database via the `fit.value()` method.
31-
32-
This method combines all of these attributes and returns a list of `FitDataset1D` objects, by loading the masked
33-
dataset adding CTI to its pre-cti data via the cti model and clocking and fitting the model image to the dataset.
34-
35-
If multiple `Dataset1D` objects were fitted simultaneously via analysis summing, the `fit.child_values()` method
36-
is instead used to load lists of the datasets, perform the fit and return a list of `FitDataset1D` objects.
37-
38-
If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible
39-
to the database, the input `use_dataset_full` can be switched in to fit the full dataset instead.
40-
41-
Parameters
42-
----------
43-
fit
44-
A `PyAutoFit` `Fit` object which contains the results of a model-fit as an entry in a sqlite database.
45-
instance
46-
A manual instance that overwrites the max log likelihood instance in fit (e.g. for drawing the instance
47-
randomly from the PDF).
48-
use_dataset_full
49-
If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible
50-
to the database, the input `use_dataset_full` can be switched in to load instead the full `Dataset1D` objects.
51-
clocker_list
52-
If input, overwrites the clocker used in the fit with a new clocker which is used to perform the fit.
53-
"""
54-
55-
from autocti.dataset_1d.fit import FitDataset1D
56-
57-
dataset_list = _dataset_1d_list_from(fit=fit, use_dataset_full=use_dataset_full)
58-
59-
if clocker_list is None:
60-
if not fit.children:
61-
clocker_list = [fit.value(name="clocker")]
62-
else:
63-
clocker_list = fit.child_values(name="clocker")
64-
65-
if instance is not None:
66-
cti = instance.cti
67-
else:
68-
cti = fit.instance.cti
69-
70-
post_cti_data_list = [
71-
clocker.add_cti(data=dataset.pre_cti_data, cti=cti)
72-
for dataset, clocker in zip(dataset_list, clocker_list)
73-
]
74-
75-
return [
76-
FitDataset1D(
77-
dataset=dataset,
78-
post_cti_data=post_cti_data,
79-
)
80-
for dataset, post_cti_data in zip(dataset_list, post_cti_data_list)
81-
]
82-
83-
84-
class FitDataset1DAgg(AggBase):
85-
def __init__(
86-
self,
87-
aggregator: af.Aggregator,
88-
use_dataset_full: bool = False,
89-
clocker_list: Optional[List[AbstractClocker]] = None,
90-
):
91-
"""
92-
Interfaces with an `PyAutoFit` aggregator object to create instances of `Dataset1D` objects from the results
93-
of a model-fit.
94-
95-
The results of a model-fit can be stored in a sqlite database, including the following attributes of the fit:
96-
97-
- The masked dataset (e.g. data / noise map / pre cti data) as .fits files (contained in `dataset` folder).
98-
- The clocker used to add CTI in the fit (`dataset/clocker.json`).
99-
- The settings used for clocking CIT (contained in `dataset/settings_cti.json`).
100-
101-
The `aggregator` contains the path to each of these files, and they can be loaded individually. This class
102-
can load them all at once and create a `FitDataset1D` object via the `_fit_dataset_1d_from` method.
103-
104-
This class's methods returns generators which create the instances of the `FitDataset1D` objects. This ensures
105-
that large sets of results can be efficiently loaded from the hard-disk and do not require storing all
106-
`Dataset1D` instances in the memory at once.
107-
108-
For example, if the `aggregator` contains 3 model-fits, this class can be used to create a generator which
109-
creates instances of the corresponding 3 `Dataset1D` objects.
110-
111-
If multiple `Dataset1D` objects were fitted simultaneously via analysis summing, the `fit.child_values()` method
112-
is instead used to load lists of the datasets, perform the fit and return a list of `FitDataset1D` objects.
113-
114-
If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible
115-
to the database, the input `use_dataset_full` can be switched in to fit the full dataset instead.
116-
117-
This can be done manually, but this object provides a more concise API.
118-
119-
Parameters
120-
----------
121-
aggregator
122-
A `PyAutoFit` aggregator object which can load the results of model-fits.
123-
use_dataset_full
124-
If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore
125-
accessible to the database, the input `use_dataset_full` can be switched in to load instead the
126-
full `Dataset1D` objects.
127-
clocker_list
128-
If input, overwrites the clocker used in the fit with a new clocker which is used to perform the fit.
129-
"""
130-
super().__init__(
131-
aggregator=aggregator,
132-
use_dataset_full=use_dataset_full,
133-
clocker_list=clocker_list,
134-
)
135-
136-
def object_via_gen_from(
137-
self, fit, instance: Optional[af.ModelInstance] = None
138-
) -> List[FitDataset1D]:
139-
"""
140-
Returns a generator of `FitDataset1D` objects from an input aggregator.
141-
142-
See `__init__` for a description of how the `FitDataset1D` objects are created by this method.
143-
144-
If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible
145-
to the database, the input `use_dataset_full` can be switched in to fit the full dataset instead.
146-
147-
Parameters
148-
----------
149-
fit
150-
A `PyAutoFit` `Fit` object which contains the results of a model-fit as an entry in a sqlite database.
151-
cti
152-
The CTI model used to add CTI to the dataset to perform the fit.
153-
"""
154-
return _fit_dataset_1d_list_from(
155-
fit=fit,
156-
instance=instance,
157-
use_dataset_full=self.use_dataset_full,
158-
clocker_list=self.clocker_list,
159-
)
1+
from __future__ import annotations
2+
from typing import TYPE_CHECKING, List, Optional
3+
4+
from autocti.aggregator.abstract import AggBase
5+
6+
if TYPE_CHECKING:
7+
from autocti.clocker.abstract import AbstractClocker
8+
from autocti.dataset_1d.fit import FitDataset1D
9+
10+
import autofit as af
11+
12+
from autocti.aggregator.dataset_1d import _dataset_1d_list_from
13+
14+
15+
def _cti_list_from(source, total_datasets: int):
16+
"""
17+
Extract one CTI model per dataset from a model instance.
18+
19+
A single-analysis instance exposes ``instance.cti`` directly; a factor-graph
20+
instance (multi-dataset fit) is an indexed collection with one child
21+
instance per factor.
22+
"""
23+
if hasattr(source, "cti"):
24+
return [source.cti] * total_datasets
25+
26+
# A factor-graph instance also carries the FactorGraphModel itself as a
27+
# trailing child, so only children with a CTI model are taken.
28+
cti_list = [child.cti for child in source if hasattr(child, "cti")]
29+
30+
if len(cti_list) != total_datasets:
31+
raise ValueError(
32+
f"The instance contains {len(cti_list)} CTI models but the fit has "
33+
f"{total_datasets} datasets."
34+
)
35+
36+
return cti_list
37+
38+
39+
def _fit_dataset_1d_list_from(
40+
fit: af.Fit,
41+
instance: Optional[af.ModelInstance] = None,
42+
use_dataset_full: bool = False,
43+
clocker_list: Optional[AbstractClocker] = None,
44+
) -> List[FitDataset1D]:
45+
"""
46+
Returns a list of `FitDataset1D` object from a `PyAutoFit` sqlite database `Fit` object.
47+
48+
The results of a model-fit can be stored in a sqlite database, including the following attributes of the fit:
49+
50+
- The masked dataset (e.g. data / noise map / pre cti data) as .fits files (contained in `dataset` folder).
51+
- The clocker used to add CTI in the fit (`dataset/clocker.json`).
52+
- The settings used for clocking CIT (contained in `dataset/settings_cti.json`).
53+
54+
Each individual attribute can be loaded from the database via the `fit.value()` method.
55+
56+
This method combines all of these attributes and returns a list of `FitDataset1D` objects, by loading the masked
57+
dataset adding CTI to its pre-cti data via the cti model and clocking and fitting the model image to the dataset.
58+
59+
If multiple `Dataset1D` objects were fitted simultaneously via analysis summing, the `fit.child_values()` method
60+
is instead used to load lists of the datasets, perform the fit and return a list of `FitDataset1D` objects.
61+
62+
If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible
63+
to the database, the input `use_dataset_full` can be switched in to fit the full dataset instead.
64+
65+
Parameters
66+
----------
67+
fit
68+
A `PyAutoFit` `Fit` object which contains the results of a model-fit as an entry in a sqlite database.
69+
instance
70+
A manual instance that overwrites the max log likelihood instance in fit (e.g. for drawing the instance
71+
randomly from the PDF).
72+
use_dataset_full
73+
If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible
74+
to the database, the input `use_dataset_full` can be switched in to load instead the full `Dataset1D` objects.
75+
clocker_list
76+
If input, overwrites the clocker used in the fit with a new clocker which is used to perform the fit.
77+
"""
78+
79+
from autocti.dataset_1d.fit import FitDataset1D
80+
81+
dataset_list = _dataset_1d_list_from(fit=fit, use_dataset_full=use_dataset_full)
82+
83+
if clocker_list is None:
84+
if not fit.children:
85+
clocker_list = [fit.value(name="clocker")]
86+
else:
87+
clocker_list = fit.child_values(name="clocker")
88+
89+
cti_list = _cti_list_from(
90+
source=instance if instance is not None else fit.instance,
91+
total_datasets=len(dataset_list),
92+
)
93+
94+
post_cti_data_list = [
95+
clocker.add_cti(data=dataset.pre_cti_data, cti=cti)
96+
for dataset, clocker, cti in zip(dataset_list, clocker_list, cti_list)
97+
]
98+
99+
return [
100+
FitDataset1D(
101+
dataset=dataset,
102+
post_cti_data=post_cti_data,
103+
)
104+
for dataset, post_cti_data in zip(dataset_list, post_cti_data_list)
105+
]
106+
107+
108+
class FitDataset1DAgg(AggBase):
109+
def __init__(
110+
self,
111+
aggregator: af.Aggregator,
112+
use_dataset_full: bool = False,
113+
clocker_list: Optional[List[AbstractClocker]] = None,
114+
):
115+
"""
116+
Interfaces with an `PyAutoFit` aggregator object to create instances of `Dataset1D` objects from the results
117+
of a model-fit.
118+
119+
The results of a model-fit can be stored in a sqlite database, including the following attributes of the fit:
120+
121+
- The masked dataset (e.g. data / noise map / pre cti data) as .fits files (contained in `dataset` folder).
122+
- The clocker used to add CTI in the fit (`dataset/clocker.json`).
123+
- The settings used for clocking CIT (contained in `dataset/settings_cti.json`).
124+
125+
The `aggregator` contains the path to each of these files, and they can be loaded individually. This class
126+
can load them all at once and create a `FitDataset1D` object via the `_fit_dataset_1d_from` method.
127+
128+
This class's methods returns generators which create the instances of the `FitDataset1D` objects. This ensures
129+
that large sets of results can be efficiently loaded from the hard-disk and do not require storing all
130+
`Dataset1D` instances in the memory at once.
131+
132+
For example, if the `aggregator` contains 3 model-fits, this class can be used to create a generator which
133+
creates instances of the corresponding 3 `Dataset1D` objects.
134+
135+
If multiple `Dataset1D` objects were fitted simultaneously via analysis summing, the `fit.child_values()` method
136+
is instead used to load lists of the datasets, perform the fit and return a list of `FitDataset1D` objects.
137+
138+
If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible
139+
to the database, the input `use_dataset_full` can be switched in to fit the full dataset instead.
140+
141+
This can be done manually, but this object provides a more concise API.
142+
143+
Parameters
144+
----------
145+
aggregator
146+
A `PyAutoFit` aggregator object which can load the results of model-fits.
147+
use_dataset_full
148+
If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore
149+
accessible to the database, the input `use_dataset_full` can be switched in to load instead the
150+
full `Dataset1D` objects.
151+
clocker_list
152+
If input, overwrites the clocker used in the fit with a new clocker which is used to perform the fit.
153+
"""
154+
super().__init__(
155+
aggregator=aggregator,
156+
use_dataset_full=use_dataset_full,
157+
clocker_list=clocker_list,
158+
)
159+
160+
def object_via_gen_from(
161+
self, fit, instance: Optional[af.ModelInstance] = None
162+
) -> List[FitDataset1D]:
163+
"""
164+
Returns a generator of `FitDataset1D` objects from an input aggregator.
165+
166+
See `__init__` for a description of how the `FitDataset1D` objects are created by this method.
167+
168+
If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible
169+
to the database, the input `use_dataset_full` can be switched in to fit the full dataset instead.
170+
171+
Parameters
172+
----------
173+
fit
174+
A `PyAutoFit` `Fit` object which contains the results of a model-fit as an entry in a sqlite database.
175+
cti
176+
The CTI model used to add CTI to the dataset to perform the fit.
177+
"""
178+
return _fit_dataset_1d_list_from(
179+
fit=fit,
180+
instance=instance,
181+
use_dataset_full=self.use_dataset_full,
182+
clocker_list=self.clocker_list,
183+
)

0 commit comments

Comments
 (0)