diff --git a/.gitignore b/.gitignore index 87a6a3ca35..9fcf4562aa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,25 @@ +# Byte-compiled / optimized +__pycache__/ +*.py[cod] -*.pyc -*.o -*.so +# C extensions *.c -.cache -*.egg-info -*.*~ -environment?.?.yml +*.so + +# Distribution / packaging +build/ +*.egg-info/ +environment-*.yml +miniconda.sh + +# Unit test / coverage reports +.hypothesis/ + +# Jupyter Notebook +.ipynb_checkpoints + +# Backup files +*~ + +# macOS .DS_Store -doc/.DS_Store -doc/build -invisible_cities/cities/.ipynb_checkpoints diff --git a/.travis.yml b/.travis.yml index 1c183a008d..578fd67e54 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,3 +22,11 @@ install: script: - HYPOTHESIS_PROFILE=travis-ci bash manage.sh run_tests_par + + +before_install: + - if [ "$TRAVIS_OS_NAME" = "osx" ]; then brew install git-lfs; fi + - if [ "$TRAVIS_OS_NAME" = "osx" ]; then git lfs install; fi + +before_script: + - if [ "$TRAVIS_OS_NAME" = "osx" ]; then git lfs pull; fi diff --git a/conftest.py b/conftest.py index b45e9aa305..a1cc2b871e 100644 --- a/conftest.py +++ b/conftest.py @@ -7,8 +7,6 @@ settings.register_profile("travis-ci" , settings(max_examples = 1000, deadline=300)) settings.register_profile("hard" , settings(max_examples = 1000)) settings.register_profile("dev" , settings(max_examples = 10)) -settings.register_profile("hard_nocov", settings(max_examples = 1000, use_coverage=False)) -settings.register_profile("dev_nocov" , settings(max_examples = 10, use_coverage=False)) settings.register_profile("debug" , settings(max_examples = 10, verbosity=Verbosity.verbose)) settings.load_profile(os.getenv(u'HYPOTHESIS_PROFILE', 'dev')) diff --git a/doc/commits.rst b/doc/commits.rst new file mode 100644 index 0000000000..c101b9ef8e --- /dev/null +++ b/doc/commits.rst @@ -0,0 +1,48 @@ +How to create a good commit history +=================================== + +Commit structure +----------------- + +- Focus on a specific change. Don't try to change everything at once. + If the plan is to implement a series of things, do them one by one. + +- Do not mix changes concerning different features in one commit. + For instance, do not mix cosmetical changes with other, more relevant ones. + +- Commit frequently and in an incremental manner. + Do not be afraid of making too many commits, they can be combined afterwards + (this is known as squashing in git). + Besides, the history can be easily cleaned when the changes are small and independent of each other. + + +Commit style +-------------- + +- Just follow https://chris.beams.io/posts/git-commit/ + +This is a summary of a good commit message. +But, please at least take a look to the details in the link above. + + + | The seven rules of a great Git commit message + | + | 1. Separate subject from body with a blank line + | 2. Limit the subject line to 50 characters + | 3. Capitalize the subject line + | 4. Do not end the subject line with a period + | 5. Use the imperative mood in the subject line + | 6. Wrap the body at 72 characters + | 7. Use the body to explain what and why vs. how + +Commits after PR review +----------------------- + +- Remember that these guidelines apply also to changes requested during a review process. + Don't make commits like ``Changes requested by the reviewer``. + +- Follow the same guidelines listed above and, + if you want to refer to the discussion happening in the PR, + do it using the second part of the commit message. + You may also refer to the PR by using ``#PR_NUMBER`` or + by posting a link to a specific comment in the discussion. diff --git a/invisible_cities/cities/beersheba.py b/invisible_cities/cities/beersheba.py index 0e49413bfa..f6c1990c98 100644 --- a/invisible_cities/cities/beersheba.py +++ b/invisible_cities/cities/beersheba.py @@ -22,11 +22,12 @@ from typing import Tuple from typing import List -from typing import Callable from enum import auto from . components import city +from . components import collect +from . components import copy_mc_info from . components import print_every from . components import cdst_from_files @@ -38,6 +39,8 @@ from .. dataflow.dataflow import push from .. dataflow.dataflow import pipe +from .. database.load_db import DataSiPM + from .. reco.deconv_functions import find_nearest from .. reco.deconv_functions import cut_and_redistribute_df from .. reco.deconv_functions import drop_isolated_sensors @@ -45,15 +48,15 @@ from .. reco.deconv_functions import richardson_lucy from .. reco.deconv_functions import InterpolationMethod -from .. io. mcinfo_io import mc_info_writer from .. io.run_and_event_io import run_and_event_writer -from .. io. dst_io import _store_pandas_as_tables +from .. io. dst_io import df_writer +from .. io. dst_io import load_dst from .. evm.event_model import HitEnergy from .. types.ic_types import AutoNameEnumBase -from .. core.system_of_units_c import units +from .. core import system_of_units as units class CutType (AutoNameEnumBase): @@ -65,7 +68,8 @@ class DeconvolutionMode(AutoNameEnumBase): separate = auto() -def deconvolve_signal(psf_fname : str, +def deconvolve_signal(det_db : pd.DataFrame, + psf_fname : str, e_cut : float, n_iterations : int, iteration_tol : float, @@ -85,6 +89,7 @@ def deconvolve_signal(psf_fname : str, Parameters ---------- + det_db : Detector database. psf_fname : Point-spread function. e_cut : Cut in relative value to the max voxel over the deconvolution output. n_iterations : Number of Lucy-Richardson iterations @@ -111,16 +116,19 @@ def deconvolve_signal(psf_fname : str, bin_size = np.asarray(bin_size ) diffusion = np.asarray(diffusion ) - psfs = pd.read_hdf(psf_fname) - deconvolution = deconvolve(n_iterations, iteration_tol, sample_width, bin_size, inter_method) + psfs = load_dst(psf_fname, 'PSF', 'PSFs') + det_grid = [np.arange(det_db[var].min() + bs/2, det_db[var].max() - bs/2 + np.finfo(np.float32).eps, bs) + for var, bs in zip(dimensions, bin_size)] + deconvolution = deconvolve(n_iterations, iteration_tol, + sample_width, det_grid , inter_method) - if energy_type not in HitEnergy : + if not isinstance(energy_type , HitEnergy ): raise ValueError(f'energy_type {energy_type} is not a valid energy type.') - if inter_method not in InterpolationMethod: + if not isinstance(inter_method, InterpolationMethod): raise ValueError(f'inter_method {inter_method} is not a valid interpolation method.') - if cut_type not in CutType : + if not isinstance(cut_type , CutType ): raise ValueError(f'cut_type {cut_type} is not a valid cut type.') - if deconv_mode not in DeconvolutionMode : + if not isinstance(deconv_mode , DeconvolutionMode ): raise ValueError(f'deconv_mode {deconv_mode} is not a valid deconvolution mode.') def deconvolve_hits(df, z): @@ -294,17 +302,18 @@ def deconv_writer(h5out, compression='ZLIB4'): For a given open table returns a writer for deconvolution hits dataframe """ def write_deconv(df): - return _store_pandas_as_tables(h5out = h5out , - df = df , - compression = compression , - group_name = 'DECO' , - table_name = 'Events' , - descriptive_string = 'Deconvolved hits') + return df_writer(h5out = h5out , + df = df , + compression = compression , + group_name = 'DECO' , + table_name = 'Events' , + descriptive_string = 'Deconvolved hits', + columns_to_index = ['event'] ) return write_deconv @city -def beersheba(files_in, file_out, compression, event_range, print_mod, run_number, +def beersheba(files_in, file_out, compression, event_range, print_mod, detector_db, run_number, deconv_params = dict()): """ The city corrects Penthesilea hits energy and extracts topology information. @@ -395,7 +404,7 @@ def beersheba(files_in, file_out, compression, event_range, print_mod, run_numbe filter_events_no_hits = fl.map(check_nonempty_dataframe, args = 'cdst', out = 'cdst_passed_no_hits') - deconvolve_events = fl.map(deconvolve_signal(**deconv_params), + deconvolve_events = fl.map(deconvolve_signal(DataSiPM(detector_db, run_number), **deconv_params), args = 'cdst', out = 'deconv_dst') @@ -403,28 +412,35 @@ def beersheba(files_in, file_out, compression, event_range, print_mod, run_numbe event_count_out = fl.spy_count() events_passed_no_hits = fl.count_filter(bool, args = "cdst_passed_no_hits") + evtnum_collect = collect() + with tb.open_file(file_out, "w", filters = tbl.filters(compression)) as h5out: # Define writers - write_event_info = fl.sink(run_and_event_writer(h5out), args=("run_number", "event_number", "timestamp")) - write_mc_ = mc_info_writer(h5out) if run_number <= 0 else (lambda *_: None) - - write_mc = fl.sink( write_mc_, args = ("mc", "event_number")) - write_deconv = fl.sink( deconv_writer(h5out=h5out), args = "deconv_dst" ) - write_summary = fl.sink( summary_writer(h5out=h5out), args = "summary" ) - return push(source = cdst_from_files(files_in), - pipe = pipe(fl.slice(*event_range, close_all=True) , - print_every(print_mod) , - event_count_in.spy , - cut_sensors , - drop_sensors , - filter_events_no_hits , - events_passed_no_hits .filter , - deconvolve_events , - event_count_out.spy , - fl.fork(write_mc , - write_deconv , - write_summary , - write_event_info)) , - result = dict(events_in = event_count_in .future, - events_out = event_count_out .future, - events_pass = events_passed_no_hits.future)) + write_event_info = fl.sink(run_and_event_writer (h5out), args = ("run_number", "event_number", "timestamp")) + write_deconv = fl.sink( deconv_writer(h5out=h5out), args = "deconv_dst") + write_summary = fl.sink( summary_writer(h5out=h5out), args = "summary" ) + result = push(source = cdst_from_files(files_in), + pipe = pipe(fl.slice(*event_range, close_all=True) , + print_every(print_mod) , + event_count_in.spy , + cut_sensors , + drop_sensors , + filter_events_no_hits , + events_passed_no_hits .filter , + deconvolve_events , + event_count_out.spy , + fl.branch("event_number" , + evtnum_collect.sink) , + fl.fork(write_deconv , + write_summary , + write_event_info)) , + result = dict(events_in = event_count_in .future, + events_out = event_count_out .future, + evtnum_list = evtnum_collect .future, + events_pass = events_passed_no_hits.future)) + + if run_number <= 0: + copy_mc_info(files_in, h5out, result.evtnum_list, + detector_db, run_number) + + return result diff --git a/invisible_cities/cities/beersheba_test.py b/invisible_cities/cities/beersheba_test.py index 3a63ad904d..36734709b7 100644 --- a/invisible_cities/cities/beersheba_test.py +++ b/invisible_cities/cities/beersheba_test.py @@ -17,6 +17,7 @@ from .. evm .event_model import HitEnergy from .. core.testing_utils import assert_dataframes_close from .. core.testing_utils import assert_tables_equality +from .. database.load_db import DataSiPM def test_create_deconvolution_df(ICDATADIR): @@ -27,7 +28,7 @@ def test_create_deconvolution_df(ICDATADIR): CutType.abs, ecut, 3) for _, t in true_dst.groupby('event')]) true_dst = true_dst.loc[true_dst.E > ecut, :].reset_index(drop=True) - assert_dataframes_close(new_dst, true_dst) + assert_dataframes_close(new_dst .reset_index(drop=True), true_dst.reset_index(drop=True)) @mark.parametrize("cut_type", CutType.__members__) @@ -59,7 +60,6 @@ def test_beersheba_contains_all_tables(deconvolution_config): beersheba(**conf) with tb.open_file(PATH_OUT) as h5out: assert "MC" in h5out.root - assert "MC/extents" in h5out.root assert "MC/hits" in h5out.root assert "MC/particles" in h5out.root assert "DECO/Events" in h5out.root @@ -70,39 +70,43 @@ def test_beersheba_contains_all_tables(deconvolution_config): def test_beersheba_exact_result_joint(ICDATADIR, deconvolution_config): - true_out = os.path.join(ICDATADIR, "test_Xe2nu_NEW_exact_deconvolution_joint.h5") + true_out = os.path.join(ICDATADIR, "test_Xe2nu_NEW_exact_deconvolution_joint.NEWMC.h5") conf, PATH_OUT = deconvolution_config beersheba(**conf) - tables = ( "MC/extents" , "MC/hits" , "MC/particles" , "MC/generators", - "DECO/Events" , - "Summary/Events", - "Run/events" , "Run/runInfo" ) + tables = ("DECO/Events" , + "Summary/Events" , + "Run/events" , "Run/runInfo" , + "MC/event_mapping", "MC/generators", + "MC/hits" , "MC/particles") with tb.open_file(true_out) as true_output_file: with tb.open_file(PATH_OUT) as output_file: for table in tables: + assert hasattr(output_file.root, table) got = getattr( output_file.root, table) expected = getattr(true_output_file.root, table) assert_tables_equality(got, expected) def test_beersheba_exact_result_separate(ICDATADIR, deconvolution_config): - true_out = os.path.join(ICDATADIR, "test_Xe2nu_NEW_exact_deconvolution_separate.h5") + true_out = os.path.join(ICDATADIR, "test_Xe2nu_NEW_exact_deconvolution_separate.NEWMC.h5") conf, PATH_OUT = deconvolution_config conf['deconv_params']['deconv_mode' ] = 'separate' conf['deconv_params']['n_iterations' ] = 50 conf['deconv_params']['n_iterations_g'] = 50 beersheba(**conf) - tables = ( "MC/extents" , "MC/hits" , "MC/particles" , "MC/generators", - "DECO/Events" , - "Summary/Events", - "Run/events" , "Run/runInfo" ) + tables = ("DECO/Events" , + "Summary/Events" , + "Run/events" , "Run/runInfo" , + "MC/event_mapping", "MC/generators", + "MC/hits" , "MC/particles") with tb.open_file(true_out) as true_output_file: with tb.open_file(PATH_OUT) as output_file: for table in tables: + assert hasattr(output_file.root, table) got = getattr( output_file.root, table) expected = getattr(true_output_file.root, table) print(got) @@ -111,9 +115,8 @@ def test_beersheba_exact_result_separate(ICDATADIR, deconvolution_config): @mark.parametrize("ndim", (1, 3)) -def test_beersheba_param_dim(ICDATADIR, deconvolution_config, ndim): - true_out = os.path.join(ICDATADIR, "test_Xe2nu_NEW_exact_deconvolution_joint.h5") - conf, PATH_OUT = deconvolution_config +def test_beersheba_param_dim(deconvolution_config, ndim): + conf, _ = deconvolution_config conf['deconv_params']['n_dim' ] = ndim @@ -122,10 +125,9 @@ def test_beersheba_param_dim(ICDATADIR, deconvolution_config, ndim): @mark.parametrize("param_name", ('cut_type', 'deconv_mode', 'energy_type', 'inter_method')) -def test_deconvolve_signal_enums(ICDATADIR, deconvolution_config, param_name): - true_out = os.path.join(ICDATADIR, "test_Xe2nu_NEW_exact_deconvolution_joint.h5") - conf, PATH_OUT = deconvolution_config - conf_dict = conf['deconv_params'] +def test_deconvolve_signal_enums(deconvolution_config, param_name): + conf, _ = deconvolution_config + conf_dict = conf['deconv_params'] conf_dict.pop("q_cut") conf_dict.pop("drop_dist") @@ -138,12 +140,11 @@ def test_deconvolve_signal_enums(ICDATADIR, deconvolution_config, param_name): conf_dict[param_name] = param_name with raises(ValueError): - deconv = deconvolve_signal(**conf_dict) + deconvolve_signal(DataSiPM('new'), **conf_dict) -def test_beersheba_expandvar(ICDATADIR, deconvolution_config): - true_out = os.path.join(ICDATADIR, "test_Xe2nu_NEW_exact_deconvolution_joint.h5") - conf, PATH_OUT = deconvolution_config +def test_beersheba_expandvar(deconvolution_config): + conf, _ = deconvolution_config conf['deconv_params']['psf_fname'] = '$ICDIR/database/test_data/PSF_dst_sum_collapsed.h5' diff --git a/invisible_cities/cities/berenice.py b/invisible_cities/cities/berenice.py index 8135c5b6f9..c62bd3d0af 100644 --- a/invisible_cities/cities/berenice.py +++ b/invisible_cities/cities/berenice.py @@ -28,9 +28,9 @@ import numpy as np import tables as tb -from .. io . hist_io import hist_writer +from .. io . histogram_io import hist_writer from .. io .run_and_event_io import run_and_event_writer -from .. icaro.hst_functions import shift_to_bin_centers +from .. core .core_functions import shift_to_bin_centers from .. reco import tbl_functions as tbl from .. reco import calib_functions as cf from .. reco import calib_sensors_functions as csf diff --git a/invisible_cities/cities/berenice_test.py b/invisible_cities/cities/berenice_test.py index 911097449a..0922f803f2 100644 --- a/invisible_cities/cities/berenice_test.py +++ b/invisible_cities/cities/berenice_test.py @@ -3,8 +3,6 @@ import tables as tb import numpy as np -from pytest import mark - from . berenice import berenice from .. core.configure import configure from .. core.configure import all as all_events @@ -33,7 +31,7 @@ def test_berenice_sipmdarkcurrent(config_tmpdir, ICDATADIR): evts_out = h5out.root.Run.events[:nrequired] assert_array_equal(evts_in, evts_out) - assert 'Sensors' in h5out.root + assert 'Sensors' in h5out.root ch_in_pmt = np.array(h5in .root.Sensors.DataPMT [:]) ch_out_pmt = np.array(h5out.root.Sensors.DataPMT [:]) ch_in_sipm = np.array(h5in .root.Sensors.DataSiPM[:]) diff --git a/invisible_cities/cities/cities_test.py b/invisible_cities/cities/cities_test.py index 303dd577be..a795cfde2f 100644 --- a/invisible_cities/cities/cities_test.py +++ b/invisible_cities/cities/cities_test.py @@ -1,5 +1,4 @@ import os -import tables as tb from importlib import import_module from pytest import mark @@ -7,6 +6,7 @@ cities = "irene dorothea penthesilea esmeralda beersheba".split() +@mark.filterwarnings("ignore::UserWarning") @mark.parametrize("city", cities) def test_city_empty_input_file(config_tmpdir, ICDATADIR, city): # All cities run in Canfranc must run on an empty file diff --git a/invisible_cities/cities/command_line_execution_test.py b/invisible_cities/cities/command_line_execution_test.py index beaa647e41..9af50a8d17 100644 --- a/invisible_cities/cities/command_line_execution_test.py +++ b/invisible_cities/cities/command_line_execution_test.py @@ -12,7 +12,7 @@ # 'diomira isidora irene dorothea zaira penthesilea'.split()) # TODO understand what's wrong with isidora (in Travis) @mark.parametrize('city', - 'diomira isidora irene dorothea penthesilea zaira berenice phyllis trude esmeralda beersheba'.split()) + 'diomira isidora irene dorothea penthesilea berenice phyllis trude esmeralda beersheba hypathia'.split()) def test_command_line_run(city, tmpdir_factory): ICTDIR = getenv('ICTDIR') diff --git a/invisible_cities/cities/components.py b/invisible_cities/cities/components.py index 7f20129bbc..af7586b8b2 100644 --- a/invisible_cities/cities/components.py +++ b/invisible_cities/cities/components.py @@ -7,17 +7,23 @@ from itertools import count from itertools import repeat from enum import Enum +from typing import Callable from typing import Iterator from typing import Mapping +from typing import Generator from typing import List from typing import Dict +from typing import Tuple from typing import Union import tables as tb import numpy as np import pandas as pd import inspect +import warnings from .. dataflow import dataflow as fl +from .. dataflow.dataflow import sink +from .. dataflow.dataflow import pipe from .. evm .ic_containers import SensorData from .. evm .event_model import KrEvent from .. evm .event_model import Hit @@ -25,33 +31,38 @@ from .. evm .event_model import HitCollection from .. evm .event_model import MCInfo from .. evm .pmaps import SiPMCharge -from .. core .system_of_units_c import units +from .. core import system_of_units as units from .. core .exceptions import XYRecoFail +from .. core .exceptions import MCEventNotFound from .. core .exceptions import NoInputFiles from .. core .exceptions import NoOutputFile from .. core .exceptions import InvalidInputFileStructure from .. core .configure import EventRange from .. core .configure import event_range_help from .. core .random_sampling import NoiseSampler +from .. detsim import buffer_functions as bf from .. reco import calib_functions as cf +from .. reco import sensor_functions as sf from .. reco import calib_sensors_functions as csf from .. reco import peak_functions as pkf from .. reco import pmaps_functions as pmf from .. reco import hits_functions as hif -from .. reco .tbl_functions import get_mc_info +from .. reco import wfm_functions as wfm from .. reco .xy_algorithms import corona from .. filters.s1s2_filter import S12Selector from .. filters.s1s2_filter import pmap_filter from .. database import load_db from .. sierpe import blr +from .. io import mcinfo_io from .. io .pmaps_io import load_pmaps from .. io .hits_io import hits_from_df from .. io .dst_io import load_dst -from .. io .hits_io import load_hits -from .. io .dst_io import load_dst +from .. io .event_filter_io import event_filter_writer +from .. io .pmaps_io import pmap_writer from .. types .ic_types import xy from .. types .ic_types import NN from .. types .ic_types import NNN +from .. types .ic_types import minmax NoneType = type(None) @@ -152,6 +163,99 @@ def print_every_loop(target): return print_every_loop +def collect(): + """Return a future/sink pair for collecting streams into a list.""" + def append(l,e): + l.append(e) + return l + return fl.reduce(append, initial=[])() + + +def copy_mc_info(files_in : List[str], + h5out : tb.File , + event_numbers: List[int], + db_file : str , + run_number : int ) -> None: + """ + Copy to an output file the MC info of a list of selected events. + + Parameters + ---------- + files_in : List of strings + Name of the input files. + file_out : tables.File + The output h5 file. + event_numbers : List[int] + List of event numbers for which the MC info is copied + to the output file. + """ + + writer = mcinfo_io.mc_writer(h5out) + + copied_events = [] + for f in files_in: + if mcinfo_io.check_mc_present(f): + event_numbers_in_file = mcinfo_io.get_event_numbers_in_file(f) + event_numbers_to_copy = np.intersect1d(event_numbers , + event_numbers_in_file) + mcinfo_io.copy_mc_info(f, writer, event_numbers_to_copy, + db_file, run_number) + copied_events.extend(event_numbers_to_copy) + else: + warnings.warn(f' File does not contain MC tables.\ + Use positve run numbers for data', UserWarning) + continue + if len(np.setdiff1d(event_numbers, copied_events)) != 0: + raise MCEventNotFound(f' Some events not found in MC tables') + + +def wf_binner(max_buffer: int) -> Callable: + """ + Returns a function to be used to convert the raw + input MC sensor info into data binned according to + a set bin width, effectively + padding with zeros inbetween the separate signals. + + Parameters + ---------- + max_buffer : float + Maximum event time to be considered in nanoseconds + """ + def bin_sensors(sensors : pd.DataFrame, + bin_width: float , + t_min : float , + t_max : float ) -> Tuple[np.ndarray, pd.Series]: + return bf.bin_sensors(sensors, bin_width, t_min, t_max, max_buffer) + return bin_sensors + + +def signal_finder(buffer_len : float, + bin_width : float, + bin_threshold: int) -> Callable: + """ + Decides where there is signal-like + charge according to the configuration + and the PMT sum in order to give + a useful position for buffer selection. + Currently simple threshold on binned charge. + + Parameters + ---------- + buffer_len : float + Configured buffer length in mus + bin_width : float + Sampling width for sensors + bin_threshold : int + PE threshold for selection + """ + # The stand_off is the minumum number of samples + # necessary between candidate triggers. + stand_off = int(buffer_len / bin_width) + def find_signal(wfs: pd.Series) -> List[int]: + return bf.find_signal_start(wfs, bin_threshold, stand_off) + return find_signal + + # TODO: consider caching database def deconv_pmt(dbfile, run_number, n_baseline, selection=None): DataPMT = load_db.DataPMT(dbfile, run_number = run_number) @@ -191,13 +295,6 @@ def get_sipm_wfs(h5in, wf_type): else : raise TypeError(f"Invalid WfType: {type(wf_type)}") -def get_mc_info_safe(h5in, run_number): - if run_number <= 0: - try : return get_mc_info(h5in) - except tb.exceptions.NoSuchNodeError: pass - return - - def get_trigger_info(h5in): group = h5in.root.Trigger if "Trigger" in h5in.root else () trigger_type = group.trigger if "trigger" in group else repeat(None) @@ -209,6 +306,19 @@ def get_event_info(h5in): return h5in.root.Run.events +def get_number_of_active_pmts(detector_db, run_number): + datapmt = load_db.DataPMT(detector_db, run_number) + return np.count_nonzero(datapmt.Active.values.astype(bool)) + + +def check_nonempty_indices(s1_indices, s2_indices): + return s1_indices.size and s2_indices.size + + +def check_empty_pmap(pmap): + return bool(pmap.s1s) or bool(pmap.s2s) + + def length_of(iterable): if isinstance(iterable, tb.table.Table ): return iterable.nrows elif isinstance(iterable, tb.earray.EArray): return iterable.shape[0] @@ -228,6 +338,61 @@ def check_lengths(*iterables): raise InvalidInputFileStructure("Input data tables have different sizes") +def mcsensors_from_file(paths : List[str], + db_file : str , + run_number: int ) -> Generator: + """ + Loads the nexus MC sensor information into + a pandas DataFrame using the IC function + load_mcsensor_response_df. + Returns info event by event as a + generator in the structure expected by + the dataflow. + + paths : List of strings + List of input file names to be read + db_file : string + Name of detector database to be used + run_number : int + Run number for database + """ + + pmt_ids = load_db.DataPMT (db_file, run_number).SensorID + + for file_name in paths: + sns_resp = mcinfo_io.load_mcsensor_response_df(file_name , + return_raw = False , + db_file = db_file , + run_no = run_number) + sns_bins = mcinfo_io.get_sensor_binning(file_name) + if sns_resp.empty or sns_bins.empty: + raise MCEventNotFound(f'Sensor response info not in file {file_name}') + elif sns_bins.shape[0] != 2: + warnings.warn(f' File contains wrong number (not 2) of sensor types.\ + Simulation should be for NEW, NEXT100 or DEMOPP', UserWarning) + ## Source only valid for NEW, NEXT100 & DEMOPP + ## and for flexible geometries with Pmt/SiPM separation of sensors + PMT_name_indx = sns_bins.index.str.contains('Pmt') + pmt_binwid = sns_bins.bin_width[ PMT_name_indx] + sipm_binwid = sns_bins.bin_width[~PMT_name_indx] + + ## MC uses dummy timestamp for now + ## Only in case of evt splitting will be non zero + timestamp = 0 + + for evt in sns_resp.index.levels[0]: + pmt_indx = sns_resp.loc[evt].index.isin(pmt_ids) + pmt_resp = sns_resp.loc[evt][ pmt_indx] + sipm_resp = sns_resp.loc[evt][~pmt_indx] + + yield dict(evt = evt , + timestamp = timestamp , + pmt_binwid = pmt_binwid .iloc[0], + sipm_binwid = sipm_binwid.iloc[0], + pmt_resp = pmt_resp , + sipm_resp = sipm_resp ) + + def wf_from_files(paths, wf_type): for path in paths: with tb.open_file(path, "r") as h5in: @@ -236,7 +401,6 @@ def wf_from_files(paths, wf_type): run_number = get_run_number (h5in) pmt_wfs = get_pmt_wfs (h5in, wf_type) sipm_wfs = get_sipm_wfs (h5in, wf_type) - mc_info = get_mc_info_safe(h5in, run_number) (trg_type , trg_chann) = get_trigger_info(h5in) except tb.exceptions.NoSuchNodeError: @@ -248,12 +412,9 @@ def wf_from_files(paths, wf_type): event_number, timestamp = evtinfo.fetch_all_fields() if trtype is not None: trtype = trtype .fetch_all_fields()[0] - yield dict(pmt=pmt, sipm=sipm, mc=mc_info, - run_number=run_number, event_number=event_number, timestamp=timestamp, + yield dict(pmt=pmt, sipm=sipm, run_number=run_number, + event_number=event_number, timestamp=timestamp, trigger_type=trtype, trigger_channels=trchann) - # NB, the monte_carlo writer is different from the others: - # it needs to be given the WHOLE TABLE (rather than a - # single event) at a time. def pmap_from_files(paths): @@ -267,7 +428,6 @@ def pmap_from_files(paths): try: run_number = get_run_number(h5in) event_info = get_event_info(h5in) - mc_info = get_mc_info_safe(h5in, run_number) except tb.exceptions.NoSuchNodeError: continue except IndexError: @@ -277,11 +437,9 @@ def pmap_from_files(paths): for evtinfo in event_info: event_number, timestamp = evtinfo.fetch_all_fields() - yield dict(pmap=pmaps[event_number], mc=mc_info, - run_number=run_number, event_number=event_number, timestamp=timestamp) - # NB, the monte_carlo writer is different from the others: - # it needs to be given the WHOLE TABLE (rather than a - # single event) at a time. + yield dict(pmap=pmaps[event_number], run_number=run_number, + event_number=event_number, timestamp=timestamp) + def cdst_from_files(paths: List[str]) -> Iterator[Dict[str,Union[pd.DataFrame, MCInfo, int, float]]]: """Reader of the files, yields collected hits, @@ -300,7 +458,6 @@ def cdst_from_files(paths: List[str]) -> Iterator[Dict[str,Union[pd.DataFrame, M evts, _ = zip(*event_info[:]) bool_mask = np.in1d(evts, cdst_df.event.unique()) event_info = event_info[bool_mask] - mc_info = get_mc_info_safe(h5in, run_number) except (tb.exceptions.NoSuchNodeError, IndexError): continue check_lengths(event_info, cdst_df.event.unique()) @@ -308,14 +465,15 @@ def cdst_from_files(paths: List[str]) -> Iterator[Dict[str,Union[pd.DataFrame, M event_number, timestamp = evtinfo yield dict(cdst = cdst_df .loc[cdst_df .event==event_number], summary = summary_df.loc[summary_df.event==event_number], - mc=mc_info, run_number=run_number, + run_number=run_number, event_number=event_number, timestamp=timestamp) # NB, the monte_carlo writer is different from the others: # it needs to be given the WHOLE TABLE (rather than a # single event) at a time. def hits_and_kdst_from_files(paths: List[str]) -> Iterator[Dict[str,Union[HitCollection, pd.DataFrame, MCInfo, int, float]]]: - """Reader of the files, yields HitsCollection, pandas DataFrame with kdst info, mc_info, run_number, event_number and timestamp""" + """Reader of the files, yields HitsCollection, pandas DataFrame with + kdst info, run_number, event_number and timestamp.""" for path in paths: try: hits_df = load_dst (path, 'RECO', 'Events') @@ -327,7 +485,6 @@ def hits_and_kdst_from_files(paths: List[str]) -> Iterator[Dict[str,Union[HitCol try: run_number = get_run_number(h5in) event_info = get_event_info(h5in) - mc_info = get_mc_info_safe(h5in, run_number) except (tb.exceptions.NoSuchNodeError, IndexError): continue @@ -336,11 +493,12 @@ def hits_and_kdst_from_files(paths: List[str]) -> Iterator[Dict[str,Union[HitCol for evtinfo in event_info: event_number, timestamp = evtinfo.fetch_all_fields() hits = hits_from_df(hits_df.loc[hits_df.event == event_number]) - yield dict(hits = hits[event_number], kdst = kdst_df.loc[kdst_df.event==event_number], mc=mc_info, run_number=run_number, - event_number=event_number, timestamp=timestamp) - # NB, the monte_carlo writer is different from the others: - # it needs to be given the WHOLE TABLE (rather than a - # single event) at a time. + yield dict(hits = hits[event_number], + kdst = kdst_df.loc[kdst_df.event==event_number], + run_number = run_number, + event_number = event_number, + timestamp = timestamp) + def sensor_data(path, wf_type): with tb.open_file(path, "r") as h5in: @@ -353,6 +511,34 @@ def sensor_data(path, wf_type): ####### Transformers ######## +def build_pmap(detector_db, run_number, pmt_samp_wid, sipm_samp_wid, + s1_lmax, s1_lmin, s1_rebin_stride, s1_stride, s1_tmax, s1_tmin, + s2_lmax, s2_lmin, s2_rebin_stride, s2_stride, s2_tmax, s2_tmin, thr_sipm_s2): + s1_params = dict(time = minmax(min = s1_tmin, + max = s1_tmax), + length = minmax(min = s1_lmin, + max = s1_lmax), + stride = s1_stride, + rebin_stride = s1_rebin_stride) + + s2_params = dict(time = minmax(min = s2_tmin, + max = s2_tmax), + length = minmax(min = s2_lmin, + max = s2_lmax), + stride = s2_stride, + rebin_stride = s2_rebin_stride) + + datapmt = load_db.DataPMT(detector_db, run_number) + pmt_ids = datapmt.SensorID[datapmt.Active.astype(bool)].values + + def build_pmap(ccwf, s1_indx, s2_indx, sipmzs): # -> PMap + return pkf.get_pmap(ccwf, s1_indx, s2_indx, sipmzs, + s1_params, s2_params, thr_sipm_s2, pmt_ids, + pmt_samp_wid, sipm_samp_wid) + + return build_pmap + + def calibrate_pmts(dbfile, run_number, n_MAU, thr_MAU): DataPMT = load_db.DataPMT(dbfile, run_number = run_number) adc_to_pes = np.abs(DataPMT.adc_to_pes.values) @@ -400,6 +586,30 @@ def ccwfs_to_zs(ccwf_sum, ccwf_sum_mau): pkf.indices_and_wf_above_threshold(ccwf_sum , thr_csum_s2).indices) return ccwfs_to_zs + +def compute_pe_resolution(rms, adc_to_pes): + return np.divide(rms , + adc_to_pes , + out = np.zeros_like(adc_to_pes), + where = adc_to_pes != 0 ) + + +def simulate_sipm_response(detector, run_number, wf_length, noise_cut, filter_padding): + datasipm = load_db.DataSiPM (detector, run_number) + baselines = load_db.SiPMNoise(detector, run_number)[-1] + noise_sampler = NoiseSampler(detector, run_number, wf_length, True) + + adc_to_pes = datasipm.adc_to_pes.values + thresholds = noise_cut * adc_to_pes + baselines + single_pe_rms = datasipm.Sigma.values.astype(np.double) + pe_resolution = compute_pe_resolution(single_pe_rms, adc_to_pes) + + def simulate_sipm_response(sipmrd): + wfs = sf.simulate_sipm_response(sipmrd, noise_sampler, adc_to_pes, pe_resolution) + return wfm.noise_suppression(wfs, thresholds, filter_padding) + return simulate_sipm_response + + ####### Filters ######## def peak_classifier(**params): @@ -575,3 +785,52 @@ def waveform_integrator(limits): def integrate_wfs(wfs): return cf.spaced_integrals(wfs, limits)[:, ::2] return integrate_wfs + + +# Compound components +def compute_and_write_pmaps(detector_db, run_number, pmt_samp_wid, sipm_samp_wid, + s1_lmax, s1_lmin, s1_rebin_stride, s1_stride, s1_tmax, s1_tmin, + s2_lmax, s2_lmin, s2_rebin_stride, s2_stride, s2_tmax, s2_tmin, thr_sipm_s2, + h5out, compression, sipm_rwf_to_cal=None): + + # Filter events without signal over threshold + indices_pass = fl.map(check_nonempty_indices, + args = ("s1_indices", "s2_indices"), + out = "indices_pass") + empty_indices = fl.count_filter(bool, args = "indices_pass") + + # Build the PMap + compute_pmap = fl.map(build_pmap(detector_db, run_number, pmt_samp_wid, sipm_samp_wid, + s1_lmax, s1_lmin, s1_rebin_stride, s1_stride, s1_tmax, s1_tmin, + s2_lmax, s2_lmin, s2_rebin_stride, s2_stride, s2_tmax, s2_tmin, thr_sipm_s2), + args = ("ccwfs", "s1_indices", "s2_indices", "sipm"), + out = "pmap") + + # Filter events with zero peaks + pmaps_pass = fl.map(check_empty_pmap, args = "pmap", out = "pmaps_pass") + empty_pmaps = fl.count_filter(bool, args = "pmaps_pass") + + # Define writers... + write_pmap_ = pmap_writer (h5out, compression=compression) + write_indx_filter_ = event_filter_writer(h5out, "s12_indices", compression=compression) + write_pmap_filter_ = event_filter_writer(h5out, "empty_pmap" , compression=compression) + + # ... and make them sinks + write_pmap = sink(write_pmap_ , args=( "pmap", "event_number")) + write_indx_filter = sink(write_indx_filter_ , args=("event_number", "indices_pass")) + write_pmap_filter = sink(write_pmap_filter_ , args=("event_number", "pmaps_pass")) + + fn_list = (indices_pass, + fl.branch(write_indx_filter), + empty_indices.filter, + sipm_rwf_to_cal, + compute_pmap, + pmaps_pass, + fl.branch(write_pmap_filter), + empty_pmaps.filter, + fl.branch(write_pmap)) + + # Filter out simp_rwf_to_cal if it is not set + compute_pmaps = pipe(*filter(None, fn_list)) + + return compute_pmaps, empty_indices, empty_pmaps diff --git a/invisible_cities/cities/components_test.py b/invisible_cities/cities/components_test.py index 954197b615..cd8dc71a91 100644 --- a/invisible_cities/cities/components_test.py +++ b/invisible_cities/cities/components_test.py @@ -9,22 +9,25 @@ from pytest import mark from pytest import raises +from pytest import warns from .. core.configure import EventRange as ER from .. core.exceptions import InvalidInputFileStructure -from .. core.exceptions import ClusterEmptyList - -from .. core.system_of_units_c import units +from .. core.exceptions import MCEventNotFound +from .. core import system_of_units as units from . components import event_range +from . components import collect +from . components import copy_mc_info from . components import WfType from . components import wf_from_files from . components import pmap_from_files from . components import compute_xy_position from . components import city from . components import hits_and_kdst_from_files +from . components import mcsensors_from_file -from .. database import load_db +from .. dataflow import dataflow as fl def _create_dummy_conf_with_event_range(value): @@ -103,10 +106,8 @@ def test_compute_xy_position_depends_on_actual_run_number(): seed_charge = minimum_seed_charge + 1 charge_to_test = np.array([charge, charge, charge, seed_charge, charge, charge, charge, charge]) - try: - find_xy_pos(xys_to_test, charge_to_test) - except(ClusterEmptyList): - assert False + find_xy_pos(xys_to_test, charge_to_test) + def test_city_adds_default_detector_db(config_tmpdir): @@ -115,7 +116,7 @@ def test_city_adds_default_detector_db(config_tmpdir): 'file_out' : os.path.join(config_tmpdir, 'dummy_out')} @city def dummy_city(files_in, file_out, event_range, detector_db): - with tb.open_file(file_out, 'w') as h5out: + with tb.open_file(file_out, 'w'): pass return detector_db @@ -129,7 +130,7 @@ def test_city_does_not_overwrite_detector_db(config_tmpdir): 'file_out' : os.path.join(config_tmpdir, 'dummy_out')} @city def dummy_city(files_in, file_out, event_range, detector_db): - with tb.open_file(file_out, 'w') as h5out: + with tb.open_file(file_out, 'w'): pass return detector_db @@ -142,7 +143,7 @@ def test_city_only_pass_default_detector_db_when_expected(config_tmpdir): 'file_out' : os.path.join(config_tmpdir, 'dummy_out')} @city def dummy_city(files_in, file_out, event_range): - with tb.open_file(file_out, 'w') as h5out: + with tb.open_file(file_out, 'w'): pass dummy_city(**args) @@ -151,7 +152,7 @@ def test_hits_and_kdst_from_files(ICDATADIR): event_number = 1 timestamp = 0. num_hits = 13 - keys = ['hits', 'mc', 'kdst', 'run_number', 'event_number', 'timestamp'] + keys = ['hits', 'kdst', 'run_number', 'event_number', 'timestamp'] file_in = os.path.join(ICDATADIR , 'Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.HDST.h5') generator = hits_and_kdst_from_files([file_in]) output = next(generator) @@ -160,3 +161,78 @@ def test_hits_and_kdst_from_files(ICDATADIR): assert output['timestamp'] == timestamp assert len(output['hits'].hits) == num_hits assert type(output['kdst']) == pd.DataFrame + + +def test_collect(): + the_source = list(range(0,10)) + the_collector = collect() + the_result = fl.push(source = the_source, + pipe = fl.pipe(the_collector.sink), + result = the_collector.future) + assert the_source == the_result + + +def test_copy_mc_info_noMC(ICDATADIR, config_tmpdir): + file_in = os.path.join(ICDATADIR, 'run_2983.h5') + file_out = os.path.join(config_tmpdir, 'dummy_out.h5') + with tb.open_file(file_out, "w") as h5out: + with warns(UserWarning): + copy_mc_info([file_in], h5out, [], 'new', -6400) + + +@mark.xfail +def test_copy_mc_info_repeated_event_numbers(ICDATADIR, config_tmpdir): + file_in = os.path.join(ICDATADIR, "Kr83_nexus_v5_03_00_ACTIVE_7bar_10evts.sim.h5") + file_out = os.path.join(config_tmpdir, "dummy_out.h5") + + with tb.open_file(file_out, 'w') as h5out: + copy_mc_info([file_in, file_in], h5out, [0,1,0,9]) + events_in_h5out = h5out.root.MC.extents.cols.evt_number[:] + assert events_in_h5out.tolist() == [0,1,0,9] + + +def test_mcsensors_from_file_invalid_input_raises(ICDATADIR): + file_in = os.path.join(ICDATADIR, "nexus_new_kr83m_fast.oldformat.sim.h5") + + s = mcsensors_from_file((file_in,), db_file='new', run_number=-6400) + with raises(MCEventNotFound): + next(s) + + +def test_mcsensors_from_file_raises_warning_flex3type(ICDATADIR): + file_in = os.path.join(ICDATADIR, "NextFlex_mc_sensors.h5") + + s = mcsensors_from_file((file_in,), db_file='new', run_number=-6400) + with warns(UserWarning): + next(s) + + +def test_mcsensors_from_file_correct_yield(ICDATADIR): + evt_no = 0 + timestamp = 0 + pmt_binwid = 1 * units.ns + sipm_binwid = 1 * units.mus + npmts_hit = 12 + total_pmthits = 4303 + nsipms_hit = 313 + total_sipmhits = 389 + keys = ['evt' , 'timestamp', 'pmt_binwid', + 'sipm_binwid', 'pmt_resp' , 'sipm_resp'] + + file_in = os.path.join(ICDATADIR, "nexus_new_kr83m_full.newformat.sim.h5") + sns_gen = mcsensors_from_file([file_in], 'new', -7951) + first_evt = next(sns_gen) + + assert set(keys) == set(first_evt.keys()) + + assert first_evt[ 'evt'] == evt_no + assert first_evt[ 'timestamp'] == timestamp + assert first_evt[ 'pmt_binwid'] == pmt_binwid + assert first_evt['sipm_binwid'] == sipm_binwid + assert type(first_evt[ 'pmt_resp']) == pd.DataFrame + assert type(first_evt[ 'sipm_resp']) == pd.DataFrame + assert len(first_evt[ 'pmt_resp'].index.unique()) == npmts_hit + assert first_evt[ 'pmt_resp'].shape[0] == total_pmthits + assert len(first_evt[ 'sipm_resp'].index.unique()) == nsipms_hit + assert first_evt[ 'sipm_resp'].shape[0] == total_sipmhits + diff --git a/invisible_cities/cities/diomira.py b/invisible_cities/cities/diomira.py index 8e29ac247a..8967c121ba 100644 --- a/invisible_cities/cities/diomira.py +++ b/invisible_cities/cities/diomira.py @@ -1,6 +1,6 @@ """ ----------------------------------------------------------------------- - Diomira + Diomira ----------------------------------------------------------------------- From Germanic, Teodomiro/Teodemaro: famous among its people. @@ -33,11 +33,8 @@ from .. reco import tbl_functions as tbl from .. reco import sensor_functions as sf from .. reco import peak_functions as pkf -from .. reco import wfm_functions as wfm from .. sierpe import fee as FE -from .. core.random_sampling import NoiseSampler as SiPMsNoiseSampler from .. io.rwf_io import rwf_writer -from .. io. mcinfo_io import mc_info_writer from .. io.run_and_event_io import run_and_event_writer from .. io. event_filter_io import event_filter_writer from .. filters.trigger_filters import TriggerFilter @@ -47,78 +44,109 @@ from .. types.ic_types import minmax from .. dataflow import dataflow as fl -from .. dataflow.dataflow import push -from .. dataflow.dataflow import pipe -from .. dataflow.dataflow import fork from . components import city from . components import print_every +from . components import collect +from . components import copy_mc_info from . components import sensor_data from . components import deconv_pmt from . components import WfType from . components import wf_from_files +from . components import simulate_sipm_response +from . components import compute_pe_resolution @city -def diomira(files_in, file_out, compression, event_range, print_mod, detector_db, run_number, - sipm_noise_cut, filter_padding, trigger_type, - trigger_params = dict(), s2_params = dict(), - random_seed = None): +def diomira(files_in , file_out , compression , + event_range , print_mod , detector_db , + run_number , sipm_noise_cut, filter_padding, + trigger_type, trigger_params = dict(), + s2_params = dict(), random_seed = None): if random_seed is not None: np.random.seed(random_seed) sd = sensor_data(files_in[0], WfType.mcrd) - simulate_pmt_response_ = fl.map(simulate_pmt_response (detector_db, run_number), + simulate_pmt_response_ = fl.map(simulate_pmt_response (detector_db, + run_number ), args="pmt" , out= ("pmt_sim", "blr_sim")) - simulate_sipm_response_ = fl.map(simulate_sipm_response(detector_db, run_number, sd.SIPMWL, sipm_noise_cut, filter_padding), + simulate_sipm_response_ = fl.map(simulate_sipm_response(detector_db , + run_number , + sd.SIPMWL , + sipm_noise_cut, + filter_padding), args="sipm", out="sipm_sim" ) - trigger_filter_ = select_trigger_filter(trigger_type, trigger_params, s2_params) - emulate_trigger_ = fl.map(emulate_trigger(detector_db, run_number, trigger_type, trigger_params, s2_params), args="pmt_sim", out="trigger_sim") - trigger_pass = fl.map(trigger_filter_, args="trigger_sim", out="trigger_pass") + trigger_filter_ = select_trigger_filter(trigger_type , + trigger_params, + s2_params ) + emulate_trigger_ = fl.map(emulate_trigger(detector_db , + run_number , + trigger_type , + trigger_params, + s2_params ), + args="pmt_sim", out="trigger_sim" ) + trigger_pass = fl.map(trigger_filter_ , + args="trigger_sim", + out="trigger_pass") trigger_filter = fl.count_filter(bool, args="trigger_pass") with tb.open_file(file_out, "w", filters=tbl.filters(compression)) as h5out: RWF = partial(rwf_writer, h5out, group_name='RD') - write_pmt = fl.sink(RWF(table_name='pmtrwf' , n_sensors=sd.NPMT , waveform_length=sd. PMTWL // int(FE.t_sample)), args= "pmt_sim") - write_blr = fl.sink(RWF(table_name='pmtblr' , n_sensors=sd.NPMT , waveform_length=sd. PMTWL // int(FE.t_sample)), args= "blr_sim") - write_sipm = fl.sink(RWF(table_name='sipmrwf', n_sensors=sd.NSIPM, waveform_length=sd.SIPMWL ), args="sipm_sim") + write_pmt = fl.sink(RWF(table_name = 'pmtrwf', + n_sensors = sd.NPMT , + waveform_length = sd.PMTWL // int(FE.t_sample)), + args= "pmt_sim") + write_blr = fl.sink(RWF(table_name = 'pmtblr', + n_sensors = sd.NPMT , + waveform_length = sd.PMTWL // int(FE.t_sample)), + args= "blr_sim") + write_sipm = fl.sink(RWF(table_name = 'sipmrwf', + n_sensors = sd.NSIPM , + waveform_length = sd.SIPMWL), + args="sipm_sim") write_event_info_ = run_and_event_writer(h5out) - write_mc_ = mc_info_writer (h5out) if run_number <= 0 else (lambda *_: None) - write_evt_filter_ = event_filter_writer (h5out, "trigger", compression=compression) + write_evt_filter_ = event_filter_writer (h5out , + "trigger" , + compression = compression) - write_event_info = fl.sink(write_event_info_, args=("run_number", "event_number", "timestamp" )) - write_mc = fl.sink(write_mc_ , args=( "mc", "event_number" )) - write_evt_filter = fl.sink(write_evt_filter_, args=( "event_number", "trigger_pass")) + write_event_info = fl.sink(write_event_info_, + args=("run_number", "event_number", + "timestamp" )) + write_evt_filter = fl.sink(write_evt_filter_, + args=("event_number", "trigger_pass")) event_count_in = fl.spy_count() - return push( - source = wf_from_files(files_in, WfType.mcrd), - pipe = pipe(fl.slice(*event_range, close_all=True), - event_count_in.spy , - print_every(print_mod) , - simulate_pmt_response_ , - emulate_trigger_ , - trigger_pass , - fl.branch(write_evt_filter) , - trigger_filter.filter , - simulate_sipm_response_ , - fork(write_pmt , - write_blr , - write_sipm , - write_mc , - write_event_info) ), - result = dict(events_in = event_count_in.future, - events_filter = trigger_filter.future)) - - -def compute_pe_resolution(rms, adc_to_pes): - return np.divide(rms , - adc_to_pes , - out = np.zeros_like(adc_to_pes), - where = adc_to_pes != 0 ) + evtnum_collect = collect() + + result = fl.push(source = wf_from_files(files_in, WfType.mcrd), + pipe = fl.pipe(fl.slice(*event_range , + close_all=True) , + event_count_in.spy , + print_every(print_mod) , + simulate_pmt_response_ , + emulate_trigger_ , + trigger_pass , + fl.branch(write_evt_filter) , + trigger_filter.filter , + simulate_sipm_response_ , + fl.branch("event_number" , + evtnum_collect.sink), + fl.fork(write_pmt , + write_blr , + write_sipm , + write_event_info)) , + result = dict(events_in = event_count_in.future, + evtnum_list = evtnum_collect.future, + events_filter = trigger_filter.future)) + + if run_number <= 0: + copy_mc_info(files_in, h5out, result.evtnum_list, + detector_db, run_number) + + return result def simulate_pmt_response(detector, run_number): @@ -131,27 +159,10 @@ def simulate_pmt_response(pmtrd): rwf, blr = sf.simulate_pmt_response(0, pmtrd[np.newaxis], adc_to_pes, pe_resolution, detector, run_number) - return rwf.astype(np.int16), blr.astype(np.int16) + return np.round(rwf).astype(np.int16), np.round(blr).astype(np.int16) return simulate_pmt_response -def simulate_sipm_response(detector, run_number, wf_length, noise_cut, filter_padding): - datasipm = load_db.DataSiPM (detector, run_number) - baselines = load_db.SiPMNoise(detector, run_number)[-1] - noise_sampler = SiPMsNoiseSampler(detector, run_number, wf_length, True) - - adc_to_pes = datasipm.adc_to_pes.values - thresholds = noise_cut * adc_to_pes + baselines - single_pe_rms = datasipm.Sigma.values.astype(np.double) - pe_resolution = compute_pe_resolution(single_pe_rms, adc_to_pes) - - def simulate_sipm_response(sipmrd): - wfs = sf.simulate_sipm_response(0, sipmrd[np.newaxis], - noise_sampler, adc_to_pes, - pe_resolution) - return wfm.noise_suppression(wfs, thresholds, filter_padding) - return simulate_sipm_response - def select_trigger_filter(trigger_type, trigger_params, s2_params): if trigger_type is None: diff --git a/invisible_cities/cities/diomira_test.py b/invisible_cities/cities/diomira_test.py index 3a219f77c7..a7539ed9ae 100644 --- a/invisible_cities/cities/diomira_test.py +++ b/invisible_cities/cities/diomira_test.py @@ -1,19 +1,25 @@ import os import tables as tb import numpy as np +import pandas as pd from pytest import mark from pytest import raises -from .. core import system_of_units as units -from .. core.configure import all as all_events -from .. core.configure import configure -from .. core.testing_utils import assert_tables_equality +from .. core.configure import all as all_events +from .. core.configure import configure +from .. core.testing_utils import assert_dataframes_close +from .. core.testing_utils import assert_tables_equality +from .. database import load_db +from .. reco import tbl_functions as tbl +from .. io .mcinfo_io import get_event_numbers_in_file +from .. io .mcinfo_io import load_mchits_df +from .. io .mcinfo_io import load_mcparticles_df -from .. reco import tbl_functions as tbl -from .. sierpe import fee as FEE +from .. core import fit_functions as fitf +from .. core.core_functions import shift_to_bin_centers -from . diomira import diomira +from . diomira import diomira def test_diomira_identify_bug(ICDATADIR): @@ -58,27 +64,21 @@ def test_diomira_copy_mc_and_offset(ICDATADIR, config_tmpdir): nactual = cnt.events_in assert nrequired == nactual - with tb.open_file(PATH_IN, mode='r') as h5in, \ - tb.open_file(PATH_OUT, mode='r') as h5out: + with tb.open_file(PATH_OUT, mode='r') as h5out: # check event & run number assert h5out.root.Run.runInfo[0]['run_number'] == run_number assert h5out.root.Run.events [0]['evt_number'] == start_evt - # check mcextents - # we have to convert manually into a tuple because MCTracks[0] - # returns an object of type numpy.void where we cannot index - # using ranges like mctracks_in[1:] - mcextents_in = tuple(h5in .root.MC.extents[0]) - mcextents_out = tuple(h5out.root.MC.extents[0]) - #evt number is not equal if we redefine first event number - assert mcextents_out[0] == start_evt - for e in zip(mcextents_in[1:], mcextents_out[1:]): - np.testing.assert_array_equal(e[0],e[1]) - - # check event number is different for each event - first_evt_number = h5out.root.MC.extents[ 0][0] - last_evt_number = h5out.root.MC.extents[-1][0] - assert first_evt_number != last_evt_number + evts_in = get_event_numbers_in_file(PATH_IN ) + evts_out = get_event_numbers_in_file(PATH_OUT) + assert len(evts_out) == nrequired + assert all(evts_in[:nrequired] == evts_out) + + hits_in = load_mchits_df(PATH_IN ) + hits_out = load_mchits_df(PATH_OUT) + assert_dataframes_close(hits_in.loc[0:nrequired-1], + hits_out ) + @mark.slow def test_diomira_mismatch_between_input_and_database(ICDATADIR, output_tmpdir): @@ -124,9 +124,10 @@ def test_diomira_trigger_on_masked_pmt_raises_ValueError(ICDATADIR, output_tmpdi diomira(**conf) def test_diomira_read_multiple_files(ICDATADIR, output_tmpdir): - file_in = os.path.join(ICDATADIR , "Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts*.MCRD.h5") - file_out = os.path.join(output_tmpdir, "Kr83_nexus_v5_03_00_ACTIVE_7bar_6evts.RWF.h5") - second_file = os.path.join(ICDATADIR , "Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts_1.MCRD.h5") + file_in = os.path.join(ICDATADIR , + "Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts*.MCRD.h5") + file_out = os.path.join(output_tmpdir , + "Kr83_nexus_v5_03_00_ACTIVE_7bar_6evts.RWF.h5") nevents_per_file = 3 @@ -140,24 +141,36 @@ def test_diomira_read_multiple_files(ICDATADIR, output_tmpdir): diomira(**conf) - with tb.open_file(file_out) as h5out: - last_particle_list = h5out.root.MC.extents[:]['last_particle'] - last_hit_list = h5out.root.MC.extents[:]['last_hit' ] + first_file = os.path.join(ICDATADIR , + "Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.MCRD.h5") + second_file = os.path.join(ICDATADIR , + "Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts_1.MCRD.h5") + + particles_in1 = load_mcparticles_df( first_file) + hits_in1 = load_mchits_df ( first_file) + particles_in2 = load_mcparticles_df(second_file) + hits_in2 = load_mchits_df (second_file) + particles_out = load_mcparticles_df( file_out) + hits_out = load_mchits_df ( file_out) - assert all(x= charge_threshold_high constrain. it also contains: - Ep field that is the energy of a hit after applying drop_end_point_voxel algorithm. - - track_id denoting to which track from Tracking/Tracks dataframe the hit belong to + - track_id denoting to which track from Tracking/Tracks dataframe the hit belong to - MC info (if run number <=0) - Tracking/Tracks - summary of per track information - Summary/events - summary of per event information @@ -29,7 +29,7 @@ from .. reco import tbl_functions as tbl from .. reco import paolina_functions as plf from .. reco import hits_functions as hif -from .. reco import corrections_new as cof +from .. reco import corrections as cof from .. evm import event_model as evm from .. dataflow import dataflow as fl from .. dataflow.dataflow import push @@ -37,14 +37,17 @@ from . components import city from . components import print_every +from . components import collect +from . components import copy_mc_info from . components import hits_and_kdst_from_files from .. types. ic_types import xy -from .. io. hits_io import hits_writer -from .. io. mcinfo_io import mc_info_writer -from .. io.run_and_event_io import run_and_event_writer -from .. io. event_filter_io import event_filter_writer -from .. io. dst_io import _store_pandas_as_tables + +from .. io. hits_io import hits_writer +from .. io.run_and_event_io import run_and_event_writer +from .. io. event_filter_io import event_filter_writer +from .. io. dst_io import df_writer + types_dict_summary = OrderedDict({'event' : np.int32 , 'evt_energy' : np.float64, 'evt_charge' : np.float64, 'evt_ntrks' : np.int , 'evt_nhits' : np.int , 'evt_x_avg' : np.float64, @@ -302,34 +305,53 @@ def make_event_summary(event_number : int , return es -def track_writer(h5out, compression='ZLIB4', group_name='Tracking', table_name='Tracks', descriptive_string='Track information', str_col_length=32): +def track_writer(h5out, compression='ZLIB4'): """ For a given open table returns a writer for topology info dataframe """ def write_tracks(df): - return _store_pandas_as_tables(h5out=h5out, df=df, compression=compression, group_name=group_name, table_name=table_name, descriptive_string=descriptive_string, str_col_length=str_col_length) + return df_writer(h5out = h5out , + df = df , + compression = compression , + group_name = 'Tracking' , + table_name = 'Tracks' , + descriptive_string = 'Track information', + columns_to_index = ['event'] ) return write_tracks -def summary_writer(h5out, compression='ZLIB4', group_name='Summary', table_name='Events', descriptive_string='Event summary information', str_col_length=32): +def summary_writer(h5out, compression='ZLIB4'): """ For a given open table returns a writer for summary info dataframe """ def write_summary(df): - return _store_pandas_as_tables(h5out=h5out, df=df, compression=compression, group_name=group_name, table_name=table_name, descriptive_string=descriptive_string, str_col_length=str_col_length) + return df_writer(h5out = h5out , + df = df , + compression = compression , + group_name = 'Summary' , + table_name = 'Events' , + descriptive_string = 'Event summary information', + columns_to_index = ['event'] ) return write_summary -def kdst_from_df_writer(h5out, compression='ZLIB4', group_name='DST', table_name='Events', descriptive_string='KDST Events', str_col_length=32): +def kdst_from_df_writer(h5out, compression='ZLIB4'): """ For a given open table returns a writer for KDST dataframe info """ def write_kdst(df): - return _store_pandas_as_tables(h5out=h5out, df=df, compression=compression, group_name=group_name, table_name=table_name, descriptive_string=descriptive_string, str_col_length=str_col_length) + return df_writer(h5out = h5out , + df = df , + compression = compression , + group_name = 'DST' , + table_name = 'Events' , + descriptive_string = 'KDST Events', + columns_to_index = ['event'] ) return write_kdst @city -def esmeralda(files_in, file_out, compression, event_range, print_mod, run_number, +def esmeralda(files_in, file_out, compression, event_range, print_mod, + detector_db, run_number, cor_hits_params = dict(), paolina_params = dict()): """ @@ -390,7 +412,7 @@ def esmeralda(files_in, file_out, compression, event_range, print_mod, run_numbe - highTh - contains corrected hits table that passed h.Q >= charge_threshold_high constrain. it also contains: - Ep field that is the energy of a hit after applying drop_end_point_voxel algorithm. - - track_id denoting to which track from Tracking/Tracks dataframe the hit belong to + - track_id denoting to which track from Tracking/Tracks dataframe the hit belong to - MC info (if run number <=0) - Tracking/Tracks - summary of per track information - Summary/events - summary of per event information @@ -443,13 +465,12 @@ def esmeralda(files_in, file_out, compression, event_range, print_mod, run_numbe # Define writers... write_event_info = fl.sink(run_and_event_writer(h5out), args=("run_number", "event_number", "timestamp")) - write_mc_ = mc_info_writer(h5out) if run_number <= 0 else (lambda *_: None) write_hits_low_th = fl.sink( hits_writer (h5out, group_name='CHITS', table_name='lowTh'), args="cor_low_th_hits") write_hits_paolina = fl.sink( hits_writer (h5out, group_name='CHITS', table_name='highTh' ), args="paolina_hits" ) - write_mc = fl.sink( write_mc_ , args=("mc", "event_number")) + write_tracks = fl.sink( track_writer (h5out=h5out) , args="topology_info" ) write_summary = fl.sink( summary_writer (h5out=h5out) , args="event_info" ) write_high_th_filter = fl.sink( event_filter_writer(h5out, "high_th_select" ) , args=("event_number", "high_th_hits_passed")) @@ -457,31 +478,39 @@ def esmeralda(files_in, file_out, compression, event_range, print_mod, run_numbe write_topology_filter = fl.sink( event_filter_writer(h5out, "topology_select") , args=("event_number", "topology_passed" )) write_kdst_table = fl.sink( kdst_from_df_writer(h5out) , args="kdst" ) - return push(source = hits_and_kdst_from_files(files_in), - pipe = pipe( - fl.slice(*event_range, close_all=True) , - print_every(print_mod) , - event_count_in .spy , - fl.branch(fl.fork(write_kdst_table , - write_event_info , - write_mc )), - fl.branch(threshold_and_correct_hits_low , - filter_events_low_th , - fl.branch(write_low_th_filter) , - hits_passed_low_th.filter , - write_hits_low_th ), - threshold_and_correct_hits_high , - filter_events_high_th , - fl.branch(write_high_th_filter) , - hits_passed_high_th .filter , - copy_Ec_to_Ep_hit_attribute , - create_extract_track_blob_info , - filter_events_topology , - fl.branch(make_final_summary, write_summary) , - fl.branch(write_topology_filter) , - fl.branch(write_hits_paolina) , - events_passed_topology.filter , - event_count_out .spy , - write_tracks ), - result = dict(events_in =event_count_in .future , - events_out=event_count_out.future )) + evtnum_collect = collect() + + result = push(source = hits_and_kdst_from_files(files_in), + pipe = pipe(fl.slice(*event_range, close_all=True) , + print_every(print_mod) , + event_count_in .spy , + fl.branch(fl.fork(write_kdst_table , + write_event_info )), + fl.branch("event_number", evtnum_collect.sink), + fl.branch(threshold_and_correct_hits_low , + filter_events_low_th , + fl.branch(write_low_th_filter) , + hits_passed_low_th.filter , + write_hits_low_th ), + threshold_and_correct_hits_high , + filter_events_high_th , + fl.branch(write_high_th_filter) , + hits_passed_high_th .filter , + copy_Ec_to_Ep_hit_attribute , + create_extract_track_blob_info , + filter_events_topology , + fl.branch(make_final_summary, write_summary) , + fl.branch(write_topology_filter) , + fl.branch(write_hits_paolina) , + events_passed_topology.filter , + event_count_out .spy , + write_tracks ), + result = dict(events_in =event_count_in .future, + events_out =event_count_out.future, + evtnum_list=evtnum_collect .future)) + + if run_number <= 0: + copy_mc_info(files_in, h5out, result.evtnum_list, + detector_db, run_number) + + return result diff --git a/invisible_cities/cities/esmeralda_test.py b/invisible_cities/cities/esmeralda_test.py index c861e5e7d9..d51899ca79 100644 --- a/invisible_cities/cities/esmeralda_test.py +++ b/invisible_cities/cities/esmeralda_test.py @@ -3,15 +3,16 @@ import tables as tb import pandas as pd -from . components import get_event_info -from . components import length_of -from .. core.system_of_units_c import units -from .. core.configure import configure -from .. core.configure import all as all_events -from .. io import dst_io as dio -from . esmeralda import esmeralda -from .. core.testing_utils import assert_dataframes_close -from .. core.testing_utils import assert_tables_equality +from . components import get_event_info +from . components import length_of +from .. core import system_of_units as units +from .. core.configure import configure +from .. core.configure import all as all_events +from .. io import dst_io as dio +from .. io.mcinfo_io import get_event_numbers_in_file +from . esmeralda import esmeralda +from .. core.testing_utils import assert_dataframes_close +from .. core.testing_utils import assert_tables_equality def test_esmeralda_contains_all_tables(KrMC_hdst_filename, correction_map_MC_filename, config_tmpdir): @@ -32,7 +33,6 @@ def test_esmeralda_contains_all_tables(KrMC_hdst_filename, correction_map_MC_fil with tb.open_file(PATH_OUT) as h5out: assert "MC" in h5out.root - assert "MC/extents" in h5out.root assert "MC/hits" in h5out.root assert "MC/particles" in h5out.root assert "Tracking/Tracks" in h5out.root @@ -90,7 +90,7 @@ def test_esmeralda_filters_events(KrMC_hdst_filename_toy, correction_map_MC_file with tb.open_file(PATH_OUT) as h5out: event_info = get_event_info(h5out) assert length_of(event_info) == nevt_req - MC_num_evs = h5out.root.MC.extents[:]['evt_number'] + MC_num_evs = get_event_numbers_in_file(PATH_OUT) assert len(MC_num_evs) == nevt_req @@ -154,7 +154,8 @@ def test_esmeralda_tracks_exact(data_hdst, esmeralda_tracks, correction_map_file min_voxels = 2 , blob_radius = 21 * units.mm , max_num_hits = 10000 ))) - cnt = esmeralda(**conf) + + esmeralda(**conf) df_tracks = dio.load_dst(PATH_OUT, 'Tracking', 'Tracks' ) df_tracks_exact = pd.read_hdf(esmeralda_tracks, key = 'Tracks') @@ -162,7 +163,9 @@ def test_esmeralda_tracks_exact(data_hdst, esmeralda_tracks, correction_map_file #some events are not in df_tracks_exact events = df_tracks_exact.event.unique() df_tracks_cut = df_tracks[df_tracks.event.isin(events)] - assert_dataframes_close (df_tracks_cut[columns2], df_tracks_exact[columns2]) + + assert_dataframes_close (df_tracks_cut[columns2] .reset_index(drop=True), + df_tracks_exact[columns2].reset_index(drop=True)) #make sure out_of_map is true for events not in df_tracks_exact diff_events = list(set(df_tracks.event.unique()).difference(events)) df_summary = dio.load_dst(PATH_OUT, 'Summary', 'Events') @@ -207,7 +210,7 @@ def test_esmeralda_blob_overlap_bug(data_hdst, correction_map_filename, config_t blob_radius = 21 * units.mm , max_num_hits = 10000 ))) - cnt = esmeralda(**conf) + esmeralda(**conf) df_tracks = dio.load_dst(PATH_OUT, 'Tracking', 'Tracks') assert df_tracks['ovlp_blob_energy'].dtype == float @@ -216,7 +219,7 @@ def test_esmeralda_exact_result_all_events(ICDATADIR, KrMC_hdst_filename, correc file_in = KrMC_hdst_filename file_out = os.path.join(config_tmpdir, "exact_Kr_tracks_with_MC.h5") conf = configure('dummy invisible_cities/config/esmeralda.conf'.split()) - true_out = os.path.join(ICDATADIR, "exact_Kr_tracks_with_MC_KDST_no_filter.h5") + true_out = os.path.join(ICDATADIR, "exact_Kr_tracks_with_MC_KDST_no_filter.NEWMC.h5") nevt_req = all_events conf.update(dict(files_in = file_in , file_out = file_out , @@ -235,17 +238,19 @@ def test_esmeralda_exact_result_all_events(ICDATADIR, KrMC_hdst_filename, correc blob_radius = 21 * units.mm , max_num_hits = 10000 ))) - cnt = esmeralda(**conf) + esmeralda(**conf) - tables = ["MC/extents", "MC/hits", "MC/particles", - "Tracking/Tracks", "CHITS/lowTh", "CHITS/highTh", + tables = ["Tracking/Tracks", "CHITS/lowTh", "CHITS/highTh", "Run/events", "Run/runInfo", "DST/Events", "Summary/Events", - "Filters/low_th_select", "Filters/high_th_select", "Filters/topology_select"] + "Filters/low_th_select", "Filters/high_th_select", "Filters/topology_select", + "MC/event_mapping", "MC/generators", + "MC/hits" , "MC/particles"] with tb.open_file(true_out) as true_output_file: with tb.open_file(file_out) as output_file: for table in tables: + assert hasattr(output_file.root, table) got = getattr( output_file.root, table) expected = getattr(true_output_file.root, table) assert_tables_equality(got, expected) @@ -276,7 +281,7 @@ def test_esmeralda_bug_duplicate_hits(data_hdst, correction_map_filename, config blob_radius = 21 * units.mm , max_num_hits = 10000 ))) - cnt = esmeralda(**conf) + esmeralda(**conf) df_tracks = dio.load_dst(PATH_OUT, 'Tracking', 'Tracks') df_phits = dio.load_dst(PATH_OUT, 'CHITS' , 'highTh') @@ -309,7 +314,7 @@ def test_esmeralda_all_hits_after_drop_voxels(data_hdst, esmeralda_tracks, corre min_voxels = 2 , blob_radius = 21 * units.mm , max_num_hits = 10000 ))) - cnt = esmeralda(**conf) + esmeralda(**conf) df_phits = dio.load_dst(PATH_OUT, 'CHITS' , 'highTh') df_in_hits = dio.load_dst(PATH_IN , 'RECO' , 'Events') @@ -330,7 +335,6 @@ def test_esmeralda_filters_events_with_too_many_hits(data_hdst, correction_map_f PATH_OUT = os.path.join(config_tmpdir, "esmeralda_filters_long_events.h5") conf = configure('dummy invisible_cities/config/esmeralda.conf'.split()) nevt_req = 9 - all_hits = 601 nhits_max = 50 paolina_events = {3021898, 3021914, 3021930, 3020951, 3020961} conf.update(dict(files_in = PATH_IN , @@ -350,7 +354,8 @@ def test_esmeralda_filters_events_with_too_many_hits(data_hdst, correction_map_f min_voxels = 2 , blob_radius = 21 * units.mm , max_num_hits = nhits_max ))) - cnt = esmeralda(**conf) + + esmeralda(**conf) summary = dio.load_dst(PATH_OUT, 'Summary' , 'Events') tracks = dio.load_dst(PATH_OUT, 'Tracking', 'Tracks') diff --git a/invisible_cities/cities/hypathia.py b/invisible_cities/cities/hypathia.py new file mode 100644 index 0000000000..56de85a553 --- /dev/null +++ b/invisible_cities/cities/hypathia.py @@ -0,0 +1,168 @@ +""" +----------------------------------------------------------------------- + Hypathia +----------------------------------------------------------------------- + +From ancient Greek ‘Υπατια: highest, supreme. + +This city reads true waveforms from detsim and compute pmaps from them +without simulating the electronics. This includes: + - Rebin 1-ns waveforms to 25-ns waveforms to match those produced + by the detector. + - Produce a PMT-summed waveform. + - Apply a threshold to the PMT-summed waveform. + - Find pulses in the PMT-summed waveform. + - Match the time window of the PMT pulse with those in the SiPMs. + - Build the PMap object. +""" +import numpy as np +import tables as tb + +from functools import partial + +from .. reco import sensor_functions as sf +from .. reco import tbl_functions as tbl +from .. reco import peak_functions as pkf +from .. core. random_sampling import NoiseSampler as SiPMsNoiseSampler +from .. core import system_of_units as units +from .. io .run_and_event_io import run_and_event_writer +from .. io . trigger_io import trigger_writer +from .. io . event_filter_io import event_filter_writer + +from .. dataflow import dataflow as fl +from .. dataflow.dataflow import push +from .. dataflow.dataflow import pipe +from .. dataflow.dataflow import sink + +from . components import city +from . components import print_every +from . components import collect +from . components import copy_mc_info +from . components import zero_suppress_wfs +from . components import WfType +from . components import sensor_data +from . components import wf_from_files +from . components import get_number_of_active_pmts +from . components import compute_and_write_pmaps +from . components import simulate_sipm_response +from . components import calibrate_sipms + + +@city +def hypathia(files_in, file_out, compression, event_range, print_mod, detector_db, run_number, + sipm_noise_cut, filter_padding, thr_sipm, thr_sipm_type, pmt_wfs_rebin, pmt_pe_rms, + s1_lmin, s1_lmax, s1_tmin, s1_tmax, s1_rebin_stride, s1_stride, thr_csum_s1, + s2_lmin, s2_lmax, s2_tmin, s2_tmax, s2_rebin_stride, s2_stride, thr_csum_s2, thr_sipm_s2, + pmt_samp_wid=25*units.ns, sipm_samp_wid=1*units.mus): + if thr_sipm_type.lower() == "common": + # In this case, the threshold is a value in pes + sipm_thr = thr_sipm + + elif thr_sipm_type.lower() == "individual": + # In this case, the threshold is a percentual value + noise_sampler = SiPMsNoiseSampler(detector_db, run_number) + sipm_thr = noise_sampler.compute_thresholds(thr_sipm) + + else: + raise ValueError(f"Unrecognized thr type: {thr_sipm_type}. " + "Only valid options are 'common' and 'individual'") + + #### Define data transformations + sd = sensor_data(files_in[0], WfType.mcrd) + + # Raw WaveForm to Corrected WaveForm + mcrd_to_rwf = fl.map(rebin_pmts(pmt_wfs_rebin), + args = "pmt", + out = "rwf") + + # Add single pe fluctuation to pmts + simulate_pmt = fl.map(partial(sf.charge_fluctuation, single_pe_rms=pmt_pe_rms), + args = "rwf", + out = "ccwfs") + + # Compute pmt sum + pmt_sum = fl.map(pmts_sum, args = 'ccwfs', + out = 'pmt') + + # Find where waveform is above threshold + zero_suppress = fl.map(zero_suppress_wfs(thr_csum_s1, thr_csum_s2), + args = ("pmt", "pmt"), + out = ("s1_indices", "s2_indices", "s2_energies")) + + # SiPMs simulation + simulate_sipm_response_ = fl.map(simulate_sipm_response(detector_db, run_number, + sd.SIPMWL, sipm_noise_cut, + filter_padding), + item="sipm") + + # Sipm calibration function expects waveform as int16 + discretize_signal = fl.map(lambda rwf: np.round(rwf).astype(np.int16), + item="sipm") + + # SiPMs calibration + sipm_rwf_to_cal = fl.map(calibrate_sipms(detector_db, run_number, sipm_thr), + item = "sipm") + + event_count_in = fl.spy_count() + event_count_out = fl.spy_count() + + evtnum_collect = collect() + + with tb.open_file(file_out, "w", filters = tbl.filters(compression)) as h5out: + + # Define writers... + write_event_info_ = run_and_event_writer(h5out) + write_trigger_info_ = trigger_writer (h5out, get_number_of_active_pmts(detector_db, run_number)) + + # ... and make them sinks + write_event_info = sink(write_event_info_ , args=( "run_number", "event_number", "timestamp" )) + write_trigger_info = sink(write_trigger_info_, args=( "trigger_type", "trigger_channels" )) + + compute_pmaps, empty_indices, empty_pmaps = compute_and_write_pmaps( + detector_db, run_number, pmt_samp_wid, sipm_samp_wid, + s1_lmax, s1_lmin, s1_rebin_stride, s1_stride, s1_tmax, s1_tmin, + s2_lmax, s2_lmin, s2_rebin_stride, s2_stride, s2_tmax, s2_tmin, thr_sipm_s2, + h5out, compression, sipm_rwf_to_cal) + + result = push(source = wf_from_files(files_in, WfType.mcrd), + pipe = pipe(fl.slice(*event_range, close_all=True), + print_every(print_mod), + event_count_in.spy, + mcrd_to_rwf, + simulate_pmt, + pmt_sum, + zero_suppress, + simulate_sipm_response_, + discretize_signal, + compute_pmaps, + event_count_out.spy, + fl.branch("event_number", evtnum_collect.sink), + fl.fork(write_event_info, + write_trigger_info)), + result = dict(events_in = event_count_in .future, + events_out = event_count_out.future, + evtnum_list = evtnum_collect .future, + over_thr = empty_indices .future, + full_pmap = empty_pmaps .future)) + + if run_number <= 0: + copy_mc_info(files_in, h5out, result.evtnum_list, + detector_db, run_number) + + +def rebin_pmts(rebin_stride): + def rebin_pmts(rwf): + rebinned_wfs = rwf + if rebin_stride > 1: + # dummy data for times and widths + times = np.zeros(rwf.shape[1]) + widths = times + waveforms = rwf + _, _, rebinned_wfs = pkf.rebin_times_and_waveforms(times, widths, waveforms, rebin_stride=rebin_stride) + return rebinned_wfs + return rebin_pmts + + +def pmts_sum(rwfs): + return rwfs.sum(axis=0) + diff --git a/invisible_cities/cities/hypathia_test.py b/invisible_cities/cities/hypathia_test.py new file mode 100644 index 0000000000..736588670b --- /dev/null +++ b/invisible_cities/cities/hypathia_test.py @@ -0,0 +1,40 @@ +import os + +import tables as tb +import numpy as np + +from .. core.configure import configure +from .. core.testing_utils import assert_tables_equality + +from . hypathia import hypathia + + +def test_hypathia_exact_result(ICDATADIR, output_tmpdir): + file_in = os.path.join(ICDATADIR , "Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.MCRD.h5") + file_out = os.path.join(output_tmpdir, "exact_result_hypathia.h5") + true_output = os.path.join(ICDATADIR , "Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.hypathia.h5") + + conf = configure("hypathia invisible_cities/config/hypathia.conf".split()) + conf.update(dict(run_number = -6340, + files_in = file_in, + file_out = file_out, + event_range = (0, 2))) + + # Set a specific seed because we want the result to be + # repeatible. Go back to original state after running. + original_random_state = np.random.get_state() + np.random.seed(123456789) + hypathia(**conf) + np.random.set_state(original_random_state) + + tables = ( "PMAPS/S1" , "PMAPS/S2" , "PMAPS/S2Si" , + "PMAPS/S1Pmt" , "PMAPS/S2Pmt" , + "Run/events" , "Run/runInfo" , + "Trigger/events" , "Trigger/trigger" , + "Filters/s12_indices", "Filters/empty_pmap") + with tb.open_file(true_output) as true_output_file: + with tb.open_file(file_out) as output_file: + for table in tables: + got = getattr( output_file.root, table) + expected = getattr(true_output_file.root, table) + assert_tables_equality(got, expected) diff --git a/invisible_cities/cities/irene.py b/invisible_cities/cities/irene.py index 1d66fca786..656b07eb2a 100644 --- a/invisible_cities/cities/irene.py +++ b/invisible_cities/cities/irene.py @@ -16,21 +16,13 @@ - Match the time window of the PMT pulse with those in the SiPMs. - Build the PMap object. """ -import numpy as np import tables as tb -from .. types.ic_types import minmax -from .. database import load_db - -from .. reco import tbl_functions as tbl -from .. reco import peak_functions as pkf -from .. core.random_sampling import NoiseSampler as SiPMsNoiseSampler -from .. core.system_of_units_c import units -from .. io .pmaps_io import pmap_writer -from .. io .mcinfo_io import mc_info_writer -from .. io .run_and_event_io import run_and_event_writer -from .. io .trigger_io import trigger_writer -from .. io .event_filter_io import event_filter_writer +from .. reco import tbl_functions as tbl +from .. core.random_sampling import NoiseSampler as SiPMsNoiseSampler +from .. core import system_of_units as units +from .. io .run_and_event_io import run_and_event_writer +from .. io .trigger_io import trigger_writer from .. dataflow import dataflow as fl from .. dataflow.dataflow import push @@ -39,13 +31,16 @@ from . components import city from . components import print_every +from . components import collect +from . components import copy_mc_info from . components import deconv_pmt from . components import calibrate_pmts from . components import calibrate_sipms from . components import zero_suppress_wfs from . components import WfType from . components import wf_from_files - +from . components import get_number_of_active_pmts +from . components import compute_and_write_pmaps @city @@ -88,108 +83,50 @@ def irene(files_in, file_out, compression, event_range, print_mod, detector_db, sipm_rwf_to_cal = fl.map(calibrate_sipms(detector_db, run_number, sipm_thr), item = "sipm") - # Build the PMap - compute_pmap = fl.map(build_pmap(detector_db, run_number, pmt_samp_wid, sipm_samp_wid, - s1_lmax, s1_lmin, s1_rebin_stride, s1_stride, s1_tmax, s1_tmin, - s2_lmax, s2_lmin, s2_rebin_stride, s2_stride, s2_tmax, s2_tmin, thr_sipm_s2), - args = ("ccwfs", "s1_indices", "s2_indices", "sipm"), - out = "pmap") - - ### Define data filters - - # Filter events without signal over threshold - indices_pass = fl.map(check_nonempty_indices, args = ("s1_indices", "s2_indices"), out = "indices_pass") - empty_indices = fl.count_filter(bool, args = "indices_pass") - - # Filter events with zero peaks - pmaps_pass = fl.map(check_empty_pmap, args = "pmap", out = "pmaps_pass") - empty_pmaps = fl.count_filter(bool, args = "pmaps_pass") - event_count_in = fl.spy_count() event_count_out = fl.spy_count() + evtnum_collect = collect() + with tb.open_file(file_out, "w", filters = tbl.filters(compression)) as h5out: # Define writers... write_event_info_ = run_and_event_writer(h5out) - write_mc_ = mc_info_writer (h5out) if run_number <= 0 else (lambda *_: None) - write_pmap_ = pmap_writer (h5out, compression=compression) write_trigger_info_ = trigger_writer (h5out, get_number_of_active_pmts(detector_db, run_number)) - write_indx_filter_ = event_filter_writer (h5out, "s12_indices", compression=compression) - write_pmap_filter_ = event_filter_writer (h5out, "empty_pmap" , compression=compression) # ... and make them sinks + write_event_info = sink(write_event_info_ , args=( "run_number", "event_number", "timestamp" )) - write_mc = sink(write_mc_ , args=( "mc", "event_number" )) - write_pmap = sink(write_pmap_ , args=( "pmap", "event_number" )) write_trigger_info = sink(write_trigger_info_, args=( "trigger_type", "trigger_channels" )) - write_indx_filter = sink(write_indx_filter_ , args=( "event_number", "indices_pass")) - write_pmap_filter = sink(write_pmap_filter_ , args=( "event_number", "pmaps_pass")) - - return push(source = wf_from_files(files_in, WfType.rwf), - pipe = pipe( - fl.slice(*event_range, close_all=True), - print_every(print_mod), - event_count_in.spy, - rwf_to_cwf, - cwf_to_ccwf, - zero_suppress, - indices_pass, - fl.branch(write_indx_filter), - empty_indices.filter, - sipm_rwf_to_cal, - compute_pmap, - pmaps_pass, - fl.branch(write_pmap_filter), - empty_pmaps.filter, - event_count_out.spy, - fl.fork(write_pmap, - write_mc, - write_event_info, - write_trigger_info)), - result = dict(events_in = event_count_in .future, - events_out = event_count_out.future, - over_thr = empty_indices .future, - full_pmap = empty_pmaps .future)) - - - -def build_pmap(detector_db, run_number, pmt_samp_wid, sipm_samp_wid, - s1_lmax, s1_lmin, s1_rebin_stride, s1_stride, s1_tmax, s1_tmin, - s2_lmax, s2_lmin, s2_rebin_stride, s2_stride, s2_tmax, s2_tmin, thr_sipm_s2): - s1_params = dict(time = minmax(min = s1_tmin, - max = s1_tmax), - length = minmax(min = s1_lmin, - max = s1_lmax), - stride = s1_stride, - rebin_stride = s1_rebin_stride) - - s2_params = dict(time = minmax(min = s2_tmin, - max = s2_tmax), - length = minmax(min = s2_lmin, - max = s2_lmax), - stride = s2_stride, - rebin_stride = s2_rebin_stride) - - datapmt = load_db.DataPMT(detector_db, run_number) - pmt_ids = datapmt.SensorID[datapmt.Active.astype(bool)].values - - def build_pmap(ccwf, s1_indx, s2_indx, sipmzs): # -> PMap - return pkf.get_pmap(ccwf, s1_indx, s2_indx, sipmzs, - s1_params, s2_params, thr_sipm_s2, pmt_ids, - pmt_samp_wid, sipm_samp_wid) - - return build_pmap - - -def get_number_of_active_pmts(detector_db, run_number): - datapmt = load_db.DataPMT(detector_db, run_number) - return np.count_nonzero(datapmt.Active.values.astype(bool)) - - -def check_nonempty_indices(s1_indices, s2_indices): - return s1_indices.size and s2_indices.size - - -def check_empty_pmap(pmap): - return pmap.s1s + pmap.s2s + + + compute_pmaps, empty_indices, empty_pmaps = compute_and_write_pmaps( + detector_db, run_number, pmt_samp_wid, sipm_samp_wid, + s1_lmax, s1_lmin, s1_rebin_stride, s1_stride, s1_tmax, s1_tmin, + s2_lmax, s2_lmin, s2_rebin_stride, s2_stride, s2_tmax, s2_tmin, + thr_sipm_s2, + h5out, compression, sipm_rwf_to_cal) + + result = push(source = wf_from_files(files_in, WfType.rwf), + pipe = pipe(fl.slice(*event_range, close_all=True), + print_every(print_mod), + event_count_in.spy, + rwf_to_cwf, + cwf_to_ccwf, + zero_suppress, + compute_pmaps, + event_count_out.spy, + fl.branch("event_number", evtnum_collect.sink), + fl.fork(write_event_info, + write_trigger_info)), + result = dict(events_in = event_count_in .future, + events_out = event_count_out.future, + evtnum_list = evtnum_collect .future, + over_thr = empty_indices .future, + full_pmap = empty_pmaps .future)) + + if run_number <= 0: + copy_mc_info(files_in, h5out, result.evtnum_list, + detector_db, run_number) + + return result diff --git a/invisible_cities/cities/irene_test.py b/invisible_cities/cities/irene_test.py index 9ba965427a..e87fc769d8 100644 --- a/invisible_cities/cities/irene_test.py +++ b/invisible_cities/cities/irene_test.py @@ -1,8 +1,8 @@ import os -from collections import namedtuple import tables as tb import numpy as np +import pandas as pd from pytest import raises from pytest import mark @@ -12,12 +12,14 @@ from .. core.configure import all as all_events from .. core.configure import configure from .. core.testing_utils import exactly +from .. core.testing_utils import assert_dataframes_close from .. core.testing_utils import assert_tables_equality from .. types.ic_types import minmax from .. io.run_and_event_io import read_run_and_event from .. evm.ic_containers import S12Params as S12P +from .. io.mcinfo_io import load_mcparticles_df +from .. io.mcinfo_io import load_mchits_df -from .. database import load_db from .. database.load_db import DetDB from .. io .pmaps_io import load_pmaps @@ -56,24 +58,6 @@ def unpack_s12params(s12params): s2_lmax = s2par.length.max) -@fixture(scope='module') -def job_info_missing_pmts(config_tmpdir, ICDATADIR): - # Specifies a name for a data configuration file. Also, default number - # of events set to 1. - job_info = namedtuple("job_info", - "run_number pmt_missing pmt_active input_filename output_filename") - - run_number = 3366 - pmt_missing = [11] - pmt_active = list(filter(lambda x: x not in pmt_missing, range(12))) - - - ifilename = os.path.join(ICDATADIR , 'electrons_40keV_z250_RWF.h5') - ofilename = os.path.join(config_tmpdir, 'electrons_40keV_z250_pmaps_missing_PMT.h5') - - return job_info(run_number, pmt_missing, pmt_active, ifilename, ofilename) - - @mark.slow @mark.parametrize("thr_sipm_type thr_sipm_value".split(), (("common" , 3.5 ), @@ -84,8 +68,10 @@ def test_irene_electrons_40keV(config_tmpdir, ICDATADIR, s12params, # since they are in general test-specific # NB: avoid taking defaults for run number (test-specific) - PATH_IN = os.path.join(ICDATADIR , 'electrons_40keV_ACTIVE_10evts_RWF.h5') - PATH_OUT = os.path.join(config_tmpdir, 'electrons_40keV_ACTIVE_10evts_CWF.h5') + PATH_IN = os.path.join(ICDATADIR , + 'electrons_40keV_ACTIVE_10evts_RWF.h5') + PATH_OUT = os.path.join(config_tmpdir , + 'electrons_40keV_ACTIVE_10evts_CWF.h5') nrequired = 2 @@ -104,12 +90,11 @@ def test_irene_electrons_40keV(config_tmpdir, ICDATADIR, s12params, nactual = cnt.events_in assert nrequired == nactual + mcparticles_in = load_mcparticles_df( PATH_IN) + mcparticles_out = load_mcparticles_df(PATH_OUT) + assert_dataframes_close(mcparticles_in, mcparticles_out) with tb.open_file(PATH_IN , mode='r') as h5in, \ tb.open_file(PATH_OUT, mode='r') as h5out: - nrow = 0 - mctracks_in = h5in .root.MC.particles[nrow] - mctracks_out = h5out.root.MC.particles[nrow] - np.testing.assert_array_equal(mctracks_in, mctracks_out) # check events numbers & timestamps evts_in = h5in .root.Run.events[:nactual].astype([('evt_number', ' 0 - assert cnt.events_out <= cnt.events_in diff --git a/invisible_cities/config/PmapsGenericConfig.json b/invisible_cities/config/PmapsGenericConfig.json deleted file mode 100644 index eb34419871..0000000000 --- a/invisible_cities/config/PmapsGenericConfig.json +++ /dev/null @@ -1 +0,0 @@ -{"S1_Number_bins": [-0.5, 10.5, 11], "S1_Width_bins": [-0.01, 0.99, 40], "S1_Height_bins": [0, 200, 50], "S1_Energy_bins": [0, 1000, 200], "S1_Charge_bins": [0, 50, 50], "S1_Time_bins": [0, 1000, 1000], "S2_Number_bins": [-0.5, 10.5, 11], "S2_Width_bins": [0, 250, 50], "S2_Height_bins": [0, 500000.0, 100], "S2_Energy_bins": [0, 1200000.0, 200], "S2_Charge_bins": [0, 200000.0, 200], "S2_Time_bins": [640, 2000, 1360], "S2_NSiPM_bins": [-0.5, 1792.5, 1793], "S2_IdSiPM_bins": [-0.5, 1792.5, 1793], "S2_QSiPM_bins": [0, 1000, 100], "S2_XSiPM_bins": [-200, 200, 40], "S2_YSiPM_bins": [-200, 200, 40], "S1_Number_labels": ["S1 number (#)"], "S1_Width_labels": ["S1 width ($\\mu$s)"], "S1_Height_labels": ["S1 height (pes)"], "S1_Energy_labels": ["S1 energy (pes)"], "S1_Charge_labels": ["S1 charge (pes)"], "S1_Time_labels": ["S1 time ($\\mu$s)"], "S2_Number_labels": ["S2 number (#)"], "S2_Width_labels": ["S2 width ($\\mu$s)"], "S2_Height_labels": ["S2 height (pes)"], "S2_Energy_labels": ["S2 energy (pes)"], "S2_Charge_labels": ["S2 charge (pes)"], "S2_Time_labels": ["S2 time ($\\mu$s)"], "S2_NSiPM_labels": ["SiPM number (#)"], "S2_IdSiPM_labels": ["SiPM id"], "S2_QSiPM_labels": ["SiPM charge (pes)"], "S2_XSiPM_labels": ["X (mm)"], "S2_YSiPM_labels": ["Y (mm)"], "nPMT": 12} \ No newline at end of file diff --git a/invisible_cities/config/PmapsKrConfig.json b/invisible_cities/config/PmapsKrConfig.json deleted file mode 100644 index 753be3f7d8..0000000000 --- a/invisible_cities/config/PmapsKrConfig.json +++ /dev/null @@ -1 +0,0 @@ -{"S1_Number_bins": [-0.5, 10.5, 11], "S1_Width_bins": [-0.01, 0.99, 40], "S1_Height_bins": [0, 10, 10], "S1_Energy_bins": [0, 50, 50], "S1_Charge_bins": [0, 2, 20], "S1_Time_bins": [0, 650, 650], "S2_Number_bins": [-0.5, 10.5, 11], "S2_Width_bins": [0, 50, 50], "S2_Height_bins": [0, 8000, 100], "S2_Energy_bins": [0, 20000.0, 100], "S2_Charge_bins": [0, 3500, 100], "S2_Time_bins": [640, 1300, 660], "S2_NSiPM_bins": [-0.5, 500.5, 501], "S2_IdSiPM_bins": [-0.5, 1792.5, 1793], "S2_QSiPM_bins": [0, 100, 100], "S2_XSiPM_bins": [-200, 200, 40], "S2_YSiPM_bins": [-200, 200, 40], "S1_Number_labels": ["S1 number (#)"], "S1_Width_labels": ["S1 width ($\\mu$s)"], "S1_Height_labels": ["S1 height (pes)"], "S1_Energy_labels": ["S1 energy (pes)"], "S1_Charge_labels": ["S1 charge (pes)"], "S1_Time_labels": ["S1 time ($\\mu$s)"], "S2_Number_labels": ["S2 number (#)"], "S2_Width_labels": ["S2 width ($\\mu$s)"], "S2_Height_labels": ["S2 height (pes)"], "S2_Energy_labels": ["S2 energy (pes)"], "S2_Charge_labels": ["S2 charge (pes)"], "S2_Time_labels": ["S2 time ($\\mu$s)"], "S2_NSiPM_labels": ["SiPM number (#)"], "S2_IdSiPM_labels": ["SiPM id"], "S2_QSiPM_labels": ["SiPM charge (pes)"], "S2_XSiPM_labels": ["X (mm)"], "S2_YSiPM_labels": ["Y (mm)"], "nPMT": 12} \ No newline at end of file diff --git a/invisible_cities/config/RwfConfig.json b/invisible_cities/config/RwfConfig.json deleted file mode 100644 index 6db3aee580..0000000000 --- a/invisible_cities/config/RwfConfig.json +++ /dev/null @@ -1 +0,0 @@ -{"PMT_Baseline_bins": [2300.0, 2700.0, 400], "PMT_BaselineRMS_bins": [0.0, 10.0, 100], "PMT_nSensors_bins": [-0.5, 12.5, 13], "SiPM_Baseline_bins": [0.0, 100.0, 100], "SiPM_BaselineRMS_bins": [0.0, 10.0, 100], "SiPM_nSensors_bins": [1750.5, 1800.5, 50], "PMT_Baseline_labels": ["PMT Baseline (ADC)"], "PMT_BaselineRMS_labels": ["PMT Baseline RMS (ADC)"], "PMT_nSensors_labels": ["Number of PMTs"], "SiPM_Baseline_labels": ["SiPM Baseline (ADC)"], "SiPM_BaselineRMS_labels": ["SiPM Baseline RMS (ADC)"], "SiPM_nSensors_labels": ["Number of SiPMs"], "n_baseline": 48000} \ No newline at end of file diff --git a/invisible_cities/config/berenice.conf b/invisible_cities/config/berenice.conf index e4436b1e73..a4ffc94a37 100644 --- a/invisible_cities/config/berenice.conf +++ b/invisible_cities/config/berenice.conf @@ -1,5 +1,5 @@ -files_in = '$ICDIR/database/test_data/electrons_40keV_z250_RWF.h5' -file_out = '/tmp/electrons_40keV_z250_sipmPDF.h5' +files_in = '$ICDIR/database/test_data/electrons_40keV_z25_RWF.h5' +file_out = '/tmp/electrons_40keV_z25_sipmPDF.h5' compression = 'ZLIB4' print_mod = 1 event_range = 1 diff --git a/invisible_cities/config/hypathia.conf b/invisible_cities/config/hypathia.conf new file mode 100644 index 0000000000..fe428d799a --- /dev/null +++ b/invisible_cities/config/hypathia.conf @@ -0,0 +1,51 @@ +files_in = '$ICDIR/database/test_data/electrons_40keV_z250_MCRD.h5' + +# REPLACE /tmp with your output directory +file_out = '/tmp/electrons_40keV_z250_PMP.h5' + +# compression library +compression = 'ZLIB4' + +# run number +run_number = 0 +detector_db = 'new' + +# How frequently to print events +print_mod = 1 + +pmt_wfs_rebin = 25 +pmt_pe_rms = 0.4 +sipm_noise_cut = 1.0 * pes +filter_padding = 50 + +# max number of events to run +event_range = 0, 2 + +# Set thresholds for calibrated sum +thr_csum_s1 = 0.5 * pes +thr_csum_s2 = 2.0 * pes + +# Set MAU thresholds for SiPM +thr_sipm = 1.0 * pes +thr_sipm_type = "common" + + +# Set parameters to search for S1 +# Notice that in MC file S1 is in t=100 mus +s1_tmin = 99 * mus # position of S1 in MC files at 100 mus +s1_tmax = 101 * mus # change tmin and tmax if S1 not at 100 mus +s1_stride = 4 # minimum number of 25 ns bins in S1 searches +s1_lmin = 4 # 8 x 25 = 200 ns +s1_lmax = 40 # 20 x 25 = 500 ns +s1_rebin_stride = 1 # Do not rebin S1 by default + +# Set parameters to search for S2 +s2_tmin = 101 * mus # assumes S1 at 100 mus, change if S1 not at 100 mus +s2_tmax = 799 * mus # end of the window +s2_stride = 40 # 40 x 25 = 1 mus +s2_lmin = 80 # 100 x 25 = 2.5 mus +s2_lmax = 100000 # maximum value of S2 width +s2_rebin_stride = 40 # Rebin by default, 40 25 ns time bins to make one 1us time bin + +# Set S2Si parameters +thr_sipm_s2 = 5 * pes # Threshold for the full sipm waveform diff --git a/invisible_cities/config/irene.conf b/invisible_cities/config/irene.conf index 5c34cce170..f81ed1a38d 100644 --- a/invisible_cities/config/irene.conf +++ b/invisible_cities/config/irene.conf @@ -1,7 +1,7 @@ -files_in = '$ICDIR/database/test_data/electrons_40keV_z250_RWF.h5' +files_in = '$ICDIR/database/test_data/electrons_40keV_z25_RWF.h5' # REPLACE /tmp with your output directory -file_out = '/tmp/electrons_40keV_z250_PMP.h5' +file_out = '/tmp/electrons_40keV_z25_PMP.h5' # compression library compression = 'ZLIB4' diff --git a/invisible_cities/config/isidora.conf b/invisible_cities/config/isidora.conf index 0e25e5ce42..b73d09cd0e 100644 --- a/invisible_cities/config/isidora.conf +++ b/invisible_cities/config/isidora.conf @@ -1,8 +1,8 @@ # set_input_files -files_in = '$ICDIR/database/test_data/electrons_40keV_z250_RWF.h5' +files_in = '$ICDIR/database/test_data/electrons_40keV_z25_RWF.h5' # REPLACE /tmp with your output directory -file_out = '/tmp/electrons_40keV_z250_PMP.h5' +file_out = '/tmp/electrons_40keV_z25_PMP.h5' # compression library compression = 'ZLIB4' diff --git a/invisible_cities/config/kDSTConfig.json b/invisible_cities/config/kDSTConfig.json deleted file mode 100644 index a23b0967bf..0000000000 --- a/invisible_cities/config/kDSTConfig.json +++ /dev/null @@ -1 +0,0 @@ -{"nS2_bins": [-0.5, 10.5, 11], "S1w_bins": [-0.01, 0.99, 40], "S1h_bins": [0, 10, 10], "S1e_bins": [0, 50, 50], "S1t_bins": [0, 650, 650], "S2w_bins": [0, 50, 50], "S2h_bins": [0, 8000, 100], "S2e_bins": [0, 20000.0, 100], "S2q_bins": [0, 2000, 100], "S2t_bins": [640, 1300, 660], "Nsipm_bins": [-0.5, 500.5, 501], "DT_bins": [0, 600, 100], "Z_bins": [0, 600, 100], "X_bins": [-200, 200, 50], "Y_bins": [-200, 200, 50], "R_bins": [0, 200, 50], "Phi_bins": [-3.15, 3.15, 50], "Zrms_bins": [0, 80, 40], "Xrms_bins": [0, 200, 50], "Yrms_bins": [0, 200, 50], "nS2_labels": ["S2 number (#)"], "S1w_labels": ["S1 width ($\\mu$s)"], "S1h_labels": ["S1 height (pes)"], "S1e_labels": ["S1 energy (pes)"], "S1t_labels": ["S1 time ($\\mu$s)"], "S2w_labels": ["S2 width ($\\mu$s)"], "S2h_labels": ["S2 height (pes)"], "S2e_labels": ["S2 energy (pes)"], "S2q_labels": ["S2 charge (pes)"], "S2t_labels": ["S2 time ($\\mu$s)"], "Nsipm_labels": ["SiPM number (#)"], "DT_labels": ["S2 - S1 time ($\\mu$s)"], "Z_labels": ["Z (mm)"], "X_labels": ["X (mm)"], "Y_labels": ["Y (mm)"], "R_labels": ["R (mm)"], "Phi_labels": ["Phi (rad)"], "Zrms_labels": ["Z rms (mm)"], "Xrms_labels": ["X rms (mm)"], "Yrms_labels": ["Y rms (mm)"]} \ No newline at end of file diff --git a/invisible_cities/config/monte_carlo_city.conf b/invisible_cities/config/monte_carlo_city.conf deleted file mode 100644 index c1a78fc643..0000000000 --- a/invisible_cities/config/monte_carlo_city.conf +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/python -# monte_carlo_city.conf -# Configures a Monte Carlo simulation city, that is a city that: -# 1) simulates sensor response (PMT and SiPM) -# 2) simulates trigger - - -include('$ICDIR/config/trigger_emulation_city.conf') - -# Read MCRD - -raw_data_type = 'MCRD' # Monte Carlo Raw Data - -# cut in the noise of individual SiPMs - -sipm_noise_cut = 1.0 * pes diff --git a/invisible_cities/config/phyllis.conf b/invisible_cities/config/phyllis.conf index 1b882238c5..44c7983ceb 100644 --- a/invisible_cities/config/phyllis.conf +++ b/invisible_cities/config/phyllis.conf @@ -1,5 +1,5 @@ -files_in = '$ICDIR/database/test_data/electrons_40keV_z250_RWF.h5' -file_out = '/tmp/electrons_40keV_z250_sipmPDF.h5' +files_in = '$ICDIR/database/test_data/electrons_40keV_z25_RWF.h5' +file_out = '/tmp/electrons_40keV_z25_sipmPDF.h5' compression = 'ZLIB4' print_mod = 1 event_range = 1 diff --git a/invisible_cities/config/trude.conf b/invisible_cities/config/trude.conf index cc16622c03..9a8f375948 100644 --- a/invisible_cities/config/trude.conf +++ b/invisible_cities/config/trude.conf @@ -1,5 +1,5 @@ -files_in = '$ICDIR/database/test_data/electrons_40keV_z250_RWF.h5' -file_out = '/tmp/electrons_40keV_z250_sipmPDF.h5' +files_in = '$ICDIR/database/test_data/electrons_40keV_z25_RWF.h5' +file_out = '/tmp/electrons_40keV_z25_sipmPDF.h5' compression = 'ZLIB4' print_mod = 1 event_range = 1 diff --git a/invisible_cities/config/zaira.conf b/invisible_cities/config/zaira.conf deleted file mode 100644 index 2edfb79b3d..0000000000 --- a/invisible_cities/config/zaira.conf +++ /dev/null @@ -1,26 +0,0 @@ -files_in = '$ICDIR/database/test_data/KrDST_MC.h5' -file_out = '/tmp/run3389.corr' - -detector_db = 'new' - -dst_group = 'DST' -dst_node = 'Events' - -lifetime = 10 * ms -u_lifetime = 10 * mus - -xbins = 50 -xmin = -215 * mm -xmax = +215 * mm - -ybins = 50 -ymin = -215 * mm -ymax = +215 * mm - -rmax = 100 * mm - -zmin = 0 * mm -zmax = 200 * mm - -emin = 3e3 * pes -emax = 1e4 * pes diff --git a/invisible_cities/conftest.py b/invisible_cities/conftest.py index 2b09fdadd7..7165eeabb5 100644 --- a/invisible_cities/conftest.py +++ b/invisible_cities/conftest.py @@ -6,16 +6,16 @@ from pandas import DataFrame from collections import namedtuple -from . core .system_of_units_c import units -from . evm . pmaps_test import pmaps -from . io . pmaps_io import load_pmaps_as_df -from . io . pmaps_io import load_pmaps -from . io . pmaps_io import pmap_writer -from . io . dst_io import load_dst -from . io . hits_io import load_hits -from . io . hits_io import load_hits_skipping_NN -from . io . mcinfo_io import load_mchits -from . types.ic_types import NN +from . core import system_of_units as units +from . evm . pmaps_test import pmaps +from . io . pmaps_io import load_pmaps_as_df +from . io . pmaps_io import load_pmaps +from . io . pmaps_io import pmap_writer +from . io . dst_io import load_dst +from . io . hits_io import load_hits +from . io . hits_io import load_hits_skipping_NN +from . io .mcinfo_io import load_mchits_df +from . types.ic_types import NN tbl_data = namedtuple('tbl_data', 'filename group node') dst_data = namedtuple('dst_data', 'file_info config read true') @@ -38,11 +38,6 @@ def ICDATADIR(ICDIR): def PSFDIR(ICDIR): return os.path.join(ICDIR, "database/test_data/PSF_dst_sum_collapsed.h5") -@pytest.fixture(scope = 'session') -def irene_diomira_chain_tmpdir(tmpdir_factory): - return tmpdir_factory.mktemp('irene_diomira_tests') - - @pytest.fixture(scope = 'session') def config_tmpdir(tmpdir_factory): return tmpdir_factory.mktemp('configure_tests') @@ -58,15 +53,6 @@ def example_blr_wfs_filename(ICDATADIR): return os.path.join(ICDATADIR, "blr_examples.h5") -@pytest.fixture(scope = 'session', - params = ['electrons_40keV_z250_RWF.h5', - 'electrons_511keV_z250_RWF.h5', - 'electrons_1250keV_z250_RWF.h5', - 'electrons_2500keV_z250_RWF.h5']) -def electron_RWF_file(request, ICDATADIR): - return os.path.join(ICDATADIR, request.param) - - @pytest.fixture(scope = 'session', params = ['electrons_40keV_z250_MCRD.h5']) def electron_MCRD_file(request, ICDATADIR): @@ -88,6 +74,11 @@ def nohits_sim_file(request, ICDATADIR): def sns_only_sim_file(request, ICDATADIR): return os.path.join(ICDATADIR, request.param) +@pytest.fixture(scope = 'session', + params = ['Kr83_full_nexus_v5_03_01_ACTIVE_7bar_1evt.sim.h5']) +def full_sim_file(request, ICDATADIR): + return os.path.join(ICDATADIR, request.param) + @pytest.fixture(scope='session') def mc_all_hits_data(krypton_MCRD_file): @@ -96,40 +87,6 @@ def mc_all_hits_data(krypton_MCRD_file): efile = krypton_MCRD_file return efile, number_of_hits, evt_number -@pytest.fixture(scope='session') -def mc_particle_and_hits_nexus_data(ICDATADIR): - X = [ -4.37347144e-02, -2.50248108e-02, -3.25887166e-02, -3.25617939e-02 ] - Y = [ -1.37645766e-01, -1.67959690e-01, -1.80502057e-01, -1.80206522e-01 ] - Z = [ 2.49938721e+02, 2.49911240e+02, 2.49915543e+02, 2.49912308e+02 ] - E = [ 0.02225098, 0.00891293, 0.00582698, 0.0030091 ] - t = [ 0.00139908, 0.00198319, 0.00226054, 0.00236114 ] - - vi = np.array([ 0., 0., 250., 0.]) - vf = np.array([-3.25617939e-02, -1.80206522e-01, 2.49912308e+02, 2.36114440e-03]) - - p = np.array([-0.05745485, -0.18082699, -0.08050126]) - - efile = os.path.join(ICDATADIR, 'electrons_40keV_z250_MCRD.h5') - Ep = 0.04 - name = 'e-' - nhits = 4 - - return efile, name, vi, vf, p, Ep, nhits, X, Y, Z, E, t - - -@pytest.fixture(scope='session') -def mc_sensors_nexus_data(ICDATADIR): - pmt0_first = (0, 1) - pmt0_last = (670, 1) - pmt0_tot_samples = 54 - - sipm_id = 13016 - sipm = [(63, 3), (64, 2), (65, 1)] - - efile = os.path.join(ICDATADIR, 'Kr83_full_nexus_v5_03_01_ACTIVE_7bar_1evt.sim.h5') - - return efile, pmt0_first, pmt0_last, pmt0_tot_samples, sipm_id, sipm - def _get_pmaps_dict_and_event_numbers(filename): dict_pmaps = load_pmaps(filename) @@ -178,12 +135,6 @@ def KrMC_pmaps_dict(KrMC_pmaps_filename): return dict_pmaps, evt_numbers -@pytest.fixture(scope='session') -def KrMC_pmaps_without_ipmt_dict(KrMC_pmaps_without_ipmt_filename): - dict_pmaps, evt_numbers = _get_pmaps_dict_and_event_numbers(KrMC_pmaps_without_ipmt_filename) - return dict_pmaps, evt_numbers - - @pytest.fixture(scope='session') def correction_map_filename(ICDATADIR): test_file = "kr_emap_xy_100_100_r_6573_time.h5" @@ -638,8 +589,8 @@ def KrMC_hdst(ICDATADIR): def KrMC_true_hits(KrMC_pmaps_filename, KrMC_hdst): pmap_filename = KrMC_pmaps_filename hdst_filename = KrMC_hdst .file_info.filename - pmap_mctracks = load_mchits(pmap_filename) - hdst_mctracks = load_mchits(hdst_filename) + pmap_mctracks = load_mchits_df(pmap_filename) + hdst_mctracks = load_mchits_df(hdst_filename) return mcs_data(pmap_mctracks, hdst_mctracks) @@ -665,17 +616,6 @@ def TlMC_hits_merged(ICDATADIR): hits = load_hits(hits_file_name) return hits -@pytest.fixture(scope='session') -def corr_toy_data(ICDATADIR): - x = np.arange( 100, 200) - y = np.arange(-200, 0) - E = np.arange( 1e4, 1e4 + x.size*y.size).reshape(x.size, y.size) - U = np.arange( 1e2, 1e2 + x.size*y.size).reshape(x.size, y.size) - N = np.ones_like(U) - - corr_filename = os.path.join(ICDATADIR, "toy_corr.h5") - return corr_filename, (x, y, E, U, N) - @pytest.fixture(scope='session') def hits_toy_data(ICDATADIR): @@ -732,11 +672,16 @@ def dbnew(): def dbnext100(): return 'next100' +@pytest.fixture(scope='session') +def dbflex100(): + return 'flex100' + @pytest.fixture(scope='session', params=[db_data('demopp' , 3, 256, 3, 79), db_data('new' , 12, 1792, 3, 79), - db_data('next100', 60, 6848, 8, 79)], - ids=["demo", "new", "next100"]) + db_data('next100', 60, 3584, 0, 0), + db_data('flex100', 60, 3093, 0, 0)], + ids=["demo", "new", "next100", "flex100"]) def db(request): return request.param @@ -744,7 +689,6 @@ def db(request): def deconvolution_config(ICDIR, ICDATADIR, PSFDIR, config_tmpdir): PATH_IN = os.path.join(ICDATADIR , "test_Xe2nu_NEW_v1.2.0_cdst.5_62.h5") PATH_OUT = os.path.join(config_tmpdir, "beersheba_MC.h5") - config_path = os.path.join(ICDIR , "config/beersheba.conf") nevt_req = 3 conf = dict(files_in = PATH_IN , file_out = PATH_OUT, @@ -768,3 +712,23 @@ def deconvolution_config(ICDIR, ICDATADIR, PSFDIR, config_tmpdir): inter_method = 'cubic')) return conf, PATH_OUT + +## To make very slow tests only run with specific option +def pytest_addoption(parser): + parser.addoption( + "--runslow", action="store_true", default=False, help="run slow tests" + ) + + +def pytest_configure(config): + config.addinivalue_line("markers", "veryslow: mark test as very slow to run") + + +def pytest_collection_modifyitems(config, items): + if config.getoption("--runslow"): + # --runslow given in cli: do not skip slow tests + return + skip_slow = pytest.mark.skip(reason="need --runslow option to run") + for item in items: + if "veryslow" in item.keywords: + item.add_marker(skip_slow) diff --git a/invisible_cities/core/configure_test.py b/invisible_cities/core/configure_test.py index 957cb70e13..6885bc4658 100644 --- a/invisible_cities/core/configure_test.py +++ b/invisible_cities/core/configure_test.py @@ -66,8 +66,8 @@ """ # The values that will be fed into the above. -config_file_spec = dict(files_in = 'electrons_40keV_z250_RWF.h5', - file_out = 'electrons_40keV_z250_PMP.h5', +config_file_spec = dict(files_in = 'electrons_40keV_z25_RWF.h5', + file_out = 'electrons_40keV_z25_PMP.h5', compression = 'ZLIB4', run_number = 23, nprint = 24, @@ -189,13 +189,13 @@ def test_configure_does_not_take_multiple_arguments(default_conf, flag): iargs = " ".join(f"arg_{i}.h5" for i in range(2)) argv = f"dummy {default_conf} {flag} {iargs}".split() with raises(SystemExit): - conf = configure(argv) + configure(argv) def test_configure_raises_SystemExit_with_multiple_mutually_exclusive_options(): argv = f"dummy {default_conf} --no-files --full-files".split() with raises(SystemExit): - conf = configure(argv) + configure(argv) def test_read_config_file_special_values(config_tmpdir): @@ -206,7 +206,7 @@ def test_read_config_file_special_values(config_tmpdir): var_last = last vars_units = {all_units} """) - argv = f"dummy {default_conf} -i ifile -o ofile -r runno".split() + conf = read_config_file(filename) assert conf["var_all" ] is all assert conf["var_last" ] is last diff --git a/invisible_cities/core/core_functions.py b/invisible_cities/core/core_functions.py index c4ca5b1667..0b0577546b 100644 --- a/invisible_cities/core/core_functions.py +++ b/invisible_cities/core/core_functions.py @@ -3,7 +3,8 @@ This module includes utility functions. """ import time -import enum + +from enum import auto import numpy as np @@ -12,15 +13,11 @@ from .. types.ic_types import AutoNameEnumBase -class NormMode(enum.Enum): - first = 0 - second = 1 - sumof = 3 - mean = 4 - - -def merge_two_dicts(a,b): - return {**a, **b} +class NormMode(AutoNameEnumBase): + first = auto() + second = auto() + sumof = auto() + mean = auto() def timefunc(f): diff --git a/invisible_cities/core/core_functions_performance.py b/invisible_cities/core/core_functions_performance.py deleted file mode 100644 index 3574df28ac..0000000000 --- a/invisible_cities/core/core_functions_performance.py +++ /dev/null @@ -1,32 +0,0 @@ - -""" -Performance of some coreFunctions -""" - -from time import time -import numpy as np - -from .. reco import wfm_functions as wfm -from .. reco import peak_functions_c as cpf - - -t = np.arange(1.,100., 0.1, dtype=np.double) -e = np.exp(-t/t**2) - -t0 = time() -T, E = wfm.rebin_wf(t, e, stride=2) -t1 = time() -dt = t1 - t0 -print("rebin_wf (wfmFunctions) run in {} s".format(dt)) - -t0 = time() -T, E = wfm.rebin_waveform(t, e, stride=2) -t1 = time() -dt = t1 - t0 -print("rebin_waveform (wfmFunctions) run in {} s".format(dt)) - -t0 = time() -T, E = cpf.rebin_waveform(t, e, stride=2) -t1 = time() -dt = t1 - t0 -print("cython rebin_waveform (Reco/peakFunctions) run in {} s".format(dt)) diff --git a/invisible_cities/core/core_functions_test.py b/invisible_cities/core/core_functions_test.py index 931f029e9d..ee624e5e4e 100644 --- a/invisible_cities/core/core_functions_test.py +++ b/invisible_cities/core/core_functions_test.py @@ -34,9 +34,10 @@ def test_timefunc(capfd): # We run a function with a defined time duration (sleep) and we check # the decorator prints a correct measurement. time = 0.12 - result = core.timefunc(sleep)(time) + core.timefunc(sleep)(time) + out, err = capfd.readouterr() - time_measured = re.search('\d+\.\d+', out).group(0) + time_measured = re.search(r'\d+\.\d+', out).group(0) time_measured = float(time_measured) np.isclose(time, time_measured) @@ -329,7 +330,7 @@ def test_std_handle_empty_empty_input(): assert np.isnan(core.std_handle_empty([])) -@given(arrays(float, 10, floats(min_value=-1e5, max_value=1e5))) +@given(arrays(float, 10, elements=floats(min_value=-1e5, max_value=1e5))) def test_shift_to_bin_centers(x): x_shifted = core.shift_to_bin_centers(x) truth = [np.mean(x[i:i+2]) for i in range(x.size-1)] diff --git a/invisible_cities/core/exceptions.py b/invisible_cities/core/exceptions.py index d26d562a4d..039fa493ea 100644 --- a/invisible_cities/core/exceptions.py +++ b/invisible_cities/core/exceptions.py @@ -14,18 +14,6 @@ class InvalidInputFileStructure(ICException): class NoOutputFile(ICException): pass -class UnknownRWF(ICException): - pass - -class UnknownDST(ICException): - pass - -class ParameterNotSet(ICException): - pass - -class PeakNotFound(ICException): - pass - class XYRecoFail(ICException): pass @@ -38,12 +26,6 @@ class SipmEmptyListAboveQthr(XYRecoFail): class ClusterEmptyList(XYRecoFail): pass -class PmtNotFound(ICException): - pass - -class SipmNotFound(XYRecoFail): - pass - class SipmZeroCharge(XYRecoFail): pass @@ -56,29 +38,17 @@ class NoHits(ICException): class NoVoxels(ICException): pass -class InconsistentS12dIpmtd(ICException): - pass - -class NegativeThresholdNotAllowed(ICException): - pass - -class InitializedEmptyPmapObject(ICException): - pass - -class UnknownParameter(ICException): - pass - class SensorBinningNotFound(ICException): pass class NoParticleInfoInFile(ICException): pass -class VoxelNotInList(ICException): - pass - class TableMismatch(ICException): pass class TimeEvolutionTableMissing(ICException): pass + +class MCEventNotFound(ICException): + """ Requested event missing from input file """ diff --git a/invisible_cities/core/fit_functions_test.py b/invisible_cities/core/fit_functions_test.py index 6a8392cad8..8a20398c92 100644 --- a/invisible_cities/core/fit_functions_test.py +++ b/invisible_cities/core/fit_functions_test.py @@ -17,7 +17,6 @@ from hypothesis.strategies import integers from hypothesis.strategies import floats from . testing_utils import float_arrays -from . testing_utils import FLOAT_ARRAY from . testing_utils import random_length_float_arrays from . import core_functions as core @@ -319,7 +318,7 @@ def test_fixed_parameters(): {'fake1' : 1, 'mu' : 0})) def test_fix_wrong_parameters_raises_error(pars): with raises(ValueError): - fixed_mu = fitf.fixed_parameters(fitf.gauss, **pars) + fitf.fixed_parameters(fitf.gauss, **pars) @mark.parametrize(["func"], @@ -356,9 +355,13 @@ def test_profile1D_uniform_distribution_std(func): assert np.allclose(ye, rms, rtol=0.1) -@mark.parametrize("func xdata ydata".split(), - ((fitf.profileX, FLOAT_ARRAY(100, -1e3, 1e3), FLOAT_ARRAY(100, -1e3, 1e3)), - (fitf.profileY, FLOAT_ARRAY(100, -1e3, 1e3), FLOAT_ARRAY(100, -1e3, 1e3)))) +@given(xdata = float_arrays(size = 100, + min_value = -1e3, + max_value = +1e3), + ydata = float_arrays(size = 100, + min_value = -1e3, + max_value = +1e3)) +@mark.parametrize("func", (fitf.profileX, fitf.profileY)) def test_profile1D_custom_range(func, xdata, ydata): xrange = (-100, 100) kw = "yrange" if func.__name__.endswith("Y") else "xrange" @@ -367,14 +370,17 @@ def test_profile1D_custom_range(func, xdata, ydata): assert np.all(core.in_range(xp, *xrange)) -@mark.parametrize("func xdata ydata xrange".split(), + +@given(xdata = float_arrays(size = 100, + min_value = -100, + max_value = 100), + ydata = float_arrays(size = 100, + min_value = -500, + max_value = 0)) +@mark.parametrize("func xrange".split(), ((fitf.profileX, - FLOAT_ARRAY(100, -100, 100), - FLOAT_ARRAY(100, -500, 0), (-100, 100)), (fitf.profileY, - FLOAT_ARRAY(100, -100, 100), - FLOAT_ARRAY(100, -500, 0), (-500, 0)))) def test_profile1D_full_range_x(func, xdata, ydata, xrange): xp, yp, ye = func(xdata, ydata) @@ -394,26 +400,22 @@ def test_profile1D_one_bin_missing_x(func, xdata, ydata): assert xp.size == 99 -@mark.parametrize("func xdata ydata".split(), - ((fitf.profileX, - FLOAT_ARRAY(100, min_value=-1e10, max_value=+1e10), - FLOAT_ARRAY(100, min_value=-1e10, max_value=+1e10)), - (fitf.profileY, - FLOAT_ARRAY(100, min_value=-1e10, max_value=+1e10), - FLOAT_ARRAY(100, min_value=-1e10, max_value=+1e10)))) +@given(xdata = float_arrays(size = 100, + min_value = -1e10, + max_value = +1e10), + ydata = float_arrays(size = 100, + min_value = -1e10, + max_value = +1e10)) +@mark.parametrize("func", (fitf.profileX, fitf.profileY)) def test_number_of_bins_matches(func, xdata, ydata): N = 50 xp, yp, ye = func(xdata, ydata, N, drop_nan=False) assert xp.size == yp.size == ye.size == N -@mark.parametrize("func xdata ydata".split(), - ((fitf.profileX, - FLOAT_ARRAY(100), - FLOAT_ARRAY(100)), - (fitf.profileY, - FLOAT_ARRAY(100), - FLOAT_ARRAY(100)))) +@given(xdata = float_arrays(size = 100), + ydata = float_arrays(size = 100)) +@mark.parametrize("func", (fitf.profileX, fitf.profileY)) def test_empty_dataset_yields_nans(func, xdata, ydata): N = 50 empty = np.array([]) diff --git a/invisible_cities/core/mctrk_functions.py b/invisible_cities/core/mctrk_functions.py deleted file mode 100644 index 1b5b42254b..0000000000 --- a/invisible_cities/core/mctrk_functions.py +++ /dev/null @@ -1,173 +0,0 @@ -""" -Monte Carlo tracks functions -""" -import matplotlib.pyplot as plt -from ..reco import tbl_functions as tbl - - -def plot_track(geom_df, mchits_df, vox_size=10, zoom=False): - """Plot the hits of a mctrk. Adapted from JR plotting functions notice - that geom_df and mchits_df are pandas objects defined above if - zoom = True, the track is zoomed in (min/max dimensions are taken - from track). If zoom = False, detector dimensions are used - - """ - grdcol = 0.99 - - varr_x = mchits_df["x"].values * vox_size - varr_y = mchits_df["y"].values * vox_size - varr_z = mchits_df["z"].values * vox_size - varr_c = mchits_df["energy"].values / units.keV - - min_x = geom_df["xdet_min"] * vox_size - max_x = geom_df["xdet_max"] * vox_size - min_y = geom_df["ydet_min"] * vox_size - max_y = geom_df["ydet_max"] * vox_size - min_z = geom_df["zdet_min"] * vox_size - max_z = geom_df["zdet_max"] * vox_size - emin = 0 - emax = np.max(varr_c) - - if zoom is True: - min_x = np.min(varr_x) - max_x = np.max(varr_x) - min_y = np.min(varr_y) - max_y = np.max(varr_y) - min_z = np.min(varr_z) - max_z = np.max(varr_z) - emin = np.min(varr_c) - - # Plot the 3D voxelized track. - fig = plt.figure(1) - fig.set_figheight(6) - fig.set_figwidth(8) - - ax1 = fig.add_subplot(111, projection="3d") - s1 = ax1.scatter(varr_x, varr_y, varr_z, marker="s", linewidth=0.5, - s=2*vox_size, c=varr_c, cmap=plt.get_cmap("rainbow"), - vmin=emin, vmax=emax) - - # this disables automatic setting of alpha relative of distance to camera - s1.set_edgecolors = s1.set_facecolors = lambda *args: None - - print(" min_x ={} max_x ={}".format(min_x, max_x)) - print(" min_y ={} max_y ={}".format(min_y, max_y)) - print(" min_z ={} max_z ={}".format(min_z, max_z)) - print("min_e ={} max_e ={}".format(emin, emax)) - - ax1.set_xlim([min_x, max_x]) - ax1.set_ylim([min_y, max_y]) - ax1.set_zlim([min_z, max_z]) - - # ax1.set_xlim([0, 2 * vox_ext]); - # ax1.set_ylim([0, 2 * vox_ext]); - # ax1.set_zlim([0, 2 * vox_ext]); - ax1.set_xlabel("x (mm)") - ax1.set_ylabel("y (mm)") - ax1.set_zlabel("z (mm)") - ax1.set_title("") - - lb_x = ax1.get_xticklabels() - lb_y = ax1.get_yticklabels() - lb_z = ax1.get_zticklabels() - for lb in (lb_x + lb_y + lb_z): - lb.set_fontsize(8) - - ax1.w_xaxis.set_pane_color((1.0, 1.0, 1.0, 1.0)) - ax1.w_yaxis.set_pane_color((1.0, 1.0, 1.0, 1.0)) - ax1.w_zaxis.set_pane_color((1.0, 1.0, 1.0, 1.0)) - for axis in [ax1.w_xaxis, ax1.w_yaxis, ax1.w_zaxis]: - axis._axinfo.update({"grid": {"color": (grdcol, grdcol, grdcol, 1)}}) - - cb1 = plt.colorbar(s1) - cb1.set_label("Energy (keV)") - - -def plot_track_projections(geom_df, mchits_df, vox_size=10, zoom=False): - """Plot the projections of an MC track. Adapted from function - plot_track above notice that geom_df and mchits_df are pandas - objects defined above if zoom = True, the track is zoomed in - (min/max dimensions are taken from track). If zoom = False, - detector dimensions are used. - - For now, it is assumed that vox_sizeX = vox_sizeY = vox_sizeZ - """ - vox_sizeX = vox_size - vox_sizeY = vox_size - vox_sizeZ = vox_size - - varr_x = mchits_df["x"].values * vox_size - varr_y = mchits_df["y"].values * vox_size - varr_z = mchits_df["z"].values * vox_size - varr_c = mchits_df["energy"].values/units.keV - - min_x = geom_df["xdet_min"] * vox_size - max_x = geom_df["xdet_max"] * vox_size - min_y = geom_df["ydet_min"] * vox_size - max_y = geom_df["ydet_max"] * vox_size - min_z = geom_df["zdet_min"] * vox_size - max_z = geom_df["zdet_max"] * vox_size - - if zoom is True: - min_x = np.min(varr_x) - max_x = np.max(varr_x) - min_y = np.min(varr_y) - max_y = np.max(varr_y) - min_z = np.min(varr_z) - max_z = np.max(varr_z) - - # Plot the 2D projections. - fig = plt.figure(1) - fig.set_figheight(5.) - fig.set_figwidth(20.) - - # Create the x-y projection. - ax1 = fig.add_subplot(131) - hxy, xxy, yxy = np.histogram2d(varr_y, varr_x, - weights=varr_c, normed=False, - bins= ((1.05 * max_y - 0.95 * min_y) / vox_sizeY, - (1.05 * max_x - 0.95 * min_x) / vox_sizeX), - range=([0.95 * min_y, 1.05 * max_y], - [0.95 * min_x, 1.05 * max_x])) - - extent1 = [yxy[0], yxy[-1], xxy[0], xxy[-1]] - sp1 = ax1.imshow(hxy, extent=extent1, interpolation="none", - aspect="auto", origin="lower") - ax1.set_xlabel("x (mm)") - ax1.set_ylabel("y (mm)") - cbp1 = plt.colorbar(sp1) - cbp1.set_label("Energy (keV)") - - # Create the y-z projection. - ax2 = fig.add_subplot(132) - hyz, xyz, yyz = np.histogram2d(varr_z, varr_y, - weights=varr_c, normed=False, - bins= ((1.05 * max_z - 0.95 * min_z) / vox_sizeZ, - (1.05 * max_y - 0.95 * min_y) / vox_sizeY), - range=([0.95 * min_z, 1.05 * max_z], - [0.95 * min_y, 1.05 * max_y])) - - extent2 = [yyz[0], yyz[-1], xyz[0], xyz[-1]] - sp2 = ax2.imshow(hyz, extent=extent2, interpolation="none", - aspect="auto", origin="lower") - ax2.set_xlabel("y (mm)") - ax2.set_ylabel("z (mm)") - cbp2 = plt.colorbar(sp2) - cbp2.set_label("Energy (keV)") - - # Create the x-z projection. - ax3 = fig.add_subplot(133) - hxz, xxz, yxz = np.histogram2d(varr_z, varr_x, - weights=varr_c, normed=False, - bins= ((1.05 * max_z - 0.95 * min_z) / vox_sizeZ, - (1.05 * max_x - 0.95 * min_x) / vox_sizeX), - range=([0.95 * min_z, 1.05 * max_z], - [0.95 * min_x, 1.05 * max_x])) - - extent3 = [yxz[0], yxz[-1], xxz[0], xxz[-1]] - sp3 = ax3.imshow(hxz, extent=extent3, interpolation="none", - aspect="auto", origin="lower") - ax3.set_xlabel("x (mm)") - ax3.set_ylabel("z (mm)") - cbp3 = plt.colorbar(sp3) - cbp3.set_label("Energy (keV)") diff --git a/invisible_cities/core/stat_functions_test.py b/invisible_cities/core/stat_functions_test.py index 92b9162df7..329d5909da 100644 --- a/invisible_cities/core/stat_functions_test.py +++ b/invisible_cities/core/stat_functions_test.py @@ -19,7 +19,7 @@ def arrays_of_positive_numbers(draw): _strategies = {int: integers, float: floats} _type, strategy = draw(sampled_from(list(_strategies.items()))) size = draw(integers(1, 10)) - return draw(arrays(_type, size, strategy(0, 100))) + return draw(arrays(_type, size, elements=strategy(0, 100))) @mark.parametrize("mean", np.linspace(0.1, 3, 3)) diff --git a/invisible_cities/core/system_of_units_c.pxd b/invisible_cities/core/system_of_units_c.pxd deleted file mode 100644 index 41107f87de..0000000000 --- a/invisible_cities/core/system_of_units_c.pxd +++ /dev/null @@ -1,10 +0,0 @@ -# $Id: system_of_units.h,v 1.1 2004/04/16 17:09:03 gomez Exp $ -# ---------------------------------------------------------------------- -# HEP coherent system of Unitscoulomb -# - -cdef class SystemOfUnits: - cdef readonly double euro, millimeter, millimeter2, millimeter3, centimeter, centimeter2, cm, cm2, cm3, centimeter3, decimeter, decimeter2, decimeter3, liter, l, meter, meter2, meter3, kilometer, kilometer2, kilometer3, micrometer, nanometer, angstrom, fermi, nm, mum, micron, barn, millibarn, microbarn, nanobarn, picobarn, mm, mm2, mm3, m, m2, m3, km, km2, km3, ft, radian, milliradian, mrad, degree, steradian, rad, sr, deg, nanosecond, millisecond, second, year, day, minute, hour, s, ms, ps, mus, ns, picosecond, microsecond, hertz, kilohertz, megahertz, gigahertz, MHZ, kHz, kHZ, GHZ, eplus, e_SI, coulomb, electronvolt, megaelectronvolt, milielectronvolt, kiloelectronvolt, gigaelectronvolt, teraelectronvolt, petaelectronvolt, meV, eV, keV, MeV, GeV, TeV, PeV, eV2, joule, kilogram, gram, milligram, ton, kiloton, kg, g, mg, watt, newton, hep_pascal, pascal, Pa, kPa, MPa, GPa, bar, milibar, atmosphere, denier, ampere, milliampere, microampere, nanoampere, mA, muA, nA, megavolt, kilovolt, volt, millivolt, V, mV, kV, MV, ohm, farad, millifarad, microfarad, nanofarad, picofarad, nF, pF, weber, tesla, gauss, kilogauss, henry, kelvin, mole, mol, becquerel, curie, Bq, mBq, muBq, cks, U238ppb, Th232ppb, gray, candela, lumen, lux, perCent, perThousand, perMillion, pes, adc - - -cpdef double celsius(double tKelvin) diff --git a/invisible_cities/core/system_of_units_c.pyx b/invisible_cities/core/system_of_units_c.pyx deleted file mode 100644 index 3cf14b73b8..0000000000 --- a/invisible_cities/core/system_of_units_c.pyx +++ /dev/null @@ -1,272 +0,0 @@ -# $Id: system_of_units.h,v 1.1 2004/04/16 17:09:03 gomez Exp $ -# ---------------------------------------------------------------------- -# HEP coherent system of Unitscoulomb -# - -cdef class SystemOfUnits: - - def __init__(self): - self.euro = 1. - self.millimeter = 1. - self.millimeter2 = self.millimeter * self.millimeter - self.millimeter3 = self.millimeter * self.millimeter2 -# - self.centimeter = 10. * self.millimeter - self.centimeter2 = self.centimeter * self.centimeter - self.centimeter3 = self.centimeter * self.centimeter2 -# - self.decimeter = 100. * self.millimeter - self.decimeter2 = self.decimeter * self.decimeter - self.decimeter3 = self.decimeter * self.decimeter2 - self.liter = self.decimeter3 - self.l = self.liter -# - self.meter = 1000. * self.millimeter - self.meter2 = self.meter * self.meter - self.meter3 = self.meter * self.meter2 -# - self.kilometer = 1000. * self.meter - self.kilometer2 = self.kilometer * self.kilometer - self.kilometer3 = self.kilometer * self.kilometer2 -# - self.micrometer = 1.e-6 * self.meter - self.nanometer = 1.e-9 * self.meter - self.angstrom = 1.e-10 * self.meter - self.fermi = 1.e-15 * self.meter -# - self.nm = self.nanometer - self.mum = self.micrometer -# - self.micron = self.micrometer - self.barn = 1.e-28 * self.meter2 - self.millibarn = 1.e-3 * self.barn - self.microbarn = 1.e-6 * self.barn - self.nanobarn = 1.e-9 * self.barn - self.picobarn = 1.e-12 * self.barn -# -# # symbols - self.mm = self.millimeter - self.mm2 = self.millimeter2 - self.mm3 = self.millimeter3 -# - self.cm = self.centimeter - self.cm2 = self.centimeter2 - self.cm3 = self.centimeter3 -# - self.m = self.meter - self.m2 = self.meter2 - self.m3 = self.meter3 -# - self.km = self.kilometer - self.km2 = self.kilometer2 - self.km3 = self.kilometer3 -# - self.ft = 30.48 * self.cm -# -# # -# # Angle -# # - self.radian = 1. - self.milliradian = 1.e-3 * self.radian - self.degree = (3.14159265358979323846/180.0) * self.radian -# - self.steradian = 1. -# -# # symbols - self.rad = self.radian - self.mrad = self.milliradian - self.sr = self.steradian - self.deg = self.degree -# -# # -# # Time [T] -# # - self.nanosecond = 1. - self.second = 1.e+9 * self.nanosecond - self.millisecond = 1.e-3 * self.second - self.microsecond = 1.e-6 * self.second - self.picosecond = 1.e-12 * self.second - self.year = 3.1536e+7 * self.second - self.day = 864e2 * self.second - self.minute = 60 * self.second - self.hour = 60 * self.minute -# - self.s = self.second - self.ms = self.millisecond - self.ps = self.picosecond - self.mus = self.microsecond - self.ns = self.nanosecond -# -# # new! - self.hertz = 1./self.second - self.kilohertz = 1.e+3 * self.hertz - self.megahertz = 1.e+6 * self.hertz - self.gigahertz = 1.e+6 * self.hertz -# - self.MHZ = self.megahertz - self.kHZ = self.kilohertz - self.kHz = self.kHZ - self.GHZ = self.gigahertz -# -# # -# # Electric charge [Q] -# # - self.eplus = 1. # positron charge - self.e_SI = 1.60217733e-19 # positron charge in coulomb - self.coulomb = self.eplus/self.e_SI # coulomb = 6.24150 e+18 * eplus -# # -# # Energy [E] -# # - self.megaelectronvolt = 1. - self.electronvolt = 1.e-6 * self.megaelectronvolt - self.milielectronvolt = 1.e-3 * self.electronvolt - self.kiloelectronvolt = 1.e-3 * self.megaelectronvolt - self.gigaelectronvolt = 1.e+3 * self.megaelectronvolt - self.teraelectronvolt = 1.e+6 * self.megaelectronvolt - self.petaelectronvolt = 1.e+9 * self.megaelectronvolt -# - self.meV = self.milielectronvolt - self.eV = self.electronvolt - self.keV = self.kiloelectronvolt - self.MeV = self.megaelectronvolt - self.GeV = self.gigaelectronvolt - self.TeV = self.teraelectronvolt - self.PeV = self.petaelectronvolt - self.eV2 = self.eV*self.eV -# - self.joule = self.electronvolt/self.e_SI # joule = 6.24150 e+12 * MeV -# # -# # Mass [E][T^2][L^-2] -# # - self.kilogram = self.joule * self.second * self.second / self.meter2 - self.gram = 1.e-3 * self.kilogram - self.milligram = 1.e-3 * self.gram - self.ton = 1.e+3 * self.kilogram - self.kiloton = 1.e+3 * self.ton -# -# # symbols - self.kg = self.kilogram - self.g = self.gram - self.mg = self.milligram -# # -# # Power [E][T^-1] -# # - self.watt = self.joule/self.second # watt = 6.24150 e+3 * MeV/ns -# # -# # Force [E][L^-1] -# # - self.newton = self.joule/self.meter # newton = 6.24150 e+9 * MeV/mm -# # -# # Pressure [E][L^-3] -# # - self.hep_pascal = self.newton/self.m2 # pascal = 6.24150 e+3 * MeV/mm3 - self.pascal = self.hep_pascal - self.Pa = self.pascal - self.kPa = 1000 * self.Pa - self.MPa = 1e+6 * self.Pa - self.GPa = 1e+9 * self.Pa - self.bar = 100000 * self.pascal # bar = 6.24150 e+8 * MeV/mm3 - self.milibar = 1e-3 * self.bar - self.atmosphere = 101325 * self.pascal # atm = 6.32420 e+8 * MeV/mm3 - self.denier = self.gram / (9000 * self.meter) -# # -# # Electric current [Q][T^-1] - self.ampere = self.coulomb/self.second # ampere = 6.24150 e+9 * eplus/ns - self.milliampere = 1.e-3 * self.ampere - self.microampere = 1.e-6 * self.ampere - self.nanoampere = 1.e-9 * self.ampere - self.mA = self.milliampere - self.muA = self.microampere - self.nA = self.nanoampere -# # -# # Electric potential [E][Q^-1] -# # - self.megavolt = self.megaelectronvolt/self.eplus - self.kilovolt = 1.e-3 * self.megavolt - self.volt = 1.e-6 * self.megavolt - self.millivolt = 1.e-3 * self.volt -# - self.V = self.volt - self.mV = self.millivolt - self.kV = self.kilovolt - self.MV = self.megavolt -# -# # -# # Electric resistance [E][T][Q^-2] -# # - self.ohm = self.volt/self.ampere # ohm = 1.60217e-16*(MeV/eplus)/(eplus/ns) -# # -# # Electric capacitance [Q^2][E^-1] -# # - self.farad = self.coulomb/self.volt # farad = 6.24150e+24 * eplus/Megavolt - self.millifarad = 1.e-3 * self.farad - self.microfarad = 1.e-6 * self.farad - self.nanofarad = 1.e-9 * self.farad - self.picofarad = 1.e-12 * self.farad -# - self.nF = self.nanofarad - self.pF = self.picofarad -# # -# # Magnetic Flux [T][E][Q^-1] -# # - self.weber = self.volt * self.second # weber = 1000*megavolt*ns -# # -# # Magnetic Field [T][E][Q^-1][L^-2] -# # - self.tesla = self.volt*self.second/self.meter2 # tesla = 0.001*megavolt*ns/mm2 - self.gauss = 1.e-4 * self.tesla - self.kilogauss = 1.e-1 * self.tesla -# # -# # Inductance [T^2][E][Q^-2] -# # - self.henry = self.weber/self.ampere # henry = 1.60217e-7*MeV*(ns/eplus)**2 -# # -# # Temperature -# # - self.kelvin = 1. -# # -# # Amount of substance -# # - self.mole = 1. - self.mol = self.mole -# # -# # Activity [T^-1] -# # - self.becquerel = 1./self.second - self.curie = 3.7e+10 * self.becquerel - self.Bq = self.becquerel - self.mBq = 1e-3 * self.becquerel - self.muBq = 1e-6 * self.becquerel - self.cks = self.Bq/self.keV - self.U238ppb = self.Bq / 81. - self.Th232ppb = self.Bq / 246. -# # -# # Absorbed dose [L^2][T^-2] -# # - self.gray = self.joule/self.kilogram -# # -# # Luminous intensity [I] -# # - self.candela = 1. -# # -# # Luminous flux [I] -# # - self.lumen = self.candela * self.steradian -# # -# # Illuminance [I][L^-2] -# # - self.lux = self.lumen/self.meter2 -# # -# # Miscellaneous -# # - self.perCent = 1e-2 - self.perThousand = 1e-3 - self.perMillion = 1e-6 - self.pes = 1. - self.adc = 1 -# -cpdef double celsius(double tKelvin): - return tKelvin - 273.15 - - -units = SystemOfUnits() diff --git a/invisible_cities/core/testing_utils.py b/invisible_cities/core/testing_utils.py index 94e2040be3..faccb6b0e9 100644 --- a/invisible_cities/core/testing_utils.py +++ b/invisible_cities/core/testing_utils.py @@ -1,4 +1,5 @@ import numpy as np +import pandas as pd from pytest import approx from numpy.testing import assert_array_equal @@ -106,11 +107,18 @@ def _compare_dataframes(assertion, df1, df2, check_types=True, **kwargs): def assert_dataframes_equal(df1, df2, check_types=True, **kwargs): - _compare_dataframes(assert_array_equal, df1, df2, check_types, **kwargs) + pd.testing.assert_frame_equal(df1.sort_index(axis=1), df2.sort_index(axis=1), check_names=True, check_dtype=check_types, check_exact=True, **kwargs) +def assert_dataframes_close(df1, df2, check_types=True, rtol=None, atol=None, **kwargs): + if rtol: + check_less_precise = int(np.log10(1./rtol)) + elif atol: + check_less_precise = int(np.log10(atol)) + else: + check_less_precise = True -def assert_dataframes_close(df1, df2, check_types=True, **kwargs): - _compare_dataframes(assert_allclose, df1, df2, check_types, **kwargs) + pd.testing.assert_frame_equal(df1.sort_index(axis=1), df2.sort_index(axis=1), + check_names=True, check_dtype=check_types, check_less_precise = check_less_precise) def assert_SensorResponses_equality(sr0, sr1): diff --git a/invisible_cities/core/testing_utils_test.py b/invisible_cities/core/testing_utils_test.py index 74a354269b..2bdad4fdf7 100644 --- a/invisible_cities/core/testing_utils_test.py +++ b/invisible_cities/core/testing_utils_test.py @@ -4,7 +4,6 @@ from flaky import flaky from hypothesis import given from hypothesis.strategies import floats -from hypothesis.strategies import integers from hypothesis. extra.pandas import data_frames from hypothesis. extra.pandas import column from hypothesis. extra.pandas import range_indexes @@ -41,4 +40,4 @@ def test_assert_tables_equality(df): def test_assert_tables_equality_withNaN(): table = np.array([('Rex', 9, 81.0), ('Fido', 3, np.nan)], dtype=[('name', 'U10'), ('age', 'i4'), ('weight', 'f4')]) - assert_tables_equality(table, table) + assert_tables_equality(table, table) diff --git a/invisible_cities/daemons/asriel.py b/invisible_cities/daemons/asriel.py deleted file mode 100644 index c9e49d554a..0000000000 --- a/invisible_cities/daemons/asriel.py +++ /dev/null @@ -1,12 +0,0 @@ -from . daemon import Daemon - -class Asriel(Daemon): - - def __init__(self): - print('I am Asriel') - - def run(self): - print('Asriel runs') - - def end(self): - print('Asriel ends') diff --git a/invisible_cities/daemons/daemon.py b/invisible_cities/daemons/daemon.py deleted file mode 100644 index eef48daa02..0000000000 --- a/invisible_cities/daemons/daemon.py +++ /dev/null @@ -1,9 +0,0 @@ -class Daemon: - - """Defines an interface for daemons""" - - def run(self): - pass - - def end(self): - pass diff --git a/invisible_cities/daemons/idaemon.py b/invisible_cities/daemons/idaemon.py deleted file mode 100644 index 2dd3e5d721..0000000000 --- a/invisible_cities/daemons/idaemon.py +++ /dev/null @@ -1,10 +0,0 @@ -from importlib import import_module -import traceback - - -def summon_daemon(daemon_name): - """Take a daemon name and return a new instance of the daemon""" - module_name = 'invisible_cities.daemons.' + daemon_name - daemon_class = getattr(import_module(module_name), daemon_name.capitalize()) - return daemon_class() - diff --git a/invisible_cities/daemons/lyra.py b/invisible_cities/daemons/lyra.py deleted file mode 100644 index bf28f548cd..0000000000 --- a/invisible_cities/daemons/lyra.py +++ /dev/null @@ -1,12 +0,0 @@ -from . daemon import Daemon - -class Lyra(Daemon): - - def __init__(self): - print('I am Lyra') - - def run(self): - print('Lyra runs') - - def end(self): - print('Lyra ends') diff --git a/invisible_cities/database/download.py b/invisible_cities/database/download.py index f58ee878ea..70e31d27fd 100644 --- a/invisible_cities/database/download.py +++ b/invisible_cities/database/download.py @@ -76,7 +76,7 @@ def loadDB(dbname : str): if __name__ == '__main__': - dbname = ['NEWDB', 'DEMOPPDB', 'NEXT100DB'] + dbname = ['NEWDB', 'DEMOPPDB', 'NEXT100DB', 'Flex100DB'] if len(sys.argv) > 1: dbname = sys.argv[1] loadDB(dbname) diff --git a/invisible_cities/database/download_test.py b/invisible_cities/database/download_test.py index e8a439e887..69838dea40 100644 --- a/invisible_cities/database/download_test.py +++ b/invisible_cities/database/download_test.py @@ -8,7 +8,7 @@ from . import download as db -@mark.parametrize('dbname', 'DEMOPPDB NEWDB NEXT100DB'.split()) +@mark.parametrize('dbname', 'DEMOPPDB NEWDB NEXT100DB Flex100DB'.split()) def test_create_table_sqlite(dbname, output_tmpdir): dbfile = os.path.join(output_tmpdir, 'db.sqlite3') diff --git a/invisible_cities/database/load_db.py b/invisible_cities/database/load_db.py index b97ec076e4..0533836260 100644 --- a/invisible_cities/database/load_db.py +++ b/invisible_cities/database/load_db.py @@ -10,6 +10,7 @@ class DetDB: new = os.environ['ICTDIR'] + '/invisible_cities/database/localdb.NEWDB.sqlite3' demopp = os.environ['ICTDIR'] + '/invisible_cities/database/localdb.DEMOPPDB.sqlite3' next100 = os.environ['ICTDIR'] + '/invisible_cities/database/localdb.NEXT100DB.sqlite3' + flex100 = os.environ['ICTDIR'] + '/invisible_cities/database/localdb.Flex100DB.sqlite3' def tmap(*args): return tuple(map(*args)) @@ -121,7 +122,6 @@ def SiPMNoise(db_file, run_number=1e5): @lru_cache(maxsize=10) def PMTLowFrequencyNoise(db_file, run_number=1e5): conn = sqlite3.connect(get_db(db_file)) - cursor = conn.cursor() sqlmapping = '''select SensorID, FEBox from PMTFEMapping where MinRun <= {0} and (MaxRun >= {0} or MaxRun is NULL) diff --git a/invisible_cities/database/load_db_test.py b/invisible_cities/database/load_db_test.py index 953228b46f..970360ee96 100644 --- a/invisible_cities/database/load_db_test.py +++ b/invisible_cities/database/load_db_test.py @@ -1,5 +1,6 @@ import time import sqlite3 +import pytest from os.path import join @@ -145,6 +146,10 @@ def test_database_is_being_cached(db_fun, db): def test_frontend_mapping(db): """ Check the mapping has the expected shape etc """ + if db.detector == "next100": + pytest.skip("NEXT100 not implemented yet") + if db.detector == "flex100": + pytest.skip("Flex100 not implemented yet") run_number = 6000 fe_mapping, _ = DB.PMTLowFrequencyNoise(db.detector, run_number) @@ -159,6 +164,11 @@ def test_frontend_mapping(db): def test_pmt_noise_frequencies(db): """ Check the magnitudes and frequencies are of the expected length """ + if db.detector == "next100": + pytest.skip("NEXT100 not implemented yet") + if db.detector == "flex100": + pytest.skip("Flex100 not implemented yet") + run_number = 5000 _, frequencies = DB.PMTLowFrequencyNoise(db.detector, run_number) diff --git a/invisible_cities/database/localdb.DEMOPPDB.sqlite3 b/invisible_cities/database/localdb.DEMOPPDB.sqlite3 index 5ec4354ca5..b7389c06cf 100644 --- a/invisible_cities/database/localdb.DEMOPPDB.sqlite3 +++ b/invisible_cities/database/localdb.DEMOPPDB.sqlite3 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:543c744f37ad6c61fefea492dfb7f99117bd74b7cfab2f706cf34283de714ef1 -size 5324800 +oid sha256:32cc72d86f9511ef8901191acfa531bf3e7e19ffbd5ae5581bd60b3e9c5ca26c +size 8167424 diff --git a/invisible_cities/database/localdb.Flex100DB.sqlite3 b/invisible_cities/database/localdb.Flex100DB.sqlite3 new file mode 100644 index 0000000000..3e4f555b9c --- /dev/null +++ b/invisible_cities/database/localdb.Flex100DB.sqlite3 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e334127d9284f7dd8bfbf7a884682c6ad7a1a47658b9a059de78d558d09d1b1d +size 29347840 diff --git a/invisible_cities/database/localdb.NEWDB.sqlite3 b/invisible_cities/database/localdb.NEWDB.sqlite3 index 2eccd0b3ad..3cb000c65b 100644 --- a/invisible_cities/database/localdb.NEWDB.sqlite3 +++ b/invisible_cities/database/localdb.NEWDB.sqlite3 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:688653b704cfa16942ef6364d5ae613febcfcd166f5fa4be378ee5d7a46405c9 -size 341274624 +oid sha256:719bb95497f9e388900480bfd2b2d8f2b8b5971286e55b56c109b2fe098929ca +size 402411520 diff --git a/invisible_cities/database/localdb.NEXT100DB.sqlite3 b/invisible_cities/database/localdb.NEXT100DB.sqlite3 index d21be885ce..5d41f9c1e9 100644 --- a/invisible_cities/database/localdb.NEXT100DB.sqlite3 +++ b/invisible_cities/database/localdb.NEXT100DB.sqlite3 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:19738c2f5a59229a17625808f4de170fb63f7691b772db28a17f16255dd935d5 -size 82366464 +oid sha256:4628581c351d5184f7443bcda8e9d63f150cc112a3fd1ea4b7f5e90249bdce3b +size 34603008 diff --git a/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.CRWF.h5 b/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.CRWF.h5 new file mode 100644 index 0000000000..ba8da8ee83 --- /dev/null +++ b/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.CRWF.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67d9d23665809fb317e23632ed89c326ce25bb96ab90d7a8ed136f34dd12634b +size 2572818 diff --git a/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.NEWMC.BLR.h5 b/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.NEWMC.BLR.h5 new file mode 100644 index 0000000000..ff1bac06d3 --- /dev/null +++ b/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.NEWMC.BLR.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc4ec76df56c11dbc9967345a015dc9653d4e85aa4a6a6ff2b1c223315c9ad3e +size 2543613 diff --git a/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.NEWMC.HDST.h5 b/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.NEWMC.HDST.h5 new file mode 100644 index 0000000000..cdaba0dd59 --- /dev/null +++ b/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.NEWMC.HDST.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a58c9517007fbe247eabacf42792a43840ee50e1eaf94a64f34ce456504eab0 +size 71552 diff --git a/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.NEWMC.PMP.h5 b/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.NEWMC.PMP.h5 new file mode 100644 index 0000000000..12cdbbf0cf --- /dev/null +++ b/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.NEWMC.PMP.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:163a5958e5237fec3a22269f2426add361d15ffe07eaa4e7a9b8243e487bf55d +size 121770 diff --git a/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.hypathia.h5 b/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.hypathia.h5 new file mode 100644 index 0000000000..e643b0a5c7 --- /dev/null +++ b/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.hypathia.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e884d9db77e821f5175762df2cb9c378de74404e13aa860a339d5edb44044fa3 +size 129484 diff --git a/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_noS1.NEWMC.HDST.h5 b/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_noS1.NEWMC.HDST.h5 new file mode 100644 index 0000000000..1f52e0f6cf --- /dev/null +++ b/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_noS1.NEWMC.HDST.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc62de03c9eeaba22e7756c88b02b78a4af80ecfe0379667fc4f9be4f66ad6b0 +size 79589 diff --git a/invisible_cities/database/test_data/Kr83m_nexus_HEAD20200327.sim.0.h5 b/invisible_cities/database/test_data/Kr83m_nexus_HEAD20200327.sim.0.h5 new file mode 100644 index 0000000000..dc248b00e6 --- /dev/null +++ b/invisible_cities/database/test_data/Kr83m_nexus_HEAD20200327.sim.0.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7610cd9a35f61d434ba9b5dd60a5d82fd06354cbc0e6c28d99f52a80f1c0b6ce +size 47657 diff --git a/invisible_cities/database/test_data/Kr83m_nexus_HEAD20200327.sim.1.h5 b/invisible_cities/database/test_data/Kr83m_nexus_HEAD20200327.sim.1.h5 new file mode 100644 index 0000000000..eae49aaf64 --- /dev/null +++ b/invisible_cities/database/test_data/Kr83m_nexus_HEAD20200327.sim.1.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec9b598e957906b030025507e8135d4e0fcdb45d50303729e3ad66855e967780 +size 48560 diff --git a/invisible_cities/database/test_data/NextFlex_mc_hits.h5 b/invisible_cities/database/test_data/NextFlex_mc_hits.h5 new file mode 100644 index 0000000000..629f871e38 --- /dev/null +++ b/invisible_cities/database/test_data/NextFlex_mc_hits.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bd28f8ede95609e22210979d52826c6b973bc8f3aa11567af02b23fb969183b +size 99721 diff --git a/invisible_cities/database/test_data/NextFlex_mc_sensors.h5 b/invisible_cities/database/test_data/NextFlex_mc_sensors.h5 new file mode 100644 index 0000000000..c083c134eb --- /dev/null +++ b/invisible_cities/database/test_data/NextFlex_mc_sensors.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dc5368d37b8a3f6b930670742eea5843017fd08ec1c01396f658b090277d2ee +size 1537208 diff --git a/invisible_cities/database/test_data/bad_mc_tables.h5 b/invisible_cities/database/test_data/bad_mc_tables.h5 new file mode 100644 index 0000000000..6e51535906 --- /dev/null +++ b/invisible_cities/database/test_data/bad_mc_tables.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d16b3b4cd2f3a4a7597eef2f33ae81bb3f360380254b1732d176fa198a831820 +size 5632 diff --git a/invisible_cities/database/test_data/binned_simwfs.h5 b/invisible_cities/database/test_data/binned_simwfs.h5 new file mode 100644 index 0000000000..b792d782a1 --- /dev/null +++ b/invisible_cities/database/test_data/binned_simwfs.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7254828b6e8cbec760007e4a23ab22507c214d9c3de00a0f597f9b572d130b67 +size 2331616 diff --git a/invisible_cities/database/test_data/electrons_40keV_z250_RWF.h5 b/invisible_cities/database/test_data/electrons_40keV_z250_RWF.h5 deleted file mode 100644 index 4ac6bcb08a..0000000000 --- a/invisible_cities/database/test_data/electrons_40keV_z250_RWF.h5 +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:be13b302fad35ff701dc4a9056861e43221c0d3663394d61cdd90ac1aa11f4f1 -size 2912536 diff --git a/invisible_cities/database/test_data/exact_Kr_tracks_with_MC_KDST_no_filter.NEWMC.h5 b/invisible_cities/database/test_data/exact_Kr_tracks_with_MC_KDST_no_filter.NEWMC.h5 new file mode 100644 index 0000000000..46255242f4 --- /dev/null +++ b/invisible_cities/database/test_data/exact_Kr_tracks_with_MC_KDST_no_filter.NEWMC.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a409f276b302cee7c29e941979859774607c07307602cd86e8ca8ba799d2fef +size 152181 diff --git a/invisible_cities/database/test_data/nexus_new_kr83m_fast.newformat.sim.h5 b/invisible_cities/database/test_data/nexus_new_kr83m_fast.newformat.sim.h5 new file mode 100644 index 0000000000..11b7db4b57 --- /dev/null +++ b/invisible_cities/database/test_data/nexus_new_kr83m_fast.newformat.sim.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8773205d8890c47add685a5dbc3648826d89b9d3a9b3156c26540edf7d3a492a +size 63458 diff --git a/invisible_cities/database/test_data/nexus_new_kr83m_fast.oldformat.sim.h5 b/invisible_cities/database/test_data/nexus_new_kr83m_fast.oldformat.sim.h5 new file mode 100644 index 0000000000..1e322d4248 --- /dev/null +++ b/invisible_cities/database/test_data/nexus_new_kr83m_fast.oldformat.sim.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fe305ee83458f55d42dba2661805f3f6c72cf0fbd2f18a01e197c2cce305671 +size 50062 diff --git a/invisible_cities/database/test_data/nexus_new_kr83m_full.newformat.sim.h5 b/invisible_cities/database/test_data/nexus_new_kr83m_full.newformat.sim.h5 new file mode 100644 index 0000000000..c232ec033b --- /dev/null +++ b/invisible_cities/database/test_data/nexus_new_kr83m_full.newformat.sim.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8e4fb86f7a089b46ff05ba66181f40c3af71646661d88f064fdf22d18421432 +size 91374 diff --git a/invisible_cities/database/test_data/nexus_new_kr83m_full.oldformat.sim.h5 b/invisible_cities/database/test_data/nexus_new_kr83m_full.oldformat.sim.h5 new file mode 100644 index 0000000000..054910f20c --- /dev/null +++ b/invisible_cities/database/test_data/nexus_new_kr83m_full.oldformat.sim.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdb9c0c02e04590f264417a2e381dc5af093bacd0b079c73aa43be1d9ed10e83 +size 74351 diff --git a/invisible_cities/database/test_data/nexus_scint.newformat.sim.h5 b/invisible_cities/database/test_data/nexus_scint.newformat.sim.h5 new file mode 100644 index 0000000000..817f31c223 --- /dev/null +++ b/invisible_cities/database/test_data/nexus_scint.newformat.sim.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbdd305bc43d4a07274771c0d7aad6c3cc0fae04431274be50a5e5d6cb6e8f69 +size 44603 diff --git a/invisible_cities/database/test_data/nexus_scint.oldformat.sim.h5 b/invisible_cities/database/test_data/nexus_scint.oldformat.sim.h5 new file mode 100644 index 0000000000..8972d411bc --- /dev/null +++ b/invisible_cities/database/test_data/nexus_scint.oldformat.sim.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13ca824453d2fd69f4cedfb937ccc9a1b175bf865fede3f280531f0d586144aa +size 44475 diff --git a/invisible_cities/database/test_data/pmaps_0000_7505_trigger1_v1.1.0_20190801_krbg1600.h5 b/invisible_cities/database/test_data/pmaps_0000_7505_trigger1_v1.1.0_20190801_krbg1600.h5 new file mode 100644 index 0000000000..82a6ae7c90 --- /dev/null +++ b/invisible_cities/database/test_data/pmaps_0000_7505_trigger1_v1.1.0_20190801_krbg1600.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a669ec91d91164fcacf745976178fccaf7516c22ba185effe6f895c2a1bee169 +size 1260963 diff --git a/invisible_cities/database/test_data/run_7775_0120_trigger1_waveforms.h5 b/invisible_cities/database/test_data/run_7775_0120_trigger1_waveforms.h5 new file mode 100644 index 0000000000..430df591d7 --- /dev/null +++ b/invisible_cities/database/test_data/run_7775_0120_trigger1_waveforms.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b220c439eff2cfe2f29a23c7ffdfc9378edda35695369006d60446568603312c +size 192298409 diff --git a/invisible_cities/database/test_data/single_pe_pmts.h5 b/invisible_cities/database/test_data/single_pe_pmts.h5 new file mode 100644 index 0000000000..02c4fd7ab2 --- /dev/null +++ b/invisible_cities/database/test_data/single_pe_pmts.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a80f9f4d0e7d971d2c632f0b418cc5e2ea4d07f0f1d33be364c284cd4d4aa5a8 +size 181105 diff --git a/invisible_cities/database/test_data/test_Xe2nu_NEW_exact_deconvolution_joint.NEWMC.h5 b/invisible_cities/database/test_data/test_Xe2nu_NEW_exact_deconvolution_joint.NEWMC.h5 new file mode 100644 index 0000000000..c811cf1445 --- /dev/null +++ b/invisible_cities/database/test_data/test_Xe2nu_NEW_exact_deconvolution_joint.NEWMC.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b4ba24e2bb2f7d8c0e5592be7747e56b1c02b414357279890de0bd8cc8cb13c +size 235984 diff --git a/invisible_cities/database/test_data/test_Xe2nu_NEW_exact_deconvolution_separate.NEWMC.h5 b/invisible_cities/database/test_data/test_Xe2nu_NEW_exact_deconvolution_separate.NEWMC.h5 new file mode 100644 index 0000000000..b74246ad70 --- /dev/null +++ b/invisible_cities/database/test_data/test_Xe2nu_NEW_exact_deconvolution_separate.NEWMC.h5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a42713a8c584884cff122833398c27f30a6edc2e0134cae099a0f46d8ab0446 +size 120716 diff --git a/invisible_cities/database/test_data/test_hits_th_1pes_deconvolution.npz b/invisible_cities/database/test_data/test_hits_th_1pes_deconvolution.npz index 0441758a87..46d9ab94e4 100644 Binary files a/invisible_cities/database/test_data/test_hits_th_1pes_deconvolution.npz and b/invisible_cities/database/test_data/test_hits_th_1pes_deconvolution.npz differ diff --git a/invisible_cities/daemons/__init__.py b/invisible_cities/detsim/__init__.py similarity index 100% rename from invisible_cities/daemons/__init__.py rename to invisible_cities/detsim/__init__.py diff --git a/invisible_cities/detsim/buffer_functions.py b/invisible_cities/detsim/buffer_functions.py new file mode 100644 index 0000000000..18cab781aa --- /dev/null +++ b/invisible_cities/detsim/buffer_functions.py @@ -0,0 +1,141 @@ +import numpy as np +import pandas as pd + +from typing import Callable +from typing import List +from typing import Tuple + +from .. reco.peak_functions import indices_and_wf_above_threshold +from .. reco.peak_functions import split_in_peaks + + +def weighted_histogram(data: pd.DataFrame, bins: np.ndarray) -> np.ndarray: + return np.histogram(data.time, weights=data.charge, bins=bins)[0] + + +def bin_sensors(sensors : pd.DataFrame, + bin_width : float , + t_min : float , + t_max : float , + max_buffer: int ) -> Tuple[np.ndarray, pd.Series]: + """ + Raw data binning function. + + Parameters + ---------- + sensors : List of Waveforms for one sensor type + Should be sorted into one type/binning + t_min : float + Minimum time to be used to define bins. + t_max : float + As t_min but the maximum to be used + """ + max_time = min(t_max, t_min + max_buffer) + min_bin = np.floor(t_min / bin_width) * bin_width + max_bin = np.ceil (max_time / bin_width) * bin_width + + bins = np.arange(min_bin, max_bin + bin_width, bin_width) + bin_sensors = sensors.groupby('sensor_id').apply(weighted_histogram, bins) + return bins[:-1], bin_sensors + + +## !! to-do: clarify for non-pmt versions of next +## !! to-do: Check on integral instead of only threshold? +def find_signal_start(wfs : pd.Series, + bin_threshold: float , + stand_off : int ) -> List[int]: + """ + Finds signal in the binned waveforms and + identifies candidate triggers. + """ + eng_sum = wfs.sum() + indices = indices_and_wf_above_threshold(eng_sum, + bin_threshold).indices + ## Just using this and the stand_off for now + ## taking first above sum threshold. + ## !! To-do: make more robust with min int? or similar + all_indx = split_in_peaks(indices, stand_off) + return [pulse[0] for pulse in all_indx] + + +def pad_safe(sensors: np.ndarray, padding: Tuple) -> np.ndarray: + """Pads zeros around each sensor in a 2D array""" + if not sensors.shape[0]: + return np.empty((0, padding[0] + padding[1] + 1)) + return np.apply_along_axis(np.pad, 1, sensors, padding, "constant") + + +def buffer_calculator(buffer_len: float, pre_trigger: float, + pmt_binwid: float, sipm_binwid: float) -> Callable: + """ + Calculates the output buffers for all sensors + based on a configured buffer length and pretrigger. + Synchronising the clock between the two sensor types. + + Parameters + ---------- + buffer_len : float + Length of buffer expected in mus + pre_trigger : float + Time in buffer before identified signal in mus + pmt_binwid : float + Width in mus of PMT sample integration + sipm_binwid : float + Width in mus of SiPM sample integration + """ + pmt_buffer_samples = int(buffer_len // pmt_binwid) + sipm_buffer_samples = int(buffer_len // sipm_binwid) + sipm_pretrg = int(pre_trigger // sipm_binwid) + sipm_postrg = sipm_buffer_samples - sipm_pretrg + pmt_pretrg_base = int(pre_trigger // pmt_binwid) + pmt_postrg_base = pmt_buffer_samples - pmt_pretrg_base + + def generate_slice(trigger : int , + pmt_bins : np.ndarray, + pmt_charge : np.ndarray, + sipm_bins : np.ndarray, + sipm_charge: np.ndarray) -> Tuple: + """ + Synchronises the clocks between the SiPMs and PMTs + and slices the histograms where this synchronisation + indicates. + """ + npmt_bin = len(pmt_bins) + nsipm_bin = len(sipm_bins) + + trg_bin = np.where(sipm_bins <= pmt_bins[trigger])[0][-1] + bin_corr = (pmt_bins[trigger] - sipm_bins[trg_bin]) // pmt_binwid + pmt_pretrg = pmt_pretrg_base + int(bin_corr) + pmt_postrg = pmt_postrg_base - int(bin_corr) + + pmt_pre = 0 , trigger - pmt_pretrg + pmt_pos = npmt_bin, trigger + pmt_postrg + pmt_slice = slice(max(pmt_pre), min(pmt_pos)) + pmt_pad = -min(pmt_pre), max(0, pmt_pos[1] - npmt_bin) + + sipm_pre = 0 , trg_bin - sipm_pretrg + sipm_pos = nsipm_bin, trg_bin + sipm_postrg + sipm_slice = slice(max(sipm_pre), min(sipm_pos)) + sipm_pad = -min(sipm_pre), max(0, sipm_pos[1] - nsipm_bin) + + return (pad_safe( pmt_charge[:, pmt_slice], pmt_pad), + pad_safe(sipm_charge[:, sipm_slice], sipm_pad)) + + + def position_signal(triggers : List, + pmt_bins : np.ndarray, + pmt_charge : pd.Series, + sipm_bins : np.ndarray, + sipm_charge: pd.Series + ) -> List[Tuple[np.ndarray, np.ndarray]]: + """ + Synchronises the SiPMs and PMTs for each identified + trigger and calls the padding function to fill with + zeros where necessary. + """ + pmt_q = np.asarray(pmt_charge.tolist()) + sipm_q = np.empty((0,0))\ + if sipm_charge.empty else np.asarray(sipm_charge.tolist()) + return [generate_slice(trigger, pmt_bins, pmt_q, sipm_bins, sipm_q) + for trigger in triggers ] + return position_signal diff --git a/invisible_cities/detsim/buffer_functions_test.py b/invisible_cities/detsim/buffer_functions_test.py new file mode 100644 index 0000000000..6e783f772b --- /dev/null +++ b/invisible_cities/detsim/buffer_functions_test.py @@ -0,0 +1,111 @@ +import numpy as np +import pandas as pd + +from pytest import mark + +from .. core import system_of_units as units + +from . buffer_functions import bin_sensors +from . buffer_functions import buffer_calculator +from . buffer_functions import find_signal_start + + +def test_bin_sensors(mc_waveforms, pmt_ids, sipm_ids): + max_buffer = 10 * units.minute + + evts, pmt_binwid, sipm_binwid, all_wfs = mc_waveforms + + evt = evts[0] + wfs = all_wfs.loc[evt] + + pmts = wfs[wfs.index.isin( pmt_ids)] + sipms = wfs[wfs.index.isin(sipm_ids)] + + ## Assumes pmts the triggering sensors as in new/next-100 + bins_min = pmts.time.min() + bins_max = pmts.time.max() + pmt_binwid + pmt_bins , pmt_wf = bin_sensors(pmts , pmt_binwid, + bins_min, bins_max, max_buffer) + sipm_bins, sipm_wf = bin_sensors(sipms, sipm_binwid, + bins_min, bins_max, max_buffer) + + assert pmt_bins[ 0] >= bins_min + assert pmt_bins[-1] <= min(bins_max, max_buffer) + assert np.all(np.diff( pmt_bins) == pmt_binwid) + + assert sipm_bins[ 0] >= bins_min + assert sipm_bins[-1] <= min(bins_max, max_buffer) + assert np.all(np.diff(sipm_bins) == sipm_binwid) + + ## In current DAQ, pmts have higher sample frequency than SiPMs + assert pmt_bins[ 0] >= sipm_bins[ 0] + assert pmt_bins[-1] >= sipm_bins[-1] + + pmt_sum = pmts .charge.sum() + sipm_sum = sipms.charge.sum() + assert pmt_wf .sum().sum() == pmt_sum + assert sipm_wf.sum().sum() == sipm_sum + + +@mark.parametrize("signal_thresh", (2, 10)) +def test_find_signal_start(binned_waveforms, signal_thresh): + + pmt_bins, pmt_wfs, *_ = binned_waveforms + + buffer_length = 800 * units.mus + bin_width = np.diff(pmt_bins)[0] + stand_off = int(buffer_length / bin_width) + + pmt_sum = pmt_wfs.sum() + pulses = find_signal_start(pmt_wfs, signal_thresh, stand_off) + + assert np.all(pmt_sum[pulses] > signal_thresh) + + +def test_find_signal_start_correct_index(): + + thresh = 5 + thr_bin = 3 + simple_evt = pd.Series([np.zeros(5)] * 12) + simple_evt[1][thr_bin] = thresh + + pulses = find_signal_start(simple_evt, thresh, 5) + assert len(pulses) == 1 + assert pulses[0] == thr_bin + + +@mark.parametrize("pre_trigger signal_thresh".split(), + ((100 * units.mus, 2), + (400 * units.mus, 10))) +def test_buffer_calculator(mc_waveforms, binned_waveforms, + pre_trigger , signal_thresh): + + _, pmt_binwid, sipm_binwid, _ = mc_waveforms + + pmt_bins, pmt_wfs, sipm_bins, sipm_wfs = binned_waveforms + + buffer_length = 800 * units.mus + bin_width = np.diff(pmt_bins)[0] + stand_off = int(buffer_length / bin_width) + + pulses = find_signal_start(pmt_wfs, signal_thresh, stand_off) + + calculate_buffers = buffer_calculator(buffer_length, + pre_trigger , + pmt_binwid , + sipm_binwid ) + + buffers = calculate_buffers(pulses, *binned_waveforms) + pmt_sum = pmt_wfs.sum() + + assert len(buffers) == len(pulses) + for i, (evt_pmt, evt_sipm) in enumerate(buffers): + sipm_trg_bin = np.where(sipm_bins <= pmt_bins[pulses[i]])[0][-1] + diff_binedge = pmt_bins[pulses[i]] - sipm_bins[sipm_trg_bin] + pre_trg_samp = int(pre_trigger / pmt_binwid + diff_binedge) + + assert pmt_wfs .shape[0] == evt_pmt .shape[0] + assert sipm_wfs.shape[0] == evt_sipm.shape[0] + assert evt_pmt .shape[1] == int(buffer_length / pmt_binwid) + assert evt_sipm.shape[1] == int(buffer_length / sipm_binwid) + assert np.sum(evt_pmt, axis=0)[pre_trg_samp] == pmt_sum[pulses[i]] diff --git a/invisible_cities/detsim/conftest.py b/invisible_cities/detsim/conftest.py new file mode 100644 index 0000000000..21dbb0bc03 --- /dev/null +++ b/invisible_cities/detsim/conftest.py @@ -0,0 +1,49 @@ +import os + +import numpy as np +import pandas as pd +import tables as tb + +from pytest import fixture +from pytest import mark + +from .. core import system_of_units as units +from .. database import load_db +from .. io .mcinfo_io import get_sensor_binning +from .. io .mcinfo_io import load_mcsensor_response_df + +from . buffer_functions import bin_sensors + + +@fixture(scope="module") +def mc_waveforms(full_sim_file): + file_in = full_sim_file + wfs = load_mcsensor_response_df(file_in, db_file='new', run_no=-6400) + + sns_bins = get_sensor_binning(file_in) + pmt_binwid = sns_bins.bin_width[sns_bins.index.str.contains( 'Pmt')] + sipm_binwid = sns_bins.bin_width[sns_bins.index.str.contains('SiPM')] + return wfs.index.levels[0], pmt_binwid.iloc[0], sipm_binwid.iloc[0], wfs + + +## !! to-do: generalise for all detector configurations +@fixture(scope="module") +def pmt_ids(): + return load_db.DataPMT('new', 6400).SensorID.values + + +@fixture(scope="module") +def sipm_ids(): + return load_db.DataSiPM('new', 6400).SensorID.values + + +@fixture(scope="module") +def binned_waveforms(ICDATADIR): + binned_file = os.path.join(ICDATADIR, 'binned_simwfs.h5') + with tb.open_file(binned_file) as h5in: + pmt_bins = h5in.root.BINWIDTHS.pmt_binwid .read() + sipm_bins = h5in.root.BINWIDTHS.sipm_binwid.read() + + pmt_wf = pd.read_hdf(binned_file, 'pmtwfs') + sipm_wf = pd.read_hdf(binned_file, 'sipmwfs') + return pmt_bins, pmt_wf, sipm_bins, sipm_wf diff --git a/invisible_cities/detsim/sensor_utils.py b/invisible_cities/detsim/sensor_utils.py new file mode 100644 index 0000000000..8154c1116f --- /dev/null +++ b/invisible_cities/detsim/sensor_utils.py @@ -0,0 +1,96 @@ +import numpy as np +import pandas as pd + +from typing import Callable +from typing import List +from typing import Tuple + +from .. database.load_db import DataPMT +from .. database.load_db import DataSiPM +from .. io .mcinfo_io import get_sensor_binning + + +def trigger_times(trigger_indx: List[int] , + event_time : float, + time_bins : np.ndarray) -> List[float]: + """ + Calculates trigger time for all identified candidates + according to the nexus event time and the time within + the event where the trigger was identified. + + Parameters + ---------- + trigger_indx : List[int] + The indices in the histogrammed nexus sensor + data where triggers were identified. + event_time : float + Time of nexus event + time_bins : np.ndarray + The binning used to histogram the nexus sensor + data. + + Returns + ------- + triggered event times : List[float] + """ + return event_time + time_bins[trigger_indx] + + +def first_and_last_times(pmt_wfs : pd.DataFrame, + sipm_wfs : pd.DataFrame, + pmt_binwid : float , + sipm_binwid: float ) -> Tuple[float, float]: + """ + Returns the maximum and minimum time of an + event given the two types of detector. + """ + min_time = min(pmt_wfs.time.min(), sipm_wfs.time.min()) + max_time = max(pmt_wfs.time.max(), sipm_wfs.time.max()) + max_time += min(pmt_binwid , sipm_binwid) + return min_time, max_time + + +def sensor_order(detector_db: str, run_number : int, + length_pmt : int, length_sipm: int) -> Callable: + """ + Casts the event sensor info into the correct order + adding zeros for sensors which didn't see any signal. + """ + pmt_ids = DataPMT (detector_db, run_number).SensorID + sipm_ids = DataSiPM(detector_db, run_number).SensorID + n_pmt, n_sipm = get_n_sensors(detector_db, run_number) + pmt_shape = (n_pmt , length_pmt ) + sipm_shape = (n_sipm, length_sipm) + def ordering(sensor_order : pd.Int64Index , + sensor_resp : np.ndarray , + sensor_shape : Tuple[int, int]) -> np.ndarray: + sensors = np.zeros(sensor_shape, np.int) + sensors[sensor_order] = sensor_resp + return sensors + + def order_and_pad(pmt_resp : pd.Series , + sipm_resp : pd.Series , + evt_buffers : List[Tuple[np.ndarray, np.ndarray]] + ) -> List[Tuple]: + pmt_ord = pmt_ids [ pmt_ids.isin( pmt_resp.index)].index + sipm_ord = sipm_ids[sipm_ids.isin(sipm_resp.index)].index + + return [(ordering(pmt_ord , pmts , pmt_shape ), + ordering(sipm_ord, sipms, sipm_shape)) + for pmts, sipms in evt_buffers] + return order_and_pad + + +def get_n_sensors(detector_db: str, run_number: int) -> Tuple[int, int]: + """Get the number of sensors for this run""" + npmt = DataPMT (detector_db, run_number).shape[0] + nsipm = DataSiPM(detector_db, run_number).shape[0] + return npmt, nsipm + + +def pmt_and_sipm_bin_width(file_name: str) -> Tuple[float, float]: + """returns pmt and sipm bin widths as set in nexus""" + sns_bins = get_sensor_binning(file_name) + pmt_wid = sns_bins.bin_width[sns_bins.index.str.contains( 'Pmt')].iloc[0] + sipm_wid = sns_bins.bin_width[sns_bins.index.str.contains('SiPM')].iloc[0] + return pmt_wid, sipm_wid diff --git a/invisible_cities/detsim/sensor_utils_test.py b/invisible_cities/detsim/sensor_utils_test.py new file mode 100644 index 0000000000..059f05feb9 --- /dev/null +++ b/invisible_cities/detsim/sensor_utils_test.py @@ -0,0 +1,99 @@ +import numpy as np +import pandas as pd + +from pytest import fixture +from pytest import mark + +from .. core import system_of_units as units + +from . sensor_utils import first_and_last_times +from . sensor_utils import get_n_sensors +from . sensor_utils import sensor_order +from . sensor_utils import pmt_and_sipm_bin_width +from . sensor_utils import trigger_times + + +def test_trigger_times(): + event_time = 22 * units.ns + bin_width = 0.1 + time_bins = np.arange(0, 10, bin_width) + triggers = [15, 75] + + trgr_times = trigger_times(triggers, event_time, time_bins) + assert len(trgr_times) == len(triggers) + assert trgr_times[0] == event_time + triggers[0] * bin_width + assert trgr_times[1] == event_time + triggers[1] * bin_width + + +@mark.parametrize("bin_min bin_max bin_wid".split(), + ((22, 40, 1), (1, 5, 0.1))) +def test_first_and_last_times(bin_min, bin_max, bin_wid): + test_bins1 = np.arange(bin_min, bin_max, bin_wid) + test_bins2 = np.arange(bin_min, bin_max - 1) + + bmin, bmax = first_and_last_times(pd.DataFrame({'time': test_bins1}), + pd.DataFrame({'time': test_bins2}), + bin_wid, 1) + + assert np.isclose(bmin, bin_min) + assert np.isclose(bmax, bin_max) + + +@fixture(scope = 'function') +def sensor_info(): + id_dict = {'pmt_ids' : (5, 2, 7), 'pmt_ord' : (5, 2, 7), + 'sipm_ids': (1010, 5023), 'sipm_ord': (10, 279)} + + nsamp_pmt = 5000 + nsamp_sipm = 5 + pmt_resp = pd.Series([[1]*nsamp_pmt ]*len(id_dict[ 'pmt_ids']), + index = id_dict[ 'pmt_ids']) + sipm_resp = pd.Series([[1]*nsamp_sipm]*len(id_dict['sipm_ids']), + index = id_dict['sipm_ids']) + pmt_q = np.array( pmt_resp.tolist()) + sipm_q = np.array(sipm_resp.tolist()) + return id_dict, nsamp_pmt, nsamp_sipm, pmt_resp, sipm_resp, pmt_q, sipm_q + + +def test_sensor_order(sensor_info, pmt_ids, sipm_ids): + detector_db = 'new' + run_number = 6400 + n_pmt = len( pmt_ids) + n_sipm = len(sipm_ids) + + (id_dict , nsamp_pmt, nsamp_sipm, + pmt_resp, sipm_resp, pmt_q , sipm_q) = sensor_info + + order_sensors = sensor_order(detector_db, run_number, + nsamp_pmt , nsamp_sipm) + + padded_evt = order_sensors(pmt_resp, sipm_resp, [(pmt_q, sipm_q)]) + + pmt_out = padded_evt[0][0] + sipm_out = padded_evt[0][1] + assert pmt_out.shape == (n_pmt , nsamp_pmt) + assert sipm_out.shape == (n_sipm, nsamp_sipm) + + pmt_nonzero = np.argwhere( pmt_out.sum(axis=1) != 0) + sipm_nonzero = np.argwhere(sipm_out.sum(axis=1) != 0) + + assert np.all([pmt in id_dict[ 'pmt_ord'] for pmt in pmt_nonzero]) + assert np.all([sipm in id_dict['sipm_ord'] for sipm in sipm_nonzero]) + + +def test_get_n_sensors(pmt_ids, sipm_ids): + npmt, nsipm = get_n_sensors('new', -6400) + + assert npmt == len( pmt_ids) + assert nsipm == len(sipm_ids) + + +def test_pmt_and_sipm_bin_width(full_sim_file): + file_in = full_sim_file + + expected_pmtwid = 100 * units.ns + expected_sipmwid = 1 * units.mus + + pmt_binwid, sipm_binwid = pmt_and_sipm_bin_width(file_in) + assert pmt_binwid == expected_pmtwid + assert sipm_binwid == expected_sipmwid diff --git a/invisible_cities/evm/event_model.py b/invisible_cities/evm/event_model.py index 614a07e28e..8e7f5561dc 100644 --- a/invisible_cities/evm/event_model.py +++ b/invisible_cities/evm/event_model.py @@ -3,18 +3,11 @@ import tables as tb import numpy as np -from enum import Enum from enum import auto -from networkx import Graph -from .. types.ic_types import NN -from .. types.ic_types import minmax -from .. types.ic_types import AutoNameEnumBase -from .. core.exceptions import PeakNotFound -from .. core.exceptions import SipmEmptyList -from .. core.exceptions import SipmNotFound -from .. core.core_functions import loc_elem_1d -from .. core.system_of_units_c import units +from .. types.ic_types import NN +from .. types.ic_types import AutoNameEnumBase +from .. core import system_of_units as units from typing import List from typing import Tuple @@ -23,35 +16,6 @@ ZANODE = -9.425 * units.mm -class SensorParams: - """Transient class storing sensor parameters.""" - def __init__(self, npmt, pmtwl, nsipm, sipmwl): - self.npmt = npmt - self.pmtwl = pmtwl - self.nsipm = nsipm - self.sipmwl = sipmwl - - @property - def NPMT (self): return self.npmt - - @property - def PMTWL (self): return self.pmtwl - - @property - def NSIPM (self): return self.nsipm - - @property - def SIPMWL(self): return self.sipmwl - - def __str__(self): - s = "{0}SensorParams\n{0}".format("#"*20 + "\n") - for attr in self.__dict__: - s += "{}: {}\n".format(attr, getattr(self, attr)) - return s - - __repr__ = __str__ - - class MCInfo(NamedTuple): """Transient class storing the tables of MC true info""" extents : tb.Table @@ -82,37 +46,6 @@ def __str__(self): __repr__ = __str__ -class MCParticle: - """A MC Particle """ - - def __init__(self, particle_name, primary, - mother_indx, - initial_vertex, final_vertex, - initial_volume, final_volume, - momentum, energy, creator_proc): - self.name = particle_name - self.primary = primary - self.mother_indx = mother_indx - self.initial_vertex = initial_vertex - self.final_vertex = final_vertex - self.initial_volume = initial_volume - self.final_volume = final_volume - self.p = momentum - self.E = energy - self.process = creator_proc - self.hits = [] - - def __str__(self): - return f""" MCParticle: name = {self.name} - initial volume = {self.initial_volume}, final volume = {self.final_volume} - kinetic energy = {self.E} MeV - mother = {self.mother_indx}, creator process = {self.process} - number of hits = {len(self.hits)} - \n""" - - __repr__ = __str__ - - class HitEnergy(AutoNameEnumBase): E = auto() Ec = auto() diff --git a/invisible_cities/evm/event_model_test.py b/invisible_cities/evm/event_model_test.py index 189f80fb2e..2f709058ec 100644 --- a/invisible_cities/evm/event_model_test.py +++ b/invisible_cities/evm/event_model_test.py @@ -1,7 +1,6 @@ import numpy as np -from numpy.testing import assert_equal -from pytest import mark +from pytest import mark from hypothesis import given from hypothesis.strategies import just @@ -10,11 +9,8 @@ from hypothesis.strategies import floats from hypothesis.strategies import integers from hypothesis.strategies import composite -from hypothesis.extra.numpy import arrays -from .. types.ic_types_c import xy -from .. types.ic_types_c import minmax -from . event_model import SensorParams +from .. types.ic_types import xy from . event_model import Event from . event_model import Cluster @@ -25,15 +21,6 @@ from . event_model import KrEvent -@composite -def sensor_params_input(draw): - npmt = draw(integers()) - pmtwl = draw(integers()) - nsipm = draw(integers()) - sipmwl = draw(integers()) - return npmt, pmtwl, nsipm, sipmwl - - @composite def event_input(draw): evt_no = draw(integers()) @@ -84,17 +71,6 @@ def hits(draw): return h -@given(sensor_params_input()) -def test_sensor_params(sensor_pars): - npmt, pmtwl, nsipm, sipmwl = sensor_pars - sp = SensorParams(*sensor_pars) - - assert sp.npmt == sp.NPMT == npmt - assert sp.nsipm == sp.NSIPM == nsipm - assert sp.pmtwl == sp.PMTWL == pmtwl - assert sp.sipmwl == sp.SIPMWL == sipmwl - - @mark.parametrize("test_class", (Event, HitCollection, diff --git a/invisible_cities/evm/histos.py b/invisible_cities/evm/histos.py deleted file mode 100644 index 6d51581cf2..0000000000 --- a/invisible_cities/evm/histos.py +++ /dev/null @@ -1,173 +0,0 @@ -import numpy as np -import tables as tb - -from .. reco import tbl_functions as tbl - - -class Histogram: - def __init__(self, title, bins, labels, values=None): - """ - This class represents a histogram with is a parameter holder that - contains data grouped by bins. - - Attributes: - title = String with the histogram title - bins = List with the histogram binning. - data = Array with the accumulated entries on each bin. - out_range = Array with the Accumulated counts out of the bin range. - Values are n-dim arrays of lenght 2 (first element is - underflow, second oveflow). - errors = Array with the assigned uncertanties to each bin. - labels = List with the axis labels. - - Arguments: - bins = List containing the histogram binning. - values = Array with initial values, optional. - If not passed, then the initial bin content is set to zero. - """ - self.title = title - self.bins = bins - self.data = self.init_from_bins() - self.errors = self.init_from_bins() - self.out_range = np.zeros(shape=(2, len(self.bins))) - self.labels = labels - - if values is not None: - self.fill(np.asarray(values)) - - def init_from_bins(self): - "Encapsulation for histogram initialization to 0" - return np.zeros(shape=tuple(len(x) - 1 for x in self.bins)) - - def fill(self, additive, data_weights=None): - """ - Given datapoints, bins and adds thems to the stored bin content. - - Arguments: - additive = Array or list with data to fill the histogram. - data_weights = Array or list with weights of the data. - """ - additive = np.array(additive) - data_weights = np.ones(len(additive.T)) if data_weights is None else np.array(data_weights) - if len(data_weights) != len(additive.T): - raise ValueError("Dimensions of data and weights is not compatible.") - - binnedData, outRange = self.bin_data(additive, data_weights) - - self.data += binnedData - self.out_range += outRange - self.update_errors() - - def bin_data(self, data, data_weights): - """ - Bins the given data and computes the events out of range. - - Arguments: - data = Array with the data to be binned. - data_weights = Array with weights for the data points. - """ - binned_data, *_ = np.histogramdd(data.T, self.bins, weights=data_weights) - out_of_range = self.count_out_of_range(np.array(data, ndmin=2)) - - return binned_data, out_of_range - - def count_out_of_range(self, data): - """ - Returns an array with the number of events out of the Histogram's bin - range of the given data. - - Arguments: - data = Array with the data. - """ - out_of_range = [] - for i, bins in enumerate(self.bins): - lower_limit = bins[0] - upper_limit = bins[-1] - out_of_range.append([np.count_nonzero(data[i] < lower_limit), - np.count_nonzero(data[i] > upper_limit)]) - - return np.asarray(out_of_range).T - - def update_errors(self, errors=None): - """ - Updates the errors with the passed list/array. If nothing is passed, - then the square root of the counts is computed and assigned as error. - - Arguments: - errors = List or array of errors. - """ - self.errors = np.asarray(errors) if errors is not None else np.sqrt(self.data) - - def _check_valid_binning(self, bins): - if len(self.bins) != len(bins) or not np.all(a == b for a, b in zip(self.bins, bins)): - raise ValueError("Histogram binning is not compatible") - - def __radd__(self, other): - return self + other - - def __add__ (self, other): - if other is None: - return self - self._check_valid_binning(other.bins) - if self.title != other.title: - print(f"""Warning: Histogram titles are different. - {self.title}, {other.title}""") - if self.labels != other.labels: - print(f"""Warning: Histogram titles are different. - {self.labels}, {other.labels}""") - new_histogram = Histogram(self.title, self.bins, self.labels) - new_histogram.data = self.data + other.data - new_histogram.out_range = self.out_range + other.out_range - new_histogram.errors = np.sqrt (self.errors ** 2 + other.errors ** 2) - return new_histogram - - -class HistoManager: - def __init__(self, histograms=None): - """ - This class is a parameter holder that contains a dictionary - of Histogram objects. - - Attributes: - histos = Dictionary holding Histogram objects. - Keys are the Histograms' name. - - Arguments: - histograms = List with the initial Histogram objects. - """ - self.histos = {} - - if histograms is not None: - values = histograms.values() if isinstance(histograms, dict) else iter(histograms) - for histogram in values: - self.new_histogram(histogram) - - def new_histogram(self, histogram): - """ - Adds a new Histogram to the HistoManager. - - Arguments: - - histogram = Histogram object. - """ - self[histogram.title] = histogram - - def fill_histograms(self, additives): - """ - Fills several Histograms of the Histomanager. - - Arguments: - additives: Dictionary with keys equal to the Histograms names. - Values are the data to fill the Histogram. - """ - for histoname, additive in additives.items(): - if histoname in self.histos: - self[histoname].fill(np.asarray(additive)) - else: - print(f"Histogram with name {histoname} does not exist") - - def __getitem__(self, histoname): - return self.histos[histoname] - - def __setitem__(self, histoname, histogram): - self.histos[histoname] = histogram diff --git a/invisible_cities/evm/histos_test.py b/invisible_cities/evm/histos_test.py deleted file mode 100644 index 7f041ad47e..0000000000 --- a/invisible_cities/evm/histos_test.py +++ /dev/null @@ -1,375 +0,0 @@ -import string - -import numpy as np - -from pytest import raises -from pytest import mark - -from hypothesis import assume -from hypothesis import given -from hypothesis import settings -from hypothesis.extra.numpy import arrays -from hypothesis.strategies import composite -from hypothesis.strategies import integers -from hypothesis.strategies import floats -from hypothesis.strategies import text -from hypothesis.strategies import lists -from hypothesis.strategies import sampled_from -from hypothesis.strategies import one_of - -from .. evm.histos import HistoManager, Histogram - - -characters = string.ascii_letters + string.digits -letters = string.ascii_letters - - -def assert_histogram_equality(histogram1, histogram2): - assert np.all (a == b for a, b in zip(histogram1.bins, histogram2.bins)) - assert np.allclose(histogram1.data , histogram2.data ) - assert np.allclose(histogram1.errors , histogram2.errors ) - assert np.allclose(histogram1.out_range, histogram2.out_range) - assert histogram1.title == histogram2.title - assert histogram1.labels == histogram2.labels - - -@composite -def titles(draw): - return draw(text(letters, min_size=1)) + draw(text(characters, min_size=5)) - - -@composite -def bins_arrays(draw, dimension=0): - if dimension <= 0: - dimension = draw(sampled_from((1,2))) - - bin_lower_margins = draw(arrays(float, dimension, - floats(-1e3, 1e3, allow_nan=False, allow_infinity=False))) - bin_additive = draw(arrays(float, dimension, - floats(1.1 , 1e3, allow_nan=False, allow_infinity=False))) - bin_upper_margins = bin_lower_margins + bin_additive - - bins = [ np.linspace(bin_lower_margins[i], bin_upper_margins[i], draw(integers(2, 20))) for i, _ in enumerate(bin_lower_margins) ] - - return bins - - -@composite -def filled_histograms(draw, dimension=0, fixed_bins=None): - if fixed_bins is not None: - dimension = len(fixed_bins) - if dimension <= 0: - dimension = draw(sampled_from((1,2))) - - if fixed_bins is not None: - bins = [ np.linspace(bin_range[0], bin_range[1], bin_range[2] + 1) for bin_range in fixed_bins ] - else: - bins = draw(bins_arrays(dimension=dimension)) - - labels = draw(lists(text(characters, min_size=5), min_size=dimension, max_size=dimension)) - shape = draw(integers(50, 100)), - data = [] - for i in range(dimension): - lower_limit = bins[i][0] - draw(floats(0.5, 1e8, allow_nan=False, allow_infinity=False)) - upper_limit = bins[i][-1] + draw(floats(0.5, 1e8, allow_nan=False, allow_infinity=False)) - data.append(draw(arrays(float, shape, floats(lower_limit, upper_limit, - allow_nan=False, allow_infinity=False)))) - data = np.array(data) - args = draw(titles()), bins, labels, data - return args, Histogram(*args) - - -@composite -def empty_histograms(draw, dimension=0): - if dimension <= 0: - dimension = draw(sampled_from((1,2))) - bins = draw(bins_arrays(dimension=dimension)) - labels = draw(lists(text(characters, min_size=5), min_size=dimension, max_size=dimension)) - args = draw(titles()), bins, labels - return args, Histogram(*args) - - -@composite -def histograms_lists(draw, number=0, dimension=0, fixed_bins=None): - if number <= 0: - number = draw(integers(2, 5)) - empty_histogram = empty_histograms (dimension=dimension) - filled_histogram = filled_histograms(dimension=dimension, fixed_bins=fixed_bins) - args, histograms = zip(*[ draw(one_of(empty_histogram, filled_histogram)) for i in range(number) ]) - - titles, *_ = zip(*args) - assume(len(set(titles)) == len(titles)) - - return args, histograms - -@mark.skip(reason="Delaying elimination of solid cities") -@given(bins_arrays()) -@settings(deadline=None) -def test_histogram_initialization(bins): - label = [ 'Random distribution' ] - title = 'Test_histogram' - test_histo = Histogram(title, bins, label) - - assert np.all (a == b for a, b in zip(test_histo.bins, bins)) - assert np.allclose(test_histo.data , np.zeros(shape=tuple(len(x) - 1 for x in bins))) - assert np.allclose(test_histo.errors , np.zeros(shape=tuple(len(x) - 1 for x in bins))) - assert np.allclose(test_histo.out_range, np.zeros(shape=(2, len(bins))) ) - assert test_histo.labels == label - assert test_histo.title == title - -@mark.skip(reason="Delaying elimination of solid cities") -@given(bins_arrays()) -@settings(deadline=None) -def test_histogram_initialization_with_values(bins): - label = [ 'Random distribution' ] - title = 'Test_histogram' - data = [] - out_range = [] - for ibin in bins: - lower_limit = ibin[0] * 0.95 - upper_limit = ibin[-1] * 1.05 - data.append( np.random.uniform(lower_limit, upper_limit, 500) ) - out_range.append(np.array([ np.count_nonzero(data[-1] < ibin[0]), - np.count_nonzero(data[-1] > ibin[-1])])) - - data = np.array(data) - out_range = np.array(out_range).T - binned_data = np.histogramdd(data.T, bins)[0] - test_histo = Histogram(title, bins, label, data) - - assert np.all (a == b for a, b in zip(test_histo.bins, bins)) - assert np.allclose(test_histo.data , binned_data ) - assert np.allclose(test_histo.out_range, out_range ) - assert np.allclose(test_histo.errors , np.sqrt(binned_data)) - assert test_histo.labels == label - assert test_histo.title == title - -@mark.skip(reason="Delaying elimination of solid cities") -@given(empty_histograms()) -@settings(deadline=None) -def test_histogram_fill(empty_histogram): - _, test_histogram = empty_histogram - histobins = test_histogram.bins - n_points = np.random.randint(5, 201) - test_data = [ np.random.uniform( bins[0] * (1 + np.random.uniform()), - bins[-1] * (1 + np.random.uniform()), - n_points) for bins in histobins ] - test_histogram.fill(test_data) - out_of_range = [ np.array([ np.count_nonzero(test_data[i] < bins[0] ), - np.count_nonzero(test_data[i] > bins[-1]) ]) - for i, bins in enumerate(histobins) ] - test_out_of_range = np.array(out_of_range).T - test_data = np.histogramdd(np.asarray(test_data).T, test_histogram.bins)[0] - test_errors = np.sqrt(test_data) - - assert np.allclose(test_histogram.data , test_data ) - assert np.allclose(test_histogram.errors , test_errors ) - assert np.allclose(test_histogram.out_range, test_out_of_range) - -@mark.skip(reason="Delaying elimination of solid cities") -@given(empty_histograms()) -@settings(deadline=None) -def test_histogram_fill_with_weights(empty_histogram): - _, test_histogram = empty_histogram - histobins = test_histogram.bins - n_points = np.random.randint(50, 201) - test_data = [ np.random.uniform( bins[0] * (1 + np.random.uniform()), - bins[-1] * (1 + np.random.uniform()), - n_points) for bins in histobins ] - test_weights = np.random.uniform(1, 10, n_points) - test_histogram.fill(test_data, test_weights) - out_of_range = [ np.array([ np.count_nonzero(test_data[i] < bins[0] ), - np.count_nonzero(test_data[i] > bins[-1]) ]) - for i, bins in enumerate(histobins) ] - test_out_of_range = np.array(out_of_range).T - test_data = np.histogramdd(np.asarray(test_data).T, test_histogram.bins, weights=test_weights)[0] - test_errors = np.sqrt(test_data) - - assert np.allclose(test_histogram.data , test_data ) - assert np.allclose(test_histogram.errors , test_errors ) - assert np.allclose(test_histogram.out_range, test_out_of_range) - -@mark.skip(reason="Delaying elimination of solid cities") -def test_bin_data(): - bins = [ np.linspace(0., 5., 6) ] - label = [ 'Random distribution' ] - title = 'Test_histogram' - data = np.array([-1., 2.2, 3.2, 4.5, 6.3, 7.1, 4.9, 3.1, 0.2, 2.1, 2.2, 1.1]) - test_data = [1, 1, 3, 2, 2] - test_out_range = [[1], [2]] - - test_histo = Histogram(title, bins, label) - binned_data, out_range = test_histo.bin_data(data, data_weights=np.ones(len(data))) - - assert np.allclose(binned_data, test_data ) - assert np.allclose(out_range , test_out_range) - -@mark.skip(reason="Delaying elimination of solid cities") -def test_count_out_of_range(): - bins = [ np.linspace(0., 5., 6) ] - label = [ 'Random distribution' ] - title = 'Test_histogram' - data = np.array([-1., 2.2, 3.2, 4.5, 6.3, 7.1, 4.9, 3.1, 0.2, 2.1, 2.2, 1.1]) - test_data = [1, 1, 3, 2, 2] - test_out_range = [[1], [2]] - - test_histo = Histogram(title, bins, label) - out_range = test_histo.count_out_of_range(np.array(data, ndmin=2)) - - assert np.allclose(out_range, test_out_range) - -@mark.skip(reason="Delaying elimination of solid cities") -@given(filled_histograms()) -@settings(deadline=None) -def test_update_errors(filled_histogram): - _, test_histogram = filled_histogram - test_histogram.errors = None - test_histogram.update_errors() - assert np.allclose(test_histogram.errors, np.sqrt(test_histogram.data)) - -@mark.skip(reason="Delaying elimination of solid cities") -@given(filled_histograms()) -@settings(deadline=None) -def test_update_errors_with_values(filled_histogram): - _, test_histogram = filled_histogram - new_errors = np.random.uniform(0., 1000, size=test_histogram.data.shape) - test_histogram.update_errors(new_errors) - assert np.allclose(test_histogram.errors, new_errors) - -@mark.skip(reason="Delaying elimination of solid cities") -@given(filled_histograms(fixed_bins=[[50, 900, 20]]), - filled_histograms(fixed_bins=[[50, 900, 20]])) -@settings(deadline=None) -def test_add_histograms(first_histogram, second_histogram): - _, test_histogram1 = first_histogram - _, test_histogram2 = second_histogram - sum_histogram = test_histogram1 + test_histogram2 - - assert np.all (a == b for a, b in zip(sum_histogram.bins, test_histogram1.bins)) - assert np.all (a == b for a, b in zip(sum_histogram.bins, test_histogram2.bins)) - assert np.allclose(sum_histogram.data , test_histogram1.data + test_histogram2.data ) - assert np.allclose(sum_histogram.errors , np.sqrt(test_histogram1.errors ** 2 + test_histogram2.errors ** 2)) - assert np.allclose(sum_histogram.out_range, test_histogram1.out_range + test_histogram2.out_range ) - assert sum_histogram.labels == test_histogram1.labels - assert sum_histogram.title == test_histogram1.title - -@mark.skip(reason="Delaying elimination of solid cities") -@given(filled_histograms(fixed_bins=[[50, 900, 20]]), - filled_histograms(fixed_bins=[[50, 900, 5]])) -@settings(deadline=None) -def test_add_histograms_with_incompatible_binning_raises_ValueError(first_histogram, second_histogram): - _, test_histogram1 = first_histogram - _, test_histogram2 = second_histogram - - with raises(ValueError): - sum_histogram = test_histogram1 + test_histogram2 - -@mark.skip(reason="Delaying elimination of solid cities") -@given(filled_histograms(fixed_bins=[[50, 900, 20], [20, 180, 15]]), - filled_histograms(fixed_bins=[[50, 900, 20], [20, 180, 15]])) -@settings(deadline=None) -def test_add_histograms_2d(first_histogram, second_histogram): - _, test_histogram1 = first_histogram - _, test_histogram2 = second_histogram - sum_histogram = test_histogram1 + test_histogram2 - - assert np.all (a == b for a, b in zip(sum_histogram.bins, test_histogram1.bins)) - assert np.all (a == b for a, b in zip(sum_histogram.bins, test_histogram2.bins)) - assert np.allclose(sum_histogram.data , test_histogram1.data + test_histogram2.data ) - assert np.allclose(sum_histogram.errors , np.sqrt(test_histogram1.errors ** 2 + test_histogram2.errors ** 2)) - assert np.allclose(sum_histogram.out_range, test_histogram1.out_range + test_histogram2.out_range ) - assert sum_histogram.labels == test_histogram1.labels - assert sum_histogram.title == test_histogram1.title - -@mark.skip(reason="Delaying elimination of solid cities") -@given (histograms_lists()) -@settings(deadline=None) -def test_histomanager_initialization_with_histograms(histogram_list): - _, list_of_histograms = histogram_list - histogram_manager = HistoManager(list_of_histograms) - - for histogram in list_of_histograms: - assert_histogram_equality(histogram, histogram_manager[histogram.title]) - -@mark.skip(reason="Delaying elimination of solid cities") -def test_histomanager_initialization_without_histograms(): - histogram_manager = HistoManager() - assert len(histogram_manager.histos) == 0 - -@mark.skip(reason="Delaying elimination of solid cities") -@given(one_of(empty_histograms(), filled_histograms())) -@settings(deadline=None) -def test_new_histogram_in_histomanager(test_histogram): - _, histogram = test_histogram - histoname = histogram.title - histogram_manager = HistoManager() - histogram_manager.new_histogram(histogram) - - assert_histogram_equality(histogram, histogram_manager[histoname]) - -@mark.skip(reason="Delaying elimination of solid cities") -@given (histograms_lists()) -@settings(deadline=None) -def test_fill_histograms_in_histomanager(histogram_list): - args, list_of_histograms = histogram_list - titles, histobins, *_ = zip(*args) - histogram_manager = HistoManager(list_of_histograms) - - test_data_values = {} - old_data_values = {} - test_out_of_range = {} - old_out_of_range = {} - for i, title in enumerate(titles): - old_data_values [title] = np.copy(histogram_manager[title].data ) - old_out_of_range [title] = np.copy(histogram_manager[title].out_range) - n_points = np.random.randint(5, 201) - test_data = [ np.random.uniform( bins[0] * (1 + np.random.uniform()), - bins[-1] * (1 + np.random.uniform()), - n_points) for bins in histobins[i] ] - test_data_values [title] = test_data - out_of_range = [ np.array([ np.count_nonzero(test_data[j] < bins[0] ), - np.count_nonzero(test_data[j] > bins[-1]) ]) - for j, bins in enumerate(histobins[i]) ] - test_out_of_range[title] = np.array(out_of_range).T - - histogram_manager.fill_histograms(test_data_values) - - for histoname, data in test_data_values.items(): - histogram = histogram_manager [histoname] - old_data = np.array(old_data_values[histoname]) - test_data = np.histogramdd(np.asarray(data).T, histogram.bins)[0] - assert np.allclose(histogram.data , test_data + old_data ) - assert np.allclose(histogram.errors , np.sqrt(test_data + old_data) ) - assert np.allclose(histogram.out_range, test_out_of_range[histoname] + old_out_of_range[histoname]) - -@mark.skip(reason="Delaying elimination of solid cities") -@given (histograms_lists()) -@settings(deadline=None) -def test_getitem_histomanager(histogram_list): - args, list_of_histograms = histogram_list - titles, histobins, *_ = zip(*args) - histogram_manager = HistoManager(list_of_histograms) - - for histoname in histogram_manager.histos: - assert np.all (a == b for a, b in zip(histogram_manager.histos[histoname].bins, histogram_manager[histoname].bins)) - assert np.allclose(histogram_manager.histos[histoname].data , histogram_manager[histoname].data ) - assert np.allclose(histogram_manager.histos[histoname].errors , histogram_manager[histoname].errors ) - assert np.allclose(histogram_manager.histos[histoname].out_range, histogram_manager[histoname].out_range) - assert histogram_manager.histos[histoname].labels == histogram_manager[histoname].labels - assert histogram_manager.histos[histoname].title == histogram_manager[histoname].title - -@mark.skip(reason="Delaying elimination of solid cities") -@given(histograms_lists()) -@settings(deadline=None) -def test_setitem_histomanager(histogram_list): - args, list_of_histograms = histogram_list - titles, histobins, *_ = zip(*args) - histogram_manager1 = HistoManager() - histogram_manager2 = HistoManager() - - for histogram in list_of_histograms: - histoname = histogram.title - histogram_manager1.histos[histoname] = histogram - histogram_manager2 [histoname] = histogram - - assert_histogram_equality(histogram_manager1[histoname], histogram_manager2[histoname]) diff --git a/invisible_cities/evm/ic_containers.py b/invisible_cities/evm/ic_containers.py index 3989b3c2cc..39eb0ff89f 100644 --- a/invisible_cities/evm/ic_containers.py +++ b/invisible_cities/evm/ic_containers.py @@ -20,28 +20,13 @@ def _add_namedtuple_in_this_module(name, attribute_names): setattr(this_module, name, new_nametuple) for name, attrs in ( - ('DataVectors' , 'pmt sipm mc events trg_type trg_channels'), ('SensorData' , 'NPMT PMTWL NSIPM SIPMWL '), - ('PmapVectors' , 'pmaps events timestamps mc'), - ('RawVectors' , 'event pmtrwf sipmrwf pmt_active sipm_active'), - ('CalibParams' , 'coeff_c coeff_blr adc_to_pes_pmt adc_to_pes_sipm'), ('DeconvParams' , 'n_baseline thr_trigger'), ('CalibVectors' , 'channel_id coeff_blr coeff_c adc_to_pes adc_to_pes_sipm pmt_active'), ('S12Params' , 'time stride length rebin_stride'), - ('PmapParams' , 's1_params s2_params s1p_params s1_PMT_params s1p_PMT_params'), - ('ThresholdParams', 'thr_s1 thr_s2 thr_MAU thr_sipm thr_SIPM'), - ('CSum' , 'csum csum_mau'), - ('CCWf' , 'ccwf ccwf_mau'), - ('S12Sum' , 's1_ene s1_indx s2_ene s2_indx'), ('ZsWf' , 'indices energies'), - ('CalibratedPMT' , 'CPMT CPMT_mau'), - ('S1PMaps' , 'S1 S1_PMT S1p S1p_PMT'), - ('PMaps' , 'S1 S2 S2Si'), - ('Peak' , 't E'), ('FitFunction' , 'fn values errors chi2 pvalue cov'), ('TriggerParams' , 'trigger_channels min_number_channels charge height width'), - ('PeakData' , 'charge height width'), - ('Measurement' , 'value uncertainty'), ('SensorParams' , 'spectra peak_range min_bin_peak max_bin_peak half_peak_width p1pe_seed lim_ped'), ('PedestalParams' , 'gain gain_min gain_max sigma sigma_min sigma_max')): _add_namedtuple_in_this_module(name, attrs) diff --git a/invisible_cities/evm/nh5.py b/invisible_cities/evm/nh5.py index 2b6915aa1f..e243f4ac60 100644 --- a/invisible_cities/evm/nh5.py +++ b/invisible_cities/evm/nh5.py @@ -15,25 +15,6 @@ class EventInfo(tb.IsDescription): timestamp = tb.UInt64Col(shape=(), pos=1) -class DetectorGeometry(tb.IsDescription): - """Store geometry information for the detector.""" - x_det = tb.Float32Col(pos=1, shape=2) # xmin, xmax - y_det = tb.Float32Col(pos=2, shape=2) # ymin, ymax - z_det = tb.Float32Col(pos=3, shape=2) # zmin, zmax - r_det = tb.Float32Col(pos=4) # radius - - -class DataSensor(tb.IsDescription): - """Store metadata information for the SiPMs (position, gain, - calibration-constant, mask). - """ - channel = tb. Int32Col(pos=0) # electronic channel - position = tb.Float32Col(pos=1, shape=3) - coeff = tb.Float64Col(pos=2) - adc_to_pes = tb.Float32Col(pos=3) - noise_rms = tb.Float32Col(pos=4) - - class SensorTable(tb.IsDescription): """ Stores the Sensors group, mimicking what is saved @@ -62,6 +43,12 @@ class MCExtentInfo(tb.IsDescription): last_particle = tb.UInt64Col(pos=2) +class MCEventMap(tb.IsDescription): + """Map between event index and original event.""" + evt_number = tb.Int32Col(shape=(), pos=0) + sub_evt = tb.Int32Col(shape=(), pos=1) + + class MCHitInfo(tb.IsDescription): """Stores the simulated hits as metadata using Pytables. """ @@ -73,30 +60,6 @@ class MCHitInfo(tb.IsDescription): hit_indx = tb. Int16Col( pos=5) -class MCParticleInfo(tb.IsDescription): - """Stores the simulated particles as metadata using Pytables. - """ - particle_indx = tb. Int16Col( pos= 0) - particle_name = tb. StringCol( 20, pos= 1) - primary = tb. Int16Col( pos= 2) - mother_indx = tb. Int16Col( pos= 3) - initial_vertex = tb.Float32Col( pos= 4, shape=4) - final_vertex = tb.Float32Col( pos= 5, shape=4) - initial_volume = tb. StringCol( 20, pos= 6) - final_volume = tb. StringCol( 20, pos= 7) - momentum = tb.Float32Col( pos= 8, shape=3) - kin_energy = tb.Float32Col( pos= 9) - creator_proc = tb. StringCol(100, pos=10) - - -class SENSOR_WF(tb.IsDescription): - """Describe a true waveform (zero supressed).""" - event = tb.UInt32Col (pos=0) - ID = tb.UInt32Col (pos=1) - time_mus = tb.Float32Col(pos=2) - ene_pes = tb.Float32Col(pos=3) - - class FEE(tb.IsDescription): """Store the parameters used by the EP simulation as metadata.""" OFFSET = tb. Int16Col(pos= 1) # displaces the baseline (e.g, 700) @@ -119,12 +82,6 @@ class FEE(tb.IsDescription): f_LPF2 = tb.Float32Col(pos=18) # LPF -class DECONV_PARAM(tb.IsDescription): - N_BASELINE = tb.Int32Col(pos=0) - THR_TRIGGER = tb.Int16Col(pos=1) - ACCUM_DISCHARGE_LENGTH = tb.Int16Col(pos=2) - - class S12(tb.IsDescription): """Store for a S1/S2 The table maps a S12: @@ -196,26 +153,6 @@ class KrTable(tb.IsDescription): Yrms = tb.Float64Col(pos=24) -class XYfactors(tb.IsDescription): - x = tb.Float32Col(pos=0) - y = tb.Float32Col(pos=1) - factor = tb.Float32Col(pos=2) - uncertainty = tb.Float32Col(pos=3) - nevt = tb. UInt32Col(pos=4) - - -class Zfactors(tb.IsDescription): - z = tb.Float32Col(pos=0) - factor = tb.Float32Col(pos=1) - uncertainty = tb.Float32Col(pos=2) - - -class Tfactors(tb.IsDescription): - t = tb.Float32Col(pos=0) - factor = tb.Float32Col(pos=1) - uncertainty = tb.Float32Col(pos=2) - - class HitsTable(tb.IsDescription): event = tb. Int32Col(pos=0) time = tb.Float64Col(pos=1) diff --git a/invisible_cities/evm/pmaps.py b/invisible_cities/evm/pmaps.py index 0a2abfcc30..8d39ba173a 100644 --- a/invisible_cities/evm/pmaps.py +++ b/invisible_cities/evm/pmaps.py @@ -4,9 +4,9 @@ from enum import auto -from .. core.system_of_units_c import units -from .. core.core_functions import weighted_mean_and_std -from .. types.ic_types import AutoNameEnumBase +from .. core import system_of_units as units +from .. core.core_functions import weighted_mean_and_std +from .. types.ic_types import AutoNameEnumBase class SiPMCharge(AutoNameEnumBase): diff --git a/invisible_cities/evm/pmaps_test.py b/invisible_cities/evm/pmaps_test.py index 241ff73aa6..12fd1e122d 100644 --- a/invisible_cities/evm/pmaps_test.py +++ b/invisible_cities/evm/pmaps_test.py @@ -8,20 +8,19 @@ from hypothesis import assume from hypothesis import given -from hypothesis import settings from hypothesis.strategies import integers from hypothesis.strategies import floats from hypothesis.strategies import sampled_from from hypothesis.strategies import composite from hypothesis.extra.numpy import arrays -from .. core.core_functions import weighted_mean_and_std -from .. core.random_sampling import NoiseSampler -from .. core.system_of_units_c import units -from .. core.testing_utils import exactly -from .. core.testing_utils import assert_SensorResponses_equality -from .. core.testing_utils import assert_Peak_equality -from .. core.testing_utils import previous_float +from .. core.core_functions import weighted_mean_and_std +from .. core.random_sampling import NoiseSampler +from .. core import system_of_units as units +from .. core.testing_utils import exactly +from .. core.testing_utils import assert_SensorResponses_equality +from .. core.testing_utils import assert_Peak_equality +from .. core.testing_utils import previous_float from invisible_cities.database import load_db as DB @@ -42,9 +41,9 @@ def sensor_responses(draw, n_samples=None, subtype=None, ids=None): n_sensors = draw(integers(1, 5)) if ids is None else len(ids) n_samples = draw(integers(1, 50)) if n_samples is None else n_samples shape = n_sensors, n_samples - all_wfs = draw(arrays(float, shape, floats (wf_min, wf_max))) + all_wfs = draw(arrays(float, shape, elements = floats (wf_min, wf_max))) if ids is None: - ids = draw(arrays( int, n_sensors, integers(0, 1e3), unique=True)) + ids = draw(arrays( int, n_sensors, elements = integers(0, 1e3), unique=True)) if subtype is None: subtype = draw(sampled_from((PMTResponses, SiPMResponses))) args = np.sort(ids), all_wfs @@ -65,7 +64,7 @@ def peaks(draw, subtype=None, pmt_ids=None, with_sipms=True): _, sipm_r = draw(sensor_responses(nsamples, SiPMResponses)) times = draw(arrays(float, nsamples, - floats(min_value=0, max_value=1e3), + elements = floats(min_value=0, max_value=1e3), unique = True).map(sorted)) bin_widths = np.array([1]) @@ -131,8 +130,8 @@ def test_SensorResponses_sum_over_sensors(srs): @given(size=integers(1, 10)) def test_SensorResponses_raises_exception_when_shapes_dont_match(SR, size): with raises(ValueError): - sr = SR(np.empty(size), - np.empty((size + 1, 1))) + SR(np.empty(size), + np.empty((size + 1, 1))) @given(peaks()) @@ -298,8 +297,8 @@ def test_Peak_raises_exception_when_shapes_dont_match(PK, sr1, sr2): (ids, wfs), sr1 = sr1 _ , sr2 = sr2 n_samples = wfs.shape[1] - pk = PK(np.empty(n_samples + 1), - np.empty(n_samples + 1), sr1, sr2) + PK(np.empty(n_samples + 1), + np.empty(n_samples + 1), sr1, sr2) @given(pmaps()) diff --git a/invisible_cities/filters/s1s2_filter.py b/invisible_cities/filters/s1s2_filter.py index 1b53489daf..7a7fcf102e 100644 --- a/invisible_cities/filters/s1s2_filter.py +++ b/invisible_cities/filters/s1s2_filter.py @@ -5,9 +5,9 @@ import numpy as np -from .. types.ic_types_c import minmax -from .. evm .pmaps import _Peak -from .. evm .pmaps import PMap +from .. types.ic_types import minmax +from .. evm .pmaps import _Peak +from .. evm .pmaps import PMap class S12SelectorOutput: diff --git a/invisible_cities/icaro/histogram_plot_functions.py b/invisible_cities/icaro/histogram_plot_functions.py deleted file mode 100644 index 52af7b84f3..0000000000 --- a/invisible_cities/icaro/histogram_plot_functions.py +++ /dev/null @@ -1,189 +0,0 @@ -import numpy as np -import matplotlib.pyplot as plt - -from .. io .hist_io import get_histograms_from_file -from .. evm .histos import Histogram -from .. evm .histos import HistoManager -from .. icaro.hst_functions import shift_to_bin_centers -from .. core .core_functions import weighted_mean_and_std - - -def plot_histograms_from_file(histofile, histonames='all', group_name='HIST', plot_errors=False, out_path=None, reference_histo=None): - """ - Plots the Histograms of a given file containing an HistoManager in a 3 column plot grid. - - histofile = String. Path to the file containing the histograms. - histonames = List with histogram name to be plotted, if 'all', all histograms are plotted. - group_name = String. Name of the group were Histograms were saved. - plot_errors = Boolean. If true, plot the associated errors instead of the data. - out_path = String. Path to save the histograms in png. If not passed, histograms won't be saved. - reference_histo = String. Path to a file containing the reference histograms. - If not passed reference histograms won't be plotted. - """ - histograms = get_histograms_from_file(histofile , group_name) - if reference_histo: - reference_histo = get_histograms_from_file(reference_histo, group_name) - plot_histograms(histograms, histonames=histonames, plot_errors=plot_errors, out_path=out_path, reference_histo=reference_histo) - - -def plot_histogram(histogram, ax=None, plot_errors=False, draw_color='black', stats=True, normed=True): - """ - Plot a Histogram. - - ax = Axes object to plot the figure. If not passed, a new axes will be created. - plot_errors = Boolean. If true, plot the associated errors instead of the data. - draw_color = String with the linecolor. - stats = Boolean. If true, histogram statistics info is added to the plotself. - normed = Boolean. If true, histogram is normalized. - """ - if ax is None: - fig, ax = plt.subplots(1, 1, figsize=(8, 6)) - - bins = histogram.bins - out_range = histogram.out_range - labels = histogram.labels - title = histogram.title - if plot_errors: - entries = histogram.errors - else: - entries = histogram.data - - if len(bins) == 1: - ax.hist (shift_to_bin_centers(bins[0]), bins[0], - weights = entries, - histtype = 'step', - edgecolor = draw_color, - linewidth = 1.5, - normed=normed) - ax.grid (True) - ax.set_axisbelow(True) - ax.set_ylabel ("Entries", weight='bold', fontsize=20) - - if stats: - entries_string = f'Entries = {np.sum(entries):.0f}\n' - out_range_string = 'Out range (%) = [{0:.2f}, {1:.2f}]'.format(get_percentage(out_range[0,0], np.sum(entries)), - get_percentage(out_range[1,0], np.sum(entries))) - - if np.sum(entries) > 0: - mean, std = weighted_mean_and_std(shift_to_bin_centers(bins[0]), entries, frequentist = True, unbiased = True) - else: - mean, std = 0, 0 - - ax.annotate(entries_string + - 'Mean = {0:.2f}\n'.format(mean) + - 'RMS = {0:.2f}\n' .format(std) + - out_range_string, - xy = (0.99, 0.99), - xycoords = 'axes fraction', - fontsize = 11, - weight = 'bold', - color = 'black', - horizontalalignment = 'right', - verticalalignment = 'top') - - elif len(bins) == 2: - ax.pcolormesh(bins[0], bins[1], entries.T) - ax.set_ylabel(labels[1], weight='bold', fontsize=20) - - if stats: - entries_string = f'Entries = {np.sum(entries):.0f}\n' - out_range_stringX = 'Out range X (%) = [{0:.2f}, {1:.2f}]'.format(get_percentage(out_range[0,0], np.sum(entries)), - get_percentage(out_range[1,0], np.sum(entries))) - out_range_stringY = 'Out range Y (%) = [{0:.2f}, {1:.2f}]'.format(get_percentage(out_range[0,1], np.sum(entries)), - get_percentage(out_range[1,1], np.sum(entries))) - - if np.sum(entries) > 0: - meanX, stdX = weighted_mean_and_std(shift_to_bin_centers(bins[0]), np.sum(entries, axis = 1), frequentist = True, unbiased = True) - meanY, stdY = weighted_mean_and_std(shift_to_bin_centers(bins[1]), np.sum(entries, axis = 0), frequentist = True, unbiased = True) - else: - meanX, stdX = 0, 0 - meanY, stdY = 0, 0 - - ax.annotate(entries_string + - 'Mean X = {0:.2f}\n'.format(meanX) + 'Mean Y = {0:.2f}\n'.format(meanY) + - 'RMS X = {0:.2f}\n' .format(stdX) + 'RMS Y = {0:.2f}\n' .format(stdY) + - out_range_stringX + '\n' + out_range_stringY, - xy = (0.99, 0.99), - xycoords = 'axes fraction', - fontsize = 11, - weight = 'bold', - color = 'white', - horizontalalignment = 'right', - verticalalignment = 'top') - - elif len(bins) == 3: - ave = np .apply_along_axis(average_empty, 2, entries, shift_to_bin_centers(bins[2])) - ave = np.ma.masked_array (ave, ave < 0.00001) - - img = ax .pcolormesh (bins[0], bins[1], ave.T) - cb = plt.colorbar (img, ax=ax) - cb .set_label (labels[2], weight='bold', fontsize=20) - for label in cb.ax.yaxis.get_ticklabels(): - label.set_weight ("bold") - label.set_fontsize (16) - - ax.set_ylabel(labels[1], weight='bold', fontsize=20) - - ax.set_xlabel (labels[0], weight='bold', fontsize=20) - ax.ticklabel_format(axis='x', style='sci', scilimits=(-3,3)) - - for label in (ax.get_xticklabels() + ax.get_yticklabels()): - label.set_fontweight('bold') - label.set_fontsize (16) - ax .xaxis.offsetText.set_fontsize (14) - ax .xaxis.offsetText.set_fontweight('bold') - - -def plot_histograms(histo_manager, histonames='all', n_columns=3, plot_errors=False, out_path=None, reference_histo=None, normed=True): - """ - Plot Histograms from a HistoManager. - - histo_manager = HistoManager object containing the Histograms to be plotted. - histonames = List with histogram name to be plotted, if 'all', all histograms are plotted. - n_columns = Int. Number of columns to distribute the histograms. - plot_errors = Boolean. If true, plot the associated errors instead of the data. - out_path = String. Path to save the histograms in png. If not passed, histograms won't be saved. - reference_histo = HistoManager object containing the Histograms to be plotted as reference. - normed = Boolean. If true, histograms are normalized. - """ - if histonames == 'all': - histonames = histo_manager.histos - - if out_path is None: - n_histos = len(histonames) - n_columns = min(3, n_histos) - n_rows = int(np.ceil(n_histos / n_columns)) - - fig, axes = plt.subplots(n_rows, n_columns, figsize=(8 * n_columns, 6 * n_rows)) - - for i, histoname in enumerate(histonames): - if out_path: - fig, ax = plt.subplots(1, 1, figsize=(8, 6)) - else: - ax = axes.flatten()[i] if isinstance(axes, np.ndarray) else axes - if reference_histo: - if len(reference_histo[histoname].bins) == 1: - plot_histogram(reference_histo[histoname], ax=ax, plot_errors=plot_errors, normed=normed, draw_color='red', stats=False) - plot_histogram (histo_manager [histoname], ax=ax, plot_errors=plot_errors, normed=normed) - - if out_path: - fig.tight_layout() - fig.savefig(out_path + histoname + '.png') - fig.clf() - plt.close(fig) - if out_path is None: - fig.tight_layout() - - -def get_percentage(a, b): - """ - Given two flots, return the percentage between them. - """ - return 100 * a / b if b else -100 - - -def average_empty(x, bins): - """ - Returns the weighted mean. If all weights are 0, the mean is considered to be 0. - """ - return np.average(bins, weights=x) if np.any(x > 0.) else 0. diff --git a/invisible_cities/icaro/hst_functions.py b/invisible_cities/icaro/hst_functions.py deleted file mode 100644 index 38dbd865b8..0000000000 --- a/invisible_cities/icaro/hst_functions.py +++ /dev/null @@ -1,259 +0,0 @@ -import os -import functools -import textwrap - -import numpy as np -import matplotlib.pyplot as plt - -from .. core import fit_functions as fitf -from .. core.core_functions import shift_to_bin_centers -from .. evm.ic_containers import Measurement - - -def create_new_figure(kwargs): - if kwargs.setdefault("new_figure", True): - plt.figure() - del kwargs["new_figure"] - - -def labels(xlabel, ylabel, title=""): - """ - Set x and y labels. - """ - plt.xlabel(xlabel) - plt.ylabel(ylabel) - plt.title ( title) - - -def hbins(x, nsigma=5, nbins=10): - """Given an array x, hbins returns the number of bins - in an interval of [ - nsigma*std(x), + nsigma*std(x)] - """ - xmin = np.average(x) - nsigma * np.std(x) - xmax = np.average(x) + nsigma * np.std(x) - bins = np.linspace(xmin, xmax, nbins + 1) - return bins - - -def plot(*args, **kwargs): - """ - Create a figure and then the histogram - """ - create_new_figure(kwargs) - return plt.plot(*args, **kwargs) - - -def hist(*args, **kwargs): - """ - Create a figure and then the histogram - """ - create_new_figure(kwargs) - - y, x, p = plt.hist(*args, **kwargs) - return y, shift_to_bin_centers(x), p - - -def doublehist(data1, data2, lbls, *args, **kwargs): - """ - Create a figure and then the histogram - """ - create_new_figure(kwargs) - - h1 = hist(data1, *args, label=lbls[0], alpha=0.5, normed=True, new_figure=False, **kwargs) - h2 = hist(data2, *args, label=lbls[1], alpha=0.5, normed=True, new_figure=False, **kwargs) - return h1, h2, plt.legend() - - -def hist2d(*args, **kwargs): - """ - Create a figure and then the histogram - """ - create_new_figure(kwargs) - - z, x, y, p = plt.hist2d(*args, **kwargs) - return z, shift_to_bin_centers(x), shift_to_bin_centers(y), p - - -def pdf(data, *args, **kwargs): - """ - Create a figure and then the normalized histogram - """ - create_new_figure(kwargs) - - h = hist(data, *args, **kwargs, weights=np.ones_like(data)/len(data)) - plt.yscale("log") - plt.ylim(1e-4, 1.) - return h - - -def scatter(*args, **kwargs): - """ - Create a figure and then a scatter plot - """ - create_new_figure(kwargs) - return plt.scatter(*args, **kwargs) - - -def errorbar(*args, **kwargs): - """ - Create a figure and then a scatter plot - """ - create_new_figure(kwargs) - return plt.errorbar(*args, **kwargs) - - -# I will leave this function here so old code does not crash, -# but the user will want to use the one after that -def profile_and_scatter(x, y, z, nbin, *args, **kwargs): - """ - Create a figure and then a scatter plot - """ - create_new_figure(kwargs) - - x, y, z, ze = fitf.profileXY(x, y, z, *nbin, *args, **kwargs) - x_ = np.repeat(x, x.size) - y_ = np.tile (y, y.size) - z_ = z.flatten() - return (x, y, z, ze), plt.scatter(x_, y_, c=z_, marker="s"), plt.colorbar() - - -def hist2d_profile(x, y, z, nbinx, nbiny, xrange, yrange, **kwargs): - """ - Create a profile 2d of the data and plot it as an histogram. - """ - - x, y, z, ze = fitf.profileXY(x, y, z, nbinx, nbiny, xrange, yrange) - plot_output = display_matrix(x, y, z, **kwargs) - return ((x, y, z, ze), *plot_output) - - -def display_matrix(x, y, z, mask=None, **kwargs): - """ - Display the matrix z using the coordinates x and y as the bin centers. - """ - nx = np.size(x) - ny = np.size(y) - - dx = (np.max(x) - np.min(x)) / nx - dy = (np.max(y) - np.min(y)) / ny - - x_binning = np.linspace(np.min(x) - dx, np.max(x) + dx, nx + 1) - y_binning = np.linspace(np.min(y) - dy, np.max(y) + dy, ny + 1) - - x_ = np.repeat(x, ny) - y_ = np.tile (y, nx) - z_ = z.flatten() - - if mask is None: - mask = np.ones_like(z_, dtype=bool) - else: - mask = mask.flatten() - h = hist2d(x_[mask], y_[mask], (x_binning, - y_binning), - weights = z_[mask], - **kwargs) - return h, plt.colorbar() - - -def doublescatter(x1, y1, x2, y2, lbls, *args, **kwargs): - """ - Create a figure and then a scatter plot - """ - create_new_figure(kwargs) - - sc1 = scatter(x1, y1, *args, label=lbls[0], new_figure=False, **kwargs) - sc2 = scatter(x2, y2, *args, label=lbls[1], new_figure=False, **kwargs) - return sc1, sc2, plt.legend() - - -def covariance(x, y, **kwargs): - """ - Display the eigenvectors of the covariance matrix. - """ - create_new_figure(kwargs) - - cov = np.cov(x, y) - l, v = np.linalg.eig(cov) - lx, ly = l**0.5 - vx, vy = v.T - x0, y0 = np.mean(x), np.mean(y) - x1 = lx * vx[0] - y1 = lx * vx[1] - plt.arrow(x0, y0, x1, y1, head_width=0.1*ly, head_length=0.1*lx, fc='r', ec='r') - x1 = ly * vy[0] - y1 = ly * vy[1] - plt.arrow(x0, y0, x1, y1, head_width=0.1*lx, head_length=0.1*ly, fc='r', ec='r') - return l, v - - -def resolution(values, errors = None, E_from=41.5, E_to=2458): - """ - Compute resolution at E_from and resolution at E_to - with uncertainty propagation. - """ - if errors is None: - errors = np.zeros_like(values) - - amp , mu, sigma, *_ = values - u_amp, u_mu, u_sigma, *_ = errors - - r = 235. * sigma/mu - u_r = r * (u_mu**2/mu**2 + u_sigma**2/sigma**2)**0.5 - - scale = (E_from/E_to)**0.5 - return Measurement(r , u_r ), \ - Measurement(r * scale, u_r * scale) - - -def gausstext(values, errors, E_from=41.5, E_to=2458): - """ - Build a string to be displayed within a matplotlib plot. - """ - reso = resolution(values, errors, E_from=E_from, E_to=E_to) - E_to = "Qbb" if E_to==2458 else str(E_to) + " keV" - return textwrap.dedent(""" - $\mu$ = {0} - $\sigma$ = {1} - R = {2} % @ {3} keV - R = {4} % @ {5}""".format(measurement_string(values[1] , errors[1]), - measurement_string(values[2] , errors[2]), - measurement_string(* reso[0]), E_from, - measurement_string(* reso[1]), E_to)) - - -def save_to_folder(outputfolder, name, format="png", dpi=100): - """ - Set title and save plot in folder. - """ - plt.savefig("{}/{}.{}".format(outputfolder, name, format), dpi=dpi) - - -def plot_writer(outputfolder, format, dpi=100): - """ - Build a partial implementation of the save_to_folder function ensuring - the output folder exists. - """ - os.makedirs(outputfolder, exist_ok = True) - - return functools.partial(save_to_folder, - outputfolder, - format = format, - dpi = dpi) - - -def measurement_string(x, u_x): - """ - Display a value-uncertainty pair with the same precision. - """ - scale = int(np.floor(np.log10(u_x))) - if scale >= 2: - return "({}) · 1e{}".format(measurement_string( x/10**scale, - u_x/10**scale), - scale) - n = 1 - scale - - format = "{" + ":.{}f".format(n) + "}" - string = "{} +- {}".format(format, format) - - return string.format(np.round( x, n), - np.round(u_x, n)) diff --git a/invisible_cities/icaro/hst_functions_test.py b/invisible_cities/icaro/hst_functions_test.py deleted file mode 100644 index e673327afe..0000000000 --- a/invisible_cities/icaro/hst_functions_test.py +++ /dev/null @@ -1,20 +0,0 @@ -import numpy as np - -from numpy.testing import assert_equal -from numpy.testing import assert_allclose - -from . import hst_functions as hst - - -def test_resolution_no_errors(): - R, Rbb = hst.resolution([None, 1, 1]) - - assert_equal(R .uncertainty, 0) - assert_equal(Rbb.uncertainty, 0) - - -def test_resolution_scaling(): - _, Rbb1 = hst.resolution([None, 1, 1], E_from = 1) - _, Rbb2 = hst.resolution([None, 1, 1], E_from = 2) - - assert_allclose(Rbb1.value * 2**0.5, Rbb2.value) diff --git a/invisible_cities/icaro/hvt_mpl.py b/invisible_cities/icaro/hvt_mpl.py deleted file mode 100644 index d6569bc46f..0000000000 --- a/invisible_cities/icaro/hvt_mpl.py +++ /dev/null @@ -1,187 +0,0 @@ -"""hvt = Hits Voxels and Tracks plotting functions. - -""" -import numpy as np -import pandas as pd -import tables as tb -import matplotlib.pyplot as plt - -from matplotlib import colors as mcolors -import matplotlib.cm as cm -from mpl_toolkits.mplot3d import Axes3D -from mpl_toolkits.mplot3d.art3d import Poly3DCollection -from matplotlib.pyplot import figure, show - -from . mpl_functions import circles -from . mpl_functions import cube -from . mpl_functions import plt_scatter3d -from . mpl_functions import plt_scatter2d -from . mpl_functions import make_color_map -from . mpl_functions import set_plot_labels -from .. core.system_of_units_c import units -from .. core.core_functions import loc_elem_1d - -from .. database import load_db - -def distance(va, vb): - return np.linalg.norm(va.pos - vb.pos) - -def print_tracks(tc): - for trk_no, trk in enumerate(tc.tracks): - print('trk no = {}, number of voxels = {}, energy = {}'. - format(trk_no, trk.number_of_voxels, trk.E)) - vE = [v.E for v in trk.voxels] - print('voxel energies = {}'.format(vE)) - ba, bb = trk.blobs - print('----------------------------') - print('blob a: number of voxels = {}, seed = {}, energy = {}'. - format(ba.number_of_voxels, ba.seed, ba.E)) - print('blob b: number of voxels = {}, seed = {}, energy = {}'. - format(bb.number_of_voxels, bb.seed, bb.E)) - - blobs_sa = set(ba.voxels) - blobs_sb = set(bb.voxels) - blobs_i = blobs_sa.intersection(blobs_sb) - - print('intersection blobs a and blob b = {}'. - format(len(blobs_i))) - - - print('\n') - -def print_distance_to_track(tc, trkm): - trkm_ba, trkm_bb = trkm.blobs - for trk_no, trk in enumerate(tc.tracks): - print('trk no = {}, number of voxels = {}, energy = {}'. - format(trk_no, trk.number_of_voxels, trk.E)) - ba, bb = trk.blobs - print('----------------------------') - print('blob a: number of voxels = {}, seed = {}, energy = {}'. - format(ba.number_of_voxels, ba.seed, ba.E)) - print('blob b: number of voxels = {}, seed = {}, energy = {}'. - format(bb.number_of_voxels, bb.seed, bb.E)) - - print("""distances to reference track: - a : a = {} a : b = {} - b : a = {} b : b = {} - """.format(distance(trkm_ba.seed, ba.seed), - distance(trkm_ba.seed, bb.seed), - distance(trkm_bb.seed, ba.seed), - distance(trkm_bb.seed, bb.seed))) - - -def get_hits(hits, norm=True): - x, y, z, q = [], [], [], [] - for hit in hits: - x.append(hit.X) - y.append(hit.Y) - z.append(hit.Z) - q.append(hit.E) - if norm: - return np.array(x), np.array(y), np.array(z), np.array(q)/np.amax(q) - else: - return np.array(x), np.array(y), np.array(z), np.array(q) - - -def set_xyz_limit(ax, xview, yview, zview): - ax.set_xlim3d(xview) - ax.set_ylim3d(yview) - ax.set_zlim3d(zview) - - -def set_xy_limit(ax, xview, yview): - ax.set_xlim(xview) - ax.set_ylim(yview) - - -def plot_hits_3D(hits, norm=True, xview=(-198, 198), yview=(-198, 198), zview=(0, 500), - xsc = 10, ysc = 10, zsc = 10, - s = 30, alpha=0.3, figsize=(12, 12)): - - x, y, z, q = get_hits(hits, norm = norm) - - fig = plt.figure(figsize=figsize) - ax1 = fig.add_subplot(221, projection='3d') - set_xyz_limit(ax1, xview, yview, zview) - - ax2 = fig.add_subplot(222, projection='3d') - set_xyz_limit(ax2, xview, yview, (np.amin(z) - zsc, np.amax(z) + zsc)) - - ax3 = fig.add_subplot(223) - set_xy_limit(ax3, xview, yview) - - ax4 = fig.add_subplot(224) - set_xy_limit(ax4, (np.amin(x) -xsc, np.amax(x) + xsc), - (np.amin(y) -ysc, np.amax(y) + ysc)) - - plt_scatter3d(ax1, x, y, z, q, s, alpha) - plt_scatter3d(ax2, x, y, z, q, s, alpha) - plt_scatter2d(ax3, x, y, q, s, alpha) - plt_scatter2d(ax4, x, y, q, s, alpha) - -def make_cube(v, voxel_size, col, axes): - quads = cube(origin=(v.X, v.Y, v.Z), - width=voxel_size.X, - height=voxel_size.Y, - depth=voxel_size.Z) - collection = Poly3DCollection(quads) - collection.set_color(col) - axes.add_collection3d(collection) - -def set_color_map(fig, axes, voxels, alpha, colormap='Blues'): - scplot = axes.scatter([], [], c=[]) - fig.colorbar(scplot, ax=axes) - voxel_energy = sorted([v.E for v in voxels]) - RGBA, _ = make_color_map(voxel_energy, alpha=alpha, colormap=colormap) - return RGBA, voxel_energy - - -def draw_voxels(voxels, voxel_size, xview=(-198, 198), yview=(-198, 198), zview=(0, 500), - figsize=(10, 8), alpha=0.5, colormap='Blues'): - - canvas = figure() - canvas.set_size_inches(figsize) - axes = Axes3D(canvas) - - set_xyz_limit(axes, xview, yview, zview) - RGBA, voxel_energy = set_color_map(canvas, axes, voxels, alpha, colormap=colormap) - #print(RGBA) - for v in voxels: - c_index = loc_elem_1d(voxel_energy, v.E) - make_cube(v, voxel_size, RGBA[c_index], axes) - show() - - -def draw_voxels2(voxels, voxel_size, xview=(-198, 198), yview=(-198, 198), zview=(0, 500), - alpha=0.5, figsize=(10, 8), colormap='Blues'): - - fig = plt.figure(figsize=figsize) - axes = fig.add_subplot(111, projection='3d', aspect='equal') - - set_xyz_limit(axes, xview, yview, zview) - RGBA, voxel_energy = set_color_map(canvas, axes, voxels, alpha, colormap=colormap) - - for v in voxels: - c_index = loc_elem_1d(RGBA, v.E) - make_cube(v, voxel_size, RGBA[c_index], axes) - plt.show() - -def draw_tracks(track_container, voxel_size, - xview=(-198, 198), yview=(-198, 198), zview=(0, 500), - alpha=0.5, figsize=(10, 8)): - - fig = plt.figure(figsize=figsize) - axes = fig.add_subplot(111, projection='3d', aspect='equal') - set_xyz_limit(axes, xview, yview, zview) - RGBA = [(1.0, 0.0, 0.0, 0.5), (0.0, 0.0, 1.0, 0.5) ] - - for trk in track_container.tracks: - ba, bb = trk.blobs - for v in trk.voxels: - make_cube(v, voxel_size, RGBA[0], axes) - for v in ba.voxels: - make_cube(v, voxel_size, RGBA[1], axes) - for v in bb.voxels: - make_cube(v, voxel_size, RGBA[1], axes) - - plt.show() diff --git a/invisible_cities/icaro/mpl_functions.py b/invisible_cities/icaro/mpl_functions.py deleted file mode 100644 index fb19ee8db7..0000000000 --- a/invisible_cities/icaro/mpl_functions.py +++ /dev/null @@ -1,487 +0,0 @@ -"""A utility module for plots with matplotlib""" - -import numpy as np -import matplotlib.pyplot as plt -import matplotlib.animation -from matplotlib.patches import Circle -from matplotlib.collections import PatchCollection -from matplotlib.offsetbox import (TextArea, DrawingArea, OffsetImage, - AnnotationBbox) - - -from matplotlib import colors as mcolors -import matplotlib.cm as cm -from mpl_toolkits.mplot3d import Axes3D -from mpl_toolkits.mplot3d.art3d import Poly3DCollection -from matplotlib.pyplot import figure, show - -from .. core.system_of_units_c import units -from .. core.core_functions import define_window -from .. database import load_db - - -# matplotlib.style.use("ggplot") -#matplotlib.rc('animation', html='html5') - -# histograms, signals and shortcuts - - -def plot_vector(v, figsize=(10,10)): - """Plot vector v and return figure """ - plt.figure(figsize=(10, 10)) - ax1 = plt.subplot(1, 1, 1) - plt.plot(v) - - -def fplot_xy(x, y, figsize=(10,10)): - """Plot y vs x and return figure""" - fig = Figure(figsize=figsize) - ax1 = fig.add_subplot(1, 1, 1) - ax1.plot(x,y) - return fig - - -def plot_xy(x, y, figsize=(10,10)): - """Plot y vs x and return figure""" - plt.figure(figsize=figsize) - ax1 = plt.subplot(1, 1, 1) - plt.plot(x,y) - - -def plot_sipm_list(sipmrwf, sipm_list, x=4): - """Plot a list of SiPMs.""" - plt.figure(figsize=(12, 12)) - nmax = len(sipm_list) - y = int(nmax / x) + 1 - for i in range(0, nmax): - plt.subplot(x, y, i+1) - plt.plot(sipmrwf[sipm_list[i]]) - - -def plot_sensor_list_ene_map(run_number, swf, slist, stype='PMT', cmap='Blues'): - """Plot a map of the energies of sensors in list.""" - DataSensor = load_db.DataPMT(run_number) - radius = 10 - if stype == 'SiPM': - DataSensor = load_db.DataSiPM(run_number) - radius = 2 - xs = DataSensor.X.values - ys = DataSensor.Y.values - r = np.ones(len(xs)) * radius - col = np.zeros(len(xs)) - for i in slist: - col[i] = np.sum(swf[i]) - - plt.figure(figsize=(8, 8)) - plt.subplot(aspect="equal") - circles(xs, ys, r, c=col, alpha=0.5, ec="none", cmap=cmap) - plt.colorbar() - - plt.xlim(-198, 198) - plt.ylim(-198, 198) - - -def draw_pmt_map(run_number): - """Draws a map with the channel_id number in the positions of the PMTs. - channel_id = elecID (electronic ID) for PMTs. - xpmt = x pos - """ - DataPMT = load_db.DataPMT(run_number) - xpmt = DataPMT.X.values - ypmt = DataPMT.Y.values - channel_id = DataPMT.ChannelID.values - cid = ['{}'.format(c) for c in channel_id] - fig, ax = plt.subplots() - plt.plot(xpmt, ypmt, 'o') - - for c, x,y in zip(cid, xpmt,ypmt): - xy = (x,y) - offsetbox = TextArea(c, minimumdescent=False) - ab = AnnotationBbox(offsetbox, xy) - ax.add_artist(ab) - - plt.show() - - -def plt_scatter3d(ax, x, y, z, q, s = 30, alpha=0.3): - ax.scatter(x, y, z, c=q, s=s, alpha=alpha) - ax.set_xlabel("x (mm)") - ax.set_ylabel("y (mm)") - ax.set_zlabel("z (mm)") - - -def plt_scatter2d(ax, x, y, q, s = 30, alpha=0.3): - ax.scatter(x, y, c=q, s=s, alpha=alpha) - ax.set_xlabel("x (mm)") - ax.set_ylabel("y (mm)") - - -def make_tracking_plane_movie(slices, thrs=0.1): - """Create a video made of consecutive frames showing the response of - the tracking plane. - - Parameters - ---------- - slices : 2-dim np.ndarray - The signal of each SiPM (axis 1) for each time sample (axis 0). - thrs : float, optional - Default cut value to be applied to each slice. Defaults to 0.1. - - Returns - ------- - mov : matplotlib.animation - The movie. - """ - - fig, ax = plt.subplots() - fig.set_size_inches(10, 8) - DataSensor = load_db.DataSiPM(0) - X = DataSensor.X.values - Y = DataSensor.Y.values - - xmin, xmax = np.nanmin(X), np.nanmax(X) - ymin, ymax = np.nanmin(Y), np.nanmax(Y) - def init(): - global cbar, scplot - ax.set_xlabel("x (mm)") - ax.set_ylabel("y (mm)") - ax.set_xlim((xmin, xmax)) - ax.set_ylim((ymin, ymax)) - scplot = ax.scatter([], [], c=[]) - cbar = fig.colorbar(scplot, ax=ax) - cbar.set_label("Charge (pes)") - return (scplot,) - - def animate(i): - global cbar, scplot - slice_ = slices[i] - selection = slice_ > np.nanmax(slice_) * thrs - x, y, q = X[selection], Y[selection], slice_[selection] - cbar.remove() - fig.clear() - ax = plt.gca() - ax.set_xlabel("x (mm)") - ax.set_ylabel("y (mm)") - ax.set_xlim((xmin, xmax)) - ax.set_ylim((ymin, ymax)) - scplot = ax.scatter(x, y, c=q, marker="s", vmin=0, - vmax=np.nanmax(slices)) - cbar = fig.colorbar(scplot, ax=ax, - boundaries=np.linspace(0, np.nanmax(slices), 100)) - cbar.set_label("Charge (pes)") - return (scplot,) - - anim = matplotlib.animation.FuncAnimation(fig, animate, init_func=init, - frames=len(slices), interval=200, - blit=False) - return anim - - -def set_plot_labels(xlabel="", ylabel="", grid=True): - """Short cut to set labels in plots.""" - plt.xlabel(xlabel) - plt.ylabel(ylabel) - if grid is True: - plt.grid(which="both", axis="both") - - -# Circles! -def circles(x, y, s, c="a", vmin=None, vmax=None, **kwargs): - """Make a scatter of circles plot of x vs y, where x and y are - sequence like objects of the same lengths. The size of circles are - in data scale. - - Parameters - ---------- - x,y : scalar or array_like, shape (n, ) - Input data - s : scalar or array_like, shape (n, ) - Radius of circle in data unit. - c : color or sequence of color, optional, default : 'b' - `c` can be a single color format string, or a sequence of color - specifications of length `N`, or a sequence of `N` numbers to be - mapped to colors using the `cmap` and `norm` specified via kwargs. - Note that `c` should not be a single numeric RGB or RGBA sequence - because that is indistinguishable from an array of values - to be colormapped. (If you insist, use `color` instead.) - `c` can be a 2-D array in which the rows are RGB or RGBA, however. - vmin, vmax : scalar, optional, default: None - `vmin` and `vmax` are used in conjunction with `norm` to normalize - luminance data. If either are `None`, the min and max of the - color array is used. - kwargs : `~matplotlib.collections.Collection` properties - Eg. alpha, edgecolor(ec), facecolor(fc), linewidth(lw), linestyle(ls), - norm, cmap, transform, etc. - - Returns - ------- - paths : `~matplotlib.collections.PathCollection` - - Examples - -------- - a = np.arange(11) - circles(a, a, a*0.2, c=a, alpha=0.5, edgecolor='none') - plt.colorbar() - - License - -------- - This code is under [The BSD 3-Clause License] - (http://opensource.org/licenses/BSD-3-Clause) - - """ - - if np.isscalar(c): - kwargs.setdefault("color", c) - c = None - if "fc" in kwargs: - kwargs.setdefault("facecolor", kwargs.pop("fc")) - if "ec" in kwargs: - kwargs.setdefault("edgecolor", kwargs.pop("ec")) - if "ls" in kwargs: - kwargs.setdefault("linestyle", kwargs.pop("ls")) - if "lw" in kwargs: - kwargs.setdefault("linewidth", kwargs.pop("lw")) - - patches = [Circle((x_, y_), s_) for x_, y_, s_ in np.broadcast(x, y, s)] - collection = PatchCollection(patches, **kwargs) - if c is not None: - collection.set_array(np.asarray(c)) - collection.set_clim(vmin, vmax) - - ax = plt.gca() - ax.add_collection(collection) - ax.autoscale_view() - if c is not None: - plt.sci(collection) - return collection - -""" -Defines geometry plotting utilities objects: -- :func:`quad` -- :func:`grid` -- :func:`cube` -""" - - -__all__ = ['quad', 'grid', 'cube'] - - -def quad(plane='xy', origin=None, width=1, height=1, depth=0): - """ - Returns the vertices of a quad geometric element in counter-clockwise - order. - Parameters - ---------- - plane : array_like, optional - **{'xy', 'xz', 'yz'}**, - Construction plane of the quad. - origin: array_like, optional - Quad origin on the construction plane. - width: numeric, optional - Quad width. - height: numeric, optional - Quad height. - depth: numeric, optional - Quad depth. - Returns - ------- - ndarray - Quad vertices. - Examples - -------- - >>> quad() - array([[0, 0, 0], - [1, 0, 0], - [1, 1, 0], - [0, 1, 0]]) - """ - - u, v = (0, 0) if origin is None else origin - - plane = plane.lower() - if plane == 'xy': - vertices = ((u, v, depth), (u + width, v, depth), - (u + width, v + height, depth), (u, v + height, depth)) - elif plane == 'xz': - vertices = ((u, depth, v), (u + width, depth, v), - (u + width, depth, v + height), (u, depth, v + height)) - elif plane == 'yz': - vertices = ((depth, u, v), (depth, u + width, v), - (depth, u + width, v + height), (depth, u, v + height)) - else: - raise ValueError('"{0}" is not a supported plane!'.format(plane)) - - return np.array(vertices) - - -def grid(plane='xy', - origin=None, - width=1, - height=1, - depth=0, - width_segments=1, - height_segments=1): - """ - Returns the vertices of a grid made of quads. - Parameters - ---------- - plane : array_like, optional - **{'xy', 'xz', 'yz'}**, - Construction plane of the grid. - origin: array_like, optional - Grid origin on the construction plane. - width: numeric, optional - Grid width. - height: numeric, optional - Grid height. - depth: numeric, optional - Grid depth. - width_segments: int, optional - Grid segments, quad counts along the width. - height_segments: int, optional - Grid segments, quad counts along the height. - Returns - ------- - ndarray - Grid vertices. - Examples - -------- - >>> grid(width_segments=2, height_segments=2) - array([[[ 0. , 0. , 0. ], - [ 0.5, 0. , 0. ], - [ 0.5, 0.5, 0. ], - [ 0. , 0.5, 0. ]], - - [[ 0. , 0.5, 0. ], - [ 0.5, 0.5, 0. ], - [ 0.5, 1. , 0. ], - [ 0. , 1. , 0. ]], - - [[ 0.5, 0. , 0. ], - [ 1. , 0. , 0. ], - [ 1. , 0.5, 0. ], - [ 0.5, 0.5, 0. ]], - - [[ 0.5, 0.5, 0. ], - [ 1. , 0.5, 0. ], - [ 1. , 1. , 0. ], - [ 0.5, 1. , 0. ]]]) - """ - - u, v = (0, 0) if origin is None else origin - - w_x, h_y = width / width_segments, height / height_segments - - quads = [] - for i in range(width_segments): - for j in range(height_segments): - quads.append( - quad(plane, (i * w_x + u, j * h_y + v), w_x, h_y, depth)) - - return np.array(quads) - - -def cube(plane=None, - origin=None, - width=1, - height=1, - depth=1, - width_segments=1, - height_segments=1, - depth_segments=1): - """ - Returns the vertices of a cube made of grids. - Parameters - ---------- - plane : array_like, optional - Any combination of **{'+x', '-x', '+y', '-y', '+z', '-z'}**, - Included grids in the cube construction. - origin: array_like, optional - Cube origin. - width: numeric, optional - Cube width. - height: numeric, optional - Cube height. - depth: numeric, optional - Cube depth. - width_segments: int, optional - Cube segments, quad counts along the width. - height_segments: int, optional - Cube segments, quad counts along the height. - depth_segments: int, optional - Cube segments, quad counts along the depth. - Returns - ------- - ndarray - Cube vertices. - Examples - -------- - >>> cube() - array([[[ 0., 0., 0.], - [ 1., 0., 0.], - [ 1., 1., 0.], - [ 0., 1., 0.]], - - [[ 0., 0., 1.], - [ 1., 0., 1.], - [ 1., 1., 1.], - [ 0., 1., 1.]], - - [[ 0., 0., 0.], - [ 1., 0., 0.], - [ 1., 0., 1.], - [ 0., 0., 1.]], - - [[ 0., 1., 0.], - [ 1., 1., 0.], - [ 1., 1., 1.], - [ 0., 1., 1.]], - - [[ 0., 0., 0.], - [ 0., 1., 0.], - [ 0., 1., 1.], - [ 0., 0., 1.]], - - [[ 1., 0., 0.], - [ 1., 1., 0.], - [ 1., 1., 1.], - [ 1., 0., 1.]]]) - """ - - plane = (('+x', '-x', '+y', '-y', '+z', '-z') - if plane is None else [p.lower() for p in plane]) - u, v, w = (0, 0, 0) if origin is None else origin - - w_s, h_s, d_s = width_segments, height_segments, depth_segments - - grids = [] - if '-z' in plane: - grids.extend(grid('xy', (u, w), width, depth, v, w_s, d_s)) - if '+z' in plane: - grids.extend(grid('xy', (u, w), width, depth, v + height, w_s, d_s)) - - if '-y' in plane: - grids.extend(grid('xz', (u, v), width, height, w, w_s, h_s)) - if '+y' in plane: - grids.extend(grid('xz', (u, v), width, height, w + depth, w_s, h_s)) - - if '-x' in plane: - grids.extend(grid('yz', (w, v), depth, height, u, d_s, h_s)) - if '+x' in plane: - grids.extend(grid('yz', (w, v), depth, height, u + width, d_s, h_s)) - - return np.array(grids) - -def make_color_map(lst, alpha=0.5, colormap='inferno'): - """lst is the sequence to map to colors""" - minima = min(lst) - maxima = max(lst) - - norm = mcolors.Normalize(vmin=minima, vmax=maxima, clip=True) - mapper = cm.ScalarMappable(norm=norm, cmap=colormap) - - RGBA = [] - for v in lst: - RGBA.append(mapper.to_rgba(v, alpha)) - - return RGBA, mapper diff --git a/invisible_cities/icaro/pmaps_mpl.py b/invisible_cities/icaro/pmaps_mpl.py deleted file mode 100644 index 19b595929d..0000000000 --- a/invisible_cities/icaro/pmaps_mpl.py +++ /dev/null @@ -1,61 +0,0 @@ -"""PMAPS plotting functions. - -""" -import numpy as np -import pandas as pd -import tables as tb -import matplotlib.pyplot as plt - -from . mpl_functions import circles -from . mpl_functions import set_plot_labels -from .. core.system_of_units_c import units - -from .. database import load_db - - - -def plot_s12(s12, figsize=(6,6)): - """Plot the peaks of input S12. - - Uses the s12 interface defined in event model - """ - plt.figure(figsize=figsize) - - set_plot_labels(xlabel = "t (mus)", - ylabel = "S12 (pes)") - xy = s12.number_of_peaks - if xy == 1: - wfm = s12.peak_waveform(0) - ax1 = plt.subplot(1, 1, 1) - ax1.plot(wfm.t/units.mus, wfm.E) - else: - x = 3 - y = xy/x - if y % xy != 0: - y = int(xy/x) + 1 - for i in range(xy): - ax1 = plt.subplot(x, y, i+1) - wfm = s12.peak_waveform(i) - plt.plot(wfm.t/units.mus, wfm.E) - - -def plot_s2si_map(s2si, run_number=0, vmin=None, vmax=None, cmap='Blues'): - """Plot a map of the energies of S2Si objects.""" - - DataSensor = load_db.DataSiPM(run_number) - radius = 2 - xs = DataSensor.X.values - ys = DataSensor.Y.values - r = np.ones(len(xs)) * radius - col = np.zeros(len(xs)) - for peak_no in range(s2si.number_of_peaks): - for sipm_no in s2si.sipms_in_peak(peak_no): - col[sipm_no] += s2si.sipm_total_energy(peak_no, sipm_no) - - plt.figure(figsize=(8, 8)) - plt.subplot(aspect="equal") - circles(xs, ys, r, c=col, vmin=vmin, vmax=vmax, alpha=0.5, ec="none", cmap=cmap) - plt.colorbar() - - plt.xlim(-198, 198) - plt.ylim(-198, 198) diff --git a/invisible_cities/icaro/signal_functions_mpl.py b/invisible_cities/icaro/signal_functions_mpl.py deleted file mode 100644 index 2ea1290587..0000000000 --- a/invisible_cities/icaro/signal_functions_mpl.py +++ /dev/null @@ -1,155 +0,0 @@ -"""A utility module for plots with matplotlib""" - -import numpy as np -import matplotlib.pyplot as plt -from matplotlib.pyplot import figure, show - -from .. core.system_of_units_c import units -from .. core.core_functions import define_window -from .. database import load_db -from . mpl_functions import set_plot_labels - -# matplotlib.style.use("ggplot") -#matplotlib.rc('animation', html='html5') - -# histograms, signals and shortcuts - - -def plts(signal, signal_start=0, signal_end=1e+4, offset=5): - """Plot a signal in a give interval, control offset by hand.""" - ax1 = plt.subplot(1, 1, 1) - ymin = np.amin(signal[signal_start:signal_end]) - offset - ymax = np.amax(signal[signal_start:signal_end]) + offset - ax1.set_xlim([signal_start, signal_end]) - ax1.set_ylim([ymin, ymax]) - plt.plot(signal) - - -def plot_signal(signal_t, signal, title="signal", - signal_start=0, signal_end=1e+4, - ymax=200, t_units="", units=""): - """Given a series signal (t, signal), plot the signal.""" - - ax1 = plt.subplot(1, 1, 1) - ax1.set_xlim([signal_start, signal_end]) - ax1.set_ylim([0, ymax]) - set_plot_labels(xlabel="t ({})".format(t_units), - ylabel="signal ({})".format(units)) - plt.title(title) - plt.plot(signal_t, signal) - # plt.show() - - -def plot_signal_vs_time_mus(signal, - t_min = 0, - t_max = 1200, - signal_min = 0, - signal_max = 200, - figsize=(6,6)): - """Plot signal versus time in mus (tmin, tmax in mus). """ - plt.figure(figsize=figsize) - tstep = 25 # in ns - PMTWL = signal.shape[0] - signal_t = np.arange(0., PMTWL * tstep, tstep)/units.mus - ax1 = plt.subplot(1, 1, 1) - ax1.set_xlim([t_min, t_max]) - ax1.set_ylim([signal_min, signal_max]) - set_plot_labels(xlabel = "t (mus)", - ylabel = "signal (pes/adc)") - plt.plot(signal_t, signal) - - -def plot_waveform(pmtwf, zoom=False, window_size=800): - """Take as input a vector a single waveform and plot it""" - - first, last = 0, len(pmtwf) - if zoom: - first, last = define_window(pmtwf, window_size) - - mpl.set_plot_labels(xlabel="samples", ylabel="adc") - plt.plot(pmtwf[first:last]) - - -def plot_waveforms_overlap(wfs, zoom=False, window_size=800): - """Draw all waveforms together. If zoom is True, plot is zoomed - around peak. - """ - first, last = 0, wfs.shape[1] - if zoom: - first, last = define_window(wfs[0], window_size) - for wf in wfs: - plt.plot(wf[first:last]) - - -def plot_wfa_wfb(wfa, wfb, zoom=False, window_size=800): - """Plot together wfa and wfb, where wfa and wfb can be - RWF, CWF, BLR. - """ - plt.figure(figsize=(12, 12)) - for i in range(len(wfa)): - first, last = 0, len(wfa[i]) - if zoom: - first, last = define_window(wfa[i], window_size) - plt.subplot(3, 4, i+1) - # ax1.set_xlim([0, len_pmt]) - set_plot_labels(xlabel="samples", ylabel="adc") - plt.plot(wfa[i][first:last], label= 'WFA') - plt.plot(wfb[i][first:last], label= 'WFB') - legend = plt.legend(loc='upper right') - for label in legend.get_texts(): - label.set_fontsize('small') - - -def plot_pmt_waveforms(pmtwfdf, zoom=False, window_size=800, figsize=(10,10)): - """plot PMT wf and return figure""" - plt.figure(figsize=figsize) - for i in range(len(pmtwfdf)): - first, last = 0, len(pmtwfdf[i]) - if zoom: - first, last = define_window(pmtwfdf[i], window_size) - - ax = plt.subplot(3, 4, i+1) - set_plot_labels(xlabel="samples", ylabel="adc") - plt.plot(pmtwfdf[i][first:last]) - - -def plot_pmt_signals_vs_time_mus(pmt_signals, - pmt_active, - t_min = 0, - t_max = 1200, - signal_min = 0, - signal_max = 200, - figsize=(10,10)): - """Plot PMT signals versus time in mus and return figure.""" - - tstep = 25 - PMTWL = pmt_signals[0].shape[0] - signal_t = np.arange(0., PMTWL * tstep, tstep)/units.mus - plt.figure(figsize=figsize) - - for j, i in enumerate(pmt_active): - ax1 = plt.subplot(3, 4, j+1) - ax1.set_xlim([t_min, t_max]) - ax1.set_ylim([signal_min, signal_max]) - set_plot_labels(xlabel = "t (mus)", - ylabel = "signal (pes/adc)") - - plt.plot(signal_t, pmt_signals[i]) - - -def plot_calibrated_sum_in_mus(CSUM, - tmin=0, tmax=1200, - signal_min=-5, signal_max=200, - csum=True, csum_mau=False): - """Plots calibrated sums in mus (notice units)""" - - if csum: - plot_signal_vs_time_mus(CSUM.csum, - t_min=tmin, t_max=tmax, - signal_min=signal_min, signal_max=signal_max, - label='CSUM') - if csum_mau: - plot_signal_vs_time_mus(CSUM.csum_mau, - t_min=tmin, t_max=tmax, - signal_min=signal_min, signal_max=signal_max, - label='CSUM_MAU') diff --git a/invisible_cities/io/dst_io.py b/invisible_cities/io/dst_io.py index 23ce1d5583..51ccc8c50b 100644 --- a/invisible_cities/io/dst_io.py +++ b/invisible_cities/io/dst_io.py @@ -4,19 +4,32 @@ import warnings -from tables import NoSuchNodeError -from tables import HDF5ExtError -from .. core.exceptions import TableMismatch -from . table_io import make_table -from typing import Optional +from tables import NoSuchNodeError +from tables import HDF5ExtError +from .. core.exceptions import TableMismatch +from . table_io import make_table -def load_dst(filename, group, node): +from typing import Optional +from typing import Sequence + + +def _decode_str_columns(df): + to_string = lambda x: x.astype(bytes).str.decode('utf-8') if x.dtype == np.object else x + df = df.apply(to_string) + return df + +def load_dst(filename, group, node, evt_list=None): """load a kdst if filename, group and node correctly found""" try: with tb.open_file(filename) as h5in: try: - table = getattr(getattr(h5in.root, group), node).read() - return pd.DataFrame.from_records(table) + table = getattr(getattr(h5in.root, group), node) + if evt_list is None: + values = table.read() + else: + events = table.read(field='event') + values = table.read_coordinates(np.where(np.isin(events, evt_list))) + return _decode_str_columns(pd.DataFrame.from_records(values, columns=table.colnames)) except NoSuchNodeError: warnings.warn(f' not of kdst type: file= {filename} ', UserWarning) except HDF5ExtError: @@ -25,13 +38,13 @@ def load_dst(filename, group, node): warnings.warn(f' does not exist: file = {filename} ', UserWarning) -def load_dsts(dst_list, group, node): - dsts = [load_dst(filename, group, node) for filename in dst_list] +def load_dsts(dst_list, group, node, evt_list=None): + dsts = [load_dst(filename, group, node, evt_list) for filename in dst_list] return pd.concat(dsts, ignore_index=True) -def _make_tabledef(column_types : pd.Series, str_col_length : int=32) -> dict: +def _make_tabledef(column_types : np.dtype, str_col_length : int) -> dict: tabledef = {} - for indx, colname in enumerate(column_types.index): + for indx, colname in enumerate(column_types.names): coltype = column_types[colname].name if coltype == 'object': tabledef[colname] = tb.StringCol(str_col_length, pos=indx) @@ -39,28 +52,74 @@ def _make_tabledef(column_types : pd.Series, str_col_length : int=32) -> dict: tabledef[colname] = tb.Col.from_type(coltype, pos=indx) return tabledef -def _store_pandas_as_tables(h5out : tb.file.File, df : pd.DataFrame, group_name : str, table_name : str, compression : str='ZLIB4', descriptive_string : [str]="", str_col_length : int=32) -> None: - if len(df) == 0: - warnings.warn(f'dataframe is empty', UserWarning) +def _check_castability(arr : np.ndarray, table_types : np.dtype): + arr_types = arr.dtype + + if set(arr_types.names) != set(table_types.names): + raise TableMismatch(f'dataframe differs from already existing table structure') + + for name in arr_types.names: + if arr_types[name].name == 'object': + max_str_length = max(map(len, arr[name])) + if max_str_length > table_types[name].itemsize: + warnings.warn(f'dataframe contains strings longer than allowed', UserWarning) + + elif not np.can_cast(arr_types[name], table_types[name], casting='same_kind'): + raise TableMismatch(f'dataframe numeric types not consistent with the table existing ones') + + +def df_writer(h5out : tb.file.File , + df : pd.DataFrame , + group_name : str , + table_name : str , + compression : str = 'ZLIB4', + descriptive_string : str = "" , + str_col_length : int = 32 , + columns_to_index : Optional[Sequence[str]] = None + ) -> None: + """ The function writes a dataframe to open pytables file. + Parameters: + h5out : open pytable file for writing + df : DataFrame to be written + group_name : group name where table is to be saved) + (group is created if doesnt exist) + table_name : table name + (table is created if doesnt exist) + compression : compression type + descriptive_string : table description + str_col_length : maximum length in characters of strings + columns_to_index : list of columns to be flagged for indexing + """ if group_name not in h5out.root: group = h5out.create_group(h5out.root, group_name) - group = getattr(h5out.root, group_name) + + arr = df.to_records(index=False) + if table_name not in group: - tabledef = _make_tabledef(df.dtypes) + tabledef = _make_tabledef(arr.dtype, str_col_length=str_col_length) table = make_table(h5out, group = group_name, name = table_name, fformat = tabledef, description = descriptive_string, compression = compression) + else: + table = getattr(group, table_name) - table = getattr(group, table_name) - if not np.array_equal(df.columns, table.colnames): - raise TableMismatch(f'dataframe differs from already existing table structure') - for indx in df.index: - tablerow = table.row - for colname in table.colnames: - tablerow[colname] = df.at[indx, colname] - tablerow.append() - table.flush() + data_types = table.dtype + if len(arr) == 0: + warnings.warn(f'dataframe is empty', UserWarning) + else: + _check_castability(arr, data_types) + columns = list(data_types.names) + arr = arr[columns].astype(data_types) + table.append(arr) + table.flush() + + if columns_to_index is not None: + if set(columns_to_index).issubset(set(df.columns)): + table.set_attr('columns_to_index', columns_to_index) + else: + not_found = list(set(columns_to_index).difference(set(df.columns))) + raise KeyError(f'columns {not_found} not present in the dataframe') diff --git a/invisible_cities/io/dst_io_test.py b/invisible_cities/io/dst_io_test.py index db9cb96d98..ada9eb252e 100644 --- a/invisible_cities/io/dst_io_test.py +++ b/invisible_cities/io/dst_io_test.py @@ -1,24 +1,23 @@ import os import string +import pytest +import re + import pandas as pd import tables as tb -import numpy as np + from ..core.testing_utils import assert_dataframes_close from ..core.testing_utils import assert_dataframes_equal from ..core.exceptions import TableMismatch from . dst_io import load_dst from . dst_io import load_dsts -from . dst_io import _store_pandas_as_tables +from . dst_io import df_writer from . dst_io import _make_tabledef -import warnings -import pytest from pytest import raises from pytest import fixture -from numpy .testing import assert_raises from hypothesis import given -from hypothesis.extra.pandas import columns from hypothesis.extra.pandas import data_frames from hypothesis.extra.pandas import column from hypothesis.extra.pandas import range_indexes @@ -34,14 +33,19 @@ def empty_dataframe(columns=['int_value', 'float_value', 'bool_value', 'str_valu return df -dataframe = data_frames(index=range_indexes(min_size=1, max_size=5), columns=[column('int_value' , dtype = int ), - column('float_val' , dtype = float ), - column('bool_value', dtype = bool )]) +@fixture() +def fixed_dataframe(): + return pd.DataFrame({'int':[0, 1], 'float':[10., 20.], 'string':['aa', 'bb']}) + + +dataframe = data_frames(index=range_indexes(min_size=1, max_size=5), columns=[column('int_value' , dtype = int ), + column('float_val' , dtype = float ), + column('bool_value', dtype = bool )]) -dataframe_diff = data_frames(index=range_indexes(min_size=1, max_size=5), columns=[column('int_value' , dtype = int ), +dataframe_diff = data_frames(index=range_indexes(min_size=1, max_size=5), columns=[column('int_value' , dtype = int ), column('float_val' , dtype = float)]) -strings_dataframe = data_frames(index=range_indexes(min_size=1, max_size=5), columns=[column('str_val', elements=text(alphabet=string.ascii_letters, max_size=32))]) +strings_dataframe = data_frames(index=range_indexes(min_size=1, max_size=5), columns=[column('str_val', elements=text(alphabet=string.ascii_letters, min_size=10, max_size=32))]) def test_load_dst(KrMC_kdst): @@ -50,6 +54,15 @@ def test_load_dst(KrMC_kdst): False , rtol=1e-5) +def test_load_dst_event_list(KrMC_kdst): + event_list = [0, 4, 7] + df_read = load_dst(*KrMC_kdst[0].file_info, evt_list=event_list) + df_check = KrMC_kdst[0].true + df_check = df_check[df_check.event.isin(event_list)].reset_index(drop=True) + assert_dataframes_close(df_read, df_check, + False , rtol=1e-5) + + def test_load_dsts_single_file(KrMC_kdst): tbl = KrMC_kdst[0].file_info df_read = load_dsts([tbl.filename], tbl.group, tbl.node) @@ -58,6 +71,17 @@ def test_load_dsts_single_file(KrMC_kdst): False , rtol=1e-5) +def test_load_dsts_single_file_event_list(KrMC_kdst): + event_list = [0, 4, 7] + tbl = KrMC_kdst[0].file_info + df_read = load_dsts([tbl.filename], tbl.group, tbl.node, evt_list=event_list) + df_check = KrMC_kdst[0].true + df_check = df_check[df_check.event.isin(event_list)].reset_index(drop=True) + + assert_dataframes_close(df_read, df_check, + False , rtol=1e-5) + + def test_load_dsts_double_file(KrMC_kdst): tbl = KrMC_kdst[0].file_info df_true = KrMC_kdst[0].true @@ -69,6 +93,7 @@ def test_load_dsts_double_file(KrMC_kdst): #assert index number unique (important for saving it to pytable) assert all(~df_read.index.duplicated()) + def test_load_dsts_reads_good_kdst(ICDATADIR): good_file = "Kr83_nexus_v5_03_00_ACTIVE_7bar_10evts_KDST.h5" good_file = os.path.join(ICDATADIR, good_file) @@ -76,6 +101,7 @@ def test_load_dsts_reads_good_kdst(ICDATADIR): node = "Events" load_dsts([good_file], group, node) + def test_load_dsts_warns_not_of_kdst_type(ICDATADIR): good_file = "Kr83_nexus_v5_03_00_ACTIVE_7bar_10evts_KDST.h5" good_file = os.path.join(ICDATADIR, good_file) @@ -87,6 +113,7 @@ def test_load_dsts_warns_not_of_kdst_type(ICDATADIR): with pytest.warns(UserWarning, match='not of kdst type'): load_dsts([good_file, wrong_file], group, node) + def test_load_dsts_warns_if_not_existing_file(ICDATADIR): good_file = "Kr83_nexus_v5_03_00_ACTIVE_7bar_10evts_KDST.h5" good_file = os.path.join(ICDATADIR, good_file) @@ -98,61 +125,140 @@ def test_load_dsts_warns_if_not_existing_file(ICDATADIR): with pytest.warns(UserWarning, match='does not exist'): load_dsts([good_file, wrong_file], group, node) + +def test_load_dst_converts_from_bytes(ICDATADIR, fixed_dataframe): + filename = os.path.join(ICDATADIR, 'Kr83_full_nexus_v5_03_01_ACTIVE_7bar_1evt.sim.h5') + config = load_dst(filename, 'MC', 'configuration') + for column in config.columns: + assert all(config[column].apply(type) != bytes) + + @given(df=dataframe) -def test_store_pandas_as_tables_exact(config_tmpdir, df): +def test_df_writer_exact(config_tmpdir, df): filename = config_tmpdir + 'dataframe_to_table_exact.h5' group_name = 'test_group' table_name = 'table_name_1' with tb.open_file(filename, 'w') as h5out: - _store_pandas_as_tables(h5out, df, group_name, table_name) + df_writer(h5out, df, group_name, table_name) df_read = load_dst(filename, group_name, table_name) - assert_dataframes_close(df_read, df, False, rtol=1e-5) + assert_dataframes_close(df_read, df) + @given(df1=dataframe, df2=dataframe) -def test_store_pandas_as_tables_2df(config_tmpdir, df1, df2): +def test_df_writer_2df(config_tmpdir, df1, df2): filename = config_tmpdir + 'dataframe_to_table_exact.h5' group_name = 'test_group' table_name = 'table_name_2' with tb.open_file(filename, 'w') as h5out: - _store_pandas_as_tables(h5out, df1, group_name, table_name) - _store_pandas_as_tables(h5out, df2, group_name, table_name) + df_writer(h5out, df1, group_name, table_name) + df_writer(h5out, df2, group_name, table_name) df_read = load_dst(filename, group_name, table_name) - assert_dataframes_close(df_read, pd.concat([df1, df2]).reset_index(drop=True), False, rtol=1e-5) + assert_dataframes_close(df_read, pd.concat([df1, df2]).reset_index(drop=True)) + @given(df1=dataframe, df2=dataframe_diff) -def test_store_pandas_as_tables_raises_exception(config_tmpdir, df1, df2): +def test_df_writer_raises_exception(config_tmpdir, df1, df2): filename = config_tmpdir + 'dataframe_to_table_exact.h5' group_name = 'test_group' table_name = 'table_name_2' with tb.open_file(filename, 'w') as h5out: - _store_pandas_as_tables(h5out, df1, group_name, table_name) + df_writer(h5out, df1, group_name, table_name) with raises(TableMismatch): - _store_pandas_as_tables(h5out, df2, group_name, table_name) + df_writer(h5out, df2, group_name, table_name) + @given(df=strings_dataframe) -def test_strings_store_pandas_as_tables(config_tmpdir, df): +def test_strings_df_writer(config_tmpdir, df): filename = config_tmpdir + 'dataframe_to_table_exact.h5' group_name = 'test_group' table_name = 'table_name_str' with tb.open_file(filename, 'w') as h5out: - _store_pandas_as_tables(h5out, df, group_name, table_name) + df_writer(h5out, df, group_name, table_name) df_read = load_dst(filename, group_name, table_name) - #we have to cast from byte strings to compare with original dataframe - df_read.str_val = df_read.str_val.str.decode('utf-8') - assert_dataframes_equal(df_read, df, False) + assert_dataframes_equal(df_read, df) + def test_make_tabledef(empty_dataframe): - tabledef = _make_tabledef(empty_dataframe.dtypes) + tabledef = _make_tabledef(empty_dataframe.to_records(index=False).dtype, 32) expected_tabledef = {'int_value' : tb.Int32Col ( shape=(), dflt=0 , pos=0), 'float_value' : tb.Float32Col( shape=(), dflt=0 , pos=1), 'bool_value' : tb.BoolCol ( shape=(), dflt=False, pos=2), 'str_value' : tb.StringCol (itemsize=32, shape=(), dflt=b'' , pos=3)} assert tabledef == expected_tabledef -def test_store_pandas_as_tables_raises_warning_empty_dataframe(config_tmpdir, empty_dataframe): + +def test_df_writer_raises_warning_empty_dataframe(config_tmpdir, empty_dataframe): filename = config_tmpdir + 'dataframe_to_table_exact.h5' group_name = 'test_group' table_name = 'table_name_3' with tb.open_file(filename, 'w') as h5out: with pytest.warns(UserWarning, match='dataframe is empty'): - _store_pandas_as_tables(h5out, empty_dataframe, group_name, table_name) + df_writer(h5out, empty_dataframe, group_name, table_name) + + +@given(df=strings_dataframe) +def test_df_writer_raises_warning_long_string(config_tmpdir, df): + filename = config_tmpdir + 'dataframe_to_table_long_string.h5' + group_name = 'test_group' + table_name = 'table_name_lstr' + with tb.open_file(filename, 'w') as h5out: + with pytest.warns(UserWarning, match='dataframe contains strings longer than allowed'): + df_writer(h5out, df, group_name, table_name, str_col_length=1) + + +@given(df=dataframe) +def test_df_writer_raises_TableMismatch_inconsistent_types(config_tmpdir, df): + filename = config_tmpdir + 'dataframe_to_table_exact.h5' + group_name = 'test_group' + table_name = 'table_name_inttype' + with tb.open_file(filename, 'w') as h5out: + df_writer(h5out, df, group_name, table_name) + with raises(TableMismatch, match='dataframe numeric types not consistent with the table existing ones'): + df_writer(h5out, df.astype(float), group_name, table_name) + + +@given(df1=dataframe, df2=dataframe) +def test_df_writer_unordered_df(config_tmpdir, df1, df2): + filename = config_tmpdir + 'dataframe_to_table_exact.h5' + group_name = 'test_group' + table_name = 'table_name_unordered' + with tb.open_file(filename, 'w') as h5out: + df_writer(h5out, df1, group_name, table_name) + df2 = df2[['bool_value', 'int_value', 'float_val']] + df_writer(h5out, df2, group_name, table_name) + df_read = load_dst(filename, group_name, table_name) + assert_dataframes_equal(df_read, pd.concat([df1, df2], sort=True).reset_index(drop=True)) + + +def test_df_writer_index(config_tmpdir, fixed_dataframe): + df = fixed_dataframe + filename = config_tmpdir + 'dataframe_to_table_index.h5' + group_name = 'test_group' + table_name = 'table_name' + columns_to_index = ['int', 'float'] + with tb.open_file(filename, 'w') as h5out: + df_writer(h5out, df, group_name, table_name, columns_to_index=columns_to_index) + table = h5out.root[group_name][table_name] + assert set(columns_to_index) == set(table.attrs.columns_to_index) + + +def test_df_writer_index_error(config_tmpdir, fixed_dataframe): + df = fixed_dataframe + filename = config_tmpdir + 'dataframe_to_table_index.h5' + group_name = 'test_group' + table_name = 'table_name' + columns_to_index = ['float_'] + with tb.open_file(filename, 'w') as h5out: + with raises(KeyError, match=(re.escape(f"columns {columns_to_index} not present in the dataframe"))): + df_writer(h5out, df, group_name, table_name, columns_to_index=columns_to_index) + + +def test_df_writer_no_index(config_tmpdir, fixed_dataframe): + df = fixed_dataframe + filename = config_tmpdir + 'dataframe_to_table_index.h5' + group_name = 'test_group' + table_name = 'table_name' + with tb.open_file(filename, 'w') as h5out: + df_writer(h5out, df, group_name, table_name) + table = h5out.root[group_name][table_name] + assert 'columns_to_index' not in table.attrs diff --git a/invisible_cities/io/event_filter_io_test.py b/invisible_cities/io/event_filter_io_test.py index 4b39756460..6e40c4aaec 100644 --- a/invisible_cities/io/event_filter_io_test.py +++ b/invisible_cities/io/event_filter_io_test.py @@ -58,7 +58,7 @@ def multiple_filter_tables_file_different_lengths(ICDATADIR): @fixture(scope="session") def multiple_filter_tables(request): - return request.getfuncargvalue(request.param) + return request.getfixturevalue(request.param) @mark.parametrize("table_names", diff --git a/invisible_cities/io/hist_io.py b/invisible_cities/io/hist_io.py deleted file mode 100644 index 07238d95da..0000000000 --- a/invisible_cities/io/hist_io.py +++ /dev/null @@ -1,122 +0,0 @@ -import numpy as np -import tables as tb - -from .. reco import tbl_functions as tbl - -from .. evm.histos import HistoManager -from .. evm.histos import Histogram - - -def hist_writer(file, - *, - group_name : 'options: HIST, HIST2D', - table_name : 'options: pmt, pmtMAU, sipm, sipmMAU', - compression = 'ZLIB4', - n_sensors : 'number of pmts or sipms', - bin_centres : 'np.array of bin centres'): - try: hist_group = getattr (file.root, group_name) - except tb.NoSuchNodeError: hist_group = file.create_group(file.root, group_name) - - n_bins = len(bin_centres) - - hist_table = file.create_earray(hist_group, - table_name, - atom = tb.Int32Atom(), - shape = (0, n_sensors, n_bins), - filters = tbl.filters(compression)) - - ## The bins can be written just once at definition of the writer - file.create_array(hist_group, table_name+'_bins', bin_centres) - - def write_hist(histo : 'np.array of histograms, one for each sensor'): - hist_table.append(histo.reshape(1, n_sensors, n_bins)) - - return write_hist - -def hist_writer_var(file, *, compression='ZLIB4'): - - def write_hist(group_name : 'string with folder name to save histograms', - table_name : 'histogram name' , - entries : 'np.array with bin content' , - bins : 'list of np.array of bins' , - out_of_range: 'np.array lenght=2 with events out of range', - errors : 'np.array with bins uncertainties' , - labels : 'list with labels of the histogram' ): - - try: hist_group = getattr (file.root, group_name) - except tb.NoSuchNodeError: hist_group = file.create_group(file.root, group_name) - - if table_name in hist_group: - raise ValueError(f"Histogram {table_name} already exists") - - vlarray = file.create_vlarray(hist_group, table_name + '_bins', - atom = tb.Float64Atom(shape=()), - filters = tbl.filters(compression)) - for ibin in bins: - vlarray.append(ibin) - add_carray (hist_group, table_name , entries ) - add_carray (hist_group, table_name + '_outRange', out_of_range) - add_carray (hist_group, table_name + '_errors' , errors ) - file.create_array(hist_group, table_name + '_labels' , labels ) - - def add_carray(hist_group, table_name, var): - array_atom = tb .Atom.from_dtype(var.dtype) - array_shape = var .shape - entry = file.create_carray(hist_group, table_name, - atom = array_atom, - shape = array_shape, - filters = tbl.filters(compression)) - entry[:] = var - - return write_hist - - -def save_histomanager_to_file(histogram_manager, file_out, mode='w', group='HIST'): - """ - Saves the HistoManager and its contained Histograms to a file. - - Arguments: - file_out = String with the path of the file were the HistoManager will - be written. - mode = Writting mode. By default a new file will be created. - group = Group name to save the histograms in the file. - """ - if mode not in 'wa': - raise ValueError(f"Incompatible mode ({mode}) of writting, please use 'w' (write) or 'a' (append).") - with tb.open_file(file_out, mode, filters=tbl.filters('ZLIB4')) as h5out: - writer = hist_writer_var(h5out) - for histoname, histo in histogram_manager.histos.items(): - writer(group, histoname, - histo.data, histo.bins, histo.out_range, - histo.errors, histo.labels) - - -def get_histograms_from_file(file_input, group_name='HIST'): - histo_manager = HistoManager() - - def name_selection(x): - selection = ( ('bins' not in x) - and ('labels' not in x) - and ('errors' not in x) - and ('outRange' not in x)) - return selection - - with tb.open_file(file_input, "r") as h5in: - histogram_list = [] - group = getattr(h5in.root, group_name) - for histoname in filter(name_selection, group._v_children): - entries = np.array(getattr(group, histoname )[:]) - bins = getattr(group, histoname + '_bins' )[:] - out_range = getattr(group, histoname + '_outRange')[:] - errors = np.array(getattr(group, histoname + '_errors' )[:]) - labels = getattr(group, histoname + '_labels' )[:] - labels = [str(lab)[2:-1].replace('\\\\', '\\') for lab in labels] - - histogram = Histogram(histoname, bins, labels) - histogram.data = entries - histogram.out_range = out_range - histogram.errors = errors - - histogram_list.append(histogram) - - return HistoManager(histogram_list) diff --git a/invisible_cities/io/hist_io_test.py b/invisible_cities/io/hist_io_test.py deleted file mode 100644 index 22a15b7b46..0000000000 --- a/invisible_cities/io/hist_io_test.py +++ /dev/null @@ -1,114 +0,0 @@ -import os -import string - -from keyword import iskeyword - -import numpy as np -import tables as tb - -from pytest import raises -from pytest import mark - -from hypothesis import assume -from hypothesis import given -from hypothesis import settings -from hypothesis.extra.numpy import arrays -from hypothesis.strategies import composite -from hypothesis.strategies import integers -from hypothesis.strategies import floats -from hypothesis.strategies import text -from hypothesis.strategies import lists -from hypothesis.strategies import sampled_from -from hypothesis.strategies import one_of - -from .. evm.histos import HistoManager, Histogram -from .. io .hist_io import save_histomanager_to_file -from .. io .hist_io import get_histograms_from_file -from .. evm.histos_test import histograms_lists -from .. evm.histos_test import assert_histogram_equality - - -letters = string.ascii_letters - -@mark.skip(reason="Delaying elimination of solid cities") -@given (histograms_lists(), text(letters, min_size=1)) -@settings(deadline=None, max_examples=400) -def test_save_histomanager_to_file_write_mode(output_tmpdir, histogram_list, group): - assume(not iskeyword(group)) - args, list_of_histograms = histogram_list - histogram_manager = HistoManager(list_of_histograms) - - file_out = os.path.join (output_tmpdir , 'test_save_histogram_manager.h5') - save_histomanager_to_file(histogram_manager, file_out, group=group) - - with tb.open_file(file_out, "r") as h5in: - file_group = getattr(h5in.root, group) - for histogram in list_of_histograms: - histoname = histogram.title - saved_labels = [str(label)[2:-1].replace('\\\\', '\\') for label in getattr(file_group, histoname + "_labels")[:]] - - assert histoname in file_group - assert len(histogram.bins) == len(getattr(file_group, histoname + "_bins")[:]) - assert np.all (a == b for a, b in zip(histogram.bins, getattr(file_group, histoname + "_bins")[:])) - assert np.allclose(histogram.data , getattr(file_group, histoname )[:]) - assert np.allclose(histogram.errors , getattr(file_group, histoname + "_errors" )[:]) - assert np.allclose(histogram.out_range, getattr(file_group, histoname + "_outRange")[:]) - assert histogram.labels == saved_labels - - -@mark.skip(reason="Delaying elimination of solid cities") -@given (histograms_lists(), text(letters, min_size=1)) -@settings(deadline=None, max_examples=400) -def test_save_histomanager_to_file_append_mode(output_tmpdir, histogram_list, group): - assume(not iskeyword(group)) - args, list_of_histograms = histogram_list - histogram_manager = HistoManager(list_of_histograms[:1]) - - file_out = os.path.join(output_tmpdir, 'test_save_histogram_manager.h5') - save_histomanager_to_file (histogram_manager, file_out, mode='w', group=group) - histogram_manager = HistoManager(list_of_histograms[1:]) - save_histomanager_to_file (histogram_manager, file_out, mode='a', group=group) - - with tb.open_file(file_out, "r") as h5in: - file_group = getattr(h5in.root, group) - for histogram in list_of_histograms: - histoname = histogram.title - saved_labels = [str(label)[2:-1].replace('\\\\', '\\') for label in getattr(file_group, histoname + "_labels")[:]] - - assert histoname in file_group - assert len(histogram.bins) == len(getattr(file_group, histoname + "_bins")[:]) - assert np.all (a == b for a, b in zip(histogram.bins, getattr(file_group, histoname + "_bins")[:])) - assert np.allclose(histogram.data , getattr(file_group, histoname )[:]) - assert np.allclose(histogram.errors , getattr(file_group, histoname + "_errors" )[:]) - assert np.allclose(histogram.out_range, getattr(file_group, histoname + "_outRange")[:]) - assert histogram.labels == saved_labels - - -@mark.skip(reason="Delaying elimination of solid cities") -@given (histograms_lists(), text(letters, min_size=1), text(letters, min_size=1, max_size=1).filter(lambda x: x not in 'wa')) -@settings(deadline=None) -def test_save_histomanager_to_file_raises_ValueError(output_tmpdir, histogram_list, group, write_mode): - args, list_of_histograms = histogram_list - histogram_manager = HistoManager(list_of_histograms) - - file_out = os.path.join (output_tmpdir, 'test_save_histogram_manager.h5') - - with raises(ValueError): - save_histomanager_to_file(histogram_manager, file_out, mode=write_mode, group=group) - - -@mark.skip(reason="Delaying elimination of solid cities") -@given(histograms_lists()) -@settings(deadline=None, max_examples=400) -def test_get_histograms_from_file(output_tmpdir, histogram_list): - args, list_of_histograms = histogram_list - histogram_manager1 = HistoManager(list_of_histograms) - - file_out = os.path.join(output_tmpdir, 'test_save_histogram_manager.h5') - save_histomanager_to_file(histogram_manager1, file_out) - - histogram_manager2 = get_histograms_from_file(file_out) - - assert len(histogram_manager1.histos) == len(histogram_manager2.histos) - for histoname in histogram_manager1.histos: - assert_histogram_equality(histogram_manager1[histoname], histogram_manager2[histoname]) diff --git a/invisible_cities/io/histogram_io.py b/invisible_cities/io/histogram_io.py new file mode 100644 index 0000000000..9e456785f4 --- /dev/null +++ b/invisible_cities/io/histogram_io.py @@ -0,0 +1,30 @@ +import tables as tb + +from .. reco import tbl_functions as tbl + + +def hist_writer(file, + *, + group_name : 'options: HIST, HIST2D', + table_name : 'options: pmt, pmtMAU, sipm, sipmMAU', + compression = 'ZLIB4', + n_sensors : 'number of pmts or sipms', + bin_centres : 'np.array of bin centres'): + try: hist_group = getattr (file.root, group_name) + except tb.NoSuchNodeError: hist_group = file.create_group(file.root, group_name) + + n_bins = len(bin_centres) + + hist_table = file.create_earray(hist_group, + table_name, + atom = tb.Int32Atom(), + shape = (0, n_sensors, n_bins), + filters = tbl.filters(compression)) + + ## The bins can be written just once at definition of the writer + file.create_array(hist_group, table_name+'_bins', bin_centres) + + def write_hist(histo : 'np.array of histograms, one for each sensor'): + hist_table.append(histo.reshape(1, n_sensors, n_bins)) + + return write_hist diff --git a/invisible_cities/icaro/__init__.py b/invisible_cities/io/histogram_io_test.py similarity index 100% rename from invisible_cities/icaro/__init__.py rename to invisible_cities/io/histogram_io_test.py diff --git a/invisible_cities/io/hits_io_test.py b/invisible_cities/io/hits_io_test.py index 7161c990c7..3e3cc027c8 100644 --- a/invisible_cities/io/hits_io_test.py +++ b/invisible_cities/io/hits_io_test.py @@ -1,13 +1,11 @@ import os import numpy as np import tables as tb -import pandas as pd import time as tm from numpy.testing import assert_allclose from . dst_io import load_dst -from .. core.testing_utils import assert_dataframes_equal from .. types.ic_types import xy from .. evm.event_model import Cluster from .. evm.event_model import Hit @@ -38,10 +36,10 @@ def test_load_hits_double_ratio_e_q_equals_one(TlMC_hits): Emax.append(np.max(E)) Qmax.append(np.max(Q)) - pop = E.pop(np.argmax(E)) - pop = Q.pop(np.argmax(Q)) - Ein.extend(E) - Qin.extend(Q) + E .pop (np.argmax(E)) + Q .pop (np.argmax(Q)) + Ein .extend(E) + Qin .extend(Q) r1 = np.mean(Emax)/np.mean(Qmax) @@ -65,10 +63,10 @@ def test_load_hits_double_ratio_e_q_equals_one_skipping_NN(TlMC_hits_skipping_NN Emax.append(np.max(E)) Qmax.append(np.max(Q)) - pop = E.pop(np.argmax(E)) - pop = Q.pop(np.argmax(Q)) - Ein.extend(E) - Qin.extend(Q) + E .pop (np.argmax(E)) + Q .pop (np.argmax(Q)) + Ein .extend(E) + Qin .extend(Q) r1 = np.mean(Emax)/np.mean(Qmax) diff --git a/invisible_cities/io/indexation_test.py b/invisible_cities/io/indexation_test.py index c4b351d84b..d53fcece07 100644 --- a/invisible_cities/io/indexation_test.py +++ b/invisible_cities/io/indexation_test.py @@ -1,17 +1,15 @@ import os import tables as tb - +import pandas as pd from pytest import mark -from pytest import fixture from .. cities.components import city -from .. core .configure import configure from . hits_io import hits_writer from . kdst_io import kr_writer -from . mcinfo_io import mc_info_writer from . pmaps_io import pmap_writer +from . dst_io import df_writer @city def writer_test_city(writer, file_out, files_in, event_range, detector_db): @@ -22,22 +20,18 @@ def get_writers (self, h5out): pass def write_parameters(self, h5out): pass -@fixture(scope="session") -def init_city(ICDIR, config_tmpdir): - conf = configure(('city ' + ICDIR + 'config/city.conf').split()) - file_out = os.path.join(config_tmpdir, "empty_file.h5") - conf.update(dict(file_out = file_out)) - city = DummyCity(**conf) - return city, file_out +def _df_writer(h5out): + df = pd.DataFrame(columns=['event', 'some_value'], dtype=int) + return df_writer(h5out, df, 'DUMMY', 'dummy', columns_to_index=['event']) @mark.parametrize(" writer group node column thing".split(), [( hits_writer, "RECO" , "Events" , "event" , "hits"), ( kr_writer, "DST" , "Events" , "event" , "kr" ), - (mc_info_writer, "MC" , "extents" , "evt_number", "mc" ), ( pmap_writer, "PMAPS", "S1" , "event" , "s1" ), ( pmap_writer, "PMAPS", "S2" , "event" , "s2" ), - ( pmap_writer, "PMAPS", "S2Si" , "event" , "s2si")]) + ( pmap_writer, "PMAPS", "S2Si" , "event" , "s2si"), + ( _df_writer, "DUMMY", "dummy" , "event" , 'df' )]) def test_table_is_indexed(tmpdir_factory, writer, group, node, column, thing): tmpdir = tmpdir_factory.mktemp('indexation') file_out = os.path.join(tmpdir, f"empty_table_containing_{thing}.h5") diff --git a/invisible_cities/io/kdst_io.py b/invisible_cities/io/kdst_io.py index 38b382a38f..2efbfdcff0 100644 --- a/invisible_cities/io/kdst_io.py +++ b/invisible_cities/io/kdst_io.py @@ -1,6 +1,5 @@ from . table_io import make_table from .. evm.nh5 import KrTable -from .. evm.nh5 import XYfactors from .. evm.nh5 import PSFfactors @@ -19,45 +18,6 @@ def write_kr(kr_event): return write_kr -def xy_writer(hdf5_file, **kwargs): - xy_table = make_table(hdf5_file, - fformat = XYfactors, - **kwargs) - - def write_xy(xs, ys, fs, us, ns): - row = xy_table.row - for i, x in enumerate(xs): - for j, y in enumerate(ys): - row["x"] = x - row["y"] = y - row["factor"] = fs[i,j] - row["uncertainty"] = us[i,j] - row["nevt"] = ns[i,j] - row.append() - return write_xy - - -def xy_correction_writer(hdf5_file, * , - group = "Corrections", - table_name = "XYcorrections", - compression = 'ZLIB4'): - return xy_writer(hdf5_file, - group = group, - name = table_name, - description = "XY corrections", - compression = compression) - - -def xy_lifetime_writer(hdf5_file, * , - group = "Corrections", - table_name = "LifetimeXY", - compression = 'ZLIB4'): - return xy_writer(hdf5_file, - group = group, - name = table_name, - description = "XY-dependent lifetime values", - compression = compression) - def psf_writer(hdf5_file, **kwargs): psf_table = make_table(hdf5_file, group = "PSF", diff --git a/invisible_cities/io/kdst_io_test.py b/invisible_cities/io/kdst_io_test.py index 5e0697ba11..74f3ec1912 100644 --- a/invisible_cities/io/kdst_io_test.py +++ b/invisible_cities/io/kdst_io_test.py @@ -3,15 +3,13 @@ import numpy as np import tables as tb -from pytest import mark from numpy.testing import assert_allclose from ..core.testing_utils import assert_dataframes_equal from ..io.dst_io import load_dst from . kdst_io import kr_writer -from . kdst_io import xy_correction_writer -from . kdst_io import xy_lifetime_writer from . kdst_io import psf_writer + from ..evm.event_model import KrEvent @@ -50,36 +48,6 @@ def dump_df(write, df): assert_dataframes_equal(dst, df, False) -@mark.parametrize("writer".split(), - ((xy_correction_writer,), - (xy_lifetime_writer,))) -def test_xy_writer(config_tmpdir, corr_toy_data, writer): - output_file = os.path.join(config_tmpdir, "test_corr.h5") - - _, (x, y, F, U, N) = corr_toy_data - - group = "Corrections" - name = "XYcorrections" - - with tb.open_file(output_file, 'w') as h5out: - write = writer(h5out, - group = group, - table_name = name) - write(x, y, F, U, N) - - x, y = np.repeat(x, y.size), np.tile(y, x.size) - F, U, N = F.flatten(), U.flatten(), N.flatten() - - dst = load_dst(output_file, - group = group, - node = name) - assert_allclose(x, dst.x .values) - assert_allclose(y, dst.y .values) - assert_allclose(F, dst.factor .values) - assert_allclose(U, dst.uncertainty.values) - assert_allclose(N, dst.nevt .values) - - def test_psf_writer(config_tmpdir): output_file = os.path.join(config_tmpdir, "test_psf.h5") diff --git a/invisible_cities/io/mcinfo_io.py b/invisible_cities/io/mcinfo_io.py index 28155125bd..eb585bf019 100644 --- a/invisible_cities/io/mcinfo_io.py +++ b/invisible_cities/io/mcinfo_io.py @@ -1,365 +1,828 @@ - -import tables as tb import numpy as np +import tables as tb import pandas as pd -from .. reco import tbl_functions as tbl -from .. core import system_of_units as units -from .. core.exceptions import SensorBinningNotFound +from enum import auto +from functools import partial -from .. evm.event_model import MCParticle -from .. evm.event_model import MCHit -from .. evm.event_model import Waveform +import warnings -from .. evm.nh5 import MCGeneratorInfo -from .. evm.nh5 import MCExtentInfo -from .. evm.nh5 import MCHitInfo -from .. evm.nh5 import MCParticleInfo +from .. core import system_of_units as units +from .. core.exceptions import NoParticleInfoInFile -from .. database import load_db as DB +from .. evm.event_model import MCHit +from .. evm.event_model import MCInfo +from .. database import load_db as DB +from . dst_io import load_dst +from . dst_io import df_writer +from .. types.ic_types import AutoNameEnumBase -from typing import Mapping +from typing import Callable +from typing import Dict +from typing import List +from typing import Mapping +from typing import Optional from typing import Sequence -from typing import Tuple +from typing import Type -# use Mapping (duck type) rather than dict -units_dict = {'picosecond' : units.picosecond, 'ps' : units.picosecond, - 'nanosecond' : units.nanosecond, 'ns' : units.nanosecond, - 'microsecond': units.microsecond, 'mus': units.microsecond, - 'millisecond': units.millisecond, 'ms' : units.millisecond} +class MCTableType(AutoNameEnumBase): + configuration = auto() + events = auto() + event_mapping = auto() + extents = auto() + generators = auto() + hits = auto() + particles = auto() + sensor_positions = auto() + sns_positions = auto() + sns_response = auto() + waveforms = auto() -class mc_info_writer: - """Write MC info to file.""" - def __init__(self, h5file, compression = 'ZLIB4'): +def mc_writer(h5out : tb.file.File) -> Callable: + """ + Writes the MC tables to the output file. + + parameters + ---------- + h5out : pytables file + the file to which the MC info is to be written - self.h5file = h5file - self.compression = compression - self._create_tables() - self.reset() - self.current_tables = None + returns + ------- + write_mctables : Callable + Function which writes to the file. + """ + mcwriter_ = partial(df_writer , + h5out = h5out, + group_name = 'MC') + def write_mctables(table_dict : Dict): + """ + Writer function + + parameters + ---------- + table_dict : Dict + Dictionary with key MCTableType + and value pd.DataFrame with the + information from the input file + which has to be written. + """ + first_indx = check_last_merge_index(h5out) + 1 + for key, tbl in table_dict.items(): + if (key is MCTableType.configuration or + key is MCTableType.event_mapping ): + try: + orig_indx = tbl.file_index.unique() + new_indx = np.arange(first_indx , + first_indx + len(orig_indx)) + tbl.file_index.replace(orig_indx, new_indx, inplace=True) + except AttributeError: + tbl['file_index'] = first_indx + col_indx = 'event' if hasattr(tbl, 'event') else None + str_len = 300 if key is MCTableType.configuration else 100 + mcwriter_(df = tbl, table_name = key.name, + str_col_length = str_len, columns_to_index = col_indx) + return write_mctables + + +def check_last_merge_index(h5out : tb.file.File) -> int: + """ + Get the last file index used to index merged files + in the configuration table and mapping + + parameters + ---------- + h5out: pytables file + the file for output + + returns + ------- + indx: int + integer for the last saved file index + """ + if 'MC' in h5out.root: + if 'event_mapping' in h5out.root.MC: + return h5out.root.MC.event_mapping.cols.file_index[-1] + return -1 - self.last_written_hit = 0 - self.last_written_particle = 0 - self.first_extent_row = True - self.first_file = True +def read_mc_tables(file_in : str , + evt_arr : Optional[np.ndarray] = None, + db_file : Optional[str] = None, + run_no : Optional[int] = None) -> Dict: + """ + Reads all the MC tables present in + file_in and stores the Dataframes for + each into a dictionary. + + parameters + ---------- + file_in: str + Name of the input file + evt_arr: optional np.ndarray + list of events or None. default None for + all events. + db_file: optional str + Name of the database to be used. + Only required for old format MC files + run_no : optional int + Run number for database access + Only required for old format MC files + + returns + ------- + tbl_dict : Dict + A dictionary with key type MCTableType + and values of type pd.DataFrame + containing the MC info for all tables + sorted into new format in the case + of an old format MC file. + """ + mctbls = get_mc_tbl_list(file_in) + if evt_arr is None: + evt_arr = get_event_numbers_in_file(file_in) + tbl_dict = {} + for tbl in mctbls: + if tbl is MCTableType.hits : + hits = load_mchits_df(file_in).reset_index() + tbl_dict[tbl] = hits[hits.event_id.isin(evt_arr)] + elif tbl is MCTableType.particles : + parts = load_mcparticles_df(file_in).reset_index() + tbl_dict[tbl] = parts[parts.event_id.isin(evt_arr)] + elif tbl is MCTableType.extents : + pass + elif (tbl is MCTableType.sns_response or + tbl is MCTableType.waveforms ) : + sns = load_mcsensor_response_df(file_in, return_raw=True) + tbl_key = MCTableType.sns_response + tbl_dict[tbl_key] = sns[sns.event_id.isin(evt_arr)] + elif (tbl is MCTableType.sensor_positions or + tbl is MCTableType.sns_positions ): + pos = load_mcsensor_positions(file_in, db_file, run_no) + tbl_key = MCTableType.sns_positions + tbl_dict[tbl_key] = pos + elif tbl is MCTableType.generators : + gen = load_mcgenerators(file_in) + tbl_dict[tbl] = gen[gen.evt_number.isin(evt_arr)] + elif tbl is MCTableType.configuration : + config = load_mcconfiguration(file_in) + tbl_dict[tbl] = config + elif tbl is MCTableType.events : + pass + elif tbl is MCTableType.event_mapping : + evt_map = load_mcevent_mapping(file_in) + tbl_dict[tbl] = evt_map[evt_map.event_id.isin(evt_arr)] + else : + raise TypeError("MC table has no reader") + return tbl_dict + + +def copy_mc_info(file_in : str , + writer : Type[mc_writer] , + which_events : Optional[List[int]] = None, + db_file : Optional[str] = None, + run_number : Optional[int] = None) -> None: + """ + Copy from the input file to the output file the MC info of certain events. + + Parameters + ---------- + file_in : str + Input file name. + writer : instance of class mcinfo_io.mc_info_writer + MC info writer to h5 file. + which_events : None or list of ints + List of IDs (i.e. event numbers) that identify the events to be copied + to the output file. If None, all events in the input file are copied. + db_file : None or str + Name of the database to be used. + Only required in cas of old format MC file + run_number : None or int + Run number to be used for database access. + Only required in case of old format MC file + """ + if which_events is None: + which_events = get_event_numbers_in_file(file_in) + tbl_dict = read_mc_tables(file_in, which_events, db_file, run_number) + if MCTableType.event_mapping not in tbl_dict.keys(): + new_key = MCTableType.event_mapping + tbl_dict[new_key] = pd.DataFrame(which_events, columns=['event_id']) + writer(tbl_dict) + + +def check_mc_present(file_name : str) -> bool: + with tb.open_file(file_name) as h5in: + return hasattr(h5in.root, 'MC') + + +def is_oldformat_file(file_name : str) -> bool: + """ + Checks if the file type is pre 2020 or not + + parameters + ---------- + file_name : str + File name of the input file + + return + ------ + bool: True if MC.extents table found: pre-2020 format + False if MC.extents not found : 2020-- format + """ + with tb.open_file(file_name) as h5in: + return hasattr(h5in.root, 'MC/extents') - def reset(self): - # last visited row - self.last_row = 0 - def _create_tables(self): - """Create tables in MC group in file h5file.""" - if '/MC' in self.h5file: - MC = self.h5file.root.MC +def get_mc_tbl_list(file_name: str) -> List[MCTableType]: + """ + Returns a list of the tables in + the MC group of a given file + + parameters + ---------- + file_name : str + Name of the input file + + returns + ------- + tbl_list : List[MCTableType] + A list of the MC tables which are present + in the input file. + """ + with tb.open_file(file_name, 'r') as h5in: + mc_group = h5in.root.MC + return [MCTableType[tbl.name] for tbl in mc_group] + + +def get_event_numbers_in_file(file_name: str) -> np.ndarray: + """ + Get a list of the event numbers in the file + based on the MC tables + + parameters + ---------- + file_name : str + File name of the input file + + returns + ------- + evt_arr : np.ndarray + numpy array containing the list of all + event numbers in file. + """ + with tb.open_file(file_name, 'r') as h5in: + if is_oldformat_file(file_name): + evt_list = h5in.root.MC.extents.cols.evt_number[:] else: - MC = self.h5file.create_group(self.h5file.root, "MC") - - self.extent_table = self.h5file.create_table(MC, "extents", - description = MCExtentInfo, - title = "extents", - filters = tbl.filters(self.compression)) - - # Mark column to index after populating table - self.extent_table.set_attr('columns_to_index', ['evt_number']) - - self.hit_table = self.h5file.create_table(MC, "hits", - description = MCHitInfo, - title = "hits", - filters = tbl.filters(self.compression)) - - self.particle_table = self.h5file.create_table(MC, "particles", - description = MCParticleInfo, - title = "particles", - filters = tbl.filters(self.compression)) - - self.generator_table = self.h5file.create_table(MC, "generators", - description = MCGeneratorInfo, - title = "generators", - filters = tbl.filters(self.compression)) - - - - def __call__(self, mctables: (tb.Table, tb.Table, tb.Table, tb.Table), - evt_number: int): - if mctables is not self.current_tables: - self.current_tables = mctables - self.reset() - - extents = mctables[0] - # Note: filtered out events do not make it to evt_number, but are included in extents dataset - for iext in range(self.last_row, len(extents)): - this_row = extents[iext] - if this_row['evt_number'] == evt_number: - if iext == 0: - if self.first_file: - modified_hit = this_row['last_hit'] - modified_particle = this_row['last_particle'] - self.first_extent_row = False - self.first_file = False - else: - modified_hit = this_row['last_hit'] + self.last_written_hit + 1 - modified_particle = this_row['last_particle'] + self.last_written_particle + 1 - self.first_extent_row = False - - elif self.first_extent_row: - previous_row = extents[iext-1] - modified_hit = this_row['last_hit'] - previous_row['last_hit'] + self.last_written_hit - 1 - modified_particle = this_row['last_particle'] - previous_row['last_particle'] + self.last_written_particle - 1 - self.first_extent_row = False - self.first_file = False - else: - previous_row = extents[iext-1] - modified_hit = this_row['last_hit'] - previous_row['last_hit'] + self.last_written_hit - modified_particle = this_row['last_particle'] - previous_row['last_particle'] + self.last_written_particle - - modified_row = self.extent_table.row - modified_row['evt_number'] = evt_number - modified_row['last_hit'] = modified_hit - modified_row['last_particle'] = modified_particle - modified_row.append() - - self.last_written_hit = modified_hit - self.last_written_particle = modified_particle - - break - - self.extent_table.flush() - - hits, particles, generators = read_mcinfo_evt(mctables, evt_number, self.last_row) - self.last_row = iext + 1 - - for h in hits: - new_row = self.hit_table.row - new_row['hit_position'] = h[0] - new_row['hit_time'] = h[1] - new_row['hit_energy'] = h[2] - new_row['label'] = h[3] - new_row['particle_indx'] = h[4] - new_row['hit_indx'] = h[5] - new_row.append() - self.hit_table.flush() - - for p in particles: - new_row = self.particle_table.row - new_row['particle_indx'] = p[0] - new_row['particle_name'] = p[1] - new_row['primary'] = p[2] - new_row['mother_indx'] = p[3] - new_row['initial_vertex'] = p[4] - new_row['final_vertex'] = p[5] - new_row['initial_volume'] = p[6] - new_row['final_volume'] = p[7] - new_row['momentum'] = p[8] - new_row['kin_energy'] = p[9] - new_row['creator_proc'] = p[10] - new_row.append() - self.particle_table.flush() - - for g in generators: - new_row = self.generator_table.row - new_row['evt_number'] = g[0] - new_row['atomic_number'] = g[1] - new_row['mass_number'] = g[2] - new_row['region'] = g[3] - new_row.append() - self.generator_table.flush() - - -def load_mchits(file_name: str, - event_range=(0, int(1e9))) -> Mapping[int, Sequence[MCHit]]: + evt_list = _get_list_of_events_new(h5in) + return evt_list - with tb.open_file(file_name, mode='r') as h5in: - mchits_dict = read_mchit_info(h5in, event_range) - return mchits_dict +def _get_list_of_events_new(h5in : tb.file.File) -> np.ndarray: + mc_tbls = ['hits', 'particles', 'sns_response'] + def try_unique_evt_itr(group, itr): + for elem in itr: + try: + yield np.unique(getattr(group, elem).cols.event_id) + except tb.exceptions.NoSuchNodeError: + pass + + evt_list = list(try_unique_evt_itr(h5in.root.MC, mc_tbls)) + if len(evt_list) == 0: + raise AttributeError("At least one of MC/hits, MC/particles, \ + MC/sns_response must be present to use get_list_of_events.") + return np.unique(np.concatenate(evt_list)).astype(int) + + +def load_mcconfiguration(file_name : str) -> pd.DataFrame: + """ + Load the MC.configuration table from file into + a pd.DataFrame + + parameters + ---------- + file_name : str + Name of the file with MC info + + returns + ------- + config : pd.DataFrame + DataFrame with all nexus configuration + parameters + """ + config = load_dst(file_name, 'MC', 'configuration') + if is_oldformat_file(file_name): + config.param_key = config.param_key.str.replace(r'.*Pmt.*\_binning.*' , + 'PmtR11410_binning' ) + config.param_key = config.param_key.str.replace(r'.*SiPM.*\_binning.*', + 'SiPM_binning' ) + config = config.drop_duplicates('param_key').reset_index(drop=True) + return config + + +def load_mcsensor_positions(file_name : str, + db_file : Optional[str] = None, + run_no : Optional[int] = None) -> pd.DataFrame: + """ + Load the sensor positions stored in the MC group + into a pd.DataFrame + + parameters + ---------- + file_name : str + Name of the file containing MC info + db_file : None or str + Name of the database to be used. + Only required for pre-2020 format files + run_no : None or int + Run number to be used for database access. + Only required for pre-2020 format files + + returns + ------- + sns_pos : pd.DataFrame + DataFrame containing information about + the positions and types of sensors in + the MC simulation. + """ + if is_oldformat_file(file_name): + sns_pos = load_dst(file_name, 'MC', 'sensor_positions') + if sns_pos.shape[0] > 0: + if db_file is None or run_no is None: + warnings.warn(f' Database and file number needed', UserWarning) + return pd.DataFrame(columns=['sensor_id', 'sensor_name', + 'x', 'y', 'z']) + ## Add a column to the DataFrame so all info + ## is present like in the new format + pmt_ids = DB.DataPMT(db_file, run_no).SensorID + sns_names = get_sensor_binning(file_name).index + pmt_name = sns_names.str.contains('Pmt') + pmt_pos = sns_pos.sensor_id.isin(pmt_ids) + sns_pos.loc[pmt_pos, 'sensor_name'] = sns_names[pmt_name][0] + sns_pos.sensor_name.fillna(sns_names[~pmt_name][0], inplace=True) + else: + ## So the column names and shape are the same as 2020 format + new_cols = sns_pos.columns.tolist() + ['sensor_name'] + sns_pos = sns_pos.reindex(new_cols, axis=1) + else: + sns_pos = load_dst(file_name, 'MC', 'sns_positions' ) + return sns_pos + + +def load_mcgenerators(file_name : str) -> pd.DataFrame: + """ + Load the generator information to a pd.DataFrame + + parameters + ---------- + file_name : str + Name of the file containing MC info. + + returns + ------- + pd.DataFrame with the generator information + available for the MC events in the file. + """ + return load_dst(file_name, 'MC', 'generators') + + +def load_mcevent_mapping(file_name : str) -> pd.DataFrame: + """ + Load the event mapping information into a pd.DataFrame + + parameters + ---------- + file_name : str + Name of the file containing MC info. + + returns + ------- + pd.DataFrame with the mapping information between + configurations and files in case of a merged file. + + """ + return load_dst(file_name, 'MC', 'event_mapping') def load_mchits_df(file_name : str) -> pd.DataFrame: """ Opens file and calls read_mchits_df + + parameters + ---------- file_name : str The name of the file to be read + + returns + ------- + hits : pd.DataFrame + DataFrame with the information stored in file + about the energy deposits in the ACTIVE volume + in nexus simulations. """ - extents = pd.read_hdf(file_name, 'MC/extents') - with tb.open_file(file_name) as h5in: - hits = read_mchits_df(h5in, extents) + if is_oldformat_file(file_name): + ## Is this a pre-Feb 2020 file? + return load_mchits_dfold(file_name) + else: + return load_mchits_dfnew(file_name) + +def load_mchits_dfnew(file_name : str) -> pd.DataFrame: + """ + Returns MC hit information for 2020-- format files + + parameters + ---------- + file_name : str + Name of the file containing MC info + + returns + ------- + hits : pd.DataFrame + DataFrame with the information stored in file + about the energy deposits in the ACTIVE volume + in nexus simulations. + """ + hits = load_dst(file_name, 'MC', 'hits') + hits.set_index(['event_id', 'particle_id', 'hit_id'], + inplace=True) return hits -def read_mchits_df(h5in : tb.file.File, - extents : pd.DataFrame) -> pd.DataFrame: +def load_mchits_dfold(file_name : str) -> pd.DataFrame: """ Loads the MC hit information into a pandas DataFrame. - h5in : pytables file - A file already read into a pytables object - extents : pd.DataFrame - The extents table from the file which gives - information about the event tracture. - """ - - hits_tb = h5in.root.MC.hits - - # Generating hits DataFrame - hits = pd.DataFrame({'hit_id' : hits_tb.col('hit_indx'), - 'particle_id' : hits_tb.col('particle_indx'), - 'label' : hits_tb.col('label').astype('U13'), - 'time' : hits_tb.col('hit_time'), - 'x' : hits_tb.col('hit_position')[:, 0], - 'y' : hits_tb.col('hit_position')[:, 1], - 'z' : hits_tb.col('hit_position')[:, 2], - 'energy' : hits_tb.col('hit_energy')}) - - evt_hit_df = extents[['last_hit', 'evt_number']] - evt_hit_df.set_index('last_hit', inplace = True) - - hits = hits.merge(evt_hit_df , - left_index = True, - right_index = True, - how = 'left') - hits.rename(columns={"evt_number": "event_id"}, inplace = True) - hits.event_id.fillna(method='bfill', inplace = True) - hits.event_id = hits.event_id.astype(int) - - # Setting the indexes - hits.set_index(['event_id', 'particle_id', 'hit_id'], inplace=True) + For pre-2020 format files - return hits + parameters + ---------- + file_name : str + Name of the file containing MC info + + returns + ------- + hits : pd.DataFrame + DataFrame with the information stored in file + about the energy deposits in the ACTIVE volume + in nexus simulations. + """ + extents = load_dst(file_name, 'MC', 'extents') + with tb.open_file(file_name) as h5in: + hits_tb = h5in.root.MC.hits + # Generating hits DataFrame + hits = pd.DataFrame({'hit_id' : hits_tb.col('hit_indx') , + 'particle_id': hits_tb.col('particle_indx') , + 'x' : hits_tb.col('hit_position')[:, 0] , + 'y' : hits_tb.col('hit_position')[:, 1] , + 'z' : hits_tb.col('hit_position')[:, 2] , + 'time' : hits_tb.col('hit_time') , + 'energy' : hits_tb.col('hit_energy') , + 'label' : hits_tb.col('label').astype('U13')}) -def load_mcparticles(file_name: str, - event_range=(0, int(1e9))) -> Mapping[int, Mapping[int, MCParticle]]: + evt_hit_df = extents[['last_hit', 'evt_number']] + evt_hit_df.set_index('last_hit', inplace=True) - with tb.open_file(file_name, mode='r') as h5in: - return read_mcinfo(h5in, event_range) + hits = hits.merge(evt_hit_df , + left_index = True, + right_index = True, + how = 'left') + hits.rename(columns={"evt_number": "event_id"}, inplace=True) + hits.event_id.fillna(method='bfill', inplace=True) + hits.event_id = hits.event_id.astype(int) + + # Setting the indexes + hits.set_index(['event_id', 'particle_id', 'hit_id'], inplace=True) + + return hits + + +def cast_mchits_to_dict(hits_df: pd.DataFrame) -> Mapping[int, List[MCHit]]: + """ + Casts the mchits dataframe to an + old style mapping. + + paramerters + ----------- + hits_df : pd.DataFrame + DataFrame containing event deposit information + + returns + ------- + hit_dict : Mapping + The same hit information cast into a dictionary + using MCHit objects. + """ + hit_dict = {} + for evt, evt_hits in hits_df.groupby(level=0): + hit_dict[evt] = [MCHit( hit.iloc[0, :3].values, + *hit.iloc[0, 3:].values) + for _, hit in evt_hits.groupby(level=2)] + return hit_dict def load_mcparticles_df(file_name: str) -> pd.DataFrame: """ Opens file and calls read_mcparticles_df + + parameters + ---------- file_name : str The name of the file to be read + + returns + ------- + parts : pd.DataFrame + DataFrame containing MC particle information. """ - extents = pd.read_hdf(file_name, 'MC/extents') - with tb.open_file(file_name, mode='r') as h5in: - particles = read_mcparticles_df(h5in, extents) + if is_oldformat_file(file_name): + return load_mcparticles_dfold(file_name) + else: + return load_mcparticles_dfnew(file_name) + + +def load_mcparticles_dfnew(file_name: str) -> pd.DataFrame: + """ + Loads MC particle info from file into a pd.DataFrame + parameters + ---------- + file_name : str + Name of file containing MC info. + + returns + ------- + particles : pd.DataFrame + DataFrame containg the MC particle info + stored in file_name. + """ + particles = load_dst(file_name, 'MC', 'particles') + particles.primary = particles.primary.astype('bool') + particles.set_index(['event_id', 'particle_id'], inplace=True) return particles -def read_mcparticles_df(h5in : tb.file.File, - extents : pd.DataFrame) -> pd.DataFrame: +def load_mcparticles_dfold(file_name: str) -> pd.DataFrame: """ A reader for the MC particle output based on pandas DataFrames. + parameters + ---------- file_name: string Name of the file to be read - """ - p_tb = h5in.root.MC.particles - - # Generating parts DataFrame - parts = pd.DataFrame({'particle_id' : p_tb.col('particle_indx'), - 'particle_name' : p_tb.col('particle_name').astype('U20'), - 'primary' : p_tb.col('primary').astype('bool'), - 'mother_id' : p_tb.col('mother_indx'), - 'initial_x' : p_tb.col('initial_vertex')[:, 0], - 'initial_y' : p_tb.col('initial_vertex')[:, 1], - 'initial_z' : p_tb.col('initial_vertex')[:, 2], - 'initial_t' : p_tb.col('initial_vertex')[:, 3], - 'final_x' : p_tb.col('final_vertex')[:, 0], - 'final_y' : p_tb.col('final_vertex')[:, 1], - 'final_z' : p_tb.col('final_vertex')[:, 2], - 'final_t' : p_tb.col('final_vertex')[:, 3], - 'initial_volume' : p_tb.col('initial_volume').astype('U20'), - 'final_volume' : p_tb.col('final_volume').astype('U20'), - 'initial_momentum_x': p_tb.col('momentum')[:, 0], - 'initial_momentum_y': p_tb.col('momentum')[:, 1], - 'initial_momentum_z': p_tb.col('momentum')[:, 2], - 'kin_energy' : p_tb.col('kin_energy'), - 'creator_proc' : p_tb.col('creator_proc').astype('U20')}) - - # Adding event info - evt_part_df = extents[['last_particle', 'evt_number']] - evt_part_df.set_index('last_particle', inplace = True) - parts = parts.merge(evt_part_df , - left_index = True, - right_index = True, - how = 'left') - parts.rename(columns={"evt_number": "event_id"}, inplace = True) - parts.event_id.fillna(method='bfill', inplace = True) - parts.event_id = parts.event_id.astype(int) - - # Setting the indexes - parts.set_index(['event_id', 'particle_id'], inplace=True) - - return parts - - -def load_mcsensor_response(file_name: str, - event_range=(0, int(1e9))) -> Mapping[int, Mapping[int, Waveform]]: + returns + ------- + parts : pd.DataFrame + DataFrame containing the MC particle info + contained in file_name. + """ + extents = load_dst(file_name, 'MC', 'extents') with tb.open_file(file_name, mode='r') as h5in: - return read_mcsns_response(h5in, event_range) - - -def get_sensor_binning(file_name : str) -> Tuple: + p_tb = h5in.root.MC.particles + + # Generating parts DataFrame + parts = pd.DataFrame({'particle_id' : p_tb.col('particle_indx'), + 'particle_name' : p_tb.col('particle_name').astype('U20'), + 'primary' : p_tb.col('primary').astype('bool'), + 'mother_id' : p_tb.col('mother_indx'), + 'initial_x' : p_tb.col('initial_vertex')[:, 0], + 'initial_y' : p_tb.col('initial_vertex')[:, 1], + 'initial_z' : p_tb.col('initial_vertex')[:, 2], + 'initial_t' : p_tb.col('initial_vertex')[:, 3], + 'final_x' : p_tb.col('final_vertex')[:, 0], + 'final_y' : p_tb.col('final_vertex')[:, 1], + 'final_z' : p_tb.col('final_vertex')[:, 2], + 'final_t' : p_tb.col('final_vertex')[:, 3], + 'initial_volume' : p_tb.col('initial_volume').astype('U20'), + 'final_volume' : p_tb.col('final_volume').astype('U20'), + 'initial_momentum_x': p_tb.col('momentum')[:, 0], + 'initial_momentum_y': p_tb.col('momentum')[:, 1], + 'initial_momentum_z': p_tb.col('momentum')[:, 2], + 'kin_energy' : p_tb.col('kin_energy'), + 'creator_proc' : p_tb.col('creator_proc').astype('U20')}) + + # Adding event info + evt_part_df = extents[['last_particle', 'evt_number']] + evt_part_df.set_index('last_particle', inplace=True) + parts = parts.merge(evt_part_df , + left_index = True, + right_index = True, + how = 'left') + parts.rename(columns={"evt_number": "event_id"}, inplace=True) + parts.event_id.fillna(method='bfill', inplace=True) + parts.event_id = parts.event_id.astype(int) + + ## Add columns present in new format + missing_columns = ['final_momentum_x', 'final_momentum_y', + 'final_momentum_z', 'length', 'final_proc'] + parts = parts.reindex(parts.columns.tolist() + missing_columns, axis=1) + + # Setting the indexes + parts.set_index(['event_id', 'particle_id'], inplace=True) + + return parts + + +def get_sensor_binning(file_name : str) -> pd.DataFrame: """ Looks in the configuration table of the input file and extracts the binning used - for both types of sensitive detector. - """ - config = pd.read_hdf(file_name, 'MC/configuration') - bins = config[config.param_key.str.contains('time_binning')] - pmt_bin = bins.param_value[bins.param_key.str.contains('Pmt')].iloc[0] - pmt_bin = float(pmt_bin.split()[0]) * units_dict[pmt_bin.split()[1]] - sipm_bin = bins.param_value[bins.param_key.str.contains('SiPM')].iloc[0] - sipm_bin = float(sipm_bin.split()[0]) * units_dict[sipm_bin.split()[1]] - - return pmt_bin, sipm_bin + for all types of sensitive detector. + parameters + ---------- + file_name : str + Name of the file containing MC info. + + returns + ------- + bins : pd.DataFrame + DataFrame containing the sensor types + and the sampling width (binning) used + in full simulation. + """ + config = load_mcconfiguration(file_name).set_index('param_key') + bins = config[config.index.str.contains('binning')] + if bins.empty: + warnings.warn(f' No binning info available.', UserWarning) + return pd.DataFrame(columns=['sns_name', 'bin_width']) + ## Drop duplicates in case of merged file + bins = bins.drop('file_index', axis=1, errors='ignore') + bins = bins.loc[~bins.index.duplicated()] + bins.columns = ['bin_width'] + bins.index = bins.index.rename('sns_name') + bins.index = bins.index.str.strip('_binning') + ## Combine value and unit in configuration to get + ## binning in standard units. + bins.bin_width = bins.bin_width.str.split(expand=True).apply( + lambda x: float(x[0]) * getattr(units, x[1]), axis=1) + ## Drop protects probably out of date 2020 file NextFlex + return bins.drop(bins[bins.index.str.contains('Geom')].index) + + +def get_sensor_types(file_name : str) -> pd.DataFrame: + """ + returns a DataFrame linking sensor_ids to + sensor type names. + !! Only valid for new format data, otherwise use + !! database. + raises exception if old format file used + + parameters + ---------- + file_name : str + name of the file with nexus sensor info. -def load_mcsensor_response_df(file_name : str, - db_file : str, - run_no : int) -> Tuple: + returns + ------- + sns_pos : pd.DataFrame + Sensor position info for the MC sensors + which saw light in this simulation. + """ + if is_oldformat_file(file_name): + raise TypeError('Old format files not valid for get_sensor_types') + sns_pos = load_dst(file_name, 'MC', 'sns_positions') + sns_pos.drop(['x', 'y', 'z'], axis=1, inplace=True) + return sns_pos + + +def load_mcsensor_response_df(file_name : str , + return_raw : Optional[bool] = False, + db_file : Optional[ str] = None, + run_no : Optional[ int] = None + ) -> pd.DataFrame: """ A reader for the MC sensor output based on pandas DataFrames. - file_name: string - Name of the file to be read - db_file : string - Name of the detector database to be accessed - run_no : int - Run number for database access + parameters + ---------- + file_name : string + Name of the file to be read + return_raw : bool + Return without conversion of time_bin to + time if True + db_file : None orstring + Name of the detector database to be accessed. + Only required for pre-2020 format files. + run_no : None or int + Run number for database access. + Only required for pre-2020 format files. + + returns + ------- + sns_resp : pd.DataFrame + DataFrame containing the sensor response info + contained in file_name """ - pmt_ids = DB.DataPMT(db_file, run_no).SensorID - - pmt_bin, sipm_bin = get_sensor_binning(file_name) + if is_oldformat_file(file_name): + sns_resp = load_mcsensors_dfold(file_name) + if return_raw: + return sns_resp + assert(db_file), "Database name required for this file" + assert( run_no), "Run number required for database access" + return convert_timebin_to_time(sns_resp , + get_sensor_binning (file_name), + load_mcsensor_positions(file_name, + db_file , + run_no )) + else: + sns_resp = load_dst(file_name, 'MC', 'sns_response') + if return_raw: + return sns_resp + return convert_timebin_to_time(sns_resp , + get_sensor_binning (file_name), + load_mcsensor_positions(file_name)) + + +def load_mcsensors_dfold(file_name : str) -> pd.DataFrame: + """ + Load MC sensor info for pre-2020 format files - extents = pd.read_hdf(file_name, 'MC/extents') + parameters + ---------- + file_name : str + Name of the file containing MC info. - sns = pd.read_hdf(file_name, 'MC/waveforms') + returns + ------- + sns : pd.DataFrame + DataFrame containing the sensor response info + contained in file_name + """ + extents = load_dst(file_name, 'MC', 'extents') + sns = load_dst(file_name, 'MC', 'waveforms') evt_sns = extents[['last_sns_data', 'evt_number']] - evt_sns.set_index('last_sns_data', inplace = True) + evt_sns.set_index('last_sns_data', inplace=True) + sns = sns.merge(evt_sns , + left_index = True, + right_index = True, + how = 'left') + sns.evt_number.fillna(method='bfill', inplace=True) + sns.evt_number = sns.evt_number.astype(int) + sns.index = sns.index.astype(int) + sns.rename(columns = {'evt_number': 'event_id'}, inplace=True) + return sns - sns = sns.merge(evt_sns , - left_index = True, - right_index = True, - how = 'left') - sns.evt_number.fillna(method='bfill', inplace = True) - sns['time'] = sns[sns.sensor_id.isin(pmt_ids)].time_bin * pmt_bin - sns.time.fillna(sns.time_bin * sipm_bin, inplace = True) +def get_mc_info(h5in): + """Return MC info bank""" - sns.evt_number = sns.evt_number.astype(int) - sns.rename(columns = {'evt_number': 'event_id'}, inplace = True) - sns.set_index(['event_id', 'sensor_id', 'time_bin'], inplace = True) + extents = h5in.root.MC.extents + hits = h5in.root.MC.hits + particles = h5in.root.MC.particles - return extents.evt_number.unique(), pmt_bin, sipm_bin, sns + try: + h5in.root.MC.particles[0] + except: + raise NoParticleInfoInFile('Trying to access particle information: this file could have sensor response only.') + + if len(h5in.root.MC.hits) == 0: + hits = np.zeros((0,), dtype=('3 pd.DataFrame: + """ + Convert the time bin to an event time. + + parameters + ---------- + sns_resp : pd.DataFrame + MC sensor response information as saved in the file. + sns_bins : pd.DataFrame + MC sensor bin sample width information + sns_pos : pd.DataFrame + Sensor position and type information. + + returns + ------- + sns_merge : pd.DataFrame + Merge of the parameter information with sensor + response information but with event time info + instead of time_bin info. + """ + sns_pos.set_index('sensor_name', inplace=True) + sns_merge = sns_resp.merge(sns_pos.join(sns_bins), on='sensor_id') + sns_merge['time'] = sns_merge.bin_width * sns_merge.time_bin + sns_merge.drop(['x', 'y', 'z', 'time_bin', 'bin_width'], + axis=1, inplace=True) + sns_merge.set_index(['event_id', 'sensor_id'], inplace=True) + return sns_merge def read_mcinfo_evt (mctables: (tb.Table, tb.Table, tb.Table, tb.Table), event_number: int, last_row=0, @@ -409,68 +872,9 @@ def read_mcinfo_evt (mctables: (tb.Table, tb.Table, tb.Table, tb.Table), event_n return hit_rows, particle_rows, generator_rows -def read_mcinfo(h5f, event_range=(0, int(1e9))) -> Mapping[int, Mapping[int, Sequence[MCParticle]]]: - mc_info = tbl.get_mc_info(h5f) - - h5extents = mc_info.extents - - events_in_file = len(h5extents) - - all_events = {} - - for iext in range(*event_range): - if iext >= events_in_file: - break - - current_event = {} - evt_number = h5extents[iext]['evt_number'] - hit_rows, particle_rows, generator_rows = read_mcinfo_evt(mc_info, evt_number, iext) - - for h5particle in particle_rows: - this_particle = h5particle['particle_indx'] - current_event[this_particle] = MCParticle(h5particle['particle_name'].decode('utf-8','ignore'), - h5particle['primary'], - h5particle['mother_indx'], - h5particle['initial_vertex'], - h5particle['final_vertex'], - h5particle['initial_volume'].decode('utf-8','ignore'), - h5particle['final_volume'].decode('utf-8','ignore'), - h5particle['momentum'], - h5particle['kin_energy'], - h5particle['creator_proc'].decode('utf-8','ignore')) - - for h5hit in hit_rows: - ipart = h5hit['particle_indx'] - current_particle = current_event[ipart] - - hit = MCHit(h5hit['hit_position'], - h5hit['hit_time'], - h5hit['hit_energy'], - h5hit['label'].decode('utf-8','ignore')) - - current_particle.hits.append(hit) - - evt_number = h5extents[iext]['evt_number'] - all_events[evt_number] = current_event - - return all_events - - -def compute_mchits_dict(mcevents:Mapping[int, Mapping[int, MCParticle]]) -> Mapping[int, Sequence[MCHit]]: - """Returns all hits in the event""" - mchits_dict = {} - for event_no, particle_dict in mcevents.items(): - hits = [] - for particle_no in particle_dict.keys(): - particle = particle_dict[particle_no] - hits.extend(particle.hits) - mchits_dict[event_no] = hits - return mchits_dict - - -def read_mchit_info(h5f, event_range=(0, int(1e9))) -> Mapping[int, Sequence[MCHit]]: +def _read_mchit_info(h5f, event_range=(0, int(1e9))) -> Mapping[int, Sequence[MCHit]]: """Returns all hits in the event""" - mc_info = tbl.get_mc_info(h5f) + mc_info = get_mc_info(h5f) h5extents = mc_info.extents events_in_file = len(h5extents) @@ -480,7 +884,6 @@ def read_mchit_info(h5f, event_range=(0, int(1e9))) -> Mapping[int, Sequence[MCH if iext >= events_in_file: break - current_event = {} evt_number = h5extents[iext]['evt_number'] hit_rows, _, _ = read_mcinfo_evt(mc_info, evt_number, iext, True) @@ -495,86 +898,3 @@ def read_mchit_info(h5f, event_range=(0, int(1e9))) -> Mapping[int, Sequence[MCH all_events[evt_number] = hits return all_events - - -def read_mcsns_response(h5f, event_range=(0, 1e9)) -> Mapping[int, Mapping[int, Waveform]]: - - h5config = h5f.root.MC.configuration - - bin_width_PMT = None - bin_width_SiPM = None - for row in h5config: - param_name = row['param_key'].decode('utf-8','ignore') - if param_name.find('time_binning') >= 0: - param_value = row['param_value'].decode('utf-8','ignore') - numb, unit = param_value.split() - if param_name.find('Pmt') > 0: - bin_width_PMT = float(numb) * units_dict[unit] - elif param_name.find('SiPM') >= 0: - bin_width_SiPM = float(numb) * units_dict[unit] - - - if bin_width_PMT is None: - raise SensorBinningNotFound - if bin_width_SiPM is None: - raise SensorBinningNotFound - - - h5extents = h5f.root.MC.extents - - try: - h5f.root.MC.waveforms[0] - except IndexError: - print('Error: this file has no sensor response information.') - - h5waveforms = h5f.root.MC.waveforms - - last_line_of_event = 'last_sns_data' - events_in_file = len(h5extents) - - all_events = {} - - iwvf = 0 - if event_range[0] > 0: - iwvf = h5extents[event_range[0]-1][last_line_of_event] + 1 - - for iext in range(*event_range): - if iext >= events_in_file: - break - - current_event = {} - - iwvf_end = h5extents[iext][last_line_of_event] - current_sensor_id = h5waveforms[iwvf]['sensor_id'] - time_bins = [] - charges = [] - while iwvf <= iwvf_end: - wvf_row = h5waveforms[iwvf] - sensor_id = wvf_row['sensor_id'] - - if sensor_id == current_sensor_id: - time_bins.append(wvf_row['time_bin']) - charges. append(wvf_row['charge']) - else: - bin_width = bin_width_PMT if current_sensor_id < 1000 else bin_width_SiPM - times = np.array(time_bins) * bin_width - - current_event[current_sensor_id] = Waveform(times, charges, bin_width) - - time_bins = [] - charges = [] - time_bins.append(wvf_row['time_bin']) - charges.append(wvf_row['charge']) - - current_sensor_id = sensor_id - - iwvf += 1 - - bin_width = bin_width_PMT if current_sensor_id < 1000 else bin_width_SiPM - times = np.array(time_bins) * bin_width - current_event[current_sensor_id] = Waveform(times, charges, bin_width) - - evt_number = h5extents[iext]['evt_number'] - all_events[evt_number] = current_event - - return all_events diff --git a/invisible_cities/io/mcinfo_io_test.py b/invisible_cities/io/mcinfo_io_test.py index 65c6278155..27e8f0e4e4 100644 --- a/invisible_cities/io/mcinfo_io_test.py +++ b/invisible_cities/io/mcinfo_io_test.py @@ -3,210 +3,392 @@ import pandas as pd import tables as tb -from glob import glob -from os.path import expandvars from numpy.testing import assert_allclose -from . mcinfo_io import load_mchits +from . dst_io import load_dst from . mcinfo_io import load_mchits_df -from . mcinfo_io import read_mchits_df -from . mcinfo_io import load_mcparticles +from . mcinfo_io import cast_mchits_to_dict from . mcinfo_io import load_mcparticles_df -from . mcinfo_io import read_mcparticles_df +from . mcinfo_io import get_event_numbers_in_file from . mcinfo_io import get_sensor_binning -from . mcinfo_io import load_mcsensor_response +from . mcinfo_io import get_sensor_types +from . mcinfo_io import get_mc_tbl_list +from . mcinfo_io import is_oldformat_file +from . mcinfo_io import load_mcsensor_positions from . mcinfo_io import load_mcsensor_response_df -from . mcinfo_io import mc_info_writer -from . mcinfo_io import read_mcinfo_evt +from . mcinfo_io import MCTableType +from . mcinfo_io import copy_mc_info +from . mcinfo_io import read_mc_tables +from . mcinfo_io import mc_writer +from . mcinfo_io import _read_mchit_info -from .. core import system_of_units as units -from .. core.exceptions import NoParticleInfoInFile +from .. core import system_of_units as units +from .. core.testing_utils import assert_dataframes_equal +from .. core.testing_utils import assert_MChit_equality -from .. reco.tbl_functions import get_mc_info - -from pytest import raises +from pytest import fixture from pytest import mark -parametrize = mark.parametrize - - -@mark.serial -@parametrize('skipped_evt, in_filename, out_filename', - ((0, 'Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.MCRD.h5', 'test_kr_mcinfo_skip_evt0.h5'), - (1, 'Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.MCRD.h5', 'test_kr_mcinfo_skip_evt1.h5'), - (700000000, 'mcfile_withgeneratorinfo_3evts_MCRD.h5', 'test_background_mcinfo_skip_evt700000000.h5'), - (640000000, 'mcfile_withgeneratorinfo_3evts_MCRD.h5', 'test_background_mcinfo_skip_evt640000000.h5'))) -def test_mc_info_writer_non_consecutive_events(output_tmpdir, ICDATADIR, skipped_evt, in_filename, out_filename): - """This test includes two different input files, with three events each. They differ in terms of their event number ordering. In the first (Kr) file, event numbers are ordered in ascending order. In the second (background) file, event numbers are unique but with random ordering. In particular, the Kr file contains event numbers [0, 1, 2], while the background file contains event numbers [700000000, 640000000, 40197500], in this order. - """ - filein = os.path.join(ICDATADIR, in_filename) - fileout = os.path.join(output_tmpdir, out_filename) - - with tb.open_file(filein) as h5in: - with tb.open_file(fileout, 'w') as h5out: - - mc_writer = mc_info_writer(h5out) - events_in = h5in.root.MC.extents[:]['evt_number'] - - mc_info = get_mc_info(h5in) - - #Skip the desired event - events_to_copy = [evt for evt in events_in if evt != skipped_evt] - - for evt in events_to_copy: - mc_writer(mc_info, evt) - - events_out = h5out.root.MC.extents[:]['evt_number'] - - np.testing.assert_array_equal(events_to_copy, events_out) - - -@mark.serial -@parametrize('file_to_check, evt_to_be_read', - (('test_kr_mcinfo_skip_evt0.h5', 1), - ('test_kr_mcinfo_skip_evt1.h5', 0), - ('test_kr_mcinfo_skip_evt1.h5', 2))) -def test_mc_info_writer_output_non_consecutive_events(output_tmpdir, ICDATADIR, krypton_MCRD_file, file_to_check, evt_to_be_read): - filein = krypton_MCRD_file - filecheck = os.path.join(output_tmpdir, file_to_check) - - with tb.open_file(filein) as h5in: - with tb.open_file(filecheck) as h5filtered: - mc_info = get_mc_info(h5in) - filtered_mc_info = get_mc_info(h5filtered) - # test the content of events to be sure that they are written - # correctly - hit_rows, particle_rows, generator_rows = read_mcinfo_evt(mc_info, - evt_to_be_read) - filtered_hit_rows, filtered_particle_rows, filtered_generator_rows = read_mcinfo_evt(filtered_mc_info, - evt_to_be_read) - - for hitr, filtered_hitr in zip(hit_rows, filtered_hit_rows): - assert np.allclose(hitr['hit_position'], filtered_hitr['hit_position']) - assert np.allclose(hitr['hit_time'] , filtered_hitr['hit_time']) - assert np.allclose(hitr['hit_energy'] , filtered_hitr['hit_energy']) - assert hitr['label'] == filtered_hitr['label'] - - for partr, filtered_partr in zip(particle_rows, filtered_particle_rows): - assert np.allclose(partr['initial_vertex'] , filtered_partr['initial_vertex']) - assert np.allclose(partr['final_vertex'] , filtered_partr['final_vertex']) - assert np.allclose(partr['momentum'] , filtered_partr['momentum']) - assert np.allclose(partr['kin_energy'] , filtered_partr['kin_energy']) - assert partr['particle_name'] == filtered_partr['particle_name'] - - -@mark.serial -@parametrize('file_to_check', - ('test_background_mcinfo_skip_evt700000000.h5', - 'test_background_mcinfo_skip_evt640000000.h5')) -def test_mc_info_writer_generatoroutput_non_consecutive_events(output_tmpdir, file_to_check): - filein = os.path.join(output_tmpdir, file_to_check) - - with tb.open_file(filein) as h5in: - mc_info = get_mc_info(h5in) - - # test the content of events to be sure that the extents rows are ion sync with generators rows - evt_numbers_in_extents = h5in.root.MC.extents[:]['evt_number'] - evt_numbers_in_generators = h5in.root.MC.generators[:]['evt_number'] - - np.testing.assert_array_equal(evt_numbers_in_extents, evt_numbers_in_generators) - - -def test_mc_info_writer_reset(output_tmpdir, ICDATADIR, krypton_MCRD_file): - filein = os.path.join(ICDATADIR, krypton_MCRD_file) - fileout = os.path.join(output_tmpdir, "test_mc_info_writer_reset.h5") +from pytest import raises +from pytest import warns - with tb.open_file(filein) as h5in: - with tb.open_file(fileout, 'w') as h5out: - mc_writer = mc_info_writer(h5out) - events_in = np.unique(h5in.root.MC.extents[:]['evt_number']) +@fixture(scope='module') +def mc_particle_and_hits_nexus_data(ICDATADIR): + X = [ -4.37347144e-02, -2.50248108e-02, + -3.25887166e-02, -3.25617939e-02 ] + Y = [ -1.37645766e-01, -1.67959690e-01, + -1.80502057e-01, -1.80206522e-01 ] + Z = [ 2.49938721e+02, 2.49911240e+02, + 2.49915543e+02, 2.49912308e+02 ] + E = [ 0.02225098, 0.00891293, 0.00582698, 0.0030091 ] + t = [ 0.00139908, 0.00198319, 0.00226054, 0.00236114 ] - assert mc_writer.last_row == 0 + vi = np.array([ 0., 0., 250., 0.]) + vf = np.array([-3.25617939e-02, -1.80206522e-01, + 2.49912308e+02, 2.36114440e-03]) - mc_writer(get_mc_info(h5in), events_in[0]) - assert mc_writer.last_row == 1 + p = np.array([-0.05745485, -0.18082699, -0.08050126]) - mc_writer.reset() - assert mc_writer.last_row == 0 + efile = os.path.join(ICDATADIR, 'electrons_40keV_z250_MCRD.h5') + Ep = 0.04 + name = 'e-' + nhits = 4 + return efile, name, vi, vf, p, Ep, nhits, X, Y, Z, E, t -def test_mc_info_writer_automatic_reset(output_tmpdir, ICDATADIR, krypton_MCRD_file, electron_MCRD_file): - fileout = os.path.join(output_tmpdir, "test_mc_info_writer_automatic_reset.h5") - with tb.open_file(fileout, "w") as h5out: - mc_writer = mc_info_writer(h5out) +@fixture(scope='module') +def mc_sensors_nexus_data(ICDATADIR): + pmt0_first = (0, 1) + pmt0_last = (670, 1) + pmt0_tot_samples = 54 - with tb.open_file(krypton_MCRD_file) as h5in: - events_in = np.unique(h5in.root.MC.extents[:]['evt_number']) - mc_writer(get_mc_info(h5in), events_in[0]) + sipm_id = 13016 + sipm = [(63, 3), (64, 2), (65, 1)] - # This would not be possible without automatic reset - with tb.open_file(electron_MCRD_file) as h5in: - events_in = np.unique(h5in.root.MC.extents[:]['evt_number']) - mc_writer(get_mc_info(h5in), events_in[0]) + efile = os.path.join(ICDATADIR, 'Kr83_full_nexus_v5_03_01_ACTIVE_7bar_1evt.sim.h5') - assert h5out.root.MC.extents [:].size == 2 - assert h5out.root.MC.hits [:].size == 12 - assert h5out.root.MC.particles[:].size == 3 + return efile, pmt0_first, pmt0_last, pmt0_tot_samples, sipm_id, sipm -def test_mc_info_writer_filter_first_event_of_first_file(output_tmpdir, ICDATADIR): - files_in = os.path.join(ICDATADIR , "Kr83_nexus_v5_02_08_ACTIVE_7bar_RWF.*.h5") - input_files = sorted(glob(expandvars(files_in))) +def test_get_mc_tbl_list_bad_MC_table(ICDATADIR): + file_in = os.path.join(ICDATADIR, "bad_mc_tables.h5") - file_out = os.path.join(output_tmpdir, "Kr83_nexus_v5_02_08_ACTIVE_7bar_RWF_all.h5") + with raises(KeyError): + get_mc_tbl_list(file_in) - with tb.open_file(file_out, "w") as h5out: - mc_writer = mc_info_writer(h5out) - skip_evt = True - for filename in input_files: - with tb.open_file(filename) as h5in: - events_in = np.unique(h5in.root.MC.extents[:]['evt_number']) - for evt in events_in: - if skip_evt: - skip_evt = False - continue - mc_writer(get_mc_info(h5in), evt) +def test_get_event_numbers_in_file_correct(ICDATADIR): + file_in = os.path.join(ICDATADIR , + "Kr83_nexus_v5_03_00_ACTIVE_7bar_10evts.MCRD.h5") - last_particle_list = h5out.root.MC.extents[:]['last_particle'] - last_hit_list = h5out.root.MC.extents[:]['last_hit'] + with tb.open_file(file_in) as h5in: + true_evt_numbers = h5in.root.Run.events.cols.evt_number[:] - assert all(x None: """ rwf_table.append(waveform.reshape(1, n_sensors, waveform_length)) return write_rwf + + +def buffer_writer(h5out, *, + run_number : int , + n_sens_eng : int , + n_sens_trk : int , + length_eng : int , + length_trk : int , + group_name : Optional[str] = None, + compression: Optional[str] = 'ZLIB4' + ) -> Callable[[int, List, List], None]: + """ + Generalised buffer writer which defines a raw waveform writer + for each type of sensor as well as an event info writer. + Each call gives a list of 'triggers' to be written as + separate events in the output. + + parameters + ---------- + run_number : int + Run number to be saved in runInfo. + n_sens_eng : int + Number of sensors in the energy plane. + n_sens_trk : int + Number of sensors in the tracking plane. + length_eng : int + Number of samples per waveform for energy plane. + length_trk : int + Number of samples per waveform for tracking plane. + group_name : Optional[str] default None + Group name within root where waveforms to be saved. + Default directly in root + compression : Optional[str] default 'ZLIB4' + Compression level for output file. + + returns + ------- + write_buffers : Callable + A function which takes event information + for the tracking and energy planes and + the event timestamps and saves to file. + """ + + eng_writer = rwf_writer(h5out, + group_name = group_name, + compression = compression, + table_name = 'pmtrd', + n_sensors = n_sens_eng, + waveform_length = length_eng) + + trk_writer = rwf_writer(h5out, + group_name = group_name, + compression = compression, + table_name = 'sipmrd', + n_sensors = n_sens_trk, + waveform_length = length_trk) + + run_and_event = partial(run_and_event_writer(h5out , + compression=compression), + run_number = run_number ) + + nexus_map = make_table(h5out, 'Run', 'eventMap', MCEventMap, + "event & nexus evt for each index", compression) + + def write_buffers(nexus_evt : int , + timestamps : List[ int], + events : List[Tuple]) -> None: + """ + Write run info and event waveforms to file. + + parameters + ---------- + nexus_evt : int + Event number from MC output file. + timestamps : List[int] + List of event times + events : List[Tuple] + List of tuples containing the energy and + tracking plane info for each identified 'trigger'. + """ + + for i, (t_stamp, (eng, trk)) in enumerate(zip(timestamps, events)): + ## The exact way to log MC event splitting + ## still to be decided. + run_and_event(event_number=nexus_evt, timestamp=t_stamp) + mrow = nexus_map.row + mrow["evt_number"] = nexus_evt + mrow[ "sub_evt"] = i + mrow.append() + ## + + eng_writer(eng) + trk_writer(trk) + return write_buffers diff --git a/invisible_cities/io/rwf_io_test.py b/invisible_cities/io/rwf_io_test.py index 19feaa3c2c..29f4b7b551 100644 --- a/invisible_cities/io/rwf_io_test.py +++ b/invisible_cities/io/rwf_io_test.py @@ -3,9 +3,11 @@ import numpy as np import tables as tb -from pytest import mark +from pytest import fixture +from pytest import mark -from . rwf_io import rwf_writer +from . rwf_io import rwf_writer +from . rwf_io import buffer_writer @mark.parametrize("group_name", (None, 'RD', 'BLR')) @@ -42,3 +44,51 @@ def test_rwf_writer(config_tmpdir, group_name): table = getattr(group, table_name) assert table.shape == (nevt, nsensor, nsample) assert np.all(test_data == table.read()) + + +@mark.parametrize("event triggers".split(), + ((2, [10]), (3, [10, 1100]), (4, [20]))) +def test_buffer_writer(config_tmpdir, event, triggers): + + run_number = -6400 + n_pmt = 12 + nsamp_pmt = 100 + n_sipm = 1792 + nsamp_sipm = 10 + + buffers = [(np.random.poisson(5, (n_pmt , nsamp_pmt )), + np.random.poisson(5, (n_sipm, nsamp_sipm))) for _ in triggers] + + out_name = os.path.join(config_tmpdir, 'test_buffers.h5') + with tb.open_file(out_name, 'w') as data_out: + + buffer_writer_ = buffer_writer(data_out, + run_number = run_number, + n_sens_eng = n_pmt, + n_sens_trk = n_sipm, + length_eng = nsamp_pmt, + length_trk = nsamp_sipm) + + buffer_writer_(event, triggers, buffers) + + pmt_wf = np.array([b[0] for b in buffers]) + sipm_wf = np.array([b[1] for b in buffers]) + with tb.open_file(out_name) as h5saved: + assert 'Run' in h5saved.root + assert 'pmtrd' in h5saved.root + assert 'sipmrd' in h5saved.root + assert 'events' in h5saved.root.Run + assert 'runInfo' in h5saved.root.Run + assert 'eventMap' in h5saved.root.Run + + nsaves = len(triggers) + assert len(h5saved.root.Run.events ) == nsaves + assert len(h5saved.root.Run.eventMap) == nsaves + assert len(h5saved.root.Run.runInfo ) == nsaves + assert np.all([r[0] == run_number for r in h5saved.root.Run.runInfo]) + + assert h5saved.root.pmtrd .shape == (nsaves, n_pmt , nsamp_pmt) + assert np.all(h5saved.root.pmtrd [:] == pmt_wf) + + assert h5saved.root.sipmrd.shape == (nsaves, n_sipm, nsamp_sipm) + assert np.all(h5saved.root.sipmrd[:] == sipm_wf) diff --git a/invisible_cities/io/trigger_io.py b/invisible_cities/io/trigger_io.py index fcbdcfe0b4..8f077e191c 100644 --- a/invisible_cities/io/trigger_io.py +++ b/invisible_cities/io/trigger_io.py @@ -1,6 +1,5 @@ from functools import partial -import numpy as np import tables as tb from .. evm import nh5 as table_formats diff --git a/invisible_cities/io/voxels_io.py b/invisible_cities/io/voxels_io.py index fbd159ff24..953357ec1f 100644 --- a/invisible_cities/io/voxels_io.py +++ b/invisible_cities/io/voxels_io.py @@ -1,7 +1,6 @@ import tables -from . dst_io import load_dst from .. io.table_io import make_table from .. evm.event_model import Voxel from .. evm.event_model import VoxelCollection diff --git a/invisible_cities/reco/calib_functions.py b/invisible_cities/reco/calib_functions.py index 69ce9cc3f1..b8486c11c7 100644 --- a/invisible_cities/reco/calib_functions.py +++ b/invisible_cities/reco/calib_functions.py @@ -8,14 +8,14 @@ from scipy.signal import find_peaks_cwt from enum import auto -from .. core.system_of_units_c import units -from .. core.core_functions import in_range -from .. core.stat_functions import poisson_sigma -from .. core import fit_functions as fitf -from .. database import load_db as DB -from .. types.ic_types import AutoNameEnumBase -from .. evm.ic_containers import SensorParams -from .. evm.ic_containers import PedestalParams +from .. core import system_of_units as units +from .. core.core_functions import in_range +from .. core.stat_functions import poisson_sigma +from .. core import fit_functions as fitf +from .. database import load_db as DB +from .. types.ic_types import AutoNameEnumBase +from .. evm.ic_containers import SensorParams +from .. evm.ic_containers import PedestalParams def bin_waveforms(waveforms, bins): diff --git a/invisible_cities/reco/calib_functions_test.py b/invisible_cities/reco/calib_functions_test.py index f0d8b2950d..d954d9da13 100644 --- a/invisible_cities/reco/calib_functions_test.py +++ b/invisible_cities/reco/calib_functions_test.py @@ -9,14 +9,14 @@ from pytest import raises from scipy.signal import find_peaks_cwt -from . import calib_functions as cf -from .. reco import tbl_functions as tbl -from .. core import fit_functions as fitf -from .. core.stat_functions import poisson_sigma -from .. core.system_of_units_c import units -from .. evm.nh5 import SensorTable -from . calib_functions import SensorType -from .. cities.components import get_run_number +from . import calib_functions as cf +from .. reco import tbl_functions as tbl +from .. core import fit_functions as fitf +from .. core import system_of_units as units +from .. core.stat_functions import poisson_sigma +from .. evm.nh5 import SensorTable +from . calib_functions import SensorType +from .. cities.components import get_run_number def test_bin_waveforms(): @@ -216,7 +216,6 @@ def test_compute_seeds_from_spectrum(ICDATADIR): PATH_IN = os.path.join(ICDATADIR, 'sipmcalspectra_R6358.h5') h5in = tb.open_file(PATH_IN, 'r') - run_no = get_run_number(h5in) specsL = np.array(h5in.root.HIST.sipm_spe).sum(axis=0) specsD = np.array(h5in.root.HIST.sipm_dark).sum(axis=0) diff --git a/invisible_cities/reco/calib_sensors_functions.py b/invisible_cities/reco/calib_sensors_functions.py index 58386f975f..bb2a2d15a9 100644 --- a/invisible_cities/reco/calib_sensors_functions.py +++ b/invisible_cities/reco/calib_sensors_functions.py @@ -44,7 +44,7 @@ def wf_mode(wf): def zero_masked(fn): """ protection for mean and median - so that we get the correct answer in case of + so that we get the correct answer in case of zero suppressed data """ @wraps(fn) @@ -90,7 +90,6 @@ def subtract_baseline(wfs, *, bls_mode=BlsMode.mean): elif bls_mode is BlsMode.scipymode: return wfs - scipy_mode(wfs, axis=1) else: raise TypeError(f"Unrecognized baseline subtraction option: {bls_mode}") - return bls def calibrate_wfs(wfs, adc_to_pes): diff --git a/invisible_cities/reco/calib_sensors_functions_test.py b/invisible_cities/reco/calib_sensors_functions_test.py index e60fe58aa6..001c0dab26 100644 --- a/invisible_cities/reco/calib_sensors_functions_test.py +++ b/invisible_cities/reco/calib_sensors_functions_test.py @@ -1,5 +1,4 @@ import numpy as np -import scipy.signal as signal from functools import reduce from operator import add @@ -145,7 +144,7 @@ def test_subtract_baseline_valid_options_sanity_check(gaussian_sipm_signal, bls_ def test_subtract_baseline_raises_TypeError(wrong_bls_mode): dummy = np.empty((2, 2)) with raises(TypeError): - bls = csf.subtract_baseline(dummy, bls_mode=wrong_bls_mode) + csf.subtract_baseline(dummy, bls_mode=wrong_bls_mode) def test_calibrate_wfs(gaussian_sipm_signal_wo_baseline): diff --git a/invisible_cities/reco/corrections.py b/invisible_cities/reco/corrections.py index 42cf86c5a0..6d9ae4e1ea 100644 --- a/invisible_cities/reco/corrections.py +++ b/invisible_cities/reco/corrections.py @@ -1,192 +1,360 @@ -from functools import partial -from itertools import product +import numpy as np +import pandas as pd + +from pandas import DataFrame +from pandas import Series +from dataclasses import dataclass +from typing import Callable +from typing import Optional +from enum import auto + +from .. core.core_functions import in_range +from .. core import system_of_units as units +from .. core.exceptions import TimeEvolutionTableMissing +from .. types.ic_types import AutoNameEnumBase + + +@dataclass +class ASectorMap: # Map in chamber sector containing average of pars + chi2 : DataFrame + e0 : DataFrame + lt : DataFrame + e0u : DataFrame + ltu : DataFrame + mapinfo : Optional[Series] + t_evol : Optional[DataFrame] + +def read_maps(filename : str)->ASectorMap: -import numpy as np -from scipy.interpolate import griddata + """ + Read 'filename' variable and creates ASectorMap class. + If the map corresponds to a data run (run_number>0), + ASectorMap will also contain a DataFrame with time evolution information. + + Parameters + ---------- + filename : string + Name of the file that contains the correction maps. + + Returns + ------- + ASectorMap: + +@dataclass +class ASectorMap: + chi2 : DataFrame # chi2 value for each bin + e0 : DataFrame # geometric map + lt : DataFrame # lifetime map + e0u : DataFrame # uncertainties of geometric map + ltu : DataFrame # uncertainties of lifetime map + mapinfo : Optional[Series] # series with some info about the + t_evol : Optional[DataFrame] # time evolution of some parameters + (only for data) + """ + + chi2 = pd.read_hdf(filename, 'chi2') + e0 = pd.read_hdf(filename, 'e0') + e0u = pd.read_hdf(filename, 'e0u') + lt = pd.read_hdf(filename, 'lt') + ltu = pd.read_hdf(filename, 'ltu') + mapinfo = pd.read_hdf(filename, 'mapinfo') + + if mapinfo.run_number>0: + try: + t_evol = pd.read_hdf(filename, 'time_evolution') + except: + t_evol = None + maps = ASectorMap(chi2, e0, lt, e0u, ltu, mapinfo, t_evol) + + else: maps = ASectorMap(chi2, e0, lt, e0u, ltu, mapinfo, None) + + return maps + + +def maps_coefficient_getter(mapinfo : Series, + map_df : DataFrame) -> Callable: + """ + For a given correction map, + it returns a function that yields the values of map + for a given (X,Y) position. + + Parameters + ---------- + mapinfo : Series + Stores some information about the map + (run number, number of X-Y bins, X-Y range) + map_df : DataFrame + DataFrame of a correction map (lt or e0) + + Returns + ------- + A function that returns the value of the 'map_df' map + for a given (X,Y) position + """ + + binsx = np.linspace(mapinfo.xmin, mapinfo.xmax, mapinfo.nx + 1) + binsy = np.linspace(mapinfo.ymin, mapinfo.ymax, mapinfo.ny + 1) -from ..core import fit_functions as fitf -from ..core.exceptions import ParameterNotSet -from .. evm.ic_containers import Measurement + def get_maps_coefficient(x : np.array, y : np.array) -> np.array: + ix = np.digitize(x, binsx) - 1 + iy = np.digitize(y, binsy) - 1 + valid = in_range(x, mapinfo.xmin, mapinfo.xmax) + valid &= in_range(y, mapinfo.ymin, mapinfo.ymax) + output = np.full(len(valid), np.nan, dtype=np.float) -opt_nearest = {"interp_method": "nearest"} -opt_linear = {"interp_method": "linear" , - "default_f" : 1 , - "default_u" : 0 } -opt_cubic = {"interp_method": "cubic" , - "default_f" : 1 , - "default_u" : 0 } + output[valid] = map_df.values[iy[valid], ix[valid]] + return output + return get_maps_coefficient -class Correction: + +def correct_geometry_(CE : np.array) -> np.array: """ - Interface for accessing any kind of corrections. + Computes the geometric correction factor + for a given correction coefficient Parameters ---------- - xs : np.ndarray - Array of coordinates corresponding to each correction. - fs : np.ndarray - Array of corrections or the values used for computing them. - us : np.ndarray - Array of uncertainties or the values used for computing them. - norm_strategy : False or string - Flag to set the normalization option. Accepted values: - - False: Do not normalize. - - "max": Normalize to maximum energy encountered. - - "index": Normalize to the energy placed to index (i,j). - default_f, default_u : floats - Default correction and uncertainty for missing values (where fs = 0). - """ - - def __init__(self, - xs, fs, us, - norm_strategy = None, - norm_opts = {}, - interp_method = "nearest", - default_f = 0, - default_u = 0): - - self._xs = [np.array( x, dtype=float) for x in xs] - self._fs = np.array(fs, dtype=float) - self._us = np.array(us, dtype=float) - - self.norm_strategy = norm_strategy - self.norm_opts = norm_opts - self.interp_method = interp_method - self.default_f = default_f - self.default_u = default_u - - self._normalize ( norm_strategy, - norm_opts ) - self._init_interpolator(interp_method , default_f, default_u) - - def __call__(self, *xs): - """ - Compute the correction factor. - - Parameters - ---------- - *x: Sequence of nd.arrays - Each array is one coordinate. The number of coordinates must match - that of the `xs` array in the init method. - """ - # In order for this to work well both for arrays and scalars - arrays = len(np.shape(xs)) > 1 - if arrays: - xs = np.stack(xs, axis=1) - - value = self._get_value (xs).flatten() - uncert = self._get_uncertainty(xs).flatten() - return (Measurement(value , uncert ) if arrays else - Measurement(value[0], uncert[0])) - - def _init_interpolator(self, method, default_f, default_u): - coordinates = np.array(list(product(*self._xs))) - self._get_value = partial(griddata, - coordinates, - self._fs.flatten(), - method = method, - fill_value = default_f) - - self._get_uncertainty = partial(griddata, - coordinates, - self._us.flatten(), - method = method, - fill_value = default_u) - - def _normalize(self, strategy, opts): - if not strategy : return - - elif strategy == "const" : - if "value" not in opts: - raise ParameterNotSet("Normalization strategy 'const' requires" - "the normalization option 'value'") - f_ref = opts["value"] - u_ref = 0 - - elif strategy == "max" : - flat_index = np.argmax(self._fs) - mult_index = np.unravel_index(flat_index, self._fs.shape) - f_ref = self._fs[mult_index] - u_ref = self._us[mult_index] - - elif strategy == "center": - index = tuple(i // 2 for i in self._fs.shape) - f_ref = self._fs[index] - u_ref = self._us[index] - - elif strategy == "index" : - if "index" not in opts: - raise ParameterNotSet("Normalization strategy 'index' requires" - "the normalization option 'index'") - index = opts["index"] - f_ref = self._fs[index] - u_ref = self._us[index] - - else: - raise ValueError("Normalization strategy not recognized: {}".format(strategy)) - - assert f_ref > 0, "Invalid reference value." - - valid = (self._fs > 0) & (self._us > 0) - valid_fs = self._fs[valid].copy() - valid_us = self._us[valid].copy() - - # Redefine and propagate uncertainties as: - # u(F) = F sqrt(u(F)**2/F**2 + u(Fref)**2/Fref**2) - self._fs[ valid] = f_ref / valid_fs - self._us[ valid] = np.sqrt((valid_us / valid_fs)**2 + - ( u_ref / f_ref )**2 ) - self._us[ valid] *= self._fs[valid] - - # Set invalid to defaults - self._fs[~valid] = self.default_f - self._us[~valid] = self.default_u - - def __eq__(self, other): - for i, x in enumerate(self._xs): - if not np.allclose(x, other._xs[i]): - return False - - if not np.allclose(self._fs, other._fs): - return False - - if not np.allclose(self._us, other._us): - return False - - return True - - -class Fcorrection: - def __init__(self, f, u_f, pars): - self._f = lambda *x: f(*x, *pars) - self._u_f = lambda *x: u_f(*x, *pars) - - def __call__(self, *x): - return Measurement(self._f(*x), self._u_f(*x)) - - -def LifetimeCorrection(LT, u_LT): - fun = lambda z, LT, u_LT=0: fitf.expo(z, 1, LT) - u_fun = lambda z, LT, u_LT : z * u_LT / LT**2 * fun(z, LT) - return Fcorrection(fun, u_fun, (LT, u_LT)) - - -def LifetimeXYCorrection(pars, u_pars, xs, ys, **kwargs): - LTs = Correction((xs, ys), pars, u_pars, **kwargs) - return (lambda z, x, y: LifetimeCorrection(*LTs(x, y))(z)) - - -def LifetimeRCorrection(pars, u_pars): - def LTfun(z, r, a, b, c, u_a, u_b, u_c): - LT = a - b * r * np.exp(r / c) - return fitf.expo(z, 1, LT) - - def u_LTfun(z, r, a, b, c, u_a, u_b, u_c): - LT = a - b * r * np.exp(r / c) - u_LT = (u_a**2 + u_b**2 * np.exp(2 * r / c) + - u_c**2 * b**2 * r**2 * np.exp(2 * r / c) / c**4)**0.5 - return z * u_LT / LT**2 * LTfun(z, r, a, b, c, u_a, u_b, u_c) + CE : np.array + Array with geometric correction coefficients + + Returns + ------- + An array with geometric correction factors + """ + + return 1/CE + + +def correct_lifetime_(Z : np.array, LT : np.array) -> np.array: + """ + Computes the lifetime correction factor + for a given correction coefficient + + Parameters + ---------- + LT : np.array + Array with lifetime correction coefficients + + Returns + ------- + An array with lifetime correction factors + """ + + return np.exp(Z / LT) + + +def time_coefs_corr(time_evt : np.array, + times_evol : np.array, + par : np.array, + par_u : np.array)-> np.array: + """ + Computes a time-dependence parameter that will correct the + correction coefficient for taking into account time evolution. + + Parameters + ---------- + time_evt : np.array + Array with timestamps for each hit (is the same for all hit of the same event). + times_evol : np.array + Time intervals to perform the interpolation. + par : np.array + Time evolution of a certain parameter (e.g. lt or e0). + Each value is associated to a times_evol one. + par_u : np.array + Time evolution of the uncertainty of a certain parameter. + Each value is associated to a times_evol one. + + Returns + ------- + An array with the computed value. + """ + par_mean = np.average(par, weights = 1/par_u) + par_i = np.interp(time_evt, times_evol, par) + par_factor = par_i/par_mean + return par_factor + + +def get_df_to_z_converter(map_te: ASectorMap) -> Callable: + """ + For given map, it returns a function that provides the conversion + from drift time to z position using the mean drift velocity. + + Parameters + ---------- + map_te : AsectorMap + Correction map with time evolution of some kdst parameters. + + Returns + ------- + A function that returns z converted array for a given drift time input + array. + """ + try: + assert map_te.t_evol is not None + except(AttributeError , AssertionError): + raise TimeEvolutionTableMissing("No temp_map table provided in the map") + + dv_vs_time = map_te.t_evol.dv + dv = np.mean(dv_vs_time) + def df_to_z_converter(dt): + return dt*dv + + return df_to_z_converter + + +class norm_strategy(AutoNameEnumBase): + mean = auto() + max = auto() + kr = auto() + custom = auto() + + +def get_normalization_factor(map_e0 : ASectorMap, + norm_strat: norm_strategy = norm_strategy.max, + norm_value: Optional[float] = None + ) -> float: + """ + For given map, it returns a factor that provides the conversion + from pes time to kr energy scale using the selected normalization + strategy. + + Parameters + ---------- + map_e0 : AsectorMap + Correction map for geometric corrections. + norm_strat : norm_strategy + Normalization strategy used when correcting the energy. + norm_value : float (Optional) + Normalization scale when custom strategy is selected. + + Returns + ------- + Factor for the kr energy scale conversion. + """ + if norm_strat is norm_strategy.max: + norm_value = map_e0.e0.max().max() + elif norm_strat is norm_strategy.mean: + norm_value = np.mean(np.mean(map_e0.e0)) + elif norm_strat is norm_strategy.kr: + norm_value = 41.5575 * units.keV + elif norm_strat is norm_strategy.custom: + if norm_value is None: + s = "If custom strategy is selected for normalization" + s += " user must specify the norm_value" + raise ValueError(s) + else: + s = "None of the current available normalization" + s += " strategies was selected" + raise ValueError(s) + + return norm_value + + +def apply_all_correction_single_maps(map_e0 : ASectorMap, + map_lt : ASectorMap, + map_te : Optional[ASectorMap] = None, + apply_temp : bool = True, + norm_strat : norm_strategy = norm_strategy.max, + norm_value : Optional[float] = None + ) -> Callable: + """ + For a map for each correction, it returns a function + that provides a correction factor for a + given hit collection when (x,y,z,time) is provided. + + Parameters + ---------- + map_e0 : AsectorMap + Correction map for geometric corrections. + map_lt : AsectorMap + Correction map for lifetime corrections. + map_te : AsectorMap (optional) + Correction map with time evolution of some kdst parameters. + apply_temp : Bool + If True, time evolution will be taken into account. + norm_strat : AutoNameEnumBase + Provides the desired normalization to be used. + norm_value : Float(optional) + If norm_strat is selected to be custom, user must provide the + desired scale. + krscale_output : Bool + If true, the returned factor will take into account the scaling + from pes to the Kr energy scale. + Returns + ------- + A function that returns time correction factor without passing a map. + """ + + normalization = get_normalization_factor(map_e0, norm_strat, norm_value) + + get_xy_corr_fun = maps_coefficient_getter(map_e0.mapinfo, map_e0.e0) + get_lt_corr_fun = maps_coefficient_getter(map_lt.mapinfo, map_lt.lt) + + if apply_temp: + try: + assert map_te.t_evol is not None + except(AttributeError , AssertionError): + raise TimeEvolutionTableMissing("apply_temp is true while temp_map is not provided") + + evol_table = map_te.t_evol + temp_correct_e0 = lambda t : time_coefs_corr(t, + evol_table.ts, + evol_table.e0, + evol_table.e0u) + temp_correct_lt = lambda t : time_coefs_corr(t, + evol_table.ts, + evol_table['lt'], + evol_table.ltu) + e0evol_vs_t = temp_correct_e0 + ltevol_vs_t = temp_correct_lt + + else: + e0evol_vs_t = lambda x : np.ones_like(x) + ltevol_vs_t = lambda x : np.ones_like(x) + + def total_correction_factor(x : np.array, + y : np.array, + z : np.array, + t : np.array)-> np.array: + geo_factor = correct_geometry_(get_xy_corr_fun(x,y) * e0evol_vs_t(t)) + lt_factor = correct_lifetime_(z, get_lt_corr_fun(x,y) * ltevol_vs_t(t)) + factor = geo_factor * lt_factor * normalization + return factor + + return total_correction_factor + +def apply_all_correction(maps : ASectorMap , + apply_temp : bool = True , + norm_strat : norm_strategy = norm_strategy.max, + norm_value : Optional[float] = None + )->Callable: + """ + Returns a function to get all correction factor for a + given hit collection when (x,y,z,time) is provided, + if an unique correction map is wanted to be used + + Parameters + ---------- + maps : AsectorMap + Selected correction map for doing geometric and lifetime correction. + apply_temp : Bool + If True, time evolution will be taken into account. + norm_strat : AutoNameEnumBase + Provides the desired normalization to be used. + norm_value : Float(optional) + If norm_strat is selected to be custom, user must provide the + desired scale. + krscale_output : Bool + If true, the returned factor will take into account the scaling + from pes to the Kr energy scale. + + Returns + ------- + A function that returns complete time correction factor + """ - return Fcorrection(LTfun, u_LTfun, np.concatenate([pars, u_pars])) + return apply_all_correction_single_maps(maps, maps, maps, + apply_temp, norm_strat, + norm_value) diff --git a/invisible_cities/reco/corrections_new.py b/invisible_cities/reco/corrections_new.py deleted file mode 100644 index a444fbb535..0000000000 --- a/invisible_cities/reco/corrections_new.py +++ /dev/null @@ -1,362 +0,0 @@ -import numpy as np -import pandas as pd - -from pandas import DataFrame -from pandas import Series -from dataclasses import dataclass -from typing import Callable -from typing import List -from typing import Optional -from enum import auto - -from .. core.core_functions import in_range -from .. core import system_of_units as units -from .. core.exceptions import TimeEvolutionTableMissing -from .. types.ic_types import AutoNameEnumBase -from .. evm.event_model import Hit - - -@dataclass -class ASectorMap: # Map in chamber sector containing average of pars - chi2 : DataFrame - e0 : DataFrame - lt : DataFrame - e0u : DataFrame - ltu : DataFrame - mapinfo : Optional[Series] - t_evol : Optional[DataFrame] - -def read_maps(filename : str)->ASectorMap: - - """ - Read 'filename' variable and creates ASectorMap class. - If the map corresponds to a data run (run_number>0), - ASectorMap will also contain a DataFrame with time evolution information. - - Parameters - ---------- - filename : string - Name of the file that contains the correction maps. - - Returns - ------- - ASectorMap: - -@dataclass -class ASectorMap: - chi2 : DataFrame # chi2 value for each bin - e0 : DataFrame # geometric map - lt : DataFrame # lifetime map - e0u : DataFrame # uncertainties of geometric map - ltu : DataFrame # uncertainties of lifetime map - mapinfo : Optional[Series] # series with some info about the - t_evol : Optional[DataFrame] # time evolution of some parameters - (only for data) - """ - - chi2 = pd.read_hdf(filename, 'chi2') - e0 = pd.read_hdf(filename, 'e0') - e0u = pd.read_hdf(filename, 'e0u') - lt = pd.read_hdf(filename, 'lt') - ltu = pd.read_hdf(filename, 'ltu') - mapinfo = pd.read_hdf(filename, 'mapinfo') - - if mapinfo.run_number>0: - try: - t_evol = pd.read_hdf(filename, 'time_evolution') - except: - t_evol = None - maps = ASectorMap(chi2, e0, lt, e0u, ltu, mapinfo, t_evol) - - else: maps = ASectorMap(chi2, e0, lt, e0u, ltu, mapinfo, None) - - return maps - - -def maps_coefficient_getter(mapinfo : Series, - map_df : DataFrame) -> Callable: - """ - For a given correction map, - it returns a function that yields the values of map - for a given (X,Y) position. - - Parameters - ---------- - mapinfo : Series - Stores some information about the map - (run number, number of X-Y bins, X-Y range) - map_df : DataFrame - DataFrame of a correction map (lt or e0) - - Returns - ------- - A function that returns the value of the 'map_df' map - for a given (X,Y) position - """ - - binsx = np.linspace(mapinfo.xmin, mapinfo.xmax, mapinfo.nx + 1) - binsy = np.linspace(mapinfo.ymin, mapinfo.ymax, mapinfo.ny + 1) - - def get_maps_coefficient(x : np.array, y : np.array) -> np.array: - ix = np.digitize(x, binsx) - 1 - iy = np.digitize(y, binsy) - 1 - - valid = in_range(x, mapinfo.xmin, mapinfo.xmax) - valid &= in_range(y, mapinfo.ymin, mapinfo.ymax) - output = np.full(len(valid), np.nan, dtype=np.float) - - output[valid] = map_df.values[iy[valid], ix[valid]] - return output - - return get_maps_coefficient - - -def correct_geometry_(CE : np.array) -> np.array: - """ - Computes the geometric correction factor - for a given correction coefficient - - Parameters - ---------- - CE : np.array - Array with geometric correction coefficients - - Returns - ------- - An array with geometric correction factors - """ - - return 1/CE - - -def correct_lifetime_(Z : np.array, LT : np.array) -> np.array: - """ - Computes the lifetime correction factor - for a given correction coefficient - - Parameters - ---------- - LT : np.array - Array with lifetime correction coefficients - - Returns - ------- - An array with lifetime correction factors - """ - - return np.exp(Z / LT) - - -def time_coefs_corr(time_evt : np.array, - times_evol : np.array, - par : np.array, - par_u : np.array)-> np.array: - """ - Computes a time-dependence parameter that will correct the - correction coefficient for taking into account time evolution. - - Parameters - ---------- - time_evt : np.array - Array with timestamps for each hit (is the same for all hit of the same event). - times_evol : np.array - Time intervals to perform the interpolation. - par : np.array - Time evolution of a certain parameter (e.g. lt or e0). - Each value is associated to a times_evol one. - par_u : np.array - Time evolution of the uncertainty of a certain parameter. - Each value is associated to a times_evol one. - - Returns - ------- - An array with the computed value. - """ - par_mean = np.average(par, weights = 1/par_u) - par_i = np.interp(time_evt, times_evol, par) - par_factor = par_i/par_mean - return par_factor - - -def get_df_to_z_converter(map_te: ASectorMap) -> Callable: - """ - For given map, it returns a function that provides the conversion - from drift time to z position using the mean drift velocity. - - Parameters - ---------- - map_te : AsectorMap - Correction map with time evolution of some kdst parameters. - - Returns - ------- - A function that returns z converted array for a given drift time input - array. - """ - try: - assert map_te.t_evol is not None - except(AttributeError , AssertionError): - raise TimeEvolutionTableMissing("No temp_map table provided in the map") - - dv_vs_time = map_te.t_evol.dv - dv = np.mean(dv_vs_time) - def df_to_z_converter(dt): - return dt*dv - - return df_to_z_converter - - -class norm_strategy(AutoNameEnumBase): - mean = auto() - max = auto() - kr = auto() - custom = auto() - - -def get_normalization_factor(map_e0 : ASectorMap, - norm_strat: norm_strategy = norm_strategy.max, - norm_value: Optional[float] = None - ) -> float: - """ - For given map, it returns a factor that provides the conversion - from pes time to kr energy scale using the selected normalization - strategy. - - Parameters - ---------- - map_e0 : AsectorMap - Correction map for geometric corrections. - norm_strat : norm_strategy - Normalization strategy used when correcting the energy. - norm_value : float (Optional) - Normalization scale when custom strategy is selected. - - Returns - ------- - Factor for the kr energy scale conversion. - """ - if norm_strat is norm_strategy.max: - norm_value = map_e0.e0.max().max() - elif norm_strat is norm_strategy.mean: - norm_value = np.mean(np.mean(map_e0.e0)) - elif norm_strat is norm_strategy.kr: - norm_value = 41.5575 * units.keV - elif norm_strat is norm_strategy.custom: - if norm_value is None: - s = "If custom strategy is selected for normalization" - s += " user must specify the norm_value" - raise ValueError(s) - else: - s = "None of the current available normalization" - s += " strategies was selected" - raise ValueError(s) - - return norm_value - - -def apply_all_correction_single_maps(map_e0 : ASectorMap, - map_lt : ASectorMap, - map_te : Optional[ASectorMap] = None, - apply_temp : bool = True, - norm_strat : norm_strategy = norm_strategy.max, - norm_value : Optional[float] = None - ) -> Callable: - """ - For a map for each correction, it returns a function - that provides a correction factor for a - given hit collection when (x,y,z,time) is provided. - - Parameters - ---------- - map_e0 : AsectorMap - Correction map for geometric corrections. - map_lt : AsectorMap - Correction map for lifetime corrections. - map_te : AsectorMap (optional) - Correction map with time evolution of some kdst parameters. - apply_temp : Bool - If True, time evolution will be taken into account. - norm_strat : AutoNameEnumBase - Provides the desired normalization to be used. - norm_value : Float(optional) - If norm_strat is selected to be custom, user must provide the - desired scale. - krscale_output : Bool - If true, the returned factor will take into account the scaling - from pes to the Kr energy scale. - Returns - ------- - A function that returns time correction factor without passing a map. - """ - - normalization = get_normalization_factor(map_e0, norm_strat, norm_value) - - get_xy_corr_fun = maps_coefficient_getter(map_e0.mapinfo, map_e0.e0) - get_lt_corr_fun = maps_coefficient_getter(map_lt.mapinfo, map_lt.lt) - - if apply_temp: - try: - assert map_te.t_evol is not None - except(AttributeError , AssertionError): - raise TimeEvolutionTableMissing("apply_temp is true while temp_map is not provided") - - evol_table = map_te.t_evol - temp_correct_e0 = lambda t : time_coefs_corr(t, - evol_table.ts, - evol_table.e0, - evol_table.e0u) - temp_correct_lt = lambda t : time_coefs_corr(t, - evol_table.ts, - evol_table['lt'], - evol_table.ltu) - e0evol_vs_t = temp_correct_e0 - ltevol_vs_t = temp_correct_lt - - else: - e0evol_vs_t = lambda x : np.ones_like(x) - ltevol_vs_t = lambda x : np.ones_like(x) - - def total_correction_factor(x : np.array, - y : np.array, - z : np.array, - t : np.array)-> np.array: - geo_factor = correct_geometry_(get_xy_corr_fun(x,y) * e0evol_vs_t(t)) - lt_factor = correct_lifetime_(z, get_lt_corr_fun(x,y) * ltevol_vs_t(t)) - factor = geo_factor * lt_factor * normalization - return factor - - return total_correction_factor - -def apply_all_correction(maps : ASectorMap , - apply_temp : bool = True , - norm_strat : norm_strategy = norm_strategy.max, - norm_value : Optional[float] = None - )->Callable: - """ - Returns a function to get all correction factor for a - given hit collection when (x,y,z,time) is provided, - if an unique correction map is wanted to be used - - Parameters - ---------- - maps : AsectorMap - Selected correction map for doing geometric and lifetime correction. - apply_temp : Bool - If True, time evolution will be taken into account. - norm_strat : AutoNameEnumBase - Provides the desired normalization to be used. - norm_value : Float(optional) - If norm_strat is selected to be custom, user must provide the - desired scale. - krscale_output : Bool - If true, the returned factor will take into account the scaling - from pes to the Kr energy scale. - - Returns - ------- - A function that returns complete time correction factor - """ - - return apply_all_correction_single_maps(maps, maps, maps, - apply_temp, norm_strat, - norm_value) diff --git a/invisible_cities/reco/corrections_new_test.py b/invisible_cities/reco/corrections_new_test.py deleted file mode 100644 index 6a6b8053f0..0000000000 --- a/invisible_cities/reco/corrections_new_test.py +++ /dev/null @@ -1,320 +0,0 @@ -import os -import numpy as np -from . corrections_new import maps_coefficient_getter -from . corrections_new import read_maps -from . corrections_new import ASectorMap -from . corrections_new import correct_geometry_ -from . corrections_new import correct_lifetime_ -from . corrections_new import time_coefs_corr -from . corrections_new import get_df_to_z_converter -from . corrections_new import norm_strategy -from . corrections_new import get_normalization_factor -from . corrections_new import apply_all_correction_single_maps -from . corrections_new import apply_all_correction - -from pytest import fixture, mark -from numpy.testing import assert_allclose -from numpy.testing import assert_array_equal -from numpy.testing import assert_raises - -from hypothesis.strategies import floats -from hypothesis.strategies import integers -from hypothesis.strategies import composite -from hypothesis.strategies import lists -from hypothesis import given - -from invisible_cities.core.testing_utils import random_length_float_arrays -from invisible_cities.core.testing_utils import float_arrays -from invisible_cities.core.exceptions import TimeEvolutionTableMissing -from invisible_cities.core import system_of_units as units - -@fixture(scope='session') -def map_filename(ICDATADIR): - test_file = "kr_emap_xy_constant_values.h5" - test_file = os.path.join(ICDATADIR, test_file) - return test_file - -@fixture(scope='session') -def map_filename_MC(ICDATADIR): - test_file = "kr_emap_xy_constant_values_RN_negat.h5" - test_file = os.path.join(ICDATADIR, test_file) - return test_file - -@fixture -def toy_corrections(correction_map_filename): - xs, ys = np.meshgrid(np.linspace(-199,199,5), np.linspace(-199,199,5)) - xs = xs.flatten() - ys = ys.flatten() - zs = np.ones(25) - - ts = np.array([1.54513805e+09, 1.54514543e+09, 1.54515280e+09, 1.54516018e+09, - 1.54516755e+09, 1.54517493e+09, 1.54518230e+09, 1.54518968e+09, - 1.54519705e+09, 1.54520443e+09, 1.54521180e+09, 1.54521918e+09, - 1.54522655e+09, 1.54523393e+09, 1.54524130e+09, 1.54524868e+09, - 1.54525605e+09, 1.54526343e+09, 1.54527080e+09, 1.54527818e+09, - 1.54528555e+09, 1.54529293e+09, 1.54530030e+09, 1.54530768e+09, - 1.54531505e+09]) - e0coef = np.array([10632.51025668, 10632.51025668, 8198.00693508, 10632.51025668, - 10632.51025668, 10632.51025668, 12083.6205191 , 12215.5218439 , - 11031.20482006, 10632.51025668, 10632.51025668, 12250.67984846, - 12662.43500038, 11687.13986784, 7881.16756887, 10632.51025668, - 11005.19041964, 11454.06577668, 10605.04377619, 10632.51025668, - 10632.51025668, 10632.51025668, 10632.51025668, 10632.51025668, - 10632.51025668]) - ltcoef = np.array([3611.42910617, 3611.42910617, 2383.13016371, 3611.42910617, - 3611.42910617, 3611.42910617, 4269.56836159, 4289.18987023, - 4165.14681987, 3611.42910617, 3611.42910617, 3815.34850334, - 3666.44326169, 3680.6402539 , 2513.42432537, 3611.42910617, - 3552.99407017, 3514.83760875, 3348.95744382, 3611.42910617, - 3611.42910617, 3611.42910617, 3611.42910617, 3611.42910617, - 3611.42910617]) - correction = np.array([9.416792493e-05, 9.413238325e-05, 1.220993888e-04, 9.413193693e-05, - 9.411739173e-05, 9.409713667e-05, 8.279152513e-05, 8.189972046e-05, - 9.070379578e-05, 9.408888204e-05, 9.407901934e-05, 8.167086609e-05, - 7.900813579e-05, 8.560232940e-05, 1.269748871e-04, 9.407975724e-05, - 9.084074259e-05, 8.729552081e-05, 9.426779280e-05, 9.404039796e-05, - 9.402650568e-05, 9.402980196e-05, 9.405365918e-05, 9.401772495e-05, - 9.398598283e-05]) - - return xs, ys, zs, ts, e0coef, ltcoef, correction - -def test_maps_coefficient_getter_exact(toy_corrections, correction_map_filename): - maps = read_maps(correction_map_filename) - xs, ys, zs, _, coef_geo, coef_lt, _ = toy_corrections - get_maps_coefficient_e0= maps_coefficient_getter(maps.mapinfo, maps.e0) - CE = get_maps_coefficient_e0(xs,ys) - get_maps_coefficient_lt= maps_coefficient_getter(maps.mapinfo, maps.lt) - LT = get_maps_coefficient_lt(xs,ys) - assert_allclose (CE, coef_geo) - assert_allclose (LT, coef_lt) - -def test_read_maps_returns_ASectorMap(correction_map_filename): - maps = read_maps(correction_map_filename) - assert type(maps)==ASectorMap - -@composite -def xy_pos(draw, elements=floats(min_value=-250, max_value=250)): - size = draw(integers(min_value=1, max_value=10)) - x = draw(lists(elements,min_size=size, max_size=size)) - y = draw(lists(elements,min_size=size, max_size=size)) - return (np.array(x),np.array(y)) - -@given(xy_pos = xy_pos()) -def test_maps_coefficient_getter_gives_nans(correction_map_filename, xy_pos): - x,y = xy_pos - maps = read_maps(correction_map_filename) - mapinfo = maps.mapinfo - map_df = maps.e0 - xmin,xmax = mapinfo.xmin,mapinfo.xmax - ymin,ymax = mapinfo.ymin,mapinfo.ymax - get_maps_coefficient_e0= maps_coefficient_getter(mapinfo, map_df) - CE = get_maps_coefficient_e0(x,y) - mask_x = (x >=xmax) | (x=ymax) | (y=xmax) | (x=ymax) | (y pd.DataFrame: return drop_isolated_sensors -def deconvolution_input(sample_width : List[float], - bin_size : List[float], +def deconvolution_input(sample_width : List[float ], + det_grid : List[np.ndarray], inter_method : InterpolationMethod = InterpolationMethod.cubic ) -> Callable: """ @@ -113,7 +114,7 @@ def deconvolution_input(sample_width : List[float], Initialization parameters: sample_width : Sampling size of the sensors. - bin_size : Size of the interpolated bins. + det_grid : xy-coordinates of the detector grid, to interpolate on them inter_method : Interpolation method. Returns @@ -121,21 +122,20 @@ def deconvolution_input(sample_width : List[float], Hs : Charge input for deconvolution. inter_points : Coordinates of the deconvolution input. """ - if inter_method not in InterpolationMethod: + if not isinstance(inter_method, InterpolationMethod): raise ValueError(f'inter_method {inter_method} is not a valid interpolation method.') def deconvolution_input(data : Tuple[np.ndarray, ...], weight : np.ndarray ) -> Tuple[np.ndarray, Tuple[np.ndarray, ...]]: - ranges = [[coord.min() - 1.5 * sw, coord.max() + 1.5 * sw] for coord, sw in zip(data , sample_width)] - nbin = [np.ceil(np.diff(rang)/bs).astype('int')[0] for bs , rang in zip(bin_size , ranges)] - + ranges = [[coord.min() - 1.5 * sw, coord.max() + 1.5 * sw] for coord, sw in zip(data, sample_width)] if inter_method in (InterpolationMethod.linear, InterpolationMethod.cubic, InterpolationMethod.nearest): - allbins = [np.linspace(*rang, np.ceil(np.diff(rang)/sw)+1) for rang , sw in zip(ranges[:2], sample_width)] + allbins = [np.arange(rang[0], rang[1] + np.finfo(np.float32).eps, sw) for rang, sw in zip(ranges, sample_width)] Hs, edges = np.histogramdd(data, bins=allbins, normed=False, weights=weight) elif inter_method is InterpolationMethod.none: - Hs, edges = np.histogramdd(data, bins=nbin , normed=False, weights=weight, range=ranges) + allbins = [grid[in_range(grid, *rang)] for rang, grid in zip(ranges, det_grid)] + Hs, edges = np.histogramdd(data, bins=allbins, normed=False, weights=weight) else: raise ValueError(f'inter_method {inter_method} is not a valid interpolatin mode.') @@ -143,7 +143,7 @@ def deconvolution_input(data : Tuple[np.ndarray, ...], inter_points = tuple (inter_p.flatten() for inter_p in inter_points) if inter_method in (InterpolationMethod.linear, InterpolationMethod.cubic, InterpolationMethod.nearest): - Hs, inter_points = interpolate_signal(Hs, inter_points, edges, nbin, inter_method) + Hs, inter_points = interpolate_signal(Hs, inter_points, ranges, det_grid, inter_method) return Hs, inter_points @@ -152,8 +152,8 @@ def deconvolution_input(data : Tuple[np.ndarray, ...], def interpolate_signal(Hs : np.ndarray, inter_points : Tuple[np.ndarray, ...], - edges : Tuple[np.ndarray, ...], - nbin : List[int], + edges : List [np.ndarray ], + det_grid : List [np.ndarray ], inter_method : InterpolationMethod = InterpolationMethod.cubic ) -> Tuple[np.ndarray, Tuple[np.ndarray, ...]]: """ @@ -165,7 +165,7 @@ def interpolate_signal(Hs : np.ndarray, Hs : Distribution weights to be interpolated. inter_points : Distribution coordinates to be interpolated. edges : Edges of the coordinates. - nbin : Number of points to be interpolated in each dimension. + det_grid : xy-coordinates of the detector grid, to interpolate on them inter_method : Interpolation method. Returns @@ -173,13 +173,11 @@ def interpolate_signal(Hs : np.ndarray, H1 : Interpolated distribution weights. new_points : Interpolated coordinates. """ - coords = (shift_to_bin_centers(np.linspace(np.min(edge), np.max(edge), n + 1)) - for n, edge in zip(nbin, edges)) + coords = [grid[in_range(grid, *edge)] for edge, grid in zip(edges, det_grid)] new_points = np.meshgrid(*coords, indexing='ij') new_points = tuple (new_p.flatten() for new_p in new_points) - H1 = interpolate.griddata(inter_points, Hs.flatten(), new_points, method=inter_method.value) - H1 = np.nan_to_num (H1.reshape(nbin)) + H1 = np.nan_to_num (H1.reshape([len(c) for c in coords])) H1 = np.clip (H1, 0, None) return H1, new_points @@ -208,7 +206,7 @@ def find_nearest(array : np.ndarray, def deconvolve(n_iterations : int, iteration_tol : float, sample_width : List[float], - bin_size : List[float], + det_grid : List[np.ndarray], inter_method : InterpolationMethod = InterpolationMethod.cubic ) -> Callable: """ @@ -222,10 +220,11 @@ def deconvolve(n_iterations : int, psf : Point-spread function. Initialization parameters: - n_iterations : Number of Lucy-Richardson iterations - sample_width : Sampling size of the sensors. - bin_size : Size of the interpolated bins. - inter_method : Interpolation method. + n_iterations : Number of Lucy-Richardson iterations + iteration_tol : Stopping threshold (difference between iterations). + sample_width : Sampling size of the sensors. + det_grid : xy-coordinates of the detector grid, to interpolate on them + inter_method : Interpolation method. Returns ---------- @@ -233,7 +232,7 @@ def deconvolve(n_iterations : int, inter_pos : Coordinates of the deconvolved image. """ var_name = np.array(['xr', 'yr', 'zr']) - deconv_input = deconvolution_input(sample_width, bin_size, inter_method) + deconv_input = deconvolution_input(sample_width, det_grid, inter_method) def deconvolve(data : Tuple[np.ndarray, ...], weight : np.ndarray, diff --git a/invisible_cities/reco/deconv_functions_test.py b/invisible_cities/reco/deconv_functions_test.py index 5248b2e832..ac6bc4682f 100644 --- a/invisible_cities/reco/deconv_functions_test.py +++ b/invisible_cities/reco/deconv_functions_test.py @@ -2,30 +2,32 @@ import numpy as np import pandas as pd -from pytest import mark -from pytest import raises +from pytest import mark +from pytest import raises -from hypothesis import given -from hypothesis.strategies import floats -from hypothesis.extra.pandas import data_frames -from hypothesis.extra.pandas import column -from hypothesis.extra.pandas import range_indexes +from hypothesis import given +from hypothesis.strategies import floats +from hypothesis.extra.pandas import data_frames +from hypothesis.extra.pandas import column +from hypothesis.extra.pandas import range_indexes -from .. reco.deconv_functions import cut_and_redistribute_df -from .. reco.deconv_functions import drop_isolated_sensors -from .. reco.deconv_functions import interpolate_signal -from .. reco.deconv_functions import deconvolution_input -from .. reco.deconv_functions import deconvolve -from .. reco.deconv_functions import richardson_lucy -from .. reco.deconv_functions import InterpolationMethod +from .. reco .deconv_functions import cut_and_redistribute_df +from .. reco .deconv_functions import drop_isolated_sensors +from .. reco .deconv_functions import interpolate_signal +from .. reco .deconv_functions import deconvolution_input +from .. reco .deconv_functions import deconvolve +from .. reco .deconv_functions import richardson_lucy +from .. reco .deconv_functions import InterpolationMethod -from .. core.core_functions import in_range -from .. core.core_functions import shift_to_bin_centers -from .. core.testing_utils import assert_dataframes_close +from .. core .core_functions import in_range +from .. core .core_functions import shift_to_bin_centers +from .. core .testing_utils import assert_dataframes_close -from .. io.dst_io import load_dst +from .. io .dst_io import load_dst -from scipy.stats import multivariate_normal +from .. database.load_db import DataSiPM + +from scipy.stats import multivariate_normal @given(data_frames(columns=[column('A', dtype=float, elements=floats(1, 1e3)), @@ -65,53 +67,56 @@ def test_drop_isolated_sensors(): def test_interpolate_signal(): - ref_interpolation = np.array([0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , + ref_interpolation = np.array([0. , 0. , 0. , 0. , 0. , 0 , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0.17 , 0.183, 0.188, 0.195, 0.202, - 0.2 , 0.2 , 0.19 , 0.181, 0.169, 0. , 0. , 0.308, 0.328, - 0.344, 0.353, 0.363, 0.365, 0.356, 0.345, 0.327, 0.305, 0. , - 0. , 0.501, 0.531, 0.569, 0.583, 0.596, 0.598, 0.583, 0.566, - 0.533, 0.496, 0. , 0. , 0.693, 0.752, 0.784, 0.819, 0.836, - 0.833, 0.82 , 0.794, 0.752, 0.688, 0. , 0. , 0.813, 0.882, - 0.922, 0.958, 0.976, 0.975, 0.958, 0.927, 0.88 , 0.813, 0. , - 0. , 0.812, 0.876, 0.929, 0.958, 0.974, 0.975, 0.959, 0.923, - 0.88 , 0.822, 0. , 0. , 0.688, 0.752, 0.8 , 0.822, 0.831, - 0.833, 0.819, 0.789, 0.753, 0.693, 0. , 0. , 0.496, 0.535, - 0.567, 0.587, 0.597, 0.591, 0.581, 0.57 , 0.532, 0.504, 0. , - 0. , 0.305, 0.326, 0.346, 0.356, 0.362, 0.363, 0.356, 0.342, - 0.33 , 0.31 , 0. , 0. , 0.168, 0.18 , 0.189, 0.198, 0.202, - 0.199, 0.195, 0.192, 0.181, 0.174, 0. , 0. , 0. , 0. , + 0.201, 0.202, 0.188, 0.181, 0.168, 0. , 0. , 0.308, 0.328, + 0.344, 0.354, 0.362, 0.365, 0.357, 0.347, 0.326, 0.308, 0. , + 0. , 0.5 , 0.531, 0.569, 0.585, 0.593, 0.592, 0.58 , 0.566, + 0.543, 0.514, 0. , 0. , 0.693, 0.751, 0.786, 0.825, 0.833, + 0.827, 0.816, 0.79 , 0.757, 0.703, 0. , 0. , 0.818, 0.886, + 0.924, 0.957, 0.965, 0.973, 0.963, 0.925, 0.882, 0.818, 0. , + 0. , 0.82 , 0.884, 0.93 , 0.958, 0.969, 0.965, 0.954, 0.928, + 0.883, 0.818, 0. , 0. , 0.698, 0.752, 0.793, 0.819, 0.832, + 0.836, 0.826, 0.794, 0.752, 0.699, 0. , 0. , 0.509, 0.535, + 0.57 , 0.582, 0.6 , 0.592, 0.585, 0.568, 0.536, 0.51 , 0. , + 0. , 0.311, 0.328, 0.347, 0.36 , 0.359, 0.361, 0.356, 0.343, + 0.328, 0.312, 0.0 , 0.0 , 0.169, 0.182, 0.196, 0.196, 0.201, + 0.197, 0.194, 0.192, 0.182, 0.169, 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ]) g = multivariate_normal((0.5, 0.5), (0.05, 0.5)) - grid_x, grid_y = np.mgrid[0:1:12j, 0:1:12j] # Grid for interpolation points = np.meshgrid(np.linspace(0, 1, 6), np.linspace(0, 1, 6)) # Coordinates where g is known points = (points[0].flatten(), points[1].flatten()) values = g.pdf(list(zip(points[0], points[1]))) # Value of g at the known coordinates. n_interpolation = 12 # How many points to interpolate + grid = shift_to_bin_centers(np.linspace(-0.05, 1.05, n_interpolation + 1))# Grid for interpolation out_interpolation = interpolate_signal(values, points, - (np.linspace(-0.05, 1.05, 6), np.linspace(-0.05, 1.05, 6)), - [n_interpolation, n_interpolation], + [[-0.05, 1.05], [-0.05, 1.05]], + [grid, grid], InterpolationMethod.cubic) inter_charge = out_interpolation[0].flatten() inter_position = out_interpolation[1] ref_position = shift_to_bin_centers(np.linspace(-0.05, 1.05, n_interpolation + 1)) assert np.allclose(ref_interpolation, np.around(inter_charge, decimals=3)) - assert np.allclose(ref_position , sorted(set(inter_position[0]))) - assert np.allclose(ref_position , sorted(set(inter_position[1]))) + assert np.allclose(grid , sorted(set(inter_position[0]))) + assert np.allclose(grid , sorted(set(inter_position[1]))) def test_deconvolution_input(data_hdst, data_hdst_deconvolved): ref_interpolation = np.load(data_hdst_deconvolved) - hdst = load_dst(data_hdst, 'RECO', 'Events') - h = hdst[(hdst.event == 3021916) & (hdst.npeak == 0)] - h = h.groupby(['X', 'Y']).Q.sum().reset_index() - h = h[h.Q > 40] + hdst = load_dst(data_hdst, 'RECO', 'Events') + h = hdst[(hdst.event == 3021916) & (hdst.npeak == 0)] + h = h.groupby(['X', 'Y']).Q.sum().reset_index() + h = h[h.Q > 40] - interpolator = deconvolution_input([10., 10.], [1., 1.], InterpolationMethod.cubic) + det_db = DataSiPM('new', 0) + det_grid = [np.arange(det_db[var].min() + bs/2, det_db[var].max() - bs/2 + np.finfo(np.float32).eps, bs) + for var, bs in zip(['X', 'Y'], [1., 1.])] + interpolator = deconvolution_input([10., 10.], det_grid, InterpolationMethod.cubic) inter = interpolator((h.X, h.Y), h.Q) assert np.allclose(ref_interpolation['e_inter'], inter[0]) @@ -127,14 +132,17 @@ def test_deconvolution_input_interpolation_method(data_hdst, data_hdst_deconvolv def test_deconvolve(data_hdst, data_hdst_deconvolved): ref_interpolation = np.load (data_hdst_deconvolved) - hdst = load_dst(data_hdst, 'RECO', 'Events') - h = hdst[(hdst.event == 3021916) & (hdst.npeak == 0)] - z = h.Z.mean() - h = h.groupby(['X', 'Y']).Q.sum().reset_index() - h = h[h.Q > 40] + hdst = load_dst(data_hdst, 'RECO', 'Events') + h = hdst[(hdst.event == 3021916) & (hdst.npeak == 0)] + z = h.Z.mean() + h = h.groupby(['X', 'Y']).Q.sum().reset_index() + h = h[h.Q > 40] - deconvolutor = deconvolve(15, 0.01, [10., 10.], [1., 1.], inter_method=InterpolationMethod.cubic) + det_db = DataSiPM('new', 0) + det_grid = [np.arange(det_db[var].min() + bs/2, det_db[var].max() - bs/2 + np.finfo(np.float32).eps, bs) + for var, bs in zip(['X', 'Y'], [1., 1.])] + deconvolutor = deconvolve(15, 0.01, [10., 10.], det_grid, inter_method=InterpolationMethod.cubic) x, y = np.linspace(-49.5, 49.5, 100), np.linspace(-49.5, 49.5, 100) xx, yy = np.meshgrid(x, y) @@ -156,15 +164,19 @@ def test_deconvolve(data_hdst, data_hdst_deconvolved): def test_richardson_lucy(data_hdst, data_hdst_deconvolved): ref_interpolation = np.load (data_hdst_deconvolved) - hdst = load_dst(data_hdst, 'RECO', 'Events') - h = hdst[(hdst.event == 3021916) & (hdst.npeak == 0)] - z = h.Z.mean() - h = h.groupby(['X', 'Y']).Q.sum().reset_index() - h = h[h.Q>40] + hdst = load_dst(data_hdst, 'RECO', 'Events') + h = hdst[(hdst.event == 3021916) & (hdst.npeak == 0)] + z = h.Z.mean() + h = h.groupby(['X', 'Y']).Q.sum().reset_index() + h = h[h.Q>40] + + det_db = DataSiPM('new', 0) + det_grid = [np.arange(det_db[var].min() + bs/2, det_db[var].max() - bs/2 + np.finfo(np.float32).eps, bs) + for var, bs in zip(['X', 'Y'], [1., 1.])] - interpolator = deconvolution_input([10., 10.], [1., 1.], InterpolationMethod.cubic) - inter = interpolator((h.X, h.Y), h.Q) + interpolator = deconvolution_input([10., 10.], det_grid, InterpolationMethod.cubic) + inter = interpolator((h.X, h.Y), h.Q) x , y = np.linspace(-49.5, 49.5, 100), np.linspace(-49.5, 49.5, 100) xx, yy = np.meshgrid(x, y) @@ -181,3 +193,65 @@ def test_richardson_lucy(data_hdst, data_hdst_deconvolved): iterations=15, iter_thr=0.0001) assert np.allclose(ref_interpolation['e_deco'], deco.flatten()) + + +def test_grid_binning(data_hdst, data_hdst_deconvolved): + hdst = load_dst(data_hdst, 'RECO', 'Events') + h = hdst[(hdst.event == 3021916) & (hdst.npeak == 0)] + z = h.Z.mean() + h = h.groupby(['X', 'Y']).Q.sum().reset_index() + h = h[h.Q > 40] + + det_db = DataSiPM('new', 0) + det_grid = [np.arange(det_db[var].min() + bs/2, det_db[var].max() - bs/2 + np.finfo(np.float32).eps, bs) + for var, bs in zip(['X', 'Y'], [9., 9.])] + + deconvolutor = deconvolve(15, 0.01, [10., 10.], det_grid, inter_method=InterpolationMethod.cubic) + + x, y = np.linspace(-49.5, 49.5, 100), np.linspace(-49.5, 49.5, 100) + xx, yy = np.meshgrid(x, y) + xx, yy = xx.flatten(), yy.flatten() + + psf = {} + psf['factor'] = multivariate_normal([0., 0.], [1.027 * np.sqrt(z/10)] * 2).pdf(list(zip(xx, yy))) + psf['xr'] = xx + psf['yr'] = yy + psf['zr'] = [z] * len(xx) + psf = pd.DataFrame(psf) + + deco = deconvolutor((h.X, h.Y), h.Q, psf) + + assert np.all((deco[1][0] - det_db['X'].min() + 9/2) % 9 == 0) + assert np.all((deco[1][1] - det_db['Y'].min() + 9/2) % 9 == 0) + + +def test_nonexact_binning(data_hdst, data_hdst_deconvolved): + hdst = load_dst(data_hdst, 'RECO', 'Events') + h = hdst[(hdst.event == 3021916) & (hdst.npeak == 0)] + z = h.Z.mean() + h = h.groupby(['X', 'Y']).Q.sum().reset_index() + h = h[h.Q > 40] + + det_db = DataSiPM('new', 0) + det_grid = [np.arange(det_db[var].min() + bs/2, det_db[var].max() - bs/2 + np.finfo(np.float32).eps, bs) + for var, bs in zip(['X', 'Y'], [9., 9.])] + + deconvolutor = deconvolve(15, 0.01, [10., 10.], det_grid, inter_method=InterpolationMethod.cubic) + x, y = np.linspace(-49.5, 49.5, 100), np.linspace(-49.5, 49.5, 100) + xx, yy = np.meshgrid(x, y) + xx, yy = xx.flatten(), yy.flatten() + + psf = {} + psf['factor'] = multivariate_normal([0., 0.], [1.027 * np.sqrt(z/10)] * 2).pdf(list(zip(xx, yy))) + psf['xr'] = xx + psf['yr'] = yy + psf['zr'] = [z] * len(xx) + psf = pd.DataFrame(psf) + + deco = deconvolutor((h.X, h.Y), h.Q, psf) + + check_x = np.diff(np.sort(np.unique(deco[1][0]), axis=None)) + check_y = np.diff(np.sort(np.unique(deco[1][1]), axis=None)) + + assert(np.all(check_x % 9 == 0)) + assert(np.all(check_y % 9 == 0)) diff --git a/invisible_cities/reco/dst_functions.py b/invisible_cities/reco/dst_functions.py index 729be62aa6..a9ac9cbed0 100644 --- a/invisible_cities/reco/dst_functions.py +++ b/invisible_cities/reco/dst_functions.py @@ -1,58 +1,5 @@ import numpy as np -from . corrections import Correction -from . corrections import LifetimeXYCorrection -from .. io.dst_io import load_dst - - -def load_z_corrections(filename): - dst = load_dst(filename, "Corrections", "Zcorrections") - return Correction((dst.z.values,), dst.factor.values, dst.uncertainty.values) - - -def load_xy_corrections(filename, *, - group = "Corrections", - node = "XYcorrections", - **kwargs): - dst = load_dst(filename, group, node) - x, y = np.unique(dst.x.values), np.unique(dst.y.values) - f, u = dst.factor.values, dst.uncertainty.values - - return Correction((x, y), - f.reshape(x.size, y.size), - u.reshape(x.size, y.size), - **kwargs) - - -def load_lifetime_xy_corrections(filename, *, - group = "Corrections", - node = "LifetimeXY", - scale = 1, - **kwargs): - """ - Load the lifetime map from hdf5 file. - - Parameters - ---------- - filename: str - Path to the file containing the map. - group: str - Name of the group where the table is stored. - node: str - Name of the table containing the data. - scale: float - Scale factor for the lifetime values. - - Other kwargs are passed to the contructor of LifetimeXYCorrection. - """ - dst = load_dst(filename, group, node) - x, y = np.unique(dst.x.values), np.unique(dst.y.values) - f, u = dst.factor.values * scale, dst.uncertainty.values * scale - - return LifetimeXYCorrection(f.reshape(x.size, y.size), - u.reshape(x.size, y.size), - x, y, **kwargs) - def dst_event_id_selection(data, event_ids): """Filter a DST by a list of event IDs. Parameters diff --git a/invisible_cities/reco/dst_functions_test.py b/invisible_cities/reco/dst_functions_test.py index 11b86579dd..52494d35c5 100644 --- a/invisible_cities/reco/dst_functions_test.py +++ b/invisible_cities/reco/dst_functions_test.py @@ -1,73 +1,12 @@ -from operator import mul -from operator import truediv -from collections import namedtuple - import numpy as np import pandas as pd -from pytest import fixture -from pytest import mark - from hypothesis import given from hypothesis.strategies import integers from hypothesis.strategies import lists from hypothesis.extra.pandas import columns, data_frames -from . corrections import Correction -from . dst_functions import load_xy_corrections -from . dst_functions import load_lifetime_xy_corrections -from . dst_functions import dst_event_id_selection - -normalization_data = namedtuple("normalization_data", "node kwargs op") - -@fixture(scope = "session", - params = [False, True]) -def normalization(request): - if request.param: - node = "LifetimeXY_inverse" - kwargs = {"norm_strategy": "const", - "norm_opts" : {"value": 1}} - op = truediv - else: - node = "LifetimeXY" - kwargs = {} - op = mul - return normalization_data(node, kwargs, op) - - -def test_load_xy_corrections(corr_toy_data, normalization): - filename, true_data = corr_toy_data - x, y, E, U, _ = true_data - corr = load_xy_corrections(filename, - node = normalization.node, - **normalization.kwargs) - assert corr == Correction((x,y), E, U) - - -@mark.parametrize("scale", - (0.5, 1, 2.0)) -def test_load_lifetime_xy_corrections(corr_toy_data, normalization, scale): - filename, true_data = corr_toy_data - x, y, LT, U, _ = true_data - corr = load_lifetime_xy_corrections(filename, - node = normalization.node, - scale = scale, - **normalization.kwargs) - - LT = normalization.op(LT, scale) - U = normalization.op(U , scale) - for i in np.linspace(0, 2, 5): - # This should yield exp(i * x/x) = exp(i) - z_test = LT.flatten() * i - x_test = np.repeat(x, y.size) - y_test = np.tile (y, x.size) - (f_test, - u_test) = corr(z_test, x_test, y_test) - - f_true = np.exp(i) - u_true = z_test * U.flatten()/LT.flatten()**2 * f_test - assert np.allclose(f_test, f_true) - assert np.allclose(u_test, u_true) +from . dst_functions import dst_event_id_selection @given(data_frames(columns=columns(['event'], elements=integers(min_value=-1e5, max_value=1e5))), diff --git a/invisible_cities/reco/histogram_functions.py b/invisible_cities/reco/histogram_functions.py deleted file mode 100644 index 349fe4fb38..0000000000 --- a/invisible_cities/reco/histogram_functions.py +++ /dev/null @@ -1,66 +0,0 @@ -from .. io .hist_io import save_histomanager_to_file -from .. io .hist_io import get_histograms_from_file -from .. evm .histos import Histogram -from .. evm .histos import HistoManager - - -def create_histomanager_from_dicts(histobins_dict, histolabels_dict, init_fill_dict=None): - """ - Creates and returns an HistoManager from a dict of bins and a given of labels with identical keys. - - Arguments: - histobins_dict = Dictionary with keys equal to Histogram names and values equal to the binning. - histolabels_dict = Dictionary with keys equal to Histogram names and values equal to the axis labels. - init_fill_dict = Dictionary with keys equal to Histogram names and values equal to an initial filling. - """ - histo_manager = HistoManager() - if init_fill_dict is None: init_fill_dict = dict() - for histotitle, histobins in histobins_dict.items(): - histo_manager.new_histogram(Histogram(histotitle, - histobins, - histolabels_dict[histotitle], - init_fill_dict.get(histotitle, None))) - return histo_manager - - -def join_histograms_from_files(histofiles, group_name='HIST', join_file=None, write_mode='w'): - """ - Joins the histograms of a given list of histogram files. If possible, - Histograms with the same name will be added. - - histofiles = List of strings with the filenames to be summed. - join_file = String. If passed, saves the resulting HistoManager to this path. - """ - if not histofiles: - raise ValueError("List of files is empty") - - final_histogram_manager = get_histograms_from_file(histofiles[0], group_name) - - for file in histofiles[1:]: - added_histograms = get_histograms_from_file(file, group_name) - final_histogram_manager = join_histo_managers (final_histogram_manager, added_histograms) - - if join_file is not None: - save_histomanager_to_file(final_histogram_manager, join_file, mode=write_mode, group=group_name) - - return final_histogram_manager - - -def join_histo_managers(histo_manager1, histo_manager2): - """ - Joins two HistoManager. If they share histograms, the histograms are sumed. - - Arguments: - histo_manager1, histo_manager2 = HistoManager objects to be joined. - """ - new_histogram_manager = HistoManager() - list_of_histograms = set(histo_manager1.histos) | set(histo_manager2.histos) - for histoname in list_of_histograms: - histo1 = histo_manager1.histos.get(histoname, None) - histo2 = histo_manager2.histos.get(histoname, None) - try: - new_histogram_manager.new_histogram(histo1 + histo2) - except ValueError: - print(f"Histograms with name {histoname} have not been added due to" - " incompatible binning.") - return new_histogram_manager diff --git a/invisible_cities/reco/histogram_functions_test.py b/invisible_cities/reco/histogram_functions_test.py deleted file mode 100644 index 89725fbce6..0000000000 --- a/invisible_cities/reco/histogram_functions_test.py +++ /dev/null @@ -1,142 +0,0 @@ -import os -from collections import defaultdict - -import numpy as np - -from hypothesis import given -from hypothesis import settings -from hypothesis.strategies import lists - -from pytest import mark - -from .. reco import histogram_functions as histf - -from .. io .hist_io import save_histomanager_to_file -from .. io .hist_io import get_histograms_from_file -from .. evm.histos import HistoManager -from .. evm.histos import Histogram -from .. evm.histos_test import assert_histogram_equality -from .. evm.histos_test import histograms_lists -from .. evm.histos_test import bins_arrays - -@mark.skip(reason="Delaying elimination of solid cities") -@given(histograms_lists()) -@settings(deadline=None) -def test_join_histo_managers(histogram_list): - args, list_of_histograms = histogram_list - histogram_manager = HistoManager(list_of_histograms) - joined_histogram_manager = histf.join_histo_managers(histogram_manager, histogram_manager) - - assert len(list_of_histograms) == len(joined_histogram_manager.histos) - for histoname, histogram in joined_histogram_manager.histos.items(): - histo1 = histogram_manager[histoname] - true_histogram = Histogram(histoname, histo1.bins, histo1.labels) - true_histogram.data = 2 * histo1.data - true_histogram.errors = np.sqrt(2) * histo1.errors - true_histogram.out_range = 2 * histo1.out_range - assert_histogram_equality(histogram, true_histogram) - - -@mark.skip(reason="Delaying elimination of solid cities") -@given (histograms_lists(), histograms_lists()) -@settings(deadline=None, max_examples=700) -def test_join_histo_managers_with_different_histograms(histogram_list1, histogram_list2): - _, list_of_histograms1 = histogram_list1 - _, list_of_histograms2 = histogram_list2 - histogram_manager1 = HistoManager(list_of_histograms1) - histogram_manager2 = HistoManager(list_of_histograms2) - joined_histogram_manager = histf.join_histo_managers(histogram_manager1, histogram_manager2) - - unique_histograms = set(histogram_manager1.histos) | set(histogram_manager2.histos) - common_histograms = set(histogram_manager1.histos) & set(histogram_manager2.histos) - - remove_names = [] - for name in unique_histograms: - if name in common_histograms: - if not np.all(a == b for a, b in zip(histogram_manager1[name].bins, histogram_manager2[name].bins)): - remove_names.append(name) - list_of_names = unique_histograms - set(remove_names) - - for histoname, histogram in joined_histogram_manager.histos.items(): - assert histoname in list_of_names - if (histoname in histogram_manager1.histos) and (histoname in histogram_manager2.histos): - histo1 = histogram_manager1[histoname] - histo2 = histogram_manager2[histoname] - - true_histogram = Histogram(histoname, histo1.bins, histo1.labels) - true_histogram.data = histo1.data + histo2.data - true_histogram.errors = np.sqrt(histo1.errors ** 2 + histo2.errors ** 2) - true_histogram.out_range = histo1.out_range + histo2.out_range - - assert_histogram_equality(histogram, true_histogram) - - elif histoname in histogram_manager1.histos: - histo1 = histogram_manager1[histoname] - assert_histogram_equality(histogram, histo1) - - elif histoname in histogram_manager2.histos: - histo2 = histogram_manager2[histoname] - assert_histogram_equality(histogram, histo2) - - -@mark.skip(reason="Delaying elimination of solid cities") -@given(lists(bins_arrays(), min_size=1, max_size=5)) -@settings(deadline=None) -def test_create_histomanager_from_dicts(bins): - histobins_dict = {} - histolabels_dict = {} - histograms_dict = {} - for i, bins_element in enumerate(bins): - title = f"Histo_{i}" - labels = [ f"Xlabel_{i}", f"Ylabel_{i}" ] - histobins_dict [title] = bins_element - histolabels_dict[title] = labels - histograms_dict [title] = Histogram(title, bins_element, labels) - - histo_manager = histf.create_histomanager_from_dicts(histobins_dict, histolabels_dict) - - assert len(histograms_dict) == len(histo_manager.histos) - for histoname, histogram in histo_manager.histos.items(): - assert_histogram_equality(histogram, histograms_dict[histoname]) - - -@mark.skip(reason="Delaying elimination of solid cities") -@given (histograms_lists(), histograms_lists()) -@settings(deadline=None, max_examples=250) -def test_join_histograms_from_file(output_tmpdir, histogram_list1, histogram_list2): - _, list_of_histograms1 = histogram_list1 - _, list_of_histograms2 = histogram_list2 - histogram_manager1 = HistoManager(list_of_histograms1) - histogram_manager2 = HistoManager(list_of_histograms2) - - file_out1 = os.path.join(output_tmpdir, 'test_save_histogram_manager_1.h5') - save_histomanager_to_file(histogram_manager1, file_out1) - file_out2 = os.path.join(output_tmpdir, 'test_save_histogram_manager_2.h5') - save_histomanager_to_file(histogram_manager2, file_out2) - - joined_histogram_manager1 = histf.join_histograms_from_files([file_out1, file_out2]) - joined_histogram_manager2 = histf.join_histo_managers(histogram_manager1, histogram_manager2) - - assert len(joined_histogram_manager1.histos) == len(joined_histogram_manager2.histos) - for histoname in joined_histogram_manager1.histos: - assert_histogram_equality(joined_histogram_manager1[histoname], joined_histogram_manager2[histoname]) - - -@mark.skip(reason="Delaying elimination of solid cities") -@given (histograms_lists()) -@settings(deadline=None, max_examples=250) -def test_join_histograms_from_file_and_write(output_tmpdir, histogram_list): - _, list_of_histograms = histogram_list - histogram_manager = HistoManager(list_of_histograms) - - file_out_test = os.path.join(output_tmpdir , 'test_save_histogram_manager_1.h5') - save_histomanager_to_file(histogram_manager, file_out_test) - - file_out = os.path.join(output_tmpdir, 'test_join_histograms.h5') - _ = histf.join_histograms_from_files([file_out_test, file_out_test], join_file=file_out) - joined_histogram_manager1 = histf.get_histograms_from_file(file_out) - joined_histogram_manager2 = histf.join_histo_managers(histogram_manager, histogram_manager) - - assert len(joined_histogram_manager1.histos) == len(joined_histogram_manager2.histos) - for histoname in joined_histogram_manager1.histos: - assert_histogram_equality(joined_histogram_manager1[histoname], joined_histogram_manager2[histoname]) diff --git a/invisible_cities/reco/hits_functions_test.py b/invisible_cities/reco/hits_functions_test.py index cea05baf43..3dbe5ac057 100644 --- a/invisible_cities/reco/hits_functions_test.py +++ b/invisible_cities/reco/hits_functions_test.py @@ -1,29 +1,27 @@ import os -import numpy as np -from numpy.testing import assert_allclose -from numpy.testing import assert_almost_equal -from numpy.testing import assert_raises -from .. core.configure import configure -from .. evm.event_model import Hit -from .. evm.event_model import Cluster -from .. types.ic_types import xy -from .. core.system_of_units_c import units -from .. core.testing_utils import assert_hit_equality -from .. types.ic_types import NN -from .. cities.penthesilea import penthesilea -from .. io import hits_io as hio -from . hits_functions import merge_NN_hits -from . hits_functions import threshold_hits -from pytest import fixture -from pytest import mark -from hypothesis import given -from hypothesis import settings -from hypothesis.strategies import lists -from hypothesis.strategies import floats -from hypothesis.strategies import integers -from copy import deepcopy -from hypothesis import assume -from hypothesis.strategies import composite + +from pytest import mark +from numpy.testing import assert_almost_equal + +from .. core.configure import configure +from .. evm.event_model import Hit +from .. evm.event_model import Cluster +from .. types.ic_types import xy +from .. core import system_of_units as units +from .. core.testing_utils import assert_hit_equality +from .. types.ic_types import NN +from .. cities.penthesilea import penthesilea +from .. io import hits_io as hio +from . hits_functions import merge_NN_hits +from . hits_functions import threshold_hits +from hypothesis import given +from hypothesis import settings +from hypothesis.strategies import lists +from hypothesis.strategies import floats +from hypothesis.strategies import integers +from copy import deepcopy +from hypothesis import assume +from hypothesis.strategies import composite @composite def hit(draw, min_value=1, max_value=100): @@ -58,8 +56,11 @@ def thresholds(draw, min_value=1, max_value=1): def test_merge_NN_not_modify_input(hits): hits_org = deepcopy(hits) before_len = len(hits) - hits_merged = merge_NN_hits(hits) + + merge_NN_hits(hits) + after_len = len(hits) + assert before_len == after_len for h1, h2 in zip(hits, hits_org): assert_hit_equality(h1, h2) @@ -97,7 +98,9 @@ def test_threshold_hits_with_penthesilea(config_tmpdir, Kr_pmaps_run4628_filenam lm_radius = 0 * units.mm , new_lm_radius = 0 * units.mm , msipm = 1 ))) - cnt = penthesilea (**conf) + + penthesilea (**conf) + PATH_OUT_th2 = os.path.join(config_tmpdir, 'KrDST_4628_th2.h5') conf.update(dict(run_number = 4628, files_in = PATH_IN, @@ -109,7 +112,9 @@ def test_threshold_hits_with_penthesilea(config_tmpdir, Kr_pmaps_run4628_filenam lm_radius = 0 * units.mm , new_lm_radius = 0 * units.mm , msipm = 1 ))) - cnt = penthesilea (**conf) + + penthesilea (**conf) + hits_pent_th1 = hio.load_hits(PATH_OUT_th1) hits_pent_th2 = hio.load_hits(PATH_OUT_th2) ev_num = 1 @@ -123,7 +128,9 @@ def test_threshold_hits_with_penthesilea(config_tmpdir, Kr_pmaps_run4628_filenam def test_threshold_hits_not_modify_input(hits, th): hits_org = deepcopy(hits) before_len = len(hits) - hits_thresh = threshold_hits(hits,th) + + threshold_hits(hits,th) + after_len = len(hits) assert before_len == after_len for h1, h2 in zip(hits, hits_org): diff --git a/invisible_cities/reco/monitor_functions.py b/invisible_cities/reco/monitor_functions.py deleted file mode 100644 index f0b296090a..0000000000 --- a/invisible_cities/reco/monitor_functions.py +++ /dev/null @@ -1,343 +0,0 @@ -import glob -from collections import defaultdict - -import numpy as np -import tables as tb - -from .. database import load_db as dbf -from .. reco import histogram_functions as histf -from .. core import system_of_units as units - -from .. evm .histos import HistoManager -from .. io .pmaps_io import load_pmaps -from .. io .dst_io import load_dst -from .. reco.tbl_functions import get_rwf_vectors - - -def pmap_bins(config_dict): - """ - Generates the binning arrays and label of the monitor plots from the a - config dictionary that contains the ranges, number of bins and labels. - - Returns a dictionary with the bins and another with the labels. - """ - var_bins = {} - var_labels = {} - - for k, v in config_dict.items(): - if "_bins" in k: var_bins [k.replace("_bins" , "")] = [np.linspace(v[0], v[1], v[2] + 1)] - elif "_labels" in k: var_labels[k.replace("_labels", "")] = v - - exception = ['S1_Energy', 'S1_Number', 'S1_Time'] - bin_sel = lambda x: ('S2' not in x) and (x not in exception) - for param in filter(bin_sel, list(var_bins)): - var_bins ['S1_Energy_' + param] = var_bins ['S1_Energy'] + var_bins [param] - var_labels['S1_Energy_' + param] = var_labels['S1_Energy'] + var_labels[param] - var_bins ['S1_Time_S1_Energy'] = var_bins ['S1_Time'] + var_bins ['S1_Energy'] - var_labels ['S1_Time_S1_Energy'] = var_labels['S1_Time'] + var_labels['S1_Energy'] - - exception = ['S2_Energy', 'S2_Number', 'S2_Time'] - bin_sel = lambda x: ('S1' not in x) and (x not in exception) and ('SiPM' not in x) - for param in filter(bin_sel, list(var_bins)): - var_bins ['S2_Energy_' + param] = var_bins ['S2_Energy'] + var_bins [param] - var_labels['S2_Energy_' + param] = var_labels['S2_Energy'] + var_labels[param] - var_bins ['S2_Time_S2_Energy'] = var_bins ['S2_Time'] + var_bins ['S2_Energy'] - var_labels ['S2_Time_S2_Energy'] = var_labels['S2_Time'] + var_labels['S2_Energy'] - var_bins ['S2_Energy_S1_Energy'] = var_bins ['S2_Energy'] + var_bins ['S1_Energy'] - var_labels ['S2_Energy_S1_Energy'] = var_labels['S2_Energy'] + var_labels['S1_Energy'] - var_bins ['S2_XYSiPM'] = var_bins ['S2_XSiPM'] + var_bins ['S2_YSiPM'] - var_labels ['S2_XYSiPM'] = var_labels['S2_XSiPM'] + var_labels['S2_YSiPM'] - - for i in range(config_dict['nPMT']): - var_bins [f'PMT{i}_S2_Energy'] = var_bins ['S2_Energy'] - var_bins [f'PMT{i}_S2_Height'] = var_bins ['S2_Height'] - var_bins [f'PMT{i}_S2_Time' ] = var_bins ['S2_Time' ] - var_labels[f'PMT{i}_S2_Energy'] = [f'PMT{i} ' + var_labels['S2_Energy'][0]] - var_labels[f'PMT{i}_S2_Height'] = [f'PMT{i} ' + var_labels['S2_Height'][0]] - var_labels[f'PMT{i}_S2_Time' ] = [f'PMT{i} ' + var_labels['S2_Time' ][0]] - - del var_bins['S2_XSiPM'] - del var_bins['S2_YSiPM'] - - return var_bins, var_labels - - -def fill_pmap_var_1d(speaks, var_dict, ptype, DataSiPM=None): - """ - Fills a passed dictionary of lists with the pmap variables to monitor. - - Arguments: - speaks = List of S1 or S2s. - var_dict = Dictionary that stores the variable values. - ptype = Type of pmap ('S1' or 'S2') - DataSiPM = Database with the SiPM information. Only needed in case of 'S2' - ptype - """ - var_dict[ptype + '_Number'].append(len(speaks)) - for speak in speaks: - var_dict[ptype + '_Width' ].append(speak.width / units.mus) - var_dict[ptype + '_Height'].append(speak.height) - var_dict[ptype + '_Energy'].append(speak.total_energy) - var_dict[ptype + '_Charge'].append(speak.total_charge) - var_dict[ptype + '_Time' ].append(speak.time_at_max_energy / units.mus) - - if ptype == 'S2': - nS1 = var_dict['S1_Number'][-1] - var_dict [ptype + '_SingleS1'] .append(nS1) - if nS1 == 1: - var_dict[ptype + '_SingleS1_Energy'].append(var_dict['S1_Energy'][-1]) - - sipm_ids = speak.sipms.ids - sipm_Q = speak.sipms.sum_over_times - var_dict [ptype + '_NSiPM' ].append(len(sipm_ids)) - var_dict [ptype + '_QSiPM' ].extend(sipm_Q) - var_dict [ptype + '_IdSiPM'].extend(sipm_ids) - if len(sipm_ids) > 0: - var_dict[ptype + '_XSiPM' ].extend(DataSiPM.X.values[sipm_ids]) - var_dict[ptype + '_YSiPM' ].extend(DataSiPM.Y.values[sipm_ids]) - - -def fill_pmap_var_2d(var_dict, ptype): - """ - Makes 2d combinations of the variables stored in a dictionary that contains - the pmaps variables. - - Arguments: - var_dict = Dictionary that stores the variable values. - ptype = Type of pmap ('S1' or 'S2') - """ - param_list = ['Width', 'Height', 'Charge'] - for param in param_list: - var_dict[ptype + '_Energy_' + ptype + '_' + param] = np.array([var_dict[ptype + '_Energy'], var_dict[ptype + '_' + param]]) - var_dict [ptype + '_Time_' + ptype + '_Energy'] = np.array([var_dict[ptype + '_Time'] , var_dict[ptype + '_Energy']]) - - if ptype == 'S2': - sel = np.asarray(var_dict['S2_SingleS1']) == 1 - var_dict[ptype + '_Energy_S1_Energy'] = np.array([np.asarray(var_dict[ptype + '_Energy'])[sel], - np.asarray(var_dict[ptype + '_SingleS1_Energy'])]) - var_dict[ptype + '_XYSiPM'] = np.array([var_dict[ptype + '_XSiPM'], var_dict[ptype + '_YSiPM']]) - - -def fill_pmt_var(speaks, var_dict): - for speak in speaks: - pmts = speak.pmts - times = speak.times - energies = pmts .sum_over_times - heights = np.max (pmts.all_waveforms, axis=1) - times = np.apply_along_axis(lambda wf: times[np.argmax(wf)], axis=1, arr=pmts.all_waveforms) / units.mus - - for i in range(len(pmts.all_waveforms)): - var_dict[f'PMT{i}_S2_Energy'].append(energies[i]) - var_dict[f'PMT{i}_S2_Height'].append(heights [i]) - var_dict[f'PMT{i}_S2_Time' ].append(times [i]) - - -def fill_pmap_var(pmap, sipm_db): - var = defaultdict(list) - - fill_pmap_var_1d(pmap.s1s, var, 'S1') - fill_pmap_var_1d(pmap.s2s, var, 'S2', sipm_db) - fill_pmap_var_2d(var, 'S1') - fill_pmap_var_2d(var, 'S2') - fill_pmt_var (pmap.s2s, var) - - del var['S2_XSiPM'] - del var['S2_YSiPM'] - del var['S2_SingleS1'] - del var['S2_SingleS1_Energy'] - - return var - - -def fill_pmap_histos(in_path, detector_db, run_number, config_dict): - """ - Creates and returns an HistoManager object with the pmap histograms. - - Arguments: - in_path = String with the path to the file(s) to be monitored. - run_number = Run number of the dataset (used to obtain the SiPM database). - config_dict = Dictionary with the configuration parameters (bins, labels). - """ - var_bins, var_labels = pmap_bins(config_dict) - histo_manager = histf.create_histomanager_from_dicts(var_bins, var_labels) - SiPM_db = dbf.DataSiPM(detector_db, run_number) - - for in_file in glob.glob(in_path): - pmaps = load_pmaps(in_file) - for ti, pi in enumerate(pmaps): - var = fill_pmap_var(pmaps[pi], SiPM_db) - histo_manager.fill_histograms(var) - return histo_manager - - -def rwf_bins(config_dict): - """ - Generates the binning arrays and label of the rwf monitor plots from the a - config dictionary that contains the ranges, number of bins and labels. - - Returns a dictionary with the bins and another with the labels. - """ - var_bins = {} - var_labels = {} - - for k, v in config_dict.items(): - if "_bins" in k: var_bins [k.replace("_bins" , "")] = [np.linspace(v[0], v[1], v[2] + 1)] - elif "_labels" in k: var_labels[k.replace("_labels", "")] = v - - return var_bins, var_labels, config_dict['n_baseline'] - -def fill_rwf_var(rwf, var_dict, sensor_type): - """ - Fills a passed dictionary of lists with the rwf variables to monitor. - - Arguments: - rwf = Raw waveforms - var_dict = Dictionary that stores the variable values. - sensor_type = Type of sensor('PMT' or 'SiPM') - """ - - bls = np.mean(rwf, axis=1) - rms = np.std (rwf, axis=1) - var_dict[sensor_type + '_Baseline'] .extend(bls) - var_dict[sensor_type + '_BaselineRMS'].extend(rms) - var_dict[sensor_type + '_nSensors'] .append(len(bls)) - - -def fill_rwf_histos(in_path, config_dict): - """ - Creates and returns an HistoManager object with the waveform histograms. - - Arguments: - in_path = String with the path to the file(s) to be monitored. - config_dict = Dictionary with the configuration parameters (bins, labels) - """ - var_bins, var_labels, n_baseline = rwf_bins(config_dict) - - histo_manager = histf.create_histomanager_from_dicts(var_bins, var_labels) - - for in_file in glob.glob(in_path): - with tb.open_file(in_file, "r") as h5in: - var = defaultdict(list) - nevt, pmtrwf, sipmrwf, _ = get_rwf_vectors(h5in) - for evt in range(nevt): - fill_rwf_var(pmtrwf [evt, :, :n_baseline], var, "PMT") - fill_rwf_var(sipmrwf[evt] , var, "SiPM") - - histo_manager.fill_histograms(var) - return histo_manager - - -def kdst_bins(config_dict): - """ - Generates the binning arrays and label of the kdst monitor plots from the a - config dictionary that contains the ranges, number of bins and labels. - - Returns a dictionary with the bins and another with the labels. - """ - var_bins = {} - var_labels = {} - - for k, v in config_dict.items(): - if "_bins" in k: var_bins [k.replace("_bins" , "")] = [np.linspace(v[0], v[1], v[2] + 1)] - elif "_labels" in k: var_labels[k.replace("_labels", "")] = v - - var_bins ['S2e_Z' ] = var_bins ['Z' ] + var_bins ['S2e'] - var_labels['S2e_Z' ] = var_labels['Z' ] + var_labels['S2e'] - - var_bins ['S2q_Z' ] = var_bins ['Z' ] + var_bins ['S2q'] - var_labels['S2q_Z' ] = var_labels['Z' ] + var_labels['S2q'] - - var_bins ['S2e_R' ] = var_bins ['R' ] + var_bins ['S2e'] - var_labels['S2e_R' ] = var_labels['R' ] + var_labels['S2e'] - - var_bins ['S2q_R' ] = var_bins ['R' ] + var_bins ['S2q'] - var_labels['S2q_R' ] = var_labels['R' ] + var_labels['S2q'] - - var_bins ['S2e_Phi'] = var_bins ['Phi'] + var_bins ['S2e'] - var_labels['S2e_Phi'] = var_labels['Phi'] + var_labels['S2e'] - - var_bins ['S2q_Phi'] = var_bins ['Phi'] + var_bins ['S2q'] - var_labels['S2q_Phi'] = var_labels['Phi'] + var_labels['S2q'] - - var_bins ['S2e_X' ] = var_bins ['X' ] + var_bins ['S2e'] - var_labels['S2e_X' ] = var_labels['X' ] + var_labels['S2e'] - - var_bins ['S2q_X' ] = var_bins ['X' ] + var_bins ['S2q'] - var_labels['S2q_X' ] = var_labels['X' ] + var_labels['S2q'] - - var_bins ['S2e_Y' ] = var_bins ['Y' ] + var_bins ['S2e'] - var_labels['S2e_Y' ] = var_labels['Y' ] + var_labels['S2e'] - - var_bins ['S2q_Y' ] = var_bins ['Y' ] + var_bins ['S2q'] - var_labels['S2q_Y' ] = var_labels['Y' ] + var_labels['S2q'] - - var_bins ['XY' ] = var_bins ['X' ] + var_bins ['Y' ] - var_labels['XY' ] = var_labels['X' ] + var_labels['Y' ] - - var_bins ['S2e_XY'] = var_bins ['X' ] + var_bins ['Y' ] + var_bins ['S2e'] - var_labels['S2e_XY'] = var_labels['X' ] + var_labels['Y' ] + var_labels['S2e'] - - var_bins ['S2q_XY'] = var_bins ['X' ] + var_bins ['Y' ] + var_bins ['S2q'] - var_labels['S2q_XY'] = var_labels['X' ] + var_labels['Y' ] + var_labels['S2q'] - - return var_bins, var_labels - - -def fill_kdst_var_1d(kdst, var_dict): - """ - Fills a passed dictionary of lists with the kdst variables to monitor. - - Arguments: - kdst = kdst dataframe. - var_dict = Dictionary that stores the variable values. - """ - - var_names = kdst.keys() - - var_sel = lambda x: (x not in ['event', 'time', 'peak']) - var_ns_to_mus = ['S1t', 'S2t', 'S1w'] - - for param in filter(var_sel, list(var_names)): - values = kdst[param].values - if param in var_ns_to_mus: - values = values / units.mus - var_dict[param].extend(values) - - -def fill_kdst_var_2d(var_dict): - """ - Makes 2d combinations of the variables stored in a dictionary that contains - the kdst variables. - - Arguments: - var_dict = Dictionary that stores the variable values. - """ - param_list = ['Z', 'X', 'Y', 'R', 'Phi'] - for param in param_list: - var_dict['S2e_' + param] = np.array([var_dict[param], var_dict['S2e']]) - var_dict['S2q_' + param] = np.array([var_dict[param], var_dict['S2q']]) - var_dict ['XY' ] = np.array([var_dict['X' ], var_dict['Y' ]]) - var_dict ['S2e_XY' ] = np.array([var_dict['X' ], var_dict['Y' ], var_dict['S2e']]) - var_dict ['S2q_XY' ] = np.array([var_dict['X' ], var_dict['Y' ], var_dict['S2q']]) - -def fill_kdst_histos(in_path, config_dict): - """ - Creates and returns an HistoManager object with the kdst histograms. - - Arguments: - in_path = String with the path to the file(s) to be monitored. - config_dict = Dictionary with the configuration parameters (bins, labels) - """ - var_bins, var_labels = kdst_bins(config_dict) - - histo_manager = histf.create_histomanager_from_dicts(var_bins, var_labels) - - for in_file in glob.glob(in_path): - var = defaultdict(list) - kdst = load_dst (in_file, 'DST', 'Events') - - fill_kdst_var_1d (kdst, var) - fill_kdst_var_2d (var) - - histo_manager.fill_histograms(var) - return histo_manager diff --git a/invisible_cities/reco/monitor_functions_test.py b/invisible_cities/reco/monitor_functions_test.py deleted file mode 100644 index 231f605d8d..0000000000 --- a/invisible_cities/reco/monitor_functions_test.py +++ /dev/null @@ -1,520 +0,0 @@ -import os -from collections import defaultdict - -import numpy as np - -from hypothesis import given -from hypothesis import settings -from hypothesis import HealthCheck -from hypothesis.extra.pandas import columns, data_frames -from hypothesis.strategies import floats - -from .. reco import monitor_functions as monf -from .. reco import histogram_functions as histf -from .. database import load_db as dbf -from .. core import system_of_units as units - -from .. evm.pmaps_test import pmaps -from .. evm.pmaps_test import sensor_responses - - -@given(pmaps()) -@settings(deadline=None) -def test_fill_pmap_var_1d(dbnew, pmaps): - var_dict = defaultdict(list) - (s1s, s2s), _ = pmaps - data_sipm = dbf.DataSiPM(dbnew, 4670) - - monf.fill_pmap_var_1d(s1s, var_dict, "S1", DataSiPM=None ) - monf.fill_pmap_var_1d(s2s, var_dict, "S2", DataSiPM=data_sipm) - - assert var_dict['S1_Number'][-1] == len(s1s) - assert var_dict['S2_Number'][-1] == len(s2s) - - for i, speak in enumerate(s1s): - assert var_dict['S1_Energy'][i] == speak.total_energy - assert var_dict['S1_Width'] [i] == speak.width / units.mus - assert var_dict['S1_Height'][i] == speak.height - assert var_dict['S1_Charge'][i] == speak.total_charge - assert var_dict['S1_Time'] [i] == speak.time_at_max_energy / units.mus - - counter = 0 - for i, speak in enumerate(s2s): - assert var_dict['S2_Energy'] [i] == speak.total_energy - assert var_dict['S2_Width'] [i] == speak.width / units.mus - assert var_dict['S2_Height'] [i] == speak.height - assert var_dict['S2_Charge'] [i] == speak.total_charge - assert var_dict['S2_Time'] [i] == speak.time_at_max_energy / units.mus - assert var_dict['S2_SingleS1'][i] == len(s1s) - nsipm = len(speak.sipms.ids) - assert var_dict['S2_NSiPM'] [i] == nsipm - if len(s1s) == 1: - assert var_dict['S2_SingleS1_Energy'][i] == s1s[0].total_energy - assert np.allclose(var_dict['S2_QSiPM'] [counter:counter + nsipm], speak.sipms.sum_over_times ) - assert np.allclose(var_dict['S2_IdSiPM'][counter:counter + nsipm], speak.sipms.ids ) - assert np.allclose(var_dict['S2_XSiPM'] [counter:counter + nsipm], data_sipm.X.values[speak.sipms.ids].tolist()) - assert np.allclose(var_dict['S2_YSiPM'] [counter:counter + nsipm], data_sipm.Y.values[speak.sipms.ids].tolist()) - counter += nsipm - - -@given(pmaps()) -@settings(deadline=None) -def test_fill_pmap_var_2d(dbnew, pmaps): - var_dict = defaultdict(list) - (s1s, s2s), _ = pmaps - data_sipm = dbf.DataSiPM(dbnew, 4670) - - monf.fill_pmap_var_1d(s1s, var_dict, "S1", DataSiPM=None) - monf.fill_pmap_var_1d(s2s, var_dict, "S2", DataSiPM=data_sipm) - monf.fill_pmap_var_2d( var_dict, 'S1') - monf.fill_pmap_var_2d( var_dict, 'S2') - - for i, speak in enumerate(s1s): - assert var_dict['S1_Energy_S1_Width'] [0, i] == speak.total_energy - assert var_dict['S1_Energy_S1_Width'] [1, i] == speak.width / units.mus - assert var_dict['S1_Energy_S1_Height'][0, i] == speak.total_energy - assert var_dict['S1_Energy_S1_Height'][1, i] == speak.height - assert var_dict['S1_Energy_S1_Charge'][0, i] == speak.total_energy - assert var_dict['S1_Energy_S1_Charge'][1, i] == speak.total_charge - assert var_dict['S1_Time_S1_Energy'] [0, i] == speak.time_at_max_energy /units.mus - assert var_dict['S1_Time_S1_Energy'] [1, i] == speak.total_energy - - counter = 0 - for i, speak in enumerate(s2s): - assert var_dict['S2_Energy_S2_Width'] [0, i] == speak.total_energy - assert var_dict['S2_Energy_S2_Width'] [1, i] == speak.width / units.mus - assert var_dict['S2_Energy_S2_Height'][0, i] == speak.total_energy - assert var_dict['S2_Energy_S2_Height'][1, i] == speak.height - assert var_dict['S2_Energy_S2_Charge'][0, i] == speak.total_energy - assert var_dict['S2_Energy_S2_Charge'][1, i] == speak.total_charge - assert var_dict['S2_Time_S2_Energy'] [0, i] == speak.time_at_max_energy /units.mus - assert var_dict['S2_Time_S2_Energy'] [1, i] == speak.total_energy - if len(s1s) == 1: - assert var_dict['S2_Energy_S1_Energy'][0, i] == speak.total_energy - assert var_dict['S2_Energy_S1_Energy'][1, i] == s1s[0].total_energy - sipm_ids = speak.sipms.ids - assert np.allclose(var_dict['S2_XYSiPM'][0, counter:counter + len(sipm_ids)], data_sipm.X.values[speak.sipms.ids].tolist()) - assert np.allclose(var_dict['S2_XYSiPM'][1, counter:counter + len(sipm_ids)], data_sipm.Y.values[speak.sipms.ids].tolist()) - counter += len(sipm_ids) - - -@given(pmaps(pmt_ids=np.arange(0,11,1))) -def test_fill_pmt_var(dbnew, pmaps): - var_dict = defaultdict(list) - (_, s2s), _ = pmaps - data_sipm = dbf.DataSiPM(dbnew, 4670) - - monf.fill_pmt_var(s2s, var_dict) - - for i, speak in enumerate(s2s): - pmts = speak.pmts - times = speak.times - energies = pmts .sum_over_times - heights = np.max (pmts.all_waveforms, axis=1) - times = np.apply_along_axis(lambda wf: times[np.argmax(wf)], axis=1, arr=pmts.all_waveforms) / units.mus - npmts = len(pmts.all_waveforms) - - for j in range(npmts): - assert var_dict[f'PMT{j}_S2_Energy'][i] == energies[j] - assert var_dict[f'PMT{j}_S2_Height'][i] == heights [j] - assert var_dict[f'PMT{j}_S2_Time' ][i] == times [j] - - -def test_pmap_bins(): - test_dict = {'S1_Energy_bins' : [ 0, 10, 2], - 'S1_Width_bins' : [ 0, 6, 3], - 'S1_Time_bins' : [ 0, 5, 4], - 'S2_Energy_bins' : [ 0, 15000, 2], - 'S2_Height_bins' : [ 0, 15000, 2], - 'S2_XSiPM_bins' : [-100, 100, 4], - 'S2_YSiPM_bins' : [-100, 100, 4], - 'S2_Time_bins' : [ 5, 10, 4], - - 'S1_Energy_labels' : ['S1 energy (pes)'], - 'S1_Width_labels' : [ 'S1 width (mus)'], - 'S1_Time_labels' : [ 'S1 time (mus)'], - 'S2_Energy_labels' : ['S2 energy (pes)'], - 'S2_Height_labels' : ['S2 energy (pes)'], - 'S2_XSiPM_labels' : [ 'X (mm)'], - 'S2_YSiPM_labels' : [ 'Y (mm)'], - 'S2_Time_labels' : [ 'S2 time (mus)'], - 'nPMT' : 12} - - test_bins = {'S1_Energy': [ 0, 5, 10 ], - 'S1_Width' : [ 0, 2, 4, 6 ], - 'S1_Time' : [ 0, 1.250, 2.5, 3.75, 5.], - 'S2_Time' : [ 0, 1.250, 2.5, 3.75, 5.], - 'S2_Energy': [ 0, 7.5e3, 15e3 ], - 'S2_Height': [ 0, 7.5e3, 15e3 ], - 'S2_XSiPM' : [-100, -50, 0, 50, 100.], - 'S2_YSiPM' : [-100, -50, 0, 50, 100.], - 'S2_Time' : [ 5, 6.25, 7.5, 8.75, 10.]} - - out_bins, out_labels = monf.pmap_bins(test_dict) - - list_var = ['S1_Energy', 'S1_Width', 'S1_Time', 'S2_Time', 'S2_Energy', 'S2_Height'] - - for var_name in list_var: - assert np.allclose(out_bins[var_name], test_bins[var_name]) - assert test_dict[var_name + '_labels'] == out_labels[var_name] - - assert_bins_and_labels_ndim('S1_Energy_S1_Width' , ['S1_Energy', 'S1_Width' ], out_bins, out_labels, test_bins, test_dict) - assert_bins_and_labels_ndim('S1_Time_S1_Energy' , ['S1_Time' , 'S1_Energy'], out_bins, out_labels, test_bins, test_dict) - assert_bins_and_labels_ndim('S2_Time_S2_Energy' , ['S2_Time' , 'S2_Energy'], out_bins, out_labels, test_bins, test_dict) - assert_bins_and_labels_ndim('S2_Energy_S1_Energy', ['S2_Energy', 'S1_Energy'], out_bins, out_labels, test_bins, test_dict) - assert_bins_and_labels_ndim('S2_XYSiPM' , ['S2_XSiPM' , 'S2_YSiPM' ], out_bins, out_labels, test_bins, test_dict) - - variable_names = ['S1_Energy', 'S1_Width', 'S1_Time', 'S2_Energy', 'S2_Time', - 'S2_Height', 'S1_Energy_S1_Width', 'S1_Time_S1_Energy', - 'S2_Time_S2_Energy', 'S2_Energy_S1_Energy', 'S2_Energy_S2_Height', - 'S2_XYSiPM'] - - for i in range(test_dict['nPMT']): - energy_name = f'PMT{i}_S2_Energy' - height_name = f'PMT{i}_S2_Height' - time_name = f'PMT{i}_S2_Time' - variable_names.append(energy_name) - variable_names.append(height_name) - variable_names.append( time_name) - assert out_labels[energy_name] == [f'PMT{i} ' + test_dict['S2_Energy_labels'][0]] - assert out_labels[height_name] == [f'PMT{i} ' + test_dict['S2_Height_labels'][0]] - assert out_labels[ time_name] == [f'PMT{i} ' + test_dict['S2_Time_labels' ][0]] - - for k in out_bins: - assert k in variable_names - - -def test_fill_pmap_histos(dbnew, ICDATADIR): - test_config_dict = {'S1_Number_bins' : [-0.50, 10.50, 11], - 'S1_Width_bins' : [-0.01, 0.99, 40], - 'S1_Height_bins' : [ 0, 10, 10], - 'S1_Energy_bins' : [ 0, 50, 50], - 'S1_Charge_bins' : [ 0, 2, 20], - 'S1_Time_bins' : [ 0, 650, 650], - - 'S2_Number_bins' : [-0.50, 10.50, 11], - 'S2_Width_bins' : [ 0, 50, 50], - 'S2_Height_bins' : [ 0, 8000, 100], - 'S2_Energy_bins' : [ 0, 20e3, 100], - 'S2_Charge_bins' : [ 0, 3500, 100], - 'S2_Time_bins' : [ 640, 1300, 660], - - 'S2_NSiPM_bins' : [-0.50, 500.50, 501], - 'S2_IdSiPM_bins' : [-0.50, 1792.50, 1793], - 'S2_QSiPM_bins' : [ 0, 100, 100], - 'S2_XSiPM_bins' : [ -200, 200, 40], - 'S2_YSiPM_bins' : [ -200, 200, 40], - - 'S1_Number_labels' : [ "S1 number (#)"], - 'S1_Width_labels' : [ r"S1 width ($\mu$s)"], - 'S1_Height_labels' : [ "S1 height (pes)"], - 'S1_Energy_labels' : [ "S1 energy (pes)"], - 'S1_Charge_labels' : [ "S1 charge (pes)"], - 'S1_Time_labels' : [ r"S1 time ($\mu$s)"], - - 'S2_Number_labels' : [ "S2 number (#)"], - 'S2_Width_labels' : [ r"S2 width ($\mu$s)"], - 'S2_Height_labels' : [ "S2 height (pes)"], - 'S2_Energy_labels' : [ "S2 energy (pes)"], - 'S2_Charge_labels' : [ "S2 charge (pes)"], - 'S2_Time_labels' : [ r"S2 time ($\mu$s)"], - - 'S2_NSiPM_labels' : [ 'SiPM number (#)'], - 'S2_IdSiPM_labels' : [ 'SiPM id'], - 'S2_QSiPM_labels' : [ 'SiPM charge (pes)'], - 'S2_XSiPM_labels' : [ 'X (mm)'], - 'S2_YSiPM_labels' : [ 'Y (mm)'], - - 'nPMT' : 12} - - test_infile = "Kr_pmaps_run4628.h5" - test_infile = os.path.join(ICDATADIR, test_infile) - - run_number = 4628 - - test_histo = monf.fill_pmap_histos(test_infile, dbnew, run_number, test_config_dict) - - test_checkfile = "Kr_pmaps_histos_run4628.h5" - test_checkfile = os.path.join(ICDATADIR, test_checkfile) - check_histo = histf.get_histograms_from_file(test_checkfile) - - assert set(check_histo.histos) == set(test_histo.histos) - - for k, v in check_histo.histos.items(): - assert np.allclose(v.data , test_histo.histos[k].data ) - assert np.allclose(v.out_range, test_histo.histos[k].out_range) - assert np.allclose(v.errors , test_histo.histos[k].errors ) - assert v.title == test_histo.histos[k].title - assert v.labels == test_histo.histos[k].labels - for i, bins in enumerate(v.bins): - assert np.allclose(bins, test_histo.histos[k].bins [i]) - - -def test_fill_rwf_var(): - var_dict = defaultdict(list) - pmt_waveforms = np.random.uniform(0, 10, size=( 12, 10000)) - sipm_waveforms = np.random.uniform(0, 10, size=(1792, 10000)) - monf.fill_rwf_var( pmt_waveforms, var_dict, "PMT" ) - monf.fill_rwf_var(sipm_waveforms, var_dict, "SiPM") - - assert np.allclose(var_dict['PMT_Baseline'] , np.mean( pmt_waveforms, axis=1)) - assert np.allclose(var_dict['PMT_BaselineRMS'] , np.std ( pmt_waveforms, axis=1)) - assert np.allclose(var_dict['SiPM_Baseline'] , np.mean(sipm_waveforms, axis=1)) - assert np.allclose(var_dict['SiPM_BaselineRMS'], np.std (sipm_waveforms, axis=1)) - - -def test_rwf_bins(): - test_dict = {'PMT_Baseline_bins' : [2300, 2700, 400], - 'PMT_BaselineRMS_bins' : [ 0, 10, 100], - 'PMT_nSensors_bins' : [-0.5, 12, 100], - 'SiPM_Baseline_bins' : [ 0, 100, 100], - 'SiPM_BaselineRMS_bins' : [ 0, 10, 100], - 'SiPM_nSensors_bins' : [-0.5, 12, 100], - - - 'PMT_Baseline_labels' : ["ADCs"] , - 'PMT_BaselineRMS_labels' : ["ADCs"] , - 'PMT_nSensors_labels' : ["Number of PMTs"] , - 'SiPM_Baseline_labels' : ["ADCs"] , - 'SiPM_BaselineRMS_labels' : ["ADCs"] , - 'SiPM_nSensors_labels' : ["Number of SiPMs"], - - 'n_baseline' : 10000 } - - out_bins, out_labels, out_baseline = monf.rwf_bins(test_dict) - - bins = test_dict['PMT_Baseline_bins'] - assert np.allclose(out_bins['PMT_Baseline'] , [np.linspace(bins[0], bins[1], bins[2] + 1)]) - bins = test_dict['PMT_BaselineRMS_bins'] - assert np.allclose(out_bins['PMT_BaselineRMS'] , [np.linspace(bins[0], bins[1], bins[2] + 1)]) - bins = test_dict['PMT_nSensors_bins'] - assert np.allclose(out_bins['PMT_nSensors'] , [np.linspace(bins[0], bins[1], bins[2] + 1)]) - bins = test_dict['SiPM_Baseline_bins'] - assert np.allclose(out_bins['SiPM_Baseline'] , [np.linspace(bins[0], bins[1], bins[2] + 1)]) - bins = test_dict['SiPM_BaselineRMS_bins'] - assert np.allclose(out_bins['SiPM_BaselineRMS'], [np.linspace(bins[0], bins[1], bins[2] + 1)]) - bins = test_dict['SiPM_nSensors_bins'] - assert np.allclose(out_bins['SiPM_nSensors'] , [np.linspace(bins[0], bins[1], bins[2] + 1)]) - - assert out_labels['PMT_Baseline'] [0] == test_dict['PMT_Baseline_labels'] [0] - assert out_labels['PMT_BaselineRMS'] [0] == test_dict['PMT_BaselineRMS_labels'] [0] - assert out_labels['PMT_nSensors'] [0] == test_dict['PMT_nSensors_labels'] [0] - assert out_labels['SiPM_Baseline'] [0] == test_dict['SiPM_Baseline_labels'] [0] - assert out_labels['SiPM_BaselineRMS'][0] == test_dict['SiPM_BaselineRMS_labels'][0] - assert out_labels['SiPM_nSensors'] [0] == test_dict['SiPM_nSensors_labels'] [0] - assert out_baseline == test_dict['n_baseline'] - - -def test_fill_rwf_histos(ICDATADIR): - test_config_dict = {'PMT_Baseline_bins' : [ 2300, 2700, 400], - 'PMT_BaselineRMS_bins' : [ 0, 10, 100], - 'PMT_nSensors_bins' : [ -0.5, 12.5, 13], - 'SiPM_Baseline_bins' : [ 0, 100, 100], - 'SiPM_BaselineRMS_bins' : [ 0, 10, 100], - 'SiPM_nSensors_bins' : [1750.5, 1800.5, 50], - - 'PMT_Baseline_labels' : ["PMT Baseline (ADC)"], - 'PMT_BaselineRMS_labels' : ["PMT Baseline RMS (ADC)"], - 'PMT_nSensors_labels' : ["Number of PMTs"], - 'SiPM_Baseline_labels' : ["SiPM Baseline (ADC)"], - 'SiPM_BaselineRMS_labels' : ["SiPM Baseline RMS (ADC)"], - 'SiPM_nSensors_labels' : ["Number of SiPMs"], - - 'n_baseline' : 48000} - - test_infile = "irene_bug_Kr_ACTIVE_7bar_RWF.h5" - test_infile = os.path.join(ICDATADIR, test_infile) - - test_histo = monf.fill_rwf_histos(test_infile, test_config_dict) - - test_checkfile = "irene_bug_Kr_ACTIVE_7bar_RWF_histos.h5" - test_checkfile = os.path.join(ICDATADIR, test_checkfile) - check_histo = histf.get_histograms_from_file(test_checkfile) - - assert set(check_histo.histos) == set(test_histo.histos) - - for k, v in check_histo.histos.items(): - assert np.allclose(v.data , test_histo.histos[k].data ) - assert np.allclose(v.out_range, test_histo.histos[k].out_range) - assert np.allclose(v.errors , test_histo.histos[k].errors ) - assert v.title == test_histo.histos[k].title - assert v.labels == test_histo.histos[k].labels - for i, bins in enumerate(v.bins): - assert np.allclose(bins, test_histo.histos[k].bins[i]) - - -def test_kdst_bins(): - test_dict = {'S1e_bins' : [ 0, 10, 2], - 'S1w_bins' : [ 0, 6, 3], - 'S1t_bins' : [ 0, 5, 4], - 'S2e_bins' : [ 0, 15000, 2], - 'S2q_bins' : [ 0, 15000, 2], - 'S2h_bins' : [ 0, 15000, 2], - 'X_bins' : [-100, 100, 4], - 'Y_bins' : [-100, 100, 4], - 'Z_bins' : [-100, 100, 4], - 'R_bins' : [-100, 100, 4], - 'Phi_bins' : [-100, 100, 4], - 'S2t_bins' : [ 5, 10, 4], - - 'S1e_labels' : ['S1 energy (pes)'], - 'S1w_labels' : [ 'S1 width (mus)'], - 'S1t_labels' : [ 'S1 time (mus)'], - 'S2e_labels' : ['S2 energy (pes)'], - 'S2q_labels' : ['S2 charge (pes)'], - 'S2h_labels' : ['S2 energy (pes)'], - 'X_labels' : [ 'X (mm)'], - 'Y_labels' : [ 'Y (mm)'], - 'Z_labels' : [ 'Z (mm)'], - 'R_labels' : [ 'R (mm)'], - 'Phi_labels' : [ 'Phi (rad)'], - 'S2t_labels' : [ 'S2 time (mus)']} - - test_bins = {'S1e' : [ 0, 5, 10 ], - 'S1w' : [ 0, 2, 4, 6 ], - 'S1t' : [ 0, 1.250, 2.5, 3.75, 5.], - 'S2t' : [ 0, 1.250, 2.5, 3.75, 5.], - 'S2e' : [ 0, 7.5e3, 15e3 ], - 'S2q' : [ 0, 7.5e3, 15e3 ], - 'S2h' : [ 0, 7.5e3, 15e3 ], - 'X' : [-100, -50, 0, 50, 100.], - 'Y' : [-100, -50, 0, 50, 100.], - 'Z' : [-100, -50, 0, 50, 100.], - 'R' : [-100, -50, 0, 50, 100.], - 'Phi' : [-100, -50, 0, 50, 100.], - 'S2t' : [ 5, 6.25, 7.5, 8.75, 10.]} - - out_bins, out_labels = monf.kdst_bins(test_dict) - - for var_name in test_bins: - assert np.allclose(out_bins [var_name] , test_bins [var_name]) - assert test_dict[var_name + '_labels'] == out_labels[var_name] - - assert_bins_and_labels_ndim('S2e_X' , ['X' , 'S2e' ], out_bins, out_labels, test_bins, test_dict) - assert_bins_and_labels_ndim('S2e_Y' , ['Y' , 'S2e' ], out_bins, out_labels, test_bins, test_dict) - assert_bins_and_labels_ndim('S2e_Z' , ['Z' , 'S2e' ], out_bins, out_labels, test_bins, test_dict) - assert_bins_and_labels_ndim('S2e_R' , ['R' , 'S2e' ], out_bins, out_labels, test_bins, test_dict) - assert_bins_and_labels_ndim('S2e_Phi', ['Phi', 'S2e' ], out_bins, out_labels, test_bins, test_dict) - assert_bins_and_labels_ndim('S2q_X' , ['X' , 'S2q' ], out_bins, out_labels, test_bins, test_dict) - assert_bins_and_labels_ndim('S2q_Y' , ['Y' , 'S2q' ], out_bins, out_labels, test_bins, test_dict) - assert_bins_and_labels_ndim('S2q_Z' , ['Z' , 'S2q' ], out_bins, out_labels, test_bins, test_dict) - assert_bins_and_labels_ndim('S2q_R' , ['R' , 'S2q' ], out_bins, out_labels, test_bins, test_dict) - assert_bins_and_labels_ndim('S2q_Phi', ['Phi', 'S2q' ], out_bins, out_labels, test_bins, test_dict) - assert_bins_and_labels_ndim('XY' , ['X' , 'Y' ], out_bins, out_labels, test_bins, test_dict) - assert_bins_and_labels_ndim('S2e_XY' , ['X' , 'Y', 'S2e'], out_bins, out_labels, test_bins, test_dict) - assert_bins_and_labels_ndim('S2q_XY' , ['X' , 'Y', 'S2q'], out_bins, out_labels, test_bins, test_dict) - - variable_names = list(test_bins) - variable_names.extend(['S2e_X', 'S2e_Y' , 'S2e_Z', 'S2e_R', 'S2e_Phi', - 'S2q_X', 'S2q_Y' , 'S2q_Z', 'S2q_R', 'S2q_Phi', - 'XY' , 'S2e_XY','S2q_XY']) - for k in out_bins: - assert k in variable_names - - -kdst_variables = ['nS2', 'S1w' , 'S1h', 'S1e', 'S1t', 'S2w', 'S2h', 'S2e', 'S2q' , - 'S2t', 'Nsipm', 'DT' , 'Z' , 'X' , 'Y' , 'R' , 'Phi', 'Zrms', - 'Xrms', 'Yrms'] - - -@given(data_frames(columns=columns(kdst_variables, elements=floats(allow_nan=False)))) -@settings(deadline=None, suppress_health_check=(HealthCheck.too_slow,)) -def test_fill_kdst_var_1d(kdst): - var_dict = defaultdict(list) - monf.fill_kdst_var_1d (kdst, var_dict) - - for var in var_dict: - value = kdst[var].values - if var in ['S1t', 'S2t', 'S1w']: - value = value / units.mus - assert np.allclose(value, var_dict[var]) - - -@given(data_frames(columns=columns(kdst_variables, elements=floats(allow_nan=False)))) -@settings(deadline=None, suppress_health_check=(HealthCheck.too_slow,)) -def test_fill_kdst_var_2d(kdst): - var_dict = defaultdict(list) - monf.fill_kdst_var_1d (kdst, var_dict) - monf.fill_kdst_var_2d (var_dict) - - param_list = ['Z', 'X', 'Y', 'R', 'Phi'] - kdstE = kdst['S2e'] .values - kdstQ = kdst['S2q'] .values - for param in param_list: - valueE = [kdst[param].values, kdstE] - valueQ = [kdst[param].values, kdstQ] - assert np.allclose(valueE, var_dict['S2e_' + param]) - assert np.allclose(valueQ, var_dict['S2q_' + param]) - np.allclose([kdst['X'].values, kdst['Y'].values], var_dict['XY' ]) - np.allclose([kdst['X'].values, kdst['Y'].values , kdstE], var_dict['S2e_XY' ]) - np.allclose([kdst['X'].values, kdst['Y'].values , kdstQ], var_dict['S2q_XY' ]) - - -def test_fill_kdst_histos(ICDATADIR): - test_config_dict = {'nS2_bins' : [-0.5 , 10.5 , 11], - 'S1w_bins' : [-0.01, 0.99, 40], - 'S1h_bins' : [ 0, 10, 10], - 'S1e_bins' : [ 0, 50, 50], - 'S1t_bins' : [ 0, 650, 650], - 'S2w_bins' : [ 0, 50, 50], - 'S2h_bins' : [ 0, 8000, 100], - 'S2e_bins' : [ 0, 20000, 100], - 'S2q_bins' : [ 0, 2000, 100], - 'S2t_bins' : [ 640, 1300, 660], - 'Nsipm_bins': [-0.5 , 500.5 , 501], - 'DT_bins' : [ 0, 600, 100], - 'Z_bins' : [ 0, 600, 100], - 'X_bins' : [ -200, 200, 50], - 'Y_bins' : [ -200, 200, 50], - 'R_bins' : [ 0, 200, 50], - 'Phi_bins' : [-3.15, 3.15, 50], - 'Zrms_bins' : [ 0, 80, 40], - 'Xrms_bins' : [ 0, 200, 50], - 'Yrms_bins' : [ 0, 200, 50], - - 'nS2_labels' : [ 'S2 number (#)'], - 'S1w_labels' : [ r'S1 width ($\mu$s)'], - 'S1h_labels' : [ 'S1 height (pes)'], - 'S1e_labels' : [ 'S1 energy (pes)'], - 'S1t_labels' : [ r'S1 time ($\mu$s)'], - 'S2w_labels' : [ r'S2 width ($\mu$s)'], - 'S2h_labels' : [ 'S2 height (pes)'], - 'S2e_labels' : [ 'S2 energy (pes)'], - 'S2q_labels' : [ 'S2 charge (pes)'], - 'S2t_labels' : [ r'S2 time ($\mu$s)'], - 'Nsipm_labels': [ 'SiPM number (#)'], - 'DT_labels' : [r'S2 - S1 time ($\mu$s)'], - 'Z_labels' : [ 'Z (mm)'], - 'X_labels' : [ 'X (mm)'], - 'Y_labels' : [ 'Y (mm)'], - 'R_labels' : [ 'R (mm)'], - 'Phi_labels' : [ 'Phi (rad)'], - 'Zrms_labels' : [ 'Z rms (mm)'], - 'Xrms_labels' : [ 'X rms (mm)'], - 'Yrms_labels' : [ 'Y rms (mm)']} - - test_infile = "dst_NEXT_v1_00_05_Kr_ACTIVE_0_0_7bar_KDST_10evt_new.h5" - test_infile = os.path.join(ICDATADIR, test_infile) - - test_histo = monf.fill_kdst_histos(test_infile, test_config_dict) - - test_checkfile = "kdst_histos.h5" - test_checkfile = os.path.join(ICDATADIR, test_checkfile) - check_histo = histf.get_histograms_from_file(test_checkfile) - - assert set(check_histo.histos) == set(test_histo.histos) - - for k, v in check_histo.histos.items(): - assert np.allclose(v.data , test_histo.histos[k].data ) - assert np.allclose(v.out_range, test_histo.histos[k].out_range) - assert np.allclose(v.errors , test_histo.histos[k].errors ) - assert v.title == test_histo.histos[k].title - assert v.labels == test_histo.histos[k].labels - for i, bins in enumerate(v.bins): - assert np.allclose(bins, test_histo.histos[k].bins [i]) - - -def assert_bins_and_labels_ndim(var_name, single_vars, out_bins, out_labels, test_bins, test_dict): - for i, var in enumerate(single_vars): - assert np.allclose(out_bins [var_name][i] , test_bins[var]) - assert out_labels[var_name][i] == test_dict[var + '_labels'][0] diff --git a/invisible_cities/reco/olivia.py b/invisible_cities/reco/olivia.py deleted file mode 100644 index 9c5ad82b1a..0000000000 --- a/invisible_cities/reco/olivia.py +++ /dev/null @@ -1,42 +0,0 @@ -import sys -import os -import json - -import numpy as np - -from invisible_cities.io .hist_io import save_histomanager_to_file -from invisible_cities.core.configure import configure - -import invisible_cities.reco.monitor_functions as monf - - -data_types = ['rwf', 'pmaps', 'kdst'] - -def olivia(conf): - print(conf) - files_in = os.path.expandvars(conf.files_in) - file_out = os.path.expandvars(conf.file_out) - detector_db = conf.detector_db - run_number = int(conf.run_number) - data_type = conf.data_type - histo_config = conf.histo_config - - if data_type not in data_types: - print(f'Error: Data type {data_type} is not recognized.') - print(f'Supported data types: {data_types}') - sys.exit(2) - - with open(histo_config) as config_file: - config_dict = json.load(config_file) - if data_type == 'rwf' : - histo_manager = monf.fill_rwf_histos (files_in, config_dict) - elif data_type == 'pmaps': - histo_manager = monf.fill_pmap_histos(files_in, detector_db, run_number, config_dict) - elif data_type == 'kdst' : - histo_manager = monf.fill_kdst_histos(files_in, config_dict) - save_histomanager_to_file(histo_manager, file_out) - - -if __name__ == "__main__": - conf = configure(sys.argv).as_namespace - olivia(conf) diff --git a/invisible_cities/reco/paolina_functions.py b/invisible_cities/reco/paolina_functions.py index e6d80076b0..0f60e5de73 100644 --- a/invisible_cities/reco/paolina_functions.py +++ b/invisible_cities/reco/paolina_functions.py @@ -7,17 +7,16 @@ import numpy as np import networkx as nx -from networkx import Graph -from .. evm.event_model import Voxel -from .. core.exceptions import NoHits -from .. core.exceptions import NoVoxels -from .. evm.event_model import BHit -from .. evm.event_model import Voxel -from .. evm.event_model import Track -from .. evm.event_model import Blob -from .. evm.event_model import TrackCollection -from .. evm.event_model import HitEnergy -from .. core.system_of_units_c import units +from networkx import Graph +from .. evm.event_model import Voxel +from .. core.exceptions import NoHits +from .. core.exceptions import NoVoxels +from .. evm.event_model import BHit +from .. evm.event_model import Track +from .. evm.event_model import Blob +from .. evm.event_model import TrackCollection +from .. evm.event_model import HitEnergy +from .. core import system_of_units as units from typing import Sequence from typing import List diff --git a/invisible_cities/reco/paolina_functions_test.py b/invisible_cities/reco/paolina_functions_test.py index 7649f8f7fd..e07db46353 100644 --- a/invisible_cities/reco/paolina_functions_test.py +++ b/invisible_cities/reco/paolina_functions_test.py @@ -25,7 +25,6 @@ from hypothesis.strategies import floats from hypothesis.strategies import integers from hypothesis.strategies import builds -from hypothesis.strategies import integers from networkx.generators.random_graphs import fast_gnp_random_graph @@ -54,12 +53,13 @@ from . paolina_functions import make_tracks from . paolina_functions import get_track_energy -from .. core.exceptions import NoHits -from .. core.exceptions import NoVoxels -from .. core.system_of_units_c import units -from .. core.testing_utils import assert_bhit_equality +from .. core import system_of_units as units +from .. core.exceptions import NoHits +from .. core.exceptions import NoVoxels +from .. core.testing_utils import assert_bhit_equality -from .. io.mcinfo_io import load_mchits +from .. io.mcinfo_io import cast_mchits_to_dict +from .. io.mcinfo_io import load_mchits_df from .. io.hits_io import load_hits from .. types.ic_types import xy @@ -240,20 +240,14 @@ def test_voxels_with_no_hits(ICDATADIR): size = 15. vox_size = np.array([size,size,size],dtype=np.float16) - with tb.open_file(hit_file, mode='r') as h5in: - - h5extents = h5in.root.MC.extents - events_in_file = len(h5extents) - - for i in range(events_in_file): - if h5extents[i]['evt_number'] == evt_number: - evt_line = i - break - - hits_dict = load_mchits(hit_file, (evt_line, evt_line+1)) - voxels = voxelize_hits(hits_dict[evt_number], vox_size, strict_voxel_size=False) - for v in voxels: - assert sum(h.E for h in v.hits) == v.E + hits_df = load_mchits_df(hit_file) + hits_dict = cast_mchits_to_dict(hits_df) + hit_seq = hits_dict[evt_number] + voxels = voxelize_hits(hit_seq , + vox_size , + strict_voxel_size = False) + for v in voxels: + assert sum(h.E for h in v.hits) == v.E @given(bunch_of_hits, box_sizes) @@ -403,8 +397,8 @@ def voxels_without_hits(): return voxels -def test_length(): - voxels = voxels_without_hits() +def test_length(voxels_without_hits): + voxels = voxels_without_hits tracks = make_track_graphs(voxels) assert len(tracks) == 1 @@ -749,9 +743,9 @@ def test_blob_hits_are_inside_radius(hits, voxel_dimensions, blob_radius): assert np.linalg.norm(h.XYZ - centre_b) < blob_radius -@given(radius, min_n_of_voxels, fraction_zero_one) -def test_paolina_functions_with_voxels_without_associated_hits(blob_radius, min_voxels, fraction_zero_one): - voxels = voxels_without_hits() +@given(blob_radius=radius, min_voxels=min_n_of_voxels, fraction_zero_one=fraction_zero_one) +def test_paolina_functions_with_voxels_without_associated_hits(blob_radius, min_voxels, fraction_zero_one, voxels_without_hits): + voxels = voxels_without_hits tracks = make_track_graphs(voxels) for t in tracks: a, b = find_extrema(t) diff --git a/invisible_cities/reco/peak_functions.py b/invisible_cities/reco/peak_functions.py index e5d13c5f16..686b16c87d 100644 --- a/invisible_cities/reco/peak_functions.py +++ b/invisible_cities/reco/peak_functions.py @@ -10,13 +10,13 @@ import numpy as np -from .. core.system_of_units_c import units -from .. evm .ic_containers import ZsWf -from .. evm .pmaps import S1 -from .. evm .pmaps import S2 -from .. evm .pmaps import PMap -from .. evm .pmaps import PMTResponses -from .. evm .pmaps import SiPMResponses +from .. core import system_of_units as units +from .. evm .ic_containers import ZsWf +from .. evm .pmaps import S1 +from .. evm .pmaps import S2 +from .. evm .pmaps import PMap +from .. evm .pmaps import PMTResponses +from .. evm .pmaps import SiPMResponses def indices_and_wf_above_threshold(wf, thr): diff --git a/invisible_cities/reco/peak_functions_test.py b/invisible_cities/reco/peak_functions_test.py index bea7a6cae1..16efa9bcf7 100644 --- a/invisible_cities/reco/peak_functions_test.py +++ b/invisible_cities/reco/peak_functions_test.py @@ -13,20 +13,20 @@ from hypothesis.strategies import integers from hypothesis.extra.numpy import arrays -from ..core.testing_utils import exactly -from ..core.testing_utils import previous_float -from ..core.testing_utils import assert_Peak_equality -from ..core.testing_utils import assert_PMap_equality -from ..core.system_of_units_c import units -from ..core.fit_functions import gauss -from ..evm .pmaps import PMTResponses -from ..evm .pmaps import SiPMResponses -from ..evm .pmaps import S1 -from ..evm .pmaps import S2 -from ..evm .pmaps import PMap -from ..io .pmaps_io import load_pmaps -from ..types.ic_types_c import minmax -from . import peak_functions as pf +from ..core.testing_utils import exactly +from ..core.testing_utils import previous_float +from ..core.testing_utils import assert_Peak_equality +from ..core.testing_utils import assert_PMap_equality +from ..core import system_of_units as units +from ..core.fit_functions import gauss +from ..evm .pmaps import PMTResponses +from ..evm .pmaps import SiPMResponses +from ..evm .pmaps import S1 +from ..evm .pmaps import S2 +from ..evm .pmaps import PMap +from ..io .pmaps_io import load_pmaps +from ..types.ic_types import minmax +from . import peak_functions as pf wf_min = 0 @@ -36,21 +36,21 @@ @composite def waveforms(draw): n_samples = draw(integers(1, 50)) - return draw(arrays(float, n_samples, floats(wf_min, wf_max))) + return draw(arrays(float, n_samples, elements=floats(wf_min, wf_max))) @composite def multiple_waveforms(draw): n_sensors = draw(integers(1, 10)) n_samples = draw(integers(1, 50)) - return draw(arrays(float, (n_sensors, n_samples), floats(wf_min, wf_max))) + return draw(arrays(float, (n_sensors, n_samples), elements=floats(wf_min, wf_max))) @composite def times_and_waveforms(draw): waveforms = draw(multiple_waveforms()) n_samples = waveforms.shape[1] - times = draw(arrays(float, n_samples, floats(0, 10*n_samples), unique=True)) + times = draw(arrays(float, n_samples, elements=floats(0, 10*n_samples), unique=True)) return times, waveforms @@ -80,7 +80,7 @@ def rebinned_sliced_waveforms(draw): @composite def peak_indices(draw): size = draw(integers(10, 50)) - indices = draw(arrays(int, size, integers(0, 5 * size), unique=True)) + indices = draw(arrays(int, size, elements=integers(0, 5 * size), unique=True)) indices = np.sort(indices) stride = draw(integers(1, 5)) peaks = np.split(indices, 1 + np.where(np.diff(indices) > stride)[0]) @@ -593,7 +593,7 @@ def test_rebin_times_and_waveforms_times_are_consistent(t_and_wf, stride): widths = [1] if len(times) > 1: widths = np.append(np.diff(times), max(np.diff(times))) - + # The samples falling in the last bin cannot be so easily # compared as the other ones so I remove them. remain = times.size - times.size % stride @@ -611,6 +611,6 @@ def test_rebin_times_and_waveforms_negative_bins(ICDATADIR): pmap_file = os.path.join(ICDATADIR, 'pmaps_negative_bins.h5') pmaps = load_pmaps(pmap_file) s2 = pmaps[48490].s2s[0] - rebinned_info = pf.rebin_times_and_waveforms(s2.times , - s2.bin_widths , - s2.pmts.all_waveforms) + pf.rebin_times_and_waveforms(s2.times , + s2.bin_widths , + s2.pmts.all_waveforms) diff --git a/invisible_cities/reco/sensor_functions.py b/invisible_cities/reco/sensor_functions.py index 965e3379b9..e3c137f336 100644 --- a/invisible_cities/reco/sensor_functions.py +++ b/invisible_cities/reco/sensor_functions.py @@ -77,12 +77,12 @@ def simulate_pmt_response(event, pmtrd, adc_to_pes, pe_resolution, detector_db=' return np.array(RWF), np.array(BLRX) -def simulate_sipm_response(event, sipmrd, sipms_noise_sampler, sipm_adc_to_pes, pe_resolution): +def simulate_sipm_response(sipmrd, sipms_noise_sampler, sipm_adc_to_pes, pe_resolution): """Add noise to the sipms with the NoiseSampler class and return the noisy waveform (in adc).""" ## Fluctuate according to charge resolution - sipm_fl = np.array(tuple(map(charge_fluctuation, sipmrd[event], pe_resolution))) + sipm_fl = np.array(tuple(map(charge_fluctuation, sipmrd, pe_resolution))) # return total signal in adc counts + noise sampled from pdf spectra return wfm.to_adc(sipm_fl, sipm_adc_to_pes) + sipms_noise_sampler.sample() diff --git a/invisible_cities/reco/sensor_functions_test.py b/invisible_cities/reco/sensor_functions_test.py index 4bf0157380..b487bb6729 100644 --- a/invisible_cities/reco/sensor_functions_test.py +++ b/invisible_cities/reco/sensor_functions_test.py @@ -1,17 +1,14 @@ -import os import tables as tb import numpy as np import pandas as pd + from pytest import mark -from .. core import system_of_units as units from .. core.random_sampling import NoiseSampler as SiPMsNoiseSampler +from .. sierpe import blr +from .. database import load_db -from . import tbl_functions as tbl -from . import wfm_functions as wfm -from .. sierpe import fee as FEE -from .. sierpe import blr -from .. database import load_db +from . import wfm_functions as wfm from . sensor_functions import convert_channel_id_to_IC_id from . sensor_functions import simulate_pmt_response @@ -19,18 +16,17 @@ def test_cwf_blr(dbnew, electron_MCRD_file): """Check that CWF -> (deconv) (RWF) == BLR within 1 %. """ - run_number = 0 - DataPMT = load_db.DataPMT(dbnew, run_number) - pmt_active = np.nonzero(DataPMT.Active.values)[0].tolist() - channel_id = DataPMT.ChannelID.values - coeff_blr = abs(DataPMT.coeff_blr.values) - coeff_c = abs(DataPMT.coeff_c .values) - adc_to_pes = abs(DataPMT.adc_to_pes.values) + run_number = 0 + DataPMT = load_db.DataPMT(dbnew, run_number) + pmt_active = np.nonzero(DataPMT.Active.values)[0].tolist() + coeff_blr = abs(DataPMT.coeff_blr.values) + coeff_c = abs(DataPMT.coeff_c .values) + adc_to_pes = abs(DataPMT.adc_to_pes.values) single_pe_rms = abs(DataPMT.Sigma.values) with tb.open_file(electron_MCRD_file, 'r') as h5in: event = 0 - _, pmtrd, _ = tbl.get_rd_vectors(h5in) + pmtrd = h5in.root.pmtrd dataPMT, BLR = simulate_pmt_response(event, pmtrd, adc_to_pes, single_pe_rms) RWF = dataPMT.astype(np.int16) diff --git a/invisible_cities/reco/tbl_functions.py b/invisible_cities/reco/tbl_functions.py index ef22c3b1b0..ab3a0fdb55 100644 --- a/invisible_cities/reco/tbl_functions.py +++ b/invisible_cities/reco/tbl_functions.py @@ -13,13 +13,7 @@ import re import numpy as np import tables as tb -import pandas as pd -from argparse import Namespace - -from ..evm.event_model import SensorParams -from ..evm.event_model import MCInfo -from ..core.exceptions import NoParticleInfoInFile def filters(name): """Return the filter corresponding to a given key. @@ -52,46 +46,6 @@ def filters(name): raise ValueError("Compression option {} not found.".format(name)) -def read_FEE_table(fee_t): - """Read the FEE table and return a PD Series for the simulation - parameters and a PD series for the values of the capacitors used - in the simulation. - """ - - fa = fee_t.read() - - F = pd.Series([fa[0][ 0], fa[0][ 1], fa[0][ 2], fa[0][ 3], fa[0][ 4], - fa[0][ 5], fa[0][ 6], fa[0][ 7], fa[0][ 8], fa[0][ 9], - fa[0][10], fa[0][11], fa[0][12], fa[0][13], fa[0][14], - fa[0][15], fa[0][16], fa[0][17]], - index=["OFFSET", "CEILING", "PMT_GAIN", "FEE_GAIN", "R1", - "C1", "C2", "ZIN", "DAQ_GAIN", "NBITS", "LSB", - "NOISE_I", "NOISE_DAQ", "t_sample", "f_sample", - "f_mc", "f_LPF1", "f_LPF2"]) - FEE = Namespace() - FEE.fee_param = F - FEE.coeff_c = np.array(fa[0][18], dtype=np.double) - FEE.coeff_blr = np.array(fa[0][19], dtype=np.double) - FEE.adc_to_pes = np.array(fa[0][20], dtype=np.double) - FEE.pmt_noise_rms = np.array(fa[0][21], dtype=np.double) - return FEE - - -def table_from_params(table, params): - row = table.row - for param in table.colnames: - row[param] = params[param] - row.append() - table.flush() - - -def table_to_params(table): - params = {} - for param in table.colnames: - params[param] = table[0][param] - return params - - def get_vectors(h5f): """Return the most relevant fields stored in a raw data file. @@ -122,114 +76,8 @@ def get_vectors(h5f): if 'sipmrwf' in h5f.root.RD: sipmrwf = h5f.root.RD.sipmrwf - - return pmtrwf, pmtblr, sipmrwf - - -def get_rwf_vectors(h5in): - """Return the most relevant fields stored in a raw data file. - - Parameters - ---------- - h5f : tb.File - (Open) hdf5 file. - - Returns - ------- - NEVT : number of events in array - pmtrwf : tb.EArray - RWF array for PMTs - pmtblr : tb.EArray - BLR array for PMTs - sipmrwf : tb.EArray - RWF array for PMTs - - """ - pmtrwf, pmtblr, sipmrwf = get_vectors(h5in) - NEVT_pmt , _, _ = pmtrwf .shape - NEVT_simp, _, _ = sipmrwf.shape - - #assert NEVT_simp == NEVT_pmt - return max(NEVT_pmt, NEVT_simp), pmtrwf, sipmrwf, pmtblr - -def get_rd_vectors(h5in): - "Return MC RD vectors and sensor data." - pmtrd = h5in.root.pmtrd - sipmrd = h5in.root.sipmrd - NEVT_pmt , _, _ = pmtrd .shape - NEVT_simp, _, _ = sipmrd.shape - assert NEVT_simp == NEVT_pmt - - return NEVT_pmt, pmtrd, sipmrd - - - -def get_mc_info(h5in): - """Return MC info bank""" - - extents = h5in.root.MC.extents - hits = h5in.root.MC.hits - particles = h5in.root.MC.particles - - try: - h5in.root.MC.particles[0] - except: - raise NoParticleInfoInFile('Trying to access particle information: this file could have sensor response only.') - - if len(h5in.root.MC.hits) == 0: - hits = np.zeros((0,), dtype=('3 d -@mark.skip("Functions removed") -def test_get_neighbours(): - pos = np.array([(35.5, 55.5)]) - - exp_xs = np.array([35, 35, 35, 25, 25, 25, 45, 45, 45]) - exp_ys = np.array([55, 65, 45, 55, 65, 45, 55, 65, 45]) - expected_neighbours = np.stack((exp_xs, exp_ys), axis=1) - - found_neighbours = get_neighbours(pos, pitch = 10. * units.mm) - - number_of_sipm_found_correctly = 0 - for found in found_neighbours: - assert any(have_same_position_in_space(found, expected) for expected in expected_neighbours) - number_of_sipm_found_correctly += 1 - - assert number_of_sipm_found_correctly == 9 - - -@mark.skip("Function removed") -def test_is_masked(): - pos_masked = np.array([(0, 2), - (2, 1)]) - - sipm_masked = [(0, 2)] - sipm_alive = [(3, 2)] - - assert is_masked(sipm_masked, pos_masked) - assert not is_masked(sipm_alive , pos_masked) - - - def test_count_masked_all_active(datasipm_all_active): xy0 = np.array([0, 0], dtype=np.float) is_masked = datasipm_all_active.Active.values diff --git a/invisible_cities/sierpe/blr_test.py b/invisible_cities/sierpe/blr_test.py index 65d9134544..4f72dcdc57 100644 --- a/invisible_cities/sierpe/blr_test.py +++ b/invisible_cities/sierpe/blr_test.py @@ -1,13 +1,9 @@ -import os - from collections import namedtuple -from itertools import starmap import numpy as np import tables as tb from pytest import fixture -from pytest import approx from pytest import mark from flaky import flaky diff --git a/invisible_cities/sierpe/cblr.pyx b/invisible_cities/sierpe/cblr.pyx deleted file mode 100644 index c2335f0943..0000000000 --- a/invisible_cities/sierpe/cblr.pyx +++ /dev/null @@ -1,537 +0,0 @@ -import numpy as np -cimport numpy as np -from scipy import signal as SGN - -cpdef BLR(np.ndarray[np.int16_t, ndim=1] signal_daq, float coef, - int nm, float thr1, float thr2, float thr3): - """ - Deconvolution offline of the DAQ signal - """ - cdef int len_signal_daq - len_signal_daq = signal_daq.shape[0] - - cdef np.ndarray[np.float64_t, ndim=1] signal_i = signal_daq.astype(float) - cdef np.ndarray[np.float64_t, ndim=1] B_MAU = (1 / nm)*np.ones(nm, dtype=np.float64) - cdef np.ndarray[np.float64_t, ndim=1] MAU = np.zeros(len_signal_daq, dtype=np.float64) - cdef np.ndarray[np.float64_t, ndim=1] acum = np.zeros(len_signal_daq, dtype=np.float64) - cdef np.ndarray[np.float64_t, ndim=1] signal_r = np.zeros(len_signal_daq, dtype=np.float64) - cdef np.ndarray[np.float64_t, ndim=1] pulse_ = np.zeros(len_signal_daq, dtype=np.float64) - cdef np.ndarray[np.float64_t, ndim=1] wait_ = np.zeros(len_signal_daq, dtype=np.float64) - - cdef float BASELINE, upper, lower - - MAU[0:nm] = SGN.lfilter(B_MAU, 1, signal_daq[0:nm]) - - acum[nm] = MAU[nm] - BASELINE = MAU[nm-1] - -#---------- - -# While MAU inits BLR is switched off, thus signal_r = signal_daq - - signal_r[0:nm] = signal_daq[0:nm] - - # MAU has computed the offset using nm samples - # now loop until the end of DAQ window - - cdef int j, k - cdef float trigger_line, pulse_on, wait_over, offset - cdef float part_sum - - pulse_on = 0 - wait_over = 0 - offset = 0 - - for k in range(nm, len_signal_daq): - trigger_line = MAU[k-1] + thr1 - pulse_[k] = pulse_on - wait_ [k] = wait_over - - # condition: raw signal raises above trigger line and - # we are not in the tail - # (wait_over == 0) - - if signal_daq[k] > trigger_line and wait_over == 0: - - # if the pulse just started pulse_on = 0. - # In this case compute the offset as value - #of the MAU before pulse starts (at k-1) - - if pulse_on == 0: # pulse just started - #offset computed as the value of MAU before pulse starts - offset = MAU[k-1] - pulse_on = 1 - - #Pulse is on: Freeze the MAU - MAU[k] = signal_i[k] = MAU[k-1] #signal_i follows the MAU - - #update recovered signal, correcting by offset - - acum[k] = acum[k-1] + signal_daq[k] - offset; - signal_r[k] = signal_daq[k] + coef * acum[k] - - #signal_r[k] = signal_daq[k] + signal_daq[k]*(coef/2) + coef*acum[k-1] - #acum[k] = acum[k-1] + signal_daq[k] - offset - - else: #no signal or raw signal has dropped below threshold - # but raw signal can be negative for a while and still contribute to the - # reconstructed signal. - - if pulse_on == 1: #reconstructed signal still on - # switch the pulse off only when recovered signal - # drops below threshold - # lide the MAU, still frozen. - # keep recovering signal - - MAU[k] = signal_i[k] = MAU[k-1] - - acum[k] = acum[k-1] + signal_daq[k] - offset; - signal_r[k] = signal_daq[k] + coef * acum[k] - - #signal_r[k] = signal_daq[k] + signal_daq[k]*(coef/2) + coef*acum[k-1] - #acum[k] = acum[k-1] + signal_daq[k] - offset - - #if the recovered signal drops before trigger line - #rec pulse is over! - if signal_r[k] < trigger_line + thr2: - wait_over = 1 #start tail compensation - pulse_on = 0 #recovered pulse is over - - else: #recovered signal has droped below trigger line - #need to compensate the tail to avoid drifting due to erros in - #baseline calculatoin - - if wait_over == 1: #compensating pulse - # recovered signal and raw signal - #must be equal within a threshold - # otherwise keep compensating pluse - - if signal_daq[k-1] < signal_r[k-1] - thr3: - # raw signal still below recovered signal - # keep compensating pulse - # is the recovered signal near offset? - upper = offset + (thr3 + thr2) - lower = offset - (thr3 + thr2) - - if lower < signal_r[k-1] < upper: - # we are near offset, activate MAU. - - signal_i[k] = signal_r[k-1] - part_sum = 0 - for j in range(k-nm, k): - part_sum += signal_i[j] / nm - MAU[k] = part_sum - - #MAU[k] = np.sum(signal_i[k-nm:k])*1./nm - - else: - # rec signal not near offset MAU frozen - MAU[k] = signal_i[k] = MAU[k-1] - - # keep adding recovered signal - acum [k] = signal_daq[k] + acum[k-1] - offset; - signal_r[k] = signal_daq[k] + coef*acum[k] - #signal_r[k] = signal_daq[k] + signal_daq[k]*(coef/2) + coef*acum[k-1] - #acum[k] = acum[k-1] + signal_daq[k] - offset - - else: # raw signal above recovered signal: we are done - - wait_over = 0 - acum [k] = MAU [k-1] - signal_r[k] = signal_daq[k] - signal_i[k] = signal_r [k] - part_sum = 0 - for j in range(k-nm, k): - part_sum += signal_i[j] / nm - MAU[k] = part_sum - #MAU[k] = np.sum(signal_i[k-nm:k])*1 / nm - - else: #signal still not found - - #update MAU and signals - part_sum = 0 - for j in range(k-nm, k): - part_sum += signal_i[j] / nm - MAU[k] = part_sum - #MAU[k] = np.sum(signal_i[k-nm:k]*1) / nm - acum[k] = MAU[k-1] - signal_r[k] = signal_daq[k] - signal_i[k] = signal_r[k] - #energy = np.dot(pulse_f,(signal_r-BASELINE)) - - signal_r = signal_r - BASELINE - - #return signal_r.astype(int) - return signal_r.astype(int), MAU, pulse_, wait_ - -cpdef deconvolve_signal_acum_simple(np.ndarray[np.int16_t, ndim=1] signal_i, - int n_baseline = 500, - float coef_clean = 2.905447E-06, - float coef_blr = 1.632411E-03, - float noise_rms = 0.9, - float thr_trigger = 5, - float thr_acum = 800, - float coeff_acum = 0.9995): - - """ - The accumulator approach by Master VHB - - """ - - cdef float coef = coef_blr - cdef int nm = n_baseline - - cdef int len_signal_daq = len(signal_i) - cdef np.ndarray[np.float64_t, ndim=1] signal_r = np.zeros(len_signal_daq, - dtype = np.float64) - cdef np.ndarray[np.float64_t, ndim=1] acum = np.zeros(len_signal_daq, - dtype = np.double) - - cdef np.ndarray[np.float64_t, ndim=1] signal_daq = signal_i.astype(float) - cdef int j - cdef float baseline = 0 - for j in range(nm): - baseline += signal_daq[j] - baseline /= nm - cdef float trigger_line - trigger_line = thr_trigger * noise_rms - - signal_daq = baseline - signal_daq - - b_cf, a_cf = SGN.butter(1, coef_clean, 'high', analog=False); - signal_daq = SGN.lfilter(b_cf, a_cf, signal_daq) - - # BLR - signal_r[0:nm] = signal_daq[0:nm] - cdef int k - for k in range(nm,len_signal_daq): - # condition: raw signal raises above trigger line - if (signal_daq[k] > trigger_line) or (acum[k-1] > thr_acum): - - signal_r[k] = signal_daq[k] + signal_daq[k] * (coef / 2) + coef * acum[k-1] - acum[k] = acum[k-1] + signal_daq[k] - - else: - signal_r[k] = signal_daq[k] - # deplete the accumulator before or after the signal to avoid runoffs - if (acum[k-1] > 0): - acum[k]=acum[k-1] * coeff_acum - - return signal_r.astype(int), acum - - -cpdef deconvolve_signal_acum_v1(np.ndarray[np.int16_t, ndim=1] signal_i, - int n_baseline = 500, - float coef_clean = 2.905447E-06, - float coef_blr = 1.632411E-03, - float thr_trigger = 5, - float thr_acum = 1000, - int acum_discharge_length = 5000, - float acum_tau = 2500, - float acum_compress = 0.0025): - - """ - The accumulator approach by Master VHB - decorated and cythonized by JJGC - """ - - cdef float coef = coef_blr - cdef int nm = n_baseline - cdef int len_signal_daq = len(signal_i) - - cdef np.ndarray[np.float64_t, ndim=1] signal_r = np.zeros(len_signal_daq, - dtype = np.double) - cdef np.ndarray[np.float64_t, ndim=1] acum = np.zeros(len_signal_daq, - dtype = np.double) - # signal_daq in floats - cdef np.ndarray[np.float64_t, ndim=1] signal_daq = signal_i.astype(float) - - # compute baseline and noise - - cdef int j - cdef float baseline = 0 - for j in range(nm): - baseline += signal_daq[j] - baseline /= nm - - # reverse sign of signal and subtract baseline - signal_daq = baseline - signal_daq - - # compute noise - cdef float noise = 0 - for j in range(nm): - noise += signal_daq[j] * signal_daq[j] - noise /= nm - cdef float noise_rms = np.sqrt(noise) - - # trigger line - - cdef float trigger_line - trigger_line = thr_trigger*noise_rms - - # cleaning signal - cdef np.ndarray[np.float64_t, ndim=1] b_cf - cdef np.ndarray[np.float64_t, ndim=1] a_cf - - b_cf, a_cf = SGN.butter(1, coef_clean, 'high', analog=False); - signal_daq = SGN.lfilter(b_cf, a_cf, signal_daq) - - # compute discharge curve - cdef np.ndarray[np.float64_t, ndim=1] t_discharge - cdef np.ndarray[np.float64_t, ndim=1] exp - cdef np.ndarray[np.float64_t, ndim=1] cf - cdef np.ndarray[np.float64_t, ndim=1] discharge_curve - - cdef float d_length = float(acum_discharge_length) - t_discharge = np.arange(0, d_length, 1, dtype=np.double) - exp = np.exp(-(t_discharge - d_length / 2) / acum_tau) - cf = 1 / (1 + exp) - discharge_curve = acum_compress*(1 - cf) + (1 - acum_compress) - - # signal_r equals to signal_d (baseline suppressed and change signed) - # for the first nm samples - signal_r[0:nm] = signal_daq[0:nm] - - # print ("baseline = {}, noise (LSB_rms) = {} ".format( - # baseline, noise_rms,)) - - cdef int k - j = 0 - for k in range(nm, len_signal_daq): - - # update recovered signal - signal_r[k] = (signal_daq[k] + signal_daq[k] * (coef / 2) + - coef_blr * acum[k-1]) - - # condition: raw signal raises above trigger line - # once signal raises above trigger line condition is on until - # accumulator drops below thr_acum - if (signal_daq[k] > trigger_line) or (acum[k-1] > thr_acum): - - # update accumulator (signal_daq is already baseline subtracted) - acum[k] = acum[k-1] + signal_daq[k] - else: - j = 0 - # discharge acumulator - if acum[k-1] > 1: - acum[k] = acum[k-1] * discharge_curve[j] - if j < acum_discharge_length - 1: - j = j + 1 - else: - j = acum_discharge_length - 1 - else: - acum[k] = 0 - j = 0 - - return signal_r.astype(int), acum.astype(int) - - -cpdef deconvolve_signal_acum_v2(np.ndarray[np.int16_t, ndim=1] signal_i, - int n_baseline = 500, - float coef_clean = 2.905447E-06, - float coef_blr = 1.632411E-03, - float thr_trigger = 5, - int acum_discharge_length = 5000, - float acum_tau = 2500, - float acum_compress = 0.01): - - """ - The accumulator approach by Master VHB - decorated and cythonized by JJGC - 22.11 Compute baseline using all the waveforms - computes accumulator threshold from thr_trigger - """ - - cdef float coef = coef_blr - cdef int nm = n_baseline - cdef float thr_acum = thr_trigger/coef - cdef int len_signal_daq = len(signal_i) - - cdef np.ndarray[np.float64_t, ndim=1] signal_r = np.zeros(len_signal_daq, - dtype = np.double) - cdef np.ndarray[np.float64_t, ndim=1] acum = np.zeros(len_signal_daq, - dtype = np.double) - # signal_daq in floats - cdef np.ndarray[np.float64_t, ndim=1] signal_daq = signal_i.astype(float) - - # compute baseline and noise - - cdef int j - cdef float baseline = 0 - cdef float baseline_end = 0 - - for j in range(0,len_signal_daq): - baseline += signal_daq[j] - baseline /= len_signal_daq - - for j in range(len_signal_daq-nm, len_signal_daq): - baseline_end += signal_daq[j] - baseline_end /= nm - - # reverse sign of signal and subtract baseline - signal_daq = baseline - signal_daq - - # compute noise - cdef float noise = 0 - for j in range(nm): - noise += signal_daq[j] * signal_daq[j] - noise /= nm - cdef float noise_rms = np.sqrt(noise) - - # trigger line - - cdef float trigger_line - trigger_line = thr_trigger * noise_rms - - # cleaning signal - cdef np.ndarray[np.float64_t, ndim=1] b_cf - cdef np.ndarray[np.float64_t, ndim=1] a_cf - - b_cf, a_cf = SGN.butter(1, coef_clean, 'high', analog=False); - signal_daq = SGN.lfilter(b_cf, a_cf, signal_daq) - - # compute discharge curve - cdef np.ndarray[np.float64_t, ndim=1] t_discharge - cdef np.ndarray[np.float64_t, ndim=1] exp - cdef np.ndarray[np.float64_t, ndim=1] cf - cdef np.ndarray[np.float64_t, ndim=1] discharge_curve - - cdef float d_length = float(acum_discharge_length) - t_discharge = np.arange(0, d_length, 1, dtype=np.double) - exp = np.exp(-(t_discharge - d_length / 2) / acum_tau) - cf = 1 / (1 + exp) - discharge_curve = acum_compress*(1 - cf) + (1 - acum_compress) - - # signal_r equals to signal_d (baseline suppressed and change signed) - # for the first nm samples - signal_r[0:nm] = signal_daq[0:nm] - - # print ("baseline = {}, noise (LSB_rms) = {} ".format( - # baseline, noise_rms,)) - - cdef int k - j = 0 - for k in range(nm, len_signal_daq): - - # update recovered signal - signal_r[k] = (signal_daq[k] + signal_daq[k] * (coef / 2) + - coef_blr * acum[k-1]) - - # condition: raw signal raises above trigger line - # once signal raises above trigger line condition is on until - # accumulator drops below thr_acum - if (signal_daq[k] > trigger_line) or (acum[k-1] > thr_acum): - - # update accumulator (signal_daq is already baseline subtracted) - acum[k] = acum[k-1] + signal_daq[k] - else: - j = 0 - # discharge acumulator - if acum[k-1] > 1: - acum[k] = acum[k-1] * discharge_curve[j] - if j < acum_discharge_length - 1: - j += 1 - else: - j = acum_discharge_length - 1 - else: - acum[k] = 0 - j = 0 - # return signal and friends - return signal_r.astype(int), acum.astype(int), baseline, baseline_end, noise_rms - -cpdef deconvolve_signal_acum(np.ndarray[np.int16_t, ndim=1] signal_i, - int n_baseline = 28000, - float coef_clean = 2.905447E-06, - float coef_blr = 1.632411E-03, - float thr_trigger = 5, - int acum_discharge_length = 5000): - - """ - The accumulator approach by Master VHB - decorated and cythonized by JJGC - 22.11 Compute baseline using all the waveforms - computes accumulator threshold from thr_trigger - simplify discharge wrt previous versions: - In this verison the recovered signal and the accumulator are - always being charged. At the same time, the accumulator is being - discharged when there is no signal. This avoids runoffs - The baseline is computed using a window of 700 mus (by default) - which should be good for Na and Kr - """ - - cdef float coef = coef_blr - cdef int nm = n_baseline - cdef float thr_acum = thr_trigger/coef - cdef int len_signal_daq = len(signal_i) - - cdef np.ndarray[np.float64_t, ndim=1] signal_r = np.zeros(len_signal_daq, - dtype = np.double) - cdef np.ndarray[np.float64_t, ndim=1] acum = np.zeros(len_signal_daq, - dtype = np.double) - # signal_daq in floats - cdef np.ndarray[np.float64_t, ndim=1] signal_daq = signal_i.astype(float) - - # compute baseline and noise - - cdef int j - cdef float baseline = 0 - cdef float baseline_end = 0 - - for j in range(0,nm): - baseline += signal_daq[j] - baseline /= nm - - cdef nf = len_signal_daq - nm - - for j in range(nm, len_signal_daq): - baseline_end += signal_daq[j] - baseline_end /= nf - - # reverse sign of signal and subtract baseline - signal_daq = baseline - signal_daq - - # compute noise - cdef float noise = 0 - cdef nn = 400 # fixed at 100 mud - for j in range(0, nn): - noise += signal_daq[j] * signal_daq[j] - noise /= nn - cdef float noise_rms = np.sqrt(noise) - - # trigger line - - cdef float trigger_line - trigger_line = thr_trigger*noise_rms - - # cleaning signal - cdef np.ndarray[np.float64_t, ndim=1] b_cf - cdef np.ndarray[np.float64_t, ndim=1] a_cf - - b_cf, a_cf = SGN.butter(1, coef_clean, 'high', analog=False); - signal_daq = SGN.lfilter(b_cf, a_cf, signal_daq) - - cdef int k - j = 0 - signal_r[0] = signal_daq[0] - for k in range(1, len_signal_daq): - - # always update signal and accumulator - signal_r[k] = (signal_daq[k] + signal_daq[k] * (coef / 2) + - coef * acum[k-1]) - - acum[k] = acum[k-1] + signal_daq[k] - - if (signal_daq[k] < trigger_line) and (acum[k-1] < thr_acum): - # discharge accumulator - - if acum[k-1] > 1: - acum[k] = acum[k-1] * (1 - coef) - if j < acum_discharge_length - 1: - j = j + 1 - else: - j = acum_discharge_length - 1 - else: - acum[k] = 0 - j = 0 - # return signal and friends - return signal_r.astype(int), acum.astype(int), baseline, baseline_end, noise_rms diff --git a/invisible_cities/sierpe/fee.py b/invisible_cities/sierpe/fee.py index f42e42e574..170912cc7a 100644 --- a/invisible_cities/sierpe/fee.py +++ b/invisible_cities/sierpe/fee.py @@ -29,7 +29,7 @@ f_LPF1 = 3 * units.MHZ f_LPF2 = 10 * units.MHZ ADC_TO_PES_LPF = 24.1 # After LPF, comes out from spe area -ADC_TO_PES = 18.71 +ADC_TO_PES = 16.241 OFFSET = 2500 # offset adc CEILING = 4096 # ceiling of adc @@ -399,5 +399,13 @@ def daq_decimator(f_sample1, f_sample2, signal_in): Includes anti-aliasing filter """ + ## The default 30 point FIR filter with Hamming window of order + ## 20 * scale is used as an acceptable compromise between charge + ## losses and speed as approved by V. Herrero. A slightly better + ## charge conservation (~0.001% loss vs ~0.01%) is achieved using: + ## signal.decimate(signal_i, scale, + ## ftype=signal.dlti(*signal.butter(1, 0.8/scale)), + ## zero_phase=True) + ## but for an order 35 increase in processing time. scale = int(f_sample1 / f_sample2) - return signal.decimate(signal_in, scale, n=30, ftype='fir', zero_phase=False) + return signal.decimate(signal_in, scale, ftype='fir', zero_phase=True) diff --git a/invisible_cities/sierpe/fee_test.py b/invisible_cities/sierpe/fee_test.py index 1355cd297b..ab228ee208 100644 --- a/invisible_cities/sierpe/fee_test.py +++ b/invisible_cities/sierpe/fee_test.py @@ -1,11 +1,13 @@ -import numpy as np +import numpy as np +import tables as tb import pytest from scipy import signal from flaky import flaky -from .. core import system_of_units as units -from . import fee as FE +from .. core import system_of_units as units +from .. database import load_db +from . import fee as FE def signal_i_th(): """Generates a "theoretical" current signal (signal_i)""" @@ -74,7 +76,29 @@ def test_show_signal_decimate_signature(): assert 23 < adc_to_pes < 23.1 +def test_signal_maintained(electron_MCRD_file): + """Test that there is no appreciable charge loss in daq_decimate""" + detector_db = 'new' + run_number = -7951 + pmt_id = 0 + with tb.open_file(electron_MCRD_file) as h5in: + evt0_pmt = h5in.root.pmtrd[0] + adc_to_pes = load_db.DataPMT(detector_db, run_number).adc_to_pes.values + spe = FE.SPE() + fee = FE.FEE(detector_db, run_number, + noise_FEEPMB_rms = FE.NOISE_I , + noise_DAQ_rms = FE.NOISE_DAQ) + cc = adc_to_pes[pmt_id] / FE.ADC_TO_PES + signal_i = FE.spe_pulse_from_vector(spe, evt0_pmt[pmt_id], norm=cc) + signal_d = FE.daq_decimator(FE.f_mc, FE.f_sample, signal_i) + signal_blr = FE.signal_v_lpf(fee, signal_d) * FE.v_to_adc() + + input_adc = adc_to_pes[pmt_id] * np.sum(evt0_pmt[pmt_id]) + assert np.isclose(np.sum(signal_blr), input_adc, rtol=2e-4) + + +@pytest.mark.skip('Skipped as uses outdated functions not used in code') def test_spe_to_adc(): """Convert SPE to adc values with the current FEE Parameters must be.""" ipmt = 0 diff --git a/invisible_cities/sierpe/low_frequency_noise_test.py b/invisible_cities/sierpe/low_frequency_noise_test.py index 447db2b89d..0431f27f01 100644 --- a/invisible_cities/sierpe/low_frequency_noise_test.py +++ b/invisible_cities/sierpe/low_frequency_noise_test.py @@ -1,5 +1,3 @@ -import pytest - import numpy as np from numpy.testing import assert_allclose @@ -22,7 +20,7 @@ def test_buffer_and_limits(): buffer_sample_wid = 25e-9 frequency_bin = 500 frequencies = np.arange(15000, 20000, frequency_bin) - + freq_cont, lowf, highf = lfn.buffer_and_limits(buffer_len , buffer_sample_wid, frequencies ) @@ -55,4 +53,3 @@ def test_low_frequency_noise(dbnew): assert np.any(np.diff(pmt_noise, axis = 0)) ## then that not all are assert not np.all(np.diff(pmt_noise, axis = 0)) - diff --git a/invisible_cities/sierpe/osc_magnitudes.h5 b/invisible_cities/sierpe/osc_magnitudes.h5 deleted file mode 100644 index 9b9e771d01..0000000000 --- a/invisible_cities/sierpe/osc_magnitudes.h5 +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d6079aa1d48a754e3decb77d869cd55f4cf6ef939b469a59473aea4e4c19cf0a -size 3312 diff --git a/invisible_cities/sierpe/waveform_generator.py b/invisible_cities/sierpe/waveform_generator.py index 9abae648de..71d96d9e55 100644 --- a/invisible_cities/sierpe/waveform_generator.py +++ b/invisible_cities/sierpe/waveform_generator.py @@ -4,15 +4,12 @@ Generates toy PMT and SiPM waveforms """ -import numpy as np -import scipy.signal as signal +import numpy as np -from . import fee as FE -from . fee import FEE -from .. core.system_of_units_c import units +from . import fee as FE +from . fee import FEE from typing import NamedTuple -from typing import Tuple class Point(NamedTuple): @@ -116,7 +113,6 @@ def simulate_pmt_response(fee : FEE, wf : np.ndarray) -> WaveformPmt: signal_i = FE.spe_pulse_from_vector(spe, wf) # signal_i in current units signal_d = FE.daq_decimator(FE.f_mc, FE.f_sample, signal_i) # Decimate (DAQ decimation) signal_fee = FE.signal_v_fee(fee, signal_d, -1) * FE.v_to_adc() # Effect of FEE and transform to adc counts - signal_daq = FE.noise_adc(fee, signal_fee) # add noise daq signal_blr = FE.signal_v_lpf(fee, signal_d) * FE.v_to_adc() # signal blr is just pure MC decimated by adc in adc counts return WaveformPmt(signal_blr.astype(int), signal_fee.astype(int)) diff --git a/invisible_cities/types/ic_types.py b/invisible_cities/types/ic_types.py index b60a398202..b4771c65c9 100644 --- a/invisible_cities/types/ic_types.py +++ b/invisible_cities/types/ic_types.py @@ -1,4 +1,3 @@ -from argparse import Namespace from enum import Enum import numpy as np @@ -12,18 +11,6 @@ def __getattr__(self, _): return NN -class Counters(Namespace): - - def set(self, **kwds): - for name, value in kwds.items(): - setattr(self, name, value) - - def init(self, **kwds): - for name, value in kwds.items(): - assert name not in self - setattr(self, name, value) - - class xy: def __init__(self, x, y): self.x = x @@ -74,7 +61,7 @@ def interval(self): return (self.min, self.max) def center(self): return (self.max + self.min) / 2 def contains(self, x): - return self.min <= x < self.max + return self.min <= x <= self.max def __mul__(self, factor): return minmax(self.min * factor, self.max * factor) diff --git a/invisible_cities/types/ic_types_c.pxd b/invisible_cities/types/ic_types_c.pxd deleted file mode 100644 index ad7ea45aa8..0000000000 --- a/invisible_cities/types/ic_types_c.pxd +++ /dev/null @@ -1,8 +0,0 @@ - -cdef class xy: - """Represent a (x,y) number""" - cdef double x,y - -cdef class minmax: - """Represents a bracketed interval""" - cdef double min, max diff --git a/invisible_cities/types/ic_types_c.pyx b/invisible_cities/types/ic_types_c.pyx deleted file mode 100644 index f5cace2be0..0000000000 --- a/invisible_cities/types/ic_types_c.pyx +++ /dev/null @@ -1,115 +0,0 @@ -cimport numpy as np -import numpy as np -from cpython.object cimport Py_EQ - -cdef class xy: - """Represent a (x,y) number""" - def __init__(self, x, y): - self.x = x - self.y = y - - property pos: - def __get__(self): - return np.stack(([self.x], [self.y]), axis=1) - - property XY: - def __get__(self): - return (self.x, self.y) - - property X: - def __get__(self): - return self.x - - property Y: - def __get__(self): - return self.y - - property x: - def __get__(self): - return self.x - - property y: - def __get__(self): - return self.y - - property R: - def __get__(self): - return np.sqrt(self.x ** 2 + self.y ** 2) - - property Phi: - def __get__(self): - return np.arctan2(self.y, self.x) - - def __str__(self): - return 'xy(x={.x}, max={.y})'.format(self, self) - - def __repr__(self): - return 'xy(x={.x}, max={.y})'.format(self, self) - - def __getitem__(self, n): - if n == 0: return self.x - if n == 1: return self.y - raise IndexError - - -cdef class minmax: - - def __init__(self, min, max): - assert min <= max - self.min = min - self.max = max - - property min: - def __get__(self): - return self.min - - property max: - def __get__(self): - return self.max - - property bracket: - def __get__(self): - return self.max - self.min - - property center: - def __get__(self): - return (self.max + self.min) / 2 - - def contains(self, double x): - return self.min <= x <= self.max - - def __add__(self, scalar): - return minmax(self.min + scalar, self.max + scalar) - - def __mul__(self, factor): - return minmax(self.min * factor, self.max * factor) - - def __truediv__(self, factor): - assert factor != 0 - return self.__mul__(1./factor) - - def __sub__(self, scalar): - return minmax(self.min - scalar, self.max - scalar) - - def __richcmp__(x, y, int opt): - cdef: - minmax mm - double min, max - mm, y = (x, y) if isinstance(x, minmax) else (y,x) - min = mm.min - max = mm.max - if opt == Py_EQ: - return mm.min == y.min and mm.max == y.max - else: - assert False - - def __str__(self): - return 'minmax(min={.min}, max={.max})'.format(self, self) - - def __repr__(self): - return 'minmax(min={.min}, max={.max})'.format(self, self) - - def __getitem__(self, n): - if n == 0: return self.min - if n == 1: return self.max - raise IndexError diff --git a/invisible_cities/types/ic_types_test.py b/invisible_cities/types/ic_types_test.py index d9a502fbc1..fe5580b2fc 100644 --- a/invisible_cities/types/ic_types_test.py +++ b/invisible_cities/types/ic_types_test.py @@ -4,11 +4,8 @@ from . ic_types import minmax from . ic_types import xy -from . ic_types import Counters from . ic_types import NN from . ic_types import NNN -from . ic_types_c import xy as cxy -from . ic_types_c import minmax as cmm from pytest import raises @@ -26,20 +23,10 @@ def make_minmax(a,b): def make_xy(a,b): return xy(a,b) -def make_cminmax(a,b): - # Ensure that arguments respect the required order - if a > b: a, b = b, a - return cmm(a, b) - -def make_xyc(a,b): - return cxy(a,b) - sensible_floats = floats(min_value=0.5, max_value=1e3, allow_nan=False, allow_infinity=False) minmaxes = builds(make_minmax, sensible_floats, sensible_floats) -cmms = builds(make_cminmax, sensible_floats, sensible_floats) xys = builds(make_xy, sensible_floats, sensible_floats) -cxys = builds(make_xyc, sensible_floats, sensible_floats) @given(sensible_floats, sensible_floats) def test_minmax_interval(a,b): @@ -88,38 +75,6 @@ def test_minmax_sub(mm, f): np.isclose (lowered.max , hi - f, rtol=1e-4) -@given(cmms, sensible_floats) -def test_cminmax_add(mm, f): - lo, hi = mm - raised = mm + f - np.isclose (raised.min , lo + f, rtol=1e-4) - np.isclose (raised.max , hi + f, rtol=1e-4) - - -@given(cmms, sensible_floats) -def test_cminmax_div(mm, f): - lo, hi = mm - scaled = mm / f - np.isclose (scaled.min , lo / f, rtol=1e-4) - np.isclose (scaled.max , hi / f, rtol=1e-4) - - -@given(cmms, sensible_floats) -def test_cminmax_mul(mm, f): - lo, hi = mm - scaled = mm * f - np.isclose (scaled.min , lo * f, rtol=1e-4) - np.isclose (scaled.max , hi * f, rtol=1e-4) - - -@given(cmms, sensible_floats) -def test_cminmax_sub(mm, f): - lo, hi = mm - lowered = mm - f - np.isclose (lowered.min , lo - f, rtol=1e-4) - np.isclose (lowered.max , hi - f, rtol=1e-4) - - @given(xys, sensible_floats, sensible_floats) def test_xy(xy, a, b): ab = a, b @@ -136,22 +91,6 @@ def test_xy(xy, a, b): np.allclose(xy.pos, pos, rtol=1e-3, atol=1e-03) -@given(cxys, sensible_floats, sensible_floats) -def test_cxy(xyc, a, b): - ab = a, b - r = np.sqrt(a ** 2 + b ** 2) - phi = np.arctan2(b, a) - pos = np.stack(([a], [b]), axis=1) - np.isclose (xyc.x , a, rtol=1e-4) - np.isclose (xyc.y , b, rtol=1e-4) - np.isclose (xyc.X , a, rtol=1e-4) - np.isclose (xyc.Y , b, rtol=1e-4) - np.isclose (xyc.XY , ab, rtol=1e-4) - np.isclose (xyc.R , r, rtol=1e-4) - np.isclose (xyc.Phi, phi, rtol=1e-4) - np.allclose(xyc.pos, pos, rtol=1e-3, atol=1e-03) - - @given(text(min_size=1, max_size=10, alphabet=ascii_letters)) def test_NNN_generates_NN_for_every_attribute_name(name): nnn = NNN() diff --git a/manage.sh b/manage.sh index 6b375e09aa..d23bc0bab1 100644 --- a/manage.sh +++ b/manage.sh @@ -60,10 +60,11 @@ function install_conda { echo Conda already installed. Skipping conda installation. else echo Installing conda for $CONDA_OS + CONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-py${PYTHON_VERSION//.}_4.8.2-${CONDA_OS}-x86_64.sh" if which wget; then - wget https://repo.continuum.io/miniconda/Miniconda3-4.5.4-${CONDA_OS}-x86_64.sh -O miniconda.sh + wget ${CONDA_URL} -O miniconda.sh else - curl https://repo.continuum.io/miniconda/Miniconda3-4.5.4-${CONDA_OS}-x86_64.sh -o miniconda.sh + curl ${CONDA_URL} -o miniconda.sh fi bash miniconda.sh -b -p $HOME/miniconda CONDA_SH=$HOME/miniconda/etc/profile.d/conda.sh @@ -72,7 +73,7 @@ function install_conda { fi } -CONDA_ENV_TAG=2018-11-14 +CONDA_ENV_TAG=2020-06-16 CONDA_ENV_NAME=IC-${PYTHON_VERSION}-${CONDA_ENV_TAG} function make_environment { @@ -87,25 +88,27 @@ dependencies: # *REMEMBER TO CHANGE CONDA_ENV_TAG WHEN CHANGING VERSION NUMBERS* - cython = 0.29 - jupyter = 1.0.0 -- jupyterlab = 0.35.3 -- matplotlib = 3.0.1 -- networkx = 2.2 -- notebook = 5.7.0 -- numpy = 1.15.2 -- pandas = 0.23.4 -- seaborn = 0.9.0 +- jupyterlab = 1.2.6 +- matplotlib = 3.1.3 +- networkx = 2.4 +- notebook = 6.0.3 +- numpy = 1.18.1 +- pandas = 1.0.3 +- seaborn = 0.10.1 - pymysql = 0.9.2 -- pytables = 3.4.4 -- pytest = 3.8.2 -- scipy = 1.1.0 -- sphinx = 1.8.1 -- tornado = 5.1.1 -- flaky = 3.4.0 -- hypothesis = 3.68.0 -- pytest-xdist = 1.23.2 -- coverage = 4.5.4 +- pytables = 3.6.1 +- pytest = 5.4.2 +- scipy = 1.4.1 +- sphinx = 3.0.4 +- tornado = 6.0.4 +- flaky = 3.6.1 +- hypothesis = 5.16.1 +- pytest-xdist = 1.32.0 +- coverage = 5.0 +- pip = 20.0.2 +- setuptools = 47.1.1 - pip: - - pytest-instafail==0.4.0 + - pytest-instafail==0.4.2 EOF conda env create -f ${YML_FILENAME} diff --git a/pytest.ini b/pytest.ini index 6c5c6ee114..5fe74ca3d4 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,7 +1,10 @@ [pytest] +markers = + slow: marks tests as slow + serial: marks tests as serial filterwarnings = - ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working:DeprecationWarning + ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working:DeprecationWarning ignore:can't resolve package from __spec__ or __package__, falling back on __name__ and __path__:ImportWarning ignore::FutureWarning:scipy.signal.signaltools:3463 ignore::FutureWarning:scipy.signal.signaltools:1344 - + ignore::UserWarning:invisible_cities.io.dst_io:107