Selectioncity

.gitignore

@@ -1,13 +1,25 @@
+# Byte-compiled / optimized
+__pycache__/
+*.py[cod]
 
-*.pyc
-*.o
-*.so
+# C extensions
 *.c
-.cache
-*.egg-info
-*.*~
-environment?.?.yml
+*.so
+
+# Distribution / packaging
+build/
+*.egg-info/
+environment-*.yml
+miniconda.sh
+
+# Unit test / coverage reports
+.hypothesis/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# Backup files
+*~
+
+# macOS
 .DS_Store
-doc/.DS_Store
-doc/build
-invisible_cities/cities/.ipynb_checkpoints

.travis.yml

@@ -22,3 +22,11 @@ install:
 
 script:
   - HYPOTHESIS_PROFILE=travis-ci bash manage.sh run_tests_par
+
+
+before_install:
+  - if [ "$TRAVIS_OS_NAME" = "osx" ]; then brew install git-lfs; fi
+  - if [ "$TRAVIS_OS_NAME" = "osx" ]; then git lfs install;      fi
+
+before_script:
+  - if [ "$TRAVIS_OS_NAME" = "osx" ]; then git lfs pull; fi

conftest.py

@@ -7,8 +7,6 @@
 settings.register_profile("travis-ci" , settings(max_examples = 1000, deadline=300))
 settings.register_profile("hard"      , settings(max_examples = 1000))
 settings.register_profile("dev"       , settings(max_examples =   10))
-settings.register_profile("hard_nocov", settings(max_examples = 1000, use_coverage=False))
-settings.register_profile("dev_nocov" , settings(max_examples =   10, use_coverage=False))
 settings.register_profile("debug"     , settings(max_examples =   10,
                                                 verbosity=Verbosity.verbose))
 settings.load_profile(os.getenv(u'HYPOTHESIS_PROFILE', 'dev'))

doc/commits.rst

@@ -0,0 +1,48 @@
+How to create a good commit history
+===================================
+
+Commit structure
+-----------------
+
+- Focus on a specific change. Don't try to change everything at once.
+  If the plan is to implement a series of things, do them one by one.
+
+- Do not mix changes concerning different features in one commit.
+  For instance, do not mix cosmetical changes with other, more relevant ones.
+
+- Commit frequently and in an incremental manner.
+  Do not be afraid of making too many commits, they can be combined afterwards
+  (this is known as squashing in git).
+  Besides, the history can be easily cleaned when the changes are small and independent of each other.
+
+
+Commit style
+--------------
+
+- Just follow https://chris.beams.io/posts/git-commit/
+
+This is a summary of a good commit message.
+But, please at least take a look to the details in the link above.
+
+
+     | The seven rules of a great Git commit message
+     |
+     | 1. Separate subject from body with a blank line
+     | 2. Limit the subject line to 50 characters
+     | 3. Capitalize the subject line
+     | 4. Do not end the subject line with a period
+     | 5. Use the imperative mood in the subject line
+     | 6. Wrap the body at 72 characters
+     | 7. Use the body to explain what and why vs. how
+
+Commits after PR review
+-----------------------
+
+- Remember that these guidelines apply also to changes requested during a review process.
+  Don't make commits like ``Changes requested by the reviewer``.
+
+- Follow the same guidelines listed above and,
+  if you want to refer to the discussion happening in the PR,
+  do it using the second part of the commit message.
+  You may also refer to the PR by using ``#PR_NUMBER`` or
+  by posting a link to a specific comment in the discussion.

invisible_cities/cities/beersheba.py

@@ -22,11 +22,12 @@
 
 from typing      import Tuple
 from typing      import List
-from typing      import Callable
 
 from enum        import auto
 
 from .  components import city
+from .  components import collect
+from .  components import copy_mc_info
 from .  components import print_every
 from .  components import cdst_from_files
 
@@ -38,22 +39,24 @@
 from .. dataflow.dataflow      import push
 from .. dataflow.dataflow      import pipe
 
+from .. database.load_db       import DataSiPM
+
 from .. reco.deconv_functions  import find_nearest
 from .. reco.deconv_functions  import cut_and_redistribute_df
 from .. reco.deconv_functions  import drop_isolated_sensors
 from .. reco.deconv_functions  import deconvolve
 from .. reco.deconv_functions  import richardson_lucy
 from .. reco.deconv_functions  import InterpolationMethod
 
-from .. io.       mcinfo_io    import mc_info_writer
 from .. io.run_and_event_io    import run_and_event_writer
-from .. io.          dst_io    import _store_pandas_as_tables
+from .. io.          dst_io    import df_writer
+from .. io.          dst_io    import load_dst
 
 from .. evm.event_model        import HitEnergy
 
 from .. types.ic_types         import AutoNameEnumBase
 
-from .. core.system_of_units_c import units
+from .. core                   import system_of_units as units
 
 
 class CutType          (AutoNameEnumBase):
@@ -65,7 +68,8 @@ class DeconvolutionMode(AutoNameEnumBase):
     separate = auto()
 
 
-def deconvolve_signal(psf_fname       : str,
+def deconvolve_signal(det_db          : pd.DataFrame,
+                      psf_fname       : str,
                       e_cut           : float,
                       n_iterations    : int,
                       iteration_tol   : float,
@@ -85,6 +89,7 @@ def deconvolve_signal(psf_fname       : str,
 
     Parameters
     ----------
+    det_db          : Detector database.
     psf_fname       : Point-spread function.
     e_cut           : Cut in relative value to the max voxel over the deconvolution output.
     n_iterations    : Number of Lucy-Richardson iterations
@@ -111,16 +116,19 @@ def deconvolve_signal(psf_fname       : str,
     bin_size      = np.asarray(bin_size               )
     diffusion     = np.asarray(diffusion              )
 
-    psfs          = pd.read_hdf(psf_fname)
-    deconvolution = deconvolve(n_iterations, iteration_tol, sample_width, bin_size, inter_method)
+    psfs          = load_dst(psf_fname, 'PSF', 'PSFs')
+    det_grid      = [np.arange(det_db[var].min() + bs/2, det_db[var].max() - bs/2 + np.finfo(np.float32).eps, bs)
+                     for var, bs in zip(dimensions, bin_size)]
+    deconvolution = deconvolve(n_iterations, iteration_tol,
+                               sample_width, det_grid       , inter_method)
 
-    if energy_type  not in HitEnergy          :
+    if not isinstance(energy_type , HitEnergy          ):
         raise ValueError(f'energy_type {energy_type} is not a valid energy type.')
-    if inter_method not in InterpolationMethod:
+    if not isinstance(inter_method, InterpolationMethod):
         raise ValueError(f'inter_method {inter_method} is not a valid interpolation method.')
-    if cut_type     not in CutType            :
+    if not isinstance(cut_type    , CutType            ):
         raise ValueError(f'cut_type {cut_type} is not a valid cut type.')
-    if deconv_mode  not in DeconvolutionMode  :
+    if not isinstance(deconv_mode , DeconvolutionMode  ):
         raise ValueError(f'deconv_mode {deconv_mode} is not a valid deconvolution mode.')
 
     def deconvolve_hits(df, z):
@@ -294,17 +302,18 @@ def deconv_writer(h5out, compression='ZLIB4'):
     For a given open table returns a writer for deconvolution hits dataframe
     """
     def write_deconv(df):
-        return _store_pandas_as_tables(h5out              = h5out             ,
-                                       df                 = df                ,
-                                       compression        = compression       ,
-                                       group_name         = 'DECO'            ,
-                                       table_name         = 'Events'          ,
-                                       descriptive_string = 'Deconvolved hits')
+        return df_writer(h5out              = h5out             ,
+                         df                 = df                ,
+                         compression        = compression       ,
+                         group_name         = 'DECO'            ,
+                         table_name         = 'Events'          ,
+                         descriptive_string = 'Deconvolved hits',
+                         columns_to_index   = ['event']         )
     return write_deconv
 
 
 @city
-def beersheba(files_in, file_out, compression, event_range, print_mod, run_number,
+def beersheba(files_in, file_out, compression, event_range, print_mod, detector_db, run_number,
               deconv_params = dict()):
     """
     The city corrects Penthesilea hits energy and extracts topology information.
@@ -395,36 +404,43 @@ def beersheba(files_in, file_out, compression, event_range, print_mod, run_numbe
     filter_events_no_hits = fl.map(check_nonempty_dataframe,
                                    args = 'cdst',
                                    out  = 'cdst_passed_no_hits')
-    deconvolve_events     = fl.map(deconvolve_signal(**deconv_params),
+    deconvolve_events     = fl.map(deconvolve_signal(DataSiPM(detector_db, run_number), **deconv_params),
                                    args = 'cdst',
                                    out  = 'deconv_dst')
 
     event_count_in        = fl.spy_count()
     event_count_out       = fl.spy_count()
     events_passed_no_hits = fl.count_filter(bool, args = "cdst_passed_no_hits")
 
+    evtnum_collect        = collect()
+
     with tb.open_file(file_out, "w", filters = tbl.filters(compression)) as h5out:
         # Define writers
-        write_event_info = fl.sink(run_and_event_writer(h5out), args=("run_number", "event_number", "timestamp"))
-        write_mc_        = mc_info_writer(h5out) if run_number <= 0 else (lambda *_: None)
-
-        write_mc         = fl.sink(                   write_mc_, args = ("mc", "event_number"))
-        write_deconv     = fl.sink(  deconv_writer(h5out=h5out), args =  "deconv_dst"         )
-        write_summary    = fl.sink( summary_writer(h5out=h5out), args =  "summary"            )
-        return push(source = cdst_from_files(files_in),
-                    pipe   = pipe(fl.slice(*event_range, close_all=True)    ,
-                                  print_every(print_mod)                    ,
-                                  event_count_in.spy                        ,
-                                  cut_sensors                               ,
-                                  drop_sensors                              ,
-                                  filter_events_no_hits                     ,
-                                  events_passed_no_hits    .filter          ,
-                                  deconvolve_events                         ,
-                                  event_count_out.spy                       ,
-                                  fl.fork(write_mc        ,
-                                          write_deconv    ,
-                                          write_summary   ,
-                                          write_event_info))                ,
-                    result = dict(events_in   = event_count_in       .future,
-                                  events_out  = event_count_out      .future,
-                                  events_pass = events_passed_no_hits.future))
+        write_event_info = fl.sink(run_and_event_writer (h5out), args = ("run_number", "event_number", "timestamp"))
+        write_deconv     = fl.sink(  deconv_writer(h5out=h5out), args =  "deconv_dst")
+        write_summary    = fl.sink( summary_writer(h5out=h5out), args =  "summary"   )
+        result = push(source = cdst_from_files(files_in),
+                      pipe   = pipe(fl.slice(*event_range, close_all=True)    ,
+                                    print_every(print_mod)                    ,
+                                    event_count_in.spy                        ,
+                                    cut_sensors                               ,
+                                    drop_sensors                              ,
+                                    filter_events_no_hits                     ,
+                                    events_passed_no_hits    .filter          ,
+                                    deconvolve_events                         ,
+                                    event_count_out.spy                       ,
+                                    fl.branch("event_number"     ,
+                                              evtnum_collect.sink)            ,
+                                    fl.fork(write_deconv    ,
+                                            write_summary   ,
+                                            write_event_info))                ,
+                      result = dict(events_in   = event_count_in       .future,
+                                    events_out  = event_count_out      .future,
+                                    evtnum_list = evtnum_collect       .future,
+                                    events_pass = events_passed_no_hits.future))
+
+        if run_number <= 0:
+            copy_mc_info(files_in, h5out, result.evtnum_list,
+                         detector_db, run_number)
+
+        return result