Allow multi-processing in build_dsp

src/dspeed/cli.py

@@ -180,7 +180,7 @@ def dspeed_cli():
             lh5_tables=args.hdf5_groups,
             database=args.database,
             outputs=args.output_pars,
-            n_max=args.max_rows,
+            n_entries=args.max_rows,
             write_mode=args.writemode,
             buffer_len=args.chunk,
             block_width=args.block,

src/dspeed/vis/waveform_browser.py

@@ -5,15 +5,16 @@
 import math
 import string
 import sys
+from collections.abc import Collection, Mapping
 
 import lgdo
 import matplotlib.pyplot as plt
 import numpy as np
-import pandas
-import pint
 from cycler import cycler
 from lgdo.lh5 import LH5Iterator
+from lgdo.types import Table
 from matplotlib.lines import Line2D
+from pint import Quantity, Unit
 
 from ..processing_chain import build_processing_chain
 from ..units import unit_registry as ureg
@@ -32,22 +33,22 @@ class WaveformBrowser:
 
     def __init__(
         self,
-        files_in: str | list[str] | LH5Iterator,  # noqa: F821
-        lh5_group: str | list[str] = "",
+        raw_in: str | Collection[str] | LH5Iterator | Table,
+        lh5_group: str | Collection[str] = "",
         base_path: str = "",
-        entry_list: list[int] | list[list[int]] = None,
-        entry_mask: list[int] | list[list[int]] = None,
-        dsp_config: dict | str = None,
-        database: str | dict = None,
-        aux_values: pandas.DataFrame = None,
-        lines: str | list[str] = None,
-        styles: dict[str, list] | str = None,
-        legend: str | list[str] = None,
-        legend_opts: dict = None,
+        entry_list: Collection[int] | Collection[Collection[int]] = None,
+        entry_mask: Collection[bool] | Collection[Collection[bool]] = None,
+        dsp_config: str | Mapping = None,
+        database: str | Mapping = None,
+        aux_values: Mapping[np.ndarray] = None,
+        lines: str | Collection[str] = None,
+        styles: Mapping[str, Collection] | str = None,
+        legend: str | Collection[str] = None,
+        legend_opts: Mapping = None,
         n_drawn: int = 1,
-        x_unit: pint.Unit | str = None,
-        x_lim: tuple[float | str | pint.Quantity] = None,
-        y_lim: tuple[float | str | pint.Quantity] = None,
+        x_unit: str | Unit = None,
+        x_lim: Collection[float | str | Quantity] = None,
+        y_lim: Collection[float | str | Quantity] = None,
         norm: str = None,
         align: str = None,
         buffer_len: int = 128,
@@ -56,9 +57,9 @@ def __init__(
         """
         Parameters
         ----------
-        files_in
-            name of file or list of files to browse. Can use wildcards. Can
-            also pass an LH5Iterator
+        raw_in
+            raw data with waveforms. Can be a file or list of lh5 files
+            (requires use of lh5_group argument), an LH5Iterator
 
         lh5_group
             HDF5 base group(s) to read containing a LGDO table that contains
@@ -67,7 +68,7 @@ def __init__(
             the same group will be assigned to each file found
 
         base_path
-            base path for file. See :class:`~lgdo.lh5.LH5Store`.
+            base path for files. See :class:`~lgdo.lh5.LH5Store`.
 
         entry_list
             list of event indices to draw. If it is a nested list, use local
@@ -157,18 +158,15 @@ def __init__(
         self.next_entry = 0
 
         # data i/o initialization
-        if isinstance(files_in, LH5Iterator):
-            self.lh5_it = files_in
+        if isinstance(raw_in, LH5Iterator):
+            self.lh5_it = raw_in
         else:
-            # HACK: do not read VOV "tracelist", cannot be handled correctly by LH5Iterator
-            # remove this hack once VOV support is properly implemented
             self.lh5_it = LH5Iterator(
-                files_in,
+                raw_in,
                 lh5_group,
                 base_path=base_path,
                 entry_list=entry_list,
                 entry_mask=entry_mask,
-                field_mask={"tracelist": False},
                 buffer_len=buffer_len,
             )
 
@@ -203,7 +201,7 @@ def __init__(
             self.lines = {line: [] for line in lines}
 
         # styles
-        if isinstance(styles, (list, tuple)):
+        if isinstance(styles, Collection) and not isinstance(styles, str):
             self.styles = [None for _ in self.lines]
             for i, sty in enumerate(styles):
                 if isinstance(sty, str):
@@ -252,7 +250,7 @@ def __init__(
                 legend_format += f"{st}{{{name}:{form}{cv}}}"
             self.legend_format.append(legend_format)
 
-        self.legend_kwargs = legend_opts if isinstance(legend_opts, dict) else {}
+        self.legend_kwargs = legend_opts if isinstance(legend_opts, Mapping) else {}
 
         # make processing chain and output buffer
         outputs = list(self.lines) + list(self.legend_vals)
@@ -266,8 +264,8 @@ def __init__(
             outputs = [o for o in outputs if o not in self.aux_vals]
 
         self.proc_chain, field_mask, self.lh5_out = build_processing_chain(
-            self.lh5_in,
             dsp_config,
+            self.lh5_in,
             db_dict=database,
             outputs=outputs,
             block_width=block_width,
@@ -356,7 +354,7 @@ def clear_data(self) -> None:
         self.n_stored = 0
 
     def find_entry(
-        self, entry: int | list[int], append: bool = True, safe: bool = False
+        self, entry: int | Collection[int], append: bool = True, safe: bool = False
     ) -> None:
         """Find the requested data associated with entry in input files and
         place store it internally without drawing it.
@@ -512,13 +510,13 @@ def draw_current(self, clear: bool = True) -> None:
 
         leg_handles = []
         leg_labels = []
-        if not isinstance(self.styles, list):
+        if not isinstance(self.styles, Collection):
             styles = self.styles
 
         # draw lines
         default_style = itertools.cycle(cycler(plt.rcParams["axes.prop_cycle"]))
         for i, lines in enumerate(self.lines.values()):
-            if isinstance(self.styles, list):
+            if isinstance(self.styles, Collection):
                 styles = self.styles[i]
             else:
                 styles = self.styles
@@ -594,7 +592,7 @@ def _update_auto_limit(self, x: np.ndarray, y: np.ndarray) -> None:
 
     def draw_entry(
         self,
-        entry: int | list[int],
+        entry: int | Collection[int],
         append: bool = False,
         clear: bool = True,
         safe: bool = False,

tests/configs/icpc-dsp-config.yaml → tests/configs/icpc-dsp-config-yaml.yaml

@@ -1,38 +1,38 @@
 outputs:
- -tp_min
- -tp_max
- -wf_min
- -wf_max
- -bl_mean
- -bl_std
- -bl_slope
- -bl_intercept
- -pz_slope
- -pz_std
- -pz_mean
- -trapTmax
- -tp_0_est
- -tp_0_atrap
- -tp_10
- -tp_20
- -tp_50
- -tp_80
- -tp_90
- -tp_99
- -tp_100
- -tp_01
- -tp_95
- -A_max
- -QDrift
- -dt_eff
- -tp_aoe_max
- -tp_aoe_samp
- -trapEmax
- -trapEftp
- -cuspEmax
- -zacEmax
- -zacEftp
- -cuspEftp
+- tp_min
+- tp_max
+- wf_min
+- wf_max
+- bl_mean
+- bl_std
+- bl_slope
+- bl_intercept
+- pz_slope
+- pz_std
+- pz_mean
+- trapTmax
+- tp_0_est
+- tp_0_atrap
+- tp_10
+- tp_20
+- tp_50
+- tp_80
+- tp_90
+- tp_99
+- tp_100
+- tp_01
+- tp_95
+- A_max
+- QDrift
+- dt_eff
+- tp_aoe_max
+- tp_aoe_samp
+- trapEmax
+- trapEftp
+- cuspEmax
+- zacEmax
+- zacEftp
+- cuspEftp
 processors:
   tp_min, tp_max, wf_min, wf_max:
      function: dspeed.processors.min_max
@@ -58,7 +58,10 @@ processors:
     unit: [ADC, ADC, ADC, ADC]
   t0_kernel:
     function: dspeed.processors.t0_filter
-    args: [128*ns, 2*us, "t0_kernel(round(128*ns+2*us, wf_pz.period), 'f')"]
+    args:
+    - 128*ns/wf_pz.period
+    - 2*us/wf_pz.period
+    - t0_kernel(round((128*ns+2*us)/wf_pz.period), 'f')
     coord_grid: wf_pz
     unit: ADC
   wf_t0_filter:
@@ -122,10 +125,10 @@ processors:
   cusp_kernel:
     function: dspeed.processors.cusp_filter
     args:
-      - db.cusp.sigma
-      - round(db.cusp.flat, wf_blsub.period)
-      - db.pz.tau
-      - cusp_kernel(len(wf_blsub)-round(33.6*us - 4.8*us, wf_blsub.period), 'f')
+      - db.cusp.sigma/wf_blsub.period
+      - round(db.cusp.flat/wf_blsub.period)
+      - db.pz.tau/wf_blsub.period
+      - cusp_kernel(round(len(wf_blsub)-(33.6*us/wf_blsub.period)-(4.8*us/wf_blsub.period)), 'f')
     defaults:
       db.cusp.sigma: 20*us
       db.cusp.flat: 3*us
@@ -135,10 +138,10 @@ processors:
   wf_cusp:
     function: dspeed.processors.fft_convolve_wf
     args:
-      - wf_blsub[:len(wf_blsub)-round(33.6*us, wf_blsub.period)]
+      - wf_blsub[:len(wf_blsub)-round(33.6*us/wf_blsub.period)]
       - cusp_kernel
       - "'v'"
-      - wf_cusp(round(4.8*us, wf_blsub.period) + 1, 'f')
+      - wf_cusp(round(4.8*us/wf_blsub.period) + 1, 'f')
     unit: ADC
   cuspEmax:
     function: numpy.amax
@@ -256,7 +259,4 @@ processors:
     function: dspeed.processors.min_max
     args: [curr_av, aoe_t_min, tp_aoe_max, A_min, A_max]
     unit: [ns, ns, ADC/sample, ADC/sample]
-  tp_aoe_samp:
-    function: dspeed.processors.add
-    args: [tp_0_est, tp_aoe_max/16, tp_aoe_samp]
-    unit: ns
+  tp_aoe_samp: tp_0_est + tp_aoe_max/16

tests/processors/test_histogram.py

@@ -1,39 +1,22 @@
-import os
+import numpy as np
+import pytest
 
-from lgdo import lh5
+from dspeed.errors import DSPFatal
+from dspeed.processors import histogram
 
-from dspeed import build_dsp
 
+def test_histogram_fixed_width(compare_numba_vs_python):
+    vals = np.arange(100) * 2 / 3
+    with pytest.raises(DSPFatal):
+        histogram(vals, np.zeros(10), np.zeros(10))
 
-def test_histogram_fixed_width(lgnd_test_data, tmptestdir):
-    dsp_file = f"{tmptestdir}/LDQTA_r117_20200110T105115Z_cal_geds__numpy_test_dsp.lh5"
-    dsp_config = {
-        "outputs": ["hist_weights", "hist_borders"],
-        "processors": {
-            "hist_weights , hist_borders": {
-                "function": "histogram",
-                "module": "dspeed.processors.histogram",
-                "args": ["waveform", "hist_weights(100)", "hist_borders(101)"],
-                "unit": ["none", "ADC"],
-            }
-        },
-    }
-    build_dsp(
-        f_raw=lgnd_test_data.get_path(
-            "lh5/LDQTA_r117_20200110T105115Z_cal_geds_raw.lh5"
-        ),
-        f_dsp=dsp_file,
-        dsp_config=dsp_config,
-        write_mode="r",
-    )
-    assert os.path.exists(dsp_file)
+    hist_weights = np.zeros(66)
+    hist_edges = np.zeros(67)
+    histogram(vals, hist_weights, hist_edges)
+    assert all(hist_edges == np.arange(67))
+    assert all(hist_weights[0::2] == 2) and all(hist_weights[1::2] == 1)
 
-    df = lh5.read_as(
-        "geds/dsp/", dsp_file, "pd", field_mask=["hist_weights", "hist_borders"]
-    )
-
-    assert len(df["hist_weights"][0]) + 1 == len(df["hist_borders"][0])
-    for i in range(2, len(df["hist_borders"][0])):
-        a = df["hist_borders"][0][i - 1] - df["hist_borders"][0][i - 2]
-        b = df["hist_borders"][0][i] - df["hist_borders"][0][i - 1]
-        assert round(a, 2) == round(b, 2)
+    vals[5] = np.nan
+    histogram(vals, hist_weights, hist_edges)
+    assert all(np.isnan(hist_edges))
+    assert all(hist_weights == 0)