Develop WD2 decoding

.gitignore

@@ -160,3 +160,9 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+
+# ignore the new .txt files generated in test
+/packs/tests/data/repetitive_data/test_*.txt
+# temporary h5 files get ignored
+*tmp.h5

MULE_environment.yml

@@ -2,23 +2,23 @@ name: MULE-3.12-10-24
 channels:
   - defaults
 dependencies:
-  - python=3.12
-  - numpy
-  - iminuit
-  - matplotlib
-  - seaborn
-  - tables
-  - h5py
-  - pandas
-  - tqdm
-  - scipy
-  - cython
-  - pytables
-  - coverage
-  - pytest
-  - hypothesis
-  - jupyterlab
-  - jupyter
-  - flaky
-  - sphinx
+  - python      = 3.12
+  - numpy       = 1.26.4
+  - iminuit     = 2.18.0
+  - matplotlib  = 3.9.2
+  - seaborn     = 0.13.2
+  - tables      = 3.9.2
+  - h5py        = 3.11.0
+  - pandas      = 2.2.2
+  - tqdm        = 4.66.5
+  - scipy       = 1.13.1
+  - cython      = 3.0.11
+  - pytables    = 3.10.1
+  - coverage    = 7.6.1
+  - pytest      = 7.4.4
+  - hypothesis  = 6.111.0
+  - jupyterlab  = 4.2.5
+  - jupyter     = 1.0.0
+  - flaky       = 3.7.0
+  - sphinx      = 7.3.7
 prefix: /home/e78368jw/anaconda3/envs/MULE-3.12-10-24

bin/mule

@@ -27,10 +27,10 @@ MULE pack runtime executable
 Use 'mule --help' for more information
 ======================================''', formatter_class=argparse.RawTextHelpFormatter)
 
-parser.add_argument("pack", choices = ['acq','proc', 'test'], help = '''The pack implemented: 
-        acq  - Acquisition of data using wavedump 1
-        proc - Processing of data
-        test - Testing directory (IGNORE)
+parser.add_argument("pack", choices = ['acq','proc', 'tests'], help = '''The pack implemented: 
+        acq   - Acquisition of data using wavedump 1
+        proc  - Processing of data
+        tests - Testing directory (IGNORE)
         ''')
 parser.add_argument("config", help = 'The config file provided to the pack, this differs based on which pack youre using')
 # acquire arguments
@@ -46,4 +46,4 @@ except ModuleNotFoundError:
     traceback.print_exc()
     exit(1)
 else:
-    pack()
+    pack(args.config)

packs/acq/acq.py

@@ -1,3 +1,10 @@
-def acq():
+from packs.core.core_utils        import check_test
+
+
+def acq(config_file):
     print("This works as expected: acquisition")
     print("In here you should read the config provided")
+
+    if check_test(config_file):
+        return
+

packs/configs/process_WD2_3channel.conf

@@ -0,0 +1,11 @@
+[required]
+
+process          = 'decode'
+wavedump_edition = 2
+file_path        = '/path/to/file.bin'
+save_path        = '/path/to/file.h5'
+
+[optional]
+
+overwrite        = True
+counts           = -1  

packs/core/core_utils.py

@@ -0,0 +1,14 @@
+def flatten(xss):
+    '''
+    Flattens a 2D list
+    eg: [[0,1,2,3], [4,5,6,7]] -> [0,1,2,3,4,5,6,7]
+    '''
+    return [x for xs in xss for x in xs]
+
+def check_test(file):
+    # quick check for test config
+    if file == "test_config":
+        print("Test config executable run successfully")
+        return True
+    else:
+        return False

packs/core/io.py

@@ -0,0 +1,101 @@
+import pandas as pd
+
+import h5py
+import ast
+import configparser
+
+
+from packs.types import types
+
+
+def load_evt_info(file_path, merge = False):
+    '''
+    Loads in a processed WD .h5 file as pandas DataFrame, extracting event information tables.
+
+    Parameters
+    ----------
+
+    file_path (str)   :  Path to saved data
+    merge     (bool)  :  Flag for merging chunked data
+
+    Returns
+    -------
+
+    (pd.DataFrame)  :  Dataframe of event information
+    '''
+
+    h5_data = []
+    with h5py.File(file_path) as f:
+        # extract event info
+        evt_info = f.get('event_information')
+        for i in evt_info.keys():
+            q = evt_info.get(str(i))
+            for j in q:
+                h5_data.append(j)
+
+
+    return pd.DataFrame(map(list, h5_data), columns = (types.event_info_type).names)
+
+
+def load_rwf_info(file_path  :  str,
+                  samples    :  int) -> list:
+    '''
+    Loads in a processed WD .h5 file as pandas dataframe, extracting raw waveform tables.
+    Samples must be provided, and can be found using `load_evt_info()`.
+
+    Parameters
+    ----------
+
+    file_path (str)  :  Path to saved data
+    samples   (int)  :  Number of samples in each raw waveform
+
+    Returns
+    -------
+
+    (pd.DataFrame)  :  Dataframe of raw waveform information
+    '''
+    h5_data = []
+    with h5py.File(file_path) as f:
+        rwf_info = f.get('rwf')
+        for i in rwf_info.keys():
+            q = rwf_info.get(str(i))
+            for j in q:
+                h5_data.append(j)
+
+    return pd.DataFrame(map(list, h5_data), columns = (types.rwf_type(samples)).names)
+
+
+def read_config_file(file_path  :  str) -> dict:
+    '''
+    Read config file passed in via 'mule' and extract relevant information for pack.
+    Example:
+
+    >> mule proc config.conf
+
+    This function collects the relevant information from `config.conf` and passes it to the `proc` pack.
+
+    Parameters
+    ----------
+
+    file_path (str)  :  Path to config file
+
+    Returns
+    -------
+
+    arg_dict (dict)  :  Dictionary of relevant arguments for the pack
+    '''
+    # setup config parser
+    config = configparser.ConfigParser()
+
+    # read in arguments, require the required ones
+    config.read(file_path)
+    arg_dict = {}
+    for section in config.sections():
+        for key in config[section]:
+            # the config should be written in such a way that the python evaluator
+            # can determine its type
+            #
+            # we can setup stricter rules at some other time
+            arg_dict[key] = ast.literal_eval(config[section][key])
+
+    return arg_dict

packs/proc/proc.py

@@ -1,3 +1,28 @@
-def proc():
-    print("This works as expected: processing")
-    print("In here you should read the config provided")
+import os
+
+from packs.core.io                import read_config_file
+from packs.proc.processing_utils  import process_bin_WD2
+from packs.core.core_utils        import check_test
+
+def proc(config_file):
+    print("Starting the processing pack...")
+
+    # checks if test, if so ends run
+    if check_test(config_file):
+        return
+
+    # take full path
+    full_path = os.path.expandvars(config_file)
+
+    conf_dict = read_config_file(full_path)
+    # removing the first two components so that they can be fed into functions
+    arg_dict  = dict(list(conf_dict.items())[2:])
+    # check the method implemented, currently just process
+    match conf_dict['process']:
+        case 'decode':
+            if conf_dict['wavedump_edition'] == 2:
+                process_bin_WD2(**arg_dict)
+            else:
+                raise RuntimeError(f"wavedump edition {conf_dict['wavedump_edition']} decoding isn't currently implemented.")
+        case default:
+            raise RuntimeError(f"process {conf_dict['process']} not currently implemented.")