fix non-full buffers not being written into disk

workflow/Snakefile

@@ -57,6 +57,8 @@ include: "rules/chanlist_gen.smk"
 include: "rules/common.smk"
 include: "rules/main.smk"
 include: "rules/tcm.smk"
+
+
 # include: "rules/dsp_pars_geds.smk"
 # include: "rules/dsp_pars_spms.smk"
 # include: "rules/dsp.smk"

workflow/Snakefile-build-raw

@@ -29,8 +29,8 @@ time = datetime.now().strftime("%Y%m%dT%H%M%SZ")
 
 # Had to disable this since it tried to sync legend-metadata even though dir was existing
 # NOTE: this will attempt a clone of legend-metadata, if the directory does not exist
-#metadata = LegendMetadata(meta_path, lazy=True)
-#if "legend_metadata_version" in config:
+# metadata = LegendMetadata(meta_path, lazy=True)
+# if "legend_metadata_version" in config:
 #    metadata.checkout(config.legend_metadata_version)
 
 

workflow/rules/filelist_gen.smk

@@ -163,7 +163,10 @@ def build_filelist(
         ignore_keys = []
     if analysis_runs is None:
         analysis_runs = {}
-    ignore_suffixes = [".log", ".mac"] # allows to store DAQ logs & macros together with the binary outputs
+    ignore_suffixes = [
+        ".log",
+        ".mac",
+    ]  # allows to store DAQ logs & macros together with the binary outputs
 
     phy_filenames = []
     other_filenames = []

workflow/src/legenddataflow/scripts/filedb.py

@@ -2,9 +2,7 @@
 import logging
 from pathlib import Path
 
-import numpy as np
 from dbetto.catalog import Props
-from lgdo import lh5
 from pygama.flow.file_db import FileDB
 
 

workflow/src/legenddataflow/scripts/tier/raw_mgdo.py

@@ -141,6 +141,10 @@ def build_tier_raw_mgdo() -> None:
             if tbl.is_full():
                 store.write(tbl, tbl_name, args.output, wo_mode="append")
                 tbl.clear()
+        # write the remaining table entries when the buffer is not completely filled
+        if tbl.loc != 0:
+            store.write(tbl, tbl_name, args.output, wo_mode="append", n_rows=tbl.loc)
+            tbl.clear()
 
 
 def _tblid(id):