From fa6062a5bd7f82294abc2a606fbdc464413bf3f0 Mon Sep 17 00:00:00 2001 From: Niko Nanda Putra Nila Lay Date: Mon, 16 Jun 2025 09:01:06 +0200 Subject: [PATCH 1/2] fix non-full buffers not being written into disk --- workflow/src/legenddataflow/scripts/tier/raw_mgdo.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/workflow/src/legenddataflow/scripts/tier/raw_mgdo.py b/workflow/src/legenddataflow/scripts/tier/raw_mgdo.py index 8c5fb2d..f8a39fa 100644 --- a/workflow/src/legenddataflow/scripts/tier/raw_mgdo.py +++ b/workflow/src/legenddataflow/scripts/tier/raw_mgdo.py @@ -141,7 +141,10 @@ def build_tier_raw_mgdo() -> None: if tbl.is_full(): store.write(tbl, tbl_name, args.output, wo_mode="append") tbl.clear() - + # write the remaining table entries when the buffer is not completely filled + if tbl.loc != 0: + store.write(tbl, tbl_name, args.output, wo_mode="append", n_rows=tbl.loc) + tbl.clear() def _tblid(id): return f"ch{id:03d}/raw" From 871245c0b6da3974d044bef3ffff436e9f868dc8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 16 Jun 2025 07:03:51 +0000 Subject: [PATCH 2/2] style: pre-commit fixes --- workflow/Snakefile | 2 ++ workflow/Snakefile-build-raw | 4 ++-- workflow/rules/filelist_gen.smk | 5 ++++- workflow/src/legenddataflow/scripts/filedb.py | 2 -- workflow/src/legenddataflow/scripts/tier/raw_mgdo.py | 1 + 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 9c41405..19ff262 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -57,6 +57,8 @@ include: "rules/chanlist_gen.smk" include: "rules/common.smk" include: "rules/main.smk" include: "rules/tcm.smk" + + # include: "rules/dsp_pars_geds.smk" # include: "rules/dsp_pars_spms.smk" # include: "rules/dsp.smk" diff --git a/workflow/Snakefile-build-raw b/workflow/Snakefile-build-raw index 985eae4..ac2aaf4 100644 --- a/workflow/Snakefile-build-raw +++ b/workflow/Snakefile-build-raw @@ -29,8 +29,8 @@ time = datetime.now().strftime("%Y%m%dT%H%M%SZ") # Had to disable this since it tried to sync legend-metadata even though dir was existing # NOTE: this will attempt a clone of legend-metadata, if the directory does not exist -#metadata = LegendMetadata(meta_path, lazy=True) -#if "legend_metadata_version" in config: +# metadata = LegendMetadata(meta_path, lazy=True) +# if "legend_metadata_version" in config: # metadata.checkout(config.legend_metadata_version) diff --git a/workflow/rules/filelist_gen.smk b/workflow/rules/filelist_gen.smk index 1b9c035..c53afb3 100644 --- a/workflow/rules/filelist_gen.smk +++ b/workflow/rules/filelist_gen.smk @@ -163,7 +163,10 @@ def build_filelist( ignore_keys = [] if analysis_runs is None: analysis_runs = {} - ignore_suffixes = [".log", ".mac"] # allows to store DAQ logs & macros together with the binary outputs + ignore_suffixes = [ + ".log", + ".mac", + ] # allows to store DAQ logs & macros together with the binary outputs phy_filenames = [] other_filenames = [] diff --git a/workflow/src/legenddataflow/scripts/filedb.py b/workflow/src/legenddataflow/scripts/filedb.py index 6261167..179953b 100644 --- a/workflow/src/legenddataflow/scripts/filedb.py +++ b/workflow/src/legenddataflow/scripts/filedb.py @@ -2,9 +2,7 @@ import logging from pathlib import Path -import numpy as np from dbetto.catalog import Props -from lgdo import lh5 from pygama.flow.file_db import FileDB diff --git a/workflow/src/legenddataflow/scripts/tier/raw_mgdo.py b/workflow/src/legenddataflow/scripts/tier/raw_mgdo.py index f8a39fa..b338afd 100644 --- a/workflow/src/legenddataflow/scripts/tier/raw_mgdo.py +++ b/workflow/src/legenddataflow/scripts/tier/raw_mgdo.py @@ -146,6 +146,7 @@ def build_tier_raw_mgdo() -> None: store.write(tbl, tbl_name, args.output, wo_mode="append", n_rows=tbl.loc) tbl.clear() + def _tblid(id): return f"ch{id:03d}/raw"