diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8f713be..680ba80 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,13 +5,13 @@ ci: repos: - repo: https://github.com/adamchainz/blacken-docs - rev: "1.19.1" + rev: "1.20.0" hooks: - id: blacken-docs additional_dependencies: [black==24.*] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: "v5.0.0" + rev: "v6.0.0" hooks: - id: check-added-large-files - id: check-case-conflict @@ -31,27 +31,27 @@ repos: - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.8.6" + rev: "v0.13.3" hooks: - id: ruff args: ["--fix", "--show-fixes"] - id: ruff-format - repo: https://github.com/abravalheri/validate-pyproject - rev: "v0.23" + rev: "v0.24.1" hooks: - id: validate-pyproject additional_dependencies: ["validate-pyproject-schema-store[all]"] - repo: https://github.com/python-jsonschema/check-jsonschema - rev: "0.30.0" + rev: "0.34.0" hooks: - id: check-dependabot - id: check-github-workflows - id: check-readthedocs - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.14.1" + rev: "v1.18.2" hooks: - id: mypy files: src @@ -75,13 +75,13 @@ repos: stages: [manual] - repo: https://github.com/codespell-project/codespell - rev: "v2.3.0" + rev: "v2.4.1" hooks: - id: codespell args: ["-L", "nd,unparseable,compiletime,livetime,fom,puls"] - repo: https://github.com/shellcheck-py/shellcheck-py - rev: "v0.10.0.1" + rev: "v0.11.0.1" hooks: - id: shellcheck @@ -99,7 +99,7 @@ repos: types_or: [yaml, markdown, json, toml] - repo: https://github.com/snakemake/snakefmt - rev: v0.10.2 + rev: v0.11.2 hooks: - id: snakefmt files: Snakefile*|\.smk diff --git a/inputs/hardware/configuration/channelmaps/sp02-p%-r%-T%-all-config.yaml b/inputs/hardware/configuration/channelmaps/sp02-p%-r%-T%-all-config.yaml index 24d0104..f745d1c 100644 --- a/inputs/hardware/configuration/channelmaps/sp02-p%-r%-T%-all-config.yaml +++ b/inputs/hardware/configuration/channelmaps/sp02-p%-r%-T%-all-config.yaml @@ -11,7 +11,7 @@ P00661C: Unknown1: name: Unknown1 system: geds - location: + location: string: 1 position: 2 daq: diff --git a/inputs/hardware/configuration/channelmaps/sp03-p%-r%-T%-all-config.yaml b/inputs/hardware/configuration/channelmaps/sp03-p%-r%-T%-all-config.yaml index def05b0..4d433af 100644 --- a/inputs/hardware/configuration/channelmaps/sp03-p%-r%-T%-all-config.yaml +++ b/inputs/hardware/configuration/channelmaps/sp03-p%-r%-T%-all-config.yaml @@ -11,7 +11,7 @@ V03422: Unknown1: name: Unknown1 system: geds - location: + location: string: 1 position: 2 daq: diff --git a/workflow/src/legenddataflow/scripts/complete_run.py b/workflow/src/legenddataflow/scripts/complete_run.py index ea4a2b1..fa1d39d 100644 --- a/workflow/src/legenddataflow/scripts/complete_run.py +++ b/workflow/src/legenddataflow/scripts/complete_run.py @@ -98,23 +98,23 @@ def reformat_dict(dic, out_string="", indent_level=0, ncol=ncol, indent=indent): for key, item in dic.items(): if isinstance(item, list): out_string += ( - f'{add_spaces(indent_level+indent)}"{key}"' + f'{add_spaces(indent_level + indent)}"{key}"' + ": [\n" - + f"{add_spaces(2*indent+indent_level)}" + + f"{add_spaces(2 * indent + indent_level)}" ) for i, _item in enumerate(item): if i > 0 and (i) % ncol == 0: - out_string += f"\n{add_spaces(2*indent+indent_level)}" + out_string += f"\n{add_spaces(2 * indent + indent_level)}" out_string += f'"{_item}", ' out_string = out_string[:-2] - out_string += "\n" + f"{add_spaces(indent+indent_level)}" + "],\n" + out_string += "\n" + f"{add_spaces(indent + indent_level)}" + "],\n" elif isinstance(item, dict): - out_string += f'{add_spaces(indent+indent_level)}"{key}": ' + "{\n" + out_string += f'{add_spaces(indent + indent_level)}"{key}": ' + "{\n" out_string = reformat_dict( item, out_string, indent_level=indent_level + indent, ncol=ncol ) - out_string += "\n" + f"{add_spaces(indent_level+indent)}" + "},\n" + out_string += "\n" + f"{add_spaces(indent_level + indent)}" + "},\n" return out_string[:-2] out_string = "{\n" @@ -146,7 +146,7 @@ def build_valid_keys(input_files_regex, output_dir): for key in list(key_dict): dtype = key.split("-")[-1] out_file = ( - Path(output_dir) / f'{key.replace(f"-{dtype}", "")}-valid_{dtype}.json' + Path(output_dir) / f"{key.replace(f'-{dtype}', '')}-valid_{dtype}.json" ) out_file.parent.mkdir(parents=True, exist_ok=True) if Path(out_file).is_file(): diff --git a/workflow/src/legenddataflow/scripts/par/geds/dsp/dplms.py b/workflow/src/legenddataflow/scripts/par/geds/dsp/dplms.py index f758dc6..c41ac34 100644 --- a/workflow/src/legenddataflow/scripts/par/geds/dsp/dplms.py +++ b/workflow/src/legenddataflow/scripts/par/geds/dsp/dplms.py @@ -65,7 +65,9 @@ def par_geds_dsp_dplms() -> None: idx=idxs, )[0] t1 = time.time() - log.info(f"Time to load fft data {(t1-t0):.2f} s, total events {len(raw_fft)}") + log.info( + f"Time to load fft data {(t1 - t0):.2f} s, total events {len(raw_fft)}" + ) log.info("\nRunning event selection") peaks_kev = np.array(dplms_dict["peaks_kev"]) @@ -81,7 +83,7 @@ def par_geds_dsp_dplms() -> None: raw_cal = sto.read(args.raw_table_name, args.peak_file, idx=ids)[0] log.info( - f"Time to run event selection {(time.time()-t1):.2f} s, total events {len(raw_cal)}" + f"Time to run event selection {(time.time() - t1):.2f} s, total events {len(raw_cal)}" ) if isinstance(dsp_config, (str, list)): @@ -116,7 +118,7 @@ def par_geds_dsp_dplms() -> None: f"loadlh5('{args.lh5_path}', '{args.channel}/dplms/coefficients')" ) - log.info(f"DPLMS creation finished in {(time.time()-t0)/60} minutes") + log.info(f"DPLMS creation finished in {(time.time() - t0) / 60} minutes") else: out_dict = {} dplms_pars = Table(col_dict={"coefficients": Array([])}) diff --git a/workflow/src/legenddataflow/scripts/par/geds/dsp/eopt.py b/workflow/src/legenddataflow/scripts/par/geds/dsp/eopt.py index b67feca..ef3741f 100644 --- a/workflow/src/legenddataflow/scripts/par/geds/dsp/eopt.py +++ b/workflow/src/legenddataflow/scripts/par/geds/dsp/eopt.py @@ -116,7 +116,7 @@ def par_geds_dsp_eopt() -> None: tb_data = sto.read(args.raw_table_name, args.peak_file, idx=ids)[0] t1 = time.time() - log.info(f"Data Loaded in {(t1-t0)/60} minutes") + log.info(f"Data Loaded in {(t1 - t0) / 60} minutes") if isinstance(dsp_config, (str, list)): dsp_config = Props.read_from(dsp_config) @@ -186,7 +186,7 @@ def par_geds_dsp_eopt() -> None: db_dict["zac"]["sigma"] = f"{x[0]}*us" db_dict["etrap"]["rise"] = f"{x[0]}*us" - log.info(f"Initialising values {i+1} : {db_dict}") + log.info(f"Initialising values {i + 1} : {db_dict}") tb_out = run_one_dsp(tb_data, dsp_config, db_dict=db_dict, verbosity=0) @@ -205,7 +205,7 @@ def par_geds_dsp_eopt() -> None: sample_y_trap.append(res[out_field]) err_y_trap.append(res[out_err_field]) - log.info(f"{i+1} Finished") + log.info(f"{i + 1} Finished") if np.isnan(sample_y_cusp).all(): max_cusp = opt_dict["nan_default"] @@ -321,7 +321,7 @@ def par_geds_dsp_eopt() -> None: # db_dict.update(out_param_dict) t2 = time.time() - log.info(f"Optimiser finished in {(t2-t1)/60} minutes") + log.info(f"Optimiser finished in {(t2 - t1) / 60} minutes") out_alpha_dict = {} out_alpha_dict["cuspEmax_ctc"] = { diff --git a/workflow/src/legenddataflow/scripts/par/geds/dsp/evtsel.py b/workflow/src/legenddataflow/scripts/par/geds/dsp/evtsel.py index 1502628..70e454f 100644 --- a/workflow/src/legenddataflow/scripts/par/geds/dsp/evtsel.py +++ b/workflow/src/legenddataflow/scripts/par/geds/dsp/evtsel.py @@ -128,7 +128,7 @@ def par_geds_dsp_evtsel() -> None: Path(args.peak_file).parent.mkdir(parents=True, exist_ok=True) rng = np.random.default_rng() - rand_num = f"{rng.integers(0,99999):05d}" + rand_num = f"{rng.integers(0, 99999):05d}" temp_output = f"{args.peak_file}.{rand_num}" if peak_dict.pop("run_selection") is True: log.debug("Starting peak selection") @@ -252,7 +252,7 @@ def par_geds_dsp_evtsel() -> None: ) peak_dict["n_rows_read"] += n_rows_read_i - log.debug(f'{peak}: {peak_dict["n_rows_read"]}') + log.debug(f"{peak}: {peak_dict['n_rows_read']}") peak_dict["obj_buf_start"] += n_rows_read_i if peak_dict["n_rows_read"] >= 10000 or file == raw_files[-1]: if "e_lower_lim" not in peak_dict: @@ -352,7 +352,7 @@ def par_geds_dsp_evtsel() -> None: peak_dict["obj_buf_start"] = 0 peak_dict["n_events"] = n_wfs log.debug( - f'found {peak_dict["n_events"]} events for {peak}' + f"found {peak_dict['n_events']} events for {peak}" ) else: if ( @@ -384,7 +384,7 @@ def par_geds_dsp_evtsel() -> None: peak_dict["obj_buf"] = None peak_dict["obj_buf_start"] = 0 log.debug( - f'found {peak_dict["n_events"]} events for {peak}' + f"found {peak_dict['n_events']} events for {peak}" ) if peak_dict["n_events"] >= n_events: peak_dict["idxs"] = None @@ -395,5 +395,5 @@ def par_geds_dsp_evtsel() -> None: else: Path(temp_output).touch() - log.debug(f"event selection completed in {time.time()-t0} seconds") + log.debug(f"event selection completed in {time.time() - t0} seconds") Path(temp_output).rename(args.peak_file) diff --git a/workflow/src/legenddataflow/scripts/par/geds/dsp/nopt.py b/workflow/src/legenddataflow/scripts/par/geds/dsp/nopt.py index 814a72c..69a67fa 100644 --- a/workflow/src/legenddataflow/scripts/par/geds/dsp/nopt.py +++ b/workflow/src/legenddataflow/scripts/par/geds/dsp/nopt.py @@ -64,7 +64,7 @@ def par_geds_dsp_nopt() -> None: "args.raw_table_name", raw_files, n_rows=opt_dict["n_events"], idx=idxs )[0] t1 = time.time() - log.info(f"Time to open raw files {t1-t0:.2f} s, n. baselines {len(tb_data)}") + log.info(f"Time to open raw files {t1 - t0:.2f} s, n. baselines {len(tb_data)}") log.info(f"Select baselines {len(tb_data)}") dsp_data = run_one_dsp(tb_data, dsp_config) @@ -96,7 +96,7 @@ def par_geds_dsp_nopt() -> None: ) t2 = time.time() - log.info(f"Optimiser finished in {(t2-t0)/60} minutes") + log.info(f"Optimiser finished in {(t2 - t0) / 60} minutes") else: out_dict = {} plot_dict = {} diff --git a/workflow/src/legenddataflow/scripts/par/geds/hit/ecal.py b/workflow/src/legenddataflow/scripts/par/geds/hit/ecal.py index 3d7db4b..ed6758b 100644 --- a/workflow/src/legenddataflow/scripts/par/geds/hit/ecal.py +++ b/workflow/src/legenddataflow/scripts/par/geds/hit/ecal.py @@ -653,7 +653,7 @@ def par_geds_hit_ecal() -> None: hit_dict.update( { cal_energy_param.replace("_ctc", ""): { - "expression": f"where({cal_energy_param.replace('ctc','noctc')}>{kwarg_dict.get('dt_theshold_kev',100)}, {cal_energy_param}, {cal_energy_param.replace('ctc','noctc')})", + "expression": f"where({cal_energy_param.replace('ctc', 'noctc')}>{kwarg_dict.get('dt_theshold_kev', 100)}, {cal_energy_param}, {cal_energy_param.replace('ctc', 'noctc')})", "parameters": {}, } } diff --git a/workflow/src/legenddataflow/scripts/par/geds/pht/aoe.py b/workflow/src/legenddataflow/scripts/par/geds/pht/aoe.py index 5b52a2e..dd0c0f7 100644 --- a/workflow/src/legenddataflow/scripts/par/geds/pht/aoe.py +++ b/workflow/src/legenddataflow/scripts/par/geds/pht/aoe.py @@ -195,7 +195,7 @@ def eres_func(x): except KeyError: aoe.eres_func = {} else: - out_dict = {tstamp: None for tstamp in cal_dicts} + out_dict = dict.fromkeys(cal_dicts) aoe_plot_dict = {} aoe = None diff --git a/workflow/src/legenddataflow/scripts/par/geds/pht/ecal_part.py b/workflow/src/legenddataflow/scripts/par/geds/pht/ecal_part.py index b13eaaa..065e207 100644 --- a/workflow/src/legenddataflow/scripts/par/geds/pht/ecal_part.py +++ b/workflow/src/legenddataflow/scripts/par/geds/pht/ecal_part.py @@ -291,7 +291,7 @@ def calibrate_partition( bin_width_kev=0.2 if nruns > 3 else 0.5, ) else: - err = f"2614.511 peak not found in {cal_energy_param} fit, reduced csqr {csqr[0]/csqr[1]} not below 10, check fit" + err = f"2614.511 peak not found in {cal_energy_param} fit, reduced csqr {csqr[0] / csqr[1]} not below 10, check fit" raise ValueError(err) full_object_dict[cal_energy_param].get_energy_res_curve( @@ -327,7 +327,7 @@ def calibrate_partition( cal_dicts, { cal_energy_param.replace("_ctc", ""): { - "expression": f"where({cal_energy_param.replace('ctc', 'noctc')}>{kwarg_dict.get('dt_theshold_kev',100)}, {cal_energy_param}, {cal_energy_param.replace('ctc','noctc')})", + "expression": f"where({cal_energy_param.replace('ctc', 'noctc')}>{kwarg_dict.get('dt_theshold_kev', 100)}, {cal_energy_param}, {cal_energy_param.replace('ctc', 'noctc')})", "parameters": {}, } }, diff --git a/workflow/src/legenddataflow/scripts/par/geds/pht/lq.py b/workflow/src/legenddataflow/scripts/par/geds/pht/lq.py index 85f65c6..5a7a3d9 100644 --- a/workflow/src/legenddataflow/scripts/par/geds/pht/lq.py +++ b/workflow/src/legenddataflow/scripts/par/geds/pht/lq.py @@ -214,7 +214,7 @@ def eres_func(x): except KeyError: lq_obj.eres_func = {} else: - out_dict = {tstamp: None for tstamp in cal_dicts} + out_dict = dict.fromkeys(cal_dicts) lq_plot_dict = {} lq_obj = None diff --git a/workflow/src/legenddataflow/scripts/tier/dsp.py b/workflow/src/legenddataflow/scripts/tier/dsp.py index f04a7c6..0f5c45f 100644 --- a/workflow/src/legenddataflow/scripts/tier/dsp.py +++ b/workflow/src/legenddataflow/scripts/tier/dsp.py @@ -70,10 +70,10 @@ def build_tier_dsp() -> None: # if the dictionary only contains one __default__ key, build the channel # list from the (processable) channel map and assign the default config if list(chan_cfg_map.keys()) == ["__default__"]: - chan_cfg_map = { - chan: chan_cfg_map.__default__ - for chan in chan_map.group("analysis.processable")[True].map("name") - } + chan_cfg_map = dict.fromkeys( + chan_map.group("analysis.processable")[True].map("name"), + chan_cfg_map.__default__, + ) # now construct the dictionary of DSP configs for build_dsp() dsp_cfg_tbl_dict = {} @@ -141,7 +141,7 @@ def build_tier_dsp() -> None: outputs[entry]["channels"].append(channel.split("/")[0]) in_dict = True if in_dict is False: - outputs[f"group{len(list(outputs))+1}"] = { + outputs[f"group{len(list(outputs)) + 1}"] = { "channels": [channel.split("/")[0]], "fields": output, } @@ -167,7 +167,7 @@ def build_tier_dsp() -> None: outputs[entry]["channels"].append(channel.split("/")[0]) in_dict = True if in_dict is False: - outputs[f"group{len(list(outputs))+1}"] = { + outputs[f"group{len(list(outputs)) + 1}"] = { "channels": [channel.split("/")[0]], "fields": output, } diff --git a/workflow/src/legenddataflow/scripts/tier/hit.py b/workflow/src/legenddataflow/scripts/tier/hit.py index 1c4bd13..6b72955 100644 --- a/workflow/src/legenddataflow/scripts/tier/hit.py +++ b/workflow/src/legenddataflow/scripts/tier/hit.py @@ -54,10 +54,10 @@ def build_tier_hit() -> None: # if the mapping only contains one __default__ key, build the channel # list from the (processable) channel map and assign the default config if list(chan_cfg_map.keys()) == ["__default__"]: - chan_cfg_map = { - chan: chan_cfg_map.__default__ - for chan in chan_map.group("analysis.processable")[True].map("name") - } + chan_cfg_map = dict.fromkeys( + chan_map.group("analysis.processable")[True].map("name"), + chan_cfg_map.__default__, + ) # now construct the dictionary of hit configs for build_hit() channel_dict = {} @@ -95,7 +95,7 @@ def build_tier_hit() -> None: hit_outputs[entry]["channels"].append(channel) in_dict = True if in_dict is False: - hit_outputs[f"group{len(list(hit_outputs))+1}"] = { + hit_outputs[f"group{len(list(hit_outputs)) + 1}"] = { "channels": [channel], "fields": output, } diff --git a/workflow/src/legenddataflow/scripts/tier/raw_blind.py b/workflow/src/legenddataflow/scripts/tier/raw_blind.py index 57df92d..58bc044 100644 --- a/workflow/src/legenddataflow/scripts/tier/raw_blind.py +++ b/workflow/src/legenddataflow/scripts/tier/raw_blind.py @@ -113,7 +113,7 @@ def build_tier_raw_blind() -> None: # make some temp file to write the output to before renaming it rng = np.random.default_rng() - rand_num = f"{rng.integers(0,99999):05d}" + rand_num = f"{rng.integers(0, 99999):05d}" temp_output = f"{args.output}.{rand_num}" Path(temp_output).parent.mkdir(parents=True, exist_ok=True) diff --git a/workflow/src/legenddataflow/scripts/tier/raw_fcio.py b/workflow/src/legenddataflow/scripts/tier/raw_fcio.py index c52f441..67a42e8 100644 --- a/workflow/src/legenddataflow/scripts/tier/raw_fcio.py +++ b/workflow/src/legenddataflow/scripts/tier/raw_fcio.py @@ -63,7 +63,7 @@ def build_tier_raw_fcio() -> None: raise NotImplementedError() rng = np.random.default_rng() - rand_num = f"{rng.integers(0,99999):05d}" + rand_num = f"{rng.integers(0, 99999):05d}" temp_output = f"{args.output}.{rand_num}" build_raw(args.input, out_spec=all_config, filekey=temp_output, **settings) diff --git a/workflow/src/legenddataflow/scripts/tier/raw_mgdo.py b/workflow/src/legenddataflow/scripts/tier/raw_mgdo.py index d5491ed..a5f031d 100644 --- a/workflow/src/legenddataflow/scripts/tier/raw_mgdo.py +++ b/workflow/src/legenddataflow/scripts/tier/raw_mgdo.py @@ -74,8 +74,9 @@ def build_tier_raw_mgdo() -> None: # insert it in our dictionary if struckid not in chmap: - raise RuntimeError(f"struckid {struckid} not specified in " - "channel map but present in data.") + raise RuntimeError( + f"struckid {struckid} not specified in channel map but present in data." + ) data_dict[_tblid(chmap[struckid].daq.rawid)] = tbl if any(ch not in found_struckids for ch in chmap): diff --git a/workflow/src/legenddataflow/scripts/tier/raw_orca.py b/workflow/src/legenddataflow/scripts/tier/raw_orca.py index ca6a9f3..fc653a4 100644 --- a/workflow/src/legenddataflow/scripts/tier/raw_orca.py +++ b/workflow/src/legenddataflow/scripts/tier/raw_orca.py @@ -101,7 +101,7 @@ def build_tier_raw_orca() -> None: Props.add_to(all_config, muon_config) rng = np.random.default_rng() - rand_num = f"{rng.integers(0,99999):05d}" + rand_num = f"{rng.integers(0, 99999):05d}" temp_output = f"{args.output}.{rand_num}" build_raw(args.input, out_spec=all_config, filekey=temp_output, **settings) diff --git a/workflow/src/legenddataflow/scripts/write_filelist.py b/workflow/src/legenddataflow/scripts/write_filelist.py index efba09f..ca1e0a2 100644 --- a/workflow/src/legenddataflow/scripts/write_filelist.py +++ b/workflow/src/legenddataflow/scripts/write_filelist.py @@ -1,4 +1,4 @@ -# ruff: noqa: F821, T201 +# ruff: noqa: T201 from pathlib import Path from snakemake.script import snakemake # snakemake > 8.16